edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #39 from EdisonJwa/refactor/remove-ios-raw-unit

Browse Source 
refactor(audio): remove experimental iOS RemoteIO+WebRTC APM path
This commit is contained in:
Edison Jwa
2026-06-10 00:53:43 +09:00
committed by GitHub
parent 2f6d45fb04 2b285491d0
commit ddd977796f
10 changed files with 43 additions and 699 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/chanora_flutter/ios/Podfile.lock
+1 -1
View File
@@ -23,7 +23,7 @@ EXTERNAL SOURCES:
:path: ".symlinks/plugins/haptic_kit/ios"
SPEC CHECKSUMS:
chanora_bridge: 26252acdf9ca660ce9c132ad25cd5ad5af467b16
chanora_bridge: 27a03592058709f6f38701343eb51c3a55b02da0
Flutter: cabc95a1d2626b1b06e7179b784ebcf0c0cde467
flutter_foreground_task: a159d2c2173b33699ddb3e6c2a067045d7cebb89
haptic_kit: b22c4fbb2aa7b0d66f2891f81a9e950ad2de5758
apps/chanora_flutter/lib/src/rust/api.dart
+1 -4
View File
@@ -1339,11 +1339,8 @@ sealed class BridgeEvent with _$BridgeEvent {
/// Bridge iOS voice-processing mode.
enum BridgeIosVoiceProcessingMode {
/// Shipping VPIO path.
/// Apple VoiceProcessingIO path.
platformVoiceProcessing,
/// Experimental Sonora path.
sonoraExperimental,
}
@freezed
crates/chanora_audio/src/audio_processing.rs
+9 -47
View File
@@ -56,10 +56,8 @@ impl AudioRoute {
/// iOS voice-processing mode.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IosVoiceProcessingMode {
/// Shipping default: Apple VoiceProcessingIO owns AEC/NS/AGC.
/// Apple VoiceProcessingIO owns AEC/NS/AGC.
PlatformVoiceProcessing,
/// Experimental raw capture-processing path.
SonoraExperimental,
}
/// Processing backend selected by policy/config.
@@ -195,28 +193,20 @@ impl AudioProcessingConfig {
"bluetooth_a2dp is output-only and cannot transmit duplex voice".to_string(),
));
}
if self.ios_mode == IosVoiceProcessingMode::PlatformVoiceProcessing
&& (self.processing_backend == AudioBackend::Sonora
|| self.processing_backend == AudioBackend::WebrtcApm
|| self.aec == EffectOwner::Sonora
|| self.aec == EffectOwner::WebrtcApm
|| self.ns == EffectOwner::Sonora
|| self.ns == EffectOwner::WebrtcApm
|| self.agc == EffectOwner::Sonora
|| self.agc == EffectOwner::WebrtcApm)
if self.processing_backend == AudioBackend::Sonora
|| self.processing_backend == AudioBackend::WebrtcApm
|| self.aec == EffectOwner::Sonora
|| self.aec == EffectOwner::WebrtcApm
|| self.ns == EffectOwner::Sonora
|| self.ns == EffectOwner::WebrtcApm
|| self.agc == EffectOwner::Sonora
|| self.agc == EffectOwner::WebrtcApm
{
return Err(AudioError::InvalidAudioProcessingConfig(
"software audio processing cannot be enabled with iOS VoiceProcessingIO"
.to_string(),
));
}
if self.ios_mode == IosVoiceProcessingMode::SonoraExperimental
&& self.processing_backend != AudioBackend::WebrtcApm
{
return Err(AudioError::InvalidAudioProcessingConfig(
"ios raw processing mode requires the WebRTC APM backend".to_string(),
));
}
Ok(())
}
@@ -262,34 +252,6 @@ mod tests {
assert!(config.validate_for_ios().is_err());
}
#[test]
fn raw_processing_allows_full_webrtc_apm_chain() {
let config = AudioProcessingConfig {
ios_mode: IosVoiceProcessingMode::SonoraExperimental,
processing_backend: AudioBackend::WebrtcApm,
aec: EffectOwner::WebrtcApm,
ns: EffectOwner::WebrtcApm,
agc: EffectOwner::WebrtcApm,
..AudioProcessingConfig::default()
};
assert!(config.validate_for_ios().is_ok());
}
#[test]
fn raw_processing_rejects_non_webrtc_apm_backend() {
let config = AudioProcessingConfig {
ios_mode: IosVoiceProcessingMode::SonoraExperimental,
processing_backend: AudioBackend::PlatformVoiceProcessing,
aec: EffectOwner::WebrtcApm,
ns: EffectOwner::WebrtcApm,
agc: EffectOwner::WebrtcApm,
..AudioProcessingConfig::default()
};
assert!(config.validate_for_ios().is_err());
}
#[test]
fn disable_failed_vad_backend_demotes_to_webrtc() {
let mut config = AudioProcessingConfig {
crates/chanora_audio/src/debug_wav.rs
-9
View File
@@ -314,15 +314,6 @@ mod tests {
rec.stop();
}
#[test]
fn ios_raw_debug_wav_does_not_push_from_realtime_callback() {
let src = include_str!("ios_raw_unit.rs");
assert!(
!src.contains("push_raw_mic") && !src.contains("push_processed_mic"),
"ios raw callbacks must not call WavDebugRecorder push_*_mic until it has a preallocated handoff"
);
}
#[test]
fn wav_header_is_44_bytes() {
// Write to a temp file to test the header.
crates/chanora_audio/src/engine.rs
+6 -35
View File
@@ -395,8 +395,6 @@ unsafe impl Sync for AudioEngine {}
#[cfg(any(target_os = "ios", target_os = "macos"))]
enum IosVoiceBackend {
Vpio(crate::ios_voice_unit::IosVoiceUnit),
#[cfg(target_os = "ios")]
Raw(crate::ios_raw_unit::IosRawUnit),
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
@@ -406,14 +404,12 @@ impl IosVoiceBackend {
{
match self {
Self::Vpio(unit) => unit.restart(),
Self::Raw(unit) => unit.restart(),
}
}
#[cfg(target_os = "macos")]
{
match self {
Self::Vpio(_unit) => Ok(()),
}
let _ = self;
Ok(())
}
}
@@ -422,14 +418,12 @@ impl IosVoiceBackend {
{
match self {
Self::Vpio(unit) => unit.pause(),
Self::Raw(unit) => unit.pause(),
}
}
#[cfg(target_os = "macos")]
{
match self {
Self::Vpio(_unit) => Ok(()),
}
let _ = self;
Ok(())
}
}
@@ -438,14 +432,12 @@ impl IosVoiceBackend {
{
match self {
Self::Vpio(unit) => unit.resume(),
Self::Raw(unit) => unit.resume(),
}
}
#[cfg(target_os = "macos")]
{
match self {
Self::Vpio(_unit) => Ok(()),
}
let _ = self;
Ok(())
}
}
}
@@ -454,27 +446,6 @@ impl IosVoiceBackend {
fn open_ios_voice_backend(
params: crate::mobile_voice_backend::VoiceAudioParams,
) -> Result<IosVoiceBackend, AudioError> {
let _cfg = params.audio_processing_config.lock().unwrap().clone();
#[cfg(target_os = "ios")]
{
if _cfg.ios_mode == crate::IosVoiceProcessingMode::SonoraExperimental {
let raw_params = params.clone();
match crate::ios_raw_unit::IosRawUnit::start(raw_params) {
Ok(unit) => {
info!(target: "chanora_audio", "ios: RemoteIO/WebRTC APM backend selected");
return Ok(IosVoiceBackend::Raw(unit));
}
Err(e) => {
warn!(
target: "chanora_audio",
error = %e,
"ios: RemoteIO/WebRTC APM backend failed; falling back to VoiceProcessingIO"
);
}
}
}
}
let unit = crate::ios_voice_unit::IosVoiceUnit::start(params)?;
Ok(IosVoiceBackend::Vpio(unit))
}
crates/chanora_audio/src/ios_raw_unit.rs
-538
View File
@@ -1,538 +0,0 @@
//! Optional raw iOS RemoteIO path for the WebRTC APM experimental mode.
//!
//! Provides an alternative to `ios_voice_unit.rs` for the
//! `SonoraExperimental` processing mode. Instead of
//! `kAudioUnitSubType_VoiceProcessingIO` (which owns AEC/NS/AGC), it
//! opens `kAudioUnitSubType_RemoteIO` with voice processing explicitly
//! disabled so WebRTC APM can own the full signal path.
//!
//! ## Hard invariants enforced here
//!
//! * INV_009: Rust AEC only active when platform AEC is disabled.
//! * INV_010: VoiceProcessingIO and WebRTC APM AEC are mutually exclusive.
//! * INV_011: Software AEC backend receives both capture and render-reference.
//! * INV_012: Render reference is copied from decoded/mixed remote PCM
//! before playout.
//!
//! ## Fallback
//!
//! If RemoteIO construction fails, the caller falls back to `IosVoiceUnit`
//! (VPIO) and logs the error.
//!
//! ## Status
//!
//! Experimental / disabled by default. Only activated when the user
//! explicitly selects `SonoraExperimental` mode via the bridge API.
//!
//! ## Platform
//!
//! `kAudioUnitSubType_RemoteIO` is only available in the iOS SDK.
//! This module is gated to `target_os = "ios"`.
#[cfg(target_os = "ios")]
pub use inner::IosRawUnit;
#[cfg(target_os = "ios")]
mod inner {
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::sync::{Arc, Mutex};
use audiopus::coder::Encoder as OpusEncoder;
use coreaudio::audio_unit::audio_format::LinearPcmFlags;
use coreaudio::audio_unit::render_callback::{self, data};
use coreaudio::audio_unit::IOType;
use coreaudio::audio_unit::{AudioUnit, Element, SampleFormat, Scope, StreamFormat};
use tracing::{info, warn};
use crate::mobile_voice_backend::VoiceAudioParams;
use crate::processor::AudioProcessor;
use crate::AudioError;
const SAMPLE_RATE_HZ: f64 = 48_000.0;
// ------------------------------------------------------------------ //
// Render-reference ring buffer //
// ------------------------------------------------------------------ //
/// 4-slot ring buffer shared between the render callback (writer) and
/// the capture callback (reader for Sonora AEC3). Capacity: 4 × 10 ms
/// = 40 ms of headroom.
///
/// If the capture callback runs before the render callback has written
/// a frame it reads zeros (silence reference), which is safe — Sonora
/// AEC3 simply skips cancellation for that frame.
type RenderReferenceBuffer = crate::render_reference::RenderReferenceBuffer<480, 4>;
type RenderReferenceFrameAccumulator =
crate::render_reference::RenderReferenceFrameAccumulator<480>;
const RAW_RENDER_SCRATCH_FRAMES: usize = 1024;
// ------------------------------------------------------------------ //
// Capture pipeline state //
// ------------------------------------------------------------------ //
struct RawCaptureState {
encoder: OpusEncoder,
pcm_accum: Vec<i16>,
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx: crate::opus_voice::EncodedVoiceFrameSender,
transmit_active: Arc<AtomicBool>,
output_muted: Arc<AtomicBool>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
vad_detector: crate::vad::WebRtcFallbackVad,
silero_coreml_worker: Option<crate::vad::apple_coreml::AppleCoreMlVadWorker>,
current_vad_backend: crate::VadBackend,
capture_frame_seq: u64,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
/// Processing config — retained for route-change reloads.
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
webrtc_apm_processor: crate::processor::WebRtcApmProcessor,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
render_reference: Arc<RenderReferenceBuffer>,
pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: usize,
fallback_warned_backend: Option<crate::VadBackend>,
}
impl RawCaptureState {
fn new(
params: &VoiceAudioParams,
render_reference: Arc<RenderReferenceBuffer>,
) -> Result<Self, AudioError> {
let encoder = crate::opus_voice::new_voip_encoder("ios raw")?;
let webrtc_apm_config = params
.audio_processing_config
.lock()
.map(|cfg| crate::processor::webrtc_apm::WebRtcApmConfig::from_audio_config(&cfg))
.unwrap_or_default();
Ok(Self {
encoder,
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx: crate::opus_voice::start_out_packet_worker(
params.voice_out_tx.clone(),
params.frames_sent.clone(),
"ios-raw",
)?,
transmit_active: params.transmit_active.clone(),
output_muted: params.output_muted.clone(),
mic_gain: params.mic_gain,
voice_activity_selector: params.voice_activity_selector.clone(),
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_coreml_worker: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
capture_frame_seq: 0,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
audio_processing_config: params.audio_processing_config.clone(),
webrtc_apm_processor: crate::processor::WebRtcApmProcessor::with_config(
webrtc_apm_config,
)?,
audio_processing_stats: params.audio_processing_stats.clone(),
render_reference,
pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: 0,
fallback_warned_backend: None,
})
}
fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) {
if self.fallback_warned_backend != Some(failed_backend) {
self.fallback_warned_backend = Some(failed_backend);
if self.capture_frame_seq < 128 {
tracing::info!(
target: "chanora_audio",
backend = failed_backend.as_str(),
seq = self.capture_frame_seq,
"VAD backend warming up; using WebRTC fallback"
);
} else {
tracing::warn!(
target: "chanora_audio",
backend = failed_backend.as_str(),
"VAD backend unavailable; using WebRTC fallback for runtime detection"
);
}
}
}
fn ingest_i16(&mut self, samples: &[i16]) {
// Accumulate into 10 ms frames for VAD / Sonora processing.
let mut offset = 0;
while offset < samples.len() {
let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
let take = remaining.min(samples.len() - offset);
self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take]
.copy_from_slice(&samples[offset..offset + take]);
self.pending_10ms_len += take;
offset += take;
if self.pending_10ms_len == crate::frame::FRAME_10MS_SAMPLES {
let frame = self.pending_10ms;
self.process_10ms_capture_frame(&frame);
self.encode_complete_20ms_frames();
self.pending_10ms_len = 0;
}
}
if !self.transmit_active.load(Ordering::Relaxed) {
self.pcm_accum.clear();
return;
}
self.encode_complete_20ms_frames();
}
fn encode_complete_20ms_frames(&mut self) {
while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES {
let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES];
frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]);
self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES);
match self.encoder.encode(&frame, &mut self.opus_out[..]) {
Ok(len) => {
crate::opus_voice::send_voip_frame(
&self.voice_out_tx,
&self.opus_out,
len,
|| {
warn!(
target: "chanora_audio",
"ios raw: voice_out queue full; dropping frame"
);
},
|| {},
);
}
Err(e) => {
tracing::error!(target: "chanora_audio",
error = %e, "ios raw opus encode failed");
}
}
}
}
fn process_10ms_capture_frame(
&mut self,
samples: &[i16; crate::frame::FRAME_10MS_SAMPLES],
) {
let mut frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) {
*dst = crate::frame::i16_to_f32(src);
}
let input_dbfs = crate::frame::dbfs(&frame);
// Debug WAV mic taps are intentionally unavailable on iOS raw
// realtime callbacks until WavDebugRecorder supports a
// preallocated handoff path; the current recorder push path
// allocates per frame.
// Feed render reference to WebRTC APM before capture so AEC can adapt.
let render_ref = self.render_reference.read_latest();
self.webrtc_apm_processor.process_render(&render_ref);
self.webrtc_apm_processor.process_capture(&mut frame);
// Processed-mic debug WAV capture is disabled for the same
// realtime allocation reason as the raw-mic tap above.
let voice_activity_mode = self
.voice_activity_selector
.as_ref()
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
.unwrap_or(false);
if !voice_activity_mode {
self.silero_coreml_worker = None;
self.current_vad_backend = crate::VadBackend::Disabled;
self.fallback_warned_backend = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
let (vad_backend, vad_hangover) = self
.audio_processing_config
.try_lock()
.map(|cfg| (cfg.vad_backend, cfg.vad_hangover_ms))
.unwrap_or((
crate::VadBackend::WebrtcVad,
crate::voice_activity::VAD_HANGOVER_MS,
));
if voice_activity_mode {
self.vad_state.configure(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
}
if voice_activity_mode && vad_backend != self.current_vad_backend {
self.current_vad_backend = vad_backend;
self.fallback_warned_backend = None;
if vad_backend == crate::VadBackend::SileroOnnx {
self.silero_coreml_worker = None;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
self.audio_processing_stats.set_vad_fallback_active(true);
} else {
self.silero_coreml_worker = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
self.vad_state.reset();
}
let (vad_probability, active) = if voice_activity_mode {
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
probability: 1.0,
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
match crate::vad::callback_vad_worker_policy(
voice_activity_mode,
vad_backend,
self.silero_coreml_worker.is_some(),
) {
crate::vad::VadWorkerPolicy::UseWorker => {
let worker = self
.silero_coreml_worker
.as_ref()
.expect("policy checked worker");
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
}
crate::vad::VadWorkerPolicy::UseFallback => {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
crate::vad::VadWorkerPolicy::NotModelBacked => crate::vad::VadOutput {
probability: 1.0,
speech: true,
},
}
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
(vad.probability, self.vad_state.update(vad.speech))
} else {
(0.0, false)
};
if let Some(sel) = &self.voice_activity_selector {
sel.set_voice_activity_open(voice_activity_mode && active);
}
self.audio_processing_stats.update_capture(
input_dbfs,
crate::frame::dbfs(&frame),
vad_probability,
voice_activity_mode && active,
self.transmit_active.load(Ordering::Relaxed),
);
if !self.transmit_active.load(Ordering::Relaxed) {
return;
}
if crate::capture_accumulator::append_processed_i16_bounded(
&mut self.pcm_accum,
&frame,
self.mic_gain,
) {
self.audio_processing_stats.increment_callback_xrun();
}
}
}
// ------------------------------------------------------------------ //
// IosRawUnit //
// ------------------------------------------------------------------ //
/// Raw iOS RemoteIO audio unit for the Sonora experimental path.
pub struct IosRawUnit {
unit: AudioUnit,
}
impl IosRawUnit {
/// Open a RemoteIO AudioUnit, install render + input callbacks, start.
pub(crate) fn start(params: VoiceAudioParams) -> Result<Self, AudioError> {
// INV_010: reject if config requests VPIO (that's IosVoiceUnit's job).
{
let cfg = params.audio_processing_config.lock().unwrap();
if cfg.ios_mode == crate::IosVoiceProcessingMode::PlatformVoiceProcessing {
return Err(AudioError::InvalidAudioProcessingConfig(
"IosRawUnit requires raw WebRTC APM mode".to_string(),
));
}
}
let mut unit = AudioUnit::new_uninitialized(IOType::RemoteIO)
.map_err(|e| AudioError::Backend(format!("remoteio new: {e}")))?;
// Enable input on bus 1.
const ENABLE_IO: u32 = 2003;
let enable: u32 = 1;
unit.set_property(ENABLE_IO, Scope::Input, Element::Input, Some(&enable))
.map_err(|e| AudioError::Backend(format!("remoteio enable input: {e}")))?;
// 48 kHz Int16 mono on both buses.
let fmt = StreamFormat {
sample_rate: SAMPLE_RATE_HZ,
sample_format: SampleFormat::I16,
flags: LinearPcmFlags::IS_SIGNED_INTEGER | LinearPcmFlags::IS_PACKED,
channels: 1,
};
unit.set_stream_format(fmt, Scope::Input, Element::Output)
.map_err(|e| AudioError::StreamConfig(format!("remoteio fmt output: {e}")))?;
unit.set_stream_format(fmt, Scope::Output, Element::Input)
.map_err(|e| AudioError::StreamConfig(format!("remoteio fmt input: {e}")))?;
// Shared render-reference buffer (INV_011 / INV_012).
let render_ref_buf = RenderReferenceBuffer::new();
let render_ref_for_capture = render_ref_buf.clone();
let mut capture_state = RawCaptureState::new(&params, render_ref_for_capture)?;
unit.set_input_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
capture_state.ingest_i16(args.data.buffer);
Ok(())
})
.map_err(|e| AudioError::Backend(format!("remoteio input cb: {e}")))?;
let mut scratch = [0.0_f32; RAW_RENDER_SCRATCH_FRAMES * 2];
let mut mono = [0.0_f32; RAW_RENDER_SCRATCH_FRAMES];
let mut render_ref_accum = RenderReferenceFrameAccumulator::new();
let handler = params.handler.clone();
let output_gain = params.output_gain.clone();
let output_muted = params.output_muted.clone();
let stats_render = params.audio_processing_stats.clone();
unit.set_render_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
let out = args.data.buffer;
let n = out.len();
let process_n = n.min(RAW_RENDER_SCRATCH_FRAMES);
let stereo_n = process_n * 2;
if n > RAW_RENDER_SCRATCH_FRAMES {
stats_render.increment_callback_xrun();
}
scratch[..stereo_n].fill(0.0);
match handler.try_lock() {
Ok(mut h) => {
let _ = h.fill_buffer(&mut scratch[..stereo_n]);
}
Err(std::sync::TryLockError::WouldBlock) => {
stats_render.increment_callback_xrun();
}
Err(std::sync::TryLockError::Poisoned(e)) => {
warn!(target: "chanora_audio",
"AudioHandler poisoned (raw render): {e}");
}
}
// INV_012: copy render reference BEFORE playout.
crate::voice_render::downmix_stereo_f32_to_mono_f32(
&scratch[..stereo_n],
&mut mono[..process_n],
);
render_ref_accum.push_mono_samples(&mono[..process_n], |frame| {
render_ref_buf.write(frame);
});
let gain = f32::from_bits(output_gain.load(Ordering::Relaxed));
let muted = output_muted.load(Ordering::Relaxed);
let mix_stats = crate::voice_render::downmix_stereo_f32_to_interleaved_i16(
&scratch[..stereo_n],
&mut out[..process_n],
1,
gain,
muted,
);
if process_n < n {
out[process_n..].fill(0);
}
if mix_stats.clipped_samples > 0 {
stats_render.add_clipped_samples(mix_stats.clipped_samples);
}
stats_render.update_render(crate::frame::dbfs(&scratch[..stereo_n]), n as u32);
Ok(())
})
.map_err(|e| AudioError::Backend(format!("remoteio render cb: {e}")))?;
unit.initialize()
.map_err(|e| AudioError::Backend(format!("remoteio init: {e}")))?;
unit.start()
.map_err(|e| AudioError::Backend(format!("remoteio start: {e}")))?;
info!(
target: "chanora_audio",
sample_rate_hz = SAMPLE_RATE_HZ,
"ios RemoteIO (Sonora experimental) started"
);
Ok(Self { unit })
}
/// Restart the unit after a route change (stop → uninit → init → start).
pub fn restart(&mut self) -> Result<(), AudioError> {
self.unit
.stop()
.map_err(|e| AudioError::Backend(format!("remoteio restart stop: {e}")))?;
self.unit
.uninitialize()
.map_err(|e| AudioError::Backend(format!("remoteio restart uninit: {e}")))?;
self.unit
.initialize()
.map_err(|e| AudioError::Backend(format!("remoteio restart init: {e}")))?;
self.unit
.start()
.map_err(|e| AudioError::Backend(format!("remoteio restart start: {e}")))?;
info!(target: "chanora_audio", "ios RemoteIO restarted");
Ok(())
}
/// Pause the unit during an AVAudioSession interruption.
pub fn pause(&mut self) -> Result<(), AudioError> {
self.unit
.stop()
.map_err(|e| AudioError::Backend(format!("remoteio pause: {e}")))
}
/// Resume the unit after an interruption ends.
pub fn resume(&mut self) -> Result<(), AudioError> {
self.unit
.start()
.map_err(|e| AudioError::Backend(format!("remoteio resume: {e}")))
}
}
impl Drop for IosRawUnit {
fn drop(&mut self) {
if let Err(e) = self.unit.stop() {
warn!(target: "chanora_audio", error = %e,
"ios RemoteIO stop on drop failed");
} else {
info!(target: "chanora_audio", "ios RemoteIO stopped");
}
}
}
}
crates/chanora_audio/src/ios_voice_unit.rs
+20 -35
View File
@@ -406,38 +406,20 @@ impl IosCaptureState {
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
match crate::vad::callback_vad_worker_policy(
voice_activity_mode,
vad_backend,
self.silero_coreml_worker.is_some(),
) {
crate::vad::VadWorkerPolicy::UseWorker => {
let worker = self
.silero_coreml_worker
.as_ref()
.expect("policy checked worker");
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
if let Some(worker) = self.silero_coreml_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
}
crate::vad::VadWorkerPolicy::UseFallback => {
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(
@@ -445,10 +427,13 @@ impl IosCaptureState {
&frame,
)
}
crate::vad::VadWorkerPolicy::NotModelBacked => crate::vad::VadOutput {
probability: 1.0,
speech: true,
},
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
crates/chanora_audio/src/lib.rs
+1 -4
View File
@@ -65,10 +65,7 @@ pub(crate) mod voice_render;
mod sdl_output;
#[cfg(any(target_os = "ios", target_os = "macos"))]
mod ios_voice_unit;
#[cfg(target_os = "ios")]
pub mod ios_raw_unit;
mod ios_voice_unit;
#[cfg(target_os = "android")]
pub mod android_voice_unit;
crates/chanora_bridge/src/api.rs
+5 -23
View File
@@ -1062,10 +1062,8 @@ pub enum BridgeAudioRoute {
/// Bridge iOS voice-processing mode.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BridgeIosVoiceProcessingMode {
/// Shipping VPIO path.
/// Apple VoiceProcessingIO path.
PlatformVoiceProcessing,
/// Experimental Sonora path.
SonoraExperimental,
}
/// Bridge processing backend.
@@ -1223,7 +1221,6 @@ impl From<BridgeIosVoiceProcessingMode> for chanora_core::IosVoiceProcessingMode
fn from(mode: BridgeIosVoiceProcessingMode) -> Self {
match mode {
BridgeIosVoiceProcessingMode::PlatformVoiceProcessing => Self::PlatformVoiceProcessing,
BridgeIosVoiceProcessingMode::SonoraExperimental => Self::SonoraExperimental,
}
}
}
@@ -1234,7 +1231,6 @@ impl From<chanora_core::IosVoiceProcessingMode> for BridgeIosVoiceProcessingMode
chanora_core::IosVoiceProcessingMode::PlatformVoiceProcessing => {
Self::PlatformVoiceProcessing
}
chanora_core::IosVoiceProcessingMode::SonoraExperimental => Self::SonoraExperimental,
}
}
}
@@ -2361,25 +2357,11 @@ pub async fn set_ios_voice_processing_mode(
let config = BridgeAudioProcessingConfig {
route: BridgeAudioRoute::Speaker,
ios_mode: mode,
processing_backend: match mode {
BridgeIosVoiceProcessingMode::PlatformVoiceProcessing => {
BridgeAudioBackend::PlatformVoiceProcessing
}
BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeAudioBackend::WebrtcApm,
},
processing_backend: BridgeAudioBackend::PlatformVoiceProcessing,
vad_backend: BridgeVadBackend::SileroOnnx,
aec: match mode {
BridgeIosVoiceProcessingMode::PlatformVoiceProcessing => BridgeEffectOwner::Platform,
BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeEffectOwner::WebrtcApm,
},
ns: match mode {
BridgeIosVoiceProcessingMode::PlatformVoiceProcessing => BridgeEffectOwner::Platform,
BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeEffectOwner::WebrtcApm,
},
agc: match mode {
BridgeIosVoiceProcessingMode::PlatformVoiceProcessing => BridgeEffectOwner::Platform,
BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeEffectOwner::WebrtcApm,
},
aec: BridgeEffectOwner::Platform,
ns: BridgeEffectOwner::Platform,
agc: BridgeEffectOwner::Platform,
hpf_enabled: true,
limiter_enabled: true,
vad_hangover_ms: 500,
crates/chanora_bridge/src/frb_generated.rs
-3
View File
@@ -2409,7 +2409,6 @@ impl SseDecode for crate::api::BridgeIosVoiceProcessingMode {
let mut inner = <i32>::sse_decode(deserializer);
return match inner {
0 => crate::api::BridgeIosVoiceProcessingMode::PlatformVoiceProcessing,
1 => crate::api::BridgeIosVoiceProcessingMode::SonoraExperimental,
_ => unreachable!(
"Invalid variant for BridgeIosVoiceProcessingMode: {}",
inner
@@ -3445,7 +3444,6 @@ impl flutter_rust_bridge::IntoDart for crate::api::BridgeIosVoiceProcessingMode
fn into_dart(self) -> flutter_rust_bridge::for_generated::DartAbi {
match self {
Self::PlatformVoiceProcessing => 0.into_dart(),
Self::SonoraExperimental => 1.into_dart(),
_ => unreachable!(),
}
}
@@ -4215,7 +4213,6 @@ impl SseEncode for crate::api::BridgeIosVoiceProcessingMode {
<i32>::sse_encode(
match self {
crate::api::BridgeIosVoiceProcessingMode::PlatformVoiceProcessing => 0,
crate::api::BridgeIosVoiceProcessingMode::SonoraExperimental => 1,
_ => {
unimplemented!("");
}