From ddf858cc6c19ccbff6a3f6d34f4e16959213d15b Mon Sep 17 00:00:00 2001 From: Edison Jwa Date: Tue, 2 Jun 2026 19:39:11 +0900 Subject: [PATCH] fix(audio): address CoreML VAD review feedback --- CHANGELOG.md | 11 +++++++++++ README.md | 13 ++++++++++++- crates/chanora_audio/src/audio_processing.rs | 9 +++------ crates/chanora_audio/src/engine.rs | 1 - crates/chanora_audio/src/vad/apple_coreml.rs | 18 +++++++++--------- crates/chanora_audio/src/vad/mod.rs | 8 ++++---- docs/security/flutter-license-inventory.md | 1 + 7 files changed, 40 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db8b44c..c81b766 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,17 @@ This project is expected to follow a Conventional Commits style workflow. ## [Unreleased] +### Added + +- Apple CoreML-backed Silero VAD is now the preferred Apple voice + activity detector when the private sibling `silero-coreml` SwiftPM + package is available; WebRTC VAD remains the runtime fallback. + +### Changed + +- Apple platform floors are raised to iOS 16 and macOS 13 while the + CoreML VAD package is linked. + ## [v1.0.0-rc.1] — MVP Public release candidate This is the first release candidate for the MVP public release per diff --git a/README.md b/README.md index 5d89172..ebd2765 100644 --- a/README.md +++ b/README.md @@ -46,13 +46,24 @@ Current platform policy: | Platform | Baseline | |---|---| -| iOS / iPadOS runtime target | iOS 13+ unless Flutter, plugin, audio, or product constraints require raising it | +| iOS / iPadOS runtime target | iOS 16+ while Apple CoreML Silero VAD is linked | +| macOS runtime target | macOS 13+ while Apple CoreML Silero VAD is linked | | App Store Connect upload gate | Xcode 26+ with iOS 26 / iPadOS 26 SDK+ for upload on or after 2026-04-28 | | Android runtime target | Android API 24+ unless Flutter, plugin, audio, or product constraints require raising it | | Google Play target API | Target the Google Play-required API level on upload date | The App Store / Play Store upload gates are release requirements. They are separate from local development and internal testing requirements. +Apple CoreML VAD development requires the private `silero-coreml` SwiftPM package checked out as a sibling of this repository, so the app checkout and package checkout share the same parent directory: + +```text +workspace/ + chanora/ + silero-coreml/ +``` + +The iOS and macOS Xcode projects reference that package via `../../../../silero-coreml` from their project files. GitHub CI skips the unsigned iOS build when the sibling package is unavailable, but local Apple builds need that checkout. + --- ## Architecture Overview diff --git a/crates/chanora_audio/src/audio_processing.rs b/crates/chanora_audio/src/audio_processing.rs index 2ce2280..26adbe0 100644 --- a/crates/chanora_audio/src/audio_processing.rs +++ b/crates/chanora_audio/src/audio_processing.rs @@ -100,12 +100,6 @@ pub enum VadBackend { Disabled, } -#[cfg(target_os = "ios")] -fn default_vad_backend() -> VadBackend { - VadBackend::SileroOnnx -} - -#[cfg(not(target_os = "ios"))] fn default_vad_backend() -> VadBackend { VadBackend::SileroOnnx } @@ -114,6 +108,9 @@ impl VadBackend { /// Stable bridge/debug string. pub fn as_str(self) -> &'static str { match self { + #[cfg(any(target_os = "ios", target_os = "macos"))] + Self::SileroOnnx => "apple_coreml", + #[cfg(not(any(target_os = "ios", target_os = "macos")))] Self::SileroOnnx => "silero_vad_onnx", Self::WebrtcVad => "webrtc_vad", Self::EnergyDebug => "energy_debug", diff --git a/crates/chanora_audio/src/engine.rs b/crates/chanora_audio/src/engine.rs index 4da1443..ce32975 100644 --- a/crates/chanora_audio/src/engine.rs +++ b/crates/chanora_audio/src/engine.rs @@ -312,7 +312,6 @@ pub struct AudioEngine { voice_activity_selector: Option>, #[cfg(any(target_os = "ios", target_os = "macos", target_os = "android"))] mic_gain: f32, - #[cfg(any(target_os = "ios", target_os = "macos"))] // Streams must be dropped to stop audio. Both are `!Send` because // cpal's Stream isn't Send on some backends; we keep them in an // Option wrapped by Mutex so stop() can move them out. On Linux diff --git a/crates/chanora_audio/src/vad/apple_coreml.rs b/crates/chanora_audio/src/vad/apple_coreml.rs index 562f4c3..f28e640 100644 --- a/crates/chanora_audio/src/vad/apple_coreml.rs +++ b/crates/chanora_audio/src/vad/apple_coreml.rs @@ -94,6 +94,7 @@ pub struct AppleCoreMlVad { handle: *mut c_void, symbols: AppleSileroSymbols, accum: Vec, + frame_scratch: Box<[f32; SILERO_COREML_FRAME_16K]>, last_probability: f32, } @@ -130,6 +131,7 @@ impl AppleCoreMlVad { handle, symbols, accum: Vec::with_capacity(SILERO_COREML_FRAME_16K), + frame_scratch: Box::new([0.0; SILERO_COREML_FRAME_16K]), last_probability: 0.0, }) } @@ -148,14 +150,13 @@ impl AppleCoreMlVad { } } - fn calc_level(&mut self, audio_frame: &[f32]) -> f32 { - debug_assert_eq!(audio_frame.len(), SILERO_COREML_FRAME_16K); + fn calc_level(&mut self) -> f32 { let mut probability = self.last_probability; let rc = unsafe { (self.symbols.process)( self.handle, - audio_frame.as_ptr(), - audio_frame.len(), + self.frame_scratch.as_ptr(), + self.frame_scratch.len(), &mut probability, ) }; @@ -183,11 +184,10 @@ impl VoiceActivityDetector for AppleCoreMlVad { self.accum.extend_from_slice(samples); if self.accum.len() >= SILERO_COREML_FRAME_16K { - let audio_frame: Vec = self.accum[..SILERO_COREML_FRAME_16K].to_vec(); - self.calc_level(&audio_frame); - let overflow: Vec = self.accum.drain(SILERO_COREML_FRAME_16K..).collect(); - self.accum.clear(); - self.accum.extend_from_slice(&overflow); + self.frame_scratch + .copy_from_slice(&self.accum[..SILERO_COREML_FRAME_16K]); + self.calc_level(); + drop(self.accum.drain(..SILERO_COREML_FRAME_16K)); } VadOutput { diff --git a/crates/chanora_audio/src/vad/mod.rs b/crates/chanora_audio/src/vad/mod.rs index 35a9f32..d8b6000 100644 --- a/crates/chanora_audio/src/vad/mod.rs +++ b/crates/chanora_audio/src/vad/mod.rs @@ -1,9 +1,9 @@ //! Voice activity detection backends and helpers. //! -//! iOS capture feeds VoiceProcessingIO-processed microphone frames into -//! this module and uses the realtime-safe WebRTC fallback. Other -//! platforms may use a model-backed detector when available so -//! VoiceActivity mode never collapses back to Continuous transmit. +//! Apple capture feeds VoiceProcessingIO/CoreAudio-processed microphone +//! frames into this module and prefers Apple CoreML Silero VAD when the +//! Swift bridge is linked. WebRTC VAD remains the realtime-safe fallback; +//! non-Apple platforms may use ONNX-backed Silero when available. #[cfg(any(target_os = "ios", target_os = "macos"))] pub mod apple_coreml; diff --git a/docs/security/flutter-license-inventory.md b/docs/security/flutter-license-inventory.md index 7c6b7b4..8ee58c3 100644 --- a/docs/security/flutter-license-inventory.md +++ b/docs/security/flutter-license-inventory.md @@ -294,6 +294,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ``` + ### boolean_selector 2.1.2 ```