fix(audio): address CoreML VAD review feedback

This commit is contained in:
Edison Jwa
2026-06-02 19:39:11 +09:00
parent 966afd2b53
commit ddf858cc6c
7 changed files with 40 additions and 21 deletions
+11
View File
@@ -6,6 +6,17 @@ This project is expected to follow a Conventional Commits style workflow.
## [Unreleased] ## [Unreleased]
### Added
- Apple CoreML-backed Silero VAD is now the preferred Apple voice
activity detector when the private sibling `silero-coreml` SwiftPM
package is available; WebRTC VAD remains the runtime fallback.
### Changed
- Apple platform floors are raised to iOS 16 and macOS 13 while the
CoreML VAD package is linked.
## [v1.0.0-rc.1] — MVP Public release candidate ## [v1.0.0-rc.1] — MVP Public release candidate
This is the first release candidate for the MVP public release per This is the first release candidate for the MVP public release per
+12 -1
View File
@@ -46,13 +46,24 @@ Current platform policy:
| Platform | Baseline | | Platform | Baseline |
|---|---| |---|---|
| iOS / iPadOS runtime target | iOS 13+ unless Flutter, plugin, audio, or product constraints require raising it | | iOS / iPadOS runtime target | iOS 16+ while Apple CoreML Silero VAD is linked |
| macOS runtime target | macOS 13+ while Apple CoreML Silero VAD is linked |
| App Store Connect upload gate | Xcode 26+ with iOS 26 / iPadOS 26 SDK+ for upload on or after 2026-04-28 | | App Store Connect upload gate | Xcode 26+ with iOS 26 / iPadOS 26 SDK+ for upload on or after 2026-04-28 |
| Android runtime target | Android API 24+ unless Flutter, plugin, audio, or product constraints require raising it | | Android runtime target | Android API 24+ unless Flutter, plugin, audio, or product constraints require raising it |
| Google Play target API | Target the Google Play-required API level on upload date | | Google Play target API | Target the Google Play-required API level on upload date |
The App Store / Play Store upload gates are release requirements. They are separate from local development and internal testing requirements. The App Store / Play Store upload gates are release requirements. They are separate from local development and internal testing requirements.
Apple CoreML VAD development requires the private `silero-coreml` SwiftPM package checked out as a sibling of this repository, so the app checkout and package checkout share the same parent directory:
```text
workspace/
chanora/
silero-coreml/
```
The iOS and macOS Xcode projects reference that package via `../../../../silero-coreml` from their project files. GitHub CI skips the unsigned iOS build when the sibling package is unavailable, but local Apple builds need that checkout.
--- ---
## Architecture Overview ## Architecture Overview
+3 -6
View File
@@ -100,12 +100,6 @@ pub enum VadBackend {
Disabled, Disabled,
} }
#[cfg(target_os = "ios")]
fn default_vad_backend() -> VadBackend {
VadBackend::SileroOnnx
}
#[cfg(not(target_os = "ios"))]
fn default_vad_backend() -> VadBackend { fn default_vad_backend() -> VadBackend {
VadBackend::SileroOnnx VadBackend::SileroOnnx
} }
@@ -114,6 +108,9 @@ impl VadBackend {
/// Stable bridge/debug string. /// Stable bridge/debug string.
pub fn as_str(self) -> &'static str { pub fn as_str(self) -> &'static str {
match self { match self {
#[cfg(any(target_os = "ios", target_os = "macos"))]
Self::SileroOnnx => "apple_coreml",
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
Self::SileroOnnx => "silero_vad_onnx", Self::SileroOnnx => "silero_vad_onnx",
Self::WebrtcVad => "webrtc_vad", Self::WebrtcVad => "webrtc_vad",
Self::EnergyDebug => "energy_debug", Self::EnergyDebug => "energy_debug",
-1
View File
@@ -312,7 +312,6 @@ pub struct AudioEngine {
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>, voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
#[cfg(any(target_os = "ios", target_os = "macos", target_os = "android"))] #[cfg(any(target_os = "ios", target_os = "macos", target_os = "android"))]
mic_gain: f32, mic_gain: f32,
#[cfg(any(target_os = "ios", target_os = "macos"))]
// Streams must be dropped to stop audio. Both are `!Send` because // Streams must be dropped to stop audio. Both are `!Send` because
// cpal's Stream isn't Send on some backends; we keep them in an // cpal's Stream isn't Send on some backends; we keep them in an
// Option wrapped by Mutex so stop() can move them out. On Linux // Option wrapped by Mutex so stop() can move them out. On Linux
+9 -9
View File
@@ -94,6 +94,7 @@ pub struct AppleCoreMlVad {
handle: *mut c_void, handle: *mut c_void,
symbols: AppleSileroSymbols, symbols: AppleSileroSymbols,
accum: Vec<f32>, accum: Vec<f32>,
frame_scratch: Box<[f32; SILERO_COREML_FRAME_16K]>,
last_probability: f32, last_probability: f32,
} }
@@ -130,6 +131,7 @@ impl AppleCoreMlVad {
handle, handle,
symbols, symbols,
accum: Vec::with_capacity(SILERO_COREML_FRAME_16K), accum: Vec::with_capacity(SILERO_COREML_FRAME_16K),
frame_scratch: Box::new([0.0; SILERO_COREML_FRAME_16K]),
last_probability: 0.0, last_probability: 0.0,
}) })
} }
@@ -148,14 +150,13 @@ impl AppleCoreMlVad {
} }
} }
fn calc_level(&mut self, audio_frame: &[f32]) -> f32 { fn calc_level(&mut self) -> f32 {
debug_assert_eq!(audio_frame.len(), SILERO_COREML_FRAME_16K);
let mut probability = self.last_probability; let mut probability = self.last_probability;
let rc = unsafe { let rc = unsafe {
(self.symbols.process)( (self.symbols.process)(
self.handle, self.handle,
audio_frame.as_ptr(), self.frame_scratch.as_ptr(),
audio_frame.len(), self.frame_scratch.len(),
&mut probability, &mut probability,
) )
}; };
@@ -183,11 +184,10 @@ impl VoiceActivityDetector for AppleCoreMlVad {
self.accum.extend_from_slice(samples); self.accum.extend_from_slice(samples);
if self.accum.len() >= SILERO_COREML_FRAME_16K { if self.accum.len() >= SILERO_COREML_FRAME_16K {
let audio_frame: Vec<f32> = self.accum[..SILERO_COREML_FRAME_16K].to_vec(); self.frame_scratch
self.calc_level(&audio_frame); .copy_from_slice(&self.accum[..SILERO_COREML_FRAME_16K]);
let overflow: Vec<f32> = self.accum.drain(SILERO_COREML_FRAME_16K..).collect(); self.calc_level();
self.accum.clear(); drop(self.accum.drain(..SILERO_COREML_FRAME_16K));
self.accum.extend_from_slice(&overflow);
} }
VadOutput { VadOutput {
+4 -4
View File
@@ -1,9 +1,9 @@
//! Voice activity detection backends and helpers. //! Voice activity detection backends and helpers.
//! //!
//! iOS capture feeds VoiceProcessingIO-processed microphone frames into //! Apple capture feeds VoiceProcessingIO/CoreAudio-processed microphone
//! this module and uses the realtime-safe WebRTC fallback. Other //! frames into this module and prefers Apple CoreML Silero VAD when the
//! platforms may use a model-backed detector when available so //! Swift bridge is linked. WebRTC VAD remains the realtime-safe fallback;
//! VoiceActivity mode never collapses back to Continuous transmit. //! non-Apple platforms may use ONNX-backed Silero when available.
#[cfg(any(target_os = "ios", target_os = "macos"))] #[cfg(any(target_os = "ios", target_os = "macos"))]
pub mod apple_coreml; pub mod apple_coreml;
@@ -294,6 +294,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE. SOFTWARE.
``` ```
### boolean_selector 2.1.2 ### boolean_selector 2.1.2
``` ```