fix(audio): address CoreML VAD review feedback
This commit is contained in:
@@ -6,6 +6,17 @@ This project is expected to follow a Conventional Commits style workflow.
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- Apple CoreML-backed Silero VAD is now the preferred Apple voice
|
||||||
|
activity detector when the private sibling `silero-coreml` SwiftPM
|
||||||
|
package is available; WebRTC VAD remains the runtime fallback.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- Apple platform floors are raised to iOS 16 and macOS 13 while the
|
||||||
|
CoreML VAD package is linked.
|
||||||
|
|
||||||
## [v1.0.0-rc.1] — MVP Public release candidate
|
## [v1.0.0-rc.1] — MVP Public release candidate
|
||||||
|
|
||||||
This is the first release candidate for the MVP public release per
|
This is the first release candidate for the MVP public release per
|
||||||
|
|||||||
@@ -46,13 +46,24 @@ Current platform policy:
|
|||||||
|
|
||||||
| Platform | Baseline |
|
| Platform | Baseline |
|
||||||
|---|---|
|
|---|---|
|
||||||
| iOS / iPadOS runtime target | iOS 13+ unless Flutter, plugin, audio, or product constraints require raising it |
|
| iOS / iPadOS runtime target | iOS 16+ while Apple CoreML Silero VAD is linked |
|
||||||
|
| macOS runtime target | macOS 13+ while Apple CoreML Silero VAD is linked |
|
||||||
| App Store Connect upload gate | Xcode 26+ with iOS 26 / iPadOS 26 SDK+ for upload on or after 2026-04-28 |
|
| App Store Connect upload gate | Xcode 26+ with iOS 26 / iPadOS 26 SDK+ for upload on or after 2026-04-28 |
|
||||||
| Android runtime target | Android API 24+ unless Flutter, plugin, audio, or product constraints require raising it |
|
| Android runtime target | Android API 24+ unless Flutter, plugin, audio, or product constraints require raising it |
|
||||||
| Google Play target API | Target the Google Play-required API level on upload date |
|
| Google Play target API | Target the Google Play-required API level on upload date |
|
||||||
|
|
||||||
The App Store / Play Store upload gates are release requirements. They are separate from local development and internal testing requirements.
|
The App Store / Play Store upload gates are release requirements. They are separate from local development and internal testing requirements.
|
||||||
|
|
||||||
|
Apple CoreML VAD development requires the private `silero-coreml` SwiftPM package checked out as a sibling of this repository, so the app checkout and package checkout share the same parent directory:
|
||||||
|
|
||||||
|
```text
|
||||||
|
workspace/
|
||||||
|
chanora/
|
||||||
|
silero-coreml/
|
||||||
|
```
|
||||||
|
|
||||||
|
The iOS and macOS Xcode projects reference that package via `../../../../silero-coreml` from their project files. GitHub CI skips the unsigned iOS build when the sibling package is unavailable, but local Apple builds need that checkout.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Architecture Overview
|
## Architecture Overview
|
||||||
|
|||||||
@@ -100,12 +100,6 @@ pub enum VadBackend {
|
|||||||
Disabled,
|
Disabled,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(target_os = "ios")]
|
|
||||||
fn default_vad_backend() -> VadBackend {
|
|
||||||
VadBackend::SileroOnnx
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(not(target_os = "ios"))]
|
|
||||||
fn default_vad_backend() -> VadBackend {
|
fn default_vad_backend() -> VadBackend {
|
||||||
VadBackend::SileroOnnx
|
VadBackend::SileroOnnx
|
||||||
}
|
}
|
||||||
@@ -114,6 +108,9 @@ impl VadBackend {
|
|||||||
/// Stable bridge/debug string.
|
/// Stable bridge/debug string.
|
||||||
pub fn as_str(self) -> &'static str {
|
pub fn as_str(self) -> &'static str {
|
||||||
match self {
|
match self {
|
||||||
|
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||||
|
Self::SileroOnnx => "apple_coreml",
|
||||||
|
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||||||
Self::SileroOnnx => "silero_vad_onnx",
|
Self::SileroOnnx => "silero_vad_onnx",
|
||||||
Self::WebrtcVad => "webrtc_vad",
|
Self::WebrtcVad => "webrtc_vad",
|
||||||
Self::EnergyDebug => "energy_debug",
|
Self::EnergyDebug => "energy_debug",
|
||||||
|
|||||||
@@ -312,7 +312,6 @@ pub struct AudioEngine {
|
|||||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||||
#[cfg(any(target_os = "ios", target_os = "macos", target_os = "android"))]
|
#[cfg(any(target_os = "ios", target_os = "macos", target_os = "android"))]
|
||||||
mic_gain: f32,
|
mic_gain: f32,
|
||||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
|
||||||
// Streams must be dropped to stop audio. Both are `!Send` because
|
// Streams must be dropped to stop audio. Both are `!Send` because
|
||||||
// cpal's Stream isn't Send on some backends; we keep them in an
|
// cpal's Stream isn't Send on some backends; we keep them in an
|
||||||
// Option wrapped by Mutex so stop() can move them out. On Linux
|
// Option wrapped by Mutex so stop() can move them out. On Linux
|
||||||
|
|||||||
@@ -94,6 +94,7 @@ pub struct AppleCoreMlVad {
|
|||||||
handle: *mut c_void,
|
handle: *mut c_void,
|
||||||
symbols: AppleSileroSymbols,
|
symbols: AppleSileroSymbols,
|
||||||
accum: Vec<f32>,
|
accum: Vec<f32>,
|
||||||
|
frame_scratch: Box<[f32; SILERO_COREML_FRAME_16K]>,
|
||||||
last_probability: f32,
|
last_probability: f32,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -130,6 +131,7 @@ impl AppleCoreMlVad {
|
|||||||
handle,
|
handle,
|
||||||
symbols,
|
symbols,
|
||||||
accum: Vec::with_capacity(SILERO_COREML_FRAME_16K),
|
accum: Vec::with_capacity(SILERO_COREML_FRAME_16K),
|
||||||
|
frame_scratch: Box::new([0.0; SILERO_COREML_FRAME_16K]),
|
||||||
last_probability: 0.0,
|
last_probability: 0.0,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -148,14 +150,13 @@ impl AppleCoreMlVad {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn calc_level(&mut self, audio_frame: &[f32]) -> f32 {
|
fn calc_level(&mut self) -> f32 {
|
||||||
debug_assert_eq!(audio_frame.len(), SILERO_COREML_FRAME_16K);
|
|
||||||
let mut probability = self.last_probability;
|
let mut probability = self.last_probability;
|
||||||
let rc = unsafe {
|
let rc = unsafe {
|
||||||
(self.symbols.process)(
|
(self.symbols.process)(
|
||||||
self.handle,
|
self.handle,
|
||||||
audio_frame.as_ptr(),
|
self.frame_scratch.as_ptr(),
|
||||||
audio_frame.len(),
|
self.frame_scratch.len(),
|
||||||
&mut probability,
|
&mut probability,
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
@@ -183,11 +184,10 @@ impl VoiceActivityDetector for AppleCoreMlVad {
|
|||||||
|
|
||||||
self.accum.extend_from_slice(samples);
|
self.accum.extend_from_slice(samples);
|
||||||
if self.accum.len() >= SILERO_COREML_FRAME_16K {
|
if self.accum.len() >= SILERO_COREML_FRAME_16K {
|
||||||
let audio_frame: Vec<f32> = self.accum[..SILERO_COREML_FRAME_16K].to_vec();
|
self.frame_scratch
|
||||||
self.calc_level(&audio_frame);
|
.copy_from_slice(&self.accum[..SILERO_COREML_FRAME_16K]);
|
||||||
let overflow: Vec<f32> = self.accum.drain(SILERO_COREML_FRAME_16K..).collect();
|
self.calc_level();
|
||||||
self.accum.clear();
|
drop(self.accum.drain(..SILERO_COREML_FRAME_16K));
|
||||||
self.accum.extend_from_slice(&overflow);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
VadOutput {
|
VadOutput {
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
//! Voice activity detection backends and helpers.
|
//! Voice activity detection backends and helpers.
|
||||||
//!
|
//!
|
||||||
//! iOS capture feeds VoiceProcessingIO-processed microphone frames into
|
//! Apple capture feeds VoiceProcessingIO/CoreAudio-processed microphone
|
||||||
//! this module and uses the realtime-safe WebRTC fallback. Other
|
//! frames into this module and prefers Apple CoreML Silero VAD when the
|
||||||
//! platforms may use a model-backed detector when available so
|
//! Swift bridge is linked. WebRTC VAD remains the realtime-safe fallback;
|
||||||
//! VoiceActivity mode never collapses back to Continuous transmit.
|
//! non-Apple platforms may use ONNX-backed Silero when available.
|
||||||
|
|
||||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||||
pub mod apple_coreml;
|
pub mod apple_coreml;
|
||||||
|
|||||||
@@ -294,6 +294,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|||||||
SOFTWARE.
|
SOFTWARE.
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
### boolean_selector 2.1.2
|
### boolean_selector 2.1.2
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|||||||
Reference in New Issue
Block a user