edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: integrate chat voice and diagnostics client

Browse Source
This commit is contained in:
Edison Jwa
2026-05-23 06:51:55 +09:00
parent 7e28791ec2
commit 7d5d8c2c90
93 changed files with 10273 additions and 3347 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chanora_audio/src/vad/silero_onnx.rs
+9 -14
View File
@@ -76,6 +76,8 @@ pub struct SileroOnnxVad {
enum SileroInner {
Onnx(OnnxSession),
#[cfg(test)]
Stub,
}
struct OnnxSession {
@@ -179,9 +181,13 @@ impl SileroOnnxVad {
use ort::value::Value;
use tracing::error;
let SileroInner::Onnx(ref mut inner) = self.inner else {
return self.last_probability;
#[cfg(test)]
let inner = match self.inner {
SileroInner::Onnx(ref mut inner) => inner,
SileroInner::Stub => return self.last_probability,
};
#[cfg(not(test))]
let SileroInner::Onnx(ref mut inner) = self.inner;
debug_assert_eq!(audio_frame.len(), SILERO_FRAME_16K);
@@ -259,17 +265,6 @@ impl SileroOnnxVad {
}
}
#[cfg(target_os = "macos")]
pub(crate) fn bundled_onnxruntime_path_for_vad() -> Option<std::path::PathBuf> {
let exe = std::env::current_exe().ok()?;
let app_dir = exe.parent()?;
let framework = app_dir
.join("Frameworks")
.join("onnxruntime.framework")
.join("onnxruntime");
framework.exists().then_some(framework)
}
impl VoiceActivityDetector for SileroOnnxVad {
/// Accept one 10 ms **16 kHz** f32 mono frame (160 samples).
///
@@ -333,7 +328,7 @@ impl SileroOnnxVadWorker {
let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits()));
let latest_processed_seq = Arc::new(AtomicU64::new(u64::MAX));
let alive = Arc::new(AtomicBool::new(true));
let (tx, rx) = std::sync::mpsc::sync_channel::<SileroFrameMessage>(32);
let (tx, rx) = std::sync::mpsc::sync_channel::<SileroFrameMessage>(64);
let latest_probability_for_thread = latest_probability.clone();
let latest_processed_seq_for_thread = latest_processed_seq.clone();
let alive_for_thread = alive.clone();
crates/chanora_audio/src/vad/ten_onnx.rs
+52 -32
View File
@@ -8,6 +8,8 @@
use crate::frame::f32_to_i16;
use rustfft::{num_complex::Complex32, FftPlanner};
use super::resampler::{Downsampler48to16, INPUT_FRAME_10MS};
use super::{VadOutput, VoiceActivityDetector};
@@ -49,6 +51,8 @@ pub struct TenOnnxVad {
feature_stack: [[f32; FEATURE_LEN]; CONTEXT],
states: [[f32; HIDDEN]; 4],
mel_filters: Vec<[f32; N_BINS]>,
fft: std::sync::Arc<dyn rustfft::Fft<f32>>,
fft_buffer: Vec<Complex32>,
last_probability: f32,
last_speech: bool,
}
@@ -75,6 +79,8 @@ impl TenOnnxVad {
return None;
}
};
let mut fft_planner = FftPlanner::<f32>::new();
let fft = fft_planner.plan_fft_forward(FFT_SIZE);
tracing::info!(target: "chanora_audio", path = model_path, "TEN VAD ONNX model loaded");
Some(Self {
session,
@@ -84,6 +90,8 @@ impl TenOnnxVad {
feature_stack: [[0.0; FEATURE_LEN]; CONTEXT],
states: [[0.0; HIDDEN]; 4],
mel_filters: build_mel_filters(),
fft,
fft_buffer: vec![Complex32::ZERO; FFT_SIZE],
last_probability: 0.0,
last_speech: false,
})
@@ -104,7 +112,12 @@ impl TenOnnxVad {
self.sample_fifo.drain(..excess);
}
let feature = compute_feature(&self.mel_filters, &frame);
let feature = compute_feature(
&self.mel_filters,
self.fft.as_ref(),
&mut self.fft_buffer,
&frame,
);
self.feature_stack.copy_within(1..CONTEXT, 0);
self.feature_stack[CONTEXT - 1] = feature;
self.run_onnx();
@@ -169,19 +182,13 @@ impl TenOnnxVad {
}
}
#[cfg(target_os = "macos")]
fn bundled_onnxruntime_path() -> Option<std::path::PathBuf> {
let exe = std::env::current_exe().ok()?;
let app_dir = exe.parent()?;
let framework = app_dir
.join("Frameworks")
.join("onnxruntime.framework")
.join("onnxruntime");
framework.exists().then_some(framework)
}
fn compute_feature(mel_filters: &[[f32; N_BINS]], frame: &[f32]) -> [f32; FEATURE_LEN] {
let power = power_spectrum(frame);
fn compute_feature(
mel_filters: &[[f32; N_BINS]],
fft: &dyn rustfft::Fft<f32>,
fft_buffer: &mut [Complex32],
frame: &[f32],
) -> [f32; FEATURE_LEN] {
let power = power_spectrum(fft, fft_buffer, frame);
let mut feature = [0.0; FEATURE_LEN];
for band in 0..MEL_BANDS {
let energy = mel_filters[band]
@@ -245,33 +252,43 @@ fn build_mel_filters() -> Vec<[f32; N_BINS]> {
let left = bins[band];
let center = bins[band + 1].max(left + 1);
let right = bins[band + 2].max(center + 1).min(N_BINS - 1);
for i in left..center.min(N_BINS) {
filters[band][i] = (i - left) as f32 / (center - left) as f32;
for (i, weight) in filters[band]
.iter_mut()
.enumerate()
.take(center.min(N_BINS))
.skip(left)
{
*weight = (i - left) as f32 / (center - left) as f32;
}
for i in center..=right {
filters[band][i] = (right - i) as f32 / (right - center).max(1) as f32;
for (i, weight) in filters[band]
.iter_mut()
.enumerate()
.take(right + 1)
.skip(center)
{
*weight = (right - i) as f32 / (right - center).max(1) as f32;
}
}
filters
}
fn power_spectrum(frame: &[f32]) -> [f32; N_BINS] {
let mut windowed = [0.0_f32; FFT_SIZE];
fn power_spectrum(
fft: &dyn rustfft::Fft<f32>,
fft_buffer: &mut [Complex32],
frame: &[f32],
) -> [f32; N_BINS] {
debug_assert_eq!(fft_buffer.len(), FFT_SIZE);
fft_buffer.fill(Complex32::ZERO);
for (idx, sample) in frame.iter().take(WINDOW_16K).enumerate() {
let hann = 0.5 - 0.5 * (2.0 * std::f32::consts::PI * idx as f32 / WINDOW_16K as f32).cos();
windowed[idx] = f32_to_i16(*sample) as f32 * hann;
fft_buffer[idx].re = f32_to_i16(*sample) as f32 * hann;
}
fft.process(fft_buffer);
let mut out = [0.0_f32; N_BINS];
for (k, dst) in out.iter_mut().enumerate() {
let mut re = 0.0_f32;
let mut im = 0.0_f32;
for (n, &x) in windowed.iter().enumerate() {
let phase = -2.0 * std::f32::consts::PI * k as f32 * n as f32 / FFT_SIZE as f32;
re += x * phase.cos();
im += x * phase.sin();
}
*dst = re * re + im * im;
for (dst, bin) in out.iter_mut().zip(fft_buffer.iter()) {
*dst = bin.norm_sqr();
}
out
}
@@ -340,7 +357,7 @@ impl TenOnnxVadWorker {
let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits()));
let latest_processed_seq = Arc::new(AtomicU64::new(u64::MAX));
let alive = Arc::new(AtomicBool::new(true));
let (tx, rx) = std::sync::mpsc::sync_channel::<TenFrameMessage>(32);
let (tx, rx) = std::sync::mpsc::sync_channel::<TenFrameMessage>(128);
let prob_arc = latest_probability.clone();
let seq_arc = latest_processed_seq.clone();
let alive_arc = alive.clone();
@@ -425,8 +442,11 @@ mod tests {
#[test]
fn preprocessing_produces_finite_features() {
let filters = build_mel_filters();
let mut planner = FftPlanner::<f32>::new();
let fft = planner.plan_fft_forward(FFT_SIZE);
let mut fft_buffer = vec![Complex32::ZERO; FFT_SIZE];
let frame = vec![0.0_f32; WINDOW_16K];
let feature = compute_feature(&filters, &frame);
let feature = compute_feature(&filters, fft.as_ref(), &mut fft_buffer, &frame);
assert!(feature.iter().all(|v| v.is_finite()));
}
}