feat: continuous talk mode with voice activation (VAD)
Continuous Talk: - Voice Activity Detection with energy-based detection - Smoothing window (5 frames) to avoid false triggers - Hangover timer (15 frames / 300ms) to keep transmitting during pauses - Hysteresis: threshold * 0.7 to stop, threshold to start - Configurable threshold (default 0.005) Talk Modes: - Push-to-Talk: hold V key to talk (existing) - Continuous: mic always on, VAD auto-starts/stops transmission - Toggle between modes in Settings page Architecture: - VoiceActivation struct with state machine (Silent/Speaking/Hangover) - Subscription monitors mic input in continuous mode - MicSamples messages feed VAD, which triggers StartContinuous/StopContinuous - start_transmission/stop_transmission helper functions (shared by PTT and VAD) - Mic lifecycle managed by subscription (starts on mode enable, stops on disable) 69 tests passing, clippy clean on both builds
This commit is contained in:
@@ -265,3 +265,95 @@ impl OpusEncoderState {
|
|||||||
self.packet_id = 0;
|
self.packet_id = 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum TalkMode {
|
||||||
|
PushToTalk,
|
||||||
|
Continuous,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum VadState {
|
||||||
|
Silent,
|
||||||
|
Speaking,
|
||||||
|
Hangover,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct VoiceActivation {
|
||||||
|
threshold: f32,
|
||||||
|
hangover_frames: usize,
|
||||||
|
hangover_counter: usize,
|
||||||
|
smoothing_window: usize,
|
||||||
|
energy_history: Vec<f32>,
|
||||||
|
state: VadState,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VoiceActivation {
|
||||||
|
pub fn new(threshold: f32) -> Self {
|
||||||
|
Self {
|
||||||
|
threshold,
|
||||||
|
hangover_frames: 15,
|
||||||
|
hangover_counter: 0,
|
||||||
|
smoothing_window: 5,
|
||||||
|
energy_history: Vec::new(),
|
||||||
|
state: VadState::Silent,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn process(&mut self, samples: &[f32]) -> VadState {
|
||||||
|
let energy: f32 = samples.iter().map(|s| s * s).sum::<f32>() / samples.len() as f32;
|
||||||
|
|
||||||
|
self.energy_history.push(energy);
|
||||||
|
if self.energy_history.len() > self.smoothing_window {
|
||||||
|
self.energy_history.remove(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
let avg_energy: f32 =
|
||||||
|
self.energy_history.iter().sum::<f32>() / self.energy_history.len() as f32;
|
||||||
|
|
||||||
|
match self.state {
|
||||||
|
VadState::Silent => {
|
||||||
|
if avg_energy > self.threshold {
|
||||||
|
self.state = VadState::Speaking;
|
||||||
|
self.hangover_counter = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
VadState::Speaking => {
|
||||||
|
if avg_energy <= self.threshold * 0.7 {
|
||||||
|
self.state = VadState::Hangover;
|
||||||
|
self.hangover_counter = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
VadState::Hangover => {
|
||||||
|
if avg_energy > self.threshold {
|
||||||
|
self.state = VadState::Speaking;
|
||||||
|
self.hangover_counter = 0;
|
||||||
|
} else {
|
||||||
|
self.hangover_counter += 1;
|
||||||
|
if self.hangover_counter >= self.hangover_frames {
|
||||||
|
self.state = VadState::Silent;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.state
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn state(&self) -> VadState {
|
||||||
|
self.state
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_speaking(&self) -> bool {
|
||||||
|
matches!(self.state, VadState::Speaking | VadState::Hangover)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_threshold(&mut self, threshold: f32) {
|
||||||
|
self.threshold = threshold;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reset(&mut self) {
|
||||||
|
self.state = VadState::Silent;
|
||||||
|
self.hangover_counter = 0;
|
||||||
|
self.energy_history.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+223
-78
@@ -61,6 +61,12 @@ enum Message {
|
|||||||
#[cfg(feature = "audio")]
|
#[cfg(feature = "audio")]
|
||||||
PttReleased,
|
PttReleased,
|
||||||
#[cfg(feature = "audio")]
|
#[cfg(feature = "audio")]
|
||||||
|
ToggleTalkMode,
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
|
StartContinuous,
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
|
StopContinuous,
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
MicSamples(Vec<f32>),
|
MicSamples(Vec<f32>),
|
||||||
Noop,
|
Noop,
|
||||||
}
|
}
|
||||||
@@ -151,6 +157,12 @@ struct App {
|
|||||||
#[cfg(feature = "audio")]
|
#[cfg(feature = "audio")]
|
||||||
ptt_active: bool,
|
ptt_active: bool,
|
||||||
#[cfg(feature = "audio")]
|
#[cfg(feature = "audio")]
|
||||||
|
continuous_active: bool,
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
|
talk_mode: audio::TalkMode,
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
|
vad: Arc<Mutex<audio::VoiceActivation>>,
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
audio_send_tx: Option<tokio::sync::mpsc::Sender<Vec<u8>>>,
|
audio_send_tx: Option<tokio::sync::mpsc::Sender<Vec<u8>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -209,6 +221,12 @@ impl App {
|
|||||||
#[cfg(feature = "audio")]
|
#[cfg(feature = "audio")]
|
||||||
ptt_active: false,
|
ptt_active: false,
|
||||||
#[cfg(feature = "audio")]
|
#[cfg(feature = "audio")]
|
||||||
|
continuous_active: false,
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
|
talk_mode: audio::TalkMode::PushToTalk,
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
|
vad: Arc::new(Mutex::new(audio::VoiceActivation::new(0.005))),
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
audio_send_tx: None,
|
audio_send_tx: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -544,60 +562,13 @@ impl App {
|
|||||||
}
|
}
|
||||||
#[cfg(feature = "audio")]
|
#[cfg(feature = "audio")]
|
||||||
Message::PttPressed => {
|
Message::PttPressed => {
|
||||||
if !self.ptt_active && self.connected {
|
if !self.ptt_active && self.connected && self.talk_mode == audio::TalkMode::PushToTalk {
|
||||||
self.ptt_active = true;
|
self.ptt_active = true;
|
||||||
let mic = self.mic.clone();
|
start_transmission(
|
||||||
let encoder = self.audio_send_tx.clone();
|
self.mic.clone(),
|
||||||
let (sample_tx, sample_rx) = std::sync::mpsc::channel();
|
self.handle.clone(),
|
||||||
let h = self.handle.clone();
|
None,
|
||||||
|
);
|
||||||
tokio::spawn(async move {
|
|
||||||
{
|
|
||||||
let mut mic_guard = mic.lock().await;
|
|
||||||
if mic_guard.start(None, sample_tx).is_err() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut enc = match audio::OpusEncoderState::new() {
|
|
||||||
Ok(e) => e,
|
|
||||||
Err(_) => return,
|
|
||||||
};
|
|
||||||
|
|
||||||
let (audio_tx, mut audio_rx) = tokio::sync::mpsc::channel::<Vec<u8>>(100);
|
|
||||||
|
|
||||||
let send_handle = h.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
while let Some(data) = audio_rx.recv().await {
|
|
||||||
let mut guard = send_handle.lock().await;
|
|
||||||
if let Some(ref mut handle) = *guard {
|
|
||||||
let _ = handle
|
|
||||||
.with_connection(move |con| {
|
|
||||||
let packet = OutAudio::new(&AudioData::C2S {
|
|
||||||
id: 0,
|
|
||||||
codec: CodecType::OpusVoice,
|
|
||||||
data: &data,
|
|
||||||
});
|
|
||||||
let _ = con.send_audio(packet);
|
|
||||||
Ok::<(), ()>(())
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
loop {
|
|
||||||
match sample_rx.recv() {
|
|
||||||
Ok(samples) => {
|
|
||||||
enc.encode_and_send(&samples, &audio_tx);
|
|
||||||
}
|
|
||||||
Err(_) => break,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut mic_guard = mic.lock().await;
|
|
||||||
mic_guard.stop();
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
Task::none()
|
Task::none()
|
||||||
}
|
}
|
||||||
@@ -605,34 +576,65 @@ impl App {
|
|||||||
Message::PttReleased => {
|
Message::PttReleased => {
|
||||||
if self.ptt_active {
|
if self.ptt_active {
|
||||||
self.ptt_active = false;
|
self.ptt_active = false;
|
||||||
let mic = self.mic.clone();
|
stop_transmission(self.mic.clone(), self.handle.clone());
|
||||||
let h = self.handle.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
{
|
|
||||||
let mut mic_guard = mic.lock().await;
|
|
||||||
mic_guard.stop();
|
|
||||||
}
|
|
||||||
// Send end-of-transmission
|
|
||||||
let mut guard = h.lock().await;
|
|
||||||
if let Some(ref mut handle) = *guard {
|
|
||||||
let _ = handle
|
|
||||||
.with_connection(move |con| {
|
|
||||||
let packet = OutAudio::new(&AudioData::C2S {
|
|
||||||
id: 0,
|
|
||||||
codec: CodecType::OpusVoice,
|
|
||||||
data: &[],
|
|
||||||
});
|
|
||||||
let _ = con.send_audio(packet);
|
|
||||||
Ok::<(), ()>(())
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
Task::none()
|
Task::none()
|
||||||
}
|
}
|
||||||
#[cfg(feature = "audio")]
|
#[cfg(feature = "audio")]
|
||||||
Message::MicSamples(_) => Task::none(),
|
Message::ToggleTalkMode => {
|
||||||
|
self.talk_mode = match self.talk_mode {
|
||||||
|
audio::TalkMode::PushToTalk => audio::TalkMode::Continuous,
|
||||||
|
audio::TalkMode::Continuous => audio::TalkMode::PushToTalk,
|
||||||
|
};
|
||||||
|
if self.talk_mode == audio::TalkMode::PushToTalk && self.continuous_active {
|
||||||
|
self.continuous_active = false;
|
||||||
|
stop_transmission(self.mic.clone(), self.handle.clone());
|
||||||
|
}
|
||||||
|
Task::none()
|
||||||
|
}
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
|
Message::StartContinuous => {
|
||||||
|
if !self.continuous_active && self.connected && self.talk_mode == audio::TalkMode::Continuous {
|
||||||
|
self.continuous_active = true;
|
||||||
|
start_transmission(
|
||||||
|
self.mic.clone(),
|
||||||
|
self.handle.clone(),
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Task::none()
|
||||||
|
}
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
|
Message::StopContinuous => {
|
||||||
|
if self.continuous_active {
|
||||||
|
self.continuous_active = false;
|
||||||
|
stop_transmission(self.mic.clone(), self.handle.clone());
|
||||||
|
}
|
||||||
|
Task::none()
|
||||||
|
}
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
|
Message::MicSamples(samples) => {
|
||||||
|
if self.talk_mode == audio::TalkMode::Continuous && self.connected {
|
||||||
|
let vad = self.vad.clone();
|
||||||
|
let was_speaking = self.continuous_active;
|
||||||
|
Task::perform(
|
||||||
|
async move {
|
||||||
|
let mut vad_guard = vad.lock().await;
|
||||||
|
let state = vad_guard.process(&samples);
|
||||||
|
let is_speaking = vad_guard.is_speaking();
|
||||||
|
drop(vad_guard);
|
||||||
|
match (was_speaking, is_speaking) {
|
||||||
|
(false, true) => Some(Message::StartContinuous),
|
||||||
|
(true, false) => Some(Message::StopContinuous),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|msg| msg.unwrap_or(Message::Noop),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
Task::none()
|
||||||
|
}
|
||||||
|
}
|
||||||
Message::Noop => Task::none(),
|
Message::Noop => Task::none(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -834,6 +836,37 @@ impl App {
|
|||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
));
|
));
|
||||||
|
|
||||||
|
// Continuous talk: monitor mic input when in continuous mode
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
|
if self.talk_mode == audio::TalkMode::Continuous {
|
||||||
|
let mic = self.mic.clone();
|
||||||
|
let session_id = self.session_id;
|
||||||
|
subs.push(Subscription::run_with_id(
|
||||||
|
(session_id, "continuous"),
|
||||||
|
iced::stream::channel(100, move |mut sender| async move {
|
||||||
|
let (sample_tx, sample_rx) = std::sync::mpsc::channel();
|
||||||
|
{
|
||||||
|
let mut mic_guard = mic.lock().await;
|
||||||
|
if mic_guard.start(None, sample_tx).is_err() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
loop {
|
||||||
|
match sample_rx.recv() {
|
||||||
|
Ok(samples) => {
|
||||||
|
if sender.send(Message::MicSamples(samples)).await.is_err() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(_) => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut mic_guard = mic.lock().await;
|
||||||
|
mic_guard.stop();
|
||||||
|
}),
|
||||||
|
));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Subscription::batch(subs)
|
Subscription::batch(subs)
|
||||||
@@ -1210,6 +1243,14 @@ impl App {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn view_settings_content(&self) -> Element<'_, Message> {
|
fn view_settings_content(&self) -> Element<'_, Message> {
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
|
let talk_mode_text = match self.talk_mode {
|
||||||
|
audio::TalkMode::PushToTalk => "Push-to-Talk (hold V)",
|
||||||
|
audio::TalkMode::Continuous => "Continuous (voice activation)",
|
||||||
|
};
|
||||||
|
#[cfg(not(feature = "audio"))]
|
||||||
|
let talk_mode_text = "N/A";
|
||||||
|
|
||||||
container(
|
container(
|
||||||
column![
|
column![
|
||||||
text("Settings").size(20),
|
text("Settings").size(20),
|
||||||
@@ -1222,8 +1263,28 @@ impl App {
|
|||||||
text("Audio Input Device").size(13),
|
text("Audio Input Device").size(13),
|
||||||
text("System Default").size(12).style(text::secondary),
|
text("System Default").size(12).style(text::secondary),
|
||||||
vertical_space().height(12),
|
vertical_space().height(12),
|
||||||
|
text("Voice Activation Mode").size(13),
|
||||||
|
{
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
|
{
|
||||||
|
row![
|
||||||
|
text(talk_mode_text).size(12).style(text::secondary),
|
||||||
|
horizontal_space(),
|
||||||
|
button(text("Toggle").size(11))
|
||||||
|
.padding([4, 12])
|
||||||
|
.style(theme::secondary_button)
|
||||||
|
.on_press(Message::ToggleTalkMode),
|
||||||
|
]
|
||||||
|
.align_y(iced::Alignment::Center)
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "audio"))]
|
||||||
|
{
|
||||||
|
text(talk_mode_text).size(12).style(text::secondary)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
vertical_space().height(12),
|
||||||
text("Push-to-Talk Key").size(13),
|
text("Push-to-Talk Key").size(13),
|
||||||
text("Not configured").size(12).style(text::secondary),
|
text("V (hold to talk)").size(12).style(text::secondary),
|
||||||
]
|
]
|
||||||
.spacing(4)
|
.spacing(4)
|
||||||
.padding(16),
|
.padding(16),
|
||||||
@@ -1310,6 +1371,90 @@ async fn run_connection(con: SyncConnection, event_tx: mpsc::Sender<TsEvent>) {
|
|||||||
let _ = event_tx.send(TsEvent::Disconnected).await;
|
let _ = event_tx.send(TsEvent::Disconnected).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
|
fn start_transmission(
|
||||||
|
mic: Arc<Mutex<audio::Microphone>>,
|
||||||
|
handle: Arc<Mutex<Option<SyncConnectionHandle>>>,
|
||||||
|
_device_name: Option<&str>,
|
||||||
|
) {
|
||||||
|
let (sample_tx, sample_rx) = std::sync::mpsc::channel();
|
||||||
|
|
||||||
|
tokio::spawn(async move {
|
||||||
|
{
|
||||||
|
let mut mic_guard = mic.lock().await;
|
||||||
|
if mic_guard.start(None, sample_tx).is_err() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut enc = match audio::OpusEncoderState::new() {
|
||||||
|
Ok(e) => e,
|
||||||
|
Err(_) => return,
|
||||||
|
};
|
||||||
|
|
||||||
|
let (audio_tx, mut audio_rx) = tokio::sync::mpsc::channel::<Vec<u8>>(100);
|
||||||
|
|
||||||
|
let send_handle = handle.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
while let Some(data) = audio_rx.recv().await {
|
||||||
|
let mut guard = send_handle.lock().await;
|
||||||
|
if let Some(ref mut h) = *guard {
|
||||||
|
let _ = h
|
||||||
|
.with_connection(move |con| {
|
||||||
|
let packet = OutAudio::new(&AudioData::C2S {
|
||||||
|
id: 0,
|
||||||
|
codec: CodecType::OpusVoice,
|
||||||
|
data: &data,
|
||||||
|
});
|
||||||
|
let _ = con.send_audio(packet);
|
||||||
|
Ok::<(), ()>(())
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
loop {
|
||||||
|
match sample_rx.recv() {
|
||||||
|
Ok(samples) => {
|
||||||
|
enc.encode_and_send(&samples, &audio_tx);
|
||||||
|
}
|
||||||
|
Err(_) => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut mic_guard = mic.lock().await;
|
||||||
|
mic_guard.stop();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "audio")]
|
||||||
|
fn stop_transmission(
|
||||||
|
mic: Arc<Mutex<audio::Microphone>>,
|
||||||
|
handle: Arc<Mutex<Option<SyncConnectionHandle>>>,
|
||||||
|
) {
|
||||||
|
tokio::spawn(async move {
|
||||||
|
{
|
||||||
|
let mut mic_guard = mic.lock().await;
|
||||||
|
mic_guard.stop();
|
||||||
|
}
|
||||||
|
let mut guard = handle.lock().await;
|
||||||
|
if let Some(ref mut h) = *guard {
|
||||||
|
let _ = h
|
||||||
|
.with_connection(move |con| {
|
||||||
|
let packet = OutAudio::new(&AudioData::C2S {
|
||||||
|
id: 0,
|
||||||
|
codec: CodecType::OpusVoice,
|
||||||
|
data: &[],
|
||||||
|
});
|
||||||
|
let _ = con.send_audio(packet);
|
||||||
|
Ok::<(), ()>(())
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
fn nav_button(label: &str, page: Page, current: Page) -> Element<'static, Message> {
|
fn nav_button(label: &str, page: Page, current: Page) -> Element<'static, Message> {
|
||||||
button(text(label.to_string()).size(12))
|
button(text(label.to_string()).size(12))
|
||||||
.padding([8, 16])
|
.padding([8, 16])
|
||||||
|
|||||||
Reference in New Issue
Block a user