From 4a54f6667cc1c378f359a7df1d92f4f074776813 Mon Sep 17 00:00:00 2001 From: ReTeamSpeak Date: Wed, 13 May 2026 11:08:16 +0900 Subject: [PATCH] feat: continuous talk mode with voice activation (VAD) Continuous Talk: - Voice Activity Detection with energy-based detection - Smoothing window (5 frames) to avoid false triggers - Hangover timer (15 frames / 300ms) to keep transmitting during pauses - Hysteresis: threshold * 0.7 to stop, threshold to start - Configurable threshold (default 0.005) Talk Modes: - Push-to-Talk: hold V key to talk (existing) - Continuous: mic always on, VAD auto-starts/stops transmission - Toggle between modes in Settings page Architecture: - VoiceActivation struct with state machine (Silent/Speaking/Hangover) - Subscription monitors mic input in continuous mode - MicSamples messages feed VAD, which triggers StartContinuous/StopContinuous - start_transmission/stop_transmission helper functions (shared by PTT and VAD) - Mic lifecycle managed by subscription (starts on mode enable, stops on disable) 69 tests passing, clippy clean on both builds --- src/iced-app/src/audio.rs | 92 ++++++++++++ src/iced-app/src/main.rs | 301 ++++++++++++++++++++++++++++---------- 2 files changed, 315 insertions(+), 78 deletions(-) diff --git a/src/iced-app/src/audio.rs b/src/iced-app/src/audio.rs index 1a7debe..f548501 100644 --- a/src/iced-app/src/audio.rs +++ b/src/iced-app/src/audio.rs @@ -265,3 +265,95 @@ impl OpusEncoderState { self.packet_id = 0; } } + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TalkMode { + PushToTalk, + Continuous, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VadState { + Silent, + Speaking, + Hangover, +} + +pub struct VoiceActivation { + threshold: f32, + hangover_frames: usize, + hangover_counter: usize, + smoothing_window: usize, + energy_history: Vec, + state: VadState, +} + +impl VoiceActivation { + pub fn new(threshold: f32) -> Self { + Self { + threshold, + hangover_frames: 15, + hangover_counter: 0, + smoothing_window: 5, + energy_history: Vec::new(), + state: VadState::Silent, + } + } + + pub fn process(&mut self, samples: &[f32]) -> VadState { + let energy: f32 = samples.iter().map(|s| s * s).sum::() / samples.len() as f32; + + self.energy_history.push(energy); + if self.energy_history.len() > self.smoothing_window { + self.energy_history.remove(0); + } + + let avg_energy: f32 = + self.energy_history.iter().sum::() / self.energy_history.len() as f32; + + match self.state { + VadState::Silent => { + if avg_energy > self.threshold { + self.state = VadState::Speaking; + self.hangover_counter = 0; + } + } + VadState::Speaking => { + if avg_energy <= self.threshold * 0.7 { + self.state = VadState::Hangover; + self.hangover_counter = 0; + } + } + VadState::Hangover => { + if avg_energy > self.threshold { + self.state = VadState::Speaking; + self.hangover_counter = 0; + } else { + self.hangover_counter += 1; + if self.hangover_counter >= self.hangover_frames { + self.state = VadState::Silent; + } + } + } + } + self.state + } + + pub fn state(&self) -> VadState { + self.state + } + + pub fn is_speaking(&self) -> bool { + matches!(self.state, VadState::Speaking | VadState::Hangover) + } + + pub fn set_threshold(&mut self, threshold: f32) { + self.threshold = threshold; + } + + pub fn reset(&mut self) { + self.state = VadState::Silent; + self.hangover_counter = 0; + self.energy_history.clear(); + } +} diff --git a/src/iced-app/src/main.rs b/src/iced-app/src/main.rs index 4cf299a..6c55411 100644 --- a/src/iced-app/src/main.rs +++ b/src/iced-app/src/main.rs @@ -61,6 +61,12 @@ enum Message { #[cfg(feature = "audio")] PttReleased, #[cfg(feature = "audio")] + ToggleTalkMode, + #[cfg(feature = "audio")] + StartContinuous, + #[cfg(feature = "audio")] + StopContinuous, + #[cfg(feature = "audio")] MicSamples(Vec), Noop, } @@ -151,6 +157,12 @@ struct App { #[cfg(feature = "audio")] ptt_active: bool, #[cfg(feature = "audio")] + continuous_active: bool, + #[cfg(feature = "audio")] + talk_mode: audio::TalkMode, + #[cfg(feature = "audio")] + vad: Arc>, + #[cfg(feature = "audio")] audio_send_tx: Option>>, } @@ -209,6 +221,12 @@ impl App { #[cfg(feature = "audio")] ptt_active: false, #[cfg(feature = "audio")] + continuous_active: false, + #[cfg(feature = "audio")] + talk_mode: audio::TalkMode::PushToTalk, + #[cfg(feature = "audio")] + vad: Arc::new(Mutex::new(audio::VoiceActivation::new(0.005))), + #[cfg(feature = "audio")] audio_send_tx: None, }; @@ -544,60 +562,13 @@ impl App { } #[cfg(feature = "audio")] Message::PttPressed => { - if !self.ptt_active && self.connected { + if !self.ptt_active && self.connected && self.talk_mode == audio::TalkMode::PushToTalk { self.ptt_active = true; - let mic = self.mic.clone(); - let encoder = self.audio_send_tx.clone(); - let (sample_tx, sample_rx) = std::sync::mpsc::channel(); - let h = self.handle.clone(); - - tokio::spawn(async move { - { - let mut mic_guard = mic.lock().await; - if mic_guard.start(None, sample_tx).is_err() { - return; - } - } - - let mut enc = match audio::OpusEncoderState::new() { - Ok(e) => e, - Err(_) => return, - }; - - let (audio_tx, mut audio_rx) = tokio::sync::mpsc::channel::>(100); - - let send_handle = h.clone(); - tokio::spawn(async move { - while let Some(data) = audio_rx.recv().await { - let mut guard = send_handle.lock().await; - if let Some(ref mut handle) = *guard { - let _ = handle - .with_connection(move |con| { - let packet = OutAudio::new(&AudioData::C2S { - id: 0, - codec: CodecType::OpusVoice, - data: &data, - }); - let _ = con.send_audio(packet); - Ok::<(), ()>(()) - }) - .await; - } - } - }); - - loop { - match sample_rx.recv() { - Ok(samples) => { - enc.encode_and_send(&samples, &audio_tx); - } - Err(_) => break, - } - } - - let mut mic_guard = mic.lock().await; - mic_guard.stop(); - }); + start_transmission( + self.mic.clone(), + self.handle.clone(), + None, + ); } Task::none() } @@ -605,34 +576,65 @@ impl App { Message::PttReleased => { if self.ptt_active { self.ptt_active = false; - let mic = self.mic.clone(); - let h = self.handle.clone(); - tokio::spawn(async move { - { - let mut mic_guard = mic.lock().await; - mic_guard.stop(); - } - // Send end-of-transmission - let mut guard = h.lock().await; - if let Some(ref mut handle) = *guard { - let _ = handle - .with_connection(move |con| { - let packet = OutAudio::new(&AudioData::C2S { - id: 0, - codec: CodecType::OpusVoice, - data: &[], - }); - let _ = con.send_audio(packet); - Ok::<(), ()>(()) - }) - .await; - } - }); + stop_transmission(self.mic.clone(), self.handle.clone()); } Task::none() } #[cfg(feature = "audio")] - Message::MicSamples(_) => Task::none(), + Message::ToggleTalkMode => { + self.talk_mode = match self.talk_mode { + audio::TalkMode::PushToTalk => audio::TalkMode::Continuous, + audio::TalkMode::Continuous => audio::TalkMode::PushToTalk, + }; + if self.talk_mode == audio::TalkMode::PushToTalk && self.continuous_active { + self.continuous_active = false; + stop_transmission(self.mic.clone(), self.handle.clone()); + } + Task::none() + } + #[cfg(feature = "audio")] + Message::StartContinuous => { + if !self.continuous_active && self.connected && self.talk_mode == audio::TalkMode::Continuous { + self.continuous_active = true; + start_transmission( + self.mic.clone(), + self.handle.clone(), + None, + ); + } + Task::none() + } + #[cfg(feature = "audio")] + Message::StopContinuous => { + if self.continuous_active { + self.continuous_active = false; + stop_transmission(self.mic.clone(), self.handle.clone()); + } + Task::none() + } + #[cfg(feature = "audio")] + Message::MicSamples(samples) => { + if self.talk_mode == audio::TalkMode::Continuous && self.connected { + let vad = self.vad.clone(); + let was_speaking = self.continuous_active; + Task::perform( + async move { + let mut vad_guard = vad.lock().await; + let state = vad_guard.process(&samples); + let is_speaking = vad_guard.is_speaking(); + drop(vad_guard); + match (was_speaking, is_speaking) { + (false, true) => Some(Message::StartContinuous), + (true, false) => Some(Message::StopContinuous), + _ => None, + } + }, + |msg| msg.unwrap_or(Message::Noop), + ) + } else { + Task::none() + } + } Message::Noop => Task::none(), } } @@ -834,6 +836,37 @@ impl App { } }), )); + + // Continuous talk: monitor mic input when in continuous mode + #[cfg(feature = "audio")] + if self.talk_mode == audio::TalkMode::Continuous { + let mic = self.mic.clone(); + let session_id = self.session_id; + subs.push(Subscription::run_with_id( + (session_id, "continuous"), + iced::stream::channel(100, move |mut sender| async move { + let (sample_tx, sample_rx) = std::sync::mpsc::channel(); + { + let mut mic_guard = mic.lock().await; + if mic_guard.start(None, sample_tx).is_err() { + return; + } + } + loop { + match sample_rx.recv() { + Ok(samples) => { + if sender.send(Message::MicSamples(samples)).await.is_err() { + break; + } + } + Err(_) => break, + } + } + let mut mic_guard = mic.lock().await; + mic_guard.stop(); + }), + )); + } } Subscription::batch(subs) @@ -1210,6 +1243,14 @@ impl App { } fn view_settings_content(&self) -> Element<'_, Message> { + #[cfg(feature = "audio")] + let talk_mode_text = match self.talk_mode { + audio::TalkMode::PushToTalk => "Push-to-Talk (hold V)", + audio::TalkMode::Continuous => "Continuous (voice activation)", + }; + #[cfg(not(feature = "audio"))] + let talk_mode_text = "N/A"; + container( column![ text("Settings").size(20), @@ -1222,8 +1263,28 @@ impl App { text("Audio Input Device").size(13), text("System Default").size(12).style(text::secondary), vertical_space().height(12), + text("Voice Activation Mode").size(13), + { + #[cfg(feature = "audio")] + { + row![ + text(talk_mode_text).size(12).style(text::secondary), + horizontal_space(), + button(text("Toggle").size(11)) + .padding([4, 12]) + .style(theme::secondary_button) + .on_press(Message::ToggleTalkMode), + ] + .align_y(iced::Alignment::Center) + } + #[cfg(not(feature = "audio"))] + { + text(talk_mode_text).size(12).style(text::secondary) + } + }, + vertical_space().height(12), text("Push-to-Talk Key").size(13), - text("Not configured").size(12).style(text::secondary), + text("V (hold to talk)").size(12).style(text::secondary), ] .spacing(4) .padding(16), @@ -1310,6 +1371,90 @@ async fn run_connection(con: SyncConnection, event_tx: mpsc::Sender) { let _ = event_tx.send(TsEvent::Disconnected).await; } +#[cfg(feature = "audio")] +fn start_transmission( + mic: Arc>, + handle: Arc>>, + _device_name: Option<&str>, +) { + let (sample_tx, sample_rx) = std::sync::mpsc::channel(); + + tokio::spawn(async move { + { + let mut mic_guard = mic.lock().await; + if mic_guard.start(None, sample_tx).is_err() { + return; + } + } + + let mut enc = match audio::OpusEncoderState::new() { + Ok(e) => e, + Err(_) => return, + }; + + let (audio_tx, mut audio_rx) = tokio::sync::mpsc::channel::>(100); + + let send_handle = handle.clone(); + tokio::spawn(async move { + while let Some(data) = audio_rx.recv().await { + let mut guard = send_handle.lock().await; + if let Some(ref mut h) = *guard { + let _ = h + .with_connection(move |con| { + let packet = OutAudio::new(&AudioData::C2S { + id: 0, + codec: CodecType::OpusVoice, + data: &data, + }); + let _ = con.send_audio(packet); + Ok::<(), ()>(()) + }) + .await; + } + } + }); + + loop { + match sample_rx.recv() { + Ok(samples) => { + enc.encode_and_send(&samples, &audio_tx); + } + Err(_) => break, + } + } + + let mut mic_guard = mic.lock().await; + mic_guard.stop(); + }); +} + +#[cfg(feature = "audio")] +fn stop_transmission( + mic: Arc>, + handle: Arc>>, +) { + tokio::spawn(async move { + { + let mut mic_guard = mic.lock().await; + mic_guard.stop(); + } + let mut guard = handle.lock().await; + if let Some(ref mut h) = *guard { + let _ = h + .with_connection(move |con| { + let packet = OutAudio::new(&AudioData::C2S { + id: 0, + codec: CodecType::OpusVoice, + data: &[], + }); + let _ = con.send_audio(packet); + Ok::<(), ()>(()) + }) + .await; + } + }); +} + fn nav_button(label: &str, page: Page, current: Page) -> Element<'static, Message> { button(text(label.to_string()).size(12)) .padding([8, 16])