edison/re-teamspeak
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: continuous talk mode with voice activation (VAD)

Browse Source 
Continuous Talk:
- Voice Activity Detection with energy-based detection
- Smoothing window (5 frames) to avoid false triggers
- Hangover timer (15 frames / 300ms) to keep transmitting during pauses
- Hysteresis: threshold * 0.7 to stop, threshold to start
- Configurable threshold (default 0.005)

Talk Modes:
- Push-to-Talk: hold V key to talk (existing)
- Continuous: mic always on, VAD auto-starts/stops transmission
- Toggle between modes in Settings page

Architecture:
- VoiceActivation struct with state machine (Silent/Speaking/Hangover)
- Subscription monitors mic input in continuous mode
- MicSamples messages feed VAD, which triggers StartContinuous/StopContinuous
- start_transmission/stop_transmission helper functions (shared by PTT and VAD)
- Mic lifecycle managed by subscription (starts on mode enable, stops on disable)

69 tests passing, clippy clean on both builds
This commit is contained in:
ReTeamSpeak
2026-05-13 11:08:16 +09:00
parent 190fcf2a7e
commit 4a54f6667c
2 changed files with 315 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/iced-app/src/audio.rs
+92
View File
@@ -265,3 +265,95 @@ impl OpusEncoderState {
self.packet_id = 0;
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TalkMode {
PushToTalk,
Continuous,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VadState {
Silent,
Speaking,
Hangover,
}
pub struct VoiceActivation {
threshold: f32,
hangover_frames: usize,
hangover_counter: usize,
smoothing_window: usize,
energy_history: Vec<f32>,
state: VadState,
}
impl VoiceActivation {
pub fn new(threshold: f32) -> Self {
Self {
threshold,
hangover_frames: 15,
hangover_counter: 0,
smoothing_window: 5,
energy_history: Vec::new(),
state: VadState::Silent,
}
}
pub fn process(&mut self, samples: &[f32]) -> VadState {
let energy: f32 = samples.iter().map(|s| s * s).sum::<f32>() / samples.len() as f32;
self.energy_history.push(energy);
if self.energy_history.len() > self.smoothing_window {
self.energy_history.remove(0);
}
let avg_energy: f32 =
self.energy_history.iter().sum::<f32>() / self.energy_history.len() as f32;
match self.state {
VadState::Silent => {
if avg_energy > self.threshold {
self.state = VadState::Speaking;
self.hangover_counter = 0;
}
}
VadState::Speaking => {
if avg_energy <= self.threshold * 0.7 {
self.state = VadState::Hangover;
self.hangover_counter = 0;
}
}
VadState::Hangover => {
if avg_energy > self.threshold {
self.state = VadState::Speaking;
self.hangover_counter = 0;
} else {
self.hangover_counter += 1;
if self.hangover_counter >= self.hangover_frames {
self.state = VadState::Silent;
}
}
}
}
self.state
}
pub fn state(&self) -> VadState {
self.state
}
pub fn is_speaking(&self) -> bool {
matches!(self.state, VadState::Speaking | VadState::Hangover)
}
pub fn set_threshold(&mut self, threshold: f32) {
self.threshold = threshold;
}
pub fn reset(&mut self) {
self.state = VadState::Silent;
self.hangover_counter = 0;
self.energy_history.clear();
}
}
src/iced-app/src/main.rs
+223 -78
View File
@@ -61,6 +61,12 @@ enum Message {
#[cfg(feature = "audio")]
PttReleased,
#[cfg(feature = "audio")]
ToggleTalkMode,
#[cfg(feature = "audio")]
StartContinuous,
#[cfg(feature = "audio")]
StopContinuous,
#[cfg(feature = "audio")]
MicSamples(Vec<f32>),
Noop,
}
@@ -151,6 +157,12 @@ struct App {
#[cfg(feature = "audio")]
ptt_active: bool,
#[cfg(feature = "audio")]
continuous_active: bool,
#[cfg(feature = "audio")]
talk_mode: audio::TalkMode,
#[cfg(feature = "audio")]
vad: Arc<Mutex<audio::VoiceActivation>>,
#[cfg(feature = "audio")]
audio_send_tx: Option<tokio::sync::mpsc::Sender<Vec<u8>>>,
}
@@ -209,6 +221,12 @@ impl App {
#[cfg(feature = "audio")]
ptt_active: false,
#[cfg(feature = "audio")]
continuous_active: false,
#[cfg(feature = "audio")]
talk_mode: audio::TalkMode::PushToTalk,
#[cfg(feature = "audio")]
vad: Arc::new(Mutex::new(audio::VoiceActivation::new(0.005))),
#[cfg(feature = "audio")]
audio_send_tx: None,
};
@@ -544,60 +562,13 @@ impl App {
}
#[cfg(feature = "audio")]
Message::PttPressed => {
if !self.ptt_active && self.connected {
if !self.ptt_active && self.connected && self.talk_mode == audio::TalkMode::PushToTalk {
self.ptt_active = true;
let mic = self.mic.clone();
let encoder = self.audio_send_tx.clone();
let (sample_tx, sample_rx) = std::sync::mpsc::channel();
let h = self.handle.clone();
tokio::spawn(async move {
{
let mut mic_guard = mic.lock().await;
if mic_guard.start(None, sample_tx).is_err() {
return;
}
}
let mut enc = match audio::OpusEncoderState::new() {
Ok(e) => e,
Err(_) => return,
};
let (audio_tx, mut audio_rx) = tokio::sync::mpsc::channel::<Vec<u8>>(100);
let send_handle = h.clone();
tokio::spawn(async move {
while let Some(data) = audio_rx.recv().await {
let mut guard = send_handle.lock().await;
if let Some(ref mut handle) = *guard {
let _ = handle
.with_connection(move |con| {
let packet = OutAudio::new(&AudioData::C2S {
id: 0,
codec: CodecType::OpusVoice,
data: &data,
});
let _ = con.send_audio(packet);
Ok::<(), ()>(())
})
.await;
}
}
});
loop {
match sample_rx.recv() {
Ok(samples) => {
enc.encode_and_send(&samples, &audio_tx);
}
Err(_) => break,
}
}
let mut mic_guard = mic.lock().await;
mic_guard.stop();
});
start_transmission(
self.mic.clone(),
self.handle.clone(),
None,
);
}
Task::none()
}
@@ -605,34 +576,65 @@ impl App {
Message::PttReleased => {
if self.ptt_active {
self.ptt_active = false;
let mic = self.mic.clone();
let h = self.handle.clone();
tokio::spawn(async move {
{
let mut mic_guard = mic.lock().await;
mic_guard.stop();
}
// Send end-of-transmission
let mut guard = h.lock().await;
if let Some(ref mut handle) = *guard {
let _ = handle
.with_connection(move |con| {
let packet = OutAudio::new(&AudioData::C2S {
id: 0,
codec: CodecType::OpusVoice,
data: &[],
});
let _ = con.send_audio(packet);
Ok::<(), ()>(())
})
.await;
}
});
stop_transmission(self.mic.clone(), self.handle.clone());
}
Task::none()
}
#[cfg(feature = "audio")]
Message::MicSamples(_) => Task::none(),
Message::ToggleTalkMode => {
self.talk_mode = match self.talk_mode {
audio::TalkMode::PushToTalk => audio::TalkMode::Continuous,
audio::TalkMode::Continuous => audio::TalkMode::PushToTalk,
};
if self.talk_mode == audio::TalkMode::PushToTalk && self.continuous_active {
self.continuous_active = false;
stop_transmission(self.mic.clone(), self.handle.clone());
}
Task::none()
}
#[cfg(feature = "audio")]
Message::StartContinuous => {
if !self.continuous_active && self.connected && self.talk_mode == audio::TalkMode::Continuous {
self.continuous_active = true;
start_transmission(
self.mic.clone(),
self.handle.clone(),
None,
);
}
Task::none()
}
#[cfg(feature = "audio")]
Message::StopContinuous => {
if self.continuous_active {
self.continuous_active = false;
stop_transmission(self.mic.clone(), self.handle.clone());
}
Task::none()
}
#[cfg(feature = "audio")]
Message::MicSamples(samples) => {
if self.talk_mode == audio::TalkMode::Continuous && self.connected {
let vad = self.vad.clone();
let was_speaking = self.continuous_active;
Task::perform(
async move {
let mut vad_guard = vad.lock().await;
let state = vad_guard.process(&samples);
let is_speaking = vad_guard.is_speaking();
drop(vad_guard);
match (was_speaking, is_speaking) {
(false, true) => Some(Message::StartContinuous),
(true, false) => Some(Message::StopContinuous),
_ => None,
}
},
|msg| msg.unwrap_or(Message::Noop),
)
} else {
Task::none()
}
}
Message::Noop => Task::none(),
}
}
@@ -834,6 +836,37 @@ impl App {
}
}),
));
// Continuous talk: monitor mic input when in continuous mode
#[cfg(feature = "audio")]
if self.talk_mode == audio::TalkMode::Continuous {
let mic = self.mic.clone();
let session_id = self.session_id;
subs.push(Subscription::run_with_id(
(session_id, "continuous"),
iced::stream::channel(100, move |mut sender| async move {
let (sample_tx, sample_rx) = std::sync::mpsc::channel();
{
let mut mic_guard = mic.lock().await;
if mic_guard.start(None, sample_tx).is_err() {
return;
}
}
loop {
match sample_rx.recv() {
Ok(samples) => {
if sender.send(Message::MicSamples(samples)).await.is_err() {
break;
}
}
Err(_) => break,
}
}
let mut mic_guard = mic.lock().await;
mic_guard.stop();
}),
));
}
}
Subscription::batch(subs)
@@ -1210,6 +1243,14 @@ impl App {
}
fn view_settings_content(&self) -> Element<'_, Message> {
#[cfg(feature = "audio")]
let talk_mode_text = match self.talk_mode {
audio::TalkMode::PushToTalk => "Push-to-Talk (hold V)",
audio::TalkMode::Continuous => "Continuous (voice activation)",
};
#[cfg(not(feature = "audio"))]
let talk_mode_text = "N/A";
container(
column![
text("Settings").size(20),
@@ -1222,8 +1263,28 @@ impl App {
text("Audio Input Device").size(13),
text("System Default").size(12).style(text::secondary),
vertical_space().height(12),
text("Voice Activation Mode").size(13),
{
#[cfg(feature = "audio")]
{
row![
text(talk_mode_text).size(12).style(text::secondary),
horizontal_space(),
button(text("Toggle").size(11))
.padding([4, 12])
.style(theme::secondary_button)
.on_press(Message::ToggleTalkMode),
]
.align_y(iced::Alignment::Center)
}
#[cfg(not(feature = "audio"))]
{
text(talk_mode_text).size(12).style(text::secondary)
}
},
vertical_space().height(12),
text("Push-to-Talk Key").size(13),
text("Not configured").size(12).style(text::secondary),
text("V (hold to talk)").size(12).style(text::secondary),
]
.spacing(4)
.padding(16),
@@ -1310,6 +1371,90 @@ async fn run_connection(con: SyncConnection, event_tx: mpsc::Sender<TsEvent>) {
let _ = event_tx.send(TsEvent::Disconnected).await;
}
#[cfg(feature = "audio")]
fn start_transmission(
mic: Arc<Mutex<audio::Microphone>>,
handle: Arc<Mutex<Option<SyncConnectionHandle>>>,
_device_name: Option<&str>,
) {
let (sample_tx, sample_rx) = std::sync::mpsc::channel();
tokio::spawn(async move {
{
let mut mic_guard = mic.lock().await;
if mic_guard.start(None, sample_tx).is_err() {
return;
}
}
let mut enc = match audio::OpusEncoderState::new() {
Ok(e) => e,
Err(_) => return,
};
let (audio_tx, mut audio_rx) = tokio::sync::mpsc::channel::<Vec<u8>>(100);
let send_handle = handle.clone();
tokio::spawn(async move {
while let Some(data) = audio_rx.recv().await {
let mut guard = send_handle.lock().await;
if let Some(ref mut h) = *guard {
let _ = h
.with_connection(move |con| {
let packet = OutAudio::new(&AudioData::C2S {
id: 0,
codec: CodecType::OpusVoice,
data: &data,
});
let _ = con.send_audio(packet);
Ok::<(), ()>(())
})
.await;
}
}
});
loop {
match sample_rx.recv() {
Ok(samples) => {
enc.encode_and_send(&samples, &audio_tx);
}
Err(_) => break,
}
}
let mut mic_guard = mic.lock().await;
mic_guard.stop();
});
}
#[cfg(feature = "audio")]
fn stop_transmission(
mic: Arc<Mutex<audio::Microphone>>,
handle: Arc<Mutex<Option<SyncConnectionHandle>>>,
) {
tokio::spawn(async move {
{
let mut mic_guard = mic.lock().await;
mic_guard.stop();
}
let mut guard = handle.lock().await;
if let Some(ref mut h) = *guard {
let _ = h
.with_connection(move |con| {
let packet = OutAudio::new(&AudioData::C2S {
id: 0,
codec: CodecType::OpusVoice,
data: &[],
});
let _ = con.send_audio(packet);
Ok::<(), ()>(())
})
.await;
}
});
}
fn nav_button(label: &str, page: Page, current: Page) -> Element<'static, Message> {
button(text(label.to_string()).size(12))
.padding([8, 16])