feat(core): A.6 — supervisor reconnect with watchdog and event stream
Adds an end-to-end auto-reconnect path so a brief network outage no longer leaves the client wedged in a half-dead state. The flow has three layers, each motivated by a real failure mode observed on the Moto G live test: * `chanora_protocol::DisconnectReason` (`UserRequested` / `StreamEnded` / `Error(String)`) is reported on a `oneshot` when the per-connection task exits, so the supervisor can tell user intent apart from a real loss. * `chanora_core` spawns a supervisor task per `ChanoraSession`. It listens for the loss notifier AND runs a watchdog that issues `snapshot()` probes every 5s with a 4s timeout — three consecutive misses synthesise a `DisconnectReason::Error(...)` and trigger the reconnect path. The watchdog catches the "ghost connected" case where tsclientlib silently resets internal state but the event stream never errors. Backoff schedule: 1s, 2s, 5s, 15s, 30s, 60s (capped). On success the supervisor swaps the dead `ProtocolClient` for the new one in place and, if audio was running, restarts the audio engine bound to the new `voice_in`/`voice_out` channels. * `SessionEvent` (Connected / Lost / Reconnecting / Disconnected / AudioStarted / AudioStopped) is broadcast on a 64-slot channel. `chanora_bridge` re-exports it as `BridgeEvent` and exposes `events_stream(StreamSink)`; the Flutter side subscribes from `initState` and renders a reconnect banner with attempt count and delay. New `SnapshotProbe` exposes a clone-friendly snapshot path so the watchdog can probe without holding `&self` across awaits. Localization adds `statusReconnecting` and `statusConnectionLost` keys to `app_en.arb` and `app_zh.arb`. Verified on Moto G Stylus 5G (Android 14) against cn.teamspeak.app: killed Wi-Fi + cellular for ~70 s; watchdog declared loss at three misses, supervisor walked the backoff schedule, and the UI reconnected automatically once the radios came back. Snapshot tree re-rendered without user action.
This commit is contained in:
@@ -68,6 +68,20 @@ enum Request {
|
||||
Disconnect(oneshot::Sender<()>),
|
||||
}
|
||||
|
||||
/// Why a [`ProtocolClient`] task ended. Distinguishes a user-driven
|
||||
/// disconnect (the supervisor must NOT retry) from a network-driven
|
||||
/// loss (the supervisor should consider retrying).
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum DisconnectReason {
|
||||
/// The caller explicitly called [`ProtocolClient::disconnect`]
|
||||
/// or dropped the handle.
|
||||
UserRequested,
|
||||
/// The underlying tsclientlib event stream ended.
|
||||
StreamEnded,
|
||||
/// A protocol-layer error caused the task to abort.
|
||||
Error(String),
|
||||
}
|
||||
|
||||
/// Async handle owning a live protocol connection. Drop = disconnect.
|
||||
pub struct ProtocolClient {
|
||||
tx: mpsc::Sender<Request>,
|
||||
@@ -78,6 +92,11 @@ pub struct ProtocolClient {
|
||||
/// Wrapped in a `Mutex<Option<_>>` so the consumer can take it
|
||||
/// exactly once.
|
||||
voice_in_rx: std::sync::Mutex<Option<mpsc::Receiver<InboundVoice>>>,
|
||||
/// Fires exactly once when the connection task exits, with the
|
||||
/// reason. Used by the supervisor in `chanora_core` to drive
|
||||
/// auto-reconnect. Wrapped in a Mutex<Option<_>> so it can be
|
||||
/// taken once by the supervisor and never resurfaced.
|
||||
lost_rx: std::sync::Mutex<Option<oneshot::Receiver<DisconnectReason>>>,
|
||||
}
|
||||
|
||||
/// One inbound voice packet from a remote client.
|
||||
@@ -88,6 +107,28 @@ pub struct InboundVoice {
|
||||
pub packet: InAudioBuf,
|
||||
}
|
||||
|
||||
/// A cheap, clone-free probe handle for the watchdog. Owns its own
|
||||
/// clone of the connection task's request channel.
|
||||
#[derive(Clone)]
|
||||
pub struct SnapshotProbe {
|
||||
tx: mpsc::Sender<Request>,
|
||||
}
|
||||
|
||||
impl SnapshotProbe {
|
||||
/// Issue a single snapshot RPC. Returns the same error shape as
|
||||
/// [`ProtocolClient::snapshot`]. Suitable for use under a
|
||||
/// `tokio::time::timeout`.
|
||||
pub async fn probe(&self) -> Result<ServerSnapshot, ProtocolError> {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
self.tx
|
||||
.send(Request::Snapshot(tx))
|
||||
.await
|
||||
.map_err(|_| ProtocolError::Lost("connection task is gone".to_string()))?;
|
||||
rx.await
|
||||
.map_err(|_| ProtocolError::Lost("snapshot reply dropped".to_string()))?
|
||||
}
|
||||
}
|
||||
|
||||
impl ProtocolClient {
|
||||
/// Dial the server and wait for the initial state snapshot. The
|
||||
/// returned client is ready for [`Self::snapshot`] and
|
||||
@@ -104,6 +145,7 @@ impl ProtocolClient {
|
||||
let (voice_out_tx, voice_out_rx) = mpsc::channel::<OutPacket>(64);
|
||||
let (voice_in_tx, voice_in_rx) = mpsc::channel::<InboundVoice>(64);
|
||||
let (ready_tx, ready_rx) = oneshot::channel::<Result<(), ProtocolError>>();
|
||||
let (lost_tx, lost_rx) = oneshot::channel::<DisconnectReason>();
|
||||
|
||||
tokio::spawn(connection_task(
|
||||
cfg.clone(),
|
||||
@@ -111,6 +153,7 @@ impl ProtocolClient {
|
||||
voice_out_rx,
|
||||
voice_in_tx,
|
||||
ready_tx,
|
||||
lost_tx,
|
||||
));
|
||||
|
||||
match tokio::time::timeout(cfg.ready_timeout, ready_rx).await {
|
||||
@@ -118,6 +161,7 @@ impl ProtocolClient {
|
||||
tx,
|
||||
voice_out_tx,
|
||||
voice_in_rx: std::sync::Mutex::new(Some(voice_in_rx)),
|
||||
lost_rx: std::sync::Mutex::new(Some(lost_rx)),
|
||||
}),
|
||||
Ok(Ok(Err(e))) => Err(e),
|
||||
Ok(Err(_)) => Err(ProtocolError::Backend(
|
||||
@@ -151,11 +195,28 @@ impl ProtocolClient {
|
||||
self.voice_out_tx.clone()
|
||||
}
|
||||
|
||||
/// Clone the request channel so a watchdog can issue probes
|
||||
/// without holding a `&self` reference across the await. The
|
||||
/// returned [`SnapshotProbe`] is `Send + 'static` and dispatches
|
||||
/// a single snapshot RPC against this protocol task.
|
||||
pub fn snapshot_probe(&self) -> SnapshotProbe {
|
||||
SnapshotProbe {
|
||||
tx: self.tx.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Take the inbound-voice receiver. Returns `None` if it has
|
||||
/// already been taken; only one consumer is allowed.
|
||||
pub fn take_voice_in(&self) -> Option<mpsc::Receiver<InboundVoice>> {
|
||||
self.voice_in_rx.lock().ok().and_then(|mut g| g.take())
|
||||
}
|
||||
|
||||
/// Take the loss-notifier. Returns `None` if it has already been
|
||||
/// taken. The supervisor in `chanora_core` consumes this to
|
||||
/// drive auto-reconnect; nothing else should call it.
|
||||
pub fn take_loss_notifier(&self) -> Option<oneshot::Receiver<DisconnectReason>> {
|
||||
self.lost_rx.lock().ok().and_then(|mut g| g.take())
|
||||
}
|
||||
}
|
||||
|
||||
async fn connection_task(
|
||||
@@ -164,7 +225,20 @@ async fn connection_task(
|
||||
mut voice_out_rx: mpsc::Receiver<OutPacket>,
|
||||
voice_in_tx: mpsc::Sender<InboundVoice>,
|
||||
ready_tx: oneshot::Sender<Result<(), ProtocolError>>,
|
||||
lost_tx: oneshot::Sender<DisconnectReason>,
|
||||
) {
|
||||
// Box the lost_tx so each exit branch can move it.
|
||||
let mut lost_tx = Some(lost_tx);
|
||||
// Macro: report the disconnect reason and return from the task.
|
||||
macro_rules! exit {
|
||||
($reason:expr) => {{
|
||||
if let Some(tx) = lost_tx.take() {
|
||||
let _ = tx.send($reason);
|
||||
}
|
||||
return;
|
||||
}};
|
||||
}
|
||||
|
||||
// Resolve the hostname OURSELVES using the platform resolver.
|
||||
// tsclientlib's built-in hickory-resolver reads /etc/resolv.conf,
|
||||
// which does not exist on Android or iOS — by side-stepping it
|
||||
@@ -172,8 +246,9 @@ async fn connection_task(
|
||||
let addrs = match crate::resolver::resolve(&cfg.address).await {
|
||||
Ok(a) => a,
|
||||
Err(e) => {
|
||||
let msg = format!("{e}");
|
||||
let _ = ready_tx.send(Err(e));
|
||||
return;
|
||||
exit!(DisconnectReason::Error(msg));
|
||||
}
|
||||
};
|
||||
// Pick the first address (IPv4 preferred by the resolver's
|
||||
@@ -196,8 +271,9 @@ async fn connection_task(
|
||||
Some(s) => match Identity::new_from_str(s) {
|
||||
Ok(id) => id,
|
||||
Err(e) => {
|
||||
let _ = ready_tx.send(Err(ProtocolError::Identity(format!("{e}"))));
|
||||
return;
|
||||
let msg = format!("{e}");
|
||||
let _ = ready_tx.send(Err(ProtocolError::Identity(msg.clone())));
|
||||
exit!(DisconnectReason::Error(format!("identity: {msg}")));
|
||||
}
|
||||
},
|
||||
None => Identity::create(),
|
||||
@@ -211,8 +287,9 @@ async fn connection_task(
|
||||
let mut con = match builder.connect() {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
let _ = ready_tx.send(Err(ProtocolError::Connect(format!("{e}"))));
|
||||
return;
|
||||
let msg = format!("{e}");
|
||||
let _ = ready_tx.send(Err(ProtocolError::Connect(msg.clone())));
|
||||
exit!(DisconnectReason::Error(format!("connect: {msg}")));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -227,14 +304,14 @@ async fn connection_task(
|
||||
info!(target: "chanora_protocol", "initial state snapshot received");
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
let _ = ready_tx.send(Err(ProtocolError::DisconnectedEarly(format!("{e}"))));
|
||||
return;
|
||||
let msg = format!("{e}");
|
||||
let _ = ready_tx.send(Err(ProtocolError::DisconnectedEarly(msg.clone())));
|
||||
exit!(DisconnectReason::Error(format!("disconnected early: {msg}")));
|
||||
}
|
||||
None => {
|
||||
let _ = ready_tx.send(Err(ProtocolError::DisconnectedEarly(
|
||||
"event stream ended before snapshot".to_string(),
|
||||
)));
|
||||
return;
|
||||
let msg = "event stream ended before snapshot".to_string();
|
||||
let _ = ready_tx.send(Err(ProtocolError::DisconnectedEarly(msg.clone())));
|
||||
exit!(DisconnectReason::Error(msg));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -258,10 +335,9 @@ async fn connection_task(
|
||||
warn!(target: "chanora_protocol", error = %e, "event error during settle");
|
||||
}
|
||||
Ok(None) => {
|
||||
let _ = ready_tx.send(Err(ProtocolError::DisconnectedEarly(
|
||||
"stream closed during settle".to_string(),
|
||||
)));
|
||||
return;
|
||||
let msg = "stream closed during settle".to_string();
|
||||
let _ = ready_tx.send(Err(ProtocolError::DisconnectedEarly(msg.clone())));
|
||||
exit!(DisconnectReason::StreamEnded);
|
||||
}
|
||||
Err(_) => { /* no event available right now; keep waiting */ }
|
||||
}
|
||||
@@ -303,10 +379,12 @@ async fn connection_task(
|
||||
}
|
||||
Ok(Some(Err(e))) => {
|
||||
warn!(target: "chanora_protocol", error = %e, "event error");
|
||||
// Some errors are transient; treat persistent ones
|
||||
// as a loss after the next iteration.
|
||||
}
|
||||
Ok(None) => {
|
||||
warn!(target: "chanora_protocol", "event stream ended");
|
||||
return;
|
||||
exit!(DisconnectReason::StreamEnded);
|
||||
}
|
||||
Err(_) => { /* no event in 20 ms */ }
|
||||
}
|
||||
@@ -322,14 +400,14 @@ async fn connection_task(
|
||||
con.events().for_each(|_| future::ready(())).await;
|
||||
let _ = reply.send(());
|
||||
info!(target: "chanora_protocol", "clean disconnect");
|
||||
return;
|
||||
exit!(DisconnectReason::UserRequested);
|
||||
}
|
||||
Err(mpsc::error::TryRecvError::Empty) => {}
|
||||
Err(mpsc::error::TryRecvError::Disconnected) => {
|
||||
let _ = con.disconnect(DisconnectOptions::new());
|
||||
con.events().for_each(|_| future::ready(())).await;
|
||||
info!(target: "chanora_protocol", "handle dropped; implicit disconnect");
|
||||
return;
|
||||
exit!(DisconnectReason::UserRequested);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,7 +40,7 @@ mod adapter;
|
||||
mod dto;
|
||||
mod resolver;
|
||||
|
||||
pub use adapter::{ConnectConfig, InboundVoice, ProtocolClient};
|
||||
pub use adapter::{ConnectConfig, DisconnectReason, InboundVoice, ProtocolClient, SnapshotProbe};
|
||||
pub use dto::{ChannelInfo, ClientInfo, ServerSnapshot};
|
||||
|
||||
// Re-export the upstream voice types so chanora_audio can build outbound
|
||||
|
||||
Reference in New Issue
Block a user