feat(voice): add iOS VAD runtime support

This commit is contained in:
Edison Jwa
2026-05-21 20:51:45 +09:00
parent 171baf6e41
commit 6af4ecab0f
73 changed files with 11529 additions and 1249 deletions
+261 -193
View File
@@ -9,9 +9,9 @@
// (all carried over from v0.3.0-beta.1)
import 'dart:async';
import 'dart:io' show File, Platform, Process;
import 'package:connectivity_plus/connectivity_plus.dart';
import 'dart:io' show Platform, Process;
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
@@ -21,18 +21,54 @@ import 'package:path_provider/path_provider.dart';
import 'l10n/generated/app_localizations.dart';
import 'services/android_permissions_service.dart';
import 'services/ios_permissions_service.dart';
import 'src/rust/api.dart' as rust;
import 'src/rust/lib.dart' as rust_err;
import 'src/rust/frb_generated.dart';
import 'widgets/permission_state_banner.dart';
import 'widgets/voice_platform.dart';
import 'widgets/voice_bar.dart';
import 'widgets/voice_compact.dart';
import 'widgets/voice_settings.dart';
bool get _isMacOS => !kIsWeb && Platform.isMacOS;
const MethodChannel _iosPlatformChannel = MethodChannel('chanora/ios_platform');
const Color _appSurfaceColor = Color(0xFFFFFBFE);
const String _sileroVadAsset = 'assets/models/silero_vad.onnx';
const String _tenVadAsset = 'assets/models/ten_vad.onnx';
Future<File> _copyBundledAssetToDocuments({
required String assetPath,
required String fileName,
}) async {
final dir = await getApplicationDocumentsDirectory();
final file = File('${dir.path}/$fileName');
final data = await rootBundle.load(assetPath);
final bytes = data.buffer.asUint8List(data.offsetInBytes, data.lengthInBytes);
if (await file.exists() && await file.length() == bytes.length) {
return file;
}
await file.writeAsBytes(bytes, flush: true);
return file;
}
Future<void> _configureBundledVadModels() async {
final silero = await _copyBundledAssetToDocuments(
assetPath: _sileroVadAsset,
fileName: 'silero_vad.onnx',
);
await _copyBundledAssetToDocuments(
assetPath: _tenVadAsset,
fileName: 'ten_vad.onnx',
);
await rust.setVadModelPath(path: silero.path);
}
/// Top padding for macOS to clear traffic-light buttons.
const double _macOSTrafficLightPad = 56.0;
@@ -78,15 +114,6 @@ String? _pttDisplayLabelForKey(LogicalKeyboardKey k) {
return fallback;
}
/// True when the host is a touch-only mobile platform without a
/// hardware keyboard. Mirrors the helpers in widgets/voice_bar.dart
/// and widgets/voice_settings.dart so the AppBar + narrow-mode
/// layout in main.dart can branch consistently.
bool get _isTouchOnlyPttHost {
if (kIsWeb) return false;
return Platform.isIOS || Platform.isAndroid;
}
/// Public version string shown in the About dialog. Resolved at
/// app init by combining a hardcoded semver baseline (kept in sync
/// with the git tag and pubspec.yaml's `version:` field) with the
@@ -124,6 +151,23 @@ Future<void> main() async {
/// Wire the iOS AVAudioSession lifecycle MethodChannel.
///
rust.BridgeAudioRoute _parseBridgeAudioRoute(String s) {
switch (s) {
case 'Earpiece':
return rust.BridgeAudioRoute.earpiece;
case 'Speaker':
return rust.BridgeAudioRoute.speaker;
case 'WiredHeadset':
return rust.BridgeAudioRoute.wiredHeadset;
case 'BluetoothHfp':
return rust.BridgeAudioRoute.bluetoothHfp;
case 'BluetoothA2dp':
return rust.BridgeAudioRoute.bluetoothA2Dp;
default:
return rust.BridgeAudioRoute.unknown;
}
}
/// Swift side (AppDelegate) posts route-change and interruption
/// events through `FlutterMethodChannel` named
/// `"chanora/ios_audio_lifecycle"`. This handler dispatches them to
@@ -134,7 +178,12 @@ void _wireIosAudioLifecycle() {
try {
switch (call.method) {
case 'handleRouteChange':
rust.handleRouteChange();
final routeStr = call.arguments as String? ?? 'Unknown';
final route = _parseBridgeAudioRoute(routeStr);
rust.handleRouteChange(route: route);
break;
case 'handleMediaServicesReset':
rust.handleMediaServicesReset();
break;
case 'handleInterruptionBegan':
rust.handleInterruptionBegan();
@@ -144,6 +193,16 @@ void _wireIosAudioLifecycle() {
final shouldResume = call.arguments as bool? ?? false;
rust.handleInterruptionEnded(shouldResume: shouldResume);
break;
case 'handleWillResignActive':
case 'handleDidEnterBackground':
rust.handleInterruptionBegan();
break;
case 'handleWillEnterForeground':
rust.handleInterruptionEnded(shouldResume: true);
break;
case 'handleWillTerminate':
rust.handleInterruptionBegan();
break;
default:
// Unknown method — ignore gracefully rather than crashing.
break;
@@ -315,6 +374,7 @@ class _BetaHomeState extends State<_BetaHome> {
// (see AndroidPermissionsService for the platform branch).
final AndroidPermissionsService _androidPermissions =
AndroidPermissionsService();
final IosPermissionsService _iosPermissions = IosPermissionsService();
@override
void initState() {
@@ -325,9 +385,13 @@ class _BetaHomeState extends State<_BetaHome> {
// events as early as possible so the listen-only banner reflects
// the system state on first frame.
_androidPermissions.start();
unawaited(_iosPermissions.start());
_androidPermissions.recordAudioState.addListener(
_onRecordAudioPermissionChanged,
);
_iosPermissions.recordAudioState.addListener(
_onRecordAudioPermissionChanged,
);
WidgetsBinding.instance.addPostFrameCallback((_) {
unawaited(_requestRecordAudioOnStartup());
});
@@ -337,15 +401,34 @@ class _BetaHomeState extends State<_BetaHome> {
Future<void> _requestRecordAudioOnStartup() async {
try {
await _androidPermissions.ensureRecordAudio();
if (Platform.isAndroid) {
await _androidPermissions.ensureRecordAudio();
}
} catch (_) {
// Best-effort startup prompt only. The join path still gates on
// ensureRecordAudio() and applies the listen-only hard-mute policy.
}
}
ValueListenable<AndroidRecordAudioPermissionState>
get _activeRecordAudioState => Platform.isIOS
? _iosPermissions.recordAudioState
: _androidPermissions.recordAudioState;
Future<AndroidRecordAudioPermissionState> _ensureActiveRecordAudio() {
return Platform.isIOS
? _iosPermissions.ensureRecordAudio()
: _androidPermissions.ensureRecordAudio();
}
Future<void> _openActivePermissionSettings() {
return Platform.isIOS
? _iosPermissions.openAppSettings()
: _androidPermissions.openAppSettings();
}
void _onRecordAudioPermissionChanged() {
if (_androidPermissions.recordAudioState.value ==
if (_activeRecordAudioState.value ==
AndroidRecordAudioPermissionState.granted) {
unawaited(_clearPermissionHardMute());
}
@@ -528,16 +611,16 @@ class _BetaHomeState extends State<_BetaHome> {
final messenger = ScaffoldMessenger.of(context);
if (began) {
messenger.showSnackBar(
const SnackBar(
content: Text('Audio interrupted by system (phone call)'),
SnackBar(
content: Text(AppL10n.of(context).iosAudioInterrupted),
duration: Duration(seconds: 3),
backgroundColor: Colors.orange,
),
);
} else if (shouldResume) {
messenger.showSnackBar(
const SnackBar(
content: Text('Audio resuming'),
SnackBar(
content: Text(AppL10n.of(context).iosAudioResuming),
duration: Duration(seconds: 2),
backgroundColor: Colors.green,
),
@@ -612,10 +695,14 @@ class _BetaHomeState extends State<_BetaHome> {
_androidPermissions.recordAudioState.removeListener(
_onRecordAudioPermissionChanged,
);
_iosPermissions.recordAudioState.removeListener(
_onRecordAudioPermissionChanged,
);
// SDD-106: detach the Kotlin -> Dart MethodChannel handler so a
// late invokeMethod from the platform side cannot land on this
// disposed state.
_androidPermissions.stop();
_iosPermissions.stop();
super.dispose();
}
@@ -670,6 +757,14 @@ class _BetaHomeState extends State<_BetaHome> {
: l10n.microphonePermissionBody,
),
actions: [
if (Platform.isIOS && !isNetwork)
TextButton(
onPressed: () {
Navigator.pop(ctx);
unawaited(_openIosAppSettings());
},
child: Text(l10n.networkPermissionOpenSettings),
),
if (Platform.isMacOS)
TextButton(
onPressed: () {
@@ -691,6 +786,15 @@ class _BetaHomeState extends State<_BetaHome> {
);
}
Future<void> _openIosAppSettings() async {
try {
await _iosPlatformChannel.invokeMethod<bool>('openAppSettings');
} catch (_) {
// Best-effort affordance only; if iOS refuses the URL, the
// dialog still explained the missing microphone permission.
}
}
// ignore: unused_element
Future<void> _setPtt(bool active, {bool reportError = true}) async {
try {
@@ -718,13 +822,9 @@ class _BetaHomeState extends State<_BetaHome> {
final next = !_outputMuted;
try {
await rust.setOutputMuted(muted: next);
await rust.setHardMute(
muted: next || _inputMuted || _hardMuteByPermission,
);
if (!mounted) return;
setState(() {
_outputMuted = next;
_hardMute = next || _inputMuted || _hardMuteByPermission;
});
} catch (e) {
if (!mounted) return;
@@ -770,7 +870,11 @@ class _BetaHomeState extends State<_BetaHome> {
// Trace: SDD-106 §1 (request timing), §2 (listen-only on denial),
// §3 (path to settings on permanent denial), §6
// (TransmitModeSelector clamp); SRS-209.
final permState = await _androidPermissions.ensureRecordAudio();
final permState = Platform.isAndroid
? await _androidPermissions.ensureRecordAudio()
: Platform.isIOS
? await _iosPermissions.ensureRecordAudio()
: AndroidRecordAudioPermissionState.granted;
if (permState != AndroidRecordAudioPermissionState.granted) {
// Listen-only: clamp hard-mute. The permission_state_banner
// surfaces the path-to-grant; the user can re-attempt at any
@@ -800,6 +904,7 @@ class _BetaHomeState extends State<_BetaHome> {
});
}
}
await _configureBundledVadModels();
await rust.voiceJoin(channelId: ch.id, password: password ?? '');
if (!mounted) return;
unawaited(_onRefresh());
@@ -930,19 +1035,42 @@ class _BetaHomeState extends State<_BetaHome> {
/// Narrow-mode voice controls modal sheet (Plan E status chip
/// trigger). On mobile this is the **single** voice-controls
/// surface: route picker + inline mode radio + inline release-tail
/// slider + level meter + stats + (desktop-only) capability badge.
/// Zero navigation depth \u2014 no nested dialog.
/// slider + level meter + stats + audio processing + (desktop-only)
/// capability badge. Zero navigation depth no nested dialog.
Future<void> _onOpenVoiceDetailsSheet() async {
// Load current audio processing config for the sheet.
rust.BridgeAudioProcessingConfig audioConfig;
try {
audioConfig = await rust.getAudioProcessingConfig();
} catch (_) {
audioConfig = const rust.BridgeAudioProcessingConfig(
route: rust.BridgeAudioRoute.unknown,
iosMode: rust.BridgeIosVoiceProcessingMode.platformVoiceProcessing,
processingBackend: rust.BridgeAudioBackend.platformVoiceProcessing,
vadBackend: rust.BridgeVadBackend.sileroOnnx,
aec: rust.BridgeEffectOwner.platform,
ns: rust.BridgeEffectOwner.platform,
agc: rust.BridgeEffectOwner.platform,
hpfEnabled: true,
limiterEnabled: true,
vadHangoverMs: 500,
vadPreRollMs: 160,
vadMinTxMs: 200,
debugWavDumpEnabled: false,
);
}
if (!mounted) return;
await showVoiceDetailsSheet(
context,
audioStats: _audioStats,
transmitMode: _transmitMode,
releaseTailMs: _releaseTailMs,
pttBoundKeyLabel: _pttBoundKeyLabel,
pttLevel: _pttLevel,
pttBackendId: _pttBackendId,
pttBoundInputClass: _pttBoundInputClass,
isTouchOnly: _isTouchOnlyPttHost,
isTouchOnly: isTouchOnlyPttHost,
initialAudioConfig: audioConfig,
onModeChanged: (mode) async {
try {
await rust.setTransmitMode(mode: mode);
@@ -963,21 +1091,55 @@ class _BetaHomeState extends State<_BetaHome> {
setState(() => _error = e.toString());
}
},
onAudioConfigChanged: (config) async {
try {
await rust.setAudioProcessingConfig(config: config);
} catch (e) {
if (!mounted) return;
setState(() => _error = e.toString());
}
},
);
}
Future<void> _onOpenVoiceSettings() async {
// Load the current audio processing config before opening the dialog.
rust.BridgeAudioProcessingConfig audioConfig;
try {
audioConfig = await rust.getAudioProcessingConfig();
} catch (_) {
// If not connected yet, use a sensible default.
audioConfig = const rust.BridgeAudioProcessingConfig(
route: rust.BridgeAudioRoute.unknown,
iosMode: rust.BridgeIosVoiceProcessingMode.platformVoiceProcessing,
processingBackend: rust.BridgeAudioBackend.platformVoiceProcessing,
vadBackend: rust.BridgeVadBackend.sileroOnnx,
aec: rust.BridgeEffectOwner.platform,
ns: rust.BridgeEffectOwner.platform,
agc: rust.BridgeEffectOwner.platform,
hpfEnabled: true,
limiterEnabled: true,
vadHangoverMs: 500,
vadPreRollMs: 160,
vadMinTxMs: 200,
debugWavDumpEnabled: false,
);
}
if (!mounted) return;
final result = await showDialog<VoiceSettingsResult>(
context: context,
builder: (ctx) => VoiceSettingsDialog(
initialMode: _transmitMode,
initialReleaseTailMs: _releaseTailMs,
initialAudioConfig: audioConfig,
),
);
if (result == null) return;
try {
await rust.setTransmitMode(mode: result.mode);
await rust.setReleaseTailMs(ms: result.releaseTailMs);
await rust.setAudioProcessingConfig(config: result.audioConfig);
} catch (e) {
if (!mounted) return;
setState(() => _error = e.toString());
@@ -1083,6 +1245,27 @@ class _BetaHomeState extends State<_BetaHome> {
),
),
actions: [
TextButton(
onPressed: () async {
try {
final path = await _writeDiagnosticExport(text);
if (!ctx.mounted) return;
Navigator.of(ctx).pop();
if (!mounted) return;
ScaffoldMessenger.of(this.context).showSnackBar(
SnackBar(content: Text(l10n.diagnosticsSaved(path))),
);
} catch (e) {
if (!ctx.mounted) return;
Navigator.of(ctx).pop();
if (!mounted) return;
ScaffoldMessenger.of(this.context).showSnackBar(
SnackBar(content: Text(l10n.statusError(e.toString()))),
);
}
},
child: Text(l10n.diagnosticsSaveAction),
),
TextButton(
onPressed: () async {
await Clipboard.setData(ClipboardData(text: text));
@@ -1100,6 +1283,18 @@ class _BetaHomeState extends State<_BetaHome> {
);
}
Future<String> _writeDiagnosticExport(String text) async {
final dir = await getApplicationDocumentsDirectory();
final stamp = DateTime.now()
.toUtc()
.toIso8601String()
.replaceAll(':', '-')
.replaceAll('.', '-');
final file = File('${dir.path}/chanora-diagnostics-$stamp.txt');
await file.writeAsString(text, flush: true);
return file.path;
}
Future<void> _onConfigurePtt(BuildContext context) async {
// On the Linux GNOME-Wayland portal backend, the portal hosts
// its own system-managed binding dialog (gen2 v0.9.3 / Q3a).
@@ -1399,26 +1594,33 @@ class _BetaHomeState extends State<_BetaHome> {
const SizedBox(height: 12),
if (_phase == _Phase.idle) ...[
Expanded(
child: SingleChildScrollView(
keyboardDismissBehavior:
ScrollViewKeyboardDismissBehavior.onDrag,
child: Column(
crossAxisAlignment: CrossAxisAlignment.stretch,
children: [
_ConnectForm(
hostCtl: _hostCtl,
nickCtl: _nickCtl,
passwordCtl: _passwordCtl,
onConnect: () => _onConnect(),
onAddBookmark: _onAddCurrentBookmark,
),
const SizedBox(height: 16),
_BookmarkList(
bookmarks: _bookmarks,
onConnect: _onUseBookmark,
onDelete: _onDeleteBookmark,
),
],
child: AnimatedPadding(
duration: const Duration(milliseconds: 180),
curve: Curves.easeOut,
padding: EdgeInsets.only(
bottom: MediaQuery.viewInsetsOf(ctx).bottom,
),
child: SingleChildScrollView(
keyboardDismissBehavior:
ScrollViewKeyboardDismissBehavior.onDrag,
child: Column(
crossAxisAlignment: CrossAxisAlignment.stretch,
children: [
_ConnectForm(
hostCtl: _hostCtl,
nickCtl: _nickCtl,
passwordCtl: _passwordCtl,
onConnect: () => _onConnect(),
onAddBookmark: _onAddCurrentBookmark,
),
const SizedBox(height: 16),
_BookmarkList(
bookmarks: _bookmarks,
onConnect: _onUseBookmark,
onDelete: _onDeleteBookmark,
),
],
),
),
),
),
@@ -1450,11 +1652,14 @@ class _BetaHomeState extends State<_BetaHome> {
onConfigure: _onOpenVoiceSettings,
onPttHeldChanged: _onOnscreenPttHeldChanged,
);
// SDD-106 §2/§3 + SRS-209: listen-only banner.
// Self-hides on granted / unknown / non-Android.
final permissionBanner = PermissionStateBanner(
service: _androidPermissions,
);
// SDD-106 §2/§3 + SRS-209 + SRS-164: listen-only
// banner. Self-hides on granted / unknown.
final permissionBanner =
PermissionStateBanner.fromCallbacks(
recordAudioState: _activeRecordAudioState,
ensureRecordAudio: _ensureActiveRecordAudio,
openAppSettings: _openActivePermissionSettings,
);
final snapshotView = _SnapshotView(
snapshot: _snapshot!,
audioStats: _audioStats,
@@ -1501,7 +1706,7 @@ class _BetaHomeState extends State<_BetaHome> {
releaseTailMs: _releaseTailMs,
pttBoundKeyLabel: _pttBoundKeyLabel,
audioStats: _audioStats,
isTouchOnly: _isTouchOnlyPttHost,
isTouchOnly: isTouchOnlyPttHost,
onTap: () => _onOpenVoiceDetailsSheet(),
),
if (_inChannel &&
@@ -1552,7 +1757,10 @@ class _BetaHomeState extends State<_BetaHome> {
return Scaffold(
appBar: AppBar(title: headerTitle, actions: headerActions),
body: Padding(padding: const EdgeInsets.all(16), child: bodyContent),
body: SafeArea(
top: false,
child: Padding(padding: const EdgeInsets.all(16), child: bodyContent),
),
);
}
}
@@ -1961,146 +2169,6 @@ class _BookmarkList extends StatelessWidget {
/// Driven by the `BridgeEvent::PttCapability` stream published by
/// the `PttController` (SDD-088). The `_BetaHomeState` listener
/// updates the props on each transition.
class PttCapabilityBadge extends StatelessWidget {
/// Construct a badge.
const PttCapabilityBadge({
super.key,
required this.level,
required this.backendId,
required this.boundInputClass,
});
/// Resolved capability level as the bridge emits it
/// (`L0Focused` / `L1WindowsHook` / `L2WindowsRawInput` /
/// `L1MacOSEventTap` / `L1LinuxGnomeWaylandPortal`).
final String level;
/// Stable backend identifier (`focused`, `windows-raw-input`, …).
final String backendId;
/// Privacy-safe input class (`keyboard`, `mouse-side-button`,
/// or empty when no binding is set).
final String boundInputClass;
bool get _isFocused => level == 'L0Focused';
String _explainBodyForPlatform(AppL10n l10n) {
// Use `defaultTargetPlatform` rather than `Theme.of(context).platform`
// because the latter is influenced by debug platform overrides
// that callers may toggle in dev mode. We want the badge's
// explanation to match the actual host OS.
switch (defaultTargetPlatform) {
case TargetPlatform.windows:
return l10n.pttCapabilityExplainGoGlobalWindows;
case TargetPlatform.macOS:
return l10n.pttCapabilityExplainGoGlobalMacos;
case TargetPlatform.linux:
return l10n.pttCapabilityExplainGoGlobalLinux;
case TargetPlatform.iOS:
return l10n.pttCapabilityExplainGoGlobalIos;
default:
return l10n.pttCapabilityExplainGoGlobalGeneric;
}
}
void _openExplanationSheet(BuildContext context) {
final l10n = AppL10n.of(context);
showModalBottomSheet<void>(
context: context,
showDragHandle: true,
builder: (sheetContext) {
final theme = Theme.of(sheetContext);
return SafeArea(
child: Padding(
padding: const EdgeInsets.fromLTRB(20, 4, 20, 24),
child: Column(
mainAxisSize: MainAxisSize.min,
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text(
l10n.pttCapabilityExplainTitle,
style: theme.textTheme.titleMedium,
),
const SizedBox(height: 12),
Text(
l10n.pttCapabilityExplainFocusedHeading,
style: theme.textTheme.titleSmall,
),
const SizedBox(height: 4),
Text(
l10n.pttCapabilityExplainFocusedBody,
style: theme.textTheme.bodyMedium,
),
const SizedBox(height: 16),
Text(
_explainBodyForPlatform(l10n),
style: theme.textTheme.bodyMedium,
),
const SizedBox(height: 16),
Align(
alignment: AlignmentDirectional.centerEnd,
child: TextButton(
onPressed: () => Navigator.of(sheetContext).pop(),
child: Text(l10n.closeAction),
),
),
],
),
),
);
},
);
}
@override
Widget build(BuildContext context) {
final l10n = AppL10n.of(context);
final theme = Theme.of(context);
final badgeLabel = l10n.pttCapabilityBadge(level, backendId);
final tooltipMessage = boundInputClass.isEmpty
? badgeLabel
: '$badgeLabel\n($boundInputClass)';
return Padding(
padding: const EdgeInsets.only(bottom: 6),
child: Tooltip(
message: tooltipMessage,
child: Row(
children: [
Icon(
_isFocused ? Icons.crop_free : Icons.public,
size: 14,
color: theme.colorScheme.onSurfaceVariant,
),
const SizedBox(width: 4),
Expanded(
child: Text(
badgeLabel,
style: theme.textTheme.bodySmall?.copyWith(
color: theme.colorScheme.onSurfaceVariant,
),
),
),
// Info icon only for L0Focused — the explanation sheet
// tells the user why their PTT may not work outside the
// app window and how to grant the permission. There is
// intentionally NO 'Configure' button here: the single
// configuration entry point is the Voice Bar's
// settings gear (onConfigure on `VoiceBar`). Having two
// identical bind-key entry points just confuses users.
if (_isFocused)
IconButton(
icon: const Icon(Icons.info_outline, size: 16),
tooltip: l10n.pttCapabilityExplainTitle,
visualDensity: VisualDensity.compact,
onPressed: () => _openExplanationSheet(context),
),
],
),
),
);
}
}
class _SnapshotView extends StatelessWidget {
const _SnapshotView({
required this.snapshot,