feat: Android Oboe voice backend — WebRTC APM, VAD, HW/SW toggle, BBCode welcome, link trust, foreground task

Audio engine (Rust):
- Android Oboe: WebRTC APM (AEC/NS/AGC/HPF) + TEN/Silero ONNX VAD
- Hardware effects (JNI) with software fallback per-effect
- Render reference buffer for AEC between output/capture callbacks
- Voice activity gate: suppress transmission when speaker muted (all platforms)
- Audio focus (SDD-109) + Bluetooth SCO (SDD-110) via JNI
- ONNX Runtime 1.26 via ort 2.0.0-rc.12 (down from rc.10, ndarray 0.17)
- VAD worker channel capacity 8→32, initial seq u64::MAX (warm-up fix)
- TEN VAD default backend (was Silero)
- Platform→WebrtcApm resolution after hardware binding
- oboe-rs edisonjwa fork with get_raw_session_id()

Android Kotlin:
- AndroidAudioFocusController + AndroidBluetoothScoController
- AndroidAudioLifecycleController (route changes to Flutter)
- ProGuard rules for new controllers

Flutter UI:
- VoiceSettings: Android HW/SW toggle (Platform auto / WebRTC APM)
- VoiceStatusChip: mute warning border + Speaker muted label
- BBCode welcome message parser (BbCodeText, case-insensitive)
- Welcome message foldable (expanded by default)
- Link trust dialog (domain wildcards, SharedPreferences)
- HapticFeedback on voice sheet opener
- Server name in AppBar, version v0.1.0
- Default channel (id=1) visible, serverquery clients hidden
- flutter_foreground_task integration

Config:
- ort load-dynamic on all non-iOS (Android/Linux/Windows)
- ONNX Runtime AAR 1.26.0
- ndarray moved to common deps (was Apple-only)
This commit is contained in:
Edison Jwa
2026-05-22 09:29:57 +09:00
parent 6af4ecab0f
commit bf284018e6
37 changed files with 3676 additions and 703 deletions
@@ -0,0 +1,313 @@
import 'package:flutter/material.dart';
import 'package:url_launcher/url_launcher.dart';
import '../services/link_trust_service.dart';
final _tagRe = RegExp(
r'\[(\/?(?:b|i|u|s'
r'|color(?:=[^\]]*)?'
r'|size(?:=\d+)?'
r'|url(?:=[^\]]*)?'
r'|img(?:=[^\]]*)?'
r'|list|\*|quote|code|center|left|right'
r'))\]',
caseSensitive: false,
);
final _colorRe = RegExp(r'color=([#\w]+)');
final _sizeRe = RegExp(r'size=(\d+)');
final _urlRe = RegExp(r'url=(.+)');
final _imgRe = RegExp(r'img=(.+)');
final _urlAutoRe = RegExp(r'(?:\[url\])?(https?://[^\s\[\]]+)(?:\[/url\])?', caseSensitive: false);
final _closeUrlRe = RegExp(r'\[/url\]', caseSensitive: false);
int? _findCloseUrl(String src, int from) {
final m = _closeUrlRe.matchAsPrefix(src, from);
if (m != null) return from;
final idx = src.indexOf('[/url]', from);
if (idx >= 0) return idx;
final idxu = src.indexOf('[/URL]', from);
if (idxu >= 0) return idxu;
return null;
}
Color? _parseColor(String hex) {
try {
var h = hex.replaceFirst('#', '');
if (h.length == 6) h = 'FF$h';
if (h.length == 8) {
return Color(int.parse(h, radix: 16));
}
} catch (_) {}
return null;
}
class BbCodeText extends StatelessWidget {
const BbCodeText(this.text, {super.key, required this.linkTrust});
final String text;
final LinkTrustService linkTrust;
@override
Widget build(BuildContext context) {
if (!text.contains('[') || !text.contains(']')) {
return _plainWithAutoLinks(context, text);
}
return _render(context, text);
}
Widget _plainWithAutoLinks(BuildContext context, String src) {
final parts = <InlineSpan>[];
int last = 0;
for (final m in _urlAutoRe.allMatches(src)) {
if (m.start > last) {
parts.add(TextSpan(text: src.substring(last, m.start)));
}
final url = m.group(1)!;
parts.add(WidgetSpan(
alignment: PlaceholderAlignment.middle,
child: _LinkTap(
url: url,
linkTrust: linkTrust,
child: Text(
url,
style: const TextStyle(
color: Colors.blue,
decoration: TextDecoration.underline,
),
),
),
));
last = m.end;
}
if (last < src.length) {
parts.add(TextSpan(text: src.substring(last)));
}
if (parts.isEmpty) return Text(src);
return Text.rich(TextSpan(children: parts));
}
Widget _render(BuildContext context, String src) {
final spans = <InlineSpan>[];
final tags = <String>[];
void flush(StringBuffer buf) {
if (buf.isEmpty) return;
var t = buf.toString();
buf.clear();
bool bold = false;
bool italic = false;
bool underline = false;
bool strikethrough = false;
Color? color;
double? size;
for (final tag in tags) {
if (tag == 'b') {
bold = true;
} else if (tag == 'i') {
italic = true;
} else if (tag == 'u') {
underline = true;
} else if (tag == 's') {
strikethrough = true;
} else if (tag.startsWith('color=')) {
color = _parseColor(tag.substring(6));
} else if (tag.startsWith('size=')) {
size = double.tryParse(tag.substring(5));
}
}
spans.add(TextSpan(
text: t,
style: TextStyle(
fontWeight: bold ? FontWeight.bold : null,
fontStyle: italic ? FontStyle.italic : null,
decoration: TextDecoration.combine([
if (underline) TextDecoration.underline,
if (strikethrough) TextDecoration.lineThrough,
]),
color: color,
fontSize: size,
),
));
}
final buf = StringBuffer();
int i = 0;
while (i < src.length) {
if (src[i] != '[') {
buf.write(src[i]);
i++;
continue;
}
final m = _tagRe.matchAsPrefix(src, i);
if (m == null) {
buf.write(src[i]);
i++;
continue;
}
flush(buf);
final raw = m.group(1)!.toLowerCase();
i = m.end;
if (raw.startsWith('/')) {
final closeTag = raw.substring(1);
if (closeTag == 'url' || closeTag == 'img') {
continue;
}
tags.remove(closeTag);
continue;
}
switch (raw) {
case 'b':
case 'i':
case 'u':
case 's':
case 'list':
case 'quote':
case 'code':
case 'center':
case 'left':
case 'right':
tags.add(raw);
break;
case '*':
spans.add(const TextSpan(text: '\n \u2022 '));
break;
default:
if (raw.startsWith('color=') || raw.startsWith('size=')) {
tags.add(raw);
} else if (raw.startsWith('url=')) {
final url = _urlRe.firstMatch(raw)?.group(1) ?? '';
final closeIdx = _findCloseUrl(src, i);
String inner;
if (closeIdx != null) {
inner = src.substring(i, closeIdx);
i = closeIdx + 6;
} else {
inner = url;
}
spans.add(WidgetSpan(
alignment: PlaceholderAlignment.middle,
child: _LinkTap(
url: url.isNotEmpty ? url : inner,
linkTrust: linkTrust,
child: Text(
inner,
style: const TextStyle(
color: Colors.blue,
decoration: TextDecoration.underline,
),
),
),
));
} else if (raw.startsWith('img=')) {
final src2 = _imgRe.firstMatch(raw)?.group(1) ?? '';
if (src2.isNotEmpty) {
spans.add(WidgetSpan(
child: ClipRRect(
borderRadius: BorderRadius.circular(8),
child: Image.network(
Uri.tryParse(src2)?.toString() ?? src2,
fit: BoxFit.scaleDown,
errorBuilder: (_, __, ___) => const SizedBox.shrink(),
),
),
));
}
} else if (raw == 'url') {
final closeIdx = _findCloseUrl(src, i);
if (closeIdx != null) {
final url = src.substring(i, closeIdx).trim();
i = closeIdx + 6;
spans.add(WidgetSpan(
alignment: PlaceholderAlignment.middle,
child: _LinkTap(
url: url,
linkTrust: linkTrust,
child: Text(
url,
style: const TextStyle(
color: Colors.blue,
decoration: TextDecoration.underline,
),
),
),
));
}
}
break;
}
}
flush(buf);
if (spans.isEmpty) return Text(src);
return Text.rich(TextSpan(children: spans));
}
}
class _LinkTap extends StatefulWidget {
const _LinkTap({
required this.url,
required this.child,
required this.linkTrust,
});
final String url;
final Widget child;
final LinkTrustService linkTrust;
@override
State<_LinkTap> createState() => _LinkTapState();
}
class _LinkTapState extends State<_LinkTap> {
@override
void initState() {
super.initState();
widget.linkTrust.addListener(_onChanged);
}
@override
void dispose() {
widget.linkTrust.removeListener(_onChanged);
super.dispose();
}
void _onChanged() => mounted ? setState(() {}) : null;
Future<void> _open() async {
final uri = Uri.tryParse(widget.url);
if (uri == null) return;
final host = uri.host;
if (host.isEmpty) return;
if (widget.linkTrust.isTrusted(host)) {
await launchUrl(uri, mode: LaunchMode.externalApplication);
return;
}
if (!mounted) return;
final trust = await showLinkTrustDialog(context, host);
if (trust == null) return;
if (trust) {
await widget.linkTrust.addTrustedDomain(host);
}
await launchUrl(uri, mode: LaunchMode.externalApplication);
}
@override
Widget build(BuildContext context) {
return GestureDetector(
onTap: _open,
child: widget.child,
);
}
}
@@ -40,6 +40,8 @@ class VoiceStatusChip extends StatelessWidget {
required this.audioStats,
required this.isTouchOnly,
required this.onTap,
this.inputMuted = false,
this.outputMuted = false,
});
/// Current transmit mode.
@@ -57,6 +59,12 @@ class VoiceStatusChip extends StatelessWidget {
/// True on iOS / iPadOS / Android.
final bool isTouchOnly;
/// True when local mic is muted (hard mute or permission mute).
final bool inputMuted;
/// True when local speaker is muted.
final bool outputMuted;
/// Open the voice details modal.
final VoidCallback onTap;
@@ -95,9 +103,15 @@ class VoiceStatusChip extends StatelessWidget {
final tailText = transmitMode == rust.BridgeTransmitMode.ptt
? '$releaseTailMs${l10n.voiceReleaseTailHint} ${l10n.voiceReleaseTailLabel.toLowerCase()}'
: null;
final micText = micOn ? l10n.voiceMicOn : l10n.voiceMicOff;
final micText = inputMuted
? '${l10n.voiceMicOff} (muted)'
: outputMuted
? 'Speaker muted'
: (micOn ? l10n.voiceMicOn : l10n.voiceMicOff);
final line2 = tailText == null ? micText : '$tailText \u00b7 $micText';
final muted = inputMuted || outputMuted;
return Semantics(
button: true,
label: '${l10n.voiceSheetTitle}: $line1, $line2',
@@ -105,17 +119,24 @@ class VoiceStatusChip extends StatelessWidget {
child: Material(
type: MaterialType.transparency,
child: InkWell(
onTap: onTap,
onTap: () {
HapticFeedback.lightImpact();
onTap();
},
borderRadius: BorderRadius.circular(12),
child: ExcludeSemantics(
child: Container(
padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 8),
decoration: BoxDecoration(
color: theme.colorScheme.surfaceContainerHigh,
color: muted
? theme.colorScheme.errorContainer.withValues(alpha: 0.35)
: theme.colorScheme.surfaceContainerHigh,
borderRadius: BorderRadius.circular(12),
border: Border.all(
color: theme.colorScheme.outlineVariant,
width: 0.5,
color: muted
? theme.colorScheme.error
: theme.colorScheme.outlineVariant,
width: muted ? 1.5 : 0.5,
),
),
child: Row(
@@ -394,6 +415,7 @@ class _VoiceSheetBodyState extends State<_VoiceSheetBody> {
late bool _aecEnabled;
late bool _agcEnabled;
late bool _hpfEnabled;
late bool _preferHardware;
late rust.BridgeVadBackend _vadBackend;
@override
@@ -404,6 +426,7 @@ class _VoiceSheetBodyState extends State<_VoiceSheetBody> {
_aecEnabled = c.aec != rust.BridgeEffectOwner.off;
_agcEnabled = c.agc != rust.BridgeEffectOwner.off;
_hpfEnabled = c.hpfEnabled;
_preferHardware = c.aec == rust.BridgeEffectOwner.platform;
_vadBackend = c.vadBackend == rust.BridgeVadBackend.disabled
? rust.BridgeVadBackend.webrtcVad
: c.vadBackend;
@@ -440,23 +463,51 @@ class _VoiceSheetBodyState extends State<_VoiceSheetBody> {
final c = widget.initialAudioConfig;
final isSonora =
c.iosMode == rust.BridgeIosVoiceProcessingMode.sonoraExperimental;
// VPIO owns enabled effects on the default path. Sonora owns them only in
// the experimental raw path.
final isAndroid = Platform.isAndroid;
if (isAndroid) {
final owner = _preferHardware
? rust.BridgeEffectOwner.platform
: rust.BridgeEffectOwner.webrtcApm;
return rust.BridgeAudioProcessingConfig(
route: c.route,
iosMode: c.iosMode,
processingBackend: _preferHardware
? rust.BridgeAudioBackend.platformVoiceProcessing
: rust.BridgeAudioBackend.webrtcApm,
vadBackend: _vadBackend == rust.BridgeVadBackend.disabled
? rust.BridgeVadBackend.webrtcVad
: _vadBackend,
aec: _aecEnabled ? owner : rust.BridgeEffectOwner.off,
ns: _nsEnabled ? owner : rust.BridgeEffectOwner.off,
agc: _agcEnabled ? owner : rust.BridgeEffectOwner.off,
hpfEnabled: _hpfEnabled,
limiterEnabled: c.limiterEnabled,
vadHangoverMs: c.vadHangoverMs,
vadPreRollMs: c.vadPreRollMs,
vadMinTxMs: c.vadMinTxMs,
debugWavDumpEnabled: c.debugWavDumpEnabled,
);
}
// iOS / macOS: VPIO vs Sonora paths.
// VPIO owns enabled effects on the default path. The experimental raw path
// delegates app-side processing to WebRTC APM.
final aecOwner = isSonora
? (_aecEnabled
? rust.BridgeEffectOwner.sonora
? rust.BridgeEffectOwner.webrtcApm
: rust.BridgeEffectOwner.off)
: rust.BridgeEffectOwner.platform;
final nsOwner = isSonora
? (_nsEnabled
? rust.BridgeEffectOwner.sonora
? rust.BridgeEffectOwner.webrtcApm
: rust.BridgeEffectOwner.off)
: (_nsEnabled
? rust.BridgeEffectOwner.platform
: rust.BridgeEffectOwner.off);
final agcOwner = isSonora
? (_agcEnabled
? rust.BridgeEffectOwner.sonora
? rust.BridgeEffectOwner.webrtcApm
: rust.BridgeEffectOwner.off)
: (_agcEnabled
? rust.BridgeEffectOwner.platform
@@ -631,6 +682,22 @@ class _VoiceSheetBodyState extends State<_VoiceSheetBody> {
),
),
const SizedBox(height: 4),
// Android: hardware (JNI) vs software (WebRTC APM)
if (Platform.isAndroid) ...[
_AudioToggleRow(
label: 'Prefer hardware effects',
subtitle: _preferHardware
? 'Try JNI hardware · software fallback'
: 'Software WebRTC AEC3 · NS · AGC2',
value: _preferHardware,
onChanged: (v) {
setState(() => _preferHardware = v);
_notifyAudioConfig();
},
),
],
_AudioToggleRow(
label: 'Noise suppression',
subtitle: 'Wiener filter',
@@ -642,25 +709,40 @@ class _VoiceSheetBodyState extends State<_VoiceSheetBody> {
),
_AudioToggleRow(
label: 'Echo cancellation',
subtitle:
widget.initialAudioConfig.iosMode ==
rust.BridgeIosVoiceProcessingMode.platformVoiceProcessing
? 'Always on · managed by platform VPIO'
: 'AEC3 adaptive filter · 80 ms tail',
subtitle: () {
if (Platform.isAndroid) {
return 'WebRTC AEC3 · adaptive filter';
}
return widget.initialAudioConfig.iosMode ==
rust.BridgeIosVoiceProcessingMode.platformVoiceProcessing
? 'Always on · managed by platform VPIO'
: 'AEC3 adaptive filter · 80 ms tail';
}(),
value:
widget.initialAudioConfig.iosMode ==
rust.BridgeIosVoiceProcessingMode.platformVoiceProcessing
? true // always on in VPIO
: _aecEnabled,
() {
if (Platform.isAndroid) return _aecEnabled;
return widget.initialAudioConfig.iosMode ==
rust.BridgeIosVoiceProcessingMode.platformVoiceProcessing
? true
: _aecEnabled;
}(),
// AEC is always on in VPIO — disable the toggle.
onChanged:
widget.initialAudioConfig.iosMode ==
rust.BridgeIosVoiceProcessingMode.platformVoiceProcessing
? null
: (v) {
setState(() => _aecEnabled = v);
_notifyAudioConfig();
},
// On Android, AEC is user-selectable.
onChanged: () {
if (Platform.isAndroid) {
return (v) {
setState(() => _aecEnabled = v);
_notifyAudioConfig();
};
}
return widget.initialAudioConfig.iosMode ==
rust.BridgeIosVoiceProcessingMode.platformVoiceProcessing
? null
: (v) {
setState(() => _aecEnabled = v);
_notifyAudioConfig();
};
}(),
),
_AudioToggleRow(
label: 'Auto gain control',
@@ -18,6 +18,11 @@ import '../l10n/generated/app_localizations.dart';
import 'voice_platform.dart';
import '../src/rust/api.dart' as rust;
bool get _isAndroid {
if (kIsWeb) return false;
return Platform.isAndroid;
}
bool get _isIos {
if (kIsWeb) return false;
return Platform.isIOS;
@@ -68,6 +73,7 @@ class _VoiceSettingsDialogState extends State<VoiceSettingsDialog> {
late rust.BridgeVadBackend _vadBackend;
late rust.BridgeIosVoiceProcessingMode _iosMode;
late bool _debugWavDump;
late bool _preferHardware; // Android only: try JNI hardware effects
@override
void initState() {
@@ -86,29 +92,59 @@ class _VoiceSettingsDialogState extends State<VoiceSettingsDialog> {
: c.vadBackend;
_iosMode = c.iosMode;
_debugWavDump = c.debugWavDumpEnabled;
_preferHardware = c.aec == rust.BridgeEffectOwner.platform
|| c.ns == rust.BridgeEffectOwner.platform
|| c.agc == rust.BridgeEffectOwner.platform;
}
rust.BridgeAudioProcessingConfig _buildConfig() {
final c = widget.initialAudioConfig;
final isSonora =
_iosMode == rust.BridgeIosVoiceProcessingMode.sonoraExperimental;
// In VPIO mode, enabled effects are platform-owned. Sonora ownership is
// reserved for the experimental raw path so config validation stays honest.
if (_isAndroid) {
final owner = _preferHardware
? rust.BridgeEffectOwner.platform
: rust.BridgeEffectOwner.webrtcApm;
return rust.BridgeAudioProcessingConfig(
route: c.route,
iosMode: _iosMode,
processingBackend: _preferHardware
? rust.BridgeAudioBackend.platformVoiceProcessing
: rust.BridgeAudioBackend.webrtcApm,
vadBackend: _vadBackend == rust.BridgeVadBackend.disabled
? rust.BridgeVadBackend.webrtcVad
: _vadBackend,
aec: _aecEnabled ? owner : rust.BridgeEffectOwner.off,
ns: _nsEnabled ? owner : rust.BridgeEffectOwner.off,
agc: _agcEnabled ? owner : rust.BridgeEffectOwner.off,
hpfEnabled: _hpfEnabled,
limiterEnabled: _limiterEnabled,
vadHangoverMs: c.vadHangoverMs,
vadPreRollMs: c.vadPreRollMs,
vadMinTxMs: c.vadMinTxMs,
debugWavDumpEnabled: _debugWavDump,
);
}
// iOS / macOS: VPIO vs Sonora paths.
// In VPIO mode, enabled effects are platform-owned. The experimental raw
// path uses WebRTC APM ownership so config validation stays honest.
final aecOwner = isSonora
? (_aecEnabled
? rust.BridgeEffectOwner.sonora
? rust.BridgeEffectOwner.webrtcApm
: rust.BridgeEffectOwner.off)
: rust.BridgeEffectOwner.platform; // VPIO always owns AEC
final nsOwner = isSonora
? (_nsEnabled
? rust.BridgeEffectOwner.sonora
? rust.BridgeEffectOwner.webrtcApm
: rust.BridgeEffectOwner.off)
: (_nsEnabled
? rust.BridgeEffectOwner.platform
: rust.BridgeEffectOwner.off);
final agcOwner = isSonora
? (_agcEnabled
? rust.BridgeEffectOwner.sonora
? rust.BridgeEffectOwner.webrtcApm
: rust.BridgeEffectOwner.off)
: (_agcEnabled
? rust.BridgeEffectOwner.platform
@@ -120,7 +156,7 @@ class _VoiceSettingsDialogState extends State<VoiceSettingsDialog> {
route: c.route,
iosMode: _iosMode,
processingBackend: isSonora
? rust.BridgeAudioBackend.sonora
? rust.BridgeAudioBackend.webrtcApm
: rust.BridgeAudioBackend.platformVoiceProcessing,
vadBackend: vadBackend,
aec: aecOwner,
@@ -244,6 +280,30 @@ class _VoiceSettingsDialogState extends State<VoiceSettingsDialog> {
const SizedBox(height: 4),
],
// Android HW/SW selector
if (_isAndroid) ...[
_subHeader(theme, 'Processing backend'),
_radioTile<bool>(
value: true,
groupValue: _preferHardware,
title: const Text('Platform (auto)'),
subtitle: _tileSubtitle(
'Try hardware JNI effects · software fallback',
),
onSelected: (v) => setState(() => _preferHardware = v),
),
_radioTile<bool>(
value: false,
groupValue: _preferHardware,
title: const Text('WebRTC APM'),
subtitle: _tileSubtitle(
'Software AEC3 · NS · AGC2',
),
onSelected: (v) => setState(() => _preferHardware = v),
),
const SizedBox(height: 4),
],
// DSP toggles
_subHeader(theme, 'DSP stages'),
_switchTile(
@@ -254,12 +314,14 @@ class _VoiceSettingsDialogState extends State<VoiceSettingsDialog> {
),
_switchTile(
title: 'Echo cancellation (AEC3)',
subtitle: platformVpio
? 'Managed by platform VPIO'
: 'Adaptive NLMS · 80 ms tail',
subtitle: _isAndroid
? 'WebRTC AEC3 · adaptive filter'
: platformVpio
? 'Managed by platform VPIO'
: 'Adaptive NLMS · 80 ms tail',
value: _aecEnabled,
// AEC is always on in VPIO mode — disable the toggle.
onSelected: platformVpio ? null : (v) => _aecEnabled = v,
onSelected: (_isAndroid || !platformVpio) ? (v) => _aecEnabled = v : null,
),
_switchTile(
title: 'Auto gain control (AGC2)',