feat(voice): add iOS VAD runtime support
This commit is contained in:
Edison Jwa
@@ -9,8 +9,8 @@ import 'package:flutter_rust_bridge/flutter_rust_bridge_for_generated.dart';
|
||||
import 'package:freezed_annotation/freezed_annotation.dart' hide protected;
|
||||
part 'api.freezed.dart';
|
||||
|
||||
// These functions are ignored because they are not marked as `pub`: `log_file_path`, `log_sink`, `map_join_error_code`, `map_join_sync_state`, `open_log_file`, `permission_events`, `publish_permission_state`, `runtime`, `session`, `transmit_mode_from_u8`
|
||||
// These function are ignored because they are on traits that is not defined in current crate (put an empty `#[frb]` on it to unignore): `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `eq`, `eq`, `eq`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`
|
||||
// These functions are ignored because they are not marked as `pub`: `install_panic_diagnostic_hook`, `log_file_path`, `log_sink`, `map_join_error_code`, `map_join_sync_state`, `open_log_file`, `permission_events`, `publish_permission_state`, `runtime`, `session`, `task_join_error`, `transmit_mode_from_u8`
|
||||
// These function are ignored because they are on traits that is not defined in current crate (put an empty `#[frb]` on it to unignore): `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `eq`, `eq`, `eq`, `eq`, `eq`, `eq`, `eq`, `eq`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`
|
||||
// These functions are ignored (category: IgnoreBecauseExplicitAttribute): `from_kotlin_str`, `to_permission_gate`
|
||||
|
||||
/// Return the platform-conventional log-file path as a string, or
|
||||
@@ -43,7 +43,12 @@ Future<void> disconnect() => RustLib.instance.api.crateApiDisconnect();
|
||||
Future<bool> isConnected() => RustLib.instance.api.crateApiIsConnected();
|
||||
|
||||
/// Handle iOS AVAudioSession route changes (SDD-100).
|
||||
void handleRouteChange() => RustLib.instance.api.crateApiHandleRouteChange();
|
||||
void handleRouteChange({required BridgeAudioRoute route}) =>
|
||||
RustLib.instance.api.crateApiHandleRouteChange(route: route);
|
||||
|
||||
/// Handle iOS AVAudioSession media-services reset.
|
||||
void handleMediaServicesReset() =>
|
||||
RustLib.instance.api.crateApiHandleMediaServicesReset();
|
||||
|
||||
/// Handle iOS AVAudioSession interruption begin (SDD-101).
|
||||
void handleInterruptionBegan() =>
|
||||
@@ -221,6 +226,330 @@ Stream<BridgeEvent> eventsStream() =>
|
||||
Future<BridgeAudioStats> audioStats() =>
|
||||
RustLib.instance.api.crateApiAudioStats();
|
||||
|
||||
/// Apply the P1 audio-processing config.
|
||||
Future<void> setAudioProcessingConfig({
|
||||
required BridgeAudioProcessingConfig config,
|
||||
}) async {
|
||||
_lastAppliedAudioConfig = config;
|
||||
return RustLib.instance.api.crateApiSetAudioProcessingConfig(config: config);
|
||||
}
|
||||
|
||||
/// Read P1 audio-processing diagnostics.
|
||||
Future<BridgeAudioProcessingStats> audioProcessingStats() =>
|
||||
RustLib.instance.api.crateApiAudioProcessingStats();
|
||||
|
||||
/// Read the current audio-processing config.
|
||||
///
|
||||
/// Derives the config from [audioProcessingStats] for the route/backend
|
||||
/// fields, and returns the last value applied via [setAudioProcessingConfig]
|
||||
/// for timing/debug fields. Falls back to P1 spec defaults on first call.
|
||||
Future<BridgeAudioProcessingConfig> getAudioProcessingConfig() async {
|
||||
BridgeAudioProcessingStats? stats;
|
||||
try {
|
||||
stats = await audioProcessingStats();
|
||||
} catch (_) {}
|
||||
|
||||
final last = _lastAppliedAudioConfig;
|
||||
return BridgeAudioProcessingConfig(
|
||||
route: stats?.audioRoute ?? last?.route ?? BridgeAudioRoute.unknown,
|
||||
iosMode:
|
||||
stats?.iosVoiceProcessingMode ??
|
||||
last?.iosMode ??
|
||||
BridgeIosVoiceProcessingMode.platformVoiceProcessing,
|
||||
processingBackend:
|
||||
stats?.processingBackend ??
|
||||
last?.processingBackend ??
|
||||
BridgeAudioBackend.platformVoiceProcessing,
|
||||
vadBackend:
|
||||
stats?.vadBackend ?? last?.vadBackend ?? BridgeVadBackend.sileroOnnx,
|
||||
aec: last?.aec ?? BridgeEffectOwner.platform,
|
||||
ns: last?.ns ?? BridgeEffectOwner.platform,
|
||||
agc: last?.agc ?? BridgeEffectOwner.platform,
|
||||
hpfEnabled: last?.hpfEnabled ?? true,
|
||||
limiterEnabled: last?.limiterEnabled ?? true,
|
||||
vadHangoverMs: last?.vadHangoverMs ?? 500,
|
||||
vadPreRollMs: last?.vadPreRollMs ?? 160,
|
||||
vadMinTxMs: last?.vadMinTxMs ?? 200,
|
||||
debugWavDumpEnabled: last?.debugWavDumpEnabled ?? false,
|
||||
);
|
||||
}
|
||||
|
||||
/// Last config applied via [setAudioProcessingConfig]. Used by
|
||||
/// [getAudioProcessingConfig] to preserve timing/debug values across calls.
|
||||
BridgeAudioProcessingConfig? _lastAppliedAudioConfig;
|
||||
|
||||
/// Configure the VAD model path.
|
||||
Future<void> setVadModelPath({required String path}) =>
|
||||
RustLib.instance.api.crateApiSetVadModelPath(path: path);
|
||||
|
||||
/// Enable or disable audio debug WAV dumping.
|
||||
Future<void> enableAudioDebugWavDump({required bool enabled}) =>
|
||||
RustLib.instance.api.crateApiEnableAudioDebugWavDump(enabled: enabled);
|
||||
|
||||
/// Select the iOS voice-processing mode.
|
||||
Future<void> setIosVoiceProcessingMode({
|
||||
required BridgeIosVoiceProcessingMode mode,
|
||||
}) => RustLib.instance.api.crateApiSetIosVoiceProcessingMode(mode: mode);
|
||||
|
||||
/// Bridge processing backend.
|
||||
enum BridgeAudioBackend {
|
||||
/// Platform voice processing.
|
||||
platformVoiceProcessing,
|
||||
|
||||
/// Sonora backend.
|
||||
sonora,
|
||||
|
||||
/// WebRTC APM backend.
|
||||
webrtcApm,
|
||||
|
||||
/// No-op backend.
|
||||
noop,
|
||||
}
|
||||
|
||||
/// P1 audio-processing configuration DTO.
|
||||
class BridgeAudioProcessingConfig {
|
||||
/// Route class.
|
||||
final BridgeAudioRoute route;
|
||||
|
||||
/// iOS voice-processing mode.
|
||||
final BridgeIosVoiceProcessingMode iosMode;
|
||||
|
||||
/// Processing backend.
|
||||
final BridgeAudioBackend processingBackend;
|
||||
|
||||
/// VAD backend.
|
||||
final BridgeVadBackend vadBackend;
|
||||
|
||||
/// AEC owner.
|
||||
final BridgeEffectOwner aec;
|
||||
|
||||
/// Noise suppression owner.
|
||||
final BridgeEffectOwner ns;
|
||||
|
||||
/// AGC owner.
|
||||
final BridgeEffectOwner agc;
|
||||
|
||||
/// High-pass filter enabled.
|
||||
final bool hpfEnabled;
|
||||
|
||||
/// Limiter enabled.
|
||||
final bool limiterEnabled;
|
||||
|
||||
/// VAD hangover in ms.
|
||||
final int vadHangoverMs;
|
||||
|
||||
/// VAD pre-roll in ms.
|
||||
final int vadPreRollMs;
|
||||
|
||||
/// Minimum transmit duration in ms.
|
||||
final int vadMinTxMs;
|
||||
|
||||
/// Debug WAV dump enabled.
|
||||
final bool debugWavDumpEnabled;
|
||||
|
||||
const BridgeAudioProcessingConfig({
|
||||
required this.route,
|
||||
required this.iosMode,
|
||||
required this.processingBackend,
|
||||
required this.vadBackend,
|
||||
required this.aec,
|
||||
required this.ns,
|
||||
required this.agc,
|
||||
required this.hpfEnabled,
|
||||
required this.limiterEnabled,
|
||||
required this.vadHangoverMs,
|
||||
required this.vadPreRollMs,
|
||||
required this.vadMinTxMs,
|
||||
required this.debugWavDumpEnabled,
|
||||
});
|
||||
|
||||
@override
|
||||
int get hashCode =>
|
||||
route.hashCode ^
|
||||
iosMode.hashCode ^
|
||||
processingBackend.hashCode ^
|
||||
vadBackend.hashCode ^
|
||||
aec.hashCode ^
|
||||
ns.hashCode ^
|
||||
agc.hashCode ^
|
||||
hpfEnabled.hashCode ^
|
||||
limiterEnabled.hashCode ^
|
||||
vadHangoverMs.hashCode ^
|
||||
vadPreRollMs.hashCode ^
|
||||
vadMinTxMs.hashCode ^
|
||||
debugWavDumpEnabled.hashCode;
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) =>
|
||||
identical(this, other) ||
|
||||
other is BridgeAudioProcessingConfig &&
|
||||
runtimeType == other.runtimeType &&
|
||||
route == other.route &&
|
||||
iosMode == other.iosMode &&
|
||||
processingBackend == other.processingBackend &&
|
||||
vadBackend == other.vadBackend &&
|
||||
aec == other.aec &&
|
||||
ns == other.ns &&
|
||||
agc == other.agc &&
|
||||
hpfEnabled == other.hpfEnabled &&
|
||||
limiterEnabled == other.limiterEnabled &&
|
||||
vadHangoverMs == other.vadHangoverMs &&
|
||||
vadPreRollMs == other.vadPreRollMs &&
|
||||
vadMinTxMs == other.vadMinTxMs &&
|
||||
debugWavDumpEnabled == other.debugWavDumpEnabled;
|
||||
}
|
||||
|
||||
/// P1 audio-processing stats DTO.
|
||||
class BridgeAudioProcessingStats {
|
||||
/// Input dBFS.
|
||||
final double inputDbfs;
|
||||
|
||||
/// Render dBFS.
|
||||
final double renderDbfs;
|
||||
|
||||
/// Processed capture dBFS.
|
||||
final double processedDbfs;
|
||||
|
||||
/// Latest VAD probability.
|
||||
final double vadProbability;
|
||||
|
||||
/// VAD active.
|
||||
final bool vadActive;
|
||||
|
||||
/// Currently transmitting.
|
||||
final bool transmitting;
|
||||
|
||||
/// VAD backend.
|
||||
final BridgeVadBackend vadBackend;
|
||||
|
||||
/// Fallback VAD active.
|
||||
final bool vadFallbackActive;
|
||||
|
||||
/// Processing backend.
|
||||
final BridgeAudioBackend processingBackend;
|
||||
|
||||
/// iOS voice-processing mode.
|
||||
final BridgeIosVoiceProcessingMode iosVoiceProcessingMode;
|
||||
|
||||
/// Audio route.
|
||||
final BridgeAudioRoute audioRoute;
|
||||
|
||||
/// Actual sample rate.
|
||||
final int actualSampleRateHz;
|
||||
|
||||
/// Actual IO buffer frames.
|
||||
final int actualIoBufferFrames;
|
||||
|
||||
/// Input overruns.
|
||||
final BigInt inputOverruns;
|
||||
|
||||
/// Output underruns.
|
||||
final BigInt outputUnderruns;
|
||||
|
||||
/// Callback xruns.
|
||||
final BigInt callbackXruns;
|
||||
|
||||
/// Clipped samples.
|
||||
final BigInt clippedSamples;
|
||||
|
||||
/// Sonora enabled.
|
||||
final bool sonoraEnabled;
|
||||
|
||||
/// Platform voice processing enabled.
|
||||
final bool platformVoiceProcessingEnabled;
|
||||
|
||||
const BridgeAudioProcessingStats({
|
||||
required this.inputDbfs,
|
||||
required this.renderDbfs,
|
||||
required this.processedDbfs,
|
||||
required this.vadProbability,
|
||||
required this.vadActive,
|
||||
required this.transmitting,
|
||||
required this.vadBackend,
|
||||
required this.vadFallbackActive,
|
||||
required this.processingBackend,
|
||||
required this.iosVoiceProcessingMode,
|
||||
required this.audioRoute,
|
||||
required this.actualSampleRateHz,
|
||||
required this.actualIoBufferFrames,
|
||||
required this.inputOverruns,
|
||||
required this.outputUnderruns,
|
||||
required this.callbackXruns,
|
||||
required this.clippedSamples,
|
||||
required this.sonoraEnabled,
|
||||
required this.platformVoiceProcessingEnabled,
|
||||
});
|
||||
|
||||
@override
|
||||
int get hashCode =>
|
||||
inputDbfs.hashCode ^
|
||||
renderDbfs.hashCode ^
|
||||
processedDbfs.hashCode ^
|
||||
vadProbability.hashCode ^
|
||||
vadActive.hashCode ^
|
||||
transmitting.hashCode ^
|
||||
vadBackend.hashCode ^
|
||||
vadFallbackActive.hashCode ^
|
||||
processingBackend.hashCode ^
|
||||
iosVoiceProcessingMode.hashCode ^
|
||||
audioRoute.hashCode ^
|
||||
actualSampleRateHz.hashCode ^
|
||||
actualIoBufferFrames.hashCode ^
|
||||
inputOverruns.hashCode ^
|
||||
outputUnderruns.hashCode ^
|
||||
callbackXruns.hashCode ^
|
||||
clippedSamples.hashCode ^
|
||||
sonoraEnabled.hashCode ^
|
||||
platformVoiceProcessingEnabled.hashCode;
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) =>
|
||||
identical(this, other) ||
|
||||
other is BridgeAudioProcessingStats &&
|
||||
runtimeType == other.runtimeType &&
|
||||
inputDbfs == other.inputDbfs &&
|
||||
renderDbfs == other.renderDbfs &&
|
||||
processedDbfs == other.processedDbfs &&
|
||||
vadProbability == other.vadProbability &&
|
||||
vadActive == other.vadActive &&
|
||||
transmitting == other.transmitting &&
|
||||
vadBackend == other.vadBackend &&
|
||||
vadFallbackActive == other.vadFallbackActive &&
|
||||
processingBackend == other.processingBackend &&
|
||||
iosVoiceProcessingMode == other.iosVoiceProcessingMode &&
|
||||
audioRoute == other.audioRoute &&
|
||||
actualSampleRateHz == other.actualSampleRateHz &&
|
||||
actualIoBufferFrames == other.actualIoBufferFrames &&
|
||||
inputOverruns == other.inputOverruns &&
|
||||
outputUnderruns == other.outputUnderruns &&
|
||||
callbackXruns == other.callbackXruns &&
|
||||
clippedSamples == other.clippedSamples &&
|
||||
sonoraEnabled == other.sonoraEnabled &&
|
||||
platformVoiceProcessingEnabled ==
|
||||
other.platformVoiceProcessingEnabled;
|
||||
}
|
||||
|
||||
/// Bridge route class for P1 audio-processing policy.
|
||||
enum BridgeAudioRoute {
|
||||
/// Built-in speakerphone.
|
||||
speaker,
|
||||
|
||||
/// Built-in receiver/earpiece.
|
||||
earpiece,
|
||||
|
||||
/// Wired or USB headset.
|
||||
wiredHeadset,
|
||||
|
||||
/// Bluetooth HFP duplex route.
|
||||
bluetoothHfp,
|
||||
|
||||
/// Bluetooth A2DP output-only route.
|
||||
bluetoothA2Dp,
|
||||
|
||||
/// Unknown route.
|
||||
unknown,
|
||||
}
|
||||
|
||||
/// Statistics from the audio engine.
|
||||
class BridgeAudioStats {
|
||||
/// Number of Opus frames sent since audio started.
|
||||
@@ -402,6 +731,24 @@ class BridgeClient {
|
||||
isServerQuery == other.isServerQuery;
|
||||
}
|
||||
|
||||
/// Bridge effect owner for AEC/NS/AGC.
|
||||
enum BridgeEffectOwner {
|
||||
/// Platform-owned effect.
|
||||
platform,
|
||||
|
||||
/// Sonora-owned effect.
|
||||
sonora,
|
||||
|
||||
/// WebRTC APM-owned effect.
|
||||
webrtcApm,
|
||||
|
||||
/// Conservative route-managed setting.
|
||||
conservative,
|
||||
|
||||
/// Disabled.
|
||||
off,
|
||||
}
|
||||
|
||||
@freezed
|
||||
sealed class BridgeEvent with _$BridgeEvent {
|
||||
const BridgeEvent._();
|
||||
@@ -536,6 +883,15 @@ sealed class BridgeEvent with _$BridgeEvent {
|
||||
}) = BridgeEvent_PermissionState;
|
||||
}
|
||||
|
||||
/// Bridge iOS voice-processing mode.
|
||||
enum BridgeIosVoiceProcessingMode {
|
||||
/// Shipping VPIO path.
|
||||
platformVoiceProcessing,
|
||||
|
||||
/// Experimental Sonora path.
|
||||
sonoraExperimental,
|
||||
}
|
||||
|
||||
/// Coarse OS-reported network state. Mirrors
|
||||
/// [`chanora_core::NetworkState`] across the bridge.
|
||||
enum BridgeNetworkState {
|
||||
@@ -635,25 +991,69 @@ enum BridgeTransmitMode {
|
||||
voiceActivity,
|
||||
}
|
||||
|
||||
/// Bridge VAD backend.
|
||||
enum BridgeVadBackend {
|
||||
/// Silero ONNX VAD.
|
||||
sileroOnnx,
|
||||
|
||||
/// TEN VAD.
|
||||
tenVad,
|
||||
|
||||
/// WebRTC fallback VAD.
|
||||
webrtcVad,
|
||||
|
||||
/// Debug energy VAD.
|
||||
energyDebug,
|
||||
|
||||
/// VAD disabled.
|
||||
disabled,
|
||||
}
|
||||
|
||||
/// Bridge mirror of stable join error/status codes.
|
||||
enum BridgeVoiceJoinErrorCode {
|
||||
/// Duplicate same-target join intent was coalesced.
|
||||
duplicateSameTargetCoalesced,
|
||||
|
||||
/// A different target was requested while one is already pending.
|
||||
joinAlreadyPendingDifferentTarget,
|
||||
|
||||
/// Join denied by server policy/permission.
|
||||
joinDenied,
|
||||
|
||||
/// Join failed due to protocol-level error.
|
||||
joinProtocolFailure,
|
||||
|
||||
/// Join failed due to transport/network error.
|
||||
joinNetworkFailure,
|
||||
|
||||
/// Join timed out awaiting confirmation.
|
||||
joinTimeout,
|
||||
|
||||
/// Pending join was superseded by user leave.
|
||||
joinSupersededByLeave,
|
||||
|
||||
/// Stale join outcome was ignored.
|
||||
joinStaleOutcomeIgnored,
|
||||
|
||||
/// Authoritative membership reconciled to different channel.
|
||||
joinReconciledDifferentChannel,
|
||||
|
||||
/// Join command was rejected before send acceptance.
|
||||
joinCommandRejectedBeforeSend,
|
||||
|
||||
/// Join intent rejected while reducer synchronizing.
|
||||
joinCannotStartWhileSynchronizing,
|
||||
}
|
||||
|
||||
/// Bridge mirror of core join projection sync state.
|
||||
enum BridgeVoiceJoinSyncState {
|
||||
/// Reducer is ready to accept channel actions.
|
||||
ready,
|
||||
|
||||
/// Reducer is waiting on initial snapshot reconciliation.
|
||||
synchronizingInitialSnapshot,
|
||||
|
||||
/// Reducer is waiting on reconnect snapshot reconciliation.
|
||||
synchronizingReconnect,
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user