edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(voice): add iOS VAD runtime support

Browse Source
This commit is contained in:
Edison Jwa
2026-05-21 20:51:45 +09:00
parent 171baf6e41
commit 6af4ecab0f
73 changed files with 11529 additions and 1249 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/chanora_flutter/ios/ExportOptions/AppStore.plist
+18
View File
@@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>method</key>
<string>app-store</string>
<key>destination</key>
<string>export</string>
<key>signingStyle</key>
<string>automatic</string>
<key>stripSwiftSymbols</key>
<true/>
<key>uploadBitcode</key>
<false/>
<key>uploadSymbols</key>
<true/>
</dict>
</plist>
apps/chanora_flutter/ios/Podfile
+2 -1
View File
@@ -1,5 +1,5 @@
# Uncomment this line to define a global platform for your project
platform :ios, '13.0'
platform :ios, '15.1'
# CocoaPods analytics sends network stats synchronously affecting flutter build latency.
ENV['COCOAPODS_DISABLE_STATS'] = 'true'
@@ -39,6 +39,7 @@ target 'Runner' do
# flutter_rust_bridge can dlopen() it at runtime via FRB's
# default `chanora_bridge.framework/chanora_bridge` lookup path.
pod 'chanora_bridge', :path => '.'
pod 'onnxruntime-c', '1.22.0'
flutter_install_all_ios_pods File.dirname(File.realpath(__FILE__))
target 'RunnerTests' do
apps/chanora_flutter/ios/Podfile.lock
+15 -2
View File
@@ -5,6 +5,9 @@ PODS:
- connectivity_plus (0.0.1):
- Flutter
- Flutter (1.0.0)
- haptic_kit (1.0.0):
- Flutter
- onnxruntime-c (1.22.0)
- package_info_plus (0.4.5):
- Flutter
@@ -13,8 +16,14 @@ DEPENDENCIES:
- chanora_bridge (from `.`)
- connectivity_plus (from `.symlinks/plugins/connectivity_plus/ios`)
- Flutter (from `Flutter`)
- haptic_kit (from `.symlinks/plugins/haptic_kit/ios`)
- onnxruntime-c (= 1.22.0)
- package_info_plus (from `.symlinks/plugins/package_info_plus/ios`)
SPEC REPOS:
trunk:
- onnxruntime-c
EXTERNAL SOURCES:
audio_session:
:path: ".symlinks/plugins/audio_session/ios"
@@ -24,16 +33,20 @@ EXTERNAL SOURCES:
:path: ".symlinks/plugins/connectivity_plus/ios"
Flutter:
:path: Flutter
haptic_kit:
:path: ".symlinks/plugins/haptic_kit/ios"
package_info_plus:
:path: ".symlinks/plugins/package_info_plus/ios"
SPEC CHECKSUMS:
audio_session: 9bb7f6c970f21241b19f5a3658097ae459681ba0
chanora_bridge: af821d2c0507cb3199c91be12996bf0eb6b8bf5d
chanora_bridge: 0289413733edf8b7c937c50c3c3424b3319b94b5
connectivity_plus: cb623214f4e1f6ef8fe7403d580fdad517d2f7dd
Flutter: cabc95a1d2626b1b06e7179b784ebcf0c0cde467
haptic_kit: b22c4fbb2aa7b0d66f2891f81a9e950ad2de5758
onnxruntime-c: 7f778680e96145956c0a31945f260321eed2611a
package_info_plus: af8e2ca6888548050f16fa2f1938db7b5a5df499
PODFILE CHECKSUM: 15f58b0363434f244766f3301e00b9b1cdee096a
PODFILE CHECKSUM: a3abe93db2fc91b90387b399576e9c42a54226e0
COCOAPODS: 1.16.2
apps/chanora_flutter/ios/Runner.xcodeproj/project.pbxproj
+3 -3
View File
@@ -463,7 +463,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 13.0;
IPHONEOS_DEPLOYMENT_TARGET = 15.1;
MTL_ENABLE_DEBUG_INFO = NO;
SDKROOT = iphoneos;
SUPPORTED_PLATFORMS = iphoneos;
@@ -596,7 +596,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 13.0;
IPHONEOS_DEPLOYMENT_TARGET = 15.1;
MTL_ENABLE_DEBUG_INFO = YES;
ONLY_ACTIVE_ARCH = YES;
SDKROOT = iphoneos;
@@ -647,7 +647,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 13.0;
IPHONEOS_DEPLOYMENT_TARGET = 15.1;
MTL_ENABLE_DEBUG_INFO = NO;
SDKROOT = iphoneos;
SUPPORTED_PLATFORMS = iphoneos;
apps/chanora_flutter/ios/Runner/AppDelegate.swift
+124 -30
View File
@@ -5,6 +5,7 @@ import AVFoundation
@main
@objc class AppDelegate: FlutterAppDelegate, FlutterImplicitEngineDelegate {
private var iosAudioLifecycleChannel: FlutterMethodChannel?
private var iosPlatformChannel: FlutterMethodChannel?
override func application(
_ application: UIApplication,
@@ -134,32 +135,12 @@ import AVFoundation
object: nil
)
// Request microphone access on first launch rather than waiting
// for the user's first voice-channel join. The latter is
// surprising: the user has only tapped "connect to server" and
// suddenly iOS pops the permission prompt because joining a
// text channel happens to trigger audio engine startup. Asking
// up-front matches user expectations for a voice-chat client.
//
// The request is asynchronous and non-blocking. If the user
// denies, voice_join will surface a clearer error later when
// the audio engine fails to open the input device. The
// permission state is cached by iOS so subsequent launches
// skip the prompt.
//
// Deferred ~1 s so iOS finishes initialising the keyboard /
// text-input subsystem before the permission alert appears.
// Firing the alert too early steals focus from the not-yet-
// ready text-input layer, with the symptom that the first tap
// on a TextField does nothing (the second tap works because
// by then iOS has caught up). DispatchQueue.main.asyncAfter
// keeps everything on the main thread; the permission API
// itself must be called there too.
DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) {
AVAudioSession.sharedInstance().requestRecordPermission { granted in
NSLog("chanora_flutter: microphone permission granted=\(granted)")
}
}
NotificationCenter.default.addObserver(
self,
selector: #selector(handleMediaServicesReset(_:)),
name: AVAudioSession.mediaServicesWereResetNotification,
object: nil
)
return super.application(application, didFinishLaunchingWithOptions: launchOptions)
}
@@ -182,7 +163,7 @@ import AVFoundation
// wrong rate, causing pitch + timing artifacts).
logAudioSessionState(context: "setActive")
let s = AVAudioSession.sharedInstance()
let ins = s.currentRoute.inputs.map { "\($0.portType.rawValue)/\($0.portName)" }.joined(separator: ",")
let ins = s.currentRoute.inputs.map { $0.portType.rawValue }.joined(separator: ",")
NSLog(
"chanora_flutter: AVAudioSession actual: " +
"sampleRate=\(s.sampleRate) " +
@@ -221,9 +202,18 @@ import AVFoundation
let outputs = routeDescription.outputs.map { $0.portType.rawValue }.joined(separator: ",")
NSLog("chanora_flutter: route change reason=\(reason.rawValue) outputs=\(outputs)")
if reason == .oldDeviceUnavailable || reason == .newDeviceAvailable {
iosAudioLifecycleChannel?.invokeMethod("handleRouteChange", arguments: nil)
}
// P1: Send the detailed route class to Rust on every route change,
// not just device plug/unplug. This covers:
// - .newDeviceAvailable / .oldDeviceUnavailable (headset plug/unplug)
// - .override (speaker/earpiece toggle)
// - .categoryChange (session category changed)
// - .wakeFromSleep (device woke from sleep)
// - .routeConfigurationChange (BT HFP connect/disconnect)
// The Rust side uses the route class to recompute the processing
// policy (route_policy.rs) and reset AEC delay state if needed.
let routeClass = classifyAudioRoute(routeDescription)
NSLog("chanora_flutter: route class=\(routeClass) reason=\(reason.rawValue)")
iosAudioLifecycleChannel?.invokeMethod("handleRouteChange", arguments: routeClass)
}
@objc private func handleInterruption(_ notification: Notification) {
@@ -249,11 +239,115 @@ import AVFoundation
}
}
@objc private func handleMediaServicesReset(_ notification: Notification) {
NSLog("chanora_flutter: media services reset")
do {
let session = AVAudioSession.sharedInstance()
try session.setCategory(
.playAndRecord,
mode: .default,
options: [.defaultToSpeaker, .allowBluetoothHFP, .allowBluetoothA2DP]
)
try session.setPreferredIOBufferDuration(0.02)
try session.setPreferredSampleRate(48000.0)
try session.setActive(true, options: [])
logAudioSessionState(context: "mediaServicesWereReset")
} catch {
NSLog("chanora_flutter: AVAudioSession media-services reset rebuild failed: \(error)")
}
// P1: After rebuilding the session, send the current route class to
// Rust so it can recompute the processing policy and reset the
// AudioUnit. The Rust side handles this via ios_handle_media_services_reset
// which calls ios_restart_voice_unit.
let routeClass = classifyAudioRoute(AVAudioSession.sharedInstance().currentRoute)
NSLog("chanora_flutter: media services reset complete, route=\(routeClass)")
iosAudioLifecycleChannel?.invokeMethod("handleMediaServicesReset", arguments: routeClass)
}
override func applicationWillResignActive(_ application: UIApplication) {
iosAudioLifecycleChannel?.invokeMethod("handleWillResignActive", arguments: nil)
}
override func applicationDidEnterBackground(_ application: UIApplication) {
iosAudioLifecycleChannel?.invokeMethod("handleDidEnterBackground", arguments: nil)
}
override func applicationWillEnterForeground(_ application: UIApplication) {
iosAudioLifecycleChannel?.invokeMethod("handleWillEnterForeground", arguments: nil)
}
override func applicationWillTerminate(_ application: UIApplication) {
iosAudioLifecycleChannel?.invokeMethod("handleWillTerminate", arguments: nil)
}
func didInitializeImplicitFlutterEngine(_ engineBridge: FlutterImplicitEngineBridge) {
GeneratedPluginRegistrant.register(with: engineBridge.pluginRegistry)
iosAudioLifecycleChannel = FlutterMethodChannel(
name: "chanora/ios_audio_lifecycle",
binaryMessenger: engineBridge.applicationRegistrar.messenger()
)
iosPlatformChannel = FlutterMethodChannel(
name: "chanora/ios_platform",
binaryMessenger: engineBridge.applicationRegistrar.messenger()
)
iosPlatformChannel?.setMethodCallHandler { call, result in
switch call.method {
case "getMicrophonePermissionState":
result(self.microphonePermissionStateString())
case "requestMicrophonePermission":
AVAudioSession.sharedInstance().requestRecordPermission { granted in
DispatchQueue.main.async {
result(granted ? "Granted" : self.microphonePermissionStateString())
}
}
case "openAppSettings":
guard let url = URL(string: UIApplication.openSettingsURLString) else {
result(false)
return
}
UIApplication.shared.open(url, options: [:]) { opened in
result(opened)
}
default:
result(FlutterMethodNotImplemented)
}
}
}
private func classifyAudioRoute(_ route: AVAudioSessionRouteDescription) -> String {
for output in route.outputs {
switch output.portType {
case .builtInReceiver:
return "Earpiece"
case .builtInSpeaker:
return "Speaker"
case .headphones, .usbAudio:
return "WiredHeadset"
case .bluetoothHFP:
return "BluetoothHfp"
case .bluetoothA2DP:
return "BluetoothA2dp"
default:
break
}
}
return "Unknown"
}
private func microphonePermissionStateString() -> String {
switch AVAudioSession.sharedInstance().recordPermission {
case .granted:
return "Granted"
case .denied:
return "Denied"
case .undetermined:
return "NotDetermined"
@unknown default:
return "Unknown"
}
}
deinit {
NotificationCenter.default.removeObserver(self)
}
}
apps/chanora_flutter/ios/Runner/Info.plist
+3 -5
View File
@@ -4,6 +4,9 @@
<dict>
<key>CADisableMinimumFrameDurationOnPhone</key>
<true/>
<!-- Opt into ProMotion / high-refresh-rate CADisplayLink ranges on
supported iPhones. Flutter's iOS embedder reads this key; no
additional Flutter package is required for dynamic refresh. -->
<key>CFBundleDevelopmentRegion</key>
<string>$(DEVELOPMENT_LANGUAGE)</string>
<key>CFBundleDisplayName</key>
@@ -74,11 +77,6 @@
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
<!-- Make Chanora's Documents folder visible to the Files.app and
accessible via iTunes / Finder file-sharing. We write
diagnostic logs (chanora.log) into Documents/ so users can
export them for support. Both keys are required for the
"On My iPhone -> Chanora" listing to appear in Files.app. -->
<key>UIFileSharingEnabled</key>
<true/>
<key>LSSupportsOpeningDocumentsInPlace</key>
apps/chanora_flutter/ios/chanora_bridge.podspec
+127 -12
View File
@@ -40,7 +40,7 @@ Pod::Spec.new do |s|
s.license = { :type => 'Apache-2.0 OR MIT', :text => 'See LICENSE-APACHE / LICENSE-MIT at the repo root' }
s.author = { 'EdisonJwa' => 'me@edison.network' }
s.source = { :path => '.' }
s.platform = :ios, '13.0'
s.platform = :ios, '15.1'
# Build the Rust bridge on `pod install`. The script runs under
# bash; we use `set -e` so any failure (cargo missing, target not
@@ -51,15 +51,59 @@ Pod::Spec.new do |s|
s.prepare_command = <<-SCRIPT
set -e
REPO_ROOT="$(cd ../../.. && pwd)"
USER_NAME="$(id -un)"
USER_HOME="$(dscl . -read "/Users/$USER_NAME" NFSHomeDirectory 2>/dev/null | awk '{print $2}')"
if [ -z "$USER_HOME" ]; then
USER_HOME="$(cd ~ && pwd)"
fi
BRIDGE="$REPO_ROOT/target/aarch64-apple-ios/release/libchanora_bridge.dylib"
find_cargo() {
for candidate in \
"$USER_HOME/.cargo/bin/cargo" \
"/opt/homebrew/opt/rustup/bin/cargo" \
"/usr/local/opt/rustup/bin/cargo"
do
if [ -x "$candidate" ]; then
echo "$candidate"
return 0
fi
done
command -v cargo
}
find_rustc() {
for candidate in \
"$USER_HOME/.cargo/bin/rustc" \
"/opt/homebrew/opt/rustup/bin/rustc" \
"/usr/local/opt/rustup/bin/rustc"
do
if [ -x "$candidate" ]; then
echo "$candidate"
return 0
fi
done
command -v rustc
}
CARGO_BIN="$(find_cargo)"
RUSTC_BIN="$(find_rustc)"
ORT_FRAMEWORK="$REPO_ROOT/apps/chanora_flutter/ios/Pods/onnxruntime-c/onnxruntime.xcframework/ios-arm64/onnxruntime.framework"
ORT_LINK_DIR="$REPO_ROOT/target/onnxruntime-ios-device"
if [ -f "$ORT_FRAMEWORK/onnxruntime" ]; then
mkdir -p "$ORT_LINK_DIR"
lipo "$ORT_FRAMEWORK/onnxruntime" -thin arm64 -output "$ORT_LINK_DIR/libonnxruntime.a"
fi
echo "[chanora_bridge.podspec] cargo build aarch64-apple-ios"
cd "$REPO_ROOT"
PATH="$HOME/.cargo/bin:$PATH" \\
IPHONEOS_DEPLOYMENT_TARGET=13.0 \\
HOME="$USER_HOME" \\
CARGO_HOME="$USER_HOME/.cargo" \\
RUSTUP_HOME="$USER_HOME/.rustup" \\
RUSTUP_TOOLCHAIN="stable-aarch64-apple-darwin" \\
RUSTC="$RUSTC_BIN" \\
ORT_LIB_LOCATION="$ORT_LINK_DIR" \\
IPHONEOS_DEPLOYMENT_TARGET=15.1 \\
CMAKE_POLICY_VERSION_MINIMUM=3.5 \\
CMAKE_OSX_DEPLOYMENT_TARGET=13.0 \\
cargo build --release --target aarch64-apple-ios -p chanora_bridge
CMAKE_OSX_DEPLOYMENT_TARGET=15.1 \\
"$CARGO_BIN" build --release --target aarch64-apple-ios -p chanora_bridge
if [ ! -f "$BRIDGE" ]; then
echo "ERROR: bridge dylib not found at $BRIDGE" >&2
@@ -86,7 +130,7 @@ Pod::Spec.new do |s|
<key>CFBundleShortVersionString</key><string>1.0.0</string>
<key>CFBundleVersion</key><string>1</string>
<key>CFBundleSupportedPlatforms</key><array><string>iPhoneOS</string></array>
<key>MinimumOSVersion</key><string>13.0</string>
<key>MinimumOSVersion</key><string>15.1</string>
</dict>
</plist>
PLIST
@@ -116,15 +160,70 @@ PLIST
:script => <<-SCRIPT,
set -e
REPO_ROOT="$(cd "${PODS_TARGET_SRCROOT}/../../.." && pwd)"
BRIDGE="$REPO_ROOT/target/aarch64-apple-ios/release/libchanora_bridge.dylib"
USER_NAME="$(id -un)"
USER_HOME="$(dscl . -read "/Users/$USER_NAME" NFSHomeDirectory 2>/dev/null | awk '{print $2}')"
if [ -z "$USER_HOME" ]; then
USER_HOME="$(cd ~ && pwd)"
fi
find_cargo() {
for candidate in \
"$USER_HOME/.cargo/bin/cargo" \
"/opt/homebrew/opt/rustup/bin/cargo" \
"/usr/local/opt/rustup/bin/cargo"
do
if [ -x "$candidate" ]; then
echo "$candidate"
return 0
fi
done
command -v cargo
}
find_rustc() {
for candidate in \
"$USER_HOME/.cargo/bin/rustc" \
"/opt/homebrew/opt/rustup/bin/rustc" \
"/usr/local/opt/rustup/bin/rustc"
do
if [ -x "$candidate" ]; then
echo "$candidate"
return 0
fi
done
command -v rustc
}
CARGO_BIN="$(find_cargo)"
RUSTC_BIN="$(find_rustc)"
if [ "${PLATFORM_NAME:-iphoneos}" = "iphonesimulator" ]; then
RUST_TARGET="aarch64-apple-ios-sim"
SUPPORTED_PLATFORM="iPhoneSimulator"
ORT_SLICE="ios-arm64_x86_64-simulator"
else
RUST_TARGET="aarch64-apple-ios"
SUPPORTED_PLATFORM="iPhoneOS"
ORT_SLICE="ios-arm64"
fi
BRIDGE="$REPO_ROOT/target/$RUST_TARGET/release/libchanora_bridge.dylib"
ORT_FRAMEWORK="$REPO_ROOT/apps/chanora_flutter/ios/Pods/onnxruntime-c/onnxruntime.xcframework/$ORT_SLICE/onnxruntime.framework"
ORT_LINK_DIR="$REPO_ROOT/target/onnxruntime-$RUST_TARGET"
if [ ! -f "$ORT_FRAMEWORK/onnxruntime" ]; then
echo "ERROR: ONNX Runtime framework not found at $ORT_FRAMEWORK" >&2
exit 1
fi
mkdir -p "$ORT_LINK_DIR"
lipo "$ORT_FRAMEWORK/onnxruntime" -thin arm64 -output "$ORT_LINK_DIR/libonnxruntime.a"
echo "[chanora_bridge script_phase] cargo build aarch64-apple-ios"
echo "[chanora_bridge script_phase] cargo build $RUST_TARGET"
cd "$REPO_ROOT"
PATH="$HOME/.cargo/bin:$PATH" \\
IPHONEOS_DEPLOYMENT_TARGET=13.0 \\
HOME="$USER_HOME" \\
CARGO_HOME="$USER_HOME/.cargo" \\
RUSTUP_HOME="$USER_HOME/.rustup" \\
RUSTUP_TOOLCHAIN="stable-aarch64-apple-darwin" \\
RUSTC="$RUSTC_BIN" \\
ORT_LIB_LOCATION="$ORT_LINK_DIR" \\
IPHONEOS_DEPLOYMENT_TARGET=15.1 \\
CMAKE_POLICY_VERSION_MINIMUM=3.5 \\
CMAKE_OSX_DEPLOYMENT_TARGET=13.0 \\
cargo build --release --target aarch64-apple-ios -p chanora_bridge
CMAKE_OSX_DEPLOYMENT_TARGET=15.1 \\
"$CARGO_BIN" build --release --target "$RUST_TARGET" -p chanora_bridge
cd "$REPO_ROOT/apps/chanora_flutter/ios"
FW=Frameworks/chanora_bridge.framework
@@ -139,6 +238,22 @@ PLIST
mkdir -p "$FW"
cp "$BRIDGE" "$FW/chanora_bridge"
cat > "$FW/Info.plist" <<PLIST
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleExecutable</key><string>chanora_bridge</string>
<key>CFBundleIdentifier</key><string>app.chanora.bridge</string>
<key>CFBundleName</key><string>chanora_bridge</string>
<key>CFBundlePackageType</key><string>FMWK</string>
<key>CFBundleShortVersionString</key><string>1.0.0</string>
<key>CFBundleVersion</key><string>1</string>
<key>CFBundleSupportedPlatforms</key><array><string>$SUPPORTED_PLATFORM</string></array>
<key>MinimumOSVersion</key><string>15.1</string>
</dict>
</plist>
PLIST
install_name_tool -id "@rpath/chanora_bridge.framework/chanora_bridge" \\
"$FW/chanora_bridge"
echo "[chanora_bridge script_phase] framework refreshed"