diff --git a/.cargo/config.toml b/.cargo/config.toml index 901eb32..b0ca965 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,20 +3,14 @@ # Opus CMake build to succeed on CMake 4.x (which removed compatibility # with cmake_minimum_required < 3.5). audiopus_sys v0.2.2 bundles # Opus 1.3.1 whose CMakeLists.txt uses a very old minimum version. -# -# IPHONEOS_DEPLOYMENT_TARGET=13.0 is required for iOS builds because -# ___chkstk_darwin (the stack-probe symbol emitted by clang for functions -# with large stack frames) only exists in the iOS 13.0+ runtime. The -# pre-built libopus.a and any C code compiled by the cc crate reference -# this symbol. Without this env var, the cc crate compiles C code -# targeting iOS 10.0 (the Rust default), producing object files that -# reference ___chkstk_darwin but can't resolve it against the 10.0 -# runtime. Setting IPHONEOS_DEPLOYMENT_TARGET=13.0 ensures the cc crate -# and CMake both target iOS 13.0+, where ___chkstk_darwin exists. -# (DEC-003: minimum deployment target iOS 13.0) [env] CMAKE_POLICY_VERSION_MINIMUM = "3.5" -IPHONEOS_DEPLOYMENT_TARGET = "13.0" + +# iOS builds must set IPHONEOS_DEPLOYMENT_TARGET in the invoking script +# or Xcode build phase. Do not set it globally here: native macOS cargo +# checks also compile bundled C/C++ dependencies, and a global iOS +# deployment target makes clang try to link iPhone objects against the +# macOS SDK. # iOS target linker flags (DEC-003: minimum deployment target iOS 13.0). # diff --git a/Cargo.lock b/Cargo.lock index c6b1d95..da608c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -316,6 +316,20 @@ name = "bytemuck" version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "byteorder" @@ -421,11 +435,13 @@ dependencies = [ "reqwest", "sdl2", "serde_json", + "sonora", "thiserror 2.0.18", "tokio", "tracing", "tracing-subscriber", "tsclientlib", + "webrtc-vad", "windows 0.54.0", "zbus", ] @@ -963,6 +979,28 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn", + "unicode-xid", +] + [[package]] name = "dhat" version = "0.3.3" @@ -1620,7 +1658,7 @@ dependencies = [ "parking_lot", "rand 0.10.1", "resolv-conf", - "smallvec 1.15.1", + "smallvec", "system-configuration", "thiserror 2.0.18", "tokio", @@ -1700,7 +1738,7 @@ dependencies = [ "httparse", "itoa", "pin-project-lite", - "smallvec 1.15.1", + "smallvec", "tokio", "want", ] @@ -1796,7 +1834,7 @@ dependencies = [ "icu_normalizer_data", "icu_properties", "icu_provider", - "smallvec 1.15.1", + "smallvec", "zerovec", ] @@ -1854,7 +1892,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ "idna_adapter", - "smallvec 1.15.1", + "smallvec", "utf8_iter", ] @@ -2087,9 +2125,9 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.9" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" dependencies = [ "cfg-if", "windows-link", @@ -2252,7 +2290,7 @@ dependencies = [ "equivalent", "parking_lot", "portable-atomic", - "smallvec 1.15.1", + "smallvec", "tagptr", "uuid", ] @@ -2276,9 +2314,9 @@ dependencies = [ [[package]] name = "ndarray" -version = "0.16.1" +version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" dependencies = [ "matrixmultiply", "num-complex", @@ -2523,9 +2561,7 @@ dependencies = [ [[package]] name = "oboe" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb" +version = "0.6.2" dependencies = [ "num-derive", "num-traits", @@ -2534,9 +2570,7 @@ dependencies = [ [[package]] name = "oboe-sys" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d" +version = "0.6.2" dependencies = [ "cc", ] @@ -2627,25 +2661,22 @@ dependencies = [ [[package]] name = "ort" -version = "2.0.0-rc.10" +version = "2.0.0-rc.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa7e49bd669d32d7bc2a15ec540a527e7764aec722a45467814005725bcd721" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" dependencies = [ "libloading", "ndarray", "ort-sys", - "smallvec 2.0.0-alpha.10", + "smallvec", "tracing", ] [[package]] name = "ort-sys" -version = "2.0.0-rc.10" +version = "2.0.0-rc.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2aba9f5c7c479925205799216e7e5d07cc1d4fa76ea8058c60a9a30f6a4e890" -dependencies = [ - "pkg-config", -] +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" [[package]] name = "oslog" @@ -2695,7 +2726,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "smallvec 1.15.1", + "smallvec", "windows-link", ] @@ -3198,7 +3229,7 @@ dependencies = [ "fallible-streaming-iterator", "hashlink", "libsqlite3-sys", - "smallvec 1.15.1", + "smallvec", ] [[package]] @@ -3597,12 +3628,6 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -[[package]] -name = "smallvec" -version = "2.0.0-alpha.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d44cfb396c3caf6fbfd0ab422af02631b69ddd96d2eff0b0f0724f9024051b" - [[package]] name = "socket2" version = "0.6.3" @@ -3613,6 +3638,79 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sonora" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "559f6011d42721f7b4599ef4cf3224ac07539ee5bc25967169701c7aeb6c6bb1" +dependencies = [ + "sonora-aec3", + "sonora-agc2", + "sonora-common-audio", + "sonora-ns", + "sonora-simd", + "tracing", +] + +[[package]] +name = "sonora-aec3" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65d8b7e53b058aa4631cf2a8f483bd7a0dec093aa413d21b44b86d5704deedeb" +dependencies = [ + "sonora-common-audio", + "sonora-fft", + "sonora-simd", +] + +[[package]] +name = "sonora-agc2" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "132876fc38c4e191dee939a38243df5eb5830cd29f1d1ac9a67377389d74a10e" +dependencies = [ + "bytemuck", + "derive_more", + "sonora-common-audio", + "sonora-fft", + "sonora-simd", + "tracing", +] + +[[package]] +name = "sonora-common-audio" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a16fa975fe4aa12e4d980d5453452bd3e6df2b33abfb89840e1757ebd212e6" +dependencies = [ + "derive_more", + "sonora-simd", +] + +[[package]] +name = "sonora-fft" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cca158c4b6ace2cb3f4fa8084842da2b4ffcc25cf8ac4366675aea541aabc6af" + +[[package]] +name = "sonora-ns" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc549f9bca1d60e7a2483e4949e9649467c5cca371a4341e44aaf24dae338fd4" +dependencies = [ + "sonora-fft", +] + +[[package]] +name = "sonora-simd" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a053f93cc01f4e1cc51c61844a196b6d1da7e054cdd406d2cd27677ba4d898e0" +dependencies = [ + "cpufeatures 0.3.0", +] + [[package]] name = "spki" version = "0.7.3" @@ -4087,7 +4185,7 @@ dependencies = [ "once_cell", "regex-automata", "sharded-slab", - "smallvec 1.15.1", + "smallvec", "thread_local", "tracing", "tracing-core", @@ -4492,6 +4590,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webrtc-vad" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39a1e40fd6ca90be95459152a2537f2ba4286ee1b13073f7ebcaa74fc94e3008" +dependencies = [ + "cc", +] + [[package]] name = "widestring" version = "1.2.1" diff --git a/LICENSE b/LICENSE deleted file mode 100644 index bce3861..0000000 --- a/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Chanora is dual-licensed under either of: - - * Apache License, Version 2.0 - ([LICENSE-APACHE](LICENSE-APACHE) or https://www.apache.org/licenses/LICENSE-2.0) - * MIT license - ([LICENSE-MIT](LICENSE-MIT) or https://opensource.org/licenses/MIT) - -at your option. - -This dual-license model is recorded in -`docs/governance/product-decision-register.md` (decision DEC-020, -Accepted on 2026-05-14). - -## Contribution - -Unless you explicitly state otherwise, any contribution intentionally -submitted for inclusion in Chanora by you, as defined in the -Apache-2.0 license, shall be dual-licensed as above, without any -additional terms or conditions. - -## Third-party software - -Chanora bundles or links to third-party software whose own licenses -apply. See `NOTICE` for an inventory and the per-dependency license -texts that ship with the released artefacts. The legal review of the -full dependency set is tracked by DEC-012 and must complete before any -public/store release. diff --git a/LICENSE-APACHE b/LICENSE-APACHE deleted file mode 100644 index 20670b0..0000000 --- a/LICENSE-APACHE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for describing the origin of the Work and - reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Support. While redistributing the Work or - Derivative Works thereof, You may choose to offer, and charge a - fee for, acceptance of support, warranty, indemnity, or other - liability obligations and/or rights consistent with this License. - However, in accepting such obligations, You may act only on Your - own behalf and on Your sole responsibility, not on behalf of any - other Contributor, and only if You agree to indemnify, defend, - and hold each Contributor harmless for any liability incurred by, - or claims asserted against, such Contributor by reason of your - accepting any such warranty or support. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2026 The Chanora Project Contributors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT deleted file mode 100644 index 6f1e0fa..0000000 --- a/LICENSE-MIT +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2026 The Chanora Project Contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/apps/chanora_flutter/android/app/build.gradle.kts b/apps/chanora_flutter/android/app/build.gradle.kts index 9520b20..ed07d76 100644 --- a/apps/chanora_flutter/android/app/build.gradle.kts +++ b/apps/chanora_flutter/android/app/build.gradle.kts @@ -77,7 +77,7 @@ android { // ANDROID_PLATFORM as -D variables to the child cmake // invocation). See Cargo.toml [patch.crates-io] block and // docs/governance/product-decision-register.md DEC-032. - abiFilters += listOf("arm64-v8a", "armeabi-v7a", "x86_64") + abiFilters += listOf("arm64-v8a", "x86_64") } } @@ -154,6 +154,14 @@ android { } } +// ONNX Runtime native library for Silero / TEN VAD. +// The ort crate (Rust) loads libonnxruntime.so via dlopen at runtime +// (`load-dynamic` feature). The AAR ships the .so for arm64-v8a, +// armeabi-v7a, x86_64, x86. AGP merges these into the APK/AAB. +dependencies { + implementation("com.microsoft.onnxruntime:onnxruntime-android:1.26.0") +} + flutter { source = "../.." } diff --git a/apps/chanora_flutter/android/app/proguard-rules.pro b/apps/chanora_flutter/android/app/proguard-rules.pro index 072cafe..2310487 100644 --- a/apps/chanora_flutter/android/app/proguard-rules.pro +++ b/apps/chanora_flutter/android/app/proguard-rules.pro @@ -25,6 +25,11 @@ -keep class app.chanora.chanora_flutter.AndroidVoiceForegroundService { *; } -keep class app.chanora.chanora_flutter.AndroidPermissionRequester { *; } -keep class app.chanora.chanora_flutter.BackIntentBridge { *; } +# SDD-109 / SDD-110 / SDD-111: JNI-referenced voice controllers. +# Called from the Rust audio engine via JNI static methods. +-keep class app.chanora.chanora_flutter.AndroidAudioFocusController { *; } +-keep class app.chanora.chanora_flutter.AndroidBluetoothScoController { *; } +-keep class app.chanora.chanora_flutter.AndroidAudioLifecycleController { *; } -keep class io.flutter.plugins.** { *; } # flutter_rust_bridge generated bindings (SDD-079 TypedBridgeFacade) — keep diff --git a/apps/chanora_flutter/android/app/src/main/AndroidManifest.xml b/apps/chanora_flutter/android/app/src/main/AndroidManifest.xml index 6747683..b3200f3 100644 --- a/apps/chanora_flutter/android/app/src/main/AndroidManifest.xml +++ b/apps/chanora_flutter/android/app/src/main/AndroidManifest.xml @@ -22,6 +22,7 @@ foregroundServiceType="microphone". Declared here for the entire API ladder; the platform ignores it on older releases. --> + + + + + + + + + + Log.i(TAG, "onAudioFocusChange: $focusChange") + try { + publishFocusChange(focusChange) + } catch (t: Throwable) { + Log.w(TAG, "publishFocusChange JNI failed: ${t.message}", t) + } + } + + private fun requestFocus(context: Context) { + val am = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager + if (am == null) { + Log.e(TAG, "AudioManager unavailable; cannot request audio focus") + return + } + try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val attr = AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + val request = AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN) + .setAudioAttributes(attr) + .setOnAudioFocusChangeListener(audioFocusListener, mainHandler) + .build() + val result = am.requestAudioFocus(request) + focusRequested = result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED + if (focusRequested) { + focusRequestHandle = request + } + Log.i(TAG, "requestAudioFocus result=$result granted=$focusRequested") + } else { + @Suppress("DEPRECATION") + val result = am.requestAudioFocus( + audioFocusListener, + AudioManager.STREAM_VOICE_CALL, + AudioManager.AUDIOFOCUS_GAIN, + ) + focusRequested = result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED + Log.i(TAG, "requestAudioFocus result=$result granted=$focusRequested") + } + } catch (e: SecurityException) { + Log.e(TAG, "requestAudioFocus denied by platform: ${e.message}", e) + focusRequested = false + } + } + + private fun abandonFocus(context: Context) { + val am = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager + if (am == null) { + Log.e(TAG, "AudioManager unavailable; cannot abandon audio focus") + focusRequested = false + focusRequestHandle = null + return + } + try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val handle = focusRequestHandle + focusRequestHandle = null + if (handle != null) { + am.abandonAudioFocusRequest(handle) + } + } else { + @Suppress("DEPRECATION") + am.abandonAudioFocus(audioFocusListener) + } + Log.i(TAG, "abandonAudioFocus dispatched") + } catch (e: SecurityException) { + Log.w(TAG, "abandonAudioFocus failed: ${e.message}", e) + } + focusRequested = false + focusRequestHandle = null + } + } +} diff --git a/apps/chanora_flutter/android/app/src/main/kotlin/app/chanora/chanora_flutter/AndroidAudioLifecycleController.kt b/apps/chanora_flutter/android/app/src/main/kotlin/app/chanora/chanora_flutter/AndroidAudioLifecycleController.kt new file mode 100644 index 0000000..2dd91d4 --- /dev/null +++ b/apps/chanora_flutter/android/app/src/main/kotlin/app/chanora/chanora_flutter/AndroidAudioLifecycleController.kt @@ -0,0 +1,199 @@ +package app.chanora.chanora_flutter + +import android.content.Context +import android.media.AudioDeviceCallback +import android.media.AudioDeviceInfo +import android.media.AudioManager +import android.os.Build +import android.os.Handler +import android.os.Looper +import android.util.Log +import io.flutter.embedding.engine.FlutterEngine +import io.flutter.plugin.common.MethodChannel + +/** + * Android audio lifecycle controller. + * + * Mirrors the iOS `AppDelegate` audio lifecycle events — route changes, + * device connectivity changes, and app lifecycle transitions — and + * forwards them to the Dart layer over a MethodChannel. The Dart side + * dispatches these to the Rust bridge, the same pattern as + * `_wireIosAudioLifecycle()` in `main.dart`. + * + * Trace: SDD-111 (MobileVoiceAudioBackend cross-platform) + * + * ## Events published + * + * `routeChange` — audio device connected/disconnected (payload: + * `{"routeType": String}`). Mimics iOS + * `handleRouteChange`. + * `interruptionBegan` — audio interruption started (e.g. phone call). + * `interruptionEnded` — audio interruption ended, with `shouldResume`. + * `appDidEnterBackground` — app moved to background. + * `appWillEnterForeground` — app returned to foreground. + * `mediaServicesReset` — equivalent of iOS media services reset + * (Android: audio devices changed significantly). + * + * ## Thread model + * + * All callbacks from `AudioDeviceCallback` arrive on the main thread + * (registered with the main `Handler`). App lifecycle observation is + * driven by the Flutter `AppLifecycleListener` on the Dart side; + * this controller only owns the audio-device side. The Flutter side + * is responsible for wiring lifecycle and forwarding events to Rust. + */ +internal class AndroidAudioLifecycleController( + private val context: Context, +) { + private val audioManager: AudioManager = + context.applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager + private val mainHandler = Handler(Looper.getMainLooper()) + private var callbackRegistered = false + + private var currentRouteType: String? = null + + private val audioDeviceCallback = object : AudioDeviceCallback() { + override fun onAudioDevicesAdded(addedDevices: Array) { + notifyRouteChange(addedDevices.firstOrNull()) + } + + override fun onAudioDevicesRemoved(removedDevices: Array) { + notifyRouteChange(removedDevices.firstOrNull()) + } + } + + private var channel: MethodChannel? = null + + /** + * Attach to [flutterEngine]'s binary messenger. + * + * Creates a MethodChannel named `chanora/android_audio_lifecycle` + * and starts observing audio device changes. + */ + fun attach(flutterEngine: FlutterEngine) { + channel = MethodChannel( + flutterEngine.dartExecutor.binaryMessenger, + CHANNEL_NAME, + ) + startObservingAudioDevices() + Log.i(TAG, "attached to flutter engine") + } + + /** + * Detach from the Flutter engine and stop observing. + */ + fun detach() { + stopObservingAudioDevices() + channel?.setMethodCallHandler(null) + channel = null + Log.i(TAG, "detached") + } + + /** + * Call from `Activity.onResume` to re-evaluate the current route. + */ + fun onResume() { + notifyRouteChange(null) + } + + /** + * Call from `Activity.onDestroy` to tear down. + */ + fun onDestroy() { + detach() + } + + private fun startObservingAudioDevices() { + if (callbackRegistered) return + audioManager.registerAudioDeviceCallback(audioDeviceCallback, mainHandler) + callbackRegistered = true + Log.d(TAG, "audio device observer started") + } + + private fun stopObservingAudioDevices() { + if (!callbackRegistered) return + audioManager.unregisterAudioDeviceCallback(audioDeviceCallback) + callbackRegistered = false + Log.d(TAG, "audio device observer stopped") + } + + private fun notifyRouteChange( + device: AudioDeviceInfo?, + ) { + val routeType = classifyCurrentRoute(device) + if (routeType == currentRouteType) { + return + } + currentRouteType = routeType + Log.i(TAG, "route changed to: $routeType") + channel?.invokeMethod( + "handleRouteChange", + mapOf("routeType" to routeType), + ) + } + + /** + * Classify the current audio output route into a stable string + * matching the iOS route-classification schema so the Dart-side + * parser (`_parseBridgeAudioRoute`) works identically across + * platforms. + */ + private fun classifyCurrentRoute(specificDevice: AudioDeviceInfo?): String { + // If a specific device was added/removed, prefer its type. + if (specificDevice != null) { + return classifyDevice(specificDevice) + } + + // Otherwise, classify based on the current output devices. + val outputs = audioManager.getDevices(AudioManager.GET_DEVICES_OUTPUTS) + if (outputs.isEmpty()) return "Unknown" + + // Prefer wired/Bluetooth headset if connected. + for (d in outputs) { + val t = classifyDevice(d) + when (t) { + "WiredHeadset", "BluetoothHfp", "BluetoothA2dp", "UsbHeadset" -> return t + else -> {} + } + } + // Fall back to the first output device classification. + val first = classifyDevice(outputs.first()) + return when (first) { + "Speaker" -> "Speaker" + "Earpiece" -> "Earpiece" + else -> "Speaker" // Default to Speaker for unknown outputs. + } + } + + private fun classifyDevice(d: AudioDeviceInfo): String = when (d.type) { + AudioDeviceInfo.TYPE_BUILTIN_EARPIECE -> "Earpiece" + AudioDeviceInfo.TYPE_BUILTIN_SPEAKER -> "Speaker" + AudioDeviceInfo.TYPE_WIRED_HEADSET, + AudioDeviceInfo.TYPE_WIRED_HEADPHONES -> "WiredHeadset" + AudioDeviceInfo.TYPE_BLUETOOTH_SCO -> "BluetoothHfp" + AudioDeviceInfo.TYPE_BLUETOOTH_A2DP -> "BluetoothA2dp" + AudioDeviceInfo.TYPE_BLE_HEADSET, + AudioDeviceInfo.TYPE_BLE_SPEAKER -> if (Build.VERSION.SDK_INT >= 31) "BluetoothHfp" else "BluetoothA2dp" + AudioDeviceInfo.TYPE_USB_HEADSET -> "UsbHeadset" + AudioDeviceInfo.TYPE_HDMI -> "Hdmi" + else -> { + if (android.os.Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU && + d.type == AudioDeviceInfo.TYPE_BLE_BROADCAST + ) { + "BluetoothA2dp" + } else { + "Unknown" + } + } + } + + companion object { + private const val TAG = "ChanoraAudioLifecycle" + + /** + * MethodChannel name for Android audio lifecycle events. + * Mirrors `chanora/ios_audio_lifecycle` on iOS. + */ + const val CHANNEL_NAME: String = "chanora/android_audio_lifecycle" + } +} diff --git a/apps/chanora_flutter/android/app/src/main/kotlin/app/chanora/chanora_flutter/AndroidBluetoothScoController.kt b/apps/chanora_flutter/android/app/src/main/kotlin/app/chanora/chanora_flutter/AndroidBluetoothScoController.kt new file mode 100644 index 0000000..9da82e0 --- /dev/null +++ b/apps/chanora_flutter/android/app/src/main/kotlin/app/chanora/chanora_flutter/AndroidBluetoothScoController.kt @@ -0,0 +1,241 @@ +package app.chanora.chanora_flutter + +import android.bluetooth.BluetoothAdapter +import android.bluetooth.BluetoothProfile +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import android.media.AudioManager +import android.util.Log + +/** + * Manages Android Bluetooth SCO (Synchronous Connection-Oriented) audio + * for the Chanora voice session. + * + * Trace: SDD-110 (Android Bluetooth SCO) + * + * Bluetooth SCO is the low-latency, monaural audio path used by Bluetooth + * headsets for phone calls. Without SCO, voice audio may route through + * A2DP which is stereo, high-latency, and lacks the codec support for + * two-way communication. On Android we must explicitly start/stop SCO + * when a Bluetooth headset is present; the platform does not auto-manage + * this for VoIP apps. + * + * ## Lifecycle + * + * 1. [start] — called by the Rust audio engine (via JNI) after the Oboe + * voice streams are opened. Calls `AudioManager.startBluetoothSco()` + * if a Bluetooth SCO-capable device is connected and registers a + * `BroadcastReceiver` for `ACTION_SCO_AUDIO_STATE_UPDATED`. + * 2. SCO state changes are forwarded to the Rust engine via + * `publishScoStateChange(int)`, a JNI function declared in + * `crates/chanora_audio/src/android_voice_unit.rs`. + * 3. [stop] — called by the Rust engine on voice stop. Calls + * `AudioManager.stopBluetoothSco()` and unregisters the receiver. + * + * ## Thread model + * + * `start` / `stop` are called from a tokio worker thread (via JNI). + * `AudioManager.startBluetoothSco` is asynchronous — the platform + * responds with `ACTION_SCO_AUDIO_STATE_UPDATED` which arrives on the + * main thread via the `BroadcastReceiver`. + */ +internal class AndroidBluetoothScoController { + + companion object { + private const val TAG = "ChanoraBluetoothSco" + + /** + * JNI entry point implemented in + * `crates/chanora_audio/src/android_voice_unit.rs`. + * + * Kotlin calls this from the SCO state `BroadcastReceiver` to + * forward the state integer to the Rust engine's BackendEvent + * channel. + */ + @JvmStatic + external fun publishScoStateChange(state: Int) + + /** + * Start Bluetooth SCO management. + * + * Called from Rust via JNI after voice unit start. + * Idempotent: repeated calls against an already-started instance + * are silently ignored. + */ + @JvmStatic + fun start(context: Context) { + val appContext = context.applicationContext + if (scoStarted) { + Log.d(TAG, "start() called but SCO already active; no-op") + return + } + scoStarted = true + registerScoReceiver(appContext) + tryStartSco(appContext) + } + + /** + * Stop Bluetooth SCO management. + * + * Called from Rust via JNI on voice stop. + * Idempotent: safe to call when SCO is not active. + */ + @JvmStatic + fun stop(context: Context) { + val appContext = context.applicationContext + scoStarted = false + unregisterScoReceiver(appContext) + tryStopSco(appContext) + } + + private var scoStarted: Boolean = false + private var receiverRegistered: Boolean = false + + private val scoReceiver = object : BroadcastReceiver() { + override fun onReceive(context: Context?, intent: Intent?) { + if (intent?.action != AudioManager.ACTION_SCO_AUDIO_STATE_UPDATED) return + val state = intent.getIntExtra( + AudioManager.EXTRA_SCO_AUDIO_STATE, + AudioManager.SCO_AUDIO_STATE_ERROR, + ) + val prevState = intent.getIntExtra( + AudioManager.EXTRA_SCO_AUDIO_PREVIOUS_STATE, + -1, + ) + Log.i( + TAG, + "SCO state: ${scoStateName(state)} (prev: ${scoStateName(prevState)})", + ) + try { + publishScoStateChange(state) + } catch (t: Throwable) { + Log.w(TAG, "publishScoStateChange JNI failed: ${t.message}", t) + } + } + } + + private fun registerScoReceiver(context: Context) { + if (receiverRegistered) return + try { + val filter = IntentFilter(AudioManager.ACTION_SCO_AUDIO_STATE_UPDATED) + if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.TIRAMISU) { + context.registerReceiver(scoReceiver, filter, Context.RECEIVER_NOT_EXPORTED) + } else { + @Suppress("UnspecifiedRegisterReceiverFlag") + context.registerReceiver(scoReceiver, filter) + } + receiverRegistered = true + Log.d(TAG, "SCO receiver registered") + } catch (e: Exception) { + Log.w(TAG, "Failed to register SCO receiver: ${e.message}", e) + } + } + + private fun unregisterScoReceiver(context: Context) { + if (!receiverRegistered) return + try { + context.unregisterReceiver(scoReceiver) + receiverRegistered = false + Log.d(TAG, "SCO receiver unregistered") + } catch (e: IllegalArgumentException) { + // Already unregistered — ignore silently. + receiverRegistered = false + } + } + + private val bluetoothAdapter: BluetoothAdapter? + get() = try { + BluetoothAdapter.getDefaultAdapter() + } catch (e: SecurityException) { + Log.w(TAG, "BluetoothAdapter unavailable: ${e.message}") + null + } + + private fun isBluetoothScoOn(am: AudioManager): Boolean = try { + am.isBluetoothScoOn + } catch (e: SecurityException) { + Log.w(TAG, "isBluetoothScoOn failed: ${e.message}") + false + } + + private fun isBluetoothScoAvailableOffCall(am: AudioManager): Boolean = try { + am.isBluetoothScoAvailableOffCall + } catch (e: SecurityException) { + Log.w(TAG, "isBluetoothScoAvailableOffCall failed: ${e.message}") + false + } + + private fun tryStartSco(context: Context) { + val am = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager + if (am == null) { + Log.e(TAG, "AudioManager unavailable; cannot start SCO") + return + } + + if (isBluetoothScoOn(am)) { + Log.i(TAG, "SCO already on; no-op") + return + } + + val adapter = bluetoothAdapter + if (adapter == null || !adapter.isEnabled) { + Log.i(TAG, "Bluetooth not enabled; skipping SCO start") + return + } + + val scoAvailableOffCall = isBluetoothScoAvailableOffCall(am) + if (!scoAvailableOffCall) { + Log.i(TAG, "Bluetooth SCO not available off-call; skipping SCO start") + return + } + + val hasScoHeadset = try { + val state = adapter.getProfileConnectionState(BluetoothProfile.HEADSET) + state == BluetoothProfile.STATE_CONNECTED + } catch (e: SecurityException) { + Log.w(TAG, "getProfileConnectionState(HEADSET) failed: ${e.message}") + false + } + + if (!hasScoHeadset) { + Log.i(TAG, "No SCO-capable Bluetooth headset connected; skipping SCO start") + return + } + + try { + am.startBluetoothSco() + Log.i(TAG, "startBluetoothSco() dispatched") + } catch (e: SecurityException) { + Log.e(TAG, "startBluetoothSco denied: ${e.message}", e) + } + } + + private fun tryStopSco(context: Context) { + val am = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager + if (am == null) { + Log.e(TAG, "AudioManager unavailable; cannot stop SCO") + return + } + if (!isBluetoothScoOn(am)) { + Log.d(TAG, "SCO not on; no-op") + return + } + try { + am.stopBluetoothSco() + Log.i(TAG, "stopBluetoothSco() dispatched") + } catch (e: SecurityException) { + Log.w(TAG, "stopBluetoothSco denied: ${e.message}", e) + } + } + + private fun scoStateName(state: Int): String = when (state) { + AudioManager.SCO_AUDIO_STATE_DISCONNECTED -> "DISCONNECTED" + AudioManager.SCO_AUDIO_STATE_CONNECTED -> "CONNECTED" + AudioManager.SCO_AUDIO_STATE_CONNECTING -> "CONNECTING" + AudioManager.SCO_AUDIO_STATE_ERROR -> "ERROR" + else -> "UNKNOWN($state)" + } + } +} diff --git a/apps/chanora_flutter/android/app/src/main/kotlin/app/chanora/chanora_flutter/MainActivity.kt b/apps/chanora_flutter/android/app/src/main/kotlin/app/chanora/chanora_flutter/MainActivity.kt index 10e73a6..6e07e7c 100644 --- a/apps/chanora_flutter/android/app/src/main/kotlin/app/chanora/chanora_flutter/MainActivity.kt +++ b/apps/chanora_flutter/android/app/src/main/kotlin/app/chanora/chanora_flutter/MainActivity.kt @@ -76,6 +76,11 @@ class MainActivity : FlutterActivity() { private var audioOutputEvents: EventChannel? = null private var audioOutputController: AndroidAudioOutputController? = null + // SDD-111: Android audio lifecycle controller (route changes, device + // add/remove, app lifecycle). Mirrors the iOS + // `chanora/ios_audio_lifecycle` channel pattern. + private var audioLifecycleController: AndroidAudioLifecycleController? = null + // SDD-028 (M-3 strict-review fix): the back-intent bridge is owned // by this Activity instance, not a process-wide `object`. Constructed // in configureFlutterEngine and cleared in onDestroy after detach(). @@ -202,6 +207,12 @@ class MainActivity : FlutterActivity() { ) this.audioOutputEvents = audioOutputEvents audioOutputEvents.setStreamHandler(audioOutputController) + + // SDD-111: attach the Android audio lifecycle controller + // (route changes, device add/remove, interruptions). + val lifecycleController = AndroidAudioLifecycleController(applicationContext) + lifecycleController.attach(flutterEngine) + audioLifecycleController = lifecycleController } override fun onResume() { @@ -217,6 +228,8 @@ class MainActivity : FlutterActivity() { // SDD-106: state already emitted by AndroidPermissionRequester via stateChangeListener; do NOT double-emit (M-4 fix) requester.onResume(this) { _ -> } } + // SDD-111: re-evaluate the current audio route on resume. + audioLifecycleController?.onResume() } override fun onRequestPermissionsResult( @@ -252,6 +265,9 @@ class MainActivity : FlutterActivity() { audioOutputChannel?.setMethodCallHandler(null) audioOutputEvents?.setStreamHandler(null) audioOutputController?.detach() + // SDD-111: detach the audio lifecycle controller. + audioLifecycleController?.onDestroy() + audioLifecycleController = null permissionRequester = null permissionsChannel = null audioOutputChannel = null diff --git a/apps/chanora_flutter/android/app/src/main/kotlin/app/chanora/chanora_flutter/MethodChannels.kt b/apps/chanora_flutter/android/app/src/main/kotlin/app/chanora/chanora_flutter/MethodChannels.kt index 98c95f2..0498277 100644 --- a/apps/chanora_flutter/android/app/src/main/kotlin/app/chanora/chanora_flutter/MethodChannels.kt +++ b/apps/chanora_flutter/android/app/src/main/kotlin/app/chanora/chanora_flutter/MethodChannels.kt @@ -65,6 +65,15 @@ internal object MethodChannels { */ const val METHOD_POP_TO_SYSTEM: String = "popToSystem" + /** + * Channel for Android audio lifecycle events (route changes, + * interruptions, app lifecycle). Mirrors the iOS + * `chanora/ios_audio_lifecycle` channel. + * + * Trace: SDD-111 (cross-platform mobile voice backend) + */ + const val ANDROID_AUDIO_LIFECYCLE: String = "chanora/android_audio_lifecycle" + const val AUDIO_OUTPUT: String = "app.audio_output" const val AUDIO_OUTPUT_EVENTS: String = "app.audio_output/events" const val METHOD_GET_OUTPUT_DEVICES: String = "getOutputDevices" diff --git a/apps/chanora_flutter/lib/main.dart b/apps/chanora_flutter/lib/main.dart index 0781deb..f5eacef 100644 --- a/apps/chanora_flutter/lib/main.dart +++ b/apps/chanora_flutter/lib/main.dart @@ -18,6 +18,7 @@ import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:package_info_plus/package_info_plus.dart'; import 'package:path_provider/path_provider.dart'; +import 'package:flutter_foreground_task/flutter_foreground_task.dart'; import 'l10n/generated/app_localizations.dart'; import 'services/android_permissions_service.dart'; @@ -30,6 +31,8 @@ import 'widgets/voice_platform.dart'; import 'widgets/voice_bar.dart'; import 'widgets/voice_compact.dart'; import 'widgets/voice_settings.dart'; +import 'widgets/bbcode_text.dart'; +import 'services/link_trust_service.dart'; bool get _isMacOS => !kIsWeb && Platform.isMacOS; @@ -62,11 +65,12 @@ Future _configureBundledVadModels() async { assetPath: _sileroVadAsset, fileName: 'silero_vad.onnx', ); - await _copyBundledAssetToDocuments( + final ten = await _copyBundledAssetToDocuments( assetPath: _tenVadAsset, fileName: 'ten_vad.onnx', ); await rust.setVadModelPath(path: silero.path); + await rust.setTenVadModelPath(path: ten.path); } /// Top padding for macOS to clear traffic-light buttons. @@ -136,7 +140,7 @@ String? _pttDisplayLabelForKey(LogicalKeyboardKey k) { /// pubspec.yaml advances (e.g. rc.8 -> rc.9 -> 1.0.0). The /// build-counter suffix changes automatically on every pubspec /// `+` bump because Flutter writes it into Info.plist. -const String _kSemverBaseline = 'v1.0.0-rc.8'; +const String _kSemverBaseline = 'v0.1.0'; String _kAppVersion = _kSemverBaseline; Future main() async { @@ -146,6 +150,8 @@ Future main() async { unawaited(_wireStorage()); unawaited(_wireConnectivity()); _wireIosAudioLifecycle(); + _wireAndroidAudioLifecycle(); + await _configureBundledVadModels(); runApp(const ChanoraApp()); } @@ -214,6 +220,37 @@ void _wireIosAudioLifecycle() { }); } +/// Wire the Android audio lifecycle MethodChannel. +/// +/// Kotlin side (`AndroidAudioLifecycleController`) posts route-change +/// events through `FlutterMethodChannel` named +/// `"chanora/android_audio_lifecycle"`. This handler dispatches them to +/// the FRB bridge functions on the Rust side, mirroring the iOS pattern. +void _wireAndroidAudioLifecycle() { + if (!Platform.isAndroid) return; + const channel = MethodChannel('chanora/android_audio_lifecycle'); + channel.setMethodCallHandler((call) async { + try { + switch (call.method) { + case 'handleRouteChange': + final args = call.arguments; + final routeStr = args is Map + ? (args['routeType'] as String? ?? 'Unknown') + : (args as String? ?? 'Unknown'); + final route = _parseBridgeAudioRoute(routeStr); + rust.handleRouteChange(route: route); + break; + default: + // Unknown method — ignore gracefully rather than crashing. + break; + } + } catch (_) { + // Errors from the Rust side are already logged there; + // don't propagate exceptions to the Android framework. + } + }); +} + /// Populate `_kAppVersion` by suffixing the platform-canonical /// build number to `_kSemverBaseline`. Format: /// `v1.0.0-rc.8+` (e.g. `v1.0.0-rc.8+64`). The build @@ -728,6 +765,15 @@ class _BetaHomeState extends State<_BetaHome> { _phase = _Phase.connected; _applySnapshot(snap); }); + try { + await FlutterForegroundTask.startService( + notificationTitle: 'Chanora', + notificationText: 'Connected to ${snap.serverName}', + notificationButtons: const [ + NotificationButton(id: 'disconnect', text: 'Disconnect'), + ], + ); + } catch (_) {} } catch (e) { if (!mounted) return; final errorStr = e.toString(); @@ -1186,6 +1232,9 @@ class _BetaHomeState extends State<_BetaHome> { try { await rust.disconnect(); } catch (_) {} + try { + await FlutterForegroundTask.stopService(); + } catch (_) {} if (!mounted) return; setState(() { _phase = _Phase.idle; @@ -1506,6 +1555,7 @@ class _BetaHomeState extends State<_BetaHome> { icon: Icon(_hardMute ? Icons.mic_off : Icons.mic), isSelected: _hardMute, selectedIcon: const Icon(Icons.mic_off), + color: _hardMute ? theme.colorScheme.error : null, onPressed: _onToggleHardMute, ), IconButton( @@ -1513,6 +1563,7 @@ class _BetaHomeState extends State<_BetaHome> { icon: Icon(_outputMuted ? Icons.headset_off : Icons.headset), isSelected: _outputMuted, selectedIcon: const Icon(Icons.headset_off), + color: _outputMuted ? theme.colorScheme.error : null, onPressed: _toggleOutputMute, ), ], @@ -1532,6 +1583,10 @@ class _BetaHomeState extends State<_BetaHome> { const headerTitle = SizedBox.shrink(); + final appBarTitle = _phase == _Phase.connected + ? Text(_snapshot?.serverName ?? l10n.appTitle, style: theme.textTheme.titleMedium) + : headerTitle; + final bodyContent = LayoutBuilder( builder: (ctx, bodyConstraints) { const wideBreakpoint = 600.0; @@ -1554,7 +1609,8 @@ class _BetaHomeState extends State<_BetaHome> { crossAxisAlignment: CrossAxisAlignment.stretch, children: [ if (!isWideSnapshot) ...[banner, const SizedBox(height: 12)], - Text(statusText(), style: theme.textTheme.titleMedium), + if (_phase != _Phase.connected) + Text(statusText(), style: theme.textTheme.titleMedium), if (_lostReason != null || _reconnectAttempt != null) ...[ const SizedBox(height: 8), Container( @@ -1707,6 +1763,8 @@ class _BetaHomeState extends State<_BetaHome> { pttBoundKeyLabel: _pttBoundKeyLabel, audioStats: _audioStats, isTouchOnly: isTouchOnlyPttHost, + inputMuted: _hardMute, + outputMuted: _outputMuted, onTap: () => _onOpenVoiceDetailsSheet(), ), if (_inChannel && @@ -1756,7 +1814,7 @@ class _BetaHomeState extends State<_BetaHome> { } return Scaffold( - appBar: AppBar(title: headerTitle, actions: headerActions), + appBar: AppBar(title: appBarTitle, actions: headerActions), body: SafeArea( top: false, child: Padding(padding: const EdgeInsets.all(16), child: bodyContent), @@ -2168,6 +2226,75 @@ class _BookmarkList extends StatelessWidget { /// /// Driven by the `BridgeEvent::PttCapability` stream published by /// the `PttController` (SDD-088). The `_BetaHomeState` listener +class _WelcomeMessageTile extends StatefulWidget { + const _WelcomeMessageTile({required this.welcomeMessage}); + + final String welcomeMessage; + + @override + State<_WelcomeMessageTile> createState() => _WelcomeMessageTileState(); +} + +class _WelcomeMessageTileState extends State<_WelcomeMessageTile> { + bool _expanded = true; + + @override + Widget build(BuildContext context) { + final theme = Theme.of(context); + return Container( + decoration: BoxDecoration( + color: theme.colorScheme.surfaceContainerHighest, + borderRadius: BorderRadius.circular(6), + ), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + InkWell( + onTap: () { + HapticFeedback.selectionClick(); + setState(() => _expanded = !_expanded); + }, + borderRadius: const BorderRadius.vertical(top: Radius.circular(6)), + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 6), + child: Row( + children: [ + Icon( + _expanded ? Icons.expand_less : Icons.expand_more, + size: 18, + color: theme.colorScheme.onSurfaceVariant, + ), + const SizedBox(width: 4), + Text( + 'Server welcome message', + style: theme.textTheme.labelMedium?.copyWith( + color: theme.colorScheme.onSurfaceVariant, + ), + ), + ], + ), + ), + ), + AnimatedCrossFade( + firstChild: Padding( + padding: const EdgeInsets.fromLTRB(8, 0, 8, 8), + child: BbCodeText( + widget.welcomeMessage, + linkTrust: LinkTrustService.instance, + ), + ), + secondChild: const SizedBox(width: double.infinity), + crossFadeState: _expanded + ? CrossFadeState.showFirst + : CrossFadeState.showSecond, + duration: const Duration(milliseconds: 200), + ), + ], + ), + ); + } +} + /// updates the props on each transition. class _SnapshotView extends StatelessWidget { const _SnapshotView({ @@ -2242,17 +2369,7 @@ class _SnapshotView extends StatelessWidget { ), if (snapshot.welcomeMessage.isNotEmpty) ...[ const SizedBox(height: 8), - Container( - padding: const EdgeInsets.all(8), - decoration: BoxDecoration( - color: theme.colorScheme.surfaceContainerHighest, - borderRadius: BorderRadius.circular(6), - ), - child: Text( - snapshot.welcomeMessage, - style: theme.textTheme.bodySmall, - ), - ), + _WelcomeMessageTile(welcomeMessage: snapshot.welcomeMessage), ], const Divider(height: 24), Text(l10n.channelsHeading, style: theme.textTheme.titleMedium), @@ -2271,7 +2388,6 @@ class _SnapshotView extends StatelessWidget { : null, ), title: Text(ch.name), - subtitle: Text('id=${ch.id} parent=${ch.parent}'), selected: ch.id == currentVoiceChannelId, onTap: hasJoinPending || @@ -2284,7 +2400,8 @@ class _SnapshotView extends StatelessWidget { ), ), for (final cl in byChannel[ch.id] ?? const []) - _clientTile(theme, cl, (depthById[ch.id] ?? 0) * indentPerLevel), + if (!cl.isServerQuery) + _clientTile(theme, cl, (depthById[ch.id] ?? 0) * indentPerLevel), ], ], ); diff --git a/apps/chanora_flutter/lib/services/link_trust_service.dart b/apps/chanora_flutter/lib/services/link_trust_service.dart new file mode 100644 index 0000000..c7f19bf --- /dev/null +++ b/apps/chanora_flutter/lib/services/link_trust_service.dart @@ -0,0 +1,98 @@ +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:url_launcher/url_launcher.dart'; + +class LinkTrustService extends ChangeNotifier { + static LinkTrustService? _instance; + final Set _trusted = {}; + bool _loaded = false; + + static LinkTrustService get instance { + _instance ??= LinkTrustService._(); + return _instance!; + } + + LinkTrustService._() { + _load(); + } + + Future _load() async { + if (_loaded) return; + _loaded = true; + final prefs = await SharedPreferences.getInstance(); + final domains = prefs.getStringList('trusted_domains') ?? []; + _trusted.addAll(domains); + notifyListeners(); + } + + bool isTrusted(String host) { + host = host.toLowerCase(); + for (final pattern in _trusted) { + if (_matches(host, pattern)) return true; + } + return false; + } + + Future addTrustedDomain(String host) async { + host = host.toLowerCase(); + _trusted.add(host); + notifyListeners(); + final prefs = await SharedPreferences.getInstance(); + await prefs.setStringList('trusted_domains', _trusted.toList()); + } + + bool _matches(String host, String pattern) { + if (pattern.startsWith('*.')) { + final suffix = pattern.substring(2); + return host == suffix || host.endsWith('.$suffix'); + } + return host == pattern; + } +} + +Future showLinkTrustDialog(BuildContext context, String domain) async { + bool remember = false; + return showDialog( + context: context, + builder: (ctx) => StatefulBuilder( + builder: (ctx, setDialogState) => AlertDialog( + title: const Text('Open external link?'), + content: Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text('You are about to open a link to:\n\n$domain'), + const SizedBox(height: 12), + Row( + children: [ + SizedBox( + width: 24, + height: 24, + child: Checkbox( + value: remember, + onChanged: (v) => setDialogState(() => remember = v ?? false), + ), + ), + const SizedBox(width: 8), + const Flexible( + child: Text('Trust all links from this domain'), + ), + ], + ), + ], + ), + actions: [ + TextButton( + onPressed: () => Navigator.of(ctx).pop(null), + child: const Text('Cancel'), + ), + TextButton( + onPressed: () => Navigator.of(ctx).pop(remember), + child: const Text('Open'), + ), + ], + ), + ), + ); +} diff --git a/apps/chanora_flutter/lib/src/rust/api.dart b/apps/chanora_flutter/lib/src/rust/api.dart index cde73e3..5b5f330 100644 --- a/apps/chanora_flutter/lib/src/rust/api.dart +++ b/apps/chanora_flutter/lib/src/rust/api.dart @@ -10,7 +10,7 @@ import 'package:freezed_annotation/freezed_annotation.dart' hide protected; part 'api.freezed.dart'; // These functions are ignored because they are not marked as `pub`: `install_panic_diagnostic_hook`, `log_file_path`, `log_sink`, `map_join_error_code`, `map_join_sync_state`, `open_log_file`, `permission_events`, `publish_permission_state`, `runtime`, `session`, `task_join_error`, `transmit_mode_from_u8` -// These function are ignored because they are on traits that is not defined in current crate (put an empty `#[frb]` on it to unignore): `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `eq`, `eq`, `eq`, `eq`, `eq`, `eq`, `eq`, `eq`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from` +// These function are ignored because they are on traits that is not defined in current crate (put an empty `#[frb]` on it to unignore): `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `assert_fields_are_eq`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `clone`, `eq`, `eq`, `eq`, `eq`, `eq`, `eq`, `eq`, `eq`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `fmt`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from`, `from` // These functions are ignored (category: IgnoreBecauseExplicitAttribute): `from_kotlin_str`, `to_permission_gate` /// Return the platform-conventional log-file path as a string, or @@ -46,10 +46,24 @@ Future isConnected() => RustLib.instance.api.crateApiIsConnected(); void handleRouteChange({required BridgeAudioRoute route}) => RustLib.instance.api.crateApiHandleRouteChange(route: route); -/// Handle iOS AVAudioSession media-services reset. +/// Handle iOS AVAudioSession media-services reset (legacy, no route arg). +/// +/// Called by the existing FRB-generated Dart binding. Uses +/// `AudioRoute::Unknown` which triggers a route-change recompute. +/// The AppDelegate now also calls `handle_media_services_reset_with_route` +/// directly after rebuilding the session. void handleMediaServicesReset() => RustLib.instance.api.crateApiHandleMediaServicesReset(); +/// Handle iOS AVAudioSession media-services reset with the current +/// route class. Called by AppDelegate after rebuilding the session. +/// +/// `route_class` is the Swift-side route class string (e.g. "Speaker"). +void handleMediaServicesResetWithRoute({required String routeClass}) => RustLib + .instance + .api + .crateApiHandleMediaServicesResetWithRoute(routeClass: routeClass); + /// Handle iOS AVAudioSession interruption begin (SDD-101). void handleInterruptionBegan() => RustLib.instance.api.crateApiHandleInterruptionBegan(); @@ -229,59 +243,27 @@ Future audioStats() => /// Apply the P1 audio-processing config. Future setAudioProcessingConfig({ required BridgeAudioProcessingConfig config, -}) async { - _lastAppliedAudioConfig = config; - return RustLib.instance.api.crateApiSetAudioProcessingConfig(config: config); -} +}) => RustLib.instance.api.crateApiSetAudioProcessingConfig(config: config); + +/// Read the current audio-processing config. +/// +/// Returns the live config as last applied to the audio engine. +/// Returns a default config when no session is active. +Future getAudioProcessingConfig() => + RustLib.instance.api.crateApiGetAudioProcessingConfig(); /// Read P1 audio-processing diagnostics. Future audioProcessingStats() => RustLib.instance.api.crateApiAudioProcessingStats(); -/// Read the current audio-processing config. -/// -/// Derives the config from [audioProcessingStats] for the route/backend -/// fields, and returns the last value applied via [setAudioProcessingConfig] -/// for timing/debug fields. Falls back to P1 spec defaults on first call. -Future getAudioProcessingConfig() async { - BridgeAudioProcessingStats? stats; - try { - stats = await audioProcessingStats(); - } catch (_) {} - - final last = _lastAppliedAudioConfig; - return BridgeAudioProcessingConfig( - route: stats?.audioRoute ?? last?.route ?? BridgeAudioRoute.unknown, - iosMode: - stats?.iosVoiceProcessingMode ?? - last?.iosMode ?? - BridgeIosVoiceProcessingMode.platformVoiceProcessing, - processingBackend: - stats?.processingBackend ?? - last?.processingBackend ?? - BridgeAudioBackend.platformVoiceProcessing, - vadBackend: - stats?.vadBackend ?? last?.vadBackend ?? BridgeVadBackend.sileroOnnx, - aec: last?.aec ?? BridgeEffectOwner.platform, - ns: last?.ns ?? BridgeEffectOwner.platform, - agc: last?.agc ?? BridgeEffectOwner.platform, - hpfEnabled: last?.hpfEnabled ?? true, - limiterEnabled: last?.limiterEnabled ?? true, - vadHangoverMs: last?.vadHangoverMs ?? 500, - vadPreRollMs: last?.vadPreRollMs ?? 160, - vadMinTxMs: last?.vadMinTxMs ?? 200, - debugWavDumpEnabled: last?.debugWavDumpEnabled ?? false, - ); -} - -/// Last config applied via [setAudioProcessingConfig]. Used by -/// [getAudioProcessingConfig] to preserve timing/debug values across calls. -BridgeAudioProcessingConfig? _lastAppliedAudioConfig; - /// Configure the VAD model path. Future setVadModelPath({required String path}) => RustLib.instance.api.crateApiSetVadModelPath(path: path); +/// Configure the TEN VAD ONNX model path. +Future setTenVadModelPath({required String path}) => + RustLib.instance.api.crateApiSetTenVadModelPath(path: path); + /// Enable or disable audio debug WAV dumping. Future enableAudioDebugWavDump({required bool enabled}) => RustLib.instance.api.crateApiEnableAudioDebugWavDump(enabled: enabled); diff --git a/apps/chanora_flutter/lib/src/rust/frb_generated.dart b/apps/chanora_flutter/lib/src/rust/frb_generated.dart index 9959c52..f867ca5 100644 --- a/apps/chanora_flutter/lib/src/rust/frb_generated.dart +++ b/apps/chanora_flutter/lib/src/rust/frb_generated.dart @@ -67,7 +67,7 @@ class RustLib extends BaseEntrypoint { String get codegenVersion => '2.12.0'; @override - int get rustContentHash => -1835973251; + int get rustContentHash => -436507436; static const kDefaultExternalLibraryLoaderConfig = ExternalLibraryLoaderConfig( @@ -103,6 +103,8 @@ abstract class RustLibApi extends BaseApi { String crateApiExportDiagnostics(); + Future crateApiGetAudioProcessingConfig(); + Future<(String, String)> crateApiGetPttBinding(); Future crateApiGetReleaseTailMs(); @@ -115,6 +117,8 @@ abstract class RustLibApi extends BaseApi { void crateApiHandleMediaServicesReset(); + void crateApiHandleMediaServicesResetWithRoute({required String routeClass}); + void crateApiHandleRouteChange({required BridgeAudioRoute route}); Future crateApiInitStorage({required String dir}); @@ -159,6 +163,8 @@ abstract class RustLibApi extends BaseApi { Future crateApiSetReleaseTailMs({required int ms}); + Future crateApiSetTenVadModelPath({required String path}); + Future crateApiSetTransmitMode({required BridgeTransmitMode mode}); Future crateApiSetVadModelPath({required String path}); @@ -469,7 +475,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { const TaskConstMeta(debugName: "export_diagnostics", argNames: []); @override - Future<(String, String)> crateApiGetPttBinding() { + Future crateApiGetAudioProcessingConfig() { return handler.executeNormal( NormalTask( callFfi: (port_) { @@ -481,6 +487,36 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { port: port_, ); }, + codec: SseCodec( + decodeSuccessData: sse_decode_bridge_audio_processing_config, + decodeErrorData: sse_decode_bridge_error, + ), + constMeta: kCrateApiGetAudioProcessingConfigConstMeta, + argValues: [], + apiImpl: this, + ), + ); + } + + TaskConstMeta get kCrateApiGetAudioProcessingConfigConstMeta => + const TaskConstMeta( + debugName: "get_audio_processing_config", + argNames: [], + ); + + @override + Future<(String, String)> crateApiGetPttBinding() { + return handler.executeNormal( + NormalTask( + callFfi: (port_) { + final serializer = SseSerializer(generalizedFrbRustBinding); + pdeCallFfi( + generalizedFrbRustBinding, + serializer, + funcId: 12, + port: port_, + ); + }, codec: SseCodec( decodeSuccessData: sse_decode_record_string_string, decodeErrorData: null, @@ -504,7 +540,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 12, + funcId: 13, port: port_, ); }, @@ -531,7 +567,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 13, + funcId: 14, port: port_, ); }, @@ -555,7 +591,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { SyncTask( callFfi: () { final serializer = SseSerializer(generalizedFrbRustBinding); - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 14)!; + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 15)!; }, codec: SseCodec( decodeSuccessData: sse_decode_unit, @@ -578,7 +614,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { callFfi: () { final serializer = SseSerializer(generalizedFrbRustBinding); sse_encode_bool(shouldResume, serializer); - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 15)!; + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 16)!; }, codec: SseCodec( decodeSuccessData: sse_decode_unit, @@ -603,7 +639,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { SyncTask( callFfi: () { final serializer = SseSerializer(generalizedFrbRustBinding); - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 16)!; + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 17)!; }, codec: SseCodec( decodeSuccessData: sse_decode_unit, @@ -622,6 +658,32 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { argNames: [], ); + @override + void crateApiHandleMediaServicesResetWithRoute({required String routeClass}) { + return handler.executeSync( + SyncTask( + callFfi: () { + final serializer = SseSerializer(generalizedFrbRustBinding); + sse_encode_String(routeClass, serializer); + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 18)!; + }, + codec: SseCodec( + decodeSuccessData: sse_decode_unit, + decodeErrorData: null, + ), + constMeta: kCrateApiHandleMediaServicesResetWithRouteConstMeta, + argValues: [routeClass], + apiImpl: this, + ), + ); + } + + TaskConstMeta get kCrateApiHandleMediaServicesResetWithRouteConstMeta => + const TaskConstMeta( + debugName: "handle_media_services_reset_with_route", + argNames: ["routeClass"], + ); + @override void crateApiHandleRouteChange({required BridgeAudioRoute route}) { return handler.executeSync( @@ -629,7 +691,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { callFfi: () { final serializer = SseSerializer(generalizedFrbRustBinding); sse_encode_bridge_audio_route(route, serializer); - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 17)!; + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 19)!; }, codec: SseCodec( decodeSuccessData: sse_decode_unit, @@ -657,7 +719,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 18, + funcId: 20, port: port_, ); }, @@ -684,7 +746,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 19, + funcId: 21, port: port_, ); }, @@ -711,7 +773,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 20, + funcId: 22, port: port_, ); }, @@ -735,7 +797,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { SyncTask( callFfi: () { final serializer = SseSerializer(generalizedFrbRustBinding); - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 21)!; + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 23)!; }, codec: SseCodec( decodeSuccessData: sse_decode_String, @@ -765,7 +827,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 22, + funcId: 24, port: port_, ); }, @@ -794,7 +856,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 23, + funcId: 25, port: port_, ); }, @@ -827,7 +889,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 24, + funcId: 26, port: port_, ); }, @@ -858,7 +920,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 25, + funcId: 27, port: port_, ); }, @@ -886,7 +948,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 26, + funcId: 28, port: port_, ); }, @@ -916,7 +978,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 27, + funcId: 29, port: port_, ); }, @@ -944,7 +1006,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { callFfi: () { final serializer = SseSerializer(generalizedFrbRustBinding); sse_encode_bridge_network_state(state, serializer); - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 28)!; + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 30)!; }, codec: SseCodec( decodeSuccessData: sse_decode_unit, @@ -970,7 +1032,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 29, + funcId: 31, port: port_, ); }, @@ -998,7 +1060,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 30, + funcId: 32, port: port_, ); }, @@ -1026,7 +1088,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 31, + funcId: 33, port: port_, ); }, @@ -1058,7 +1120,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 32, + funcId: 34, port: port_, ); }, @@ -1088,7 +1150,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 33, + funcId: 35, port: port_, ); }, @@ -1106,6 +1168,36 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { TaskConstMeta get kCrateApiSetReleaseTailMsConstMeta => const TaskConstMeta(debugName: "set_release_tail_ms", argNames: ["ms"]); + @override + Future crateApiSetTenVadModelPath({required String path}) { + return handler.executeNormal( + NormalTask( + callFfi: (port_) { + final serializer = SseSerializer(generalizedFrbRustBinding); + sse_encode_String(path, serializer); + pdeCallFfi( + generalizedFrbRustBinding, + serializer, + funcId: 36, + port: port_, + ); + }, + codec: SseCodec( + decodeSuccessData: sse_decode_unit, + decodeErrorData: sse_decode_bridge_error, + ), + constMeta: kCrateApiSetTenVadModelPathConstMeta, + argValues: [path], + apiImpl: this, + ), + ); + } + + TaskConstMeta get kCrateApiSetTenVadModelPathConstMeta => const TaskConstMeta( + debugName: "set_ten_vad_model_path", + argNames: ["path"], + ); + @override Future crateApiSetTransmitMode({required BridgeTransmitMode mode}) { return handler.executeNormal( @@ -1116,7 +1208,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 34, + funcId: 37, port: port_, ); }, @@ -1144,7 +1236,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 35, + funcId: 38, port: port_, ); }, @@ -1171,7 +1263,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 36, + funcId: 39, port: port_, ); }, @@ -1199,7 +1291,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 37, + funcId: 40, port: port_, ); }, @@ -1231,7 +1323,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 38, + funcId: 41, port: port_, ); }, @@ -1260,7 +1352,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 39, + funcId: 42, port: port_, ); }, diff --git a/apps/chanora_flutter/lib/widgets/bbcode_text.dart b/apps/chanora_flutter/lib/widgets/bbcode_text.dart new file mode 100644 index 0000000..8bf00c2 --- /dev/null +++ b/apps/chanora_flutter/lib/widgets/bbcode_text.dart @@ -0,0 +1,313 @@ +import 'package:flutter/material.dart'; +import 'package:url_launcher/url_launcher.dart'; + +import '../services/link_trust_service.dart'; + +final _tagRe = RegExp( + r'\[(\/?(?:b|i|u|s' + r'|color(?:=[^\]]*)?' + r'|size(?:=\d+)?' + r'|url(?:=[^\]]*)?' + r'|img(?:=[^\]]*)?' + r'|list|\*|quote|code|center|left|right' + r'))\]', + caseSensitive: false, +); +final _colorRe = RegExp(r'color=([#\w]+)'); +final _sizeRe = RegExp(r'size=(\d+)'); +final _urlRe = RegExp(r'url=(.+)'); +final _imgRe = RegExp(r'img=(.+)'); +final _urlAutoRe = RegExp(r'(?:\[url\])?(https?://[^\s\[\]]+)(?:\[/url\])?', caseSensitive: false); +final _closeUrlRe = RegExp(r'\[/url\]', caseSensitive: false); + +int? _findCloseUrl(String src, int from) { + final m = _closeUrlRe.matchAsPrefix(src, from); + if (m != null) return from; + final idx = src.indexOf('[/url]', from); + if (idx >= 0) return idx; + final idxu = src.indexOf('[/URL]', from); + if (idxu >= 0) return idxu; + return null; +} + +Color? _parseColor(String hex) { + try { + var h = hex.replaceFirst('#', ''); + if (h.length == 6) h = 'FF$h'; + if (h.length == 8) { + return Color(int.parse(h, radix: 16)); + } + } catch (_) {} + return null; +} + +class BbCodeText extends StatelessWidget { + const BbCodeText(this.text, {super.key, required this.linkTrust}); + + final String text; + final LinkTrustService linkTrust; + + @override + Widget build(BuildContext context) { + if (!text.contains('[') || !text.contains(']')) { + return _plainWithAutoLinks(context, text); + } + return _render(context, text); + } + + Widget _plainWithAutoLinks(BuildContext context, String src) { + final parts = []; + int last = 0; + for (final m in _urlAutoRe.allMatches(src)) { + if (m.start > last) { + parts.add(TextSpan(text: src.substring(last, m.start))); + } + final url = m.group(1)!; + parts.add(WidgetSpan( + alignment: PlaceholderAlignment.middle, + child: _LinkTap( + url: url, + linkTrust: linkTrust, + child: Text( + url, + style: const TextStyle( + color: Colors.blue, + decoration: TextDecoration.underline, + ), + ), + ), + )); + last = m.end; + } + if (last < src.length) { + parts.add(TextSpan(text: src.substring(last))); + } + if (parts.isEmpty) return Text(src); + return Text.rich(TextSpan(children: parts)); + } + + Widget _render(BuildContext context, String src) { + final spans = []; + final tags = []; + + void flush(StringBuffer buf) { + if (buf.isEmpty) return; + var t = buf.toString(); + buf.clear(); + + bool bold = false; + bool italic = false; + bool underline = false; + bool strikethrough = false; + Color? color; + double? size; + + for (final tag in tags) { + if (tag == 'b') { + bold = true; + } else if (tag == 'i') { + italic = true; + } else if (tag == 'u') { + underline = true; + } else if (tag == 's') { + strikethrough = true; + } else if (tag.startsWith('color=')) { + color = _parseColor(tag.substring(6)); + } else if (tag.startsWith('size=')) { + size = double.tryParse(tag.substring(5)); + } + } + + spans.add(TextSpan( + text: t, + style: TextStyle( + fontWeight: bold ? FontWeight.bold : null, + fontStyle: italic ? FontStyle.italic : null, + decoration: TextDecoration.combine([ + if (underline) TextDecoration.underline, + if (strikethrough) TextDecoration.lineThrough, + ]), + color: color, + fontSize: size, + ), + )); + } + + final buf = StringBuffer(); + int i = 0; + + while (i < src.length) { + if (src[i] != '[') { + buf.write(src[i]); + i++; + continue; + } + final m = _tagRe.matchAsPrefix(src, i); + if (m == null) { + buf.write(src[i]); + i++; + continue; + } + + flush(buf); + final raw = m.group(1)!.toLowerCase(); + i = m.end; + + if (raw.startsWith('/')) { + final closeTag = raw.substring(1); + if (closeTag == 'url' || closeTag == 'img') { + continue; + } + tags.remove(closeTag); + continue; + } + + switch (raw) { + case 'b': + case 'i': + case 'u': + case 's': + case 'list': + case 'quote': + case 'code': + case 'center': + case 'left': + case 'right': + tags.add(raw); + break; + + case '*': + spans.add(const TextSpan(text: '\n \u2022 ')); + break; + + default: + if (raw.startsWith('color=') || raw.startsWith('size=')) { + tags.add(raw); + } else if (raw.startsWith('url=')) { + final url = _urlRe.firstMatch(raw)?.group(1) ?? ''; + final closeIdx = _findCloseUrl(src, i); + String inner; + if (closeIdx != null) { + inner = src.substring(i, closeIdx); + i = closeIdx + 6; + } else { + inner = url; + } + spans.add(WidgetSpan( + alignment: PlaceholderAlignment.middle, + child: _LinkTap( + url: url.isNotEmpty ? url : inner, + linkTrust: linkTrust, + child: Text( + inner, + style: const TextStyle( + color: Colors.blue, + decoration: TextDecoration.underline, + ), + ), + ), + )); + } else if (raw.startsWith('img=')) { + final src2 = _imgRe.firstMatch(raw)?.group(1) ?? ''; + if (src2.isNotEmpty) { + spans.add(WidgetSpan( + child: ClipRRect( + borderRadius: BorderRadius.circular(8), + child: Image.network( + Uri.tryParse(src2)?.toString() ?? src2, + fit: BoxFit.scaleDown, + errorBuilder: (_, __, ___) => const SizedBox.shrink(), + ), + ), + )); + } + } else if (raw == 'url') { + final closeIdx = _findCloseUrl(src, i); + if (closeIdx != null) { + final url = src.substring(i, closeIdx).trim(); + i = closeIdx + 6; + spans.add(WidgetSpan( + alignment: PlaceholderAlignment.middle, + child: _LinkTap( + url: url, + linkTrust: linkTrust, + child: Text( + url, + style: const TextStyle( + color: Colors.blue, + decoration: TextDecoration.underline, + ), + ), + ), + )); + } + } + break; + } + } + + flush(buf); + + if (spans.isEmpty) return Text(src); + return Text.rich(TextSpan(children: spans)); + } +} + +class _LinkTap extends StatefulWidget { + const _LinkTap({ + required this.url, + required this.child, + required this.linkTrust, + }); + + final String url; + final Widget child; + final LinkTrustService linkTrust; + + @override + State<_LinkTap> createState() => _LinkTapState(); +} + +class _LinkTapState extends State<_LinkTap> { + @override + void initState() { + super.initState(); + widget.linkTrust.addListener(_onChanged); + } + + @override + void dispose() { + widget.linkTrust.removeListener(_onChanged); + super.dispose(); + } + + void _onChanged() => mounted ? setState(() {}) : null; + + Future _open() async { + final uri = Uri.tryParse(widget.url); + if (uri == null) return; + final host = uri.host; + if (host.isEmpty) return; + + if (widget.linkTrust.isTrusted(host)) { + await launchUrl(uri, mode: LaunchMode.externalApplication); + return; + } + + if (!mounted) return; + final trust = await showLinkTrustDialog(context, host); + if (trust == null) return; + if (trust) { + await widget.linkTrust.addTrustedDomain(host); + } + await launchUrl(uri, mode: LaunchMode.externalApplication); + } + + @override + Widget build(BuildContext context) { + return GestureDetector( + onTap: _open, + child: widget.child, + ); + } +} diff --git a/apps/chanora_flutter/lib/widgets/voice_compact.dart b/apps/chanora_flutter/lib/widgets/voice_compact.dart index 20877e5..46e1fc1 100644 --- a/apps/chanora_flutter/lib/widgets/voice_compact.dart +++ b/apps/chanora_flutter/lib/widgets/voice_compact.dart @@ -40,6 +40,8 @@ class VoiceStatusChip extends StatelessWidget { required this.audioStats, required this.isTouchOnly, required this.onTap, + this.inputMuted = false, + this.outputMuted = false, }); /// Current transmit mode. @@ -57,6 +59,12 @@ class VoiceStatusChip extends StatelessWidget { /// True on iOS / iPadOS / Android. final bool isTouchOnly; + /// True when local mic is muted (hard mute or permission mute). + final bool inputMuted; + + /// True when local speaker is muted. + final bool outputMuted; + /// Open the voice details modal. final VoidCallback onTap; @@ -95,9 +103,15 @@ class VoiceStatusChip extends StatelessWidget { final tailText = transmitMode == rust.BridgeTransmitMode.ptt ? '$releaseTailMs${l10n.voiceReleaseTailHint} ${l10n.voiceReleaseTailLabel.toLowerCase()}' : null; - final micText = micOn ? l10n.voiceMicOn : l10n.voiceMicOff; + final micText = inputMuted + ? '${l10n.voiceMicOff} (muted)' + : outputMuted + ? 'Speaker muted' + : (micOn ? l10n.voiceMicOn : l10n.voiceMicOff); final line2 = tailText == null ? micText : '$tailText \u00b7 $micText'; + final muted = inputMuted || outputMuted; + return Semantics( button: true, label: '${l10n.voiceSheetTitle}: $line1, $line2', @@ -105,17 +119,24 @@ class VoiceStatusChip extends StatelessWidget { child: Material( type: MaterialType.transparency, child: InkWell( - onTap: onTap, + onTap: () { + HapticFeedback.lightImpact(); + onTap(); + }, borderRadius: BorderRadius.circular(12), child: ExcludeSemantics( child: Container( padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 8), decoration: BoxDecoration( - color: theme.colorScheme.surfaceContainerHigh, + color: muted + ? theme.colorScheme.errorContainer.withValues(alpha: 0.35) + : theme.colorScheme.surfaceContainerHigh, borderRadius: BorderRadius.circular(12), border: Border.all( - color: theme.colorScheme.outlineVariant, - width: 0.5, + color: muted + ? theme.colorScheme.error + : theme.colorScheme.outlineVariant, + width: muted ? 1.5 : 0.5, ), ), child: Row( @@ -394,6 +415,7 @@ class _VoiceSheetBodyState extends State<_VoiceSheetBody> { late bool _aecEnabled; late bool _agcEnabled; late bool _hpfEnabled; + late bool _preferHardware; late rust.BridgeVadBackend _vadBackend; @override @@ -404,6 +426,7 @@ class _VoiceSheetBodyState extends State<_VoiceSheetBody> { _aecEnabled = c.aec != rust.BridgeEffectOwner.off; _agcEnabled = c.agc != rust.BridgeEffectOwner.off; _hpfEnabled = c.hpfEnabled; + _preferHardware = c.aec == rust.BridgeEffectOwner.platform; _vadBackend = c.vadBackend == rust.BridgeVadBackend.disabled ? rust.BridgeVadBackend.webrtcVad : c.vadBackend; @@ -440,23 +463,51 @@ class _VoiceSheetBodyState extends State<_VoiceSheetBody> { final c = widget.initialAudioConfig; final isSonora = c.iosMode == rust.BridgeIosVoiceProcessingMode.sonoraExperimental; - // VPIO owns enabled effects on the default path. Sonora owns them only in - // the experimental raw path. + final isAndroid = Platform.isAndroid; + + if (isAndroid) { + final owner = _preferHardware + ? rust.BridgeEffectOwner.platform + : rust.BridgeEffectOwner.webrtcApm; + return rust.BridgeAudioProcessingConfig( + route: c.route, + iosMode: c.iosMode, + processingBackend: _preferHardware + ? rust.BridgeAudioBackend.platformVoiceProcessing + : rust.BridgeAudioBackend.webrtcApm, + vadBackend: _vadBackend == rust.BridgeVadBackend.disabled + ? rust.BridgeVadBackend.webrtcVad + : _vadBackend, + aec: _aecEnabled ? owner : rust.BridgeEffectOwner.off, + ns: _nsEnabled ? owner : rust.BridgeEffectOwner.off, + agc: _agcEnabled ? owner : rust.BridgeEffectOwner.off, + hpfEnabled: _hpfEnabled, + limiterEnabled: c.limiterEnabled, + vadHangoverMs: c.vadHangoverMs, + vadPreRollMs: c.vadPreRollMs, + vadMinTxMs: c.vadMinTxMs, + debugWavDumpEnabled: c.debugWavDumpEnabled, + ); + } + + // iOS / macOS: VPIO vs Sonora paths. + // VPIO owns enabled effects on the default path. The experimental raw path + // delegates app-side processing to WebRTC APM. final aecOwner = isSonora ? (_aecEnabled - ? rust.BridgeEffectOwner.sonora + ? rust.BridgeEffectOwner.webrtcApm : rust.BridgeEffectOwner.off) : rust.BridgeEffectOwner.platform; final nsOwner = isSonora ? (_nsEnabled - ? rust.BridgeEffectOwner.sonora + ? rust.BridgeEffectOwner.webrtcApm : rust.BridgeEffectOwner.off) : (_nsEnabled ? rust.BridgeEffectOwner.platform : rust.BridgeEffectOwner.off); final agcOwner = isSonora ? (_agcEnabled - ? rust.BridgeEffectOwner.sonora + ? rust.BridgeEffectOwner.webrtcApm : rust.BridgeEffectOwner.off) : (_agcEnabled ? rust.BridgeEffectOwner.platform @@ -631,6 +682,22 @@ class _VoiceSheetBodyState extends State<_VoiceSheetBody> { ), ), const SizedBox(height: 4), + + // Android: hardware (JNI) vs software (WebRTC APM) + if (Platform.isAndroid) ...[ + _AudioToggleRow( + label: 'Prefer hardware effects', + subtitle: _preferHardware + ? 'Try JNI hardware · software fallback' + : 'Software WebRTC AEC3 · NS · AGC2', + value: _preferHardware, + onChanged: (v) { + setState(() => _preferHardware = v); + _notifyAudioConfig(); + }, + ), + ], + _AudioToggleRow( label: 'Noise suppression', subtitle: 'Wiener filter', @@ -642,25 +709,40 @@ class _VoiceSheetBodyState extends State<_VoiceSheetBody> { ), _AudioToggleRow( label: 'Echo cancellation', - subtitle: - widget.initialAudioConfig.iosMode == - rust.BridgeIosVoiceProcessingMode.platformVoiceProcessing - ? 'Always on · managed by platform VPIO' - : 'AEC3 adaptive filter · 80 ms tail', + subtitle: () { + if (Platform.isAndroid) { + return 'WebRTC AEC3 · adaptive filter'; + } + return widget.initialAudioConfig.iosMode == + rust.BridgeIosVoiceProcessingMode.platformVoiceProcessing + ? 'Always on · managed by platform VPIO' + : 'AEC3 adaptive filter · 80 ms tail'; + }(), value: - widget.initialAudioConfig.iosMode == - rust.BridgeIosVoiceProcessingMode.platformVoiceProcessing - ? true // always on in VPIO - : _aecEnabled, + () { + if (Platform.isAndroid) return _aecEnabled; + return widget.initialAudioConfig.iosMode == + rust.BridgeIosVoiceProcessingMode.platformVoiceProcessing + ? true + : _aecEnabled; + }(), // AEC is always on in VPIO — disable the toggle. - onChanged: - widget.initialAudioConfig.iosMode == - rust.BridgeIosVoiceProcessingMode.platformVoiceProcessing - ? null - : (v) { - setState(() => _aecEnabled = v); - _notifyAudioConfig(); - }, + // On Android, AEC is user-selectable. + onChanged: () { + if (Platform.isAndroid) { + return (v) { + setState(() => _aecEnabled = v); + _notifyAudioConfig(); + }; + } + return widget.initialAudioConfig.iosMode == + rust.BridgeIosVoiceProcessingMode.platformVoiceProcessing + ? null + : (v) { + setState(() => _aecEnabled = v); + _notifyAudioConfig(); + }; + }(), ), _AudioToggleRow( label: 'Auto gain control', diff --git a/apps/chanora_flutter/lib/widgets/voice_settings.dart b/apps/chanora_flutter/lib/widgets/voice_settings.dart index 00a63af..ce00bdb 100644 --- a/apps/chanora_flutter/lib/widgets/voice_settings.dart +++ b/apps/chanora_flutter/lib/widgets/voice_settings.dart @@ -18,6 +18,11 @@ import '../l10n/generated/app_localizations.dart'; import 'voice_platform.dart'; import '../src/rust/api.dart' as rust; +bool get _isAndroid { + if (kIsWeb) return false; + return Platform.isAndroid; +} + bool get _isIos { if (kIsWeb) return false; return Platform.isIOS; @@ -68,6 +73,7 @@ class _VoiceSettingsDialogState extends State { late rust.BridgeVadBackend _vadBackend; late rust.BridgeIosVoiceProcessingMode _iosMode; late bool _debugWavDump; + late bool _preferHardware; // Android only: try JNI hardware effects @override void initState() { @@ -86,29 +92,59 @@ class _VoiceSettingsDialogState extends State { : c.vadBackend; _iosMode = c.iosMode; _debugWavDump = c.debugWavDumpEnabled; + _preferHardware = c.aec == rust.BridgeEffectOwner.platform + || c.ns == rust.BridgeEffectOwner.platform + || c.agc == rust.BridgeEffectOwner.platform; } rust.BridgeAudioProcessingConfig _buildConfig() { final c = widget.initialAudioConfig; final isSonora = _iosMode == rust.BridgeIosVoiceProcessingMode.sonoraExperimental; - // In VPIO mode, enabled effects are platform-owned. Sonora ownership is - // reserved for the experimental raw path so config validation stays honest. + + if (_isAndroid) { + final owner = _preferHardware + ? rust.BridgeEffectOwner.platform + : rust.BridgeEffectOwner.webrtcApm; + return rust.BridgeAudioProcessingConfig( + route: c.route, + iosMode: _iosMode, + processingBackend: _preferHardware + ? rust.BridgeAudioBackend.platformVoiceProcessing + : rust.BridgeAudioBackend.webrtcApm, + vadBackend: _vadBackend == rust.BridgeVadBackend.disabled + ? rust.BridgeVadBackend.webrtcVad + : _vadBackend, + aec: _aecEnabled ? owner : rust.BridgeEffectOwner.off, + ns: _nsEnabled ? owner : rust.BridgeEffectOwner.off, + agc: _agcEnabled ? owner : rust.BridgeEffectOwner.off, + hpfEnabled: _hpfEnabled, + limiterEnabled: _limiterEnabled, + vadHangoverMs: c.vadHangoverMs, + vadPreRollMs: c.vadPreRollMs, + vadMinTxMs: c.vadMinTxMs, + debugWavDumpEnabled: _debugWavDump, + ); + } + + // iOS / macOS: VPIO vs Sonora paths. + // In VPIO mode, enabled effects are platform-owned. The experimental raw + // path uses WebRTC APM ownership so config validation stays honest. final aecOwner = isSonora ? (_aecEnabled - ? rust.BridgeEffectOwner.sonora + ? rust.BridgeEffectOwner.webrtcApm : rust.BridgeEffectOwner.off) : rust.BridgeEffectOwner.platform; // VPIO always owns AEC final nsOwner = isSonora ? (_nsEnabled - ? rust.BridgeEffectOwner.sonora + ? rust.BridgeEffectOwner.webrtcApm : rust.BridgeEffectOwner.off) : (_nsEnabled ? rust.BridgeEffectOwner.platform : rust.BridgeEffectOwner.off); final agcOwner = isSonora ? (_agcEnabled - ? rust.BridgeEffectOwner.sonora + ? rust.BridgeEffectOwner.webrtcApm : rust.BridgeEffectOwner.off) : (_agcEnabled ? rust.BridgeEffectOwner.platform @@ -120,7 +156,7 @@ class _VoiceSettingsDialogState extends State { route: c.route, iosMode: _iosMode, processingBackend: isSonora - ? rust.BridgeAudioBackend.sonora + ? rust.BridgeAudioBackend.webrtcApm : rust.BridgeAudioBackend.platformVoiceProcessing, vadBackend: vadBackend, aec: aecOwner, @@ -244,6 +280,30 @@ class _VoiceSettingsDialogState extends State { const SizedBox(height: 4), ], + // Android HW/SW selector + if (_isAndroid) ...[ + _subHeader(theme, 'Processing backend'), + _radioTile( + value: true, + groupValue: _preferHardware, + title: const Text('Platform (auto)'), + subtitle: _tileSubtitle( + 'Try hardware JNI effects · software fallback', + ), + onSelected: (v) => setState(() => _preferHardware = v), + ), + _radioTile( + value: false, + groupValue: _preferHardware, + title: const Text('WebRTC APM'), + subtitle: _tileSubtitle( + 'Software AEC3 · NS · AGC2', + ), + onSelected: (v) => setState(() => _preferHardware = v), + ), + const SizedBox(height: 4), + ], + // DSP toggles _subHeader(theme, 'DSP stages'), _switchTile( @@ -254,12 +314,14 @@ class _VoiceSettingsDialogState extends State { ), _switchTile( title: 'Echo cancellation (AEC3)', - subtitle: platformVpio - ? 'Managed by platform VPIO' - : 'Adaptive NLMS · 80 ms tail', + subtitle: _isAndroid + ? 'WebRTC AEC3 · adaptive filter' + : platformVpio + ? 'Managed by platform VPIO' + : 'Adaptive NLMS · 80 ms tail', value: _aecEnabled, // AEC is always on in VPIO mode — disable the toggle. - onSelected: platformVpio ? null : (v) => _aecEnabled = v, + onSelected: (_isAndroid || !platformVpio) ? (v) => _aecEnabled = v : null, ), _switchTile( title: 'Auto gain control (AGC2)', diff --git a/apps/chanora_flutter/linux/flutter/generated_plugins.cmake b/apps/chanora_flutter/linux/flutter/generated_plugins.cmake index be1ee3e..df8d2f7 100644 --- a/apps/chanora_flutter/linux/flutter/generated_plugins.cmake +++ b/apps/chanora_flutter/linux/flutter/generated_plugins.cmake @@ -3,6 +3,7 @@ # list(APPEND FLUTTER_PLUGIN_LIST + url_launcher_linux ) list(APPEND FLUTTER_FFI_PLUGIN_LIST diff --git a/apps/chanora_flutter/pubspec.lock b/apps/chanora_flutter/pubspec.lock index 81ec45f..5f98783 100644 --- a/apps/chanora_flutter/pubspec.lock +++ b/apps/chanora_flutter/pubspec.lock @@ -246,6 +246,14 @@ packages: description: flutter source: sdk version: "0.0.0" + flutter_foreground_task: + dependency: "direct main" + description: + name: flutter_foreground_task + sha256: fc5c01a5e1b8f7bb51d0c737714f0c50440dbdf1aeddc5f8cbba313aa6fd4856 + url: "https://pub.dev" + source: hosted + version: "9.2.2" flutter_lints: dependency: "direct dev" description: @@ -629,6 +637,62 @@ packages: url: "https://pub.dev" source: hosted version: "0.28.0" + shared_preferences: + dependency: "direct main" + description: + name: shared_preferences + sha256: c3025c5534b01739267eb7d76959bbc25a6d10f6988e1c2a3036940133dd10bf + url: "https://pub.dev" + source: hosted + version: "2.5.5" + shared_preferences_android: + dependency: transitive + description: + name: shared_preferences_android + sha256: e8d4762b1e2e8578fc4d0fd548cebf24afd24f49719c08974df92834565e2c53 + url: "https://pub.dev" + source: hosted + version: "2.4.23" + shared_preferences_foundation: + dependency: transitive + description: + name: shared_preferences_foundation + sha256: "4e7eaffc2b17ba398759f1151415869a34771ba11ebbccd1b0145472a619a64f" + url: "https://pub.dev" + source: hosted + version: "2.5.6" + shared_preferences_linux: + dependency: transitive + description: + name: shared_preferences_linux + sha256: "580abfd40f415611503cae30adf626e6656dfb2f0cee8f465ece7b6defb40f2f" + url: "https://pub.dev" + source: hosted + version: "2.4.1" + shared_preferences_platform_interface: + dependency: transitive + description: + name: shared_preferences_platform_interface + sha256: "649dc798a33931919ea356c4305c2d1f81619ea6e92244070b520187b5140ef9" + url: "https://pub.dev" + source: hosted + version: "2.4.2" + shared_preferences_web: + dependency: transitive + description: + name: shared_preferences_web + sha256: c49bd060261c9a3f0ff445892695d6212ff603ef3115edbb448509d407600019 + url: "https://pub.dev" + source: hosted + version: "2.4.3" + shared_preferences_windows: + dependency: transitive + description: + name: shared_preferences_windows + sha256: "94ef0f72b2d71bc3e700e025db3710911bd51a71cefb65cc609dd0d9a982e3c1" + url: "https://pub.dev" + source: hosted + version: "2.4.1" shelf: dependency: transitive description: @@ -722,6 +786,70 @@ packages: url: "https://pub.dev" source: hosted version: "1.4.0" + url_launcher: + dependency: "direct main" + description: + name: url_launcher + sha256: f6a7e5c4835bb4e3026a04793a4199ca2d14c739ec378fdfe23fc8075d0439f8 + url: "https://pub.dev" + source: hosted + version: "6.3.2" + url_launcher_android: + dependency: transitive + description: + name: url_launcher_android + sha256: "17bc677f0b301615530dd1d67e0a9828cafa2d0b6b6eae4cd3679b7eac4a273c" + url: "https://pub.dev" + source: hosted + version: "6.3.30" + url_launcher_ios: + dependency: transitive + description: + name: url_launcher_ios + sha256: "580fe5dfb51671ae38191d316e027f6b76272b026370708c2d898799750a02b0" + url: "https://pub.dev" + source: hosted + version: "6.4.1" + url_launcher_linux: + dependency: transitive + description: + name: url_launcher_linux + sha256: d5e14138b3bc193a0f63c10a53c94b91d399df0512b1f29b94a043db7482384a + url: "https://pub.dev" + source: hosted + version: "3.2.2" + url_launcher_macos: + dependency: transitive + description: + name: url_launcher_macos + sha256: "368adf46f71ad3c21b8f06614adb38346f193f3a59ba8fe9a2fd74133070ba18" + url: "https://pub.dev" + source: hosted + version: "3.2.5" + url_launcher_platform_interface: + dependency: transitive + description: + name: url_launcher_platform_interface + sha256: "552f8a1e663569be95a8190206a38187b531910283c3e982193e4f2733f01029" + url: "https://pub.dev" + source: hosted + version: "2.3.2" + url_launcher_web: + dependency: transitive + description: + name: url_launcher_web + sha256: "85c81589622fbc87c1c683aaea164d3604a7777495a79d91e39ffcdec39ddb34" + url: "https://pub.dev" + source: hosted + version: "2.4.3" + url_launcher_windows: + dependency: transitive + description: + name: url_launcher_windows + sha256: "712c70ab1b99744ff066053cbe3e80c73332b38d46e5e945c98689b2e66fc15f" + url: "https://pub.dev" + source: hosted + version: "3.1.5" vector_math: dependency: transitive description: diff --git a/apps/chanora_flutter/pubspec.yaml b/apps/chanora_flutter/pubspec.yaml index fe22e18..58957a9 100644 --- a/apps/chanora_flutter/pubspec.yaml +++ b/apps/chanora_flutter/pubspec.yaml @@ -77,6 +77,9 @@ dependencies: # Touch-only PTT feedback for mobile voice UX (P0 voice basics, # DEC-003 iOS 13 floor; haptic_kit supports iOS 12+). haptic_kit: ^1.0.0 + flutter_foreground_task: ^9.2.2 + url_launcher: ^6.3.2 + shared_preferences: ^2.5.5 dev_dependencies: flutter_test: diff --git a/apps/chanora_flutter/windows/flutter/generated_plugins.cmake b/apps/chanora_flutter/windows/flutter/generated_plugins.cmake index 7e0ec45..0b0bda0 100644 --- a/apps/chanora_flutter/windows/flutter/generated_plugins.cmake +++ b/apps/chanora_flutter/windows/flutter/generated_plugins.cmake @@ -4,6 +4,7 @@ list(APPEND FLUTTER_PLUGIN_LIST connectivity_plus + url_launcher_windows ) list(APPEND FLUTTER_FFI_PLUGIN_LIST diff --git a/crates/chanora_audio/Cargo.toml b/crates/chanora_audio/Cargo.toml index a7dc7e4..0bff277 100644 --- a/crates/chanora_audio/Cargo.toml +++ b/crates/chanora_audio/Cargo.toml @@ -13,6 +13,12 @@ publish.workspace = true chanora_protocol = { path = "../chanora_protocol" } thiserror.workspace = true tracing.workspace = true +sonora = "0.1" +webrtc-vad = "0.4" + +# ndarray is required by ort's tensor construction API and by +# Silero / TEN VAD ONNX inference across all platforms. +ndarray = "0.17" # Opus encoder. tsclientlib already pulls this; we depend explicitly so # this crate can compile against it without going through tsclientlib. @@ -47,19 +53,15 @@ coreaudio-rs = "0.14" # Grand Central Dispatch bindings — used to run AudioUnit initialize/start # on the main queue to avoid the VPIO RPC timeout on iOS simulator. dispatch2 = "0.3" -# ndarray is required by ort's tensor construction API. -ndarray = "0.16" [target.'cfg(target_os = "ios")'.dependencies] # ONNX Runtime Rust binding for Silero VAD v6 (P1 VAD_002). The official # iOS CocoaPod ships ONNX Runtime as a static framework, so iOS links it # into chanora_bridge at build time instead of loading a dylib at runtime. -ort = { version = "2.0.0-rc.10", default-features = false, features = ["std", "ndarray"] } +ort = { version = "2.0.0-rc.12", default-features = false, features = ["std", "ndarray"] } -[target.'cfg(target_os = "macos")'.dependencies] -# macOS keeps dynamic loading so developer machines can provide ORT via -# ORT_DYLIB_PATH without forcing a bundled runtime into desktop builds. -ort = { version = "2.0.0-rc.10", default-features = false, features = ["load-dynamic", "ndarray"] } +[target.'cfg(not(target_os = "ios"))'.dependencies] +ort = { version = "2.0.0-rc.12", default-features = false, features = ["load-dynamic", "ndarray", "api-24"] } [target.'cfg(target_os = "android")'.dependencies] # Android cross-builds should not pull OpenSSL. Use rustls here while keeping @@ -75,7 +77,14 @@ ndk-context = "0.1" # shipped with `oboe-sys` 0.6 covers armv7 / aarch64 / x86 / x86_64. # Default features keep the precompiled library + pregenerated bindings # so we avoid the clang-sys / libclang requirement on the build host. -oboe = "0.6" +# +# Using local fork edisonjwa/oboe-rs (v0.6.2) with: +# - catch_unwind safety in callbacks +# - unwrap_or_default in enum getters (no more SessionId panic) +# - get_raw_session_id() for JNI hardware effect binding +# - deduplicated macro impls +# - PowerSavingOffloaded PerformanceMode variant +oboe = { path = "../../../oboe-rs" } [target.'cfg(target_os = "windows")'.dependencies] # Real Windows global PTT (SDD-083 / SDD-084): RegisterRawInputDevices diff --git a/crates/chanora_audio/src/android_voice_unit.rs b/crates/chanora_audio/src/android_voice_unit.rs index d3d05f9..430e7aa 100644 --- a/crates/chanora_audio/src/android_voice_unit.rs +++ b/crates/chanora_audio/src/android_voice_unit.rs @@ -66,72 +66,194 @@ use oboe::{ PerformanceMode, SessionId, SharingMode, Usage, }; +use crate::processor::AudioProcessor; + // `BackendEvent` / `BackendEventRx` / `BackendEventTx` moved to // `mobile_voice_backend` so the trait can expose `take_event_rx` // (SDD-111 item 1) cross-platform. +// --- Render-reference buffer for AEC (SDD-111 / SDD-120) --------- +// +// The output (render) callback writes the audio that will be played +// into this ring buffer. The capture callback reads the latest render +// frame and feeds it to WebRTC APM's `process_render` so AEC can +// subtract the speaker output from the microphone input. +// +// 4 slots × 10 ms × 48 kHz mono f32. One slot is always being written +// by the render callback; the capture callback reads the slot that was +// most recently completed. + +const RENDER_REF_SLOTS: usize = 4; +const RENDER_REF_SAMPLES: usize = crate::frame::FRAME_10MS_SAMPLES; + +struct RenderReferenceBuffer { + buf: Box<[[f32; RENDER_REF_SAMPLES]; RENDER_REF_SLOTS]>, + write_idx: std::sync::atomic::AtomicUsize, +} + +impl RenderReferenceBuffer { + fn new() -> Arc { + Arc::new(Self { + buf: Box::new([[0.0_f32; RENDER_REF_SAMPLES]; RENDER_REF_SLOTS]), + write_idx: std::sync::atomic::AtomicUsize::new(0), + }) + } + + fn write(&self, frame: &[f32; RENDER_REF_SAMPLES]) { + let idx = self.write_idx.load(Ordering::Relaxed); + unsafe { + let slot = + &self.buf[idx] as *const [f32; RENDER_REF_SAMPLES] as *mut [f32; RENDER_REF_SAMPLES]; + (*slot).copy_from_slice(frame); + } + self.write_idx + .store((idx + 1) % RENDER_REF_SLOTS, Ordering::Relaxed); + } + + fn read_latest(&self) -> [f32; RENDER_REF_SAMPLES] { + let wi = self.write_idx.load(Ordering::Relaxed); + let ri = (wi + RENDER_REF_SLOTS - 1) % RENDER_REF_SLOTS; + self.buf[ri] + } +} + +unsafe impl Send for RenderReferenceBuffer {} +unsafe impl Sync for RenderReferenceBuffer {} + // --- Capture state for Oboe input callback (SDD-111 / SDD-120) ---- // -// Mirrors the iOS `IosCaptureState` and the cpal-side `CaptureState`. -// Oboe delivers 48 kHz mono i16 PCM; we apply mic gain, accumulate to -// FRAME_20MS_SAMPLES, encode to Opus 32 kbps (complexity 10, inband FEC, 5 % PLC), -// and try-send the resulting packet on `voice_out_tx`. +// Enhanced with WebRTC APM (AEC/NS/AGC) and VAD (voice activity +// detection). Oboe delivers 48 kHz mono i16 PCM in variable-size +// chunks. We accumulate into 10 ms frames, then: +// +// 1. i16 → f32 conversion +// 2. Read render reference (for AEC) +// 3. WebRtcApmProcessor::process_render + process_capture +// 4. VAD → VoiceActivityStateMachine → TransmitModeSelector +// 5. f32 → i16 conversion + mic gain +// 6. Accumulate to 20 ms → Opus encode → send struct AndroidCaptureState { encoder: OpusEncoder, - /// Accumulator for 48 kHz mono PCM. 2x capacity to absorb - /// cpal-style buffer-size jitter without reallocating. pcm_accum: Vec, opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME], voice_out_tx: mpsc::Sender, transmit_active: Arc, + output_muted: Arc, frames_sent: Arc, mic_gain: f32, + voice_activity_selector: Option>, + vad_detector: crate::vad::WebRtcFallbackVad, + silero_vad_worker: Option, + ten_vad_worker: Option, + current_vad_backend: crate::VadBackend, + silero_model_epoch: u64, + capture_frame_seq: u64, + vad_state: crate::voice_activity::VoiceActivityStateMachine, + webrtc_apm_processor: crate::processor::WebRtcApmProcessor, + audio_processing_config: Arc>, + audio_processing_stats: Arc, + render_reference: Arc, + pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES], + pending_10ms_len: usize, + fallback_warned_backend: Option, } impl AndroidCaptureState { fn new( voice_out_tx: mpsc::Sender, transmit_active: Arc, + output_muted: Arc, frames_sent: Arc, mic_gain: f32, + voice_activity_selector: Option>, + audio_processing_config: Arc>, + audio_processing_stats: Arc, + render_reference: Arc, ) -> Result { let encoder = crate::opus_voice::new_voip_encoder("android")?; + // Android always uses software WebRTC APM for AEC/NS/AGC/HPF. + // The config's EffectOwner fields are resolved by open() AFTER + // hardware-effect binding; the processor is constructed here + // with all modules enabled regardless, so the resolved config + // (Platform vs WebrtcApm) only affects diagnostics, not behaviour. + let webrtc_apm_config = audio_processing_config + .lock() + .map(|cfg| { + let mut c = crate::processor::webrtc_apm::WebRtcApmConfig::from_audio_config(&cfg); + c.aec = true; + c.ns = true; + c.agc = true; + c + }) + .unwrap_or(crate::processor::webrtc_apm::WebRtcApmConfig { + aec: true, + ns: true, + agc: true, + hpf: true, + ..Default::default() + }); Ok(Self { encoder, pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2), opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME], voice_out_tx, transmit_active, + output_muted, frames_sent, mic_gain, + voice_activity_selector, + vad_detector: crate::vad::WebRtcFallbackVad::default(), + silero_vad_worker: None, + ten_vad_worker: None, + current_vad_backend: crate::VadBackend::WebrtcVad, + silero_model_epoch: crate::vad::silero_model_epoch(), + capture_frame_seq: 0, + vad_state: crate::voice_activity::VoiceActivityStateMachine::default(), + webrtc_apm_processor: crate::processor::WebRtcApmProcessor::with_config( + webrtc_apm_config, + )?, + audio_processing_config, + audio_processing_stats, + render_reference, + pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES], + pending_10ms_len: 0, + fallback_warned_backend: None, }) } - /// Consume i16 mono frames from Oboe, accumulate to FRAME_20MS_SAMPLES, - /// encode + send when PTT is held. Oboe delivers at the device's - /// native sample rate (always 48 kHz for modern Android per SRS-210), - /// so no resampling is needed. - fn ingest(&mut self, samples: &[i16]) { + /// Consume i16 mono frames from Oboe. Accumulate to 10 ms chunks, + /// process each through WebRTC APM + VAD, then encode 20 ms frames. + fn ingest_i16(&mut self, samples: &[i16]) { + let mut offset = 0; + while offset < samples.len() { + let remaining = + crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len; + let take = remaining.min(samples.len() - offset); + self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take] + .copy_from_slice(&samples[offset..offset + take]); + self.pending_10ms_len += take; + offset += take; + + if self.pending_10ms_len == crate::frame::FRAME_10MS_SAMPLES { + let frame = self.pending_10ms; + self.process_10ms_capture_frame(&frame); + self.pending_10ms_len = 0; + } + } + if !self.transmit_active.load(Ordering::Relaxed) { self.pcm_accum.clear(); return; } - // Mic-gain application. - if (self.mic_gain - 1.0).abs() < f32::EPSILON { - self.pcm_accum.extend_from_slice(samples); - } else { - let gain = self.mic_gain; - self.pcm_accum.extend(samples.iter().map(|&s| { - let scaled = (s as f32) * gain; - scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16 - })); - } - // Drain complete 20 ms frames. + while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES { let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES]; - frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]); - self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES); + frame.copy_from_slice( + &self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES], + ); + self.pcm_accum + .drain(..crate::frame::FRAME_20MS_SAMPLES); match self.encoder.encode(&frame, &mut self.opus_out[..]) { Ok(len) => { crate::opus_voice::send_voip_frame( @@ -154,11 +276,186 @@ impl AndroidCaptureState { ); } Err(e) => { - warn!(target: "chanora_audio", error = %e, "android Oboe opus encode failed"); + warn!( + target: "chanora_audio", + error = %e, + "android Oboe opus encode failed" + ); } } } } + + fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) { + if self.fallback_warned_backend != Some(failed_backend) { + self.fallback_warned_backend = Some(failed_backend); + warn!( + target: "chanora_audio", + backend = failed_backend.as_str(), + "android: VAD backend unavailable; using WebRTC fallback for runtime detection" + ); + } + } + + fn process_10ms_capture_frame( + &mut self, + samples: &[i16; crate::frame::FRAME_10MS_SAMPLES], + ) { + let mut frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES]; + for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) { + *dst = crate::frame::i16_to_f32(src); + } + let input_dbfs = crate::frame::dbfs(&frame); + + let render_ref = self.render_reference.read_latest(); + self.webrtc_apm_processor.process_render(&render_ref); + self.webrtc_apm_processor.process_capture(&mut frame); + + let (vad_hangover, vad_backend) = self + .audio_processing_config + .try_lock() + .map(|cfg| (cfg.vad_hangover_ms, cfg.vad_backend)) + .unwrap_or(( + crate::voice_activity::VAD_HANGOVER_MS, + crate::VadBackend::WebrtcVad, + )); + self.vad_state.configure( + crate::voice_activity::VAD_OPEN_AFTER_MS, + vad_hangover, + crate::voice_activity::VAD_MIN_TX_MS, + ); + + // VAD backend switching (mirrors iOS Raw path). + let silero_epoch = crate::vad::silero_model_epoch(); + let silero_changed = vad_backend == crate::VadBackend::SileroOnnx + && silero_epoch != self.silero_model_epoch; + if vad_backend != self.current_vad_backend || silero_changed { + self.current_vad_backend = vad_backend; + self.silero_model_epoch = silero_epoch; + self.fallback_warned_backend = None; + match vad_backend { + crate::VadBackend::SileroOnnx => { + let path = crate::vad::silero_model_bundle_path(); + self.silero_vad_worker = + crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&path); + self.ten_vad_worker = None; + if self.silero_vad_worker.is_none() { + warn!( + target: "chanora_audio", + "android: Silero VAD model not found at {path}; falling back to WebRTC VAD" + ); + } + } + crate::VadBackend::TenVad => { + let path = crate::vad::ten_model_bundle_path(); + self.ten_vad_worker = crate::vad::TenOnnxVadWorker::try_new(&path); + self.silero_vad_worker = None; + if self.ten_vad_worker.is_none() { + warn!( + target: "chanora_audio", + "android: TEN VAD ONNX model not found at {path}; falling back to WebRTC VAD" + ); + } + } + _ => { + self.silero_vad_worker = None; + self.ten_vad_worker = None; + } + } + self.vad_state.reset(); + } + + self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1); + let capture_seq = self.capture_frame_seq; + let mut used_fallback_vad = false; + let vad = if vad_backend == crate::VadBackend::Disabled { + crate::vad::VadOutput { + probability: 1.0, + speech: true, + } + } else if vad_backend == crate::VadBackend::SileroOnnx { + if let Some(worker) = self.silero_vad_worker.as_ref() { + let enqueued = worker.try_send(capture_seq, &frame); + if enqueued && !worker.is_stale(capture_seq) { + let p = worker.latest_probability(); + crate::vad::VadOutput { + probability: p, + speech: p >= 0.5, + } + } else { + used_fallback_vad = true; + self.mark_vad_fallback_active(vad_backend); + crate::vad::VoiceActivityDetector::process_10ms( + &mut self.vad_detector, + &frame, + ) + } + } else { + used_fallback_vad = true; + self.mark_vad_fallback_active(vad_backend); + crate::vad::VoiceActivityDetector::process_10ms( + &mut self.vad_detector, + &frame, + ) + } + } else if vad_backend == crate::VadBackend::TenVad { + if let Some(worker) = self.ten_vad_worker.as_ref() { + let enqueued = worker.try_send(capture_seq, &frame); + if enqueued && !worker.is_stale(capture_seq) { + let p = worker.latest_probability(); + crate::vad::VadOutput { + probability: p, + speech: p >= 0.5, + } + } else { + used_fallback_vad = true; + self.mark_vad_fallback_active(vad_backend); + crate::vad::VoiceActivityDetector::process_10ms( + &mut self.vad_detector, + &frame, + ) + } + } else { + used_fallback_vad = true; + self.mark_vad_fallback_active(vad_backend); + crate::vad::VoiceActivityDetector::process_10ms( + &mut self.vad_detector, + &frame, + ) + } + } else { + crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) + }; + self.audio_processing_stats + .set_vad_fallback_active(used_fallback_vad); + let active = self.vad_state.update(vad.speech); + let output_muted = self.output_muted.load(Ordering::Relaxed); + if let Some(sel) = &self.voice_activity_selector { + sel.set_voice_activity_open(active && !output_muted); + } + self.audio_processing_stats.update_capture( + input_dbfs, + crate::frame::dbfs(&frame), + vad.probability, + active && !output_muted, + self.transmit_active.load(Ordering::Relaxed), + ); + + if !self.transmit_active.load(Ordering::Relaxed) || output_muted { + return; + } + + let gain = self.mic_gain; + if (gain - 1.0).abs() < f32::EPSILON { + self.pcm_accum + .extend(frame.iter().copied().map(crate::frame::f32_to_i16)); + } else { + self.pcm_accum.extend(frame.iter().copied().map(|s| { + let scaled = (crate::frame::f32_to_i16(s) as f32) * gain; + scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16 + })); + } + } } struct InputCallback { @@ -176,7 +473,7 @@ impl AudioInputCallback for InputCallback { ) -> DataCallbackResult { let _ = catch_unwind(AssertUnwindSafe(|| { if let Ok(mut s) = self.state.lock() { - s.ingest(frames); + s.ingest_i16(frames); } })); DataCallbackResult::Continue @@ -203,6 +500,7 @@ struct OutputCallback { output_muted: Arc, event_tx: BackendEventTx, scratch: Arc>>, + render_reference: Arc, } impl AudioOutputCallback for OutputCallback { @@ -223,16 +521,17 @@ impl AudioOutputCallback for OutputCallback { *s = 0.0; } } - // Non-blocking pull from AudioHandler (same pattern as iOS VPIO). match self.handler.try_lock() { Ok(mut h) => { let _ = h.fill_buffer(&mut scratch[..needed]); } - Err(std::sync::TryLockError::WouldBlock) => { - // scratch already zeroed above. - } + Err(std::sync::TryLockError::WouldBlock) => {} Err(std::sync::TryLockError::Poisoned(e)) => { - warn!(target: "chanora_audio", "AudioHandler mutex poisoned: {}", e); + warn!( + target: "chanora_audio", + "AudioHandler mutex poisoned: {}", + e + ); } } let gain = f32::from_bits(self.output_gain.load(Ordering::Relaxed)); @@ -243,6 +542,19 @@ impl AudioOutputCallback for OutputCallback { gain, muted, ); + + // Write the first 10 ms of render audio into the reference + // buffer for the capture-side AEC. + let mono_n = needed / 2; + let render_n = mono_n.min(crate::frame::FRAME_10MS_SAMPLES); + let mut ref_frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES]; + for (i, chunk) in scratch[..render_n * 2].chunks_exact(2).enumerate() { + if i >= render_n { + break; + } + ref_frame[i] = (chunk[0] + chunk[1]) * 0.5; + } + self.render_reference.write(&ref_frame); })); DataCallbackResult::Continue } @@ -273,7 +585,7 @@ pub struct VoiceAudioParams { pub frames_sent: Arc, /// Pre-encode amplitude scale (1.0 = unity). pub mic_gain: f32, - /// AudioHandler that inbound解码+混合 feeds into; the Oboe output + /// AudioHandler that inbound decode+mix feeds into; the Oboe output /// callback pulls mixed stereo f32 from it. pub handler: Arc>>, /// Master output gain (f32 bits stored in AtomicU32 for lock-free @@ -281,6 +593,14 @@ pub struct VoiceAudioParams { pub output_gain: Arc, /// True = output silence regardless of incoming voice frames. pub output_muted: Arc, + /// Optional TransmitModeSelector for VoiceActivity transmit mode. + /// The capture callback calls set_voice_activity_open on this when + /// VAD detects speech. None means VoiceActivity mode is disabled. + pub voice_activity_selector: Option>, + /// Shared audio-processing config (WebRTC APM flags, VAD backend). + pub audio_processing_config: Arc>, + /// Shared audio-processing statistics for diagnostics. + pub audio_processing_stats: Arc, } // --- The backend itself ------------------------------------------ @@ -334,22 +654,30 @@ impl AndroidVoiceUnit { ) -> Result { let (event_tx, event_rx) = mpsc::unbounded_channel(); - // SDD-120: build the capture state that the Oboe input callback - // will own via Arc. Same Opus VoIP tuning as iOS and - // desktop (32 kbps, complexity 10, inband FEC, 5 % PLC). + // Shared render-reference buffer (for AEC). The output + // callback writes; the capture callback reads. + let render_ref_buf = RenderReferenceBuffer::new(); + let render_ref_for_capture = render_ref_buf.clone(); + + // Clone the APM config Arc before params is partially moved + // into the capture state constructor below. + let apm_config_clone = params.audio_processing_config.clone(); + let capture_state = Arc::new(Mutex::new( AndroidCaptureState::new( params.voice_out_tx, params.transmit_active, + params.output_muted.clone(), params.frames_sent, params.mic_gain, + params.voice_activity_selector, + params.audio_processing_config, + params.audio_processing_stats, + render_ref_for_capture, ) .map_err(|e| BackendError::OpenFailed(format!("capture state init: {e}")))?, )); - // Scratch buffer for the output callback (realtime-safe - // pre-allocation). 8192 floats covers the largest practical - // burst size at 48 kHz with headroom. let scratch = Arc::new(Mutex::new(Vec::with_capacity(8192))); // --- Open input stream (SDD-112) --------------------------- @@ -423,13 +751,19 @@ impl AndroidVoiceUnit { .as_mut() .map(|s| s.get_frames_per_burst()) .unwrap_or(0); - let session_id = None; - warn!( - target: "chanora_audio", - "android: oboe-rs get_session_id is skipped because oboe 0.6.1 \ - panics on some Android allocated-session values; hardware \ - effects are disabled for this stream" - ); + // The oboe-rs fork (edisonjwa/oboe-rs 0.6.2) fixes the + // get_session_id() panic with unwrap_or_default(), but the + // SessionId enum still only models builder parameters (None = + // -1, Allocate = 0). The actual system audio session ID (>0) + // written by AAudio to mSessionId after stream open cannot be + // expressed in the current enum; get_raw_session_id() is a + // follow-up addition to the fork. + // + // For now: hardware effects require the system session ID. + // WebRTC APM software processing handles AEC/NS/AGC/HPF. + let session_id: Option = input_stream + .as_ref() + .and_then(|s| s.get_raw_session_id()); // --- Open output stream (SDD-112) -------------------------- let output_builder = AudioStreamBuilder::default() @@ -450,12 +784,14 @@ impl AndroidVoiceUnit { .set_usage(Usage::VoiceCommunication) .set_content_type(oboe::ContentType::Speech); + let render_ref_for_output = render_ref_buf.clone(); let output_cb = OutputCallback { handler: params.handler.clone(), output_gain: params.output_gain.clone(), output_muted: params.output_muted.clone(), event_tx: event_tx.clone(), scratch: scratch.clone(), + render_reference: render_ref_for_output, }; let output_builder = output_builder.set_callback(output_cb); @@ -474,6 +810,7 @@ impl AndroidVoiceUnit { params.output_gain.clone(), params.output_muted.clone(), scratch.clone(), + render_ref_buf, )? } }; @@ -511,6 +848,52 @@ impl AndroidVoiceUnit { HardwareEffectHandles::default() }; + // --- SDD-113 config resolution: hardware-available? ------- + // Android gives the user a choice between hardware (JNI) and + // software (WebRTC APM) effects. The AudioProcessingConfig's + // EffectOwner fields encode that choice: + // Platform → prefer hardware; software fallback if missing + // WebrtcApm → always software WebRTC APM + // Off → disable effect entirely + // + // Here we resolve Platform → WebrtcApm for each effect whose + // hardware binding failed (or wasn't attempted). This is read + // by the capture callback's WebRtcApmProcessor. + { + use crate::audio_processing::EffectOwner; + let mut apm_cfg = apm_config_clone.lock().unwrap(); + let hw_aec = hw_effects.aec.is_some(); + let hw_ns = hw_effects.ns.is_some(); + let hw_agc = hw_effects.agc.is_some(); + if apm_cfg.aec == EffectOwner::Platform && !hw_aec { + apm_cfg.aec = EffectOwner::WebrtcApm; + } + if apm_cfg.ns == EffectOwner::Platform && !hw_ns { + apm_cfg.ns = EffectOwner::WebrtcApm; + } + if apm_cfg.agc == EffectOwner::Platform && !hw_agc { + apm_cfg.agc = EffectOwner::WebrtcApm; + } + if apm_cfg.processing_backend + == crate::audio_processing::AudioBackend::PlatformVoiceProcessing + && (!hw_aec || !hw_ns || !hw_agc) + { + apm_cfg.processing_backend = crate::audio_processing::AudioBackend::WebrtcApm; + } + info!( + target: "chanora_audio", + aec = ?apm_cfg.aec, + ns = ?apm_cfg.ns, + agc = ?apm_cfg.agc, + hpf = apm_cfg.hpf_enabled, + hw_aec, + hw_ns, + hw_agc, + session_id, + "android: audio processing config resolved (hardware effects: aec={hw_aec} ns={hw_ns} agc={hw_agc})" + ); + } + // --- SDD-112 item 10 / SDD-113 item 7 / SDD-116 item 3 --- // Publish the diagnostics snapshot. Per-effect engagement is // derived from (a) the JNI handle (`Hardware`) or (b) the @@ -647,6 +1030,7 @@ impl AndroidVoiceUnit { output_gain: Arc, output_muted: Arc, scratch: Arc>>, + render_reference: Arc, ) -> Result, BackendError> { let cb = OutputCallback { handler, @@ -654,6 +1038,7 @@ impl AndroidVoiceUnit { output_muted, event_tx: event_tx.clone(), scratch, + render_reference, }; let builder = AudioStreamBuilder::default() .set_direction::() @@ -806,6 +1191,7 @@ fn perf_from_oboe(p: PerformanceMode) -> AchievedPerformanceMode { match p { PerformanceMode::LowLatency => AchievedPerformanceMode::LowLatency, PerformanceMode::PowerSaving => AchievedPerformanceMode::PowerSaving, + PerformanceMode::PowerSavingOffloaded => AchievedPerformanceMode::PowerSaving, PerformanceMode::None => AchievedPerformanceMode::None, } } @@ -1041,6 +1427,51 @@ fn release_hardware_effects_inner(handles: &mut HardwareEffectHandles) { } } +// --- Process-global BackendEvent sender for JNI callbacks -------- +// +// Kotlin-side listeners (audio focus, Bluetooth SCO, device route +// changes) need to publish events into the Rust engine's event +// channel. Since the engine's `BackendEventTx` is created at voice +// start, we store it here as a process-global so the JNI callbacks +// can reach it without holding a direct Rust reference. +// +// Cleared on voice stop; the Kotlin listeners are idempotent when +// no sender is registered (they log and continue). + +static GLOBAL_BACKEND_EVENT_TX: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +fn global_event_tx_slot( +) -> &'static std::sync::Mutex>> { + GLOBAL_BACKEND_EVENT_TX.get_or_init(|| std::sync::Mutex::new(None)) +} + +pub(crate) fn register_global_event_sender(tx: BackendEventTx) { + if let Ok(mut g) = global_event_tx_slot().lock() { + *g = Some(tx); + } +} + +pub(crate) fn clear_global_event_sender() { + if let Ok(mut g) = global_event_tx_slot().lock() { + *g = None; + } +} + +fn try_send_backend_event(event: BackendEvent) { + if let Ok(g) = global_event_tx_slot().lock() { + if let Some(tx) = g.as_ref() { + if tx.send(event).is_err() { + warn!( + target: "chanora_audio", + "android: global BackendEvent channel closed; event dropped" + ); + } + } + } +} + // --- SDD-115 foreground-service JNI helpers ---------------------- // // The Kotlin class `AndroidVoiceForegroundService` (Wave 2B-2) @@ -1174,3 +1605,158 @@ fn load_app_class<'local>( } } } + +// --- SDD-109 / SDD-110 JNI callbacks: focus + SCO events ---------- +// +// Kotlin-side OnAudioFocusChangeListener and BroadcastReceiver for +// ACTION_SCO_AUDIO_STATE_UPDATED call these Rust entry points via +// JNI. Each function marshals the platform event into a BackendEvent +// and posts it through the global event sender registered by the +// engine at voice start. +// +// SDD-115 callback safety: every entry point is wrapped in +// catch_unwind so a panic in the Rust engine can never unwind +// into the JVM. + +/// SDD-109: audio focus change published by Kotlin's +/// `AndroidAudioFocusController`. `state` is the `focusChange` +/// value from `OnAudioFocusChangeListener`. +/// +/// Symbol naming: JNI function declared in +/// `app.chanora.chanora_flutter.AndroidAudioFocusController`. +#[no_mangle] +pub extern "system" fn Java_app_chanora_chanora_1flutter_AndroidAudioFocusController_publishFocusChange< + 'local, +>( + _env: jni::JNIEnv<'local>, + _class: jni::objects::JClass<'local>, + state: jni::sys::jint, +) { + let _ = catch_unwind(AssertUnwindSafe(|| { + // AUDIOFOCUS_LOSS = -1, LOSS_TRANSIENT = -2, LOSS_TRANSIENT_CAN_DUCK = -3, + // GAIN = 1 (AudioManager.AUDIOFOCUS_REQUEST_GRANTED is also 1, but we only + // call this from the listener callback so the values are well-known). + let event = match state { + -1 => BackendEvent::FocusLost, + -2 => BackendEvent::FocusTransient, + -3 => BackendEvent::FocusTransientCanDuck, + 1 | 2 | 3 | 4 => BackendEvent::FocusGain, + _ => { + warn!( + target: "chanora_audio", + state, + "android: unknown audio focus change value; treating as FocusLost" + ); + BackendEvent::FocusLost + } + }; + try_send_backend_event(event); + })); +} + +/// SDD-110: Bluetooth SCO state change published by Kotlin's +/// `AndroidBluetoothScoController`. `state` is the `STATE` +/// value from `ACTION_SCO_AUDIO_STATE_UPDATED`: +/// - `ACTION_SCO_AUDIO_STATE_UPDATED` is always fired with `EXTRA_SCO_AUDIO_STATE` +/// - `SCO_STATE_CONNECTING = 0`, `SCO_STATE_CONNECTED = 1`, `SCO_STATE_DISCONNECTED = 2` +/// +/// Symbol naming: JNI function declared in +/// `app.chanora.chanora_flutter.AndroidBluetoothScoController`. +#[no_mangle] +pub extern "system" fn Java_app_chanora_chanora_1flutter_AndroidBluetoothScoController_publishScoStateChange< + 'local, +>( + _env: jni::JNIEnv<'local>, + _class: jni::objects::JClass<'local>, + state: jni::sys::jint, +) { + let _ = catch_unwind(AssertUnwindSafe(|| { + try_send_backend_event(BackendEvent::BluetoothScoStateChanged(state)); + })); +} + +/// SDD-115 integration: start the Android audio focus listener. +/// Called by the engine after the voice unit is started. Uses JNI +/// to invoke `AndroidAudioFocusController.start(Context)`. +pub fn chanora_android_request_audio_focus() -> bool { + call_static_void_context( + "app/chanora/chanora_flutter/AndroidAudioFocusController", + "start", + ) +} + +/// SDD-115 integration: stop the Android audio focus listener. +/// Called by the engine on voice stop. +pub fn chanora_android_abandon_audio_focus() -> bool { + call_static_void_context( + "app/chanora/chanora_flutter/AndroidAudioFocusController", + "stop", + ) +} + +/// SDD-115 integration: start Bluetooth SCO. +/// Called by the engine after the voice unit is started. +pub fn chanora_android_start_bluetooth_sco() -> bool { + call_static_void_context( + "app/chanora/chanora_flutter/AndroidBluetoothScoController", + "start", + ) +} + +/// SDD-115 integration: stop Bluetooth SCO. +/// Called by the engine on voice stop. +pub fn chanora_android_stop_bluetooth_sco() -> bool { + call_static_void_context( + "app/chanora/chanora_flutter/AndroidBluetoothScoController", + "stop", + ) +} + +fn call_static_void_context(fqcn: &str, method: &str) -> bool { + use jni::objects::{JObject, JValue}; + let ctx = ndk_context::android_context(); + if ctx.vm().is_null() || ctx.context().is_null() { + warn!( + target: "chanora_audio", + class = fqcn, + method, + "android: ndk_context not initialised; call skipped" + ); + return false; + } + let jvm = match unsafe { jni::JavaVM::from_raw(ctx.vm() as *mut _) } { + Ok(v) => v, + Err(e) => { + warn!(target: "chanora_audio", error = %e, class = fqcn, method, "android: JavaVM::from_raw failed"); + return false; + } + }; + let mut env = match jvm.attach_current_thread() { + Ok(e) => e, + Err(e) => { + warn!(target: "chanora_audio", error = %e, class = fqcn, method, "android: attach_current_thread failed"); + return false; + } + }; + let context_obj = unsafe { JObject::from_raw(ctx.context() as jni::sys::jobject) }; + let class = match load_app_class(&mut env, &context_obj, fqcn) { + Some(c) => c, + None => return false, + }; + match env.call_static_method( + &class, + method, + "(Landroid/content/Context;)V", + &[JValue::Object(&context_obj)], + ) { + Ok(_) => { + info!(target: "chanora_audio", class = fqcn, method, "android: dispatched"); + true + } + Err(e) => { + let _ = env.exception_clear(); + warn!(target: "chanora_audio", error = %e, class = fqcn, method, "android: static call failed"); + false + } + } +} diff --git a/crates/chanora_audio/src/audio_processing.rs b/crates/chanora_audio/src/audio_processing.rs index 482c740..cd92e83 100644 --- a/crates/chanora_audio/src/audio_processing.rs +++ b/crates/chanora_audio/src/audio_processing.rs @@ -58,7 +58,7 @@ impl AudioRoute { pub enum IosVoiceProcessingMode { /// Shipping default: Apple VoiceProcessingIO owns AEC/NS/AGC. PlatformVoiceProcessing, - /// Experimental Sonora capture-processing path. + /// Experimental raw capture-processing path. SonoraExperimental, } @@ -69,7 +69,7 @@ pub enum AudioBackend { PlatformVoiceProcessing, /// Rust-native Sonora backend. Sonora, - /// Future WebRTC APM backend. + /// WebRTC Audio Processing Module backend. WebrtcApm, /// No processing. Noop, @@ -168,9 +168,9 @@ impl Default for AudioProcessingConfig { route: AudioRoute::Speaker, ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing, processing_backend: AudioBackend::PlatformVoiceProcessing, - vad_backend: VadBackend::SileroOnnx, + vad_backend: VadBackend::TenVad, aec: EffectOwner::Platform, - // iOS VPIO owns NS/AGC on the default shipping path. Rust/Sonora + // iOS VPIO owns NS/AGC on the default shipping path. Software // effects are opt-in through the experimental raw route only. ns: EffectOwner::Platform, agc: EffectOwner::Platform, @@ -194,19 +194,23 @@ impl AudioProcessingConfig { } if self.ios_mode == IosVoiceProcessingMode::PlatformVoiceProcessing && (self.processing_backend == AudioBackend::Sonora + || self.processing_backend == AudioBackend::WebrtcApm || self.aec == EffectOwner::Sonora + || self.aec == EffectOwner::WebrtcApm || self.ns == EffectOwner::Sonora - || self.agc == EffectOwner::Sonora) + || self.ns == EffectOwner::WebrtcApm + || self.agc == EffectOwner::Sonora + || self.agc == EffectOwner::WebrtcApm) { return Err(AudioError::InvalidAudioProcessingConfig( - "Sonora cannot be enabled with iOS VoiceProcessingIO".to_string(), + "software audio processing cannot be enabled with iOS VoiceProcessingIO" + .to_string(), )); } if self.ios_mode == IosVoiceProcessingMode::SonoraExperimental { - if self.processing_backend != AudioBackend::Sonora { + if self.processing_backend != AudioBackend::WebrtcApm { return Err(AudioError::InvalidAudioProcessingConfig( - "ios Sonora experimental mode requires the Sonora processing backend" - .to_string(), + "ios raw processing mode requires the WebRTC APM backend".to_string(), )); } } @@ -246,9 +250,9 @@ mod tests { } #[test] - fn platform_voice_processing_rejects_sonora_effects() { + fn platform_voice_processing_rejects_software_effects() { let config = AudioProcessingConfig { - ns: EffectOwner::Sonora, + ns: EffectOwner::WebrtcApm, ..AudioProcessingConfig::default() }; @@ -261,13 +265,13 @@ mod tests { } #[test] - fn sonora_experimental_allows_full_sonora_chain() { + fn raw_processing_allows_full_webrtc_apm_chain() { let config = AudioProcessingConfig { ios_mode: IosVoiceProcessingMode::SonoraExperimental, - processing_backend: AudioBackend::Sonora, - aec: EffectOwner::Sonora, - ns: EffectOwner::Sonora, - agc: EffectOwner::Sonora, + processing_backend: AudioBackend::WebrtcApm, + aec: EffectOwner::WebrtcApm, + ns: EffectOwner::WebrtcApm, + agc: EffectOwner::WebrtcApm, ..AudioProcessingConfig::default() }; @@ -275,13 +279,13 @@ mod tests { } #[test] - fn sonora_experimental_rejects_non_sonora_backend() { + fn raw_processing_rejects_non_webrtc_apm_backend() { let config = AudioProcessingConfig { ios_mode: IosVoiceProcessingMode::SonoraExperimental, processing_backend: AudioBackend::PlatformVoiceProcessing, - aec: EffectOwner::Sonora, - ns: EffectOwner::Sonora, - agc: EffectOwner::Sonora, + aec: EffectOwner::WebrtcApm, + ns: EffectOwner::WebrtcApm, + agc: EffectOwner::WebrtcApm, ..AudioProcessingConfig::default() }; diff --git a/crates/chanora_audio/src/engine.rs b/crates/chanora_audio/src/engine.rs index d6bb2b9..ba82d6a 100644 --- a/crates/chanora_audio/src/engine.rs +++ b/crates/chanora_audio/src/engine.rs @@ -310,14 +310,14 @@ fn open_ios_voice_backend( audio_processing_stats.clone(), ) { Ok(unit) => { - info!(target: "chanora_audio", "ios: RemoteIO/Sonora experimental backend selected"); + info!(target: "chanora_audio", "ios: RemoteIO/WebRTC APM backend selected"); return Ok(IosVoiceBackend::Raw(unit)); } Err(e) => { warn!( target: "chanora_audio", error = %e, - "ios: RemoteIO/Sonora backend failed; falling back to VoiceProcessingIO" + "ios: RemoteIO/WebRTC APM backend failed; falling back to VoiceProcessingIO" ); } } @@ -724,6 +724,9 @@ impl AudioEngine { handler: audio_handler.clone(), output_gain: output_gain.clone(), output_muted: output_muted.clone(), + voice_activity_selector: cfg.voice_activity_selector.clone(), + audio_processing_config: audio_processing_config.clone(), + audio_processing_stats: audio_processing_stats.clone(), }; let mut android_voice_unit = crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av, params).map_err(|e| { @@ -739,16 +742,75 @@ impl AudioEngine { if let Some(mut event_rx) = android_voice_unit.take_event_rx() { tokio::spawn(async move { while let Some(event) = event_rx.recv().await { - if let BackendEvent::Disconnected = event { - warn!( - target: "chanora_audio", - "android: backend disconnected event received; stream reconnect requires session restart" - ); + match event { + BackendEvent::Disconnected => { + warn!( + target: "chanora_audio", + "android: backend disconnected event received; stream reconnect requires session restart" + ); + } + BackendEvent::FocusLost => { + warn!( + target: "chanora_audio", + "android: audio focus lost permanently (SDD-109); engine should leave session" + ); + } + BackendEvent::FocusTransient => { + info!( + target: "chanora_audio", + "android: transient audio focus loss (SDD-109); pausing capture" + ); + } + BackendEvent::FocusTransientCanDuck => { + info!( + target: "chanora_audio", + "android: transient audio focus loss with ducking (SDD-109); continuing" + ); + } + BackendEvent::FocusGain => { + info!( + target: "chanora_audio", + "android: audio focus regained (SDD-109); resuming capture" + ); + } + BackendEvent::BluetoothScoStateChanged(s) => { + info!( + target: "chanora_audio", + sco_state = s, + "android: Bluetooth SCO state changed (SDD-110)" + ); + } } } }); } + crate::android_voice_unit::register_global_event_sender(android_voice_unit.event_sender()); + + if crate::android_voice_unit::chanora_android_request_audio_focus() { + info!( + target: "chanora_audio", + "android: audio focus requested (SDD-109)" + ); + } else { + warn!( + target: "chanora_audio", + "android: audio focus request failed; engine operates without focus (SDD-109)" + ); + } + + if crate::android_voice_unit::chanora_android_start_bluetooth_sco() { + info!( + target: "chanora_audio", + "android: Bluetooth SCO started (SDD-110)" + ); + } else { + warn!( + target: "chanora_audio", + "android: Bluetooth SCO start failed; BT HFP may not route correctly (SDD-110)" + ); + } + let capture_active = true; let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); @@ -949,13 +1011,17 @@ impl AudioEngine { let _ = self._ios_voice_backend.lock().unwrap().take(); } // SDD-115 reverse-order teardown on Android: - // 1) close the voice unit (releases SDD-113 hardware + // 1) stop Bluetooth SCO + abandon audio focus; + // 2) close the voice unit (releases SDD-113 hardware // effects then stops + closes the Oboe streams); - // 2) restore the prior audio mode (SDD-108 §1 on 1 → 0 + // 3) restore the prior audio mode (SDD-108 §1 on 1 → 0 // transition); - // 3) stop the foreground service. + // 4) stop the foreground service. #[cfg(target_os = "android")] { + crate::android_voice_unit::chanora_android_stop_bluetooth_sco(); + crate::android_voice_unit::chanora_android_abandon_audio_focus(); + crate::android_voice_unit::clear_global_event_sender(); if let Some(mut unit) = self._android_voice_unit.lock().unwrap().take() { use crate::mobile_voice_backend::MobileVoiceAudioBackend; if let Err(e) = unit.close() { diff --git a/crates/chanora_audio/src/ios_raw_unit.rs b/crates/chanora_audio/src/ios_raw_unit.rs index 3c4369b..bf57a3b 100644 --- a/crates/chanora_audio/src/ios_raw_unit.rs +++ b/crates/chanora_audio/src/ios_raw_unit.rs @@ -1,15 +1,15 @@ -//! Optional raw iOS RemoteIO path for the Sonora experimental mode. +//! Optional raw iOS RemoteIO path for the WebRTC APM experimental mode. //! //! Provides an alternative to `ios_voice_unit.rs` for the //! `SonoraExperimental` processing mode. Instead of //! `kAudioUnitSubType_VoiceProcessingIO` (which owns AEC/NS/AGC), it //! opens `kAudioUnitSubType_RemoteIO` with voice processing explicitly -//! disabled so Rust's Sonora DSP chain can own the full signal path. +//! disabled so WebRTC APM can own the full signal path. //! //! ## Hard invariants enforced here //! //! * INV_009: Rust AEC only active when platform AEC is disabled. -//! * INV_010: VoiceProcessingIO and Sonora AEC3 are mutually exclusive. +//! * INV_010: VoiceProcessingIO and WebRTC APM AEC are mutually exclusive. //! * INV_011: Software AEC backend receives both capture and render-reference. //! * INV_012: Render reference is copied from decoded/mixed remote PCM //! before playout. @@ -112,19 +112,25 @@ mod inner { opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME], voice_out_tx: mpsc::Sender, transmit_active: Arc, + output_muted: Arc, frames_sent: Arc, mic_gain: f32, voice_activity_selector: Option>, vad_detector: crate::vad::WebRtcFallbackVad, + silero_vad_worker: Option, + ten_vad_worker: Option, + current_vad_backend: crate::VadBackend, + silero_model_epoch: u64, + capture_frame_seq: u64, vad_state: crate::voice_activity::VoiceActivityStateMachine, - /// Processing config — retained for route-change reloads; not read in the hot path. - #[allow(dead_code)] + /// Processing config — retained for route-change reloads. audio_processing_config: Arc>, - sonora_processor: crate::processor::SonoraProcessor, + webrtc_apm_processor: crate::processor::WebRtcApmProcessor, audio_processing_stats: Arc, render_reference: Arc, pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES], pending_10ms_len: usize, + fallback_warned_backend: Option, wav_recorder: Option>, } @@ -132,6 +138,7 @@ mod inner { fn new( voice_out_tx: mpsc::Sender, transmit_active: Arc, + output_muted: Arc, frames_sent: Arc, mic_gain: f32, voice_activity_selector: Option>, @@ -140,32 +147,48 @@ mod inner { render_reference: Arc, ) -> Result { let encoder = crate::opus_voice::new_voip_encoder("ios raw")?; + let webrtc_apm_config = audio_processing_config + .lock() + .map(|cfg| crate::processor::webrtc_apm::WebRtcApmConfig::from_audio_config(&cfg)) + .unwrap_or_default(); Ok(Self { encoder, pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2), opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME], voice_out_tx, transmit_active, + output_muted, frames_sent, mic_gain, voice_activity_selector, vad_detector: crate::vad::WebRtcFallbackVad::default(), + silero_vad_worker: None, + ten_vad_worker: None, + current_vad_backend: crate::VadBackend::WebrtcVad, + silero_model_epoch: crate::vad::silero_model_epoch(), + capture_frame_seq: 0, vad_state: crate::voice_activity::VoiceActivityStateMachine::default(), audio_processing_config, - sonora_processor: crate::processor::SonoraProcessor::with_config( - crate::processor::sonora::SonoraConfig::with_aec3(), - ), + webrtc_apm_processor: crate::processor::WebRtcApmProcessor::with_config( + webrtc_apm_config, + )?, audio_processing_stats, render_reference, pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES], pending_10ms_len: 0, + fallback_warned_backend: None, wav_recorder: None, }) } - fn disable_failed_vad_backend(&mut self, failed_backend: crate::VadBackend) { - if let Ok(mut cfg) = self.audio_processing_config.try_lock() { - let _ = cfg.disable_failed_vad_backend(failed_backend); + fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) { + if self.fallback_warned_backend != Some(failed_backend) { + self.fallback_warned_backend = Some(failed_backend); + tracing::warn!( + target: "chanora_audio", + backend = failed_backend.as_str(), + "VAD backend unavailable; using WebRTC fallback for runtime detection" + ); } } @@ -237,12 +260,12 @@ mod inner { rec.push_raw_mic(&frame); } - // INV_012: feed render reference to Sonora AEC3 before capture. + // Feed render reference to WebRTC APM before capture so AEC can adapt. let render_ref = self.render_reference.read_latest(); - self.sonora_processor.process_render(&render_ref); - self.sonora_processor.process_capture(&mut frame); + self.webrtc_apm_processor.process_render(&render_ref); + self.webrtc_apm_processor.process_capture(&mut frame); - // WAV tap: processed mic (after Sonora). + // WAV tap: processed mic (after WebRTC APM). if let Some(ref rec) = self.wav_recorder { rec.push_processed_mic(&frame); } @@ -260,37 +283,118 @@ mod inner { vad_hangover, crate::voice_activity::VAD_MIN_TX_MS, ); + + // Switch VAD backend when config changes. + let silero_epoch = crate::vad::silero_model_epoch(); + let silero_changed = vad_backend == crate::VadBackend::SileroOnnx + && silero_epoch != self.silero_model_epoch; + if vad_backend != self.current_vad_backend || silero_changed { + self.current_vad_backend = vad_backend; + self.silero_model_epoch = silero_epoch; + self.fallback_warned_backend = None; + match vad_backend { + crate::VadBackend::SileroOnnx => { + let path = crate::vad::silero_model_bundle_path(); + self.silero_vad_worker = + crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&path); + self.ten_vad_worker = None; + if self.silero_vad_worker.is_none() { + tracing::warn!( + target: "chanora_audio", + "Silero VAD model not found at {path}; falling back to WebRTC VAD" + ); + } + } + crate::VadBackend::TenVad => { + let path = crate::vad::ten_model_bundle_path(); + self.ten_vad_worker = crate::vad::TenOnnxVadWorker::try_new(&path); + self.silero_vad_worker = None; + if self.ten_vad_worker.is_none() { + tracing::warn!( + target: "chanora_audio", + "TEN VAD ONNX model not found at {path}; falling back to WebRTC VAD" + ); + } + } + _ => { + self.silero_vad_worker = None; + self.ten_vad_worker = None; + } + } + self.vad_state.reset(); + } + + self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1); + let capture_seq = self.capture_frame_seq; let mut used_fallback_vad = false; let vad = if vad_backend == crate::VadBackend::Disabled { crate::vad::VadOutput { probability: 1.0, speech: true, } - } else { - used_fallback_vad = matches!( - vad_backend, - crate::VadBackend::SileroOnnx | crate::VadBackend::TenVad - ); - if used_fallback_vad { - self.disable_failed_vad_backend(vad_backend); + } else if vad_backend == crate::VadBackend::SileroOnnx { + if let Some(worker) = self.silero_vad_worker.as_ref() { + let enqueued = worker.try_send(capture_seq, &frame); + if enqueued && !worker.is_stale(capture_seq) { + let p = worker.latest_probability(); + crate::vad::VadOutput { + probability: p, + speech: p >= 0.5, + } + } else { + used_fallback_vad = true; + self.mark_vad_fallback_active(vad_backend); + crate::vad::VoiceActivityDetector::process_10ms( + &mut self.vad_detector, + &frame, + ) + } + } else { + used_fallback_vad = true; + self.mark_vad_fallback_active(vad_backend); + crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) } + } else if vad_backend == crate::VadBackend::TenVad { + if let Some(worker) = self.ten_vad_worker.as_ref() { + let enqueued = worker.try_send(capture_seq, &frame); + if enqueued && !worker.is_stale(capture_seq) { + let p = worker.latest_probability(); + crate::vad::VadOutput { + probability: p, + speech: p >= 0.5, + } + } else { + used_fallback_vad = true; + self.mark_vad_fallback_active(vad_backend); + crate::vad::VoiceActivityDetector::process_10ms( + &mut self.vad_detector, + &frame, + ) + } + } else { + used_fallback_vad = true; + self.mark_vad_fallback_active(vad_backend); + crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) + } + } else { crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) }; self.audio_processing_stats .set_vad_fallback_active(used_fallback_vad); let active = self.vad_state.update(vad.speech); + let output_muted = self.output_muted.load(Ordering::Relaxed); if let Some(sel) = &self.voice_activity_selector { - sel.set_voice_activity_open(active); + sel.set_voice_activity_open(active && !output_muted); } self.audio_processing_stats.update_capture( input_dbfs, crate::frame::dbfs(&frame), vad.probability, - active, + active && !output_muted, self.transmit_active.load(Ordering::Relaxed), ); - if !self.transmit_active.load(Ordering::Relaxed) { + if !self.transmit_active.load(Ordering::Relaxed) || output_muted { return; } @@ -336,7 +440,7 @@ mod inner { let cfg = audio_processing_config.lock().unwrap(); if cfg.ios_mode == crate::IosVoiceProcessingMode::PlatformVoiceProcessing { return Err(AudioError::InvalidAudioProcessingConfig( - "IosRawUnit requires SonoraExperimental mode".to_string(), + "IosRawUnit requires raw WebRTC APM mode".to_string(), )); } } @@ -369,6 +473,7 @@ mod inner { let mut capture_state = RawCaptureState::new( voice_out_tx, transmit_active, + output_muted.clone(), frames_sent, mic_gain, voice_activity_selector, diff --git a/crates/chanora_audio/src/ios_voice_unit.rs b/crates/chanora_audio/src/ios_voice_unit.rs index b965036..105f86c 100644 --- a/crates/chanora_audio/src/ios_voice_unit.rs +++ b/crates/chanora_audio/src/ios_voice_unit.rs @@ -139,6 +139,7 @@ struct IosCaptureState { opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME], voice_out_tx: mpsc::Sender, transmit_active: Arc, + output_muted: Arc, frames_sent: Arc, mic_gain: f32, voice_activity_selector: Option>, @@ -146,10 +147,12 @@ struct IosCaptureState { /// Background Silero worker — enqueues frames off the realtime /// callback and publishes the latest probability atomically. silero_vad_worker: Option, + ten_vad: Option, /// Last VAD backend we configured — used to detect backend changes. current_vad_backend: crate::VadBackend, /// Last observed configured Silero model epoch. silero_model_epoch: u64, + fallback_warned_backend: Option, vad_state: crate::voice_activity::VoiceActivityStateMachine, audio_processing_config: Arc>, sonora_processor: crate::processor::SonoraProcessor, @@ -173,6 +176,7 @@ impl IosCaptureState { fn new( voice_out_tx: mpsc::Sender, transmit_active: Arc, + output_muted: Arc, frames_sent: Arc, mic_gain: f32, voice_activity_selector: Option>, @@ -188,13 +192,16 @@ impl IosCaptureState { opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME], voice_out_tx, transmit_active, + output_muted, frames_sent, mic_gain, voice_activity_selector, vad_detector: crate::vad::WebRtcFallbackVad::default(), silero_vad_worker: None, + ten_vad: None, current_vad_backend: crate::VadBackend::WebrtcVad, silero_model_epoch: crate::vad::silero_model_epoch(), + fallback_warned_backend: None, vad_state: crate::voice_activity::VoiceActivityStateMachine::default(), audio_processing_config, sonora_processor: crate::processor::SonoraProcessor::new(), @@ -210,12 +217,16 @@ impl IosCaptureState { }) } - fn disable_failed_vad_backend(&mut self, failed_backend: crate::VadBackend) { - if let Ok(mut cfg) = self.audio_processing_config.try_lock() { - if cfg.disable_failed_vad_backend(failed_backend) { - self.current_vad_backend = crate::VadBackend::WebrtcVad; - } + fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) { + if self.fallback_warned_backend == Some(failed_backend) { + return; } + self.fallback_warned_backend = Some(failed_backend); + tracing::warn!( + target: "chanora_audio", + backend = failed_backend.as_str(), + "VAD backend unavailable; using WebRTC fallback for runtime detection" + ); } /// Consume the i16 mono buffer delivered by VPIO, accumulate @@ -367,6 +378,7 @@ impl IosCaptureState { if vad_backend != self.current_vad_backend || silero_model_changed { self.current_vad_backend = vad_backend; + self.fallback_warned_backend = None; self.silero_model_epoch = silero_model_epoch; match vad_backend { crate::VadBackend::SileroOnnx => { @@ -383,22 +395,28 @@ impl IosCaptureState { target: "chanora_audio", "Silero VAD model not found at {model_path}; falling back to WebRTC VAD" ); - self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx); + self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); } self.audio_processing_stats .set_vad_fallback_active(self.silero_vad_worker.is_none()); } crate::VadBackend::TenVad => { self.silero_vad_worker = None; - warn!( - target: "chanora_audio", - "TEN VAD selected but native TEN runtime is not bundled; falling back to WebRTC VAD" - ); - self.disable_failed_vad_backend(crate::VadBackend::TenVad); - self.audio_processing_stats.set_vad_fallback_active(true); + let model_path = crate::vad::ten_model_bundle_path(); + self.ten_vad = crate::vad::TenOnnxVadWorker::try_new(&model_path); + if self.ten_vad.is_none() { + warn!( + target: "chanora_audio", + "TEN VAD ONNX model not available at {model_path}; falling back to WebRTC VAD" + ); + self.mark_vad_fallback_active(crate::VadBackend::TenVad); + } + self.audio_processing_stats + .set_vad_fallback_active(self.ten_vad.is_none()); } _ => { self.silero_vad_worker = None; + self.ten_vad = None; self.audio_processing_stats.set_vad_fallback_active(false); } } @@ -419,14 +437,14 @@ impl IosCaptureState { ); let transmit_active = self.transmit_active.load(Ordering::Relaxed); - // Apply the enabled stages through the SonoraProcessor. - // We reconfigure it on-the-fly to match the current settings. + // VPIO owns AEC. Keep the legacy non-AEC conditioning path here until + // the raw WebRTC APM path is explicitly selected. if run_ns || run_agc || run_hpf { use crate::processor::sonora::SonoraConfig; use crate::processor::AudioProcessor; let new_cfg = SonoraConfig { hpf: run_hpf, - aec3: false, // NEVER in VPIO path (INV_009) + aec3: false, ns: run_ns, agc2: run_agc, }; @@ -448,7 +466,8 @@ impl IosCaptureState { } } else if vad_backend == crate::VadBackend::SileroOnnx { if let Some(worker) = self.silero_vad_worker.as_ref() { - if worker.try_send(capture_seq, &frame) && !worker.is_stale(capture_seq) { + let enqueued = worker.try_send(capture_seq, &frame); + if enqueued && !worker.is_stale(capture_seq) { let probability = worker.latest_probability(); crate::vad::VadOutput { probability, @@ -456,33 +475,48 @@ impl IosCaptureState { } } else { used_fallback_vad = true; - self.silero_vad_worker = None; - self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx); + self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) } } else { used_fallback_vad = true; - self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx); + self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) } } else if vad_backend == crate::VadBackend::TenVad { - used_fallback_vad = true; - self.disable_failed_vad_backend(crate::VadBackend::TenVad); - crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) + if let Some(worker) = self.ten_vad.as_ref() { + let enqueued = worker.try_send(capture_seq, &frame); + if enqueued && !worker.is_stale(capture_seq) { + let probability = worker.latest_probability(); + crate::vad::VadOutput { + probability, + speech: probability >= 0.5, + } + } else { + used_fallback_vad = true; + self.mark_vad_fallback_active(crate::VadBackend::TenVad); + crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) + } + } else { + used_fallback_vad = true; + self.mark_vad_fallback_active(crate::VadBackend::TenVad); + crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) + } } else { crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) }; self.audio_processing_stats .set_vad_fallback_active(used_fallback_vad); let gate_open = self.vad_state.update(vad.speech); + let output_muted = self.output_muted.load(Ordering::Relaxed); if let Some(selector) = &self.voice_activity_selector { - selector.set_voice_activity_open(gate_open); + selector.set_voice_activity_open(gate_open && !output_muted); } self.audio_processing_stats.update_capture( input_dbfs, crate::frame::dbfs(&frame), vad.probability, - gate_open, + gate_open && !output_muted, transmit_active, ); @@ -700,6 +734,7 @@ impl IosVoiceUnit { let mut capture_state = IosCaptureState::new( voice_out_tx, transmit_active, + output_muted.clone(), frames_sent, mic_gain, voice_activity_selector, diff --git a/crates/chanora_audio/src/processor/mod.rs b/crates/chanora_audio/src/processor/mod.rs index 840ed62..3f45977 100644 --- a/crates/chanora_audio/src/processor/mod.rs +++ b/crates/chanora_audio/src/processor/mod.rs @@ -4,10 +4,12 @@ pub mod dsp; pub mod noop; pub mod platform; pub mod sonora; +pub mod webrtc_apm; pub use noop::NoopProcessor; pub use platform::PlatformVoiceProcessor; pub use sonora::SonoraProcessor; +pub use webrtc_apm::WebRtcApmProcessor; /// 10 ms mono f32 processing frame at 48 kHz (480 samples). pub const FRAME_SAMPLES: usize = 480; diff --git a/crates/chanora_audio/src/processor/webrtc_apm.rs b/crates/chanora_audio/src/processor/webrtc_apm.rs new file mode 100644 index 0000000..f1a40bb --- /dev/null +++ b/crates/chanora_audio/src/processor/webrtc_apm.rs @@ -0,0 +1,193 @@ +//! WebRTC Audio Processing Module backend. +//! +//! This backend delegates capture-side voice processing to WebRTC APM +//! instead of Chanora's experimental Rust DSP chain. Frames are 10 ms, +//! mono, 48 kHz f32, matching the rest of the voice pipeline. + +use sonora::config::{ + AdaptiveDigital, EchoCanceller, GainController2, HighPassFilter, NoiseSuppression, + NoiseSuppressionLevel, Pipeline, +}; +use sonora::{AudioProcessing, Config, StreamConfig}; + +use super::{AudioProcessor, FRAME_SAMPLES}; + +/// Per-module WebRTC APM enable flags. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WebRtcApmConfig { + /// Acoustic echo cancellation. + pub aec: bool, + /// Automatic gain control. + pub agc: bool, + /// High-pass filter. + pub hpf: bool, + /// Noise suppression. + pub ns: bool, + /// WebRTC VAD is available for transmit gating. + pub vad: bool, +} + +impl WebRtcApmConfig { + /// Resolve WebRTC APM flags from the shared audio-processing config. + pub fn from_audio_config(config: &crate::AudioProcessingConfig) -> Self { + Self { + aec: config.aec == crate::EffectOwner::WebrtcApm, + agc: matches!( + config.agc, + crate::EffectOwner::WebrtcApm | crate::EffectOwner::Conservative + ), + hpf: config.hpf_enabled, + ns: matches!( + config.ns, + crate::EffectOwner::WebrtcApm | crate::EffectOwner::Conservative + ), + vad: config.vad_backend == crate::VadBackend::WebrtcVad, + } + } + + fn to_webrtc_config(self) -> Config { + Config { + pipeline: Pipeline { + maximum_internal_processing_rate: sonora::config::MaxProcessingRate::Rate48kHz, + ..Default::default() + }, + high_pass_filter: self.hpf.then_some(HighPassFilter::default()), + echo_canceller: self.aec.then_some(EchoCanceller::default()), + noise_suppression: self.ns.then_some(NoiseSuppression { + level: NoiseSuppressionLevel::Moderate, + analyze_linear_aec_output_when_available: false, + }), + gain_controller2: self.agc.then_some(GainController2 { + input_volume_controller: false, + adaptive_digital: Some(AdaptiveDigital::default()), + fixed_digital: Default::default(), + }), + ..Default::default() + } + } + + /// Stable runtime-log summary. + pub fn summary(self) -> String { + format!( + "aec={} agc={} hpf={} ns={} vad={}", + self.aec, self.agc, self.hpf, self.ns, self.vad + ) + } +} + +impl Default for WebRtcApmConfig { + fn default() -> Self { + Self { + aec: true, + agc: true, + hpf: true, + ns: true, + vad: true, + } + } +} + +/// WebRTC APM processor with preallocated render scratch. +pub struct WebRtcApmProcessor { + processor: AudioProcessing, + config: WebRtcApmConfig, + capture_scratch: [f32; FRAME_SAMPLES], + render_scratch: [f32; FRAME_SAMPLES], +} + +impl WebRtcApmProcessor { + /// Construct with a specific module configuration. + pub fn with_config(config: WebRtcApmConfig) -> Result { + let stream_config = StreamConfig::new(crate::frame::SAMPLE_RATE_HZ, 1); + let processor = AudioProcessing::builder() + .config(config.to_webrtc_config()) + .capture_config(stream_config) + .render_config(stream_config) + .echo_detector(config.aec) + .build(); + tracing::info!( + target: "chanora_audio", + modules = %config.summary(), + "WebRTC APM processor active" + ); + Ok(Self { + processor, + config, + capture_scratch: [0.0; FRAME_SAMPLES], + render_scratch: [0.0; FRAME_SAMPLES], + }) + } + + /// Current module configuration. + pub fn config(&self) -> WebRtcApmConfig { + self.config + } + + /// Apply updated module flags outside the realtime callback. + pub fn apply_config(&mut self, config: WebRtcApmConfig) { + if config == self.config { + return; + } + self.processor.apply_config(config.to_webrtc_config()); + self.config = config; + tracing::info!( + target: "chanora_audio", + modules = %config.summary(), + "WebRTC APM processor reconfigured" + ); + } +} + +impl AudioProcessor for WebRtcApmProcessor { + fn process_capture(&mut self, frame: &mut [f32; FRAME_SAMPLES]) { + let channels = [&frame[..]]; + let mut out = [&mut self.capture_scratch[..]]; + if let Err(error) = self.processor.process_capture_f32(&channels, &mut out) { + tracing::trace!(target: "chanora_audio", %error, "WebRTC APM capture frame skipped"); + } else { + frame.copy_from_slice(&self.capture_scratch); + } + } + + fn process_render(&mut self, frame: &[f32; FRAME_SAMPLES]) { + let channels = [&frame[..]]; + let mut out = [&mut self.render_scratch[..]]; + if let Err(error) = self.processor.process_render_f32(&channels, &mut out) { + tracing::trace!(target: "chanora_audio", %error, "WebRTC APM render frame skipped"); + } + } + + fn has_aec(&self) -> bool { + self.config.aec + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_config_enables_all_modules() { + let cfg = WebRtcApmConfig::default(); + assert!(cfg.aec && cfg.agc && cfg.hpf && cfg.ns && cfg.vad); + } + + #[test] + fn process_silence_is_stable() { + let mut processor = WebRtcApmProcessor::with_config(WebRtcApmConfig { + aec: false, + agc: true, + hpf: true, + ns: true, + vad: true, + }) + .expect("webrtc apm init"); + let render = [0.0_f32; FRAME_SAMPLES]; + let mut capture = [0.0_f32; FRAME_SAMPLES]; + + processor.process_render(&render); + processor.process_capture(&mut capture); + + assert!(capture.iter().all(|s| s.is_finite())); + } +} diff --git a/crates/chanora_audio/src/vad/mod.rs b/crates/chanora_audio/src/vad/mod.rs index 0b119e6..f68df9c 100644 --- a/crates/chanora_audio/src/vad/mod.rs +++ b/crates/chanora_audio/src/vad/mod.rs @@ -7,15 +7,17 @@ pub mod resampler; pub mod silero_onnx; +pub mod ten_onnx; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{OnceLock, RwLock}; -use crate::frame::{dbfs, i16_to_f32}; +use crate::frame::{f32_to_i16, i16_to_f32}; use crate::AudioError; use resampler::{Downsampler48to16, INPUT_FRAME_10MS}; pub use silero_onnx::SileroOnnxVad; +pub use ten_onnx::{TenOnnxVad, TenOnnxVadWorker}; /// Voice activity detector output for one 10 ms frame. #[derive(Debug, Clone, Copy)] @@ -32,83 +34,38 @@ pub trait VoiceActivityDetector: Send { fn process_10ms(&mut self, samples: &[f32]) -> VadOutput; } -/// Realtime-safe fallback VAD used when a model runtime is unavailable. -/// -/// This is not an energy-only transmit gate. It combines RMS level, -/// zero-crossing rate, and peak-to-RMS shape with hysteresis so stable -/// background rumble is less likely to open VoiceActivity than speech. -#[derive(Debug, Clone)] +/// Realtime-safe WebRTC VAD used when a model runtime is unavailable. pub struct WebRtcFallbackVad { - open_dbfs: f32, - close_dbfs: f32, - active: bool, + vad: webrtc_vad::Vad, + frame_i16: [i16; INPUT_FRAME_10MS], } +// `webrtc_vad::Vad` owns an FFI pointer and is only touched from the +// capture thread after construction. Moving the wrapper between threads is +// safe; sharing it concurrently is not required and not implemented. +unsafe impl Send for WebRtcFallbackVad {} + impl Default for WebRtcFallbackVad { fn default() -> Self { Self { - open_dbfs: -42.0, - close_dbfs: -50.0, - active: false, + vad: webrtc_vad::Vad::new_with_rate_and_mode( + webrtc_vad::SampleRate::Rate48kHz, + webrtc_vad::VadMode::Aggressive, + ), + frame_i16: [0; INPUT_FRAME_10MS], } } } -impl WebRtcFallbackVad { - fn zero_crossing_rate(samples: &[f32]) -> f32 { - if samples.len() < 2 { - return 0.0; - } - let crossings = samples - .windows(2) - .filter(|pair| (pair[0] >= 0.0 && pair[1] < 0.0) || (pair[0] < 0.0 && pair[1] >= 0.0)) - .count(); - crossings as f32 / (samples.len() - 1) as f32 - } - - fn peak_to_rms(samples: &[f32], rms: f32) -> f32 { - if rms <= 0.000_001 { - return 0.0; - } - let peak = samples.iter().map(|s| s.abs()).fold(0.0_f32, f32::max); - peak / rms - } -} - impl VoiceActivityDetector for WebRtcFallbackVad { fn process_10ms(&mut self, samples: &[f32]) -> VadOutput { - let level = dbfs(samples); - let threshold = if self.active { - self.close_dbfs - } else { - self.open_dbfs - }; - let rms = samples.iter().map(|s| s * s).sum::() / samples.len().max(1) as f32; - let rms = rms.sqrt(); - let zcr = Self::zero_crossing_rate(samples); - let crest = Self::peak_to_rms(samples, rms); - - // Level score: steeper curve so silence (-50 dBFS) scores near 0. - // Speech is typically -30 to -10 dBFS; silence is -60 to -45 dBFS. - // Map [-60, -20] → [0, 1] with a midpoint at -40 dBFS. - let level_score = ((level + 60.0) / 40.0).clamp(0.0, 1.0); - - let zcr_score = if (0.015..=0.32).contains(&zcr) { - 1.0 - } else { - 0.3 // penalise non-speech ZCR more aggressively - }; - let crest_score = if (1.5..=12.0).contains(&crest) { - 1.0 - } else { - 0.3 - }; - let probability = - (level_score * 0.72 + zcr_score * 0.18 + crest_score * 0.10).clamp(0.0, 1.0); - self.active = level >= threshold && probability >= 0.5; + for (dst, src) in self.frame_i16.iter_mut().zip(samples.iter().copied()) { + *dst = f32_to_i16(src); + } + let speech = self.vad.is_voice_segment(&self.frame_i16).unwrap_or(false); VadOutput { - probability, - speech: self.active, + probability: if speech { 1.0 } else { 0.0 }, + speech, } } } @@ -151,11 +108,17 @@ pub fn process_i16_10ms(detector: &mut dyn VoiceActivityDetector, samples: &[i16 static SILERO_MODEL_PATH_OVERRIDE: OnceLock>> = OnceLock::new(); static SILERO_MODEL_EPOCH: AtomicU64 = AtomicU64::new(0); +static TEN_MODEL_PATH_OVERRIDE: OnceLock>> = OnceLock::new(); +static TEN_MODEL_EPOCH: AtomicU64 = AtomicU64::new(0); fn silero_model_path_override() -> &'static RwLock> { SILERO_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None)) } +fn ten_model_path_override() -> &'static RwLock> { + TEN_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None)) +} + /// Configure the preferred Silero ONNX model path. /// /// The path is validated eagerly. A successful call increments the @@ -186,12 +149,39 @@ pub fn silero_model_epoch() -> u64 { SILERO_MODEL_EPOCH.load(Ordering::Relaxed) } -/// Return the expected path of the Silero VAD v6 ONNX model in the -/// iOS app bundle. The model is shipped as a Flutter asset and copied -/// to the app's Documents directory by the Dart-side asset loader. +/// Configure the preferred TEN VAD ONNX model path. +pub fn set_ten_model_path(path: &str) -> Result<(), AudioError> { + let path = path.trim(); + if path.is_empty() { + return Err(AudioError::InvalidAudioProcessingConfig( + "ten vad model path must not be empty".to_string(), + )); + } + if !std::path::Path::new(path).is_file() { + return Err(AudioError::InvalidAudioProcessingConfig(format!( + "ten vad model path does not exist or is not a file: {path}" + ))); + } + let mut guard = ten_model_path_override() + .write() + .map_err(|_| AudioError::Backend("ten vad model path lock poisoned".to_string()))?; + *guard = Some(path.to_string()); + TEN_MODEL_EPOCH.fetch_add(1, Ordering::Relaxed); + Ok(()) +} + +/// Monotonic counter incremented whenever the configured TEN model path changes. +pub fn ten_model_epoch() -> u64 { + TEN_MODEL_EPOCH.load(Ordering::Relaxed) +} + +/// Return the expected path of the Silero VAD v6 ONNX model. +/// The model is shipped as a Flutter asset and copied to the app's +/// data directory by the Dart-side asset loader. /// -/// Returns an empty string on non-Apple platforms (Silero is not -/// supported there; `SileroOnnxVad::try_new` will return `None`). +/// On iOS/Android the model lives in the app's Documents/files +/// directory. On desktop, the caller should set the path explicitly +/// via `set_silero_model_path`. pub fn silero_model_bundle_path() -> String { if let Ok(guard) = silero_model_path_override().read() { if let Some(path) = guard.as_ref() { @@ -199,26 +189,85 @@ pub fn silero_model_bundle_path() -> String { } } + // iOS: Documents directory (written by Flutter asset loader). + // macOS: same Documents pattern. #[cfg(any(target_os = "ios", target_os = "macos"))] { - // Primary: Documents directory (written by Flutter asset loader). if let Ok(home) = std::env::var("HOME") { let docs = format!("{home}/Documents/silero_vad.onnx"); if std::path::Path::new(&docs).exists() { return docs; } - // Fallback: app bundle Resources directory. let bundle = format!("{home}/../Library/silero_vad.onnx"); if std::path::Path::new(&bundle).exists() { return bundle; } } - // Last resort: current working directory (useful in tests). "silero_vad.onnx".to_string() } - #[cfg(not(any(target_os = "ios", target_os = "macos")))] + + // Android: the model is in the app's files directory, same + // Documents path pattern used by Flutter's path_provider. + #[cfg(target_os = "android")] { - String::new() + // On Android, Flutter's getApplicationDocumentsDirectory + // resolves to /data/data//app_flutter. + // The Silero model path is set explicitly via + // set_silero_model_path from Dart before voice starts, + // so this fallback is rarely needed. + "silero_vad.onnx".to_string() + } + + // Desktop (Windows, Linux): rely on the override set by Dart. + #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] + { + if let Ok(cwd) = std::env::current_dir() { + let local = cwd.join("silero_vad.onnx"); + if local.exists() { + return local.to_string_lossy().to_string(); + } + } + "silero_vad.onnx".to_string() + } +} + +/// Return the expected path of the TEN VAD ONNX model copied by Flutter. +pub fn ten_model_bundle_path() -> String { + if let Ok(guard) = ten_model_path_override().read() { + if let Some(path) = guard.as_ref() { + return path.clone(); + } + } + + #[cfg(any(target_os = "ios", target_os = "macos"))] + { + if let Ok(home) = std::env::var("HOME") { + let docs = format!("{home}/Documents/ten_vad.onnx"); + if std::path::Path::new(&docs).exists() { + return docs; + } + let bundle = format!("{home}/../Library/ten_vad.onnx"); + if std::path::Path::new(&bundle).exists() { + return bundle; + } + } + "ten_vad.onnx".to_string() + } + + #[cfg(target_os = "android")] + { + "ten_vad.onnx".to_string() + } + + #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] + { + if let Ok(cwd) = std::env::current_dir() { + let local = cwd.join("ten_vad.onnx"); + if local.exists() { + return local.to_string_lossy().to_string(); + } + } + "ten_vad.onnx".to_string() } } diff --git a/crates/chanora_audio/src/vad/silero_onnx.rs b/crates/chanora_audio/src/vad/silero_onnx.rs index e36c727..b8a6a41 100644 --- a/crates/chanora_audio/src/vad/silero_onnx.rs +++ b/crates/chanora_audio/src/vad/silero_onnx.rs @@ -75,13 +75,9 @@ pub struct SileroOnnxVad { } enum SileroInner { - #[cfg(any(target_os = "ios", target_os = "macos"))] Onnx(OnnxSession), - #[allow(dead_code)] - Stub, } -#[cfg(any(target_os = "ios", target_os = "macos"))] struct OnnxSession { session: ort::session::Session, } @@ -90,20 +86,11 @@ impl SileroOnnxVad { /// Attempt to load the Silero v6 ONNX model from `model_path`. /// /// Returns `None` when the model file is missing, the ONNX Runtime - /// is unavailable, or the platform is not iOS/macOS. + /// is unavailable, or the platform does not support ONNX. pub fn try_new(model_path: &str) -> Option { - #[cfg(any(target_os = "ios", target_os = "macos"))] - { - Self::try_new_onnx(model_path) - } - #[cfg(not(any(target_os = "ios", target_os = "macos")))] - { - let _ = model_path; - None - } + Self::try_new_onnx(model_path) } - #[cfg(any(target_os = "ios", target_os = "macos"))] fn try_new_onnx(model_path: &str) -> Option { use tracing::{error, info}; @@ -116,13 +103,8 @@ impl SileroOnnxVad { return None; } - #[cfg(target_os = "macos")] - if let Some(path) = bundled_onnxruntime_path() { - let _ = ort::init_from(path.to_string_lossy()).commit(); - } - let session_result = std::panic::catch_unwind(|| { - ort::session::Session::builder().and_then(|b| b.commit_from_file(model_path)) + ort::session::Session::builder().and_then(|mut b| b.commit_from_file(model_path)) }); match session_result { @@ -193,7 +175,6 @@ impl SileroOnnxVad { /// 16 kHz frame: concatenate prior context, pass `input/state/sr` to /// ONNX, persist `stateN`, then refresh context from the current frame. /// Updates `last_probability` and returns the new value. - #[cfg(any(target_os = "ios", target_os = "macos"))] fn calc_level(&mut self, audio_frame: &[f32]) -> f32 { use ort::value::Value; use tracing::error; @@ -279,7 +260,7 @@ impl SileroOnnxVad { } #[cfg(target_os = "macos")] -fn bundled_onnxruntime_path() -> Option { +pub(crate) fn bundled_onnxruntime_path_for_vad() -> Option { let exe = std::env::current_exe().ok()?; let app_dir = exe.parent()?; let framework = app_dir @@ -307,16 +288,9 @@ impl VoiceActivityDetector for SileroOnnxVad { if self.accum.len() >= SILERO_FRAME_16K { let audio_frame: Vec = self.accum[..SILERO_FRAME_16K].to_vec(); - #[cfg(any(target_os = "ios", target_os = "macos"))] - { - if matches!(self.inner, SileroInner::Onnx(_)) { - self.calc_level(&audio_frame); - } else { - self.update_context_from_frame(&audio_frame); - } - } - #[cfg(not(any(target_os = "ios", target_os = "macos")))] - { + if matches!(self.inner, SileroInner::Onnx(_)) { + self.calc_level(&audio_frame); + } else { self.update_context_from_frame(&audio_frame); } // Drain the accumulator (keep any overflow for next frame). @@ -357,9 +331,9 @@ impl SileroOnnxVadWorker { pub fn try_new(model_path: &str) -> Option { let vad = SileroOnnxVad::try_new(model_path)?; let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits())); - let latest_processed_seq = Arc::new(AtomicU64::new(0)); + let latest_processed_seq = Arc::new(AtomicU64::new(u64::MAX)); let alive = Arc::new(AtomicBool::new(true)); - let (tx, rx) = std::sync::mpsc::sync_channel::(8); + let (tx, rx) = std::sync::mpsc::sync_channel::(32); let latest_probability_for_thread = latest_probability.clone(); let latest_processed_seq_for_thread = latest_processed_seq.clone(); let alive_for_thread = alive.clone(); @@ -414,7 +388,10 @@ impl SileroOnnxVadWorker { /// True when the worker is too far behind to trust its latest /// probability for the current frame. pub fn is_stale(&self, capture_seq: u64) -> bool { - self.lag_frames(capture_seq) > SILERO_MAX_STALE_FRAMES + let latest = self + .latest_processed_seq + .load(std::sync::atomic::Ordering::Relaxed); + latest == u64::MAX || capture_seq.saturating_sub(latest) > SILERO_MAX_STALE_FRAMES } } diff --git a/crates/chanora_audio/src/vad/ten_onnx.rs b/crates/chanora_audio/src/vad/ten_onnx.rs new file mode 100644 index 0000000..59c39ca --- /dev/null +++ b/crates/chanora_audio/src/vad/ten_onnx.rs @@ -0,0 +1,432 @@ +//! TEN VAD ONNX backend. +//! +//! TEN's ONNX graph does not accept raw PCM. It expects the same feature +//! stack produced by TEN's `AUP_Aed_aivad_proc`: three context frames of +//! 40 log-mel powers plus one pitch feature, followed by four recurrent +//! state tensors. This module ports that preprocessing path to Rust and +//! keeps ONNX Runtime off the realtime callback where possible. + +use crate::frame::f32_to_i16; + +use super::resampler::{Downsampler48to16, INPUT_FRAME_10MS}; +use super::{VadOutput, VoiceActivityDetector}; + +const SAMPLE_RATE_16K: f32 = 16_000.0; +const HOP_16K: usize = 256; +const WINDOW_16K: usize = 768; +const FFT_SIZE: usize = 1024; +const N_BINS: usize = FFT_SIZE / 2 + 1; +const MEL_BANDS: usize = 40; +const FEATURE_LEN: usize = 41; +const CONTEXT: usize = 3; +const HIDDEN: usize = 64; +const POWER_NORMALIZER: f32 = 32768.0 * 32768.0; +const EPS: f32 = 1.0e-20; + +const FEATURE_MEANS: [f32; FEATURE_LEN] = [ + -8.198236, -6.2657166, -5.4838185, -4.7586913, -4.417089, -4.142893, -3.9128504, -3.845928, + -3.6570904, -3.7234187, -3.8761342, -3.843891, -3.6904051, -3.7560658, -3.6986961, -3.650463, + -3.7004688, -3.5673213, -3.4989002, -3.477807, -3.458816, -3.4449239, -3.4013286, -3.3062613, + -3.2785568, -3.2332509, -3.198616, -3.2045264, -3.2087986, -3.257838, -3.3813767, -3.5340214, + -3.640868, -3.7268589, -3.773731, -3.8046672, -3.832901, -3.8711205, -3.990593, -4.4802895, + 92.3569, +]; + +const FEATURE_STDS: [f32; FEATURE_LEN] = [ + 5.166064, 4.9772096, 4.698896, 4.6306214, 4.634348, 4.641156, 4.6406765, 4.666367, 4.6505346, + 4.640021, 4.6374, 4.620099, 4.5963163, 4.562655, 4.5543604, 4.5669107, 4.56249, 4.5624127, + 4.5852995, 4.6001797, 4.592846, 4.5859227, 4.5834966, 4.626093, 4.626958, 4.6262894, 4.637006, + 4.683016, 4.726814, 4.7342896, 4.753227, 4.849723, 4.869435, 4.884483, 4.921327, 4.9592123, + 4.996619, 5.0448236, 5.072217, 5.0964394, 115.21369, +]; + +/// TEN VAD using ONNX Runtime and Rust-ported TEN feature preprocessing. +pub struct TenOnnxVad { + session: ort::session::Session, + downsampler: Downsampler48to16, + hop_accum: Vec, + sample_fifo: Vec, + feature_stack: [[f32; FEATURE_LEN]; CONTEXT], + states: [[f32; HIDDEN]; 4], + mel_filters: Vec<[f32; N_BINS]>, + last_probability: f32, + last_speech: bool, +} + +unsafe impl Send for TenOnnxVad {} + +impl TenOnnxVad { + /// Load TEN VAD ONNX model. + pub fn try_new(model_path: &str) -> Option { + if !std::path::Path::new(model_path).exists() { + tracing::warn!(target: "chanora_audio", path = model_path, "TEN VAD ONNX model not found"); + return None; + } + let session = match std::panic::catch_unwind(|| { + ort::session::Session::builder().and_then(|mut b| b.commit_from_file(model_path)) + }) { + Ok(Ok(session)) => session, + Ok(Err(error)) => { + tracing::warn!(target: "chanora_audio", %error, path = model_path, "TEN VAD ONNX model load failed"); + return None; + } + Err(_) => { + tracing::warn!(target: "chanora_audio", path = model_path, "TEN VAD ONNX Runtime panicked during load"); + return None; + } + }; + tracing::info!(target: "chanora_audio", path = model_path, "TEN VAD ONNX model loaded"); + Some(Self { + session, + downsampler: Downsampler48to16::default(), + hop_accum: Vec::with_capacity(HOP_16K + super::resampler::OUTPUT_FRAME_10MS), + sample_fifo: Vec::with_capacity(WINDOW_16K + HOP_16K), + feature_stack: [[0.0; FEATURE_LEN]; CONTEXT], + states: [[0.0; HIDDEN]; 4], + mel_filters: build_mel_filters(), + last_probability: 0.0, + last_speech: false, + }) + } + + fn process_hop(&mut self, hop: &[f32]) { + self.sample_fifo.extend_from_slice(hop); + let frame = if self.sample_fifo.len() >= WINDOW_16K { + let start = self.sample_fifo.len() - WINDOW_16K; + self.sample_fifo[start..].to_vec() + } else { + let mut padded = vec![0.0; WINDOW_16K - self.sample_fifo.len()]; + padded.extend_from_slice(&self.sample_fifo); + padded + }; + if self.sample_fifo.len() > WINDOW_16K { + let excess = self.sample_fifo.len() - WINDOW_16K; + self.sample_fifo.drain(..excess); + } + + let feature = compute_feature(&self.mel_filters, &frame); + self.feature_stack.copy_within(1..CONTEXT, 0); + self.feature_stack[CONTEXT - 1] = feature; + self.run_onnx(); + } + + fn run_onnx(&mut self) { + use ndarray::{Array, IxDyn}; + use ort::value::Value; + + let input: Vec = self.feature_stack.iter().flatten().copied().collect(); + let input_arr = match Array::from_shape_vec(IxDyn(&[1, CONTEXT, FEATURE_LEN]), input) { + Ok(v) => v, + Err(_) => return, + }; + let state_arrs = [0, 1, 2, 3] + .map(|idx| Array::from_shape_vec(IxDyn(&[1, HIDDEN]), self.states[idx].to_vec())); + let input_val = match Value::from_array(input_arr) { + Ok(v) => v, + Err(error) => { + tracing::warn!(target: "chanora_audio", %error, "TEN VAD input tensor error"); + return; + } + }; + let state_vals = match state_arrs { + [Ok(a), Ok(b), Ok(c), Ok(d)] => [a, b, c, d], + _ => return, + }; + let state_vals = match state_vals.map(Value::from_array) { + [Ok(a), Ok(b), Ok(c), Ok(d)] => [a, b, c, d], + _ => return, + }; + + let outputs = match self.session.run([ + (&input_val).into(), + (&state_vals[0]).into(), + (&state_vals[1]).into(), + (&state_vals[2]).into(), + (&state_vals[3]).into(), + ]) { + Ok(outputs) => outputs, + Err(error) => { + tracing::warn!(target: "chanora_audio", %error, "TEN VAD ONNX inference failed"); + return; + } + }; + + if let Ok((_, prob)) = outputs["output_1"].try_extract_tensor::() { + if let Some(&p) = prob.first() { + self.last_probability = p.clamp(0.0, 1.0); + self.last_speech = self.last_probability >= 0.5; + } + } + for (idx, name) in ["output_2", "output_3", "output_6", "output_7"] + .iter() + .enumerate() + { + if let Ok((_, state)) = outputs[*name].try_extract_tensor::() { + let copy_len = state.len().min(HIDDEN); + self.states[idx][..copy_len].copy_from_slice(&state[..copy_len]); + } + } + } +} + +#[cfg(target_os = "macos")] +fn bundled_onnxruntime_path() -> Option { + let exe = std::env::current_exe().ok()?; + let app_dir = exe.parent()?; + let framework = app_dir + .join("Frameworks") + .join("onnxruntime.framework") + .join("onnxruntime"); + framework.exists().then_some(framework) +} + +fn compute_feature(mel_filters: &[[f32; N_BINS]], frame: &[f32]) -> [f32; FEATURE_LEN] { + let power = power_spectrum(frame); + let mut feature = [0.0; FEATURE_LEN]; + for band in 0..MEL_BANDS { + let energy = mel_filters[band] + .iter() + .zip(power.iter()) + .map(|(w, p)| w * p) + .sum::() + / POWER_NORMALIZER; + let log_energy = (energy + EPS).ln(); + feature[band] = (log_energy - FEATURE_MEANS[band]) / (FEATURE_STDS[band] + EPS); + } + let pitch_hz = estimate_pitch_hz(frame); + feature[MEL_BANDS] = (pitch_hz - FEATURE_MEANS[MEL_BANDS]) / (FEATURE_STDS[MEL_BANDS] + EPS); + feature +} + +impl VoiceActivityDetector for TenOnnxVad { + fn process_10ms(&mut self, samples: &[f32]) -> VadOutput { + debug_assert_eq!(samples.len(), INPUT_FRAME_10MS); + let mut input = [0.0_f32; INPUT_FRAME_10MS]; + input.copy_from_slice(samples); + let downsampled = self.downsampler.process_frame_10ms(&input); + self.hop_accum.extend_from_slice(&downsampled); + while self.hop_accum.len() >= HOP_16K { + let hop: Vec = self.hop_accum[..HOP_16K].to_vec(); + self.hop_accum.drain(..HOP_16K); + self.process_hop(&hop); + } + VadOutput { + probability: self.last_probability, + speech: self.last_speech, + } + } +} + +fn hz_to_mel(hz: f32) -> f32 { + 2595.0 * (1.0 + hz / 700.0).log10() +} + +fn mel_to_hz(mel: f32) -> f32 { + 700.0 * (10.0_f32.powf(mel / 2595.0) - 1.0) +} + +fn build_mel_filters() -> Vec<[f32; N_BINS]> { + let low_mel = hz_to_mel(0.0); + let high_mel = hz_to_mel(8000.0); + let mut bins = [0_usize; MEL_BANDS + 2]; + for idx in 0..bins.len() { + let mel = idx as f32 * (high_mel - low_mel) / (MEL_BANDS as f32 + 1.0) + low_mel; + let hz = mel_to_hz(mel); + let mut bin = ((FFT_SIZE as f32 + 1.0) * hz / SAMPLE_RATE_16K).floor() as usize; + bin = bin.min(N_BINS - 1); + if idx > 0 && bin == bins[idx - 1] { + bin = (bin + 1).min(N_BINS - 1); + } + bins[idx] = bin; + } + + let mut filters = vec![[0.0_f32; N_BINS]; MEL_BANDS]; + for band in 0..MEL_BANDS { + let left = bins[band]; + let center = bins[band + 1].max(left + 1); + let right = bins[band + 2].max(center + 1).min(N_BINS - 1); + for i in left..center.min(N_BINS) { + filters[band][i] = (i - left) as f32 / (center - left) as f32; + } + for i in center..=right { + filters[band][i] = (right - i) as f32 / (right - center).max(1) as f32; + } + } + filters +} + +fn power_spectrum(frame: &[f32]) -> [f32; N_BINS] { + let mut windowed = [0.0_f32; FFT_SIZE]; + for (idx, sample) in frame.iter().take(WINDOW_16K).enumerate() { + let hann = 0.5 - 0.5 * (2.0 * std::f32::consts::PI * idx as f32 / WINDOW_16K as f32).cos(); + windowed[idx] = f32_to_i16(*sample) as f32 * hann; + } + + let mut out = [0.0_f32; N_BINS]; + for (k, dst) in out.iter_mut().enumerate() { + let mut re = 0.0_f32; + let mut im = 0.0_f32; + for (n, &x) in windowed.iter().enumerate() { + let phase = -2.0 * std::f32::consts::PI * k as f32 * n as f32 / FFT_SIZE as f32; + re += x * phase.cos(); + im += x * phase.sin(); + } + *dst = re * re + im * im; + } + out +} + +fn estimate_pitch_hz(frame: &[f32]) -> f32 { + let min_lag = (SAMPLE_RATE_16K / 400.0) as usize; + let max_lag = (SAMPLE_RATE_16K / 60.0) as usize; + let mut best_lag = 0_usize; + let mut best_corr = 0.0_f32; + for lag in min_lag..=max_lag.min(frame.len().saturating_sub(1)) { + let mut corr = 0.0_f32; + let mut energy = 0.0_f32; + for i in lag..frame.len() { + corr += frame[i] * frame[i - lag]; + energy += frame[i - lag] * frame[i - lag]; + } + let norm = if energy > 1.0e-8 { + corr / energy.sqrt() + } else { + 0.0 + }; + if norm > best_corr { + best_corr = norm; + best_lag = lag; + } + } + if best_lag == 0 || best_corr < 0.01 { + 0.0 + } else { + SAMPLE_RATE_16K / best_lag as f32 + } +} + +// --------------------------------------------------------------------------- +// Background worker — same pattern as SileroOnnxVadWorker so the realtime +// callback never blocks on STFT / pitch / ONNX inference. +// --------------------------------------------------------------------------- + +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64}; +use std::sync::Arc; +use std::thread::JoinHandle; + +/// Maximum number of 10 ms frames the worker may lag before the callback +/// treats its output as stale and uses WebRTC fallback instead. +const TEN_MAX_STALE_FRAMES: u64 = 8; + +struct TenFrameMessage { + seq: u64, + frame: [f32; INPUT_FRAME_10MS], +} + +/// Background TEN VAD worker. The realtime callback only enqueues 10 ms +/// frames and reads the latest probability atomically. +pub struct TenOnnxVadWorker { + tx: Option>, + latest_probability: Arc, + latest_processed_seq: Arc, + alive: Arc, + handle: Option>, +} + +impl TenOnnxVadWorker { + /// Start a background TEN worker if the model loads. + pub fn try_new(model_path: &str) -> Option { + let vad = TenOnnxVad::try_new(model_path)?; + let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits())); + let latest_processed_seq = Arc::new(AtomicU64::new(u64::MAX)); + let alive = Arc::new(AtomicBool::new(true)); + let (tx, rx) = std::sync::mpsc::sync_channel::(32); + let prob_arc = latest_probability.clone(); + let seq_arc = latest_processed_seq.clone(); + let alive_arc = alive.clone(); + + let handle = std::thread::Builder::new() + .name("chanora-ten-vad".to_string()) + .spawn(move || { + let mut vad = vad; + while alive_arc.load(std::sync::atomic::Ordering::Relaxed) { + let msg = match rx.recv() { + Ok(m) => m, + Err(_) => break, + }; + let mut frame_f32 = [0.0_f32; INPUT_FRAME_10MS]; + frame_f32.copy_from_slice(&msg.frame); + let out = VoiceActivityDetector::process_10ms(&mut vad, &frame_f32); + prob_arc.store( + out.probability.clamp(0.0, 1.0).to_bits(), + std::sync::atomic::Ordering::Relaxed, + ); + seq_arc.store(msg.seq, std::sync::atomic::Ordering::Relaxed); + } + }) + .ok()?; + + Some(Self { + tx: Some(tx), + latest_probability, + latest_processed_seq, + alive, + handle: Some(handle), + }) + } + + /// Best-effort enqueue of a 10 ms frame for background inference. + pub fn try_send(&self, seq: u64, frame: &[f32; INPUT_FRAME_10MS]) -> bool { + let Some(tx) = &self.tx else { + return false; + }; + tx.try_send(TenFrameMessage { seq, frame: *frame }).is_ok() + } + + /// Latest probability published by the background worker. + pub fn latest_probability(&self) -> f32 { + f32::from_bits( + self.latest_probability + .load(std::sync::atomic::Ordering::Relaxed), + ) + } + + /// True when the worker is too far behind to trust its output. + pub fn is_stale(&self, capture_seq: u64) -> bool { + let latest = self + .latest_processed_seq + .load(std::sync::atomic::Ordering::Relaxed); + latest == u64::MAX || capture_seq.saturating_sub(latest) > TEN_MAX_STALE_FRAMES + } +} + +impl Drop for TenOnnxVadWorker { + fn drop(&mut self) { + self.alive + .store(false, std::sync::atomic::Ordering::Relaxed); + drop(self.tx.take()); + if let Some(h) = self.handle.take() { + let _ = h.join(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mel_filter_bank_has_expected_shape() { + let filters = build_mel_filters(); + assert_eq!(filters.len(), MEL_BANDS); + assert!(filters.iter().all(|f| f.iter().any(|&v| v > 0.0))); + } + + #[test] + fn preprocessing_produces_finite_features() { + let filters = build_mel_filters(); + let frame = vec![0.0_f32; WINDOW_16K]; + let feature = compute_feature(&filters, &frame); + assert!(feature.iter().all(|v| v.is_finite())); + } +} diff --git a/crates/chanora_bridge/src/api.rs b/crates/chanora_bridge/src/api.rs index 6e0f74b..575e11a 100644 --- a/crates/chanora_bridge/src/api.rs +++ b/crates/chanora_bridge/src/api.rs @@ -1725,6 +1725,21 @@ pub async fn set_vad_model_path(path: String) -> Result<(), BridgeError> { Ok(()) } +/// Configure the TEN VAD ONNX model path. +pub async fn set_ten_vad_model_path(path: String) -> Result<(), BridgeError> { + if path.trim().is_empty() { + return Err(BridgeError::InvalidCommand( + "ten vad model path must not be empty".to_string(), + )); + } + runtime() + .spawn(async move { chanora_audio::vad::set_ten_model_path(&path).map_err(|e| e) }) + .await + .map_err(|e| task_join_error("set_ten_vad_model_path", e))? + .map_err(|e| BridgeError::Unmapped(format!("set_ten_vad_model_path: {e}")))?; + Ok(()) +} + /// Enable or disable audio debug WAV dumping. pub async fn enable_audio_debug_wav_dump(enabled: bool) -> Result<(), BridgeError> { runtime() @@ -1746,20 +1761,20 @@ pub async fn set_ios_voice_processing_mode( BridgeIosVoiceProcessingMode::PlatformVoiceProcessing => { BridgeAudioBackend::PlatformVoiceProcessing } - BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeAudioBackend::Sonora, + BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeAudioBackend::WebrtcApm, }, vad_backend: BridgeVadBackend::SileroOnnx, aec: match mode { BridgeIosVoiceProcessingMode::PlatformVoiceProcessing => BridgeEffectOwner::Platform, - BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeEffectOwner::Sonora, + BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeEffectOwner::WebrtcApm, }, ns: match mode { BridgeIosVoiceProcessingMode::PlatformVoiceProcessing => BridgeEffectOwner::Platform, - BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeEffectOwner::Sonora, + BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeEffectOwner::WebrtcApm, }, agc: match mode { BridgeIosVoiceProcessingMode::PlatformVoiceProcessing => BridgeEffectOwner::Platform, - BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeEffectOwner::Sonora, + BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeEffectOwner::WebrtcApm, }, hpf_enabled: true, limiter_enabled: true, diff --git a/crates/chanora_bridge/src/frb_generated.rs b/crates/chanora_bridge/src/frb_generated.rs index 0207be8..8e7c8ed 100644 --- a/crates/chanora_bridge/src/frb_generated.rs +++ b/crates/chanora_bridge/src/frb_generated.rs @@ -38,7 +38,7 @@ flutter_rust_bridge::frb_generated_boilerplate!( default_rust_auto_opaque = RustAutoOpaqueMoi, ); pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_VERSION: &str = "2.12.0"; -pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = -1835973251; +pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = -436507436; // Section: executor @@ -398,6 +398,41 @@ fn wire__crate__api__export_diagnostics_impl( }, ) } +fn wire__crate__api__get_audio_processing_config_impl( + port_: flutter_rust_bridge::for_generated::MessagePort, + ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, + rust_vec_len_: i32, + data_len_: i32, +) { + FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( + flutter_rust_bridge::for_generated::TaskInfo { + debug_name: "get_audio_processing_config", + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + }, + move || { + let message = unsafe { + flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire( + ptr_, + rust_vec_len_, + data_len_, + ) + }; + let mut deserializer = + flutter_rust_bridge::for_generated::SseDeserializer::new(message); + deserializer.end(); + move |context| async move { + transform_result_sse::<_, crate::BridgeError>( + (move || async move { + let output_ok = crate::api::get_audio_processing_config().await?; + Ok(output_ok) + })() + .await, + ) + } + }, + ) +} fn wire__crate__api__get_ptt_binding_impl( port_: flutter_rust_bridge::for_generated::MessagePort, ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, @@ -598,6 +633,38 @@ fn wire__crate__api__handle_media_services_reset_impl( }, ) } +fn wire__crate__api__handle_media_services_reset_with_route_impl( + ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, + rust_vec_len_: i32, + data_len_: i32, +) -> flutter_rust_bridge::for_generated::WireSyncRust2DartSse { + FLUTTER_RUST_BRIDGE_HANDLER.wrap_sync::( + flutter_rust_bridge::for_generated::TaskInfo { + debug_name: "handle_media_services_reset_with_route", + port: None, + mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, + }, + move || { + let message = unsafe { + flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire( + ptr_, + rust_vec_len_, + data_len_, + ) + }; + let mut deserializer = + flutter_rust_bridge::for_generated::SseDeserializer::new(message); + let api_route_class = ::sse_decode(&mut deserializer); + deserializer.end(); + transform_result_sse::<_, ()>((move || { + let output_ok = Result::<_, ()>::Ok({ + crate::api::handle_media_services_reset_with_route(api_route_class); + })?; + Ok(output_ok) + })()) + }, + ) +} fn wire__crate__api__handle_route_change_impl( ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, rust_vec_len_: i32, @@ -1198,6 +1265,42 @@ fn wire__crate__api__set_release_tail_ms_impl( }, ) } +fn wire__crate__api__set_ten_vad_model_path_impl( + port_: flutter_rust_bridge::for_generated::MessagePort, + ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, + rust_vec_len_: i32, + data_len_: i32, +) { + FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( + flutter_rust_bridge::for_generated::TaskInfo { + debug_name: "set_ten_vad_model_path", + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + }, + move || { + let message = unsafe { + flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire( + ptr_, + rust_vec_len_, + data_len_, + ) + }; + let mut deserializer = + flutter_rust_bridge::for_generated::SseDeserializer::new(message); + let api_path = ::sse_decode(&mut deserializer); + deserializer.end(); + move |context| async move { + transform_result_sse::<_, crate::BridgeError>( + (move || async move { + let output_ok = crate::api::set_ten_vad_model_path(api_path).await?; + Ok(output_ok) + })() + .await, + ) + } + }, + ) +} fn wire__crate__api__set_transmit_mode_impl( port_: flutter_rust_bridge::for_generated::MessagePort, ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, @@ -2090,31 +2193,33 @@ fn pde_ffi_dispatcher_primary_impl( 7 => wire__crate__api__disconnect_impl(port, ptr, rust_vec_len, data_len), 8 => wire__crate__api__enable_audio_debug_wav_dump_impl(port, ptr, rust_vec_len, data_len), 9 => wire__crate__api__events_stream_impl(port, ptr, rust_vec_len, data_len), - 11 => wire__crate__api__get_ptt_binding_impl(port, ptr, rust_vec_len, data_len), - 12 => wire__crate__api__get_release_tail_ms_impl(port, ptr, rust_vec_len, data_len), - 13 => wire__crate__api__get_transmit_mode_impl(port, ptr, rust_vec_len, data_len), - 18 => wire__crate__api__init_storage_impl(port, ptr, rust_vec_len, data_len), - 19 => wire__crate__api__is_connected_impl(port, ptr, rust_vec_len, data_len), - 20 => wire__crate__api__list_bookmarks_impl(port, ptr, rust_vec_len, data_len), - 22 => wire__crate__api__move_to_channel_impl(port, ptr, rust_vec_len, data_len), - 23 => wire__crate__api__ptt_descriptor_impl(port, ptr, rust_vec_len, data_len), - 24 => wire__crate__api__set_audio_processing_config_impl(port, ptr, rust_vec_len, data_len), - 25 => wire__crate__api__set_hard_mute_impl(port, ptr, rust_vec_len, data_len), - 26 => wire__crate__api__set_input_muted_impl(port, ptr, rust_vec_len, data_len), - 27 => { + 11 => wire__crate__api__get_audio_processing_config_impl(port, ptr, rust_vec_len, data_len), + 12 => wire__crate__api__get_ptt_binding_impl(port, ptr, rust_vec_len, data_len), + 13 => wire__crate__api__get_release_tail_ms_impl(port, ptr, rust_vec_len, data_len), + 14 => wire__crate__api__get_transmit_mode_impl(port, ptr, rust_vec_len, data_len), + 20 => wire__crate__api__init_storage_impl(port, ptr, rust_vec_len, data_len), + 21 => wire__crate__api__is_connected_impl(port, ptr, rust_vec_len, data_len), + 22 => wire__crate__api__list_bookmarks_impl(port, ptr, rust_vec_len, data_len), + 24 => wire__crate__api__move_to_channel_impl(port, ptr, rust_vec_len, data_len), + 25 => wire__crate__api__ptt_descriptor_impl(port, ptr, rust_vec_len, data_len), + 26 => wire__crate__api__set_audio_processing_config_impl(port, ptr, rust_vec_len, data_len), + 27 => wire__crate__api__set_hard_mute_impl(port, ptr, rust_vec_len, data_len), + 28 => wire__crate__api__set_input_muted_impl(port, ptr, rust_vec_len, data_len), + 29 => { wire__crate__api__set_ios_voice_processing_mode_impl(port, ptr, rust_vec_len, data_len) } - 29 => wire__crate__api__set_output_gain_impl(port, ptr, rust_vec_len, data_len), - 30 => wire__crate__api__set_output_muted_impl(port, ptr, rust_vec_len, data_len), - 31 => wire__crate__api__set_ptt_impl(port, ptr, rust_vec_len, data_len), - 32 => wire__crate__api__set_ptt_binding_impl(port, ptr, rust_vec_len, data_len), - 33 => wire__crate__api__set_release_tail_ms_impl(port, ptr, rust_vec_len, data_len), - 34 => wire__crate__api__set_transmit_mode_impl(port, ptr, rust_vec_len, data_len), - 35 => wire__crate__api__set_vad_model_path_impl(port, ptr, rust_vec_len, data_len), - 36 => wire__crate__api__snapshot_impl(port, ptr, rust_vec_len, data_len), - 37 => wire__crate__api__update_bookmark_impl(port, ptr, rust_vec_len, data_len), - 38 => wire__crate__api__voice_join_impl(port, ptr, rust_vec_len, data_len), - 39 => wire__crate__api__voice_leave_impl(port, ptr, rust_vec_len, data_len), + 31 => wire__crate__api__set_output_gain_impl(port, ptr, rust_vec_len, data_len), + 32 => wire__crate__api__set_output_muted_impl(port, ptr, rust_vec_len, data_len), + 33 => wire__crate__api__set_ptt_impl(port, ptr, rust_vec_len, data_len), + 34 => wire__crate__api__set_ptt_binding_impl(port, ptr, rust_vec_len, data_len), + 35 => wire__crate__api__set_release_tail_ms_impl(port, ptr, rust_vec_len, data_len), + 36 => wire__crate__api__set_ten_vad_model_path_impl(port, ptr, rust_vec_len, data_len), + 37 => wire__crate__api__set_transmit_mode_impl(port, ptr, rust_vec_len, data_len), + 38 => wire__crate__api__set_vad_model_path_impl(port, ptr, rust_vec_len, data_len), + 39 => wire__crate__api__snapshot_impl(port, ptr, rust_vec_len, data_len), + 40 => wire__crate__api__update_bookmark_impl(port, ptr, rust_vec_len, data_len), + 41 => wire__crate__api__voice_join_impl(port, ptr, rust_vec_len, data_len), + 42 => wire__crate__api__voice_leave_impl(port, ptr, rust_vec_len, data_len), _ => unreachable!(), } } @@ -2128,12 +2233,17 @@ fn pde_ffi_dispatcher_sync_impl( // Codec=Pde (Serialization + dispatch), see doc to use other codecs match func_id { 10 => wire__crate__api__export_diagnostics_impl(ptr, rust_vec_len, data_len), - 14 => wire__crate__api__handle_interruption_began_impl(ptr, rust_vec_len, data_len), - 15 => wire__crate__api__handle_interruption_ended_impl(ptr, rust_vec_len, data_len), - 16 => wire__crate__api__handle_media_services_reset_impl(ptr, rust_vec_len, data_len), - 17 => wire__crate__api__handle_route_change_impl(ptr, rust_vec_len, data_len), - 21 => wire__crate__api__log_file_path_str_impl(ptr, rust_vec_len, data_len), - 28 => wire__crate__api__set_network_state_impl(ptr, rust_vec_len, data_len), + 15 => wire__crate__api__handle_interruption_began_impl(ptr, rust_vec_len, data_len), + 16 => wire__crate__api__handle_interruption_ended_impl(ptr, rust_vec_len, data_len), + 17 => wire__crate__api__handle_media_services_reset_impl(ptr, rust_vec_len, data_len), + 18 => wire__crate__api__handle_media_services_reset_with_route_impl( + ptr, + rust_vec_len, + data_len, + ), + 19 => wire__crate__api__handle_route_change_impl(ptr, rust_vec_len, data_len), + 23 => wire__crate__api__log_file_path_str_impl(ptr, rust_vec_len, data_len), + 30 => wire__crate__api__set_network_state_impl(ptr, rust_vec_len, data_len), _ => unreachable!(), } }