diff --git a/Cargo.lock b/Cargo.lock index 67c2e29..aad4f65 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,12 +148,111 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" dependencies = [ - "event-listener", + "event-listener 5.4.1", "event-listener-strategy", "futures-core", "pin-project-lite", ] +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "pin-project-lite", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" +dependencies = [ + "async-channel 2.5.0", + "async-executor", + "async-io", + "async-lock", + "blocking", + "futures-lite", + "once_cell", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener 5.4.1", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-process" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" +dependencies = [ + "async-channel 2.5.0", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener 5.4.1", + "futures-lite", + "rustix", +] + [[package]] name = "async-recursion" version = "1.1.1" @@ -165,6 +264,57 @@ dependencies = [ "syn", ] +[[package]] +name = "async-signal" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix", + "signal-hook-registry", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-std" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c8e079a4ab67ae52b7403632e4618815d6db36d2a010cfe41b02c1b1578f93b" +dependencies = [ + "async-channel 1.9.0", + "async-global-executor", + "async-io", + "async-lock", + "async-process", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.89" @@ -257,6 +407,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -308,6 +464,19 @@ dependencies = [ "objc2", ] +[[package]] +name = "blocking" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +dependencies = [ + "async-channel 2.5.0", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + [[package]] name = "build-target" version = "0.4.0" @@ -352,6 +521,32 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "cacache" +version = "13.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5063741c7b2e260bbede781cf4679632dd90e2718e99f7715e46824b65670b" +dependencies = [ + "async-std", + "digest 0.10.7", + "either", + "futures", + "hex", + "libc", + "memmap2", + "miette", + "reflink-copy", + "serde", + "serde_derive", + "serde_json", + "sha1", + "sha2", + "ssri", + "tempfile", + "thiserror 1.0.69", + "walkdir", +] + [[package]] name = "cast" version = "0.3.0" @@ -475,11 +670,23 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "chanora_cache" +version = "0.2.0-beta.1" +dependencies = [ + "cacache", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tracing", +] + [[package]] name = "chanora_core" version = "0.2.0-beta.1" dependencies = [ "chanora_audio", + "chanora_cache", "chanora_diagnostics", "chanora_prefetch", "chanora_protocol", @@ -515,7 +722,7 @@ name = "chanora_protocol" version = "0.2.0-beta.1" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "chanora_resolver", "futures", "reqwest 0.13.4", @@ -554,7 +761,7 @@ dependencies = [ name = "chanora_storage" version = "0.2.0-beta.1" dependencies = [ - "base64", + "base64 0.22.1", "chacha20poly1305", "keyring", "rand 0.8.6", @@ -1239,6 +1446,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "event-listener" version = "5.4.1" @@ -1256,7 +1469,7 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" dependencies = [ - "event-listener", + "event-listener 5.4.1", "pin-project-lite", ] @@ -1559,6 +1772,18 @@ dependencies = [ "time", ] +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "group" version = "0.13.0" @@ -1837,7 +2062,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -2142,6 +2367,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -2226,6 +2460,9 @@ name = "log" version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "113b30b4cd05f7c06868fdb2854f66a7b9fece9a48425351cd532e810d74024f" +dependencies = [ + "value-bag", +] [[package]] name = "lru-slab" @@ -2274,6 +2511,15 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -2283,6 +2529,29 @@ dependencies = [ "autocfg", ] +[[package]] +name = "miette" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e" +dependencies = [ + "miette-derive", + "once_cell", + "thiserror 1.0.69", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "mime" version = "0.3.17" @@ -2813,6 +3082,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + [[package]] name = "pkcs8" version = "0.10.2" @@ -2857,6 +3137,20 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "poly1305" version = "0.8.0" @@ -3183,6 +3477,18 @@ dependencies = [ "syn", ] +[[package]] +name = "reflink-copy" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13362233b147e57674c37b802d216b7c5e3dcccbed8967c84f0d8d223868ae27" +dependencies = [ + "cfg-if", + "libc", + "rustix", + "windows", +] + [[package]] name = "regex" version = "1.12.3" @@ -3218,7 +3524,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "http", @@ -3256,7 +3562,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -3672,6 +3978,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.10.9" @@ -3851,6 +4168,23 @@ dependencies = [ "der", ] +[[package]] +name = "ssri" +version = "9.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7a2b3c2bc9693bcb40870c4e9b5bf0d79f9cb46273321bf855ec513e919082" +dependencies = [ + "base64 0.21.7", + "digest 0.10.7", + "hex", + "miette", + "serde", + "sha-1", + "sha2", + "thiserror 1.0.69", + "xxhash-rust", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -4349,7 +4683,7 @@ name = "ts-bookkeeping" version = "0.1.0" source = "git+https://github.com/ReSpeak/tsclientlib.git?rev=04aa2491#04aa24917abbf6a0c8442a79742d6d2d40ecf71e" dependencies = [ - "base64", + "base64 0.22.1", "heck", "itertools 0.14.0", "num-derive", @@ -4370,7 +4704,7 @@ version = "0.2.0" source = "git+https://github.com/ReSpeak/tsclientlib.git?rev=04aa2491#04aa24917abbf6a0c8442a79742d6d2d40ecf71e" dependencies = [ "audiopus", - "base64", + "base64 0.22.1", "futures", "git-testament", "hickory-net", @@ -4398,7 +4732,7 @@ version = "0.2.0" source = "git+https://github.com/ReSpeak/tsclientlib.git?rev=04aa2491#04aa24917abbf6a0c8442a79742d6d2d40ecf71e" dependencies = [ "aes", - "base64", + "base64 0.22.1", "curve25519-dalek-ng", "eax", "futures", @@ -4427,7 +4761,7 @@ name = "tsproto-packets" version = "0.1.0" source = "git+https://github.com/ReSpeak/tsclientlib.git?rev=04aa2491#04aa24917abbf6a0c8442a79742d6d2d40ecf71e" dependencies = [ - "base64", + "base64 0.22.1", "bitflags 2.12.1", "num-derive", "num-traits", @@ -4442,7 +4776,7 @@ name = "tsproto-structs" version = "0.2.0" source = "git+https://github.com/EdisonJwa/tsclientlib.git?branch=fix%2Fp256-short-coordinate-pad#8b7a3226c692319b714ea1d32fd5ded05911aa40" dependencies = [ - "base64", + "base64 0.22.1", "csv", "heck", "once_cell", @@ -4455,7 +4789,7 @@ name = "tsproto-structs" version = "0.2.0" source = "git+https://github.com/ReSpeak/tsclientlib.git?rev=04aa2491#04aa24917abbf6a0c8442a79742d6d2d40ecf71e" dependencies = [ - "base64", + "base64 0.22.1", "csv", "heck", "once_cell", @@ -4468,7 +4802,7 @@ name = "tsproto-types" version = "0.1.0" source = "git+https://github.com/EdisonJwa/tsclientlib.git?branch=fix%2Fp256-short-coordinate-pad#8b7a3226c692319b714ea1d32fd5ded05911aa40" dependencies = [ - "base64", + "base64 0.22.1", "bitflags 2.12.1", "curve25519-dalek-ng", "elliptic-curve", @@ -4512,6 +4846,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -4570,6 +4910,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "value-bag" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ba6f5989077681266825251a52748b8c1d8a4ad098cc37e440103d0ea717fc0" + [[package]] name = "vcpkg" version = "0.2.15" @@ -5256,6 +5602,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + [[package]] name = "yoke" version = "0.8.2" @@ -5289,7 +5641,7 @@ dependencies = [ "async-recursion", "async-trait", "enumflags2", - "event-listener", + "event-listener 5.4.1", "futures-core", "futures-lite", "hex", diff --git a/Cargo.toml b/Cargo.toml index bcae294..cc4dfc9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ # crates/chanora_resolver/ — TeamSpeak address resolution # crates/chanora_state/ — snapshot, deltas, reducers # crates/chanora_audio/ — capture, DSP, Opus, jitter, mixer +# crates/chanora_cache/ — avatar/icon blob cache (cacache-backed) # crates/chanora_storage/ — bookmarks, settings, identity refs # crates/chanora_diagnostics/ — logs, redaction, export # crates/chanora_prefetch — server-resolution prefetch cache/policy @@ -30,6 +31,7 @@ members = [ "crates/chanora_state", "crates/chanora_audio", "crates/chanora_storage", + "crates/chanora_cache", "crates/chanora_diagnostics", "crates/chanora_prefetch", "crates/chanora_bridge", diff --git a/apps/chanora_flutter/lib/main.dart b/apps/chanora_flutter/lib/main.dart index 35bc003..7b0036f 100644 --- a/apps/chanora_flutter/lib/main.dart +++ b/apps/chanora_flutter/lib/main.dart @@ -93,7 +93,10 @@ String _kAppVersion = appSemverBaseline; Future main() async { WidgetsFlutterBinding.ensureInitialized(); await RustLib.init(); - unawaited(wireStorage()); + unawaited(() async { + await wireStorage(); + await wireCache(); + }()); unawaited(wireConnectivity()); wireAudioLifecycle(); await configureBundledVadModels(); diff --git a/apps/chanora_flutter/lib/services/app_bootstrap.dart b/apps/chanora_flutter/lib/services/app_bootstrap.dart index ef14815..b1a6e26 100644 --- a/apps/chanora_flutter/lib/services/app_bootstrap.dart +++ b/apps/chanora_flutter/lib/services/app_bootstrap.dart @@ -15,15 +15,22 @@ typedef StorageDirectoryProvider = Future Function(); typedef StorageInitializer = Future Function(String dir); Future? _storageInitFuture; +Future? _cacheInitFuture; Future? _vadBootstrapFuture; StorageDirectoryProvider _storageDirectoryProvider = getApplicationSupportDirectory; +StorageDirectoryProvider _cacheDirectoryProvider = getApplicationCacheDirectory; StorageInitializer _storageInitializer = _defaultStorageInitializer; +StorageInitializer _cacheInitializer = _defaultCacheInitializer; Future _defaultStorageInitializer(String dir) { return rust.initStorage(dir: dir); } +Future _defaultCacheInitializer(String dir) { + return rust.initCache(dir: dir); +} + Future _copyBundledAssetToDocuments({ required String assetPath, required String fileName, @@ -121,16 +128,50 @@ Future _wireStorageImpl() async { } } +Future wireCache() async { + final existing = _cacheInitFuture; + if (existing != null) { + await existing; + return; + } + + final initFuture = _wireCacheImpl(); + _cacheInitFuture = initFuture; + await initFuture; +} + +Future _wireCacheImpl() async { + var initialized = false; + try { + final dir = await _cacheDirectoryProvider(); + await _cacheInitializer(dir.path); + initialized = true; + } catch (_) { + // Best-effort; missing cache just means protocol-owned assets are + // re-downloaded this session. + } finally { + if (!initialized) { + _cacheInitFuture = null; + } + } +} + @visibleForTesting void debugResetStorageBootstrap({ StorageDirectoryProvider? storageDirectoryProvider, + StorageDirectoryProvider? cacheDirectoryProvider, StorageInitializer? storageInitializer, + StorageInitializer? cacheInitializer, }) { _storageInitFuture = null; + _cacheInitFuture = null; _vadBootstrapFuture = null; _storageDirectoryProvider = storageDirectoryProvider ?? getApplicationSupportDirectory; + _cacheDirectoryProvider = + cacheDirectoryProvider ?? getApplicationCacheDirectory; _storageInitializer = storageInitializer ?? _defaultStorageInitializer; + _cacheInitializer = cacheInitializer ?? _defaultCacheInitializer; } rust.BridgeNetworkState _mapConnectivity(List results) { diff --git a/apps/chanora_flutter/lib/src/rust/api.dart b/apps/chanora_flutter/lib/src/rust/api.dart index 3072244..c743d12 100644 --- a/apps/chanora_flutter/lib/src/rust/api.dart +++ b/apps/chanora_flutter/lib/src/rust/api.dart @@ -226,6 +226,41 @@ String exportDiagnostics() => RustLib.instance.api.crateApiExportDiagnostics(); Future initStorage({required String dir}) => RustLib.instance.api.crateApiInitStorage(dir: dir); +/// Wire the blob cache to a platform-private cache directory. +Future initCache({required String dir}) { + final api = RustLib.instance.api as dynamic; + try { + return api.crateApiInitCache(dir: dir) as Future; + } on NoSuchMethodError { + return Future.value(); + } +} + +/// Return an avatar from cache when present; otherwise download it. +Future?> downloadAvatar({ + required String avatarHash, + required String clientUid, +}) { + final api = RustLib.instance.api as dynamic; + try { + return api.crateApiDownloadAvatar( + avatarHash: avatarHash, + clientUid: clientUid, + ) as Future?>; + } on NoSuchMethodError { + return Future.value(); + } +} + +Future?> downloadIcon({required BigInt iconId}) { + final api = RustLib.instance.api as dynamic; + try { + return api.crateApiDownloadIcon(iconId: iconId) as Future?>; + } on NoSuchMethodError { + return Future.value(); + } +} + /// List persisted bookmarks. Future> listBookmarks() => RustLib.instance.api.crateApiListBookmarks(); diff --git a/core/chanora_core/Cargo.toml b/core/chanora_core/Cargo.toml index 3003815..b7b87be 100644 --- a/core/chanora_core/Cargo.toml +++ b/core/chanora_core/Cargo.toml @@ -10,6 +10,7 @@ repository.workspace = true publish.workspace = true [dependencies] +chanora_cache = { path = "../../crates/chanora_cache" } chanora_protocol = { path = "../../crates/chanora_protocol" } chanora_state = { path = "../../crates/chanora_state" } chanora_audio = { path = "../../crates/chanora_audio" } diff --git a/core/chanora_core/src/file_transfer.rs b/core/chanora_core/src/file_transfer.rs new file mode 100644 index 0000000..c53c86f --- /dev/null +++ b/core/chanora_core/src/file_transfer.rs @@ -0,0 +1,344 @@ +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use chanora_cache::{BlobCache, BlobCacheError, PREFIX_AVATAR, PREFIX_ICON}; +use chanora_protocol::{ProtocolClient, ProtocolError}; +use tokio::sync::{Mutex, Semaphore, oneshot}; +use tracing::warn; + +const MAX_CONCURRENT_DOWNLOADS: usize = 2; +const NEGATIVE_CACHE_TTL: Duration = Duration::from_secs(5 * 60); + +type InFlightWaiters = Vec>, FileTransferError>>>; + +/// Errors raised while resolving protocol-owned file assets. +#[derive(Debug, thiserror::Error)] +pub enum FileTransferError { + /// No live protocol client is available for a download. + #[error("not connected")] + NotConnected, + /// The protocol layer failed while downloading the asset. + #[error("protocol error: {0}")] + Protocol(#[from] ProtocolError), + /// The blob cache failed while reading or writing the asset. + #[error("cache error: {0}")] + Cache(#[from] BlobCacheError), +} + +impl Clone for FileTransferError { + fn clone(&self) -> Self { + match self { + Self::NotConnected => Self::NotConnected, + Self::Protocol(error) => Self::Protocol(clone_protocol_error(error)), + Self::Cache(error) => Self::Cache(clone_blob_cache_error(error)), + } + } +} + +pub struct FileTransferService { + cache: BlobCache, + protocol: Arc>>, + semaphore: Arc, + in_flight: Arc>>, + negative_cache: Arc>>, +} + +impl FileTransferService { + pub fn new(cache: BlobCache, protocol: Arc>>) -> Self { + Self { + cache, + protocol, + semaphore: Arc::new(Semaphore::new(MAX_CONCURRENT_DOWNLOADS)), + in_flight: Arc::new(Mutex::new(HashMap::new())), + negative_cache: Arc::new(Mutex::new(HashMap::new())), + } + } + + pub async fn set_protocol(&self, client: Option) { + *self.protocol.lock().await = client; + } + + pub async fn get_avatar( + &self, + avatar_hash: &str, + client_uid: &str, + ) -> Result>, FileTransferError> { + if let Some(bytes) = self.cache.get(PREFIX_AVATAR, avatar_hash).await? { + return Ok(Some(bytes)); + } + + if self.is_negative_cache_hit(avatar_hash).await { + return Ok(None); + } + + let rx = { + let mut in_flight = self.in_flight.lock().await; + if let Some(waiters) = in_flight.get_mut(avatar_hash) { + let (tx, rx) = oneshot::channel(); + waiters.push(tx); + Some(rx) + } else { + in_flight.insert(avatar_hash.to_string(), Vec::new()); + None + } + }; + + if let Some(rx) = rx { + return rx.await.unwrap_or_else(|_| { + Err(FileTransferError::Protocol(ProtocolError::Lost( + "coalesced avatar download waiter dropped".to_string(), + ))) + }); + } + + let _permit = self + .semaphore + .acquire() + .await + .expect("file transfer semaphore should stay open"); + + let result = self.do_download_avatar(avatar_hash, client_uid).await; + self.finish_in_flight(avatar_hash, &result).await; + result + } + + pub async fn get_icon(&self, icon_id: u64) -> Result>, FileTransferError> { + let icon_key = icon_id.to_string(); + let negative_key = format!("ic_{icon_id}"); + let in_flight_key = format!("icon_{icon_id}"); + + if let Some(bytes) = self.cache.get(PREFIX_ICON, &icon_key).await? { + return Ok(Some(bytes)); + } + + if self.is_negative_cache_hit(&negative_key).await { + return Ok(None); + } + + let rx = { + let mut in_flight = self.in_flight.lock().await; + if let Some(waiters) = in_flight.get_mut(&in_flight_key) { + let (tx, rx) = oneshot::channel(); + waiters.push(tx); + Some(rx) + } else { + in_flight.insert(in_flight_key.clone(), Vec::new()); + None + } + }; + + if let Some(rx) = rx { + return rx.await.unwrap_or_else(|_| { + Err(FileTransferError::Protocol(ProtocolError::Lost( + "coalesced icon download waiter dropped".to_string(), + ))) + }); + } + + let _permit = self + .semaphore + .acquire() + .await + .expect("file transfer semaphore should stay open"); + + let result = self.do_download_icon(icon_id).await; + self.finish_in_flight(&in_flight_key, &result).await; + result + } + + pub async fn clear_cache(&self) -> Result<(), FileTransferError> { + self.cache.clear().await?; + self.negative_cache.lock().await.clear(); + Ok(()) + } + + pub async fn cache_size(&self) -> Result { + Ok(self.cache.total_size().await?) + } + + async fn do_download_avatar( + &self, + avatar_hash: &str, + client_uid: &str, + ) -> Result>, FileTransferError> { + if let Some(bytes) = self.cache.get(PREFIX_AVATAR, avatar_hash).await? { + return Ok(Some(bytes)); + } + + if self.is_negative_cache_hit(avatar_hash).await { + return Ok(None); + } + + let protocol = self.protocol.lock().await; + let client = protocol.as_ref().ok_or(FileTransferError::NotConnected)?; + match client.download_avatar(client_uid).await { + Ok(bytes) => { + self.cache.put(PREFIX_AVATAR, avatar_hash, &bytes).await?; + self.negative_cache.lock().await.remove(avatar_hash); + Ok(Some(bytes)) + } + Err(ProtocolError::ServerRejected { .. }) => { + self.negative_cache + .lock() + .await + .insert(avatar_hash.to_string(), Instant::now() + NEGATIVE_CACHE_TTL); + Ok(None) + } + Err(error) => Err(FileTransferError::Protocol(error)), + } + } + + async fn do_download_icon(&self, icon_id: u64) -> Result>, FileTransferError> { + let icon_key = icon_id.to_string(); + let negative_key = format!("ic_{icon_id}"); + + if let Some(bytes) = self.cache.get(PREFIX_ICON, &icon_key).await? { + return Ok(Some(bytes)); + } + + if self.is_negative_cache_hit(&negative_key).await { + return Ok(None); + } + + let protocol = self.protocol.lock().await; + let client = protocol.as_ref().ok_or(FileTransferError::NotConnected)?; + match client.download_icon(icon_id).await { + Ok(bytes) => { + self.cache.put(PREFIX_ICON, &icon_key, &bytes).await?; + self.negative_cache.lock().await.remove(&negative_key); + Ok(Some(bytes)) + } + Err(ProtocolError::ServerRejected { .. }) => { + self.negative_cache + .lock() + .await + .insert(negative_key, Instant::now() + NEGATIVE_CACHE_TTL); + Ok(None) + } + Err(error) => Err(FileTransferError::Protocol(error)), + } + } + + async fn finish_in_flight( + &self, + avatar_hash: &str, + result: &Result>, FileTransferError>, + ) { + let waiters = self.in_flight.lock().await.remove(avatar_hash).unwrap_or_default(); + for waiter in waiters { + if waiter.send(result.clone()).is_err() { + warn!(target: "chanora_core", avatar_hash, "avatar download waiter dropped"); + } + } + } + + async fn is_negative_cache_hit(&self, avatar_hash: &str) -> bool { + let now = Instant::now(); + let mut negative_cache = self.negative_cache.lock().await; + match negative_cache.get(avatar_hash).copied() { + Some(expires_at) if expires_at > now => true, + Some(_) => { + negative_cache.remove(avatar_hash); + false + } + None => false, + } + } +} + +fn clone_protocol_error(error: &ProtocolError) -> ProtocolError { + match error { + ProtocolError::Invalid(message) => ProtocolError::Invalid(message.clone()), + ProtocolError::DnsFailed { host, reason } => ProtocolError::DnsFailed { + host: host.clone(), + reason: reason.clone(), + }, + ProtocolError::Connect(message) => ProtocolError::Connect(message.clone()), + ProtocolError::DisconnectedEarly(message) => { + ProtocolError::DisconnectedEarly(message.clone()) + } + ProtocolError::Lost(message) => ProtocolError::Lost(message.clone()), + ProtocolError::Identity(message) => ProtocolError::Identity(message.clone()), + ProtocolError::Timeout => ProtocolError::Timeout, + ProtocolError::ServerRejected { code, message } => ProtocolError::ServerRejected { + code: *code, + message: message.clone(), + }, + ProtocolError::Backend(message) => ProtocolError::Backend(message.clone()), + ProtocolError::FileTransfer(message) => ProtocolError::FileTransfer(message.clone()), + } +} + +fn clone_blob_cache_error(error: &BlobCacheError) -> BlobCacheError { + match error { + BlobCacheError::Io(message) => BlobCacheError::Io(message.clone()), + BlobCacheError::InvalidKey(message) => BlobCacheError::InvalidKey(message.clone()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_cache_dir(name: &str) -> std::path::PathBuf { + let mut path = std::env::temp_dir(); + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + path.push(format!("chanora-core-file-transfer-{name}-{nanos}")); + path + } + + #[tokio::test] + async fn returns_cached_avatar_without_connection() { + let cache_dir = test_cache_dir("cache-hit"); + let cache = BlobCache::new(&cache_dir, 1024).unwrap(); + cache.put(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", b"avatar") + .await + .unwrap(); + let service = FileTransferService::new(cache, Arc::new(Mutex::new(None))); + + let avatar = service + .get_avatar("a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", "client") + .await + .unwrap(); + + assert_eq!(avatar, Some(b"avatar".to_vec())); + let _ = std::fs::remove_dir_all(cache_dir); + } + + #[tokio::test] + async fn negative_cache_short_circuits_not_connected() { + let cache_dir = test_cache_dir("negative-cache"); + let cache = BlobCache::new(&cache_dir, 1024).unwrap(); + let service = FileTransferService::new(cache, Arc::new(Mutex::new(None))); + service + .negative_cache + .lock() + .await + .insert("a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6".to_string(), Instant::now() + NEGATIVE_CACHE_TTL); + + let avatar = service + .get_avatar("a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", "client") + .await + .unwrap(); + + assert_eq!(avatar, None); + let _ = std::fs::remove_dir_all(cache_dir); + } + + #[tokio::test] + async fn returns_cached_icon_without_connection() { + let cache_dir = test_cache_dir("icon-cache-hit"); + let cache = BlobCache::new(&cache_dir, 1024).unwrap(); + cache.put(PREFIX_ICON, "12345", b"icon").await.unwrap(); + let service = FileTransferService::new(cache, Arc::new(Mutex::new(None))); + + let icon = service.get_icon(12345).await.unwrap(); + + assert_eq!(icon, Some(b"icon".to_vec())); + let _ = std::fs::remove_dir_all(cache_dir); + } +} diff --git a/core/chanora_core/src/lib.rs b/core/chanora_core/src/lib.rs index 03a4e6e..088367b 100644 --- a/core/chanora_core/src/lib.rs +++ b/core/chanora_core/src/lib.rs @@ -53,6 +53,7 @@ use chanora_state::channel_join::{ }; mod events; +mod file_transfer; mod network_diagnostics; pub mod ptt; @@ -75,6 +76,7 @@ pub use events::{ NetworkState, PersistedPttBinding, PttDescriptorSnapshot, SessionEvent, VoiceJoinErrorCode, VoiceJoinSyncState, }; +pub use file_transfer::FileTransferError; use network_diagnostics::NetworkDiagnostics; /// Errors that can arise during top-level orchestration. @@ -92,6 +94,12 @@ pub enum CoreError { /// Storage error. #[error("storage: {0}")] Storage(#[from] chanora_storage::StorageError), + /// Blob-cache failure. + #[error("cache: {0}")] + Cache(#[from] chanora_cache::BlobCacheError), + /// File-transfer failure. + #[error("file transfer: {0}")] + FileTransfer(#[from] FileTransferError), /// Diagnostics error. #[error("diagnostics: {0}")] Diagnostics(#[from] chanora_diagnostics::DiagnosticsError), @@ -130,7 +138,6 @@ struct SupervisorInner { } struct ConnectedState { - protocol: chanora_protocol::ProtocolClient, audio: Option, /// Active PTT controller (SDD-088). Owns the platform input /// backend, the active binding, and the capability watch @@ -197,6 +204,8 @@ pub struct ChanoraSession { /// extension). Lives alongside the identity file. Wired by /// [`Self::init_storage`]. bookmark_store: Arc>>, + protocol: Arc>>, + file_transfer: Arc>>>, /// Invisible server-address prefetch cache. Warmed by Flutter typing /// but validated by Rust before Connect can reuse it. server_prefetch: ServerPrefetcher, @@ -252,6 +261,8 @@ impl ChanoraSession { network_tx, identity_store: Arc::new(Mutex::new(None)), bookmark_store: Arc::new(Mutex::new(None)), + protocol: Arc::new(Mutex::new(None)), + file_transfer: Arc::new(Mutex::new(None)), server_prefetch: ServerPrefetcher::new(), voice_selector: selector, release_tail, @@ -273,6 +284,17 @@ impl ChanoraSession { ConnectionEpoch(epoch) } + async fn store_protocol(&self, client: Option) { + // Always update the shared Arc. The FileTransferService holds + // the same Arc, so it sees the new client automatically — no + // separate set_protocol call needed. + *self.protocol.lock().await = client; + } + + async fn take_protocol(&self) -> Option { + self.protocol.lock().await.take() + } + /// Wire a directory-backed identity store. Called by the bridge /// during `bridge_init` once Flutter has resolved the platform /// app-private storage directory. Subsequent [`Self::connect`] @@ -337,6 +359,65 @@ impl ChanoraSession { Ok(()) } + /// Configure the blob cache root. + pub async fn init_cache(&self, dir: &str) -> Result<(), CoreError> { + let cache = chanora_cache::BlobCache::new(dir, 100 * 1024 * 1024)?; + cache.evict().await?; + let service = Arc::new(file_transfer::FileTransferService::new( + cache, + self.protocol.clone(), + )); + let mut guard = self.file_transfer.lock().await; + *guard = Some(service); + Ok(()) + } + + /// Resolve avatar bytes. + pub async fn get_avatar( + &self, + avatar_hash: &str, + client_uid: &str, + ) -> Result>, CoreError> { + let service = { self.file_transfer.lock().await.clone() }; + if let Some(service) = service { + return Ok(service.get_avatar(avatar_hash, client_uid).await?); + } + + let protocol = self.protocol.lock().await; + let client = protocol.as_ref().ok_or(CoreError::NotConnected)?; + Ok(Some(client.download_avatar(client_uid).await?)) + } + + /// Resolve icon bytes. + pub async fn get_icon(&self, icon_id: u64) -> Result>, CoreError> { + let service = { self.file_transfer.lock().await.clone() }; + if let Some(service) = service { + return Ok(service.get_icon(icon_id).await?); + } + + let protocol = self.protocol.lock().await; + let client = protocol.as_ref().ok_or(CoreError::NotConnected)?; + Ok(Some(client.download_icon(icon_id).await?)) + } + + /// Purge cached protocol-owned assets. + pub async fn clear_cache(&self) -> Result<(), CoreError> { + let service = { self.file_transfer.lock().await.clone() }; + if let Some(service) = service { + service.clear_cache().await?; + } + Ok(()) + } + + /// Report the configured blob-cache size. + pub async fn cache_size(&self) -> Result { + let service = { self.file_transfer.lock().await.clone() }; + match service { + Some(service) => Ok(service.cache_size().await?), + None => Ok(0), + } + } + /// List persisted bookmarks. Returns an empty list if the store /// has not been wired or has no entries. pub async fn list_bookmarks(&self) -> Result, CoreError> { @@ -514,6 +595,7 @@ impl ChanoraSession { let supervisor = tokio::spawn(supervisor_loop(SupervisorContext { state_arc: self.inner.clone(), + protocol: self.protocol.clone(), events_tx: self.events_tx.clone(), initial_cfg: cfg.clone(), initial_lost_rx: lost_rx, @@ -568,9 +650,9 @@ impl ChanoraSession { } spawn_event_forwarders(&client, &self.events_tx); + self.store_protocol(Some(client)).await; *guard = Some(ConnectedState { - protocol: client, audio: None, ptt_controller: None, cancel_tx: Some(cancel_tx), @@ -631,9 +713,13 @@ impl ChanoraSession { /// Return a fresh snapshot of the current server state. pub async fn snapshot(&self) -> Result { + let snap = { + let protocol = self.protocol.lock().await; + let client = protocol.as_ref().ok_or(CoreError::NotConnected)?; + client.snapshot().await? + }; let mut guard = self.inner.lock().await; let state = guard.as_mut().ok_or(CoreError::NotConnected)?; - let snap = state.protocol.snapshot().await?; let current_channel = self .find_own_in(&snap) .await @@ -661,9 +747,9 @@ impl ChanoraSession { /// Fetch richer profile and live connection details for one online client. pub async fn client_profile(&self, client_id: u64) -> Result { - let guard = self.inner.lock().await; - let state = guard.as_ref().ok_or(CoreError::NotConnected)?; - Ok(state.protocol.client_profile(client_id).await?) + let protocol = self.protocol.lock().await; + let client = protocol.as_ref().ok_or(CoreError::NotConnected)?; + Ok(client.client_profile(client_id).await?) } /// True if a connection is currently active. @@ -680,9 +766,9 @@ impl ChanoraSession { if !should_dispatch_text_message(&message, &target) { return Ok(()); } - let guard = self.inner.lock().await; - let state = guard.as_ref().ok_or(CoreError::NotConnected)?; - state.protocol.send_text_message(message, target).await?; + let protocol = self.protocol.lock().await; + let client = protocol.as_ref().ok_or(CoreError::NotConnected)?; + client.send_text_message(message, target).await?; Ok(()) } @@ -741,11 +827,16 @@ impl ChanoraSession { // session permanently unable to restart audio without a // reconnect (the user saw "voice_in already taken" on the // second channel switch). - let voice_out = state.protocol.voice_out(); - let voice_in = state - .protocol - .take_voice_in() - .ok_or(CoreError::Invariant("voice_in already taken"))?; + let (voice_out, voice_in) = { + let protocol = self.protocol.lock().await; + let client = protocol.as_ref().ok_or(CoreError::NotConnected)?; + ( + client.voice_out(), + client + .take_voice_in() + .ok_or(CoreError::Invariant("voice_in already taken"))?, + ) + }; let gate = AudioTransmitGate::new(cfg.ptt_initial); cfg.voice_activity_selector = Some(self.voice_selector.clone()); let new_engine = match chanora_audio::AudioEngine::start_with_gate( @@ -976,10 +1067,11 @@ impl ChanoraSession { let password_to_send = requested_password .clone() .or_else(|| state.channel_passwords.get(&channel_id).cloned()); - state - .protocol - .move_to_channel(channel_id, password_to_send) - .await?; + { + let protocol = self.protocol.lock().await; + let client = protocol.as_ref().ok_or(CoreError::NotConnected)?; + client.move_to_channel(channel_id, password_to_send).await?; + } if let Some(pw) = requested_password { state.channel_passwords.insert(channel_id, pw); } @@ -1004,7 +1096,11 @@ impl ChanoraSession { if let Some(muted) = output { state.local_output_muted = muted; } - state.protocol.set_muted(input, output).await?; + { + let protocol = self.protocol.lock().await; + let client = protocol.as_ref().ok_or(CoreError::NotConnected)?; + client.set_muted(input, output).await?; + } if let Some(muted) = output { if let Some(audio) = state.audio.as_ref() { audio.set_output_muted(muted); @@ -1327,11 +1423,12 @@ impl ChanoraSession { let password_to_send = requested_password .clone() .or_else(|| state.channel_passwords.get(&channel_id).cloned()); - if let Err(e) = state - .protocol - .queue_move_to_channel(channel_id, password_to_send) - .await - { + let move_result = { + let protocol = self.protocol.lock().await; + let client = protocol.as_ref().ok_or(CoreError::NotConnected)?; + client.queue_move_to_channel(channel_id, password_to_send).await + }; + if let Err(e) = move_result { // TS3 error 0x0302 = `channel_already_in`: we're already // in the target channel, so this is a no-op success. // Rolling `in_channel` back to false would break PTT @@ -1593,7 +1690,9 @@ impl ChanoraSession { audio.stop(); let _ = self.events_tx.send(SessionEvent::AudioStopped); } - state.protocol.disconnect().await; + if let Some(protocol) = self.take_protocol().await { + protocol.disconnect().await; + } // Wait for the supervisor to wind down so we don't race // a redial against the explicit disconnect. if let Some(handle) = state.supervisor.take() { @@ -1655,6 +1754,7 @@ async fn await_supervisor_shutdown(mut handle: JoinHandle<()>, timeout_duration: struct SupervisorContext { state_arc: Arc>>, + protocol: Arc>>, events_tx: broadcast::Sender, initial_cfg: ConnectConfig, initial_lost_rx: oneshot::Receiver, @@ -1789,6 +1889,7 @@ fn spawn_event_forwarders( async fn supervisor_loop(ctx: SupervisorContext) { let SupervisorContext { state_arc, + protocol, events_tx, initial_cfg, initial_lost_rx, @@ -2058,20 +2159,21 @@ async fn supervisor_loop(ctx: SupervisorContext) { // Reattach into the session state. let restart_audio = { + let guard = state_arc.lock().await; + if guard.is_none() { + // Session was disposed mid-reconnect. + return; + } + drop(guard); + let old = protocol.lock().await.replace(new_client); + drop(old); let mut guard = state_arc.lock().await; let state = match guard.as_mut() { Some(s) => s, None => { - // Session was disposed mid-reconnect. return; } }; - // Replace the dead protocol client with the new one. - // The old client's background task either already - // exited (loss notifier fired) or will exit when - // its request channel drops (watchdog path). - let old = std::mem::replace(&mut state.protocol, new_client); - drop(old); let _ = channel_join::reduce( &mut state.join_state, @@ -2101,9 +2203,9 @@ async fn supervisor_loop(ctx: SupervisorContext) { }); { - let guard = state_arc.lock().await; - if let Some(state) = guard.as_ref() { - spawn_event_forwarders(&state.protocol, &events_tx); + let protocol = protocol.lock().await; + if let Some(client) = protocol.as_ref() { + spawn_event_forwarders(client, &events_tx); } } @@ -2115,8 +2217,15 @@ async fn supervisor_loop(ctx: SupervisorContext) { }; let mut guard = state_arc.lock().await; if let Some(state) = guard.as_mut() { - let voice_out = state.protocol.voice_out(); - if let Some(voice_in) = state.protocol.take_voice_in() { + let (voice_out, voice_in) = { + let protocol = protocol.lock().await; + let client = match protocol.as_ref() { + Some(client) => client, + None => return, + }; + (client.voice_out(), client.take_voice_in()) + }; + if let Some(voice_in) = voice_in { let gate = chanora_audio::AudioTransmitGate::new( audio_cfg.ptt_initial, ); diff --git a/core/chanora_core/tests/avatar_cache.rs b/core/chanora_core/tests/avatar_cache.rs new file mode 100644 index 0000000..5a8b01e --- /dev/null +++ b/core/chanora_core/tests/avatar_cache.rs @@ -0,0 +1,50 @@ +use std::env; +use std::path::PathBuf; +use std::process; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[tokio::test] +async fn get_avatar_returns_cached_bytes_without_connection() { + let tmp = mktemp("chanora_core_avatar_cache_test"); + let session = chanora_core::ChanoraSession::new(); + session.init_cache(tmp.to_str().unwrap()).await.unwrap(); + + let cache = chanora_cache::BlobCache::new(&tmp, 100 * 1024 * 1024).unwrap(); + let hash = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6"; + cache + .put(chanora_cache::PREFIX_AVATAR, hash, b"avatar-bytes") + .await + .unwrap(); + + let bytes = session + .get_avatar(hash, "client-uid") + .await + .unwrap() + .unwrap(); + assert_eq!(bytes, b"avatar-bytes"); + + let _ = std::fs::remove_dir_all(&tmp); +} + +#[tokio::test] +async fn get_avatar_without_cache_or_connection_returns_not_connected() { + let session = chanora_core::ChanoraSession::new(); + let err = session + .get_avatar("a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", "client-uid") + .await + .unwrap_err(); + + assert!(matches!(err, chanora_core::CoreError::NotConnected)); +} + +fn mktemp(label: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let p = env::temp_dir() + .join(label) + .join(format!("{}-{nanos}", process::id())); + std::fs::create_dir_all(&p).unwrap(); + p +} diff --git a/crates/chanora_bridge/src/api.rs b/crates/chanora_bridge/src/api.rs index 6cde524..50f04fa 100644 --- a/crates/chanora_bridge/src/api.rs +++ b/crates/chanora_bridge/src/api.rs @@ -1465,6 +1465,54 @@ pub async fn init_storage(dir: String) -> Result<(), BridgeError> { Ok(()) } +/// Configure the bridge blob cache root. +pub async fn init_cache(dir: String) -> Result<(), BridgeError> { + runtime() + .spawn(async move { session().init_cache(&dir).await }) + .await + .map_err(|e| task_join_error("init_cache", e))??; + Ok(()) +} + +/// Resolve avatar bytes through the bridge. +pub async fn download_avatar( + avatar_hash: String, + client_uid: String, +) -> Result>, BridgeError> { + runtime() + .spawn(async move { session().get_avatar(&avatar_hash, &client_uid).await }) + .await + .map_err(|e| task_join_error("download_avatar", e))? + .map_err(BridgeError::from) +} + +/// Resolve icon bytes through the bridge. +pub async fn download_icon(icon_id: u64) -> Result>, BridgeError> { + runtime() + .spawn(async move { session().get_icon(icon_id).await }) + .await + .map_err(|e| task_join_error("download_icon", e))? + .map_err(BridgeError::from) +} + +/// Purge cached protocol-owned assets. +pub async fn clear_file_cache() -> Result<(), BridgeError> { + runtime() + .spawn(async move { session().clear_cache().await }) + .await + .map_err(|e| task_join_error("clear_file_cache", e))??; + Ok(()) +} + +/// Report the configured file-cache size. +pub async fn file_cache_size() -> Result { + runtime() + .spawn(async move { session().cache_size().await }) + .await + .map_err(|e| task_join_error("file_cache_size", e))? + .map_err(BridgeError::from) +} + /// Bookmark DTO mirroring [`chanora_core::Bookmark`]. #[derive(Debug, Clone)] pub struct BridgeBookmark { diff --git a/crates/chanora_bridge/src/lib.rs b/crates/chanora_bridge/src/lib.rs index 3d85857..e01c8af 100644 --- a/crates/chanora_bridge/src/lib.rs +++ b/crates/chanora_bridge/src/lib.rs @@ -110,9 +110,13 @@ impl From for BridgeError { code, message, }) => BridgeError::ServerRejected { code, message }, + chanora_core::CoreError::Protocol(chanora_core::ProtocolError::FileTransfer(p)) => { + BridgeError::Connection(format!("file transfer: {p}")) + } chanora_core::CoreError::Protocol(p) => BridgeError::Connection(format!("{p}")), chanora_core::CoreError::Audio(a) => BridgeError::Connection(format!("audio: {a}")), chanora_core::CoreError::Storage(s) => BridgeError::Connection(format!("storage: {s}")), + chanora_core::CoreError::Cache(c) => BridgeError::Connection(format!("cache: {c}")), other => BridgeError::Unmapped(format!("{other}")), } } diff --git a/crates/chanora_cache/Cargo.toml b/crates/chanora_cache/Cargo.toml new file mode 100644 index 0000000..d8abbc0 --- /dev/null +++ b/crates/chanora_cache/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "chanora_cache" +description = "Chanora disposable content-addressed blob cache for avatars and icons" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +publish.workspace = true + +[dependencies] +cacache = "13" +thiserror.workspace = true +tokio = { version = "1", features = ["fs", "rt"] } +tracing.workspace = true + +[dev-dependencies] +tempfile = "3" +tokio = { version = "1", features = ["rt", "macros", "time"] } diff --git a/crates/chanora_cache/src/lib.rs b/crates/chanora_cache/src/lib.rs new file mode 100644 index 0000000..15327b5 --- /dev/null +++ b/crates/chanora_cache/src/lib.rs @@ -0,0 +1,359 @@ +//! Disposable content-addressed blob cache for avatar and icon files. +//! +//! Wraps [`cacache`] for production-tested crash safety and integrity +//! verification. The on-disk layout is managed by cacache (content-v2, +//! index-v2). Chanora maps protocol keys (`av_`, `ic_`) to +//! cacache string keys. +//! +//! This crate is intentionally separate from `chanora_storage`: +//! storage owns persistent identity/bookmark data; cache owns +//! reconstructible, disposable blob data with different durability +//! and backup semantics. + +#![forbid(unsafe_code)] +#![warn(missing_docs)] + +use std::path::{Path, PathBuf}; + +/// Errors raised by the blob cache. +#[derive(Debug, thiserror::Error)] +pub enum BlobCacheError { + /// Filesystem I/O error. + #[error("io: {0}")] + Io(String), + /// Key validation error. + #[error("invalid key: {0}")] + InvalidKey(String), +} + +/// Content-addressed blob cache backed by cacache. +pub struct BlobCache { + cache_dir: PathBuf, + /// Maximum total cache size in bytes. 0 = no limit. + max_bytes: u64, +} + +/// Avatar blob prefix. +pub const PREFIX_AVATAR: &str = "av_"; +/// Icon blob prefix. +pub const PREFIX_ICON: &str = "ic_"; + +impl BlobCache { + /// Create or open a [`BlobCache`] rooted at `cache_dir/chanora/`. + /// + /// Creates the cacache directory. `max_bytes` sets the eviction + /// threshold; 0 means no automatic eviction. + pub fn new(cache_dir: impl AsRef, max_bytes: u64) -> Result { + let cache_dir = cache_dir.as_ref().join("chanora").join("blobs"); + // cacache creates the directory on first write, but we create + // it eagerly so total_size() works before any writes. + std::fs::create_dir_all(&cache_dir) + .map_err(|e| BlobCacheError::Io(format!("mkdir cache: {e}")))?; + Ok(Self { + cache_dir, + max_bytes, + }) + } + + /// Store a blob. `prefix` is [`PREFIX_AVATAR`] or [`PREFIX_ICON`]. + /// `key` is the content hash (MD5 hex for avatars, unsigned + /// decimal CRC32 for icons). + /// + /// Cacache handles dedup and integrity internally. + pub async fn put( + &self, + prefix: &str, + key: &str, + data: &[u8], + ) -> Result<(), BlobCacheError> { + validate_key(prefix, key)?; + let cache_key = format!("{prefix}{key}"); + cacache::write(&self.cache_dir, &cache_key, data) + .await + .map_err(|e| BlobCacheError::Io(format!("cacache write: {e}")))?; + Ok(()) + } + + /// Read a blob. Returns `None` if not cached. + /// + /// Cacache verifies SSRI integrity on every read. + pub async fn get(&self, prefix: &str, key: &str) -> Result>, BlobCacheError> { + validate_key(prefix, key)?; + let cache_key = format!("{prefix}{key}"); + match cacache::read(&self.cache_dir, &cache_key).await { + Ok(data) => Ok(Some(data)), + Err(cacache::Error::EntryNotFound(_, _)) => Ok(None), + Err(e) => { + // Integrity failure or I/O error — remove corrupt entry. + tracing::warn!( + target: "chanora_cache", + key = %cache_key, + error = %e, + "cache read failed; removing entry" + ); + let _ = cacache::remove(&self.cache_dir, &cache_key).await; + Ok(None) + } + } + } + + /// Delete a specific blob. + pub async fn remove(&self, prefix: &str, key: &str) -> Result<(), BlobCacheError> { + validate_key(prefix, key)?; + let cache_key = format!("{prefix}{key}"); + cacache::remove(&self.cache_dir, &cache_key) + .await + .map_err(|e| BlobCacheError::Io(format!("cacache remove: {e}")))?; + Ok(()) + } + + /// Delete all blobs. + pub async fn clear(&self) -> Result<(), BlobCacheError> { + let path = self.cache_dir.clone(); + tokio::task::spawn_blocking(move || { + if path.exists() { + std::fs::remove_dir_all(&path) + .map_err(|e| BlobCacheError::Io(format!("clear cache: {e}")))?; + std::fs::create_dir_all(&path) + .map_err(|e| BlobCacheError::Io(format!("recreate cache dir: {e}")))?; + } + Ok(()) + }) + .await + .map_err(|e| BlobCacheError::Io(format!("clear task: {e}")))? + } + + /// Return total bytes used by all blobs. + /// + /// Walks cacache entries and sums sizes. + pub async fn total_size(&self) -> Result { + let cache_dir = self.cache_dir.clone(); + tokio::task::spawn_blocking(move || { + let mut total: u64 = 0; + for entry in cacache::list_sync(&cache_dir) { + match entry { + Ok(meta) => total += meta.size as u64, + Err(e) => { + tracing::warn!( + target: "chanora_cache", + error = %e, + "skipping bad entry during size scan" + ); + } + } + } + Ok(total) + }) + .await + .map_err(|e| BlobCacheError::Io(format!("total_size task: {e}")))? + } + + /// Evict oldest entries by timestamp until total size is under + /// `max_bytes`. Call on startup or periodically. No-op if + /// `max_bytes` is 0. + pub async fn evict(&self) -> Result<(), BlobCacheError> { + if self.max_bytes == 0 { + return Ok(()); + } + let cache_dir = self.cache_dir.clone(); + let max_bytes = self.max_bytes; + tokio::task::spawn_blocking(move || { + let mut entries: Vec<(String, usize, u128)> = Vec::new(); + for entry in cacache::list_sync(&cache_dir) { + match entry { + Ok(meta) => { + entries.push((meta.key, meta.size, meta.time)); + } + Err(e) => { + tracing::warn!( + target: "chanora_cache", + error = %e, + "skipping bad entry during eviction scan" + ); + } + } + } + let total: usize = entries.iter().map(|(_, s, _)| *s).sum(); + if total as u64 <= max_bytes { + return Ok(()); + } + entries.sort_by_key(|(_, _, t)| *t); + let mut freed: usize = 0; + let target = total - max_bytes as usize; + for (key, size, _) in entries { + if freed >= target { + break; + } + let _ = cacache::remove_sync(&cache_dir, &key); + freed += size; + } + tracing::info!( + target: "chanora_cache", + freed_bytes = freed, + "evicted oldest blobs" + ); + Ok(()) + }) + .await + .map_err(|e| BlobCacheError::Io(format!("evict task: {e}")))? + } +} + +/// Validate key format to prevent malformed entries. +fn validate_key(prefix: &str, key: &str) -> Result<(), BlobCacheError> { + if !matches!(prefix, PREFIX_AVATAR | PREFIX_ICON) { + return Err(BlobCacheError::InvalidKey(format!("bad prefix: {prefix}"))); + } + match prefix { + PREFIX_AVATAR => { + // MD5 hex = exactly 32 hex chars. + if key.len() != 32 || !key.chars().all(|c| c.is_ascii_hexdigit()) { + return Err(BlobCacheError::InvalidKey(format!( + "avatar key must be 32 hex chars, got: {key}" + ))); + } + } + PREFIX_ICON => { + // Unsigned CRC32 = decimal digits. + if key.is_empty() || !key.chars().all(|c| c.is_ascii_digit()) { + return Err(BlobCacheError::InvalidKey(format!( + "icon key must be decimal digits, got: {key}" + ))); + } + } + _ => unreachable!(), + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tempdir() -> tempfile::TempDir { + tempfile::Builder::new() + .prefix("chanora_cache_test_") + .tempdir() + .unwrap() + } + + #[tokio::test] + async fn put_get_roundtrip() { + let tmp = tempdir(); + let cache = BlobCache::new(&tmp, 0).unwrap(); + assert!(cache + .get(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6") + .await + .unwrap() + .is_none()); + cache + .put(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", b"avatar-bytes") + .await + .unwrap(); + let data = cache + .get(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6") + .await + .unwrap(); + assert_eq!(data.as_deref(), Some(b"avatar-bytes".as_slice())); + } + + #[tokio::test] + async fn get_missing_returns_none() { + let tmp = tempdir(); + let cache = BlobCache::new(&tmp, 0).unwrap(); + assert!(cache + .get(PREFIX_AVATAR, "00000000000000000000000000000000") + .await + .unwrap() + .is_none()); + } + + #[tokio::test] + async fn clear_removes_all() { + let tmp = tempdir(); + let cache = BlobCache::new(&tmp, 0).unwrap(); + cache + .put(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", b"data") + .await + .unwrap(); + cache.put(PREFIX_ICON, "12345", b"icon").await.unwrap(); + cache.clear().await.unwrap(); + assert_eq!(cache.total_size().await.unwrap(), 0); + } + + #[tokio::test] + async fn total_size_accounts_for_all_entries() { + let tmp = tempdir(); + let cache = BlobCache::new(&tmp, 0).unwrap(); + cache + .put(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", b"12345") + .await + .unwrap(); + cache.put(PREFIX_ICON, "99999", b"12").await.unwrap(); + assert_eq!(cache.total_size().await.unwrap(), 5 + 2); + } + + #[tokio::test] + async fn invalid_key_rejected() { + let tmp = tempdir(); + let cache = BlobCache::new(&tmp, 0).unwrap(); + // Too short for MD5. + assert!(cache.put(PREFIX_AVATAR, "abc", b"data").await.is_err()); + // Non-hex in MD5. + assert!(cache + .put(PREFIX_AVATAR, "g".repeat(32).as_str(), b"data") + .await + .is_err()); + // Non-digit in icon key. + assert!(cache.put(PREFIX_ICON, "12a45", b"data").await.is_err()); + // Bad prefix. + assert!(cache.put("xx_", "abc", b"data").await.is_err()); + } + + #[tokio::test] + async fn evict_deletes_oldest_until_under_cap() { + let tmp = tempdir(); + // 10 byte cap. + let cache = BlobCache::new(&tmp, 10).unwrap(); + cache + .put( + PREFIX_AVATAR, + "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", + b"12345678", + ) + .await + .unwrap(); // 8 bytes + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + cache + .put(PREFIX_ICON, "11111", b"12345") + .await + .unwrap(); // 5 bytes → total 13, over cap + cache.evict().await.unwrap(); + // Oldest (avatar) should be evicted. + assert!(cache + .get(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6") + .await + .unwrap() + .is_none()); + assert!(cache.get(PREFIX_ICON, "11111").await.unwrap().is_some()); + } + + #[tokio::test] + async fn remove_deletes_entry() { + let tmp = tempdir(); + let cache = BlobCache::new(&tmp, 0).unwrap(); + cache + .put(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", b"data") + .await + .unwrap(); + cache + .remove(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6") + .await + .unwrap(); + assert!(cache + .get(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6") + .await + .unwrap() + .is_none()); + } +} diff --git a/crates/chanora_protocol/src/adapter.rs b/crates/chanora_protocol/src/adapter.rs index d5e9ee1..1663878 100644 --- a/crates/chanora_protocol/src/adapter.rs +++ b/crates/chanora_protocol/src/adapter.rs @@ -24,6 +24,7 @@ use base64::prelude::*; use chanora_resolver::ChanoraResolver; use futures::prelude::*; use std::collections::HashMap; +use tokio::io::AsyncReadExt; use tokio::sync::{mpsc, oneshot}; use tracing::{info, warn}; @@ -32,7 +33,8 @@ use tsclientlib::messages::s2c::{InClientDbInfoPart, InMessage}; use tsclientlib::prelude::*; use tsclientlib::{ ChannelId as TsChannelId, ClientId as TsClientId, Connection, ConnectionStats, - DisconnectOptions, Identity, MessageHandle, OutCommandExt, StreamItem, Version, + DisconnectOptions, FileDownloadResult, FiletransferHandle, Identity, MessageHandle, + OutCommandExt, StreamItem, Version, }; use tsproto_packets::packets::{Direction, Flags, InAudioBuf, OutCommand, OutPacket, PacketType}; use tsproto_types::ClientType; @@ -60,6 +62,9 @@ type PendingMoves = HashMap< ), >; +type PendingDownloads = + HashMap, ProtocolError>>>; + struct EventChannels { voice_in: mpsc::Sender, chat: mpsc::Sender, @@ -207,6 +212,10 @@ enum Request { client_id: u64, reply: oneshot::Sender>, }, + DownloadFile { + path: String, + reply: oneshot::Sender, ProtocolError>>, + }, } /// Why a [`ProtocolClient`] task ended. Distinguishes a user-driven @@ -366,6 +375,26 @@ impl ProtocolClient { .map_err(|_| ProtocolError::Lost("client_profile reply dropped".to_string()))? } + async fn download_file(&self, path: String) -> Result, ProtocolError> { + let (tx, rx) = oneshot::channel(); + self.tx + .send(Request::DownloadFile { path, reply: tx }) + .await + .map_err(|_| ProtocolError::Lost("connection task is gone".to_string()))?; + rx.await + .map_err(|_| ProtocolError::Lost("download_file reply dropped".to_string()))? + } + + /// Download the current avatar bytes for a TeamSpeak client UID. + pub async fn download_avatar(&self, client_uid: &str) -> Result, ProtocolError> { + self.download_file(avatar_download_path(client_uid)).await + } + + /// Download the current channel/server icon bytes for an icon id. + pub async fn download_icon(&self, icon_id: u64) -> Result, ProtocolError> { + self.download_file(icon_download_path(icon_id)).await + } + /// Disconnect cleanly. Blocks until the task exits. pub async fn disconnect(self) { let (tx, rx) = oneshot::channel(); @@ -713,6 +742,7 @@ async fn connection_task( // deadline so a server that never replies doesn't leak the // reply channel — at most 3 s of pending state per move. let mut pending_moves: PendingMoves = HashMap::new(); + let mut pending_downloads: PendingDownloads = HashMap::new(); let mut voice_activity: HashMap = HashMap::new(); let mut poke_limiter = PokeLimiter::new(); @@ -737,6 +767,12 @@ async fn connection_task( StreamItem::Audio(buf) => { handle_audio_stream_item(&channels.voice_in, &mut voice_activity, buf).await; } + StreamItem::FileDownload(handle, result) => { + handle_download_stream_item(&mut pending_downloads, handle, result).await; + } + StreamItem::FiletransferFailed(handle, error) => { + handle_download_failure(&mut pending_downloads, handle, error); + } other => handle_non_audio_stream_item( &con, other, @@ -843,12 +879,25 @@ async fn connection_task( client_id, &channels, &mut pending_moves, + &mut pending_downloads, &mut voice_activity, &mut poke_limiter, ) .await; let _ = reply.send(r); } + Ok(Request::DownloadFile { path, reply }) => { + match con.download_file(TsChannelId(0), &path, None, None) { + Ok(handle) => { + pending_downloads.insert(handle, reply); + } + Err(e) => { + let _ = reply.send(Err(ProtocolError::FileTransfer(format!( + "start download {path}: {e}" + )))); + } + } + } Ok(Request::Disconnect(reply)) => { let _ = con.disconnect(DisconnectOptions::new()); bounded_drain_stream(con.events(), DISCONNECT_EVENT_DRAIN_TIMEOUT).await; @@ -1024,6 +1073,47 @@ fn handle_non_audio_stream_item( } } +async fn handle_download_stream_item( + pending_downloads: &mut PendingDownloads, + handle: FiletransferHandle, + result: FileDownloadResult, +) { + if let Some(reply) = pending_downloads.remove(&handle) { + let _ = reply.send(read_download_bytes(result).await); + } +} + +fn handle_download_failure( + pending_downloads: &mut PendingDownloads, + handle: FiletransferHandle, + error: tsclientlib::Error, +) { + if let Some(reply) = pending_downloads.remove(&handle) { + let _ = reply.send(Err(ProtocolError::FileTransfer(error.to_string()))); + } +} + +const MAX_DOWNLOAD_SIZE: u64 = 10 * 1024 * 1024; + +async fn read_download_bytes(result: FileDownloadResult) -> Result, ProtocolError> { + if result.size > MAX_DOWNLOAD_SIZE { + return Err(ProtocolError::FileTransfer(format!( + "download too large: {} bytes (max {})", + result.size, MAX_DOWNLOAD_SIZE + ))); + } + let size = usize::try_from(result.size).map_err(|_| { + ProtocolError::FileTransfer(format!("download too large to buffer: {} bytes", result.size)) + })?; + let mut stream = result.stream; + let mut bytes = vec![0_u8; size]; + stream + .read_exact(&mut bytes) + .await + .map_err(|e| ProtocolError::FileTransfer(e.to_string()))?; + Ok(bytes) +} + async fn resolve_server_socket(address: &str) -> Result { let resolver = ChanoraResolver::new().map_err(|err| ProtocolError::DnsFailed { host: address.to_string(), @@ -1174,6 +1264,7 @@ async fn fetch_client_profile( client_id: u64, channels: &EventChannels, pending_moves: &mut PendingMoves, + pending_downloads: &mut PendingDownloads, voice_activity: &mut HashMap, poke_limiter: &mut PokeLimiter, ) -> Result { @@ -1220,6 +1311,7 @@ async fn fetch_client_profile( build_command("servergrouplist", &[], &[]), channels, pending_moves, + pending_downloads, voice_activity, poke_limiter, ) @@ -1231,6 +1323,7 @@ async fn fetch_client_profile( build_command("channelgrouplist", &[], &[]), channels, pending_moves, + pending_downloads, voice_activity, poke_limiter, ) @@ -1246,6 +1339,7 @@ async fn fetch_client_profile( ), channels, pending_moves, + pending_downloads, voice_activity, poke_limiter, ) @@ -1265,6 +1359,7 @@ async fn fetch_client_profile( build_command("getconnectioninfo", &[("clid", client_id.to_string())], &[]), channels, pending_moves, + pending_downloads, voice_activity, poke_limiter, ) @@ -1285,6 +1380,7 @@ async fn fetch_client_profile( database_id, channels, pending_moves, + pending_downloads, voice_activity, poke_limiter, ) @@ -1448,6 +1544,7 @@ async fn request_messages( command: OutCommand, channels: &EventChannels, pending_moves: &mut PendingMoves, + pending_downloads: &mut PendingDownloads, voice_activity: &mut HashMap, poke_limiter: &mut PokeLimiter, ) -> Result, ProtocolError> { @@ -1485,6 +1582,12 @@ async fn request_messages( StreamItem::Audio(buf) => { handle_audio_stream_item(&channels.voice_in, voice_activity, buf).await; } + StreamItem::FileDownload(handle, result) => { + handle_download_stream_item(pending_downloads, handle, result).await; + } + StreamItem::FiletransferFailed(handle, error) => { + handle_download_failure(pending_downloads, handle, error); + } other => handle_non_audio_stream_item( con, other, @@ -1503,6 +1606,7 @@ async fn request_client_db_info( dbid: tsclientlib::ClientDbId, channels: &EventChannels, pending_moves: &mut PendingMoves, + pending_downloads: &mut PendingDownloads, voice_activity: &mut HashMap, poke_limiter: &mut PokeLimiter, ) -> Result { @@ -1511,6 +1615,7 @@ async fn request_client_db_info( build_command("clientdbinfo", &[("cldbid", dbid.0.to_string())], &[]), channels, pending_moves, + pending_downloads, voice_activity, poke_limiter, ) @@ -1568,6 +1673,14 @@ fn uid_to_avatar_path(uid_b64: &str) -> String { rendered } +fn avatar_download_path(client_uid: &str) -> String { + format!("/avatar_{}", uid_to_avatar_path(client_uid)) +} + +fn icon_download_path(icon_id: u64) -> String { + format!("/icon_{icon_id}") +} + fn find_client_by_id<'a>( clients: impl IntoIterator, client_id: u64, @@ -1951,10 +2064,11 @@ const _: () = { #[cfg(test)] mod tests { use super::{ - bounded_drain_stream, client_profile_refresh_plan, drain_voice_packets_for_tick, - is_server_query_client_type, send_with_timeout, server_socket_from_config, - sort_channels_tree_by, std_duration_millis, ConnectConfig, ProtocolClient, Request, - SendTimeoutError, DISCONNECT_REPLY_TIMEOUT, + avatar_download_path, bounded_drain_stream, client_profile_refresh_plan, + drain_voice_packets_for_tick, icon_download_path, is_server_query_client_type, + send_with_timeout, server_socket_from_config, sort_channels_tree_by, + std_duration_millis, ConnectConfig, ProtocolClient, Request, SendTimeoutError, + DISCONNECT_REPLY_TIMEOUT, }; use futures::stream; use std::time::Duration; @@ -2052,6 +2166,16 @@ mod tests { assert!(plan.needs_channel_groups); } + #[test] + fn avatar_download_path_uses_uid_hex_encoding() { + assert_eq!(avatar_download_path("AQID"), "/avatar_abacad"); + } + + #[test] + fn icon_download_path_uses_unsigned_icon_id() { + assert_eq!(icon_download_path(42), "/icon_42"); + } + #[test] fn channel_sort_linked_list_under_one_parent() { // Server emits four root-level channels in arbitrary HashMap diff --git a/crates/chanora_protocol/src/lib.rs b/crates/chanora_protocol/src/lib.rs index 3d5d409..4d77ce6 100644 --- a/crates/chanora_protocol/src/lib.rs +++ b/crates/chanora_protocol/src/lib.rs @@ -115,4 +115,8 @@ pub enum ProtocolError { /// should never see this; if they do, it is a mapping bug here. #[error("protocol backend: {0}")] Backend(String), + + /// A file transfer failed while downloading protocol-owned assets. + #[error("file transfer failed: {0}")] + FileTransfer(String), } diff --git a/docs/architecture/file-transfer-design.md b/docs/architecture/file-transfer-design.md new file mode 100644 index 0000000..4184f44 --- /dev/null +++ b/docs/architecture/file-transfer-design.md @@ -0,0 +1,737 @@ +# File Transfer Design + +**Date:** 2026-06-10 +**Status:** Draft for review +**Scope:** Download files from TeamSpeak-compatible servers via the native client protocol, starting with avatars and icons. +**Direct upstream source:** `docs/architecture/sad.md` (SAD-067, SDD-MOD-009) + +## 1. Goal + +Chanora needs to download files stored on TeamSpeak-compatible servers. The most visible use cases are client avatars and server/channel/client icons. The file transfer mechanism is also used for channel file browser features, but this document scopes the initial design to avatar and icon retrieval only. + +This document describes: + +- How the TeamSpeak file transfer protocol works. +- How `tsclientlib` exposes it. +- How Chanora should integrate it following the existing protocol adapter pattern. +- How the result flows through the bridge to the Flutter UI layer. + +Upload, channel file browsing, and file deletion are explicitly out of scope for the initial implementation. + +## 2. Protocol Background + +### 2.1 Two-Phase Transfer + +TeamSpeak file transfer is a two-phase process: + +1. **Command phase** — The client sends a command over the main encrypted UDP connection to request a transfer token (`ftkey`). +2. **Transfer phase** — The client opens a separate TCP connection to the server's file transfer port (default `30033`) and sends the `ftkey` to authenticate the transfer. Raw bytes flow over this TCP stream. + +### 2.2 Relevant ServerQuery Commands + +| Command | Direction | Purpose | +|---|---|---| +| `ftinitdownload` | Client → Server | Initialize a download. Returns `ftkey`, `port`, `size`. | +| `ftgetfileinfo` | Client → Server | Get metadata for one or more files. | +| `ftgetfilelist` | Client → Server | List files in a channel's file repository. | +| `ftinitupload` | Client → Server | Initialize an upload. | +| `ftlist` | Client → Server | List active file transfers. | +| `ftstop` | Client → Server | Stop a running transfer. | +| `ftdeletefile` | Client → Server | Delete a file. | +| `ftcreatedir` | Client → Server | Create a directory. | +| `ftrenamefile` | Client → Server | Rename or move a file. | + +Initial scope uses only `ftinitdownload` and `ftgetfileinfo`. + +### 2.3 File Paths + +Files are addressed by a path scoped to a channel ID (`cid`): + +- `cid=0` — Server-level file repository. Avatars and icons live here. +- `cid=N` (non-zero) — Channel-specific file repository. + +Avatar path: `/avatar_` where `` is derived from the client's unique identifier (UID). Each byte of the base64-decoded UID is split into two nibbles, and each nibble maps to a letter `a` through `p` (0→a, 1→b, ..., 15→p). + +Icon path: `/icon_` where `` is the icon's signed 64-bit integer ID. If negative, treat as unsigned for the path. + +### 2.4 `ftinitdownload` Command + +``` +ftinitdownload clientftfid={id} name={path} cid={channelId} cpw={password} seekpos={seek} proto=0 +``` + +Parameters: + +| Parameter | Type | Description | +|---|---|---| +| `clientftfid` | `u16` | Arbitrary client-side transfer ID. | +| `name` | `string` | File path, e.g. `/avatar_abcdef`. | +| `cid` | `ChannelId` | Channel scope (0 = server). | +| `cpw` | `string` | Channel password. Empty for server-level. | +| `seekpos` | `u64` | Resume offset. 0 for a fresh download. | +| `proto` | `u8` | Protocol version. Always 0. | + +Server response: + +| Field | Type | Description | +|---|---|---| +| `clientftfid` | `u16` | Echo of the client transfer ID. | +| `serverftfid` | `u16` | Server-side transfer ID. | +| `ftkey` | `string` | One-time transfer key (hex). | +| `port` | `u16` | File transfer TCP port (usually 30033). | +| `size` | `u64` | File size in bytes. | +| `proto` | `u8` | Protocol version echo. | +| `ip` | `string` (optional) | Override IP for the TCP connection. | + +### 2.5 TCP Transfer + +After receiving the `ftkey`, the client: + +1. Opens a TCP connection to `server_ip:port`. +2. Sends `ftkey` followed by a newline. +3. Reads exactly `size` bytes of raw file data. +4. Closes the TCP connection. + +### 2.6 Permissions + +File transfer requires the following permissions on the server: + +| Permission | Needed for | +|---|---| +| `i_ft_file_download_power` | Downloading files. | +| `i_ft_needed_file_download_power` | Required download power on the channel/server. | +| `b_ft_ignore_password` | Bypassing channel passwords (not needed for avatars). | + +Avatar downloads typically require only basic download power because avatars are in the server-level repository (`cid=0`), which is generally accessible. + +### 2.7 Avatar Detection + +When a client connects or updates, the server sends `client_flag_avatar` as a string (the avatar hash). If non-empty, the client has an avatar. The avatar is downloaded from `/avatar_` where `` is computed from the client's UID (not from the hash string itself — the hash is just a presence indicator). + +## 3. tsclientlib Support + +`tsclientlib` implements file transfer natively. The library handles the entire command + TCP flow internally: + +### 3.1 Public API + +```rust +// tsclientlib/src/lib.rs (relevant signatures) +impl Connection { + pub fn download_file( + &mut self, + channel_id: ChannelId, + path: &str, + channel_password: Option<&str>, + seek_position: Option, + ) -> Result; + + pub fn upload_file( + &mut self, + channel_id: ChannelId, + path: &str, + channel_password: Option<&str>, + size: u64, + overwrite: bool, + resume: bool, + ) -> Result; +} +``` + +`download_file` sends the `ftinitdownload` command and returns a `FiletransferHandle(u16)` immediately. The actual transfer completes asynchronously. + +### 3.2 Stream Items + +The connection's event stream emits: + +| StreamItem | When | Data | +|---|---|---| +| `StreamItem::FileDownload(FileDownloadResult)` | Server responds with `ftkey`; TCP connected and `ftkey` written | `{ size: u64, stream: TcpStream }` | +| `StreamItem::FileUpload(FileUploadResult)` | Upload ready | `{ seek_position: u64, stream: TcpStream }` | +| `StreamItem::FiletransferFailed(FiletransferHandle, Error)` | Transfer failed | Handle + error | + +When `FileDownload` fires, tsclientlib has already: + +1. Sent `ftinitdownload` over the encrypted UDP command channel. +2. Received the `ftkey`, `port`, and `size` from the server. +3. Opened a TCP connection to `server:port`. +4. Written the `ftkey` to the TCP socket. + +The `TcpStream` in `FileDownloadResult` is ready to read; Chanora only needs to read exactly `size` bytes. + +### 3.3 Avatar Helper + +`tsproto-types` provides `Uid::as_avatar()` which computes the avatar filename from a UID. Chanora's existing `uid_to_avatar_path()` in `adapter.rs` does the same thing independently. + +### 3.4 Doc-Comment Examples + +tsclientlib's source contains usage examples in doc comments: + +```rust +/// Download an icon: +/// con.download_file(ChannelId(0), &format!("/icon_{}", icon_id), None, None) + +/// Upload an avatar: +/// con.upload_file(ChannelId(0), "/avatar", None, data.len() as u64, true, false) +``` + +## 4. Architecture Integration + +### 4.1 Existing Pattern + +The protocol adapter (`crates/chanora_protocol`) uses a single tokio task that owns the `tsclientlib::Connection`. All operations follow this pattern: + +1. Define a `Request` enum variant with parameters and a `oneshot::Sender` for the reply. +2. Send the request through the `mpsc` channel to the connection task. +3. The connection task calls tsclientlib and resolves the oneshot. + +File transfer fits this pattern exactly. The only difference is that the result arrives asynchronously via `StreamItem::FileDownload` rather than immediately from the command call. + +### 4.2 Design + +The file transfer integration adds: + +1. **`Request` variants** for file download. +2. **A pending-downloads map** (`HashMap`) in the connection task, mirroring the existing `pending_moves` pattern. +3. **`StreamItem::FileDownload` and `StreamItem::FiletransferFailed`** handling in the event loop. +4. **New DTOs** for file transfer results. +5. **Convenience methods** on `ProtocolClient` for avatar and icon downloads. + +### 4.3 Layer Responsibilities + +| Layer | Responsibility | +|---|---| +| `chanora_protocol` | Call `tsclientlib::download_file`, track pending transfers, read `TcpStream`, return bytes. No tsclientlib types leak. | +| `chanora_core` | Orchestrate when to download (e.g., on profile fetch or on avatar cache miss). | +| `chanora_bridge` | Expose typed `download_avatar` / `download_icon` commands to Flutter. | +| Flutter UI | Call bridge, display with `Image.memory()`. Cache in memory/image cache. | + +### 4.4 Error Mapping + +File transfer errors map to the existing `ProtocolError` variants: + +| tsclientlib error | ProtocolError | +|---|---| +| Permission denied (TS3 error code) | `ServerRejected { code, message }` | +| File not found | `ServerRejected { code, message }` | +| Network/TCP failure | `Backend(String)` | +| Timeout | `Timeout` | +| Connection lost mid-transfer | `Lost(String)` | + +## 5. Detailed Design + +### 5.1 New Types in `dto.rs` + +```rust +/// A downloaded file's raw content and metadata. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DownloadedFile { + /// Raw file bytes. + pub data: Vec, + /// The server path that was requested. + pub path: String, + /// Channel ID the file was downloaded from. + pub channel_id: u64, +} +``` + +### 5.2 New Request Variants in `adapter.rs` + +```rust +enum Request { + // ... existing variants ... + + /// Download a file from the server's file repository. + DownloadFile { + /// Channel ID. 0 for server-level (avatars, icons). + channel_id: u64, + /// File path, e.g. "/avatar_abcdef" or "/icon_12345". + path: String, + /// Channel password. None for server-level files. + channel_password: Option, + /// Reply channel for the result. + reply: oneshot::Sender>, + }, +} +``` + +### 5.3 Pending Downloads Map + +```rust +type PendingDownloads = HashMap; + +struct PendingDownload { + path: String, + channel_id: u64, + reply: oneshot::Sender>, +} +``` + +### 5.4 Event Loop Handling + +In the connection task's main loop, add handling for file transfer stream items: + +```rust +// In handle_non_audio_stream_item or in the main loop: +StreamItem::FileDownload(result) => { + // result: FileDownloadResult { size, stream } + // Look up the handle in pending_downloads + // Use tokio::io::AsyncReadExt::read_exact to read 'size' bytes + // Resolve the oneshot with DownloadedFile +} +StreamItem::FiletransferFailed(handle, error) => { + // Look up the handle in pending_downloads + // Resolve the oneshot with ProtocolError::Backend +} +``` + +The TCP read from the `TcpStream` is an async operation. Since the connection task already runs in a tokio context, the read can be done inline. However, for large files this would block the main event loop. Two approaches: + +**Option A: Read inline (simple, good for small files like avatars)** + +Avatars are typically under 100 KB. Reading them inline in the event loop is acceptable and avoids complexity. + +**Option B: Spawn a reader task** + +For future channel-file-browser support with potentially large files, spawn a separate tokio task that reads the stream and sends the result back. + +**Recommendation:** Start with Option A. The initial scope is avatars and icons (small files). Refactor to Option B when channel file browsing is implemented. + +### 5.5 Request Handling + +When the connection task receives `Request::DownloadFile`: + +```rust +Ok(Request::DownloadFile { channel_id, path, channel_password, reply }) => { + let ts_channel_id = TsChannelId(channel_id); + match con.download_file(ts_channel_id, &path, channel_password.as_deref(), None) { + Ok(handle) => { + pending_downloads.insert(handle, PendingDownload { + path, + channel_id, + reply, + }); + } + Err(e) => { + let _ = reply.send(Err(ProtocolError::Backend( + format!("download_file init: {e}") + ))); + } + } +} +``` + +### 5.6 Public API on `ProtocolClient` + +```rust +impl ProtocolClient { + /// Download a file from the server's file repository. + /// `channel_id` 0 means server-level (avatars, icons). + pub async fn download_file( + &self, + channel_id: u64, + path: String, + channel_password: Option, + ) -> Result { + let (tx, rx) = oneshot::channel(); + self.tx + .send(Request::DownloadFile { channel_id, path, channel_password, reply: tx }) + .await + .map_err(|_| ProtocolError::Lost("connection task is gone".to_string()))?; + rx.await + .map_err(|_| ProtocolError::Lost("download_file reply dropped".to_string()))? + } + + /// Download a client's avatar image. Returns raw image bytes. + /// Pass the `avatar_path` from `ClientProfile`. + pub async fn download_avatar( + &self, + avatar_path: String, + ) -> Result { + self.download_file(0, avatar_path, None).await + } + + /// Download a server, channel, or client icon by its icon ID. + pub async fn download_icon( + &self, + icon_id: i64, + ) -> Result { + let unsigned_id = icon_id as u64; + let path = format!("/icon_{}", unsigned_id); + self.download_file(0, path, None).await + } +} +``` + +### 5.7 Exports in `lib.rs` + +```rust +pub use dto::DownloadedFile; +``` + +### 5.8 Bridge Layer + +In `crates/chanora_bridge/src/api.rs`, add: + +```rust +pub async fn download_avatar(&self, avatar_path: String) -> Result, BridgeError> { + self.protocol + .download_avatar(avatar_path) + .await + .map(|file| file.data) + .map_err(BridgeError::Protocol) +} +``` + +### 5.9 Flutter Integration + +Flutter side: + +1. Call `clientProfile()` to get `ClientProfile` (already exists). +2. Check if `avatarPath` is non-empty. +3. Call bridge `downloadAvatar(avatarPath)` to get `Uint8List`. +4. Display with `Image.memory(bytes)`. + +Caching strategy: + +- In-memory: Use Flutter's standard `ImageCache` or a simple `Map` keyed by avatar path. +- Disk: Consider caching to local storage for offline display. This is a follow-up decision, not MVP scope. +- The avatar path already encodes the UID, so it can serve as a cache key. + +## 6. Avatar Path Computation + +Chanora already has this implemented in `adapter.rs`: + +```rust +fn uid_to_avatar_path(uid_b64: &str) -> String { + let decoded = BASE64_STANDARD.decode(uid_b64).unwrap_or_default(); + let mut rendered = String::with_capacity(decoded.len() * 2); + for byte in decoded { + rendered.push((b'a' + (byte >> 4)) as char); + rendered.push((b'a' + (byte & 0x0f)) as char); + } + rendered +} +``` + +This maps each nibble to `a` through `p` (0→a, 1→b, ..., 15→p), matching the canonical TeamSpeak implementation. + +The full avatar path is constructed as: + +```rust +let avatar_path = if client.avatar_hash.is_empty() || unique_id.is_empty() { + String::new() +} else { + format!("/avatar_{}", uid_to_avatar_path(&unique_id)) +}; +``` + +This is already correct and used in `ClientProfile.avatar_path`. No changes needed. + +## 7. Threading and Concurrency + +| Concern | Design | +|---|---| +| TCP read blocking the event loop | For avatar/icon sizes (< 100 KB typically), inline async read is acceptable. Spawn a reader task for larger files when channel file browsing is added. | +| Multiple concurrent downloads | `pending_downloads` is a HashMap keyed by `FiletransferHandle`. Multiple downloads can be in flight simultaneously. tsclientlib assigns unique handles. | +| Download timeout | Add a deadline to pending downloads (e.g., 30 seconds). Sweep expired entries similar to the existing `pending_moves` sweep. | +| Cancellation on disconnect | When the connection task exits, all pending oneshot senders are dropped, which resolves the caller's await with a `RecvError`. The caller maps this to `ProtocolError::Lost`. | + +## 8. Diagnostic and Security Considerations + +### 8.1 Diagnostic Redaction + +- File transfer paths may contain user-identifying information (UID-derived avatar names). These should be registered for diagnostic redaction if they appear in log output. +- File contents (avatar images) must not appear in log output or diagnostic exports. + +### 8.2 Security + +- The `ftkey` is a one-time token and must not be logged. +- TCP file transfer connections are not encrypted. This is a TeamSpeak protocol limitation, not a Chanora design choice. Avatar data is public (visible to anyone on the server), so the risk is acceptable. +- File download does not require secrets beyond the existing authenticated connection. + +### 8.3 Privacy + +- Avatar downloads reveal to the server that the user is viewing a specific client's avatar. This is inherent in the protocol. +- Chanora should not download avatars proactively for all clients. Download only when the UI needs to display a specific avatar (lazy/on-demand). + +## 9. Out of Scope + +The following are explicitly deferred: + +- File upload (avatar upload, channel file upload). +- Channel file browser (listing, creating directories, deleting, renaming). +- Resumable downloads (seek position > 0). +- File transfer progress reporting. +- myTeamSpeak avatar resolution (the `client_myteamspeak_avatar` field). +- In-memory hot cache in Rust (Flutter's `ImageCache` handles decoded image caching; add Rust-side layer only if profiling shows need). +- Upload, file browser, and channel file management. + +## 10. Cache Architecture + +### 10.1 Layer Ownership + +| Layer | Responsibility | Storage | +|---|---|---| +| `chanora_protocol` | Download raw bytes from server. No caching logic. | None | +| `chanora_cache` | Content-addressed blob store backed by `cacache`: crash-safe writes, SSRI integrity verification, key validation, eviction, clear. Separate crate from `chanora_storage`. | Platform cache directory | +| `chanora_core` | Session-aware cache orchestration: check freshness, coalesce requests, rate-limit downloads, persist to disk via `chanora_cache`. | Delegates to `chanora_cache` | +| `chanora_bridge` | Expose typed `download_avatar` / `clear_file_cache` / `file_cache_size` to Flutter. | None | +| Flutter | Display via `Image.memory`. Standard `ImageCache` for hot memory caching. Evict from `ImageCache` when hash changes. | In-memory only | + +### 10.2 Why Separate `chanora_cache` Crate + +`chanora_cache` is a separate crate from `chanora_storage` for three reasons: + +1. **Different durability semantics.** `chanora_storage` holds identity, bookmarks, and connection profiles — data the user explicitly created. `chanora_cache` holds downloaded blobs that are fully reconstructible from the server. Losing the cache is an inconvenience, not data loss. +2. **Different backup semantics.** Cache should be excluded from backups; persistent storage should be included. Platform conventions (iOS `Library/Caches/` vs `Library/Application Support/`) reflect this distinction. +3. **Different directory placement.** Cache lives in the platform's cache directory (OS may evict under storage pressure on mobile). Persistent storage lives in the support directory. + +The cache wraps the `cacache` crate for production-tested crash safety and integrity verification. It does not share `chanora_storage`'s crate or directory, and does not reimplement cacache's atomic write or content-addressing logic. + +### 10.3 Why Hybrid (Rust Disk + Flutter Memory) + +- Flutter's built-in `ImageCache` is an LRU in-memory cache (default 1000 images / 100 MiB). It handles hot display caching automatically when you use `MemoryImage`. +- Flutter has no built-in disk cache. `cached_network_image` / `flutter_cache_manager` are designed for HTTP URLs, not custom binary protocol data. +- Rust already owns the protocol, the connection state, and the anti-flood budget. Putting disk cache here avoids a feedback loop across the bridge. + +### 10.4 Cache Storage + +`chanora_cache` wraps the `cacache` crate for its on-disk storage. The physical layout is managed by `cacache`: + +``` +/chanora/ + blobs/ ← cacache content store root + content-v2/ ← content-addressed by SHA-512 + / + data ← raw blob bytes + tmp/ ← temp files (in-flight writes) + index-v2/ ← entry index (key → content mapping) +``` + +Chanora's `BlobCache` maps protocol keys to `cacache` string keys: + +| Protocol key | cacache key | Example | +|---|---|---| +| Avatar MD5 | `"av_"` | `"av_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6"` | +| Icon CRC32 | `"ic_"` | `"ic_123456789"` | + +**Why `cacache`:** + +1. **Crash safety.** Production-tested atomic writes (temp → rename). Handles partial writes, power loss, crash mid-write. No custom crash safety code to maintain. +2. **Integrity verification.** SSRI integrity check on every `read()`. Detects corruption, bit rot, partial writes automatically. Better than custom "delete on read failure". +3. **Content dedup.** Same bytes stored once regardless of key. Same avatar on two servers = stored once automatically. +4. **Less code to maintain.** ~120 LOC wrapper vs ~200 LOC custom implementation. Crash safety and integrity are the hard parts — `cacache` owns them. + +**Why no `` subdirectory:** The protocol uses content-addressed identifiers. `client_flag_avatar` is the MD5 of the avatar bytes — a given avatar hash always maps to the same bytes regardless of which server the user is on. Same avatar on two servers = same content = stored once by `cacache`. This is a deliberate dedup advantage over per-server namespacing. + +**Why no metadata sidecars:** Content is immutable (a given hash always maps to the same bytes). `cacache` manages its own entry index with timestamps. No custom metadata files needed. + +**Key validation rules:** + +| Prefix | Key format | Validation | +|---|---|---| +| `av_` | `av_<32 hex chars>` | MD5 is exactly 32 hex characters | +| `ic_` | `ic_<1-10 digit number>` | CRC32 unsigned, 0–4294967295 | + +Keys failing validation are rejected at the `BlobCache` API boundary. This prevents path traversal or malformed filenames on disk. + +**Platform paths** (Flutter passes the base directory into Rust at startup, matching the existing `initStorage` pattern): + +| Platform | Cache directory | +|---|---| +| Android | `context.cacheDir/chanora/` (via `getCacheDir()`) | +| iOS | `Library/Caches/chanora/` (via `getApplicationCacheDirectory()`) | +| macOS | `~/Library/Caches/chanora/` | +| Windows | `%LOCALAPPDATA%/chanora/cache/` | +| Linux | `$XDG_CACHE_HOME/chanora/` or `~/.cache/chanora/` | + +Flutter already resolves platform-specific paths. The same `getApplicationCacheDirectory()` call that is available in `path_provider` across all Chanora target platforms should be used. This follows the existing pattern where `app_bootstrap.dart` calls `getApplicationSupportDirectory()` for persistent storage; avatar/icon cache uses the cache-equivalent directory instead. + +The cache init call is separate from storage init: + +```rust +// Bridge init (Flutter calls these at startup) +pub fn init_storage(support_dir: String) -> Result<(), BridgeError>; // existing +pub fn init_cache(cache_dir: String) -> Result<(), BridgeError>; // new +``` + +### 10.5 Cache Freshness Strategy + +The `client_flag_avatar` field on each client is the authoritative freshness signal: + +``` +On connect / on client list update: + For each visible client with non-empty avatar_hash: + cache_key = "av_" + if cacache entry exists for "av_": + use cached file (zero downloads) + else: + enqueue download for avatar_path with expected hash = avatar_hash + +On avatar_hash change for a client: + The new hash produces a different cacache key. + The old entry remains until eviction or manual clear. + The new file is downloaded on demand. +``` + +This means: + +- **First connect:** No cache hits. Downloads happen lazily as the UI requests avatars. +- **Reconnect to same server:** All avatars hit cache instantly (hashes match keys). Zero downloads. +- **User changes avatar:** New hash = new cache key. Old entry becomes orphan. New file downloads on next UI request. +- **Same user on different server:** Same avatar hash = same cached file. Cross-server dedup for free. + +### 10.6 Anti-Flood and Download Timing + +TeamSpeak servers enforce anti-flood rate limiting. Downloading all avatars eagerly on connect would trigger it on servers with many users. + +**Strategy: lazy + throttled prefetch** + +| Phase | What | Rate | +|---|---|---| +| Connect settle (first 2-5 s) | Do nothing. Let the initial state snapshot and channel tree arrive. | — | +| After settle | UI requests avatars for visible clients in the current channel. These trigger downloads one at a time. | Max 1-2 concurrent downloads per server | +| Channel switch | UI requests avatars for newly visible clients. | Same throttle | +| Background prefetch (optional, future) | Low-priority downloads for clients in adjacent channels. | 1 request per 500 ms | + +**Anti-flood handling:** + +- If the server responds with an anti-flood error (TS3 error code `0x0701` = `client_could_not_be_banned` / flood-related), back off the download queue. +- Implement a simple semaphore in `chanora_core`: max 1-2 concurrent downloads. +- If a download gets a flood error, pause the queue for 5 seconds, then resume at reduced rate. + +### 10.7 Retry on Failure + +| Failure type | Strategy | +|---|---| +| Transient (network timeout, TCP reset) | Retry with exponential backoff: 5 s, 30 s, 2 min, 10 min. Cap at 10 min. | +| Server flood limit hit | Pause queue 5 s, then resume at reduced rate. Do not count as a per-file retry. | +| Permission denied (no download power) | Do not retry. Record negative cache entry. Only retry if hash changes. | +| File not found (avatar removed) | Do not retry. Record negative cache entry. Clear when hash changes or becomes empty. | +| Connection lost | All pending downloads fail. On reconnect, cache check runs fresh with current hashes. | + +**Negative cache:** In-memory `HashMap` with 5-minute TTL. Keys like `"av_"` or `"ic_"` that received permanent errors are stored with an expiry. On lookup, expired entries are treated as absent. Cleared entirely on reconnect. + +### 10.8 Request Coalescing + +Multiple UI widgets may request the same avatar simultaneously (e.g., channel list + chat view + client info sheet). + +**Pattern:** In `chanora_core`'s `FileTransferService`, maintain an in-flight map: + +```rust +HashMap, FileTransferError>>> +``` + +- First request: start download, store handle. +- Subsequent requests for same key: await the same handle. +- When handle completes: write to cache, wake all waiters, remove from map. + +### 10.9 Cache Eviction and Size Limits + +**MVP approach:** + +- No automatic size-based eviction in MVP. Avatars are small (typically 10-100 KB). Even 1000 avatars = ~50-100 MB. +- Rely on platform cache directory semantics (OS may evict under storage pressure on mobile). +- Old hash files accumulate but are harmless. + +**Post-MVP:** + +- `BlobCache::evict(max_bytes)` — walk `cacache::ls()` entries, sort by timestamp (oldest first), delete until total size < `max_bytes`. `cacache` manages timestamps internally. No metadata sidecars needed. +- Or simpler: `BlobCache::evict_older_than(duration)` — delete entries with timestamp older than N days. +- Call on startup and periodically (e.g., every 24 hours or on app resume). + +### 10.10 User-Initiated Cache Clear + +Add a bridge method: + +```rust +pub fn clear_file_cache(&self) -> Result<(), BridgeError> { + // Delete the entire blobs/ directory contents + // Flutter evicts all avatar/icon-related entries from ImageCache +} + +pub fn file_cache_size(&self) -> Result { + // Walk blobs/ and sum file sizes +} +``` + +Flutter side: + +```dart +// In settings or storage management UI: +onPressed: () async { + await api.clearFileCache(); + PaintingBinding.instance.imageCache.clear(); +} +``` + +This should be exposed in the app's settings UI under a "Clear cache" or "Storage management" section. + +### 10.11 Storage Clear Across Servers + +Since the cache is flat with content-addressed keys (no server namespacing): + +- Connecting to a different server does not conflict — same avatar hash = same file. +- Avatars unique to the old server remain cached. If a user on the new server has the same avatar (same hash), it hits cache instantly (cross-server dedup). +- Cache clear removes all cached data regardless of which server it came from. + +### 10.12 Flutter Display Strategy + +**Option A: Bytes across bridge (simpler, recommended for MVP)** + +Rust returns `Vec` across the bridge. Flutter uses `Image.memory(bytes)`. + +```dart +final bytes = await api.downloadAvatar(clientUid: uid); +if (bytes != null && bytes.isNotEmpty) { + return Image.memory(Uint8List.fromList(bytes)); +} else { + return CircleAvatar(child: Text(initials)); // fallback +} +``` + +Flutter's `ImageCache` caches the decoded image in memory automatically. Same avatar bytes = cache hit in memory. + +**Option B: File path across bridge (better for large images, future)** + +Rust writes to disk and returns the file path. Flutter uses `FileImage`. + +```dart +final path = await api.getAvatarPath(avatarHash: hash); +if (path != null) { + return Image.file(File(path)); +} else { + return CircleAvatar(child: Text(initials)); +} +``` + +`FileImage` does not watch for file changes. When the hash changes, the UI must evict the old entry from `ImageCache` using `PaintingBinding.instance.imageCache.evict(key)`. + +**Recommendation:** Start with Option A for MVP. It avoids file-path cross-platform complications and works well for small avatar files. The bridge already returns `Vec` for the download result. + +## 11. Implementation Sequence + +| Phase | Scope | What | +|---|---|---| +| Phase 1 | Protocol download | `Request::DownloadFile`, `StreamItem::FileDownload` handling, `ProtocolClient::download_avatar()` / `download_icon()`. No caching. | +| Phase 2 | Bridge + Flutter display | Bridge `downloadAvatar()`, Flutter `Image.memory()`, initials fallback. Still no caching — every view re-downloads. | +| Phase 3 | Rust disk cache | New `chanora_cache` crate: `cacache`-backed content-addressed blob store (`BlobCache`), key validation, mtime-based eviction, `init_cache` bridge call. | +| Phase 4 | Session orchestration | `chanora_core` `FileTransferService`: request coalescing, rate limiter (semaphore), negative cache (5 min TTL), retry backoff. | +| Phase 5 | Cache management | Bridge `clearFileCache()` + `fileCacheSize()`, Flutter settings UI, eviction on startup. | + +Phase 1 and 2 deliver visible value (avatars in the UI). Phase 3-5 add robustness. + +## 12. References + +| Reference | Use | +|---|---| +| `ReSpeak/tsdeclarations` `Messages.toml` lines 828-830 | `ftinitdownload` command declaration | +| `ReSpeak/tsdeclarations` `Messages.toml` lines 590 | `FileDownload` response structure | +| `ReSpeak/tsdeclarations` `ts3protocol.md` | Low-level TeamSpeak protocol specification | +| `ReSpeak/tsclientlib` `src/lib.rs` lines 956-1005 | `download_file` / `upload_file` public API | +| `ReSpeak/tsclientlib` `src/lib.rs` lines 1371-1427 | `StreamItem::FileDownload` handling | +| `ReSpeak/tsclientlib` `src/lib.rs` lines 1630-1672 | Outgoing init commands | +| `Multivit4min/TS3-NodeJS-Library` `src/transport/FileTransfer.ts` | Reference TCP transfer implementation | +| `Speckmops/ts3admin.class` `lib/ts3admin.class.php` lines 1352-1370 | Reference avatar download flow | +| `docs/architecture/sad.md` SAD-067, SDD-MOD-009 | Protocol adapter boundary rules | +| `crates/chanora_protocol/src/adapter.rs` lines 1561-1568 | Existing `uid_to_avatar_path` implementation | diff --git a/docs/architecture/file-transfer-implementation-plan.md b/docs/architecture/file-transfer-implementation-plan.md new file mode 100644 index 0000000..ba821e5 --- /dev/null +++ b/docs/architecture/file-transfer-implementation-plan.md @@ -0,0 +1,1315 @@ +# File Transfer Implementation Plan + +> Status: **Draft — revised after cache research (2026-06-10)** +> Created: 2026-06-10 +> Revised: 2026-06-10 — cache architecture overhauled per crate comparison, cross-platform dir research, and Oracle consultation +> Upstream: `docs/architecture/file-transfer-design.md`, `docs/architecture/file-transfer-research.md` + +## 0. Overview + +This document decomposes the file transfer system (avatar/icon download + caching) into five sequential phases with concrete file-level changes, function signatures, and verification criteria. + +**Scope**: Download only. No upload, no file browser. + +**Architecture recap**: Content-addressed blob store backed by `cacache`. Protocol keys (`av_`, `ic_`) map to cacache string keys. Physical storage is SHA-512 content-addressed. Same content on any server = stored once = zero duplication. + +--- + +## 1. Phase Summary + +| Phase | Crate | What | Delivers | Est. LOC | +|---|---|---|---|---| +| 1 | `chanora_protocol` | `Request::DownloadFile`, `StreamItem::FileDownload` handling, `ProtocolClient::download_avatar()` / `download_icon()` | Raw download across the protocol boundary | ~180 | +| 2 | `chanora_bridge` + Flutter | Bridge `downloadAvatar()`, Flutter `Image.memory()` + initials fallback | Avatars visible in the UI (no caching — every view re-downloads) | ~100 | +| 3 | `chanora_cache` (NEW crate) | `BlobCache` — flat content-addressed blob store, no metadata files, eviction by file mtime | Disk cache, zero duplication, no deps | ~150 | +| 4 | `chanora_core` | `FileTransferService` — request coalescing, rate limiter (1-2 concurrent), negative cache, retry backoff | Robust production download pipeline | ~250 | +| 5 | `chanora_bridge` + Flutter | `clearAvatarCache()`, `clearIconCache()`, settings UI, startup eviction | Cache management | ~80 | + +Phases 1 + 2 deliver visible value (avatars in the UI). Phases 3–5 add robustness. + +--- + +## 2. Phase 1 — Protocol Download + +### 2.1 Goal + +Add a `DownloadFile` request variant to the protocol adapter's `Request` enum. Handle `StreamItem::FileDownload` in the connection task's event loop. Expose `download_avatar()` and `download_icon()` on `ProtocolClient`. + +### 2.2 Files to Modify + +#### `crates/chanora_protocol/src/adapter.rs` + +**2.2.1 Add `Request::DownloadFile` variant** (after `FetchClientProfile` at line ~209) + +```rust +/// Download a file from the server's file transfer subsystem. +/// Returns the complete file bytes on success. +DownloadFile { + /// Channel ID the file resides in (0 for server-level files). + channel_id: u64, + /// File path on the server (e.g. "/avatar_" or "/icon_"). + path: String, + /// Optional channel password for password-protected channels. + password: Option, + /// Reply channel for the result. + reply: oneshot::Sender, ProtocolError>>, +}, +``` + +**2.2.2 Add `ProtocolClient::download_file()` method** (after `send_text_message` at line ~545) + +```rust +/// Download a file from the server. Returns the complete file bytes. +pub async fn download_file( + &self, + channel_id: u64, + path: String, + password: Option, +) -> Result, ProtocolError> { + let (tx, rx) = oneshot::channel(); + self.tx + .send(Request::DownloadFile { + channel_id, + path, + password, + reply: tx, + }) + .await + .map_err(|_| ProtocolError::Lost("connection task is gone".to_string()))?; + rx.await + .map_err(|_| ProtocolError::Lost("download_file reply dropped".to_string()))? +} +``` + +**2.2.3 Add convenience wrappers** (after `download_file`) + +```rust +/// Download a client avatar. `uid_hex` is the hex-encoded client UID +/// (as produced by `uid_to_avatar_path`). +pub async fn download_avatar( + &self, + client_uid: &str, +) -> Result, ProtocolError> { + let path = uid_to_avatar_path(client_uid); + self.download_file(0, path, None).await +} + +/// Download an icon by its unsigned CRC32 ID. +pub async fn download_icon( + &self, + icon_id: u64, +) -> Result, ProtocolError> { + let path = format!("/icon_{icon_id}"); + self.download_file(0, path, None).await +} +``` + +**2.2.4 Handle `Request::DownloadFile` in the main loop** (in the `rx.try_recv()` match block, after `FetchClientProfile` around line 840) + +```rust +Ok(Request::DownloadFile { + channel_id, + path, + password, + reply, +}) => { + let r = download_file_from_server( + &mut con, + channel_id, + &path, + password.as_deref(), + ).await; + let _ = reply.send(r); +} +``` + +**2.2.5 Add `download_file_from_server` function** + +```rust +/// Execute a file download via tsclientlib's download_file API. +/// Reads the entire TCP stream into a Vec. +async fn download_file_from_server( + con: &mut Connection, + channel_id: u64, + path: &str, + password: Option<&str>, +) -> Result, ProtocolError> { + let pw = password.unwrap_or(""); + let mut stream = con + .download_file(TsChannelId(channel_id), path, pw, 0) + .await + .map_err(|e| ProtocolError::Backend(format!("download_file init: {e}")))?; + + let mut buf = Vec::new(); + while let Some(item) = stream.next().await { + match item { + Ok(StreamItem::FileDownload(result)) => { + // Read the TcpStream into buf. + let mut reader = tokio::io::BufReader::new(result.stream); + let size_hint = result.size as usize; + if size_hint > 0 && size_hint < 32 * 1024 * 1024 { + buf.reserve(size_hint); + } + tokio::io::copy_to_bytes(&mut reader) + .await + .map_err(|e| ProtocolError::Backend(format!("download read: {e}")))?; + // Note: actual implementation needs to accumulate into buf, + // not use copy_to_bytes directly. See implementation note below. + } + Ok(_) => {} // other stream items during download + Err(e) => { + return Err(ProtocolError::Backend(format!("download stream: {e}"))); + } + } + } + Ok(buf) +} +``` + +> **Implementation note**: The exact tsclientlib `download_file` API signature and `StreamItem::FileDownload` handling needs to be verified against the pinned revision. The function above shows the intent; the actual implementation may differ based on whether `download_file` returns a `TcpStream` directly or wraps it in a `StreamItem`. The research doc (§3) confirmed: `download_file(ChannelId, path, password, seek)` → `StreamItem::FileDownload(FileDownloadResult { size, stream: TcpStream })`. The implementation reads `TcpStream` bytes until EOF. + +**2.2.6 Handle `StreamItem::FileDownload` in `handle_non_audio_stream_item`** + +Currently `handle_non_audio_stream_item` ignores `StreamItem::FileDownload` (the catch-all `_ => {}` at line 1023). Once we have async download requests dispatched through the main loop, file download responses arrive here. + +However, the tsclientlib `download_file` call is synchronous-blocking (it initiates the transfer and returns a stream handle). The `StreamItem::FileDownload` appears on the event stream as a *notification* that the server accepted the transfer. The actual data transfer happens on a separate TCP connection. + +The exact integration pattern depends on tsclientlib's API: +- If `download_file()` returns a `impl Stream>`, we drain it inside `download_file_from_server`. +- If the file download items arrive on the main `con.events()` stream, we need a pending-download tracking map (similar to `pending_moves`). + +**Decision**: Defer the exact tsclientlib integration to implementation time. The research doc confirmed the API shape; the implementation agent will verify against the actual pinned tsclientlib source. + +### 2.3 Verification + +- [ ] `cargo test -p chanora_protocol` passes +- [ ] `cargo clippy -p chanora_protocol` clean +- [ ] Manual: connect to server, call `download_avatar(uid)` → returns `Vec` with image data +- [ ] Error path: download nonexistent avatar → `ProtocolError` returned cleanly + +### 2.4 Risks + +| Risk | Mitigation | +|---|---| +| tsclientlib `download_file` API differs from assumed shape | Implementation agent reads pinned tsclientlib source before writing code | +| File transfer TCP stream blocks the main event loop | `download_file_from_server` runs as async; the main loop continues on the next iteration | +| Large file OOM | Cap download at 32 MB; return `ProtocolError::Backend` if exceeded | + +--- + +## 3. Phase 2 — Bridge + Flutter Display + +### 3.1 Goal + +Expose avatar download through the Flutter/Rust bridge. Display avatars in the Flutter UI using `Image.memory()`. No caching yet — every avatar view triggers a fresh download. + +### 3.2 Files to Modify + +#### `crates/chanora_bridge/src/api.rs` + +**3.2.1 Add bridge function `download_avatar`** (after `client_profile` around line 660) + +```rust +/// Download a client avatar by UID. Returns raw image bytes (PNG/JPEG). +/// Returns `None` if the client has no avatar or the download fails +/// without a protocol-level error. +pub async fn download_avatar(client_uid: String) -> Result>, BridgeError> { + runtime() + .spawn(async move { session().download_avatar(&client_uid).await }) + .await + .map_err(|e| task_join_error("download_avatar", e))? + .map(|opt| opt.map(|bytes| bytes)) +} +``` + +**3.2.2 Add bridge function `download_icon`** + +```rust +/// Download an icon by its unsigned CRC32 ID. Returns raw image bytes. +pub async fn download_icon(icon_id: u64) -> Result>, BridgeError> { + runtime() + .spawn(async move { session().download_icon(icon_id).await }) + .await + .map_err(|e| task_join_error("download_icon", e))? +} +``` + +#### `crates/chanora_core/src/lib.rs` (or wherever `ChanoraSession` lives) + +**3.2.3 Add session methods** + +```rust +pub async fn download_avatar(&self, client_uid: &str) -> Result>, CoreError> { + let handle = self.protocol_handle()?; + match handle.download_avatar(client_uid).await { + Ok(bytes) => Ok(Some(bytes)), + Err(ProtocolError::ServerRejected { .. }) => Ok(None), + Err(e) => Err(CoreError::from(e)), + } +} + +pub async fn download_icon(&self, icon_id: u64) -> Result>, CoreError> { + let handle = self.protocol_handle()?; + match handle.download_icon(icon_id).await { + Ok(bytes) => Ok(Some(bytes)), + Err(ProtocolError::ServerRejected { .. }) => Ok(None), + Err(e) => Err(CoreError::from(e)), + } +} +``` + +#### Flutter side (new files) + +**3.2.4 `apps/chanora_flutter/lib/widgets/avatar_widget.dart`** + +```dart +import 'dart:typed_data'; +import 'package:flutter/material.dart'; +import 'package:chanora_flutter/src/rust/api.dart' as rust; + +/// Widget that displays a client avatar or initials fallback. +/// Phase 2: no caching — re-downloads on every build. +class ClientAvatar extends StatefulWidget { + final String clientUid; + final String displayName; + final double radius; + + const ClientAvatar({ + super.key, + required this.clientUid, + required this.displayName, + this.radius = 20, + }); + + @override + State createState() => _ClientAvatarState(); +} + +class _ClientAvatarState extends State { + Uint8List? _bytes; + bool _loading = false; + + @override + void initState() { + super.initState(); + _loadAvatar(); + } + + @override + void didUpdateWidget(ClientAvatar old) { + super.didUpdateWidget(old); + if (old.clientUid != widget.clientUid) { + _bytes = null; + _loadAvatar(); + } + } + + Future _loadAvatar() async { + if (_loading) return; + setState(() => _loading = true); + try { + final bytes = await rust.downloadAvatar(clientUid: widget.clientUid); + if (mounted && bytes != null) { + setState(() => _bytes = bytes); + } + } catch (_) { + // Silently fall back to initials. + } finally { + if (mounted) { + setState(() => _loading = false); + } + } + } + + String get _initials { + final parts = widget.displayName.trim().split(RegExp(r'\s+')); + if (parts.length >= 2) { + return '${parts[0][0]}${parts[1][0]}'.toUpperCase(); + } + return widget.displayName.isNotEmpty + ? widget.displayName[0].toUpperCase() + : '?'; + } + + @override + Widget build(BuildContext context) { + if (_bytes != null) { + return CircleAvatar( + radius: widget.radius, + backgroundImage: MemoryImage(_bytes!), + ); + } + return CircleAvatar( + radius: widget.radius, + child: Text(_initials), + ); + } +} +``` + +### 3.3 Integration Points + +The `ClientAvatar` widget replaces any existing hardcoded `CircleAvatar` in: +- Client list rows +- Chat message headers +- User profile panels + +**Phase 2 limitation**: Every `initState` triggers a download. Scrolling a list of 100 clients = 100 downloads. This is acceptable for development but NOT for release. Phase 4 (session orchestration) fixes this with coalescing and rate limiting. + +### 3.4 Verification + +- [ ] `flutter_rust_bridge_codegen generate` succeeds (new bridge functions) +- [ ] Flutter app builds without errors +- [ ] Connect to server with avatar users → avatars render +- [ ] Client without avatar → initials fallback renders +- [ ] Scrolling client list does not crash (though it will be slow/chatty) + +--- + +## 4. Phase 3 — Content-Addressed Blob Cache (`chanora_cache`) + +### 4.1 Goal + +Create a new `chanora_cache` crate with a `BlobCache` — a content-addressed store for avatar/icon blobs, backed by the `cacache` crate for production-tested crash safety and integrity verification. Uses the platform **cache directory** (not support directory). + +### 4.2 Design Decisions (post-research) + +| Decision | Choice | Rationale | +|---|---|---| +| Crate placement | New `chanora_cache` crate | Cache is disposable; `chanora_storage` owns persistent identity/bookmark data with different durability and backup semantics | +| Directory | Platform cache dir via `getApplicationCacheDirectory()` | OS may evict under storage pressure; semantically correct for reconstructible data | +| Backing store | `cacache` crate | Production-tested crash safety, SSRI integrity verification on read, async-native, less code to maintain | +| Metadata | None — no JSON sidecar files | `cacache` manages its own content-addressed storage internally. No custom metadata needed. | +| Eviction | `cacache::ls()` + custom mtime sweep | `cacache` has no built-in LRU eviction. Walk entries, sort by mtime, delete oldest until under cap. | +| Integrity | `cacache` SSRI on every read | Detects corruption, bit rot, partial writes automatically. Better than custom "delete on read failure". | +| In-memory hot layer | None for MVP | Flutter's `ImageCache` already caches decoded images. Add Rust-side layer only if profiling shows need | +| Dependencies | `cacache` (~6 transitive deps) | `sha2` already pulled in by `chacha20poly1305` via `chanora_storage`. Crash safety and integrity worth the deps. | + +### 4.3 Storage Layout (on-disk, managed by `cacache`) + +``` +/chanora/ + blobs/ ← cacache content store root + content-v2/ + / ← content-addressed by SHA-512 + data ← raw blob bytes + tmp/ ← cacache temp files (in-flight writes) + index-v2/ ← cacache entry index (key → content mapping) +``` + +`cacache` manages this layout internally. Chanora's `BlobCache` maps protocol keys (`av_`, `ic_`) to `cacache` string keys. Content dedup is automatic — same bytes stored once regardless of key. + +The directory is opaque to Chanora. Inspection is via `BlobCache` API or `cacache::ls()`. + +### 4.4 Files to Create + +#### `crates/chanora_cache/Cargo.toml` (NEW) + +```toml +[package] +name = "chanora_cache" +description = "Chanora disposable content-addressed blob cache for avatars and icons" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +publish.workspace = true + +[dependencies] +cacache = "13" +thiserror.workspace = true +tokio.workspace = true +tracing.workspace = true + +[dev-dependencies] +tempfile = "3" +``` + +#### `crates/chanora_cache/src/lib.rs` (NEW — ~120 LOC) + +```rust +//! Disposable content-addressed blob cache for avatar and icon files. +//! +//! Wraps `cacache` for production-tested crash safety and integrity +//! verification. The on-disk layout is managed by cacache (content-v2, +//! index-v2). Chanora maps protocol keys (av_, ic_) to +//! cacache string keys. +//! +//! This crate is intentionally separate from `chanora_storage`: +//! storage owns persistent identity/bookmark data; cache owns +//! reconstructible, disposable blob data with different durability +//! and backup semantics. + +#![forbid(unsafe_code)] +#![warn(missing_docs)] + +use std::path::{Path, PathBuf}; +use tracing::{info, warn}; + +/// Errors raised by the blob cache. +#[derive(Debug, thiserror::Error)] +pub enum BlobCacheError { + /// Filesystem I/O error. + #[error("io: {0}")] + Io(String), + /// Key validation error. + #[error("invalid key: {0}")] + InvalidKey(String), +} + +/// Content-addressed blob cache backed by cacache. +pub struct BlobCache { + cache_dir: PathBuf, + /// Maximum total cache size in bytes. 0 = no limit. + max_bytes: u64, +} + +/// Avatar blob prefix. +pub const PREFIX_AVATAR: &str = "av_"; +/// Icon blob prefix. +pub const PREFIX_ICON: &str = "ic_"; + +impl BlobCache { + /// Create or open a BlobCache rooted at `cache_dir/chanora/`. + /// Creates the cacache directory. `max_bytes` sets the + /// eviction threshold; 0 means no automatic eviction. + pub fn new(cache_dir: impl AsRef, max_bytes: u64) -> Result { + let cache_dir = cache_dir.as_ref().join("chanora").join("blobs"); + // cacache creates the directory on first write, but we + // create it eagerly so total_size() works before any writes. + std::fs::create_dir_all(&cache_dir) + .map_err(|e| BlobCacheError::Io(format!("mkdir cache: {e}")))?; + Ok(Self { cache_dir, max_bytes }) + } + + /// Store a blob. `prefix` is `PREFIX_AVATAR` or `PREFIX_ICON`. + /// `key` is the content hash (MD5 hex for avatars, unsigned + /// decimal CRC32 for icons). + /// + /// cacache handles dedup and integrity internally. + /// This is async because cacache is async-native. + pub async fn put( + &self, + prefix: &str, + key: &str, + data: &[u8], + ) -> Result<(), BlobCacheError> { + validate_key(prefix, key)?; + let cache_key = format!("{prefix}{key}"); + cacache::write(&self.cache_dir, &cache_key, data) + .await + .map_err(|e| BlobCacheError::Io(format!("cacache write: {e}")))?; + Ok(()) + } + + /// Read a blob. Returns `None` if not cached. + /// cacache verifies SSRI integrity on every read. + pub async fn get(&self, prefix: &str, key: &str) -> Result>, BlobCacheError> { + validate_key(prefix, key)?; + let cache_key = format!("{prefix}{key}"); + match cacache::read(&self.cache_dir, &cache_key).await { + Ok(data) => Ok(Some(data)), + Err(cacache::Error::EntryNotFound(_, _)) => Ok(None), + Err(e) => { + // Integrity failure or I/O error — remove corrupt entry. + warn!(target: "chanora_cache", key = %cache_key, error = %e, + "cache read failed; removing entry"); + let _ = cacache::remove_entry(&self.cache_dir, &cache_key).await; + Ok(None) + } + } + } + + /// Delete a specific blob. + pub async fn remove(&self, prefix: &str, key: &str) -> Result<(), BlobCacheError> { + validate_key(prefix, key)?; + let cache_key = format!("{prefix}{key}"); + cacache::remove_entry(&self.cache_dir, &cache_key) + .await + .map_err(|e| BlobCacheError::Io(format!("cacache remove: {e}")))?; + Ok(()) + } + + /// Delete all blobs. + pub async fn clear(&self) -> Result<(), BlobCacheError> { + // Remove and recreate the cache directory. + let path = self.cache_dir.clone(); + tokio::task::spawn_blocking(move || { + if path.exists() { + std::fs::remove_dir_all(&path) + .map_err(|e| BlobCacheError::Io(format!("clear cache: {e}")))?; + std::fs::create_dir_all(&path) + .map_err(|e| BlobCacheError::Io(format!("recreate cache dir: {e}")))?; + } + Ok(()) + }) + .await + .map_err(|e| BlobCacheError::Io(format!("clear task: {e}")))? + } + + /// Return total bytes used by all blobs. + /// Walks cacache entries and sums sizes. + pub async fn total_size(&self) -> Result { + let mut total: u64 = 0; + let mut stream = cacache::ls(&self.cache_dir).await + .map_err(|e| BlobCacheError::Io(format!("cacache ls: {e}")))?; + while let Some(entry) = stream.next().await { + match entry { + Ok(meta) => total += meta.size.unwrap_or(0), + Err(e) => { + warn!(target: "chanora_cache", error = %e, "skipping bad entry during size scan"); + } + } + } + Ok(total) + } + + /// Evict oldest entries by mtime until total size is under + /// `max_bytes`. Call on startup or periodically. No-op if + /// `max_bytes` is 0. + pub async fn evict(&self) -> Result<(), BlobCacheError> { + if self.max_bytes == 0 { + return Ok(()); + } + let mut entries: Vec<(String, u64, std::time::SystemTime)> = Vec::new(); + let mut stream = cacache::ls(&self.cache_dir).await + .map_err(|e| BlobCacheError::Io(format!("cacache ls: {e}")))?; + while let Some(entry) = stream.next().await { + match entry { + Ok(meta) => { + let size = meta.size.unwrap_or(0); + let mtime = meta.time.into_system_time(); + entries.push((meta.key, size, mtime)); + } + Err(e) => { + warn!(target: "chanora_cache", error = %e, "skipping bad entry during eviction scan"); + } + } + } + let total: u64 = entries.iter().map(|(_, s, _)| *s).sum(); + if total <= self.max_bytes { + return Ok(()); + } + // Sort by mtime ascending (oldest first). + entries.sort_by_key(|(_, _, t)| *t); + let mut freed: u64 = 0; + let target = total - self.max_bytes; + for (key, size, _) in entries { + if freed >= target { + break; + } + let _ = cacache::remove_entry(&self.cache_dir, &key).await; + freed += size; + } + info!(target: "chanora_cache", freed_bytes = freed, "evicted oldest blobs"); + Ok(()) + } +} + +/// Validate key format to prevent malformed entries. +fn validate_key(prefix: &str, key: &str) -> Result<(), BlobCacheError> { + if !matches!(prefix, PREFIX_AVATAR | PREFIX_ICON) { + return Err(BlobCacheError::InvalidKey(format!("bad prefix: {prefix}"))); + } + match prefix { + PREFIX_AVATAR => { + // MD5 hex = exactly 32 hex chars. + if key.len() != 32 || !key.chars().all(|c| c.is_ascii_hexdigit()) { + return Err(BlobCacheError::InvalidKey( + format!("avatar key must be 32 hex chars, got: {key}") + )); + } + } + PREFIX_ICON => { + // Unsigned CRC32 = decimal digits. + if key.is_empty() || !key.chars().all(|c| c.is_ascii_digit()) { + return Err(BlobCacheError::InvalidKey( + format!("icon key must be decimal digits, got: {key}") + )); + } + } + _ => unreachable!(), + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tempdir() -> tempfile::TempDir { + tempfile::Builder::new() + .prefix("chanora_cache_test_") + .tempdir() + .unwrap() + } + + #[tokio::test] + async fn put_get_roundtrip() { + let tmp = tempdir(); + let cache = BlobCache::new(&tmp, 0).unwrap(); + assert!(cache.get(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6").await.unwrap().is_none()); + cache.put(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", b"avatar-bytes").await.unwrap(); + let data = cache.get(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6").await.unwrap(); + assert_eq!(data.as_deref(), Some(b"avatar-bytes".as_slice())); + } + + #[tokio::test] + async fn get_missing_returns_none() { + let tmp = tempdir(); + let cache = BlobCache::new(&tmp, 0).unwrap(); + assert!(cache.get(PREFIX_AVATAR, "00000000000000000000000000000000").await.unwrap().is_none()); + } + + #[tokio::test] + async fn clear_removes_all() { + let tmp = tempdir(); + let cache = BlobCache::new(&tmp, 0).unwrap(); + cache.put(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", b"data").await.unwrap(); + cache.put(PREFIX_ICON, "12345", b"icon").await.unwrap(); + cache.clear().await.unwrap(); + assert_eq!(cache.total_size().await.unwrap(), 0); + } + + #[tokio::test] + async fn total_size_accounts_for_all_entries() { + let tmp = tempdir(); + let cache = BlobCache::new(&tmp, 0).unwrap(); + cache.put(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", b"12345").await.unwrap(); + cache.put(PREFIX_ICON, "99999", b"12").await.unwrap(); + assert_eq!(cache.total_size().await.unwrap(), 5 + 2); + } + + #[tokio::test] + async fn invalid_key_rejected() { + let tmp = tempdir(); + let cache = BlobCache::new(&tmp, 0).unwrap(); + // Too short for MD5. + assert!(cache.put(PREFIX_AVATAR, "abc", b"data").await.is_err()); + // Non-hex in MD5. + assert!(cache.put(PREFIX_AVATAR, "g".repeat(32).as_str(), b"data").await.is_err()); + // Non-digit in icon key. + assert!(cache.put(PREFIX_ICON, "12a45", b"data").await.is_err()); + // Bad prefix. + assert!(cache.put("xx_", "abc", b"data").await.is_err()); + } + + #[tokio::test] + async fn evict_deletes_oldest_until_under_cap() { + let tmp = tempdir(); + // 10 byte cap. + let cache = BlobCache::new(&tmp, 10).unwrap(); + cache.put(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", b"12345678").await.unwrap(); // 8 bytes + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + cache.put(PREFIX_ICON, "11111", b"12345").await.unwrap(); // 5 bytes → total 13, over cap + cache.evict().await.unwrap(); + // Oldest (avatar) should be evicted. + assert!(cache.get(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6").await.unwrap().is_none()); + assert!(cache.get(PREFIX_ICON, "11111").await.unwrap().is_some()); + } + + #[tokio::test] + async fn corrupt_entry_removed_on_read() { + let tmp = tempdir(); + let cache = BlobCache::new(&tmp, 0).unwrap(); + // Write valid entry. + cache.put(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", b"good").await.unwrap(); + // Corrupt the underlying content file (simulate bit rot). + // cacache stores content under content-v2//data. + // We find and corrupt it to test integrity check. + let content_dir = tmp.path().join("chanora").join("blobs").join("content-v2"); + if let Ok(mut entries) = std::fs::read_dir(&content_dir) { + while let Some(Ok(entry)) = entries.next() { + let data_path = entry.path().join("data"); + if data_path.exists() { + std::fs::write(&data_path, b"corrupt").unwrap(); + break; + } + } + } + // Read should detect corruption and return None. + let result = cache.get(PREFIX_AVATAR, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6").await.unwrap(); + assert!(result.is_none()); + } +} +``` + +### 4.5 Wire BlobCache into Session + +The `BlobCache` is constructed once during a new `init_cache` bridge call (separate from `init_storage`) using the platform cache directory. All `BlobCache` methods are async (cacache is async-native). + +#### `crates/chanora_bridge/src/api.rs` + +```rust +/// Wire the blob cache to a platform cache directory. Call once +/// on app start after Flutter resolves `getApplicationCacheDirectory()`. +pub async fn init_cache(dir: String) -> Result<(), BridgeError> { + runtime() + .spawn(async move { session().init_cache(&dir).await }) + .await + .map_err(|e| task_join_error("init_cache", e))??; + Ok(()) +} +``` + +#### Flutter init (in `app_bootstrap.dart`) + +```dart +// Existing: +final supportDir = await getApplicationSupportDirectory(); +await rust.initStorage(dir: supportDir.path); + +// NEW: separate cache directory +final cacheDir = await getApplicationCacheDirectory(); +await rust.initCache(dir: cacheDir.path); +``` + +#### `chanora_core::ChanoraSession` + +```rust +blob_cache: OnceCell, + +pub async fn init_cache(&self, dir: &str) -> Result<(), CoreError> { + // 100 MB cap for avatars + icons. + let cache = chanora_cache::BlobCache::new(dir, 100 * 1024 * 1024)?; + // Evict on startup. + cache.evict().await?; + self.blob_cache.set(cache).map_err(|_| CoreError::AlreadyConnected)?; + Ok(()) +} +``` + +#### Download method with cache + +```rust +pub async fn get_avatar(&self, avatar_hash: &str, client_uid: &str) -> Result>, CoreError> { + // 1. Check cache + if let Some(cache) = self.blob_cache.get() { + if let Some(bytes) = cache.get(chanora_cache::PREFIX_AVATAR, avatar_hash).await? { + return Ok(Some(bytes)); + } + } + // 2. Download + let handle = self.protocol_handle()?; + let bytes = handle.download_avatar(client_uid).await?; + // 3. Store in cache + if let Some(cache) = self.blob_cache.get() { + cache.put(chanora_cache::PREFIX_AVATAR, avatar_hash, &bytes).await?; + } + Ok(Some(bytes)) +} +``` + +### 4.6 Verification + +- [ ] `cargo test -p chanora_cache` — all 8 async tests pass +- [ ] `cargo clippy -p chanora_cache` clean +- [ ] Cache directory appears under `/chanora/blobs/` after download +- [ ] Second download of same avatar hits cache (no TCP transfer) +- [ ] Same avatar from different server uses same content (cacache dedup) +- [ ] `evict()` deletes oldest entries when over cap +- [ ] Corrupted content detected by cacache integrity check → entry removed +- [ ] Invalid keys are rejected +- [ ] `clear()` empties cache + +--- + +## 5. Phase 4 — Session Orchestration (`chanora_core`) + +### 5.1 Goal + +Add a `FileTransferService` to `chanora_core` that sits between the bridge and the raw protocol download. Provides: + +1. **Request coalescing** — 10 concurrent `get_avatar` calls for the same hash = 1 download, 10 waiters. +2. **Concurrency throttle** — At most 2 simultaneous file transfers (anti-flood). +3. **Negative cache** — Remember "avatar X does not exist" for 5 minutes to avoid re-downloading 404s. +4. **Retry with exponential backoff** — 5s → 30s → 2min → 10min for transient failures. + +### 5.2 Design + +``` +Bridge (download_avatar) + ↓ +FileTransferService::get_avatar(hash, uid) + ↓ + 1. Check BlobCache → hit? return bytes + 2. Check negative cache → hit? return None + 3. Check in-flight map → already downloading? await existing handle + 4. Acquire concurrency semaphore (max 2) + 5. Download via ProtocolClient + 6. Store in BlobCache + 7. Notify all waiters +``` + +### 5.3 Files to Create/Modify + +#### `crates/chanora_core/src/file_transfer.rs` (NEW — ~250 LOC) + +```rust +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::{Semaphore, SemaphorePermit, oneshot, Mutex}; + +pub struct FileTransferService { + cache: chanora_cache::BlobCache, + protocol: Arc>>, + /// Max concurrent downloads. + semaphore: Arc, + /// In-flight downloads: (prefix+key) → Vec>> + in_flight: Arc>, FileTransferError>>>>>>, + /// Negative cache: key → instant when entry expires. + negative_cache: Arc>>, + /// Negative cache TTL. + negative_ttl: std::time::Duration, +} + +#[derive(Debug, thiserror::Error)] +pub enum FileTransferError { + #[error("not connected")] + NotConnected, + #[error("protocol: {0}")] + Protocol(#[from] chanora_protocol::ProtocolError), + #[error("cache: {0}")] + Cache(#[from] chanora_cache::BlobCacheError), +} + +impl FileTransferService { + pub fn new(cache: chanora_cache::BlobCache) -> Self { + Self { + cache, + protocol: Arc::new(Mutex::new(None)), + semaphore: Arc::new(Semaphore::new(2)), + in_flight: Arc::new(Mutex::new(HashMap::new())), + negative_cache: Arc::new(Mutex::new(HashMap::new())), + negative_ttl: std::time::Duration::from_secs(300), // 5 min + } + } + + pub async fn set_protocol(&self, client: Option) { + *self.protocol.lock().await = client; + } + + /// Get an avatar by hash + UID. Coalesces concurrent requests. + pub async fn get_avatar( + &self, + avatar_hash: &str, + client_uid: &str, + ) -> Result>, FileTransferError> { + let cache_key = format!("av_{avatar_hash}"); + // 1. Disk cache hit? + if let Some(bytes) = self.cache.get("av_", avatar_hash).await? { + return Ok(Some(bytes)); + } + // 2. Negative cache hit? + { + let neg = self.negative_cache.lock().await; + if let Some(expires) = neg.get(&cache_key) { + if std::time::Instant::now() < *expires { + return Ok(None); + } + } + } + // 3. Already in-flight? + { + let mut in_flight = self.in_flight.lock().await; + if let Some(waiters) = in_flight.get_mut(&cache_key) { + let (tx, rx) = oneshot::channel(); + waiters.push(tx); + drop(in_flight); + return rx.await + .map_err(|_| FileTransferError::NotConnected)? + .map(|opt| opt); + } + // First requester — register in-flight. + in_flight.insert(cache_key.clone(), Vec::new()); + } + // 4. Acquire semaphore + download. + let _permit = self.semaphore.acquire().await + .map_err(|_| FileTransferError::NotConnected)?; + let result = self.do_download_avatar(avatar_hash, client_uid).await; + // 5. Notify waiters. + { + let mut in_flight = self.in_flight.lock().await; + if let Some(waiters) = in_flight.remove(&cache_key) { + for tx in waiters { + let _ = tx.send(result.clone()); + } + } + } + result + } + + async fn do_download_avatar( + &self, + avatar_hash: &str, + client_uid: &str, + ) -> Result>, FileTransferError> { + let guard = self.protocol.lock().await; + let proto = guard.as_ref().ok_or(FileTransferError::NotConnected)?; + let cloned = proto.clone(); + drop(guard); + + match cloned.download_avatar(client_uid).await { + Ok(bytes) => { + self.cache.put("av_", avatar_hash, &bytes).await?; + Ok(Some(bytes)) + } + Err(chanora_protocol::ProtocolError::ServerRejected { .. }) => { + // Avatar doesn't exist on server — negative cache. + let mut neg = self.negative_cache.lock().await; + neg.insert( + format!("av_{avatar_hash}"), + std::time::Instant::now() + self.negative_ttl, + ); + Ok(None) + } + Err(e) => Err(FileTransferError::Protocol(e)), + } + } + + /// Get an icon by CRC32. Same coalescing logic as avatars. + pub async fn get_icon( + &self, + icon_id: u64, + ) -> Result>, FileTransferError> { + let cache_key = format!("ic_{icon_id}"); + if let Some(bytes) = self.cache.get("ic_", &icon_id.to_string()).await? { + return Ok(Some(bytes)); + } + { + let neg = self.negative_cache.lock().await; + if let Some(expires) = neg.get(&cache_key) { + if std::time::Instant::now() < *expires { + return Ok(None); + } + } + } + // Same coalescing pattern as get_avatar... + // (Implementation mirrors get_avatar with icon-specific path) + let _permit = self.semaphore.acquire().await + .map_err(|_| FileTransferError::NotConnected)?; + let guard = self.protocol.lock().await; + let proto = guard.as_ref().ok_or(FileTransferError::NotConnected)?; + let cloned = proto.clone(); + drop(guard); + + match cloned.download_icon(icon_id).await { + Ok(bytes) => { + self.cache.put("ic_", &icon_id.to_string(), &bytes).await?; + Ok(Some(bytes)) + } + Err(chanora_protocol::ProtocolError::ServerRejected { .. }) => { + let mut neg = self.negative_cache.lock().await; + neg.insert(cache_key, std::time::Instant::now() + self.negative_ttl); + Ok(None) + } + Err(e) => Err(FileTransferError::Protocol(e)), + } + } +} +``` + +### 5.4 Wire into ChanoraSession + +```rust +// In ChanoraSession: +file_transfer: OnceCell, + +// After connect: +file_transfer.set_protocol(Some(protocol_client)).await; + +// After disconnect: +file_transfer.set_protocol(None).await; +``` + +### 5.5 Bridge Updates + +Replace Phase 2's raw `download_avatar` with cache-aware version: + +```rust +pub async fn get_avatar( + avatar_hash: String, + client_uid: String, +) -> Result>, BridgeError> { + runtime() + .spawn(async move { + session().file_transfer() + .get_avatar(&avatar_hash, &client_uid) + .await + }) + .await + .map_err(|e| task_join_error("get_avatar", e))? + .map_err(BridgeError::from) +} +``` + +### 5.6 Flutter Widget Update + +Update `ClientAvatar` to pass `avatar_hash` (from `client_flag_avatar` property in the snapshot): + +```dart +class ClientAvatar extends StatefulWidget { + final String clientUid; + final String avatarHash; // client_flag_avatar (MD5 hex) + final String displayName; + final double radius; + // ... +} + +Future _loadAvatar() async { + if (_loading || widget.avatarHash.isEmpty) return; + setState(() => _loading = true); + try { + final bytes = await rust.getAvatar( + avatarHash: widget.avatarHash, + clientUid: widget.clientUid, + ); + if (mounted && bytes != null) { + setState(() => _bytes = bytes); + } + } catch (_) { + // Fall back to initials. + } finally { + if (mounted) setState(() => _loading = false); + } +} +``` + +### 5.7 ProtocolDelta / BridgeEvent Updates + +Add avatar hash to `ClientJoined` and `ClientUpdated` events so Flutter knows the `client_flag_avatar` value: + +#### `crates/chanora_protocol/src/dto.rs` — add `avatar_hash` to `ClientInfo`: + +```rust +pub struct ClientInfo { + // ... existing fields ... + /// MD5 hash of the client's avatar bytes (client_flag_avatar). + /// Empty string means no avatar. + pub avatar_hash: String, +} +``` + +#### `crates/chanora_protocol/src/adapter.rs` — populate `avatar_hash` in `build_snapshot`: + +Read `client_flag_avatar` from tsclientlib state when building `ClientInfo`. + +#### `crates/chanora_bridge/src/api.rs` — add `avatar_hash` to `BridgeClient`: + +```rust +pub struct BridgeClient { + // ... existing fields ... + /// MD5 hash of the client's avatar (client_flag_avatar). Empty = no avatar. + pub avatar_hash: String, +} +``` + +#### `BridgeEvent::ClientJoined` / `BridgeEvent::ClientUpdated` — add `avatar_hash` field + +This is the key change that lets the Flutter widget know *which* avatar to display for each client. + +### 5.8 Verification + +- [ ] 10 concurrent `get_avatar` calls for same hash → 1 download, 10 responses +- [ ] Non-existent avatar → returns `None`, negative cache prevents re-download +- [ ] Semaphore limits concurrent downloads to 2 +- [ ] Cache hit returns bytes without network call +- [ ] `cargo test -p chanora_core` passes +- [ ] Flutter UI shows avatars without excessive network traffic + +--- + +## 6. Phase 5 — Cache Management + +### 6.1 Goal + +Expose cache management through the bridge. Add a "Clear Cache" button to settings. + +### 6.2 Files to Modify + +#### `crates/chanora_bridge/src/api.rs` + +```rust +/// Clear all cached avatar and icon data. +pub async fn clear_file_cache() -> Result<(), BridgeError> { + runtime() + .spawn(async { session().clear_file_cache().await }) + .await + .map_err(|e| task_join_error("clear_file_cache", e))? +} + +/// Get total cache size in bytes. +pub async fn file_cache_size() -> Result { + runtime() + .spawn(async { session().file_cache_size() }) + .await + .map_err(|e| task_join_error("file_cache_size", e))? +} +``` + +#### `crates/chanora_core/src/lib.rs` + +```rust +pub async fn clear_file_cache(&self) -> Result<(), CoreError> { + if let Some(ft) = self.file_transfer.as_ref() { + ft.cache.clear()?; + } + Ok(()) +} + +pub fn file_cache_size(&self) -> Result { + if let Some(ft) = self.file_transfer.as_ref() { + Ok(ft.cache.total_size()?) + } else { + Ok(0) + } +} +``` + +#### Flutter settings UI + +Add to settings page: + +```dart +ListTile( + title: const Text('Clear avatar cache'), + subtitle: FutureBuilder( + future: _cacheSizeLabel(), + builder: (_, snap) => Text(snap.data ?? 'Calculating...'), + ), + trailing: IconButton( + icon: const Icon(Icons.delete_outline), + onPressed: () async { + await rust.clearFileCache(); + PaintingBinding.instance.imageCache.clear(); + setState(() {}); // Refresh size display + }, + ), +), +``` + +### 6.3 Post-MVP Eviction (not in this plan) + +- LRU eviction on startup: delete blobs older than N days +- Configurable cache size limit +- Per-server metadata sidecar files (`refs/_.json`) + +### 6.4 Verification + +- [ ] `clear_file_cache()` empties `blobs/` directory +- [ ] `file_cache_size()` returns 0 after clear +- [ ] Flutter settings shows cache size +- [ ] After clear, avatars re-download on next view + +--- + +## 7. Cross-Phase Dependency Map + +``` +Phase 1 (protocol download) + ↓ +Phase 2 (bridge + Flutter display) — uses Phase 1's download_avatar + ↓ +Phase 3 (BlobCache) — used by Phase 4 + ↓ +Phase 4 (FileTransferService) — uses Phase 1 + Phase 3 + ↓ +Phase 5 (cache management) — uses Phase 3 + Phase 4 +``` + +Phases 1 and 2 can be tested end-to-end (avatars in UI) before any caching exists. +Phases 3 and 4 can be developed in parallel but Phase 4 depends on Phase 3's `BlobCache`. +Phase 5 depends on Phase 3. + +--- + +## 8. Testing Strategy + +### Unit Tests + +| Crate | Test | Phase | +|---|---|---| +| `chanora_cache` | `BlobCache` put/get/clear/evict/key validation | 3 | +| `chanora_core` | `FileTransferService` coalescing, negative cache, semaphore | 4 | +| `chanora_protocol` | `uid_to_avatar_path` (existing), `download_file_from_server` (mock) | 1 | + +### Integration Tests + +| Test | What | Phase | +|---|---|---| +| Download avatar from real server | Connect, call download_avatar, verify bytes are valid PNG/JPEG | 1 | +| Cache round-trip | Download → cache hit → disconnect → reconnect → cache hit | 3+4 | +| Concurrent coalescing | 10 tasks request same avatar → 1 download | 4 | +| Negative cache | Request non-existent avatar → get None → immediate None on retry | 4 | + +### Manual Verification + +| Scenario | Expected | Phase | +|---|---|---| +| Connect to server with 50+ users | Avatars load progressively, no flood | 2+4 | +| Scroll client list rapidly | No duplicate downloads, smooth scrolling | 4 | +| Connect to 2 servers with overlapping users | Same avatar = 1 blob file | 3 | +| Settings → Clear Cache | Cache empties, avatars re-download | 5 | + +--- + +## 9. Out of Scope (Deferred) + +| Item | Reason | When | +|---|---|---| +| Icon download + channel/client/server icon display | Avatar is higher priority; icons follow the same pipeline | Phase 4+ | +| File upload | Not in MVP | Post-MVP | +| File browser | Not in MVP | Post-MVP | +| myTeamSpeak avatar integration | Requires cloud OAuth; indefinitely deferred | Post-MVP | +| Per-server metadata sidecars | Nice-to-have for cache inspection | Phase 5+ | +| Thumbnail generation | Avatars are small; no thumbnails needed | Post-MVP | +| Animated avatars | TS3 does not support animated avatars | N/A | + +--- + +## 10. Implementation Notes + +### 10.1 tsclientlib API Verification + +The implementation agent for Phase 1 MUST read the pinned tsclientlib source before writing any code. Key files: + +- `tsclientlib/src/lib.rs` lines 956–1005 — `download_file` public API +- `tsclientlib/src/lib.rs` lines 1371–1427 — `StreamItem::FileDownload` handling +- Local clone at `/var/folders/.../T/opencode/tsclientlib-rs-research/` + +### 10.2 FRB Code Generation + +Phase 2 and 5 add new bridge functions. After modifying `api.rs`, run: + +```bash +flutter_rust_bridge_codegen generate +``` + +### 10.3 avatar_hash in ProtocolDelta + +Phase 4 adds `avatar_hash` to client events. This requires changes in: + +1. `chanora_protocol::dto::ClientInfo` — add field +2. `chanora_protocol::adapter::build_snapshot` — populate from tsclientlib state +3. `chanora_protocol::adapter::forward_delta` — extract `client_flag_avatar` from PropertyChanged events +4. `chanora_core::SessionEvent::ClientJoined/Updated` — add field +5. `chanora_bridge::BridgeEvent::ClientJoined/Updated` — add field +6. Flutter `ClientJoinedEvent` / `ClientUpdatedEvent` — read field + +This is a cross-cutting change. It should be done as a separate, atomic commit before Phase 4's Flutter widget updates. + +### 10.4 Atomic Commits + +Recommended commit boundaries: + +1. `feat(protocol): add DownloadFile request and file download support` (Phase 1) +2. `feat(bridge): add download_avatar bridge function` (Phase 2 bridge) +3. `feat(ui): add ClientAvatar widget with initials fallback` (Phase 2 Flutter) +4. `feat(cache): add chanora_cache crate with cacache-backed blob cache` (Phase 3) +5. `feat(core): add avatar_hash to client events and DTOs` (Phase 4 data plumbing) +6. `feat(core): add FileTransferService with coalescing and rate limiting` (Phase 4) +7. `feat(ui): update ClientAvatar to use cached downloads` (Phase 4 Flutter) +8. `feat(bridge): add clear_file_cache and file_cache_size` (Phase 5 bridge) +9. `feat(ui): add cache management to settings` (Phase 5 Flutter) diff --git a/docs/architecture/file-transfer-research.md b/docs/architecture/file-transfer-research.md new file mode 100644 index 0000000..3115a48 --- /dev/null +++ b/docs/architecture/file-transfer-research.md @@ -0,0 +1,770 @@ +# File Transfer Cache Research + +**Date:** 2026-06-10 +**Status:** Research complete, design implications noted (updated with TeaSpeak server findings) +**Companion to:** `docs/architecture/file-transfer-design.md` +**Purpose:** Factual findings from protocol analysis, existing client implementations, and cross-platform research that inform the cache architecture decision. + +--- + +## 1. TS3 Protocol Identity Semantics + +### 1.1 Avatar Identity + +| Aspect | Value | +|---|---| +| Protocol field | `client_flag_avatar` | +| Type | `TYPE_STRING` (TeaSpeakLibrary `PropertyDefinition.h:200`) | +| Meaning | MD5 hash of the avatar file bytes | +| Scope | Per-client per-server — a user can have different avatars on different servers | +| Freshness | Automatically up-to-date for any client "in view" (`FLAG_CLIENT_VIEW`) | +| Empty value | No avatar set | + +**Key fact:** Identical avatar image bytes produce the **same** `client_flag_avatar` hash on any TS3 server. The hash is a content fingerprint, not a server-assigned identifier. + +**Download path on server:** `/avatar_` — the filename is derived from the client's unique identifier (UID), not from the content hash. The content hash is communicated separately via `client_flag_avatar`. + +**Sources:** +- TeaSpeakLibrary `PropertyDefinition.h:200`: `PropertyDescription{CLIENT_FLAG_AVATAR, "client_flag_avatar", "", TYPE_STRING, FLAG_CLIENT_VIEW | FLAG_SAVE | FLAG_USER_EDITABLE}` +- TS3AudioBot avatar upload: computes MD5 of image bytes, then sets `client_flag_avatar` to that hash ([`TS3AudioBot/TSLib/TsBaseFunctions.cs:324-341`](https://github.com/Splamy/TS3AudioBot/blob/a69a38d8cba5a4d671dbe06505506f6b46f1d947/TSLib/TsBaseFunctions.cs#L324-L341)) +- TS3 NodeJS Library: avatar filename is `avatar_${clientBase64HashClientUID}` ([`TS3-NodeJS-Library/src/node/Client.ts:300-313`](https://github.com/Multivit4min/TS3-NodeJS-Library/blob/0c69b7ee80fa5b74e9175cf4ae3018346f7eb300/src/node/Client.ts#L300-L313)) +- TS3 PHP Framework: avatar name derivation from UID ([`ts3phpframework/src/Node/Client.php:288-307`](https://github.com/planetteamspeak/ts3phpframework/blob/87046b3d493c4d3d8064c639ea4269571192e476/src/Node/Client.php#L288-L307)) + +### 1.2 Icon Identity + +| Aspect | Value | +|---|---| +| Protocol fields | `channel_icon_id`, `client_icon_id`, `virtualserver_icon_id` | +| Type | `TYPE_UNSIGNED_NUMBER` (TeaSpeakLibrary `PropertyDefinition.h:83,146,217`) | +| Meaning | CRC32 (unsigned) of the icon file bytes | +| Scope | Per-entity per-server — but CRC32 is content-derived | +| Download path | `/icon_` | + +**Key fact:** Identical icon bytes produce the **same** CRC32 on any TS3 server. The icon ID is a content fingerprint. The upload process computes `crc32.unsigned(data)` and stores at `/icon_`. + +**CRC32 collision caveat:** CRC32 is only 32 bits. Different icon content can theoretically produce the same CRC32. Qint's `filecache.rs` explicitly notes this: "there could be collisions because only CRC-32 is used." ForChanora's purposes (small icons, not security-critical), this is acceptable. + +**Sources:** +- TeaSpeakLibrary `PropertyDefinition.h:83,146,217`: all icon IDs are `TYPE_UNSIGNED_NUMBER` +- TS3 NodeJS Library `uploadIcon()`: computes `crc32.unsigned(data)`, uploads to `/icon_` ([`TS3-NodeJS-Library/src/TeamSpeak.ts:2234-2241`](https://github.com/Multivit4min/TS3-NodeJS-Library/blob/0c69b7ee80fa5b74e9175cf4ae3018346f7eb300/src/TeamSpeak.ts#L2234-L2241)) +- TS3 PHP Framework: icon path uses `/icon_` ([`ts3phpframework/src/Node/Node.php:137-146`](https://github.com/planetteamspeak/ts3phpframework/blob/87046b3d493c4d3d8064c639ea4269571192e476/src/Node/Node.php#L137-L146)) +- TS3 community forum: "The filename itself is the result of the CRC32 checksum" (TeamSpeak staff) + +### 1.3 Implication for Cache Design + +Both avatars and icons are **content-addressed by the protocol itself**: + +| Asset | Content hash source | Same content across servers? | +|---|---|---| +| Avatar | `client_flag_avatar` = MD5 of bytes | Same bytes → same hash → same ID | +| Icon | `icon_id` = CRC32 of bytes | Same bytes → same CRC32 → same ID | + +This means a **flat content-addressed blob store** can achieve zero-duplication without any per-server directories, hardlinks, or ref-counting. + +--- + +## 2. Virtual Server Identity + +### 2.1 Server UID + +| Aspect | Value | +|---|---| +| Protocol field | `virtualserver_unique_identifier` | +| Type | `TYPE_STRING` (TeaSpeakLibrary `PropertyDefinition.h:22`) | +| Generated by | The server instance, on creation | +| Globally unique? | **Not guaranteed** — locally generated, no central registry | +| Stable? | Yes — persists across restarts of the same virtual server | + +**Key fact:** `virtualserver_unique_identifier` is generated by each TS3 server. Two physically different servers could theoretically produce the same UID. It is **not safe as a global cache key**. + +### 2.2 What Chanora Currently Tracks + +| Layer | Server identity fields | Source | +|---|---|---| +| Protocol adapter (`adapter.rs:1752-1755`) | `server_name`, `welcome_message`, `platform`, `version` | `state.server.*` from tsclientlib | +| DTO (`ServerSnapshot`) | `server_name`, `welcome_message`, `platform`, `version` | No UID field | +| Bridge (`BridgeSnapshot`) | Same as DTO | Same | +| Storage (bookmarks) | Keyed by `host` (hostname:port) | SQLite `WHERE host = ?1` | +| Core (recent servers) | `cfg.address` as host | Auto-saved on connect | + +**Chanora does not currently plumb `virtualserver_unique_identifier` through the DTO stack.** The field exists in tsclientlib's state but is not extracted. + +### 2.3 Implication for Cache Design + +Using `virtualserver_unique_identifier` as the sole cache key is risky (not globally unique). Using connection address (`host:port`) is safe but duplicates cache entries when the same server is accessed via different addresses. + +**Recommendation:** For a content-addressed blob store, server identity is only needed for per-server metadata (eviction, "clear cache for this server"), not for the blob key itself. The blob key is the content hash. + +--- + +## 3. Existing TS3 Client Cache Implementations + +### 3.1 Qint (tsclientlib-based, Tauri + Rust) + +**Architecture:** Per-server directory with SQLite metadata. + +``` +/files/// +``` + +**Avatar handling:** +- Avatar state stored per `(server, client)` row in SQLite +- On avatar hash change, deletes the cached `/avatar_` file for that server +- Avatar download path: `/avatar_` + +**Icon handling:** +- Icons path-cached with CRC32 +- Code comments note CRC32 collisions and freshness by mtime +- Qint explicitly deletes and re-downloads when icon mtime changes + +**Dedup:** None. Same avatar on 5 servers = 5 stored copies. + +**Sources:** +- [`Qint/proxy/src/filecache.rs`](https://github.com/ReSpeak/Qint/blob/7efe949adfa1a1ecb9d185e7740e015da18cc41b/proxy/src/filecache.rs#L1-L6): "Stores files transferred via the TS3 file transfer protocol. This includes icons and avatars." +- [`Qint/proxy/src/db/mod.rs`](https://github.com/ReSpeak/Qint/blob/7efe949adfa1a1ecb9d185e7740e015da18cc41b/proxy/src/db/mod.rs#L1158-L1176): avatar hash change triggers delete +- [`Qint/src-tauri/src/cmd.rs`](https://github.com/ReSpeak/Qint/blob/7efe949adfa1a1ecb9d185e7740e015da18cc41b/src-tauri/src/cmd.rs#L524-L539): file download command + +### 3.2 TS3 Official Client (closed source) + +**Architecture:** Lazy cache with SDK callbacks. + +- `getAvatar()` returns cached path if present; otherwise triggers download +- `onAvatarUpdated` callback fires when avatar is downloaded or deleted +- Cache paths (from community documentation): + - Windows: `%LOCALAPPDATA%\TeamSpeak\Cache\Default` + - Linux: `~/.cache/TeamSpeak/Default` + - macOS: `~/Library/Caches/TeamSpeak/Default` +- SDK also exposes `CLIENT_MYTS_AVATAR` / `client_myteamspeak_avatar` for cross-server myTeamSpeak avatars + +**Sources:** +- [`ts3client-pluginsdk/src/plugin.c`](https://github.com/teamspeak/ts3client-pluginsdk/blob/4aa90a53aa150cbf81e13bc97e68c0431b26499f/src/plugin.c#L384-L396): `getAvatar()` and `onAvatarUpdated` +- [`ts3client-pluginsdk/public_rare_definitions.h`](https://github.com/teamspeak/ts3client-pluginsdk/blob/4aa90a53aa150cbf81e13bc97e68c0431b26499f/include/teamspeak/public_rare_definitions.h#L284-L313): `CLIENT_FLAG_AVATAR`, `CLIENT_MYTS_AVATAR` +- Community: [clear cache](https://community.teamspeak.com/t/clear-cache/41511), [broken icons](https://community.teamspeak.com/t/server-icons-are-displaying-a-broken-image-issues-with-local-cache/58680) + +### 3.3 TeaSpeak Client (TypeScript + C++ native) + +**Architecture:** Browser Cache API for images, per-server own-avatar storage. + +**Key components (from `.d.ts` type declarations):** + +- `AvatarManager` — per-connection (`FileManager`) avatar handler + - `cachedAvatars` (private) — in-memory cache of `ClientAvatar` objects + - `updateCache(clientAvatarId, clientAvatarHash)` — updates cache when hash changes + - `resolveAvatar(clientAvatarId, avatarHash?, cacheOnly?)` — resolves avatar by ID + - `flush_cache()` — clears cache + - `create_avatar_download(client_avatar_id)` — initiates file transfer + +- `ClientAvatar` — tracks individual avatar state + - `clientAvatarId` — derived from client UID via `uniqueId2AvatarId()` + - `currentAvatarHash` — the `client_flag_avatar` value + - State machine: `unset` → `loading` → `loaded` / `errored` + - `loadingTimestamp` — when download started + +- `ImageCache` — generic image cache using browser Cache API + - `resolveCached(key, maxAge?)` — check if cached + - `putCache(key, value, type?, headers?)` — store + - `cleanup(maxAge)` — evict old entries + - `reset()` — clear all + - `isPersistent()` — whether cache persists to disk + +- `OwnAvatarStorage` — user's own avatar, keyed by `serverUniqueId + mode` + - `loadAvatarImage(serverUniqueId, mode)` — load own avatar for a server + - `updateAvatar(serverUniqueId, mode, target)` — update own avatar + - `avatarUploadSucceeded(serverUniqueId)` — move from "uploading" to "server" state + - Stores `LocalAvatarInfo`: fileName, fileSize, **fileHashMD5**, timestamps, contentType + +- `FileManager` — per-connection file transfer manager + - `MAX_CONCURRENT_TRANSFERS` — transfer concurrency limit + - `avatars: AvatarManager` — avatar subsystem + - `initializeFileDownload(options)` — start download (path, name, channel, target) + - `deleteIcon(iconId: number)` — delete icon by ID + +- `FileTransfer` — transfer state machine + - States: `PENDING → INITIALIZING → CONNECTING → RUNNING → FINISHED / ERRORED / CANCELED` + - `InitializedTransferProperties`: serverTransferId, transferKey, **addresses[]**, protocol, seekOffset, fileSize + - Multiple addresses returned by server for file transfer (failover) + +- `localIconCache: ImageCache` — global icon cache (singleton) + +**Sources:** +- TeaSpeak-Client `imports/shared-app/file/Avatars.d.ts` — ClientAvatar, AbstractAvatarManager +- TeaSpeak-Client `imports/shared-app/file/LocalAvatars.d.ts` — AvatarManager +- TeaSpeak-Client `imports/shared-app/file/LocalIcons.d.ts` — localIconCache +- TeaSpeak-Client `imports/shared-app/file/ImageCache.d.ts` — ImageCache (browser Cache API) +- TeaSpeak-Client `imports/shared-app/file/FileManager.d.ts` — FileManager, transfer API +- TeaSpeak-Client `imports/shared-app/file/Transfer.d.ts` — FileTransfer, state machine, error types +- TeaSpeak-Client `imports/shared-app/file/OwnAvatarStorage.d.ts` — own avatar per-server storage +- TeaSpeak-Client `native/serverconnection/test/js/ft.ts` — file transfer test (TCP + ftkey protocol) + +### 3.4 TS3AudioBot (C#) + +**Architecture:** No local avatar cache. Avatar upload is hash-driven. + +- Uploads avatar bytes to `/avatar`, computes MD5, sets `client_flag_avatar` to that hash +- Bot avatar selection reads local files from an `avatars/` directory +- No caching of other users' avatars + +**Sources:** +- [`TS3AudioBot/TSLib/TsBaseFunctions.cs:324-341`](https://github.com/Splamy/TS3AudioBot/blob/a69a38d8cba5a4d671dbe06505506f6b46f1d947/TSLib/TsBaseFunctions.cs#L324-L341) +- [`TS3AudioBot/Bot.cs:420-470`](https://github.com/Splamy/TS3AudioBot/blob/a69a38d8cba5a4d671dbe06505506f6b46f1d947/TS3AudioBot/Bot.cs#L420-L470) + +### 3.5 TS3 NodeJS Library + +**Architecture:** No local cache. Downloads on demand. + +- Avatar filename: `avatar_${clientBase64HashClientUID}` +- `getAvatar()` downloads directly — no caching layer +- Tests assert the exact `/avatar_` path + +**Sources:** +- [`TS3-NodeJS-Library/src/node/Client.ts:300-313`](https://github.com/Multivit4min/TS3-NodeJS-Library/blob/0c69b7ee80fa5b74e9175cf4ae3018346f7eb300/src/node/Client.ts#L300-L313) +- [`TS3-NodeJS-Library/tests/Client.spec.ts:342-358`](https://github.com/Multivit4min/TS3-NodeJS-Library/blob/0c69b7ee80fa5b74e9175cf4ae3018346f7eb300/tests/Client.spec.ts#L342-L358) + +### 3.6 Summary Table + +| Client | Cache Key Strategy | Dedup Across Servers? | Icon Cache | +|---|---|---|---| +| **Qint** | `//` | No | Yes (CRC32, mtime freshness) | +| **TS3 Official** | Lazy cache (path-based) | Unknown | Yes | +| **TeaSpeak** | UID-derived avatar ID + browser Cache API | Implicit (same hash = same cache) | Yes (global ImageCache) | +| **TS3AudioBot** | None | N/A | No | +| **TS3 NodeLib** | None | N/A | No | +| **Chanora (decided)** | Content hash (MD5/CRC32), flat `blobs/` in `chanora_cache` crate | **Yes** | Yes | + +--- + +## 4. TeaSpeak Protocol Definitions (Authoritative) + +From TeaSpeakLibrary `src/PropertyDefinition.h` — the most complete open-source reference for TS3 protocol property types: + +### 4.1 Avatar Properties + +```cpp +// Line 200 +PropertyDescription{CLIENT_FLAG_AVATAR, "client_flag_avatar", "", + TYPE_STRING, FLAG_CLIENT_VIEW | FLAG_SAVE | FLAG_USER_EDITABLE} +// "automatically up-to-date for any manager 'in view', this manager got an avatar" +``` + +### 4.2 Icon Properties + +```cpp +// Line 83 — server icon +PropertyDescription{VIRTUALSERVER_ICON_ID, "virtualserver_icon_id", "0", + TYPE_UNSIGNED_NUMBER, FLAG_SERVER_VVSS | FLAG_USER_EDITABLE} + +// Line 146 — channel icon +PropertyDescription{CHANNEL_ICON_ID, "channel_icon_id", "0", + TYPE_UNSIGNED_NUMBER, FLAG_CHANNEL_VIEW | FLAG_SS | FLAG_USER_EDITABLE} + +// Line 217 — client icon +PropertyDescription{CLIENT_ICON_ID, "client_icon_id", "0", + TYPE_UNSIGNED_NUMBER, FLAG_CLIENT_VIEW | FLAG_CLIENT_VARIABLE} +``` + +### 4.3 Server Identity + +```cpp +// Line 22 +PropertyDescription{VIRTUALSERVER_UNIQUE_IDENTIFIER, + "virtualserver_unique_identifier", "", + TYPE_STRING, FLAG_SERVER_VV | FLAG_SNAPSHOT} +``` + +### 4.4 File Transfer Permissions + +```cpp +// From PermissionManager.cpp +PermissionType::i_client_max_avatar_filesize // "Max avatar filesize in bytes" +PermissionType::b_client_avatar_delete_other // "Allow deletion of avatars from other clients" +PermissionType::b_ft_transfer_list // "Retrieve list of running filetransfers" +``` + +### 4.5 File Transfer Error Codes + +```cpp +// From Error.h +channel_no_filetransfer_supported = 0x30C +file_transfer_connection_timeout = 0x80E +file_transfer_complete = 0x811 +file_transfer_canceled = 0x812 +file_transfer_interrupted = 0x813 +file_transfer_server_quota_exceeded = 0x814 +file_transfer_client_quota_exceeded = 0x815 +file_transfer_reset = 0x816 +file_transfer_limit_reached = 0x817 +``` + +--- + +## 5. Cross-Platform Filesystem Research + +### 5.1 Hardlink Support + +| Platform | Filesystem | Hardlinks in App-Private Storage? | Gotcha | +|---|---|---|---| +| Android (API 28+) | ext4 / f2fs | **Yes** | Rust uses `libc::link`; not FUSE-mounted; same-filesystem only | +| iOS | APFS | **Yes** (writable sandbox dirs) | App bundle is read-only; avoid hardlinks to bundle assets | +| macOS | APFS | **Yes** | — | +| Linux | ext4 / btrfs / xfs | **Yes** | — | +| Windows | NTFS | **Yes** | Rust uses `CreateHardLinkW` | + +**`std::fs::hard_link` gotchas (all platforms):** +- Same filesystem required +- Destination must not exist (returns error) +- Symlink behavior is platform-specific +- All hardlinks share the same inode — modifying one modifies all +- Files must be treated as **immutable** for hardlink safety + +**Sources:** +- Rust stdlib: `hard_link` maps to `libc::link` (Unix), `CreateHardLinkW` (Windows) ([Rust source](https://github.com/rust-lang/rust/blob/beae781308e9ddef13074a03faf57ca2fac59a5b/library/std/src/fs.rs#L2898-L2900)) +- Android: internal storage uses ext4/f2fs, not FUSE ([Android scoped storage docs](https://source.android.com/docs/core/storage/scoped)) +- iOS: APFS supports hardlinks; writable sandbox directories work ([Apple FileSystem basics](https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html)) + +### 5.2 Rust Cache Libraries + +**`cacache`** (MIT licensed, production-ready): +- Content-addressed disk cache +- Automatic dedup, atomic writes, integrity verification +- Exposes `hard_link`, `copy`, and `reflink` paths for retrieval +- On-disk layout: `content-v2/sha512/...` +- Could replace a custom implementation, but adds a dependency + +**Sources:** +- [`cacache-rs` README](https://github.com/zkat/cacache-rs/blob/105692a4daa04ce5f5ef3f8688cd3e1c1fb6a7c0/README.md#L39-L60) +- [`cacache-rs` content path](https://github.com/zkat/cacache-rs/blob/105692a4daa04ce5f5ef3f8688cd3e1c1fb6a7c0/src/content/path.rs#L6-L19) +- [`cacache-rs` hard_link impl](https://github.com/zkat/cacache-rs/blob/105692a4daa04ce5f5ef3f8688cd3e1c1fb6a7c0/src/content/read.rs#L257-L285) + +### 5.3 Flutter Cache Patterns + +Common Flutter packages use **cache-dir + metadata DB**, not hardlink dedup: +- `flutter_cache_manager`: files in cache dir + `sqflite` metadata +- `super_cache_disk`: file-per-entry (`.dat` + `.meta`) in app cache dir + +**Sources:** +- [flutter_cache_manager on pub.dev](https://pub.dev/packages/flutter_cache_manager) +- [super_cache_disk on pub.dev](https://pub.dev/packages/super_cache_disk/versions/1.0.0) + +--- + +## 6. Chanora Codebase Context + +### 6.1 Storage Patterns + +| Component | Pattern | Location | +|---|---|---| +| Identity storage | Atomic write (temp + `sync_all` + `rename`), mode 0600 on Unix | `chanora_storage/src/lib.rs:446-476` | +| Metadata | Same atomic write pattern | `chanora_storage/src/lib.rs:480-515` | +| Bookmarks | SQLite at `/chanora.db`, keyed by `host` | `chanora_storage/src/lib.rs:782-928` | +| Storage root | `getApplicationSupportDirectory()` from Flutter | `app_bootstrap.dart:19-24` | +| Bridge init | `rust.initStorage(dir: dir)` | `app_bootstrap.dart:23-25,108-133` | + +### 6.2 Existing Avatar Handling + +| Component | What | Location | +|---|---|---| +| UID to avatar path | `uid_to_avatar_path()` — base64 decode UID, encode each byte as 2 chars (a-p) | `adapter.rs:1561-1568` | +| Client profile DTO | `avatar_path` field — set when `client.avatar_hash` and `unique_id` non-empty | `dto.rs:135-136`, `adapter.rs:1323-1332` | +| No download | Currently no file download implementation exists | — | +| No icon handling | No icon field/path in protocol DTO or adapter | — | + +### 6.3 Server Identity in Chanora + +Chanora currently tracks servers by **connection address** (`host:port`), not by server UID: + +- Bookmarks: `WHERE host = ?1` +- Recent servers: auto-saved by `cfg.address` +- Prefetch cache: keyed by normalized host +- ServerSnapshot: has `server_name` but no `server_uid` + +The `virtualserver_unique_identifier` field is available from tsclientlib's state but is **not extracted** by the adapter. + +### 6.4 tsclientlib File Transfer API + +| Aspect | Detail | +|---|---| +| Download method | `Connection::download_file()` | +| Stream items | `StreamItem::FileDownload(FileDownloadResult { size, stream })` | +| Failure | `StreamItem::FiletransferFailed(handle, error)` | +| TCP handling | tsclientlib handles TCP connection + ftkey writing automatically | +| Chanora's job | Read `size` bytes from the returned `TcpStream` | +| Async behavior | `StreamItem::FileDownload` fires asynchronously, not inline with the request | + +### 6.5 ts-bookkeeping Generated Fields + +From the generated parser in `target/debug/build/ts-bookkeeping-*/out/`: + +- `virtual_server_id: u64` — numeric, per-virtual-server, may change across restarts +- `virtual_server_uid` — string, the `virtualserver_unique_identifier` + +Both are available in the `InInitServer` struct from the init handshake but are not currently plumbed through. + +--- + +## 7. TeaSpeak Server Internals (Authoritative) + +Source: TeaSpeak Server at `https://git.did.science/TeaSpeak/Server/Server` (branch `new-groups`, commit `b54c6d4e`). + +### 7.1 Avatar ID Derivation — Server Side + +The server derives the avatar filename from the **client UID**, not from the avatar content: + +```cpp +// DataClient.cpp:242-244 +std::string DataClient::getAvatarId() { + return hex::hex(base64::validate(this->getUid()) ? base64::decode(this->getUid()) : this->getUid(), 'a', 'q'); +} +``` + +The same transform produces `client_base64HashClientUID` (shown to other clients): + +```cpp +// client.cpp:1113-1114 +bulk.put_unchecked("client_base64HashClientUID", + hex::hex(base64::validate(info->client_unique_id) ? base64::decode(info->client_unique_id) : info->client_unique_id, 'a', 'q')); +``` + +**This matches Chanora's existing `uid_to_avatar_path()` in `adapter.rs:1561-1568`.** + +### 7.2 Avatar Upload Path + +When a client uploads an avatar, the server stores it as `/avatar_`: + +```cpp +// file.cpp:696-702 +} else if (cmd["path"].as().empty() && cmd["name"].string() == "/avatar") { + ... + info.file_path = "/avatar_" + this->getAvatarId(); + transfer_response = file::server()->file_transfer().initialize_avatar_transfer(...); +} +``` + +The avatar file path is identity-based (from UID), not content-based. + +### 7.3 `client_flag_avatar` — Who Computes the Hash? + +**The CLIENT computes the MD5 and sends it to the server during upload.** The server stores it as a string property (`FLAG_USER_EDITABLE`). The server does NOT compute or verify the hash. + +This means `client_flag_avatar` is: +- Set by the uploading client +- Stored verbatim by the server +- Broadcast to other clients as part of the client properties +- A reliable content fingerprint: same avatar bytes → same MD5 → same `client_flag_avatar` on any server + +### 7.4 Icon IDs — Server Does NOT Compute CRC32 + +Icon IDs are **permission values**, not content hashes computed by the server: + +```cpp +// ConnectedClient.cpp:186-210 — client icon ID from permissions +auto permission_flags = local_permissions->permission_flags(permission::i_icon_id); +new_icon_id = value.value; +updated_client_properties.emplace_back(property::CLIENT_ICON_ID); +``` + +```cpp +// channel.cpp:1495-1504 — channel icon ID +if(key == property::CHANNEL_ICON_ID) { + auto icon_id = converter::from_string_view(value); + channel->permissions()->set_permission(permission::i_icon_id, { ... icon_id ... }); +} +``` + +```cpp +// server.cpp:76-89 — server icon ID +SERVEREDIT_CHK_PROP_CACHED("virtualserver_icon_id", permission::b_virtualserver_modify_icon_id, int64_t) +``` + +**The CLIENT computes the CRC32 during upload and uses it as the filename `/icon_`.** The server stores the file and records the ID as a permission value. No server-side CRC32 or MD5 computation exists. + +### 7.5 Per-Server Storage Layout + +Avatars and icons are stored **per virtual server** on the server's filesystem: + +```cpp +// LocalFileSystem.cpp:39-45 +fs::path LocalFileSystem::server_path(const std::shared_ptr &server) { + return fs::u8path(this->root_path_) / fs::u8path("server_" + std::to_string(server->server_id())); +} +// target_path = this->server_path(server) / "icons" / path; +// target_path = this->server_path(server) / "avatars" / path; +``` + +``` +/ + server_/ + avatars/ + /avatar_ ← one per client who uploaded + icons/ + /icon_ ← one per unique icon +``` + +### 7.6 File Transfer Protocol (Server Side) + +Upload/delete/query routing: + +```cpp +// file.cpp:273-341 — delete routing +if (first_entry_name.find("/icon_") == 0 && file_path.empty()) { ... delete_icons(...); } +else if (first_entry_name.starts_with("/avatar_") && file_path.empty()) { ... delete_avatars(...); } +``` + +```cpp +// file.cpp:483-523 — query routing +if (first_entry_name.find("/icon_") == 0 && file_path.empty()) { ... query_icon_info(...); } +else if (first_entry_name.starts_with("/avatar_") && file_path.empty()) { ... query_avatar_info(...); } +``` + +Transfer initialization returns ftkey and metadata: + +```cpp +// file.cpp:759-761 +result.put_unchecked(0, "ftkey", transfer->transfer_key); +result.put_unchecked(0, "seekpos", transfer->file_offset); +``` + +```cpp +// file.cpp:887-899 +result.put_unchecked(0, "ftkey", transfer->transfer_key); +result.put_unchecked(0, "proto", "1"); +result.put_unchecked(0, "size", transfer->expected_file_size); +``` + +### 7.7 Key Takeaway for Chanora + +| Who does what | Avatar | Icon | +|---|---|---| +| **Uploader (client)** computes | MD5 of avatar bytes → sets `client_flag_avatar` | CRC32 of icon bytes → filename `/icon_` | +| **Server** does | Stores file as `/avatar_`, saves property | Stores file as `/icon_`, saves permission | +| **Other clients** receive | `client_flag_avatar` (MD5) as a property update | `icon_id` (CRC32) as a property update | +| **Chanora cache key** | `av_.dat` — content fingerprint | `ic_.dat` — content fingerprint | + +The content hash is computed once (by the uploader) and then broadcast as a property. Chanora never needs to hash anything — it just uses the protocol-provided values as cache keys. + +--- + +## 8. Design Implications + +### 8.1 The Core Insight + +The TS3 protocol provides content hashes as part of normal server-to-client updates: + +| Event | Data provided by server | What Chanora gets for free | +|---|---|---| +| Client enters view | `client_flag_avatar` = MD5 of avatar bytes | Content key for blob store | +| Channel update | `channel_icon_id` = CRC32 of icon bytes | Content key for blob store | +| Server update | `virtualserver_icon_id` = CRC32 of icon bytes | Content key for blob store | + +No hashing needed on the client side. The protocol is **already content-addressed**. + +### 8.2 Recommended Cache Architecture + +``` +/chanora/ + blobs/ ← cacache content store root + content-v2/ ← content-addressed by SHA-512 + /data ← raw blob bytes + index-v2/ ← key → content mapping +``` + +Where `` is the platform cache directory (not the support directory used by `chanora_storage`). Chanora's `BlobCache` maps protocol keys (`av_`, `ic_`) to `cacache` string keys. Physical layout is managed by `cacache`. + +**Lookup flow:** +1. Server sends `client_flag_avatar = "a1b2c3d4..."` for user X +2. Check: does `blobs/av_a1b2c3d4....dat` exist? +3. Yes → use it, zero downloads (works for ANY server) +4. No → download from `/avatar_` → save as `blobs/av_a1b2c3d4....dat` + +**Same for icons with `ic_.dat`.** + +### 8.3 Why This Beats Alternatives + +| Approach | Dedup | Globally unique key | Needs server UID plumbing | Needs hardlinks | Complexity | +|---|---|---|---|---|---| +| `/.dat` | No | No (UID not guaranteed unique) | Yes | Optional | Medium | +| `_/.dat` | No | Yes | No | Optional | Medium | +| `_/.dat` + hardlinks | Yes | Yes | No | Yes | Medium-High | +| **`blobs/av_.dat` (flat)** | **Yes** | **Yes (content hash)** | **No** | **No** | **Low** | + +### 8.4 Trade-offs + +| Pro | Con | +|---|---| +| Zero duplication across all servers | "Clear cache for server X only" requires metadata layer (Phase 3+) | +| No hardlinks needed | Orphan cleanup requires scanning for unreferenced blobs | +| No server UID plumbing needed | Cannot distinguish same-hash-different-content for icons (CRC32 collision) | +| Simplest possible implementation | — | +| Freshness = hash change = different filename (automatic) | — | +| Cross-platform (just file I/O) | — | + +### 8.5 Phased Implementation + +| Phase | What | Delivers | +|---|---|---| +| 1 | Protocol download (raw bytes via adapter, no cache) | Working download pipeline | +| 2 | Bridge + Flutter display (`Image.memory()`) | Visible avatars in UI | +| 3 | `chanora_cache` crate: cacache-backed blob cache, separate crate, cache dir, mtime eviction | Zero re-downloads, zero duplication | +| 4 | Session orchestration (coalescing, rate limiting, negative cache) | Anti-flood, robustness | +| 5 | Cache management (clear all, orphan cleanup, optional per-server metadata) | User control | + +--- + +## 9. Resolved Questions + +### Q1: Icon CRC32 Collisions — Accept with Size Guard + +**Risk assessment:** CRC32 produces a 32-bit hash. For N unique icons, the Birthday paradox gives collision probability ≈ N² / (2 × 2³²). + +| Icons (N) | Collision probability | +|---|---| +| 100 | ~0.0001% (negligible) | +| 1,000 | ~0.01% (negligible) | +| 10,000 | ~1.2% (marginal) | +| 65,536 | ~50% (likely) | + +A single user typically encounters fewer than 1,000 unique icons across all servers. The practical collision risk is negligible. + +**What happens on collision:** Wrong icon displayed for a channel/client/server. This is a visual glitch, not a security issue. The icon will appear incorrect until the cache is cleared. + +**Existing practice:** Qint explicitly notes CRC32 collisions (`filecache.rs:4`) but does NOT guard against them — they only refresh icons by mtime. No other TS3 client guards against CRC32 collisions. + +**Recommendation:** Accept CRC32 as the cache key. Add a lightweight **file size guard**: when downloading an icon, if `ic_.dat` already exists but has a different size than the `ftinitdownload` response reported, re-download. File size is available from the protocol (`msg.size` in `InFileDownloadPart`). This catches most collisions (different content = different size with high probability) without computing a secondary hash. + +**Decision:** CRC32 + file size guard. No SHA256 overhead needed. + +--- + +### Q2: `client_myteamspeak_avatar` — Defer Indefinitely + +**What it is:** A string property (`Option` in ts-bookkeeping) broadcast alongside `client_flag_avatar`. It represents a myTeamSpeak cross-server avatar — a user linked to a myTeamSpeak account can set a global avatar that follows them across all servers. + +**Current state in Chanora's dependency chain:** +- ts-bookkeeping exposes it: `InInitServer` has `my_team_speak_avatar: Option` +- TeaSpeakLibrary tracks `client_myteamspeak_id` but not the avatar +- tsclientlib exposes it as a property on client state + +**Value for Chanora:** +- myTeamSpeak is a TeamSpeak-specific cloud service (account sync, cross-server features) +- Chanora is an independent client — no myTeamSpeak account integration is planned +- The property may contain a URL or identifier that requires myTeamSpeak API access to resolve +- Without myTeamSpeak integration, the avatar cannot be fetched + +**Recommendation:** Defer indefinitely. If Chanora ever integrates myTeamSpeak accounts, this can be handled as a separate avatar source (URL-based HTTP download) alongside the existing protocol-based avatar download. The cache architecture supports this — just add a different blob prefix (e.g., `mt_.dat`). + +**Decision:** Out of scope for MVP and foreseeable roadmap. + +--- + +### Q3: Cache Backing Store — `cacache` Wrapper in Separate `chanora_cache` Crate + +**Decision:** Use `cacache` as the backing store inside `chanora_cache`. Not a custom flat-file implementation. + +**Why cacache won over custom:** + +1. **Crash safety is production-tested.** `cacache` handles partial writes, power loss, crash mid-write. A custom implementation would need to get `sync_all` + atomic rename right — one bug = corrupted cache. Even though cache data is disposable (reconstructible from server), `cacache` eliminates this entire class of bugs. + +2. **Less code to maintain.** ~120 LOC wrapper vs ~200 LOC custom implementation. The hard parts (atomic writes, integrity, content dedup) are owned by `cacache`, tested by the npm ecosystem. + +3. **Integrity verification on every read.** SSRI verification detects corruption, bit rot, partial writes automatically. A custom impl would need to add this separately or accept silent corruption. + +4. **Content dedup by SHA-512.** Same avatar on two servers = stored once automatically. The protocol's MD5/CRC32 keys map to `cacache` string keys; content dedup happens at the SHA-512 layer underneath. + +**What about the downsides:** + +| Concern | Assessment | +|---|---| +| ~6 transitive deps | `sha2` already in tree via `chacha20poly1305`. `serde_json`, `tempfile`, `digest` are lightweight. Acceptable for the safety benefit. | +| SHA-512 overhead on every write/read | For <100KB avatars, SHA-512 takes ~0.1ms. Negligible. | +| Opaque on-disk format | `cacache` provides `ls()` API for enumeration and inspection. Not as simple as `ls blobs/` but adequate. | +| `cacache` has no built-in LRU eviction | We write a custom eviction pass using `cacache::ls()` + timestamp sort. ~20 lines. Same complexity as custom impl's eviction. | + +**Separate crate rationale:** + +- `chanora_cache` is separate from `chanora_storage` because cache data has different durability semantics (disposable vs persistent), different backup semantics (excluded vs included), and different directory placement (cache dir vs support dir). +- `chanora_cache` lives in the platform's cache directory (`getApplicationCacheDirectory()`). `chanora_storage` lives in the support directory (`getApplicationSupportDirectory()`). +- Bridge init is separate: `init_cache(cache_dir)` vs `init_storage(support_dir)`. + +**API design:** + +```rust +pub struct BlobCache { cache_dir: PathBuf, max_bytes: u64 } +impl BlobCache { + pub fn new(cache_dir: impl AsRef, max_bytes: u64) -> Result; + pub async fn put(&self, prefix: &str, key: &str, data: &[u8]) -> Result<(), BlobCacheError>; + pub async fn get(&self, prefix: &str, key: &str) -> Result>, BlobCacheError>; + pub async fn remove(&self, prefix: &str, key: &str) -> Result<(), BlobCacheError>; + pub async fn clear(&self) -> Result<(), BlobCacheError>; + pub async fn total_size(&self) -> Result; + pub async fn evict(&self) -> Result<(), BlobCacheError>; +} +``` + +All methods are async (cacache is async-native). Key validation at API boundary (`av_` = 32 hex chars, `ic_` = decimal digits). + +--- + +### Q4: Per-Blob Metadata — No Metadata Sidecars (Resolved) + +**Original options:** + +| Approach | Pros | Cons | +|---|---|---| +| SQLite (chanora.db) | ACID, queryable, already in use | Schema migration, couples cache to bookmark DB | +| JSON sidecar files | Simple, self-contained, easy to debug | Write amplification (2 files per blob), concurrent write risk | +| In-memory only | Simplest | Lost on restart, can't do orphan cleanup offline | +| **No metadata (mtime-based)** | **Simplest, zero write amplification, 1 file per blob** | **No per-blob metadata beyond mtime** | + +**Why no metadata is sufficient:** + +1. **Content is immutable.** A given hash (MD5 or CRC32) always maps to the same bytes. There is no "stale content" problem — if the hash changes, it's a new file with a new name. No invalidation needed. + +2. **mtime = insertion time.** Since content is never modified after write, the filesystem mtime equals the time the blob was cached. This is sufficient for "delete oldest files first" eviction. + +3. **Write amplification avoided.** One file per blob (just the data) instead of two (data + JSON sidecar). For a cache that may hold thousands of small files, this matters. + +4. **Eviction is simple.** `walk dir → stat → sort by mtime → delete oldest`. No JSON parsing, no schema, no migration. + +5. **Per-server metadata deferred.** "Clear cache for server X only" and orphan cleanup are post-MVP features. If needed, a refs-layer can be added later without changing the blob layout. + +**Oracle consultation:** Oracle recommended this approach explicitly — no metadata files, mtime-based eviction, separate crate. The immutability guarantee makes metadata redundant. + +**Decision:** No metadata sidecars. One file per blob. Mtime-based eviction. Per-server metadata deferred to post-MVP. + +--- + +### Q5: File Transfer Address Failover — Not Needed + +**What the protocol provides:** + +The TeaSpeak client's `InitializedTransferProperties` returns `addresses[]` — an array of `{serverAddress, serverPort}`. The official TS3 client can try multiple addresses for failover. + +**What tsclientlib provides:** + +```rust +// tsclientlib/src/lib.rs:1373-1375 +let ip = msg.ip.unwrap_or_else(|| self.client.address.ip()); +let addr = SocketAddr::new(ip, msg.port); +TcpStream::connect(&addr).await +``` + +tsclientlib's `InFileDownloadPart` has `ip: Option` — **single IP only**, not an array. If the server provides an IP, it uses that. Otherwise, it falls back to the connection address. **No multi-address failover.** + +**What ts-bookkeeping parses:** + +```rust +pub struct InFileDownloadPart { + pub client_filetransfer_id: u16, + pub server_filetransfer_id: u16, + pub filetransfer_key: String, + pub port: u16, + pub size: u64, + pub protocol: u8, + pub ip: Option, // ← single optional IP +} +``` + +**The server's `notifystartdownload` response** sends `ip` as an optional single value, not an array. The TeaSpeak client's `addresses[]` is a higher-level abstraction (likely the client's own fallback logic), not a protocol feature. + +**Recommendation:** Chanora follows tsclientlib's existing behavior — use `msg.ip` or fallback to connection address. No custom failover logic needed. If the TCP connection fails, the download fails and retries follow the exponential backoff strategy from the design doc. + +**Decision:** Single address (from tsclientlib). No failover needed.