diff --git a/.gitignore b/.gitignore index d8a2f09a..9c975231 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ tools/trace_replay/work/ __pycache__/ *.py[cod] /.gradle +/.trace-work/ diff --git a/tools/device_bench/disagg/README.md b/tools/device_bench/disagg/README.md new file mode 100644 index 00000000..b03a6001 --- /dev/null +++ b/tools/device_bench/disagg/README.md @@ -0,0 +1,80 @@ +# Disaggregated-mode device perf harness + +A/B/C/D performance matrix and profiling harness for MobileGL's transport arms +(`monolith` / `inproc` / `spawn+shm` / `spawn+tcp`), built for the adb-attached +Android bench device. Written from Git Bash on a Windows host; requires +`MSYS_NO_PATHCONV=1` (the scripts set it themselves). + +Prerequisites: the disaggregated trace APK installed +(`./gradlew -p android-plugin :app:assembleTraceRelease -Pmobilegl.buildDisaggregated=ON +-Pmobilegl.buildDisaggregatedInproc=ON -Pmobilegl.pipePush=ON -Pmobilegl.debuggableRelease=true`, +then sign with the debug keystore and `adb install -r`), and trace fixtures under +`tools/trace_replay/fixtures/`. + +All scripts honor `SER=` (default `2f7cbe2e`) and +`PERF_WORK=` (default `/.trace-work/perf-compare`, git-ignored). + +## run_matrix.sh — perf matrix driver + +``` +bash run_matrix.sh [smoke|prep|openra|rd12|all] +``` + +Interleaved A/B/C/D rounds (monolith -> inproc -> shm -> tcp), N reps per arm +(`OPENRA_REPS` / `RD12_REPS`, default 3). Interleaving is the substitute for +frequency pinning on unrooted devices. Per run it force-stops the app, clears +stale results, starts/stops the tcp server for the tcp arm, verifies the arm +markers in logcat (`Config: MOBILEGL_TRANSPORT=...`, `spawn ARMED`, +`run-ahead ARMED` — a run without them is invalid), and pulls +`result.json` / `benchmark.json` / PipeStats dumps into +`$PERF_WORK//-r/`. + +benchmark mode never snapshots, so ssim stays -1; the matrix adds one +non-benchmark ssim round per workload/arm for the golden gate. + +## make_report.py — aggregate into REPORT.md + +``` +python make_report.py [results_dir] +``` + +Renders raw per-rep tables, best-of-N aggregates with spread, the ssim gate, +per-run arm verification, and merged client+server PipeStats rollups. +Note the spawn-arm traps: client-side `frames=0` is structural (use the server +dump), and logcat `MGPipe stats:` mixes non-counter lines — trust the JSON dumps. + +## profile_matrix.sh + run_profile_batch.sh — profiling lane + +Same runner with `PROF=1`: adds simpleperf (`perf_event_paranoid=-1` device) +for the client and server processes, ftrace sched capture for exact wakeup +latencies (shell-writable on this device), and the /proc sampler below. + +``` +bash run_profile_batch.sh # the canned batch; edit the run list inside +PROF=1 ONE=tcp WL=rd12 REP=1 bash profile_matrix.sh one # single arm +``` + +`ONE` / `WL` / `REP` / `EXTRA_ENV` / `EXTRA_SERVER_ENV` select and parameterize +a single run; e.g. `EXTRA_SERVER_ENV=MOBILEGL_IPC_SPIN_US=2000` for the +deep-idle-exit A/B. + +## sampler.c — device-side /proc sampler + +`sampler `. Per-tid blocked-syscall +histogram (`/proc//syscall`) + CPU/switch counters. Exists because strace +attach is SELinux-blocked for shell uid on the bench device, while same-uid +`/proc` reads via `run-as` work. Build for the device with the NDK +(`aarch64-linux-android*-clang`), push, run under `run-as`. + +## netbench.sh — Wi-Fi TCP throughput baseline + +WSL -> phone bulk transfer over `toybox nc` sink, 3 reps. The `nc` listener +must keep stdin open or it exits on EOF and RSTs the connection. + +## phone_strace.sh / fdprobe.sh / wifi_hang_repro.sh — cross-host hang kit + +Repro and evidence-capture for the cross-host control-plane hang (WSL client -> +phone server wedges at eglMakeCurrent; bytes ACKed, rx queue drained, session +child never sees POLLIN). `phone_strace.sh` attaches the device +`/system/bin/strace` as the app uid via `run-as` (shell-uid ptrace is blocked); +`fdprobe.sh` reconciles `/proc//fd` socket inodes against `/proc/net/tcp`. diff --git a/tools/device_bench/disagg/fdprobe.sh b/tools/device_bench/disagg/fdprobe.sh new file mode 100644 index 00000000..13220770 --- /dev/null +++ b/tools/device_bench/disagg/fdprobe.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Repro hang, then map child fds -> socket inodes and dump per-socket rx queues + supervisor fds. +set -u +BASE=~/wifi-perf +export MOBILEGL_TRANSPORT=spawn +export MOBILEGL_IPC_CONTROL=tcp://192.168.21.181:40613 +export MOBILEGL_IPC_DATA=stream +export MOBILEGL_IPC_TOKEN=wifibench-token-2026-09 +export MOBILEGL_IPC_RUN_AHEAD=1 +export MOBILEGL_IPC_COLD_START_MS=90000 +export MOBILEGL_PIPE_STATS=1 +cd ~/MobileGL +rm -rf "$BASE/out/fdprobe"; mkdir -p "$BASE/out/fdprobe" +./build-split/tools/trace_replay/mobilegl_trace_replay \ + --trace "$BASE/traces/openra.trace" --backend DirectGLES \ + --mobilegl-library ./build-split/libMobileGL.so --pbuffer-surface \ + --benchmark --benchmark-result="$BASE/out/fdprobe/benchmark.json" \ + --output "$BASE/out/fdprobe" > "$BASE/out/fdprobe/mobilegl.log" 2>&1 & +CPID=$! +echo "client pid=$CPID" +for i in $(seq 1 60); do + grep -q "MobileGL initialized" "$BASE/out/fdprobe/mobilegl.client.log" 2>/dev/null && break + sleep 1 +done +sleep 8 +echo "=== hang confirmed; dumping ===" +echo "=== CLIENT ss ===" +ss -tn dst 192.168.21.181:40613 +echo "=== CLIENT bytes ===" +ss -ti dst 192.168.21.181:40613 | grep -E 'bytes_sent|ESTAB' +sleep 1 +kill $CPID 2>/dev/null +echo DONE_LOCAL diff --git a/tools/device_bench/disagg/make_report.py b/tools/device_bench/disagg/make_report.py new file mode 100644 index 00000000..5916b583 --- /dev/null +++ b/tools/device_bench/disagg/make_report.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +"""Render the perf-compare report tables from summary.json + per-run artefacts. + +Usage: make_report.py [results_dir] +Defaults to /.trace-work/perf-compare (run_matrix.sh's output root). +""" +import json +import re +import statistics +import sys +from pathlib import Path + +ROOT = (Path(sys.argv[1]).resolve() if len(sys.argv) > 1 + else Path(__file__).resolve().parents[3] / ".trace-work" / "perf-compare") +ARMS = ["monolith", "inproc", "shm", "tcp"] +ARM_LABEL = { + "monolith": "monolith", + "inproc": "inproc", + "shm": "spawn+shm", + "tcp": "spawn+tcp-local", +} + + +def fmt(v, nd=3): + if v is None: + return "-" + if isinstance(v, float): + return f"{v:.{nd}f}" + return str(v) + + +def load_json(path): + try: + return json.loads(Path(path).read_text(encoding="utf-8")) + except Exception: + return None + + +def load(): + return json.loads((ROOT / "summary.json").read_text(encoding="utf-8")) + + +def perf_table(data, wl): + runs = [r for r in data["runs"] if r["workload"] == wl and r["mode"] == "benchmark"] + reps = sorted({r["rep"] for r in runs}, key=lambda x: int(x)) + lines = [ + f"### {wl} — raw perf runs (benchmark lane)", + "| rep | arm | status | fps | mean ms | median ms | p95 ms | frames | tail s | dur s | temp d°C |", + "|---|---|---|---|---|---|---|---|---|---|---|", + ] + for rep in reps: + for arm in ARMS: + r = next((x for x in runs if x["rep"] == rep and x["arm"] == arm), None) + if r is None: + lines.append(f"| {rep} | {ARM_LABEL[arm]} | MISSING | - | - | - | - | - | - | - | - |") + continue + lines.append( + f"| {rep} | {ARM_LABEL[arm]} | {r['status']} | {fmt(r['fps'],1)} | {fmt(r['mean_ms'])} " + f"| {fmt(r['median_ms'])} | {fmt(r['p95_ms'])} | {fmt(r['total_frames'],0)} " + f"| {fmt(r['total_seconds'],2)} | {fmt(r['duration_s'],0)} | {fmt(r['battery_temp_dc'],0)} |" + ) + return lines + + +def agg_table(data, wl): + aggs = [a for a in data["aggregates"] if a["workload"] == wl] + by_arm = {a["arm"]: a for a in aggs} + ns = ", ".join(str(by_arm[a]["n_ok"]) for a in ARMS if a in by_arm) + lines = [ + f"### {wl} — per-arm aggregates (best-of-N by mean; N={ns})", + "| arm | best fps | best mean ms | best median ms | best p95 ms | median-of-means ms | spread (min..max mean) | spread % |", + "|---|---|---|---|---|---|---|---|", + ] + for arm in ARMS: + a = by_arm.get(arm) + if not a: + lines.append(f"| {ARM_LABEL[arm]} | - | - | - | - | - | - | - |") + continue + lo, hi = a["spread_mean_ms"] + lines.append( + f"| {ARM_LABEL[arm]} | {fmt(a['best_fps'],1)} | {fmt(a['best_mean_ms'])} | {fmt(a['best_median_ms'])} " + f"| {fmt(a['best_p95_ms'])} | {fmt(a['median_of_mean_ms'])} | {fmt(lo)}..{fmt(hi)} | {fmt(a['spread_pct'],1)} |" + ) + return lines + + +def ssim_table(data, wl): + runs = [r for r in data["runs"] if r["workload"] == wl and r["mode"] == "ssim"] + lines = [ + f"### {wl} — ssim lane (non-benchmark run to target_call, compared to golden)", + "| arm | status | passed | ssim | message |", + "|---|---|---|---|---|", + ] + for arm in ARMS: + r = next((x for x in runs if x["arm"] == arm), None) + if r is None: + lines.append(f"| {ARM_LABEL[arm]} | MISSING | - | - | - |") + continue + msg = (r.get("message") or "")[:90] + lines.append(f"| {ARM_LABEL[arm]} | {r['status']} | {r['passed']} | {fmt(r.get('ssim'),6)} | {msg} |") + return lines + + +def marker_short(v): + tm = v.get("TRANSPORT_MARKER", v.get("SPLIT_MARKER", "?")) + if tm.endswith(":OK"): + return tm.rsplit(":", 1)[0] + ":OK" + if tm.endswith(":MISSING"): + return tm.rsplit(":", 1)[0] + ":MISSING" + return tm + + +def verify_table(data, wl): + runs = [r for r in data["runs"] if r["workload"] == wl and r["mode"] == "benchmark"] + lines = [ + f"### {wl} — arm verification (per run)", + "| rep | arm | transport marker | 2nd-process proof | run-ahead | lockstep/DISARMED | PipeStats lines | Fatal |", + "|---|---|---|---|---|---|---|---|", + ] + reps = sorted({r["rep"] for r in runs}, key=lambda x: int(x)) + for rep in reps: + for arm in ARMS: + r = next((x for x in runs if x["rep"] == rep and x["arm"] == arm), None) + if r is None: + continue + v = r.get("verify", {}) + second = v.get("SECOND_PROCESS_SPAWN_ARMED", v.get("TCP_SESSION", "-")) + second = "OK" if second.startswith("OK") else second + ra = "OK" if v.get("RUN_AHEAD_ARMED", "").startswith("OK") else v.get("RUN_AHEAD_ARMED", "?") + lock = "lockstep!" if v.get("LOCKSTEP_WARNING", "").startswith("PRESENT") else ( + "DISARMED!" if v.get("DISARMED", "").startswith("PRESENT") else "none") + lines.append( + f"| {rep} | {ARM_LABEL[arm]} | {marker_short(v)} | {second} | {ra} | {lock} " + f"| {v.get('PIPESTATS_LINES','-')} | {v.get('FATAL_LINES','-')} |" + ) + return lines + + +def linkmetrics_rollup(path): + """Sum wait_replies / stage_bytes / transport_wait_ns over kind=frame lines.""" + if not path.exists(): + return None + tot = {"frames": 0, "wait_replies": 0, "stage_bytes": 0, "transport_wait_ns": 0} + seen = set() + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if "P65LinkMetrics kind=frame " not in line: + continue + m_frame = re.search(r"frame=(\d+)", line) + if not m_frame: + continue + fid = int(m_frame.group(1)) + if fid in seen: + continue + seen.add(fid) + tot["frames"] += 1 + for k in ("wait_replies", "stage_bytes", "transport_wait_ns"): + m = re.search(k + r"=(\d+)", line) + if m: + tot[k] += int(m.group(1)) + return tot if seen else None + + +def pipestats_rollup(wl_dir, arm, rep): + """Merge client/server pipestats JSON dumps for one run into one counter dict.""" + d = wl_dir / f"{arm}-r{rep}" + if not d.is_dir(): + return None + dumps = [load_json(d / f) for f in ("pipestats.client.json", "pipestats.server.json", "tcp-server-pipestats.json")] + dumps = [x for x in dumps if x] + if not dumps: + return None + frames = max((x.get("frames", 0) or 0) for x in dumps) + by = lambda key, sub: sum((x.get(key, {}) or {}).get(sub, 0) or 0 for x in dumps) + wire = {} + for x in dumps: + for k, v in (x.get("wire", {}) or {}).items(): + wire[k] = wire.get(k, 0) + (v or 0) + lm = linkmetrics_rollup(d / "linkmetrics-lines.txt") + return { + "frames": frames, + "stage_seg": by("bytes", "stage-segment-bytes"), + "stage_buf": by("bytes", "stage-buffer"), + "stage_tex": by("bytes", "stage-texture"), + "stage_pmap": by("bytes", "persistent-map-push"), + "wire_records": by("calls", "wire-records"), + "draws": by("calls", "draws"), + "srv": wire.get("server-waits", 0), + "srvpark": wire.get("server-parks", 0), + "cli": wire.get("client-waits", 0), + "clipark": wire.get("client-parks", 0), + "maxrec": wire.get("max-record-bytes", 0), + "maxcap": wire.get("max-record-bytes-cap", 0), + "ringwraps": wire.get("ring-wraps", 0), + "ringwaits": wire.get("ring-waits", 0), + "lm": lm, + } + + +def pipiestats_table(wl): + lines = [ + f"### {wl} — PipeStats run totals (client+server JSON dumps merged) & LinkMetrics", + "| arm | rep | frames | srv | srvpark | cli | clipark | stage seg B | stage buf B | stage tex B | wire-records | maxrec/cap | " + "LM frames | LM wait_replies | LM stage B | LM transport-wait ms |", + "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|", + ] + wl_dir = ROOT / wl + if not wl_dir.is_dir(): + return ["(no data)"] + for arm in ARMS: + for rep in ("1", "2", "3"): + s = pipestats_rollup(wl_dir, arm, rep) + if s is None: + continue + lm = s["lm"] or {} + tw = lm.get("transport_wait_ns") + lines.append( + f"| {ARM_LABEL[arm]} | {rep} | {s['frames']} | {s['srv']} | {s['srvpark']} | {s['cli']} | {s['clipark']} " + f"| {s['stage_seg']} | {s['stage_buf']} | {s['stage_tex']} | {s['wire_records']} " + f"| {s['maxrec']}/{s['maxcap']} " + f"| {lm.get('frames','-')} | {lm.get('wait_replies','-')} | {lm.get('stage_bytes','-')} " + f"| {fmt(tw/1e6,1) if tw is not None else '-'} |" + ) + return lines + + +def main(): + data = load() + out = [ + "# MobileGL transport perf comparison — 2f7cbe2e (Qualcomm Adreno, Android 16, aarch64, no root)", + "", + "Arms: `monolith` / `inproc` / `spawn+shm` (fork control, shm data) / `spawn+tcp-local` " + "(tcp://127.0.0.1:40613 control, stream data). Interleaved A-B-C-D rounds, no root/no frequency " + "pinning — spread columns show run-to-run scatter.", + "", + "Perf numbers are the device's own benchmark summary over the trailing `tailFrames` window. " + "openra's 128-frame trace fits entirely in the default tail of 200 (so its `mean` carries the " + "one-off setup spikes of frames 1 and 5); rd12's 251 frames are summarized over the last 200, " + "which excludes the 6.2 s first-frame shader/asset setup.", + "", + ] + for wl in ("openra", "rd12"): + if not any(r["workload"] == wl for r in data["runs"]): + continue + out.append(f"## Workload: {wl}") + out.append("") + out += perf_table(data, wl) + out.append("") + out += agg_table(data, wl) + out.append("") + out += ssim_table(data, wl) + out.append("") + out += verify_table(data, wl) + out.append("") + out += pipiestats_table(wl) + out.append("") + (ROOT / "REPORT.md").write_text("\n".join(out) + "\n", encoding="utf-8") + print(f"wrote {ROOT / 'REPORT.md'}") + + +if __name__ == "__main__": + main() diff --git a/tools/device_bench/disagg/netbench.sh b/tools/device_bench/disagg/netbench.sh new file mode 100644 index 00000000..d0102246 --- /dev/null +++ b/tools/device_bench/disagg/netbench.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Wi-Fi TCP throughput WSL -> phone, 3 reps, sink to /dev/null, nc restarted per rep. +set -u +PORT=5213 +for rep in 1 2 3; do + adb -s 2f7cbe2e shell "pkill -f 'toybox nc' 2>/dev/null; nohup toybox nc -4 -l -p $PORT -q 180 >/dev/null 2>/data/local/tmp/nc.err {sent/dt:.1f} MiB/s ({sent*8/dt/1000:.0f} Mbps)') +except Exception as e: + print(f'rep: FAIL {e}') +EOF +" + adb -s 2f7cbe2e shell "pkill -f 'toybox nc' 2>/dev/null" >/dev/null + sleep 1 +done +echo NETBENCH_DONE diff --git a/tools/device_bench/disagg/phone_strace.sh b/tools/device_bench/disagg/phone_strace.sh new file mode 100644 index 00000000..7dc60acd --- /dev/null +++ b/tools/device_bench/disagg/phone_strace.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Phone-side: attach strace (as app uid) to the session child for 6s, dump head. +set -u +PID=$(adb -s 2f7cbe2e shell "ps -A -o PID,NAME,ELAPSED | awk '\$2==\"libMobileGLServer.so\" && \$1>15000 {print \$1}'" | tr -d '\r' | head -1) +echo "target child pid=$PID" +[ -z "$PID" ] && { echo NO_CHILD; exit 1; } +adb -s 2f7cbe2e shell "run-as top.mobilegl.plugin.trace sh -c '/system/bin/strace -p $PID -f -tt -e trace=recvfrom,recvmsg,read,write,writev,futex,ppoll,pselect6,epoll_wait,epoll_pwait -s 80 -o /data/data/top.mobilegl.plugin.trace/files/st.txt 2>/data/data/top.mobilegl.plugin.trace/files/st.err & S=\$!; sleep 6; kill \$S 2>/dev/null; wait \$S 2>/dev/null; echo ===ERR===; cat /data/data/top.mobilegl.plugin.trace/files/st.err; echo ===ST===; wc -l /data/data/top.mobilegl.plugin.trace/files/st.txt; head -150 /data/data/top.mobilegl.plugin.trace/files/st.txt'" 2>&1 diff --git a/tools/device_bench/disagg/profile_matrix.sh b/tools/device_bench/disagg/profile_matrix.sh new file mode 100644 index 00000000..26efc1aa --- /dev/null +++ b/tools/device_bench/disagg/profile_matrix.sh @@ -0,0 +1,527 @@ +#!/usr/bin/env bash +# Four-arm MobileGL transport performance matrix on device 2f7cbe2e. +# Arms (interleaved A B C D rounds): monolith / inproc / spawn+shm / spawn+tcp-localhost. +# +# Usage: run_matrix.sh [smoke|prep|openra|rd12|all] +# smoke - one openra monolith run to validate the pipeline, then exit +# prep - extract + push the rd12 fixture only +# openra - openra: 4 arms x 3 perf reps (interleaved) + 1 ssim rep/arm (interleaved) +# rd12 - minecraft rd12: perf reps (auto-drops to 2 if round 1 > 300s/run) + 1 ssim rep/arm +# all - openra + rd12 + summary (default) +# +# benchmark=true runs record fps/mean/median/p95 but never snapshot, so result.json +# carries ssim=-1; a separate non-benchmark round per workload supplies the ssim gate. +set -u +export MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL='*' + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +if command -v cygpath >/dev/null 2>&1; then REPO_ROOT_WIN=$(cygpath -w "$REPO_ROOT"); else REPO_ROOT_WIN="$REPO_ROOT"; fi +ROOT="${PERF_WORK:-$REPO_ROOT/.trace-work/perf-compare}" +SER="${SER:-2f7cbe2e}" +PKG=top.mobilegl.plugin.trace +APP=/data/user/0/$PKG/files/trace-replay +FILES=/data/user/0/$PKG/files +LOG="$ROOT/run_matrix.log" +FIXTURES="$REPO_ROOT/tools/trace_replay/fixtures" +OPENRA_TIMEOUT=${OPENRA_TIMEOUT:-900} +RD12_TIMEOUT=${RD12_TIMEOUT:-900} +OPENRA_REPS=${OPENRA_REPS:-3} +RD12_REPS=${RD12_REPS:-3} +ARMS=(monolith inproc shm tcp) + +shq() { adb -s "$SER" shell "$1"; } +execout(){ adb -s "$SER" exec-out "$1"; } + +log() { local line; line="[$(date '+%F %T')] $*"; echo "$line" >>"$LOG"; echo "$line"; } + +arm_env() { + local stats="MOBILEGL_PIPE_STATS=1;MOBILEGL_PIPE_STATS_FILE=$APP/output/pipestats.json" + local extra="${EXTRA_ENV:-}" + case "$1" in + monolith) echo "$stats$extra" ;; + inproc) echo "MOBILEGL_TRANSPORT=inproc;MOBILEGL_IPC_RUN_AHEAD=1;$stats$extra" ;; + shm) echo "MOBILEGL_TRANSPORT=spawn;MOBILEGL_IPC_CONTROL=fork;MOBILEGL_IPC_DATA=auto;MOBILEGL_IPC_RUN_AHEAD=1;$stats$extra" ;; + tcp) echo "MOBILEGL_TRANSPORT=spawn;MOBILEGL_IPC_CONTROL=tcp://127.0.0.1:40613;MOBILEGL_IPC_DATA=stream;MOBILEGL_IPC_RUN_AHEAD=1;$stats$extra" ;; + esac +} + +# Pull one run-as file to host; drops it when absent or when exec-out folded the +# device-side "No such file" error into the stdout stream (same rule as +# trace-replay-ci.sh's collect_role_logs). +pull() { + local src="$1" dst="$2" + rm -f "$dst" + execout "run-as $PKG cat $src" >"$dst" 2>/dev/null || true + if [ ! -s "$dst" ] || head -c 4096 "$dst" | grep -q "No such file"; then rm -f "$dst"; fi +} + +# ---- PROFILING HOOKS (profile harness only; the source tree is untouched) ---- +PROF_PIDS="$ROOT/.prof.pids" + +# prof_start : launched right after am start (background). Attaches simpleperf +# (cpu-clock + callgraph) to the client pid and the spawn server pid, starts the +# /proc-syscall sampler on both, and arms ftrace sched_* for wakeup-latency capture. +prof_start() { + local pdir="$1" waited=0 app="" server="" arm="${2:-}" + mkdir -p "$pdir" + : >"$PROF_PIDS" + while [ "$waited" -lt 30 ]; do + app=$(shq "pidof $PKG" 2>/dev/null | tr -d '\r' | awk '{print $1}') + [ -n "${app:-}" ] && [ "$app" -gt 0 ] 2>/dev/null && break + sleep 1; waited=$((waited + 1)) + done + if [ -z "${app:-}" ]; then echo "no app pid" >"$pdir/prof-err.txt"; return 1; fi + echo "app=$app" >"$pdir/prof-pids.txt" + # ftrace sched capture (shell can write tracing_on; ring keeps the MOST RECENT events) + shq "echo 2048 > /sys/kernel/tracing/buffer_size_kb; + echo 1 > /sys/kernel/tracing/events/sched/sched_waking/enable; + echo 1 > /sys/kernel/tracing/events/sched/sched_wakeup/enable; + echo 1 > /sys/kernel/tracing/events/sched/sched_switch/enable; + echo > /sys/kernel/tracing/trace; + echo 1 > /sys/kernel/tracing/tracing_on" >/dev/null 2>&1 + # sampler binary into the app's files dir + shq "run-as $PKG sh -c 'cp /data/local/tmp/sampler files/sampler 2>/dev/null; chmod 755 files/sampler'" >/dev/null 2>&1 + # spawn arms only: the server lives in a child process (pid is printed by the client log). + if [ "$arm" != "monolith" ] && [ "$arm" != "inproc" ] && [ -n "$arm" ]; then + waited=0 + while [ "$waited" -lt 25 ] && [ -z "$server" ]; do + local line s + line=$(execout "run-as $PKG cat $APP/output/mobilegl.client.log" 2>/dev/null | tr -d '\r') + s=$(printf '%s' "$line" | grep -ao "runs in pid [0-9]*" | tail -1) + [ -n "$s" ] && server="${s##* }" + if [ -z "$server" ]; then + s=$(printf '%s' "$line" | grep -ao "pid=[0-9]* dial" | tail -1) + [ -n "$s" ] && server="${s#pid=}" && server="${server%% *}" + fi + [ -n "$server" ] && break + sleep 1; waited=$((waited + 1)) + done + fi + echo "app=$app server=${server:-none} arm=$arm" >>"$pdir/prof-pids.txt" + # simpleperf on the client (runs until prof_stop sends SIGINT; 600s safety cap) + adb -s "$SER" shell "run-as $PKG simpleperf record -e cpu-clock -f 4000 -g --duration 600 -p $app -o $FILES/perf-client.data" \ + >"$pdir/perf-client.log" 2>&1 & + echo "perf-client $!" >>"$PROF_PIDS" + if [ -n "$server" ]; then + adb -s "$SER" shell "run-as $PKG simpleperf record -e cpu-clock -f 4000 -g --duration 600 -p $server -o $FILES/perf-server.data" \ + >"$pdir/perf-server.log" 2>&1 & + echo "perf-server $!" >>"$PROF_PIDS" + fi + # /proc syscall samplers: 2ms samples of every thread's blocked syscall + adb -s "$SER" shell "run-as $PKG sh -c 'files/sampler $app 590 2 files/sampler-client.csv'" \ + >"$pdir/sampler-client.log" 2>&1 & + echo "sampler-client $!" >>"$PROF_PIDS" + if [ -n "$server" ]; then + adb -s "$SER" shell "run-as $PKG sh -c 'files/sampler $server 590 2 files/sampler-server.csv'" \ + >"$pdir/sampler-server.log" 2>&1 & + echo "sampler-server $!" >>"$PROF_PIDS" + fi + echo "started app=$app server=${server:-none}" +} + +# prof_stop : terminate profilers, wait for simpleperf to finalize, pull artifacts. +prof_stop() { + local pdir="$1" + [ -d "$pdir" ] || return 0 + shq "run-as $PKG pkill -TERM sampler" >/dev/null 2>&1 || true + sleep 1 + shq "run-as $PKG pkill -INT simpleperf" >/dev/null 2>&1 || true + # simpleperf needs time to write the PERFILE2 trailer; poll the magic instead of a fixed sleep. + local w=0 magic="" + while [ "$w" -lt 60 ]; do + magic=$(execout "run-as $PKG sh -c 'head -c 8 $FILES/perf-client.data 2>/dev/null'" 2>/dev/null | tr -d '\0\r\n') + [ "$magic" = "PERFILE2" ] && break + shq "run-as $PKG pkill -INT simpleperf" >/dev/null 2>&1 || true + sleep 2; w=$((w + 2)) + done + sleep 2 + shq "echo 0 > /sys/kernel/tracing/tracing_on; + echo 0 > /sys/kernel/tracing/events/sched/sched_waking/enable; + echo 0 > /sys/kernel/tracing/events/sched/sched_wakeup/enable; + echo 0 > /sys/kernel/tracing/events/sched/sched_switch/enable" >/dev/null 2>&1 || true + pull "$FILES/perf-client.data" "$pdir/perf-client.data" + pull "$FILES/perf-server.data" "$pdir/perf-server.data" + pull "$FILES/sampler-client.csv" "$pdir/sampler-client.csv" + pull "$FILES/sampler-server.csv" "$pdir/sampler-server.csv" + execout "cat /sys/kernel/tracing/trace" >"$pdir/ftrace-sched.txt" 2>/dev/null || true + shq "run-as $PKG pkill -TERM sampler; run-as $PKG pkill -TERM simpleperf" >/dev/null 2>&1 || true + if [ -f "$PROF_PIDS" ]; then + while read -r _ job; do [ -n "${job:-}" ] && kill "$job" 2>/dev/null || true; done <"$PROF_PIDS" + rm -f "$PROF_PIDS" + fi + ls -la "$pdir" >"$pdir/manifest.txt" 2>&1 +} + +extract_lines() { + # $1 dir; pulls the two counter-line families out of every pulled log on the host. + local dir="$1" f + : >"$dir/pipestats-lines.txt" + : >"$dir/linkmetrics-lines.txt" + for f in "$dir"/logcat.txt "$dir"/mobilegl.client.log "$dir"/mobilegl.server.log \ + "$dir"/mgl.client.log "$dir"/mgl.server.log; do + [ -f "$f" ] || continue + grep -a "MGPipe stats:" "$f" 2>/dev/null | sed "s|^|[$(basename "$f")] |" >>"$dir/pipestats-lines.txt" + grep -a "P65LinkMetrics" "$f" 2>/dev/null | sed "s|^|[$(basename "$f")] |" >>"$dir/linkmetrics-lines.txt" + done +} + +verify_run() { + local dir="$1" arm="$2" + local all="$dir/all-marker-evidence.txt" + : >"$all" + local f + for f in "$dir"/logcat.txt "$dir"/mobilegl.client.log "$dir"/mobilegl.server.log \ + "$dir"/mgl.client.log "$dir"/mgl.server.log; do + [ -f "$f" ] && grep -aE "Config: MOBILEGL_TRANSPORT|spawn ARMED|control=tcp data=stream|run-ahead ARMED|running lockstep|DISARMED|Config: IPC|MGPipe stats:" "$f" 2>/dev/null | sed "s|^|[$(basename "$f")] |" >>"$all" + done + local v="$dir/verify.txt" nstats nfatal + nstats=$(grep -ac "MGPipe stats:" "$all" 2>/dev/null || true); [ -n "$nstats" ] || nstats=0 + nfatal=$(grep -ac "Fatal{" "$all" 2>/dev/null || true); [ -n "$nfatal" ] || nfatal=0 + { + echo "arm=$arm" + case "$arm" in + inproc) + grep -aq "Config: MOBILEGL_TRANSPORT=inproc - the MGPipe record stream" "$all" \ + && echo "TRANSPORT_MARKER=inproc:OK" || echo "TRANSPORT_MARKER=inproc:MISSING" ;; + shm) + grep -aq "Config: MOBILEGL_TRANSPORT=spawn - the MGPipe record stream" "$all" \ + && echo "TRANSPORT_MARKER=spawn:OK" || echo "TRANSPORT_MARKER=spawn:MISSING" + grep -aq "spawn ARMED - the server role runs in pid" "$all" \ + && echo "SECOND_PROCESS_SPAWN_ARMED=OK: $(grep -a "spawn ARMED" "$all" | head -1)" \ + || echo "SECOND_PROCESS_SPAWN_ARMED=MISSING" ;; + tcp) + grep -aq "Config: MOBILEGL_TRANSPORT=spawn - the MGPipe record stream" "$all" \ + && echo "TRANSPORT_MARKER=spawn:OK" || echo "TRANSPORT_MARKER=spawn:MISSING" + grep -aq "control=tcp data=stream server=" "$all" \ + && echo "TCP_SESSION=OK: $(grep -a "control=tcp data=stream" "$all" | head -1)" \ + || echo "TCP_SESSION=MISSING" ;; + monolith) + if grep -aqE "Config: MOBILEGL_TRANSPORT=(inproc|spawn)" "$all"; then + echo "SPLIT_MARKER=UNEXPECTED_PRESENT" + else + echo "SPLIT_MARKER=absent (monolith:OK)" + fi ;; + esac + if grep -aq "run-ahead ARMED" "$all"; then + echo "RUN_AHEAD_ARMED=OK: $(grep -a "run-ahead ARMED" "$all" | head -1)" + else + echo "RUN_AHEAD_ARMED=MISSING" + fi + if grep -aq "running lockstep" "$all"; then + echo "LOCKSTEP_WARNING=PRESENT: $(grep -a "running lockstep" "$all" | head -1)" + else + echo "LOCKSTEP_WARNING=absent" + fi + if grep -aqi "DISARMED" "$all"; then + echo "DISARMED=PRESENT: $(grep -ai "DISARMED" "$all" | head -1)" + else + echo "DISARMED=absent" + fi + echo "PIPESTATS_LINES=$nstats" + echo "FATAL_LINES=$nfatal" + } >"$v" +} + +# run_one [crop_x crop_y crop_w crop_h] +run_one() { + local wl="$1" arm="$2" rep="$3" trace="$4" golden="$5" w="$6" h="$7" target="$8" timeout_s="$9" mode="${10}" + local cx="${11:-0}" cy="${12:-0}" cw="${13:-0}" ch="${14:-0}" + local dir="$ROOT/$wl/$arm-r$rep" + rm -rf "$dir"; mkdir -p "$dir" + local epoch0 dur temp0 am_out status="" detail="" passflag="" + if [ "$mode" = "benchmark" ]; then passflag="--ez benchmark true --es benchmark_result_path $APP/output/benchmark.json"; fi + epoch0=$(date +%s) + log "BEGIN $wl/$arm rep$rep mode=$mode (timeout ${timeout_s}s)" + shq "am force-stop $PKG" >/dev/null 2>&1 || true + shq "run-as $PKG sh -c 'rm -f $APP/output/result.json $APP/output/benchmark.json $APP/output/mobilegl.client.log $APP/output/mobilegl.server.log $APP/output/pipestats.client.json $APP/output/pipestats.server.json $APP/output/retrace.log $FILES/mgl.client.log $FILES/mgl.server.log'" >/dev/null 2>&1 || true + adb -s "$SER" logcat -c >/dev/null 2>&1 || true + temp0=$(adb -s "$SER" shell dumpsys battery 2>/dev/null | tr -d '\r' | sed -n 's/.*temperature: \([0-9]*\).*/\1/p' | head -1) + + if [ "$arm" = "tcp" ]; then + if ! python "$REPO_ROOT_WIN/tools/trace_replay/tcp_device_server.py" start \ + --serial "$SER" --package "$PKG" --backend DirectGLES --listen tcp://127.0.0.1:40613 \ + --env "MOBILEGL_PIPE_STATS=1" --env "MOBILEGL_PIPE_STATS_FILE=$FILES/pipestats.json" \ + ${EXTRA_SERVER_ENV:+--env "$EXTRA_SERVER_ENV"} \ + >"$dir/server-start.log" 2>&1; then + dur=$(( $(date +%s) - epoch0 )); status="server_start_failed" + { echo "arm=$arm rep=$rep workload=$wl duration_s=$dur"; cat "$dir/server-start.log"; } >"$dir/meta.txt" + echo "$status" >"$dir/STATUS"; log "FAIL $wl/$arm rep$rep: tcp server start"; return 1 + fi + local waited=0 listening=0 + while [ "$waited" -lt 30 ]; do + if adb -s "$SER" logcat -d -s MobileGL:* MobileGLServer:* 2>/dev/null | tr -d '\r' | grep -q "listening on tcp://127.0.0.1:40613"; then + listening=1; break + fi + sleep 1; waited=$((waited + 1)) + done + if [ "$listening" -ne 1 ]; then + dur=$(( $(date +%s) - epoch0 )); status="server_not_listening" + adb -s "$SER" logcat -d -s MobileGL:* MobileGLServer:* >"$dir/server-logcat.txt" 2>/dev/null || true + { echo "arm=$arm rep=$rep workload=$wl duration_s=$dur listening_wait_s=$waited"; } >"$dir/meta.txt" + echo "$status" >"$dir/STATUS"; log "FAIL $wl/$arm rep$rep: tcp server not listening after ${waited}s"; return 1 + fi + log "tcp supervisor listening after ${waited}s" + fi + + local envarg + envarg=$(arm_env "$arm") + am_out=$(shq "am start -a top.mobilegl.plugin.TRACE_REPLAY -n $PKG/top.mobilegl.plugin.trace.TraceReplayActivity --es trace_path $trace --es golden_path $golden --es output_dir $APP/output --es backend DirectGLES --el target_call $target --ei width $w --ei height $h --es ssim_threshold 0.99 --ez use_pbuffer true $passflag --ei crop_x $cx --ei crop_y $cy --ei crop_width $cw --ei crop_height $ch --es mobilegl_env '${envarg}'" 2>&1) + printf '%s\n' "$am_out" >"$dir/am-start.txt" + if printf '%s' "$am_out" | grep -qE "Error type|Exception|does not exist|Unknown"; then + dur=$(( $(date +%s) - epoch0 )); status="am_start_failed" + { echo "arm=$arm rep=$rep workload=$wl duration_s=$dur"; echo "$am_out"; } >"$dir/meta.txt" + echo "$status" >"$dir/STATUS"; log "FAIL $wl/$arm rep$rep: am start error: $am_out"; return 1 + fi + + local deadline=$(( epoch0 + timeout_s )) got=0 + # ---- PROF hooks: attach simpleperf (client + server) and the /proc sampler ---- + if [ "${PROF:-0}" = "1" ]; then + local pdir="$ROOT/prof-$arm-r$rep" + mkdir -p "$pdir" + echo "$pdir" >"$ROOT/PROF_LAST_DIR" + ( prof_start "$pdir" "$arm" ) >"$pdir/prof-start.log" 2>&1 & + fi + while [ "$(date +%s)" -lt "$deadline" ]; do + if execout "run-as $PKG cat $APP/output/result.json" 2>/dev/null | head -c 400 | grep -q '"passed"'; then + got=1; break + fi + if [ $(( $(date +%s) - epoch0 )) -gt 25 ] && [ -z "$(adb -s "$SER" shell pidof $PKG 2>/dev/null | tr -d '\r' | tr -d ' ')" ]; then + status="process_died"; break + fi + sleep 4 + done + if [ "$got" -eq 1 ]; then status="finished"; sleep 3; elif [ -z "$status" ]; then status="timeout"; fi + if [ "${PROF:-0}" = "1" ] && [ -f "$ROOT/PROF_LAST_DIR" ]; then + prof_stop "$(cat "$ROOT/PROF_LAST_DIR")" >>"$(cat "$ROOT/PROF_LAST_DIR")/prof-start.log" 2>&1 || true + fi + + dur=$(( $(date +%s) - epoch0 )) + adb -s "$SER" logcat -d >"$dir/logcat.txt" 2>/dev/null || true + pull "$APP/output/result.json" "$dir/result.json" + pull "$APP/output/benchmark.json" "$dir/benchmark.json" + pull "$APP/output/mobilegl.client.log" "$dir/mobilegl.client.log" + pull "$APP/output/mobilegl.server.log" "$dir/mobilegl.server.log" + pull "$APP/output/pipestats.client.json" "$dir/pipestats.client.json" + pull "$APP/output/pipestats.server.json" "$dir/pipestats.server.json" + pull "$APP/output/retrace.log" "$dir/retrace.log" + if [ "$arm" = "tcp" ]; then + pull "$FILES/mgl.client.log" "$dir/mgl.client.log" + pull "$FILES/mgl.server.log" "$dir/mgl.server.log" + pull "$FILES/pipestats.server.json" "$dir/tcp-server-pipestats.json" + fi + + local passed="none" + if [ -f "$dir/result.json" ]; then + passed=$(sed -n 's/.*"passed": *\([a-z]*\).*/\1/p' "$dir/result.json" | head -1) + [ -n "$passed" ] || passed="unparsed" + fi + if [ "$passed" != "true" ] && [ "$status" = "finished" ]; then status="result_not_passed"; fi + { echo "arm=$arm rep=$rep workload=$wl duration_s=$dur battery_temp_dc=$temp0 status=$status passed=$passed mode=$mode" + echo "--- am start ---"; printf '%s\n' "$am_out"; } >"$dir/meta.txt" + echo "$status" >"$dir/STATUS" + extract_lines "$dir" + verify_run "$dir" "$arm" + local vlines + vlines=$(tr '\n' '; ' <"$dir/verify.txt") + log "END $wl/$arm rep$rep mode=$mode status=$status passed=$passed dur=${dur}s temp=$temp0 :: $vlines" + [ "$status" = "finished" ] || return 1 + return 0 +} + +check_openra() { + if ! shq "run-as $PKG sh -c 'test -f $APP/input/trace.trace && test -f $APP/input/golden.png'" >/dev/null 2>&1; then + log "FATAL: openra fixture missing on device ($APP/input)"; exit 1 + fi +} + +prep_rd12() { + local host_trace="$ROOT/fixtures/trace.trace" + local host_golden="$FIXTURES/minecraft-1.21.4-rd12-odinlite-in-world.0004660351.png" + [ -f "$host_trace" ] || { tar xzf "$FIXTURES/minecraft-1.21.4-rd12-odinlite-in-world.tgz" -C "$ROOT/fixtures" || exit 1; } + [ -f "$host_golden" ] || { log "FATAL: rd12 golden missing: $host_golden"; exit 1; } + shq "run-as $PKG sh -c 'mkdir -p $APP/input-rd12'" || true + log "pushing rd12 fixture ($(wc -c <"$host_trace") bytes)" + adb -s "$SER" push "$(cygpath -w "$host_trace")" /data/local/tmp/mgl-rd12.trace >/dev/null || exit 1 + adb -s "$SER" push "$(cygpath -w "$host_golden")" /data/local/tmp/mgl-rd12-golden.png >/dev/null || exit 1 + if ! shq "run-as $PKG sh -c 'cp /data/local/tmp/mgl-rd12.trace $APP/input-rd12/trace.trace'"; then + log "run-as cp failed; streaming via cat pipe" + shq "cat /data/local/tmp/mgl-rd12.trace | run-as $PKG sh -c 'cat > $APP/input-rd12/trace.trace'" || exit 1 + fi + shq "run-as $PKG sh -c 'cp /data/local/tmp/mgl-rd12-golden.png $APP/input-rd12/golden.png'" \ + || shq "cat /data/local/tmp/mgl-rd12-golden.png | run-as $PKG sh -c 'cat > $APP/input-rd12/golden.png'" || exit 1 + shq "rm -f /data/local/tmp/mgl-rd12.trace /data/local/tmp/mgl-rd12-golden.png" >/dev/null 2>&1 || true + local size hostsize + size=$(shq "run-as $PKG sh -c 'wc -c < $APP/input-rd12/trace.trace'" | tr -d '\r ') + hostsize=$(wc -c <"$host_trace") + if [ "$size" != "$hostsize" ]; then log "FATAL: rd12 trace size mismatch device=$size host=$hostsize"; exit 1; fi + log "rd12 fixture on device: $size bytes" +} + +round_max_duration() { + local wl="$1" round="$2" max=0 a d + for a in "${ARMS[@]}"; do + d=$(sed -n 's/.*duration_s=\([0-9]*\).*/\1/p' "$ROOT/$wl/$a-r$round/meta.txt" 2>/dev/null | head -1) + [ -n "${d:-}" ] && [ "$d" -gt "$max" ] && max=$d + done + echo "$max" +} + +run_perf() { + local wl="$1" trace="$2" golden="$3" w="$4" h="$5" target="$6" reps="$7" timeout_s="$8" crop="$9" + local round=1 + while [ "$round" -le "$reps" ]; do + local arm + for arm in "${ARMS[@]}"; do + run_one "$wl" "$arm" "$round" "$trace" "$golden" "$w" "$h" "$target" "$timeout_s" benchmark $crop || true + done + if [ "$wl" = "rd12" ] && [ "$round" -eq 1 ] && [ "$reps" -gt 2 ]; then + local maxd; maxd=$(round_max_duration "$wl" 1) + if [ "$maxd" -gt 300 ]; then + reps=2 + log "rd12 round1 max run ${maxd}s > 300s -> reduced to 2 reps/arm (will be noted in report)" + fi + fi + round=$((round + 1)) + done +} + +run_ssim() { + # One non-benchmark run per arm, interleaved: replays to target_call, snapshots and + # compares against the golden. Supplies the ssim column the benchmark lane cannot. + local wl="$1" trace="$2" golden="$3" w="$4" h="$5" target="$6" timeout_s="$7" crop="$8" + local arm + for arm in "${ARMS[@]}"; do + run_one "$wl" "$arm" "ssim" "$trace" "$golden" "$w" "$h" "$target" "$timeout_s" ssim $crop || true + done +} + +summarize() { + local winroot="$ROOT" + command -v cygpath >/dev/null 2>&1 && winroot=$(cygpath -w "$ROOT") + python - "$winroot" <<'PYEOF' +import json, re, sys, statistics +from pathlib import Path +root = Path(sys.argv[1]) +rows = [] +for wl_dir in sorted(p for p in root.iterdir() if p.is_dir() and p.name in ("openra", "rd12")): + for run_dir in sorted(wl_dir.glob("*-r*")): + m = re.fullmatch(r"(.+)-r(\d+|ssim)", run_dir.name) + if not m: + continue + arm, rep = m.group(1), m.group(2) + meta = (run_dir / "meta.txt").read_text(encoding="utf-8", errors="replace") if (run_dir / "meta.txt").exists() else "" + dur = re.search(r"duration_s=(\d+)", meta) + temp = re.search(r"battery_temp_dc=(\d+)", meta) + mode = re.search(r"mode=(\w+)", meta) + status = (run_dir / "STATUS").read_text().strip() if (run_dir / "STATUS").exists() else "missing" + bench = res = None + if (run_dir / "benchmark.json").exists(): + try: + bench = json.loads((run_dir / "benchmark.json").read_text(encoding="utf-8", errors="replace")) + except Exception: + bench = None + if (run_dir / "result.json").exists(): + try: + res = json.loads((run_dir / "result.json").read_text(encoding="utf-8", errors="replace")) + except Exception: + res = None + verify = {} + if (run_dir / "verify.txt").exists(): + for line in (run_dir / "verify.txt").read_text(encoding="utf-8", errors="replace").splitlines(): + if "=" in line: + k, v = line.split("=", 1) + verify[k] = v + rows.append({ + "workload": wl_dir.name, "arm": arm, "rep": rep, + "mode": mode.group(1) if mode else "?", + "status": status, + "duration_s": int(dur.group(1)) if dur else None, + "battery_temp_dc": int(temp.group(1)) if temp else None, + "fps": bench.get("fps") if bench else None, + "mean_ms": bench.get("meanFrameMs") if bench else None, + "median_ms": bench.get("medianFrameMs") if bench else None, + "p95_ms": bench.get("p95FrameMs") if bench else None, + "total_frames": bench.get("totalFrames") if bench else None, + "total_seconds": bench.get("totalSeconds") if bench else None, + "passed": res.get("passed") if res else None, + "ssim": res.get("ssim") if res else None, + "message": (res.get("message") if res else None), + "verify": verify, + }) + +agg = [] +for wl in sorted({r["workload"] for r in rows}): + arms = {} + for r in rows: + if (r["workload"] == wl and r["mode"] == "benchmark" and r["status"] == "finished" + and r["mean_ms"] and r["mean_ms"] > 0): + arms.setdefault(r["arm"], []).append(r) + for arm, runs in arms.items(): + means = sorted(r["mean_ms"] for r in runs) + best = min(runs, key=lambda r: r["mean_ms"]) + agg.append({ + "workload": wl, "arm": arm, "n_ok": len(runs), + "best_mean_ms": best["mean_ms"], "best_median_ms": best["median_ms"], + "best_p95_ms": best["p95_ms"], "best_fps": best["fps"], + "median_of_mean_ms": statistics.median(means), + "spread_mean_ms": [means[0], means[-1]], + "spread_pct": round(100.0 * (means[-1] - means[0]) / statistics.median(means), 2) if len(means) > 1 else 0.0, + }) +out = {"runs": rows, "aggregates": agg} +(root / "summary.json").write_text(json.dumps(out, indent=2), encoding="utf-8") +print(f"wrote {root / 'summary.json'} ({len(rows)} runs, {len(agg)} aggregates)") +PYEOF +} + +main() { + local mode="${1:-all}" + mkdir -p "$ROOT" + log "=== run_matrix mode=$mode ===" + adb -s "$SER" get-state >/dev/null 2>&1 || { log "FATAL: device $SER not connected"; exit 1; } + shq "input keyevent KEYCODE_WAKEUP" >/dev/null 2>&1 || true + shq "wm dismiss-keyguard" >/dev/null 2>&1 || true + shq "svc stayon usb" >/dev/null 2>&1 || true + check_openra + case "$mode" in + smoke) + run_one openra monolith 0 "$APP/input/trace.trace" "$APP/input/golden.png" 640 480 31249 "$OPENRA_TIMEOUT" benchmark 1 1 638 478 || true + summarize ;; + prep) prep_rd12 ;; + one) + # one single run: ONE= [WL=openra|rd12] [REP=n] + local wl="${WL:-openra}" arm="${ONE:-tcp}" rep="${REP:-99}" + if [ "$wl" = "openra" ]; then + run_one openra "$arm" "$rep" "$APP/input/trace.trace" "$APP/input/golden.png" 640 480 31249 "$OPENRA_TIMEOUT" benchmark 1 1 638 478 || true + else + prep_rd12 + run_one rd12 "$arm" "$rep" "$APP/input-rd12/trace.trace" "$APP/input-rd12/golden.png" 854 480 4660351 "$RD12_TIMEOUT" benchmark 0 0 0 0 || true + fi + summarize ;; + ssimonly) + run_ssim openra "$APP/input/trace.trace" "$APP/input/golden.png" 640 480 31249 "$OPENRA_TIMEOUT" "1 1 638 478" + summarize ;; + openra) + run_perf openra "$APP/input/trace.trace" "$APP/input/golden.png" 640 480 31249 "$OPENRA_REPS" "$OPENRA_TIMEOUT" "1 1 638 478" + run_ssim openra "$APP/input/trace.trace" "$APP/input/golden.png" 640 480 31249 "$OPENRA_TIMEOUT" "1 1 638 478" + summarize ;; + rd12) + prep_rd12 + run_perf rd12 "$APP/input-rd12/trace.trace" "$APP/input-rd12/golden.png" 854 480 4660351 "$RD12_REPS" "$RD12_TIMEOUT" "0 0 0 0" + run_ssim rd12 "$APP/input-rd12/trace.trace" "$APP/input-rd12/golden.png" 854 480 4660351 "$RD12_TIMEOUT" "0 0 0 0" + shq "am force-stop $PKG" >/dev/null 2>&1 || true + summarize ;; + all) + run_perf openra "$APP/input/trace.trace" "$APP/input/golden.png" 640 480 31249 "$OPENRA_REPS" "$OPENRA_TIMEOUT" "1 1 638 478" + run_ssim openra "$APP/input/trace.trace" "$APP/input/golden.png" 640 480 31249 "$OPENRA_TIMEOUT" "1 1 638 478" + prep_rd12 + run_perf rd12 "$APP/input-rd12/trace.trace" "$APP/input-rd12/golden.png" 854 480 4660351 "$RD12_REPS" "$RD12_TIMEOUT" "0 0 0 0" + run_ssim rd12 "$APP/input-rd12/trace.trace" "$APP/input-rd12/golden.png" 854 480 4660351 "$RD12_TIMEOUT" "0 0 0 0" + shq "am force-stop $PKG" >/dev/null 2>&1 || true + summarize ;; + *) echo "unknown mode: $mode" >&2; exit 2 ;; + esac + log "=== run_matrix mode=$mode DONE ===" +} + +main "${1:-all}" diff --git a/tools/device_bench/disagg/run_matrix.sh b/tools/device_bench/disagg/run_matrix.sh new file mode 100644 index 00000000..4a4d221e --- /dev/null +++ b/tools/device_bench/disagg/run_matrix.sh @@ -0,0 +1,418 @@ +#!/usr/bin/env bash +# Four-arm MobileGL transport performance matrix on device 2f7cbe2e. +# Arms (interleaved A B C D rounds): monolith / inproc / spawn+shm / spawn+tcp-localhost. +# +# Usage: run_matrix.sh [smoke|prep|openra|rd12|all] +# smoke - one openra monolith run to validate the pipeline, then exit +# prep - extract + push the rd12 fixture only +# openra - openra: 4 arms x 3 perf reps (interleaved) + 1 ssim rep/arm (interleaved) +# rd12 - minecraft rd12: perf reps (auto-drops to 2 if round 1 > 300s/run) + 1 ssim rep/arm +# all - openra + rd12 + summary (default) +# +# benchmark=true runs record fps/mean/median/p95 but never snapshot, so result.json +# carries ssim=-1; a separate non-benchmark round per workload supplies the ssim gate. +set -u +export MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL='*' + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +if command -v cygpath >/dev/null 2>&1; then REPO_ROOT_WIN=$(cygpath -w "$REPO_ROOT"); else REPO_ROOT_WIN="$REPO_ROOT"; fi +ROOT="${PERF_WORK:-$REPO_ROOT/.trace-work/perf-compare}" +SER="${SER:-2f7cbe2e}" +PKG=top.mobilegl.plugin.trace +APP=/data/user/0/$PKG/files/trace-replay +FILES=/data/user/0/$PKG/files +LOG="$ROOT/run_matrix.log" +FIXTURES="$REPO_ROOT/tools/trace_replay/fixtures" +OPENRA_TIMEOUT=${OPENRA_TIMEOUT:-900} +RD12_TIMEOUT=${RD12_TIMEOUT:-900} +OPENRA_REPS=${OPENRA_REPS:-3} +RD12_REPS=${RD12_REPS:-3} +ARMS=(monolith inproc shm tcp) + +shq() { adb -s "$SER" shell "$1"; } +execout(){ adb -s "$SER" exec-out "$1"; } + +log() { local line; line="[$(date '+%F %T')] $*"; echo "$line" >>"$LOG"; echo "$line"; } + +arm_env() { + local stats="MOBILEGL_PIPE_STATS=1;MOBILEGL_PIPE_STATS_FILE=$APP/output/pipestats.json" + case "$1" in + monolith) echo "$stats" ;; + inproc) echo "MOBILEGL_TRANSPORT=inproc;MOBILEGL_IPC_RUN_AHEAD=1;$stats" ;; + shm) echo "MOBILEGL_TRANSPORT=spawn;MOBILEGL_IPC_CONTROL=fork;MOBILEGL_IPC_DATA=auto;MOBILEGL_IPC_RUN_AHEAD=1;$stats" ;; + tcp) echo "MOBILEGL_TRANSPORT=spawn;MOBILEGL_IPC_CONTROL=tcp://127.0.0.1:40613;MOBILEGL_IPC_DATA=stream;MOBILEGL_IPC_RUN_AHEAD=1;$stats" ;; + esac +} + +# Pull one run-as file to host; drops it when absent or when exec-out folded the +# device-side "No such file" error into the stdout stream (same rule as +# trace-replay-ci.sh's collect_role_logs). +pull() { + local src="$1" dst="$2" + rm -f "$dst" + execout "run-as $PKG cat $src" >"$dst" 2>/dev/null || true + if [ ! -s "$dst" ] || head -c 4096 "$dst" | grep -q "No such file"; then rm -f "$dst"; fi +} + +extract_lines() { + # $1 dir; pulls the two counter-line families out of every pulled log on the host. + local dir="$1" f + : >"$dir/pipestats-lines.txt" + : >"$dir/linkmetrics-lines.txt" + for f in "$dir"/logcat.txt "$dir"/mobilegl.client.log "$dir"/mobilegl.server.log \ + "$dir"/mgl.client.log "$dir"/mgl.server.log; do + [ -f "$f" ] || continue + grep -a "MGPipe stats:" "$f" 2>/dev/null | sed "s|^|[$(basename "$f")] |" >>"$dir/pipestats-lines.txt" + grep -a "P65LinkMetrics" "$f" 2>/dev/null | sed "s|^|[$(basename "$f")] |" >>"$dir/linkmetrics-lines.txt" + done +} + +verify_run() { + local dir="$1" arm="$2" + local all="$dir/all-marker-evidence.txt" + : >"$all" + local f + for f in "$dir"/logcat.txt "$dir"/mobilegl.client.log "$dir"/mobilegl.server.log \ + "$dir"/mgl.client.log "$dir"/mgl.server.log; do + [ -f "$f" ] && grep -aE "Config: MOBILEGL_TRANSPORT|spawn ARMED|control=tcp data=stream|run-ahead ARMED|running lockstep|DISARMED|Config: IPC|MGPipe stats:" "$f" 2>/dev/null | sed "s|^|[$(basename "$f")] |" >>"$all" + done + local v="$dir/verify.txt" nstats nfatal + nstats=$(grep -ac "MGPipe stats:" "$all" 2>/dev/null || true); [ -n "$nstats" ] || nstats=0 + nfatal=$(grep -ac "Fatal{" "$all" 2>/dev/null || true); [ -n "$nfatal" ] || nfatal=0 + { + echo "arm=$arm" + case "$arm" in + inproc) + grep -aq "Config: MOBILEGL_TRANSPORT=inproc - the MGPipe record stream" "$all" \ + && echo "TRANSPORT_MARKER=inproc:OK" || echo "TRANSPORT_MARKER=inproc:MISSING" ;; + shm) + grep -aq "Config: MOBILEGL_TRANSPORT=spawn - the MGPipe record stream" "$all" \ + && echo "TRANSPORT_MARKER=spawn:OK" || echo "TRANSPORT_MARKER=spawn:MISSING" + grep -aq "spawn ARMED - the server role runs in pid" "$all" \ + && echo "SECOND_PROCESS_SPAWN_ARMED=OK: $(grep -a "spawn ARMED" "$all" | head -1)" \ + || echo "SECOND_PROCESS_SPAWN_ARMED=MISSING" ;; + tcp) + grep -aq "Config: MOBILEGL_TRANSPORT=spawn - the MGPipe record stream" "$all" \ + && echo "TRANSPORT_MARKER=spawn:OK" || echo "TRANSPORT_MARKER=spawn:MISSING" + grep -aq "control=tcp data=stream server=" "$all" \ + && echo "TCP_SESSION=OK: $(grep -a "control=tcp data=stream" "$all" | head -1)" \ + || echo "TCP_SESSION=MISSING" ;; + monolith) + if grep -aqE "Config: MOBILEGL_TRANSPORT=(inproc|spawn)" "$all"; then + echo "SPLIT_MARKER=UNEXPECTED_PRESENT" + else + echo "SPLIT_MARKER=absent (monolith:OK)" + fi ;; + esac + if grep -aq "run-ahead ARMED" "$all"; then + echo "RUN_AHEAD_ARMED=OK: $(grep -a "run-ahead ARMED" "$all" | head -1)" + else + echo "RUN_AHEAD_ARMED=MISSING" + fi + if grep -aq "running lockstep" "$all"; then + echo "LOCKSTEP_WARNING=PRESENT: $(grep -a "running lockstep" "$all" | head -1)" + else + echo "LOCKSTEP_WARNING=absent" + fi + if grep -aqi "DISARMED" "$all"; then + echo "DISARMED=PRESENT: $(grep -ai "DISARMED" "$all" | head -1)" + else + echo "DISARMED=absent" + fi + echo "PIPESTATS_LINES=$nstats" + echo "FATAL_LINES=$nfatal" + } >"$v" +} + +# run_one [crop_x crop_y crop_w crop_h] +run_one() { + local wl="$1" arm="$2" rep="$3" trace="$4" golden="$5" w="$6" h="$7" target="$8" timeout_s="$9" mode="${10}" + local cx="${11:-0}" cy="${12:-0}" cw="${13:-0}" ch="${14:-0}" + local dir="$ROOT/$wl/$arm-r$rep" + rm -rf "$dir"; mkdir -p "$dir" + local epoch0 dur temp0 am_out status="" detail="" passflag="" + if [ "$mode" = "benchmark" ]; then passflag="--ez benchmark true --es benchmark_result_path $APP/output/benchmark.json"; fi + epoch0=$(date +%s) + log "BEGIN $wl/$arm rep$rep mode=$mode (timeout ${timeout_s}s)" + shq "am force-stop $PKG" >/dev/null 2>&1 || true + shq "run-as $PKG sh -c 'rm -f $APP/output/result.json $APP/output/benchmark.json $APP/output/mobilegl.client.log $APP/output/mobilegl.server.log $APP/output/pipestats.client.json $APP/output/pipestats.server.json $APP/output/retrace.log $FILES/mgl.client.log $FILES/mgl.server.log'" >/dev/null 2>&1 || true + adb -s "$SER" logcat -c >/dev/null 2>&1 || true + temp0=$(adb -s "$SER" shell dumpsys battery 2>/dev/null | tr -d '\r' | sed -n 's/.*temperature: \([0-9]*\).*/\1/p' | head -1) + + if [ "$arm" = "tcp" ]; then + if ! python "$REPO_ROOT_WIN/tools/trace_replay/tcp_device_server.py" start \ + --serial "$SER" --package "$PKG" --backend DirectGLES --listen tcp://127.0.0.1:40613 \ + --env "MOBILEGL_PIPE_STATS=1" --env "MOBILEGL_PIPE_STATS_FILE=$FILES/pipestats.json" \ + >"$dir/server-start.log" 2>&1; then + dur=$(( $(date +%s) - epoch0 )); status="server_start_failed" + { echo "arm=$arm rep=$rep workload=$wl duration_s=$dur"; cat "$dir/server-start.log"; } >"$dir/meta.txt" + echo "$status" >"$dir/STATUS"; log "FAIL $wl/$arm rep$rep: tcp server start"; return 1 + fi + local waited=0 listening=0 + while [ "$waited" -lt 30 ]; do + if adb -s "$SER" logcat -d -s MobileGL:* MobileGLServer:* 2>/dev/null | tr -d '\r' | grep -q "listening on tcp://127.0.0.1:40613"; then + listening=1; break + fi + sleep 1; waited=$((waited + 1)) + done + if [ "$listening" -ne 1 ]; then + dur=$(( $(date +%s) - epoch0 )); status="server_not_listening" + adb -s "$SER" logcat -d -s MobileGL:* MobileGLServer:* >"$dir/server-logcat.txt" 2>/dev/null || true + { echo "arm=$arm rep=$rep workload=$wl duration_s=$dur listening_wait_s=$waited"; } >"$dir/meta.txt" + echo "$status" >"$dir/STATUS"; log "FAIL $wl/$arm rep$rep: tcp server not listening after ${waited}s"; return 1 + fi + log "tcp supervisor listening after ${waited}s" + fi + + local envarg + envarg=$(arm_env "$arm") + am_out=$(shq "am start -a top.mobilegl.plugin.TRACE_REPLAY -n $PKG/top.mobilegl.plugin.trace.TraceReplayActivity --es trace_path $trace --es golden_path $golden --es output_dir $APP/output --es backend DirectGLES --el target_call $target --ei width $w --ei height $h --es ssim_threshold 0.99 --ez use_pbuffer true $passflag --ei crop_x $cx --ei crop_y $cy --ei crop_width $cw --ei crop_height $ch --es mobilegl_env '${envarg}'" 2>&1) + printf '%s\n' "$am_out" >"$dir/am-start.txt" + if printf '%s' "$am_out" | grep -qE "Error type|Exception|does not exist|Unknown"; then + dur=$(( $(date +%s) - epoch0 )); status="am_start_failed" + { echo "arm=$arm rep=$rep workload=$wl duration_s=$dur"; echo "$am_out"; } >"$dir/meta.txt" + echo "$status" >"$dir/STATUS"; log "FAIL $wl/$arm rep$rep: am start error: $am_out"; return 1 + fi + + local deadline=$(( epoch0 + timeout_s )) got=0 + while [ "$(date +%s)" -lt "$deadline" ]; do + if execout "run-as $PKG cat $APP/output/result.json" 2>/dev/null | head -c 400 | grep -q '"passed"'; then + got=1; break + fi + if [ $(( $(date +%s) - epoch0 )) -gt 25 ] && [ -z "$(adb -s "$SER" shell pidof $PKG 2>/dev/null | tr -d '\r' | tr -d ' ')" ]; then + status="process_died"; break + fi + sleep 4 + done + if [ "$got" -eq 1 ]; then status="finished"; sleep 3; elif [ -z "$status" ]; then status="timeout"; fi + + dur=$(( $(date +%s) - epoch0 )) + adb -s "$SER" logcat -d >"$dir/logcat.txt" 2>/dev/null || true + pull "$APP/output/result.json" "$dir/result.json" + pull "$APP/output/benchmark.json" "$dir/benchmark.json" + pull "$APP/output/mobilegl.client.log" "$dir/mobilegl.client.log" + pull "$APP/output/mobilegl.server.log" "$dir/mobilegl.server.log" + pull "$APP/output/pipestats.client.json" "$dir/pipestats.client.json" + pull "$APP/output/pipestats.server.json" "$dir/pipestats.server.json" + pull "$APP/output/retrace.log" "$dir/retrace.log" + if [ "$arm" = "tcp" ]; then + pull "$FILES/mgl.client.log" "$dir/mgl.client.log" + pull "$FILES/mgl.server.log" "$dir/mgl.server.log" + pull "$FILES/pipestats.server.json" "$dir/tcp-server-pipestats.json" + fi + + local passed="none" + if [ -f "$dir/result.json" ]; then + passed=$(sed -n 's/.*"passed": *\([a-z]*\).*/\1/p' "$dir/result.json" | head -1) + [ -n "$passed" ] || passed="unparsed" + fi + if [ "$passed" != "true" ] && [ "$status" = "finished" ]; then status="result_not_passed"; fi + { echo "arm=$arm rep=$rep workload=$wl duration_s=$dur battery_temp_dc=$temp0 status=$status passed=$passed mode=$mode" + echo "--- am start ---"; printf '%s\n' "$am_out"; } >"$dir/meta.txt" + echo "$status" >"$dir/STATUS" + extract_lines "$dir" + verify_run "$dir" "$arm" + local vlines + vlines=$(tr '\n' '; ' <"$dir/verify.txt") + log "END $wl/$arm rep$rep mode=$mode status=$status passed=$passed dur=${dur}s temp=$temp0 :: $vlines" + [ "$status" = "finished" ] || return 1 + return 0 +} + +check_openra() { + if ! shq "run-as $PKG sh -c 'test -f $APP/input/trace.trace && test -f $APP/input/golden.png'" >/dev/null 2>&1; then + log "FATAL: openra fixture missing on device ($APP/input)"; exit 1 + fi +} + +prep_rd12() { + local host_trace="$ROOT/fixtures/trace.trace" + local host_golden="$FIXTURES/minecraft-1.21.4-rd12-odinlite-in-world.0004660351.png" + [ -f "$host_trace" ] || { tar xzf "$FIXTURES/minecraft-1.21.4-rd12-odinlite-in-world.tgz" -C "$ROOT/fixtures" || exit 1; } + [ -f "$host_golden" ] || { log "FATAL: rd12 golden missing: $host_golden"; exit 1; } + shq "run-as $PKG sh -c 'mkdir -p $APP/input-rd12'" || true + log "pushing rd12 fixture ($(wc -c <"$host_trace") bytes)" + adb -s "$SER" push "$(cygpath -w "$host_trace")" /data/local/tmp/mgl-rd12.trace >/dev/null || exit 1 + adb -s "$SER" push "$(cygpath -w "$host_golden")" /data/local/tmp/mgl-rd12-golden.png >/dev/null || exit 1 + if ! shq "run-as $PKG sh -c 'cp /data/local/tmp/mgl-rd12.trace $APP/input-rd12/trace.trace'"; then + log "run-as cp failed; streaming via cat pipe" + shq "cat /data/local/tmp/mgl-rd12.trace | run-as $PKG sh -c 'cat > $APP/input-rd12/trace.trace'" || exit 1 + fi + shq "run-as $PKG sh -c 'cp /data/local/tmp/mgl-rd12-golden.png $APP/input-rd12/golden.png'" \ + || shq "cat /data/local/tmp/mgl-rd12-golden.png | run-as $PKG sh -c 'cat > $APP/input-rd12/golden.png'" || exit 1 + shq "rm -f /data/local/tmp/mgl-rd12.trace /data/local/tmp/mgl-rd12-golden.png" >/dev/null 2>&1 || true + local size hostsize + size=$(shq "run-as $PKG sh -c 'wc -c < $APP/input-rd12/trace.trace'" | tr -d '\r ') + hostsize=$(wc -c <"$host_trace") + if [ "$size" != "$hostsize" ]; then log "FATAL: rd12 trace size mismatch device=$size host=$hostsize"; exit 1; fi + log "rd12 fixture on device: $size bytes" +} + +round_max_duration() { + local wl="$1" round="$2" max=0 a d + for a in "${ARMS[@]}"; do + d=$(sed -n 's/.*duration_s=\([0-9]*\).*/\1/p' "$ROOT/$wl/$a-r$round/meta.txt" 2>/dev/null | head -1) + [ -n "${d:-}" ] && [ "$d" -gt "$max" ] && max=$d + done + echo "$max" +} + +run_perf() { + local wl="$1" trace="$2" golden="$3" w="$4" h="$5" target="$6" reps="$7" timeout_s="$8" crop="$9" + local round=1 + while [ "$round" -le "$reps" ]; do + local arm + for arm in "${ARMS[@]}"; do + run_one "$wl" "$arm" "$round" "$trace" "$golden" "$w" "$h" "$target" "$timeout_s" benchmark $crop || true + done + if [ "$wl" = "rd12" ] && [ "$round" -eq 1 ] && [ "$reps" -gt 2 ]; then + local maxd; maxd=$(round_max_duration "$wl" 1) + if [ "$maxd" -gt 300 ]; then + reps=2 + log "rd12 round1 max run ${maxd}s > 300s -> reduced to 2 reps/arm (will be noted in report)" + fi + fi + round=$((round + 1)) + done +} + +run_ssim() { + # One non-benchmark run per arm, interleaved: replays to target_call, snapshots and + # compares against the golden. Supplies the ssim column the benchmark lane cannot. + local wl="$1" trace="$2" golden="$3" w="$4" h="$5" target="$6" timeout_s="$7" crop="$8" + local arm + for arm in "${ARMS[@]}"; do + run_one "$wl" "$arm" "ssim" "$trace" "$golden" "$w" "$h" "$target" "$timeout_s" ssim $crop || true + done +} + +summarize() { + local winroot="$ROOT" + command -v cygpath >/dev/null 2>&1 && winroot=$(cygpath -w "$ROOT") + python - "$winroot" <<'PYEOF' +import json, re, sys, statistics +from pathlib import Path +root = Path(sys.argv[1]) +rows = [] +for wl_dir in sorted(p for p in root.iterdir() if p.is_dir() and p.name in ("openra", "rd12")): + for run_dir in sorted(wl_dir.glob("*-r*")): + m = re.fullmatch(r"(.+)-r(\d+|ssim)", run_dir.name) + if not m: + continue + arm, rep = m.group(1), m.group(2) + meta = (run_dir / "meta.txt").read_text(encoding="utf-8", errors="replace") if (run_dir / "meta.txt").exists() else "" + dur = re.search(r"duration_s=(\d+)", meta) + temp = re.search(r"battery_temp_dc=(\d+)", meta) + mode = re.search(r"mode=(\w+)", meta) + status = (run_dir / "STATUS").read_text().strip() if (run_dir / "STATUS").exists() else "missing" + bench = res = None + if (run_dir / "benchmark.json").exists(): + try: + bench = json.loads((run_dir / "benchmark.json").read_text(encoding="utf-8", errors="replace")) + except Exception: + bench = None + if (run_dir / "result.json").exists(): + try: + res = json.loads((run_dir / "result.json").read_text(encoding="utf-8", errors="replace")) + except Exception: + res = None + verify = {} + if (run_dir / "verify.txt").exists(): + for line in (run_dir / "verify.txt").read_text(encoding="utf-8", errors="replace").splitlines(): + if "=" in line: + k, v = line.split("=", 1) + verify[k] = v + rows.append({ + "workload": wl_dir.name, "arm": arm, "rep": rep, + "mode": mode.group(1) if mode else "?", + "status": status, + "duration_s": int(dur.group(1)) if dur else None, + "battery_temp_dc": int(temp.group(1)) if temp else None, + "fps": bench.get("fps") if bench else None, + "mean_ms": bench.get("meanFrameMs") if bench else None, + "median_ms": bench.get("medianFrameMs") if bench else None, + "p95_ms": bench.get("p95FrameMs") if bench else None, + "total_frames": bench.get("totalFrames") if bench else None, + "total_seconds": bench.get("totalSeconds") if bench else None, + "passed": res.get("passed") if res else None, + "ssim": res.get("ssim") if res else None, + "message": (res.get("message") if res else None), + "verify": verify, + }) + +agg = [] +for wl in sorted({r["workload"] for r in rows}): + arms = {} + for r in rows: + if (r["workload"] == wl and r["mode"] == "benchmark" and r["status"] == "finished" + and r["mean_ms"] and r["mean_ms"] > 0): + arms.setdefault(r["arm"], []).append(r) + for arm, runs in arms.items(): + means = sorted(r["mean_ms"] for r in runs) + best = min(runs, key=lambda r: r["mean_ms"]) + agg.append({ + "workload": wl, "arm": arm, "n_ok": len(runs), + "best_mean_ms": best["mean_ms"], "best_median_ms": best["median_ms"], + "best_p95_ms": best["p95_ms"], "best_fps": best["fps"], + "median_of_mean_ms": statistics.median(means), + "spread_mean_ms": [means[0], means[-1]], + "spread_pct": round(100.0 * (means[-1] - means[0]) / statistics.median(means), 2) if len(means) > 1 else 0.0, + }) +out = {"runs": rows, "aggregates": agg} +(root / "summary.json").write_text(json.dumps(out, indent=2), encoding="utf-8") +print(f"wrote {root / 'summary.json'} ({len(rows)} runs, {len(agg)} aggregates)") +PYEOF +} + +main() { + local mode="${1:-all}" + mkdir -p "$ROOT" + log "=== run_matrix mode=$mode ===" + adb -s "$SER" get-state >/dev/null 2>&1 || { log "FATAL: device $SER not connected"; exit 1; } + shq "input keyevent KEYCODE_WAKEUP" >/dev/null 2>&1 || true + shq "wm dismiss-keyguard" >/dev/null 2>&1 || true + shq "svc stayon usb" >/dev/null 2>&1 || true + check_openra + case "$mode" in + smoke) + run_one openra monolith 0 "$APP/input/trace.trace" "$APP/input/golden.png" 640 480 31249 "$OPENRA_TIMEOUT" benchmark 1 1 638 478 || true + summarize ;; + prep) prep_rd12 ;; + one) + # one single run: ONE= [WL=openra|rd12] [REP=n] + local wl="${WL:-openra}" arm="${ONE:-tcp}" rep="${REP:-99}" + if [ "$wl" = "openra" ]; then + run_one openra "$arm" "$rep" "$APP/input/trace.trace" "$APP/input/golden.png" 640 480 31249 "$OPENRA_TIMEOUT" benchmark 1 1 638 478 || true + else + prep_rd12 + run_one rd12 "$arm" "$rep" "$APP/input-rd12/trace.trace" "$APP/input-rd12/golden.png" 854 480 4660351 "$RD12_TIMEOUT" benchmark 0 0 0 0 || true + fi + summarize ;; + ssimonly) + run_ssim openra "$APP/input/trace.trace" "$APP/input/golden.png" 640 480 31249 "$OPENRA_TIMEOUT" "1 1 638 478" + summarize ;; + openra) + run_perf openra "$APP/input/trace.trace" "$APP/input/golden.png" 640 480 31249 "$OPENRA_REPS" "$OPENRA_TIMEOUT" "1 1 638 478" + run_ssim openra "$APP/input/trace.trace" "$APP/input/golden.png" 640 480 31249 "$OPENRA_TIMEOUT" "1 1 638 478" + summarize ;; + rd12) + prep_rd12 + run_perf rd12 "$APP/input-rd12/trace.trace" "$APP/input-rd12/golden.png" 854 480 4660351 "$RD12_REPS" "$RD12_TIMEOUT" "0 0 0 0" + run_ssim rd12 "$APP/input-rd12/trace.trace" "$APP/input-rd12/golden.png" 854 480 4660351 "$RD12_TIMEOUT" "0 0 0 0" + shq "am force-stop $PKG" >/dev/null 2>&1 || true + summarize ;; + all) + run_perf openra "$APP/input/trace.trace" "$APP/input/golden.png" 640 480 31249 "$OPENRA_REPS" "$OPENRA_TIMEOUT" "1 1 638 478" + run_ssim openra "$APP/input/trace.trace" "$APP/input/golden.png" 640 480 31249 "$OPENRA_TIMEOUT" "1 1 638 478" + prep_rd12 + run_perf rd12 "$APP/input-rd12/trace.trace" "$APP/input-rd12/golden.png" 854 480 4660351 "$RD12_REPS" "$RD12_TIMEOUT" "0 0 0 0" + run_ssim rd12 "$APP/input-rd12/trace.trace" "$APP/input-rd12/golden.png" 854 480 4660351 "$RD12_TIMEOUT" "0 0 0 0" + shq "am force-stop $PKG" >/dev/null 2>&1 || true + summarize ;; + *) echo "unknown mode: $mode" >&2; exit 2 ;; + esac + log "=== run_matrix mode=$mode DONE ===" +} + +main "${1:-all}" diff --git a/tools/device_bench/disagg/run_profile_batch.sh b/tools/device_bench/disagg/run_profile_batch.sh new file mode 100644 index 00000000..806a1455 --- /dev/null +++ b/tools/device_bench/disagg/run_profile_batch.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Batch profile runs for the MobileGL split-mode attribution report. +# Sequential: the device is exclusive. Each run = one arm with PROF=1. +set -u +export MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL='*' +cd "$(dirname "$0")" + +run() { + local wl="$1" arm="$2" rep="$3" spin="${4:-}" + echo "===== RUN wl=$wl arm=$arm rep=$rep spin=$spin $(date +%T) =====" + if [ -n "$spin" ]; then + PROF=1 ONE="$arm" WL="$wl" REP="$rep" \ + EXTRA_ENV=";MOBILEGL_IPC_SPIN_US=$spin" \ + EXTRA_SERVER_ENV="MOBILEGL_IPC_SPIN_US=$spin" \ + bash profile_matrix.sh one || echo "RUN FAILED $wl/$arm/$rep" + else + PROF=1 ONE="$arm" WL="$wl" REP="$rep" \ + bash profile_matrix.sh one || echo "RUN FAILED $wl/$arm/$rep" + fi +} + +run rd12 monolith 92 +run rd12 inproc 93 +run rd12 shm 94 +run rd12 tcp 95 2000 +run openra monolith 96 +run openra inproc 97 +echo "===== ALL DONE $(date +%T) =====" diff --git a/tools/device_bench/disagg/sampler.c b/tools/device_bench/disagg/sampler.c new file mode 100644 index 00000000..f9f63e06 --- /dev/null +++ b/tools/device_bench/disagg/sampler.c @@ -0,0 +1,166 @@ +// Device-side /proc sampler: per-tid syscall occupancy + CPU + switch counters. +// strace attach is SELinux-blocked on this device; /proc//syscall is +// readable same-uid via run-as, which gives the blocked-syscall histogram. +// Usage: sampler +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +static volatile sig_atomic_t g_stop = 0; +static void onsig(int s) { (void)s; g_stop = 1; } + +#define MAXT 128 +#define MAXSY 400 + +struct T { + int tid; + char name[32]; + long long cpu0; // utime+stime at start (ticks) + long long vcs0; // nr_voluntary_switches at start + long long run; // samples with syscall "running" + long long sys[MAXSY]; // samples while in syscall nr + long long openerr; +}; + +static long long readfile(const char *p, char *buf, int n) { + int fd = open(p, O_RDONLY); + if (fd < 0) return -1; + long long r = read(fd, buf, n - 1); + close(fd); + if (r > 0) buf[r] = 0; + return r; +} + +int main(int argc, char **argv) { + if (argc < 4) { fprintf(stderr, "usage: sampler pid dur_s interval_ms out\n"); return 2; } + int pid = atoi(argv[1]); + int durMs = atoi(argv[2]) * 1000; + int intMs = argc > 4 ? atoi(argv[3]) : 2; + const char *out = argc > 4 ? argv[4] : "sampler.csv"; + + struct T ts[MAXT]; + int n = 0; + char path[256], buf[8192]; + + // initial scan + for (int pass = 0; pass < 2; pass++) { + snprintf(path, sizeof path, "/proc/%d/task", pid); + DIR *d = opendir(path); + if (!d) { perror("opendir task"); return 1; } + struct dirent *e; + n = 0; + while ((e = readdir(d)) && n < MAXT) { + if (e->d_name[0] == '.') continue; + int tid = atoi(e->d_name); + if (tid <= 0) continue; + struct T *t = &ts[n++]; + memset(t, 0, sizeof *t); + t->tid = tid; + snprintf(path, sizeof path, "/proc/%d/task/%d/comm", pid, tid); + if (readfile(path, buf, sizeof buf) > 0) { + strncpy(t->name, buf, sizeof t->name - 1); + char *nl = strchr(t->name, '\n'); if (nl) *nl = 0; + } + snprintf(path, sizeof path, "/proc/%d/task/%d/stat", pid, tid); + long long ut = 0, st = 0; + if (readfile(path, buf, sizeof buf) > 0) { + // fields: 1 pid, 2 comm, 3 state ... 14 utime, 15 stime. + // tokens after ')' start at field 3, so utime is f==11, stime f==12. + char *r = strrchr(buf, ')'); + if (r) { + int f = 0; char *sp = r + 1; + while (*sp) { + if (f == 11) ut = atoll(sp); + if (f == 12) { st = atoll(sp); break; } + while (*sp && *sp != ' ') sp++; + while (*sp == ' ') sp++; + f++; + } + } + } + t->cpu0 = ut + st; + snprintf(path, sizeof path, "/proc/%d/task/%d/sched", pid, tid); + t->vcs0 = -1; + if (readfile(path, buf, sizeof buf) > 0) { + char *v = strstr(buf, "nr_voluntary_switches"); + if (v) { + char *col = strchr(v, ':'); + if (col) t->vcs0 = atoll(col + 1); + } + } + } + closedir(d); + if (n > 1) break; // first pass may catch a dying thread set + usleep(50000); + } + + struct timespec t0, now; + clock_gettime(CLOCK_MONOTONIC, &t0); + long long elapsed = 0; + signal(SIGTERM, onsig); + signal(SIGINT, onsig); + struct timespec slp = {.tv_sec = 0, .tv_nsec = (long)intMs * 1000000L}; + while (elapsed < durMs && !g_stop) { + for (int i = 0; i < n; i++) { + snprintf(path, sizeof path, "/proc/%d/task/%d/syscall", pid, ts[i].tid); + long long r = readfile(path, buf, sizeof buf); + if (r <= 0) { ts[i].openerr++; continue; } + if (!strncmp(buf, "running", 7)) { ts[i].run++; continue; } + long nr = -2; + if (buf[0] != '-') nr = strtol(buf, NULL, 10); + if (nr < 0) { ts[i].run++; continue; } // -1 means running + if (nr >= MAXSY) nr = MAXSY - 1; + ts[i].sys[nr]++; + } + nanosleep(&slp, NULL); + clock_gettime(CLOCK_MONOTONIC, &now); + elapsed = (now.tv_sec - t0.tv_sec) * 1000LL + (now.tv_nsec - t0.tv_nsec) / 1000000LL; + } + + FILE *f = fopen(out, "w"); + if (!f) { perror("fopen out"); return 1; } + fprintf(f, "# pid=%d dur_ms=%lld interval_ms=%d samples_per_thread~=%lld\n", + pid, elapsed, intMs, elapsed / intMs); + fprintf(f, "tid,name,cpu_ticks_delta,vcs_delta,run_samples,open_err,sys_hist\n"); + for (int i = 0; i < n; i++) { + struct T *t = &ts[i]; + long long ut = 0, st = 0; + snprintf(path, sizeof path, "/proc/%d/task/%d/stat", pid, t->tid); + if (readfile(path, buf, sizeof buf) > 0) { + char *r = strrchr(buf, ')'); + if (r) { + int f2 = 0; char *sp = r + 1; + while (*sp) { + if (f2 == 11) ut = atoll(sp); + if (f2 == 12) { st = atoll(sp); break; } + while (*sp && *sp != ' ') sp++; + while (*sp == ' ') sp++; + f2++; + } + } + } + long long vcs = -1; + snprintf(path, sizeof path, "/proc/%d/task/%d/sched", pid, t->tid); + if (readfile(path, buf, sizeof buf) > 0) { + char *v = strstr(buf, "nr_voluntary_switches"); + if (v) { + char *col = strchr(v, ':'); + if (col) vcs = atoll(col + 1); + } + } + fprintf(f, "%d,%s,%lld,%lld,%lld,%lld,\"", t->tid, t->name, + (ut + st) - t->cpu0, (vcs >= 0 && t->vcs0 >= 0) ? vcs - t->vcs0 : -1, + t->run, t->openerr); + for (int s = 0; s < MAXSY; s++) + if (t->sys[s]) fprintf(f, "%d:%lld;", s, t->sys[s]); + fprintf(f, "\"\n"); + } + fclose(f); + return 0; +} diff --git a/tools/device_bench/disagg/wifi_hang_repro.sh b/tools/device_bench/disagg/wifi_hang_repro.sh new file mode 100644 index 00000000..e760ee55 --- /dev/null +++ b/tools/device_bench/disagg/wifi_hang_repro.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Start one openra client and leave it hanging on MakeCurrent for stack capture. +set -u +BASE=~/wifi-perf +export MOBILEGL_TRANSPORT=spawn +export MOBILEGL_IPC_CONTROL=tcp://192.168.21.181:40613 +export MOBILEGL_IPC_DATA=stream +export MOBILEGL_IPC_TOKEN=wifibench-token-2026-09 +export MOBILEGL_IPC_RUN_AHEAD=1 +export MOBILEGL_IPC_COLD_START_MS=180000 +export MOBILEGL_PIPE_STATS=1 +cd ~/MobileGL +rm -rf "$BASE/out/hang"; mkdir -p "$BASE/out/hang" +./build-split/tools/trace_replay/mobilegl_trace_replay \ + --trace "$BASE/traces/openra.trace" --backend DirectGLES \ + --mobilegl-library ./build-split/libMobileGL.so --pbuffer-surface \ + --benchmark --benchmark-result="$BASE/out/hang/benchmark.json" \ + --output "$BASE/out/hang" > "$BASE/out/hang/mobilegl.log" 2>&1 & +echo "client pid=$!" +sleep 40 +echo "SLEEP_DONE - client still running: $(jobs -p)"