[Feat] (TraceReplay): benchmark mode with per-frame timing for device fixtures

tools/trace_replay/CMakeLists.txt

@@ -225,7 +225,8 @@ add_library(mobilegl_trace_glretrace_common STATIC
         "${APITRACE_ROOT}/retrace/metric_helper.cpp"
         "${APITRACE_ROOT}/retrace/metric_writer.cpp"
         "${MOBILEGL_TRACE_ROOT}/apitrace_fbo_dump.cpp"
-        "${MOBILEGL_TRACE_ROOT}/apitrace_glws_egl.cpp")
+        "${MOBILEGL_TRACE_ROOT}/apitrace_glws_egl.cpp"
+        "${MOBILEGL_TRACE_SHARED_CPP_DIR}/trace_benchmark.cpp")
 if(APPLE)
     set(MOBILEGL_TRACE_APPLE_FRAMEWORKS
             "-framework Cocoa"
@@ -239,7 +240,8 @@ target_include_directories(mobilegl_trace_glretrace_common PUBLIC
         "${APITRACE_GENERATED_DIR}"
         "${APITRACE_ROOT}/dispatch"
         "${APITRACE_ROOT}/helpers"
-        "${APITRACE_ROOT}/retrace")
+        "${APITRACE_ROOT}/retrace"
+        "${MOBILEGL_TRACE_SHARED_CPP_DIR}")
 target_compile_definitions(mobilegl_trace_glretrace_common PRIVATE
         exit=mobilegl_apitrace_exit
         main=mobilegl_apitrace_main)

tools/trace_replay/README.md

@@ -280,6 +280,72 @@ unflushed persistent maps, e.g. the Create fixtures), pass
 sundial-lite fixture), pass `--ez avoid_angle_llvmpipe_explicit_lod_bias true` so
 the replay runs with `MOBILEGL_AVOID_EXPLICIT_LOD_BIAS=1`.
 
+## Benchmark mode (frame timing)
+
+Benchmark mode reuses the same fixtures as a performance harness instead of a
+correctness one: it replays the trace from the first call to the last, takes a
+wall-clock timestamp at every frame boundary, and takes no snapshot and runs no
+SSIM comparison. `passed` then only means the replay reached the end of the trace
+without an error.
+
+Frame times include GPU completion by default, because a swap boundary on a tiled
+mobile GPU returns long before the tiler is done and would otherwise time CPU
+submission alone. That is what `benchmark_finish` / `--benchmark-finish` controls:
+on (the default) it issues a `glFinish` through the replayed context at every
+frame boundary, which serializes CPU/GPU overlap - pessimistic against a real
+running game, but deterministic and comparable between backends and revisions.
+Turn it off to measure CPU-side submission only.
+
+The mean/median/p95 are computed over the last `benchmark_tail_frames` frames
+(default 200, clamped to the frames actually recorded); the head of a trace is
+dominated by shader compiles and first-use uploads. The full per-frame array is in
+the timing JSON, next to the headline numbers, which also appear in `result.json`.
+
+Whole device runs, one line per run plus the best of the repeats:
+
+```sh
+python tools/trace_replay/run_android_retrace_local.py --benchmark \
+  --case minecraft-1.21.4-fabric-iris-photon-in-world --backend DirectVulkan \
+  --benchmark-repeats 3
+```
+
+`--benchmark-tail-frames N`, `--benchmark-no-finish` and
+`--benchmark-timeout-seconds N` are available; only the first repeat installs the
+APK and pushes the trace. Each run's `benchmark.json` is kept next to the case
+result as `benchmark-run<N>.json`.
+
+The Activity takes the same settings directly:
+
+```sh
+adb shell am start -a top.mobilegl.plugin.TRACE_REPLAY \
+  -n $PKG/top.mobilegl.plugin.trace.TraceReplayActivity \
+  --es trace_path $APP_DIR/input/openra.trace \
+  --es output_dir $APP_DIR/output \
+  --es backend DirectGLES \
+  --ei width 640 --ei height 480 \
+  --ez benchmark true \
+  --ei benchmark_tail_frames 200 \
+  --ez benchmark_finish true \
+  --es benchmark_result_path $APP_DIR/output/benchmark.json
+adb exec-out run-as $PKG cat files/trace-replay/output/benchmark.json > benchmark.json
+```
+
+`golden_path` and `target_call` are not needed in benchmark mode. The Linux CLI
+takes the same options:
+
+```sh
+./mobilegl_trace_replay --trace openra.trace --output out --backend DirectGLES \
+  --benchmark --benchmark-tail-frames=200 --benchmark-finish=1 \
+  --benchmark-result=out/benchmark.json
+```
+
+Two caveats when reading the numbers. Frame times are taken at `eglSwapBuffers`,
+so a trace that ends frames with `glFrameTerminatorGREMEDY` instead is not timed.
+And on a window surface the swap can block on the compositor, which pins frame
+times to the display refresh; replay against the pbuffer surface
+(`--ez use_pbuffer true`) to measure the renderer rather than the presentation
+path.
+
 ## Reproducing the Android DirectGLES lane on Linux (ANGLE on lavapipe)
 
 The APK workflow's DirectGLES lane is not the same stack as the Linux one, which

tools/trace_replay/apitrace_glws_egl.cpp

@@ -2,6 +2,7 @@
 #include "retrace.hpp"
 
 #include "apitrace_fbo_dump.hpp"
+#include "trace_benchmark.hpp"
 
 #include <EGL/egl.h>
 #include <EGL/eglext.h>
@@ -414,6 +415,9 @@ public:
         char callNo[32];
         snprintf(callNo, sizeof(callNo), "%u", retrace::callNo);
         gEgl.swapBuffers(gDisplay, surface);
+        // Frame boundary: retrace_eglSwapBuffers() has already run frame_complete() and
+        // handed the frame to us. No-op unless benchmark mode armed the timer.
+        mobilegl_trace::benchmark::OnFrameBoundary();
 #if defined(__APPLE__)
         PumpMacOSEvents();
         if (window && !windowShown) {

tools/trace_replay/run_android_retrace_local.py

@@ -119,7 +119,7 @@ def render_summary():
     shutil.copyfile(SUMMARY_DIR / SUMMARY_HTML, SUMMARY_DIR / "index.html")
 
 
-def run_case(case, backend):
+def run_case(case, backend, extra_args=None, timeout_seconds=None):
     backend_info = BACKENDS[backend]
     apk = find_trace_apk()
     trace_archive = FIXTURES / case["trace_archive"]
@@ -176,8 +176,9 @@ def run_case(case, backend):
         "--crop-height",
         str(case["crop_height"]),
         "--timeout-seconds",
-        str(case["timeout_seconds"]),
+        str(timeout_seconds if timeout_seconds is not None else case["timeout_seconds"]),
     ]
+    command.extend(extra_args or [])
     if alternate is not None:
         command[command.index("--target-call"):command.index("--target-call")] = ["--alternate-golden", bash_path(alternate)]
     if backend_info["use_pbuffer"]:
@@ -204,12 +205,118 @@ def run_case(case, backend):
     return result.returncode
 
 
+def read_benchmark(case, backend, run_index):
+    """Reads the benchmark.json the run just pulled and files it under the run number."""
+    result_dir = RESULT_ROOT / f"{safe_case(case['name'])}-{backend}"
+    source = result_dir / "benchmark.json"
+    if not source.exists():
+        return None
+    try:
+        report = json.loads(source.read_text(encoding="utf-8"))
+    except (OSError, ValueError) as error:
+        print(f"failed to read {source}: {error}", file=sys.stderr)
+        return None
+    shutil.copyfile(source, result_dir / f"benchmark-run{run_index}.json")
+    return report
+
+
+def format_benchmark(report):
+    return (
+        f"frames={report.get('totalFrames', -1)}"
+        f" total={report.get('totalSeconds', -1):.1f}s"
+        f" tail={report.get('tailFrames', -1)}"
+        f" mean={report.get('meanFrameMs', -1):.3f}ms"
+        f" median={report.get('medianFrameMs', -1):.3f}ms"
+        f" p95={report.get('p95FrameMs', -1):.3f}ms"
+        f" fps={report.get('fps', -1):.1f}"
+    )
+
+
+def run_benchmark_case(case, backend, args):
+    """Runs the case as a frame-timing benchmark `--benchmark-repeats` times.
+
+    Only the first run installs the APK and pushes the trace; the repeats reuse what is
+    already on the device, so the numbers are not paying for an adb push each time.
+    """
+    label = f"{case['name']} / {backend}"
+    reports = []
+    failures = 0
+    for run_index in range(1, args.benchmark_repeats + 1):
+        extra_args = [
+            "--benchmark",
+            "--benchmark-tail-frames",
+            str(args.benchmark_tail_frames),
+            "--benchmark-finish",
+            "0" if args.benchmark_no_finish else "1",
+        ]
+        if run_index > 1:
+            extra_args.append("--reuse-fixture")
+        # The previous repeat's file would otherwise be read back as this run's result.
+        stale = RESULT_ROOT / f"{safe_case(case['name'])}-{backend}" / "benchmark.json"
+        if stale.exists():
+            stale.unlink()
+        rc = run_case(case, backend, extra_args=extra_args, timeout_seconds=args.benchmark_timeout_seconds)
+        report = read_benchmark(case, backend, run_index)
+        if rc != 0 or report is None:
+            print(f"{label} run {run_index}/{args.benchmark_repeats}: FAILED (exit {rc})", flush=True)
+            failures += 1
+            continue
+        reports.append((run_index, report))
+        print(
+            f"{label} run {run_index}/{args.benchmark_repeats}: {format_benchmark(report)}",
+            flush=True,
+        )
+
+    def mean_frame_ms(entry):
+        # A run that recorded no frames reports -1; it must not win "best" by being smallest.
+        mean = entry[1].get("meanFrameMs", -1)
+        return mean if mean > 0 else float("inf")
+
+    if reports:
+        best_index, best = min(reports, key=mean_frame_ms)
+        print(
+            f"{label} best of {args.benchmark_repeats} (run {best_index}): {format_benchmark(best)}",
+            flush=True,
+        )
+    return 1 if failures else 0
+
+
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument("--case", action="append", dest="cases", help="Case name to run; may be repeated.")
     parser.add_argument("--backend", action="append", choices=sorted(BACKENDS), help="Backend to run; may be repeated.")
     parser.add_argument("--all", action="store_true", help="Run every case in the APK workflow matrix.")
     parser.add_argument("--keep-results", action="store_true", help="Do not clear the previous result root.")
+    parser.add_argument(
+        "--benchmark",
+        action="store_true",
+        help="Replay each case end to end as a frame-timing benchmark instead of comparing "
+             "one frame against its golden.",
+    )
+    parser.add_argument(
+        "--benchmark-repeats",
+        type=int,
+        default=3,
+        help="Benchmark runs per case/backend; the best (lowest mean frame time) is reported.",
+    )
+    parser.add_argument(
+        "--benchmark-tail-frames",
+        type=int,
+        default=200,
+        help="Frames at the end of the run the statistics are computed over.",
+    )
+    parser.add_argument(
+        "--benchmark-no-finish",
+        action="store_true",
+        help="Do not glFinish at every frame boundary, so frame times measure CPU submission "
+             "only instead of GPU completion.",
+    )
+    parser.add_argument(
+        "--benchmark-timeout-seconds",
+        type=int,
+        default=900,
+        help="Per-run timeout; a benchmark replays the whole trace, not just up to target_call.",
+    )
     return parser.parse_args()
 
 
@@ -221,12 +328,20 @@ def main():
     if not selected_cases:
         print("No cases selected. Use --all or --case NAME.", file=sys.stderr)
         return 2
+    if args.benchmark and args.benchmark_repeats < 1:
+        print("--benchmark-repeats must be at least 1.", file=sys.stderr)
+        return 2
     if not args.keep_results and RESULT_ROOT.exists():
         shutil.rmtree(RESULT_ROOT)
     RESULT_ROOT.mkdir(parents=True, exist_ok=True)
     failures = 0
     for case in selected_cases:
         for backend in selected_backends:
+            if args.benchmark:
+                print(f"=== Android benchmark: {case['name']} / {backend} ===", flush=True)
+                # No SSIM verdicts to render here; the summary page is for the correctness lane.
+                failures += run_benchmark_case(case, backend, args)
+                continue
             print(f"=== Android retrace: {case['name']} / {backend} ===", flush=True)
             rc = run_case(case, backend)
             try:

tools/trace_replay/trace_replay_cli.cpp

@@ -29,6 +29,15 @@ void PrintUsage(const char *argv0) {
             << "  --crop-width N            Compare crop width\n"
             << "  --crop-height N           Compare crop height\n"
             << "  --coherent-as-flush       Set MOBILEGL_COHERENT_AS_FLUSH=1 for the replay\n"
+            << "  --benchmark               Frame-timing mode: replay the whole trace, record a\n"
+            << "                            wall-clock timestamp at every frame boundary, and skip\n"
+            << "                            the snapshot and the golden comparison. --target-call is\n"
+            << "                            not required in this mode.\n"
+            << "  --benchmark-tail-frames=N Frames at the end of the run the mean/median/p95 are\n"
+            << "                            computed over (default: 200, clamped to the frame count)\n"
+            << "  --benchmark-finish=0/1    glFinish at every frame boundary so a frame time covers\n"
+            << "                            GPU completion and not just CPU submission (default: 1)\n"
+            << "  --benchmark-result=PATH   Timing JSON output (default: OUTPUT/benchmark.json)\n"
             << "  --dump-fbo-attachments CALL:DIR[:FBO,FBO,...]\n"
             << "                            At CALL, write every colour attachment and the depth\n"
             << "                            attachment of every live framebuffer object into DIR as\n"
@@ -77,6 +86,23 @@ bool ReadEnvFlag(const char *name) {
     return value != nullptr && std::string(value) == "1";
 }
 
+// The benchmark options are documented as --name=VALUE. The space-separated spelling every
+// other option here uses is accepted too, so --benchmark-tail-frames=300 and
+// --benchmark-tail-frames 300 both work; hasInlineValue says which one was given.
+bool MatchOption(const std::string &arg, const char *name, std::string &inlineValue, bool &hasInlineValue) {
+    if (arg == name) {
+        hasInlineValue = false;
+        return true;
+    }
+    const std::string prefix = std::string(name) + "=";
+    if (arg.compare(0, prefix.size(), prefix) == 0) {
+        inlineValue = arg.substr(prefix.size());
+        hasInlineValue = true;
+        return true;
+    }
+    return false;
+}
+
 bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) {
     request.backend = "DirectGLES";
     request.fixIterationRPSubgroupScratch = ReadEnvFlag("MOBILEGL_FIX_ITERATIONRP_SUBGROUP_SCRATCH");
@@ -85,6 +111,8 @@ bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) {
 
     for (int i = 1; i < argc; ++i) {
         const std::string arg = argv[i];
+        std::string optionValue;
+        bool optionHasValue = false;
         if (arg == "--trace") {
             if (!ReadValue(argc, argv, i, request.tracePath)) return false;
         } else if (arg == "--golden") {
@@ -129,6 +157,17 @@ bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) {
             if (!ReadInt(argc, argv, i, request.cropHeight)) return false;
         } else if (arg == "--coherent-as-flush") {
             request.coherentAsFlush = true;
+        } else if (arg == "--benchmark") {
+            request.benchmark = true;
+        } else if (MatchOption(arg, "--benchmark-tail-frames", optionValue, optionHasValue)) {
+            if (!optionHasValue && !ReadValue(argc, argv, i, optionValue)) return false;
+            request.benchmarkTailFrames = std::atoi(optionValue.c_str());
+        } else if (MatchOption(arg, "--benchmark-finish", optionValue, optionHasValue)) {
+            if (!optionHasValue && !ReadValue(argc, argv, i, optionValue)) return false;
+            request.benchmarkFinish = optionValue != "0";
+        } else if (MatchOption(arg, "--benchmark-result", optionValue, optionHasValue)) {
+            if (!optionHasValue && !ReadValue(argc, argv, i, optionValue)) return false;
+            request.benchmarkResultPath = optionValue;
         } else if (arg == "--dump-fbo-attachments") {
             std::string dumpPoint;
             if (!ReadValue(argc, argv, i, dumpPoint)) return false;
@@ -153,7 +192,7 @@ bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) {
         std::cerr << "--output is required\n";
         return false;
     }
-    if (request.targetCall < 0) {
+    if (request.targetCall < 0 && !request.benchmark) {
         std::cerr << "--target-call is required\n";
         return false;
     }
@@ -161,6 +200,10 @@ bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) {
         std::cerr << "--hold-ms must be non-negative\n";
         return false;
     }
+    if (request.benchmarkTailFrames <= 0) {
+        std::cerr << "--benchmark-tail-frames must be positive\n";
+        return false;
+    }
     return true;
 }
 
@@ -181,9 +224,15 @@ int main(int argc, char **argv) {
         mobilegl_trace::WriteResultJson(request, result);
     }
 
-    std::cout << result.message << "\n"
-              << "result: " << result.resultPath << "\n"
-              << "actual: " << result.actualPath << "\n"
-              << "diff: " << result.diffPath << "\n";
+    if (request.benchmark) {
+        std::cout << result.message << "\n"
+                  << "result: " << result.resultPath << "\n"
+                  << "benchmark: " << result.benchmarkResultPath << "\n";
+    } else {
+        std::cout << result.message << "\n"
+                  << "result: " << result.resultPath << "\n"
+                  << "actual: " << result.actualPath << "\n"
+                  << "diff: " << result.diffPath << "\n";
+    }
     return result.passed ? 0 : result.statusCode;
 }