[Feat] (TraceReplay): benchmark mode with per-frame timing for device fixtures

android-plugin/app/src/trace/cpp/CMakeLists.txt

@@ -218,7 +218,8 @@ add_library(glretrace_common STATIC
         "${APITRACE_ROOT}/retrace/metric_backend_opengl.cpp"
         "${APITRACE_ROOT}/retrace/metric_helper.cpp"
         "${APITRACE_ROOT}/retrace/metric_writer.cpp"
-        apitrace_glws_android.cpp)
+        apitrace_glws_android.cpp
+        trace_benchmark.cpp)
 target_include_directories(glretrace_common PUBLIC
         "${APITRACE_GENERATED_DIR}"
         "${APITRACE_ROOT}/dispatch"

android-plugin/app/src/trace/cpp/apitrace_glws_android.cpp

@@ -1,6 +1,7 @@
 #include "apitrace_fbo_dump.hpp"
 #include "glws.hpp"
 #include "retrace.hpp"
+#include "trace_benchmark.hpp"
 
 #include <android/native_window.h>
 #include <EGL/egl.h>
@@ -186,6 +187,9 @@ public:
             char callNo[32];
             snprintf(callNo, sizeof(callNo), "%u", retrace::callNo);
             gEgl.swapBuffers(gDisplay, surface);
+            // Frame boundary: retrace_eglSwapBuffers() has already run frame_complete() and
+            // handed the frame to us. No-op unless benchmark mode armed the timer.
+            mobilegl_trace::benchmark::OnFrameBoundary();
             HoldAfterTargetPresent(callNo);
         }
     }

android-plugin/app/src/trace/cpp/trace_benchmark.cpp

@@ -0,0 +1,94 @@
+#include "trace_benchmark.hpp"
+
+#include <chrono>
+#include <cstdlib>
+#include <utility>
+#include <dlfcn.h>
+
+namespace mobilegl_trace {
+namespace benchmark {
+namespace {
+
+using Clock = std::chrono::steady_clock;
+using GlFinishFn = void (*)();
+
+constexpr std::size_t kFrameReserve = 4096;
+
+bool gEnabled = false;
+bool gFinishEachFrame = false;
+bool gResolvedGlFinish = false;
+GlFinishFn gGlFinish = nullptr;
+Clock::time_point gStart;
+Clock::time_point gLastBoundary;
+std::vector<double> gFrameMs;
+
+// Same resolution order the glws layers use for MobileGL's entry points: the replay driver
+// already dlopen()ed the library with RTLD_GLOBAL before retrace started, so RTLD_NOLOAD
+// finds that handle instead of loading a second copy, and RTLD_DEFAULT is the fallback for
+// the case where it was linked in rather than dlopen()ed.
+GlFinishFn ResolveGlFinish() {
+    void *handle = nullptr;
+    const char *library = std::getenv("MOBILEGL_TRACE_LIBRARY");
+    if (library != nullptr && library[0] != '\0') {
+        handle = dlopen(library, RTLD_NOW | RTLD_GLOBAL | RTLD_NOLOAD);
+    }
+    if (handle == nullptr) {
+        handle = dlopen("libMobileGL.so", RTLD_NOW | RTLD_GLOBAL | RTLD_NOLOAD);
+    }
+    if (handle != nullptr) {
+        void *symbol = dlsym(handle, "glFinish");
+        if (symbol != nullptr) {
+            return reinterpret_cast<GlFinishFn>(symbol);
+        }
+    }
+    return reinterpret_cast<GlFinishFn>(dlsym(RTLD_DEFAULT, "glFinish"));
+}
+
+} // namespace
+
+void Begin(bool finishEachFrame) {
+    gFrameMs.clear();
+    gFrameMs.reserve(kFrameReserve);
+    gFinishEachFrame = finishEachFrame;
+    gResolvedGlFinish = false;
+    gGlFinish = nullptr;
+    gStart = Clock::now();
+    gLastBoundary = gStart;
+    gEnabled = true;
+}
+
+void OnFrameBoundary() {
+    if (!gEnabled) {
+        return;
+    }
+    if (gFinishEachFrame) {
+        // Resolved on the first boundary rather than in Begin(): a context only exists once
+        // the trace has created one, and glFinish before that would be pointless anyway.
+        if (!gResolvedGlFinish) {
+            gGlFinish = ResolveGlFinish();
+            gResolvedGlFinish = true;
+        }
+        if (gGlFinish != nullptr) {
+            gGlFinish();
+        }
+    }
+    const Clock::time_point now = Clock::now();
+    gFrameMs.push_back(std::chrono::duration<double, std::milli>(now - gLastBoundary).count());
+    gLastBoundary = now;
+}
+
+Report End() {
+    Report report;
+    if (!gEnabled) {
+        return report;
+    }
+    gEnabled = false;
+    gFinishEachFrame = false;
+    report.totalSeconds = std::chrono::duration<double>(Clock::now() - gStart).count();
+    report.frameMs = std::move(gFrameMs);
+    gFrameMs.clear();
+    return report;
+}
+
+} // namespace benchmark
+} // namespace mobilegl_trace

android-plugin/app/src/trace/cpp/trace_benchmark.hpp

@@ -0,0 +1,45 @@
+#pragma once
+
+#include <vector>
+
+namespace mobilegl_trace {
+namespace benchmark {
+
+// Per-frame wall-clock timing for the retrace loop, shared by the Android replay runner
+// and the desktop CLI. Disarmed unless Begin() armed it, and the frame-boundary hook is a
+// single bool test in that case, so the correctness harness pays nothing for it.
+//
+// Retrace runs --singlethread, so all of this is deliberately plain globals: Begin(),
+// OnFrameBoundary() and End() are only ever reached from the one retrace thread.
+
+// Arms timing for the retrace that is about to run.
+//
+// finishEachFrame issues a full glFinish through the replayed context at every frame
+// boundary, so a recorded frame time covers GPU completion and not just CPU submission.
+// That matters on tiled mobile GPUs, where a swap without a sync returns long before the
+// tiler is done and the numbers degenerate into "how fast can we feed the driver". The
+// price is that finishing every frame serializes CPU/GPU overlap, so the absolute frame
+// times are pessimistic against a real running game - they are deterministic and
+// comparable between backends and revisions, which is what a benchmark fixture is for.
+// With finishEachFrame off the run measures CPU-side submission only.
+void Begin(bool finishEachFrame);
+
+// Frame-boundary hook. Called from the platform glws swapBuffers override, which is where
+// apitrace's replay loop advances the frame: retrace_eglSwapBuffers() calls
+// frame_complete() and then Drawable::swapBuffers().
+void OnFrameBoundary();
+
+struct Report {
+    // Wall time of every completed frame, in milliseconds.
+    std::vector<double> frameMs;
+    // Begin() to End(), in seconds. Covers trace parsing and the leading partial frame too,
+    // which is why it is reported next to the per-frame statistics rather than derived from
+    // them.
+    double totalSeconds = 0.0;
+};
+
+// Disarms timing and hands back what was recorded.
+Report End();
+
+} // namespace benchmark
+} // namespace mobilegl_trace

android-plugin/app/src/trace/cpp/trace_replay_core.cpp

@@ -3,10 +3,13 @@
 #include <dlfcn.h>
 #include "apitrace_exit.hpp"
 #include "png.h"
+#include "trace_benchmark.hpp"
 
 #include <algorithm>
 #include <cerrno>
 #include <chrono>
+#include <cmath>
+#include <cstddef>
 #include <cstdint>
 #include <cstdio>
 #include <cstdlib>
@@ -421,32 +424,30 @@ std::string SnapshotCallSet(const Request& request) {
 }
 
 int RunRetraceMain(const Request& request) {
-    std::string prefix = request.outputDir + "/actual.";
+    std::vector<std::string> args;
-    std::string callSet = SnapshotCallSet(request);
+    args.emplace_back("mobilegl-glretrace");
+    args.emplace_back("-b");
+    args.emplace_back("--singlethread");
+    args.emplace_back("--no-context-check");
+    if (!request.benchmark) {
+        // The snapshot callset is also what stops the replay: apitrace exits once it has
+        // dumped the last call in -S. Benchmark mode wants the whole trace, so the -s/-S
+        // pair is left off entirely, which drops the readback and the PNG encode with it.
+        args.emplace_back("--snapshot-alpha");
+        args.emplace_back("-s");
+        args.emplace_back(request.outputDir + "/actual.");
+        args.emplace_back("-S");
+        args.emplace_back(SnapshotCallSet(request));
+    }
+    args.emplace_back(request.tracePath);
 
-    std::string arg0 = "mobilegl-glretrace";
+    std::vector<char*> argv;
-    std::string argBenchmark = "-b";
+    argv.reserve(args.size() + 1);
-    std::string argSingleThread = "--singlethread";
+    for (std::string& arg : args) {
-    std::string argNoContextCheck = "--no-context-check";
+        argv.push_back(arg.data());
-    std::string argSnapshotAlpha = "--snapshot-alpha";
+    }
-    std::string argSnapshotPrefix = "-s";
+    argv.push_back(nullptr);
-    std::string argSnapshotCall = "-S";
+    return MOBILEGL_APITRACE_RETRACE_MAIN(static_cast<int>(args.size()), argv.data());
-    std::string tracePath = request.tracePath;
-
-    char* argv[] = {
-            arg0.data(),
-            argBenchmark.data(),
-            argSingleThread.data(),
-            argNoContextCheck.data(),
-            argSnapshotAlpha.data(),
-            argSnapshotPrefix.data(),
-            prefix.data(),
-            argSnapshotCall.data(),
-            callSet.data(),
-            tracePath.data(),
-            nullptr,
-    };
-    return MOBILEGL_APITRACE_RETRACE_MAIN(10, argv);
 }
 
 bool RunRetrace(const Request& request, Result& result) {
@@ -475,6 +476,12 @@ bool RunRetrace(const Request& request, Result& result) {
         return false;
     }
 
+    if (request.benchmark) {
+        // Nothing was snapshotted, so there is nothing to collect or compare here; the
+        // caller turns the recorded frame times into the result instead.
+        return true;
+    }
+
     std::string snapshotPath = SnapshotPathForCall(request);
     if (!Exists(snapshotPath)) {
         result.statusCode = STATUS_RETRACE_FAILED;
@@ -786,6 +793,82 @@ bool CompareWithGolden(const Request& request, Result& result) {
     return result.passed;
 }
 
+std::string BenchmarkResultPath(const Request& request) {
+    return request.benchmarkResultPath.empty() ? request.outputDir + "/benchmark.json"
+                                               : request.benchmarkResultPath;
+}
+
+// Folds the recorded frame times into the headline numbers. Everything but totalSeconds and
+// the frame count is computed over the trailing benchmarkTailFrames frames only.
+void SummarizeBenchmark(const Request& request, const benchmark::Report& report, Result& result) {
+    result.benchmarkFrames = static_cast<long long>(report.frameMs.size());
+    result.benchmarkTotalSeconds = report.totalSeconds;
+    result.benchmarkTailFrames = 0;
+    if (report.frameMs.empty()) {
+        return;
+    }
+
+    const int requestedTail =
+            request.benchmarkTailFrames > 0 ? request.benchmarkTailFrames : kDefaultBenchmarkTailFrames;
+    const std::size_t tail =
+            std::min(static_cast<std::size_t>(requestedTail), report.frameMs.size());
+    result.benchmarkTailFrames = static_cast<int>(tail);
+
+    std::vector<double> window(report.frameMs.end() - static_cast<std::ptrdiff_t>(tail),
+                               report.frameMs.end());
+    double sum = 0.0;
+    for (double frameMs : window) {
+        sum += frameMs;
+    }
+    result.benchmarkMeanMs = sum / static_cast<double>(tail);
+
+    std::sort(window.begin(), window.end());
+    result.benchmarkMedianMs = (tail % 2 == 1)
+                                       ? window[tail / 2]
+                                       : 0.5 * (window[tail / 2 - 1] + window[tail / 2]);
+    // Nearest-rank p95, so the reported value is always an observed frame time.
+    std::size_t rank = static_cast<std::size_t>(std::ceil(0.95 * static_cast<double>(tail)));
+    if (rank == 0) {
+        rank = 1;
+    }
+    result.benchmarkP95Ms = window[rank - 1];
+    result.benchmarkFps = result.benchmarkMeanMs > 0.0 ? 1000.0 / result.benchmarkMeanMs : -1.0;
+}
+
+bool WriteBenchmarkJson(const Request& request,
+                        const Result& result,
+                        const benchmark::Report& report) {
+    std::ofstream file(result.benchmarkResultPath, std::ios::out | std::ios::trunc);
+    if (!file) {
+        return false;
+    }
+    file << "{\n";
+    file << "  \"tracePath\": \"" << JsonEscape(request.tracePath) << "\",\n";
+    file << "  \"backend\": \"" << JsonEscape(request.backend) << "\",\n";
+    file << "  \"benchmarkFinish\": " << (request.benchmarkFinish ? "true" : "false") << ",\n";
+    file << "  \"width\": " << request.width << ",\n";
+    file << "  \"height\": " << request.height << ",\n";
+    file << "  \"totalFrames\": " << result.benchmarkFrames << ",\n";
+    file << "  \"tailFrames\": " << result.benchmarkTailFrames << ",\n";
+    file << std::fixed << std::setprecision(6);
+    file << "  \"totalSeconds\": " << result.benchmarkTotalSeconds << ",\n";
+    file << std::setprecision(3);
+    file << "  \"meanFrameMs\": " << result.benchmarkMeanMs << ",\n";
+    file << "  \"medianFrameMs\": " << result.benchmarkMedianMs << ",\n";
+    file << "  \"p95FrameMs\": " << result.benchmarkP95Ms << ",\n";
+    file << "  \"fps\": " << result.benchmarkFps << ",\n";
+    file << "  \"frameTimesMs\": [";
+    for (std::size_t i = 0; i < report.frameMs.size(); ++i) {
+        if (i > 0) {
+            file << ", ";
+        }
+        file << report.frameMs[i];
+    }
+    file << "]\n";
+    file << "}\n";
+    return static_cast<bool>(file);
+}
+
 } // namespace
 
 extern "C" [[noreturn]] void mobilegl_apitrace_exit(int status) {
@@ -838,7 +921,25 @@ bool WriteResultJson(const Request& request, const Result& result) {
     file << "  \"deriveNumSubgroups\": " << (request.deriveNumSubgroups ? "true" : "false") << ",\n";
     file << "  \"iterationRPFixBarrier\": " << (request.iterationRPFixBarrier ? "true" : "false") << ",\n";
     file << "  \"holdMs\": " << request.holdMs << ",\n";
-    file << "  \"mismatchPixels\": " << result.mismatchPixels << "\n";
+    file << "  \"mismatchPixels\": " << result.mismatchPixels;
+    if (request.benchmark) {
+        // Headline numbers only; the per-frame array lives in benchmarkResultPath.
+        file << ",\n";
+        file << "  \"benchmark\": true,\n";
+        file << "  \"benchmarkResultPath\": \"" << JsonEscape(result.benchmarkResultPath) << "\",\n";
+        file << "  \"benchmarkFinish\": " << (request.benchmarkFinish ? "true" : "false") << ",\n";
+        file << "  \"benchmarkFrames\": " << result.benchmarkFrames << ",\n";
+        file << "  \"benchmarkTailFrames\": " << result.benchmarkTailFrames << ",\n";
+        file << std::setprecision(6);
+        file << "  \"benchmarkTotalSeconds\": " << result.benchmarkTotalSeconds << ",\n";
+        file << std::setprecision(3);
+        file << "  \"benchmarkMeanFrameMs\": " << result.benchmarkMeanMs << ",\n";
+        file << "  \"benchmarkMedianFrameMs\": " << result.benchmarkMedianMs << ",\n";
+        file << "  \"benchmarkP95FrameMs\": " << result.benchmarkP95Ms << ",\n";
+        file << "  \"benchmarkFps\": " << result.benchmarkFps << "\n";
+    } else {
+        file << "\n";
+    }
     file << "}\n";
     return true;
 }
@@ -848,6 +949,9 @@ Result RunTraceReplay(const Request& request) {
     result.resultPath = request.outputDir + "/result.json";
     result.actualPath = request.outputDir + "/actual.png";
     result.diffPath = request.diffPath;
+    if (request.benchmark) {
+        result.benchmarkResultPath = BenchmarkResultPath(request);
+    }
     const std::string mobileGlLogPath = request.outputDir + "/mobilegl.log";
 
     if (!EnsureDirectory(request.outputDir)) {
@@ -868,7 +972,8 @@ Result RunTraceReplay(const Request& request) {
         return result;
     }
 
-    if (request.targetCall < 0) {
+    // Benchmark mode never snapshots, so it has no target call to stop at.
+    if (!request.benchmark && request.targetCall < 0) {
         result.statusCode = STATUS_INVALID_ARGUMENT;
         result.message = "target_call must be set for dump-images style replay";
         return result;
@@ -883,6 +988,40 @@ Result RunTraceReplay(const Request& request) {
         return result;
     }
 
+    if (request.benchmark) {
+        benchmark::Begin(request.benchmarkFinish);
+        const bool retraced = RunRetrace(request, result);
+        const benchmark::Report report = benchmark::End();
+        SummarizeBenchmark(request, report, result);
+        // Written even when the retrace failed: a partial timing series says where the
+        // replay got to, which is exactly what is wanted when triaging one.
+        const bool wroteJson = WriteBenchmarkJson(request, result, report);
+        HoldAfterRetrace(request);
+        if (!retraced) {
+            return result;
+        }
+        if (!wroteJson) {
+            result.statusCode = STATUS_IO_ERROR;
+            result.message = "benchmark completed but failed to write " + result.benchmarkResultPath;
+            return result;
+        }
+        // "Passed" in benchmark mode means the replay ran the trace to the end without
+        // error; there is no golden to be right or wrong about.
+        result.passed = true;
+        result.statusCode = STATUS_OK;
+        std::ostringstream message;
+        message << std::fixed << std::setprecision(3)
+                << "benchmark completed; frames=" << result.benchmarkFrames
+                << ", tailFrames=" << result.benchmarkTailFrames
+                << ", meanMs=" << result.benchmarkMeanMs
+                << ", medianMs=" << result.benchmarkMedianMs
+                << ", p95Ms=" << result.benchmarkP95Ms
+                << ", fps=" << result.benchmarkFps
+                << ", benchmarkResultPath=" << result.benchmarkResultPath;
+        result.message = message.str();
+        return result;
+    }
+
     if (!RunRetrace(request, result)) {
         HoldAfterRetrace(request);
         return result;

android-plugin/app/src/trace/cpp/trace_replay_core.hpp

@@ -15,6 +15,8 @@ enum StatusCode {
     STATUS_COMPARE_FAILED = 6,
 };
 
+constexpr int kDefaultBenchmarkTailFrames = 200;
+
 struct Request {
     std::string tracePath;
     std::string goldenPath;
@@ -30,6 +32,19 @@ struct Request {
     // Named GL_TEXTURE_2D dump points, each `CALL,TEXTURE,LEVEL,DIR`. Debug-only; the replay
     // behaves exactly as before when this is empty.
     std::vector<std::string> texture2dDumps;
+    // Benchmark (frame-timing) mode. Off by default. When on, the replay runs the whole
+    // trace from start to finish and records a wall-clock timestamp at every frame
+    // boundary; no snapshot is taken and no golden comparison runs.
+    bool benchmark = false;
+    // Number of trailing frames the summary statistics are computed over. Clamped to the
+    // number of frames actually recorded. The tail is what is comparable between runs: the
+    // head of a trace is dominated by shader compiles and first-use uploads.
+    int benchmarkTailFrames = kDefaultBenchmarkTailFrames;
+    // glFinish through the replayed context at every frame boundary, so a frame time
+    // includes GPU completion instead of only CPU submission. See trace_benchmark.hpp.
+    bool benchmarkFinish = true;
+    // Where the timing JSON goes. Defaults to <outputDir>/benchmark.json.
+    std::string benchmarkResultPath;
     int targetFrame = -1;
     long long targetCall = -1;
     int width = 0;
@@ -60,6 +75,16 @@ struct Result {
     std::string matchedGoldenPath;
     double ssim = -1.0;
     long long mismatchPixels = -1;
+    // Benchmark headline numbers. Left at the defaults below unless the request asked for
+    // benchmark mode; benchmarkResultPath then names the JSON with the per-frame array.
+    std::string benchmarkResultPath;
+    long long benchmarkFrames = -1;
+    int benchmarkTailFrames = 0;
+    double benchmarkTotalSeconds = -1.0;
+    double benchmarkMeanMs = -1.0;
+    double benchmarkMedianMs = -1.0;
+    double benchmarkP95Ms = -1.0;
+    double benchmarkFps = -1.0;
 };
 
 Result RunTraceReplay(const Request& request);

android-plugin/app/src/trace/cpp/trace_replay_jni.cpp

@@ -125,7 +125,11 @@ Java_top_mobilegl_plugin_trace_TraceReplayActivity_nativeRunTraceReplay(JNIEnv*
                                                                         jboolean fixIterationRPSubgroupScratch,
                                                                         jboolean deriveNumSubgroups,
                                                                         jboolean iterationRPFixBarrier,
-                                                                        jstring texture2dDumps) {
+                                                                        jstring texture2dDumps,
+                                                                        jboolean benchmarkMode,
+                                                                        jint benchmarkTailFrames,
+                                                                        jboolean benchmarkFinish,
+                                                                        jstring benchmarkResultPath) {
     mobilegl_trace::Request request;
     request.tracePath = ToString(env, tracePath);
     request.goldenPath = ToString(env, goldenPath);
@@ -156,6 +160,12 @@ Java_top_mobilegl_plugin_trace_TraceReplayActivity_nativeRunTraceReplay(JNIEnv*
     request.fixIterationRPSubgroupScratch = fixIterationRPSubgroupScratch == JNI_TRUE;
     request.deriveNumSubgroups = deriveNumSubgroups == JNI_TRUE;
     request.iterationRPFixBarrier = iterationRPFixBarrier == JNI_TRUE;
+    request.benchmark = benchmarkMode == JNI_TRUE;
+    request.benchmarkTailFrames = benchmarkTailFrames > 0
+                                          ? benchmarkTailFrames
+                                          : mobilegl_trace::kDefaultBenchmarkTailFrames;
+    request.benchmarkFinish = benchmarkFinish == JNI_TRUE;
+    request.benchmarkResultPath = ToString(env, benchmarkResultPath);
 
     ScopedTraceReplayState replayState;
     mobilegl_trace_set_requested_size(request.width, request.height);

android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java

@@ -119,7 +119,11 @@ public final class TraceReplayActivity extends Activity {
                 request.fixIterationRPSubgroupScratch,
                 request.deriveNumSubgroups,
                 request.iterationRPFixBarrier,
-                request.texture2dDumps
+                request.texture2dDumps,
+                request.benchmark,
+                request.benchmarkTailFrames,
+                request.benchmarkFinish,
+                request.benchmarkResultPath
         );
         Log.i(TAG, result.toString());
         TraceReplayResult finalResult = result;
@@ -155,7 +159,11 @@ public final class TraceReplayActivity extends Activity {
             boolean fixIterationRPSubgroupScratch,
             boolean deriveNumSubgroups,
             boolean iterationRPFixBarrier,
-            String texture2dDumps
+            String texture2dDumps,
+            boolean benchmark,
+            int benchmarkTailFrames,
+            boolean benchmarkFinish,
+            String benchmarkResultPath
     );
 
     private static final class TraceReplayRequest {
@@ -184,6 +192,13 @@ public final class TraceReplayActivity extends Activity {
         final boolean deriveNumSubgroups;
         final boolean iterationRPFixBarrier;
         final String texture2dDumps;
+        // Benchmark (frame-timing) mode. Replays the whole trace, times every frame
+        // boundary, and skips the snapshot and the SSIM comparison; "passed" then only
+        // means the replay ran to the end without error.
+        final boolean benchmark;
+        final int benchmarkTailFrames;
+        final boolean benchmarkFinish;
+        final String benchmarkResultPath;
 
         private TraceReplayRequest(
                 String tracePath,
@@ -210,7 +225,11 @@ public final class TraceReplayActivity extends Activity {
                 boolean fixIterationRPSubgroupScratch,
                 boolean deriveNumSubgroups,
                 boolean iterationRPFixBarrier,
-                    String texture2dDumps
+                    String texture2dDumps,
+                boolean benchmark,
+                int benchmarkTailFrames,
+                boolean benchmarkFinish,
+                String benchmarkResultPath
         ) {
             this.tracePath = tracePath;
             this.goldenPath = goldenPath;
@@ -237,11 +256,17 @@ public final class TraceReplayActivity extends Activity {
             this.deriveNumSubgroups = deriveNumSubgroups;
             this.iterationRPFixBarrier = iterationRPFixBarrier;
             this.texture2dDumps = texture2dDumps;
+            this.benchmark = benchmark;
+            this.benchmarkTailFrames = benchmarkTailFrames;
+            this.benchmarkFinish = benchmarkFinish;
+            this.benchmarkResultPath = benchmarkResultPath;
         }
 
         static TraceReplayRequest from(Intent intent, File filesDir, String defaultBackend) {
             String outputDir = readString(intent, "output_dir", new File(filesDir, "trace-replay").getAbsolutePath());
             String diffPath = readString(intent, "diff_path", "");
+            String benchmarkResultPath =
+                    readString(intent, "benchmark_result_path", outputDir + "/benchmark.json");
             return new TraceReplayRequest(
                     readString(intent, "trace_path", ""),
                     readString(intent, "golden_path", ""),
@@ -267,7 +292,11 @@ public final class TraceReplayActivity extends Activity {
                     intent.getBooleanExtra("fix_iterationrp_subgroup_scratch", false),
                     intent.getBooleanExtra("derive_num_subgroups", false),
                     intent.getBooleanExtra("iterationrp_fix_barrier", false),
-                    readString(intent, "texture_2d_dumps", "")
+                    readString(intent, "texture_2d_dumps", ""),
+                    intent.getBooleanExtra("benchmark", false),
+                    intent.getIntExtra("benchmark_tail_frames", 200),
+                    intent.getBooleanExtra("benchmark_finish", true),
+                    benchmarkResultPath
             );
         }
 

android-plugin/trace-replay-ci.sh

@@ -32,6 +32,10 @@ Usage:
     [--avoid-angle-llvmpipe-explicit-lod-bias] \
     [--coherent-as-flush] \
     [--dump-texture-2d CALL,TEXTURE,LEVEL,DIR] \
+    [--benchmark] \
+    [--benchmark-tail-frames N] \
+    [--benchmark-finish 0|1] \
+    [--reuse-fixture] \
     --timeout-seconds N
 
 Set MOBILEGL_USE_ANGLE=1 to run DirectGLES replay with packaged ANGLE
@@ -50,6 +54,12 @@ sample with an explicit LOD that ANGLE llvmpipe cannot take a LOD bias on
 (MOBILEGL_AVOID_EXPLICIT_LOD_BIAS=1).
 Pass --coherent-as-flush for traces whose engine writes persistent
 GL_MAP_FLUSH_EXPLICIT_BIT maps it never flushes (MOBILEGL_COHERENT_AS_FLUSH=1).
+Pass --benchmark to replay the whole trace as a frame-timing benchmark instead of
+snapshotting one frame and comparing it against the golden. The run then also
+copies benchmark.json (per-frame times plus mean/median/p95) out of the app, and
+"passed" only means the replay reached the end of the trace without an error.
+Pass --reuse-fixture to skip re-extracting and re-pushing the trace, for repeat
+runs of a case whose fixture is already in /data/local/tmp.
 EOF
 }
 
@@ -109,6 +119,10 @@ avoid_angle_llvmpipe_sampler_mipmap_min_filter=0
 avoid_angle_llvmpipe_explicit_lod_bias=0
 coherent_as_flush=0
 texture_2d_dumps=""
+benchmark=0
+benchmark_tail_frames=200
+benchmark_finish=1
+reuse_fixture=0
 timeout_seconds=""
 
 while [ "$#" -gt 0 ]; do
@@ -150,6 +164,10 @@ while [ "$#" -gt 0 ]; do
       ;;
     --coherent-as-flush) coherent_as_flush=1; shift 1 ;;
     --dump-texture-2d) texture_2d_dumps="$(next_arg "$@")"; shift 2 ;;
+    --benchmark) benchmark=1; shift 1 ;;
+    --benchmark-tail-frames) benchmark_tail_frames="$(next_arg "$@")"; shift 2 ;;
+    --benchmark-finish) benchmark_finish="$(next_arg "$@")"; shift 2 ;;
+    --reuse-fixture) reuse_fixture=1; shift 1 ;;
     --timeout-seconds) timeout_seconds="$(next_arg "$@")"; shift 2 ;;
     -h|--help) usage; exit 0 ;;
     *) die "unknown argument: $1" ;;
@@ -339,7 +357,11 @@ run_retrace() {
   fi
 
   mkdir -p "${result_dir}"
-  "${ADB}" install -r "$(host_path_for_adb "${apk_file}")"
+  # A repeat run of a case the previous invocation already installed and pushed only needs
+  # the on-device copy back into a fresh app output directory.
+  if [ "${reuse_fixture}" -eq 0 ]; then
+    "${ADB}" install -r "$(host_path_for_adb "${apk_file}")"
+  fi
   copy_fixture_to_app
   adb_device_path shell am force-stop "${package_name}"
   "${ADB}" logcat -c
@@ -378,6 +400,16 @@ run_retrace() {
   if [ -n "${texture_2d_dumps}" ]; then
     set -- "$@" --es texture_2d_dumps "${texture_2d_dumps}"
   fi
+  if [ "${benchmark}" -eq 1 ]; then
+    set -- "$@" --ez benchmark true
+    set -- "$@" --ei benchmark_tail_frames "${benchmark_tail_frames}"
+    set -- "$@" --es benchmark_result_path "${app_dir}/output/benchmark.json"
+    if [ "${benchmark_finish}" = "0" ]; then
+      set -- "$@" --ez benchmark_finish false
+    else
+      set -- "$@" --ez benchmark_finish true
+    fi
+  fi
   set -- "$@" \
     --es output_dir "${app_dir}/output" \
     --es diff_path "${app_dir}/output/${safe_case}-diff.png" \
@@ -434,10 +466,16 @@ run_retrace() {
   fi
   adb_device_path exec-out run-as "${package_name}" cat "${app_dir}/output/result.json" > "${result_dir}/result.json"
   cat "${result_dir}/result.json"
-  copy_app_artifact "${app_dir}/output/actual.png" "${result_dir}/${safe_case}-${backend}-actual.png"
+  # A benchmark run takes no snapshot, so there is no actual/diff pair to copy.
-  copy_app_artifact "${app_dir}/output/${safe_case}-diff.png" "${result_dir}/${safe_case}-${backend}-diff.png"
+  if [ "${benchmark}" -eq 0 ]; then
+    copy_app_artifact "${app_dir}/output/actual.png" "${result_dir}/${safe_case}-${backend}-actual.png"
+    copy_app_artifact "${app_dir}/output/${safe_case}-diff.png" "${result_dir}/${safe_case}-${backend}-diff.png"
+  fi
   copy_app_artifact "${app_dir}/output/retrace.log" "${result_dir}/retrace.log"
   copy_app_artifact "${app_dir}/output/mobilegl.log" "${result_dir}/mobilegl.log"
+  if [ "${benchmark}" -eq 1 ]; then
+    copy_app_artifact "${app_dir}/output/benchmark.json" "${result_dir}/benchmark.json"
+  fi
   copy_texture_2d_dumps
 
   # A replay that wrote result.json but did not pass used to print nothing but
@@ -468,6 +506,8 @@ run_retrace() {
 }
 
 mkdir -p "${fixture_root}" "${result_root}"
-prepare_fixture
+if [ "${reuse_fixture}" -eq 0 ]; then
+  prepare_fixture
+fi
 
 run_retrace

tools/trace_replay/CMakeLists.txt

@@ -225,7 +225,8 @@ add_library(mobilegl_trace_glretrace_common STATIC
         "${APITRACE_ROOT}/retrace/metric_helper.cpp"
         "${APITRACE_ROOT}/retrace/metric_writer.cpp"
         "${MOBILEGL_TRACE_ROOT}/apitrace_fbo_dump.cpp"
-        "${MOBILEGL_TRACE_ROOT}/apitrace_glws_egl.cpp")
+        "${MOBILEGL_TRACE_ROOT}/apitrace_glws_egl.cpp"
+        "${MOBILEGL_TRACE_SHARED_CPP_DIR}/trace_benchmark.cpp")
 if(APPLE)
     set(MOBILEGL_TRACE_APPLE_FRAMEWORKS
             "-framework Cocoa"
@@ -239,7 +240,8 @@ target_include_directories(mobilegl_trace_glretrace_common PUBLIC
         "${APITRACE_GENERATED_DIR}"
         "${APITRACE_ROOT}/dispatch"
         "${APITRACE_ROOT}/helpers"
-        "${APITRACE_ROOT}/retrace")
+        "${APITRACE_ROOT}/retrace"
+        "${MOBILEGL_TRACE_SHARED_CPP_DIR}")
 target_compile_definitions(mobilegl_trace_glretrace_common PRIVATE
         exit=mobilegl_apitrace_exit
         main=mobilegl_apitrace_main)

tools/trace_replay/README.md

@@ -280,6 +280,72 @@ unflushed persistent maps, e.g. the Create fixtures), pass
 sundial-lite fixture), pass `--ez avoid_angle_llvmpipe_explicit_lod_bias true` so
 the replay runs with `MOBILEGL_AVOID_EXPLICIT_LOD_BIAS=1`.
 
+## Benchmark mode (frame timing)
+
+Benchmark mode reuses the same fixtures as a performance harness instead of a
+correctness one: it replays the trace from the first call to the last, takes a
+wall-clock timestamp at every frame boundary, and takes no snapshot and runs no
+SSIM comparison. `passed` then only means the replay reached the end of the trace
+without an error.
+
+Frame times include GPU completion by default, because a swap boundary on a tiled
+mobile GPU returns long before the tiler is done and would otherwise time CPU
+submission alone. That is what `benchmark_finish` / `--benchmark-finish` controls:
+on (the default) it issues a `glFinish` through the replayed context at every
+frame boundary, which serializes CPU/GPU overlap - pessimistic against a real
+running game, but deterministic and comparable between backends and revisions.
+Turn it off to measure CPU-side submission only.
+
+The mean/median/p95 are computed over the last `benchmark_tail_frames` frames
+(default 200, clamped to the frames actually recorded); the head of a trace is
+dominated by shader compiles and first-use uploads. The full per-frame array is in
+the timing JSON, next to the headline numbers, which also appear in `result.json`.
+
+Whole device runs, one line per run plus the best of the repeats:
+
+```sh
+python tools/trace_replay/run_android_retrace_local.py --benchmark \
+  --case minecraft-1.21.4-fabric-iris-photon-in-world --backend DirectVulkan \
+  --benchmark-repeats 3
+```
+
+`--benchmark-tail-frames N`, `--benchmark-no-finish` and
+`--benchmark-timeout-seconds N` are available; only the first repeat installs the
+APK and pushes the trace. Each run's `benchmark.json` is kept next to the case
+result as `benchmark-run<N>.json`.
+
+The Activity takes the same settings directly:
+
+```sh
+adb shell am start -a top.mobilegl.plugin.TRACE_REPLAY \
+  -n $PKG/top.mobilegl.plugin.trace.TraceReplayActivity \
+  --es trace_path $APP_DIR/input/openra.trace \
+  --es output_dir $APP_DIR/output \
+  --es backend DirectGLES \
+  --ei width 640 --ei height 480 \
+  --ez benchmark true \
+  --ei benchmark_tail_frames 200 \
+  --ez benchmark_finish true \
+  --es benchmark_result_path $APP_DIR/output/benchmark.json
+adb exec-out run-as $PKG cat files/trace-replay/output/benchmark.json > benchmark.json
+```
+
+`golden_path` and `target_call` are not needed in benchmark mode. The Linux CLI
+takes the same options:
+
+```sh
+./mobilegl_trace_replay --trace openra.trace --output out --backend DirectGLES \
+  --benchmark --benchmark-tail-frames=200 --benchmark-finish=1 \
+  --benchmark-result=out/benchmark.json
+```
+
+Two caveats when reading the numbers. Frame times are taken at `eglSwapBuffers`,
+so a trace that ends frames with `glFrameTerminatorGREMEDY` instead is not timed.
+And on a window surface the swap can block on the compositor, which pins frame
+times to the display refresh; replay against the pbuffer surface
+(`--ez use_pbuffer true`) to measure the renderer rather than the presentation
+path.
+
 ## Reproducing the Android DirectGLES lane on Linux (ANGLE on lavapipe)
 
 The APK workflow's DirectGLES lane is not the same stack as the Linux one, which

tools/trace_replay/apitrace_glws_egl.cpp

@@ -2,6 +2,7 @@
 #include "retrace.hpp"
 
 #include "apitrace_fbo_dump.hpp"
+#include "trace_benchmark.hpp"
 
 #include <EGL/egl.h>
 #include <EGL/eglext.h>
@@ -414,6 +415,9 @@ public:
         char callNo[32];
         snprintf(callNo, sizeof(callNo), "%u", retrace::callNo);
         gEgl.swapBuffers(gDisplay, surface);
+        // Frame boundary: retrace_eglSwapBuffers() has already run frame_complete() and
+        // handed the frame to us. No-op unless benchmark mode armed the timer.
+        mobilegl_trace::benchmark::OnFrameBoundary();
 #if defined(__APPLE__)
         PumpMacOSEvents();
         if (window && !windowShown) {

tools/trace_replay/run_android_retrace_local.py

@@ -119,7 +119,7 @@ def render_summary():
     shutil.copyfile(SUMMARY_DIR / SUMMARY_HTML, SUMMARY_DIR / "index.html")
 
 
-def run_case(case, backend):
+def run_case(case, backend, extra_args=None, timeout_seconds=None):
     backend_info = BACKENDS[backend]
     apk = find_trace_apk()
     trace_archive = FIXTURES / case["trace_archive"]
@@ -176,8 +176,9 @@ def run_case(case, backend):
         "--crop-height",
         str(case["crop_height"]),
         "--timeout-seconds",
-        str(case["timeout_seconds"]),
+        str(timeout_seconds if timeout_seconds is not None else case["timeout_seconds"]),
     ]
+    command.extend(extra_args or [])
     if alternate is not None:
         command[command.index("--target-call"):command.index("--target-call")] = ["--alternate-golden", bash_path(alternate)]
     if backend_info["use_pbuffer"]:
@@ -204,12 +205,118 @@ def run_case(case, backend):
     return result.returncode
 
 
+def read_benchmark(case, backend, run_index):
+    """Reads the benchmark.json the run just pulled and files it under the run number."""
+    result_dir = RESULT_ROOT / f"{safe_case(case['name'])}-{backend}"
+    source = result_dir / "benchmark.json"
+    if not source.exists():
+        return None
+    try:
+        report = json.loads(source.read_text(encoding="utf-8"))
+    except (OSError, ValueError) as error:
+        print(f"failed to read {source}: {error}", file=sys.stderr)
+        return None
+    shutil.copyfile(source, result_dir / f"benchmark-run{run_index}.json")
+    return report
+
+
+def format_benchmark(report):
+    return (
+        f"frames={report.get('totalFrames', -1)}"
+        f" total={report.get('totalSeconds', -1):.1f}s"
+        f" tail={report.get('tailFrames', -1)}"
+        f" mean={report.get('meanFrameMs', -1):.3f}ms"
+        f" median={report.get('medianFrameMs', -1):.3f}ms"
+        f" p95={report.get('p95FrameMs', -1):.3f}ms"
+        f" fps={report.get('fps', -1):.1f}"
+    )
+
+
+def run_benchmark_case(case, backend, args):
+    """Runs the case as a frame-timing benchmark `--benchmark-repeats` times.
+
+    Only the first run installs the APK and pushes the trace; the repeats reuse what is
+    already on the device, so the numbers are not paying for an adb push each time.
+    """
+    label = f"{case['name']} / {backend}"
+    reports = []
+    failures = 0
+    for run_index in range(1, args.benchmark_repeats + 1):
+        extra_args = [
+            "--benchmark",
+            "--benchmark-tail-frames",
+            str(args.benchmark_tail_frames),
+            "--benchmark-finish",
+            "0" if args.benchmark_no_finish else "1",
+        ]
+        if run_index > 1:
+            extra_args.append("--reuse-fixture")
+        # The previous repeat's file would otherwise be read back as this run's result.
+        stale = RESULT_ROOT / f"{safe_case(case['name'])}-{backend}" / "benchmark.json"
+        if stale.exists():
+            stale.unlink()
+        rc = run_case(case, backend, extra_args=extra_args, timeout_seconds=args.benchmark_timeout_seconds)
+        report = read_benchmark(case, backend, run_index)
+        if rc != 0 or report is None:
+            print(f"{label} run {run_index}/{args.benchmark_repeats}: FAILED (exit {rc})", flush=True)
+            failures += 1
+            continue
+        reports.append((run_index, report))
+        print(
+            f"{label} run {run_index}/{args.benchmark_repeats}: {format_benchmark(report)}",
+            flush=True,
+        )
+
+    def mean_frame_ms(entry):
+        # A run that recorded no frames reports -1; it must not win "best" by being smallest.
+        mean = entry[1].get("meanFrameMs", -1)
+        return mean if mean > 0 else float("inf")
+
+    if reports:
+        best_index, best = min(reports, key=mean_frame_ms)
+        print(
+            f"{label} best of {args.benchmark_repeats} (run {best_index}): {format_benchmark(best)}",
+            flush=True,
+        )
+    return 1 if failures else 0
+
+
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument("--case", action="append", dest="cases", help="Case name to run; may be repeated.")
     parser.add_argument("--backend", action="append", choices=sorted(BACKENDS), help="Backend to run; may be repeated.")
     parser.add_argument("--all", action="store_true", help="Run every case in the APK workflow matrix.")
     parser.add_argument("--keep-results", action="store_true", help="Do not clear the previous result root.")
+    parser.add_argument(
+        "--benchmark",
+        action="store_true",
+        help="Replay each case end to end as a frame-timing benchmark instead of comparing "
+             "one frame against its golden.",
+    )
+    parser.add_argument(
+        "--benchmark-repeats",
+        type=int,
+        default=3,
+        help="Benchmark runs per case/backend; the best (lowest mean frame time) is reported.",
+    )
+    parser.add_argument(
+        "--benchmark-tail-frames",
+        type=int,
+        default=200,
+        help="Frames at the end of the run the statistics are computed over.",
+    )
+    parser.add_argument(
+        "--benchmark-no-finish",
+        action="store_true",
+        help="Do not glFinish at every frame boundary, so frame times measure CPU submission "
+             "only instead of GPU completion.",
+    )
+    parser.add_argument(
+        "--benchmark-timeout-seconds",
+        type=int,
+        default=900,
+        help="Per-run timeout; a benchmark replays the whole trace, not just up to target_call.",
+    )
     return parser.parse_args()
 
 
@@ -221,12 +328,20 @@ def main():
     if not selected_cases:
         print("No cases selected. Use --all or --case NAME.", file=sys.stderr)
         return 2
+    if args.benchmark and args.benchmark_repeats < 1:
+        print("--benchmark-repeats must be at least 1.", file=sys.stderr)
+        return 2
     if not args.keep_results and RESULT_ROOT.exists():
         shutil.rmtree(RESULT_ROOT)
     RESULT_ROOT.mkdir(parents=True, exist_ok=True)
     failures = 0
     for case in selected_cases:
         for backend in selected_backends:
+            if args.benchmark:
+                print(f"=== Android benchmark: {case['name']} / {backend} ===", flush=True)
+                # No SSIM verdicts to render here; the summary page is for the correctness lane.
+                failures += run_benchmark_case(case, backend, args)
+                continue
             print(f"=== Android retrace: {case['name']} / {backend} ===", flush=True)
             rc = run_case(case, backend)
             try:

tools/trace_replay/trace_replay_cli.cpp

@@ -29,6 +29,15 @@ void PrintUsage(const char *argv0) {
             << "  --crop-width N            Compare crop width\n"
             << "  --crop-height N           Compare crop height\n"
             << "  --coherent-as-flush       Set MOBILEGL_COHERENT_AS_FLUSH=1 for the replay\n"
+            << "  --benchmark               Frame-timing mode: replay the whole trace, record a\n"
+            << "                            wall-clock timestamp at every frame boundary, and skip\n"
+            << "                            the snapshot and the golden comparison. --target-call is\n"
+            << "                            not required in this mode.\n"
+            << "  --benchmark-tail-frames=N Frames at the end of the run the mean/median/p95 are\n"
+            << "                            computed over (default: 200, clamped to the frame count)\n"
+            << "  --benchmark-finish=0/1    glFinish at every frame boundary so a frame time covers\n"
+            << "                            GPU completion and not just CPU submission (default: 1)\n"
+            << "  --benchmark-result=PATH   Timing JSON output (default: OUTPUT/benchmark.json)\n"
             << "  --dump-fbo-attachments CALL:DIR[:FBO,FBO,...]\n"
             << "                            At CALL, write every colour attachment and the depth\n"
             << "                            attachment of every live framebuffer object into DIR as\n"
@@ -77,6 +86,23 @@ bool ReadEnvFlag(const char *name) {
     return value != nullptr && std::string(value) == "1";
 }
 
+// The benchmark options are documented as --name=VALUE. The space-separated spelling every
+// other option here uses is accepted too, so --benchmark-tail-frames=300 and
+// --benchmark-tail-frames 300 both work; hasInlineValue says which one was given.
+bool MatchOption(const std::string &arg, const char *name, std::string &inlineValue, bool &hasInlineValue) {
+    if (arg == name) {
+        hasInlineValue = false;
+        return true;
+    }
+    const std::string prefix = std::string(name) + "=";
+    if (arg.compare(0, prefix.size(), prefix) == 0) {
+        inlineValue = arg.substr(prefix.size());
+        hasInlineValue = true;
+        return true;
+    }
+    return false;
+}
+
 bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) {
     request.backend = "DirectGLES";
     request.fixIterationRPSubgroupScratch = ReadEnvFlag("MOBILEGL_FIX_ITERATIONRP_SUBGROUP_SCRATCH");
@@ -85,6 +111,8 @@ bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) {
 
     for (int i = 1; i < argc; ++i) {
         const std::string arg = argv[i];
+        std::string optionValue;
+        bool optionHasValue = false;
         if (arg == "--trace") {
             if (!ReadValue(argc, argv, i, request.tracePath)) return false;
         } else if (arg == "--golden") {
@@ -129,6 +157,17 @@ bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) {
             if (!ReadInt(argc, argv, i, request.cropHeight)) return false;
         } else if (arg == "--coherent-as-flush") {
             request.coherentAsFlush = true;
+        } else if (arg == "--benchmark") {
+            request.benchmark = true;
+        } else if (MatchOption(arg, "--benchmark-tail-frames", optionValue, optionHasValue)) {
+            if (!optionHasValue && !ReadValue(argc, argv, i, optionValue)) return false;
+            request.benchmarkTailFrames = std::atoi(optionValue.c_str());
+        } else if (MatchOption(arg, "--benchmark-finish", optionValue, optionHasValue)) {
+            if (!optionHasValue && !ReadValue(argc, argv, i, optionValue)) return false;
+            request.benchmarkFinish = optionValue != "0";
+        } else if (MatchOption(arg, "--benchmark-result", optionValue, optionHasValue)) {
+            if (!optionHasValue && !ReadValue(argc, argv, i, optionValue)) return false;
+            request.benchmarkResultPath = optionValue;
         } else if (arg == "--dump-fbo-attachments") {
             std::string dumpPoint;
             if (!ReadValue(argc, argv, i, dumpPoint)) return false;
@@ -153,7 +192,7 @@ bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) {
         std::cerr << "--output is required\n";
         return false;
     }
-    if (request.targetCall < 0) {
+    if (request.targetCall < 0 && !request.benchmark) {
         std::cerr << "--target-call is required\n";
         return false;
     }
@@ -161,6 +200,10 @@ bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) {
         std::cerr << "--hold-ms must be non-negative\n";
         return false;
     }
+    if (request.benchmarkTailFrames <= 0) {
+        std::cerr << "--benchmark-tail-frames must be positive\n";
+        return false;
+    }
     return true;
 }
 
@@ -181,9 +224,15 @@ int main(int argc, char **argv) {
         mobilegl_trace::WriteResultJson(request, result);
     }
 
-    std::cout << result.message << "\n"
+    if (request.benchmark) {
-              << "result: " << result.resultPath << "\n"
+        std::cout << result.message << "\n"
-              << "actual: " << result.actualPath << "\n"
+                  << "result: " << result.resultPath << "\n"
-              << "diff: " << result.diffPath << "\n";
+                  << "benchmark: " << result.benchmarkResultPath << "\n";
+    } else {
+        std::cout << result.message << "\n"
+                  << "result: " << result.resultPath << "\n"
+                  << "actual: " << result.actualPath << "\n"
+                  << "diff: " << result.diffPath << "\n";
+    }
     return result.passed ? 0 : result.statusCode;
 }