From c1d6a3c9086c867a9c7e306f025d01d4a7118b67 Mon Sep 17 00:00:00 2001 From: Swung0x48 Date: Wed, 26 Aug 2026 02:17:35 -0400 Subject: [PATCH] [Feat] (TraceReplay): benchmark mode with per-frame timing for device fixtures --- .../app/src/trace/cpp/CMakeLists.txt | 3 +- .../src/trace/cpp/apitrace_glws_android.cpp | 4 + .../app/src/trace/cpp/trace_benchmark.cpp | 94 +++++++++ .../app/src/trace/cpp/trace_benchmark.hpp | 45 ++++ .../app/src/trace/cpp/trace_replay_core.cpp | 193 +++++++++++++++--- .../app/src/trace/cpp/trace_replay_core.hpp | 25 +++ .../app/src/trace/cpp/trace_replay_jni.cpp | 12 +- .../plugin/trace/TraceReplayActivity.java | 37 +++- android-plugin/trace-replay-ci.sh | 48 ++++- tools/trace_replay/CMakeLists.txt | 6 +- tools/trace_replay/README.md | 66 ++++++ tools/trace_replay/apitrace_glws_egl.cpp | 4 + .../trace_replay/run_android_retrace_local.py | 119 ++++++++++- tools/trace_replay/trace_replay_cli.cpp | 59 +++++- 14 files changed, 669 insertions(+), 46 deletions(-) create mode 100644 android-plugin/app/src/trace/cpp/trace_benchmark.cpp create mode 100644 android-plugin/app/src/trace/cpp/trace_benchmark.hpp diff --git a/android-plugin/app/src/trace/cpp/CMakeLists.txt b/android-plugin/app/src/trace/cpp/CMakeLists.txt index fa97470a..6474d73d 100644 --- a/android-plugin/app/src/trace/cpp/CMakeLists.txt +++ b/android-plugin/app/src/trace/cpp/CMakeLists.txt @@ -218,7 +218,8 @@ add_library(glretrace_common STATIC "${APITRACE_ROOT}/retrace/metric_backend_opengl.cpp" "${APITRACE_ROOT}/retrace/metric_helper.cpp" "${APITRACE_ROOT}/retrace/metric_writer.cpp" - apitrace_glws_android.cpp) + apitrace_glws_android.cpp + trace_benchmark.cpp) target_include_directories(glretrace_common PUBLIC "${APITRACE_GENERATED_DIR}" "${APITRACE_ROOT}/dispatch" diff --git a/android-plugin/app/src/trace/cpp/apitrace_glws_android.cpp b/android-plugin/app/src/trace/cpp/apitrace_glws_android.cpp index b02b7605..e62960a0 100644 --- a/android-plugin/app/src/trace/cpp/apitrace_glws_android.cpp +++ b/android-plugin/app/src/trace/cpp/apitrace_glws_android.cpp @@ -1,6 +1,7 @@ #include "apitrace_fbo_dump.hpp" #include "glws.hpp" #include "retrace.hpp" +#include "trace_benchmark.hpp" #include #include @@ -186,6 +187,9 @@ public: char callNo[32]; snprintf(callNo, sizeof(callNo), "%u", retrace::callNo); gEgl.swapBuffers(gDisplay, surface); + // Frame boundary: retrace_eglSwapBuffers() has already run frame_complete() and + // handed the frame to us. No-op unless benchmark mode armed the timer. + mobilegl_trace::benchmark::OnFrameBoundary(); HoldAfterTargetPresent(callNo); } } diff --git a/android-plugin/app/src/trace/cpp/trace_benchmark.cpp b/android-plugin/app/src/trace/cpp/trace_benchmark.cpp new file mode 100644 index 00000000..669023ef --- /dev/null +++ b/android-plugin/app/src/trace/cpp/trace_benchmark.cpp @@ -0,0 +1,94 @@ +#include "trace_benchmark.hpp" + +#include +#include +#include +#include + +namespace mobilegl_trace { +namespace benchmark { +namespace { + +using Clock = std::chrono::steady_clock; +using GlFinishFn = void (*)(); + +constexpr std::size_t kFrameReserve = 4096; + +bool gEnabled = false; +bool gFinishEachFrame = false; +bool gResolvedGlFinish = false; +GlFinishFn gGlFinish = nullptr; +Clock::time_point gStart; +Clock::time_point gLastBoundary; +std::vector gFrameMs; + +// Same resolution order the glws layers use for MobileGL's entry points: the replay driver +// already dlopen()ed the library with RTLD_GLOBAL before retrace started, so RTLD_NOLOAD +// finds that handle instead of loading a second copy, and RTLD_DEFAULT is the fallback for +// the case where it was linked in rather than dlopen()ed. +GlFinishFn ResolveGlFinish() { + void *handle = nullptr; + const char *library = std::getenv("MOBILEGL_TRACE_LIBRARY"); + if (library != nullptr && library[0] != '\0') { + handle = dlopen(library, RTLD_NOW | RTLD_GLOBAL | RTLD_NOLOAD); + } + if (handle == nullptr) { + handle = dlopen("libMobileGL.so", RTLD_NOW | RTLD_GLOBAL | RTLD_NOLOAD); + } + if (handle != nullptr) { + void *symbol = dlsym(handle, "glFinish"); + if (symbol != nullptr) { + return reinterpret_cast(symbol); + } + } + return reinterpret_cast(dlsym(RTLD_DEFAULT, "glFinish")); +} + +} // namespace + +void Begin(bool finishEachFrame) { + gFrameMs.clear(); + gFrameMs.reserve(kFrameReserve); + gFinishEachFrame = finishEachFrame; + gResolvedGlFinish = false; + gGlFinish = nullptr; + gStart = Clock::now(); + gLastBoundary = gStart; + gEnabled = true; +} + +void OnFrameBoundary() { + if (!gEnabled) { + return; + } + if (gFinishEachFrame) { + // Resolved on the first boundary rather than in Begin(): a context only exists once + // the trace has created one, and glFinish before that would be pointless anyway. + if (!gResolvedGlFinish) { + gGlFinish = ResolveGlFinish(); + gResolvedGlFinish = true; + } + if (gGlFinish != nullptr) { + gGlFinish(); + } + } + const Clock::time_point now = Clock::now(); + gFrameMs.push_back(std::chrono::duration(now - gLastBoundary).count()); + gLastBoundary = now; +} + +Report End() { + Report report; + if (!gEnabled) { + return report; + } + gEnabled = false; + gFinishEachFrame = false; + report.totalSeconds = std::chrono::duration(Clock::now() - gStart).count(); + report.frameMs = std::move(gFrameMs); + gFrameMs.clear(); + return report; +} + +} // namespace benchmark +} // namespace mobilegl_trace diff --git a/android-plugin/app/src/trace/cpp/trace_benchmark.hpp b/android-plugin/app/src/trace/cpp/trace_benchmark.hpp new file mode 100644 index 00000000..39410e70 --- /dev/null +++ b/android-plugin/app/src/trace/cpp/trace_benchmark.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include + +namespace mobilegl_trace { +namespace benchmark { + +// Per-frame wall-clock timing for the retrace loop, shared by the Android replay runner +// and the desktop CLI. Disarmed unless Begin() armed it, and the frame-boundary hook is a +// single bool test in that case, so the correctness harness pays nothing for it. +// +// Retrace runs --singlethread, so all of this is deliberately plain globals: Begin(), +// OnFrameBoundary() and End() are only ever reached from the one retrace thread. + +// Arms timing for the retrace that is about to run. +// +// finishEachFrame issues a full glFinish through the replayed context at every frame +// boundary, so a recorded frame time covers GPU completion and not just CPU submission. +// That matters on tiled mobile GPUs, where a swap without a sync returns long before the +// tiler is done and the numbers degenerate into "how fast can we feed the driver". The +// price is that finishing every frame serializes CPU/GPU overlap, so the absolute frame +// times are pessimistic against a real running game - they are deterministic and +// comparable between backends and revisions, which is what a benchmark fixture is for. +// With finishEachFrame off the run measures CPU-side submission only. +void Begin(bool finishEachFrame); + +// Frame-boundary hook. Called from the platform glws swapBuffers override, which is where +// apitrace's replay loop advances the frame: retrace_eglSwapBuffers() calls +// frame_complete() and then Drawable::swapBuffers(). +void OnFrameBoundary(); + +struct Report { + // Wall time of every completed frame, in milliseconds. + std::vector frameMs; + // Begin() to End(), in seconds. Covers trace parsing and the leading partial frame too, + // which is why it is reported next to the per-frame statistics rather than derived from + // them. + double totalSeconds = 0.0; +}; + +// Disarms timing and hands back what was recorded. +Report End(); + +} // namespace benchmark +} // namespace mobilegl_trace diff --git a/android-plugin/app/src/trace/cpp/trace_replay_core.cpp b/android-plugin/app/src/trace/cpp/trace_replay_core.cpp index 8bd0a8b7..da7e51ce 100644 --- a/android-plugin/app/src/trace/cpp/trace_replay_core.cpp +++ b/android-plugin/app/src/trace/cpp/trace_replay_core.cpp @@ -3,10 +3,13 @@ #include #include "apitrace_exit.hpp" #include "png.h" +#include "trace_benchmark.hpp" #include #include #include +#include +#include #include #include #include @@ -421,32 +424,30 @@ std::string SnapshotCallSet(const Request& request) { } int RunRetraceMain(const Request& request) { - std::string prefix = request.outputDir + "/actual."; - std::string callSet = SnapshotCallSet(request); + std::vector args; + args.emplace_back("mobilegl-glretrace"); + args.emplace_back("-b"); + args.emplace_back("--singlethread"); + args.emplace_back("--no-context-check"); + if (!request.benchmark) { + // The snapshot callset is also what stops the replay: apitrace exits once it has + // dumped the last call in -S. Benchmark mode wants the whole trace, so the -s/-S + // pair is left off entirely, which drops the readback and the PNG encode with it. + args.emplace_back("--snapshot-alpha"); + args.emplace_back("-s"); + args.emplace_back(request.outputDir + "/actual."); + args.emplace_back("-S"); + args.emplace_back(SnapshotCallSet(request)); + } + args.emplace_back(request.tracePath); - std::string arg0 = "mobilegl-glretrace"; - std::string argBenchmark = "-b"; - std::string argSingleThread = "--singlethread"; - std::string argNoContextCheck = "--no-context-check"; - std::string argSnapshotAlpha = "--snapshot-alpha"; - std::string argSnapshotPrefix = "-s"; - std::string argSnapshotCall = "-S"; - std::string tracePath = request.tracePath; - - char* argv[] = { - arg0.data(), - argBenchmark.data(), - argSingleThread.data(), - argNoContextCheck.data(), - argSnapshotAlpha.data(), - argSnapshotPrefix.data(), - prefix.data(), - argSnapshotCall.data(), - callSet.data(), - tracePath.data(), - nullptr, - }; - return MOBILEGL_APITRACE_RETRACE_MAIN(10, argv); + std::vector argv; + argv.reserve(args.size() + 1); + for (std::string& arg : args) { + argv.push_back(arg.data()); + } + argv.push_back(nullptr); + return MOBILEGL_APITRACE_RETRACE_MAIN(static_cast(args.size()), argv.data()); } bool RunRetrace(const Request& request, Result& result) { @@ -475,6 +476,12 @@ bool RunRetrace(const Request& request, Result& result) { return false; } + if (request.benchmark) { + // Nothing was snapshotted, so there is nothing to collect or compare here; the + // caller turns the recorded frame times into the result instead. + return true; + } + std::string snapshotPath = SnapshotPathForCall(request); if (!Exists(snapshotPath)) { result.statusCode = STATUS_RETRACE_FAILED; @@ -786,6 +793,82 @@ bool CompareWithGolden(const Request& request, Result& result) { return result.passed; } +std::string BenchmarkResultPath(const Request& request) { + return request.benchmarkResultPath.empty() ? request.outputDir + "/benchmark.json" + : request.benchmarkResultPath; +} + +// Folds the recorded frame times into the headline numbers. Everything but totalSeconds and +// the frame count is computed over the trailing benchmarkTailFrames frames only. +void SummarizeBenchmark(const Request& request, const benchmark::Report& report, Result& result) { + result.benchmarkFrames = static_cast(report.frameMs.size()); + result.benchmarkTotalSeconds = report.totalSeconds; + result.benchmarkTailFrames = 0; + if (report.frameMs.empty()) { + return; + } + + const int requestedTail = + request.benchmarkTailFrames > 0 ? request.benchmarkTailFrames : kDefaultBenchmarkTailFrames; + const std::size_t tail = + std::min(static_cast(requestedTail), report.frameMs.size()); + result.benchmarkTailFrames = static_cast(tail); + + std::vector window(report.frameMs.end() - static_cast(tail), + report.frameMs.end()); + double sum = 0.0; + for (double frameMs : window) { + sum += frameMs; + } + result.benchmarkMeanMs = sum / static_cast(tail); + + std::sort(window.begin(), window.end()); + result.benchmarkMedianMs = (tail % 2 == 1) + ? window[tail / 2] + : 0.5 * (window[tail / 2 - 1] + window[tail / 2]); + // Nearest-rank p95, so the reported value is always an observed frame time. + std::size_t rank = static_cast(std::ceil(0.95 * static_cast(tail))); + if (rank == 0) { + rank = 1; + } + result.benchmarkP95Ms = window[rank - 1]; + result.benchmarkFps = result.benchmarkMeanMs > 0.0 ? 1000.0 / result.benchmarkMeanMs : -1.0; +} + +bool WriteBenchmarkJson(const Request& request, + const Result& result, + const benchmark::Report& report) { + std::ofstream file(result.benchmarkResultPath, std::ios::out | std::ios::trunc); + if (!file) { + return false; + } + file << "{\n"; + file << " \"tracePath\": \"" << JsonEscape(request.tracePath) << "\",\n"; + file << " \"backend\": \"" << JsonEscape(request.backend) << "\",\n"; + file << " \"benchmarkFinish\": " << (request.benchmarkFinish ? "true" : "false") << ",\n"; + file << " \"width\": " << request.width << ",\n"; + file << " \"height\": " << request.height << ",\n"; + file << " \"totalFrames\": " << result.benchmarkFrames << ",\n"; + file << " \"tailFrames\": " << result.benchmarkTailFrames << ",\n"; + file << std::fixed << std::setprecision(6); + file << " \"totalSeconds\": " << result.benchmarkTotalSeconds << ",\n"; + file << std::setprecision(3); + file << " \"meanFrameMs\": " << result.benchmarkMeanMs << ",\n"; + file << " \"medianFrameMs\": " << result.benchmarkMedianMs << ",\n"; + file << " \"p95FrameMs\": " << result.benchmarkP95Ms << ",\n"; + file << " \"fps\": " << result.benchmarkFps << ",\n"; + file << " \"frameTimesMs\": ["; + for (std::size_t i = 0; i < report.frameMs.size(); ++i) { + if (i > 0) { + file << ", "; + } + file << report.frameMs[i]; + } + file << "]\n"; + file << "}\n"; + return static_cast(file); +} + } // namespace extern "C" [[noreturn]] void mobilegl_apitrace_exit(int status) { @@ -838,7 +921,25 @@ bool WriteResultJson(const Request& request, const Result& result) { file << " \"deriveNumSubgroups\": " << (request.deriveNumSubgroups ? "true" : "false") << ",\n"; file << " \"iterationRPFixBarrier\": " << (request.iterationRPFixBarrier ? "true" : "false") << ",\n"; file << " \"holdMs\": " << request.holdMs << ",\n"; - file << " \"mismatchPixels\": " << result.mismatchPixels << "\n"; + file << " \"mismatchPixels\": " << result.mismatchPixels; + if (request.benchmark) { + // Headline numbers only; the per-frame array lives in benchmarkResultPath. + file << ",\n"; + file << " \"benchmark\": true,\n"; + file << " \"benchmarkResultPath\": \"" << JsonEscape(result.benchmarkResultPath) << "\",\n"; + file << " \"benchmarkFinish\": " << (request.benchmarkFinish ? "true" : "false") << ",\n"; + file << " \"benchmarkFrames\": " << result.benchmarkFrames << ",\n"; + file << " \"benchmarkTailFrames\": " << result.benchmarkTailFrames << ",\n"; + file << std::setprecision(6); + file << " \"benchmarkTotalSeconds\": " << result.benchmarkTotalSeconds << ",\n"; + file << std::setprecision(3); + file << " \"benchmarkMeanFrameMs\": " << result.benchmarkMeanMs << ",\n"; + file << " \"benchmarkMedianFrameMs\": " << result.benchmarkMedianMs << ",\n"; + file << " \"benchmarkP95FrameMs\": " << result.benchmarkP95Ms << ",\n"; + file << " \"benchmarkFps\": " << result.benchmarkFps << "\n"; + } else { + file << "\n"; + } file << "}\n"; return true; } @@ -848,6 +949,9 @@ Result RunTraceReplay(const Request& request) { result.resultPath = request.outputDir + "/result.json"; result.actualPath = request.outputDir + "/actual.png"; result.diffPath = request.diffPath; + if (request.benchmark) { + result.benchmarkResultPath = BenchmarkResultPath(request); + } const std::string mobileGlLogPath = request.outputDir + "/mobilegl.log"; if (!EnsureDirectory(request.outputDir)) { @@ -868,7 +972,8 @@ Result RunTraceReplay(const Request& request) { return result; } - if (request.targetCall < 0) { + // Benchmark mode never snapshots, so it has no target call to stop at. + if (!request.benchmark && request.targetCall < 0) { result.statusCode = STATUS_INVALID_ARGUMENT; result.message = "target_call must be set for dump-images style replay"; return result; @@ -883,6 +988,40 @@ Result RunTraceReplay(const Request& request) { return result; } + if (request.benchmark) { + benchmark::Begin(request.benchmarkFinish); + const bool retraced = RunRetrace(request, result); + const benchmark::Report report = benchmark::End(); + SummarizeBenchmark(request, report, result); + // Written even when the retrace failed: a partial timing series says where the + // replay got to, which is exactly what is wanted when triaging one. + const bool wroteJson = WriteBenchmarkJson(request, result, report); + HoldAfterRetrace(request); + if (!retraced) { + return result; + } + if (!wroteJson) { + result.statusCode = STATUS_IO_ERROR; + result.message = "benchmark completed but failed to write " + result.benchmarkResultPath; + return result; + } + // "Passed" in benchmark mode means the replay ran the trace to the end without + // error; there is no golden to be right or wrong about. + result.passed = true; + result.statusCode = STATUS_OK; + std::ostringstream message; + message << std::fixed << std::setprecision(3) + << "benchmark completed; frames=" << result.benchmarkFrames + << ", tailFrames=" << result.benchmarkTailFrames + << ", meanMs=" << result.benchmarkMeanMs + << ", medianMs=" << result.benchmarkMedianMs + << ", p95Ms=" << result.benchmarkP95Ms + << ", fps=" << result.benchmarkFps + << ", benchmarkResultPath=" << result.benchmarkResultPath; + result.message = message.str(); + return result; + } + if (!RunRetrace(request, result)) { HoldAfterRetrace(request); return result; diff --git a/android-plugin/app/src/trace/cpp/trace_replay_core.hpp b/android-plugin/app/src/trace/cpp/trace_replay_core.hpp index 01267c93..8c86a71b 100644 --- a/android-plugin/app/src/trace/cpp/trace_replay_core.hpp +++ b/android-plugin/app/src/trace/cpp/trace_replay_core.hpp @@ -15,6 +15,8 @@ enum StatusCode { STATUS_COMPARE_FAILED = 6, }; +constexpr int kDefaultBenchmarkTailFrames = 200; + struct Request { std::string tracePath; std::string goldenPath; @@ -30,6 +32,19 @@ struct Request { // Named GL_TEXTURE_2D dump points, each `CALL,TEXTURE,LEVEL,DIR`. Debug-only; the replay // behaves exactly as before when this is empty. std::vector texture2dDumps; + // Benchmark (frame-timing) mode. Off by default. When on, the replay runs the whole + // trace from start to finish and records a wall-clock timestamp at every frame + // boundary; no snapshot is taken and no golden comparison runs. + bool benchmark = false; + // Number of trailing frames the summary statistics are computed over. Clamped to the + // number of frames actually recorded. The tail is what is comparable between runs: the + // head of a trace is dominated by shader compiles and first-use uploads. + int benchmarkTailFrames = kDefaultBenchmarkTailFrames; + // glFinish through the replayed context at every frame boundary, so a frame time + // includes GPU completion instead of only CPU submission. See trace_benchmark.hpp. + bool benchmarkFinish = true; + // Where the timing JSON goes. Defaults to /benchmark.json. + std::string benchmarkResultPath; int targetFrame = -1; long long targetCall = -1; int width = 0; @@ -60,6 +75,16 @@ struct Result { std::string matchedGoldenPath; double ssim = -1.0; long long mismatchPixels = -1; + // Benchmark headline numbers. Left at the defaults below unless the request asked for + // benchmark mode; benchmarkResultPath then names the JSON with the per-frame array. + std::string benchmarkResultPath; + long long benchmarkFrames = -1; + int benchmarkTailFrames = 0; + double benchmarkTotalSeconds = -1.0; + double benchmarkMeanMs = -1.0; + double benchmarkMedianMs = -1.0; + double benchmarkP95Ms = -1.0; + double benchmarkFps = -1.0; }; Result RunTraceReplay(const Request& request); diff --git a/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp b/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp index 9b136039..f6d61b79 100644 --- a/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp +++ b/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp @@ -125,7 +125,11 @@ Java_top_mobilegl_plugin_trace_TraceReplayActivity_nativeRunTraceReplay(JNIEnv* jboolean fixIterationRPSubgroupScratch, jboolean deriveNumSubgroups, jboolean iterationRPFixBarrier, - jstring texture2dDumps) { + jstring texture2dDumps, + jboolean benchmarkMode, + jint benchmarkTailFrames, + jboolean benchmarkFinish, + jstring benchmarkResultPath) { mobilegl_trace::Request request; request.tracePath = ToString(env, tracePath); request.goldenPath = ToString(env, goldenPath); @@ -156,6 +160,12 @@ Java_top_mobilegl_plugin_trace_TraceReplayActivity_nativeRunTraceReplay(JNIEnv* request.fixIterationRPSubgroupScratch = fixIterationRPSubgroupScratch == JNI_TRUE; request.deriveNumSubgroups = deriveNumSubgroups == JNI_TRUE; request.iterationRPFixBarrier = iterationRPFixBarrier == JNI_TRUE; + request.benchmark = benchmarkMode == JNI_TRUE; + request.benchmarkTailFrames = benchmarkTailFrames > 0 + ? benchmarkTailFrames + : mobilegl_trace::kDefaultBenchmarkTailFrames; + request.benchmarkFinish = benchmarkFinish == JNI_TRUE; + request.benchmarkResultPath = ToString(env, benchmarkResultPath); ScopedTraceReplayState replayState; mobilegl_trace_set_requested_size(request.width, request.height); diff --git a/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java b/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java index eecdacb4..86754130 100644 --- a/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java +++ b/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java @@ -119,7 +119,11 @@ public final class TraceReplayActivity extends Activity { request.fixIterationRPSubgroupScratch, request.deriveNumSubgroups, request.iterationRPFixBarrier, - request.texture2dDumps + request.texture2dDumps, + request.benchmark, + request.benchmarkTailFrames, + request.benchmarkFinish, + request.benchmarkResultPath ); Log.i(TAG, result.toString()); TraceReplayResult finalResult = result; @@ -155,7 +159,11 @@ public final class TraceReplayActivity extends Activity { boolean fixIterationRPSubgroupScratch, boolean deriveNumSubgroups, boolean iterationRPFixBarrier, - String texture2dDumps + String texture2dDumps, + boolean benchmark, + int benchmarkTailFrames, + boolean benchmarkFinish, + String benchmarkResultPath ); private static final class TraceReplayRequest { @@ -184,6 +192,13 @@ public final class TraceReplayActivity extends Activity { final boolean deriveNumSubgroups; final boolean iterationRPFixBarrier; final String texture2dDumps; + // Benchmark (frame-timing) mode. Replays the whole trace, times every frame + // boundary, and skips the snapshot and the SSIM comparison; "passed" then only + // means the replay ran to the end without error. + final boolean benchmark; + final int benchmarkTailFrames; + final boolean benchmarkFinish; + final String benchmarkResultPath; private TraceReplayRequest( String tracePath, @@ -210,7 +225,11 @@ public final class TraceReplayActivity extends Activity { boolean fixIterationRPSubgroupScratch, boolean deriveNumSubgroups, boolean iterationRPFixBarrier, - String texture2dDumps + String texture2dDumps, + boolean benchmark, + int benchmarkTailFrames, + boolean benchmarkFinish, + String benchmarkResultPath ) { this.tracePath = tracePath; this.goldenPath = goldenPath; @@ -237,11 +256,17 @@ public final class TraceReplayActivity extends Activity { this.deriveNumSubgroups = deriveNumSubgroups; this.iterationRPFixBarrier = iterationRPFixBarrier; this.texture2dDumps = texture2dDumps; + this.benchmark = benchmark; + this.benchmarkTailFrames = benchmarkTailFrames; + this.benchmarkFinish = benchmarkFinish; + this.benchmarkResultPath = benchmarkResultPath; } static TraceReplayRequest from(Intent intent, File filesDir, String defaultBackend) { String outputDir = readString(intent, "output_dir", new File(filesDir, "trace-replay").getAbsolutePath()); String diffPath = readString(intent, "diff_path", ""); + String benchmarkResultPath = + readString(intent, "benchmark_result_path", outputDir + "/benchmark.json"); return new TraceReplayRequest( readString(intent, "trace_path", ""), readString(intent, "golden_path", ""), @@ -267,7 +292,11 @@ public final class TraceReplayActivity extends Activity { intent.getBooleanExtra("fix_iterationrp_subgroup_scratch", false), intent.getBooleanExtra("derive_num_subgroups", false), intent.getBooleanExtra("iterationrp_fix_barrier", false), - readString(intent, "texture_2d_dumps", "") + readString(intent, "texture_2d_dumps", ""), + intent.getBooleanExtra("benchmark", false), + intent.getIntExtra("benchmark_tail_frames", 200), + intent.getBooleanExtra("benchmark_finish", true), + benchmarkResultPath ); } diff --git a/android-plugin/trace-replay-ci.sh b/android-plugin/trace-replay-ci.sh index 08c0639f..d5119dfe 100644 --- a/android-plugin/trace-replay-ci.sh +++ b/android-plugin/trace-replay-ci.sh @@ -32,6 +32,10 @@ Usage: [--avoid-angle-llvmpipe-explicit-lod-bias] \ [--coherent-as-flush] \ [--dump-texture-2d CALL,TEXTURE,LEVEL,DIR] \ + [--benchmark] \ + [--benchmark-tail-frames N] \ + [--benchmark-finish 0|1] \ + [--reuse-fixture] \ --timeout-seconds N Set MOBILEGL_USE_ANGLE=1 to run DirectGLES replay with packaged ANGLE @@ -50,6 +54,12 @@ sample with an explicit LOD that ANGLE llvmpipe cannot take a LOD bias on (MOBILEGL_AVOID_EXPLICIT_LOD_BIAS=1). Pass --coherent-as-flush for traces whose engine writes persistent GL_MAP_FLUSH_EXPLICIT_BIT maps it never flushes (MOBILEGL_COHERENT_AS_FLUSH=1). +Pass --benchmark to replay the whole trace as a frame-timing benchmark instead of +snapshotting one frame and comparing it against the golden. The run then also +copies benchmark.json (per-frame times plus mean/median/p95) out of the app, and +"passed" only means the replay reached the end of the trace without an error. +Pass --reuse-fixture to skip re-extracting and re-pushing the trace, for repeat +runs of a case whose fixture is already in /data/local/tmp. EOF } @@ -109,6 +119,10 @@ avoid_angle_llvmpipe_sampler_mipmap_min_filter=0 avoid_angle_llvmpipe_explicit_lod_bias=0 coherent_as_flush=0 texture_2d_dumps="" +benchmark=0 +benchmark_tail_frames=200 +benchmark_finish=1 +reuse_fixture=0 timeout_seconds="" while [ "$#" -gt 0 ]; do @@ -150,6 +164,10 @@ while [ "$#" -gt 0 ]; do ;; --coherent-as-flush) coherent_as_flush=1; shift 1 ;; --dump-texture-2d) texture_2d_dumps="$(next_arg "$@")"; shift 2 ;; + --benchmark) benchmark=1; shift 1 ;; + --benchmark-tail-frames) benchmark_tail_frames="$(next_arg "$@")"; shift 2 ;; + --benchmark-finish) benchmark_finish="$(next_arg "$@")"; shift 2 ;; + --reuse-fixture) reuse_fixture=1; shift 1 ;; --timeout-seconds) timeout_seconds="$(next_arg "$@")"; shift 2 ;; -h|--help) usage; exit 0 ;; *) die "unknown argument: $1" ;; @@ -339,7 +357,11 @@ run_retrace() { fi mkdir -p "${result_dir}" - "${ADB}" install -r "$(host_path_for_adb "${apk_file}")" + # A repeat run of a case the previous invocation already installed and pushed only needs + # the on-device copy back into a fresh app output directory. + if [ "${reuse_fixture}" -eq 0 ]; then + "${ADB}" install -r "$(host_path_for_adb "${apk_file}")" + fi copy_fixture_to_app adb_device_path shell am force-stop "${package_name}" "${ADB}" logcat -c @@ -378,6 +400,16 @@ run_retrace() { if [ -n "${texture_2d_dumps}" ]; then set -- "$@" --es texture_2d_dumps "${texture_2d_dumps}" fi + if [ "${benchmark}" -eq 1 ]; then + set -- "$@" --ez benchmark true + set -- "$@" --ei benchmark_tail_frames "${benchmark_tail_frames}" + set -- "$@" --es benchmark_result_path "${app_dir}/output/benchmark.json" + if [ "${benchmark_finish}" = "0" ]; then + set -- "$@" --ez benchmark_finish false + else + set -- "$@" --ez benchmark_finish true + fi + fi set -- "$@" \ --es output_dir "${app_dir}/output" \ --es diff_path "${app_dir}/output/${safe_case}-diff.png" \ @@ -434,10 +466,16 @@ run_retrace() { fi adb_device_path exec-out run-as "${package_name}" cat "${app_dir}/output/result.json" > "${result_dir}/result.json" cat "${result_dir}/result.json" - copy_app_artifact "${app_dir}/output/actual.png" "${result_dir}/${safe_case}-${backend}-actual.png" - copy_app_artifact "${app_dir}/output/${safe_case}-diff.png" "${result_dir}/${safe_case}-${backend}-diff.png" + # A benchmark run takes no snapshot, so there is no actual/diff pair to copy. + if [ "${benchmark}" -eq 0 ]; then + copy_app_artifact "${app_dir}/output/actual.png" "${result_dir}/${safe_case}-${backend}-actual.png" + copy_app_artifact "${app_dir}/output/${safe_case}-diff.png" "${result_dir}/${safe_case}-${backend}-diff.png" + fi copy_app_artifact "${app_dir}/output/retrace.log" "${result_dir}/retrace.log" copy_app_artifact "${app_dir}/output/mobilegl.log" "${result_dir}/mobilegl.log" + if [ "${benchmark}" -eq 1 ]; then + copy_app_artifact "${app_dir}/output/benchmark.json" "${result_dir}/benchmark.json" + fi copy_texture_2d_dumps # A replay that wrote result.json but did not pass used to print nothing but @@ -468,6 +506,8 @@ run_retrace() { } mkdir -p "${fixture_root}" "${result_root}" -prepare_fixture +if [ "${reuse_fixture}" -eq 0 ]; then + prepare_fixture +fi run_retrace diff --git a/tools/trace_replay/CMakeLists.txt b/tools/trace_replay/CMakeLists.txt index c048f244..9d7d1dc5 100644 --- a/tools/trace_replay/CMakeLists.txt +++ b/tools/trace_replay/CMakeLists.txt @@ -225,7 +225,8 @@ add_library(mobilegl_trace_glretrace_common STATIC "${APITRACE_ROOT}/retrace/metric_helper.cpp" "${APITRACE_ROOT}/retrace/metric_writer.cpp" "${MOBILEGL_TRACE_ROOT}/apitrace_fbo_dump.cpp" - "${MOBILEGL_TRACE_ROOT}/apitrace_glws_egl.cpp") + "${MOBILEGL_TRACE_ROOT}/apitrace_glws_egl.cpp" + "${MOBILEGL_TRACE_SHARED_CPP_DIR}/trace_benchmark.cpp") if(APPLE) set(MOBILEGL_TRACE_APPLE_FRAMEWORKS "-framework Cocoa" @@ -239,7 +240,8 @@ target_include_directories(mobilegl_trace_glretrace_common PUBLIC "${APITRACE_GENERATED_DIR}" "${APITRACE_ROOT}/dispatch" "${APITRACE_ROOT}/helpers" - "${APITRACE_ROOT}/retrace") + "${APITRACE_ROOT}/retrace" + "${MOBILEGL_TRACE_SHARED_CPP_DIR}") target_compile_definitions(mobilegl_trace_glretrace_common PRIVATE exit=mobilegl_apitrace_exit main=mobilegl_apitrace_main) diff --git a/tools/trace_replay/README.md b/tools/trace_replay/README.md index eecc33a9..160bd7e6 100644 --- a/tools/trace_replay/README.md +++ b/tools/trace_replay/README.md @@ -280,6 +280,72 @@ unflushed persistent maps, e.g. the Create fixtures), pass sundial-lite fixture), pass `--ez avoid_angle_llvmpipe_explicit_lod_bias true` so the replay runs with `MOBILEGL_AVOID_EXPLICIT_LOD_BIAS=1`. +## Benchmark mode (frame timing) + +Benchmark mode reuses the same fixtures as a performance harness instead of a +correctness one: it replays the trace from the first call to the last, takes a +wall-clock timestamp at every frame boundary, and takes no snapshot and runs no +SSIM comparison. `passed` then only means the replay reached the end of the trace +without an error. + +Frame times include GPU completion by default, because a swap boundary on a tiled +mobile GPU returns long before the tiler is done and would otherwise time CPU +submission alone. That is what `benchmark_finish` / `--benchmark-finish` controls: +on (the default) it issues a `glFinish` through the replayed context at every +frame boundary, which serializes CPU/GPU overlap - pessimistic against a real +running game, but deterministic and comparable between backends and revisions. +Turn it off to measure CPU-side submission only. + +The mean/median/p95 are computed over the last `benchmark_tail_frames` frames +(default 200, clamped to the frames actually recorded); the head of a trace is +dominated by shader compiles and first-use uploads. The full per-frame array is in +the timing JSON, next to the headline numbers, which also appear in `result.json`. + +Whole device runs, one line per run plus the best of the repeats: + +```sh +python tools/trace_replay/run_android_retrace_local.py --benchmark \ + --case minecraft-1.21.4-fabric-iris-photon-in-world --backend DirectVulkan \ + --benchmark-repeats 3 +``` + +`--benchmark-tail-frames N`, `--benchmark-no-finish` and +`--benchmark-timeout-seconds N` are available; only the first repeat installs the +APK and pushes the trace. Each run's `benchmark.json` is kept next to the case +result as `benchmark-run.json`. + +The Activity takes the same settings directly: + +```sh +adb shell am start -a top.mobilegl.plugin.TRACE_REPLAY \ + -n $PKG/top.mobilegl.plugin.trace.TraceReplayActivity \ + --es trace_path $APP_DIR/input/openra.trace \ + --es output_dir $APP_DIR/output \ + --es backend DirectGLES \ + --ei width 640 --ei height 480 \ + --ez benchmark true \ + --ei benchmark_tail_frames 200 \ + --ez benchmark_finish true \ + --es benchmark_result_path $APP_DIR/output/benchmark.json +adb exec-out run-as $PKG cat files/trace-replay/output/benchmark.json > benchmark.json +``` + +`golden_path` and `target_call` are not needed in benchmark mode. The Linux CLI +takes the same options: + +```sh +./mobilegl_trace_replay --trace openra.trace --output out --backend DirectGLES \ + --benchmark --benchmark-tail-frames=200 --benchmark-finish=1 \ + --benchmark-result=out/benchmark.json +``` + +Two caveats when reading the numbers. Frame times are taken at `eglSwapBuffers`, +so a trace that ends frames with `glFrameTerminatorGREMEDY` instead is not timed. +And on a window surface the swap can block on the compositor, which pins frame +times to the display refresh; replay against the pbuffer surface +(`--ez use_pbuffer true`) to measure the renderer rather than the presentation +path. + ## Reproducing the Android DirectGLES lane on Linux (ANGLE on lavapipe) The APK workflow's DirectGLES lane is not the same stack as the Linux one, which diff --git a/tools/trace_replay/apitrace_glws_egl.cpp b/tools/trace_replay/apitrace_glws_egl.cpp index 08ce8a14..16efa595 100644 --- a/tools/trace_replay/apitrace_glws_egl.cpp +++ b/tools/trace_replay/apitrace_glws_egl.cpp @@ -2,6 +2,7 @@ #include "retrace.hpp" #include "apitrace_fbo_dump.hpp" +#include "trace_benchmark.hpp" #include #include @@ -414,6 +415,9 @@ public: char callNo[32]; snprintf(callNo, sizeof(callNo), "%u", retrace::callNo); gEgl.swapBuffers(gDisplay, surface); + // Frame boundary: retrace_eglSwapBuffers() has already run frame_complete() and + // handed the frame to us. No-op unless benchmark mode armed the timer. + mobilegl_trace::benchmark::OnFrameBoundary(); #if defined(__APPLE__) PumpMacOSEvents(); if (window && !windowShown) { diff --git a/tools/trace_replay/run_android_retrace_local.py b/tools/trace_replay/run_android_retrace_local.py index 47a34fef..51a8cd24 100644 --- a/tools/trace_replay/run_android_retrace_local.py +++ b/tools/trace_replay/run_android_retrace_local.py @@ -119,7 +119,7 @@ def render_summary(): shutil.copyfile(SUMMARY_DIR / SUMMARY_HTML, SUMMARY_DIR / "index.html") -def run_case(case, backend): +def run_case(case, backend, extra_args=None, timeout_seconds=None): backend_info = BACKENDS[backend] apk = find_trace_apk() trace_archive = FIXTURES / case["trace_archive"] @@ -176,8 +176,9 @@ def run_case(case, backend): "--crop-height", str(case["crop_height"]), "--timeout-seconds", - str(case["timeout_seconds"]), + str(timeout_seconds if timeout_seconds is not None else case["timeout_seconds"]), ] + command.extend(extra_args or []) if alternate is not None: command[command.index("--target-call"):command.index("--target-call")] = ["--alternate-golden", bash_path(alternate)] if backend_info["use_pbuffer"]: @@ -204,12 +205,118 @@ def run_case(case, backend): return result.returncode +def read_benchmark(case, backend, run_index): + """Reads the benchmark.json the run just pulled and files it under the run number.""" + result_dir = RESULT_ROOT / f"{safe_case(case['name'])}-{backend}" + source = result_dir / "benchmark.json" + if not source.exists(): + return None + try: + report = json.loads(source.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + print(f"failed to read {source}: {error}", file=sys.stderr) + return None + shutil.copyfile(source, result_dir / f"benchmark-run{run_index}.json") + return report + + +def format_benchmark(report): + return ( + f"frames={report.get('totalFrames', -1)}" + f" total={report.get('totalSeconds', -1):.1f}s" + f" tail={report.get('tailFrames', -1)}" + f" mean={report.get('meanFrameMs', -1):.3f}ms" + f" median={report.get('medianFrameMs', -1):.3f}ms" + f" p95={report.get('p95FrameMs', -1):.3f}ms" + f" fps={report.get('fps', -1):.1f}" + ) + + +def run_benchmark_case(case, backend, args): + """Runs the case as a frame-timing benchmark `--benchmark-repeats` times. + + Only the first run installs the APK and pushes the trace; the repeats reuse what is + already on the device, so the numbers are not paying for an adb push each time. + """ + label = f"{case['name']} / {backend}" + reports = [] + failures = 0 + for run_index in range(1, args.benchmark_repeats + 1): + extra_args = [ + "--benchmark", + "--benchmark-tail-frames", + str(args.benchmark_tail_frames), + "--benchmark-finish", + "0" if args.benchmark_no_finish else "1", + ] + if run_index > 1: + extra_args.append("--reuse-fixture") + # The previous repeat's file would otherwise be read back as this run's result. + stale = RESULT_ROOT / f"{safe_case(case['name'])}-{backend}" / "benchmark.json" + if stale.exists(): + stale.unlink() + rc = run_case(case, backend, extra_args=extra_args, timeout_seconds=args.benchmark_timeout_seconds) + report = read_benchmark(case, backend, run_index) + if rc != 0 or report is None: + print(f"{label} run {run_index}/{args.benchmark_repeats}: FAILED (exit {rc})", flush=True) + failures += 1 + continue + reports.append((run_index, report)) + print( + f"{label} run {run_index}/{args.benchmark_repeats}: {format_benchmark(report)}", + flush=True, + ) + + def mean_frame_ms(entry): + # A run that recorded no frames reports -1; it must not win "best" by being smallest. + mean = entry[1].get("meanFrameMs", -1) + return mean if mean > 0 else float("inf") + + if reports: + best_index, best = min(reports, key=mean_frame_ms) + print( + f"{label} best of {args.benchmark_repeats} (run {best_index}): {format_benchmark(best)}", + flush=True, + ) + return 1 if failures else 0 + + def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("--case", action="append", dest="cases", help="Case name to run; may be repeated.") parser.add_argument("--backend", action="append", choices=sorted(BACKENDS), help="Backend to run; may be repeated.") parser.add_argument("--all", action="store_true", help="Run every case in the APK workflow matrix.") parser.add_argument("--keep-results", action="store_true", help="Do not clear the previous result root.") + parser.add_argument( + "--benchmark", + action="store_true", + help="Replay each case end to end as a frame-timing benchmark instead of comparing " + "one frame against its golden.", + ) + parser.add_argument( + "--benchmark-repeats", + type=int, + default=3, + help="Benchmark runs per case/backend; the best (lowest mean frame time) is reported.", + ) + parser.add_argument( + "--benchmark-tail-frames", + type=int, + default=200, + help="Frames at the end of the run the statistics are computed over.", + ) + parser.add_argument( + "--benchmark-no-finish", + action="store_true", + help="Do not glFinish at every frame boundary, so frame times measure CPU submission " + "only instead of GPU completion.", + ) + parser.add_argument( + "--benchmark-timeout-seconds", + type=int, + default=900, + help="Per-run timeout; a benchmark replays the whole trace, not just up to target_call.", + ) return parser.parse_args() @@ -221,12 +328,20 @@ def main(): if not selected_cases: print("No cases selected. Use --all or --case NAME.", file=sys.stderr) return 2 + if args.benchmark and args.benchmark_repeats < 1: + print("--benchmark-repeats must be at least 1.", file=sys.stderr) + return 2 if not args.keep_results and RESULT_ROOT.exists(): shutil.rmtree(RESULT_ROOT) RESULT_ROOT.mkdir(parents=True, exist_ok=True) failures = 0 for case in selected_cases: for backend in selected_backends: + if args.benchmark: + print(f"=== Android benchmark: {case['name']} / {backend} ===", flush=True) + # No SSIM verdicts to render here; the summary page is for the correctness lane. + failures += run_benchmark_case(case, backend, args) + continue print(f"=== Android retrace: {case['name']} / {backend} ===", flush=True) rc = run_case(case, backend) try: diff --git a/tools/trace_replay/trace_replay_cli.cpp b/tools/trace_replay/trace_replay_cli.cpp index a8345942..d5a05b78 100644 --- a/tools/trace_replay/trace_replay_cli.cpp +++ b/tools/trace_replay/trace_replay_cli.cpp @@ -29,6 +29,15 @@ void PrintUsage(const char *argv0) { << " --crop-width N Compare crop width\n" << " --crop-height N Compare crop height\n" << " --coherent-as-flush Set MOBILEGL_COHERENT_AS_FLUSH=1 for the replay\n" + << " --benchmark Frame-timing mode: replay the whole trace, record a\n" + << " wall-clock timestamp at every frame boundary, and skip\n" + << " the snapshot and the golden comparison. --target-call is\n" + << " not required in this mode.\n" + << " --benchmark-tail-frames=N Frames at the end of the run the mean/median/p95 are\n" + << " computed over (default: 200, clamped to the frame count)\n" + << " --benchmark-finish=0/1 glFinish at every frame boundary so a frame time covers\n" + << " GPU completion and not just CPU submission (default: 1)\n" + << " --benchmark-result=PATH Timing JSON output (default: OUTPUT/benchmark.json)\n" << " --dump-fbo-attachments CALL:DIR[:FBO,FBO,...]\n" << " At CALL, write every colour attachment and the depth\n" << " attachment of every live framebuffer object into DIR as\n" @@ -77,6 +86,23 @@ bool ReadEnvFlag(const char *name) { return value != nullptr && std::string(value) == "1"; } +// The benchmark options are documented as --name=VALUE. The space-separated spelling every +// other option here uses is accepted too, so --benchmark-tail-frames=300 and +// --benchmark-tail-frames 300 both work; hasInlineValue says which one was given. +bool MatchOption(const std::string &arg, const char *name, std::string &inlineValue, bool &hasInlineValue) { + if (arg == name) { + hasInlineValue = false; + return true; + } + const std::string prefix = std::string(name) + "="; + if (arg.compare(0, prefix.size(), prefix) == 0) { + inlineValue = arg.substr(prefix.size()); + hasInlineValue = true; + return true; + } + return false; +} + bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) { request.backend = "DirectGLES"; request.fixIterationRPSubgroupScratch = ReadEnvFlag("MOBILEGL_FIX_ITERATIONRP_SUBGROUP_SCRATCH"); @@ -85,6 +111,8 @@ bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) { for (int i = 1; i < argc; ++i) { const std::string arg = argv[i]; + std::string optionValue; + bool optionHasValue = false; if (arg == "--trace") { if (!ReadValue(argc, argv, i, request.tracePath)) return false; } else if (arg == "--golden") { @@ -129,6 +157,17 @@ bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) { if (!ReadInt(argc, argv, i, request.cropHeight)) return false; } else if (arg == "--coherent-as-flush") { request.coherentAsFlush = true; + } else if (arg == "--benchmark") { + request.benchmark = true; + } else if (MatchOption(arg, "--benchmark-tail-frames", optionValue, optionHasValue)) { + if (!optionHasValue && !ReadValue(argc, argv, i, optionValue)) return false; + request.benchmarkTailFrames = std::atoi(optionValue.c_str()); + } else if (MatchOption(arg, "--benchmark-finish", optionValue, optionHasValue)) { + if (!optionHasValue && !ReadValue(argc, argv, i, optionValue)) return false; + request.benchmarkFinish = optionValue != "0"; + } else if (MatchOption(arg, "--benchmark-result", optionValue, optionHasValue)) { + if (!optionHasValue && !ReadValue(argc, argv, i, optionValue)) return false; + request.benchmarkResultPath = optionValue; } else if (arg == "--dump-fbo-attachments") { std::string dumpPoint; if (!ReadValue(argc, argv, i, dumpPoint)) return false; @@ -153,7 +192,7 @@ bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) { std::cerr << "--output is required\n"; return false; } - if (request.targetCall < 0) { + if (request.targetCall < 0 && !request.benchmark) { std::cerr << "--target-call is required\n"; return false; } @@ -161,6 +200,10 @@ bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) { std::cerr << "--hold-ms must be non-negative\n"; return false; } + if (request.benchmarkTailFrames <= 0) { + std::cerr << "--benchmark-tail-frames must be positive\n"; + return false; + } return true; } @@ -181,9 +224,15 @@ int main(int argc, char **argv) { mobilegl_trace::WriteResultJson(request, result); } - std::cout << result.message << "\n" - << "result: " << result.resultPath << "\n" - << "actual: " << result.actualPath << "\n" - << "diff: " << result.diffPath << "\n"; + if (request.benchmark) { + std::cout << result.message << "\n" + << "result: " << result.resultPath << "\n" + << "benchmark: " << result.benchmarkResultPath << "\n"; + } else { + std::cout << result.message << "\n" + << "result: " << result.resultPath << "\n" + << "actual: " << result.actualPath << "\n" + << "diff: " << result.diffPath << "\n"; + } return result.passed ? 0 : result.statusCode; }