[Feat] (TraceReplay): benchmark mode with per-frame timing for device fixtures

This commit is contained in:
2026-08-26 02:17:35 -04:00
parent 386bd7e461
commit c1d6a3c908
14 changed files with 669 additions and 46 deletions
@@ -218,7 +218,8 @@ add_library(glretrace_common STATIC
"${APITRACE_ROOT}/retrace/metric_backend_opengl.cpp"
"${APITRACE_ROOT}/retrace/metric_helper.cpp"
"${APITRACE_ROOT}/retrace/metric_writer.cpp"
apitrace_glws_android.cpp)
apitrace_glws_android.cpp
trace_benchmark.cpp)
target_include_directories(glretrace_common PUBLIC
"${APITRACE_GENERATED_DIR}"
"${APITRACE_ROOT}/dispatch"
@@ -1,6 +1,7 @@
#include "apitrace_fbo_dump.hpp"
#include "glws.hpp"
#include "retrace.hpp"
#include "trace_benchmark.hpp"
#include <android/native_window.h>
#include <EGL/egl.h>
@@ -186,6 +187,9 @@ public:
char callNo[32];
snprintf(callNo, sizeof(callNo), "%u", retrace::callNo);
gEgl.swapBuffers(gDisplay, surface);
// Frame boundary: retrace_eglSwapBuffers() has already run frame_complete() and
// handed the frame to us. No-op unless benchmark mode armed the timer.
mobilegl_trace::benchmark::OnFrameBoundary();
HoldAfterTargetPresent(callNo);
}
}
@@ -0,0 +1,94 @@
#include "trace_benchmark.hpp"
#include <chrono>
#include <cstdlib>
#include <utility>
#include <dlfcn.h>
namespace mobilegl_trace {
namespace benchmark {
namespace {
using Clock = std::chrono::steady_clock;
using GlFinishFn = void (*)();
constexpr std::size_t kFrameReserve = 4096;
bool gEnabled = false;
bool gFinishEachFrame = false;
bool gResolvedGlFinish = false;
GlFinishFn gGlFinish = nullptr;
Clock::time_point gStart;
Clock::time_point gLastBoundary;
std::vector<double> gFrameMs;
// Same resolution order the glws layers use for MobileGL's entry points: the replay driver
// already dlopen()ed the library with RTLD_GLOBAL before retrace started, so RTLD_NOLOAD
// finds that handle instead of loading a second copy, and RTLD_DEFAULT is the fallback for
// the case where it was linked in rather than dlopen()ed.
GlFinishFn ResolveGlFinish() {
void *handle = nullptr;
const char *library = std::getenv("MOBILEGL_TRACE_LIBRARY");
if (library != nullptr && library[0] != '\0') {
handle = dlopen(library, RTLD_NOW | RTLD_GLOBAL | RTLD_NOLOAD);
}
if (handle == nullptr) {
handle = dlopen("libMobileGL.so", RTLD_NOW | RTLD_GLOBAL | RTLD_NOLOAD);
}
if (handle != nullptr) {
void *symbol = dlsym(handle, "glFinish");
if (symbol != nullptr) {
return reinterpret_cast<GlFinishFn>(symbol);
}
}
return reinterpret_cast<GlFinishFn>(dlsym(RTLD_DEFAULT, "glFinish"));
}
} // namespace
void Begin(bool finishEachFrame) {
gFrameMs.clear();
gFrameMs.reserve(kFrameReserve);
gFinishEachFrame = finishEachFrame;
gResolvedGlFinish = false;
gGlFinish = nullptr;
gStart = Clock::now();
gLastBoundary = gStart;
gEnabled = true;
}
void OnFrameBoundary() {
if (!gEnabled) {
return;
}
if (gFinishEachFrame) {
// Resolved on the first boundary rather than in Begin(): a context only exists once
// the trace has created one, and glFinish before that would be pointless anyway.
if (!gResolvedGlFinish) {
gGlFinish = ResolveGlFinish();
gResolvedGlFinish = true;
}
if (gGlFinish != nullptr) {
gGlFinish();
}
}
const Clock::time_point now = Clock::now();
gFrameMs.push_back(std::chrono::duration<double, std::milli>(now - gLastBoundary).count());
gLastBoundary = now;
}
Report End() {
Report report;
if (!gEnabled) {
return report;
}
gEnabled = false;
gFinishEachFrame = false;
report.totalSeconds = std::chrono::duration<double>(Clock::now() - gStart).count();
report.frameMs = std::move(gFrameMs);
gFrameMs.clear();
return report;
}
} // namespace benchmark
} // namespace mobilegl_trace
@@ -0,0 +1,45 @@
#pragma once
#include <vector>
namespace mobilegl_trace {
namespace benchmark {
// Per-frame wall-clock timing for the retrace loop, shared by the Android replay runner
// and the desktop CLI. Disarmed unless Begin() armed it, and the frame-boundary hook is a
// single bool test in that case, so the correctness harness pays nothing for it.
//
// Retrace runs --singlethread, so all of this is deliberately plain globals: Begin(),
// OnFrameBoundary() and End() are only ever reached from the one retrace thread.
// Arms timing for the retrace that is about to run.
//
// finishEachFrame issues a full glFinish through the replayed context at every frame
// boundary, so a recorded frame time covers GPU completion and not just CPU submission.
// That matters on tiled mobile GPUs, where a swap without a sync returns long before the
// tiler is done and the numbers degenerate into "how fast can we feed the driver". The
// price is that finishing every frame serializes CPU/GPU overlap, so the absolute frame
// times are pessimistic against a real running game - they are deterministic and
// comparable between backends and revisions, which is what a benchmark fixture is for.
// With finishEachFrame off the run measures CPU-side submission only.
void Begin(bool finishEachFrame);
// Frame-boundary hook. Called from the platform glws swapBuffers override, which is where
// apitrace's replay loop advances the frame: retrace_eglSwapBuffers() calls
// frame_complete() and then Drawable::swapBuffers().
void OnFrameBoundary();
struct Report {
// Wall time of every completed frame, in milliseconds.
std::vector<double> frameMs;
// Begin() to End(), in seconds. Covers trace parsing and the leading partial frame too,
// which is why it is reported next to the per-frame statistics rather than derived from
// them.
double totalSeconds = 0.0;
};
// Disarms timing and hands back what was recorded.
Report End();
} // namespace benchmark
} // namespace mobilegl_trace
@@ -3,10 +3,13 @@
#include <dlfcn.h>
#include "apitrace_exit.hpp"
#include "png.h"
#include "trace_benchmark.hpp"
#include <algorithm>
#include <cerrno>
#include <chrono>
#include <cmath>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
@@ -421,32 +424,30 @@ std::string SnapshotCallSet(const Request& request) {
}
int RunRetraceMain(const Request& request) {
std::string prefix = request.outputDir + "/actual.";
std::string callSet = SnapshotCallSet(request);
std::vector<std::string> args;
args.emplace_back("mobilegl-glretrace");
args.emplace_back("-b");
args.emplace_back("--singlethread");
args.emplace_back("--no-context-check");
if (!request.benchmark) {
// The snapshot callset is also what stops the replay: apitrace exits once it has
// dumped the last call in -S. Benchmark mode wants the whole trace, so the -s/-S
// pair is left off entirely, which drops the readback and the PNG encode with it.
args.emplace_back("--snapshot-alpha");
args.emplace_back("-s");
args.emplace_back(request.outputDir + "/actual.");
args.emplace_back("-S");
args.emplace_back(SnapshotCallSet(request));
}
args.emplace_back(request.tracePath);
std::string arg0 = "mobilegl-glretrace";
std::string argBenchmark = "-b";
std::string argSingleThread = "--singlethread";
std::string argNoContextCheck = "--no-context-check";
std::string argSnapshotAlpha = "--snapshot-alpha";
std::string argSnapshotPrefix = "-s";
std::string argSnapshotCall = "-S";
std::string tracePath = request.tracePath;
char* argv[] = {
arg0.data(),
argBenchmark.data(),
argSingleThread.data(),
argNoContextCheck.data(),
argSnapshotAlpha.data(),
argSnapshotPrefix.data(),
prefix.data(),
argSnapshotCall.data(),
callSet.data(),
tracePath.data(),
nullptr,
};
return MOBILEGL_APITRACE_RETRACE_MAIN(10, argv);
std::vector<char*> argv;
argv.reserve(args.size() + 1);
for (std::string& arg : args) {
argv.push_back(arg.data());
}
argv.push_back(nullptr);
return MOBILEGL_APITRACE_RETRACE_MAIN(static_cast<int>(args.size()), argv.data());
}
bool RunRetrace(const Request& request, Result& result) {
@@ -475,6 +476,12 @@ bool RunRetrace(const Request& request, Result& result) {
return false;
}
if (request.benchmark) {
// Nothing was snapshotted, so there is nothing to collect or compare here; the
// caller turns the recorded frame times into the result instead.
return true;
}
std::string snapshotPath = SnapshotPathForCall(request);
if (!Exists(snapshotPath)) {
result.statusCode = STATUS_RETRACE_FAILED;
@@ -786,6 +793,82 @@ bool CompareWithGolden(const Request& request, Result& result) {
return result.passed;
}
std::string BenchmarkResultPath(const Request& request) {
return request.benchmarkResultPath.empty() ? request.outputDir + "/benchmark.json"
: request.benchmarkResultPath;
}
// Folds the recorded frame times into the headline numbers. Everything but totalSeconds and
// the frame count is computed over the trailing benchmarkTailFrames frames only.
void SummarizeBenchmark(const Request& request, const benchmark::Report& report, Result& result) {
result.benchmarkFrames = static_cast<long long>(report.frameMs.size());
result.benchmarkTotalSeconds = report.totalSeconds;
result.benchmarkTailFrames = 0;
if (report.frameMs.empty()) {
return;
}
const int requestedTail =
request.benchmarkTailFrames > 0 ? request.benchmarkTailFrames : kDefaultBenchmarkTailFrames;
const std::size_t tail =
std::min(static_cast<std::size_t>(requestedTail), report.frameMs.size());
result.benchmarkTailFrames = static_cast<int>(tail);
std::vector<double> window(report.frameMs.end() - static_cast<std::ptrdiff_t>(tail),
report.frameMs.end());
double sum = 0.0;
for (double frameMs : window) {
sum += frameMs;
}
result.benchmarkMeanMs = sum / static_cast<double>(tail);
std::sort(window.begin(), window.end());
result.benchmarkMedianMs = (tail % 2 == 1)
? window[tail / 2]
: 0.5 * (window[tail / 2 - 1] + window[tail / 2]);
// Nearest-rank p95, so the reported value is always an observed frame time.
std::size_t rank = static_cast<std::size_t>(std::ceil(0.95 * static_cast<double>(tail)));
if (rank == 0) {
rank = 1;
}
result.benchmarkP95Ms = window[rank - 1];
result.benchmarkFps = result.benchmarkMeanMs > 0.0 ? 1000.0 / result.benchmarkMeanMs : -1.0;
}
bool WriteBenchmarkJson(const Request& request,
const Result& result,
const benchmark::Report& report) {
std::ofstream file(result.benchmarkResultPath, std::ios::out | std::ios::trunc);
if (!file) {
return false;
}
file << "{\n";
file << " \"tracePath\": \"" << JsonEscape(request.tracePath) << "\",\n";
file << " \"backend\": \"" << JsonEscape(request.backend) << "\",\n";
file << " \"benchmarkFinish\": " << (request.benchmarkFinish ? "true" : "false") << ",\n";
file << " \"width\": " << request.width << ",\n";
file << " \"height\": " << request.height << ",\n";
file << " \"totalFrames\": " << result.benchmarkFrames << ",\n";
file << " \"tailFrames\": " << result.benchmarkTailFrames << ",\n";
file << std::fixed << std::setprecision(6);
file << " \"totalSeconds\": " << result.benchmarkTotalSeconds << ",\n";
file << std::setprecision(3);
file << " \"meanFrameMs\": " << result.benchmarkMeanMs << ",\n";
file << " \"medianFrameMs\": " << result.benchmarkMedianMs << ",\n";
file << " \"p95FrameMs\": " << result.benchmarkP95Ms << ",\n";
file << " \"fps\": " << result.benchmarkFps << ",\n";
file << " \"frameTimesMs\": [";
for (std::size_t i = 0; i < report.frameMs.size(); ++i) {
if (i > 0) {
file << ", ";
}
file << report.frameMs[i];
}
file << "]\n";
file << "}\n";
return static_cast<bool>(file);
}
} // namespace
extern "C" [[noreturn]] void mobilegl_apitrace_exit(int status) {
@@ -838,7 +921,25 @@ bool WriteResultJson(const Request& request, const Result& result) {
file << " \"deriveNumSubgroups\": " << (request.deriveNumSubgroups ? "true" : "false") << ",\n";
file << " \"iterationRPFixBarrier\": " << (request.iterationRPFixBarrier ? "true" : "false") << ",\n";
file << " \"holdMs\": " << request.holdMs << ",\n";
file << " \"mismatchPixels\": " << result.mismatchPixels << "\n";
file << " \"mismatchPixels\": " << result.mismatchPixels;
if (request.benchmark) {
// Headline numbers only; the per-frame array lives in benchmarkResultPath.
file << ",\n";
file << " \"benchmark\": true,\n";
file << " \"benchmarkResultPath\": \"" << JsonEscape(result.benchmarkResultPath) << "\",\n";
file << " \"benchmarkFinish\": " << (request.benchmarkFinish ? "true" : "false") << ",\n";
file << " \"benchmarkFrames\": " << result.benchmarkFrames << ",\n";
file << " \"benchmarkTailFrames\": " << result.benchmarkTailFrames << ",\n";
file << std::setprecision(6);
file << " \"benchmarkTotalSeconds\": " << result.benchmarkTotalSeconds << ",\n";
file << std::setprecision(3);
file << " \"benchmarkMeanFrameMs\": " << result.benchmarkMeanMs << ",\n";
file << " \"benchmarkMedianFrameMs\": " << result.benchmarkMedianMs << ",\n";
file << " \"benchmarkP95FrameMs\": " << result.benchmarkP95Ms << ",\n";
file << " \"benchmarkFps\": " << result.benchmarkFps << "\n";
} else {
file << "\n";
}
file << "}\n";
return true;
}
@@ -848,6 +949,9 @@ Result RunTraceReplay(const Request& request) {
result.resultPath = request.outputDir + "/result.json";
result.actualPath = request.outputDir + "/actual.png";
result.diffPath = request.diffPath;
if (request.benchmark) {
result.benchmarkResultPath = BenchmarkResultPath(request);
}
const std::string mobileGlLogPath = request.outputDir + "/mobilegl.log";
if (!EnsureDirectory(request.outputDir)) {
@@ -868,7 +972,8 @@ Result RunTraceReplay(const Request& request) {
return result;
}
if (request.targetCall < 0) {
// Benchmark mode never snapshots, so it has no target call to stop at.
if (!request.benchmark && request.targetCall < 0) {
result.statusCode = STATUS_INVALID_ARGUMENT;
result.message = "target_call must be set for dump-images style replay";
return result;
@@ -883,6 +988,40 @@ Result RunTraceReplay(const Request& request) {
return result;
}
if (request.benchmark) {
benchmark::Begin(request.benchmarkFinish);
const bool retraced = RunRetrace(request, result);
const benchmark::Report report = benchmark::End();
SummarizeBenchmark(request, report, result);
// Written even when the retrace failed: a partial timing series says where the
// replay got to, which is exactly what is wanted when triaging one.
const bool wroteJson = WriteBenchmarkJson(request, result, report);
HoldAfterRetrace(request);
if (!retraced) {
return result;
}
if (!wroteJson) {
result.statusCode = STATUS_IO_ERROR;
result.message = "benchmark completed but failed to write " + result.benchmarkResultPath;
return result;
}
// "Passed" in benchmark mode means the replay ran the trace to the end without
// error; there is no golden to be right or wrong about.
result.passed = true;
result.statusCode = STATUS_OK;
std::ostringstream message;
message << std::fixed << std::setprecision(3)
<< "benchmark completed; frames=" << result.benchmarkFrames
<< ", tailFrames=" << result.benchmarkTailFrames
<< ", meanMs=" << result.benchmarkMeanMs
<< ", medianMs=" << result.benchmarkMedianMs
<< ", p95Ms=" << result.benchmarkP95Ms
<< ", fps=" << result.benchmarkFps
<< ", benchmarkResultPath=" << result.benchmarkResultPath;
result.message = message.str();
return result;
}
if (!RunRetrace(request, result)) {
HoldAfterRetrace(request);
return result;
@@ -15,6 +15,8 @@ enum StatusCode {
STATUS_COMPARE_FAILED = 6,
};
constexpr int kDefaultBenchmarkTailFrames = 200;
struct Request {
std::string tracePath;
std::string goldenPath;
@@ -30,6 +32,19 @@ struct Request {
// Named GL_TEXTURE_2D dump points, each `CALL,TEXTURE,LEVEL,DIR`. Debug-only; the replay
// behaves exactly as before when this is empty.
std::vector<std::string> texture2dDumps;
// Benchmark (frame-timing) mode. Off by default. When on, the replay runs the whole
// trace from start to finish and records a wall-clock timestamp at every frame
// boundary; no snapshot is taken and no golden comparison runs.
bool benchmark = false;
// Number of trailing frames the summary statistics are computed over. Clamped to the
// number of frames actually recorded. The tail is what is comparable between runs: the
// head of a trace is dominated by shader compiles and first-use uploads.
int benchmarkTailFrames = kDefaultBenchmarkTailFrames;
// glFinish through the replayed context at every frame boundary, so a frame time
// includes GPU completion instead of only CPU submission. See trace_benchmark.hpp.
bool benchmarkFinish = true;
// Where the timing JSON goes. Defaults to <outputDir>/benchmark.json.
std::string benchmarkResultPath;
int targetFrame = -1;
long long targetCall = -1;
int width = 0;
@@ -60,6 +75,16 @@ struct Result {
std::string matchedGoldenPath;
double ssim = -1.0;
long long mismatchPixels = -1;
// Benchmark headline numbers. Left at the defaults below unless the request asked for
// benchmark mode; benchmarkResultPath then names the JSON with the per-frame array.
std::string benchmarkResultPath;
long long benchmarkFrames = -1;
int benchmarkTailFrames = 0;
double benchmarkTotalSeconds = -1.0;
double benchmarkMeanMs = -1.0;
double benchmarkMedianMs = -1.0;
double benchmarkP95Ms = -1.0;
double benchmarkFps = -1.0;
};
Result RunTraceReplay(const Request& request);
@@ -125,7 +125,11 @@ Java_top_mobilegl_plugin_trace_TraceReplayActivity_nativeRunTraceReplay(JNIEnv*
jboolean fixIterationRPSubgroupScratch,
jboolean deriveNumSubgroups,
jboolean iterationRPFixBarrier,
jstring texture2dDumps) {
jstring texture2dDumps,
jboolean benchmarkMode,
jint benchmarkTailFrames,
jboolean benchmarkFinish,
jstring benchmarkResultPath) {
mobilegl_trace::Request request;
request.tracePath = ToString(env, tracePath);
request.goldenPath = ToString(env, goldenPath);
@@ -156,6 +160,12 @@ Java_top_mobilegl_plugin_trace_TraceReplayActivity_nativeRunTraceReplay(JNIEnv*
request.fixIterationRPSubgroupScratch = fixIterationRPSubgroupScratch == JNI_TRUE;
request.deriveNumSubgroups = deriveNumSubgroups == JNI_TRUE;
request.iterationRPFixBarrier = iterationRPFixBarrier == JNI_TRUE;
request.benchmark = benchmarkMode == JNI_TRUE;
request.benchmarkTailFrames = benchmarkTailFrames > 0
? benchmarkTailFrames
: mobilegl_trace::kDefaultBenchmarkTailFrames;
request.benchmarkFinish = benchmarkFinish == JNI_TRUE;
request.benchmarkResultPath = ToString(env, benchmarkResultPath);
ScopedTraceReplayState replayState;
mobilegl_trace_set_requested_size(request.width, request.height);
@@ -119,7 +119,11 @@ public final class TraceReplayActivity extends Activity {
request.fixIterationRPSubgroupScratch,
request.deriveNumSubgroups,
request.iterationRPFixBarrier,
request.texture2dDumps
request.texture2dDumps,
request.benchmark,
request.benchmarkTailFrames,
request.benchmarkFinish,
request.benchmarkResultPath
);
Log.i(TAG, result.toString());
TraceReplayResult finalResult = result;
@@ -155,7 +159,11 @@ public final class TraceReplayActivity extends Activity {
boolean fixIterationRPSubgroupScratch,
boolean deriveNumSubgroups,
boolean iterationRPFixBarrier,
String texture2dDumps
String texture2dDumps,
boolean benchmark,
int benchmarkTailFrames,
boolean benchmarkFinish,
String benchmarkResultPath
);
private static final class TraceReplayRequest {
@@ -184,6 +192,13 @@ public final class TraceReplayActivity extends Activity {
final boolean deriveNumSubgroups;
final boolean iterationRPFixBarrier;
final String texture2dDumps;
// Benchmark (frame-timing) mode. Replays the whole trace, times every frame
// boundary, and skips the snapshot and the SSIM comparison; "passed" then only
// means the replay ran to the end without error.
final boolean benchmark;
final int benchmarkTailFrames;
final boolean benchmarkFinish;
final String benchmarkResultPath;
private TraceReplayRequest(
String tracePath,
@@ -210,7 +225,11 @@ public final class TraceReplayActivity extends Activity {
boolean fixIterationRPSubgroupScratch,
boolean deriveNumSubgroups,
boolean iterationRPFixBarrier,
String texture2dDumps
String texture2dDumps,
boolean benchmark,
int benchmarkTailFrames,
boolean benchmarkFinish,
String benchmarkResultPath
) {
this.tracePath = tracePath;
this.goldenPath = goldenPath;
@@ -237,11 +256,17 @@ public final class TraceReplayActivity extends Activity {
this.deriveNumSubgroups = deriveNumSubgroups;
this.iterationRPFixBarrier = iterationRPFixBarrier;
this.texture2dDumps = texture2dDumps;
this.benchmark = benchmark;
this.benchmarkTailFrames = benchmarkTailFrames;
this.benchmarkFinish = benchmarkFinish;
this.benchmarkResultPath = benchmarkResultPath;
}
static TraceReplayRequest from(Intent intent, File filesDir, String defaultBackend) {
String outputDir = readString(intent, "output_dir", new File(filesDir, "trace-replay").getAbsolutePath());
String diffPath = readString(intent, "diff_path", "");
String benchmarkResultPath =
readString(intent, "benchmark_result_path", outputDir + "/benchmark.json");
return new TraceReplayRequest(
readString(intent, "trace_path", ""),
readString(intent, "golden_path", ""),
@@ -267,7 +292,11 @@ public final class TraceReplayActivity extends Activity {
intent.getBooleanExtra("fix_iterationrp_subgroup_scratch", false),
intent.getBooleanExtra("derive_num_subgroups", false),
intent.getBooleanExtra("iterationrp_fix_barrier", false),
readString(intent, "texture_2d_dumps", "")
readString(intent, "texture_2d_dumps", ""),
intent.getBooleanExtra("benchmark", false),
intent.getIntExtra("benchmark_tail_frames", 200),
intent.getBooleanExtra("benchmark_finish", true),
benchmarkResultPath
);
}