From a1e2007b8283a22aaa3a7f858382f771e78e3d75 Mon Sep 17 00:00:00 2001 From: Swung0x48 Date: Sat, 20 Jun 2026 19:04:52 +0800 Subject: [PATCH] [Fix] (trace-replay): support alternate golden images - compare actual output against primary and alternate golden images - record the matched golden path in trace replay results - allow APK and Linux retrace fixtures to pass alternate golden paths - keep nostalgia validation accepting both Mesa and PC goldens --- .github/workflows/apk.yml | 2 + .../app/src/trace/cpp/trace_replay_core.cpp | 129 ++++++++++++++---- .../app/src/trace/cpp/trace_replay_core.hpp | 3 + .../app/src/trace/cpp/trace_replay_jni.cpp | 5 + .../plugin/trace/TraceReplayActivity.java | 6 + android-plugin/trace-replay-ci.sh | 37 ++++- tools/trace_replay/CMakeLists.txt | 3 + tools/trace_replay/run_trace_case.cmake | 5 + tools/trace_replay/trace_replay_cli.cpp | 7 + 9 files changed, 167 insertions(+), 30 deletions(-) diff --git a/.github/workflows/apk.yml b/.github/workflows/apk.yml index a4005ca1..020287be 100644 --- a/.github/workflows/apk.yml +++ b/.github/workflows/apk.yml @@ -287,6 +287,7 @@ jobs: trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-nostalgia-in-world.tgz trace_file: trace.trace golden: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-nostalgia-in-world.0000153808-linux-mesa.png + alternate_golden: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-nostalgia-in-world.0000153808.png target_call: 153808 width: 854 height: 480 @@ -429,6 +430,7 @@ jobs: --trace-archive "${{ matrix.case.trace_archive }}" \ --trace-file "${{ matrix.case.trace_file }}" \ --golden "${{ matrix.case.golden }}" \ + --alternate-golden "${{ matrix.case.alternate_golden || '' }}" \ --target-call "${{ matrix.case.target_call }}" \ --width "${{ matrix.case.width }}" \ --height "${{ matrix.case.height }}" \ diff --git a/android-plugin/app/src/trace/cpp/trace_replay_core.cpp b/android-plugin/app/src/trace/cpp/trace_replay_core.cpp index b814fba9..d3b0392c 100644 --- a/android-plugin/app/src/trace/cpp/trace_replay_core.cpp +++ b/android-plugin/app/src/trace/cpp/trace_replay_core.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -545,27 +546,29 @@ bool WriteDifferenceImage(const Result& result, return WritePngRgba(result.diffPath, diff, error); } -bool CompareWithGolden(const Request& request, Result& result) { - if (request.goldenPath.empty()) { - result.passed = true; - result.statusCode = STATUS_OK; - result.message = "retrace completed; golden_path was not provided"; - result.mismatchPixels = 0; - return true; - } - if (!Exists(request.goldenPath)) { - result.statusCode = STATUS_INVALID_ARGUMENT; - result.message = "golden_path does not exist or is not a regular file"; +struct GoldenComparison { + std::string path; + RgbaImage image; + long long mismatchPixels = std::numeric_limits::max(); + int x0 = 0; + int y0 = 0; + int compareWidth = 0; + int compareHeight = 0; +}; + +bool CompareAgainstOneGolden(const Request& request, + const RgbaImage& actual, + const std::string& goldenPath, + int fuzz, + GoldenComparison& comparison, + std::string& error) { + if (!Exists(goldenPath)) { + error = "golden_path does not exist or is not a regular file: " + goldenPath; return false; } - RgbaImage actual; RgbaImage golden; - std::string pngError; - if (!ReadPngRgba(result.actualPath, actual, pngError) || - !ReadPngRgba(request.goldenPath, golden, pngError)) { - result.statusCode = STATUS_COMPARE_FAILED; - result.message = pngError.empty() ? "failed to decode actual or golden PNG" : pngError; + if (!ReadPngRgba(goldenPath, golden, error)) { return false; } @@ -573,11 +576,11 @@ bool CompareWithGolden(const Request& request, Result& result) { int y0 = request.cropY; if (request.cropWidth <= 0 && request.cropHeight <= 0 && (actual.width != golden.width || actual.height != golden.height)) { - result.statusCode = STATUS_COMPARE_FAILED; std::ostringstream message; message << "actual image size " << actual.width << "x" << actual.height - << " does not match golden image size " << golden.width << "x" << golden.height; - result.message = message.str(); + << " does not match golden image size " << golden.width << "x" << golden.height + << ": " << goldenPath; + error = message.str(); return false; } @@ -589,12 +592,10 @@ bool CompareWithGolden(const Request& request, Result& result) { y0 + compareHeight > actual.height || x0 + compareWidth > golden.width || y0 + compareHeight > golden.height) { - result.statusCode = STATUS_INVALID_ARGUMENT; - result.message = "compare crop is outside actual or golden image bounds"; + error = "compare crop is outside actual or golden image bounds: " + goldenPath; return false; } - const int fuzz = std::max(0, std::min(100, request.fuzzPercent)) * 255 / 100; long long mismatch = 0; for (int y = 0; y < compareHeight; ++y) { for (int x = 0; x < compareWidth; ++x) { @@ -613,20 +614,83 @@ bool CompareWithGolden(const Request& request, Result& result) { } } + comparison.path = goldenPath; + comparison.image = std::move(golden); + comparison.mismatchPixels = mismatch; + comparison.x0 = x0; + comparison.y0 = y0; + comparison.compareWidth = compareWidth; + comparison.compareHeight = compareHeight; + return true; +} + +bool CompareWithGolden(const Request& request, Result& result) { + std::vector goldenPaths; + if (!request.goldenPath.empty()) { + goldenPaths.push_back(request.goldenPath); + } + for (const auto& alternateGoldenPath : request.alternateGoldenPaths) { + if (!alternateGoldenPath.empty()) { + goldenPaths.push_back(alternateGoldenPath); + } + } + + if (goldenPaths.empty()) { + result.passed = true; + result.statusCode = STATUS_OK; + result.message = "retrace completed; golden_path was not provided"; + result.mismatchPixels = 0; + return true; + } + + RgbaImage actual; + std::string pngError; + if (!ReadPngRgba(result.actualPath, actual, pngError)) { + result.statusCode = STATUS_COMPARE_FAILED; + result.message = pngError.empty() ? "failed to decode actual PNG" : pngError; + return false; + } + + const int fuzz = std::max(0, std::min(100, request.fuzzPercent)) * 255 / 100; + GoldenComparison bestComparison; + std::string comparisonError; + bool hasComparison = false; + for (const auto& goldenPath : goldenPaths) { + GoldenComparison comparison; + std::string error; + if (!CompareAgainstOneGolden(request, actual, goldenPath, fuzz, comparison, error)) { + comparisonError = error; + continue; + } + if (!hasComparison || comparison.mismatchPixels < bestComparison.mismatchPixels) { + bestComparison = std::move(comparison); + hasComparison = true; + } + } + + if (!hasComparison) { + result.statusCode = STATUS_COMPARE_FAILED; + result.message = comparisonError.empty() ? "failed to compare against any golden PNG" : comparisonError; + return false; + } + std::string diffError; - if (!WriteDifferenceImage(result, actual, golden, x0, y0, compareWidth, compareHeight, fuzz, diffError)) { + if (!WriteDifferenceImage(result, actual, bestComparison.image, bestComparison.x0, bestComparison.y0, + bestComparison.compareWidth, bestComparison.compareHeight, fuzz, diffError)) { result.statusCode = STATUS_IO_ERROR; result.message = diffError.empty() ? "failed to write diff PNG" : diffError; return false; } - result.mismatchPixels = mismatch; - result.passed = mismatch <= request.tolerance; + result.mismatchPixels = bestComparison.mismatchPixels; + result.matchedGoldenPath = bestComparison.path; + result.passed = bestComparison.mismatchPixels <= request.tolerance; result.statusCode = result.passed ? STATUS_OK : STATUS_COMPARE_FAILED; std::ostringstream message; - message << "retrace completed; mismatchPixels=" << mismatch + message << "retrace completed; mismatchPixels=" << bestComparison.mismatchPixels << ", tolerance=" << request.tolerance - << ", fuzzPercent=" << request.fuzzPercent; + << ", fuzzPercent=" << request.fuzzPercent + << ", matchedGoldenPath=" << bestComparison.path; result.message = message.str(); return result.passed; } @@ -648,6 +712,15 @@ bool WriteResultJson(const Request& request, const Result& result) { file << " \"message\": \"" << JsonEscape(result.message) << "\",\n"; file << " \"tracePath\": \"" << JsonEscape(request.tracePath) << "\",\n"; file << " \"goldenPath\": \"" << JsonEscape(request.goldenPath) << "\",\n"; + file << " \"alternateGoldenPaths\": ["; + for (std::size_t i = 0; i < request.alternateGoldenPaths.size(); ++i) { + if (i > 0) { + file << ", "; + } + file << "\"" << JsonEscape(request.alternateGoldenPaths[i]) << "\""; + } + file << "],\n"; + file << " \"matchedGoldenPath\": \"" << JsonEscape(result.matchedGoldenPath) << "\",\n"; file << " \"actualPath\": \"" << JsonEscape(result.actualPath) << "\",\n"; file << " \"diffPath\": \"" << JsonEscape(result.diffPath) << "\",\n"; file << " \"backend\": \"" << JsonEscape(request.backend) << "\",\n"; diff --git a/android-plugin/app/src/trace/cpp/trace_replay_core.hpp b/android-plugin/app/src/trace/cpp/trace_replay_core.hpp index de191bb6..6882fe8d 100644 --- a/android-plugin/app/src/trace/cpp/trace_replay_core.hpp +++ b/android-plugin/app/src/trace/cpp/trace_replay_core.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include namespace mobilegl_trace { @@ -17,6 +18,7 @@ enum StatusCode { struct Request { std::string tracePath; std::string goldenPath; + std::vector alternateGoldenPaths; std::string outputDir; std::string diffPath; std::string backend; @@ -40,6 +42,7 @@ struct Result { std::string resultPath; std::string actualPath; std::string diffPath; + std::string matchedGoldenPath; long long mismatchPixels = -1; }; diff --git a/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp b/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp index f0cc92e0..02a7c03f 100644 --- a/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp +++ b/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp @@ -54,6 +54,7 @@ Java_top_mobilegl_plugin_trace_TraceReplayActivity_nativeRunTraceReplay(JNIEnv* jobject surface, jstring tracePath, jstring goldenPath, + jstring alternateGoldenPath, jstring outputDir, jstring diffPath, jstring backend, @@ -70,6 +71,10 @@ Java_top_mobilegl_plugin_trace_TraceReplayActivity_nativeRunTraceReplay(JNIEnv* mobilegl_trace::Request request; request.tracePath = ToString(env, tracePath); request.goldenPath = ToString(env, goldenPath); + std::string alternateGolden = ToString(env, alternateGoldenPath); + if (!alternateGolden.empty()) { + request.alternateGoldenPaths.push_back(alternateGolden); + } request.outputDir = ToString(env, outputDir); request.diffPath = ToString(env, diffPath); request.backend = ToString(env, backend); diff --git a/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java b/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java index 8c150617..b18578cd 100644 --- a/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java +++ b/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java @@ -97,6 +97,7 @@ public final class TraceReplayActivity extends Activity { surface, request.tracePath, request.goldenPath, + request.alternateGoldenPath, request.outputDir, request.diffPath, request.backend, @@ -123,6 +124,7 @@ public final class TraceReplayActivity extends Activity { Surface surface, String tracePath, String goldenPath, + String alternateGoldenPath, String outputDir, String diffPath, String backend, @@ -141,6 +143,7 @@ public final class TraceReplayActivity extends Activity { private static final class TraceReplayRequest { final String tracePath; final String goldenPath; + final String alternateGoldenPath; final String outputDir; final String diffPath; final String backend; @@ -158,6 +161,7 @@ public final class TraceReplayActivity extends Activity { private TraceReplayRequest( String tracePath, String goldenPath, + String alternateGoldenPath, String outputDir, String diffPath, String backend, @@ -174,6 +178,7 @@ public final class TraceReplayActivity extends Activity { ) { this.tracePath = tracePath; this.goldenPath = goldenPath; + this.alternateGoldenPath = alternateGoldenPath; this.outputDir = outputDir; this.diffPath = diffPath; this.backend = backend; @@ -195,6 +200,7 @@ public final class TraceReplayActivity extends Activity { return new TraceReplayRequest( readString(intent, "trace_path", ""), readString(intent, "golden_path", ""), + readString(intent, "alternate_golden_path", ""), outputDir, diffPath, readString(intent, "backend", defaultBackend), diff --git a/android-plugin/trace-replay-ci.sh b/android-plugin/trace-replay-ci.sh index 869247de..d0808243 100644 --- a/android-plugin/trace-replay-ci.sh +++ b/android-plugin/trace-replay-ci.sh @@ -17,6 +17,7 @@ Usage: --trace-archive FILE \ --trace-file FILE_IN_ARCHIVE \ --golden FILE \ + [--alternate-golden FILE] \ --target-call N \ --width N \ --height N \ @@ -60,6 +61,7 @@ case_name="" trace_archive="" trace_file="" golden_path="" +alternate_golden_path="" target_call="" width="" height="" @@ -82,6 +84,15 @@ while [ "$#" -gt 0 ]; do --trace-archive) trace_archive="$(next_arg "$@")"; shift 2 ;; --trace-file) trace_file="$(next_arg "$@")"; shift 2 ;; --golden) golden_path="$(next_arg "$@")"; shift 2 ;; + --alternate-golden) + if [ "$#" -lt 2 ] || [ "${2#--}" != "$2" ]; then + alternate_golden_path="" + shift 1 + else + alternate_golden_path="$2" + shift 2 + fi + ;; --target-call) target_call="$(next_arg "$@")"; shift 2 ;; --width) width="$(next_arg "$@")"; shift 2 ;; --height) height="$(next_arg "$@")"; shift 2 ;; @@ -120,6 +131,9 @@ require_value "${timeout_seconds}" "--timeout-seconds" test -f "${apk_file}" || die "APK does not exist: ${apk_file}" test -f "${trace_archive}" || die "trace archive does not exist: ${trace_archive}" test -f "${golden_path}" || die "golden image does not exist: ${golden_path}" +if [ -n "${alternate_golden_path}" ]; then + test -f "${alternate_golden_path}" || die "alternate golden image does not exist: ${alternate_golden_path}" +fi safe_case="$(printf '%s' "${case_name}" | sed 's/[^A-Za-z0-9._-]/_/g')" @@ -134,32 +148,50 @@ prepare_fixture() { "${ADB}" push "${extracted_trace}" "/data/local/tmp/mobilegl-${safe_case}.trace" "${ADB}" push "${golden_path}" "/data/local/tmp/mobilegl-${safe_case}.golden.png" + if [ -n "${alternate_golden_path}" ]; then + "${ADB}" push "${alternate_golden_path}" "/data/local/tmp/mobilegl-${safe_case}.alternate-golden.png" + fi "${ADB}" shell chmod 0644 "/data/local/tmp/mobilegl-${safe_case}.trace" "/data/local/tmp/mobilegl-${safe_case}.golden.png" + if [ -n "${alternate_golden_path}" ]; then + "${ADB}" shell chmod 0644 "/data/local/tmp/mobilegl-${safe_case}.alternate-golden.png" + fi } copy_fixture_to_app() { trace_tmp="/data/local/tmp/mobilegl-${safe_case}.trace" golden_tmp="/data/local/tmp/mobilegl-${safe_case}.golden.png" + alternate_golden_tmp="/data/local/tmp/mobilegl-${safe_case}.alternate-golden.png" "${ADB}" shell run-as "${package_name}" rm -rf files/trace-replay "${ADB}" shell run-as "${package_name}" mkdir -p files/trace-replay/input files/trace-replay/output "${ADB}" shell run-as "${package_name}" cp "${trace_tmp}" files/trace-replay/input/trace.trace "${ADB}" shell run-as "${package_name}" cp "${golden_tmp}" files/trace-replay/input/golden.png + if [ -n "${alternate_golden_path}" ]; then + "${ADB}" shell run-as "${package_name}" cp "${alternate_golden_tmp}" files/trace-replay/input/alternate-golden.png + fi } run_retrace() { app_dir="/data/user/0/${package_name}/files/trace-replay" result_dir="${result_root}/${safe_case}-${backend}" + alternate_golden_app_path="" + if [ -n "${alternate_golden_path}" ]; then + alternate_golden_app_path="${app_dir}/input/alternate-golden.png" + fi mkdir -p "${result_dir}" "${ADB}" install -r "${apk_file}" copy_fixture_to_app "${ADB}" shell am force-stop "${package_name}" "${ADB}" logcat -c - "${ADB}" shell am start -W -a top.mobilegl.plugin.TRACE_REPLAY \ + set -- am start -W -a top.mobilegl.plugin.TRACE_REPLAY \ -n "${package_name}/top.mobilegl.plugin.trace.TraceReplayActivity" \ --es trace_path "${app_dir}/input/trace.trace" \ - --es golden_path "${app_dir}/input/golden.png" \ + --es golden_path "${app_dir}/input/golden.png" + if [ -n "${alternate_golden_app_path}" ]; then + set -- "$@" --es alternate_golden_path "${alternate_golden_app_path}" + fi + set -- "$@" \ --es output_dir "${app_dir}/output" \ --es diff_path "${app_dir}/output/${safe_case}-diff.png" \ --es backend "${backend}" \ @@ -172,6 +204,7 @@ run_retrace() { --ei crop_width "${crop_width}" \ --ei crop_height "${crop_height}" \ --ei fuzz_percent "${fuzz_percent}" + "${ADB}" shell "$@" for _ in $(seq 1 "${timeout_seconds}"); do if "${ADB}" shell run-as "${package_name}" ls files/trace-replay/output/result.json >/dev/null 2>&1; then diff --git a/tools/trace_replay/CMakeLists.txt b/tools/trace_replay/CMakeLists.txt index 3cd89dae..80c72478 100644 --- a/tools/trace_replay/CMakeLists.txt +++ b/tools/trace_replay/CMakeLists.txt @@ -269,6 +269,7 @@ function(add_trace_replay_test CASE_NAME BACKEND) TRACE_ARCHIVE TRACE_FILE GOLDEN + ALTERNATE_GOLDEN TARGET_CALL WIDTH HEIGHT @@ -314,6 +315,7 @@ function(add_trace_replay_test CASE_NAME BACKEND) -DTRACE_ARCHIVE=${TRACE_CASE_TRACE_ARCHIVE} -DTRACE_FILE=${TRACE_CASE_TRACE_FILE} -DTRACE_GOLDEN=${TRACE_CASE_GOLDEN} + -DTRACE_ALTERNATE_GOLDEN=${TRACE_CASE_ALTERNATE_GOLDEN} -DTRACE_BACKEND=${BACKEND} -DTRACE_TARGET_CALL=${TRACE_CASE_TARGET_CALL} -DTRACE_WIDTH=${TRACE_CASE_WIDTH} @@ -469,6 +471,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-nostalgia-in-wor TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-nostalgia-in-world.tgz TRACE_FILE trace.trace GOLDEN ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-nostalgia-in-world.0000153808-linux-mesa.png + ALTERNATE_GOLDEN ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-nostalgia-in-world.0000153808.png TARGET_CALL 153808 WIDTH 854 HEIGHT 480 diff --git a/tools/trace_replay/run_trace_case.cmake b/tools/trace_replay/run_trace_case.cmake index 546a55e6..c9c6c5d9 100644 --- a/tools/trace_replay/run_trace_case.cmake +++ b/tools/trace_replay/run_trace_case.cmake @@ -22,6 +22,10 @@ endif() if(NOT DEFINED TRACE_FUZZ_PERCENT OR "${TRACE_FUZZ_PERCENT}" STREQUAL "") set(TRACE_FUZZ_PERCENT 20) endif() +set(alternate_golden_args) +if(DEFINED TRACE_ALTERNATE_GOLDEN AND NOT "${TRACE_ALTERNATE_GOLDEN}" STREQUAL "") + list(APPEND alternate_golden_args --alternate-golden "${TRACE_ALTERNATE_GOLDEN}") +endif() if(EXISTS "${TRACE_OUTPUT_DIR}") file(REMOVE_RECURSE "${TRACE_OUTPUT_DIR}") @@ -50,6 +54,7 @@ execute_process( COMMAND "${TRACE_REPLAY_EXE}" --trace "${trace_path}" --golden "${TRACE_GOLDEN}" + ${alternate_golden_args} --diff "${TRACE_OUTPUT_DIR}/output/${TRACE_CASE_NAME}-diff.png" --output "${TRACE_OUTPUT_DIR}/output" --backend "${TRACE_BACKEND}" diff --git a/tools/trace_replay/trace_replay_cli.cpp b/tools/trace_replay/trace_replay_cli.cpp index af95fb72..2b1672a1 100644 --- a/tools/trace_replay/trace_replay_cli.cpp +++ b/tools/trace_replay/trace_replay_cli.cpp @@ -14,6 +14,7 @@ void PrintUsage(const char *argv0) { << "\n" << "Options:\n" << " --golden PATH Golden PNG to compare against\n" + << " --alternate-golden PATH Additional acceptable golden PNG\n" << " --diff PATH Difference PNG output path\n" << " --backend NAME DirectGLES or DirectVulkan (default: DirectGLES)\n" << " --mobilegl-library PATH libMobileGL.so path (default: libMobileGL.so)\n" @@ -64,6 +65,12 @@ bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) { if (!ReadValue(argc, argv, i, request.tracePath)) return false; } else if (arg == "--golden") { if (!ReadValue(argc, argv, i, request.goldenPath)) return false; + } else if (arg == "--alternate-golden") { + std::string alternateGolden; + if (!ReadValue(argc, argv, i, alternateGolden)) return false; + if (!alternateGolden.empty()) { + request.alternateGoldenPaths.push_back(alternateGolden); + } } else if (arg == "--diff") { if (!ReadValue(argc, argv, i, request.diffPath)) return false; } else if (arg == "--output") {