[Fix] (trace-replay): support alternate golden images

- compare actual output against primary and alternate golden images

- record the matched golden path in trace replay results

- allow APK and Linux retrace fixtures to pass alternate golden paths

- keep nostalgia validation accepting both Mesa and PC goldens

.github/workflows/apk.yml

@@ -287,6 +287,7 @@ jobs:
             trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-nostalgia-in-world.tgz
             trace_file: trace.trace
             golden: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-nostalgia-in-world.0000153808-linux-mesa.png
+            alternate_golden: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-nostalgia-in-world.0000153808.png
             target_call: 153808
             width: 854
             height: 480
@@ -429,6 +430,7 @@ jobs:
             --trace-archive "${{ matrix.case.trace_archive }}" \
             --trace-file "${{ matrix.case.trace_file }}" \
             --golden "${{ matrix.case.golden }}" \
+            --alternate-golden "${{ matrix.case.alternate_golden || '' }}" \
             --target-call "${{ matrix.case.target_call }}" \
             --width "${{ matrix.case.width }}" \
             --height "${{ matrix.case.height }}" \

android-plugin/app/src/trace/cpp/trace_replay_core.cpp

@@ -18,6 +18,7 @@
 #include <memory>
 #include <sstream>
 #include <string>
+#include <utility>
 #include <vector>
 #include <fcntl.h>
 #include <sys/stat.h>
@@ -545,27 +546,29 @@ bool WriteDifferenceImage(const Result& result,
     return WritePngRgba(result.diffPath, diff, error);
 }
 
-bool CompareWithGolden(const Request& request, Result& result) {
+struct GoldenComparison {
-    if (request.goldenPath.empty()) {
+    std::string path;
-        result.passed = true;
+    RgbaImage image;
-        result.statusCode = STATUS_OK;
+    long long mismatchPixels = std::numeric_limits<long long>::max();
-        result.message = "retrace completed; golden_path was not provided";
+    int x0 = 0;
-        result.mismatchPixels = 0;
+    int y0 = 0;
-        return true;
+    int compareWidth = 0;
-    }
+    int compareHeight = 0;
-    if (!Exists(request.goldenPath)) {
+};
-        result.statusCode = STATUS_INVALID_ARGUMENT;
+
-        result.message = "golden_path does not exist or is not a regular file";
+bool CompareAgainstOneGolden(const Request& request,
+                             const RgbaImage& actual,
+                             const std::string& goldenPath,
+                             int fuzz,
+                             GoldenComparison& comparison,
+                             std::string& error) {
+    if (!Exists(goldenPath)) {
+        error = "golden_path does not exist or is not a regular file: " + goldenPath;
         return false;
     }
 
-    RgbaImage actual;
     RgbaImage golden;
-    std::string pngError;
+    if (!ReadPngRgba(goldenPath, golden, error)) {
-    if (!ReadPngRgba(result.actualPath, actual, pngError) ||
-        !ReadPngRgba(request.goldenPath, golden, pngError)) {
-        result.statusCode = STATUS_COMPARE_FAILED;
-        result.message = pngError.empty() ? "failed to decode actual or golden PNG" : pngError;
         return false;
     }
 
@@ -573,11 +576,11 @@ bool CompareWithGolden(const Request& request, Result& result) {
     int y0 = request.cropY;
     if (request.cropWidth <= 0 && request.cropHeight <= 0 &&
         (actual.width != golden.width || actual.height != golden.height)) {
-        result.statusCode = STATUS_COMPARE_FAILED;
         std::ostringstream message;
         message << "actual image size " << actual.width << "x" << actual.height
-                << " does not match golden image size " << golden.width << "x" << golden.height;
+                << " does not match golden image size " << golden.width << "x" << golden.height
-        result.message = message.str();
+                << ": " << goldenPath;
+        error = message.str();
         return false;
     }
 
@@ -589,12 +592,10 @@ bool CompareWithGolden(const Request& request, Result& result) {
         y0 + compareHeight > actual.height ||
         x0 + compareWidth > golden.width ||
         y0 + compareHeight > golden.height) {
-        result.statusCode = STATUS_INVALID_ARGUMENT;
+        error = "compare crop is outside actual or golden image bounds: " + goldenPath;
-        result.message = "compare crop is outside actual or golden image bounds";
         return false;
     }
 
-    const int fuzz = std::max(0, std::min(100, request.fuzzPercent)) * 255 / 100;
     long long mismatch = 0;
     for (int y = 0; y < compareHeight; ++y) {
         for (int x = 0; x < compareWidth; ++x) {
@@ -613,20 +614,83 @@ bool CompareWithGolden(const Request& request, Result& result) {
         }
     }
 
+    comparison.path = goldenPath;
+    comparison.image = std::move(golden);
+    comparison.mismatchPixels = mismatch;
+    comparison.x0 = x0;
+    comparison.y0 = y0;
+    comparison.compareWidth = compareWidth;
+    comparison.compareHeight = compareHeight;
+    return true;
+}
+
+bool CompareWithGolden(const Request& request, Result& result) {
+    std::vector<std::string> goldenPaths;
+    if (!request.goldenPath.empty()) {
+        goldenPaths.push_back(request.goldenPath);
+    }
+    for (const auto& alternateGoldenPath : request.alternateGoldenPaths) {
+        if (!alternateGoldenPath.empty()) {
+            goldenPaths.push_back(alternateGoldenPath);
+        }
+    }
+
+    if (goldenPaths.empty()) {
+        result.passed = true;
+        result.statusCode = STATUS_OK;
+        result.message = "retrace completed; golden_path was not provided";
+        result.mismatchPixels = 0;
+        return true;
+    }
+
+    RgbaImage actual;
+    std::string pngError;
+    if (!ReadPngRgba(result.actualPath, actual, pngError)) {
+        result.statusCode = STATUS_COMPARE_FAILED;
+        result.message = pngError.empty() ? "failed to decode actual PNG" : pngError;
+        return false;
+    }
+
+    const int fuzz = std::max(0, std::min(100, request.fuzzPercent)) * 255 / 100;
+    GoldenComparison bestComparison;
+    std::string comparisonError;
+    bool hasComparison = false;
+    for (const auto& goldenPath : goldenPaths) {
+        GoldenComparison comparison;
+        std::string error;
+        if (!CompareAgainstOneGolden(request, actual, goldenPath, fuzz, comparison, error)) {
+            comparisonError = error;
+            continue;
+        }
+        if (!hasComparison || comparison.mismatchPixels < bestComparison.mismatchPixels) {
+            bestComparison = std::move(comparison);
+            hasComparison = true;
+        }
+    }
+
+    if (!hasComparison) {
+        result.statusCode = STATUS_COMPARE_FAILED;
+        result.message = comparisonError.empty() ? "failed to compare against any golden PNG" : comparisonError;
+        return false;
+    }
+
     std::string diffError;
-    if (!WriteDifferenceImage(result, actual, golden, x0, y0, compareWidth, compareHeight, fuzz, diffError)) {
+    if (!WriteDifferenceImage(result, actual, bestComparison.image, bestComparison.x0, bestComparison.y0,
+                              bestComparison.compareWidth, bestComparison.compareHeight, fuzz, diffError)) {
         result.statusCode = STATUS_IO_ERROR;
         result.message = diffError.empty() ? "failed to write diff PNG" : diffError;
         return false;
     }
 
-    result.mismatchPixels = mismatch;
+    result.mismatchPixels = bestComparison.mismatchPixels;
-    result.passed = mismatch <= request.tolerance;
+    result.matchedGoldenPath = bestComparison.path;
+    result.passed = bestComparison.mismatchPixels <= request.tolerance;
     result.statusCode = result.passed ? STATUS_OK : STATUS_COMPARE_FAILED;
     std::ostringstream message;
-    message << "retrace completed; mismatchPixels=" << mismatch
+    message << "retrace completed; mismatchPixels=" << bestComparison.mismatchPixels
             << ", tolerance=" << request.tolerance
-            << ", fuzzPercent=" << request.fuzzPercent;
+            << ", fuzzPercent=" << request.fuzzPercent
+            << ", matchedGoldenPath=" << bestComparison.path;
     result.message = message.str();
     return result.passed;
 }
@@ -648,6 +712,15 @@ bool WriteResultJson(const Request& request, const Result& result) {
     file << "  \"message\": \"" << JsonEscape(result.message) << "\",\n";
     file << "  \"tracePath\": \"" << JsonEscape(request.tracePath) << "\",\n";
     file << "  \"goldenPath\": \"" << JsonEscape(request.goldenPath) << "\",\n";
+    file << "  \"alternateGoldenPaths\": [";
+    for (std::size_t i = 0; i < request.alternateGoldenPaths.size(); ++i) {
+        if (i > 0) {
+            file << ", ";
+        }
+        file << "\"" << JsonEscape(request.alternateGoldenPaths[i]) << "\"";
+    }
+    file << "],\n";
+    file << "  \"matchedGoldenPath\": \"" << JsonEscape(result.matchedGoldenPath) << "\",\n";
     file << "  \"actualPath\": \"" << JsonEscape(result.actualPath) << "\",\n";
     file << "  \"diffPath\": \"" << JsonEscape(result.diffPath) << "\",\n";
     file << "  \"backend\": \"" << JsonEscape(request.backend) << "\",\n";

android-plugin/app/src/trace/cpp/trace_replay_core.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <string>
+#include <vector>
 
 namespace mobilegl_trace {
 
@@ -17,6 +18,7 @@ enum StatusCode {
 struct Request {
     std::string tracePath;
     std::string goldenPath;
+    std::vector<std::string> alternateGoldenPaths;
     std::string outputDir;
     std::string diffPath;
     std::string backend;
@@ -40,6 +42,7 @@ struct Result {
     std::string resultPath;
     std::string actualPath;
     std::string diffPath;
+    std::string matchedGoldenPath;
     long long mismatchPixels = -1;
 };
 

android-plugin/app/src/trace/cpp/trace_replay_jni.cpp

@@ -54,6 +54,7 @@ Java_top_mobilegl_plugin_trace_TraceReplayActivity_nativeRunTraceReplay(JNIEnv*
                                                                         jobject surface,
                                                                         jstring tracePath,
                                                                         jstring goldenPath,
+                                                                        jstring alternateGoldenPath,
                                                                         jstring outputDir,
                                                                         jstring diffPath,
                                                                         jstring backend,
@@ -70,6 +71,10 @@ Java_top_mobilegl_plugin_trace_TraceReplayActivity_nativeRunTraceReplay(JNIEnv*
     mobilegl_trace::Request request;
     request.tracePath = ToString(env, tracePath);
     request.goldenPath = ToString(env, goldenPath);
+    std::string alternateGolden = ToString(env, alternateGoldenPath);
+    if (!alternateGolden.empty()) {
+        request.alternateGoldenPaths.push_back(alternateGolden);
+    }
     request.outputDir = ToString(env, outputDir);
     request.diffPath = ToString(env, diffPath);
     request.backend = ToString(env, backend);

android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java

@@ -97,6 +97,7 @@ public final class TraceReplayActivity extends Activity {
                 surface,
                 request.tracePath,
                 request.goldenPath,
+                request.alternateGoldenPath,
                 request.outputDir,
                 request.diffPath,
                 request.backend,
@@ -123,6 +124,7 @@ public final class TraceReplayActivity extends Activity {
             Surface surface,
             String tracePath,
             String goldenPath,
+            String alternateGoldenPath,
             String outputDir,
             String diffPath,
             String backend,
@@ -141,6 +143,7 @@ public final class TraceReplayActivity extends Activity {
     private static final class TraceReplayRequest {
         final String tracePath;
         final String goldenPath;
+        final String alternateGoldenPath;
         final String outputDir;
         final String diffPath;
         final String backend;
@@ -158,6 +161,7 @@ public final class TraceReplayActivity extends Activity {
         private TraceReplayRequest(
                 String tracePath,
                 String goldenPath,
+                String alternateGoldenPath,
                 String outputDir,
                 String diffPath,
                 String backend,
@@ -174,6 +178,7 @@ public final class TraceReplayActivity extends Activity {
         ) {
             this.tracePath = tracePath;
             this.goldenPath = goldenPath;
+            this.alternateGoldenPath = alternateGoldenPath;
             this.outputDir = outputDir;
             this.diffPath = diffPath;
             this.backend = backend;
@@ -195,6 +200,7 @@ public final class TraceReplayActivity extends Activity {
             return new TraceReplayRequest(
                     readString(intent, "trace_path", ""),
                     readString(intent, "golden_path", ""),
+                    readString(intent, "alternate_golden_path", ""),
                     outputDir,
                     diffPath,
                     readString(intent, "backend", defaultBackend),

android-plugin/trace-replay-ci.sh

@@ -17,6 +17,7 @@ Usage:
     --trace-archive FILE \
     --trace-file FILE_IN_ARCHIVE \
     --golden FILE \
+    [--alternate-golden FILE] \
     --target-call N \
     --width N \
     --height N \
@@ -60,6 +61,7 @@ case_name=""
 trace_archive=""
 trace_file=""
 golden_path=""
+alternate_golden_path=""
 target_call=""
 width=""
 height=""
@@ -82,6 +84,15 @@ while [ "$#" -gt 0 ]; do
     --trace-archive) trace_archive="$(next_arg "$@")"; shift 2 ;;
     --trace-file) trace_file="$(next_arg "$@")"; shift 2 ;;
     --golden) golden_path="$(next_arg "$@")"; shift 2 ;;
+    --alternate-golden)
+      if [ "$#" -lt 2 ] || [ "${2#--}" != "$2" ]; then
+        alternate_golden_path=""
+        shift 1
+      else
+        alternate_golden_path="$2"
+        shift 2
+      fi
+      ;;
     --target-call) target_call="$(next_arg "$@")"; shift 2 ;;
     --width) width="$(next_arg "$@")"; shift 2 ;;
     --height) height="$(next_arg "$@")"; shift 2 ;;
@@ -120,6 +131,9 @@ require_value "${timeout_seconds}" "--timeout-seconds"
 test -f "${apk_file}" || die "APK does not exist: ${apk_file}"
 test -f "${trace_archive}" || die "trace archive does not exist: ${trace_archive}"
 test -f "${golden_path}" || die "golden image does not exist: ${golden_path}"
+if [ -n "${alternate_golden_path}" ]; then
+  test -f "${alternate_golden_path}" || die "alternate golden image does not exist: ${alternate_golden_path}"
+fi
 
 safe_case="$(printf '%s' "${case_name}" | sed 's/[^A-Za-z0-9._-]/_/g')"
 
@@ -134,32 +148,50 @@ prepare_fixture() {
 
   "${ADB}" push "${extracted_trace}" "/data/local/tmp/mobilegl-${safe_case}.trace"
   "${ADB}" push "${golden_path}" "/data/local/tmp/mobilegl-${safe_case}.golden.png"
+  if [ -n "${alternate_golden_path}" ]; then
+    "${ADB}" push "${alternate_golden_path}" "/data/local/tmp/mobilegl-${safe_case}.alternate-golden.png"
+  fi
   "${ADB}" shell chmod 0644 "/data/local/tmp/mobilegl-${safe_case}.trace" "/data/local/tmp/mobilegl-${safe_case}.golden.png"
+  if [ -n "${alternate_golden_path}" ]; then
+    "${ADB}" shell chmod 0644 "/data/local/tmp/mobilegl-${safe_case}.alternate-golden.png"
+  fi
 }
 
 copy_fixture_to_app() {
   trace_tmp="/data/local/tmp/mobilegl-${safe_case}.trace"
   golden_tmp="/data/local/tmp/mobilegl-${safe_case}.golden.png"
+  alternate_golden_tmp="/data/local/tmp/mobilegl-${safe_case}.alternate-golden.png"
 
   "${ADB}" shell run-as "${package_name}" rm -rf files/trace-replay
   "${ADB}" shell run-as "${package_name}" mkdir -p files/trace-replay/input files/trace-replay/output
   "${ADB}" shell run-as "${package_name}" cp "${trace_tmp}" files/trace-replay/input/trace.trace
   "${ADB}" shell run-as "${package_name}" cp "${golden_tmp}" files/trace-replay/input/golden.png
+  if [ -n "${alternate_golden_path}" ]; then
+    "${ADB}" shell run-as "${package_name}" cp "${alternate_golden_tmp}" files/trace-replay/input/alternate-golden.png
+  fi
 }
 
 run_retrace() {
   app_dir="/data/user/0/${package_name}/files/trace-replay"
   result_dir="${result_root}/${safe_case}-${backend}"
+  alternate_golden_app_path=""
+  if [ -n "${alternate_golden_path}" ]; then
+    alternate_golden_app_path="${app_dir}/input/alternate-golden.png"
+  fi
 
   mkdir -p "${result_dir}"
   "${ADB}" install -r "${apk_file}"
   copy_fixture_to_app
   "${ADB}" shell am force-stop "${package_name}"
   "${ADB}" logcat -c
-  "${ADB}" shell am start -W -a top.mobilegl.plugin.TRACE_REPLAY \
+  set -- am start -W -a top.mobilegl.plugin.TRACE_REPLAY \
     -n "${package_name}/top.mobilegl.plugin.trace.TraceReplayActivity" \
     --es trace_path "${app_dir}/input/trace.trace" \
-    --es golden_path "${app_dir}/input/golden.png" \
+    --es golden_path "${app_dir}/input/golden.png"
+  if [ -n "${alternate_golden_app_path}" ]; then
+    set -- "$@" --es alternate_golden_path "${alternate_golden_app_path}"
+  fi
+  set -- "$@" \
     --es output_dir "${app_dir}/output" \
     --es diff_path "${app_dir}/output/${safe_case}-diff.png" \
     --es backend "${backend}" \
@@ -172,6 +204,7 @@ run_retrace() {
     --ei crop_width "${crop_width}" \
     --ei crop_height "${crop_height}" \
     --ei fuzz_percent "${fuzz_percent}"
+  "${ADB}" shell "$@"
 
   for _ in $(seq 1 "${timeout_seconds}"); do
     if "${ADB}" shell run-as "${package_name}" ls files/trace-replay/output/result.json >/dev/null 2>&1; then

tools/trace_replay/CMakeLists.txt

@@ -269,6 +269,7 @@ function(add_trace_replay_test CASE_NAME BACKEND)
             TRACE_ARCHIVE
             TRACE_FILE
             GOLDEN
+            ALTERNATE_GOLDEN
             TARGET_CALL
             WIDTH
             HEIGHT
@@ -314,6 +315,7 @@ function(add_trace_replay_test CASE_NAME BACKEND)
             -DTRACE_ARCHIVE=${TRACE_CASE_TRACE_ARCHIVE}
             -DTRACE_FILE=${TRACE_CASE_TRACE_FILE}
             -DTRACE_GOLDEN=${TRACE_CASE_GOLDEN}
+            -DTRACE_ALTERNATE_GOLDEN=${TRACE_CASE_ALTERNATE_GOLDEN}
             -DTRACE_BACKEND=${BACKEND}
             -DTRACE_TARGET_CALL=${TRACE_CASE_TARGET_CALL}
             -DTRACE_WIDTH=${TRACE_CASE_WIDTH}
@@ -469,6 +471,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-nostalgia-in-wor
         TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-nostalgia-in-world.tgz
         TRACE_FILE trace.trace
         GOLDEN ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-nostalgia-in-world.0000153808-linux-mesa.png
+        ALTERNATE_GOLDEN ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-nostalgia-in-world.0000153808.png
         TARGET_CALL 153808
         WIDTH 854
         HEIGHT 480

tools/trace_replay/run_trace_case.cmake

@@ -22,6 +22,10 @@ endif()
 if(NOT DEFINED TRACE_FUZZ_PERCENT OR "${TRACE_FUZZ_PERCENT}" STREQUAL "")
     set(TRACE_FUZZ_PERCENT 20)
 endif()
+set(alternate_golden_args)
+if(DEFINED TRACE_ALTERNATE_GOLDEN AND NOT "${TRACE_ALTERNATE_GOLDEN}" STREQUAL "")
+    list(APPEND alternate_golden_args --alternate-golden "${TRACE_ALTERNATE_GOLDEN}")
+endif()
 
 if(EXISTS "${TRACE_OUTPUT_DIR}")
     file(REMOVE_RECURSE "${TRACE_OUTPUT_DIR}")
@@ -50,6 +54,7 @@ execute_process(
         COMMAND "${TRACE_REPLAY_EXE}"
         --trace "${trace_path}"
         --golden "${TRACE_GOLDEN}"
+        ${alternate_golden_args}
         --diff "${TRACE_OUTPUT_DIR}/output/${TRACE_CASE_NAME}-diff.png"
         --output "${TRACE_OUTPUT_DIR}/output"
         --backend "${TRACE_BACKEND}"

tools/trace_replay/trace_replay_cli.cpp

@@ -14,6 +14,7 @@ void PrintUsage(const char *argv0) {
             << "\n"
             << "Options:\n"
             << "  --golden PATH             Golden PNG to compare against\n"
+            << "  --alternate-golden PATH   Additional acceptable golden PNG\n"
             << "  --diff PATH               Difference PNG output path\n"
             << "  --backend NAME            DirectGLES or DirectVulkan (default: DirectGLES)\n"
             << "  --mobilegl-library PATH   libMobileGL.so path (default: libMobileGL.so)\n"
@@ -64,6 +65,12 @@ bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) {
             if (!ReadValue(argc, argv, i, request.tracePath)) return false;
         } else if (arg == "--golden") {
             if (!ReadValue(argc, argv, i, request.goldenPath)) return false;
+        } else if (arg == "--alternate-golden") {
+            std::string alternateGolden;
+            if (!ReadValue(argc, argv, i, alternateGolden)) return false;
+            if (!alternateGolden.empty()) {
+                request.alternateGoldenPaths.push_back(alternateGolden);
+            }
         } else if (arg == "--diff") {
             if (!ReadValue(argc, argv, i, request.diffPath)) return false;
         } else if (arg == "--output") {