diff --git a/.github/workflows/apk.yml b/.github/workflows/apk.yml index c2f47cb5..c259dbfc 100644 --- a/.github/workflows/apk.yml +++ b/.github/workflows/apk.yml @@ -158,12 +158,10 @@ jobs: target_call: 31249 width: 640 height: 480 - tolerance: 3072 crop_x: 1 crop_y: 1 crop_width: 638 crop_height: 478 - fuzz_percent: 20 timeout_seconds: 180 - name: minecraft-1.21.4-startup trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-startup.tgz @@ -172,12 +170,10 @@ jobs: target_call: 92195 width: 854 height: 480 - tolerance: 4100 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 180 - name: minecraft-1.21.4-main-menu trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-main-menu.tgz @@ -186,12 +182,10 @@ jobs: target_call: 481787 width: 854 height: 480 - tolerance: 4100 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 180 - name: minecraft-1.21.4-in-world trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-in-world.tgz @@ -200,12 +194,10 @@ jobs: target_call: 280000 width: 854 height: 480 - tolerance: 4100 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 900 - name: minecraft-1.21.4-fabric-sodium-in-world trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-sodium-in-world.tgz @@ -214,12 +206,10 @@ jobs: target_call: 923340 width: 854 height: 480 - tolerance: 4100 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 1800 - name: minecraft-1.21.4-fabric-iris-bsl-in-world trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-bsl-in-world.tgz @@ -228,12 +218,10 @@ jobs: target_call: 110725 width: 854 height: 480 - tolerance: 4100 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 900 - name: minecraft-1.21.4-fabric-iris-makeup-ultrafast-in-world trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-makeup-ultrafast-in-world.tgz @@ -242,12 +230,10 @@ jobs: target_call: 95322 width: 854 height: 480 - tolerance: 4100 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 900 - name: minecraft-1.21.4-fabric-iris-super-duper-vanilla-in-world trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-super-duper-vanilla-in-world.tgz @@ -256,12 +242,10 @@ jobs: target_call: 141559 width: 854 height: 480 - tolerance: 4100 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 900 - name: minecraft-1.21.4-fabric-iris-sundial-lite-in-world trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-sundial-lite-in-world.tgz @@ -270,12 +254,10 @@ jobs: target_call: 150023 width: 854 height: 480 - tolerance: 8000 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 900 - name: minecraft-1.21.4-fabric-iris-complementary-reimagined-in-world trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-complementary-reimagined-in-world.tgz @@ -284,12 +266,10 @@ jobs: target_call: 151297 width: 854 height: 480 - tolerance: 4100 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 900 - name: minecraft-1.21.4-fabric-iris-complementary-unbound-in-world trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-complementary-unbound-in-world.tgz @@ -298,12 +278,10 @@ jobs: target_call: 146559 width: 854 height: 480 - tolerance: 4100 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 900 - name: minecraft-1.21.4-fabric-iris-mellow-in-world trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-mellow-in-world.tgz @@ -312,12 +290,10 @@ jobs: target_call: 96143 width: 854 height: 480 - tolerance: 4100 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 900 - name: minecraft-1.21.4-fabric-iris-nostalgia-in-world trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-nostalgia-in-world.tgz @@ -327,12 +303,10 @@ jobs: target_call: 153808 width: 854 height: 480 - tolerance: 4100 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 900 - name: minecraft-1.21.4-fabric-iris-bliss-in-world trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-bliss-in-world.tgz @@ -341,12 +315,10 @@ jobs: target_call: 113511 width: 854 height: 480 - tolerance: 4100 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 900 - name: minecraft-1.21.4-fabric-iris-chocapic-v6-lite-in-world trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-chocapic-v6-lite-in-world.tgz @@ -355,12 +327,10 @@ jobs: target_call: 125124 width: 854 height: 480 - tolerance: 4100 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 900 - name: minecraft-1.21.4-fabric-iris-iterationt-in-world trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-iterationt-in-world.tgz @@ -369,12 +339,10 @@ jobs: target_call: 1173280 width: 854 height: 480 - tolerance: 4100 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 2400 - name: minecraft-1.21.4-fabric-iris-iterationt-nodsa-in-world trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-iterationt-nodsa-in-world.tgz @@ -383,12 +351,10 @@ jobs: target_call: 1606997 width: 854 height: 480 - tolerance: 4100 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 2400 - name: minecraft-1.21.4-fabric-iris-photon-v1.1-in-world trace_archive: tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-photon-v1.1-in-world.tgz @@ -397,12 +363,10 @@ jobs: target_call: 159866 width: 854 height: 480 - tolerance: 4100 crop_x: 0 crop_y: 0 crop_width: 0 crop_height: 0 - fuzz_percent: 20 timeout_seconds: 900 steps: @@ -472,12 +436,11 @@ jobs: --target-call "${{ matrix.case.target_call }}" \ --width "${{ matrix.case.width }}" \ --height "${{ matrix.case.height }}" \ - --tolerance "${{ matrix.case.tolerance }}" \ + --ssim-threshold "${{ matrix.case.ssim_threshold || '0.99' }}" \ --crop-x "${{ matrix.case.crop_x }}" \ --crop-y "${{ matrix.case.crop_y }}" \ --crop-width "${{ matrix.case.crop_width }}" \ --crop-height "${{ matrix.case.crop_height }}" \ - --fuzz-percent "${{ matrix.case.fuzz_percent }}" \ --timeout-seconds "${{ matrix.case.timeout_seconds }}" - name: Collect emulator diagnostics diff --git a/android-plugin/TRACE_REPLAY.md b/android-plugin/TRACE_REPLAY.md index 3f0f16f2..9aca0a21 100644 --- a/android-plugin/TRACE_REPLAY.md +++ b/android-plugin/TRACE_REPLAY.md @@ -32,8 +32,7 @@ target_frame target frame index, or -1 target_call target call number, or -1 width optional replay surface width override height optional replay surface height override -tolerance allowed mismatching pixel count after fuzz is applied -fuzz_percent per-channel fuzz percentage; default is 20 +ssim_threshold minimum SSIM required to pass; default is 0.99 crop_x optional compare crop x crop_y optional compare crop y crop_width optional compare crop width @@ -48,7 +47,7 @@ Implementation notes: - The native runner validates inputs, sets `MOBILEGL_BACKEND_TYPE`, loads `libMobileGL.so`, runs apitrace GL retrace, writes `actual.png`, and writes `result.json`. - The runner uses a MobileGL-backed EGL window-system shim. GLX calls in PC traces are consumed by apitrace's GLX retrace frontend and mapped onto this EGL shim; the Android runner does not require or call a MobileGL GLX implementation. - `DirectGLES` replays on an EGL pbuffer by default, avoiding Android `SurfaceView` lifetime coupling. `DirectVulkan` still uses the Activity surface because it needs a native window-backed Vulkan swapchain. -- Golden comparison is implemented in native C++ with libpng RGBA decode. The Java Activity only passes arguments and displays the native result, so the replay/compare core is not tied to Android UI or Bitmap APIs and can be ported to Linux. +- Golden comparison is implemented in native C++ with libpng RGBA decode and SSIM validation. The Java Activity only passes arguments and displays the native result, so the replay/compare core is not tied to Android UI or Bitmap APIs and can be ported to Linux. - The plugin profile still excludes `libtrace_replay_runner.so`; normal plugin APK behavior is preserved. - Set `MOBILEGL_RETRACE_USE_ANGLE=1` when running `trace-replay-ci.sh` to pass `use_angle=true` for DirectGLES. The APK must include `libEGL_angle.so` and `libGLESv2_angle.so` under its x86_64 native libraries. The Activity passes Android's `nativeLibraryDir` to native code as `MOBILEGL_RETRACE_ANGLE_DIR`. @@ -68,7 +67,6 @@ adb shell am start -a top.mobilegl.plugin.TRACE_REPLAY \ --es diff_path /data/user/0/top.mobilegl.plugin.espryt.trace/files/trace-replay/output/app-diff.png \ --es backend DirectGLES \ --el target_call 31249 \ - --ei tolerance 0 \ - --ei fuzz_percent 20 + --es ssim_threshold 0.99 adb shell run-as top.mobilegl.plugin.espryt.trace cat files/trace-replay/output/result.json ``` diff --git a/android-plugin/app/src/trace/cpp/trace_replay_core.cpp b/android-plugin/app/src/trace/cpp/trace_replay_core.cpp index ee880586..7d25bf81 100644 --- a/android-plugin/app/src/trace/cpp/trace_replay_core.cpp +++ b/android-plugin/app/src/trace/cpp/trace_replay_core.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -520,7 +521,6 @@ bool WriteDifferenceImage(const Result& result, int y0, int compareWidth, int compareHeight, - int fuzz, std::string& error) { if (result.diffPath.empty()) { return true; @@ -548,7 +548,7 @@ bool WriteDifferenceImage(const Result& result, ChannelValue(golden, imageX, imageY, 1)); int db = std::abs(ChannelValue(actual, imageX, imageY, 2) - ChannelValue(golden, imageX, imageY, 2)); - bool different = dr > fuzz || dg > fuzz || db > fuzz; + bool different = dr != 0 || dg != 0 || db != 0; std::uint8_t* dst = diff.pixels.data() + (static_cast(imageY) * diff.width + imageX) * 4; if (different) { @@ -569,17 +569,71 @@ bool WriteDifferenceImage(const Result& result, struct GoldenComparison { std::string path; RgbaImage image; - long long mismatchPixels = std::numeric_limits::max(); + double ssim = -1.0; + long long mismatchPixels = 0; int x0 = 0; int y0 = 0; int compareWidth = 0; int compareHeight = 0; }; +double ComputeChannelSsim(const RgbaImage& actual, + const RgbaImage& golden, + int x0, + int y0, + int compareWidth, + int compareHeight, + unsigned channel) { + const double count = static_cast(compareWidth) * static_cast(compareHeight); + double sumA = 0.0; + double sumG = 0.0; + double sumAA = 0.0; + double sumGG = 0.0; + double sumAG = 0.0; + + for (int y = 0; y < compareHeight; ++y) { + for (int x = 0; x < compareWidth; ++x) { + const double a = ChannelValue(actual, x0 + x, y0 + y, channel); + const double g = ChannelValue(golden, x0 + x, y0 + y, channel); + sumA += a; + sumG += g; + sumAA += a * a; + sumGG += g * g; + sumAG += a * g; + } + } + + const double meanA = sumA / count; + const double meanG = sumG / count; + const double varianceA = std::max(0.0, sumAA / count - meanA * meanA); + const double varianceG = std::max(0.0, sumGG / count - meanG * meanG); + const double covariance = sumAG / count - meanA * meanG; + + constexpr double kC1 = 6.5025; // (0.01 * 255)^2 + constexpr double kC2 = 58.5225; // (0.03 * 255)^2 + const double luminance = (2.0 * meanA * meanG + kC1) / + (meanA * meanA + meanG * meanG + kC1); + const double contrastStructure = (2.0 * covariance + kC2) / + (varianceA + varianceG + kC2); + return luminance * contrastStructure; +} + +double ComputeRgbSsim(const RgbaImage& actual, + const RgbaImage& golden, + int x0, + int y0, + int compareWidth, + int compareHeight) { + double sum = 0.0; + for (unsigned channel = 0; channel < 3; ++channel) { + sum += ComputeChannelSsim(actual, golden, x0, y0, compareWidth, compareHeight, channel); + } + return sum / 3.0; +} + bool CompareAgainstOneGolden(const Request& request, const RgbaImage& actual, const std::string& goldenPath, - int fuzz, GoldenComparison& comparison, std::string& error) { if (!Exists(goldenPath)) { @@ -616,27 +670,28 @@ bool CompareAgainstOneGolden(const Request& request, return false; } - long long mismatch = 0; + long long exactMismatch = 0; for (int y = 0; y < compareHeight; ++y) { for (int x = 0; x < compareWidth; ++x) { bool different = false; for (unsigned c = 0; c < 3; ++c) { int a = ChannelValue(actual, x0 + x, y0 + y, c); int g = ChannelValue(golden, x0 + x, y0 + y, c); - if (std::abs(a - g) > fuzz) { + if (a != g) { different = true; break; } } if (different) { - ++mismatch; + ++exactMismatch; } } } comparison.path = goldenPath; comparison.image = std::move(golden); - comparison.mismatchPixels = mismatch; + comparison.ssim = ComputeRgbSsim(actual, comparison.image, x0, y0, compareWidth, compareHeight); + comparison.mismatchPixels = exactMismatch; comparison.x0 = x0; comparison.y0 = y0; comparison.compareWidth = compareWidth; @@ -659,6 +714,7 @@ bool CompareWithGolden(const Request& request, Result& result) { result.passed = true; result.statusCode = STATUS_OK; result.message = "retrace completed; golden_path was not provided"; + result.ssim = 1.0; result.mismatchPixels = 0; return true; } @@ -671,18 +727,17 @@ bool CompareWithGolden(const Request& request, Result& result) { return false; } - const int fuzz = std::max(0, std::min(100, request.fuzzPercent)) * 255 / 100; GoldenComparison bestComparison; std::string comparisonError; bool hasComparison = false; for (const auto& goldenPath : goldenPaths) { GoldenComparison comparison; std::string error; - if (!CompareAgainstOneGolden(request, actual, goldenPath, fuzz, comparison, error)) { + if (!CompareAgainstOneGolden(request, actual, goldenPath, comparison, error)) { comparisonError = error; continue; } - if (!hasComparison || comparison.mismatchPixels < bestComparison.mismatchPixels) { + if (!hasComparison || comparison.ssim > bestComparison.ssim) { bestComparison = std::move(comparison); hasComparison = true; } @@ -696,20 +751,22 @@ bool CompareWithGolden(const Request& request, Result& result) { std::string diffError; if (!WriteDifferenceImage(result, actual, bestComparison.image, bestComparison.x0, bestComparison.y0, - bestComparison.compareWidth, bestComparison.compareHeight, fuzz, diffError)) { + bestComparison.compareWidth, bestComparison.compareHeight, diffError)) { result.statusCode = STATUS_IO_ERROR; result.message = diffError.empty() ? "failed to write diff PNG" : diffError; return false; } + result.ssim = bestComparison.ssim; result.mismatchPixels = bestComparison.mismatchPixels; result.matchedGoldenPath = bestComparison.path; - result.passed = bestComparison.mismatchPixels <= request.tolerance; + result.passed = bestComparison.ssim >= request.ssimThreshold; result.statusCode = result.passed ? STATUS_OK : STATUS_COMPARE_FAILED; std::ostringstream message; - message << "retrace completed; mismatchPixels=" << bestComparison.mismatchPixels - << ", tolerance=" << request.tolerance - << ", fuzzPercent=" << request.fuzzPercent + message << std::fixed << std::setprecision(6) + << "retrace completed; ssim=" << bestComparison.ssim + << ", ssimThreshold=" << request.ssimThreshold + << ", mismatchPixels=" << bestComparison.mismatchPixels << ", matchedGoldenPath=" << bestComparison.path; result.message = message.str(); return result.passed; @@ -753,8 +810,9 @@ bool WriteResultJson(const Request& request, const Result& result) { file << " \"cropY\": " << request.cropY << ",\n"; file << " \"cropWidth\": " << request.cropWidth << ",\n"; file << " \"cropHeight\": " << request.cropHeight << ",\n"; - file << " \"tolerance\": " << request.tolerance << ",\n"; - file << " \"fuzzPercent\": " << request.fuzzPercent << ",\n"; + file << std::fixed << std::setprecision(9); + file << " \"ssim\": " << result.ssim << ",\n"; + file << " \"ssimThreshold\": " << request.ssimThreshold << ",\n"; file << " \"useAngle\": " << (UseAngleForRequest(request) ? "true" : "false") << ",\n"; file << " \"mismatchPixels\": " << result.mismatchPixels << "\n"; file << "}\n"; diff --git a/android-plugin/app/src/trace/cpp/trace_replay_core.hpp b/android-plugin/app/src/trace/cpp/trace_replay_core.hpp index 39f1db67..ac1aade7 100644 --- a/android-plugin/app/src/trace/cpp/trace_replay_core.hpp +++ b/android-plugin/app/src/trace/cpp/trace_replay_core.hpp @@ -32,8 +32,7 @@ struct Request { int cropY = 0; int cropWidth = 0; int cropHeight = 0; - int tolerance = 0; - int fuzzPercent = 20; + double ssimThreshold = 0.99; bool useAngle = false; }; @@ -45,6 +44,7 @@ struct Result { std::string actualPath; std::string diffPath; std::string matchedGoldenPath; + double ssim = -1.0; long long mismatchPixels = -1; }; diff --git a/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp b/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp index cd73d964..4b759df2 100644 --- a/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp +++ b/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp @@ -93,12 +93,11 @@ Java_top_mobilegl_plugin_trace_TraceReplayActivity_nativeRunTraceReplay(JNIEnv* jlong targetCall, jint width, jint height, - jint tolerance, + jdouble ssimThreshold, jint cropX, jint cropY, jint cropWidth, jint cropHeight, - jint fuzzPercent, jstring angleLibraryDir, jboolean useAngle) { mobilegl_trace::Request request; @@ -116,12 +115,11 @@ Java_top_mobilegl_plugin_trace_TraceReplayActivity_nativeRunTraceReplay(JNIEnv* request.targetCall = targetCall; request.width = width; request.height = height; - request.tolerance = tolerance; + request.ssimThreshold = ssimThreshold; request.cropX = cropX; request.cropY = cropY; request.cropWidth = cropWidth; request.cropHeight = cropHeight; - request.fuzzPercent = fuzzPercent; request.useAngle = useAngle == JNI_TRUE; ScopedTraceReplayState replayState; diff --git a/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java b/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java index 54bc8e91..13ba2317 100644 --- a/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java +++ b/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java @@ -106,12 +106,11 @@ public final class TraceReplayActivity extends Activity { request.targetCall, request.width, request.height, - request.tolerance, + request.ssimThreshold, request.cropX, request.cropY, request.cropWidth, request.cropHeight, - request.fuzzPercent, request.angleLibraryDir, request.useAngle ); @@ -135,12 +134,11 @@ public final class TraceReplayActivity extends Activity { long targetCall, int width, int height, - int tolerance, + double ssimThreshold, int cropX, int cropY, int cropWidth, int cropHeight, - int fuzzPercent, String angleLibraryDir, boolean useAngle ); @@ -156,12 +154,11 @@ public final class TraceReplayActivity extends Activity { final long targetCall; final int width; final int height; - final int tolerance; + final double ssimThreshold; final int cropX; final int cropY; final int cropWidth; final int cropHeight; - final int fuzzPercent; final String angleLibraryDir; final boolean useAngle; @@ -176,12 +173,11 @@ public final class TraceReplayActivity extends Activity { long targetCall, int width, int height, - int tolerance, + double ssimThreshold, int cropX, int cropY, int cropWidth, int cropHeight, - int fuzzPercent, String angleLibraryDir, boolean useAngle ) { @@ -195,12 +191,11 @@ public final class TraceReplayActivity extends Activity { this.targetCall = targetCall; this.width = width; this.height = height; - this.tolerance = tolerance; + this.ssimThreshold = ssimThreshold; this.cropX = cropX; this.cropY = cropY; this.cropWidth = cropWidth; this.cropHeight = cropHeight; - this.fuzzPercent = fuzzPercent; this.angleLibraryDir = angleLibraryDir; this.useAngle = useAngle; } @@ -219,12 +214,11 @@ public final class TraceReplayActivity extends Activity { intent.getLongExtra("target_call", -1L), intent.getIntExtra("width", 0), intent.getIntExtra("height", 0), - intent.getIntExtra("tolerance", 0), + readDouble(intent, "ssim_threshold", 0.99), intent.getIntExtra("crop_x", 0), intent.getIntExtra("crop_y", 0), intent.getIntExtra("crop_width", 0), intent.getIntExtra("crop_height", 0), - intent.getIntExtra("fuzz_percent", 20), readString(intent, "angle_library_dir", nativeLibraryDir), intent.getBooleanExtra("use_angle", false) ); @@ -234,6 +228,18 @@ public final class TraceReplayActivity extends Activity { String value = intent.getStringExtra(key); return value == null ? fallback : value; } + + private static double readDouble(Intent intent, String key, double fallback) { + String stringValue = intent.getStringExtra(key); + if (stringValue != null && !stringValue.isEmpty()) { + try { + return Double.parseDouble(stringValue); + } catch (NumberFormatException ignored) { + return fallback; + } + } + return intent.getDoubleExtra(key, fallback); + } } public static final class TraceReplayResult { diff --git a/android-plugin/trace-replay-ci.sh b/android-plugin/trace-replay-ci.sh index ccfd330d..85d15ff5 100644 --- a/android-plugin/trace-replay-ci.sh +++ b/android-plugin/trace-replay-ci.sh @@ -21,12 +21,11 @@ Usage: --target-call N \ --width N \ --height N \ - --tolerance N \ + --ssim-threshold N \ --crop-x N \ --crop-y N \ --crop-width N \ --crop-height N \ - --fuzz-percent N \ --timeout-seconds N Set MOBILEGL_RETRACE_USE_ANGLE=1 to run DirectGLES replay with packaged ANGLE @@ -68,12 +67,11 @@ alternate_golden_path="" target_call="" width="" height="" -tolerance="" +ssim_threshold="" crop_x="" crop_y="" crop_width="" crop_height="" -fuzz_percent="" timeout_seconds="" while [ "$#" -gt 0 ]; do @@ -99,12 +97,11 @@ while [ "$#" -gt 0 ]; do --target-call) target_call="$(next_arg "$@")"; shift 2 ;; --width) width="$(next_arg "$@")"; shift 2 ;; --height) height="$(next_arg "$@")"; shift 2 ;; - --tolerance) tolerance="$(next_arg "$@")"; shift 2 ;; + --ssim-threshold) ssim_threshold="$(next_arg "$@")"; shift 2 ;; --crop-x) crop_x="$(next_arg "$@")"; shift 2 ;; --crop-y) crop_y="$(next_arg "$@")"; shift 2 ;; --crop-width) crop_width="$(next_arg "$@")"; shift 2 ;; --crop-height) crop_height="$(next_arg "$@")"; shift 2 ;; - --fuzz-percent) fuzz_percent="$(next_arg "$@")"; shift 2 ;; --timeout-seconds) timeout_seconds="$(next_arg "$@")"; shift 2 ;; -h|--help) usage; exit 0 ;; *) die "unknown argument: $1" ;; @@ -123,12 +120,11 @@ require_value "${golden_path}" "--golden" require_value "${target_call}" "--target-call" require_value "${width}" "--width" require_value "${height}" "--height" -require_value "${tolerance}" "--tolerance" +require_value "${ssim_threshold}" "--ssim-threshold" require_value "${crop_x}" "--crop-x" require_value "${crop_y}" "--crop-y" require_value "${crop_width}" "--crop-width" require_value "${crop_height}" "--crop-height" -require_value "${fuzz_percent}" "--fuzz-percent" require_value "${timeout_seconds}" "--timeout-seconds" test -f "${apk_file}" || die "APK does not exist: ${apk_file}" @@ -228,12 +224,11 @@ run_retrace() { --el target_call "${target_call}" \ --ei width "${width}" \ --ei height "${height}" \ - --ei tolerance "${tolerance}" \ + --es ssim_threshold "${ssim_threshold}" \ --ei crop_x "${crop_x}" \ --ei crop_y "${crop_y}" \ --ei crop_width "${crop_width}" \ - --ei crop_height "${crop_height}" \ - --ei fuzz_percent "${fuzz_percent}" + --ei crop_height "${crop_height}" "${ADB}" shell "$@" app_exited=0 diff --git a/tools/trace_replay/CMakeLists.txt b/tools/trace_replay/CMakeLists.txt index 80c72478..4ff26c65 100644 --- a/tools/trace_replay/CMakeLists.txt +++ b/tools/trace_replay/CMakeLists.txt @@ -273,12 +273,11 @@ function(add_trace_replay_test CASE_NAME BACKEND) TARGET_CALL WIDTH HEIGHT - TOLERANCE + SSIM_THRESHOLD CROP_X CROP_Y CROP_WIDTH - CROP_HEIGHT - FUZZ_PERCENT) + CROP_HEIGHT) cmake_parse_arguments(TRACE_CASE "" "${oneValueArgs}" "" ${ARGN}) foreach(required TRACE_ARCHIVE GOLDEN TARGET_CALL WIDTH HEIGHT) if(NOT TRACE_CASE_${required}) @@ -288,8 +287,8 @@ function(add_trace_replay_test CASE_NAME BACKEND) if(NOT TRACE_CASE_TRACE_FILE) set(TRACE_CASE_TRACE_FILE trace.trace) endif() - if(NOT TRACE_CASE_TOLERANCE) - set(TRACE_CASE_TOLERANCE 0) + if(NOT TRACE_CASE_SSIM_THRESHOLD) + set(TRACE_CASE_SSIM_THRESHOLD 0.99) endif() if(NOT TRACE_CASE_CROP_X) set(TRACE_CASE_CROP_X 0) @@ -303,9 +302,6 @@ function(add_trace_replay_test CASE_NAME BACKEND) if(NOT TRACE_CASE_CROP_HEIGHT) set(TRACE_CASE_CROP_HEIGHT 0) endif() - if(NOT TRACE_CASE_FUZZ_PERCENT) - set(TRACE_CASE_FUZZ_PERCENT 20) - endif() add_test( NAME MobileGLTraceReplay.${CASE_NAME}.${BACKEND} COMMAND "${CMAKE_COMMAND}" @@ -320,12 +316,11 @@ function(add_trace_replay_test CASE_NAME BACKEND) -DTRACE_TARGET_CALL=${TRACE_CASE_TARGET_CALL} -DTRACE_WIDTH=${TRACE_CASE_WIDTH} -DTRACE_HEIGHT=${TRACE_CASE_HEIGHT} - -DTRACE_TOLERANCE=${TRACE_CASE_TOLERANCE} + -DTRACE_SSIM_THRESHOLD=${TRACE_CASE_SSIM_THRESHOLD} -DTRACE_CROP_X=${TRACE_CASE_CROP_X} -DTRACE_CROP_Y=${TRACE_CASE_CROP_Y} -DTRACE_CROP_WIDTH=${TRACE_CASE_CROP_WIDTH} -DTRACE_CROP_HEIGHT=${TRACE_CASE_CROP_HEIGHT} - -DTRACE_FUZZ_PERCENT=${TRACE_CASE_FUZZ_PERCENT} -DTRACE_OUTPUT_DIR=${CMAKE_CURRENT_BINARY_DIR}/${CASE_NAME}/${BACKEND} -DTRACE_ARTIFACT_DIR=${CMAKE_CURRENT_BINARY_DIR}/${CASE_NAME}/actual-images -P ${MOBILEGL_TRACE_ROOT}/run_trace_case.cmake) @@ -350,12 +345,11 @@ add_trace_replay_test_for_backends(OpenRA TARGET_CALL 31249 WIDTH 640 HEIGHT 480 - TOLERANCE 3072 CROP_X 1 CROP_Y 1 CROP_WIDTH 638 CROP_HEIGHT 478 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-startup TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-startup.tgz @@ -364,8 +358,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-startup TARGET_CALL 92195 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-main-menu TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-main-menu.tgz @@ -374,8 +367,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-main-menu TARGET_CALL 481787 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-in-world TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-in-world.tgz @@ -384,8 +376,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-in-world TARGET_CALL 280000 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-sodium-in-world TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-sodium-in-world.tgz @@ -394,8 +385,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-sodium-in-world TARGET_CALL 923340 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-bsl-in-world TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-bsl-in-world.tgz @@ -404,8 +394,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-bsl-in-world TARGET_CALL 110725 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-makeup-ultrafast-in-world TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-makeup-ultrafast-in-world.tgz @@ -414,8 +403,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-makeup-ultrafast TARGET_CALL 95322 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-super-duper-vanilla-in-world TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-super-duper-vanilla-in-world.tgz @@ -424,8 +412,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-super-duper-vani TARGET_CALL 141559 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-sundial-lite-in-world TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-sundial-lite-in-world.tgz @@ -434,8 +421,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-sundial-lite-in- TARGET_CALL 150023 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-complementary-reimagined-in-world TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-complementary-reimagined-in-world.tgz @@ -444,8 +430,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-complementary-re TARGET_CALL 151297 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-complementary-unbound-in-world TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-complementary-unbound-in-world.tgz @@ -454,8 +439,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-complementary-un TARGET_CALL 146559 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-mellow-in-world TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-mellow-in-world.tgz @@ -464,8 +448,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-mellow-in-world TARGET_CALL 96143 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-nostalgia-in-world TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-nostalgia-in-world.tgz @@ -475,8 +458,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-nostalgia-in-wor TARGET_CALL 153808 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-bliss-in-world TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-bliss-in-world.tgz @@ -485,8 +467,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-bliss-in-world TARGET_CALL 113511 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-chocapic-v6-lite-in-world TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-chocapic-v6-lite-in-world.tgz @@ -495,8 +476,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-chocapic-v6-lite TARGET_CALL 125124 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-iterationt-in-world TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-iterationt-in-world.tgz @@ -505,8 +485,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-iterationt-in-wo TARGET_CALL 1173280 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-iterationt-nodsa-in-world TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-iterationt-nodsa-in-world.tgz @@ -515,8 +494,7 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-iterationt-nodsa TARGET_CALL 1606997 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-photon-v1.1-in-world TRACE_ARCHIVE ${MOBILEGL_TRACE_ROOT}/fixtures/minecraft-1.21.4-fabric-iris-photon-v1.1-in-world.tgz @@ -525,5 +503,4 @@ add_trace_replay_test_for_backends(minecraft-1.21.4-fabric-iris-photon-v1.1-in-w TARGET_CALL 159866 WIDTH 854 HEIGHT 480 - TOLERANCE 4100 - FUZZ_PERCENT 20) +) diff --git a/tools/trace_replay/README.md b/tools/trace_replay/README.md index 692a8e1e..4dce83f8 100644 --- a/tools/trace_replay/README.md +++ b/tools/trace_replay/README.md @@ -94,8 +94,7 @@ build-test/tools/trace_replay/mobilegl_trace_replay \ --crop-y 1 \ --crop-width 638 \ --crop-height 478 \ - --tolerance 20 \ - --fuzz-percent 20 + --ssim-threshold 0.99 ``` ## Android device replay @@ -140,8 +139,7 @@ adb shell am start -W -a top.mobilegl.plugin.TRACE_REPLAY \ --ei crop_y 1 \ --ei crop_width 638 \ --ei crop_height 478 \ - --ei tolerance 20 \ - --ei fuzz_percent 20 + --es ssim_threshold 0.99 ``` Read back the result and images: diff --git a/tools/trace_replay/run_openra_replay.cmake b/tools/trace_replay/run_openra_replay.cmake deleted file mode 100644 index 640ed9ef..00000000 --- a/tools/trace_replay/run_openra_replay.cmake +++ /dev/null @@ -1,97 +0,0 @@ -foreach(required TRACE_REPLAY_EXE MOBILEGL_LIBRARY OPENRA_TRACE_ARCHIVE OPENRA_GOLDEN OPENRA_OUTPUT_DIR OPENRA_BACKEND) - if(NOT DEFINED ${required} OR "${${required}}" STREQUAL "") - message(FATAL_ERROR "${required} is required") - endif() -endforeach() - -if(EXISTS "${OPENRA_OUTPUT_DIR}") - file(REMOVE_RECURSE "${OPENRA_OUTPUT_DIR}") -endif() -file(MAKE_DIRECTORY "${OPENRA_OUTPUT_DIR}/input") -file(MAKE_DIRECTORY "${OPENRA_OUTPUT_DIR}/output") - -execute_process( - COMMAND "${CMAKE_COMMAND}" -E tar xzf "${OPENRA_TRACE_ARCHIVE}" - WORKING_DIRECTORY "${OPENRA_OUTPUT_DIR}/input" - RESULT_VARIABLE extract_result - OUTPUT_VARIABLE extract_stdout - ERROR_VARIABLE extract_stderr) -if(NOT extract_result EQUAL 0) - message(STATUS "${extract_stdout}") - message(STATUS "${extract_stderr}") - message(FATAL_ERROR "failed to extract ${OPENRA_TRACE_ARCHIVE}") -endif() - -set(openra_trace "${OPENRA_OUTPUT_DIR}/input/openra.trace") -if(NOT EXISTS "${openra_trace}") - message(FATAL_ERROR "extracted OpenRA trace was not found at ${openra_trace}") -endif() - -execute_process( - COMMAND "${TRACE_REPLAY_EXE}" - --trace "${openra_trace}" - --golden "${OPENRA_GOLDEN}" - --output "${OPENRA_OUTPUT_DIR}/output" - --backend "${OPENRA_BACKEND}" - --mobilegl-library "${MOBILEGL_LIBRARY}" - --target-call 31249 - --width 640 - --height 480 - --tolerance 20 - --crop-x 1 - --crop-y 1 - --crop-width 638 - --crop-height 478 - --fuzz-percent 20 - RESULT_VARIABLE replay_result - OUTPUT_VARIABLE replay_stdout - ERROR_VARIABLE replay_stderr) - -message(STATUS "${replay_stdout}") -message(STATUS "${replay_stderr}") - -set(retrace_log "${OPENRA_OUTPUT_DIR}/output/retrace.log") -set(mobilegl_log "${OPENRA_OUTPUT_DIR}/output/mobilegl.log") -if(EXISTS "${retrace_log}") - file(STRINGS "${retrace_log}" gl_identity_lines REGEX "MOBILEGL_TRACE_GL_") - foreach(line IN LISTS gl_identity_lines) - message(STATUS "${line}") - endforeach() -endif() - -set(result_json "${OPENRA_OUTPUT_DIR}/output/result.json") -if(DEFINED OPENRA_ARTIFACT_DIR AND NOT "${OPENRA_ARTIFACT_DIR}" STREQUAL "") - file(MAKE_DIRECTORY "${OPENRA_ARTIFACT_DIR}") - set(actual_png "${OPENRA_OUTPUT_DIR}/output/actual.png") - if(EXISTS "${actual_png}") - file(COPY_FILE "${actual_png}" "${OPENRA_ARTIFACT_DIR}/openra-${OPENRA_BACKEND}-actual.png") - endif() - if(EXISTS "${result_json}") - file(COPY_FILE "${result_json}" "${OPENRA_ARTIFACT_DIR}/openra-${OPENRA_BACKEND}-result.json") - endif() - if(EXISTS "${retrace_log}") - file(COPY_FILE "${retrace_log}" "${OPENRA_ARTIFACT_DIR}/openra-${OPENRA_BACKEND}-retrace.log") - endif() - if(EXISTS "${mobilegl_log}") - file(COPY_FILE "${mobilegl_log}" "${OPENRA_ARTIFACT_DIR}/openra-${OPENRA_BACKEND}-mobilegl.log") - endif() -endif() - -if(EXISTS "${result_json}") - file(READ "${result_json}" result_contents) - message(STATUS "${result_contents}") -else() - if(EXISTS "${retrace_log}") - file(READ "${retrace_log}" retrace_log_contents) - message(STATUS "${retrace_log_contents}") - endif() - if(EXISTS "${mobilegl_log}") - file(READ "${mobilegl_log}" mobilegl_log_contents) - message(STATUS "${mobilegl_log_contents}") - endif() - message(FATAL_ERROR "trace replay did not write ${result_json}") -endif() - -if(NOT replay_result EQUAL 0) - message(FATAL_ERROR "OpenRA ${OPENRA_BACKEND} trace replay failed with status ${replay_result}") -endif() diff --git a/tools/trace_replay/run_trace_case.cmake b/tools/trace_replay/run_trace_case.cmake index c9c6c5d9..6631388b 100644 --- a/tools/trace_replay/run_trace_case.cmake +++ b/tools/trace_replay/run_trace_case.cmake @@ -4,8 +4,8 @@ foreach(required TRACE_REPLAY_EXE MOBILEGL_LIBRARY TRACE_ARCHIVE TRACE_FILE TRAC endif() endforeach() -if(NOT DEFINED TRACE_TOLERANCE OR "${TRACE_TOLERANCE}" STREQUAL "") - set(TRACE_TOLERANCE 0) +if(NOT DEFINED TRACE_SSIM_THRESHOLD OR "${TRACE_SSIM_THRESHOLD}" STREQUAL "") + set(TRACE_SSIM_THRESHOLD 0.99) endif() if(NOT DEFINED TRACE_CROP_X OR "${TRACE_CROP_X}" STREQUAL "") set(TRACE_CROP_X 0) @@ -19,9 +19,6 @@ endif() if(NOT DEFINED TRACE_CROP_HEIGHT OR "${TRACE_CROP_HEIGHT}" STREQUAL "") set(TRACE_CROP_HEIGHT 0) endif() -if(NOT DEFINED TRACE_FUZZ_PERCENT OR "${TRACE_FUZZ_PERCENT}" STREQUAL "") - set(TRACE_FUZZ_PERCENT 20) -endif() set(alternate_golden_args) if(DEFINED TRACE_ALTERNATE_GOLDEN AND NOT "${TRACE_ALTERNATE_GOLDEN}" STREQUAL "") list(APPEND alternate_golden_args --alternate-golden "${TRACE_ALTERNATE_GOLDEN}") @@ -62,12 +59,11 @@ execute_process( --target-call "${TRACE_TARGET_CALL}" --width "${TRACE_WIDTH}" --height "${TRACE_HEIGHT}" - --tolerance "${TRACE_TOLERANCE}" + --ssim-threshold "${TRACE_SSIM_THRESHOLD}" --crop-x "${TRACE_CROP_X}" --crop-y "${TRACE_CROP_Y}" --crop-width "${TRACE_CROP_WIDTH}" --crop-height "${TRACE_CROP_HEIGHT}" - --fuzz-percent "${TRACE_FUZZ_PERCENT}" RESULT_VARIABLE replay_result OUTPUT_VARIABLE replay_stdout ERROR_VARIABLE replay_stderr) diff --git a/tools/trace_replay/skills/mismatch-retrace-debugging.md b/tools/trace_replay/skills/mismatch-retrace-debugging.md index 30e7332b..71826efe 100644 --- a/tools/trace_replay/skills/mismatch-retrace-debugging.md +++ b/tools/trace_replay/skills/mismatch-retrace-debugging.md @@ -40,8 +40,9 @@ The golden and target environments must use the same trace, surface size, target call numbers, and comparison crop. They do not need to report the same GL vendor, renderer, version, or extension set. The only requirement for the golden environment is that it can replay the case and pass the fixture's golden image -test. Disable overlays and avoid tolerance/fuzz while localizing the first -divergent call unless the known golden already has small nondeterministic noise. +test. Disable overlays and use a strict SSIM threshold while localizing the +first divergent call unless the known golden already has small nondeterministic +noise. ## Prepare two replay environments @@ -80,8 +81,7 @@ mkdir -p "$WORK/$CASE" --target-call "$TARGET_CALL" \ --width "$WIDTH" \ --height "$HEIGHT" \ - --tolerance 0 \ - --fuzz-percent 0 > "$WORK/$CASE/target-retrace.log" 2>&1 || true + --ssim-threshold 1.0 > "$WORK/$CASE/target-retrace.log" 2>&1 || true ``` Keep target GL identity lines such as `MOBILEGL_TRACE_GL_VENDOR`, @@ -90,7 +90,7 @@ Keep target GL identity lines such as `MOBILEGL_TRACE_GL_VENDOR`, reject a golden environment because its GL strings differ from the target. For Android, use `android-plugin/trace-replay-ci.sh` with the same trace, -golden, target call, width, height, crop, tolerance, and fuzz values. Pull +golden, target call, width, height, crop, and SSIM threshold values. Pull `result.json`, `actual.png`, `diff.png`, `retrace.log`, and `logcat.txt` before continuing. @@ -151,8 +151,7 @@ for call in 1000 1250 1430 1700; do --target-call "$call" \ --width "$WIDTH" \ --height "$HEIGHT" \ - --tolerance 0 \ - --fuzz-percent 0 || true + --ssim-threshold 1.0 || true done ``` diff --git a/tools/trace_replay/skills/trace-fixture-authoring.md b/tools/trace_replay/skills/trace-fixture-authoring.md index 4d803d12..cb03d387 100644 --- a/tools/trace_replay/skills/trace-fixture-authoring.md +++ b/tools/trace_replay/skills/trace-fixture-authoring.md @@ -173,8 +173,7 @@ add_trace_replay_test_for_backends(case-name TARGET_CALL 0 WIDTH 854 HEIGHT 480 - TOLERANCE 20 - FUZZ_PERCENT 20) + SSIM_THRESHOLD 0.99) ``` Optional crop: @@ -191,13 +190,13 @@ Optional crop: Edit `.github/workflows/apk.yml` `TRACE_REPLAY_CASES`: ```text -case-name|tools/trace_replay/fixtures/case-name.tgz|trace.trace|tools/trace_replay/fixtures/case-name.0000000000.png|0|854|480|20|0|0|0|0|20|900 +case-name|tools/trace_replay/fixtures/case-name.tgz|trace.trace|tools/trace_replay/fixtures/case-name.0000000000.png|0|854|480|0.99|0|0|0|0|900 ``` Field order: ```text -case|trace archive|trace file|golden|target call|width|height|tolerance|crop x|crop y|crop width|crop height|fuzz percent|timeout seconds +case|trace archive|trace file|golden|target call|width|height|ssim threshold|crop x|crop y|crop width|crop height|timeout seconds ``` Update `tools/trace_replay/README.md` with one fixture sentence and one golden @@ -236,12 +235,11 @@ cmake \ -DTRACE_TARGET_CALL="$TARGET_CALL" \ -DTRACE_WIDTH="$WIDTH" \ -DTRACE_HEIGHT="$HEIGHT" \ - -DTRACE_TOLERANCE=20 \ + -DTRACE_SSIM_THRESHOLD=0.99 \ -DTRACE_CROP_X=0 \ -DTRACE_CROP_Y=0 \ -DTRACE_CROP_WIDTH=0 \ -DTRACE_CROP_HEIGHT=0 \ - -DTRACE_FUZZ_PERCENT=20 \ -DTRACE_OUTPUT_DIR="$WORK/$CASE/linux-DirectGLES" \ -DTRACE_ARTIFACT_DIR="$WORK/$CASE/linux-artifacts" \ -P "$REPO/tools/trace_replay/run_trace_case.cmake" @@ -292,12 +290,11 @@ sh "$REPO/android-plugin/trace-replay-ci.sh" \ --target-call "$TARGET_CALL" \ --width "$WIDTH" \ --height "$HEIGHT" \ - --tolerance 20 \ + --ssim-threshold 0.99 \ --crop-x 0 \ --crop-y 0 \ --crop-width 0 \ --crop-height 0 \ - --fuzz-percent 20 \ --timeout-seconds 900 ``` diff --git a/tools/trace_replay/trace_replay_cli.cpp b/tools/trace_replay/trace_replay_cli.cpp index 2b1672a1..36da7714 100644 --- a/tools/trace_replay/trace_replay_cli.cpp +++ b/tools/trace_replay/trace_replay_cli.cpp @@ -20,8 +20,7 @@ void PrintUsage(const char *argv0) { << " --mobilegl-library PATH libMobileGL.so path (default: libMobileGL.so)\n" << " --width N Replay surface width override\n" << " --height N Replay surface height override\n" - << " --tolerance N Allowed mismatch pixels\n" - << " --fuzz-percent N Per-channel fuzz threshold, 0-100 (default: 20)\n" + << " --ssim-threshold N Minimum SSIM required to pass (default: 0.99)\n" << " --crop-x N Compare crop x\n" << " --crop-y N Compare crop y\n" << " --crop-width N Compare crop width\n" @@ -55,9 +54,17 @@ bool ReadLongLong(int argc, char **argv, int &index, long long &out) { return true; } +bool ReadDouble(int argc, char **argv, int &index, double &out) { + std::string value; + if (!ReadValue(argc, argv, index, value)) { + return false; + } + out = std::strtod(value.c_str(), nullptr); + return true; +} + bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) { request.backend = "DirectGLES"; - request.fuzzPercent = 20; for (int i = 1; i < argc; ++i) { const std::string arg = argv[i]; @@ -87,8 +94,8 @@ bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) { if (!ReadInt(argc, argv, i, request.width)) return false; } else if (arg == "--height") { if (!ReadInt(argc, argv, i, request.height)) return false; - } else if (arg == "--tolerance") { - if (!ReadInt(argc, argv, i, request.tolerance)) return false; + } else if (arg == "--ssim-threshold") { + if (!ReadDouble(argc, argv, i, request.ssimThreshold)) return false; } else if (arg == "--crop-x") { if (!ReadInt(argc, argv, i, request.cropX)) return false; } else if (arg == "--crop-y") { @@ -97,8 +104,6 @@ bool ParseArgs(int argc, char **argv, mobilegl_trace::Request &request) { if (!ReadInt(argc, argv, i, request.cropWidth)) return false; } else if (arg == "--crop-height") { if (!ReadInt(argc, argv, i, request.cropHeight)) return false; - } else if (arg == "--fuzz-percent") { - if (!ReadInt(argc, argv, i, request.fuzzPercent)) return false; } else if (arg == "--help" || arg == "-h") { return false; } else {