diff --git a/.github/workflows/apk.yml b/.github/workflows/apk.yml index 09fc1690..beece794 100644 --- a/.github/workflows/apk.yml +++ b/.github/workflows/apk.yml @@ -445,6 +445,19 @@ jobs: --crop-height "${{ matrix.case.crop_height }}" \ --timeout-seconds "${{ matrix.case.timeout_seconds }}" + - name: Collect retrace summary inputs + if: always() + run: | + safe_case="$(printf '%s' '${{ matrix.case.name }}' | sed 's/[^A-Za-z0-9._-]/_/g')" + result_dir="android-retrace-result/${safe_case}-${{ matrix.backend.name }}" + mkdir -p "${result_dir}" + if [ -s "${{ matrix.case.golden }}" ]; then + cp "${{ matrix.case.golden }}" "${result_dir}/${safe_case}-${{ matrix.backend.name }}-golden.png" + fi + if [ -n "${{ matrix.case.alternate_golden || '' }}" ] && [ -s "${{ matrix.case.alternate_golden || '' }}" ]; then + cp "${{ matrix.case.alternate_golden || '' }}" "${result_dir}/${safe_case}-${{ matrix.backend.name }}-alternate-golden.png" + fi + - name: Collect emulator diagnostics if: always() run: | @@ -470,3 +483,53 @@ jobs: name: MobileGL-android-retrace-result-${{ env.date_today }}-${{ env.short_sha }}-${{ matrix.backend.name }}-${{ matrix.case.name }} path: android-retrace-result/** if-no-files-found: warn + + retrace-summary: + name: retrace summary + runs-on: ubuntu-latest + needs: retrace + if: always() + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set artifact metadata + run: | + echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV" + echo "short_sha=${GITHUB_SHA::7}" >> "$GITHUB_ENV" + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + + - name: Download Android retrace results + uses: actions/download-artifact@v4 + with: + pattern: MobileGL-android-retrace-result-* + path: retrace-artifacts + + - name: Install PDF renderer + run: | + npm install --no-save --no-package-lock playwright + npx playwright install --with-deps chromium + + - name: Render retrace summary + run: | + node tools/trace_replay/render_retrace_summary.mjs \ + --input retrace-artifacts \ + --output-dir android-retrace-summary \ + --title "MobileGL Android retrace overview" \ + --group-label "Android Emulator" \ + --html mobilegl-android-retrace-overview.html \ + --pdf mobilegl-android-retrace-overview.pdf + + - name: Upload Android retrace summary + uses: actions/upload-artifact@v4 + with: + name: MobileGL-android-retrace-summary-${{ env.date_today }}-${{ env.short_sha }} + path: android-retrace-summary/** + if-no-files-found: error diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 54b976df..86628bb1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -418,3 +418,53 @@ jobs: build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/** if-no-files-found: warn + retrace-summary: + name: retrace summary + runs-on: ubuntu-latest + needs: retrace + if: always() + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set artifact metadata + run: | + echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV" + echo "short_sha=${GITHUB_SHA::7}" >> "$GITHUB_ENV" + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + + - name: Download retrace results + uses: actions/download-artifact@v4 + with: + pattern: retrace-result-* + path: retrace-artifacts + + - name: Install PDF renderer + run: | + npm install --no-save --no-package-lock playwright + npx playwright install --with-deps chromium + + - name: Render retrace summary + run: | + node tools/trace_replay/render_retrace_summary.mjs \ + --input retrace-artifacts \ + --output-dir retrace-summary \ + --title "MobileGL Linux retrace overview" \ + --group-label "Linux" \ + --html mobilegl-linux-retrace-overview.html \ + --pdf mobilegl-linux-retrace-overview.pdf + + - name: Upload retrace summary + uses: actions/upload-artifact@v4 + with: + name: retrace-summary-${{ env.date_today }}-${{ env.short_sha }} + path: retrace-summary/** + if-no-files-found: error + diff --git a/tools/trace_replay/render_retrace_summary.mjs b/tools/trace_replay/render_retrace_summary.mjs new file mode 100644 index 00000000..833b3738 --- /dev/null +++ b/tools/trace_replay/render_retrace_summary.mjs @@ -0,0 +1,562 @@ +#!/usr/bin/env node + +import { copyFile, mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const BACKENDS = ["DirectGLES", "DirectVulkan"]; +const CASES = [ + "OpenRA", + "minecraft-1.21.4-startup", + "minecraft-1.21.4-main-menu", + "minecraft-1.21.4-in-world", + "minecraft-1.21.4-fabric-sodium-in-world", + "minecraft-1.21.4-fabric-iris-bsl-in-world", + "minecraft-1.21.4-fabric-iris-makeup-ultrafast-in-world", + "minecraft-1.21.4-fabric-iris-super-duper-vanilla-in-world", + "minecraft-1.21.4-fabric-iris-sundial-lite-in-world", + "minecraft-1.21.4-fabric-iris-complementary-reimagined-in-world", + "minecraft-1.21.4-fabric-iris-complementary-unbound-in-world", + "minecraft-1.21.4-fabric-iris-mellow-in-world", + "minecraft-1.21.4-fabric-iris-nostalgia-in-world", + "minecraft-1.21.4-fabric-iris-bliss-in-world", + "minecraft-1.21.4-fabric-iris-chocapic-v6-lite-in-world", + "minecraft-1.21.4-fabric-iris-iterationt-in-world", + "minecraft-1.21.4-fabric-iris-iterationt-nodsa-in-world", + "minecraft-1.21.4-fabric-iris-photon-v1.1-in-world", +]; + +function usage() { + console.error(`Usage: + node tools/trace_replay/render_retrace_summary.mjs \\ + --input DIR [--input DIR ...] \\ + --output-dir DIR \\ + [--title TITLE] [--group-label LABEL] [--html NAME] [--pdf NAME] + +When --pdf is provided, the script requires Playwright to be installed in the +current Node environment.`); +} + +function parseArgs(argv) { + const args = { + inputs: [], + outputDir: "", + title: "MobileGL retrace overview", + groupLabel: "retrace", + htmlName: "mobilegl-retrace-overview.html", + pdfName: "", + }; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + const value = argv[i + 1]; + if (arg === "--input" && value) { + args.inputs.push(path.resolve(value)); + i += 1; + } else if (arg === "--output-dir" && value) { + args.outputDir = path.resolve(value); + i += 1; + } else if (arg === "--title" && value) { + args.title = value; + i += 1; + } else if (arg === "--group-label" && value) { + args.groupLabel = value; + i += 1; + } else if (arg === "--html" && value) { + args.htmlName = value; + i += 1; + } else if (arg === "--pdf" && value) { + args.pdfName = value; + i += 1; + } else if (arg === "--help" || arg === "-h") { + usage(); + process.exit(0); + } else { + throw new Error(`Unknown or incomplete argument: ${arg}`); + } + } + if (args.inputs.length === 0 || !args.outputDir) { + usage(); + process.exit(2); + } + return args; +} + +async function exists(filePath) { + try { + await stat(filePath); + return true; + } catch { + return false; + } +} + +async function isUsablePng(filePath) { + try { + const info = await stat(filePath); + return info.isFile() && filePath.toLowerCase().endsWith(".png") && info.size > 1024; + } catch { + return false; + } +} + +async function walk(root) { + const results = []; + if (!(await exists(root))) return results; + const entries = await readdir(root, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(root, entry.name); + if (entry.isDirectory()) { + results.push(...await walk(fullPath)); + } else if (entry.isFile()) { + results.push(fullPath); + } + } + return results; +} + +function safeCase(caseName) { + return caseName.replace(/[^A-Za-z0-9._-]/g, "_"); +} + +function shortCase(caseName) { + return caseName + .replace(/^minecraft-1\.21\.4-/, "mc-") + .replaceAll("fabric-iris-", "") + .replaceAll("fabric-", "") + .replace(/-in-world$/, ""); +} + +function normalizeSlashes(value) { + return value.split(path.sep).join("/"); +} + +function inferCaseBackend(filePath) { + const text = normalizeSlashes(filePath); + const backend = BACKENDS.find((candidate) => text.includes(candidate)) ?? ""; + const sortedCases = [...CASES].sort((a, b) => b.length - a.length); + const caseName = sortedCases.find((candidate) => + text.includes(candidate) || text.includes(safeCase(candidate)) + ) ?? ""; + return { caseName, backend }; +} + +function inferGroup(filePath, defaultGroup) { + const text = normalizeSlashes(filePath); + for (const token of ["HA27Q3LQ", "JNHUVK4XL7ZLWK7L"]) { + if (text.includes(token)) return token; + } + return defaultGroup; +} + +function scoreImage(filePath, caseName, backend, kind) { + const text = normalizeSlashes(filePath); + const name = path.basename(filePath).toLowerCase(); + const lowerKind = kind.toLowerCase(); + if (lowerKind === "golden" && name.includes("alternate-golden")) return 0; + const matchesKind = + name === `${lowerKind}.png` || + name.endsWith(`-${lowerKind}.png`) || + name.endsWith(`_${lowerKind}.png`); + if (!matchesKind) return 0; + + let score = 0; + if (text.includes(caseName) || text.includes(safeCase(caseName))) score += 4; + if (text.includes(backend)) score += 3; + score += 3; + if (name === `${caseName}-${backend}-${kind}.png`.toLowerCase()) score += 4; + if (name === `${safeCase(caseName)}-${backend}-${kind}.png`.toLowerCase()) score += 4; + if (name === `${kind}.png`) score += 4; + return score; +} + +async function findImage(pngs, caseName, backend, kind, nearFile) { + const exactNames = [ + `${caseName}-${backend}-${kind}.png`, + `${safeCase(caseName)}-${backend}-${kind}.png`, + `${caseName}-${kind}.png`, + `${safeCase(caseName)}-${kind}.png`, + `${kind}.png`, + ]; + + if (nearFile) { + let current = path.dirname(nearFile); + for (let depth = 0; depth < 5; depth += 1) { + for (const name of exactNames) { + const candidate = path.join(current, name); + if (await isUsablePng(candidate)) return candidate; + } + const parent = path.dirname(current); + if (parent === current) break; + current = parent; + } + } + + const scored = []; + for (const png of pngs) { + const score = scoreImage(png, caseName, backend, kind); + if (score >= 7) scored.push({ score, png }); + } + scored.sort((a, b) => b.score - a.score || a.png.length - b.png.length); + return scored[0]?.png ?? ""; +} + +async function collectRows(inputs, defaultGroup) { + const files = []; + for (const input of inputs) files.push(...await walk(input)); + const pngs = []; + for (const file of files) { + if (await isUsablePng(file)) pngs.push(file); + } + + const rows = new Map(); + + for (const resultJson of files.filter((file) => path.basename(file) === "result.json" || file.endsWith("-result.json"))) { + let { caseName, backend } = inferCaseBackend(resultJson); + let result = {}; + try { + result = JSON.parse(await readFile(resultJson, "utf8")); + backend = backend || result.backend || ""; + } catch (error) { + result = { passed: false, message: `failed to parse result.json: ${error.message}` }; + } + if (!caseName || !BACKENDS.includes(backend)) continue; + const group = inferGroup(resultJson, defaultGroup); + const key = `${group}\0${backend}\0${caseName}`; + const matchedGolden = result.matchedGoldenPath || ""; + const alternateGoldens = Array.isArray(result.alternateGoldenPaths) ? result.alternateGoldenPaths : []; + const useAlternateGolden = alternateGoldens.some((alternate) => + path.basename(String(alternate)) === path.basename(String(matchedGolden)) + ) || String(matchedGolden).includes("alternate-golden"); + rows.set(key, { + group, + backend, + caseName, + status: result.passed ? "PASS" : "FAIL", + ssim: result.ssim === undefined || result.ssim === null ? "" : String(result.ssim), + message: result.message || "", + actual: await findImage(pngs, caseName, backend, "actual", resultJson), + golden: await findImage(pngs, caseName, backend, useAlternateGolden ? "alternate-golden" : "golden", resultJson), + diff: await findImage(pngs, caseName, backend, "diff", resultJson), + }); + } + + // Preserve visibility for jobs that uploaded logs/diagnostics but never wrote result.json. + const directories = new Set(files.map((file) => path.dirname(file))); + for (const directory of directories) { + const { caseName, backend } = inferCaseBackend(directory); + if (!caseName || !BACKENDS.includes(backend)) continue; + const group = inferGroup(directory, defaultGroup); + const key = `${group}\0${backend}\0${caseName}`; + if (rows.has(key)) continue; + rows.set(key, { + group, + backend, + caseName, + status: "FAIL", + ssim: "", + message: "result.json was not found", + actual: await findImage(pngs, caseName, backend, "actual", directory), + golden: await findImage(pngs, caseName, backend, "golden", directory), + diff: await findImage(pngs, caseName, backend, "diff", directory), + }); + } + + return [...rows.values()]; +} + +function htmlEscape(value) { + return String(value) + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """); +} + +async function copyAsset(source, outputDir, assetName) { + if (!source) return ""; + if (!(await isUsablePng(source))) return ""; + const assetsDir = path.join(outputDir, "assets"); + await mkdir(assetsDir, { recursive: true }); + const destination = path.join(assetsDir, assetName); + await copyFile(source, destination); + return normalizeSlashes(path.relative(outputDir, destination)); +} + +function statusText(row) { + if (!row) return "NO RESULT"; + if (row.status === "FIXTURE_MISSING") return "MISSING FIXTURE"; + if (row.ssim) { + const value = Number(row.ssim); + const text = Number.isFinite(value) ? value.toFixed(4) : row.ssim; + return `${row.status} SSIM ${htmlEscape(text)}`; + } + return row.status; +} + +async function renderHtml(rows, outputDir, title, groupLabel, htmlName) { + await mkdir(outputDir, { recursive: true }); + const groups = [...new Set(rows.map((row) => row.group))].sort(); + if (groups.length === 0) groups.push(groupLabel); + const rowMap = new Map(rows.map((row) => [`${row.group}\0${row.backend}\0${row.caseName}`, row])); + + const cells = new Map(); + for (const row of rows) { + const base = `${safeCase(row.group)}-${row.backend}-${safeCase(row.caseName)}`; + cells.set(`${row.group}\0${row.backend}\0${row.caseName}`, { + ...row, + actualRel: await copyAsset(row.actual, outputDir, `${base}-actual.png`), + goldenRel: await copyAsset(row.golden, outputDir, `${base}-golden.png`), + diffRel: await copyAsset(row.diff, outputDir, `${base}-diff.png`), + }); + } + + const thumb = (src, label) => ` +
+
${label}
+ ${src ? `${label}` : `
NO IMAGE
`} +
`; + + const backendCell = (group, backend, caseName) => { + const row = cells.get(`${group}\0${backend}\0${caseName}`) ?? { + status: "NO_RESULT", + actualRel: "", + goldenRel: "", + diffRel: "", + }; + const statusClass = row.status === "PASS" ? "pass" : row.status === "NO_RESULT" || row.status === "FIXTURE_MISSING" ? "missing" : "fail"; + return ` +
+
${backend} ${statusText(row)}
+
+ ${thumb(row.actualRel, "actual")} + ${thumb(row.goldenRel, "golden")} + ${thumb(row.diffRel, "diff")} +
+
`; + }; + + const groupSections = groups.map((group) => { + const groupRows = rows.filter((row) => row.group === group); + const passed = groupRows.filter((row) => row.status === "PASS").length; + const failed = groupRows.filter((row) => row.status === "FAIL").length; + const missing = groupRows.filter((row) => row.status !== "PASS" && row.status !== "FAIL").length; + return ` +
+

${htmlEscape(title)} - ${htmlEscape(group)}

+

Each backend cell shows actual / golden / diff. Failed cases are red; missing or absent results are orange. PASS ${passed}, FAIL ${failed}, MISSING ${missing}.

+
+
case
+
DirectGLES
+
DirectVulkan
+
+ ${CASES.map((caseName, index) => ` +
+ + ${backendCell(group, "DirectGLES", caseName)} + ${backendCell(group, "DirectVulkan", caseName)} +
`).join("")} +
`; + }).join(""); + + const html = ` + + + + +${htmlEscape(title)} + + + +
+${groupSections} +
+ + +`; + const output = path.join(outputDir, htmlName); + await writeFile(output, html, "utf8"); + return output; +} + +function csvEscape(value) { + const text = String(value ?? ""); + return /[",\n]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text; +} + +async function writeCsv(rows, outputDir) { + const header = ["Group", "Backend", "Case", "Status", "SSIM", "Message", "Actual", "Golden", "Diff"]; + const sorted = [...rows].sort((a, b) => + a.group.localeCompare(b.group) || + BACKENDS.indexOf(a.backend) - BACKENDS.indexOf(b.backend) || + CASES.indexOf(a.caseName) - CASES.indexOf(b.caseName) + ); + const serialize = (row) => [ + row.group, + row.backend, + row.caseName, + row.status, + row.ssim, + row.message, + row.actual, + row.golden, + row.diff, + ].map(csvEscape).join(","); + await writeFile(path.join(outputDir, "summary.csv"), [header.join(","), ...sorted.map(serialize)].join("\n"), "utf8"); + await writeFile(path.join(outputDir, "failures.csv"), [header.join(","), ...sorted.filter((row) => row.status !== "PASS").map(serialize)].join("\n"), "utf8"); +} + +async function renderPdf(htmlPath, pdfPath) { + let chromium; + try { + ({ chromium } = await import("playwright")); + } catch (error) { + throw new Error("Playwright is required for --pdf. Install it with: npm install playwright && npx playwright install chromium"); + } + const browser = await chromium.launch(); + try { + const page = await browser.newPage({ viewport: { width: 1170, height: 2380 }, deviceScaleFactor: 1 }); + await page.goto(pathToFileURL(htmlPath).href, { waitUntil: "networkidle" }); + await page.pdf({ + path: pdfPath, + printBackground: true, + preferCSSPageSize: true, + }); + } finally { + await browser.close(); + } +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + await mkdir(args.outputDir, { recursive: true }); + const rows = await collectRows(args.inputs, args.groupLabel); + const htmlPath = await renderHtml(rows, args.outputDir, args.title, args.groupLabel, args.htmlName); + await writeCsv(rows, args.outputDir); + console.log(htmlPath); + if (args.pdfName) { + const pdfPath = path.join(args.outputDir, args.pdfName); + await renderPdf(htmlPath, pdfPath); + console.log(pdfPath); + } +} + +main().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/tools/trace_replay/run_trace_case.cmake b/tools/trace_replay/run_trace_case.cmake index 6631388b..d1905c9b 100644 --- a/tools/trace_replay/run_trace_case.cmake +++ b/tools/trace_replay/run_trace_case.cmake @@ -87,6 +87,12 @@ if(DEFINED TRACE_ARTIFACT_DIR AND NOT "${TRACE_ARTIFACT_DIR}" STREQUAL "") if(EXISTS "${actual_png}") file(COPY_FILE "${actual_png}" "${TRACE_ARTIFACT_DIR}/${TRACE_CASE_NAME}-${TRACE_BACKEND}-actual.png") endif() + if(EXISTS "${TRACE_GOLDEN}") + file(COPY_FILE "${TRACE_GOLDEN}" "${TRACE_ARTIFACT_DIR}/${TRACE_CASE_NAME}-${TRACE_BACKEND}-golden.png") + endif() + if(DEFINED TRACE_ALTERNATE_GOLDEN AND NOT "${TRACE_ALTERNATE_GOLDEN}" STREQUAL "" AND EXISTS "${TRACE_ALTERNATE_GOLDEN}") + file(COPY_FILE "${TRACE_ALTERNATE_GOLDEN}" "${TRACE_ARTIFACT_DIR}/${TRACE_CASE_NAME}-${TRACE_BACKEND}-alternate-golden.png") + endif() if(EXISTS "${result_json}") file(COPY_FILE "${result_json}" "${TRACE_ARTIFACT_DIR}/${TRACE_CASE_NAME}-${TRACE_BACKEND}-result.json") endif()