diff --git a/.github/workflows/apk.yml b/.github/workflows/apk.yml index e358792d..d2e6ee6b 100644 --- a/.github/workflows/apk.yml +++ b/.github/workflows/apk.yml @@ -35,10 +35,21 @@ on: - '.github/workflows/hygiene.yml' - '.github/actions/setup-test-runner/**' workflow_dispatch: + inputs: + android_full_matrix: + description: Replay every trace case on the monolith and spawn emulator lanes too + type: boolean + default: false env: # Software-rendered CI submissions may exceed the interactive GPU hang budget. MOBILEGL_GPU_HANG_BUDGET_MS: "0" + # The emulator replays every trace case on inproc, the transport the launcher ships. On + # monolith and spawn it replays the android_smoke cases of trace_cases.json, because + # test.yml replays every case on all three transports on Linux. A manual run with + # android_full_matrix replays every case on all three emulator lanes. A push has no inputs, + # so it gets the reduced lanes. + MOBILEGL_ANDROID_LANES: ${{ inputs.android_full_matrix && 'full' || 'inproc-full' }} # ONE LIVE RUN PER BRANCH, as in test.yml: a superseded push is canceled rather than left queued # behind its successor; a manual dispatch gets a group of its own; dev queues rather than cancels. @@ -372,7 +383,7 @@ jobs: matrix_format=github-apk-transport-shard-matrix fi android="$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}" \ - --shard-seconds "${RETRACE_SHARD_SECONDS}")" + --shard-seconds "${RETRACE_SHARD_SECONDS}" --android-lanes "${MOBILEGL_ANDROID_LANES}")" # An empty `include` is not an error to GitHub: it skips the retrace job with no red. shards="$(printf '%s' "${android}" | python3 -c 'import json, sys; print(len(json.load(sys.stdin)["include"]))')" if [ "${shards}" -lt 1 ]; then @@ -387,7 +398,11 @@ jobs: android="$(printf '%s' "${android}" | python3 android-plugin/retrace_shard_budget.py \ --cap-seconds "${RETRACE_SHARD_DEADLINE_CAP_SECONDS}")" echo "android=${android}" >> "$GITHUB_OUTPUT" - echo "Android retrace: ${shards} shard(s) from ${matrix_format} at ${RETRACE_SHARD_SECONDS} s per shard" + lanes="" + if [ "${matrix_format}" = "github-apk-transport-shard-matrix" ]; then + lanes=" (${MOBILEGL_ANDROID_LANES} lanes)" + fi + echo "Android retrace: ${shards} shard(s) from ${matrix_format}${lanes} at ${RETRACE_SHARD_SECONDS} s per shard" # One cache entry holds every Git LFS fixture of the CI cases, and each retrace job restores # that entry itself. This job only makes sure the entry exists. On a hit it downloads @@ -857,10 +872,24 @@ jobs: pattern: MobileGL-android-retrace-result-spawn-acceptance-spawn-* path: retrace-artifacts/spawn-acceptance + # The reduced monolith and spawn lanes replay the android_smoke cases only, so their + # summaries list those cases and no others. A branch outside the split shape replays + # every case on monolith, its only lane. + - name: Select the cases of the reduced lanes + if: ${{ !cancelled() }} + run: | + reduced_lane_cases="" + if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ] && [ "${MOBILEGL_ANDROID_LANES}" = "inproc-full" ]; then + reduced_lane_cases="$(python3 tools/trace_replay/trace_cases.py --format android-smoke-names)" + test -n "${reduced_lane_cases}" + fi + echo "REDUCED_LANE_CASES=${reduced_lane_cases}" >> "$GITHUB_ENV" + - name: Render monolith control summary if: ${{ !cancelled() }} run: | node tools/trace_replay/render_retrace_summary.mjs \ + --cases "${REDUCED_LANE_CASES}" \ --input retrace-artifacts/monolith-control \ --output-dir android-retrace-summary/monolith-control \ --title "MobileGL Android monolith control" \ @@ -881,6 +910,7 @@ jobs: if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }} run: | node tools/trace_replay/render_retrace_summary.mjs \ + --cases "${REDUCED_LANE_CASES}" \ --input retrace-artifacts/spawn-acceptance \ --output-dir android-retrace-summary/spawn-acceptance \ --title "MobileGL Android spawn acceptance" \ diff --git a/android-plugin/TRACE_REPLAY.md b/android-plugin/TRACE_REPLAY.md index 698bbdca..c988cfd5 100644 --- a/android-plugin/TRACE_REPLAY.md +++ b/android-plugin/TRACE_REPLAY.md @@ -102,6 +102,16 @@ matrix whose shards, taken apart, are not exactly the per-case matrix. The fixtu cases must be in `tools/trace_replay/fixtures` before the runner starts; the workflow restores them from one cache entry (`.github/scripts/trace-fixture-bundle.sh`). +Unless a manual run sets `android_full_matrix`, the workflow passes `--android-lanes +inproc-full`: the inproc lane, the transport the launcher ships, replays every CI case, and the +monolith and spawn lanes replay the cases `trace_cases.json` lists under `android_smoke`. The +Linux retrace jobs of `test.yml` replay every case on all three transports, so those two +emulator lanes keep the cases that prove the Android side of each arm: the server started from +the APK's native library directory, the pbuffer surface of the spawn lane, the monolith build +of the same commit as a control, and every per-case replay setting at least once. A manual run +with `android_full_matrix` replays every case on all three lanes. A branch that runs only the +monolith lane replays every case on it. + Between cases the runner force-stops the app, checks that no process of the app's uid is left, clears the app data, removes the case's files from `/data/local/tmp` and from the host, and checks that the device still answers. It restarts the emulator when it does not, before the diff --git a/docs/ci/ci-cleanup-log.md b/docs/ci/ci-cleanup-log.md index adc1ace0..a09a263b 100644 --- a/docs/ci/ci-cleanup-log.md +++ b/docs/ci/ci-cleanup-log.md @@ -68,6 +68,7 @@ them wrong. | 2026-11-15 | per-case emulator jobs `retrace (, )` of each lane | MobileGL APK / `retrace` | one emulator boot per replay; no case, setting or arm proof is dropped | shard jobs that replay the cases of one lane, transport and backend on one boot through `android-plugin/trace_replay_shard.py`, with app data cleared between cases, a cold boot after a case that did not pass, and one verdict per case | | 2026-11-15 | per-case `trace fixture ()` jobs, their per-case cache entries and artifacts, `.github/scripts/trace-fixture-cache.sh` | Test and MobileGL APK / `trace-fixtures` | 40 jobs and 40 cache entries per workflow for files that change rarely | one content-addressed cache entry for all Git LFS fixtures (`.github/scripts/trace-fixture-bundle.sh`); a replay job restores it and downloads a case only when the entry does not hold it | | 2026-11-15 | `remove-artifact-clutter` job | Test and MobileGL APK | it deleted the fixture artifacts and the green result artifacts after the run; the fixture artifacts no longer exist | the delete step at the end of `retrace-summary`; on Linux `retention-days: 1` on the summary inputs as well, `retrace-split` and `retrace-shipped` upload results only for a failed shard (their green results were deleted unread before), and `retrace-verify` keeps its results for every shard as before | +| 2026-11-15 | emulator replays of the cases outside `android_smoke` on the monolith control and spawn acceptance lanes (56 of 79 rows per lane; the 12 smoke cases and the whole inproc lane stay) | MobileGL APK / `retrace` | the 237 emulator replays of a push held the 20 runner slots for most of an hour, and the launcher ships inproc; what goes is the same-commit Android monolith control and the Android spawn evidence (server exec from the native library directory, the pbuffer surface) for those 56 rows | Test replays every case on monolith, inproc and spawn on Linux; the Android inproc lane replays every case; `workflow_dispatch` with `android_full_matrix` runs the full emulator matrix; `trace_cases.py --self-test-transport-matrices` checks that the reduced lanes are the full matrix less exactly those rows | ## Kept after investigation diff --git a/tools/trace_replay/render_retrace_summary.mjs b/tools/trace_replay/render_retrace_summary.mjs index 30497569..a64f7efa 100644 --- a/tools/trace_replay/render_retrace_summary.mjs +++ b/tools/trace_replay/render_retrace_summary.mjs @@ -16,14 +16,15 @@ const BACKENDS = ["DirectGLES", "DirectVulkan"]; const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)); const DEFAULT_FIXTURE_ROOT = path.join(SCRIPT_DIR, "fixtures"); const TRACE_CASES = JSON.parse(readFileSync(path.join(SCRIPT_DIR, "trace_cases.json"), "utf8")); -const CASES = TRACE_CASES.cases.map((traceCase) => traceCase.name); +// Every case of the manifest, or the ones --cases names when a lane replays a subset. +let CASES = TRACE_CASES.cases.map((traceCase) => traceCase.name); function usage() { console.error(`Usage: node tools/trace_replay/render_retrace_summary.mjs \\ --input DIR [--input DIR ...] \\ --output-dir DIR \\ - [--title TITLE] [--group-label LABEL] [--html NAME]`); + [--title TITLE] [--group-label LABEL] [--html NAME] [--cases NAME,NAME,...]`); } function parseArgs(argv) { @@ -52,6 +53,13 @@ function parseArgs(argv) { } else if (arg === "--html" && value) { args.htmlName = value; i += 1; + } else if (arg === "--cases" && value !== undefined) { + // An empty list keeps every case, so a caller can pass the option unconditionally. + const names = value.split(",").filter((name) => name); + const unknown = names.filter((name) => !CASES.includes(name)); + if (unknown.length) throw new Error(`--cases names unknown trace case(s): ${unknown.join(", ")}`); + if (names.length) CASES = CASES.filter((name) => names.includes(name)); + i += 1; } else if (arg === "--help" || arg === "-h") { usage(); process.exit(0); diff --git a/tools/trace_replay/trace_cases.json b/tools/trace_replay/trace_cases.json index e4c47c5f..fe59f2cb 100644 --- a/tools/trace_replay/trace_cases.json +++ b/tools/trace_replay/trace_cases.json @@ -1,4 +1,18 @@ { + "android_smoke": [ + "OpenRA", + "minecraft-1.21.4-startup", + "minecraft-1.21.4-main-menu", + "minecraft-1.21.4-fabric-sodium-in-world", + "minecraft-1.21.4-fabric-iris-bsl-in-world", + "minecraft-1.21.4-fabric-iris-complementary-reimagined-in-world", + "minecraft-1.21.4-fabric-iris-bliss-in-world", + "minecraft-1.21.4-fabric-iris-sundial-lite-in-world", + "minecraft-1.21.1-neoforge-create-indirect-in-world-align1024", + "minecraft-1.21.1-neoforge-create-instancing-in-world", + "improved-transparency-minecraft-26.3", + "minecraft-1.21.4-fabric-iris-iterationrp-in-world" + ], "defaults": { "trace_file": "trace.trace", "width": 854, diff --git a/tools/trace_replay/trace_cases.py b/tools/trace_replay/trace_cases.py index 9c963a96..eedb8044 100644 --- a/tools/trace_replay/trace_cases.py +++ b/tools/trace_replay/trace_cases.py @@ -14,6 +14,9 @@ from pathlib import Path TRACE_CASES_JSON = Path(__file__).with_name("trace_cases.json") + +# The top-level keys of the manifest. An unknown one is refused, as an unknown case key is. +KNOWN_MANIFEST_KEYS = frozenset({"defaults", "cases", "android_smoke"}) CI_BACKENDS = ("DirectGLES", "DirectVulkan") # Every key a case or the defaults block may carry. An UNKNOWN key is a hard error rather than a @@ -127,7 +130,26 @@ def load_trace_case_manifest(path=TRACE_CASES_JSON): merged["split"] = split validate_backend_overrides(merged) cases.append(merged) - return {"defaults": defaults, "cases": cases} + unknown_top = sorted(set(manifest) - KNOWN_MANIFEST_KEYS) + if unknown_top: + raise ValueError(f"unknown top-level key(s) in the manifest: {', '.join(unknown_top)}") + # "android_smoke" names the cases the emulator replays on the monolith and spawn lanes + # when those lanes are reduced. A name that is not a CI case would shrink the subset with + # no red, so it is refused here. + smoke = manifest.get("android_smoke", []) + if not isinstance(smoke, list) or not all(isinstance(name, str) for name in smoke): + raise ValueError("android_smoke must be a list of case names") + if len(set(smoke)) != len(smoke): + raise ValueError("android_smoke repeats a case name") + ci_names = {case["name"] for case in cases if case.get("ci", True)} + not_ci = [name for name in smoke if name not in ci_names] + if not_ci: + raise ValueError(f"android_smoke names case(s) that are not CI cases: {', '.join(not_ci)}") + return {"defaults": defaults, "cases": cases, "android_smoke": smoke} + + +def android_smoke_names(path=TRACE_CASES_JSON): + return load_trace_case_manifest(path)["android_smoke"] def validate_backend_overrides(case): @@ -400,11 +422,21 @@ def github_apk_matrix(cases): } -def github_transport_matrix(cases, *, apk=False): +# The Android lanes of the transport matrix. "full" replays every case on every transport. +# "inproc-full" replays every case on inproc, the transport the launcher ships, and only the +# manifest's android_smoke cases on monolith and spawn: the Linux retrace jobs replay every +# case on all three transports, so those two emulator lanes keep the cases that prove the +# Android side of each arm. +ANDROID_LANES = ("full", "inproc-full") + + +def github_transport_matrix(cases, *, apk=False, android_smoke=None): """Run the entire original matrix as control and as acceptance, once per transport. The transports use the same D/P artifact and each case's existing backend list. The legacy split:true subset does not limit this main acceptance matrix. + With `android_smoke`, an APK matrix keeps every case on inproc and only the named + cases on monolith and spawn. Nested APK case/backend metadata is deliberately left untouched. THE SPAWN LANE IS APK-ONLY, and that is a statement about where the evidence is missing @@ -423,12 +455,19 @@ def github_transport_matrix(cases, *, apk=False): build = github_apk_matrix if apk else github_test_matrix if not cases: raise ValueError("transport matrix needs at least one trace case") + reduced = cases + if android_smoke is not None: + if not apk: + raise ValueError("only the APK transport matrix has reduced lanes") + reduced = [case for case in cases if case["name"] in android_smoke] + if not reduced: + raise ValueError("the reduced Android lanes are empty: no case is in android_smoke") lanes = [ - (cases, "monolith-control", "monolith"), + (reduced, "monolith-control", "monolith"), (cases, "inproc-acceptance", "inproc"), ] if apk: - lanes.append((cases, "spawn-acceptance", "spawn")) + lanes.append((reduced, "spawn-acceptance", "spawn")) return {"include": [ {**row, "lane": lane, "transport": transport} for selected, lane, transport in lanes @@ -481,6 +520,29 @@ def self_test_transport_matrices(): print(f"transport matrix {'APK' if apk else 'Linux'}: {len(controls)} unchanged controls, " f"{len(acceptance)} inproc acceptance entries, {len(spawned)} spawn acceptance " f"entries; exact metadata and negative controls OK") + # The reduced Android lanes are the full matrix less the monolith and spawn rows of the + # cases outside android_smoke: nothing else is dropped, changed or added. + smoke = android_smoke_names() + assert smoke, "android_smoke is empty, so the reduced monolith and spawn lanes would be too" + full = github_transport_matrix(cases, apk=True)["include"] + reduced = github_transport_matrix(cases, apk=True, android_smoke=smoke)["include"] + kept = [row for row in full if row["transport"] == "inproc" or row["case"]["name"] in smoke] + assert reduced == kept, "the reduced Android lanes are not the full matrix filtered" + for transport in ("monolith", "spawn"): + names = {row["case"]["name"] for row in reduced if row["transport"] == transport} + assert names == set(smoke), f"the reduced {transport} lane is not the smoke cases: {names}" + full_inproc = [row for row in full if row["transport"] == "inproc"] + assert [row for row in reduced if row["transport"] == "inproc"] == full_inproc, \ + "the inproc lane was reduced" + for bad in ({"apk": False, "android_smoke": smoke}, + {"apk": True, "android_smoke": ["no-such-case"]}): + try: + github_transport_matrix(cases, **bad) + except ValueError: + continue + raise AssertionError(f"github_transport_matrix accepted {bad}") + print(f"transport matrix APK reduced: {len(reduced)} of {len(full)} rows; inproc keeps all " + f"{len(full_inproc)}, monolith and spawn keep the {len(smoke)} smoke cases") def self_test_split_matrix(cases=None): @@ -690,7 +752,8 @@ def shard_matrix(rows, hints, *, default_kind=HINT_KIND_MONOLITH, shards=None, s return {"include": out} -def per_case_matrix(format_name, cases, *, transports=SPLIT_TRANSPORTS, keep_inproc=()): +def per_case_matrix(format_name, cases, *, transports=SPLIT_TRANSPORTS, keep_inproc=(), + android_smoke=None): """The per-case matrix a shard format regroups, and the hint kind of its transport-less rows.""" if format_name == "github-test-shard-matrix": return github_test_matrix(cases), HINT_KIND_MONOLITH @@ -703,7 +766,8 @@ def per_case_matrix(format_name, cases, *, transports=SPLIT_TRANSPORTS, keep_inp if format_name == "github-apk-shard-matrix": return github_apk_matrix(cases), HINT_KIND_MONOLITH if format_name == "github-apk-transport-shard-matrix": - return github_transport_matrix(cases, apk=True), HINT_KIND_MONOLITH + return (github_transport_matrix(cases, apk=True, android_smoke=android_smoke), + HINT_KIND_MONOLITH) raise ValueError(f"not a shard format: {format_name}") @@ -721,10 +785,11 @@ ANDROID_SHARD_FORMATS = ( def github_shard_matrix(format_name, cases, *, shards=DEFAULT_SHARDS_PER_CELL, shard_seconds=DEFAULT_ANDROID_SHARD_SECONDS, - transports=SPLIT_TRANSPORTS, keep_inproc=(), hints_path=TRACE_DURATION_HINTS_JSON): + transports=SPLIT_TRANSPORTS, keep_inproc=(), hints_path=TRACE_DURATION_HINTS_JSON, + android_smoke=None): """A shard format: the Linux ones lay out by shards per cell, the Android ones by seconds.""" matrix, default_kind = per_case_matrix(format_name, cases, transports=transports, - keep_inproc=keep_inproc) + keep_inproc=keep_inproc, android_smoke=android_smoke) if format_name in ANDROID_SHARD_FORMATS: return shard_matrix(matrix["include"], load_duration_hints("android", hints_path), default_kind=default_kind, shard_seconds=shard_seconds) @@ -740,6 +805,8 @@ def self_test_shard_matrices(): layouts += [("github-split-shard-matrix", narrowed, {"shards": count}) for count in (1, 4, 100)] layouts += [(name, {}, {"shard_seconds": seconds}) for name in ANDROID_SHARD_FORMATS for seconds in (60, DEFAULT_ANDROID_SHARD_SECONDS, 10 ** 9)] + layouts += [("github-apk-transport-shard-matrix", {"android_smoke": android_smoke_names()}, + {"shard_seconds": seconds}) for seconds in (60, DEFAULT_ANDROID_SHARD_SECONDS, 10 ** 9)] for name, selection, layout in layouts: matrix, _ = per_case_matrix(name, cases, **selection) rows = matrix["include"] @@ -762,7 +829,9 @@ def self_test_shard_matrices(): if layout.get("shards") == 1 or layout.get("shard_seconds") == 10 ** 9: assert len(built) == len(cells), f"{name} {layout}: expected one shard per cell" seconds = [shard["seconds"] for shard in built] - print(f"shard matrix {name} {selection or ''} {layout}: {len(rows)} rows in {len(built)} " + shown = {key: (f"{len(value)} smoke cases" if key == "android_smoke" else value) + for key, value in selection.items()} + print(f"shard matrix {name} {shown or ''} {layout}: {len(rows)} rows in {len(built)} " f"shards over {len(cells)} cells, {min(seconds)}-{max(seconds)} s hinted per shard") # The check itself must be able to fail: a dropped, a repeated and a moved case. if len(built) >= 2 and any(len(shard["names"]) >= 2 for shard in built): @@ -886,12 +955,16 @@ def parse_args(): help="Linux shard formats: shards per (lane, transport, backend) cell") parser.add_argument("--shard-seconds", type=int, default=DEFAULT_ANDROID_SHARD_SECONDS, help="Android shard formats: hinted replay seconds one shard aims for") + parser.add_argument("--android-lanes", choices=ANDROID_LANES, default="full", + help="APK transport formats: inproc-full keeps every case on inproc and " + "the manifest's android_smoke cases on monolith and spawn") parser.add_argument("--duration-hints", default=str(TRACE_DURATION_HINTS_JSON), help="shard formats: the JSON of replay seconds the packing weighs cases by") parser.add_argument( "--format", choices=( "names", + "android-smoke-names", "github-test-matrix", "github-test-transport-matrix", "github-verify-matrix", @@ -924,8 +997,11 @@ def main(): cases = load_trace_cases() if args.ci: cases = ci_trace_cases(cases) + android_smoke = android_smoke_names() if args.android_lanes == "inproc-full" else None if args.format == "names": print(json.dumps([case["name"] for case in cases], separators=(",", ":"))) + elif args.format == "android-smoke-names": + print(",".join(android_smoke_names())) elif args.format == "github-test-matrix": print(json.dumps(github_test_matrix(cases), separators=(",", ":"))) elif args.format == "github-test-transport-matrix": @@ -941,14 +1017,16 @@ def main(): elif args.format == "github-apk-matrix": print(json.dumps(github_apk_matrix(cases), separators=(",", ":"))) elif args.format == "github-apk-transport-matrix": - print(json.dumps(github_transport_matrix(cases, apk=True), separators=(",", ":"))) + print(json.dumps(github_transport_matrix(cases, apk=True, android_smoke=android_smoke), + separators=(",", ":"))) elif args.format in LINUX_SHARD_FORMATS + ANDROID_SHARD_FORMATS: transports = tuple(t for t in args.transports.split(",") if t) keep = tuple(c for c in args.keep_inproc.split(",") if c) print(json.dumps( github_shard_matrix(args.format, cases, shards=args.shards, shard_seconds=args.shard_seconds, transports=transports, - keep_inproc=keep, hints_path=args.duration_hints), + keep_inproc=keep, hints_path=args.duration_hints, + android_smoke=android_smoke), separators=(",", ":"))) elif args.format == "fixture-manifest": # One "" line per fixture file, for the fixture bundle script.