#!/usr/bin/env python3 # MobileGL - scripts/ci/retrace_shard.py # Copyright (c) 2025-2026 MobileGL-Dev # Licensed under the GNU Lesser General Public License v3.0: # https://www.gnu.org/licenses/gpl-3.0.txt # https://www.gnu.org/licenses/lgpl-3.0.txt # SPDX-License-Identifier: LGPL-3.0-only # End of Source File Header """Replay the cases of one retrace shard, one after another, and give each its own verdict. A shard is some of the cases of one (lane, transport, backend) cell, replayed by one CI job (tools/trace_replay/trace_cases.py, shard_matrix). Each case runs through its own ctest entry with the environment and the checks of its lane (case_plan), in a process group and a driver cache directory of its own, so no case sees what another left behind. A failing case does not stop the shard. The shard fails when a case failed, or when the verdicts it recorded are not exactly the cases it was given. For every case it prints a log group, appends a row to the step summary and records a verdict in /results/shard-results.json; a failed case also gets an error annotation. The replay output the lane uploads is moved to /results//, the JUnit files are in /junit/, and the extracted trace is deleted. Usage: retrace_shard.py run --profile {retrace,split,verify} --backend B [--transport T] [--lane L] --build-dir DIR --result-dir DIR [--names-env VAR | --names JSON] [--split-reuse] [--server-path PATH] [--shard N/M] retrace_shard.py --self-test """ import argparse import json import os import re import shutil import signal import subprocess import sys import tempfile import time import xml.etree.ElementTree as ET from pathlib import Path SCRIPTS = Path(__file__).resolve().parent ITERATIONRP_CASE = "minecraft-1.21.4-fabric-iris-iterationrp-in-world" BLENDED_DEPTH_WRITE_CASE = "improved-transparency-minecraft-26.3" WIRE_EVIDENCE_CASE = "OpenRA" CONTROL_CASE = "OpenRA" CONTROL_BACKEND = "DirectGLES" CONTROL_NAME = "OpenRA (corrupted-snapshot control)" # The comparator runs the long cases several times slower than a plain replay, and a spawned # server can take as long, so the verify and split lanes raise ctest's 1500 s default. LONG_TIMEOUT = ("--timeout", "10800") SPLIT_REUSE_EXPECT_IPC = "strict=0 role-split-state=1 run-ahead=1" SPLIT_REUSE_EXPECT_IPC_REASON = ( "retrace-split dropped its inproc rows because the inproc-acceptance retrace rows replay the " "same library with strict=1 role-split-state=1 run-ahead=1, which differs from this leg's " "defaults only in strict while these hold. A default changed: restore the inproc rows (the " "trace-cases job's split-spawn-shards without --transports spawn) or update the expectation.") # GitHub shows about ten error annotations of a step. The first failures are annotated as they # happen, so a shard killed later still shows them; the rest wait for the roll-up that names # every failed case. IMMEDIATE_ANNOTATIONS = 8 def test_regex(case, backend): return rf"^MobileGLTraceReplay\.{case}\.{backend}$" def test_name(case, backend): return f"MobileGLTraceReplay.{case}.{backend}" def case_plan(profile, transport, backend, case, *, split_reuse=False, server_path=""): """What one case of a lane runs with: the environment it adds, ctest's arguments, its checks. retrace the monolith-control and inproc-acceptance lanes of the retrace job. The job itself exports MOBILEGL_TRANSPORT and the three IPC knobs for every case alike. split the retrace-split job, inproc and spawn. verify the retrace-verify job; its library has no transport. """ if profile not in ("retrace", "split", "verify"): raise ValueError(f"unknown profile: {profile}") env = {} if backend == "DirectVulkan": env["MOBILEGL_MAGMA_R11G11B10F_FALLBACK"] = "1" # The blended depth-write quirk enables itself only on Qualcomm, which no runner has, # so the case it exists for forces it on; ForceOn bypasses only the vendor gate. if case == BLENDED_DEPTH_WRITE_CASE: env["MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE"] = "1" # The iterationRP repairs default to device detection and the software driver is on no # list, so the one case that needs them forces them. The verify lane does not set it. if case == ITERATIONRP_CASE and profile != "verify": env["MOBILEGL_ITRP_FIX"] = "ForceOn" ctest = ["-V", "--no-tests=error"] evidence = None census = False if profile == "retrace": if transport == "inproc": # These rows stand in for retrace-split's dropped inproc rows, so they prove the # configuration they ran with from the client's own log. env["MGTRACE_EXPECT_IPC"] = "strict=1 role-split-state=1 run-ahead=1" census = True if transport != "monolith": evidence = [] elif profile == "split": if transport not in ("inproc", "spawn"): raise ValueError(f"the split profile runs inproc or spawn, not {transport!r}") if not server_path.startswith("/"): # The replay's working directory is not the workspace and the launcher resolves the # image relative to nothing. The inproc arm reads it only to decide it needs none. raise ValueError("the split profile needs an absolute --server-path") env["MOBILEGL_TRANSPORT"] = transport env["MOBILEGL_IPC_SERVER_PATH"] = server_path evidence = [] if case == WIRE_EVIDENCE_CASE: # This case also counts the wire's draw records and the server's draws. That turns # on the library's per-frame counters, so the others keep the plain configuration. env["MGTRACE_WIRE_EVIDENCE"] = "1" evidence = ["--require-wire"] if split_reuse: env["MGTRACE_EXPECT_IPC"] = SPLIT_REUSE_EXPECT_IPC env["MGTRACE_EXPECT_IPC_REASON"] = SPLIT_REUSE_EXPECT_IPC_REASON ctest += LONG_TIMEOUT census = True else: env["MOBILEGL_PIPE_VERIFY"] = "1" ctest += LONG_TIMEOUT ctest += ["-R", test_regex(case, backend)] return {"env": env, "ctest": ctest, "evidence": evidence, "census": census, "control": has_control(profile, backend, case)} def control_plan(): """The verify lane's negative control: the same replay with one snapshot field corrupted. It must fail, and for the comparator's reason. It adds only these two variables and raises no core limit, since the failure is the expected outcome. """ return {"env": {"MOBILEGL_PIPE_VERIFY": "1", "MOBILEGL_PIPE_VERIFY_CORRUPT": "GetRenderStateParameters"}, "ctest": ["-V", "--no-tests=error", *LONG_TIMEOUT, "-R", test_regex(CONTROL_CASE, CONTROL_BACKEND)]} def has_control(profile, backend, case): return profile == "verify" and case == CONTROL_CASE and backend == CONTROL_BACKEND def expected_results(profile, backend, names): """The verdicts a shard owes, in order.""" out = [] for name in names: out.append(name) if has_control(profile, backend, name): out.append(CONTROL_NAME) return out def step_ignores_sigpipe(): """Whether the process that started this script, the step's shell, ignores SIGPIPE.""" try: with open(f"/proc/{os.getppid()}/status", encoding="ascii") as status: for line in status: if line.startswith("SigIgn:"): return bool(int(line.split()[1], 16) & (1 << (signal.SIGPIPE - 1))) except (OSError, ValueError, IndexError): pass return False def raise_core_limit(): # What `ulimit -c unlimited` does; the runner's core handler honors each process's limit. import resource resource.setrlimit(resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY)) class Shard: def __init__(self, args, names, base_env): self.args = args self.names = names self.base_env = dict(base_env) self.build_dir = Path(args.build_dir).resolve() self.result_dir = Path(args.result_dir).resolve() self.results_root = self.result_dir / "results" self.junit_dir = self.result_dir / "junit" self.log_dir = self.result_dir / "logs" self.cache_root = self.result_dir / "driver-cache" self.records = [] self.failures = 0 self.child = None self.lane = args.lane or "-" self.transport = args.transport or "-" self.where = f"{self.lane} / {self.transport} / {args.backend}" # --- output ------------------------------------------------------------------------------- def say(self, text=""): print(text, flush=True) def summary(self, text): path = os.environ.get("GITHUB_STEP_SUMMARY") if path: with open(path, "a", encoding="utf-8") as file: file.write(text + "\n") def annotate(self, title, message): escaped = message.replace("%", "%25").replace("\r", "").replace("\n", "%0A") # A property value also escapes the two characters that delimit properties. title = title.replace("%", "%25").replace(":", "%3A").replace(",", "%2C") self.say(f"::error title={title}::{escaped}") def write_results(self): self.results_root.mkdir(parents=True, exist_ok=True) payload = {"profile": self.args.profile, "lane": self.args.lane, "transport": self.args.transport, "backend": self.args.backend, "shard": self.args.shard, "cases": self.names, "results": self.records} target = self.results_root / "shard-results.json" scratch = target.with_suffix(".tmp") scratch.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") scratch.replace(target) def record(self, name, status, seconds, reasons, *, ssim="", cores=()): """One verdict: "passed", "failed", or "skipped" for a control whose case already failed.""" entry = {"case": name, "status": status, "seconds": round(seconds, 1), "ssim": ssim, "reasons": list(reasons), "cores": list(cores), "annotated": False} if status == "failed": self.failures += 1 entry["annotated"] = self.failures <= IMMEDIATE_ANNOTATIONS self.records.append(entry) self.write_results() note = "; ".join(reasons) if cores: note = (note + "; " if note else "") + "core: " + ", ".join(cores) shown = "**failed**" if status == "failed" else status self.summary(f"| {name} | {shown} | {seconds:.0f} | {ssim} | {note.replace('|', '/')[:300]} |") if entry["annotated"]: self.annotate(f"retrace failed: {self.where} / {name}", note or "failed") # --- one process ---------------------------------------------------------------------------- def run_ctest(self, plan, *, junit=None, transcript=None, core_limit=True, cache_home=None): env = dict(self.base_env) env.update(plan["env"]) if cache_home is not None: env["XDG_CACHE_HOME"] = str(cache_home) command = [self.args.ctest, *plan["ctest"]] if junit is not None: command += ["--output-junit", str(junit)] self.say("+ " + " ".join(f"{key}={value!r}" for key, value in sorted(plan["env"].items()))) self.say("+ " + " ".join(command)) options = {"cwd": self.build_dir, "env": env, "start_new_session": True} if core_limit: options["preexec_fn"] = raise_core_limit # Python ignores SIGPIPE itself and by default hands a child the default action. The # replay gets what a step's own child would get instead, so a write to a closed pipe # or socket ends the same way as when the step ran ctest directly. options["restore_signals"] = not step_ignores_sigpipe() sys.stdout.flush() if transcript is None: self.child = subprocess.Popen(command, **options) status = self.child.wait() else: # The transcript is both shown and kept, as `2>&1 | tee` would. self.child = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options) with open(transcript, "wb") as file: for chunk in iter(lambda: self.child.stdout.read1(65536), b""): file.write(chunk) sys.stdout.buffer.write(chunk) sys.stdout.buffer.flush() status = self.child.wait() self.reap() return status def reap(self, grace=5.0): """End whatever the finished replay left running, so it cannot run under the next case. A spawned server exits by itself when its client is gone, so the group gets a moment first. True when something had to be killed. """ child, self.child = self.child, None if child is None: return False deadline = time.monotonic() + grace try: while True: os.killpg(child.pid, 0) if time.monotonic() >= deadline: break time.sleep(0.25) os.killpg(child.pid, signal.SIGKILL) except (ProcessLookupError, PermissionError): return False self.say(f"killed the processes ctest (pid {child.pid}) left running") return True # --- one case ------------------------------------------------------------------------------- def fresh_cache_home(self): """An empty driver cache directory, as on a runner that has replayed nothing yet. The software drivers keep compiled shaders on disk under the user cache directory. Each case gets a new one, so a case never starts from shaders another case compiled. On a hosted runner the default location is emptied as well, in case a process ignores the redirect; a self-hosted machine's own cache is left alone. """ self.cache_root.mkdir(parents=True, exist_ok=True) cache_home = Path(tempfile.mkdtemp(prefix="case-", dir=self.cache_root)) if self.base_env.get("RUNNER_ENVIRONMENT") != "self-hosted": default_home = Path(self.base_env.get("HOME", "/nonexistent")) / ".cache" for stale in default_home.glob("mesa_shader_cache*"): self.say(f"removing {stale} left by an earlier process") shutil.rmtree(stale, ignore_errors=True) return cache_home def drop_cache_home(self, cache_home): entries = sum(1 for path in cache_home.rglob("*") if path.is_file()) self.say(f"driver cache: {entries} file(s) written under {cache_home}, removed") shutil.rmtree(cache_home, ignore_errors=True) def junit_problem(self, junit, name): """Why this JUnit file does not show exactly the one entry that was asked for, or ''.""" if not junit.is_file(): return f"ctest wrote no {junit.name}" try: entries = [case.get("name") for case in ET.parse(junit).getroot().iter("testcase")] except ET.ParseError as error: return f"{junit.name} does not parse: {error}" if entries != [name]: return f"ctest ran {entries or 'no entry'} where exactly {name} was asked for" return "" def check_evidence(self, plan, case): """Run the lane's evidence check; its error lines are folded into the case's annotation.""" evidence = self.build_dir / case / self.args.backend / "output" / "split-evidence.txt" command = [sys.executable, str(SCRIPTS / "retrace_split_evidence.py"), str(evidence), "--transport", self.args.transport, "--case", case, "--backend", self.args.backend, *plan["evidence"]] done = subprocess.run(command, cwd=self.build_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) text = done.stdout.strip() self.say(re.sub(r"^::error::", "split evidence: ", text, flags=re.M)) if done.returncode == 0: return "" return "split evidence: " + re.sub(r"^::error::", "", text, flags=re.M).replace("\n", " ")[:600] def census(self, case): """The refusal census from the library's own logs, recorded and never a verdict. Both roles are read: under spawn the applier's refusals are raised in the server's process. ctest's transcript cannot stand in, since the console sink is compiled out. """ output = self.build_dir / case / self.args.backend / "output" log = output / "mobilegl.client.log" server_log = output / "mobilegl.server.log" evidence = output / "split-evidence.txt" self.say(evidence.read_text(errors="replace").rstrip() if evidence.is_file() else "no split-evidence.txt") if not log.is_file(): self.say(f"no {log} - the replay wrote no library log") return files = [log] + ([server_log] if server_log.is_file() else []) self.say("census over: " + " ".join(str(path) for path in files)) lines = [line for path in files for line in path.read_text(errors="replace").splitlines()] self.say(f"Fatal{{ lines: {sum(1 for line in lines if 'Fatal{' in line)}") counts = {} for line in lines: for match in re.findall(r'Fatal\{[A-Za-z]*, "[^"]*"', line): counts[match] = counts.get(match, 0) + 1 for match, count in sorted(counts.items(), key=lambda item: (-item[1], item[0]))[:20]: self.say(f"{count:7d} {match}") transport_line = next((line for line in log.read_text(errors="replace").splitlines() if "MOBILEGL_TRANSPORT=" in line), None) self.say(transport_line or f"no transport line in {log}") def collect(self, case): """Move what the lane uploads into the result directory and delete the extracted trace.""" source = self.build_dir / case target = self.results_root / case shutil.rmtree(source / self.args.backend / "input", ignore_errors=True) for relative in ("actual-images", f"{self.args.backend}/output"): if (source / relative).is_dir(): (target / relative).parent.mkdir(parents=True, exist_ok=True) shutil.rmtree(target / relative, ignore_errors=True) shutil.move(str(source / relative), str(target / relative)) shutil.rmtree(source, ignore_errors=True) def ssim_of(self, case): try: result = json.loads((self.build_dir / case / self.args.backend / "output" / "result.json") .read_text(errors="replace")) except (OSError, ValueError): return "" value = result.get("ssim") return "" if value is None else str(value) def run_case(self, case): plan = case_plan(self.args.profile, self.args.transport, self.args.backend, case, split_reuse=self.args.split_reuse, server_path=self.args.server_path) self.say(f"::group::{case} ({self.where})") started = time.monotonic() cores_before = set(Path("/tmp").glob("core.*")) junit = self.junit_dir / f"{case}.xml" reasons = [] cache_home = self.fresh_cache_home() try: try: status = self.run_ctest(plan, junit=junit, cache_home=cache_home) except (OSError, subprocess.SubprocessError) as error: # The case still gets its verdict, and the cases after it still run. self.child = None status = None reasons.append(f"ctest could not be run: {error}") if status is not None and status > 0: reasons.append(f"ctest exit {status}") elif status is not None and status < 0: reasons.append(f"ctest ended by signal {-status}") problem = self.junit_problem(junit, test_name(case, self.args.backend)) if problem: reasons.append(problem) if plan["evidence"] is not None: # The transport assertions ran on this case and found their evidence. A failed # replay is already red, and its missing evidence file would only repeat that. if status == 0: problem = self.check_evidence(plan, case) if problem: reasons.append(problem) if plan["census"]: try: self.census(case) except OSError as error: self.say(f"refusal census skipped: {error}") finally: self.drop_cache_home(cache_home) ssim = self.ssim_of(case) self.collect(case) cores = sorted(path.name for path in set(Path("/tmp").glob("core.*")) - cores_before) seconds = time.monotonic() - started self.say(f"{case}: {'passed' if not reasons else 'FAILED: ' + '; '.join(reasons)} in {seconds:.0f} s") self.say("::endgroup::") self.record(case, "failed" if reasons else "passed", seconds, reasons, ssim=ssim, cores=cores) return not reasons def run_control(self, verified_passed): """With a snapshot field corrupted the same replay must fail, or the lane proves nothing. It replays into the case directory the verified run used. That run's output has already been moved to the result directory, and the corrupted run's output is deleted, so only the verified run is uploaded under the case's name. """ self.say(f"::group::{CONTROL_NAME}") if not verified_passed: self.say("not run: the verified replay it is compared with failed") self.say("::endgroup::") self.record(CONTROL_NAME, "skipped", 0, ["not run: the verified replay failed"]) return started = time.monotonic() reasons = [] transcript = self.log_dir / "verify-control.txt" cache_home = self.fresh_cache_home() status = None try: status = self.run_ctest(control_plan(), transcript=transcript, core_limit=False, cache_home=cache_home) except (OSError, subprocess.SubprocessError) as error: self.child = None reasons.append(f"ctest could not be run: {error}") finally: self.drop_cache_home(cache_home) shutil.rmtree(self.build_dir / CONTROL_CASE, ignore_errors=True) if status is None: pass elif status == 0: reasons.append( "MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left the OpenRA retrace " "green, so the comparator is not comparing and the verify retrace lane proves nothing") else: # Its red must carry the comparator's divergence on the corrupted field. done = subprocess.run([sys.executable, str(SCRIPTS / "retrace_verify_reason.py"), str(transcript)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) text = re.sub(r"^::error::", "", done.stdout.strip(), flags=re.M) self.say(text) if done.returncode != 0: reasons.append(text.replace("\n", " ")[:600]) else: self.say(f"the corrupted field turned the retrace red, as it must (ctest exit {status})") seconds = time.monotonic() - started self.say("::endgroup::") self.record(CONTROL_NAME, "failed" if reasons else "passed", seconds, reasons) # --- the shard -------------------------------------------------------------------------------- def run(self): for directory in (self.results_root, self.junit_dir, self.log_dir): directory.mkdir(parents=True, exist_ok=True) expected = expected_results(self.args.profile, self.args.backend, self.names) self.say(f"retrace shard {self.args.shard or ''} ({self.where}): {len(self.names)} case(s), " f"{len(expected)} verdict(s) owed") self.summary(f"### retrace shard {self.args.shard or ''} ({self.where})\n\n" "| case | result | seconds | SSIM | note |\n|---|---|---|---|---|") self.write_results() for case in self.names: passed = self.run_case(case) if case_plan(self.args.profile, self.args.transport, self.args.backend, case, split_reuse=self.args.split_reuse, server_path=self.args.server_path)["control"]: self.run_control(passed) return self.finish(expected) def finish(self, expected): problems = verdict_problems(expected, self.records) # Counted again from what ctest wrote, apart from the loop's own bookkeeping. ran = sum(1 for case in self.names if not self.junit_problem(self.junit_dir / f"{case}.xml", test_name(case, self.args.backend))) passed_cases = sum(1 for entry in self.records if entry["status"] == "passed" and entry["case"] in self.names) if ran < passed_cases: problems.append(f"{passed_cases} case(s) recorded as passed but only {ran} JUnit file(s) " f"show their entry ran") failed = [entry for entry in self.records if entry["status"] == "failed"] if failed: names = ", ".join(entry["case"] for entry in failed) self.annotate(f"retrace shard {self.args.shard or ''}: {len(failed)} of {len(expected)} failed", f"{self.where}: {names}") for entry in failed: if not entry["annotated"]: self.annotate(f"retrace failed: {self.where} / {entry['case']}", "; ".join(entry["reasons"]) or "failed") for problem in problems: self.annotate(f"retrace shard {self.args.shard or ''}: verdicts do not match the cases", problem) passed = sum(1 for entry in self.records if entry["status"] == "passed") closing = f"{passed} passed, {len(failed)} failed, {len(expected)} owed" if problems: closing += "; " + "; ".join(problems) self.summary(f"\n{closing}\n") self.say(f"retrace shard ({self.where}): {closing}") return 1 if failed or problems else 0 def verdict_problems(expected, records): """Why the recorded verdicts are not exactly the ones owed; empty when they are.""" recorded = [entry["case"] for entry in records] problems = [] if len(recorded) != len(expected): problems.append(f"{len(recorded)} verdict(s) recorded for {len(expected)} owed") if recorded != expected: missing = [name for name in expected if name not in recorded] extra = [name for name in recorded if name not in expected] if missing: problems.append("no verdict for " + ", ".join(missing)) if extra: problems.append("verdict for a case that was not assigned: " + ", ".join(extra)) if not missing and not extra and len(recorded) == len(expected): problems.append("verdicts are out of order or repeated") return problems def run(args): if args.names is not None: names = json.loads(args.names) else: names = json.loads(os.environ.get(args.names_env) or "null") if not isinstance(names, list) or not names or not all(isinstance(name, str) and name for name in names): print(f"::error::retrace shard: no case list (looked in --names and ${args.names_env})") return 2 if len(set(names)) != len(names): print(f"::error::retrace shard: the case list repeats a case: {names}") return 2 if args.profile == "retrace": # The job exports the transport for this lane; the plan and the library must agree on it. exported = os.environ.get("MOBILEGL_TRANSPORT", "") if exported != args.transport: print(f"::error::retrace shard: --transport is {args.transport!r} but the job exports " f"MOBILEGL_TRANSPORT={exported!r}") return 2 try: case_plan(args.profile, args.transport, args.backend, names[0], split_reuse=args.split_reuse, server_path=args.server_path) except ValueError as error: print(f"::error::retrace shard: {error}") return 2 shard = Shard(args, names, os.environ) def stop(signum, _frame): shard.reap(grace=0) shard.say(f"retrace shard: stopped by signal {signum} after {len(shard.records)} verdict(s)") sys.exit(128 + signum) signal.signal(signal.SIGTERM, stop) signal.signal(signal.SIGINT, stop) return shard.run() def self_test(): """The plan of every lane against the values the lanes are defined by, and the verdict check.""" oit, itrp, other = BLENDED_DEPTH_WRITE_CASE, ITERATIONRP_CASE, "minecraft-1.21.4-in-world" server = "/w/build-linux/libMobileGLServer.so" regex = lambda case, backend: ["-R", rf"^MobileGLTraceReplay\.{case}\.{backend}$"] plain, long = ["-V", "--no-tests=error"], ["-V", "--no-tests=error", "--timeout", "10800"] vk = {"MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "1"} expect_strict = {"MGTRACE_EXPECT_IPC": "strict=1 role-split-state=1 run-ahead=1"} reuse = {"MGTRACE_EXPECT_IPC": SPLIT_REUSE_EXPECT_IPC, "MGTRACE_EXPECT_IPC_REASON": SPLIT_REUSE_EXPECT_IPC_REASON} split = lambda transport: {"MOBILEGL_TRANSPORT": transport, "MOBILEGL_IPC_SERVER_PATH": server} table = [ # profile, transport, backend, case, reuse -> env, ctest, evidence, census, control ("retrace", "monolith", "DirectGLES", other, False, {}, plain, None, False, False), ("retrace", "monolith", "DirectVulkan", other, False, vk, plain, None, False, False), ("retrace", "monolith", "DirectVulkan", oit, False, {**vk, "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE": "1"}, plain, None, False, False), ("retrace", "monolith", "DirectGLES", oit, False, {}, plain, None, False, False), ("retrace", "monolith", "DirectVulkan", itrp, False, {**vk, "MOBILEGL_ITRP_FIX": "ForceOn"}, plain, None, False, False), ("retrace", "inproc", "DirectGLES", other, False, expect_strict, plain, [], True, False), ("retrace", "inproc", "DirectVulkan", itrp, False, {**vk, "MOBILEGL_ITRP_FIX": "ForceOn", **expect_strict}, plain, [], True, False), ("retrace", "inproc", "DirectGLES", "OpenRA", False, expect_strict, plain, [], True, False), ("split", "spawn", "DirectGLES", other, True, {**split("spawn"), **reuse}, long, [], True, False), ("split", "spawn", "DirectGLES", other, False, split("spawn"), long, [], True, False), ("split", "spawn", "DirectVulkan", oit, True, {**vk, "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE": "1", **split("spawn"), **reuse}, long, [], True, False), ("split", "spawn", "DirectVulkan", itrp, True, {**vk, "MOBILEGL_ITRP_FIX": "ForceOn", **split("spawn"), **reuse}, long, [], True, False), ("split", "spawn", "DirectGLES", "OpenRA", True, {**split("spawn"), "MGTRACE_WIRE_EVIDENCE": "1", **reuse}, long, ["--require-wire"], True, False), ("split", "inproc", "DirectVulkan", "OpenRA", True, {**vk, **split("inproc"), "MGTRACE_WIRE_EVIDENCE": "1", **reuse}, long, ["--require-wire"], True, False), ("split", "inproc", "DirectGLES", other, False, split("inproc"), long, [], True, False), ("verify", "", "DirectGLES", other, False, {"MOBILEGL_PIPE_VERIFY": "1"}, long, None, False, False), ("verify", "", "DirectGLES", "OpenRA", False, {"MOBILEGL_PIPE_VERIFY": "1"}, long, None, False, True), ("verify", "", "DirectVulkan", "OpenRA", False, {"MOBILEGL_PIPE_VERIFY": "1", **vk}, long, None, False, False), ("verify", "", "DirectVulkan", oit, False, {"MOBILEGL_PIPE_VERIFY": "1", **vk, "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE": "1"}, long, None, False, False), ("verify", "", "DirectVulkan", itrp, False, {"MOBILEGL_PIPE_VERIFY": "1", **vk}, long, None, False, False), ] for profile, transport, backend, case, split_reuse, env, ctest, evidence, census, control in table: plan = case_plan(profile, transport, backend, case, split_reuse=split_reuse, server_path=server) wanted = {"env": env, "ctest": ctest + regex(case, backend), "evidence": evidence, "census": census, "control": control} assert plan == wanted, f"{profile} {transport} {backend} {case}: {plan} != {wanted}" control = control_plan() assert control["env"] == {"MOBILEGL_PIPE_VERIFY": "1", "MOBILEGL_PIPE_VERIFY_CORRUPT": "GetRenderStateParameters"} assert control["ctest"] == long + regex("OpenRA", "DirectGLES") for bad in (("split", "spawn", ""), ("split", "spawn", "relative/server"), ("split", "monolith", server), ("pull", "monolith", server)): try: case_plan(bad[0], bad[1], "DirectGLES", other, server_path=bad[2]) except ValueError: continue raise AssertionError(f"case_plan accepted {bad}") assert expected_results("verify", "DirectGLES", ["a", "OpenRA", "b"]) == ["a", "OpenRA", CONTROL_NAME, "b"] assert expected_results("verify", "DirectVulkan", ["OpenRA"]) == ["OpenRA"] assert expected_results("split", "DirectGLES", ["OpenRA"]) == ["OpenRA"] entry = lambda name: {"case": name, "status": "passed"} assert verdict_problems(["a", "b"], [entry("a"), entry("b")]) == [] assert verdict_problems(["a", "b"], [entry("a")]), "a case without a verdict passed the check" assert verdict_problems(["a", "b"], []), "a shard that ran nothing passed the check" assert verdict_problems(["a"], [entry("a"), entry("a")]), "a repeated verdict passed the check" assert verdict_problems(["a", "b"], [entry("a"), entry("c")]), "an unassigned case passed the check" assert verdict_problems(["a", "b"], [entry("b"), entry("a")]), "out-of-order verdicts passed the check" print(f"retrace_shard: {len(table)} plans, the control plan and the verdict check OK") return 0 def main(argv=None): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("command", nargs="?", choices=("run",)) parser.add_argument("--self-test", action="store_true") parser.add_argument("--profile", choices=("retrace", "split", "verify")) parser.add_argument("--backend") parser.add_argument("--transport", default="") parser.add_argument("--lane", default="") parser.add_argument("--shard", default="", help="N/M, for the report only") parser.add_argument("--build-dir", help="the directory ctest runs in") parser.add_argument("--result-dir", help="where verdicts, replay output and JUnit files go") parser.add_argument("--names", help="the cases, as a JSON list") parser.add_argument("--names-env", default="SHARD_NAMES", help="the variable holding the JSON list") parser.add_argument("--split-reuse", action="store_true", help="split profile: the library is the retrace job's, so prove the IPC defaults") parser.add_argument("--server-path", default="", help="split profile: absolute path of the server image") parser.add_argument("--ctest", default="ctest") args = parser.parse_args(argv) if args.self_test: return self_test() if args.command != "run" or not (args.profile and args.backend and args.build_dir and args.result_dir): parser.error("run needs --profile, --backend, --build-dir and --result-dir") return run(args) if __name__ == "__main__": sys.exit(main())