[CI, Tools] (...): replay trace cases in shards from one fixture cache

This commit is contained in:
2026-11-15 11:47:18 -04:00
parent 222f6209a0
commit 697bcc6d45
20 changed files with 4176 additions and 884 deletions
+332
View File
@@ -518,6 +518,311 @@ def self_test_split_matrix(cases=None):
f"kept); all {len(dropped)} dropped inproc rows are inproc-acceptance rows")
# --- Shards: several cases of one cell replayed by one CI job ---------------------------------
#
# A cell is everything a per-case matrix row carries except its case: the backend, and the lane
# and transport where the matrix has them. A shard row keeps the cell's keys unchanged and adds
# the cases it replays, so the job that runs it reads the same fields a per-case leg reads.
# Every shard matrix is built from the per-case matrix it replaces and is refused unless its
# rows, taken apart again, are exactly that matrix.
TRACE_DURATION_HINTS_JSON = Path(__file__).with_name("trace_duration_hints.json")
# Shards per cell on Linux, the one tunable of that layout. A cell with little to replay gets
# fewer: a shard costs about half a minute of setup, so it is not split below
# MIN_SHARD_SECONDS of replay.
DEFAULT_SHARDS_PER_CELL = 4
MIN_SHARD_SECONDS = 90
# Replay seconds one emulator shard aims for, the one tunable of the Android layout. A cell is
# split into as many shards as that takes.
DEFAULT_ANDROID_SHARD_SECONDS = 720
# The hint a row without a transport uses. The verify lane has timings of its own.
HINT_KIND_MONOLITH = "monolith"
HINT_KIND_VERIFY = "verify"
# A shard's case names reach a shell command line, so the characters are restricted.
SHARD_SAFE_NAME = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-")
def load_duration_hints(platform, path=TRACE_DURATION_HINTS_JSON):
"""{case: {backend: {kind: seconds}}} for "linux" or "android"; empty when there is none."""
path = Path(path)
if not path.is_file():
return {}
with path.open("r", encoding="utf-8") as file:
table = json.load(file).get(platform, {}).get("seconds", {})
for case, backends in table.items():
for backend, kinds in backends.items():
for kind, seconds in kinds.items():
if isinstance(seconds, bool) or not isinstance(seconds, (int, float)) or seconds <= 0:
raise ValueError(f"duration hint for {case} {backend} {kind} must be a positive "
f"number: {seconds!r}")
return table
def median(values):
ordered = sorted(values)
if not ordered:
return None
middle = len(ordered) // 2
if len(ordered) % 2:
return ordered[middle]
return (ordered[middle - 1] + ordered[middle]) / 2
def hint_seconds(hints, case, backend, kind):
"""The hinted replay time of one row, or the median of its backend and kind, or of everything."""
try:
return hints[case][backend][kind]
except KeyError:
pass
same_cell = [kinds[kind] for backends in hints.values()
for name, kinds in backends.items() if name == backend and kind in kinds]
everything = [seconds for backends in hints.values() for kinds in backends.values()
for seconds in kinds.values()]
return median(same_cell) or median(everything) or 30
def row_case_name(row):
case = row["case"]
return case["name"] if isinstance(case, dict) else case
def row_backend_name(row):
backend = row["backend"]
return backend["name"] if isinstance(backend, dict) else backend
def row_cell(row):
return {key: value for key, value in row.items() if key != "case"}
def canonical(value):
return json.dumps(value, sort_keys=True, separators=(",", ":"))
def shard_rows_as_cases(shard_rows):
"""Take shard rows apart into the per-case rows they stand for."""
added = ("shard", "shards", "names", "cases", "seconds")
rows = []
for shard in shard_rows:
cell = {key: value for key, value in shard.items() if key not in added}
for case in shard.get("cases", shard["names"]):
rows.append({**cell, "case": case})
return rows
def check_shard_union(rows, shard_rows):
"""Refuse a shard matrix that is not exactly the per-case matrix `rows`, regrouped."""
expected = sorted(canonical(row) for row in rows)
actual = sorted(canonical(row) for row in shard_rows_as_cases(shard_rows))
if expected != actual:
missing = sorted(set(expected) - set(actual))
extra = sorted(set(actual) - set(expected))
raise ValueError(
f"the shard matrix is not the per-case matrix regrouped: {len(expected)} per-case rows, "
f"{len(actual)} rows in the shards; missing {missing[:3]}, unexpected {extra[:3]}")
if len(set(expected)) != len(expected):
raise ValueError("the per-case matrix repeats a row, so a shard would replay it twice")
cells = {}
for shard in shard_rows:
if not shard["names"]:
raise ValueError(f"empty shard: {shard}")
if "cases" in shard and [case["name"] for case in shard["cases"]] != shard["names"]:
raise ValueError(f"a shard's names and cases disagree: {shard['names']}")
cell = canonical({key: value for key, value in shard.items()
if key not in ("shard", "shards", "names", "cases", "seconds")})
cells.setdefault(cell, []).append((shard["shard"], shard["shards"]))
for cell, numbers in cells.items():
count = len(numbers)
if sorted(numbers) != [(index, count) for index in range(1, count + 1)]:
raise ValueError(f"shards of {cell} are not numbered 1..{count} of {count}: {numbers}")
def shard_matrix(rows, hints, *, default_kind=HINT_KIND_MONOLITH, shards=None, shard_seconds=None):
"""Regroup a per-case matrix into shards of one cell each.
Exactly one of `shards` (shards per cell, capped so that none holds less than
MIN_SHARD_SECONDS of hinted replay) and `shard_seconds` (hinted replay per shard) sets the
layout. Cases go longest first into the shard with the least hinted time so far, which is
deterministic and keeps the shards of a cell even; inside a shard they run in matrix order.
"""
if (shards is None) == (shard_seconds is None):
raise ValueError("give exactly one of shards and shard_seconds")
if shards is not None and shards < 1:
raise ValueError(f"shards per cell must be at least 1: {shards}")
if shard_seconds is not None and shard_seconds < 1:
raise ValueError(f"seconds per shard must be at least 1: {shard_seconds}")
cells = {}
for index, row in enumerate(rows):
name = row_case_name(row)
if not name or not set(name) <= SHARD_SAFE_NAME:
raise ValueError(f"case name {name!r} cannot be passed on a command line; allowed "
f"characters are letters, digits, '.', '_' and '-'")
seconds = hint_seconds(hints, name, row_backend_name(row),
row.get("transport", default_kind))
cells.setdefault(canonical(row_cell(row)), []).append((index, row, seconds))
out = []
for members in cells.values():
total = sum(seconds for _, _, seconds in members)
if shards is not None:
count = min(shards, int(total // MIN_SHARD_SECONDS))
else:
count = -(-int(total) // shard_seconds)
count = max(1, min(count, len(members)))
bins = [[] for _ in range(count)]
loads = [0] * count
for member in sorted(members, key=lambda item: (-item[2], item[0])):
target = loads.index(min(loads))
bins[target].append(member)
loads[target] += member[2]
for number, members_of_shard in enumerate(bins, start=1):
members_of_shard.sort(key=lambda item: item[0])
shard = {**row_cell(members_of_shard[0][1]), "shard": number, "shards": count,
"names": [row_case_name(row) for _, row, _ in members_of_shard]}
if isinstance(members_of_shard[0][1]["case"], dict):
shard["cases"] = [row["case"] for _, row, _ in members_of_shard]
shard["seconds"] = round(sum(seconds for _, _, seconds in members_of_shard))
out.append(shard)
check_shard_union(rows, out)
return {"include": out}
def per_case_matrix(format_name, cases, *, transports=SPLIT_TRANSPORTS, keep_inproc=()):
"""The per-case matrix a shard format regroups, and the hint kind of its transport-less rows."""
if format_name == "github-test-shard-matrix":
return github_test_matrix(cases), HINT_KIND_MONOLITH
if format_name == "github-test-transport-shard-matrix":
return github_transport_matrix(cases), HINT_KIND_MONOLITH
if format_name == "github-verify-shard-matrix":
return github_verify_matrix(cases), HINT_KIND_VERIFY
if format_name == "github-split-shard-matrix":
return github_split_matrix(cases, transports, keep_inproc), HINT_KIND_MONOLITH
if format_name == "github-apk-shard-matrix":
return github_apk_matrix(cases), HINT_KIND_MONOLITH
if format_name == "github-apk-transport-shard-matrix":
return github_transport_matrix(cases, apk=True), HINT_KIND_MONOLITH
raise ValueError(f"not a shard format: {format_name}")
LINUX_SHARD_FORMATS = (
"github-test-shard-matrix",
"github-test-transport-shard-matrix",
"github-verify-shard-matrix",
"github-split-shard-matrix",
)
ANDROID_SHARD_FORMATS = (
"github-apk-shard-matrix",
"github-apk-transport-shard-matrix",
)
def github_shard_matrix(format_name, cases, *, shards=DEFAULT_SHARDS_PER_CELL,
shard_seconds=DEFAULT_ANDROID_SHARD_SECONDS,
transports=SPLIT_TRANSPORTS, keep_inproc=(), hints_path=TRACE_DURATION_HINTS_JSON):
"""A shard format: the Linux ones lay out by shards per cell, the Android ones by seconds."""
matrix, default_kind = per_case_matrix(format_name, cases, transports=transports,
keep_inproc=keep_inproc)
if format_name in ANDROID_SHARD_FORMATS:
return shard_matrix(matrix["include"], load_duration_hints("android", hints_path),
default_kind=default_kind, shard_seconds=shard_seconds)
return shard_matrix(matrix["include"], load_duration_hints("linux", hints_path),
default_kind=default_kind, shards=shards)
def self_test_shard_matrices():
"""Every shard format replays exactly the rows of the per-case matrix it replaces."""
cases = ci_trace_cases(load_trace_cases())
narrowed = {"transports": ("spawn",), "keep_inproc": ("OpenRA",)}
layouts = [(name, {}, {"shards": count}) for name in LINUX_SHARD_FORMATS for count in (1, 4, 100)]
layouts += [("github-split-shard-matrix", narrowed, {"shards": count}) for count in (1, 4, 100)]
layouts += [(name, {}, {"shard_seconds": seconds}) for name in ANDROID_SHARD_FORMATS
for seconds in (60, DEFAULT_ANDROID_SHARD_SECONDS, 10 ** 9)]
for name, selection, layout in layouts:
matrix, _ = per_case_matrix(name, cases, **selection)
rows = matrix["include"]
built = github_shard_matrix(name, cases, **selection, **layout)["include"]
again = github_shard_matrix(name, cases, **selection, **layout)["include"]
assert built == again, f"{name} {layout}: the layout is not deterministic"
# Compared here as well as inside shard_matrix, with nothing shared but the JSON form.
replayed = sorted(
canonical({**{key: value for key, value in shard.items()
if key not in ("shard", "shards", "names", "cases", "seconds")},
"case": case})
for shard in built for case in shard.get("cases", shard["names"]))
assert replayed == sorted(canonical(row) for row in rows), \
f"{name} {layout}: the shards do not replay the per-case matrix"
assert sum(len(shard["names"]) for shard in built) == len(rows)
cells = {canonical(row_cell(row)) for row in rows}
assert len({canonical({key: value for key, value in shard.items()
if key not in ("shard", "shards", "names", "cases", "seconds")})
for shard in built}) == len(cells), f"{name} {layout}: a shard spans two cells"
if layout.get("shards") == 1 or layout.get("shard_seconds") == 10 ** 9:
assert len(built) == len(cells), f"{name} {layout}: expected one shard per cell"
seconds = [shard["seconds"] for shard in built]
print(f"shard matrix {name} {selection or ''} {layout}: {len(rows)} rows in {len(built)} "
f"shards over {len(cells)} cells, {min(seconds)}-{max(seconds)} s hinted per shard")
# The check itself must be able to fail: a dropped, a repeated and a moved case.
if len(built) >= 2 and any(len(shard["names"]) >= 2 for shard in built):
index = next(i for i, shard in enumerate(built) if len(shard["names"]) >= 2)
for label, damage in (("dropped", lambda names: names[1:]),
("repeated", lambda names: names + names[:1])):
broken = [dict(shard) for shard in built]
broken[index]["names"] = damage(list(broken[index]["names"]))
if "cases" in broken[index]:
broken[index]["cases"] = damage(list(broken[index]["cases"]))
try:
check_shard_union(rows, broken)
except ValueError:
pass
else:
raise AssertionError(f"{name} {layout}: a {label} case passed the union check")
other = next((i for i, shard in enumerate(built)
if row_backend_name(shard) != row_backend_name(built[index])), None)
if other is not None:
broken = [dict(shard) for shard in built]
for key in ("names", "cases"):
if key in broken[index]:
moved = list(broken[index][key])
broken[other][key] = list(broken[other][key]) + moved[:1]
broken[index][key] = moved[1:]
try:
check_shard_union(rows, broken)
except ValueError:
pass
else:
raise AssertionError(f"{name} {layout}: a case moved to another backend's "
f"shard passed the union check")
# Hints only weigh the packing. With none at all, and with a case no hint names, the rows
# replayed are the same.
rows = github_transport_matrix(cases)["include"]
for hints in ({}, {"no-such-case": {"DirectGLES": {"monolith": 5}}}):
check_shard_union(rows, shard_matrix(rows, hints, shards=4)["include"])
assert hint_seconds({}, "x", "DirectGLES", "monolith") == 30
assert hint_seconds({"a": {"DirectGLES": {"monolith": 10}}, "b": {"DirectGLES": {"monolith": 20}},
"c": {"DirectVulkan": {"monolith": 90}}}, "x", "DirectGLES", "monolith") == 15
for bad in ({"shards": 0}, {"shard_seconds": 0}, {}, {"shards": 4, "shard_seconds": 60}):
try:
shard_matrix(rows, {}, **bad)
except ValueError:
continue
raise AssertionError(f"shard_matrix accepted {bad}")
try:
shard_matrix([{"backend": "DirectGLES", "case": "bad name; rm"}], {}, shards=1)
except ValueError:
pass
else:
raise AssertionError("a case name with shell characters was accepted")
# The verify lane's corrupted-snapshot control replays OpenRA on DirectGLES, so that row has
# to be in the verify matrix for the control to have a shard to run in.
verify_rows = github_verify_matrix(cases)["include"]
assert {"backend": "DirectGLES", "case": "OpenRA"} in verify_rows, \
"the verify matrix has no OpenRA DirectGLES row, so its negative control would not run"
print("shard matrices: union, numbering, determinism and negative controls OK")
def cmake_quote(value):
return '"' + str(value).replace("\\", "/").replace('"', '\\"') + '"'
@@ -570,11 +875,19 @@ def parse_args():
parser.add_argument("--self-test-transport-matrices", action="store_true")
parser.add_argument("--self-test-split-matrix", action="store_true",
help="prove the spawn-only split matrix drops only inproc-acceptance rows")
parser.add_argument("--self-test-shard-matrices", action="store_true",
help="prove every shard matrix replays exactly its per-case matrix")
parser.add_argument("--fixture-root", default="tools/trace_replay/fixtures")
parser.add_argument("--transports", default=",".join(SPLIT_TRANSPORTS),
help="github-split-matrix: the comma-separated split transports to emit")
parser.add_argument("--keep-inproc", default="",
help="github-split-matrix: comma-separated split cases whose inproc rows stay")
parser.add_argument("--shards", type=int, default=DEFAULT_SHARDS_PER_CELL,
help="Linux shard formats: shards per (lane, transport, backend) cell")
parser.add_argument("--shard-seconds", type=int, default=DEFAULT_ANDROID_SHARD_SECONDS,
help="Android shard formats: hinted replay seconds one shard aims for")
parser.add_argument("--duration-hints", default=str(TRACE_DURATION_HINTS_JSON),
help="shard formats: the JSON of replay seconds the packing weighs cases by")
parser.add_argument(
"--format",
choices=(
@@ -586,7 +899,10 @@ def parse_args():
"github-apk",
"github-apk-matrix",
"github-apk-transport-matrix",
*LINUX_SHARD_FORMATS,
*ANDROID_SHARD_FORMATS,
"fixture-files",
"fixture-manifest",
"cmake",
),
default="names",
@@ -602,6 +918,9 @@ def main():
if args.self_test_split_matrix:
self_test_split_matrix()
return 0
if args.self_test_shard_matrices:
self_test_shard_matrices()
return 0
cases = load_trace_cases()
if args.ci:
cases = ci_trace_cases(cases)
@@ -623,6 +942,19 @@ def main():
print(json.dumps(github_apk_matrix(cases), separators=(",", ":")))
elif args.format == "github-apk-transport-matrix":
print(json.dumps(github_transport_matrix(cases, apk=True), separators=(",", ":")))
elif args.format in LINUX_SHARD_FORMATS + ANDROID_SHARD_FORMATS:
transports = tuple(t for t in args.transports.split(",") if t)
keep = tuple(c for c in args.keep_inproc.split(",") if c)
print(json.dumps(
github_shard_matrix(args.format, cases, shards=args.shards,
shard_seconds=args.shard_seconds, transports=transports,
keep_inproc=keep, hints_path=args.duration_hints),
separators=(",", ":")))
elif args.format == "fixture-manifest":
# One "<case><TAB><file>" line per fixture file, for the fixture bundle script.
for case in cases:
for file in fixture_files(case, args.fixture_root):
print(f"{case['name']}\t{file}")
elif args.format == "fixture-files":
if not args.case_name:
print("--case is required for --format fixture-files", file=sys.stderr)