Files
MobileGL/scripts/ci/spawn_lane_parity.py
T

362 lines
22 KiB
Python

#!/usr/bin/env python3
"""Exit gate 9.2: integration-spawn's case-name set equals integration-split's.
WHY A GATE AND NOT A COMMENT. The two lanes are registered from one macro, so today they agree
by construction - but a macro is one edit away from a second hand-written block, and a spawn lane
covering LESS than the split lane goes green while proving less. Nothing about a smaller green
lane looks wrong. This asserts the sets rather than the counts, because a count can agree while
the membership does not.
ARM PREFIXES ARE STRIPPED BEFORE COMPARING. The lanes deliberately differ in name -
`DirectGLES.Split.X` against `DirectGLES.Spawn.X` - and the sub-lane tails (`F1.`, `Ct.`) are
part of neither the case nor the arm, so both come off.
ONE SANCTIONED ASYMMETRY, LISTED HERE AND NOWHERE ELSE. `integration-split` also labels the
PUSH-MONOLITH arm (the CMakeLists explains why: the label means "the set the disaggregation gate
runs", not "runs inproc"), and that arm's scenario is monolith-only by construction. It cannot
have a spawn counterpart, so it is named as an exception rather than allowed to widen the
tolerance for everything else.
P7 PACKAGE L ADDS THE SAME QUESTION FOR MAGMA, IN TWO TIERS, AND THE NORMALISATION IS
DIFFERENT ON PURPOSE. The Espryt half above strips the sub-lane tails `F1.` and `Ct.` because
they name neither the case nor the arm. The Magma half must NOT: `RunAhead.Credit1.` and
`RunAhead.Credit3.` are two entries of the same case, differing in the credit the lane pins, and
a normaliser that dropped the tail would fold them together and then report an equal-sized set
that had silently lost one. So there the arm - and only the arm, the SECOND dotted segment - is
what comes off.
"""
import argparse
import importlib.util
import json
from pathlib import Path
import re
import subprocess
import sys
# Registered only on the push-monolith arm, which has no spawn counterpart by construction.
MONOLITH_ONLY = {"MonolithAttachmentClearScenario"}
CASE = re.compile(r"DirectGLES\.[A-Za-z0-9]+\.(?:F1\.|Ct\.)?([A-Za-z0-9_]+Scenario\.[A-Za-z0-9_]+)")
# `<Backend>.<Arm>.<everything else>` -> `<Backend>.<everything else>`. The arm is the only
# segment two arms of one lane are allowed to differ in.
ARM = re.compile(r"^(DirectGLES|DirectVulkan)\.(Split|Spawn|Tcp|TcpDevice)\.(.+)$")
TEST_LINE = re.compile(r"^\s*Test\s+#\d+:\s*(\S+)\s*$", re.MULTILINE)
# `ctest -N -L <tcp label>` lists the FIXTURE entries too, because every tcp entry declares
# FIXTURES_REQUIRED mobilegl-tcp and ctest pulls the setup/cleanup in with them. They are the
# supervisor, not cases, and they exist on exactly one arm by construction. Named here so that
# any OTHER unparseable name is still an error rather than a silent drop.
FIXTURE_ENTRIES = {"TcpServer.Start", "TcpServer.Stop"}
# THE MAGMA TIER'S SANCTIONED ASYMMETRIES, LISTED HERE AND NOWHERE ELSE - the same discipline
# MONOLITH_ONLY above gets. Both are inproc-only BY CONSTRUCTION, not by omission:
# MagmaRunAheadScenario arms its hold through ServerLoopInstance().SetBeforeRetireHookForTesting,
# a hook on the CALLING process's server loop. With the server in another
# process the hold never activates and the wait times out.
# MagmaWireCacheScenario SetUp asserts runtime.transportName == "inproc" outright: the lane reads
# the server's own view/attachment identity out of this process.
# A cross-process hold and a cross-process cache peek are server-side knobs neither exists yet;
# until they do, naming the two scenarios here keeps the tolerance from widening for anything else.
# DescriptorPoolGrowthScenario (P11 M2, the `.DescPool.` entry): both cases read the renderer's
# pool census (DirectVulkan::GetDescriptorPoolCensus), an in-process read -
# on spawn and tcp the pools live in the server process and the census
# answers unavailable, so a copy there checks pixels under a name claiming
# the pools. The pixels do run on every arm: the Espryt entries and the
# Magma informational and full-suite tiers.
MAGMA_INPROC_ONLY = ("MagmaRunAheadScenario.", "MagmaWireCacheScenario.", "DescriptorPoolGrowthScenario.")
# P7 wave 2-B2: THE SANCTIONED ASYMMETRY THAT IS NOT INPROC-ONLY - these exist on split AND
# spawn and cannot exist on tcp. Every MGITEST_MAGMA_FORCE_* knob is read BY THE SERVER
# (WireFramebuffer.inc's getenv readers run on the apply thread), and a ctest ENVIRONMENT
# property only reaches the process ctest starts, the CLIENT. Under inproc the server is a
# thread of that process and under spawn ServerSpawn.cpp copies ::environ into the child, so
# both see it; under tcp the server is the LANE-WIDE TcpServer.Start fixture, started once from
# the fixture's own os.environ before any case runs, so nothing a per-test property says can
# reach it. A tcp copy would run the DEFAULT arm under a name claiming otherwise.
#
# MEASURED (a temporary fatal in each of the three readers): all five knob=1 entries died on
# split and on spawn and ALL FIVE PASSED on tcp.
#
# These are TAILS, not scenarios - the normaliser above keeps the sub-lane tail for the Magma
# tier on purpose, so `.MsResolve1.` names the knob entry and `.MsResolve0.` (the default, which
# does keep its tcp entry) is untouched.
#
# P7 wave 2-B3 (ID-P7-34) adds `.StaleSerial.`: MGITEST_MAGMA_FORCE_STALE_BUFFER_SERIAL is read by
# the server too (VkBufferManager.cpp), so the streamed subdata-then-draw red-once has the same
# two arms and the same tcp absence. One list, one mechanism, for every server-side knob.
# `.MsFlip1.` (B2 review round 2) is the shader-resolve knob twin of the knob-free `.MsFlip.`,
# which keeps its tcp entry.
# P7 wave 4 M2: MagmaWireReclaimScenario reads the SERVER's wbuf[] PipeStats gauges off the server's
# private log with the stats channel and MOBILEGL_IPC_WIRE_DEFERRED_MB=8 in the entry's environment -
# server-process environment again, so the same two arms and the same tcp absence. Its three cases
# are three tails (one key each) under the `.Reclaim.` prefix.
# P7 gate 5 (g5-msrbo review round, g5-msprobe): `.MsResolveBug.` / `.MsFlipBug.` set
# MGITEST_MAGMA_DEPTH_RESOLVE_PROBE=bug, which the SERVER's depth/stencil resolve arm choice reads
# in place of running its resolve probe - the knob-free `.MsResolve0.` / `.MsFlip.` keep their tcp
# entries (MsResolve0's split/spawn copies also carry a client-read expectation marker for the
# real probe's verdict, which the tcp copy cannot: same name on all three arms, no exception).
# `.MsResolveElide.` (g5-msprobe critic) sets MGITEST_MAGMA_DEPTH_RESOLVE_PROBE=elide-subject, which
# the same server-side arm choice reads to run the REAL probe with its render-pass resolve left out.
# Codex closeout finding 2 (cf-magma): the `.ImageUnitPrivate.` entries set
# MGITEST_MAGMA_FORCE_PRIVATE_IMAGE_PLACEHOLDER=1, which the SERVER's UniformManager reads to bind a
# unit-private placeholder for an invalid image unit instead of a null descriptor - the
# knob-free `.ImageUnitWindow.` copies of the same two cases keep their tcp entries. Two cases, so two
# tails (one case each), as for `.Reclaim.`.
# P10 B: the `.PrimGenReroute.` entries set MOBILEGL_MAGMA_PRIMGEN_QUERY_REROUTE=1, which the
# SERVER's VulkanRenderer reads to pin the GL_PRIMITIVES_GENERATED reroute on - the knob-free
# `.PrimGen.` copies (the bring-up probe's Auto verdict) keep their tcp entries. Eight cases, eight
# tails.
# P11 M2: the `.SplitRecording64.` entries set MOBILEGL_MAGMA_MAX_DRAWS_PER_COMMAND_BUFFER=64, which
# the SERVER's VulkanRenderer reads (SplitOversizedRecording), and assert the line it latches - the
# knob-free `.Oversized.` copies (the default budget, dev's control) keep their tcp entries. Eight
# cases, eight tails.
SPLIT_RECORDING_CASES = ("EveryDrawOfTheFrameLandsAcrossCommandBufferSplits",
"ADepthOccluderDrawnBeforeTheSplitHidesADrawAfterIt",
"AStencilMarkMadeBeforeTheSplitGatesADrawAfterIt",
"BlendingAccumulatesOntoPixelsWrittenBeforeTheSplit",
"AMultisampleTargetKeepsItsSamplesAcrossTheSplit",
"OcclusionQueriesOpenAcrossTheSplitCountBothSides",
"APrimitivesGeneratedQueryOpenAcrossTheSplitCountsBothSides",
"ATransformFeedbackCaptureActiveAcrossTheSplitAppendsEveryDraw")
PRIMGEN_REROUTE_CASES = ("CountsADrawMadeWithNoCaptureSpan",
"CountsUnderRasterizerDiscardWithNoCaptureSpan",
"CountsATessellatedPatchWithNoCaptureSpan",
"ASpanMixingActiveAndInactiveDrawsAccumulatesBoth",
"APausedSpanCountsACpuPricedDrawExactlyOnce",
"APausedSpanCountsATessellatedPatchExactlyOnce",
"APausedSpanCountsAnInstancedDrawExactlyOnce",
"TheRerouteIsActuallyArmedWhenTheEnvironmentPinsItOn")
MAGMA_SERVER_ENV_KNOB_NO_TCP = (".ShaderMip1.", ".ShaderMip2.", ".DepthMip.",
".DefaultBlitShape1.", ".MsResolve1.", ".StaleSerial.", ".MsFlip1.",
".MsResolveBug.", ".MsFlipBug.", ".MsResolveElide.",
".ImageUnitPrivate.InvalidImageUnitScenario.AStoreThroughAnInvalidUnitIsLoadedThroughNeitherAnotherInvalidUnitNorItself",
".ImageUnitPrivate.InvalidImageUnitScenario.AStoreThroughAnEmptyUnitIsLoadedThroughNeitherAnotherEmptyUnitNorItself",
".Reclaim.MagmaWireReclaimScenario.RespecifiesWithNoDrawBetweenKeepTheLiveStoreCountBounded",
".Reclaim.MagmaWireReclaimScenario.RespecifyAndDrawEachStoreInOneFrameStaysWithinTheDeferredBudget",
".Reclaim.MagmaWireReclaimScenario.ManySmallRespecifyAndDrawRoundsStayUnderTheStoreCountCeiling",
".Reclaim.MagmaWireReclaimScenario.ADrawAfterTheEarlyReclaimFollowsTheNewStoreNotTheMemoizedHandle",
".Reclaim.MagmaWireReclaimScenario.DescriptorSetsRewindInsideOneLongFrameAfterTheirSubmitRetires") + tuple(
f".PrimGenReroute.PrimitivesGeneratedNoXfbScenario.{case}" for case in PRIMGEN_REROUTE_CASES) + tuple(
f".SplitRecording64.OversizedRecordingSplitScenario.{case}" for case in SPLIT_RECORDING_CASES) + (
# P8-F: the mip shader-shape decline, forced by MGITEST_MAGMA_FORCE_SHADER_MIPMAP (server-read).
".ShaderMipDecline.",)
# THE MIRROR SHAPE - entries that exist on tcp ALONE, by construction. Each tail must match at least
# one tcp entry, or the exception names nothing. P11 A1 put `.AdoptTier0.` here (over shared segments
# MOBILEGL_IPC_ADOPT_TIER=0 died at the handshake); P11 B2 made that knob an ASK that falls back to T2
# by name, so the Magma `.AdoptTier0.` lane now runs on all three arms with one case set and is
# compared like any other. The mechanism stays for the next tcp-only lane.
MAGMA_TCP_ONLY = ()
def lane_names(build_dir, label):
"""Every ctest entry name under an ANCHORED label (`ctest -L` is a regex, not a name)."""
out = subprocess.run(["ctest", "-N", "-L", "^" + label + "$"], cwd=build_dir,
capture_output=True, text=True, check=True).stdout
return {m.group(1) for m in TEST_LINE.finditer(out)}
def arm_keys(names):
return {f"{m.group(1)}.{m.group(3)}" for m in (ARM.match(n) for n in names) if m}
def lane_cases(build_dir, label):
out = subprocess.run(["ctest", "-N", "-L", "^" + label + "$"], cwd=build_dir,
capture_output=True, text=True, check=True).stdout
return {m.group(1) for m in CASE.finditer(out)}
def whole_build_discovery(build_dir):
"""`ctest --show-only=json-v1` over EVERY entry: a lock is a property of the registration, and
an entry that reaches the supervisor under some other label is just as able to collide."""
out = subprocess.run(["ctest", "--show-only=json-v1"], cwd=build_dir,
capture_output=True, text=True, check=True).stdout
return json.loads(out)
def tcp_lock_check(build_dir):
"""Every entry on the loopback TCP supervisor holds RESOURCE_LOCK mobilegl-tcp (the rule and
its history live beside junit_tally.tcp_lock_violations). Returns True on failure."""
spec = importlib.util.spec_from_file_location("junit_tally", Path(__file__).with_name("junit_tally.py"))
tally = importlib.util.module_from_spec(spec)
spec.loader.exec_module(tally)
document = whole_build_discovery(build_dir)
reaching = [t for t in document.get("tests", [])
if tally.TCP_FIXTURE in tally._list_property(t, "FIXTURES_REQUIRED")
or tally.TCP_LANE_LABEL in tally._list_property(t, "LABELS")]
bad = tally.tcp_lock_violations(document)
print(f"tcp supervisor lock: {len(reaching)} entrie(s) reach the loopback supervisor, "
f"{len(bad)} without RESOURCE_LOCK {tally.TCP_FIXTURE}")
if bad:
print(f"::error::entries that reach the one-session TCP supervisor (FIXTURES_REQUIRED "
f"{tally.TCP_FIXTURE}, or label {tally.TCP_LANE_LABEL}) must hold RESOURCE_LOCK "
f"{tally.TCP_FIXTURE}; without it `ctest -j` runs them beside another Tcp case and "
f"the second client dies Refuse{{Busy}}: " + ", ".join(bad), file=sys.stderr)
return True
return False
def compare_arms(build_dir, tier, labels, inproc_only=(), no_tcp=(), tcp_only=()):
"""The three arms of one tier must name the same set after the arm segment comes off.
`inproc_only` drops a key from the comparison for EVERY non-split arm; `no_tcp` drops it for
the tcp arm alone, which is a different shape and needs to be: a server-side test knob does
reach a spawned server and does not reach the tcp lane's shared fixture, so the entry is
required on spawn and forbidden on tcp. Folding it into `inproc_only` would stop requiring
it on spawn, which is where it does most of its work.
Returns True on failure, the way main() below counts them."""
sets = {}
for arm, label in labels.items():
names = lane_names(build_dir, label) - FIXTURE_ENTRIES
sets[arm] = arm_keys(names)
print(f"{tier}: {label} has {len(names)} case entrie(s), "
f"{len(sets[arm])} after normalisation")
if len(names) != len(sets[arm]):
print(f"::error::{label}: {len(names) - len(sets[arm])} entry name(s) do not parse "
f"as <Backend>.<Arm>.<case>, so this comparison silently drops them: "
+ ", ".join(sorted(n for n in names if not ARM.match(n))), file=sys.stderr)
return True
if not any(sets.values()):
print(f"::note::{tier}: no entries under {sorted(labels.values())} - nothing to compare")
return False
failed = False
reference = "split"
if tcp_only and "tcp" in sets:
for tail in tcp_only:
hits = sorted(k for k in sets["tcp"] if tail in k)
if not hits:
print(f"::error::{tier}: the tcp-only tail {tail!r} matches no tcp entry - an exception "
f"about nothing.", file=sys.stderr)
return True
elsewhere = sorted(k for arm, keys in sets.items() if arm != "tcp" for k in keys if tail in k)
if elsewhere:
print(f"::error::{tier}: the tcp-only tail {tail!r} also names non-tcp entries "
f"{elsewhere}.", file=sys.stderr)
return True
print(f"{tier}: {len(hits)} tcp-only entrie(s) excluded from the comparison ({tail})")
sets["tcp"] = {k for k in sets["tcp"] if not any(only in k for only in tcp_only)}
comparable = {k for k in sets[reference]
if not any(only in k for only in inproc_only)}
if len(comparable) != len(sets[reference]):
print(f"{tier}: {len(sets[reference]) - len(comparable)} inproc-only entrie(s) excluded "
f"from the comparison by name ({', '.join(inproc_only)})")
if no_tcp:
dropped = {k for k in comparable if any(only in k for only in no_tcp)}
# EVERY TAIL MUST NAME SOMETHING (review round 2). A tail that matches no split key is
# silently inert - a renamed prefix would leave the exception in the list and the
# entry it was about riding under some other name - so an unmatched tail is an error,
# and so is a tail that suddenly matches a different number of entries than the one
# knob case it was written for.
for tail in no_tcp:
hits = sorted(k for k in comparable if tail in k)
if len(hits) != 1:
print(f"::error::{tier}: the server-env-knob tail {tail!r} matches {len(hits)} "
f"split entrie(s) {hits}, not exactly one. Each tail names ONE knob case; a "
f"tail that names none is an exception about nothing, and one that names "
f"more has let a second case ride off the tcp arm unexamined.",
file=sys.stderr)
return True
if dropped:
print(f"{tier}: {len(dropped)} server-env-knob entrie(s) excluded from the TCP "
f"comparison only ({', '.join(no_tcp)}) - the knob is read on the server and "
f"the tcp server is the lane fixture, not a per-case process")
for arm, keys in sets.items():
if arm == reference:
continue
expected = comparable
if arm == "tcp" and no_tcp:
expected = {k for k in comparable if not any(only in k for only in no_tcp)}
missing, extra = sorted(expected - keys), sorted(keys - expected)
if missing or extra:
print(f"::error::{tier}: the {arm} arm does not match the {reference} arm: "
f"missing={missing}, extra={extra}. Every arm of a tier is emitted from one "
f"loop over the same case list, so a difference here is a registration bug, "
f"not a coverage decision.", file=sys.stderr)
failed = True
return failed
def main():
parser = argparse.ArgumentParser()
parser.add_argument("build_dir")
parser.add_argument("--require-device", action="store_true",
help="Also require the configured integration-tcp-device lane")
args = parser.parse_args()
split = lane_cases(args.build_dir, "integration-split")
spawn = lane_cases(args.build_dir, "integration-spawn")
if not split or not spawn:
print(f"::error::a lane is EMPTY (split={len(split)}, spawn={len(spawn)}). An empty set "
f"compares equal to nothing and would pass this gate silently, which is the one "
f"way it could stop meaning anything.", file=sys.stderr)
return 1
split_comparable = {c for c in split if c.split(".", 1)[0] not in MONOLITH_ONLY}
missing = sorted(split_comparable - spawn)
extra = sorted(spawn - split_comparable)
print(f"spawn-lane parity: split {len(split)} ({len(split_comparable)} comparable), "
f"spawn {len(spawn)}")
if missing:
print("::error::cases in integration-split with no integration-spawn counterpart: " +
", ".join(missing) + ". Exit gate 9.2 requires the sets to be equal - a spawn lane "
"that covers less goes green while proving less. Register through "
"mgl_itest_register_split_arms, which emits both arms from one filter.",
file=sys.stderr)
if extra:
print("::error::cases in integration-spawn that integration-split does not run: " +
", ".join(extra) + ". The split arm is the control; a case only the spawn arm runs "
"has no baseline to be compared against.", file=sys.stderr)
failed = bool(missing or extra)
labels = ["integration-tcp"]
if args.require_device:
labels.append("integration-tcp-device")
for label in labels:
cases = lane_cases(args.build_dir, label)
missing, extra = sorted(spawn - cases), sorted(cases - spawn)
print(f"{label} parity: spawn {len(spawn)}, tcp {len(cases)}")
if not cases or missing or extra:
print(f"::error::{label}: missing={missing}, extra={extra}", file=sys.stderr)
failed = True
# P7 package L, exit gate 1's half of the same question: the Magma arms.
# THE EXCLUSION IS PER TIER, not global. In the GATED tier the two scenarios are registered
# on the inproc arm ALONE (the CMakeLists says why), so they are absent from the other arms
# and have to come off the reference set. In the FULL-SUITE census they are registered on
# every arm - it is a whole-binary filter, and there they simply SKIP for want of their
# lane marker - so excluding them there would turn a present-on-all-arms case into a
# spurious "extra".
# The server-env knob entries are gated-tier only for the same per-tier reason: the
# full-suite census is a whole-binary filter with no knob in its environment at all, so
# there is nothing there to exclude.
failed |= compare_arms(args.build_dir, "magma gated tier",
{"split": "integration-magma-split",
"spawn": "integration-magma-spawn",
"tcp": "integration-magma-tcp"},
inproc_only=MAGMA_INPROC_ONLY,
no_tcp=MAGMA_SERVER_ENV_KNOB_NO_TCP,
tcp_only=MAGMA_TCP_ONLY)
failed |= compare_arms(args.build_dir, "magma informational tier",
{"split": "integration-magma-all-split",
"spawn": "integration-magma-all-spawn",
"tcp": "integration-magma-all-tcp"})
# Tier 3, the whole-suite census. It is expected to be RED and is in no gating step - but
# its three arms must still name the same set, because the census's whole value is that a
# case red on spawn and green on inproc is a two-process fact, and that comparison needs
# both arms to have run the same case.
failed |= compare_arms(args.build_dir, "magma full-suite tier",
{"split": "integration-magma-full-split",
"spawn": "integration-magma-full-spawn",
"tcp": "integration-magma-full-tcp"})
# The arms above can only be compared if they can all RUN: every tcp entry shares one
# single-session supervisor, so the lock is part of what registering a tcp arm means.
failed |= tcp_lock_check(args.build_dir)
return 1 if failed else 0
if __name__ == "__main__":
sys.exit(main())