#!/usr/bin/env python3 # MobileGL - scripts/ci/spawn_lane_parity.py # Copyright (c) 2025-2026 MobileGL-Dev # Licensed under the GNU Lesser General Public License v3.0: # https://www.gnu.org/licenses/gpl-3.0.txt # https://www.gnu.org/licenses/lgpl-3.0.txt # SPDX-License-Identifier: LGPL-3.0-only # End of Source File Header """Exit gate 9.2: integration-spawn's case-name set equals integration-split's. WHY A GATE AND NOT A COMMENT. The two lanes are registered from one macro, so today they agree by construction - but a macro is one edit away from a second hand-written block, and a spawn lane covering LESS than the split lane goes green while proving less. Nothing about a smaller green lane looks wrong. This asserts the sets rather than the counts, because a count can agree while the membership does not. ARM PREFIXES ARE STRIPPED BEFORE COMPARING. The lanes deliberately differ in name - `DirectGLES.Split.X` against `DirectGLES.Spawn.X` - and the sub-lane tails (`F1.`, `Ct.`) are part of neither the case nor the arm, so both come off. ONE SANCTIONED ASYMMETRY, LISTED HERE AND NOWHERE ELSE. `integration-split` also labels the PUSH-MONOLITH arm (the CMakeLists explains why: the label means "the set the disaggregation gate runs", not "runs inproc"), and that arm's scenario is monolith-only by construction. It cannot have a spawn counterpart, so it is named as an exception rather than allowed to widen the tolerance for everything else. P7 PACKAGE L ADDS THE SAME QUESTION FOR MAGMA, IN TWO TIERS, AND THE NORMALIZATION IS DIFFERENT ON PURPOSE. The Espryt half above strips the sub-lane tails `F1.` and `Ct.` because they name neither the case nor the arm. The Magma half must NOT: `RunAhead.Credit1.` and `RunAhead.Credit3.` are two entries of the same case, differing in the credit the lane pins, and a normalizer that dropped the tail would fold them together and then report an equal-sized set that had silently lost one. So there the arm - and only the arm, the SECOND dotted segment - is what comes off. """ import argparse import importlib.util import json from pathlib import Path import re import subprocess import sys # Registered only on the push-monolith arm, which has no spawn counterpart by construction. MONOLITH_ONLY = {"MonolithAttachmentClearScenario"} CASE = re.compile(r"DirectGLES\.[A-Za-z0-9]+\.(?:F1\.|Ct\.)?([A-Za-z0-9_]+Scenario\.[A-Za-z0-9_]+)") # `..` -> `.`. The arm is the only # segment two arms of one lane are allowed to differ in. ARM = re.compile(r"^(DirectGLES|DirectVulkan)\.(Split|Spawn|Tcp|TcpDevice)\.(.+)$") TEST_LINE = re.compile(r"^\s*Test\s+#\d+:\s*(\S+)\s*$", re.MULTILINE) # `ctest -N -L ` lists the FIXTURE entries too, because every tcp entry declares # FIXTURES_REQUIRED mobilegl-tcp and ctest pulls the setup/cleanup in with them. They are the # supervisor, not cases, and they exist on exactly one arm by construction. Named here so that # any OTHER unparseable name is still an error rather than a silent drop. FIXTURE_ENTRIES = {"TcpServer.Start", "TcpServer.Stop"} # THE MAGMA TIER'S SANCTIONED ASYMMETRIES, LISTED HERE AND NOWHERE ELSE - the same discipline # MONOLITH_ONLY above gets. Both are inproc-only BY CONSTRUCTION, not by omission: # MagmaRunAheadScenario arms its hold through ServerLoopInstance().SetBeforeRetireHookForTesting, # a hook on the CALLING process's server loop. With the server in another # process the hold never activates and the wait times out. # MagmaWireCacheScenario SetUp asserts runtime.transportName == "inproc" outright: the lane reads # the server's own view/attachment identity out of this process. # A cross-process hold and a cross-process cache peek are server-side knobs neither exists yet; # until they do, naming the two scenarios here keeps the tolerance from widening for anything else. # DescriptorPoolGrowthScenario (P11 M2, the `.DescPool.` entry): both cases read the renderer's # pool census (DirectVulkan::GetDescriptorPoolCensus), an in-process read - # on spawn and tcp the pools live in the server process and the census # answers unavailable, so a copy there checks pixels under a name claiming # the pools. The pixels do run on every arm: the Espryt entries and the # Magma informational and full-suite tiers. # `.Mip3DValidation.` (P8-SV, a TAIL): WireMipShapeScenario's 3D chain under the Khronos validation # layer, red on a vkCmdBlitImage VUID in the entry's own output and proving the # layer loaded from /proc/self/maps - both need the server's Vulkan instance # in this process. The knob-free `.Mip3D.` entries keep all three arms. # `.SyncValidation.` (P14, a TAIL): TextureUploadBetweenDrawsScenario under the Khronos synchronization # validator, red on a SYNC-HAZARD line in the entry's own output and proving the # layer loaded from /proc/self/maps - the `.Mip3DValidation.` reason exactly. The # knob-free `.UploadBetweenDraws.` entries keep all three arms. MAGMA_INPROC_ONLY = ("MagmaRunAheadScenario.", "MagmaWireCacheScenario.", "DescriptorPoolGrowthScenario.", ".Mip3DValidation.", ".SyncValidation.") # P7 wave 2-B2: THE SANCTIONED ASYMMETRY THAT IS NOT INPROC-ONLY - these exist on split AND # spawn and cannot exist on tcp. Every MGITEST_MAGMA_FORCE_* knob is read BY THE SERVER # (WireFramebuffer.cpp's getenv readers run on the apply thread), and a ctest ENVIRONMENT # property only reaches the process ctest starts, the CLIENT. Under inproc the server is a # thread of that process and under spawn ServerSpawn.cpp copies ::environ into the child, so # both see it; under tcp the server is the LANE-WIDE TcpServer.Start fixture, started once from # the fixture's own os.environ before any case runs, so nothing a per-test property says can # reach it. A tcp copy would run the DEFAULT arm under a name claiming otherwise. # # MEASURED (a temporary fatal in each of the three readers): all five knob=1 entries died on # split and on spawn and ALL FIVE PASSED on tcp. # # These are TAILS, not scenarios - the normalizer above keeps the sub-lane tail for the Magma # tier on purpose, so `.MsResolve1.` names the knob entry and `.MsResolve0.` (the default, which # does keep its tcp entry) is untouched. # # P7 wave 2-B3 (ID-P7-34) adds `.StaleSerial.`: MGITEST_MAGMA_FORCE_STALE_BUFFER_SERIAL is read by # the server too (VkBufferManager.cpp), so the streamed subdata-then-draw red-once has the same # two arms and the same tcp absence. One list, one mechanism, for every server-side knob. # `.MsFlip1.` (B2 review round 2) is the shader-resolve knob twin of the knob-free `.MsFlip.`, # which keeps its tcp entry. # P7 wave 4 M2: MagmaWireReclaimScenario reads the SERVER's wbuf[] PipeStats gauges off the server's # private log with the stats channel and MOBILEGL_IPC_WIRE_DEFERRED_MB=8 in the entry's environment - # server-process environment again, so the same two arms and the same tcp absence. Its three cases # are three tails (one key each) under the `.Reclaim.` prefix. # P7 gate 5 (g5-msrbo review round, g5-msprobe): `.MsResolveBug.` / `.MsFlipBug.` set # MGITEST_MAGMA_DEPTH_RESOLVE_PROBE=bug, which the SERVER's depth/stencil resolve arm choice reads # in place of running its resolve probe - the knob-free `.MsResolve0.` / `.MsFlip.` keep their tcp # entries (MsResolve0's split/spawn copies also carry a client-read expectation marker for the # real probe's verdict, which the tcp copy cannot: same name on all three arms, no exception). # `.MsResolveElide.` (g5-msprobe critic) sets MGITEST_MAGMA_DEPTH_RESOLVE_PROBE=elide-subject, which # the same server-side arm choice reads to run the REAL probe with its render-pass resolve left out. # Codex closeout finding 2 (cf-magma): the `.ImageUnitPrivate.` entries set # MGITEST_MAGMA_FORCE_PRIVATE_IMAGE_PLACEHOLDER=1, which the SERVER's UniformManager reads to bind a # unit-private placeholder for an invalid image unit instead of a null descriptor - the # knob-free `.ImageUnitWindow.` copies of the same two cases keep their tcp entries. Two cases, so two # tails (one case each), as for `.Reclaim.`. # P10 B: the `.PrimGenReroute.` entries set MOBILEGL_MAGMA_PRIMGEN_QUERY_REROUTE=1, which the # SERVER's VulkanRenderer reads to pin the GL_PRIMITIVES_GENERATED reroute on - the knob-free # `.PrimGen.` copies (the bring-up probe's Auto verdict) keep their tcp entries. Eight cases, eight # tails. # P11 M2: the `.SplitRecording64.` entries set MOBILEGL_MAGMA_MAX_DRAWS_PER_COMMAND_BUFFER=64, which # the SERVER's VulkanRenderer reads (SplitOversizedRecording), and assert the line it latches - the # knob-free `.Oversized.` copies (the default budget, dev's control) keep their tcp entries. Eight # cases, eight tails. SPLIT_RECORDING_CASES = ("EveryDrawOfTheFrameLandsAcrossCommandBufferSplits", "ADepthOccluderDrawnBeforeTheSplitHidesADrawAfterIt", "AStencilMarkMadeBeforeTheSplitGatesADrawAfterIt", "BlendingAccumulatesOntoPixelsWrittenBeforeTheSplit", "AMultisampleTargetKeepsItsSamplesAcrossTheSplit", "OcclusionQueriesOpenAcrossTheSplitCountBothSides", "APrimitivesGeneratedQueryOpenAcrossTheSplitCountsBothSides", "ATransformFeedbackCaptureActiveAcrossTheSplitAppendsEveryDraw") PRIMGEN_REROUTE_CASES = ("CountsADrawMadeWithNoCaptureSpan", "CountsUnderRasterizerDiscardWithNoCaptureSpan", "CountsATessellatedPatchWithNoCaptureSpan", "ASpanMixingActiveAndInactiveDrawsAccumulatesBoth", "APausedSpanCountsACpuPricedDrawExactlyOnce", "APausedSpanCountsATessellatedPatchExactlyOnce", "APausedSpanCountsAnInstancedDrawExactlyOnce", "TheRerouteIsActuallyArmedWhenTheEnvironmentPinsItOn") MAGMA_SERVER_ENV_KNOB_NO_TCP = (".ShaderMip1.", ".ShaderMip2.", ".DepthMip.", ".DefaultBlitShape1.", ".MsResolve1.", ".StaleSerial.", ".MsFlip1.", ".MsResolveBug.", ".MsFlipBug.", ".MsResolveElide.", ".ImageUnitPrivate.InvalidImageUnitScenario.AStoreThroughAnInvalidUnitIsLoadedThroughNeitherAnotherInvalidUnitNorItself", ".ImageUnitPrivate.InvalidImageUnitScenario.AStoreThroughAnEmptyUnitIsLoadedThroughNeitherAnotherEmptyUnitNorItself", ".Reclaim.MagmaWireReclaimScenario.RespecifiesWithNoDrawBetweenKeepTheLiveStoreCountBounded", ".Reclaim.MagmaWireReclaimScenario.RespecifyAndDrawEachStoreInOneFrameStaysWithinTheDeferredBudget", ".Reclaim.MagmaWireReclaimScenario.ManySmallRespecifyAndDrawRoundsStayUnderTheStoreCountCeiling", ".Reclaim.MagmaWireReclaimScenario.ADrawAfterTheEarlyReclaimFollowsTheNewStoreNotTheMemoizedHandle", ".Reclaim.MagmaWireReclaimScenario.DescriptorSetsRewindInsideOneLongFrameAfterTheirSubmitRetires") + tuple( f".PrimGenReroute.PrimitivesGeneratedNoXfbScenario.{case}" for case in PRIMGEN_REROUTE_CASES) + tuple( f".SplitRecording64.OversizedRecordingSplitScenario.{case}" for case in SPLIT_RECORDING_CASES) + ( # P8-F: the mip shader-shape decline, forced by MGITEST_MAGMA_FORCE_SHADER_MIPMAP (server-read). ".ShaderMipDecline.", # P8-SV: the 1D depth chain's decline, forced by the same server-read knob. ".ShaderMipDepth1D.",) # The stream-refusal assertion is TCP-only. The shared-segment fallback assertion # is required on split/spawn and listed in no_tcp above. Neither transport should # register the other assertion merely to skip it. MAGMA_TCP_ONLY = (".AdoptTier0.LargeArenaAdoptionScenario.AStreamSessionRefusesTheAdoptTierOnceByName",) # The shared-segment fallback assertion is spawn-only on Magma: the split arm is one process and # grants T0 in process (745f59bc), so it has no fallback to read. MAGMA_SPAWN_ONLY = (".AdoptTier0.LargeArenaAdoptionScenario.ASessionThatCannotRunT0FallsBackToT2OnceByName",) def lane_names(build_dir, label): """Every ctest entry name under an ANCHORED label (`ctest -L` is a regex, not a name).""" out = subprocess.run(["ctest", "-N", "-L", "^" + label + "$"], cwd=build_dir, capture_output=True, text=True, check=True).stdout return {m.group(1) for m in TEST_LINE.finditer(out)} def arm_keys(names): return {f"{m.group(1)}.{m.group(3)}" for m in (ARM.match(n) for n in names) if m} def lane_cases(build_dir, label): out = subprocess.run(["ctest", "-N", "-L", "^" + label + "$"], cwd=build_dir, capture_output=True, text=True, check=True).stdout return {m.group(1) for m in CASE.finditer(out)} def whole_build_discovery(build_dir): """`ctest --show-only=json-v1` over EVERY entry: a lock is a property of the registration, and an entry that reaches the supervisor under some other label is just as able to collide.""" out = subprocess.run(["ctest", "--show-only=json-v1"], cwd=build_dir, capture_output=True, text=True, check=True).stdout return json.loads(out) def tcp_lock_check(build_dir): """Every entry on the loopback TCP supervisor holds RESOURCE_LOCK mobilegl-tcp (the rule and its history live beside junit_tally.tcp_lock_violations). Returns True on failure.""" spec = importlib.util.spec_from_file_location("junit_tally", Path(__file__).with_name("junit_tally.py")) tally = importlib.util.module_from_spec(spec) spec.loader.exec_module(tally) document = whole_build_discovery(build_dir) reaching = [t for t in document.get("tests", []) if tally.TCP_FIXTURE in tally._list_property(t, "FIXTURES_REQUIRED") or tally.TCP_LANE_LABEL in tally._list_property(t, "LABELS")] bad = tally.tcp_lock_violations(document) print(f"tcp supervisor lock: {len(reaching)} entrie(s) reach the loopback supervisor, " f"{len(bad)} without RESOURCE_LOCK {tally.TCP_FIXTURE}") if bad: print(f"::error::entries that reach the one-session TCP supervisor (FIXTURES_REQUIRED " f"{tally.TCP_FIXTURE}, or label {tally.TCP_LANE_LABEL}) must hold RESOURCE_LOCK " f"{tally.TCP_FIXTURE}; without it `ctest -j` runs them beside another Tcp case and " f"the second client dies Refuse{{Busy}}: " + ", ".join(bad), file=sys.stderr) return True return False def compare_arms(build_dir, tier, labels, inproc_only=(), no_tcp=(), only_on=None): """The three arms of one tier must name the same set after the arm segment comes off. `inproc_only` drops a key from the comparison for EVERY non-split arm; `no_tcp` drops it for the tcp arm alone, which is a different shape and needs to be: a server-side test knob does reach a spawned server and does not reach the tcp lane's shared fixture, so the entry is required on spawn and forbidden on tcp. Folding it into `inproc_only` would stop requiring it on spawn, which is where it does most of its work. Returns True on failure, the way main() below counts them.""" sets = {} for arm, label in labels.items(): names = lane_names(build_dir, label) - FIXTURE_ENTRIES sets[arm] = arm_keys(names) print(f"{tier}: {label} has {len(names)} case entrie(s), " f"{len(sets[arm])} after normalization") if len(names) != len(sets[arm]): print(f"::error::{label}: {len(names) - len(sets[arm])} entry name(s) do not parse " f"as .., so this comparison silently drops them: " + ", ".join(sorted(n for n in names if not ARM.match(n))), file=sys.stderr) return True if not any(sets.values()): print(f"::note::{tier}: no entries under {sorted(labels.values())} - nothing to compare") return False failed = False reference = "split" # `only_on` maps an arm to the tails registered on that arm alone. Each tail must name # something on its arm and nothing on any other; then it comes off the comparison. for only_arm, tails in (only_on or {}).items(): if only_arm not in sets: continue for tail in tails: hits = sorted(k for k in sets[only_arm] if tail in k) if not hits: print(f"::error::{tier}: the {only_arm}-only tail {tail!r} matches no {only_arm} entry - " f"an exception about nothing.", file=sys.stderr) return True elsewhere = sorted(k for arm, keys in sets.items() if arm != only_arm for k in keys if tail in k) if elsewhere: print(f"::error::{tier}: the {only_arm}-only tail {tail!r} also names non-{only_arm} " f"entries {elsewhere}.", file=sys.stderr) return True print(f"{tier}: {len(hits)} {only_arm}-only entrie(s) excluded from the comparison ({tail})") sets[only_arm] = {k for k in sets[only_arm] if not any(only in k for only in tails)} comparable = {k for k in sets[reference] if not any(only in k for only in inproc_only)} if len(comparable) != len(sets[reference]): print(f"{tier}: {len(sets[reference]) - len(comparable)} inproc-only entrie(s) excluded " f"from the comparison by name ({', '.join(inproc_only)})") if no_tcp: dropped = {k for k in comparable if any(only in k for only in no_tcp)} # EVERY TAIL MUST NAME SOMETHING (review round 2). A tail that matches no split key is # silently inert - a renamed prefix would leave the exception in the list and the # entry it was about riding under some other name - so an unmatched tail is an error, # and so is a tail that suddenly matches a different number of entries than the one # knob case it was written for. for tail in no_tcp: hits = sorted(k for k in comparable if tail in k) if len(hits) != 1: print(f"::error::{tier}: the server-env-knob tail {tail!r} matches {len(hits)} " f"split entrie(s) {hits}, not exactly one. Each tail names ONE knob case; a " f"tail that names none is an exception about nothing, and one that names " f"more has let a second case ride off the tcp arm unexamined.", file=sys.stderr) return True if dropped: print(f"{tier}: {len(dropped)} server-env-knob entrie(s) excluded from the TCP " f"comparison only ({', '.join(no_tcp)}) - the knob is read on the server and " f"the tcp server is the lane fixture, not a per-case process") for arm, keys in sets.items(): if arm == reference: continue expected = comparable if arm == "tcp" and no_tcp: expected = {k for k in comparable if not any(only in k for only in no_tcp)} missing, extra = sorted(expected - keys), sorted(keys - expected) if missing or extra: print(f"::error::{tier}: the {arm} arm does not match the {reference} arm: " f"missing={missing}, extra={extra}. Every arm of a tier is emitted from one " f"loop over the same case list, so a difference here is a registration bug, " f"not a coverage decision.", file=sys.stderr) failed = True return failed def main(): parser = argparse.ArgumentParser() parser.add_argument("build_dir") parser.add_argument("--require-device", action="store_true", help="Also require the configured integration-tcp-device lane") args = parser.parse_args() split = lane_cases(args.build_dir, "integration-split") spawn = lane_cases(args.build_dir, "integration-spawn") if not split or not spawn: print(f"::error::a lane is EMPTY (split={len(split)}, spawn={len(spawn)}). An empty set " f"compares equal to nothing and would pass this gate silently, which is the one " f"way it could stop meaning anything.", file=sys.stderr) return 1 split_comparable = {c for c in split if c.split(".", 1)[0] not in MONOLITH_ONLY} missing = sorted(split_comparable - spawn) extra = sorted(spawn - split_comparable) print(f"spawn-lane parity: split {len(split)} ({len(split_comparable)} comparable), " f"spawn {len(spawn)}") if missing: print("::error::cases in integration-split with no integration-spawn counterpart: " + ", ".join(missing) + ". Exit gate 9.2 requires the sets to be equal - a spawn lane " "that covers less goes green while proving less. Register through " "mgl_itest_register_split_arms, which emits both arms from one filter.", file=sys.stderr) if extra: print("::error::cases in integration-spawn that integration-split does not run: " + ", ".join(extra) + ". The split arm is the control; a case only the spawn arm runs " "has no baseline to be compared against.", file=sys.stderr) failed = bool(missing or extra) labels = ["integration-tcp"] if args.require_device: labels.append("integration-tcp-device") for label in labels: cases = lane_cases(args.build_dir, label) missing, extra = sorted(spawn - cases), sorted(cases - spawn) print(f"{label} parity: spawn {len(spawn)}, tcp {len(cases)}") if not cases or missing or extra: print(f"::error::{label}: missing={missing}, extra={extra}", file=sys.stderr) failed = True # P7 package L, exit gate 1's half of the same question: the Magma arms. # THE EXCLUSION IS PER TIER, not global. In the GATED tier the two scenarios are registered # on the inproc arm ALONE (the CMakeLists says why), so they are absent from the other arms # and have to come off the reference set. In the FULL-SUITE census they are registered on # every arm - it is a whole-binary filter, and there they simply SKIP for want of their # lane marker - so excluding them there would turn a present-on-all-arms case into a # spurious "extra". # The server-env knob entries are gated-tier only for the same per-tier reason: the # full-suite census is a whole-binary filter with no knob in its environment at all, so # there is nothing there to exclude. failed |= compare_arms(args.build_dir, "magma gated tier", {"split": "integration-magma-split", "spawn": "integration-magma-spawn", "tcp": "integration-magma-tcp"}, inproc_only=MAGMA_INPROC_ONLY, no_tcp=MAGMA_SERVER_ENV_KNOB_NO_TCP, only_on={"tcp": MAGMA_TCP_ONLY, "spawn": MAGMA_SPAWN_ONLY}) failed |= compare_arms(args.build_dir, "magma informational tier", {"split": "integration-magma-all-split", "spawn": "integration-magma-all-spawn", "tcp": "integration-magma-all-tcp"}) # Tier 3, the whole-suite census. It is expected to be RED and is in no gating step - but # its three arms must still name the same set, because the census's whole value is that a # case red on spawn and green on inproc is a two-process fact, and that comparison needs # both arms to have run the same case. failed |= compare_arms(args.build_dir, "magma full-suite tier", {"split": "integration-magma-full-split", "spawn": "integration-magma-full-spawn", "tcp": "integration-magma-full-tcp"}) # The arms above can only be compared if they can all RUN: every tcp entry shares one # single-session supervisor, so the lock is part of what registering a tcp arm means. failed |= tcp_lock_check(args.build_dir) return 1 if failed else 0 if __name__ == "__main__": sys.exit(main())