[CI] (.github/workflows, docs/ci, tools/trace_replay): drop retrace-split's repeated inproc rows

- with build-linux reused, retrace-split runs its spawn rows plus OpenRA's inproc rows (the wire evidence); the other 75 inproc rows are replayed by the retrace job's inproc-acceptance rows on the same library
- trace_cases.py: github-split-matrix takes --transports and --keep-inproc; the self-test proves the spawn rows unchanged and every dropped inproc row an inproc-acceptance row; only a narrowed matrix that comes out empty raises, so the workflow's named error for an empty full subset still prints
- trace-cases: the spawn-only subset is only a saving, so a failed self-test or an empty result is a warning, the output is not written and retrace-split takes the full subset
- run_trace_case.cmake: MGTRACE_EXPECT_IPC (a harness name) requires the client's IPC line to carry the listed tokens; retrace-split legs prove strict=0 role-split-state=1 run-ahead=1 and a changed default reds every leg with the rows to restore; the acceptance rows prove strict=1 role-split-state=1 run-ahead=1 and gain the refusal census; canned tests in test_run_trace_case.py

tools/trace_replay/run_trace_case.cmake

@@ -85,6 +85,15 @@ if(NOT "$ENV{MGTRACE_WIRE_EVIDENCE}" STREQUAL "" AND NOT "$ENV{MGTRACE_WIRE_EVID
     set(ENV{MOBILEGL_PIPE_STATS} 1)
     set(ENV{MOBILEGL_PIPE_STATS_PERIOD} 1)
 endif()
+# MGTRACE_EXPECT_IPC, another harness name, lists key=value tokens the client's one-line IPC
+# configuration must carry (checked in the transport block below); a leg that leaves knobs at their
+# defaults uses it to prove what those defaults resolved to. A monolith run has no such line.
+if(NOT "$ENV{MGTRACE_EXPECT_IPC}" STREQUAL ""
+   AND ("$ENV{MOBILEGL_TRANSPORT}" STREQUAL "" OR "$ENV{MOBILEGL_TRANSPORT}" STREQUAL "monolith"))
+    message(FATAL_ERROR
+            "MGTRACE_EXPECT_IPC is set for ${TRACE_CASE_NAME} ${TRACE_BACKEND} but MOBILEGL_TRANSPORT is "
+            "'$ENV{MOBILEGL_TRANSPORT}': a monolith run logs no IPC configuration to check.")
+endif()
 
 if(EXISTS "${TRACE_OUTPUT_DIR}")
     file(REMOVE_RECURSE "${TRACE_OUTPUT_DIR}")
@@ -497,6 +506,37 @@ if(DEFINED ENV{MOBILEGL_TRANSPORT} AND NOT "$ENV{MOBILEGL_TRANSPORT}" STREQUAL "
                     "(magma-b3.md §2). The clamp in VulkanRenderer::OnSubmitsCompletedUpTo is what makes "
                     "this unreachable; a line here means it was removed or bypassed.")
         endif()
+        # The IPC configuration the client resolved, against MGTRACE_EXPECT_IPC's tokens. Each token
+        # must appear whole on the last "Config: IPC" line, so run-ahead=1 never matches
+        # run-ahead=10. MGTRACE_EXPECT_IPC_REASON, when set, says why the leg relies on them.
+        set(split_ipc_expected "-")
+        if(NOT "$ENV{MGTRACE_EXPECT_IPC}" STREQUAL "")
+            set(split_ipc_expected "$ENV{MGTRACE_EXPECT_IPC}")
+            file(STRINGS "${mobilegl_log}" split_ipc_lines REGEX "Config: IPC ")
+            if(NOT split_ipc_lines)
+                message(FATAL_ERROR "${split_case}: ${mobilegl_log} carries no 'Config: IPC' line, so the "
+                                    "configuration MGTRACE_EXPECT_IPC names cannot be checked.")
+            endif()
+            list(GET split_ipc_lines -1 split_ipc_line)
+            string(REPLACE " " ";" split_ipc_tokens "$ENV{MGTRACE_EXPECT_IPC}")
+            set(split_ipc_missing "")
+            foreach(token IN LISTS split_ipc_tokens)
+                if(NOT "${token}" STREQUAL "")
+                    string(FIND " ${split_ipc_line} " " ${token} " split_ipc_at)
+                    if(split_ipc_at EQUAL -1)
+                        list(APPEND split_ipc_missing "${token}")
+                    endif()
+                endif()
+            endforeach()
+            if(split_ipc_missing)
+                string(REPLACE ";" " " split_ipc_missing "${split_ipc_missing}")
+                message(FATAL_ERROR
+                        "${split_case}: the client resolved '${split_ipc_line}', which lacks "
+                        "${split_ipc_missing} (MGTRACE_EXPECT_IPC='$ENV{MGTRACE_EXPECT_IPC}'). "
+                        "$ENV{MGTRACE_EXPECT_IPC_REASON}")
+            endif()
+            message(STATUS "MGPipe split: ${split_case} IPC expectation met ($ENV{MGTRACE_EXPECT_IPC})")
+        endif()
         # The wire volume, on the legs that asked for it (MGTRACE_WIRE_EVIDENCE above). The client
         # counts each record at the codec commit and prints one frame-sent line per Present, keyed
         # by raw opcode; the server role's backend counts its draws and prints one stats line per
@@ -569,6 +609,7 @@ if(DEFINED ENV{MOBILEGL_TRANSPORT} AND NOT "$ENV{MOBILEGL_TRANSPORT}" STREQUAL "
                 "case=${TRACE_CASE_NAME}\nbackend=${TRACE_BACKEND}\ntransport=$ENV{MOBILEGL_TRANSPORT}\n"
                 "control=${split_control}\nmarker=${split_expected_marker}\nsecond=${split_second_found}\n"
                 "server_log=1\nfatal_lines=${split_fatal_count}\nfloor_lines=${split_floor_count}\n"
+                "ipc_expected=${split_ipc_expected}\n"
                 "wire=${trace_wire_evidence}\ndraw_opcode=${split_draw_opcode}\n"
                 "wire_draw_records=${split_wire_records}\nserver_draws=${split_server_draws}\n")
         file(WRITE "${TRACE_OUTPUT_DIR}/output/split-evidence.txt" "${split_evidence}")

tools/trace_replay/test_run_trace_case.py

@@ -261,6 +261,40 @@ class RunTraceCaseTests(unittest.TestCase):
         self.assertNotEqual(quiet_check([str(self.evidence), '--transport', 'inproc']), 0,
                             'a leg whose transport block was skipped must fail the checker')
 
+    # MGTRACE_EXPECT_IPC: the client's IPC line must carry every token, whole.
+    IPC = 'Config: IPC ring=8MiB verb-barrier=1 run-ahead={ra} present-credit=1 strict={st} audit=0 role-split-state={rs} x'
+
+    def ipc_logs(self, ra='1', st='0', rs='1', line=True):
+        logs = good_logs('spawn')
+        if line:
+            logs['mobilegl.client.log'] += lines(self.IPC.format(ra=ra, st=st, rs=rs))
+        return logs
+
+    def ipc_env(self, expect='strict=0 role-split-state=1 run-ahead=1'):
+        env = self.split_env('spawn')
+        env.update({'MGTRACE_EXPECT_IPC': expect, 'MGTRACE_EXPECT_IPC_REASON': 'restore the inproc rows'})
+        return env
+
+    def test_i1_the_expected_defaults_are_met(self):
+        output = self.expect_green(self.ipc_env(), self.ipc_logs(), 'spawn')
+        self.assertIn('IPC expectation met', output)
+        self.assertEqual(checker.parse(self.evidence.read_text())['ipc_expected'],
+                         'strict=0 role-split-state=1 run-ahead=1')
+
+    def test_i2_a_changed_default_names_the_reason(self):
+        self.expect_red(self.ipc_env(), self.ipc_logs(rs='0'), 'lacks role-split-state=1')
+        self.expect_red(self.ipc_env(), self.ipc_logs(rs='0'), 'restore the inproc rows')
+
+    def test_i3_no_ipc_line(self):
+        self.expect_red(self.ipc_env(), self.ipc_logs(line=False), "carries no 'Config: IPC' line")
+
+    def test_i4_a_token_is_matched_whole(self):
+        self.expect_red(self.ipc_env(), self.ipc_logs(ra='10'), 'lacks run-ahead=1')
+
+    def test_i5_an_expectation_on_a_monolith_run(self):
+        self.expect_red({'MOBILEGL_TRANSPORT': 'monolith', 'MGTRACE_EXPECT_IPC': 'strict=1'}, {},
+                        'a monolith run logs no IPC configuration to check')
+
     # The verify block, which shares the script.
     def test_v1_verify_without_the_armed_line(self):
         self.expect_red({'MOBILEGL_PIPE_VERIFY': '1'}, {'mobilegl.log': lines('Config: loaded')},

tools/trace_replay/trace_cases.py

@@ -351,7 +351,7 @@ def github_verify_matrix(cases):
 SPLIT_TRANSPORTS = ("inproc", "spawn")
 
 
-def github_split_matrix(cases):
+def github_split_matrix(cases, transports=SPLIT_TRANSPORTS, keep_inproc=()):
     """{backend, case, transport} for the split arm.
 
     THREE DIMENSIONS, NOT TWO. The job used to take {backend, case} and hard-code
@@ -359,15 +359,31 @@ def github_split_matrix(cases):
     backend list is the case's own ci_backends - the same list the monolith matrix uses - so a
     case restricted to one backend stays restricted here instead of being registered with a
     backend it does not run.
+
+    `transports` narrows the arms (test.yml drops inproc when the inproc-acceptance retrace rows
+    already replay the same library); `keep_inproc` names split cases whose inproc rows stay
+    anyway - OpenRA's, which carry the wire-volume evidence. A narrowed matrix that comes out
+    empty is an error; the full one is returned empty, and test.yml names that error itself.
     """
-    return {
-        "include": [
-            {"backend": backend, "case": case["name"], "transport": transport}
-            for case in split_trace_cases(cases)
-            for backend in ci_backends(case)
-            for transport in SPLIT_TRANSPORTS
-        ]
-    }
+    unknown = [t for t in transports if t not in SPLIT_TRANSPORTS]
+    if unknown or not transports:
+        raise ValueError(f"split transports must be a non-empty subset of {SPLIT_TRANSPORTS}: {transports}")
+    split_cases = split_trace_cases(cases)
+    split_names = {case["name"] for case in split_cases}
+    missing = [name for name in keep_inproc if name not in split_names]
+    if missing:
+        raise ValueError(f"--keep-inproc names cases that are not split CI cases: {missing}")
+    rows = [
+        {"backend": backend, "case": case["name"], "transport": transport}
+        for case in split_cases
+        for backend in ci_backends(case)
+        for transport in SPLIT_TRANSPORTS
+        if transport in transports or (transport == "inproc" and case["name"] in keep_inproc)
+    ]
+    narrowed = tuple(transports) != SPLIT_TRANSPORTS or bool(keep_inproc)
+    if not rows and narrowed:
+        raise ValueError("the narrowed split matrix is empty")
+    return {"include": rows}
 
 
 def github_apk_matrix(cases):
@@ -393,9 +409,10 @@ def github_transport_matrix(cases, *, apk=False):
 
     THE SPAWN LANE IS APK-ONLY, and that is a statement about where the evidence is missing
     rather than about what spawn can do. On Linux the spawn arm already has a job of its own -
-    test.yml's `retrace-split`, which is {backend, case, transport} over inproc AND spawn and
-    whose legs prove in-test that the transport resolved and, on OpenRA, that draw records
-    crossed the wire (run_trace_case.cmake's evidence). Adding spawn to THIS matrix would pay
+    test.yml's `retrace-split`, which is {backend, case, transport} over spawn (and inproc too
+    when it cannot reuse this matrix's inproc rows' library) and whose legs prove in-test that
+    the transport resolved and, on OpenRA, that draw records crossed the wire
+    (run_trace_case.cmake's evidence). Adding spawn to THIS matrix would pay
     for a second full Linux sweep that proves nothing the first one does not, and test.yml's
     `retrace` job has neither the server image staged nor MOBILEGL_IPC_SERVER_PATH set - so the
     lane would not merely be redundant, it would be red.
@@ -466,6 +483,41 @@ def self_test_transport_matrices():
               f"entries; exact metadata and negative controls OK")
 
 
+def self_test_split_matrix(cases=None):
+    """retrace-split may drop its inproc rows only because the inproc-acceptance rows replay them."""
+    if cases is None:
+        cases = ci_trace_cases(load_trace_cases())
+    full = github_split_matrix(cases)["include"]
+    spawn_only = github_split_matrix(cases, transports=("spawn",), keep_inproc=("OpenRA",))["include"]
+    assert [r for r in spawn_only if r["transport"] == "spawn"] == [r for r in full if r["transport"] == "spawn"], \
+        "the spawn-only matrix lost or changed a spawn row"
+    assert any(r["transport"] == "spawn" for r in spawn_only), "the spawn-only matrix has no spawn row"
+    kept = [r for r in spawn_only if r["transport"] == "inproc"]
+    assert kept and all(r["case"] == "OpenRA" for r in kept), "the kept inproc rows are not OpenRA's"
+    assert all(r in full for r in kept), "a kept inproc row is not a row of the full matrix"
+    acceptance = {(r["backend"], r["case"]) for r in github_transport_matrix(cases)["include"]
+                  if r["lane"] == "inproc-acceptance"}
+    dropped = [(r["backend"], r["case"]) for r in full if r["transport"] == "inproc" and r not in kept]
+    uncovered = [row for row in dropped if row not in acceptance]
+    assert not uncovered, f"inproc rows dropped from retrace-split that no acceptance row replays: {uncovered}"
+    for bad in ({"transports": ("unix",)}, {"transports": ()}, {"keep_inproc": ("no-such-case",)}):
+        try:
+            github_split_matrix(cases, **bad)
+        except ValueError:
+            continue
+        raise AssertionError(f"github_split_matrix accepted {bad}")
+    try:
+        github_split_matrix([], transports=("spawn",))
+    except ValueError:
+        pass
+    else:
+        raise AssertionError("an empty narrowed split matrix silently passed")
+    assert github_split_matrix([]) == {"include": []}, \
+        "the full split matrix must come back empty, for the workflow's named error"
+    print(f"split matrix: {len(full)} rows, {len(spawn_only)} with inproc dropped ({len(kept)} OpenRA inproc "
+          f"kept); all {len(dropped)} dropped inproc rows are inproc-acceptance rows")
+
+
 def cmake_quote(value):
     return '"' + str(value).replace("\\", "/").replace('"', '\\"') + '"'
 
@@ -516,7 +568,13 @@ def parse_args():
     parser.add_argument("--case", dest="case_name", help="Trace case name.")
     parser.add_argument("--ci", action="store_true", help="Only include cases enabled for CI.")
     parser.add_argument("--self-test-transport-matrices", action="store_true")
+    parser.add_argument("--self-test-split-matrix", action="store_true",
+                        help="prove the spawn-only split matrix drops only inproc-acceptance rows")
     parser.add_argument("--fixture-root", default="tools/trace_replay/fixtures")
+    parser.add_argument("--transports", default=",".join(SPLIT_TRANSPORTS),
+                        help="github-split-matrix: the comma-separated split transports to emit")
+    parser.add_argument("--keep-inproc", default="",
+                        help="github-split-matrix: comma-separated split cases whose inproc rows stay")
     parser.add_argument(
         "--format",
         choices=(
@@ -541,6 +599,9 @@ def main():
     if args.self_test_transport_matrices:
         self_test_transport_matrices()
         return 0
+    if args.self_test_split_matrix:
+        self_test_split_matrix()
+        return 0
     cases = load_trace_cases()
     if args.ci:
         cases = ci_trace_cases(cases)
@@ -553,7 +614,9 @@ def main():
     elif args.format == "github-verify-matrix":
         print(json.dumps(github_verify_matrix(cases), separators=(",", ":")))
     elif args.format == "github-split-matrix":
-        print(json.dumps(github_split_matrix(cases), separators=(",", ":")))
+        transports = tuple(t for t in args.transports.split(",") if t)
+        keep = tuple(c for c in args.keep_inproc.split(",") if c)
+        print(json.dumps(github_split_matrix(cases, transports, keep), separators=(",", ":")))
     elif args.format == "github-apk":
         print(json.dumps([github_apk_case(case) for case in cases], separators=(",", ":")))
     elif args.format == "github-apk-matrix":