[Test] (.github/workflows, scripts/ci, tools/trace_replay): assert each split leg's transport in-test

- run_trace_case.cmake writes output/split-evidence.txt only after its transport assertions pass - MGTRACE_WIRE_EVIDENCE (a harness name) turns on the library's counters and requires DrawVbo records on the wire and draws on the server role - retrace-split's OpenRA legs ask for wire evidence; every split leg, and every inproc retrace row, checks its evidence file with retrace_split_evidence.py - test_run_trace_case.py runs the real script on canned logs: each assertion goes red on its bad log, and pipe-gates runs it
2026-11-07 01:53:24 +09:00 · 2026-10-17 14:08:15 -04:00
parent ca5411cdf6
commit bc0111c99e
4 changed files with 531 additions and 0 deletions
@@ -2771,6 +2771,12 @@ jobs:
            export MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1
          fi
          ctest -V --no-tests=error -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$'             --output-junit "${RUNNER_TEMP}/retrace.xml"
+          # A split row's transport assertions ran on this leg and found their evidence.
+          if [ "${MOBILEGL_TRANSPORT}" != monolith ]; then
+            python3 "${GITHUB_WORKSPACE}/scripts/ci/retrace_split_evidence.py" \
+              '${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt' \
+              --transport "${MOBILEGL_TRANSPORT}" --case '${{ matrix.case }}' --backend '${{ matrix.backend }}'
+          fi

      - name: Annotate failed entries
        if: failure()
@@ -3226,8 +3232,20 @@ jobs:
          # resolves the image relative to nothing. Harmless on the inproc arm, which reads it
          # only to decide it does not need it.
          export MOBILEGL_IPC_SERVER_PATH="${GITHUB_WORKSPACE}/build-linux/libMobileGLServer.so"
+          # OpenRA's legs also count the wire's draw records and the server's draws; that turns on
+          # the library's per-frame counters, so the other cases keep the plain configuration.
+          wire=()
+          if [ '${{ matrix.case }}' = 'OpenRA' ]; then
+            export MGTRACE_WIRE_EVIDENCE=1
+            wire=(--require-wire)
+          fi
          ctest -V --no-tests=error --timeout 10800 \
            -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$'
+          # The transport assertions ran on this leg and found their evidence.
+          python3 "${GITHUB_WORKSPACE}/scripts/ci/retrace_split_evidence.py" \
+            '${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt' \
+            --transport '${{ matrix.transport }}' --case '${{ matrix.case }}' \
+            --backend '${{ matrix.backend }}' "${wire[@]}"

      # Negative control: copy a library without MG_Remote (the push monolith FCL embeds) over the
      # frozen path and require the same replay to fail. OpenRA scores ssim 1.000 either way, so
@@ -3283,6 +3301,7 @@ jobs:
          # half it cannot see.
          log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.client.log"
          server_log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.server.log"
+          cat "${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt" 2>/dev/null || echo "no split-evidence.txt"
          if [ ! -f "${log}" ]; then
            echo "no ${log} - the replay wrote no library log"
            exit 0
@@ -3507,6 +3526,7 @@ jobs:
          python3 scripts/ci/tcp_lane_tools_test.py
          python3 tools/trace_replay/test_run_tcp_matrix.py
          python3 tools/trace_replay/test_compare_actuals.py
+          python3 tools/trace_replay/test_run_trace_case.py

      # Warning only for now: the disaggregation documents are still being written, and a
      # lint that fails a rewrite in progress teaches people to ignore it. It becomes
@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+# MobileGL - scripts/ci/retrace_split_evidence.py
+# Copyright (c) 2025-2026 MobileGL-Dev
+# Licensed under the GNU Lesser General Public License v3.0:
+#   https://www.gnu.org/licenses/gpl-3.0.txt
+#   https://www.gnu.org/licenses/lgpl-3.0.txt
+# SPDX-License-Identifier: LGPL-3.0-only
+# End of Source File Header
+
+"""Check the evidence file a split retrace leg leaves behind.
+
+tools/trace_replay/run_trace_case.cmake writes output/split-evidence.txt only after every
+assertion of its MOBILEGL_TRANSPORT block passed: the client log carried the transport marker
+(and, under spawn, the separate server pid), a server log existed, no Fatal{ and no MGWIRE-FLOOR
+line, and, on the legs that asked for it, draw records on the wire and draws on the server. The
+leg's ctest exit status alone cannot tell "the assertions passed" from "the assertions never ran",
+so the workflow runs this after ctest: a missing file means the transport block was skipped.
+
+Usage: retrace_split_evidence.py FILE --transport T [--case C] [--backend B] [--require-wire]
+"""
+import argparse
+from pathlib import Path
+import sys
+
+
+def parse(text):
+    values = {}
+    for line in text.splitlines():
+        if '=' in line:
+            key, value = line.split('=', 1)
+            values[key.strip()] = value.strip()
+    return values
+
+
+def count(values, key):
+    try:
+        return int(values.get(key, ''))
+    except ValueError:
+        return None
+
+
+def problems_in(values, transport, case=None, backend=None, require_wire=False):
+    problems = []
+    if values.get('transport') != transport:
+        problems.append(f"transport is '{values.get('transport')}', expected '{transport}'")
+    if case is not None and values.get('case') != case:
+        problems.append(f"case is '{values.get('case')}', expected '{case}'")
+    if backend is not None and values.get('backend') != backend:
+        problems.append(f"backend is '{values.get('backend')}', expected '{backend}'")
+    if not values.get('marker'):
+        problems.append('no transport marker recorded')
+    if values.get('server_log') != '1':
+        problems.append('no server log recorded')
+    for key in ('fatal_lines', 'floor_lines'):
+        if count(values, key) != 0:
+            problems.append(f"{key} is '{values.get(key)}', expected 0")
+    tcp = values.get('control', '-').startswith('tcp://')
+    if transport == 'spawn' and values.get('second', '-') == '-':
+        problems.append('spawn leg recorded no server-process sentence')
+    wire = values.get('wire') == 'ON'
+    if require_wire and not wire:
+        problems.append('wire evidence was required and the leg did not collect it')
+    if wire:
+        records = count(values, 'wire_draw_records')
+        if records is None or records < 1:
+            problems.append(f"wire_draw_records is '{values.get('wire_draw_records')}', expected at least 1")
+        draws = values.get('server_draws')
+        if not (tcp and draws == 'n/a'):
+            if count(values, 'server_draws') is None or count(values, 'server_draws') < 1:
+                problems.append(f"server_draws is '{draws}', expected at least 1")
+    return problems
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('evidence', type=Path)
+    parser.add_argument('--transport', required=True)
+    parser.add_argument('--case')
+    parser.add_argument('--backend')
+    parser.add_argument('--require-wire', action='store_true')
+    args = parser.parse_args(argv)
+    if not args.evidence.is_file():
+        print(f'::error::{args.evidence} does not exist: run_trace_case.cmake writes it only after its '
+              f'MOBILEGL_TRANSPORT assertions passed, so on this leg they did not run or did not finish')
+        return 1
+    values = parse(args.evidence.read_text(errors='replace'))
+    problems = problems_in(values, args.transport, args.case, args.backend, args.require_wire)
+    if problems:
+        print(f'::error::{args.evidence}: ' + '; '.join(problems))
+        return 1
+    print(f"split evidence: {values.get('case')} {values.get('backend')} transport={values.get('transport')} "
+          f"second='{values.get('second')}' wire={values.get('wire')} "
+          f"wire_draw_records={values.get('wire_draw_records')} server_draws={values.get('server_draws')}")
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
@@ -67,6 +67,25 @@ if(DEFINED TRACE_IPC_CONTROL AND NOT "${TRACE_IPC_CONTROL}" STREQUAL "")
    set(ENV{MOBILEGL_IPC_LOG_FORWARD} 1)
 endif()

+# Wire-volume evidence, only on the legs that ask for it. MGTRACE_WIRE_EVIDENCE is a harness name
+# the library never reads. On a split transport it turns on the library's own per-frame counters,
+# so the checks after the replay can count the draw records the client put on the wire and the
+# draws the server role's backend ran. Both knobs are set outright: a longer summary period from
+# the caller would leave the server's counter line missing on a short trace. It stays off unless
+# asked for, because the counters are part of the configuration under test.
+set(trace_wire_evidence OFF)
+if(NOT "$ENV{MGTRACE_WIRE_EVIDENCE}" STREQUAL "" AND NOT "$ENV{MGTRACE_WIRE_EVIDENCE}" STREQUAL "0")
+    if("$ENV{MOBILEGL_TRANSPORT}" STREQUAL "" OR "$ENV{MOBILEGL_TRANSPORT}" STREQUAL "monolith")
+        message(FATAL_ERROR
+                "MGTRACE_WIRE_EVIDENCE is set for ${TRACE_CASE_NAME} ${TRACE_BACKEND} but "
+                "MOBILEGL_TRANSPORT is '$ENV{MOBILEGL_TRANSPORT}': a run with no wire has no wire "
+                "volume to count, so the request cannot be met.")
+    endif()
+    set(trace_wire_evidence ON)
+    set(ENV{MOBILEGL_PIPE_STATS} 1)
+    set(ENV{MOBILEGL_PIPE_STATS_PERIOD} 1)
+endif()
+
 if(EXISTS "${TRACE_OUTPUT_DIR}")
    file(REMOVE_RECURSE "${TRACE_OUTPUT_DIR}")
 endif()
@@ -478,5 +497,83 @@ if(DEFINED ENV{MOBILEGL_TRANSPORT} AND NOT "$ENV{MOBILEGL_TRANSPORT}" STREQUAL "
                    "(magma-b3.md §2). The clamp in VulkanRenderer::OnSubmitsCompletedUpTo is what makes "
                    "this unreachable; a line here means it was removed or bypassed.")
        endif()
+        # The wire volume, on the legs that asked for it (MGTRACE_WIRE_EVIDENCE above). The client
+        # counts each record at the codec commit and prints one frame-sent line per Present, keyed
+        # by raw opcode; the server role's backend counts its draws and prints one stats line per
+        # Present. Under inproc and spawn the client has no backend of its own, so draws on both
+        # sides mean the picture was drawn from records that crossed the wire. A tcp server is
+        # launched outside this script with an environment that is not ours, so its count is
+        # not required.
+        set(split_draw_opcode "-")
+        set(split_wire_records "-")
+        set(split_server_draws "-")
+        if(trace_wire_evidence)
+            set(split_wire_table "${CMAKE_CURRENT_LIST_DIR}/../../MobileGL/MG_Pipe/generated/PipeWire.inc")
+            set(split_draw_op_line "")
+            if(EXISTS "${split_wire_table}")
+                file(STRINGS "${split_wire_table}" split_draw_op_line REGEX "^[ \t]*DrawVbo = [0-9]+,")
+            endif()
+            if(NOT split_draw_op_line MATCHES "DrawVbo = ([0-9]+),")
+                message(FATAL_ERROR "${split_case}: cannot find the DrawVbo opcode in ${split_wire_table}, "
+                                    "so the wire's draw records cannot be counted.")
+            endif()
+            set(split_draw_opcode "${CMAKE_MATCH_1}")
+            set(split_wire_records 0)
+            file(STRINGS "${mobilegl_log}" split_sent_lines REGEX "P65LinkMetrics kind=frame-sent ")
+            foreach(line IN LISTS split_sent_lines)
+                if(line MATCHES "by_op=([0-9=,]+)")
+                    if(",${CMAKE_MATCH_1}" MATCHES ",${split_draw_opcode}=([0-9]+)")
+                        math(EXPR split_wire_records "${split_wire_records} + ${CMAKE_MATCH_1}")
+                    endif()
+                endif()
+            endforeach()
+            if(split_wire_records EQUAL 0)
+                message(FATAL_ERROR
+                        "${split_case}: MOBILEGL_TRANSPORT=$ENV{MOBILEGL_TRANSPORT} committed no DrawVbo "
+                        "record to the wire: ${mobilegl_log} carries no frame-sent line counting opcode "
+                        "${split_draw_opcode}. Either the transport resolved and the emit table did not put "
+                        "a draw on it, or MOBILEGL_PIPE_STATS did not reach the client.")
+            endif()
+            if("$ENV{MOBILEGL_IPC_CONTROL}" MATCHES "^tcp://")
+                set(split_server_draws "n/a")
+            else()
+                set(split_server_draws 0)
+                file(STRINGS "${mobilegl_server_log}" split_stats_lines REGEX "MGPipe stats:.* draws=[0-9]+ ")
+                foreach(line IN LISTS split_stats_lines)
+                    if(line MATCHES " draws=([0-9]+) ")
+                        math(EXPR split_server_draws "${split_server_draws} + ${CMAKE_MATCH_1}")
+                    endif()
+                endforeach()
+                if(split_server_draws EQUAL 0)
+                    message(FATAL_ERROR
+                            "${split_case}: MOBILEGL_TRANSPORT=$ENV{MOBILEGL_TRANSPORT}: the server role's "
+                            "backend executed no draw - ${mobilegl_server_log} carries no MGPipe stats line "
+                            "with draws above zero, so nothing shows the records were drawn on that side.")
+                endif()
+            endif()
+        endif()
+        # The per-leg evidence that every assertion above ran and found what it needed. It is
+        # written only here, after the last of them, into an output directory wiped at the top of
+        # this script, so a file that exists is this run's; CI reads it back after ctest.
+        if("${split_expected_second}" STREQUAL "")
+            set(split_second_found "-")
+        else()
+            set(split_second_found "${split_expected_second}")
+        endif()
+        if("$ENV{MOBILEGL_IPC_CONTROL}" STREQUAL "")
+            set(split_control "-")
+        else()
+            set(split_control "$ENV{MOBILEGL_IPC_CONTROL}")
+        endif()
+        string(CONCAT split_evidence
+                "case=${TRACE_CASE_NAME}\nbackend=${TRACE_BACKEND}\ntransport=$ENV{MOBILEGL_TRANSPORT}\n"
+                "control=${split_control}\nmarker=${split_expected_marker}\nsecond=${split_second_found}\n"
+                "server_log=1\nfatal_lines=${split_fatal_count}\nfloor_lines=${split_floor_count}\n"
+                "wire=${trace_wire_evidence}\ndraw_opcode=${split_draw_opcode}\n"
+                "wire_draw_records=${split_wire_records}\nserver_draws=${split_server_draws}\n")
+        file(WRITE "${TRACE_OUTPUT_DIR}/output/split-evidence.txt" "${split_evidence}")
+        message(STATUS "MGPipe split evidence: ${split_case} transport=$ENV{MOBILEGL_TRANSPORT} "
+                       "marker found, second='${split_second_found}', wire=${trace_wire_evidence}, "
+                       "wire_draw_records=${split_wire_records}, server_draws=${split_server_draws}")
    endif()
 endif()
@@ -0,0 +1,316 @@
+#!/usr/bin/env python3
+# MobileGL - tools/trace_replay/test_run_trace_case.py
+# Copyright (c) 2025-2026 MobileGL-Dev
+# Licensed under the GNU Lesser General Public License v3.0:
+#   https://www.gnu.org/licenses/gpl-3.0.txt
+#   https://www.gnu.org/licenses/lgpl-3.0.txt
+# SPDX-License-Identifier: LGPL-3.0-only
+# End of Source File Header
+
+"""Canned-log tests of run_trace_case.cmake's split and verify assertions.
+
+Each case runs the script exactly as ctest does (`cmake -D... -P run_trace_case.cmake`) with a
+stub replay that copies canned library logs into the output directory. A bad log must turn the
+case red with the script's own sentence and leave no split-evidence.txt; a good one must pass and
+leave evidence that scripts/ci/retrace_split_evidence.py accepts. No build, GPU or trace needed.
+"""
+import contextlib
+import io
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+import unittest
+
+HERE = Path(__file__).resolve().parent
+SCRIPT = HERE / 'run_trace_case.cmake'
+CHECKER = HERE.parents[1] / 'scripts' / 'ci' / 'retrace_split_evidence.py'
+sys.path.insert(0, str(CHECKER.parent))
+import retrace_split_evidence as checker  # noqa: E402
+
+# The stub replay. It refuses to run when the wire counters disagree with what the leg asked for,
+# so removing the script's MOBILEGL_PIPE_STATS export turns every wire case red, and exporting it
+# on a leg that did not ask turns that leg red too.
+STUB = r'''#!/usr/bin/env python3
+import json, os, shutil, sys
+args = sys.argv[1:]
+out = args[args.index('--output') + 1]
+split = os.environ.get('MOBILEGL_TRANSPORT', '') not in ('', 'monolith')
+wire = os.environ.get('MGTRACE_WIRE_EVIDENCE', '') not in ('', '0')
+stats = os.environ.get('MOBILEGL_PIPE_STATS', '')
+if wire and split and (stats != '1' or os.environ.get('MOBILEGL_PIPE_STATS_PERIOD') != '1'):
+    sys.stderr.write('stub: MOBILEGL_PIPE_STATS did not reach the replay\n')
+    sys.exit(3)
+if not wire and stats:
+    sys.stderr.write('stub: MOBILEGL_PIPE_STATS reached a leg that did not ask for it\n')
+    sys.exit(4)
+canned = os.environ.get('STUB_CANNED_DIR', '')
+if canned:
+    for name in os.listdir(canned):
+        shutil.copy(os.path.join(canned, name), os.path.join(out, name))
+with open(os.path.join(out, 'retrace.log'), 'w') as log:
+    log.write('stub replay\n')
+with open(os.path.join(out, 'result.json'), 'w') as result:
+    json.dump({'passed': True, 'ssim': 1.0}, result)
+'''
+
+PREFIX = '[12:00:00] [Linux mobilegl_trace_/INFO]: '
+MARKER = {
+    'inproc': 'Config: MOBILEGL_TRANSPORT=inproc - the MGPipe record stream crosses a real ring to an apply thread',
+    'spawn': 'Config: MOBILEGL_TRANSPORT=spawn - the MGPipe record stream crosses a real ring to an apply '
+             'thread in ANOTHER PROCESS',
+}
+SPAWN_ARMED = 'MG_Remote client: spawn ARMED - the server role runs in pid 4242, reached at /tmp/x'
+TCP_ARMED = 'MG_Remote client: control=tcp data=stream server=127.0.0.1:1 pid=4243 dial=connect'
+SENT = ['P65LinkMetrics kind=frame-sent frame=1 records=40 by_op=2=4,57=1,59=30,67=1',
+        'P65LinkMetrics kind=frame-sent frame=2 records=31 by_op=57=1,59=28,67=1']
+STATS_ON = 'MGPipe stats: counters ON (MOBILEGL_PIPE_STATS), summary every 1 frames'
+STATS = ['MGPipe stats: frames=1 window=1 draws=30 draws/f=30.00 acc=12 acc/draw=0.40 bytes/f[a=1]',
+         'MGPipe stats: frames=2 window=1 draws=28 draws/f=28.00 acc=10 acc/draw=0.36 bytes/f[a=1]']
+SERVER_START = 'MG_Remote server: apply thread started'
+
+
+def quiet_check(argv):
+    """The checker's exit status, with its ::error:: lines kept off a green run's annotations."""
+    with contextlib.redirect_stdout(io.StringIO()):
+        return checker.main(argv)
+
+
+def lines(*rows):
+    return ''.join(PREFIX + row + '\n' for row in rows)
+
+
+def good_logs(transport):
+    client = [MARKER['spawn' if transport == 'tcp' else transport]]
+    if transport == 'spawn':
+        client.append(SPAWN_ARMED)
+    if transport == 'tcp':
+        client.append(TCP_ARMED)
+    client += SENT
+    server = [SERVER_START, STATS_ON] + ([] if transport == 'tcp' else STATS)
+    return {'mobilegl.client.log': lines(*client), 'mobilegl.server.log': lines(*server)}
+
+
+def squash(text):
+    return re.sub(r'\s+', ' ', text)
+
+
+@unittest.skipIf(os.name == 'nt', 'the stub replay is a POSIX executable script')
+class RunTraceCaseTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory(prefix='run-trace-case-')
+        self.root = Path(self.temp.name)
+        payload = self.root / 't.trace'
+        payload.write_text('not a real trace\n')
+        self.archive = self.root / 't.tar.gz'
+        with tarfile.open(self.archive, 'w:gz') as tar:
+            tar.add(payload, arcname='t.trace')
+        self.stub = self.root / 'stub_replay'
+        self.stub.write_text(STUB)
+        self.stub.chmod(0o755)
+        self.out = self.root / 'case'
+
+    def tearDown(self):
+        self.temp.cleanup()
+
+    def run_case(self, env, logs, tcp=False):
+        canned = self.root / 'canned'
+        shutil.rmtree(canned, ignore_errors=True)
+        canned.mkdir()
+        for name, text in logs.items():
+            (canned / name).write_text(text)
+        environment = {key: value for key, value in os.environ.items()
+                       if not key.startswith(('MOBILEGL_', 'MGTRACE_'))}
+        environment.update(env)
+        environment['STUB_CANNED_DIR'] = str(canned)
+        command = ['cmake', f'-DTRACE_REPLAY_EXE={self.stub}', f'-DMOBILEGL_LIBRARY={self.root}/libMobileGL.so',
+                   f'-DTRACE_ARCHIVE={self.archive}', '-DTRACE_FILE=t.trace',
+                   f'-DTRACE_GOLDEN={self.root}/golden.png', f'-DTRACE_OUTPUT_DIR={self.out}',
+                   f'-DTRACE_ARTIFACT_DIR={self.root}/artifacts', '-DTRACE_BACKEND=DirectGLES',
+                   '-DTRACE_CASE_NAME=Canned', '-DTRACE_TARGET_CALL=1', '-DTRACE_WIDTH=1', '-DTRACE_HEIGHT=1']
+        if tcp:
+            command += ['-DTRACE_IPC_CONTROL=tcp://127.0.0.1:1', '-DTRACE_IPC_TOKEN=x']
+        command += ['-P', str(SCRIPT)]
+        result = subprocess.run(command, env=environment, capture_output=True, text=True)
+        return result.returncode, squash(result.stdout + result.stderr)
+
+    def split_env(self, transport, wire=False):
+        env = {'MOBILEGL_TRANSPORT': 'spawn' if transport == 'tcp' else transport,
+               'MOBILEGL_IPC_SERVER_PATH': str(self.root / 'libMobileGLServer.so')}
+        if wire:
+            env['MGTRACE_WIRE_EVIDENCE'] = '1'
+        return env
+
+    @property
+    def evidence(self):
+        return self.out / 'output' / 'split-evidence.txt'
+
+    def expect_red(self, env, logs, sentence, tcp=False):
+        rc, output = self.run_case(env, logs, tcp=tcp)
+        self.assertNotEqual(rc, 0, output)
+        self.assertIn(squash(sentence), output)
+        self.assertFalse(self.evidence.exists(), 'a red leg must leave no evidence file')
+
+    def expect_green(self, env, logs, transport, tcp=False, wire=False):
+        rc, output = self.run_case(env, logs, tcp=tcp)
+        self.assertEqual(rc, 0, output)
+        self.assertEqual(quiet_check([str(self.evidence), '--transport', transport, '--case', 'Canned',
+                                       '--backend', 'DirectGLES'] + (['--require-wire'] if wire else [])), 0)
+        return output
+
+    # Red: each assertion of the transport block, fed the log that must trip it.
+    def test_r1_a_library_without_a_transport_writes_no_client_log(self):
+        self.expect_red(self.split_env('inproc'), {'mobilegl.log': lines('Config: loaded')},
+                        'so there is no evidence the transport ever resolved')
+
+    def test_r2_the_env_dump_line_is_not_the_marker(self):
+        logs = {'mobilegl.client.log': lines('Config: Accepted env variable: MOBILEGL_TRANSPORT=inproc'),
+                'mobilegl.server.log': lines(SERVER_START)}
+        self.expect_red(self.split_env('inproc'), logs, 'never reported resolving it')
+
+    def test_r3_spawn_without_a_separate_server_pid(self):
+        logs = good_logs('spawn')
+        logs['mobilegl.client.log'] = lines(MARKER['spawn'], *SENT)
+        self.expect_red(self.split_env('spawn'), logs, 'spawn ARMED - the server role runs in pid')
+
+    def test_r4_tcp_without_endpoint_and_pid(self):
+        logs = good_logs('tcp')
+        logs['mobilegl.client.log'] = lines(MARKER['spawn'], *SENT)
+        self.expect_red(self.split_env('tcp'), logs, 'TCP session has no endpoint and server pid arm proof',
+                        tcp=True)
+
+    def test_r5_no_server_log(self):
+        logs = good_logs('inproc')
+        del logs['mobilegl.server.log']
+        self.expect_red(self.split_env('inproc'), logs, 'but the run wrote no')
+
+    def test_r6_a_server_fatal(self):
+        logs = good_logs('inproc')
+        logs['mobilegl.server.log'] += lines('MGPipe: Fatal{UnmigratedVerb, "X"}')
+        self.expect_red(self.split_env('inproc'), logs, 'MGPipe Fatal(s) under MOBILEGL_TRANSPORT')
+
+    def test_r7_a_floor_line(self):
+        logs = good_logs('inproc')
+        logs['mobilegl.server.log'] += lines('MGWIRE-FLOOR unsound-serial-complete serial=3')
+        self.expect_red(self.split_env('inproc'), logs, 'MGWIRE-FLOOR unsound-serial-complete line(s)')
+
+    def test_r8_no_frame_sent_lines(self):
+        logs = good_logs('inproc')
+        logs['mobilegl.client.log'] = lines(MARKER['inproc'])
+        self.expect_red(self.split_env('inproc', wire=True), logs, 'committed no DrawVbo record to the wire')
+
+    def test_r9_frames_sent_without_a_draw(self):
+        logs = good_logs('inproc')
+        logs['mobilegl.client.log'] = lines(MARKER['inproc'], 'P65LinkMetrics kind=frame-sent frame=1 '
+                                                              'records=2 by_op=57=1,67=1,159=4,590=2')
+        self.expect_red(self.split_env('inproc', wire=True), logs, 'committed no DrawVbo record to the wire')
+
+    def test_r10_the_server_drew_nothing(self):
+        logs = good_logs('inproc')
+        logs['mobilegl.server.log'] = lines(SERVER_START, STATS_ON,
+                                            'MGPipe stats: frames=1 window=1 draws=0 draws/f=0.00 acc=0')
+        self.expect_red(self.split_env('inproc', wire=True), logs, 'backend executed no draw')
+
+    def test_r11_an_unknown_transport(self):
+        env = self.split_env('inproc')
+        env['MOBILEGL_TRANSPORT'] = 'unix:/x'
+        self.expect_red(env, good_logs('inproc'), 'this runner has no marker for it')
+
+    def test_r12_wire_evidence_without_a_split_transport(self):
+        self.expect_red({'MGTRACE_WIRE_EVIDENCE': '1', 'MOBILEGL_TRANSPORT': 'monolith'}, {},
+                        'a run with no wire has no wire volume to count')
+
+    def test_r13_the_stub_refuses_counters_on_a_leg_that_did_not_ask(self):
+        env = self.split_env('inproc')
+        env['MOBILEGL_PIPE_STATS'] = '1'
+        self.expect_red(env, good_logs('inproc'), 'stub: MOBILEGL_PIPE_STATS reached a leg that did not ask for it')
+
+    # Green: the same logs as a real leg writes, on every arm.
+    def test_g1_inproc_with_wire(self):
+        output = self.expect_green(self.split_env('inproc', wire=True), good_logs('inproc'), 'inproc', wire=True)
+        self.assertIn('wire_draw_records=58, server_draws=58', output)
+
+    def test_g1b_inproc_marker_only(self):
+        self.expect_green(self.split_env('inproc'), good_logs('inproc'), 'inproc')
+        values = checker.parse(self.evidence.read_text())
+        self.assertEqual((values['wire'], values['wire_draw_records']), ('OFF', '-'))
+        self.assertNotEqual(quiet_check([str(self.evidence), '--transport', 'inproc', '--require-wire']), 0)
+
+    def test_g2_spawn_with_wire(self):
+        self.expect_green(self.split_env('spawn', wire=True), good_logs('spawn'), 'spawn', wire=True)
+
+    def test_g3_tcp_with_wire(self):
+        self.expect_green(self.split_env('tcp', wire=True), good_logs('tcp'), 'spawn', tcp=True, wire=True)
+        self.assertEqual(checker.parse(self.evidence.read_text())['server_draws'], 'n/a')
+
+    def test_g4_monolith_leaves_no_evidence(self):
+        rc, output = self.run_case({'MOBILEGL_TRANSPORT': 'monolith'}, {'mobilegl.log': lines('Config: loaded')})
+        self.assertEqual(rc, 0, output)
+        self.assertFalse(self.evidence.exists())
+        self.assertNotEqual(quiet_check([str(self.evidence), '--transport', 'inproc']), 0,
+                            'a leg whose transport block was skipped must fail the checker')
+
+    # The verify block, which shares the script.
+    def test_v1_verify_without_the_armed_line(self):
+        self.expect_red({'MOBILEGL_PIPE_VERIFY': '1'}, {'mobilegl.log': lines('Config: loaded')},
+                        'never reported "MGPipe: verify armed"')
+
+    def test_v2_verify_with_a_divergence(self):
+        logs = {'mobilegl.log': lines('MGPipe: verify armed',
+                                      'MGPipe: Fatal{PipeVerifyDiffer, "GetRenderStateParameters@Clear"}')}
+        self.expect_red({'MOBILEGL_PIPE_VERIFY': '1'}, logs, 'MGPipe Fatal(s) under MOBILEGL_PIPE_VERIFY')
+
+    def test_v3_verify_green(self):
+        rc, output = self.run_case({'MOBILEGL_PIPE_VERIFY': '1'}, {'mobilegl.log': lines('MGPipe: verify armed')})
+        self.assertEqual(rc, 0, output)
+        self.assertIn('armed, zero divergences', output)
+
+
+class EvidenceCheckerTests(unittest.TestCase):
+    GOOD = ('case=Canned\nbackend=DirectGLES\ntransport=spawn\ncontrol=-\nmarker=m\n'
+            'second=spawn ARMED - the server role runs in pid \nserver_log=1\nfatal_lines=0\nfloor_lines=0\n'
+            'wire=ON\ndraw_opcode=59\nwire_draw_records=58\nserver_draws=58\n')
+
+    def check(self, text, *args):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / 'split-evidence.txt'
+            if text is not None:
+                path.write_text(text)
+            return quiet_check([str(path)] + list(args))
+
+    def test_good_file_passes(self):
+        self.assertEqual(self.check(self.GOOD, '--transport', 'spawn', '--require-wire'), 0)
+
+    def test_missing_file_fails(self):
+        self.assertEqual(self.check(None, '--transport', 'spawn'), 1)
+
+    def test_transport_mismatch_fails(self):
+        self.assertEqual(self.check(self.GOOD.replace('transport=spawn', 'transport=inproc'),
+                                    '--transport', 'spawn'), 1)
+
+    def test_case_mismatch_fails(self):
+        self.assertEqual(self.check(self.GOOD, '--transport', 'spawn', '--case', 'OpenRA'), 1)
+
+    def test_zero_wire_records_fail(self):
+        self.assertEqual(self.check(self.GOOD.replace('wire_draw_records=58', 'wire_draw_records=0'),
+                                    '--transport', 'spawn'), 1)
+
+    def test_zero_server_draws_fail(self):
+        self.assertEqual(self.check(self.GOOD.replace('server_draws=58', 'server_draws=0'),
+                                    '--transport', 'spawn'), 1)
+
+    def test_spawn_without_second_sentence_fails(self):
+        self.assertEqual(self.check(self.GOOD.replace('second=spawn ARMED - the server role runs in pid ',
+                                                      'second=-'), '--transport', 'spawn'), 1)
+
+    def test_a_fatal_count_fails(self):
+        self.assertEqual(self.check(self.GOOD.replace('fatal_lines=0', 'fatal_lines=2'),
+                                    '--transport', 'spawn'), 1)
+
+
+if __name__ == '__main__':
+    unittest.main()