From bc0111c99e6c72e96b174ef8d5dd38c4c874d9ec Mon Sep 17 00:00:00 2001 From: Swung0x48 Date: Sat, 17 Oct 2026 12:18:28 -0400 Subject: [PATCH] [Test] (.github/workflows, scripts/ci, tools/trace_replay): assert each split leg's transport in-test - run_trace_case.cmake writes output/split-evidence.txt only after its transport assertions pass - MGTRACE_WIRE_EVIDENCE (a harness name) turns on the library's counters and requires DrawVbo records on the wire and draws on the server role - retrace-split's OpenRA legs ask for wire evidence; every split leg, and every inproc retrace row, checks its evidence file with retrace_split_evidence.py - test_run_trace_case.py runs the real script on canned logs: each assertion goes red on its bad log, and pipe-gates runs it --- .github/workflows/test.yml | 20 ++ scripts/ci/retrace_split_evidence.py | 98 +++++++ tools/trace_replay/run_trace_case.cmake | 97 +++++++ tools/trace_replay/test_run_trace_case.py | 316 ++++++++++++++++++++++ 4 files changed, 531 insertions(+) create mode 100644 scripts/ci/retrace_split_evidence.py create mode 100644 tools/trace_replay/test_run_trace_case.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d79f9f83..536daf6e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2771,6 +2771,12 @@ jobs: export MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1 fi ctest -V --no-tests=error -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$' --output-junit "${RUNNER_TEMP}/retrace.xml" + # A split row's transport assertions ran on this leg and found their evidence. + if [ "${MOBILEGL_TRANSPORT}" != monolith ]; then + python3 "${GITHUB_WORKSPACE}/scripts/ci/retrace_split_evidence.py" \ + '${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt' \ + --transport "${MOBILEGL_TRANSPORT}" --case '${{ matrix.case }}' --backend '${{ matrix.backend }}' + fi - name: Annotate failed entries if: failure() @@ -3226,8 +3232,20 @@ jobs: # resolves the image relative to nothing. Harmless on the inproc arm, which reads it # only to decide it does not need it. export MOBILEGL_IPC_SERVER_PATH="${GITHUB_WORKSPACE}/build-linux/libMobileGLServer.so" + # OpenRA's legs also count the wire's draw records and the server's draws; that turns on + # the library's per-frame counters, so the other cases keep the plain configuration. + wire=() + if [ '${{ matrix.case }}' = 'OpenRA' ]; then + export MGTRACE_WIRE_EVIDENCE=1 + wire=(--require-wire) + fi ctest -V --no-tests=error --timeout 10800 \ -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$' + # The transport assertions ran on this leg and found their evidence. + python3 "${GITHUB_WORKSPACE}/scripts/ci/retrace_split_evidence.py" \ + '${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt' \ + --transport '${{ matrix.transport }}' --case '${{ matrix.case }}' \ + --backend '${{ matrix.backend }}' "${wire[@]}" # Negative control: copy a library without MG_Remote (the push monolith FCL embeds) over the # frozen path and require the same replay to fail. OpenRA scores ssim 1.000 either way, so @@ -3283,6 +3301,7 @@ jobs: # half it cannot see. log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.client.log" server_log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.server.log" + cat "${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt" 2>/dev/null || echo "no split-evidence.txt" if [ ! -f "${log}" ]; then echo "no ${log} - the replay wrote no library log" exit 0 @@ -3507,6 +3526,7 @@ jobs: python3 scripts/ci/tcp_lane_tools_test.py python3 tools/trace_replay/test_run_tcp_matrix.py python3 tools/trace_replay/test_compare_actuals.py + python3 tools/trace_replay/test_run_trace_case.py # Warning only for now: the disaggregation documents are still being written, and a # lint that fails a rewrite in progress teaches people to ignore it. It becomes diff --git a/scripts/ci/retrace_split_evidence.py b/scripts/ci/retrace_split_evidence.py new file mode 100644 index 00000000..937daedd --- /dev/null +++ b/scripts/ci/retrace_split_evidence.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# MobileGL - scripts/ci/retrace_split_evidence.py +# Copyright (c) 2025-2026 MobileGL-Dev +# Licensed under the GNU Lesser General Public License v3.0: +# https://www.gnu.org/licenses/gpl-3.0.txt +# https://www.gnu.org/licenses/lgpl-3.0.txt +# SPDX-License-Identifier: LGPL-3.0-only +# End of Source File Header + +"""Check the evidence file a split retrace leg leaves behind. + +tools/trace_replay/run_trace_case.cmake writes output/split-evidence.txt only after every +assertion of its MOBILEGL_TRANSPORT block passed: the client log carried the transport marker +(and, under spawn, the separate server pid), a server log existed, no Fatal{ and no MGWIRE-FLOOR +line, and, on the legs that asked for it, draw records on the wire and draws on the server. The +leg's ctest exit status alone cannot tell "the assertions passed" from "the assertions never ran", +so the workflow runs this after ctest: a missing file means the transport block was skipped. + +Usage: retrace_split_evidence.py FILE --transport T [--case C] [--backend B] [--require-wire] +""" +import argparse +from pathlib import Path +import sys + + +def parse(text): + values = {} + for line in text.splitlines(): + if '=' in line: + key, value = line.split('=', 1) + values[key.strip()] = value.strip() + return values + + +def count(values, key): + try: + return int(values.get(key, '')) + except ValueError: + return None + + +def problems_in(values, transport, case=None, backend=None, require_wire=False): + problems = [] + if values.get('transport') != transport: + problems.append(f"transport is '{values.get('transport')}', expected '{transport}'") + if case is not None and values.get('case') != case: + problems.append(f"case is '{values.get('case')}', expected '{case}'") + if backend is not None and values.get('backend') != backend: + problems.append(f"backend is '{values.get('backend')}', expected '{backend}'") + if not values.get('marker'): + problems.append('no transport marker recorded') + if values.get('server_log') != '1': + problems.append('no server log recorded') + for key in ('fatal_lines', 'floor_lines'): + if count(values, key) != 0: + problems.append(f"{key} is '{values.get(key)}', expected 0") + tcp = values.get('control', '-').startswith('tcp://') + if transport == 'spawn' and values.get('second', '-') == '-': + problems.append('spawn leg recorded no server-process sentence') + wire = values.get('wire') == 'ON' + if require_wire and not wire: + problems.append('wire evidence was required and the leg did not collect it') + if wire: + records = count(values, 'wire_draw_records') + if records is None or records < 1: + problems.append(f"wire_draw_records is '{values.get('wire_draw_records')}', expected at least 1") + draws = values.get('server_draws') + if not (tcp and draws == 'n/a'): + if count(values, 'server_draws') is None or count(values, 'server_draws') < 1: + problems.append(f"server_draws is '{draws}', expected at least 1") + return problems + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('evidence', type=Path) + parser.add_argument('--transport', required=True) + parser.add_argument('--case') + parser.add_argument('--backend') + parser.add_argument('--require-wire', action='store_true') + args = parser.parse_args(argv) + if not args.evidence.is_file(): + print(f'::error::{args.evidence} does not exist: run_trace_case.cmake writes it only after its ' + f'MOBILEGL_TRANSPORT assertions passed, so on this leg they did not run or did not finish') + return 1 + values = parse(args.evidence.read_text(errors='replace')) + problems = problems_in(values, args.transport, args.case, args.backend, args.require_wire) + if problems: + print(f'::error::{args.evidence}: ' + '; '.join(problems)) + return 1 + print(f"split evidence: {values.get('case')} {values.get('backend')} transport={values.get('transport')} " + f"second='{values.get('second')}' wire={values.get('wire')} " + f"wire_draw_records={values.get('wire_draw_records')} server_draws={values.get('server_draws')}") + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/trace_replay/run_trace_case.cmake b/tools/trace_replay/run_trace_case.cmake index b7fd2b7e..6536b798 100644 --- a/tools/trace_replay/run_trace_case.cmake +++ b/tools/trace_replay/run_trace_case.cmake @@ -67,6 +67,25 @@ if(DEFINED TRACE_IPC_CONTROL AND NOT "${TRACE_IPC_CONTROL}" STREQUAL "") set(ENV{MOBILEGL_IPC_LOG_FORWARD} 1) endif() +# Wire-volume evidence, only on the legs that ask for it. MGTRACE_WIRE_EVIDENCE is a harness name +# the library never reads. On a split transport it turns on the library's own per-frame counters, +# so the checks after the replay can count the draw records the client put on the wire and the +# draws the server role's backend ran. Both knobs are set outright: a longer summary period from +# the caller would leave the server's counter line missing on a short trace. It stays off unless +# asked for, because the counters are part of the configuration under test. +set(trace_wire_evidence OFF) +if(NOT "$ENV{MGTRACE_WIRE_EVIDENCE}" STREQUAL "" AND NOT "$ENV{MGTRACE_WIRE_EVIDENCE}" STREQUAL "0") + if("$ENV{MOBILEGL_TRANSPORT}" STREQUAL "" OR "$ENV{MOBILEGL_TRANSPORT}" STREQUAL "monolith") + message(FATAL_ERROR + "MGTRACE_WIRE_EVIDENCE is set for ${TRACE_CASE_NAME} ${TRACE_BACKEND} but " + "MOBILEGL_TRANSPORT is '$ENV{MOBILEGL_TRANSPORT}': a run with no wire has no wire " + "volume to count, so the request cannot be met.") + endif() + set(trace_wire_evidence ON) + set(ENV{MOBILEGL_PIPE_STATS} 1) + set(ENV{MOBILEGL_PIPE_STATS_PERIOD} 1) +endif() + if(EXISTS "${TRACE_OUTPUT_DIR}") file(REMOVE_RECURSE "${TRACE_OUTPUT_DIR}") endif() @@ -478,5 +497,83 @@ if(DEFINED ENV{MOBILEGL_TRANSPORT} AND NOT "$ENV{MOBILEGL_TRANSPORT}" STREQUAL " "(magma-b3.md ยง2). The clamp in VulkanRenderer::OnSubmitsCompletedUpTo is what makes " "this unreachable; a line here means it was removed or bypassed.") endif() + # The wire volume, on the legs that asked for it (MGTRACE_WIRE_EVIDENCE above). The client + # counts each record at the codec commit and prints one frame-sent line per Present, keyed + # by raw opcode; the server role's backend counts its draws and prints one stats line per + # Present. Under inproc and spawn the client has no backend of its own, so draws on both + # sides mean the picture was drawn from records that crossed the wire. A tcp server is + # launched outside this script with an environment that is not ours, so its count is + # not required. + set(split_draw_opcode "-") + set(split_wire_records "-") + set(split_server_draws "-") + if(trace_wire_evidence) + set(split_wire_table "${CMAKE_CURRENT_LIST_DIR}/../../MobileGL/MG_Pipe/generated/PipeWire.inc") + set(split_draw_op_line "") + if(EXISTS "${split_wire_table}") + file(STRINGS "${split_wire_table}" split_draw_op_line REGEX "^[ \t]*DrawVbo = [0-9]+,") + endif() + if(NOT split_draw_op_line MATCHES "DrawVbo = ([0-9]+),") + message(FATAL_ERROR "${split_case}: cannot find the DrawVbo opcode in ${split_wire_table}, " + "so the wire's draw records cannot be counted.") + endif() + set(split_draw_opcode "${CMAKE_MATCH_1}") + set(split_wire_records 0) + file(STRINGS "${mobilegl_log}" split_sent_lines REGEX "P65LinkMetrics kind=frame-sent ") + foreach(line IN LISTS split_sent_lines) + if(line MATCHES "by_op=([0-9=,]+)") + if(",${CMAKE_MATCH_1}" MATCHES ",${split_draw_opcode}=([0-9]+)") + math(EXPR split_wire_records "${split_wire_records} + ${CMAKE_MATCH_1}") + endif() + endif() + endforeach() + if(split_wire_records EQUAL 0) + message(FATAL_ERROR + "${split_case}: MOBILEGL_TRANSPORT=$ENV{MOBILEGL_TRANSPORT} committed no DrawVbo " + "record to the wire: ${mobilegl_log} carries no frame-sent line counting opcode " + "${split_draw_opcode}. Either the transport resolved and the emit table did not put " + "a draw on it, or MOBILEGL_PIPE_STATS did not reach the client.") + endif() + if("$ENV{MOBILEGL_IPC_CONTROL}" MATCHES "^tcp://") + set(split_server_draws "n/a") + else() + set(split_server_draws 0) + file(STRINGS "${mobilegl_server_log}" split_stats_lines REGEX "MGPipe stats:.* draws=[0-9]+ ") + foreach(line IN LISTS split_stats_lines) + if(line MATCHES " draws=([0-9]+) ") + math(EXPR split_server_draws "${split_server_draws} + ${CMAKE_MATCH_1}") + endif() + endforeach() + if(split_server_draws EQUAL 0) + message(FATAL_ERROR + "${split_case}: MOBILEGL_TRANSPORT=$ENV{MOBILEGL_TRANSPORT}: the server role's " + "backend executed no draw - ${mobilegl_server_log} carries no MGPipe stats line " + "with draws above zero, so nothing shows the records were drawn on that side.") + endif() + endif() + endif() + # The per-leg evidence that every assertion above ran and found what it needed. It is + # written only here, after the last of them, into an output directory wiped at the top of + # this script, so a file that exists is this run's; CI reads it back after ctest. + if("${split_expected_second}" STREQUAL "") + set(split_second_found "-") + else() + set(split_second_found "${split_expected_second}") + endif() + if("$ENV{MOBILEGL_IPC_CONTROL}" STREQUAL "") + set(split_control "-") + else() + set(split_control "$ENV{MOBILEGL_IPC_CONTROL}") + endif() + string(CONCAT split_evidence + "case=${TRACE_CASE_NAME}\nbackend=${TRACE_BACKEND}\ntransport=$ENV{MOBILEGL_TRANSPORT}\n" + "control=${split_control}\nmarker=${split_expected_marker}\nsecond=${split_second_found}\n" + "server_log=1\nfatal_lines=${split_fatal_count}\nfloor_lines=${split_floor_count}\n" + "wire=${trace_wire_evidence}\ndraw_opcode=${split_draw_opcode}\n" + "wire_draw_records=${split_wire_records}\nserver_draws=${split_server_draws}\n") + file(WRITE "${TRACE_OUTPUT_DIR}/output/split-evidence.txt" "${split_evidence}") + message(STATUS "MGPipe split evidence: ${split_case} transport=$ENV{MOBILEGL_TRANSPORT} " + "marker found, second='${split_second_found}', wire=${trace_wire_evidence}, " + "wire_draw_records=${split_wire_records}, server_draws=${split_server_draws}") endif() endif() diff --git a/tools/trace_replay/test_run_trace_case.py b/tools/trace_replay/test_run_trace_case.py new file mode 100644 index 00000000..f2acc431 --- /dev/null +++ b/tools/trace_replay/test_run_trace_case.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +# MobileGL - tools/trace_replay/test_run_trace_case.py +# Copyright (c) 2025-2026 MobileGL-Dev +# Licensed under the GNU Lesser General Public License v3.0: +# https://www.gnu.org/licenses/gpl-3.0.txt +# https://www.gnu.org/licenses/lgpl-3.0.txt +# SPDX-License-Identifier: LGPL-3.0-only +# End of Source File Header + +"""Canned-log tests of run_trace_case.cmake's split and verify assertions. + +Each case runs the script exactly as ctest does (`cmake -D... -P run_trace_case.cmake`) with a +stub replay that copies canned library logs into the output directory. A bad log must turn the +case red with the script's own sentence and leave no split-evidence.txt; a good one must pass and +leave evidence that scripts/ci/retrace_split_evidence.py accepts. No build, GPU or trace needed. +""" +import contextlib +import io +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +SCRIPT = HERE / 'run_trace_case.cmake' +CHECKER = HERE.parents[1] / 'scripts' / 'ci' / 'retrace_split_evidence.py' +sys.path.insert(0, str(CHECKER.parent)) +import retrace_split_evidence as checker # noqa: E402 + +# The stub replay. It refuses to run when the wire counters disagree with what the leg asked for, +# so removing the script's MOBILEGL_PIPE_STATS export turns every wire case red, and exporting it +# on a leg that did not ask turns that leg red too. +STUB = r'''#!/usr/bin/env python3 +import json, os, shutil, sys +args = sys.argv[1:] +out = args[args.index('--output') + 1] +split = os.environ.get('MOBILEGL_TRANSPORT', '') not in ('', 'monolith') +wire = os.environ.get('MGTRACE_WIRE_EVIDENCE', '') not in ('', '0') +stats = os.environ.get('MOBILEGL_PIPE_STATS', '') +if wire and split and (stats != '1' or os.environ.get('MOBILEGL_PIPE_STATS_PERIOD') != '1'): + sys.stderr.write('stub: MOBILEGL_PIPE_STATS did not reach the replay\n') + sys.exit(3) +if not wire and stats: + sys.stderr.write('stub: MOBILEGL_PIPE_STATS reached a leg that did not ask for it\n') + sys.exit(4) +canned = os.environ.get('STUB_CANNED_DIR', '') +if canned: + for name in os.listdir(canned): + shutil.copy(os.path.join(canned, name), os.path.join(out, name)) +with open(os.path.join(out, 'retrace.log'), 'w') as log: + log.write('stub replay\n') +with open(os.path.join(out, 'result.json'), 'w') as result: + json.dump({'passed': True, 'ssim': 1.0}, result) +''' + +PREFIX = '[12:00:00] [Linux mobilegl_trace_/INFO]: ' +MARKER = { + 'inproc': 'Config: MOBILEGL_TRANSPORT=inproc - the MGPipe record stream crosses a real ring to an apply thread', + 'spawn': 'Config: MOBILEGL_TRANSPORT=spawn - the MGPipe record stream crosses a real ring to an apply ' + 'thread in ANOTHER PROCESS', +} +SPAWN_ARMED = 'MG_Remote client: spawn ARMED - the server role runs in pid 4242, reached at /tmp/x' +TCP_ARMED = 'MG_Remote client: control=tcp data=stream server=127.0.0.1:1 pid=4243 dial=connect' +SENT = ['P65LinkMetrics kind=frame-sent frame=1 records=40 by_op=2=4,57=1,59=30,67=1', + 'P65LinkMetrics kind=frame-sent frame=2 records=31 by_op=57=1,59=28,67=1'] +STATS_ON = 'MGPipe stats: counters ON (MOBILEGL_PIPE_STATS), summary every 1 frames' +STATS = ['MGPipe stats: frames=1 window=1 draws=30 draws/f=30.00 acc=12 acc/draw=0.40 bytes/f[a=1]', + 'MGPipe stats: frames=2 window=1 draws=28 draws/f=28.00 acc=10 acc/draw=0.36 bytes/f[a=1]'] +SERVER_START = 'MG_Remote server: apply thread started' + + +def quiet_check(argv): + """The checker's exit status, with its ::error:: lines kept off a green run's annotations.""" + with contextlib.redirect_stdout(io.StringIO()): + return checker.main(argv) + + +def lines(*rows): + return ''.join(PREFIX + row + '\n' for row in rows) + + +def good_logs(transport): + client = [MARKER['spawn' if transport == 'tcp' else transport]] + if transport == 'spawn': + client.append(SPAWN_ARMED) + if transport == 'tcp': + client.append(TCP_ARMED) + client += SENT + server = [SERVER_START, STATS_ON] + ([] if transport == 'tcp' else STATS) + return {'mobilegl.client.log': lines(*client), 'mobilegl.server.log': lines(*server)} + + +def squash(text): + return re.sub(r'\s+', ' ', text) + + +@unittest.skipIf(os.name == 'nt', 'the stub replay is a POSIX executable script') +class RunTraceCaseTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix='run-trace-case-') + self.root = Path(self.temp.name) + payload = self.root / 't.trace' + payload.write_text('not a real trace\n') + self.archive = self.root / 't.tar.gz' + with tarfile.open(self.archive, 'w:gz') as tar: + tar.add(payload, arcname='t.trace') + self.stub = self.root / 'stub_replay' + self.stub.write_text(STUB) + self.stub.chmod(0o755) + self.out = self.root / 'case' + + def tearDown(self): + self.temp.cleanup() + + def run_case(self, env, logs, tcp=False): + canned = self.root / 'canned' + shutil.rmtree(canned, ignore_errors=True) + canned.mkdir() + for name, text in logs.items(): + (canned / name).write_text(text) + environment = {key: value for key, value in os.environ.items() + if not key.startswith(('MOBILEGL_', 'MGTRACE_'))} + environment.update(env) + environment['STUB_CANNED_DIR'] = str(canned) + command = ['cmake', f'-DTRACE_REPLAY_EXE={self.stub}', f'-DMOBILEGL_LIBRARY={self.root}/libMobileGL.so', + f'-DTRACE_ARCHIVE={self.archive}', '-DTRACE_FILE=t.trace', + f'-DTRACE_GOLDEN={self.root}/golden.png', f'-DTRACE_OUTPUT_DIR={self.out}', + f'-DTRACE_ARTIFACT_DIR={self.root}/artifacts', '-DTRACE_BACKEND=DirectGLES', + '-DTRACE_CASE_NAME=Canned', '-DTRACE_TARGET_CALL=1', '-DTRACE_WIDTH=1', '-DTRACE_HEIGHT=1'] + if tcp: + command += ['-DTRACE_IPC_CONTROL=tcp://127.0.0.1:1', '-DTRACE_IPC_TOKEN=x'] + command += ['-P', str(SCRIPT)] + result = subprocess.run(command, env=environment, capture_output=True, text=True) + return result.returncode, squash(result.stdout + result.stderr) + + def split_env(self, transport, wire=False): + env = {'MOBILEGL_TRANSPORT': 'spawn' if transport == 'tcp' else transport, + 'MOBILEGL_IPC_SERVER_PATH': str(self.root / 'libMobileGLServer.so')} + if wire: + env['MGTRACE_WIRE_EVIDENCE'] = '1' + return env + + @property + def evidence(self): + return self.out / 'output' / 'split-evidence.txt' + + def expect_red(self, env, logs, sentence, tcp=False): + rc, output = self.run_case(env, logs, tcp=tcp) + self.assertNotEqual(rc, 0, output) + self.assertIn(squash(sentence), output) + self.assertFalse(self.evidence.exists(), 'a red leg must leave no evidence file') + + def expect_green(self, env, logs, transport, tcp=False, wire=False): + rc, output = self.run_case(env, logs, tcp=tcp) + self.assertEqual(rc, 0, output) + self.assertEqual(quiet_check([str(self.evidence), '--transport', transport, '--case', 'Canned', + '--backend', 'DirectGLES'] + (['--require-wire'] if wire else [])), 0) + return output + + # Red: each assertion of the transport block, fed the log that must trip it. + def test_r1_a_library_without_a_transport_writes_no_client_log(self): + self.expect_red(self.split_env('inproc'), {'mobilegl.log': lines('Config: loaded')}, + 'so there is no evidence the transport ever resolved') + + def test_r2_the_env_dump_line_is_not_the_marker(self): + logs = {'mobilegl.client.log': lines('Config: Accepted env variable: MOBILEGL_TRANSPORT=inproc'), + 'mobilegl.server.log': lines(SERVER_START)} + self.expect_red(self.split_env('inproc'), logs, 'never reported resolving it') + + def test_r3_spawn_without_a_separate_server_pid(self): + logs = good_logs('spawn') + logs['mobilegl.client.log'] = lines(MARKER['spawn'], *SENT) + self.expect_red(self.split_env('spawn'), logs, 'spawn ARMED - the server role runs in pid') + + def test_r4_tcp_without_endpoint_and_pid(self): + logs = good_logs('tcp') + logs['mobilegl.client.log'] = lines(MARKER['spawn'], *SENT) + self.expect_red(self.split_env('tcp'), logs, 'TCP session has no endpoint and server pid arm proof', + tcp=True) + + def test_r5_no_server_log(self): + logs = good_logs('inproc') + del logs['mobilegl.server.log'] + self.expect_red(self.split_env('inproc'), logs, 'but the run wrote no') + + def test_r6_a_server_fatal(self): + logs = good_logs('inproc') + logs['mobilegl.server.log'] += lines('MGPipe: Fatal{UnmigratedVerb, "X"}') + self.expect_red(self.split_env('inproc'), logs, 'MGPipe Fatal(s) under MOBILEGL_TRANSPORT') + + def test_r7_a_floor_line(self): + logs = good_logs('inproc') + logs['mobilegl.server.log'] += lines('MGWIRE-FLOOR unsound-serial-complete serial=3') + self.expect_red(self.split_env('inproc'), logs, 'MGWIRE-FLOOR unsound-serial-complete line(s)') + + def test_r8_no_frame_sent_lines(self): + logs = good_logs('inproc') + logs['mobilegl.client.log'] = lines(MARKER['inproc']) + self.expect_red(self.split_env('inproc', wire=True), logs, 'committed no DrawVbo record to the wire') + + def test_r9_frames_sent_without_a_draw(self): + logs = good_logs('inproc') + logs['mobilegl.client.log'] = lines(MARKER['inproc'], 'P65LinkMetrics kind=frame-sent frame=1 ' + 'records=2 by_op=57=1,67=1,159=4,590=2') + self.expect_red(self.split_env('inproc', wire=True), logs, 'committed no DrawVbo record to the wire') + + def test_r10_the_server_drew_nothing(self): + logs = good_logs('inproc') + logs['mobilegl.server.log'] = lines(SERVER_START, STATS_ON, + 'MGPipe stats: frames=1 window=1 draws=0 draws/f=0.00 acc=0') + self.expect_red(self.split_env('inproc', wire=True), logs, 'backend executed no draw') + + def test_r11_an_unknown_transport(self): + env = self.split_env('inproc') + env['MOBILEGL_TRANSPORT'] = 'unix:/x' + self.expect_red(env, good_logs('inproc'), 'this runner has no marker for it') + + def test_r12_wire_evidence_without_a_split_transport(self): + self.expect_red({'MGTRACE_WIRE_EVIDENCE': '1', 'MOBILEGL_TRANSPORT': 'monolith'}, {}, + 'a run with no wire has no wire volume to count') + + def test_r13_the_stub_refuses_counters_on_a_leg_that_did_not_ask(self): + env = self.split_env('inproc') + env['MOBILEGL_PIPE_STATS'] = '1' + self.expect_red(env, good_logs('inproc'), 'stub: MOBILEGL_PIPE_STATS reached a leg that did not ask for it') + + # Green: the same logs as a real leg writes, on every arm. + def test_g1_inproc_with_wire(self): + output = self.expect_green(self.split_env('inproc', wire=True), good_logs('inproc'), 'inproc', wire=True) + self.assertIn('wire_draw_records=58, server_draws=58', output) + + def test_g1b_inproc_marker_only(self): + self.expect_green(self.split_env('inproc'), good_logs('inproc'), 'inproc') + values = checker.parse(self.evidence.read_text()) + self.assertEqual((values['wire'], values['wire_draw_records']), ('OFF', '-')) + self.assertNotEqual(quiet_check([str(self.evidence), '--transport', 'inproc', '--require-wire']), 0) + + def test_g2_spawn_with_wire(self): + self.expect_green(self.split_env('spawn', wire=True), good_logs('spawn'), 'spawn', wire=True) + + def test_g3_tcp_with_wire(self): + self.expect_green(self.split_env('tcp', wire=True), good_logs('tcp'), 'spawn', tcp=True, wire=True) + self.assertEqual(checker.parse(self.evidence.read_text())['server_draws'], 'n/a') + + def test_g4_monolith_leaves_no_evidence(self): + rc, output = self.run_case({'MOBILEGL_TRANSPORT': 'monolith'}, {'mobilegl.log': lines('Config: loaded')}) + self.assertEqual(rc, 0, output) + self.assertFalse(self.evidence.exists()) + self.assertNotEqual(quiet_check([str(self.evidence), '--transport', 'inproc']), 0, + 'a leg whose transport block was skipped must fail the checker') + + # The verify block, which shares the script. + def test_v1_verify_without_the_armed_line(self): + self.expect_red({'MOBILEGL_PIPE_VERIFY': '1'}, {'mobilegl.log': lines('Config: loaded')}, + 'never reported "MGPipe: verify armed"') + + def test_v2_verify_with_a_divergence(self): + logs = {'mobilegl.log': lines('MGPipe: verify armed', + 'MGPipe: Fatal{PipeVerifyDiffer, "GetRenderStateParameters@Clear"}')} + self.expect_red({'MOBILEGL_PIPE_VERIFY': '1'}, logs, 'MGPipe Fatal(s) under MOBILEGL_PIPE_VERIFY') + + def test_v3_verify_green(self): + rc, output = self.run_case({'MOBILEGL_PIPE_VERIFY': '1'}, {'mobilegl.log': lines('MGPipe: verify armed')}) + self.assertEqual(rc, 0, output) + self.assertIn('armed, zero divergences', output) + + +class EvidenceCheckerTests(unittest.TestCase): + GOOD = ('case=Canned\nbackend=DirectGLES\ntransport=spawn\ncontrol=-\nmarker=m\n' + 'second=spawn ARMED - the server role runs in pid \nserver_log=1\nfatal_lines=0\nfloor_lines=0\n' + 'wire=ON\ndraw_opcode=59\nwire_draw_records=58\nserver_draws=58\n') + + def check(self, text, *args): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'split-evidence.txt' + if text is not None: + path.write_text(text) + return quiet_check([str(path)] + list(args)) + + def test_good_file_passes(self): + self.assertEqual(self.check(self.GOOD, '--transport', 'spawn', '--require-wire'), 0) + + def test_missing_file_fails(self): + self.assertEqual(self.check(None, '--transport', 'spawn'), 1) + + def test_transport_mismatch_fails(self): + self.assertEqual(self.check(self.GOOD.replace('transport=spawn', 'transport=inproc'), + '--transport', 'spawn'), 1) + + def test_case_mismatch_fails(self): + self.assertEqual(self.check(self.GOOD, '--transport', 'spawn', '--case', 'OpenRA'), 1) + + def test_zero_wire_records_fail(self): + self.assertEqual(self.check(self.GOOD.replace('wire_draw_records=58', 'wire_draw_records=0'), + '--transport', 'spawn'), 1) + + def test_zero_server_draws_fail(self): + self.assertEqual(self.check(self.GOOD.replace('server_draws=58', 'server_draws=0'), + '--transport', 'spawn'), 1) + + def test_spawn_without_second_sentence_fails(self): + self.assertEqual(self.check(self.GOOD.replace('second=spawn ARMED - the server role runs in pid ', + 'second=-'), '--transport', 'spawn'), 1) + + def test_a_fatal_count_fails(self): + self.assertEqual(self.check(self.GOOD.replace('fatal_lines=0', 'fatal_lines=2'), + '--transport', 'spawn'), 1) + + +if __name__ == '__main__': + unittest.main()