[MG_Remote, MG_Pipe, MG_Test, CI, tools, docs] (P6.5): the first wave completed and reviewed - archive v2, publication boundary, half-close, AcceptPair/role fixes, device re-validation

scripts/ci/junit_tally.py

@@ -37,7 +37,7 @@ ARM_PREFIXES = {
 TCP_ARM = re.compile(r'control=tcp data=stream server=\S+ pid=[1-9][0-9]*\b')
 
 
-def require_tcp_proof(path, prefix, discovery):
+def require_tcp_proof(path, prefix, discovery, require_run_ahead=False):
     helper_path = Path(__file__).resolve().parents[2] / 'MobileGL/MG_IntegrationTest/Harness/split_log_paths.py'
     spec = importlib.util.spec_from_file_location('split_log_paths', helper_path)
     helper = importlib.util.module_from_spec(spec)
@@ -56,6 +56,9 @@ def require_tcp_proof(path, prefix, discovery):
         output = helper.read_role_logs(logs[name])
         if not TCP_ARM.search(output):
             raise ValueError(f"{case.get('name')}: missing TCP/stream endpoint and server pid arm proof")
+        if require_run_ahead and ('run-ahead ARMED' not in output or 'running lockstep' in output
+                                  or 'run-ahead DISARMED' in output):
+            raise ValueError(f"{case.get('name')}: TCP runtime did not keep requested run-ahead armed")
 
 
 def tally(path, prefix=None):
@@ -79,14 +82,18 @@ def main():
     for flag, prefix in ARM_PREFIXES.items():
         arms.add_argument(flag, dest='prefix', action='store_const', const=prefix)
     parser.add_argument('--discovery', help='CTest --show-only=json-v1 output (required for TCP proof)')
+    parser.add_argument('--require-run-ahead', action='store_true',
+                        help='Additionally prove actual run-ahead, without fallback or demotion, on each TCP case')
     args = parser.parse_args()
     if args.prefix in ('DirectGLES.Tcp.', 'DirectGLES.TcpDevice.') and not args.discovery:
         parser.error('TCP arm proof requires --discovery to locate each private role log')
+    if args.require_run_ahead and args.prefix not in ('DirectGLES.Tcp.', 'DirectGLES.TcpDevice.'):
+        parser.error('--require-run-ahead requires a TCP arm selection')
     try:
         prefix = args.prefix
         passed, failed, skipped = tally(args.junit, prefix=prefix)
         if prefix in ('DirectGLES.Tcp.', 'DirectGLES.TcpDevice.'):
-            require_tcp_proof(args.junit, prefix, args.discovery)
+            require_tcp_proof(args.junit, prefix, args.discovery, args.require_run_ahead)
     except Exception as exc:  # a malformed file is not "zero of everything"
         print(f"junit_tally: cannot prove {args.junit}: {exc}", file=sys.stderr)
         return 1

scripts/ci/link_seam_purity.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+"""P6.5 ownership and dispatch boundary, with explicit compatibility limits.
+
+The old raw constructors and Session::Control/Shm test/bootstrap accessors remain
+source compatible. Ring endpoints are cached, non-owning setup handles, shared
+by both data planes. This gate does not pretend that their type names disappeared;
+it rejects ownership and concrete-link dispatch outside Transport instead.
+"""
+from __future__ import annotations
+import argparse
+from pathlib import Path
+import re
+
+ROOT = Path(__file__).resolve().parents[2]
+SOURCES = [
+    "MobileGL/MG_Remote/Client/ClientSession.h",
+    "MobileGL/MG_Remote/Client/ClientSession.cpp",
+    "MobileGL/MG_Remote/Server/ServerSession.h",
+    "MobileGL/MG_Remote/Server/ServerSession.cpp",
+    "MobileGL/MG_Remote/Server/ServerLoop.cpp",
+    "MobileGL/MG_Remote/Wire/PipeWireCodec.h",
+    "MobileGL/MG_Remote/Wire/PipeWireCodec.cpp",
+    "MobileGL/MG_Impl/Pipe/ResourceTracker.h",
+]
+PHYSICAL = r"(?:SessionSegments|ReplySlotPool|RingProducer|RingConsumer|EventRingProducer|EventRingConsumer)"
+
+def code_only(source: str) -> str:
+    return re.sub(r"/\*.*?\*/|//[^\n]*", "", source, flags=re.S)
+
+def violations(path: str, text: str) -> list[str]:
+    code = code_only(text)
+    found: list[str] = []
+    rules = [
+        (rf"\b{PHYSICAL}\s+m_\w+\s*(?:[;{{=])", "physical data-plane ownership outside Transport"),
+        (r"\bm_shm\s*\.", "session bypasses its link for segment access"),
+        (r"(?:new\s+(?:Transport::)?|make_unique\s*<\s*(?:Transport::)?)(?:ShmLink|StreamLink)\b", "concrete link selected outside Transport factory"),
+        (r'#\s*include\s*[<"][^>"\n]*(?:ShmLink|StreamLink)\.h', "concrete link included above Transport"),
+    ]
+    if path.endswith("ServerLoop.cpp"):
+        rules.append((r"\bRingControl\b|(?:\.|->)Control\(\)", "apply loop reads a physical control page"))
+    if path.endswith("PipeWireCodec.h"):
+        rules.append((r"\bRingControl\s*\*\s*m_", "codec stores physical page instead of setup progress/signals"))
+    if path.endswith("ResourceTracker.h"):
+        rules.append((r"Segments\(\)\.Resolve", "writeback bypasses ILink span resolution"))
+    for pattern, message in rules:
+        for match in re.finditer(pattern, code):
+            line = code.count("\n", 0, match.start()) + 1
+            found.append(f"{path}:{line}: {message}")
+    return found
+
+def self_test() -> None:
+    controls = [
+        (SOURCES[0], "Transport::SessionSegments m_segments;"),
+        (SOURCES[0], "Transport::ReplySlotPool m_replies;"),
+        (SOURCES[1], "m_shm.Close();"),
+        (SOURCES[1], "auto link = std::make_unique<Transport::StreamLink>();"),
+        (SOURCES[4], "auto& page = session.Control();"),
+        (SOURCES[5], "Transport::RingControl* m_control = nullptr;"),
+        (SOURCES[-1], "session->Segments().Resolve(4, offset, size);"),
+    ]
+    for path, source in controls:
+        assert violations(path, source), f"negative control did not turn red: {source}"
+    assert not violations(SOURCES[0], "Transport::RingProducer* m_cmd = nullptr;")
+    print(f"link seam negative controls: {len(controls)} named failures observed")
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--self-test", action="store_true")
+    args = parser.parse_args()
+    if args.self_test:
+        self_test()
+    failures = [failure for name in SOURCES for failure in violations(name, (ROOT / name).read_text(encoding="utf-8"))]
+    if failures:
+        print("\n".join(failures))
+        return 1
+    print("link seam ownership/dispatch gate: PASS (cached endpoints and legacy test constructors retained)")
+    return 0
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/ci/p65_link_metrics.py

@@ -75,9 +75,27 @@ def summarize(path: Path, warmup: int, throughput: float | None = None) -> dict:
     return result
 
 
+def summarize_server(path: Path, warmup: int) -> dict:
+    frames = []
+    for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
+        if "P65ServerMetrics " not in line:
+            continue
+        frame = dict(FIELDS.findall(line.split("P65ServerMetrics ", 1)[1]))
+        if frame.get("valid") == "1" and int(frame["frame"]) > warmup:
+            frames.append(frame)
+    if not frames:
+        raise ValueError(f"{path}: no valid server CPU frames after warmup={warmup}")
+    cpu = sum(int(f["apply_thread_cpu_ns"]) for f in frames)
+    wall = sum(int(f["wall_ns"]) for f in frames)
+    return {"server_log": str(path), "server_frames": len(frames),
+            "apply_thread_cpu_ms_per_frame": cpu / len(frames) / 1e6,
+            "server_fps": len(frames) * 1e9 / wall if wall else None}
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--arm", action="append", required=True, metavar="LABEL:LOG")
+    parser.add_argument("--server-arm", action="append", default=[], metavar="LABEL:LOG")
     parser.add_argument("--warmup-frames", type=int, default=0)
     parser.add_argument("--link-mib-per-second", type=float)
     parser.add_argument("--output", type=Path)
@@ -93,11 +111,22 @@ def main() -> int:
             report[label] = summarize(Path(filename), args.warmup_frames, args.link_mib_per_second)
         except (OSError, ValueError, KeyError) as error:
             parser.error(str(error))
+    for arm in args.server_arm:
+        label, sep, filename = arm.partition(":")
+        if not sep or label not in report:
+            parser.error("each --server-arm must name an existing client arm")
+        try:
+            report[label].update(summarize_server(Path(filename), args.warmup_frames))
+        except (OSError, ValueError, KeyError) as error:
+            parser.error(str(error))
     if "spawn" in report:
         baseline = report["spawn"]["client_thread_cpu_ms_per_frame"]
         for label, arm in report.items():
             if label != "spawn":
                 arm["client_thread_cpu_delta_vs_spawn_ms"] = arm["client_thread_cpu_ms_per_frame"] - baseline
+                if "apply_thread_cpu_ms_per_frame" in arm and "apply_thread_cpu_ms_per_frame" in report["spawn"]:
+                    arm["apply_thread_cpu_delta_vs_spawn_ms"] = (arm["apply_thread_cpu_ms_per_frame"] -
+                                                                report["spawn"]["apply_thread_cpu_ms_per_frame"])
     text = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
     if args.output:
         args.output.write_text(text, encoding="utf-8")

scripts/ci/tcp_lane_tools_test.py

@@ -35,6 +35,15 @@ class Accounting(unittest.TestCase):
             role = root / 'case.client.log'
             role.write_text('control=tcp data=stream server=127.0.0.1:40613 pid=42\n')
             tally.require_tcp_proof(junit, 'DirectGLES.Tcp.', discovery)
+            with self.assertRaises(ValueError):
+                tally.require_tcp_proof(junit, 'DirectGLES.Tcp.', discovery, require_run_ahead=True)
+            armed = 'control=tcp data=stream server=127.0.0.1:40613 pid=42\nrun-ahead ARMED\n'
+            role.write_text(armed)
+            tally.require_tcp_proof(junit, 'DirectGLES.Tcp.', discovery, require_run_ahead=True)
+            for fallback in ('running lockstep', 'run-ahead DISARMED'):
+                role.write_text(armed + fallback)
+                with self.assertRaises(ValueError):
+                    tally.require_tcp_proof(junit, 'DirectGLES.Tcp.', discovery, require_run_ahead=True)
             for bad in ('', 'control=tcp data=shm server=127.0.0.1:40613 pid=42',
                         'control=tcp data=stream server=127.0.0.1:40613 pid=0',
                         'spawn ARMED - the server role runs in pid 42'):
@@ -53,6 +62,38 @@ class Accounting(unittest.TestCase):
             self.assertEqual(parity.main(), 0)
 
 
+class DeviceIdleExemption(unittest.TestCase):
+    def test_stop_restores_only_the_setting_this_task_changed(self):
+        spec = importlib.util.spec_from_file_location('tcp_device_server', ROOT / 'tools/trace_replay/tcp_device_server.py')
+        device = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(device)
+        package = 'top.mobilegl.plugin.trace'
+        for initially_allowed in (False, True):
+            allowed = {package} if initially_allowed else set()
+            def shell(_adb, command):
+                if len(command) == 3:
+                    return '\n'.join('user,' + name + ',10325' for name in sorted(allowed))
+                change = command[3]
+                if change.startswith('+'):
+                    allowed.add(change[1:])
+                else:
+                    allowed.discard(change[1:])
+                return ''
+            with tempfile.TemporaryDirectory() as directory, patch.object(device, 'shell', side_effect=shell):
+                state = Path(directory) / 'original.json'
+                device.allow_idle([], 'phone', package, state)
+                # A restarted supervisor must retain the first original state.
+                device.allow_idle([], 'phone', package, state)
+                self.assertIn(package, allowed)
+                self.assertEqual(json.loads(state.read_text())['originally_whitelisted'], initially_allowed)
+                with self.assertRaises(ValueError):
+                    device.restore_idle([], 'different-phone', package, state)
+                self.assertTrue(state.exists())
+                device.restore_idle([], 'phone', package, state)
+                self.assertEqual(package in allowed, initially_allowed)
+                self.assertFalse(state.exists())
+
+
 @unittest.skipUnless(sys.platform.startswith('linux'), 'supervisor fixture uses Linux /proc')
 class Supervisor(unittest.TestCase):
     def test_readiness_does_not_consume_a_connection_and_stop_owns_only_its_pid(self):
@@ -82,6 +123,10 @@ class Supervisor(unittest.TestCase):
                 unrelated = root / 'unrelated.json'
                 unrelated.write_text(json.dumps({'pid': os.getpid(), 'start': 'not-this-process'}))
                 fixture.stop(unrelated)
+                already_exited = root / 'already-exited.json'
+                already_exited.write_text(json.dumps({'pid': 2147483647, 'start': None}))
+                fixture.stop(already_exited)
+                self.assertFalse(already_exited.exists())
             finally:
                 fixture.stop(state)
             with self.assertRaises(OSError):

scripts/ci/tcp_log_forwarding_control.py

@@ -47,6 +47,8 @@ def run_arm(build, out, case, forward):
         env['VK_ICD_FILENAMES'] = '/usr/share/vulkan/icd.d/lvp_icd.json'
     peer_env = dict(env, MOBILEGL_IPC_ROLE='server', MOBILEGL_IPC_DIAL='no',
                     MOBILEGL_IPC_LOG_FORWARD=str(int(forward)), MOBILEGL_LOG_FILE_PATH=str(out / 'peer.log'))
+    for name in ('MOBILEGL_TRANSPORT', 'MOBILEGL_IPC_SERVER_PATH', 'MOBILEGL_IPC_RING_MB', 'MOBILEGL_IPC_STAGE_MB'):
+        peer_env.pop(name, None)
     with (out / 'supervisor.log').open('wb') as output:
         server = subprocess.Popen([str(build / 'libMobileGLServer.so'), endpoint, '--serve'],
                                   env=peer_env, stdout=output, stderr=output, start_new_session=True)
@@ -79,12 +81,15 @@ def run_arm(build, out, case, forward):
             pass
         server.wait(timeout=10)
     passed = failed = skipped = 0
+    marker_assertion_failed = False
     if (out / 'result.xml').exists():
         for entry in ET.parse(out / 'result.xml').getroot().iter('testcase'):
             if entry.find('skipped') is not None or entry.get('status') == 'notrun':
                 skipped += 1
             elif entry.find('failure') is not None:
                 failed += 1
+                marker_assertion_failed = any(marker in (failure.get('message', '') + (failure.text or ''))
+                                              for failure in entry.findall('failure'))
             else:
                 passed += 1
     client_log = read(out / 'client.client.log')
@@ -93,6 +98,7 @@ def run_arm(build, out, case, forward):
             'marker_in_client': marker in client_log,
             'marker_in_server': marker in read(out / 'peer.server.log'),
             'marker_forwarded': marker in read(out / 'client.server.log'),
+            'marker_assertion_failed': marker_assertion_failed,
             'output': str(out)}
 
 
@@ -132,6 +138,7 @@ def main():
                                  and on.get('tcp_armed') and on.get('marker_in_server') and on.get('marker_forwarded')
                                  and off.get('failed') == 1 and not off.get('skipped') and off.get('tcp_armed')
                                  and off.get('marker_in_server') and not off.get('marker_forwarded')
+                                 and off.get('marker_assertion_failed')
                                  and not off.get('marker_in_client'))
         results[case] = arms
         print(case + ': ' + ('PASS' if arms['proved'] else 'NOT PROVEN'), flush=True)

scripts/ci/tcp_server_fixture.py

@@ -26,8 +26,12 @@ def stop(state):
     saved = json.loads(state.read_text())
     pid = saved['pid']
     # PID reuse must never let a stale fixture file kill an unrelated process.
-    if process_start(pid) == saved['start']:
-        os.killpg(pid, signal.SIGTERM)
+    if saved['start'] is not None and process_start(pid) == saved['start']:
+        try:
+            os.killpg(pid, signal.SIGTERM)
+        except ProcessLookupError:
+            state.unlink(missing_ok=True)
+            return
         for _ in range(50):
             if process_start(pid) != saved['start']:
                 break
@@ -37,7 +41,7 @@ def stop(state):
                 os.killpg(pid, signal.SIGKILL)
             except ProcessLookupError:
                 pass
-    state.unlink()
+    state.unlink(missing_ok=True)
 
 
 def start(args):
@@ -54,8 +58,10 @@ def start(args):
     if args.state.exists():
         raise RuntimeError(f'stale fixture state; run stop first: {args.state}')
     env = dict(os.environ, MOBILEGL_IPC_ROLE='server', MOBILEGL_IPC_DIAL='no',
-               MOBILEGL_IPC_TOKEN='', MOBILEGL_IPC_LOG_FORWARD='1')
-    for key in ('MOBILEGL_IPC_CONTROL', 'MOBILEGL_IPC_ENDPOINT', 'MOBILEGL_BACKEND_TYPE'):
+               MOBILEGL_IPC_TOKEN='', MOBILEGL_IPC_LOG_FORWARD='1', MOBILEGL_TEST_SERVER_COUNTERS='1',
+               MOBILEGL_PIPE_STATS='1', MOBILEGL_PIPE_STATS_PERIOD='1')
+    for key in ('MOBILEGL_TRANSPORT', 'MOBILEGL_IPC_SERVER_PATH', 'MOBILEGL_IPC_RING_MB', 'MOBILEGL_IPC_STAGE_MB',
+                'MOBILEGL_IPC_CONTROL', 'MOBILEGL_IPC_ENDPOINT', 'MOBILEGL_BACKEND_TYPE'):
         env.pop(key, None)
     log_path = args.state.with_suffix('.log')
     with log_path.open('wb') as log:

scripts/ci/tcp_supervisor_smoke.py

@@ -84,7 +84,8 @@ def supervisor(server, log, same_build=False):
     env = dict(os.environ, MOBILEGL_IPC_ROLE='server', MOBILEGL_IPC_DIAL='no',
                MOBILEGL_IPC_TOKEN='p65-smoke-token', MOBILEGL_IPC_REQUIRE_SAME_BUILD=str(int(same_build)),
                MOBILEGL_IPC_LOG_FORWARD='0', MOBILEGL_LOG_FILE_PATH=str(log.with_suffix('.library.log')))
-    for name in ('MOBILEGL_IPC_CONTROL', 'MOBILEGL_IPC_ENDPOINT', 'MOBILEGL_BACKEND_TYPE'):
+    for name in ('MOBILEGL_TRANSPORT', 'MOBILEGL_IPC_SERVER_PATH', 'MOBILEGL_IPC_RING_MB', 'MOBILEGL_IPC_STAGE_MB',
+                 'MOBILEGL_IPC_CONTROL', 'MOBILEGL_IPC_ENDPOINT', 'MOBILEGL_BACKEND_TYPE'):
         env.pop(name, None)
     with log.open('wb') as output:
         child = subprocess.Popen([server, f'tcp://127.0.0.1:{port}', '--serve'], env=env,
@@ -154,16 +155,16 @@ def main():
                     busy = receive(second, schema)
                     assert busy.get('code') == 7, busy
                     evidence['busy'] = busy
-            token = exchange(port, schema, hello(schema, flatbuffers, token='wrong-token'))
+            layout = exchange(port, schema, hello(schema, flatbuffers))
+            assert layout.get('code') == 2 and layout.get('expected', 0), layout
+            fingerprint = layout['expected']
+            evidence['wrong_layout'] = layout
+            token = exchange(port, schema, hello(schema, flatbuffers, fingerprint=fingerprint, token='wrong-token'))
             assert token.get('code') == 4, token
             evidence['wrong_token'] = token
             version = exchange(port, schema, hello(schema, flatbuffers, major=99))
             assert version.get('code') == 1, version
             evidence['wire_major_99'] = version
-            layout = exchange(port, schema, hello(schema, flatbuffers))
-            assert layout.get('code') == 2 and layout.get('expected', 0), layout
-            fingerprint = layout['expected']
-            evidence['wrong_layout'] = layout
             accepted = exchange(port, schema, hello(schema, flatbuffers, fingerprint=fingerprint))
             assert accepted.get('welcome', 0) and accepted['welcome'] != os.getpid(), accepted
             assert accepted.get('fingerprint') == fingerprint, accepted