[Fix] (CI): read role logs and retry device disconnects

android-plugin/trace-replay-ci.sh

@@ -278,7 +278,12 @@ is_angle_surface_lost() {
 is_infrastructure_failure() {
   diagnostics_dir="$1"
   adb_state="$(cat "${diagnostics_dir}/adb-state.txt" 2>/dev/null || true)"
-  if [ "${adb_state}" != "device" ]; then
+  # Diagnostics can take several adb calls: a device that was online at their
+  # start may disappear while we collect them. Keep the polling observation and
+  # check the live state as well, rather than trusting that earlier snapshot.
+  live_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
+  if [ -f "${diagnostics_dir}/adb-disconnected.txt" ] ||
+     [ "${adb_state}" != "device" ] || [ "${live_adb_state}" != "device" ]; then
     echo "trace-replay-ci.sh: Android device is unavailable (state: ${adb_state:-unknown})" >&2
     record_infrastructure_reason "device-unavailable"
     return 0
@@ -492,25 +497,35 @@ run_retrace() {
 
   app_exited=0
   saw_app_process=0
+  poll_started="$(date +%s)"
   for _ in $(seq 1 "${timeout_seconds}"); do
     if adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then
       break
     fi
     if adb_device_path shell pidof "${package_name}" >/dev/null 2>&1; then
       saw_app_process=1
-    elif [ "${saw_app_process}" -eq 1 ]; then
-      app_exited=1
-      break
+    else
+      poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
+      if [ "${poll_adb_state}" != "device" ]; then
+        printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt"
+        break
+      fi
+      if [ "${saw_app_process}" -eq 1 ]; then
+        app_exited=1
+        break
+      fi
     fi
     sleep 1
   done
 
   collect_run_diagnostics "${result_dir}"
   if ! adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then
-    if [ "${app_exited}" -eq 1 ]; then
+    if [ -f "${result_dir}/adb-disconnected.txt" ]; then
+      echo "trace-replay-ci.sh: device disconnected while waiting for result.json" >&2
+    elif [ "${app_exited}" -eq 1 ]; then
       echo "trace-replay-ci.sh: app process exited before result.json was created" >&2
     else
-      echo "trace-replay-ci.sh: result.json was not created after ${timeout_seconds}s" >&2
+      echo "trace-replay-ci.sh: result.json was not created after $(($(date +%s) - poll_started))s (budget ${timeout_seconds}s)" >&2
     fi
     if [ -s "${result_dir}/retrace.log" ]; then
       echo "trace-replay-ci.sh: retrace.log:" >&2

scripts/ci/runtime_mode_proof.py

@@ -70,21 +70,11 @@ def logs_proof(discovery, junit):
     logs = helper.marker_log_paths(lane)
     proof = {}
     for name in sorted(RSP_CASES):
-        # P6 splits the library log per role: MOBILEGL_LOG_FILE_PATH is a BASE NAME and the
-        # files that exist are <base>.client.log / <base>.server.log. Reading the base directly
-        # is FileNotFoundError after the split - the same class of reader the role-split
-        # landing already fixed everywhere else.
-        path = Path(logs[name])
-        if not helper.any_role_file(path):
-            raise RuntimeError(
-                f'{name}: no role log under {path} '
-                f'(looked for {helper.role_paths(path)}); the library never opened the sink')
-        text = helper.read_role_logs(path)
-        proof[name] = {
-            'private_log': str(path),
-            'role_logs': helper.role_paths(path),
-            **inproc_log_proof(text),
-        }
+        # MOBILEGL_LOG_FILE_PATH is a base name. In split builds neither role
+        # writes that literal path, and the server half owns applier failures.
+        paths = [Path(path) for path in helper.role_paths(logs[name])]
+        text = '\n'.join(path.read_text(encoding='utf-8', errors='replace') for path in paths)
+        proof[name] = {'private_logs': [str(path) for path in paths], **inproc_log_proof(text)}
     return proof
 
 
@@ -104,6 +94,32 @@ def self_test():
             raise AssertionError('invalid runtime log passed')
     with tempfile.TemporaryDirectory(prefix='mobilegl-runtime-proof-') as directory:
         build = Path(directory)
+        discovery = build / 'lane.json'
+        junit = build / 'lane.xml'
+        discovery.write_text(json.dumps({'tests': [
+            {'name': name, 'properties': [{'name': 'ENVIRONMENT',
+             'value': ['MOBILEGL_LOG_FILE_PATH=' + str(build / (name + '.log'))]}]}
+            for name in sorted(RSP_CASES)]}))
+        junit.write_text('<testsuite>' + ''.join(f'<testcase name="{name}" />' for name in sorted(RSP_CASES)) + '</testsuite>')
+        for name in RSP_CASES:
+            (build / (name + '.client.log')).write_text(good)
+            (build / (name + '.server.log')).write_text('server role\n')
+        logs_proof(discovery, junit)
+        server = build / (sorted(RSP_CASES)[0] + '.server.log')
+        server.write_text('Fatal{ProtocolCorruption, server-side example}\n')
+        try:
+            logs_proof(discovery, junit)
+        except RuntimeError:
+            pass
+        else:
+            raise AssertionError('server-side Fatal passed the runtime log proof')
+        server.unlink()
+        try:
+            logs_proof(discovery, junit)
+        except FileNotFoundError:
+            pass
+        else:
+            raise AssertionError('missing server log passed the runtime log proof')
         (build / 'libMobileGL.so').write_bytes(b'checker fixture, not a real runtime')
         normal = ''.join(f'00000000 T fixture{i}\n' for i in range(1000))
         remote = normal + '00000000 T MG_Remote_fixture\n00000000 T MGPipeApply_fixture\n'

scripts/ci/test_trace_infrastructure.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""Exercise the actual APK retry classifier with saved and live adb states."""
+import pathlib
+import subprocess
+import tempfile
+import unittest
+
+SCRIPT = pathlib.Path(__file__).resolve().parents[2] / "android-plugin/trace-replay-ci.sh"
+
+
+class TraceInfrastructureTest(unittest.TestCase):
+    def classify(self, saved="device", live="device", disconnected=False, logcat=""):
+        source = SCRIPT.read_text()
+        functions = source[source.index("record_infrastructure_reason() {"):source.index("copy_app_artifact() {")]
+        with tempfile.TemporaryDirectory() as directory:
+            root = pathlib.Path(directory)
+            (root / "adb-state.txt").write_text(saved)
+            (root / "logcat.txt").write_text(logcat)
+            if disconnected:
+                (root / "adb-disconnected.txt").write_text("offline")
+            shell = functions + '\nadb_device_path() { printf "%s\\n" "$LIVE"; }\n' + \
+                'result_root="$1"; LIVE="$2"; is_infrastructure_failure "$1"\n'
+            result = subprocess.run(["sh", "-c", shell, "classifier", directory, live], capture_output=True)
+            reason = root / "infrastructure-failure-reason.txt"
+            return result.returncode, reason.read_text().strip() if reason.exists() else ""
+
+    def test_device_disappears_during_diagnostics(self):
+        self.assertEqual(self.classify(live="offline"), (0, "device-unavailable"))
+
+    def test_polling_disconnect_survives_reconnect(self):
+        self.assertEqual(self.classify(disconnected=True), (0, "device-unavailable"))
+
+    def test_saved_disconnect_survives_reconnect(self):
+        self.assertEqual(self.classify(saved=""), (0, "device-unavailable"))
+
+    def test_online_application_failure_stays_failure(self):
+        self.assertEqual(self.classify(logcat="Fatal signal 11 (trace_replay)\n"), (1, ""))
+
+    def test_system_server_crash_remains_retryable(self):
+        self.assertEqual(self.classify(logcat="Fatal signal 11 (system_server)\n"), (0, "system-server-crash"))
+
+
+if __name__ == "__main__":
+    unittest.main()