diff --git a/android-plugin/trace-replay-ci.sh b/android-plugin/trace-replay-ci.sh
index 199097ad..7d67b014 100644
--- a/android-plugin/trace-replay-ci.sh
+++ b/android-plugin/trace-replay-ci.sh
@@ -278,7 +278,12 @@ is_angle_surface_lost() {
is_infrastructure_failure() {
diagnostics_dir="$1"
adb_state="$(cat "${diagnostics_dir}/adb-state.txt" 2>/dev/null || true)"
- if [ "${adb_state}" != "device" ]; then
+ # Diagnostics can take several adb calls: a device that was online at their
+ # start may disappear while we collect them. Keep the polling observation and
+ # check the live state as well, rather than trusting that earlier snapshot.
+ live_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
+ if [ -f "${diagnostics_dir}/adb-disconnected.txt" ] ||
+ [ "${adb_state}" != "device" ] || [ "${live_adb_state}" != "device" ]; then
echo "trace-replay-ci.sh: Android device is unavailable (state: ${adb_state:-unknown})" >&2
record_infrastructure_reason "device-unavailable"
return 0
@@ -492,25 +497,35 @@ run_retrace() {
app_exited=0
saw_app_process=0
+ poll_started="$(date +%s)"
for _ in $(seq 1 "${timeout_seconds}"); do
if adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then
break
fi
if adb_device_path shell pidof "${package_name}" >/dev/null 2>&1; then
saw_app_process=1
- elif [ "${saw_app_process}" -eq 1 ]; then
- app_exited=1
- break
+ else
+ poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
+ if [ "${poll_adb_state}" != "device" ]; then
+ printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt"
+ break
+ fi
+ if [ "${saw_app_process}" -eq 1 ]; then
+ app_exited=1
+ break
+ fi
fi
sleep 1
done
collect_run_diagnostics "${result_dir}"
if ! adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then
- if [ "${app_exited}" -eq 1 ]; then
+ if [ -f "${result_dir}/adb-disconnected.txt" ]; then
+ echo "trace-replay-ci.sh: device disconnected while waiting for result.json" >&2
+ elif [ "${app_exited}" -eq 1 ]; then
echo "trace-replay-ci.sh: app process exited before result.json was created" >&2
else
- echo "trace-replay-ci.sh: result.json was not created after ${timeout_seconds}s" >&2
+ echo "trace-replay-ci.sh: result.json was not created after $(($(date +%s) - poll_started))s (budget ${timeout_seconds}s)" >&2
fi
if [ -s "${result_dir}/retrace.log" ]; then
echo "trace-replay-ci.sh: retrace.log:" >&2
diff --git a/scripts/ci/runtime_mode_proof.py b/scripts/ci/runtime_mode_proof.py
index 2045fdd8..237a103b 100644
--- a/scripts/ci/runtime_mode_proof.py
+++ b/scripts/ci/runtime_mode_proof.py
@@ -70,21 +70,11 @@ def logs_proof(discovery, junit):
logs = helper.marker_log_paths(lane)
proof = {}
for name in sorted(RSP_CASES):
- # P6 splits the library log per role: MOBILEGL_LOG_FILE_PATH is a BASE NAME and the
- # files that exist are .client.log / .server.log. Reading the base directly
- # is FileNotFoundError after the split - the same class of reader the role-split
- # landing already fixed everywhere else.
- path = Path(logs[name])
- if not helper.any_role_file(path):
- raise RuntimeError(
- f'{name}: no role log under {path} '
- f'(looked for {helper.role_paths(path)}); the library never opened the sink')
- text = helper.read_role_logs(path)
- proof[name] = {
- 'private_log': str(path),
- 'role_logs': helper.role_paths(path),
- **inproc_log_proof(text),
- }
+ # MOBILEGL_LOG_FILE_PATH is a base name. In split builds neither role
+ # writes that literal path, and the server half owns applier failures.
+ paths = [Path(path) for path in helper.role_paths(logs[name])]
+ text = '\n'.join(path.read_text(encoding='utf-8', errors='replace') for path in paths)
+ proof[name] = {'private_logs': [str(path) for path in paths], **inproc_log_proof(text)}
return proof
@@ -104,6 +94,32 @@ def self_test():
raise AssertionError('invalid runtime log passed')
with tempfile.TemporaryDirectory(prefix='mobilegl-runtime-proof-') as directory:
build = Path(directory)
+ discovery = build / 'lane.json'
+ junit = build / 'lane.xml'
+ discovery.write_text(json.dumps({'tests': [
+ {'name': name, 'properties': [{'name': 'ENVIRONMENT',
+ 'value': ['MOBILEGL_LOG_FILE_PATH=' + str(build / (name + '.log'))]}]}
+ for name in sorted(RSP_CASES)]}))
+ junit.write_text('' + ''.join(f'' for name in sorted(RSP_CASES)) + '')
+ for name in RSP_CASES:
+ (build / (name + '.client.log')).write_text(good)
+ (build / (name + '.server.log')).write_text('server role\n')
+ logs_proof(discovery, junit)
+ server = build / (sorted(RSP_CASES)[0] + '.server.log')
+ server.write_text('Fatal{ProtocolCorruption, server-side example}\n')
+ try:
+ logs_proof(discovery, junit)
+ except RuntimeError:
+ pass
+ else:
+ raise AssertionError('server-side Fatal passed the runtime log proof')
+ server.unlink()
+ try:
+ logs_proof(discovery, junit)
+ except FileNotFoundError:
+ pass
+ else:
+ raise AssertionError('missing server log passed the runtime log proof')
(build / 'libMobileGL.so').write_bytes(b'checker fixture, not a real runtime')
normal = ''.join(f'00000000 T fixture{i}\n' for i in range(1000))
remote = normal + '00000000 T MG_Remote_fixture\n00000000 T MGPipeApply_fixture\n'
diff --git a/scripts/ci/test_trace_infrastructure.py b/scripts/ci/test_trace_infrastructure.py
new file mode 100644
index 00000000..f06997b0
--- /dev/null
+++ b/scripts/ci/test_trace_infrastructure.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""Exercise the actual APK retry classifier with saved and live adb states."""
+import pathlib
+import subprocess
+import tempfile
+import unittest
+
+SCRIPT = pathlib.Path(__file__).resolve().parents[2] / "android-plugin/trace-replay-ci.sh"
+
+
+class TraceInfrastructureTest(unittest.TestCase):
+ def classify(self, saved="device", live="device", disconnected=False, logcat=""):
+ source = SCRIPT.read_text()
+ functions = source[source.index("record_infrastructure_reason() {"):source.index("copy_app_artifact() {")]
+ with tempfile.TemporaryDirectory() as directory:
+ root = pathlib.Path(directory)
+ (root / "adb-state.txt").write_text(saved)
+ (root / "logcat.txt").write_text(logcat)
+ if disconnected:
+ (root / "adb-disconnected.txt").write_text("offline")
+ shell = functions + '\nadb_device_path() { printf "%s\\n" "$LIVE"; }\n' + \
+ 'result_root="$1"; LIVE="$2"; is_infrastructure_failure "$1"\n'
+ result = subprocess.run(["sh", "-c", shell, "classifier", directory, live], capture_output=True)
+ reason = root / "infrastructure-failure-reason.txt"
+ return result.returncode, reason.read_text().strip() if reason.exists() else ""
+
+ def test_device_disappears_during_diagnostics(self):
+ self.assertEqual(self.classify(live="offline"), (0, "device-unavailable"))
+
+ def test_polling_disconnect_survives_reconnect(self):
+ self.assertEqual(self.classify(disconnected=True), (0, "device-unavailable"))
+
+ def test_saved_disconnect_survives_reconnect(self):
+ self.assertEqual(self.classify(saved=""), (0, "device-unavailable"))
+
+ def test_online_application_failure_stays_failure(self):
+ self.assertEqual(self.classify(logcat="Fatal signal 11 (trace_replay)\n"), (1, ""))
+
+ def test_system_server_crash_remains_retryable(self):
+ self.assertEqual(self.classify(logcat="Fatal signal 11 (system_server)\n"), (0, "system-server-crash"))
+
+
+if __name__ == "__main__":
+ unittest.main()