diff --git a/android-plugin/trace-replay-ci.sh b/android-plugin/trace-replay-ci.sh index 199097ad..7d67b014 100644 --- a/android-plugin/trace-replay-ci.sh +++ b/android-plugin/trace-replay-ci.sh @@ -278,7 +278,12 @@ is_angle_surface_lost() { is_infrastructure_failure() { diagnostics_dir="$1" adb_state="$(cat "${diagnostics_dir}/adb-state.txt" 2>/dev/null || true)" - if [ "${adb_state}" != "device" ]; then + # Diagnostics can take several adb calls: a device that was online at their + # start may disappear while we collect them. Keep the polling observation and + # check the live state as well, rather than trusting that earlier snapshot. + live_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)" + if [ -f "${diagnostics_dir}/adb-disconnected.txt" ] || + [ "${adb_state}" != "device" ] || [ "${live_adb_state}" != "device" ]; then echo "trace-replay-ci.sh: Android device is unavailable (state: ${adb_state:-unknown})" >&2 record_infrastructure_reason "device-unavailable" return 0 @@ -492,25 +497,35 @@ run_retrace() { app_exited=0 saw_app_process=0 + poll_started="$(date +%s)" for _ in $(seq 1 "${timeout_seconds}"); do if adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then break fi if adb_device_path shell pidof "${package_name}" >/dev/null 2>&1; then saw_app_process=1 - elif [ "${saw_app_process}" -eq 1 ]; then - app_exited=1 - break + else + poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)" + if [ "${poll_adb_state}" != "device" ]; then + printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt" + break + fi + if [ "${saw_app_process}" -eq 1 ]; then + app_exited=1 + break + fi fi sleep 1 done collect_run_diagnostics "${result_dir}" if ! adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then - if [ "${app_exited}" -eq 1 ]; then + if [ -f "${result_dir}/adb-disconnected.txt" ]; then + echo "trace-replay-ci.sh: device disconnected while waiting for result.json" >&2 + elif [ "${app_exited}" -eq 1 ]; then echo "trace-replay-ci.sh: app process exited before result.json was created" >&2 else - echo "trace-replay-ci.sh: result.json was not created after ${timeout_seconds}s" >&2 + echo "trace-replay-ci.sh: result.json was not created after $(($(date +%s) - poll_started))s (budget ${timeout_seconds}s)" >&2 fi if [ -s "${result_dir}/retrace.log" ]; then echo "trace-replay-ci.sh: retrace.log:" >&2 diff --git a/scripts/ci/runtime_mode_proof.py b/scripts/ci/runtime_mode_proof.py index 2045fdd8..237a103b 100644 --- a/scripts/ci/runtime_mode_proof.py +++ b/scripts/ci/runtime_mode_proof.py @@ -70,21 +70,11 @@ def logs_proof(discovery, junit): logs = helper.marker_log_paths(lane) proof = {} for name in sorted(RSP_CASES): - # P6 splits the library log per role: MOBILEGL_LOG_FILE_PATH is a BASE NAME and the - # files that exist are .client.log / .server.log. Reading the base directly - # is FileNotFoundError after the split - the same class of reader the role-split - # landing already fixed everywhere else. - path = Path(logs[name]) - if not helper.any_role_file(path): - raise RuntimeError( - f'{name}: no role log under {path} ' - f'(looked for {helper.role_paths(path)}); the library never opened the sink') - text = helper.read_role_logs(path) - proof[name] = { - 'private_log': str(path), - 'role_logs': helper.role_paths(path), - **inproc_log_proof(text), - } + # MOBILEGL_LOG_FILE_PATH is a base name. In split builds neither role + # writes that literal path, and the server half owns applier failures. + paths = [Path(path) for path in helper.role_paths(logs[name])] + text = '\n'.join(path.read_text(encoding='utf-8', errors='replace') for path in paths) + proof[name] = {'private_logs': [str(path) for path in paths], **inproc_log_proof(text)} return proof @@ -104,6 +94,32 @@ def self_test(): raise AssertionError('invalid runtime log passed') with tempfile.TemporaryDirectory(prefix='mobilegl-runtime-proof-') as directory: build = Path(directory) + discovery = build / 'lane.json' + junit = build / 'lane.xml' + discovery.write_text(json.dumps({'tests': [ + {'name': name, 'properties': [{'name': 'ENVIRONMENT', + 'value': ['MOBILEGL_LOG_FILE_PATH=' + str(build / (name + '.log'))]}]} + for name in sorted(RSP_CASES)]})) + junit.write_text('' + ''.join(f'' for name in sorted(RSP_CASES)) + '') + for name in RSP_CASES: + (build / (name + '.client.log')).write_text(good) + (build / (name + '.server.log')).write_text('server role\n') + logs_proof(discovery, junit) + server = build / (sorted(RSP_CASES)[0] + '.server.log') + server.write_text('Fatal{ProtocolCorruption, server-side example}\n') + try: + logs_proof(discovery, junit) + except RuntimeError: + pass + else: + raise AssertionError('server-side Fatal passed the runtime log proof') + server.unlink() + try: + logs_proof(discovery, junit) + except FileNotFoundError: + pass + else: + raise AssertionError('missing server log passed the runtime log proof') (build / 'libMobileGL.so').write_bytes(b'checker fixture, not a real runtime') normal = ''.join(f'00000000 T fixture{i}\n' for i in range(1000)) remote = normal + '00000000 T MG_Remote_fixture\n00000000 T MGPipeApply_fixture\n' diff --git a/scripts/ci/test_trace_infrastructure.py b/scripts/ci/test_trace_infrastructure.py new file mode 100644 index 00000000..f06997b0 --- /dev/null +++ b/scripts/ci/test_trace_infrastructure.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Exercise the actual APK retry classifier with saved and live adb states.""" +import pathlib +import subprocess +import tempfile +import unittest + +SCRIPT = pathlib.Path(__file__).resolve().parents[2] / "android-plugin/trace-replay-ci.sh" + + +class TraceInfrastructureTest(unittest.TestCase): + def classify(self, saved="device", live="device", disconnected=False, logcat=""): + source = SCRIPT.read_text() + functions = source[source.index("record_infrastructure_reason() {"):source.index("copy_app_artifact() {")] + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + (root / "adb-state.txt").write_text(saved) + (root / "logcat.txt").write_text(logcat) + if disconnected: + (root / "adb-disconnected.txt").write_text("offline") + shell = functions + '\nadb_device_path() { printf "%s\\n" "$LIVE"; }\n' + \ + 'result_root="$1"; LIVE="$2"; is_infrastructure_failure "$1"\n' + result = subprocess.run(["sh", "-c", shell, "classifier", directory, live], capture_output=True) + reason = root / "infrastructure-failure-reason.txt" + return result.returncode, reason.read_text().strip() if reason.exists() else "" + + def test_device_disappears_during_diagnostics(self): + self.assertEqual(self.classify(live="offline"), (0, "device-unavailable")) + + def test_polling_disconnect_survives_reconnect(self): + self.assertEqual(self.classify(disconnected=True), (0, "device-unavailable")) + + def test_saved_disconnect_survives_reconnect(self): + self.assertEqual(self.classify(saved=""), (0, "device-unavailable")) + + def test_online_application_failure_stays_failure(self): + self.assertEqual(self.classify(logcat="Fatal signal 11 (trace_replay)\n"), (1, "")) + + def test_system_server_crash_remains_retryable(self): + self.assertEqual(self.classify(logcat="Fatal signal 11 (system_server)\n"), (0, "system-server-crash")) + + +if __name__ == "__main__": + unittest.main()