[android-plugin, scripts] (P7): a passed APK replay whose device went offline before its logs were copied asks for the infrastructure retry

android-plugin/trace-replay-ci.sh

@@ -301,6 +301,20 @@ is_infrastructure_failure() {
   return 1
 }
 
+# A replay that PASSED and then lost its logs is not evidence either way. Sampled on the APK
+# workflow (5 of 7 failed legs): result.json said "passed": true, then "error: device offline"
+# before the role logs were copied, and the --require-inproc/--require-spawn proof read an empty
+# mobilegl.log and failed the leg with exit 1 - charging the emulator's disappearance to the trace
+# instead of asking for the one infrastructure retry. True only when a proof needs the log
+# ($2 = 1), the log is empty, and the classifier above says the infrastructure failed; an online
+# device with an empty log stays the proof's failure.
+passed_replay_lost_its_logs() {
+  lost_logs_dir="$1"
+  [ "$2" -eq 1 ] || return 1
+  [ ! -s "${lost_logs_dir}/mobilegl.log" ] || return 1
+  is_infrastructure_failure "${lost_logs_dir}"
+}
+
 copy_app_artifact() {
   source_path="$1"
   destination_path="$2"
@@ -589,6 +603,10 @@ run_retrace() {
   # reads "trace replay failed" over a passed result.json. host_path_for_adb converts
   # explicitly, so the verdict no longer depends on the caller's environment.
   "${PYTHON}" -c 'import json, sys; result = json.load(open(sys.argv[1], encoding="utf-8")); sys.exit(0 if result.get("passed") else f"trace replay failed: {result}")' "$(host_path_for_adb "${result_dir}/result.json")" || return "$?"
+  if passed_replay_lost_its_logs "${result_dir}" "$((require_inproc | require_spawn))"; then
+    echo "trace-replay-ci.sh: the replay passed but its logs were never copied, so the transport proof has nothing to read; requesting one infrastructure retry" >&2
+    exit "${INFRASTRUCTURE_FAILURE_EXIT_CODE}"
+  fi
   if [ "${require_inproc}" -eq 1 ]; then
     "${PYTHON}" - "$(host_path_for_adb "${result_dir}/mobilegl.log")" "$(host_path_for_adb "${result_dir}/transport-proof.json")" <<'PY'
 import json

scripts/ci/test_trace_infrastructure.py

@@ -39,6 +39,43 @@ class TraceInfrastructureTest(unittest.TestCase):
     def test_system_server_crash_remains_retryable(self):
         self.assertEqual(self.classify(logcat="Fatal signal 11 (system_server)\n"), (0, "system-server-crash"))
 
+    def lost_logs(self, live="device", log="", needs_log=1):
+        """passed_replay_lost_its_logs over a result dir whose replay PASSED (the caller has
+        already checked result.json) with the given mobilegl.log content and live adb state."""
+        source = SCRIPT.read_text()
+        functions = source[source.index("record_infrastructure_reason() {"):source.index("copy_app_artifact() {")]
+        with tempfile.TemporaryDirectory() as directory:
+            root = pathlib.Path(directory)
+            (root / "adb-state.txt").write_text("device")
+            (root / "logcat.txt").write_text("")
+            (root / "mobilegl.log").write_text(log)
+            shell = functions + '\nadb_device_path() { printf "%s\\n" "$LIVE"; }\n' + \
+                'result_root="$1"; LIVE="$2"; passed_replay_lost_its_logs "$1" "$3"\n'
+            result = subprocess.run(["sh", "-c", shell, "lost-logs", directory, live, str(needs_log)],
+                                    capture_output=True)
+            reason = root / "infrastructure-failure-reason.txt"
+            return result.returncode, reason.read_text().strip() if reason.exists() else ""
+
+    def test_a_passed_replay_whose_device_went_offline_before_its_logs_is_retried(self):
+        # APK workflow: result.json passed, then "error: device offline" before collect_role_logs,
+        # and the inproc proof failed the leg with exit 1 on an empty mobilegl.log.
+        self.assertEqual(self.lost_logs(live="offline"), (0, "device-unavailable"))
+
+    def test_an_empty_log_on_a_live_device_stays_the_proofs_failure(self):
+        self.assertEqual(self.lost_logs(live="device"), (1, ""))
+
+    def test_a_log_that_arrived_is_judged_by_the_proof_not_retried(self):
+        self.assertEqual(self.lost_logs(live="offline", log="Config: IPC run-ahead=1\n"), (1, ""))
+
+    def test_a_leg_that_needs_no_proof_is_not_retried_for_its_log(self):
+        self.assertEqual(self.lost_logs(live="offline", needs_log=0), (1, ""))
+
+    def test_the_lost_log_check_runs_before_the_transport_proofs(self):
+        source = SCRIPT.read_text()
+        call = source.index('if passed_replay_lost_its_logs "${result_dir}"')
+        self.assertLess(source.index('sys.exit(0 if result.get("passed")'), call)
+        self.assertLess(call, source.index('if [ "${require_inproc}" -eq 1 ]; then\n    "${PYTHON}"'))
+
 
 if __name__ == "__main__":
     unittest.main()