[Fix] (CI): stabilize Android retrace jobs

.github/workflows/apk.yml

@@ -227,6 +227,8 @@ jobs:
     env:
       AVD_NAME: mobilegl-ci
       ANDROID_AVD_HOME: ${{ github.workspace }}/.android/avd
+      ANDROID_HOME: ${{ github.workspace }}/.android/sdk
+      ANDROID_SDK_ROOT: ${{ github.workspace }}/.android/sdk
     steps:
       - name: Checkout repo
         uses: actions/checkout@v6
@@ -245,11 +247,11 @@ jobs:
         with:
           path: |
             ${{ env.ANDROID_AVD_HOME }}
-            /usr/local/lib/android/sdk/emulator
+            ${{ env.ANDROID_SDK_ROOT }}/emulator
-            /usr/local/lib/android/sdk/platform-tools
+            ${{ env.ANDROID_SDK_ROOT }}/platform-tools
-            /usr/local/lib/android/sdk/platforms/android-35
+            ${{ env.ANDROID_SDK_ROOT }}/platforms/android-35
-            /usr/local/lib/android/sdk/system-images/android-35/google_apis/x86_64
+            ${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64
-          key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v1-${{ hashFiles('android-plugin/run-avd-ci.sh') }}
+          key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v2-${{ hashFiles('android-plugin/run-avd-ci.sh') }}
 
       - name: Create AVD
         if: steps.android-avd-cache.outputs.cache-hit != 'true'
@@ -274,6 +276,8 @@ jobs:
     env:
       AVD_NAME: mobilegl-ci
       ANDROID_AVD_HOME: ${{ github.workspace }}/.android/avd
+      ANDROID_HOME: ${{ github.workspace }}/.android/sdk
+      ANDROID_SDK_ROOT: ${{ github.workspace }}/.android/sdk
     strategy:
       fail-fast: false
       max-parallel: 4
@@ -288,7 +292,7 @@ jobs:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@v1.0
         with:
-          swap-size-gb: 16
+          swap-size-gb: 8
 
       - name: Checkout repo
         uses: actions/checkout@v6
@@ -324,11 +328,11 @@ jobs:
         with:
           path: |
             ${{ env.ANDROID_AVD_HOME }}
-            /usr/local/lib/android/sdk/emulator
+            ${{ env.ANDROID_SDK_ROOT }}/emulator
-            /usr/local/lib/android/sdk/platform-tools
+            ${{ env.ANDROID_SDK_ROOT }}/platform-tools
-            /usr/local/lib/android/sdk/platforms/android-35
+            ${{ env.ANDROID_SDK_ROOT }}/platforms/android-35
-            /usr/local/lib/android/sdk/system-images/android-35/google_apis/x86_64
+            ${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64
-          key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v1-${{ hashFiles('android-plugin/run-avd-ci.sh') }}
+          key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v2-${{ hashFiles('android-plugin/run-avd-ci.sh') }}
 
       - name: Download retrace APK
         uses: actions/download-artifact@v8
@@ -378,27 +382,51 @@ jobs:
           if [ "${{ matrix.case.coherent_as_flush || false }}" = "true" ]; then
             extra_retrace_args+=(--coherent-as-flush)
           fi
-          timeout "$(( ${{ matrix.case.timeout_seconds }} + 300 ))" sh android-plugin/trace-replay-ci.sh \
+
-            --apk-file "${apk_file}" \
+          run_retrace() {
-            --package top.mobilegl.plugin.trace \
+            timeout "$(( ${{ matrix.case.timeout_seconds }} + 300 ))" sh android-plugin/trace-replay-ci.sh \
-            --backend "${{ matrix.backend.name }}" \
+              --apk-file "${apk_file}" \
-            --result-root android-retrace-result \
+              --package top.mobilegl.plugin.trace \
-            --fixture-root android-retrace-fixture \
+              --backend "${{ matrix.backend.name }}" \
-            --case "${{ matrix.case.name }}" \
+              --result-root android-retrace-result \
-            --trace-archive "${{ matrix.case.trace_archive }}" \
+              --fixture-root android-retrace-fixture \
-            --trace-file "${{ matrix.case.trace_file }}" \
+              --case "${{ matrix.case.name }}" \
-            --golden "${{ matrix.case.golden }}" \
+              --trace-archive "${{ matrix.case.trace_archive }}" \
-            --alternate-golden "${{ matrix.case.alternate_golden || '' }}" \
+              --trace-file "${{ matrix.case.trace_file }}" \
-            --target-call "${{ matrix.case.target_call }}" \
+              --golden "${{ matrix.case.golden }}" \
-            --width "${{ matrix.case.width }}" \
+              --alternate-golden "${{ matrix.case.alternate_golden || '' }}" \
-            --height "${{ matrix.case.height }}" \
+              --target-call "${{ matrix.case.target_call }}" \
-            --ssim-threshold "${{ matrix.case.ssim_threshold || '0.99' }}" \
+              --width "${{ matrix.case.width }}" \
-            --crop-x "${{ matrix.case.crop_x }}" \
+              --height "${{ matrix.case.height }}" \
-            --crop-y "${{ matrix.case.crop_y }}" \
+              --ssim-threshold "${{ matrix.case.ssim_threshold || '0.99' }}" \
-            --crop-width "${{ matrix.case.crop_width }}" \
+              --crop-x "${{ matrix.case.crop_x }}" \
-            --crop-height "${{ matrix.case.crop_height }}" \
+              --crop-y "${{ matrix.case.crop_y }}" \
-            --timeout-seconds "${{ matrix.case.timeout_seconds }}" \
+              --crop-width "${{ matrix.case.crop_width }}" \
-            "${extra_retrace_args[@]}"
+              --crop-height "${{ matrix.case.crop_height }}" \
+              --timeout-seconds "${{ matrix.case.timeout_seconds }}" \
+              "${extra_retrace_args[@]}"
+          }
+
+          retrace_status=0
+          run_retrace || retrace_status=$?
+          if [ "${retrace_status}" -eq 75 ]; then
+            echo "::warning::Android emulator infrastructure failed; restarting it and retrying this retrace once."
+            sh android-plugin/run-avd-ci.sh stop \
+              --avd-name "${AVD_NAME}" \
+              --emulator-log "${EMULATOR_LOG}" \
+              --pid-file "${EMULATOR_PID_FILE}"
+            adb kill-server || true
+            sleep 2
+            sh android-plugin/run-avd-ci.sh start \
+              --avd-name "${AVD_NAME}" \
+              --gpu "${{ matrix.backend.gpu }}" \
+              --emulator-log "${EMULATOR_LOG}" \
+              --pid-file "${EMULATOR_PID_FILE}" \
+              --boot-timeout 300
+            run_retrace
+          elif [ "${retrace_status}" -ne 0 ]; then
+            exit "${retrace_status}"
+          fi
 
       - name: Collect retrace summary inputs
         if: always()

android-plugin/trace-replay-ci.sh

@@ -3,6 +3,7 @@ set -eu
 
 ADB="${ADB:-adb}"
 PYTHON="${PYTHON:-python3}"
+INFRASTRUCTURE_FAILURE_EXIT_CODE=75
 
 usage() {
   cat <<'EOF'
@@ -171,6 +172,12 @@ app_dir="/data/user/0/${package_name}/files/trace-replay"
 collect_run_diagnostics() {
   diagnostics_dir="$1"
   mkdir -p "${diagnostics_dir}"
+  adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
+  printf '%s\n' "${adb_state}" > "${diagnostics_dir}/adb-state.txt"
+  : > "${diagnostics_dir}/logcat.txt"
+  if [ "${adb_state}" != "device" ]; then
+    return
+  fi
   "${ADB}" logcat -d -t 2000 > "${diagnostics_dir}/logcat.txt" || true
   adb_device_path shell pidof "${package_name}" > "${diagnostics_dir}/pidof.txt" 2>&1 || true
   adb_device_path shell dumpsys activity activities > "${diagnostics_dir}/activity.txt" 2>&1 || true
@@ -179,6 +186,20 @@ collect_run_diagnostics() {
   adb_device_path exec-out run-as "${package_name}" cat "${app_dir}/output/mobilegl.log" > "${diagnostics_dir}/mobilegl.log" || true
 }
 
+is_infrastructure_failure() {
+  diagnostics_dir="$1"
+  adb_state="$(cat "${diagnostics_dir}/adb-state.txt" 2>/dev/null || true)"
+  if [ "${adb_state}" != "device" ]; then
+    echo "trace-replay-ci.sh: Android device is unavailable (state: ${adb_state:-unknown})" >&2
+    return 0
+  fi
+  if grep -Eq 'Fatal signal [0-9]+.*[(]system_server[)]|F system_server[ :]' "${diagnostics_dir}/logcat.txt"; then
+    echo "trace-replay-ci.sh: Android system_server crashed during retrace" >&2
+    return 0
+  fi
+  return 1
+}
+
 copy_app_artifact() {
   source_path="$1"
   destination_path="$2"
@@ -310,6 +331,10 @@ run_retrace() {
     grep -E 'MobileGLTraceRunner|AndroidRuntime|FATAL EXCEPTION|trace_replay|MobileGL|libc' "${result_dir}/logcat.txt" | tail -200 >&2 || true
     echo "trace-replay-ci.sh: app trace-replay files:" >&2
     cat "${result_dir}/app-files.txt" >&2 || true
+    if is_infrastructure_failure "${result_dir}"; then
+      echo "trace-replay-ci.sh: requesting one infrastructure retry" >&2
+      exit "${INFRASTRUCTURE_FAILURE_EXIT_CODE}"
+    fi
     exit 1
   fi
   adb_device_path exec-out run-as "${package_name}" cat "${app_dir}/output/result.json" > "${result_dir}/result.json"