[Fix] (CI): stabilize Android retrace jobs

2026-09-07 19:58:32 +09:00 · 2026-07-18 19:52:04 -04:00
parent 626c7f26fd
commit fc0688c223
2 changed files with 85 additions and 32 deletions
@@ -227,6 +227,8 @@ jobs:
    env:
      AVD_NAME: mobilegl-ci
      ANDROID_AVD_HOME: ${{ github.workspace }}/.android/avd
+      ANDROID_HOME: ${{ github.workspace }}/.android/sdk
+      ANDROID_SDK_ROOT: ${{ github.workspace }}/.android/sdk
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
@@ -245,11 +247,11 @@ jobs:
        with:
          path: |
            ${{ env.ANDROID_AVD_HOME }}
-            /usr/local/lib/android/sdk/emulator
-            /usr/local/lib/android/sdk/platform-tools
-            /usr/local/lib/android/sdk/platforms/android-35
-            /usr/local/lib/android/sdk/system-images/android-35/google_apis/x86_64
-          key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v1-${{ hashFiles('android-plugin/run-avd-ci.sh') }}
+            ${{ env.ANDROID_SDK_ROOT }}/emulator
+            ${{ env.ANDROID_SDK_ROOT }}/platform-tools
+            ${{ env.ANDROID_SDK_ROOT }}/platforms/android-35
+            ${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64
+          key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v2-${{ hashFiles('android-plugin/run-avd-ci.sh') }}

      - name: Create AVD
        if: steps.android-avd-cache.outputs.cache-hit != 'true'
@@ -274,6 +276,8 @@ jobs:
    env:
      AVD_NAME: mobilegl-ci
      ANDROID_AVD_HOME: ${{ github.workspace }}/.android/avd
+      ANDROID_HOME: ${{ github.workspace }}/.android/sdk
+      ANDROID_SDK_ROOT: ${{ github.workspace }}/.android/sdk
    strategy:
      fail-fast: false
      max-parallel: 4
@@ -288,7 +292,7 @@ jobs:
      - name: Set Swap Space
        uses: pierotofy/set-swap-space@v1.0
        with:
-          swap-size-gb: 16
+          swap-size-gb: 8

      - name: Checkout repo
        uses: actions/checkout@v6
@@ -324,11 +328,11 @@ jobs:
        with:
          path: |
            ${{ env.ANDROID_AVD_HOME }}
-            /usr/local/lib/android/sdk/emulator
-            /usr/local/lib/android/sdk/platform-tools
-            /usr/local/lib/android/sdk/platforms/android-35
-            /usr/local/lib/android/sdk/system-images/android-35/google_apis/x86_64
-          key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v1-${{ hashFiles('android-plugin/run-avd-ci.sh') }}
+            ${{ env.ANDROID_SDK_ROOT }}/emulator
+            ${{ env.ANDROID_SDK_ROOT }}/platform-tools
+            ${{ env.ANDROID_SDK_ROOT }}/platforms/android-35
+            ${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64
+          key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v2-${{ hashFiles('android-plugin/run-avd-ci.sh') }}

      - name: Download retrace APK
        uses: actions/download-artifact@v8
@@ -378,27 +382,51 @@ jobs:
          if [ "${{ matrix.case.coherent_as_flush || false }}" = "true" ]; then
            extra_retrace_args+=(--coherent-as-flush)
          fi
-          timeout "$(( ${{ matrix.case.timeout_seconds }} + 300 ))" sh android-plugin/trace-replay-ci.sh \
-            --apk-file "${apk_file}" \
-            --package top.mobilegl.plugin.trace \
-            --backend "${{ matrix.backend.name }}" \
-            --result-root android-retrace-result \
-            --fixture-root android-retrace-fixture \
-            --case "${{ matrix.case.name }}" \
-            --trace-archive "${{ matrix.case.trace_archive }}" \
-            --trace-file "${{ matrix.case.trace_file }}" \
-            --golden "${{ matrix.case.golden }}" \
-            --alternate-golden "${{ matrix.case.alternate_golden || '' }}" \
-            --target-call "${{ matrix.case.target_call }}" \
-            --width "${{ matrix.case.width }}" \
-            --height "${{ matrix.case.height }}" \
-            --ssim-threshold "${{ matrix.case.ssim_threshold || '0.99' }}" \
-            --crop-x "${{ matrix.case.crop_x }}" \
-            --crop-y "${{ matrix.case.crop_y }}" \
-            --crop-width "${{ matrix.case.crop_width }}" \
-            --crop-height "${{ matrix.case.crop_height }}" \
-            --timeout-seconds "${{ matrix.case.timeout_seconds }}" \
-            "${extra_retrace_args[@]}"
+
+          run_retrace() {
+            timeout "$(( ${{ matrix.case.timeout_seconds }} + 300 ))" sh android-plugin/trace-replay-ci.sh \
+              --apk-file "${apk_file}" \
+              --package top.mobilegl.plugin.trace \
+              --backend "${{ matrix.backend.name }}" \
+              --result-root android-retrace-result \
+              --fixture-root android-retrace-fixture \
+              --case "${{ matrix.case.name }}" \
+              --trace-archive "${{ matrix.case.trace_archive }}" \
+              --trace-file "${{ matrix.case.trace_file }}" \
+              --golden "${{ matrix.case.golden }}" \
+              --alternate-golden "${{ matrix.case.alternate_golden || '' }}" \
+              --target-call "${{ matrix.case.target_call }}" \
+              --width "${{ matrix.case.width }}" \
+              --height "${{ matrix.case.height }}" \
+              --ssim-threshold "${{ matrix.case.ssim_threshold || '0.99' }}" \
+              --crop-x "${{ matrix.case.crop_x }}" \
+              --crop-y "${{ matrix.case.crop_y }}" \
+              --crop-width "${{ matrix.case.crop_width }}" \
+              --crop-height "${{ matrix.case.crop_height }}" \
+              --timeout-seconds "${{ matrix.case.timeout_seconds }}" \
+              "${extra_retrace_args[@]}"
+          }
+
+          retrace_status=0
+          run_retrace || retrace_status=$?
+          if [ "${retrace_status}" -eq 75 ]; then
+            echo "::warning::Android emulator infrastructure failed; restarting it and retrying this retrace once."
+            sh android-plugin/run-avd-ci.sh stop \
+              --avd-name "${AVD_NAME}" \
+              --emulator-log "${EMULATOR_LOG}" \
+              --pid-file "${EMULATOR_PID_FILE}"
+            adb kill-server || true
+            sleep 2
+            sh android-plugin/run-avd-ci.sh start \
+              --avd-name "${AVD_NAME}" \
+              --gpu "${{ matrix.backend.gpu }}" \
+              --emulator-log "${EMULATOR_LOG}" \
+              --pid-file "${EMULATOR_PID_FILE}" \
+              --boot-timeout 300
+            run_retrace
+          elif [ "${retrace_status}" -ne 0 ]; then
+            exit "${retrace_status}"
+          fi

      - name: Collect retrace summary inputs
        if: always()
@@ -3,6 +3,7 @@ set -eu

 ADB="${ADB:-adb}"
 PYTHON="${PYTHON:-python3}"
+INFRASTRUCTURE_FAILURE_EXIT_CODE=75

 usage() {
  cat <<'EOF'
@@ -171,6 +172,12 @@ app_dir="/data/user/0/${package_name}/files/trace-replay"
 collect_run_diagnostics() {
  diagnostics_dir="$1"
  mkdir -p "${diagnostics_dir}"
+  adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
+  printf '%s\n' "${adb_state}" > "${diagnostics_dir}/adb-state.txt"
+  : > "${diagnostics_dir}/logcat.txt"
+  if [ "${adb_state}" != "device" ]; then
+    return
+  fi
  "${ADB}" logcat -d -t 2000 > "${diagnostics_dir}/logcat.txt" || true
  adb_device_path shell pidof "${package_name}" > "${diagnostics_dir}/pidof.txt" 2>&1 || true
  adb_device_path shell dumpsys activity activities > "${diagnostics_dir}/activity.txt" 2>&1 || true
@@ -179,6 +186,20 @@ collect_run_diagnostics() {
  adb_device_path exec-out run-as "${package_name}" cat "${app_dir}/output/mobilegl.log" > "${diagnostics_dir}/mobilegl.log" || true
 }

+is_infrastructure_failure() {
+  diagnostics_dir="$1"
+  adb_state="$(cat "${diagnostics_dir}/adb-state.txt" 2>/dev/null || true)"
+  if [ "${adb_state}" != "device" ]; then
+    echo "trace-replay-ci.sh: Android device is unavailable (state: ${adb_state:-unknown})" >&2
+    return 0
+  fi
+  if grep -Eq 'Fatal signal [0-9]+.*[(]system_server[)]|F system_server[ :]' "${diagnostics_dir}/logcat.txt"; then
+    echo "trace-replay-ci.sh: Android system_server crashed during retrace" >&2
+    return 0
+  fi
+  return 1
+}
+
 copy_app_artifact() {
  source_path="$1"
  destination_path="$2"
@@ -310,6 +331,10 @@ run_retrace() {
    grep -E 'MobileGLTraceRunner|AndroidRuntime|FATAL EXCEPTION|trace_replay|MobileGL|libc' "${result_dir}/logcat.txt" | tail -200 >&2 || true
    echo "trace-replay-ci.sh: app trace-replay files:" >&2
    cat "${result_dir}/app-files.txt" >&2 || true
+    if is_infrastructure_failure "${result_dir}"; then
+      echo "trace-replay-ci.sh: requesting one infrastructure retry" >&2
+      exit "${INFRASTRUCTURE_FAILURE_EXIT_CODE}"
+    fi
    exit 1
  fi
  adb_device_path exec-out run-as "${package_name}" cat "${app_dir}/output/result.json" > "${result_dir}/result.json"