diff --git a/.github/workflows/apk.yml b/.github/workflows/apk.yml index bc27f575..fc9ffc85 100644 --- a/.github/workflows/apk.yml +++ b/.github/workflows/apk.yml @@ -227,6 +227,8 @@ jobs: env: AVD_NAME: mobilegl-ci ANDROID_AVD_HOME: ${{ github.workspace }}/.android/avd + ANDROID_HOME: ${{ github.workspace }}/.android/sdk + ANDROID_SDK_ROOT: ${{ github.workspace }}/.android/sdk steps: - name: Checkout repo uses: actions/checkout@v6 @@ -245,11 +247,11 @@ jobs: with: path: | ${{ env.ANDROID_AVD_HOME }} - /usr/local/lib/android/sdk/emulator - /usr/local/lib/android/sdk/platform-tools - /usr/local/lib/android/sdk/platforms/android-35 - /usr/local/lib/android/sdk/system-images/android-35/google_apis/x86_64 - key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v1-${{ hashFiles('android-plugin/run-avd-ci.sh') }} + ${{ env.ANDROID_SDK_ROOT }}/emulator + ${{ env.ANDROID_SDK_ROOT }}/platform-tools + ${{ env.ANDROID_SDK_ROOT }}/platforms/android-35 + ${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64 + key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v2-${{ hashFiles('android-plugin/run-avd-ci.sh') }} - name: Create AVD if: steps.android-avd-cache.outputs.cache-hit != 'true' @@ -274,6 +276,8 @@ jobs: env: AVD_NAME: mobilegl-ci ANDROID_AVD_HOME: ${{ github.workspace }}/.android/avd + ANDROID_HOME: ${{ github.workspace }}/.android/sdk + ANDROID_SDK_ROOT: ${{ github.workspace }}/.android/sdk strategy: fail-fast: false max-parallel: 4 @@ -288,7 +292,7 @@ jobs: - name: Set Swap Space uses: pierotofy/set-swap-space@v1.0 with: - swap-size-gb: 16 + swap-size-gb: 8 - name: Checkout repo uses: actions/checkout@v6 @@ -324,11 +328,11 @@ jobs: with: path: | ${{ env.ANDROID_AVD_HOME }} - /usr/local/lib/android/sdk/emulator - /usr/local/lib/android/sdk/platform-tools - /usr/local/lib/android/sdk/platforms/android-35 - /usr/local/lib/android/sdk/system-images/android-35/google_apis/x86_64 - key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v1-${{ hashFiles('android-plugin/run-avd-ci.sh') }} + ${{ env.ANDROID_SDK_ROOT }}/emulator + ${{ env.ANDROID_SDK_ROOT }}/platform-tools + ${{ env.ANDROID_SDK_ROOT }}/platforms/android-35 + ${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64 + key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v2-${{ hashFiles('android-plugin/run-avd-ci.sh') }} - name: Download retrace APK uses: actions/download-artifact@v8 @@ -378,27 +382,51 @@ jobs: if [ "${{ matrix.case.coherent_as_flush || false }}" = "true" ]; then extra_retrace_args+=(--coherent-as-flush) fi - timeout "$(( ${{ matrix.case.timeout_seconds }} + 300 ))" sh android-plugin/trace-replay-ci.sh \ - --apk-file "${apk_file}" \ - --package top.mobilegl.plugin.trace \ - --backend "${{ matrix.backend.name }}" \ - --result-root android-retrace-result \ - --fixture-root android-retrace-fixture \ - --case "${{ matrix.case.name }}" \ - --trace-archive "${{ matrix.case.trace_archive }}" \ - --trace-file "${{ matrix.case.trace_file }}" \ - --golden "${{ matrix.case.golden }}" \ - --alternate-golden "${{ matrix.case.alternate_golden || '' }}" \ - --target-call "${{ matrix.case.target_call }}" \ - --width "${{ matrix.case.width }}" \ - --height "${{ matrix.case.height }}" \ - --ssim-threshold "${{ matrix.case.ssim_threshold || '0.99' }}" \ - --crop-x "${{ matrix.case.crop_x }}" \ - --crop-y "${{ matrix.case.crop_y }}" \ - --crop-width "${{ matrix.case.crop_width }}" \ - --crop-height "${{ matrix.case.crop_height }}" \ - --timeout-seconds "${{ matrix.case.timeout_seconds }}" \ - "${extra_retrace_args[@]}" + + run_retrace() { + timeout "$(( ${{ matrix.case.timeout_seconds }} + 300 ))" sh android-plugin/trace-replay-ci.sh \ + --apk-file "${apk_file}" \ + --package top.mobilegl.plugin.trace \ + --backend "${{ matrix.backend.name }}" \ + --result-root android-retrace-result \ + --fixture-root android-retrace-fixture \ + --case "${{ matrix.case.name }}" \ + --trace-archive "${{ matrix.case.trace_archive }}" \ + --trace-file "${{ matrix.case.trace_file }}" \ + --golden "${{ matrix.case.golden }}" \ + --alternate-golden "${{ matrix.case.alternate_golden || '' }}" \ + --target-call "${{ matrix.case.target_call }}" \ + --width "${{ matrix.case.width }}" \ + --height "${{ matrix.case.height }}" \ + --ssim-threshold "${{ matrix.case.ssim_threshold || '0.99' }}" \ + --crop-x "${{ matrix.case.crop_x }}" \ + --crop-y "${{ matrix.case.crop_y }}" \ + --crop-width "${{ matrix.case.crop_width }}" \ + --crop-height "${{ matrix.case.crop_height }}" \ + --timeout-seconds "${{ matrix.case.timeout_seconds }}" \ + "${extra_retrace_args[@]}" + } + + retrace_status=0 + run_retrace || retrace_status=$? + if [ "${retrace_status}" -eq 75 ]; then + echo "::warning::Android emulator infrastructure failed; restarting it and retrying this retrace once." + sh android-plugin/run-avd-ci.sh stop \ + --avd-name "${AVD_NAME}" \ + --emulator-log "${EMULATOR_LOG}" \ + --pid-file "${EMULATOR_PID_FILE}" + adb kill-server || true + sleep 2 + sh android-plugin/run-avd-ci.sh start \ + --avd-name "${AVD_NAME}" \ + --gpu "${{ matrix.backend.gpu }}" \ + --emulator-log "${EMULATOR_LOG}" \ + --pid-file "${EMULATOR_PID_FILE}" \ + --boot-timeout 300 + run_retrace + elif [ "${retrace_status}" -ne 0 ]; then + exit "${retrace_status}" + fi - name: Collect retrace summary inputs if: always() diff --git a/android-plugin/trace-replay-ci.sh b/android-plugin/trace-replay-ci.sh index ce50aad1..a879d114 100644 --- a/android-plugin/trace-replay-ci.sh +++ b/android-plugin/trace-replay-ci.sh @@ -3,6 +3,7 @@ set -eu ADB="${ADB:-adb}" PYTHON="${PYTHON:-python3}" +INFRASTRUCTURE_FAILURE_EXIT_CODE=75 usage() { cat <<'EOF' @@ -171,6 +172,12 @@ app_dir="/data/user/0/${package_name}/files/trace-replay" collect_run_diagnostics() { diagnostics_dir="$1" mkdir -p "${diagnostics_dir}" + adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)" + printf '%s\n' "${adb_state}" > "${diagnostics_dir}/adb-state.txt" + : > "${diagnostics_dir}/logcat.txt" + if [ "${adb_state}" != "device" ]; then + return + fi "${ADB}" logcat -d -t 2000 > "${diagnostics_dir}/logcat.txt" || true adb_device_path shell pidof "${package_name}" > "${diagnostics_dir}/pidof.txt" 2>&1 || true adb_device_path shell dumpsys activity activities > "${diagnostics_dir}/activity.txt" 2>&1 || true @@ -179,6 +186,20 @@ collect_run_diagnostics() { adb_device_path exec-out run-as "${package_name}" cat "${app_dir}/output/mobilegl.log" > "${diagnostics_dir}/mobilegl.log" || true } +is_infrastructure_failure() { + diagnostics_dir="$1" + adb_state="$(cat "${diagnostics_dir}/adb-state.txt" 2>/dev/null || true)" + if [ "${adb_state}" != "device" ]; then + echo "trace-replay-ci.sh: Android device is unavailable (state: ${adb_state:-unknown})" >&2 + return 0 + fi + if grep -Eq 'Fatal signal [0-9]+.*[(]system_server[)]|F system_server[ :]' "${diagnostics_dir}/logcat.txt"; then + echo "trace-replay-ci.sh: Android system_server crashed during retrace" >&2 + return 0 + fi + return 1 +} + copy_app_artifact() { source_path="$1" destination_path="$2" @@ -310,6 +331,10 @@ run_retrace() { grep -E 'MobileGLTraceRunner|AndroidRuntime|FATAL EXCEPTION|trace_replay|MobileGL|libc' "${result_dir}/logcat.txt" | tail -200 >&2 || true echo "trace-replay-ci.sh: app trace-replay files:" >&2 cat "${result_dir}/app-files.txt" >&2 || true + if is_infrastructure_failure "${result_dir}"; then + echo "trace-replay-ci.sh: requesting one infrastructure retry" >&2 + exit "${INFRASTRUCTURE_FAILURE_EXIT_CODE}" + fi exit 1 fi adb_device_path exec-out run-as "${package_name}" cat "${app_dir}/output/result.json" > "${result_dir}/result.json"