name: MobileGL APK on: push: branches: - dev - Feat/Backend-Direct-GLES - Feat/Backend-Direct-Vulkan # TEMPORARY, remove before merging the MGPipe work into dev: the disaggregation # branch runs the full lane on every push so a phase's landing is not gated on # someone remembering to dispatch the workflow by hand. - feat/disaggregated - codex/fix-github-ci - ci/round-30min - cleanup # A push that only touches these paths changes nothing this workflow builds or replays. The # Test workflow keeps running for docs: it hosts the doc citation lint. The Android build # compiles none of the unit, integration or benchmark trees, with one exception that keeps # MobileGL/MG_Benchmark from being listed whole: MG_Util/SelfTest/DriverBenchJni.cpp # includes MG_Benchmark/Driver/DriverBenchCases.inc. paths-ignore: - 'docs/**' - '**/*.md' - 'MobileGL/MG_Test/**' - 'MobileGL/MG_IntegrationTest/**' - 'MobileGL/MG_Benchmark/CMakeLists.txt' - 'MobileGL/MG_Benchmark/SanityBench.cpp' - 'MobileGL/MG_Benchmark/Buffer/**' - 'MobileGL/MG_Benchmark/Container/**' - 'MobileGL/MG_Benchmark/Program/**' - 'MobileGL/MG_Benchmark/ShaderCache/**' - 'MobileGL/MG_Benchmark/Transpile/**' - 'tools/cts/**' - 'tools/device_bench/**' - '.github/workflows/test.yml' - '.github/workflows/hygiene.yml' - '.github/actions/setup-test-runner/**' workflow_dispatch: inputs: android_full_matrix: description: Replay every trace case on the monolith and spawn emulator lanes too type: boolean default: false env: # Software-rendered CI submissions may exceed the interactive GPU hang budget. MOBILEGL_GPU_HANG_BUDGET_MS: "0" # The emulator replays every trace case on inproc, the transport the launcher ships. On # monolith and spawn it replays the android_smoke cases of trace_cases.json, because # test.yml replays every case on all three transports on Linux. A manual run with # android_full_matrix replays every case on all three emulator lanes. A push has no inputs, # so it gets the reduced lanes. MOBILEGL_ANDROID_LANES: ${{ inputs.android_full_matrix && 'full' || 'inproc-full' }} # ONE LIVE RUN PER BRANCH, as in test.yml: a superseded push is canceled rather than left queued # behind its successor; a manual dispatch gets a group of its own; dev queues rather than cancels. concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.ref }} cancel-in-progress: ${{ github.event_name == 'push' && github.ref != 'refs/heads/dev' }} jobs: build: runs-on: ubuntu-26.04 permissions: actions: write contents: read env: CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "true" CCACHE_DIR: ${{ github.workspace }}/.ccache # One build's objects (three native configurations) take about 1 GB. The limit leaves # half of that again for the objects a push replaces, and is what the cleanup before # the save trims the entry to. CCACHE_MAXSIZE: 1500M CCACHE_NOHASHDIR: "true" MOBILEGL_CMAKE_COMPILER_LAUNCHER: ccache MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'ci/round-30min' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }} steps: - name: Checkout repo uses: actions/checkout@v6 # Only the submodules the two APKs are built from, shallow and in parallel, in place of a # recursive checkout of every submodule in the tree: the ones the root CMakeLists.txt # configures, and apitrace with the directories its thirdparty/CMakeLists.txt requires, # for the trace APK. The step fails when a file the build reads from one of them did # not arrive (scripts/ci/checkout_submodules.sh). - name: Check out build submodules timeout-minutes: 15 run: bash scripts/ci/checkout_submodules.sh retrace - name: Set artifact metadata run: | echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV" - name: Set up JDK uses: actions/setup-java@v5 with: distribution: zulu java-version: '17' - name: Android replay lifecycle regression run: python3 tools/trace_replay/test_android_lifecycle.py - name: Setup Gradle uses: gradle/actions/setup-gradle@v6 with: gradle-version: 8.10.2 - name: Restore ccache uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-apk-${{ github.job }}-ccache-v1 restore-keys: | ${{ runner.os }}-apk-${{ github.job }}-ccache- - name: Install ccache timeout-minutes: 15 run: | sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 update sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 install -y ccache ccache --version # The restored entry carries the counters of every earlier build; zero them so the # statistics printed after the build are this run's. ccache --zero-stats - name: Setup Android SDK id: netsetup84 continue-on-error: true timeout-minutes: 15 uses: android-actions/setup-android@v4 with: accept-android-sdk-licenses: false packages: platform-tools cmdline-tools-version: 13114758 - name: Setup Android SDK (retry) timeout-minutes: 15 if: ${{ steps.netsetup84.outcome == 'failure' }} uses: android-actions/setup-android@v4 with: accept-android-sdk-licenses: false packages: platform-tools cmdline-tools-version: 13114758 - name: Accept Android SDK licenses run: yes | sdkmanager --licenses >/dev/null - name: Install Android NDK timeout-minutes: 30 run: | for attempt in 1 2 3; do sdkmanager "ndk;27.3.13750724" && break [ "$attempt" = 3 ] && exit 1 echo "sdkmanager attempt $attempt failed; retrying"; sleep 20 done echo "ndk.dir=$ANDROID_HOME/ndk/27.3.13750724" >> android-plugin/local.properties - name: Update glslang external sources working-directory: 3rdparty/glslang run: python update_glslang_sources.py - name: Build plugin APK run: >- gradle --no-daemon -p android-plugin :app:assemblePluginRelease -Pmobilegl.apkSuffix="${GITHUB_SHA}" -Pmobilegl.logLevel=MOBILEGL_LOG_LEVEL_INFO -Pmobilegl.buildDisaggregated="${MOBILEGL_CI_DISAGGREGATED}" -Pmobilegl.buildDisaggregatedInproc="${MOBILEGL_CI_DISAGGREGATED}" --parallel --max-workers "$(nproc)" env: SIGNING_STORE_PASSWORD: ${{ secrets.SIGNING_STORE_PASSWORD }} SIGNING_KEY_ALIAS: ${{ secrets.SIGNING_KEY_ALIAS }} SIGNING_KEY_PASSWORD: ${{ secrets.SIGNING_KEY_PASSWORD }} - name: Download ANGLE x86_64 libraries run: | angle_dir="android-plugin/app/src/trace/jniLibs/x86_64" rm -rf "${angle_dir}" mkdir -p "${angle_dir}" package_angle_variant() { variant="$1" commit="$2" egl_sha="$3" gles_sha="$4" source_dir="${RUNNER_TEMP}/mobilegl-angle-${variant}" base="https://raw.githubusercontent.com/FCL-Team/FoldCraftLauncher/${commit}/FCLauncher/src/main/jniLibs/x86_64" mkdir -p "${source_dir}" curl -L --fail --retry 3 -o "${source_dir}/libEGL_angle.so" "${base}/libEGL_angle.so" curl -L --fail --retry 3 -o "${source_dir}/libGLESv2_angle.so" "${base}/libGLESv2_angle.so" echo "${egl_sha} ${source_dir}/libEGL_angle.so" | sha256sum -c - echo "${gles_sha} ${source_dir}/libGLESv2_angle.so" | sha256sum -c - for library in libEGL_angle libGLESv2_angle; do filename="${library}_${variant}.so" cp "${source_dir}/${library}.so" "${angle_dir}/${filename}" done } package_angle_variant \ ec889e6ea831 \ f2a3d510dffd8f6540a52e1a7d0c5787d151075b \ c41828768d089899fa058ec0bee711a91be88347f29bdb935223da6be1149c40 \ e4f820d99f94365c66df868c7740fef142fe5c0cd7c941790a9e30638857ca4d package_angle_variant \ 90a62123d794 \ bdcc96ac11c79001018ae4375eb73cb54a9f682f \ d0f4298ccc770cc801fc52e21733521646161e8a4adb3bd0052d9a1b57ee0ca8 \ 66fdc867e552192d553d59095ea2e3cef4829de65c356f1fd826027b1905972e - name: Build retrace APK run: >- gradle --no-daemon -p android-plugin :app:assembleTraceRelease -Pmobilegl.apkSuffix="${GITHUB_SHA}" -Pmobilegl.abis=all -Pmobilegl.debuggableRelease=true -Pmobilegl.logLevel=MOBILEGL_LOG_LEVEL_INFO -Pmobilegl.buildDisaggregated="${MOBILEGL_CI_DISAGGREGATED}" -Pmobilegl.buildDisaggregatedInproc="${MOBILEGL_CI_DISAGGREGATED}" --parallel --max-workers "$(nproc)" env: SIGNING_STORE_PASSWORD: ${{ secrets.SIGNING_STORE_PASSWORD }} SIGNING_KEY_ALIAS: ${{ secrets.SIGNING_KEY_ALIAS }} SIGNING_KEY_PASSWORD: ${{ secrets.SIGNING_KEY_PASSWORD }} - name: Show ccache stats if: always() run: ccache --show-stats # A build served from the cache stores nothing, so nothing would ever evict the objects # of older source states. Trim to CCACHE_MAXSIZE, least recently used first, before the # entry is saved; the objects this build hit were just touched and stay. - name: Trim ccache to its size limit run: | ccache --cleanup ccache --show-stats | grep -i 'cache size' || true du -sh "${CCACHE_DIR}" # Rewrite one rolling entry per job and branch. The upload carries the objects this # build used plus what fits under the size limit, but Actions cache keys are # immutable, so this branch's superseded blob is released first; --ref keeps the # delete off other branches' entries. Running after the build means a failed build # leaves the entry untouched. - name: Release superseded ccache entry env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ runner.os }}-apk-${{ github.job }}-ccache-v1 run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true - name: Save ccache continue-on-error: true uses: actions/cache/save@v5 with: path: .ccache key: ${{ runner.os }}-apk-${{ github.job }}-ccache-v1 - name: Verify APK metadata and packaging run: | AAPT2="$(find "$ANDROID_HOME/build-tools" -name aapt2 -type f | sort -V | tail -n 1)" plugin_apk="android-plugin/app/build/outputs/apk/plugin/release/MobileGL-plugin-release-${GITHUB_SHA}.apk" trace_apk="android-plugin/app/build/outputs/apk/trace/release/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk" test -f "${plugin_apk}" test -f "${trace_apk}" bash .github/scripts/validate-plugin-apks.sh "$AAPT2" "$plugin_apk" "$trace_apk" - name: Verify signed APKs run: | APKSIGNER="$(find "$ANDROID_HOME/build-tools" -name apksigner -type f | sort -V | tail -n 1)" mapfile -t APKS < <(printf '%s\n' \ "android-plugin/app/build/outputs/apk/plugin/release/MobileGL-plugin-release-${GITHUB_SHA}.apk" \ "android-plugin/app/build/outputs/apk/trace/release/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk") for APK in "${APKS[@]}"; do if [[ ! -f "$APK" ]]; then echo "::error::Expected release APK was not produced: $APK" exit 1 fi done for APK in "${APKS[@]}"; do if [[ "$APK" == *-unsigned.apk ]]; then echo "::error::Unsigned release APK produced: $APK" exit 1 fi "$APKSIGNER" verify --verbose "$APK" done - name: Upload plugin APK uses: actions/upload-artifact@v7 with: name: MobileGL-plugin-${{ env.date_today }}-${{ github.sha }} path: android-plugin/app/build/outputs/apk/plugin/release/MobileGL-plugin-release-${{ github.sha }}.apk archive: false if-no-files-found: error - name: Upload retrace APK uses: actions/upload-artifact@v7 with: name: MobileGL-retrace-apk-${{ env.date_today }}-${{ github.sha }} path: android-plugin/app/build/outputs/apk/trace/release/MobileGL-plugin-trace-release-${{ github.sha }}.apk archive: false if-no-files-found: error trace-cases: name: trace case matrix runs-on: ubuntu-26.04 env: MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'ci/round-30min' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }} # How much replay one emulator job aims for, in seconds as the duration hints count them. # A (lane, transport, backend) cell is cut into as many shards as that takes, and its # cases are spread so the shards of a cell come out even (tools/trace_replay/trace_cases.py, # shard_matrix). A smaller value gives more and shorter jobs, each of which pays the # emulator boot again; every case is replayed once per cell whatever the value. RETRACE_SHARD_SECONDS: "720" # The most a shard runner may spend on its cases. A shard gets the time in which every # one of its cases could run to the end of its own time budget, up to this cap. RETRACE_SHARD_DEADLINE_CAP_SECONDS: "7200" outputs: android: ${{ steps.trace-cases.outputs.android }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Load trace cases id: trace-cases run: | python3 tools/trace_replay/trace_cases.py --self-test-transport-matrices # The retrace jobs replay shards: several cases of one (lane, transport, backend) cell # per emulator boot. The self-test proves every shard matrix replays exactly the rows # of the per-case matrix it is built from, and trace_cases.py refuses to print one # that does not. python3 tools/trace_replay/trace_cases.py --self-test-shard-matrices matrix_format=github-apk-shard-matrix if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then matrix_format=github-apk-transport-shard-matrix fi android="$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}" \ --shard-seconds "${RETRACE_SHARD_SECONDS}" --android-lanes "${MOBILEGL_ANDROID_LANES}")" # An empty `include` is not an error to GitHub: it skips the retrace job with no red. shards="$(printf '%s' "${android}" | python3 -c 'import json, sys; print(len(json.load(sys.stdin)["include"]))')" if [ "${shards}" -lt 1 ]; then echo "::error::the Android retrace matrix is empty, so the retrace job would be skipped with no red" exit 1 fi # The cases of a shard share one job, so a case that hangs spends time the cases after # it need. retrace_shard_budget.py gives every shard its own deadline_seconds, the # time in which each of its cases could run to the end of its time budget, capped so # that a shard whose replays all hang cannot hold a runner for hours, and the # timeout_minutes of its job. The retrace job reads both from the matrix. android="$(printf '%s' "${android}" | python3 android-plugin/retrace_shard_budget.py \ --cap-seconds "${RETRACE_SHARD_DEADLINE_CAP_SECONDS}")" echo "android=${android}" >> "$GITHUB_OUTPUT" lanes="" if [ "${matrix_format}" = "github-apk-transport-shard-matrix" ]; then lanes=" (${MOBILEGL_ANDROID_LANES} lanes)" fi echo "Android retrace: ${shards} shard(s) from ${matrix_format}${lanes} at ${RETRACE_SHARD_SECONDS} s per shard" # One cache entry holds every Git LFS fixture of the CI cases, and each retrace job restores # that entry itself. This job only makes sure the entry exists. On a hit it downloads # nothing. On a miss it starts from the newest older bundle, fetches from the mirror the # cases whose pointers changed, saves the entry under the new key and confirms it is there, # so that no retrace job has to download fixtures itself. The key, the path and the steps # are .github/scripts/trace-fixture-bundle.sh's, and test.yml has the same job for the same # entry: the two workflows of one push look the key up at the same time, and on a miss both # fetch and both save. The save that loses is not an error, since the entry it wanted to # write is the one the confirming step then finds. The timeout bounds a fixture download # that stalls, which would otherwise hold every retrace job behind it. trace-fixtures: name: trace fixtures runs-on: ubuntu-26.04 timeout-minutes: 30 permissions: actions: read contents: read outputs: key: ${{ steps.bundle.outputs.key }} path: ${{ steps.bundle.outputs.path }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Derive the fixture bundle key id: bundle run: bash .github/scripts/trace-fixture-bundle.sh key - name: Look up the fixture bundle id: lookup uses: actions/cache/restore@v5 with: path: ${{ steps.bundle.outputs.path }} key: ${{ steps.bundle.outputs.key }} lookup-only: true - name: Restore the newest older bundle if: steps.lookup.outputs.cache-hit != 'true' uses: actions/cache/restore@v5 with: path: ${{ steps.bundle.outputs.path }} key: ${{ steps.bundle.outputs.key }} restore-keys: ${{ steps.bundle.outputs.restore-prefix }} - name: Complete the fixture bundle if: steps.lookup.outputs.cache-hit != 'true' run: bash .github/scripts/trace-fixture-bundle.sh ensure - name: Save the fixture bundle if: steps.lookup.outputs.cache-hit != 'true' continue-on-error: true uses: actions/cache/save@v5 with: path: ${{ steps.bundle.outputs.path }} key: ${{ steps.bundle.outputs.key }} # The Test workflow of the same push may be the one whose save won. Its entry is as good, # once it is listed. - name: Wait for the fixture bundle entry if: steps.lookup.outputs.cache-hit != 'true' env: GH_TOKEN: ${{ github.token }} run: bash .github/scripts/trace-fixture-bundle.sh wait '${{ steps.bundle.outputs.key }}' - name: Confirm the fixture bundle entry if: steps.lookup.outputs.cache-hit != 'true' uses: actions/cache/restore@v5 with: path: ${{ steps.bundle.outputs.path }} key: ${{ steps.bundle.outputs.key }} lookup-only: true fail-on-cache-miss: true android-avd: name: android avd image runs-on: ubuntu-26.04 env: AVD_NAME: mobilegl-ci ANDROID_AVD_HOME: ${{ github.workspace }}/.android/avd ANDROID_HOME: ${{ github.workspace }}/.android/sdk ANDROID_SDK_ROOT: ${{ github.workspace }}/.android/sdk steps: - name: Checkout repo uses: actions/checkout@v6 # The emulator shard loop and the artifact cleanup, driven by fakes: what each case is # run with, a verdict for every case, and what stays after a failure. It runs in this # job because the retrace jobs wait for it and the APK build does not. - name: Emulator shard runner regression run: | python3 android-plugin/test_trace_replay_shard.py python3 android-plugin/test_retrace_artifact_cleanup.py # This job only makes sure the cache entry exists; the retrace jobs restore it. A hit # downloads nothing and skips every step below; a miss builds the AVD, and the post step # of this action saves it. - name: Look up Android AVD cache id: android-avd-cache uses: actions/cache@v5 with: lookup-only: true path: | ${{ env.ANDROID_AVD_HOME }} ${{ env.ANDROID_SDK_ROOT }}/emulator ${{ env.ANDROID_SDK_ROOT }}/platform-tools ${{ env.ANDROID_SDK_ROOT }}/platforms/android-35 ${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64 key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v4-${{ hashFiles('android-plugin/run-avd-ci.sh') }} - name: Setup Android SDK id: netsetup336 if: steps.android-avd-cache.outputs.cache-hit != 'true' continue-on-error: true timeout-minutes: 15 uses: android-actions/setup-android@v4 with: accept-android-sdk-licenses: false packages: platform-tools cmdline-tools-version: 13114758 - name: Setup Android SDK (retry) timeout-minutes: 15 if: ${{ steps.netsetup336.outcome == 'failure' }} uses: android-actions/setup-android@v4 with: accept-android-sdk-licenses: false packages: platform-tools cmdline-tools-version: 13114758 - name: Accept Android SDK licenses if: steps.android-avd-cache.outputs.cache-hit != 'true' run: yes | sdkmanager --licenses >/dev/null - name: Create AVD if: steps.android-avd-cache.outputs.cache-hit != 'true' run: | sh android-plugin/run-avd-ci.sh create \ --api-level 35 \ --target google_apis \ --arch x86_64 \ --profile pixel_6 \ --avd-name "${AVD_NAME}" - name: Enable KVM if: steps.android-avd-cache.outputs.cache-hit != 'true' run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm # Boot the new AVD once so the cached AVD is past its first-boot setup (measured 37 s first # boot, 26 s for the next cold boot). A quick-boot snapshot was tried instead: it doubled the # cache restore and tripled "device offline" retries, so retrace jobs cold-boot. If the # pre-boot fails, the AVD is created afresh so the cache never holds a half-booted one. - name: Pre-boot AVD if: steps.android-avd-cache.outputs.cache-hit != 'true' run: | log="${RUNNER_TEMP}/avd-preboot.log" pid="${RUNNER_TEMP}/avd-preboot.pid" if ! sh android-plugin/run-avd-ci.sh preboot --avd-name "${AVD_NAME}" --gpu software \ --emulator-log "${log}" --pid-file "${pid}" --boot-timeout 300; then echo "::warning::the AVD pre-boot failed; caching a never-booted AVD instead" tail -50 "${log}" || true sh android-plugin/run-avd-ci.sh stop --avd-name "${AVD_NAME}" --emulator-log "${log}" --pid-file "${pid}" timeout 60 tail --pid="$(cat "${pid}")" -f /dev/null || true sh android-plugin/run-avd-ci.sh create --api-level 35 --target google_apis --arch x86_64 \ --profile pixel_6 --avd-name "${AVD_NAME}" fi du -sh "${ANDROID_AVD_HOME}" # One job replays a shard: some of the cases of one lane, transport and backend, one after # another on one emulator boot (android-plugin/trace_replay_shard.py). The shard's step # summary has a row per case, and a case that did not pass has an error annotation of its own. retrace: name: retrace (${{ matrix.backend.name }} / ${{ matrix.transport || 'monolith' }}, shard ${{ matrix.shard }}/${{ matrix.shards }}) runs-on: ubuntu-26.04 needs: - build - android-avd - trace-cases - trace-fixtures if: ${{ !cancelled() && needs.build.result == 'success' && needs.android-avd.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }} # Paired with --deadline-seconds below, and far above what a shard takes when its replays # end by themselves. Both come from the matrix. The deadline is the smaller of the time in # which every case of the shard could run to the end of its own time budget # (trace_replay_shard.py, needed_seconds) and RETRACE_SHARD_DEADLINE_CAP_SECONDS. With # budgets of up to 1800 s a case the cap is what a shard of several cases gets: about # three replays can hang to the end of their budgets before a later case is recorded as # not-run. The minutes between the deadline and this limit are the setup before the # replays and the uploads after them. timeout-minutes: ${{ matrix.timeout_minutes }} env: MOBILEGL_CI_TRANSPORT: ${{ matrix.transport || 'monolith' }} MOBILEGL_CI_LANE: ${{ matrix.lane || 'monolith-control' }} AVD_NAME: mobilegl-ci ANDROID_AVD_HOME: ${{ github.workspace }}/.android/avd ANDROID_HOME: ${{ github.workspace }}/.android/sdk ANDROID_SDK_ROOT: ${{ github.workspace }}/.android/sdk # The shard's cases in run order, as the JSON array of case objects the shard runner # takes: every field of the case in trace_cases.json, with the fixture paths in place. MOBILEGL_SHARD_CASES: ${{ toJSON(matrix.cases) }} # Set the repository variable to 1 to cold boot the emulator before every case of a # shard, should one boot not survive a whole shard of software-rendered replays. MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE: ${{ vars.MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE || '0' }} # The emulator renders on the host with a software rasterizer that can keep compiled # shaders on disk in the runner's home directory, where neither clearing the app's data # nor restarting the emulator reaches. Off, so every case of a shard compiles its # shaders as a job with a fresh machine does. MESA_SHADER_CACHE_DISABLE: "true" strategy: fail-fast: false # No max-parallel: the account's limit of 20 concurrent jobs, shared with Test, already # bounds this matrix, and a cap below the number of shards only makes the round longer. matrix: ${{ fromJSON(needs.trace-cases.outputs.android) }} steps: - name: Checkout repo uses: actions/checkout@v6 # First, so the steps that run whatever happened (diagnostics, stopping the emulator, the # result upload) have these even when a step before the emulator launch failed. - name: Set artifact metadata run: | echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV" echo "EMULATOR_LOG=${RUNNER_TEMP}/mobilegl-emulator.log" >> "$GITHUB_ENV" echo "EMULATOR_PID_FILE=${RUNNER_TEMP}/mobilegl-emulator.pid" >> "$GITHUB_ENV" - name: Set swap space uses: ./.github/actions/set-swap-space with: swap-size-gb: 8 # A miss does not fail the job: the entry may have been evicted since trace-fixtures # confirmed it (a re-run of failed jobs on an older run does not run that job again). - name: Restore trace fixtures uses: actions/cache/restore@v5 with: path: ${{ needs.trace-fixtures.outputs.path }} key: ${{ needs.trace-fixtures.outputs.key }} # Moves this shard's fixtures into place and checks them against their Git LFS pointers. # A case the restored bundle does not hold is downloaded here, with a warning annotation; # the timeout bounds a download that stalls. - name: Install trace fixtures timeout-minutes: 15 run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }} - name: Restore Android AVD cache id: android-avd-cache uses: actions/cache/restore@v5 with: path: | ${{ env.ANDROID_AVD_HOME }} ${{ env.ANDROID_SDK_ROOT }}/emulator ${{ env.ANDROID_SDK_ROOT }}/platform-tools ${{ env.ANDROID_SDK_ROOT }}/platforms/android-35 ${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64 key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v4-${{ hashFiles('android-plugin/run-avd-ci.sh') }} # The cache holds everything a retrace job runs (emulator, adb, image, AVD); the SDK manager # is only needed to create the AVD when the cache missed. - name: Use the cached SDK tools if: steps.android-avd-cache.outputs.cache-hit == 'true' run: | echo "${ANDROID_SDK_ROOT}/platform-tools" >> "$GITHUB_PATH" echo "${ANDROID_SDK_ROOT}/emulator" >> "$GITHUB_PATH" - name: Setup Android SDK id: netsetup415 if: steps.android-avd-cache.outputs.cache-hit != 'true' continue-on-error: true timeout-minutes: 15 uses: android-actions/setup-android@v4 with: accept-android-sdk-licenses: false packages: platform-tools cmdline-tools-version: 13114758 - name: Setup Android SDK (retry) timeout-minutes: 15 if: ${{ steps.netsetup415.outcome == 'failure' }} uses: android-actions/setup-android@v4 with: accept-android-sdk-licenses: false packages: platform-tools cmdline-tools-version: 13114758 - name: Accept Android SDK licenses if: steps.android-avd-cache.outputs.cache-hit != 'true' run: yes | sdkmanager --licenses >/dev/null - name: Download retrace APK uses: actions/download-artifact@v8 with: name: MobileGL-plugin-trace-release-${{ github.sha }}.apk path: android-retrace-apks - name: Enable KVM run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm - name: Create AVD if: steps.android-avd-cache.outputs.cache-hit != 'true' run: | sh android-plugin/run-avd-ci.sh create \ --api-level 35 \ --target google_apis \ --arch x86_64 \ --profile pixel_6 \ --avd-name "${AVD_NAME}" - name: Launch emulator run: | sh android-plugin/run-avd-ci.sh start \ --avd-name "${AVD_NAME}" \ --gpu "${{ matrix.backend.gpu }}" \ --emulator-log "${EMULATOR_LOG}" \ --pid-file "${EMULATOR_PID_FILE}" \ --boot-timeout 300 # One emulator boot replays every case of the shard in order, each in a fresh app process # with cleared app data and with the settings its case, backend and transport need. The # runner keeps going after a failing case, on a cold boot so that case cannot touch the # next one's verdict, retries a case once on a restarted emulator when the emulator # itself failed, and exits non-zero when a case failed or a verdict is missing. The # deadline stops it while the steps below can still upload what it has: the job timeout # less the setup above and the uploads below. A case it stops or never starts is # recorded as not-run, which is red. - name: Retrace and validate env: # Lets the artifact cleanup tell which job a shard's verdicts came from. MOBILEGL_SHARD_JOB_ID: ${{ job.check_run_id }} run: | sh android-plugin/trace-replay-shard.sh \ --apk-file "android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk" \ --backend "${{ matrix.backend.name }}" \ --gpu "${{ matrix.backend.gpu }}" \ --transport "${MOBILEGL_CI_TRANSPORT}" \ --lane "${MOBILEGL_CI_LANE}" \ --cases-env MOBILEGL_SHARD_CASES \ --deadline-seconds "${{ matrix.deadline_seconds }}" # Runs whatever happened above, without the device: every case of the shard gets its # lane.json and golden copies for the summary, and a not-run verdict when the runner was # killed or never started. - name: Collect retrace summary inputs if: always() env: MOBILEGL_SHARD_JOB_ID: ${{ job.check_run_id }} run: | sh android-plugin/trace-replay-shard.sh \ --apk-file "android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk" \ --backend "${{ matrix.backend.name }}" \ --gpu "${{ matrix.backend.gpu }}" \ --transport "${MOBILEGL_CI_TRANSPORT}" \ --lane "${MOBILEGL_CI_LANE}" \ --cases-env MOBILEGL_SHARD_CASES \ --finalize - name: Collect emulator diagnostics if: always() run: | mkdir -p android-retrace-result/diagnostics adb devices -l > android-retrace-result/diagnostics/adb-devices.txt || true timeout 30 adb logcat -d -t 1000 > android-retrace-result/diagnostics/logcat.txt || true # What the guest was given; replay time inside the emulator depends on it. timeout 30 adb shell 'nproc; grep -E "MemTotal|SwapTotal" /proc/meminfo' \ > android-retrace-result/diagnostics/guest-resources.txt 2>&1 || true cp "${ANDROID_AVD_HOME}/${AVD_NAME}.avd/config.ini" android-retrace-result/diagnostics/avd-config.ini 2>/dev/null || true # Expected to list nothing: the host rasterizer's on-disk shader cache is disabled. du -sh "${XDG_CACHE_HOME:-${HOME}/.cache}"/mesa_shader_cache* \ > android-retrace-result/diagnostics/host-shader-cache.txt 2>&1 || true # The log of the last boot. The shard runner keeps the log of every boot it ended # as diagnostics/emulator-boot--*.log before a restart truncates it. if [ -f "${EMULATOR_LOG}" ]; then cp "${EMULATOR_LOG}" android-retrace-result/diagnostics/emulator.log fi # A vanished emulator looks identical whether the host OOM killer took # qemu or the renderer faulted. These two say which. free -h > android-retrace-result/diagnostics/host-memory.txt 2>&1 || true sudo dmesg -T 2>/dev/null | tail -300 > android-retrace-result/diagnostics/host-dmesg.txt || true - name: Stop emulator if: always() run: | sh android-plugin/run-avd-ci.sh stop \ --avd-name "${AVD_NAME}" \ --emulator-log "${EMULATOR_LOG}" \ --pid-file "${EMULATOR_PID_FILE}" - name: Upload Android retrace result id: upload-retrace-result if: always() continue-on-error: true uses: actions/upload-artifact@v7 with: # The summary job downloads by the lane and transport at the front of this name, and # its renderer reads the backend and the case from the paths below it, so the name # ends in the shard number and holds no case name. name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-shard${{ matrix.shard }} path: android-retrace-result/** if-no-files-found: warn # A re-run of a failed job replaces the artifact its first attempt kept. overwrite: true # The artifact service sometimes times out (ETIMEDOUT on CreateArtifact, one job in 231 on # 2026-10-18), which turned a passing case red; one delayed retry absorbs that. - name: Wait before retrying the upload if: always() && steps.upload-retrace-result.outcome == 'failure' run: sleep 30 - name: Upload Android retrace result (retry) if: always() && steps.upload-retrace-result.outcome == 'failure' uses: actions/upload-artifact@v7 with: name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-shard${{ matrix.shard }} path: android-retrace-result/** if-no-files-found: warn overwrite: true retrace-summary: name: retrace summary runs-on: ubuntu-26.04 needs: retrace # Also runs for a canceled run, where it only downloads the results and deletes the # intermediate artifacts; the rendering steps are skipped then. The checkout and the # downloads say so themselves, because a step with no condition is skipped once its job is # canceled. if: always() permissions: actions: write contents: read env: MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'ci/round-30min' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }} steps: - name: Checkout repo if: ${{ success() || cancelled() }} uses: actions/checkout@v6 - name: Set up Node.js if: ${{ !cancelled() }} uses: actions/setup-node@v7 with: node-version: '22' - name: Download Android monolith control results if: ${{ success() || cancelled() }} uses: actions/download-artifact@v8 with: pattern: MobileGL-android-retrace-result-monolith-control-monolith-* path: retrace-artifacts/monolith-control - name: Download Android inproc acceptance results if: ${{ (success() || cancelled()) && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }} uses: actions/download-artifact@v8 with: pattern: MobileGL-android-retrace-result-inproc-acceptance-inproc-* path: retrace-artifacts/inproc-acceptance - name: Download Android spawn acceptance results if: ${{ (success() || cancelled()) && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }} uses: actions/download-artifact@v8 with: pattern: MobileGL-android-retrace-result-spawn-acceptance-spawn-* path: retrace-artifacts/spawn-acceptance # The reduced monolith and spawn lanes replay the android_smoke cases only, so their # summaries list those cases and no others. A branch outside the split shape replays # every case on monolith, its only lane. - name: Select the cases of the reduced lanes if: ${{ !cancelled() }} run: | reduced_lane_cases="" if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ] && [ "${MOBILEGL_ANDROID_LANES}" = "inproc-full" ]; then reduced_lane_cases="$(python3 tools/trace_replay/trace_cases.py --format android-smoke-names)" test -n "${reduced_lane_cases}" fi echo "REDUCED_LANE_CASES=${reduced_lane_cases}" >> "$GITHUB_ENV" - name: Render monolith control summary if: ${{ !cancelled() }} run: | node tools/trace_replay/render_retrace_summary.mjs \ --cases "${REDUCED_LANE_CASES}" \ --input retrace-artifacts/monolith-control \ --output-dir android-retrace-summary/monolith-control \ --title "MobileGL Android monolith control" \ --group-label "Android Emulator / monolith control" \ --html mobilegl-android-monolith-control.html - name: Render inproc acceptance summary if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }} run: | node tools/trace_replay/render_retrace_summary.mjs \ --input retrace-artifacts/inproc-acceptance \ --output-dir android-retrace-summary/inproc-acceptance \ --title "MobileGL Android inproc acceptance" \ --group-label "Android Emulator / inproc acceptance" \ --html mobilegl-android-inproc-acceptance.html - name: Render spawn acceptance summary if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }} run: | node tools/trace_replay/render_retrace_summary.mjs \ --cases "${REDUCED_LANE_CASES}" \ --input retrace-artifacts/spawn-acceptance \ --output-dir android-retrace-summary/spawn-acceptance \ --title "MobileGL Android spawn acceptance" \ --group-label "Android Emulator / spawn acceptance" \ --html mobilegl-android-spawn-acceptance.html - name: Upload Android monolith control summary if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 with: name: MobileGL-android-monolith-control-summary-${{ github.sha }} path: android-retrace-summary/monolith-control/mobilegl-android-monolith-control.html archive: false if-no-files-found: error - name: Upload Android inproc acceptance summary if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }} uses: actions/upload-artifact@v7 with: name: MobileGL-android-inproc-acceptance-summary-${{ github.sha }} path: android-retrace-summary/inproc-acceptance/mobilegl-android-inproc-acceptance.html archive: false if-no-files-found: error - name: Upload Android spawn acceptance summary if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }} uses: actions/upload-artifact@v7 with: name: MobileGL-android-spawn-acceptance-summary-${{ github.sha }} path: android-retrace-summary/spawn-acceptance/mobilegl-android-spawn-acceptance.html archive: false if-no-files-found: error # The summaries embed every image they show, so the per-shard results are intermediates # from here on. This is the last step of the workflow rather than a job of its own, which # would wait for a runner and start from an empty machine. It keeps what a failure needs, # read from the shard-status.json of the results downloaded above: the results of every # shard that ran a case which did not pass somewhere, on all backends and transports, and # every result when a job failed without saying which cases it ran. Deletes are paced # below GitHub's limit for mutating requests, and a refused delete is a warning. Skipped # when a summary step failed, so that re-running this job still finds the results, and # when a retrace job did not succeed in a run that was not canceled: every result stays # then, so a shard stopped by its job limit keeps its diagnostics and a re-run of the # failed jobs renders complete summaries. - name: Delete intermediate Android retrace artifacts if: ${{ (success() && needs.retrace.result == 'success') || cancelled() }} env: GH_TOKEN: ${{ github.token }} run: | python3 android-plugin/retrace_artifact_cleanup.py \ --downloads retrace-artifacts/monolith-control \ --downloads retrace-artifacts/inproc-acceptance \ --downloads retrace-artifacts/spawn-acceptance