swung0x48/MobileGL
1
0
Fork 0
mirror of https://github.com/MobileGL-Dev/MobileGL synced 2026-11-16 06:23:23 +09:00
Code Issues Packages Projects Releases Wiki Activity
Files
697bcc6d456b53959ce0ed98ce5a579f45c984fb
MobileGL/.github/workflows/apk.yml
T

937 lines
43 KiB
YAML
Raw Blame History

name: MobileGL APK
on:
push:
branches:
- dev
- Feat/Backend-Direct-GLES
- Feat/Backend-Direct-Vulkan
# TEMPORARY, remove before merging the MGPipe work into dev: the disaggregation
# branch runs the full lane on every push so a phase's landing is not gated on
# someone remembering to dispatch the workflow by hand.
- feat/disaggregated
- codex/fix-github-ci
- cleanup
# A push that only touches these paths changes nothing this workflow builds or replays. The
# Test workflow keeps running for docs: it hosts the doc citation lint. The Android build
# compiles none of the unit, integration or benchmark trees, with one exception that keeps
# MobileGL/MG_Benchmark from being listed whole: MG_Util/SelfTest/DriverBenchJni.cpp
# includes MG_Benchmark/Driver/DriverBenchCases.inc.
paths-ignore:
- 'docs/**'
- '**/*.md'
- 'MobileGL/MG_Test/**'
- 'MobileGL/MG_IntegrationTest/**'
- 'MobileGL/MG_Benchmark/CMakeLists.txt'
- 'MobileGL/MG_Benchmark/SanityBench.cpp'
- 'MobileGL/MG_Benchmark/Buffer/**'
- 'MobileGL/MG_Benchmark/Container/**'
- 'MobileGL/MG_Benchmark/Program/**'
- 'MobileGL/MG_Benchmark/ShaderCache/**'
- 'MobileGL/MG_Benchmark/Transpile/**'
- 'tools/cts/**'
- 'tools/device_bench/**'
- '.github/workflows/test.yml'
- '.github/workflows/hygiene.yml'
- '.github/actions/setup-test-runner/**'
workflow_dispatch:
env:
# Software-rendered CI submissions may exceed the interactive GPU hang budget.
MOBILEGL_GPU_HANG_BUDGET_MS: "0"
# ONE LIVE RUN PER BRANCH, as in test.yml: a superseded push is canceled rather than left queued
# behind its successor; a manual dispatch gets a group of its own; dev queues rather than cancels.
concurrency:
group: ${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.ref }}
cancel-in-progress: ${{ github.event_name == 'push' && github.ref != 'refs/heads/dev' }}
jobs:
build:
runs-on: ubuntu-26.04
permissions:
actions: write
contents: read
env:
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_COMPRESS: "true"
CCACHE_DIR: ${{ github.workspace }}/.ccache
# One build's objects (three native configurations) take about 1 GB. The limit leaves
# half of that again for the objects a push replaces, and is what the cleanup before
# the save trims the entry to.
CCACHE_MAXSIZE: 1500M
CCACHE_NOHASHDIR: "true"
MOBILEGL_CMAKE_COMPILER_LAUNCHER: ccache
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
# Only the submodules the two APKs are built from, shallow and in parallel, in place of a
# recursive checkout of every submodule in the tree: the ones the root CMakeLists.txt
# configures, and apitrace with the eight directories its thirdparty/CMakeLists.txt
# requires, for the trace APK. Each row names a submodule and one file the build reads
# from it, and a row whose file is missing afterwards fails the step: without
# 3rdparty/flatbuffers the root CMakeLists.txt would go on and build a library without
# MG_Remote instead of stopping. A submodule the build starts to use needs a row here.
- name: Check out build submodules
timeout-minutes: 15
run: |
top=(
"3rdparty/glslang CMakeLists.txt"
"3rdparty/SPIRV-Cross CMakeLists.txt"
"3rdparty/SPIRV-Reflect CMakeLists.txt"
"3rdparty/VulkanMemoryAllocator CMakeLists.txt"
"3rdparty/Vulkan-Headers CMakeLists.txt"
"3rdparty/Vulkan-Utility-Libraries CMakeLists.txt"
"3rdparty/xxHash build/cmake/CMakeLists.txt"
"3rdparty/asio include/asio.hpp"
"3rdparty/flatbuffers include/flatbuffers/flatbuffers.h"
"include/ska flat_hash_map.hpp"
"3rdparty/apitrace thirdparty/CMakeLists.txt"
)
apitrace=(
"thirdparty/brotli LICENSE"
"thirdparty/gtest LICENSE"
"thirdparty/libbacktrace LICENSE"
"thirdparty/libpng LICENSE"
"thirdparty/snappy COPYING"
"thirdparty/zlib README"
"thirdparty/zstd LICENSE"
"thirdparty/directxmath LICENSE"
)
# update <directory> <row>...: the update is idempotent, so a fetch that failed on
# the network is tried again, three times in all. Then every row's file must exist.
update() {
local dir="$1" paths=() row attempt updated=0 missing=0
shift
for row in "$@"; do
paths+=("${row%% *}")
done
for attempt in 1 2 3; do
if git -C "${dir}" submodule update --init --depth=1 --jobs 8 -- "${paths[@]}"; then
updated=1
break
fi
echo "the submodule update in ${dir} failed (attempt ${attempt} of 3)" >&2
sleep $((attempt * 5))
done
[ "${updated}" -eq 1 ] || return 1
for row in "$@"; do
if [ ! -f "${dir}/${row%% *}/${row#* }" ]; then
echo "::error::${dir}/${row%% *}/${row#* } is missing after the submodule update, so the APK build cannot use ${row%% *}"
missing=1
fi
done
return "${missing}"
}
update . "${top[@]}"
update 3rdparty/apitrace "${apitrace[@]}"
- name: Set artifact metadata
run: |
echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
- name: Set up JDK
uses: actions/setup-java@v5
with:
distribution: zulu
java-version: '17'
- name: Android replay lifecycle regression
run: python3 tools/trace_replay/test_android_lifecycle.py
- name: Setup Gradle
uses: gradle/actions/setup-gradle@v6
with:
gradle-version: 8.10.2
- name: Restore ccache
uses: actions/cache/restore@v5
with:
path: .ccache
key: ${{ runner.os }}-apk-${{ github.job }}-ccache-v1
restore-keys: |
${{ runner.os }}-apk-${{ github.job }}-ccache-
- name: Install ccache
timeout-minutes: 15
run: |
sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 update
sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 install -y ccache
ccache --version
# The restored entry carries the counters of every earlier build; zero them so the
# statistics printed after the build are this run's.
ccache --zero-stats
- name: Setup Android SDK
id: netsetup84
continue-on-error: true
timeout-minutes: 15
uses: android-actions/setup-android@v4
with:
accept-android-sdk-licenses: false
packages: platform-tools
cmdline-tools-version: 13114758
- name: Setup Android SDK (retry)
timeout-minutes: 15
if: ${{ steps.netsetup84.outcome == 'failure' }}
uses: android-actions/setup-android@v4
with:
accept-android-sdk-licenses: false
packages: platform-tools
cmdline-tools-version: 13114758
- name: Accept Android SDK licenses
run: yes | sdkmanager --licenses >/dev/null
- name: Install Android NDK
timeout-minutes: 30
run: |
for attempt in 1 2 3; do
sdkmanager "ndk;27.3.13750724" && break
[ "$attempt" = 3 ] && exit 1
echo "sdkmanager attempt $attempt failed; retrying"; sleep 20
done
echo "ndk.dir=$ANDROID_HOME/ndk/27.3.13750724" >> android-plugin/local.properties
- name: Update glslang external sources
working-directory: 3rdparty/glslang
run: python update_glslang_sources.py
- name: Build plugin APK
run: >-
gradle --no-daemon -p android-plugin :app:assemblePluginRelease
-Pmobilegl.apkSuffix="${GITHUB_SHA}" -Pmobilegl.logLevel=MOBILEGL_LOG_LEVEL_INFO
-Pmobilegl.buildDisaggregated="${MOBILEGL_CI_DISAGGREGATED}"
-Pmobilegl.buildDisaggregatedInproc="${MOBILEGL_CI_DISAGGREGATED}"
--parallel --max-workers "$(nproc)"
env:
SIGNING_STORE_PASSWORD: ${{ secrets.SIGNING_STORE_PASSWORD }}
SIGNING_KEY_ALIAS: ${{ secrets.SIGNING_KEY_ALIAS }}
SIGNING_KEY_PASSWORD: ${{ secrets.SIGNING_KEY_PASSWORD }}
- name: Download ANGLE x86_64 libraries
run: |
angle_dir="android-plugin/app/src/trace/jniLibs/x86_64"
rm -rf "${angle_dir}"
mkdir -p "${angle_dir}"
package_angle_variant() {
variant="$1"
commit="$2"
egl_sha="$3"
gles_sha="$4"
source_dir="${RUNNER_TEMP}/mobilegl-angle-${variant}"
base="https://raw.githubusercontent.com/FCL-Team/FoldCraftLauncher/${commit}/FCLauncher/src/main/jniLibs/x86_64"
mkdir -p "${source_dir}"
curl -L --fail --retry 3 -o "${source_dir}/libEGL_angle.so" "${base}/libEGL_angle.so"
curl -L --fail --retry 3 -o "${source_dir}/libGLESv2_angle.so" "${base}/libGLESv2_angle.so"
echo "${egl_sha} ${source_dir}/libEGL_angle.so" | sha256sum -c -
echo "${gles_sha} ${source_dir}/libGLESv2_angle.so" | sha256sum -c -
for library in libEGL_angle libGLESv2_angle; do
filename="${library}_${variant}.so"
cp "${source_dir}/${library}.so" "${angle_dir}/${filename}"
done
}
package_angle_variant \
ec889e6ea831 \
f2a3d510dffd8f6540a52e1a7d0c5787d151075b \
c41828768d089899fa058ec0bee711a91be88347f29bdb935223da6be1149c40 \
e4f820d99f94365c66df868c7740fef142fe5c0cd7c941790a9e30638857ca4d
package_angle_variant \
90a62123d794 \
bdcc96ac11c79001018ae4375eb73cb54a9f682f \
d0f4298ccc770cc801fc52e21733521646161e8a4adb3bd0052d9a1b57ee0ca8 \
66fdc867e552192d553d59095ea2e3cef4829de65c356f1fd826027b1905972e
- name: Build retrace APK
run: >-
gradle --no-daemon -p android-plugin :app:assembleTraceRelease
-Pmobilegl.apkSuffix="${GITHUB_SHA}" -Pmobilegl.abis=all
-Pmobilegl.debuggableRelease=true -Pmobilegl.logLevel=MOBILEGL_LOG_LEVEL_INFO
-Pmobilegl.buildDisaggregated="${MOBILEGL_CI_DISAGGREGATED}"
-Pmobilegl.buildDisaggregatedInproc="${MOBILEGL_CI_DISAGGREGATED}"
--parallel --max-workers "$(nproc)"
env:
SIGNING_STORE_PASSWORD: ${{ secrets.SIGNING_STORE_PASSWORD }}
SIGNING_KEY_ALIAS: ${{ secrets.SIGNING_KEY_ALIAS }}
SIGNING_KEY_PASSWORD: ${{ secrets.SIGNING_KEY_PASSWORD }}
- name: Show ccache stats
if: always()
run: ccache --show-stats
# A build served from the cache stores nothing, so nothing would ever evict the objects
# of older source states. Trim to CCACHE_MAXSIZE, least recently used first, before the
# entry is saved; the objects this build hit were just touched and stay.
- name: Trim ccache to its size limit
run: |
ccache --cleanup
ccache --show-stats | grep -i 'cache size' || true
du -sh "${CCACHE_DIR}"
# Rewrite one rolling entry per job and branch. The upload carries the objects this
# build used plus what fits under the size limit, but Actions cache keys are
# immutable, so this branch's superseded blob is released first; --ref keeps the
# delete off other branches' entries. Running after the build means a failed build
# leaves the entry untouched.
- name: Release superseded ccache entry
env:
GH_TOKEN: ${{ github.token }}
CACHE_KEY: ${{ runner.os }}-apk-${{ github.job }}-ccache-v1
run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true
- name: Save ccache
continue-on-error: true
uses: actions/cache/save@v5
with:
path: .ccache
key: ${{ runner.os }}-apk-${{ github.job }}-ccache-v1
- name: Verify APK metadata and packaging
run: |
AAPT2="$(find "$ANDROID_HOME/build-tools" -name aapt2 -type f | sort -V | tail -n 1)"
plugin_apk="android-plugin/app/build/outputs/apk/plugin/release/MobileGL-plugin-release-${GITHUB_SHA}.apk"
trace_apk="android-plugin/app/build/outputs/apk/trace/release/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk"
test -f "${plugin_apk}"
test -f "${trace_apk}"
bash .github/scripts/validate-plugin-apks.sh "$AAPT2" "$plugin_apk" "$trace_apk"
- name: Verify signed APKs
run: |
APKSIGNER="$(find "$ANDROID_HOME/build-tools" -name apksigner -type f | sort -V | tail -n 1)"
mapfile -t APKS < <(printf '%s\n' \
"android-plugin/app/build/outputs/apk/plugin/release/MobileGL-plugin-release-${GITHUB_SHA}.apk" \
"android-plugin/app/build/outputs/apk/trace/release/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk")
for APK in "${APKS[@]}"; do
if [[ ! -f "$APK" ]]; then
echo "::error::Expected release APK was not produced: $APK"
exit 1
fi
done
for APK in "${APKS[@]}"; do
if [[ "$APK" == *-unsigned.apk ]]; then
echo "::error::Unsigned release APK produced: $APK"
exit 1
fi
"$APKSIGNER" verify --verbose "$APK"
done
- name: Upload plugin APK
uses: actions/upload-artifact@v7
with:
name: MobileGL-plugin-${{ env.date_today }}-${{ github.sha }}
path: android-plugin/app/build/outputs/apk/plugin/release/MobileGL-plugin-release-${{ github.sha }}.apk
archive: false
if-no-files-found: error
- name: Upload retrace APK
uses: actions/upload-artifact@v7
with:
name: MobileGL-retrace-apk-${{ env.date_today }}-${{ github.sha }}
path: android-plugin/app/build/outputs/apk/trace/release/MobileGL-plugin-trace-release-${{ github.sha }}.apk
archive: false
if-no-files-found: error
trace-cases:
name: trace case matrix
runs-on: ubuntu-26.04
env:
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
# How much replay one emulator job aims for, in seconds as the duration hints count them.
# A (lane, transport, backend) cell is cut into as many shards as that takes, and its
# cases are spread so the shards of a cell come out even (tools/trace_replay/trace_cases.py,
# shard_matrix). A smaller value gives more and shorter jobs, each of which pays the
# emulator boot again; every case is replayed once per cell whatever the value.
RETRACE_SHARD_SECONDS: "720"
# The most a shard runner may spend on its cases. A shard gets the time in which every
# one of its cases could run to the end of its own time budget, up to this cap.
RETRACE_SHARD_DEADLINE_CAP_SECONDS: "7200"
outputs:
android: ${{ steps.trace-cases.outputs.android }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Load trace cases
id: trace-cases
run: |
python3 tools/trace_replay/trace_cases.py --self-test-transport-matrices
# The retrace jobs replay shards: several cases of one (lane, transport, backend) cell
# per emulator boot. The self-test proves every shard matrix replays exactly the rows
# of the per-case matrix it is built from, and trace_cases.py refuses to print one
# that does not.
python3 tools/trace_replay/trace_cases.py --self-test-shard-matrices
matrix_format=github-apk-shard-matrix
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then
matrix_format=github-apk-transport-shard-matrix
fi
android="$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}" \
--shard-seconds "${RETRACE_SHARD_SECONDS}")"
# An empty `include` is not an error to GitHub: it skips the retrace job with no red.
shards="$(printf '%s' "${android}" | python3 -c 'import json, sys; print(len(json.load(sys.stdin)["include"]))')"
if [ "${shards}" -lt 1 ]; then
echo "::error::the Android retrace matrix is empty, so the retrace job would be skipped with no red"
exit 1
fi
# The cases of a shard share one job, so a case that hangs spends time the cases after
# it need. retrace_shard_budget.py gives every shard its own deadline_seconds, the
# time in which each of its cases could run to the end of its time budget, capped so
# that a shard whose replays all hang cannot hold a runner for hours, and the
# timeout_minutes of its job. The retrace job reads both from the matrix.
android="$(printf '%s' "${android}" | python3 android-plugin/retrace_shard_budget.py \
--cap-seconds "${RETRACE_SHARD_DEADLINE_CAP_SECONDS}")"
echo "android=${android}" >> "$GITHUB_OUTPUT"
echo "Android retrace: ${shards} shard(s) from ${matrix_format} at ${RETRACE_SHARD_SECONDS} s per shard"
# One cache entry holds every Git LFS fixture of the CI cases, and each retrace job restores
# that entry itself. This job only makes sure the entry exists. On a hit it downloads
# nothing. On a miss it starts from the newest older bundle, fetches from the mirror the
# cases whose pointers changed, saves the entry under the new key and confirms it is there,
# so that no retrace job has to download fixtures itself. The key, the path and the steps
# are .github/scripts/trace-fixture-bundle.sh's, and test.yml has the same job for the same
# entry: the two workflows of one push look the key up at the same time, and on a miss both
# fetch and both save. The save that loses is not an error, since the entry it wanted to
# write is the one the confirming step then finds. The timeout bounds a fixture download
# that stalls, which would otherwise hold every retrace job behind it.
trace-fixtures:
name: trace fixtures
runs-on: ubuntu-26.04
timeout-minutes: 30
permissions:
actions: read
contents: read
outputs:
key: ${{ steps.bundle.outputs.key }}
path: ${{ steps.bundle.outputs.path }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Derive the fixture bundle key
id: bundle
run: bash .github/scripts/trace-fixture-bundle.sh key
- name: Look up the fixture bundle
id: lookup
uses: actions/cache/restore@v5
with:
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
lookup-only: true
- name: Restore the newest older bundle
if: steps.lookup.outputs.cache-hit != 'true'
uses: actions/cache/restore@v5
with:
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
restore-keys: ${{ steps.bundle.outputs.restore-prefix }}
- name: Complete the fixture bundle
if: steps.lookup.outputs.cache-hit != 'true'
run: bash .github/scripts/trace-fixture-bundle.sh ensure
- name: Save the fixture bundle
if: steps.lookup.outputs.cache-hit != 'true'
continue-on-error: true
uses: actions/cache/save@v5
with:
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
# The Test workflow of the same push may be the one whose save won. Its entry is as good,
# once it is listed.
- name: Wait for the fixture bundle entry
if: steps.lookup.outputs.cache-hit != 'true'
env:
GH_TOKEN: ${{ github.token }}
run: bash .github/scripts/trace-fixture-bundle.sh wait '${{ steps.bundle.outputs.key }}'
- name: Confirm the fixture bundle entry
if: steps.lookup.outputs.cache-hit != 'true'
uses: actions/cache/restore@v5
with:
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
lookup-only: true
fail-on-cache-miss: true
android-avd:
name: android avd image
runs-on: ubuntu-26.04
env:
AVD_NAME: mobilegl-ci
ANDROID_AVD_HOME: ${{ github.workspace }}/.android/avd
ANDROID_HOME: ${{ github.workspace }}/.android/sdk
ANDROID_SDK_ROOT: ${{ github.workspace }}/.android/sdk
steps:
- name: Checkout repo
uses: actions/checkout@v6
# The emulator shard loop and the artifact cleanup, driven by fakes: what each case is
# run with, a verdict for every case, and what stays after a failure. It runs in this
# job because the retrace jobs wait for it and the APK build does not.
- name: Emulator shard runner regression
run: |
python3 android-plugin/test_trace_replay_shard.py
python3 android-plugin/test_retrace_artifact_cleanup.py
# This job only makes sure the cache entry exists; the retrace jobs restore it. A hit
# downloads nothing and skips every step below; a miss builds the AVD, and the post step
# of this action saves it.
- name: Look up Android AVD cache
id: android-avd-cache
uses: actions/cache@v5
with:
lookup-only: true
path: |
${{ env.ANDROID_AVD_HOME }}
${{ env.ANDROID_SDK_ROOT }}/emulator
${{ env.ANDROID_SDK_ROOT }}/platform-tools
${{ env.ANDROID_SDK_ROOT }}/platforms/android-35
${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64
key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v4-${{ hashFiles('android-plugin/run-avd-ci.sh') }}
- name: Setup Android SDK
id: netsetup336
if: steps.android-avd-cache.outputs.cache-hit != 'true'
continue-on-error: true
timeout-minutes: 15
uses: android-actions/setup-android@v4
with:
accept-android-sdk-licenses: false
packages: platform-tools
cmdline-tools-version: 13114758
- name: Setup Android SDK (retry)
timeout-minutes: 15
if: ${{ steps.netsetup336.outcome == 'failure' }}
uses: android-actions/setup-android@v4
with:
accept-android-sdk-licenses: false
packages: platform-tools
cmdline-tools-version: 13114758
- name: Accept Android SDK licenses
if: steps.android-avd-cache.outputs.cache-hit != 'true'
run: yes | sdkmanager --licenses >/dev/null
- name: Create AVD
if: steps.android-avd-cache.outputs.cache-hit != 'true'
run: |
sh android-plugin/run-avd-ci.sh create \
--api-level 35 \
--target google_apis \
--arch x86_64 \
--profile pixel_6 \
--avd-name "${AVD_NAME}"
- name: Enable KVM
if: steps.android-avd-cache.outputs.cache-hit != 'true'
run: |
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
sudo udevadm control --reload-rules
sudo udevadm trigger --name-match=kvm
# Boot the new AVD once so the cached AVD is past its first-boot setup (measured 37 s first
# boot, 26 s for the next cold boot). A quick-boot snapshot was tried instead: it doubled the
# cache restore and tripled "device offline" retries, so retrace jobs cold-boot. If the
# pre-boot fails, the AVD is created afresh so the cache never holds a half-booted one.
- name: Pre-boot AVD
if: steps.android-avd-cache.outputs.cache-hit != 'true'
run: |
log="${RUNNER_TEMP}/avd-preboot.log"
pid="${RUNNER_TEMP}/avd-preboot.pid"
if ! sh android-plugin/run-avd-ci.sh preboot --avd-name "${AVD_NAME}" --gpu software \
--emulator-log "${log}" --pid-file "${pid}" --boot-timeout 300; then
echo "::warning::the AVD pre-boot failed; caching a never-booted AVD instead"
tail -50 "${log}" || true
sh android-plugin/run-avd-ci.sh stop --avd-name "${AVD_NAME}" --emulator-log "${log}" --pid-file "${pid}"
timeout 60 tail --pid="$(cat "${pid}")" -f /dev/null || true
sh android-plugin/run-avd-ci.sh create --api-level 35 --target google_apis --arch x86_64 \
--profile pixel_6 --avd-name "${AVD_NAME}"
fi
du -sh "${ANDROID_AVD_HOME}"
# One job replays a shard: some of the cases of one lane, transport and backend, one after
# another on one emulator boot (android-plugin/trace_replay_shard.py). The shard's step
# summary has a row per case, and a case that did not pass has an error annotation of its own.
retrace:
name: retrace (${{ matrix.backend.name }} / ${{ matrix.transport || 'monolith' }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
runs-on: ubuntu-26.04
needs:
- build
- android-avd
- trace-cases
- trace-fixtures
if: ${{ !cancelled() && needs.build.result == 'success' && needs.android-avd.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }}
# Paired with --deadline-seconds below, and far above what a shard takes when its replays
# end by themselves. Both come from the matrix. The deadline is the smaller of the time in
# which every case of the shard could run to the end of its own time budget
# (trace_replay_shard.py, needed_seconds) and RETRACE_SHARD_DEADLINE_CAP_SECONDS. With
# budgets of up to 1800 s a case the cap is what a shard of several cases gets: about
# three replays can hang to the end of their budgets before a later case is recorded as
# not-run. The minutes between the deadline and this limit are the setup before the
# replays and the uploads after them.
timeout-minutes: ${{ matrix.timeout_minutes }}
env:
MOBILEGL_CI_TRANSPORT: ${{ matrix.transport || 'monolith' }}
MOBILEGL_CI_LANE: ${{ matrix.lane || 'monolith-control' }}
AVD_NAME: mobilegl-ci
ANDROID_AVD_HOME: ${{ github.workspace }}/.android/avd
ANDROID_HOME: ${{ github.workspace }}/.android/sdk
ANDROID_SDK_ROOT: ${{ github.workspace }}/.android/sdk
# The shard's cases in run order, as the JSON array of case objects the shard runner
# takes: every field of the case in trace_cases.json, with the fixture paths in place.
MOBILEGL_SHARD_CASES: ${{ toJSON(matrix.cases) }}
# Set the repository variable to 1 to cold boot the emulator before every case of a
# shard, should one boot not survive a whole shard of software-rendered replays.
MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE: ${{ vars.MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE || '0' }}
# The emulator renders on the host with a software rasterizer that can keep compiled
# shaders on disk in the runner's home directory, where neither clearing the app's data
# nor restarting the emulator reaches. Off, so every case of a shard compiles its
# shaders as a job with a fresh machine does.
MESA_SHADER_CACHE_DISABLE: "true"
strategy:
fail-fast: false
# No max-parallel: the account's limit of 20 concurrent jobs, shared with Test, already
# bounds this matrix, and a cap below the number of shards only makes the round longer.
matrix: ${{ fromJSON(needs.trace-cases.outputs.android) }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
# First, so the steps that run whatever happened (diagnostics, stopping the emulator, the
# result upload) have these even when a step before the emulator launch failed.
- name: Set artifact metadata
run: |
echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
echo "EMULATOR_LOG=${RUNNER_TEMP}/mobilegl-emulator.log" >> "$GITHUB_ENV"
echo "EMULATOR_PID_FILE=${RUNNER_TEMP}/mobilegl-emulator.pid" >> "$GITHUB_ENV"
- name: Set swap space
uses: ./.github/actions/set-swap-space
with:
swap-size-gb: 8
# A miss does not fail the job: the entry may have been evicted since trace-fixtures
# confirmed it (a re-run of failed jobs on an older run does not run that job again).
- name: Restore trace fixtures
uses: actions/cache/restore@v5
with:
path: ${{ needs.trace-fixtures.outputs.path }}
key: ${{ needs.trace-fixtures.outputs.key }}
# Moves this shard's fixtures into place and checks them against their Git LFS pointers.
# A case the restored bundle does not hold is downloaded here, with a warning annotation;
# the timeout bounds a download that stalls.
- name: Install trace fixtures
timeout-minutes: 15
run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
- name: Restore Android AVD cache
id: android-avd-cache
uses: actions/cache/restore@v5
with:
path: |
${{ env.ANDROID_AVD_HOME }}
${{ env.ANDROID_SDK_ROOT }}/emulator
${{ env.ANDROID_SDK_ROOT }}/platform-tools
${{ env.ANDROID_SDK_ROOT }}/platforms/android-35
${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64
key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v4-${{ hashFiles('android-plugin/run-avd-ci.sh') }}
# The cache holds everything a retrace job runs (emulator, adb, image, AVD); the SDK manager
# is only needed to create the AVD when the cache missed.
- name: Use the cached SDK tools
if: steps.android-avd-cache.outputs.cache-hit == 'true'
run: |
echo "${ANDROID_SDK_ROOT}/platform-tools" >> "$GITHUB_PATH"
echo "${ANDROID_SDK_ROOT}/emulator" >> "$GITHUB_PATH"
- name: Setup Android SDK
id: netsetup415
if: steps.android-avd-cache.outputs.cache-hit != 'true'
continue-on-error: true
timeout-minutes: 15
uses: android-actions/setup-android@v4
with:
accept-android-sdk-licenses: false
packages: platform-tools
cmdline-tools-version: 13114758
- name: Setup Android SDK (retry)
timeout-minutes: 15
if: ${{ steps.netsetup415.outcome == 'failure' }}
uses: android-actions/setup-android@v4
with:
accept-android-sdk-licenses: false
packages: platform-tools
cmdline-tools-version: 13114758
- name: Accept Android SDK licenses
if: steps.android-avd-cache.outputs.cache-hit != 'true'
run: yes | sdkmanager --licenses >/dev/null
- name: Download retrace APK
uses: actions/download-artifact@v8
with:
name: MobileGL-plugin-trace-release-${{ github.sha }}.apk
path: android-retrace-apks
- name: Enable KVM
run: |
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
sudo udevadm control --reload-rules
sudo udevadm trigger --name-match=kvm
- name: Create AVD
if: steps.android-avd-cache.outputs.cache-hit != 'true'
run: |
sh android-plugin/run-avd-ci.sh create \
--api-level 35 \
--target google_apis \
--arch x86_64 \
--profile pixel_6 \
--avd-name "${AVD_NAME}"
- name: Launch emulator
run: |
sh android-plugin/run-avd-ci.sh start \
--avd-name "${AVD_NAME}" \
--gpu "${{ matrix.backend.gpu }}" \
--emulator-log "${EMULATOR_LOG}" \
--pid-file "${EMULATOR_PID_FILE}" \
--boot-timeout 300
# One emulator boot replays every case of the shard in order, each in a fresh app process
# with cleared app data and with the settings its case, backend and transport need. The
# runner keeps going after a failing case, on a cold boot so that case cannot touch the
# next one's verdict, retries a case once on a restarted emulator when the emulator
# itself failed, and exits non-zero when a case failed or a verdict is missing. The
# deadline stops it while the steps below can still upload what it has: the job timeout
# less the setup above and the uploads below. A case it stops or never starts is
# recorded as not-run, which is red.
- name: Retrace and validate
env:
# Lets the artifact cleanup tell which job a shard's verdicts came from.
MOBILEGL_SHARD_JOB_ID: ${{ job.check_run_id }}
run: |
sh android-plugin/trace-replay-shard.sh \
--apk-file "android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk" \
--backend "${{ matrix.backend.name }}" \
--gpu "${{ matrix.backend.gpu }}" \
--transport "${MOBILEGL_CI_TRANSPORT}" \
--lane "${MOBILEGL_CI_LANE}" \
--cases-env MOBILEGL_SHARD_CASES \
--deadline-seconds "${{ matrix.deadline_seconds }}"
# Runs whatever happened above, without the device: every case of the shard gets its
# lane.json and golden copies for the summary, and a not-run verdict when the runner was
# killed or never started.
- name: Collect retrace summary inputs
if: always()
env:
MOBILEGL_SHARD_JOB_ID: ${{ job.check_run_id }}
run: |
sh android-plugin/trace-replay-shard.sh \
--apk-file "android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk" \
--backend "${{ matrix.backend.name }}" \
--gpu "${{ matrix.backend.gpu }}" \
--transport "${MOBILEGL_CI_TRANSPORT}" \
--lane "${MOBILEGL_CI_LANE}" \
--cases-env MOBILEGL_SHARD_CASES \
--finalize
- name: Collect emulator diagnostics
if: always()
run: |
mkdir -p android-retrace-result/diagnostics
adb devices -l > android-retrace-result/diagnostics/adb-devices.txt || true
timeout 30 adb logcat -d -t 1000 > android-retrace-result/diagnostics/logcat.txt || true
# What the guest was given; replay time inside the emulator depends on it.
timeout 30 adb shell 'nproc; grep -E "MemTotal|SwapTotal" /proc/meminfo' \
> android-retrace-result/diagnostics/guest-resources.txt 2>&1 || true
cp "${ANDROID_AVD_HOME}/${AVD_NAME}.avd/config.ini" android-retrace-result/diagnostics/avd-config.ini 2>/dev/null || true
# Expected to list nothing: the host rasterizer's on-disk shader cache is disabled.
du -sh "${XDG_CACHE_HOME:-${HOME}/.cache}"/mesa_shader_cache* \
> android-retrace-result/diagnostics/host-shader-cache.txt 2>&1 || true
# The log of the last boot. The shard runner keeps the log of every boot it ended
# as diagnostics/emulator-boot-<n>-*.log before a restart truncates it.
if [ -f "${EMULATOR_LOG}" ]; then
cp "${EMULATOR_LOG}" android-retrace-result/diagnostics/emulator.log
fi
# A vanished emulator looks identical whether the host OOM killer took
# qemu or the renderer faulted. These two say which.
free -h > android-retrace-result/diagnostics/host-memory.txt 2>&1 || true
sudo dmesg -T 2>/dev/null | tail -300 > android-retrace-result/diagnostics/host-dmesg.txt || true
- name: Stop emulator
if: always()
run: |
sh android-plugin/run-avd-ci.sh stop \
--avd-name "${AVD_NAME}" \
--emulator-log "${EMULATOR_LOG}" \
--pid-file "${EMULATOR_PID_FILE}"
- name: Upload Android retrace result
id: upload-retrace-result
if: always()
continue-on-error: true
uses: actions/upload-artifact@v7
with:
# The summary job downloads by the lane and transport at the front of this name, and
# its renderer reads the backend and the case from the paths below it, so the name
# ends in the shard number and holds no case name.
name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-shard${{ matrix.shard }}
path: android-retrace-result/**
if-no-files-found: warn
# A re-run of a failed job replaces the artifact its first attempt kept.
overwrite: true
# The artifact service sometimes times out (ETIMEDOUT on CreateArtifact, one job in 231 on
# 2026-10-18), which turned a passing case red; one delayed retry absorbs that.
- name: Wait before retrying the upload
if: always() && steps.upload-retrace-result.outcome == 'failure'
run: sleep 30
- name: Upload Android retrace result (retry)
if: always() && steps.upload-retrace-result.outcome == 'failure'
uses: actions/upload-artifact@v7
with:
name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-shard${{ matrix.shard }}
path: android-retrace-result/**
if-no-files-found: warn
overwrite: true
retrace-summary:
name: retrace summary
runs-on: ubuntu-26.04
needs: retrace
# Also runs for a canceled run, where it only downloads the results and deletes the
# intermediate artifacts; the rendering steps are skipped then. The checkout and the
# downloads say so themselves, because a step with no condition is skipped once its job is
# canceled.
if: always()
permissions:
actions: write
contents: read
env:
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
steps:
- name: Checkout repo
if: ${{ success() || cancelled() }}
uses: actions/checkout@v6
- name: Set up Node.js
if: ${{ !cancelled() }}
uses: actions/setup-node@v7
with:
node-version: '22'
- name: Download Android monolith control results
if: ${{ success() || cancelled() }}
uses: actions/download-artifact@v8
with:
pattern: MobileGL-android-retrace-result-monolith-control-monolith-*
path: retrace-artifacts/monolith-control
- name: Download Android inproc acceptance results
if: ${{ (success() || cancelled()) && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
uses: actions/download-artifact@v8
with:
pattern: MobileGL-android-retrace-result-inproc-acceptance-inproc-*
path: retrace-artifacts/inproc-acceptance
- name: Download Android spawn acceptance results
if: ${{ (success() || cancelled()) && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
uses: actions/download-artifact@v8
with:
pattern: MobileGL-android-retrace-result-spawn-acceptance-spawn-*
path: retrace-artifacts/spawn-acceptance
- name: Render monolith control summary
if: ${{ !cancelled() }}
run: |
node tools/trace_replay/render_retrace_summary.mjs \
--input retrace-artifacts/monolith-control \
--output-dir android-retrace-summary/monolith-control \
--title "MobileGL Android monolith control" \
--group-label "Android Emulator / monolith control" \
--html mobilegl-android-monolith-control.html
- name: Render inproc acceptance summary
if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
run: |
node tools/trace_replay/render_retrace_summary.mjs \
--input retrace-artifacts/inproc-acceptance \
--output-dir android-retrace-summary/inproc-acceptance \
--title "MobileGL Android inproc acceptance" \
--group-label "Android Emulator / inproc acceptance" \
--html mobilegl-android-inproc-acceptance.html
- name: Render spawn acceptance summary
if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
run: |
node tools/trace_replay/render_retrace_summary.mjs \
--input retrace-artifacts/spawn-acceptance \
--output-dir android-retrace-summary/spawn-acceptance \
--title "MobileGL Android spawn acceptance" \
--group-label "Android Emulator / spawn acceptance" \
--html mobilegl-android-spawn-acceptance.html
- name: Upload Android monolith control summary
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v7
with:
name: MobileGL-android-monolith-control-summary-${{ github.sha }}
path: android-retrace-summary/monolith-control/mobilegl-android-monolith-control.html
archive: false
if-no-files-found: error
- name: Upload Android inproc acceptance summary
if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
uses: actions/upload-artifact@v7
with:
name: MobileGL-android-inproc-acceptance-summary-${{ github.sha }}
path: android-retrace-summary/inproc-acceptance/mobilegl-android-inproc-acceptance.html
archive: false
if-no-files-found: error
- name: Upload Android spawn acceptance summary
if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
uses: actions/upload-artifact@v7
with:
name: MobileGL-android-spawn-acceptance-summary-${{ github.sha }}
path: android-retrace-summary/spawn-acceptance/mobilegl-android-spawn-acceptance.html
archive: false
if-no-files-found: error
# The summaries embed every image they show, so the per-shard results are intermediates
# from here on. This is the last step of the workflow rather than a job of its own, which
# would wait for a runner and start from an empty machine. It keeps what a failure needs,
# read from the shard-status.json of the results downloaded above: the results of every
# shard that ran a case which did not pass somewhere, on all backends and transports, and
# every result when a job failed without saying which cases it ran. Deletes are paced
# below GitHub's limit for mutating requests, and a refused delete is a warning. Skipped
# when a summary step failed, so that re-running this job still finds the results, and
# when a retrace job did not succeed in a run that was not canceled: every result stays
# then, so a shard stopped by its job limit keeps its diagnostics and a re-run of the
# failed jobs renders complete summaries.
- name: Delete intermediate Android retrace artifacts
if: ${{ (success() && needs.retrace.result == 'success') || cancelled() }}
env:
GH_TOKEN: ${{ github.token }}
run: |
python3 android-plugin/retrace_artifact_cleanup.py \
--downloads retrace-artifacts/monolith-control \
--downloads retrace-artifacts/inproc-acceptance \
--downloads retrace-artifacts/spawn-acceptance
Reference in New Issue View Git Blame Copy Permalink