swung0x48/MobileGL
1
0
Fork 0
mirror of https://github.com/MobileGL-Dev/MobileGL synced 2026-11-16 06:23:23 +09:00
Code Issues Packages Projects Releases Wiki Activity

[CI, Tools] (...): replay trace cases in shards from one fixture cache

Browse Source
This commit is contained in:
swung0x48
2026-11-15 11:47:18 -04:00
parent 222f6209a0
commit 697bcc6d45
20 changed files with 4176 additions and 884 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/trace-fixture-bundle.sh
+304
View File
@@ -0,0 +1,304 @@
#!/usr/bin/env bash
# One cache entry that holds every Git LFS trace fixture of the CI cases.
#
# key derive the actions/cache key, restore prefix and path of the bundle
# ensure after a cache miss: complete the bundle directory, fetching only the
# cases whose files are missing from it or do not match their pointers
# wait <key> [secs] wait until the cache entry <key> is listed for this ref (needs GH_TOKEN)
# install <case>... move the fixtures of these cases from the restored bundle directory into
# the fixture directory, fetch the cases it does not hold, and check every
# file against its pointer
#
# The key is content-addressed on the pointer oids tracked at HEAD, which a plain checkout
# reads without smudging. The bundle directory is the same path for every revision, so
# `ensure` can start from an older bundle. Fixtures stored in Git itself rather than in Git
# LFS (OpenRA) are not in the bundle; `install` checks that they are real files.
set -euo pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=trace-fixture-lib.sh
. "${script_dir}/trace-fixture-lib.sh"
# Bump when the key derivation or the bundle layout changes in a way that must invalidate
# old entries; the content digest alone would not notice a format change.
key_schema="v1"
bundle_dir="trace-fixture-bundle"
fixture_dir="tools/trace_replay/fixtures"
python_bin="${PYTHON:-python3}"
if ! command -v "${python_bin}" >/dev/null 2>&1 && command -v python >/dev/null 2>&1; then
python_bin=python
fi
usage() {
echo "usage: $0 <key|ensure|wait <key> [seconds]|install <trace-case>...>" >&2
exit 2
}
# Writes "name=value" to $GITHUB_OUTPUT when running under Actions, and to stdout as well so
# the script stays runnable off-CI.
emit_output() {
local name="$1"
local value="$2"
if [ -n "${GITHUB_OUTPUT:-}" ]; then
printf '%s=%s\n' "${name}" "${value}" >> "${GITHUB_OUTPUT}"
fi
printf '%s=%s\n' "${name}" "${value}"
}
# "<case><TAB><file>" for every fixture file of every CI case.
ci_fixture_manifest() {
"${python_bin}" tools/trace_replay/trace_cases.py --ci --format fixture-manifest \
--fixture-root "${fixture_dir}" | tr -d '\r'
}
is_lfs_fixture() {
get_lfs_metadata "$1" >/dev/null 2>&1
}
# A fixture that Git stores directly must be a real file, not a pointer left by a wrong
# attribute.
check_plain_fixture() {
local file="$1"
if [ ! -s "${file}" ]; then
echo "fixture file is missing or empty: ${file}" >&2
return 1
fi
if head -n 1 "${file}" | grep -q "version https://git-lfs.github.com/spec/v1"; then
echo "fixture should not be stored as an LFS pointer: ${file}" >&2
return 1
fi
}
# Puts the tracked pointer files back. The smudge filter is skipped so that a clone with Git
# LFS set up does not download the objects here.
restore_pointers() {
GIT_LFS_SKIP_SMUDGE=1 git checkout -- "$@"
}
verify_lfs_fixture() {
local on_disk="$1"
local tracked="$2"
local metadata expected_oid expected_size
metadata="$(get_lfs_metadata "${tracked}")" || return 1
read -r expected_oid expected_size <<< "${metadata}"
verify_fixture_file "${on_disk}" "${tracked}" "${expected_oid}" "${expected_size}"
}
# Moves the given Git LFS fixtures of one case from the bundle directory into the fixture
# directory. Succeeds when every one of them is then in place and matches its pointer.
take_from_bundle() {
local file bundled
local incomplete=0
for file in "$@"; do
bundled="${bundle_dir}/$(basename "${file}")"
# Two cases may share a file; the second finds it already in place.
if [ -f "${bundled}" ]; then
mv -f "${bundled}" "${file}"
fi
if ! verify_lfs_fixture "${file}" "${file}" 2>/dev/null; then
incomplete=1
fi
done
return "${incomplete}"
}
# Downloads the given Git LFS fixtures of one case over their pointers: the mirrors first,
# Git LFS for what no mirror serves. The fetch script verifies what it downloaded.
fetch_case_fixtures() {
local case_name="$1"
shift
local file
for file in "$@"; do
rm -f "${file}" "${file}.tmp"
done
restore_pointers "$@" || return 1
bash "${script_dir}/fetch-trace-fixture-lfs.sh" "${case_name}" "${fixture_dir}"
}
[ "$#" -ge 1 ] || usage
command_name="$1"
shift
case "${command_name}" in
key)
[ "$#" -eq 0 ] || usage
# One "<basename> <oid> <size>" line per bundled file, sorted, each file once.
manifest=""
count=0
while IFS= read -r file; do
[ -n "${file}" ] || continue
if ! metadata="$(get_lfs_metadata "${file}" 2>/dev/null)"; then
continue
fi
read -r expected_oid expected_size <<< "${metadata}"
manifest+="$(basename "${file}") ${expected_oid} ${expected_size}"$'\n'
count=$((count + 1))
done < <(ci_fixture_manifest | cut -f2 | LC_ALL=C sort -u)
if [ "${count}" -eq 0 ]; then
echo "no Git LFS fixture found for the CI trace cases; the bundle would be empty" >&2
exit 1
fi
duplicates="$(printf '%s' "${manifest}" | cut -d' ' -f1 | LC_ALL=C sort | uniq -d)"
if [ -n "${duplicates}" ]; then
echo "two fixture files share a name, so the bundle directory cannot hold both: ${duplicates}" >&2
exit 1
fi
digest="$(printf '%s' "${manifest}" | sha256sum | awk '{ print substr($1, 1, 16) }')"
emit_output "key" "trace-fixture-bundle-${key_schema}-${digest}"
emit_output "restore-prefix" "trace-fixture-bundle-${key_schema}-"
emit_output "path" "${bundle_dir}"
emit_output "files" "${count}"
;;
ensure)
[ "$#" -eq 0 ] || usage
mkdir -p "${bundle_dir}" "${fixture_dir}"
mapfile -t manifest_lines < <(ci_fixture_manifest)
declare -A case_seen=()
case_order=()
for line in "${manifest_lines[@]}"; do
case_name="${line%%$'\t'*}"
if [ -z "${case_seen[${case_name}]:-}" ]; then
case_seen["${case_name}"]=1
case_order+=("${case_name}")
fi
done
fetched=0
reused=0
bundled_files=()
for case_name in "${case_order[@]}"; do
lfs_files=()
for line in "${manifest_lines[@]}"; do
[ "${line%%$'\t'*}" = "${case_name}" ] || continue
file="${line#*$'\t'}"
if is_lfs_fixture "${file}"; then
lfs_files+=("${file}")
else
check_plain_fixture "${file}"
fi
done
[ "${#lfs_files[@]}" -gt 0 ] || continue
# Take what an older bundle already holds, and fetch the case if that is not all of it.
if take_from_bundle "${lfs_files[@]}"; then
reused=$((reused + 1))
else
echo "Fetching trace fixtures of ${case_name}: the bundle does not hold them"
fetch_case_fixtures "${case_name}" "${lfs_files[@]}"
fetched=$((fetched + 1))
fi
bundled_files+=("${lfs_files[@]}")
done
# The bundle is rebuilt from the verified files, so nothing an older bundle carried and
# no case names any more is saved again.
rm -rf "${bundle_dir}"
mkdir -p "${bundle_dir}"
declare -A file_seen=()
saved=0
for file in "${bundled_files[@]}"; do
[ -z "${file_seen[${file}]:-}" ] || continue
file_seen["${file}"]=1
verify_lfs_fixture "${file}" "${file}"
mv -f "${file}" "${bundle_dir}/$(basename "${file}")"
saved=$((saved + 1))
done
if [ "${saved}" -eq 0 ]; then
echo "the trace fixture bundle came out empty" >&2
exit 1
fi
# Leave the working tree as a checkout has it: pointers in the fixture directory.
restore_pointers "${!file_seen[@]}"
echo "Trace fixture bundle: ${saved} file(s), ${reused} case(s) from the restored bundle, ${fetched} case(s) fetched."
emit_output "fetched" "${fetched}"
;;
wait)
[ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage
key="$1"
limit="${2:-300}"
# Another workflow of the same push may be the one saving this key. The replay jobs
# restore the entry, so it has to be listed before this job ends. The caller checks the
# entry itself afterwards; this only waits, and never fails.
#
# The listing is narrowed to the ref of this run. The key is content-addressed, so another
# branch may already hold an entry under it, and that entry cannot be restored from here:
# counting it would end the wait while the entry of this ref is still being uploaded.
query=(-f "key=${key}" -f per_page=100)
if [ -n "${GITHUB_REF:-}" ]; then
query+=(-f "ref=${GITHUB_REF}")
fi
waited=0
refused=0
while :; do
if listed="$(gh api -X GET "repos/${GITHUB_REPOSITORY}/actions/caches" "${query[@]}" \
--jq '[.actions_caches[] | select(.key == "'"${key}"'")] | length' 2>/dev/null)"; then
if [ "${listed:-0}" -ge 1 ]; then
echo "cache entry ${key} is listed after ${waited} s"
break
fi
else
refused=$((refused + 1))
if [ "${refused}" -ge 3 ]; then
echo "the cache listing cannot be read (no token, or no actions: read permission); not waiting"
break
fi
fi
if [ "${waited}" -ge "${limit}" ]; then
echo "cache entry ${key} is not listed after ${waited} s"
break
fi
sleep 10
waited=$((waited + 10))
done
;;
install)
[ "$#" -ge 1 ] || usage
installed=0
fetched=0
for case_name in "$@"; do
mapfile -t files < <(trace_fixture_files "${case_name}" "${fixture_dir}" "${python_bin}")
if [ "${#files[@]}" -eq 0 ]; then
echo "no fixture files declared for trace case: ${case_name}" >&2
exit 1
fi
lfs_files=()
for file in "${files[@]}"; do
if is_lfs_fixture "${file}"; then
lfs_files+=("${file}")
else
check_plain_fixture "${file}"
fi
done
[ "${#lfs_files[@]}" -gt 0 ] || continue
# A bundle that was not restored (the cache entry was evicted, or its download failed)
# or that lacks a file costs this job a download of the case, not the replay.
if ! take_from_bundle "${lfs_files[@]}"; then
echo "::warning::the restored fixture bundle does not hold the trace fixtures of ${case_name}; fetching them"
if ! fetch_case_fixtures "${case_name}" "${lfs_files[@]}"; then
echo "::error::the trace fixtures of ${case_name} are not in the restored bundle and could not be fetched"
exit 1
fi
fetched=$((fetched + 1))
for file in "${lfs_files[@]}"; do
if ! verify_lfs_fixture "${file}" "${file}"; then
echo "::error::trace fixture ${file} of ${case_name} does not match its Git LFS pointer after the fetch"
exit 1
fi
done
fi
installed=$((installed + ${#lfs_files[@]}))
done
# The rest of the bundle belongs to other shards.
rm -rf "${bundle_dir}"
echo "Installed and verified ${installed} Git LFS fixture file(s) for $# trace case(s), ${fetched} case(s) fetched."
;;
*)
usage
;;
esac
.github/scripts/trace-fixture-cache.sh
-117
View File
@@ -1,117 +0,0 @@
#!/usr/bin/env bash
# Cache-side helper for trace fixtures.
#
# key <case> [fixture-dir] derive the actions/cache key and path list
# verify <case> [fixture-dir] check restored fixtures against their pointers
# reset <case> [fixture-dir] drop restored fixtures, leaving the pointers
#
# The cache key is content-addressed on the Git LFS pointer oids tracked at
# HEAD, which are readable from a plain checkout without smudging. Fixture
# content therefore maps 1:1 onto a key: unchanged content hits, changed
# content is a new key and thus a miss, and the download path handles it. The
# key deliberately carries no restore-keys prefix in the workflow - a fixture
# that does not match the pointer exactly must never be restored.
set -euo pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=trace-fixture-lib.sh
. "${script_dir}/trace-fixture-lib.sh"
# Bump when the key derivation changes in a way that must invalidate old
# entries; the content digest alone would not notice a format change.
key_schema="v1"
if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then
echo "usage: $0 <key|verify|reset> <trace-case> [fixture-dir]" >&2
exit 2
fi
command_name="$1"
case_name="$2"
fixture_dir="${3:-tools/trace_replay/fixtures}"
python_bin="${PYTHON:-python3}"
if ! command -v "${python_bin}" >/dev/null 2>&1 && command -v python >/dev/null 2>&1; then
python_bin=python
fi
mapfile -t files < <(trace_fixture_files "${case_name}" "${fixture_dir}" "${python_bin}")
if [ "${#files[@]}" -eq 0 ]; then
echo "no fixture files declared for trace case: ${case_name}" >&2
exit 1
fi
# Writes "name=value" to $GITHUB_OUTPUT when running under Actions, and to
# stdout otherwise so the script stays runnable (and testable) off-CI.
emit_output() {
local name="$1"
local value="$2"
if [ -n "${GITHUB_OUTPUT:-}" ]; then
if [[ "${value}" == *$'\n'* ]]; then
local delimiter="ghadelim_$(date +%s%N)_$$"
{
printf '%s<<%s\n' "${name}" "${delimiter}"
printf '%s\n' "${value}"
printf '%s\n' "${delimiter}"
} >> "${GITHUB_OUTPUT}"
else
printf '%s=%s\n' "${name}" "${value}" >> "${GITHUB_OUTPUT}"
fi
fi
printf '%s=%s\n' "${name}" "${value}"
}
sanitize_case() {
printf '%s' "$1" | sed 's/[^A-Za-z0-9._-]/_/g'
}
case "${command_name}" in
key)
manifest=""
for file in "${files[@]}"; do
# A case whose fixtures are committed directly rather than through Git LFS
# (OpenRA) has no pointer oid to key on, and nothing to download either.
# Report it as uncacheable so the workflow skips the cache entirely.
if ! metadata="$(get_lfs_metadata "${file}" 2>/dev/null)"; then
echo "trace case ${case_name} is not stored in Git LFS; skipping fixture cache" >&2
emit_output "cacheable" "false"
emit_output "key" ""
exit 0
fi
read -r expected_oid expected_size <<< "${metadata}"
manifest+="$(basename "${file}") ${expected_oid} ${expected_size}"$'\n'
done
digest="$(printf '%s' "${manifest}" | sha256sum | awk '{ print substr($1, 1, 16) }')"
safe_case="$(sanitize_case "${case_name}")"
emit_output "cacheable" "true"
emit_output "key" "trace-fixture-${key_schema}-${safe_case}-${digest}"
emit_output "paths" "$(printf '%s\n' "${files[@]}")"
;;
verify)
for file in "${files[@]}"; do
metadata="$(get_lfs_metadata "${file}")"
read -r expected_oid expected_size <<< "${metadata}"
verify_fixture_file "${file}" "${file}" "${expected_oid}" "${expected_size}"
done
echo "Verified ${#files[@]} fixture file(s) for ${case_name} against the tracked Git LFS pointers."
;;
reset)
# Put the working tree back to the pointer files a fresh checkout would
# have, so that a rejected cache entry falls through to exactly the same
# download path a cache miss takes.
for file in "${files[@]}"; do
rm -f "${file}" "${file}.tmp"
done
git checkout -- "${files[@]}"
echo "Reset ${#files[@]} fixture file(s) for ${case_name} to their tracked Git LFS pointers."
;;
*)
echo "unknown command: ${command_name}" >&2
exit 2
;;
esac
.github/scripts/trace-fixture-lib.sh
+1 -1
View File
@@ -2,7 +2,7 @@
# Shared helpers for trace-fixture handling: reading the in-tree Git LFS pointer
# metadata and verifying a fixture file against it. Sourced by
# fetch-trace-fixture-lfs.sh (verify after download) and by
# trace-fixture-cache.sh (cache key derivation and verify after cache restore),
# trace-fixture-bundle.sh (cache key derivation and verify after cache restore),
# so both paths agree on what a valid fixture is.
# Reads the Git LFS pointer tracked at HEAD for a fixture path and prints
.github/workflows/apk.yml
+350 -301
View File
@@ -12,11 +12,28 @@ on:
- feat/disaggregated
- codex/fix-github-ci
- cleanup
# A docs-only push builds nothing this workflow tests (231 emulator legs per run). The Test
# workflow keeps running for docs: it hosts the doc citation lint.
# A push that only touches these paths changes nothing this workflow builds or replays. The
# Test workflow keeps running for docs: it hosts the doc citation lint. The Android build
# compiles none of the unit, integration or benchmark trees, with one exception that keeps
# MobileGL/MG_Benchmark from being listed whole: MG_Util/SelfTest/DriverBenchJni.cpp
# includes MG_Benchmark/Driver/DriverBenchCases.inc.
paths-ignore:
- 'docs/**'
- '**/*.md'
- 'MobileGL/MG_Test/**'
- 'MobileGL/MG_IntegrationTest/**'
- 'MobileGL/MG_Benchmark/CMakeLists.txt'
- 'MobileGL/MG_Benchmark/SanityBench.cpp'
- 'MobileGL/MG_Benchmark/Buffer/**'
- 'MobileGL/MG_Benchmark/Container/**'
- 'MobileGL/MG_Benchmark/Program/**'
- 'MobileGL/MG_Benchmark/ShaderCache/**'
- 'MobileGL/MG_Benchmark/Transpile/**'
- 'tools/cts/**'
- 'tools/device_bench/**'
- '.github/workflows/test.yml'
- '.github/workflows/hygiene.yml'
- '.github/actions/setup-test-runner/**'
workflow_dispatch:
env:
@@ -39,7 +56,10 @@ jobs:
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_COMPRESS: "true"
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 4G
# One build's objects (three native configurations) take about 1 GB. The limit leaves
# half of that again for the objects a push replaces, and is what the cleanup before
# the save trims the entry to.
CCACHE_MAXSIZE: 1500M
CCACHE_NOHASHDIR: "true"
MOBILEGL_CMAKE_COMPILER_LAUNCHER: ccache
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
@@ -47,8 +67,67 @@ jobs:
steps:
- name: Checkout repo
uses: actions/checkout@v6
with:
submodules: recursive
# Only the submodules the two APKs are built from, shallow and in parallel, in place of a
# recursive checkout of every submodule in the tree: the ones the root CMakeLists.txt
# configures, and apitrace with the eight directories its thirdparty/CMakeLists.txt
# requires, for the trace APK. Each row names a submodule and one file the build reads
# from it, and a row whose file is missing afterwards fails the step: without
# 3rdparty/flatbuffers the root CMakeLists.txt would go on and build a library without
# MG_Remote instead of stopping. A submodule the build starts to use needs a row here.
- name: Check out build submodules
timeout-minutes: 15
run: |
top=(
"3rdparty/glslang CMakeLists.txt"
"3rdparty/SPIRV-Cross CMakeLists.txt"
"3rdparty/SPIRV-Reflect CMakeLists.txt"
"3rdparty/VulkanMemoryAllocator CMakeLists.txt"
"3rdparty/Vulkan-Headers CMakeLists.txt"
"3rdparty/Vulkan-Utility-Libraries CMakeLists.txt"
"3rdparty/xxHash build/cmake/CMakeLists.txt"
"3rdparty/asio include/asio.hpp"
"3rdparty/flatbuffers include/flatbuffers/flatbuffers.h"
"include/ska flat_hash_map.hpp"
"3rdparty/apitrace thirdparty/CMakeLists.txt"
)
apitrace=(
"thirdparty/brotli LICENSE"
"thirdparty/gtest LICENSE"
"thirdparty/libbacktrace LICENSE"
"thirdparty/libpng LICENSE"
"thirdparty/snappy COPYING"
"thirdparty/zlib README"
"thirdparty/zstd LICENSE"
"thirdparty/directxmath LICENSE"
)
# update <directory> <row>...: the update is idempotent, so a fetch that failed on
# the network is tried again, three times in all. Then every row's file must exist.
update() {
local dir="$1" paths=() row attempt updated=0 missing=0
shift
for row in "$@"; do
paths+=("${row%% *}")
done
for attempt in 1 2 3; do
if git -C "${dir}" submodule update --init --depth=1 --jobs 8 -- "${paths[@]}"; then
updated=1
break
fi
echo "the submodule update in ${dir} failed (attempt ${attempt} of 3)" >&2
sleep $((attempt * 5))
done
[ "${updated}" -eq 1 ] || return 1
for row in "$@"; do
if [ ! -f "${dir}/${row%% *}/${row#* }" ]; then
echo "::error::${dir}/${row%% *}/${row#* } is missing after the submodule update, so the APK build cannot use ${row%% *}"
missing=1
fi
done
return "${missing}"
}
update . "${top[@]}"
update 3rdparty/apitrace "${apitrace[@]}"
- name: Set artifact metadata
run: |
@@ -82,6 +161,9 @@ jobs:
sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 update
sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 install -y ccache
ccache --version
# The restored entry carries the counters of every earlier build; zero them so the
# statistics printed after the build are this run's.
ccache --zero-stats
- name: Setup Android SDK
id: netsetup84
@@ -183,11 +265,20 @@ jobs:
if: always()
run: ccache --show-stats
# Rewrite one rolling entry per job and branch. The upload stays cumulative - it
# carries every object restored at the top of this run plus the few TUs that
# changed - but Actions cache keys are immutable, so this branch's superseded
# blob is released first; --ref keeps the delete off other branches' entries.
# Running after the build means a failed build leaves the entry untouched.
# A build served from the cache stores nothing, so nothing would ever evict the objects
# of older source states. Trim to CCACHE_MAXSIZE, least recently used first, before the
# entry is saved; the objects this build hit were just touched and stay.
- name: Trim ccache to its size limit
run: |
ccache --cleanup
ccache --show-stats | grep -i 'cache size' || true
du -sh "${CCACHE_DIR}"
# Rewrite one rolling entry per job and branch. The upload carries the objects this
# build used plus what fits under the size limit, but Actions cache keys are
# immutable, so this branch's superseded blob is released first; --ref keeps the
# delete off other branches' entries. Running after the build means a failed build
# leaves the entry untouched.
- name: Release superseded ccache entry
env:
GH_TOKEN: ${{ github.token }}
@@ -252,9 +343,17 @@ jobs:
runs-on: ubuntu-26.04
env:
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
# How much replay one emulator job aims for, in seconds as the duration hints count them.
# A (lane, transport, backend) cell is cut into as many shards as that takes, and its
# cases are spread so the shards of a cell come out even (tools/trace_replay/trace_cases.py,
# shard_matrix). A smaller value gives more and shorter jobs, each of which pays the
# emulator boot again; every case is replayed once per cell whatever the value.
RETRACE_SHARD_SECONDS: "720"
# The most a shard runner may spend on its cases. A shard gets the time in which every
# one of its cases could run to the end of its own time budget, up to this cap.
RETRACE_SHARD_DEADLINE_CAP_SECONDS: "7200"
outputs:
android: ${{ steps.trace-cases.outputs.android }}
names: ${{ steps.trace-cases.outputs.names }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
@@ -263,77 +362,105 @@ jobs:
id: trace-cases
run: |
python3 tools/trace_replay/trace_cases.py --self-test-transport-matrices
matrix_format=github-apk-matrix
# The retrace jobs replay shards: several cases of one (lane, transport, backend) cell
# per emulator boot. The self-test proves every shard matrix replays exactly the rows
# of the per-case matrix it is built from, and trace_cases.py refuses to print one
# that does not.
python3 tools/trace_replay/trace_cases.py --self-test-shard-matrices
matrix_format=github-apk-shard-matrix
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then
matrix_format=github-apk-transport-matrix
matrix_format=github-apk-transport-shard-matrix
fi
echo "android=$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}")" >> "$GITHUB_OUTPUT"
echo "names=$(python3 tools/trace_replay/trace_cases.py --ci --format names)" >> "$GITHUB_OUTPUT"
android="$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}" \
--shard-seconds "${RETRACE_SHARD_SECONDS}")"
# An empty `include` is not an error to GitHub: it skips the retrace job with no red.
shards="$(printf '%s' "${android}" | python3 -c 'import json, sys; print(len(json.load(sys.stdin)["include"]))')"
if [ "${shards}" -lt 1 ]; then
echo "::error::the Android retrace matrix is empty, so the retrace job would be skipped with no red"
exit 1
fi
# The cases of a shard share one job, so a case that hangs spends time the cases after
# it need. retrace_shard_budget.py gives every shard its own deadline_seconds, the
# time in which each of its cases could run to the end of its time budget, capped so
# that a shard whose replays all hang cannot hold a runner for hours, and the
# timeout_minutes of its job. The retrace job reads both from the matrix.
android="$(printf '%s' "${android}" | python3 android-plugin/retrace_shard_budget.py \
--cap-seconds "${RETRACE_SHARD_DEADLINE_CAP_SECONDS}")"
echo "android=${android}" >> "$GITHUB_OUTPUT"
echo "Android retrace: ${shards} shard(s) from ${matrix_format} at ${RETRACE_SHARD_SECONDS} s per shard"
# One cache entry holds every Git LFS fixture of the CI cases, and each retrace job restores
# that entry itself. This job only makes sure the entry exists. On a hit it downloads
# nothing. On a miss it starts from the newest older bundle, fetches from the mirror the
# cases whose pointers changed, saves the entry under the new key and confirms it is there,
# so that no retrace job has to download fixtures itself. The key, the path and the steps
# are .github/scripts/trace-fixture-bundle.sh's, and test.yml has the same job for the same
# entry: the two workflows of one push look the key up at the same time, and on a miss both
# fetch and both save. The save that loses is not an error, since the entry it wanted to
# write is the one the confirming step then finds. The timeout bounds a fixture download
# that stalls, which would otherwise hold every retrace job behind it.
trace-fixtures:
name: trace fixture (${{ matrix.case }})
name: trace fixtures
runs-on: ubuntu-26.04
needs: trace-cases
strategy:
fail-fast: false
max-parallel: 4
matrix:
case: ${{ fromJSON(needs.trace-cases.outputs.names) }}
timeout-minutes: 30
permissions:
actions: read
contents: read
outputs:
key: ${{ steps.bundle.outputs.key }}
path: ${{ steps.bundle.outputs.path }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Derive trace fixture cache key
id: fixture-key
run: bash .github/scripts/trace-fixture-cache.sh key '${{ matrix.case }}'
- name: Derive the fixture bundle key
id: bundle
run: bash .github/scripts/trace-fixture-bundle.sh key
- name: Restore trace fixture cache
id: fixture-cache
if: steps.fixture-key.outputs.cacheable == 'true'
- name: Look up the fixture bundle
id: lookup
uses: actions/cache/restore@v5
with:
path: ${{ steps.fixture-key.outputs.paths }}
key: ${{ steps.fixture-key.outputs.key }}
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
lookup-only: true
- name: Verify restored trace fixture
id: fixture-verify
if: steps.fixture-cache.outputs.cache-hit == 'true'
run: |
if bash .github/scripts/trace-fixture-cache.sh verify '${{ matrix.case }}'; then
echo "ok=true" >> "$GITHUB_OUTPUT"
else
echo "ok=false" >> "$GITHUB_OUTPUT"
echo "::warning::Cached fixture for ${{ matrix.case }} failed verification; falling back to the download path"
bash .github/scripts/trace-fixture-cache.sh reset '${{ matrix.case }}'
fi
- name: Restore the newest older bundle
if: steps.lookup.outputs.cache-hit != 'true'
uses: actions/cache/restore@v5
with:
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
restore-keys: ${{ steps.bundle.outputs.restore-prefix }}
- name: Fetch trace fixture
if: steps.fixture-verify.outputs.ok != 'true'
run: bash .github/scripts/fetch-trace-fixture-lfs.sh '${{ matrix.case }}'
- name: Complete the fixture bundle
if: steps.lookup.outputs.cache-hit != 'true'
run: bash .github/scripts/trace-fixture-bundle.sh ensure
- name: Save trace fixture cache
if: steps.fixture-key.outputs.cacheable == 'true' && steps.fixture-cache.outputs.cache-hit != 'true'
- name: Save the fixture bundle
if: steps.lookup.outputs.cache-hit != 'true'
continue-on-error: true
uses: actions/cache/save@v5
with:
path: ${{ steps.fixture-key.outputs.paths }}
key: ${{ steps.fixture-key.outputs.key }}
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
- name: Stage trace fixture
run: |
safe_case="$(printf '%s' '${{ matrix.case }}' | sed 's/[^A-Za-z0-9._-]/_/g')"
stage_dir="trace-fixtures/${safe_case}"
mkdir -p "${stage_dir}"
python3 tools/trace_replay/trace_cases.py --format fixture-files --case '${{ matrix.case }}' |
while IFS= read -r file; do
cp "${file}" "${stage_dir}/"
done
# The Test workflow of the same push may be the one whose save won. Its entry is as good,
# once it is listed.
- name: Wait for the fixture bundle entry
if: steps.lookup.outputs.cache-hit != 'true'
env:
GH_TOKEN: ${{ github.token }}
run: bash .github/scripts/trace-fixture-bundle.sh wait '${{ steps.bundle.outputs.key }}'
- name: Upload trace fixture
uses: actions/upload-artifact@v7
- name: Confirm the fixture bundle entry
if: steps.lookup.outputs.cache-hit != 'true'
uses: actions/cache/restore@v5
with:
name: MobileGL-trace-fixture-${{ matrix.case }}
path: trace-fixtures/**
if-no-files-found: error
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
lookup-only: true
fail-on-cache-miss: true
android-avd:
name: android avd image
@@ -347,8 +474,33 @@ jobs:
- name: Checkout repo
uses: actions/checkout@v6
# The emulator shard loop and the artifact cleanup, driven by fakes: what each case is
# run with, a verdict for every case, and what stays after a failure. It runs in this
# job because the retrace jobs wait for it and the APK build does not.
- name: Emulator shard runner regression
run: |
python3 android-plugin/test_trace_replay_shard.py
python3 android-plugin/test_retrace_artifact_cleanup.py
# This job only makes sure the cache entry exists; the retrace jobs restore it. A hit
# downloads nothing and skips every step below; a miss builds the AVD, and the post step
# of this action saves it.
- name: Look up Android AVD cache
id: android-avd-cache
uses: actions/cache@v5
with:
lookup-only: true
path: |
${{ env.ANDROID_AVD_HOME }}
${{ env.ANDROID_SDK_ROOT }}/emulator
${{ env.ANDROID_SDK_ROOT }}/platform-tools
${{ env.ANDROID_SDK_ROOT }}/platforms/android-35
${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64
key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v4-${{ hashFiles('android-plugin/run-avd-ci.sh') }}
- name: Setup Android SDK
id: netsetup336
if: steps.android-avd-cache.outputs.cache-hit != 'true'
continue-on-error: true
timeout-minutes: 15
uses: android-actions/setup-android@v4
@@ -367,20 +519,9 @@ jobs:
cmdline-tools-version: 13114758
- name: Accept Android SDK licenses
if: steps.android-avd-cache.outputs.cache-hit != 'true'
run: yes | sdkmanager --licenses >/dev/null
- name: Restore Android AVD cache
id: android-avd-cache
uses: actions/cache@v5
with:
path: |
${{ env.ANDROID_AVD_HOME }}
${{ env.ANDROID_SDK_ROOT }}/emulator
${{ env.ANDROID_SDK_ROOT }}/platform-tools
${{ env.ANDROID_SDK_ROOT }}/platforms/android-35
${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64
key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v4-${{ hashFiles('android-plugin/run-avd-ci.sh') }}
- name: Create AVD
if: steps.android-avd-cache.outputs.cache-hit != 'true'
run: |
@@ -418,16 +559,27 @@ jobs:
fi
du -sh "${ANDROID_AVD_HOME}"
# One job replays a shard: some of the cases of one lane, transport and backend, one after
# another on one emulator boot (android-plugin/trace_replay_shard.py). The shard's step
# summary has a row per case, and a case that did not pass has an error annotation of its own.
retrace:
name: retrace (${{ matrix.backend.name }} / ${{ matrix.transport || 'monolith' }}, ${{ matrix.case.name }})
name: retrace (${{ matrix.backend.name }} / ${{ matrix.transport || 'monolith' }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
runs-on: ubuntu-26.04
needs:
- build
- android-avd
- trace-cases
- trace-fixtures
if: ${{ !cancelled() && needs.build.result == 'success' && needs.android-avd.result == 'success' && needs.trace-cases.result == 'success' }}
timeout-minutes: 75
if: ${{ !cancelled() && needs.build.result == 'success' && needs.android-avd.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }}
# Paired with --deadline-seconds below, and far above what a shard takes when its replays
# end by themselves. Both come from the matrix. The deadline is the smaller of the time in
# which every case of the shard could run to the end of its own time budget
# (trace_replay_shard.py, needed_seconds) and RETRACE_SHARD_DEADLINE_CAP_SECONDS. With
# budgets of up to 1800 s a case the cap is what a shard of several cases gets: about
# three replays can hang to the end of their budgets before a later case is recorded as
# not-run. The minutes between the deadline and this limit are the setup before the
# replays and the uploads after them.
timeout-minutes: ${{ matrix.timeout_minutes }}
env:
MOBILEGL_CI_TRANSPORT: ${{ matrix.transport || 'monolith' }}
MOBILEGL_CI_LANE: ${{ matrix.lane || 'monolith-control' }}
@@ -435,38 +587,53 @@ jobs:
ANDROID_AVD_HOME: ${{ github.workspace }}/.android/avd
ANDROID_HOME: ${{ github.workspace }}/.android/sdk
ANDROID_SDK_ROOT: ${{ github.workspace }}/.android/sdk
# The shard's cases in run order, as the JSON array of case objects the shard runner
# takes: every field of the case in trace_cases.json, with the fixture paths in place.
MOBILEGL_SHARD_CASES: ${{ toJSON(matrix.cases) }}
# Set the repository variable to 1 to cold boot the emulator before every case of a
# shard, should one boot not survive a whole shard of software-rendered replays.
MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE: ${{ vars.MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE || '0' }}
# The emulator renders on the host with a software rasterizer that can keep compiled
# shaders on disk in the runner's home directory, where neither clearing the app's data
# nor restarting the emulator reaches. Off, so every case of a shard compiles its
# shaders as a job with a fresh machine does.
MESA_SHADER_CACHE_DISABLE: "true"
strategy:
fail-fast: false
# The account runs at most 20 jobs at once, shared with Test. These ~230 emulator jobs are
# the longer of the two runs, so they may take the whole pool: what is minimized is when the
# later of APK and Test finishes, not Test alone.
max-parallel: 20
# No max-parallel: the account's limit of 20 concurrent jobs, shared with Test, already
# bounds this matrix, and a cap below the number of shards only makes the round longer.
matrix: ${{ fromJSON(needs.trace-cases.outputs.android) }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
# First, so the steps that run whatever happened (diagnostics, stopping the emulator, the
# result upload) have these even when a step before the emulator launch failed.
- name: Set artifact metadata
run: |
echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
echo "EMULATOR_LOG=${RUNNER_TEMP}/mobilegl-emulator.log" >> "$GITHUB_ENV"
echo "EMULATOR_PID_FILE=${RUNNER_TEMP}/mobilegl-emulator.pid" >> "$GITHUB_ENV"
- name: Set swap space
uses: ./.github/actions/set-swap-space
with:
swap-size-gb: 8
- name: Download trace fixture
uses: actions/download-artifact@v8
# A miss does not fail the job: the entry may have been evicted since trace-fixtures
# confirmed it (a re-run of failed jobs on an older run does not run that job again).
- name: Restore trace fixtures
uses: actions/cache/restore@v5
with:
name: MobileGL-trace-fixture-${{ matrix.case.name }}
path: trace-fixture-download
path: ${{ needs.trace-fixtures.outputs.path }}
key: ${{ needs.trace-fixtures.outputs.key }}
- name: Install trace fixture
run: |
mkdir -p tools/trace_replay/fixtures
find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;
- name: Set artifact metadata
run: |
echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
echo "EMULATOR_LOG=${RUNNER_TEMP}/mobilegl-emulator.log" >> "$GITHUB_ENV"
echo "EMULATOR_PID_FILE=${RUNNER_TEMP}/mobilegl-emulator.pid" >> "$GITHUB_ENV"
# Moves this shard's fixtures into place and checks them against their Git LFS pointers.
# A case the restored bundle does not hold is downloaded here, with a warning annotation;
# the timeout bounds a download that stalls.
- name: Install trace fixtures
timeout-minutes: 15
run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
- name: Restore Android AVD cache
id: android-avd-cache
@@ -543,135 +710,44 @@ jobs:
--pid-file "${EMULATOR_PID_FILE}" \
--boot-timeout 300
# One emulator boot replays every case of the shard in order, each in a fresh app process
# with cleared app data and with the settings its case, backend and transport need. The
# runner keeps going after a failing case, on a cold boot so that case cannot touch the
# next one's verdict, retries a case once on a restarted emulator when the emulator
# itself failed, and exits non-zero when a case failed or a verdict is missing. The
# deadline stops it while the steps below can still upload what it has: the job timeout
# less the setup above and the uploads below. A case it stops or never starts is
# recorded as not-run, which is red.
- name: Retrace and validate
env:
MOBILEGL_ESPRYT_USE_ANGLE: ${{ matrix.backend.name == 'DirectGLES' && '1' || '0' }}
MOBILEGL_TRACE_ANGLE_VARIANT: ${{ matrix.case.name == 'minecraft-1.21.4-fabric-iris-bliss-in-world' && '90a62123d794' || 'ec889e6ea831' }}
MOBILEGL_MAGMA_R11G11B10F_FALLBACK: ${{ matrix.backend.name == 'DirectVulkan' && '1' || '0' }}
MOBILEGL_ITRP_FIX: ${{ matrix.case.name == 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' && 'ForceOn' || 'Auto' }}
# Lets the artifact cleanup tell which job a shard's verdicts came from.
MOBILEGL_SHARD_JOB_ID: ${{ job.check_run_id }}
run: |
apk_file="android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk"
test -f "${apk_file}"
extra_retrace_args=()
if [ "${MOBILEGL_CI_TRANSPORT}" = "inproc" ]; then
extra_retrace_args+=(
--env "MOBILEGL_TRANSPORT=inproc;MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1;MOBILEGL_GPU_HANG_BUDGET_MS=${MOBILEGL_GPU_HANG_BUDGET_MS}"
--require-inproc
)
elif [ "${MOBILEGL_CI_TRANSPORT}" = "spawn" ]; then
# THE SAME CLIENT KNOBS AS inproc, so the two acceptance lanes differ in exactly one
# thing: which PROCESS the server role runs in. They are scrubbed out of the CHILD's
# environment by the launcher and read by the client, which is where they belong.
#
# MOBILEGL_IPC_SERVER_PATH IS DELIBERATELY ABSENT. nativeLibraryDir carries an
# install-time hash, so nothing off the device can spell it; TraceReplayActivity
# resolves it from getApplicationInfo() when the env asks for spawn, and an explicit
# value would override that. --use-pbuffer is NOT optional here: a spawned
# server cannot be handed the client's window, because an ANativeWindow* is a pointer
# into the CLIENT's process and SetWindowHandle is refused on the way across with
# Fatal{UnmigratedSurface, "AndroidNativeWindow:use-MOBILEGL_IPC_SURFACE=server"} (Rule H, permanent;
# a windowed client uses MOBILEGL_IPC_SURFACE=server instead).
extra_retrace_args+=(
--env "MOBILEGL_TRANSPORT=spawn;MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1;MOBILEGL_GPU_HANG_BUDGET_MS=${MOBILEGL_GPU_HANG_BUDGET_MS}"
--use-pbuffer
--require-spawn
)
else
extra_retrace_args+=(--env "MOBILEGL_TRANSPORT=monolith;MOBILEGL_GPU_HANG_BUDGET_MS=${MOBILEGL_GPU_HANG_BUDGET_MS}")
fi
# Bliss needs the newer signed ANGLE variant plus sampler mipmap
# min-filter downgrading on ANGLE llvmpipe.
if [ "${{ matrix.backend.name }}" = "DirectGLES" ] && [ "${{ matrix.case.name }}" = "minecraft-1.21.4-fabric-iris-bliss-in-world" ]; then
extra_retrace_args+=(--avoid-angle-llvmpipe-sampler-mipmap-min-filter)
fi
if [ "${{ matrix.backend.name }}" = "DirectGLES" ] && [ "${{ matrix.case.avoid_angle_llvmpipe_explicit_lod_bias || false }}" = "true" ]; then
extra_retrace_args+=(--avoid-angle-llvmpipe-explicit-lod-bias)
fi
if [ "${{ matrix.case.coherent_as_flush || false }}" = "true" ]; then
extra_retrace_args+=(--coherent-as-flush)
fi
run_retrace() {
timeout "$(( ${{ matrix.case.timeout_seconds }} + 300 ))" sh android-plugin/trace-replay-ci.sh \
--apk-file "${apk_file}" \
--package top.mobilegl.plugin.trace \
--backend "${{ matrix.backend.name }}" \
--result-root android-retrace-result \
--fixture-root android-retrace-fixture \
--case "${{ matrix.case.name }}" \
--trace-archive "${{ matrix.case.trace_archive }}" \
--trace-file "${{ matrix.case.trace_file }}" \
--golden "${{ matrix.case.golden }}" \
--alternate-golden "${{ matrix.case.alternate_golden || '' }}" \
--target-call "${{ matrix.case.target_call }}" \
--width "${{ matrix.case.width }}" \
--height "${{ matrix.case.height }}" \
--ssim-threshold "${{ matrix.case.ssim_threshold || '0.99' }}" \
--crop-x "${{ matrix.case.crop_x }}" \
--crop-y "${{ matrix.case.crop_y }}" \
--crop-width "${{ matrix.case.crop_width }}" \
--crop-height "${{ matrix.case.crop_height }}" \
--timeout-seconds "${{ matrix.case.timeout_seconds }}" \
"${extra_retrace_args[@]}"
}
retrace_status=0
run_retrace || retrace_status=$?
if [ "${retrace_status}" -eq 75 ]; then
echo "::warning::Android emulator infrastructure failed; restarting it and retrying this retrace once."
# Surface-lost is retried rather than failed, so it would otherwise
# be invisible. Report it per job - a healthy run prints nothing and
# a rate spike shows up as a row per affected case.
reason_file="android-retrace-result/infrastructure-failure-reason.txt"
surface_lost_retries=0
if [ -f "${reason_file}" ]; then
surface_lost_retries="$(grep -c 'angle-surface-lost' "${reason_file}" || true)"
fi
if [ "${surface_lost_retries}" -gt 0 ]; then
echo "surface-lost retries: ${surface_lost_retries} (${{ matrix.backend.name }}, ${{ matrix.case.name }})" \
>> "${GITHUB_STEP_SUMMARY}"
fi
# The restart truncates EMULATOR_LOG, and the attempt that lost the
# emulator is the one worth reading - the retry usually only shows
# the wreckage. Keep the first attempt's log before it is clobbered.
if [ -f "${EMULATOR_LOG}" ]; then
cp "${EMULATOR_LOG}" "${EMULATOR_LOG}.first-attempt" || true
fi
sh android-plugin/run-avd-ci.sh stop \
--avd-name "${AVD_NAME}" \
--emulator-log "${EMULATOR_LOG}" \
--pid-file "${EMULATOR_PID_FILE}"
adb kill-server || true
sleep 2
sh android-plugin/run-avd-ci.sh start \
--avd-name "${AVD_NAME}" \
--gpu "${{ matrix.backend.gpu }}" \
--emulator-log "${EMULATOR_LOG}" \
--pid-file "${EMULATOR_PID_FILE}" \
--boot-timeout 300
run_retrace
elif [ "${retrace_status}" -ne 0 ]; then
exit "${retrace_status}"
fi
sh android-plugin/trace-replay-shard.sh \
--apk-file "android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk" \
--backend "${{ matrix.backend.name }}" \
--gpu "${{ matrix.backend.gpu }}" \
--transport "${MOBILEGL_CI_TRANSPORT}" \
--lane "${MOBILEGL_CI_LANE}" \
--cases-env MOBILEGL_SHARD_CASES \
--deadline-seconds "${{ matrix.deadline_seconds }}"
# Runs whatever happened above, without the device: every case of the shard gets its
# lane.json and golden copies for the summary, and a not-run verdict when the runner was
# killed or never started.
- name: Collect retrace summary inputs
if: always()
env:
MOBILEGL_SHARD_JOB_ID: ${{ job.check_run_id }}
run: |
safe_case="$(printf '%s' '${{ matrix.case.name }}' | sed 's/[^A-Za-z0-9._-]/_/g')"
result_dir="android-retrace-result/${safe_case}-${{ matrix.backend.name }}"
mkdir -p "${result_dir}"
python3 - "${result_dir}/lane.json" <<'PY'
import json, os, sys
from pathlib import Path
Path(sys.argv[1]).write_text(json.dumps({"lane": os.environ["MOBILEGL_CI_LANE"],
"transport": os.environ["MOBILEGL_CI_TRANSPORT"]}, indent=2))
PY
if [ -s "${{ matrix.case.golden }}" ]; then
cp "${{ matrix.case.golden }}" "${result_dir}/${safe_case}-${{ matrix.backend.name }}-golden.png"
fi
if [ -n "${{ matrix.case.alternate_golden || '' }}" ] && [ -s "${{ matrix.case.alternate_golden || '' }}" ]; then
cp "${{ matrix.case.alternate_golden || '' }}" "${result_dir}/${safe_case}-${{ matrix.backend.name }}-alternate-golden.png"
fi
sh android-plugin/trace-replay-shard.sh \
--apk-file "android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk" \
--backend "${{ matrix.backend.name }}" \
--gpu "${{ matrix.backend.gpu }}" \
--transport "${MOBILEGL_CI_TRANSPORT}" \
--lane "${MOBILEGL_CI_LANE}" \
--cases-env MOBILEGL_SHARD_CASES \
--finalize
- name: Collect emulator diagnostics
if: always()
@@ -679,12 +755,18 @@ jobs:
mkdir -p android-retrace-result/diagnostics
adb devices -l > android-retrace-result/diagnostics/adb-devices.txt || true
timeout 30 adb logcat -d -t 1000 > android-retrace-result/diagnostics/logcat.txt || true
# What the guest was given; replay time inside the emulator depends on it.
timeout 30 adb shell 'nproc; grep -E "MemTotal|SwapTotal" /proc/meminfo' \
> android-retrace-result/diagnostics/guest-resources.txt 2>&1 || true
cp "${ANDROID_AVD_HOME}/${AVD_NAME}.avd/config.ini" android-retrace-result/diagnostics/avd-config.ini 2>/dev/null || true
# Expected to list nothing: the host rasterizer's on-disk shader cache is disabled.
du -sh "${XDG_CACHE_HOME:-${HOME}/.cache}"/mesa_shader_cache* \
> android-retrace-result/diagnostics/host-shader-cache.txt 2>&1 || true
# The log of the last boot. The shard runner keeps the log of every boot it ended
# as diagnostics/emulator-boot-<n>-*.log before a restart truncates it.
if [ -f "${EMULATOR_LOG}" ]; then
cp "${EMULATOR_LOG}" android-retrace-result/diagnostics/emulator.log
fi
if [ -f "${EMULATOR_LOG}.first-attempt" ]; then
cp "${EMULATOR_LOG}.first-attempt" android-retrace-result/diagnostics/emulator-first-attempt.log
fi
# A vanished emulator looks identical whether the host OOM killer took
# qemu or the renderer faulted. These two say which.
free -h > android-retrace-result/diagnostics/host-memory.txt 2>&1 || true
@@ -704,9 +786,14 @@ jobs:
continue-on-error: true
uses: actions/upload-artifact@v7
with:
name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-${{ matrix.case.name }}
# The summary job downloads by the lane and transport at the front of this name, and
# its renderer reads the backend and the case from the paths below it, so the name
# ends in the shard number and holds no case name.
name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-shard${{ matrix.shard }}
path: android-retrace-result/**
if-no-files-found: warn
# A re-run of a failed job replaces the artifact its first attempt kept.
overwrite: true
# The artifact service sometimes times out (ETIMEDOUT on CreateArtifact, one job in 231 on
# 2026-10-18), which turned a passing case red; one delayed retry absorbs that.
@@ -718,7 +805,7 @@ jobs:
if: always() && steps.upload-retrace-result.outcome == 'failure'
uses: actions/upload-artifact@v7
with:
name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-${{ matrix.case.name }}
name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-shard${{ matrix.shard }}
path: android-retrace-result/**
if-no-files-found: warn
overwrite: true
@@ -727,44 +814,51 @@ jobs:
name: retrace summary
runs-on: ubuntu-26.04
needs: retrace
if: ${{ !cancelled() }}
# Also runs for a canceled run, where it only downloads the results and deletes the
# intermediate artifacts; the rendering steps are skipped then. The checkout and the
# downloads say so themselves, because a step with no condition is skipped once its job is
# canceled.
if: always()
permissions:
actions: write
contents: read
env:
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
steps:
- name: Checkout repo
if: ${{ success() || cancelled() }}
uses: actions/checkout@v6
- name: Set artifact metadata
run: |
echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
- name: Set up Node.js
if: ${{ !cancelled() }}
uses: actions/setup-node@v7
with:
node-version: '22'
- name: Download Android monolith control results
if: ${{ success() || cancelled() }}
uses: actions/download-artifact@v8
with:
pattern: MobileGL-android-retrace-result-monolith-control-monolith-*
path: retrace-artifacts/monolith-control
- name: Download Android inproc acceptance results
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
if: ${{ (success() || cancelled()) && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
uses: actions/download-artifact@v8
with:
pattern: MobileGL-android-retrace-result-inproc-acceptance-inproc-*
path: retrace-artifacts/inproc-acceptance
- name: Download Android spawn acceptance results
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
if: ${{ (success() || cancelled()) && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
uses: actions/download-artifact@v8
with:
pattern: MobileGL-android-retrace-result-spawn-acceptance-spawn-*
path: retrace-artifacts/spawn-acceptance
- name: Render monolith control summary
if: ${{ !cancelled() }}
run: |
node tools/trace_replay/render_retrace_summary.mjs \
--input retrace-artifacts/monolith-control \
@@ -774,7 +868,7 @@ jobs:
--html mobilegl-android-monolith-control.html
- name: Render inproc acceptance summary
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
run: |
node tools/trace_replay/render_retrace_summary.mjs \
--input retrace-artifacts/inproc-acceptance \
@@ -784,11 +878,17 @@ jobs:
--html mobilegl-android-inproc-acceptance.html
- name: Render spawn acceptance summary
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
run: |
node tools/trace_replay/render_retrace_summary.mjs --input retrace-artifacts/spawn-acceptance --output-dir android-retrace-summary/spawn-acceptance --title "MobileGL Android spawn acceptance" --group-label "Android Emulator / spawn acceptance" --html mobilegl-android-spawn-acceptance.html
node tools/trace_replay/render_retrace_summary.mjs \
--input retrace-artifacts/spawn-acceptance \
--output-dir android-retrace-summary/spawn-acceptance \
--title "MobileGL Android spawn acceptance" \
--group-label "Android Emulator / spawn acceptance" \
--html mobilegl-android-spawn-acceptance.html
- name: Upload Android monolith control summary
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v7
with:
name: MobileGL-android-monolith-control-summary-${{ github.sha }}
@@ -797,7 +897,7 @@ jobs:
if-no-files-found: error
- name: Upload Android inproc acceptance summary
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
uses: actions/upload-artifact@v7
with:
name: MobileGL-android-inproc-acceptance-summary-${{ github.sha }}
@@ -806,7 +906,7 @@ jobs:
if-no-files-found: error
- name: Upload Android spawn acceptance summary
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
uses: actions/upload-artifact@v7
with:
name: MobileGL-android-spawn-acceptance-summary-${{ github.sha }}
@@ -814,74 +914,23 @@ jobs:
archive: false
if-no-files-found: error
remove-artifact-clutter:
name: remove artifact clutter
runs-on: ubuntu-26.04
needs: retrace-summary
if: always()
permissions:
actions: write
steps:
# The summaries embed every image they show, so the per-shard results are intermediates
# from here on. This is the last step of the workflow rather than a job of its own, which
# would wait for a runner and start from an empty machine. It keeps what a failure needs,
# read from the shard-status.json of the results downloaded above: the results of every
# shard that ran a case which did not pass somewhere, on all backends and transports, and
# every result when a job failed without saying which cases it ran. Deletes are paced
# below GitHub's limit for mutating requests, and a refused delete is a warning. Skipped
# when a summary step failed, so that re-running this job still finds the results, and
# when a retrace job did not succeed in a run that was not canceled: every result stays
# then, so a shard stopped by its job limit keeps its diagnostics and a re-run of the
# failed jobs renders complete summaries.
- name: Delete intermediate Android retrace artifacts
if: ${{ (success() && needs.retrace.result == 'success') || cancelled() }}
env:
GH_TOKEN: ${{ github.token }}
run: |
# Fail before deletion if either paginated inventory cannot be read.
# 50 rows per page: the jobs endpoint answers HTTP 502 for 90+ rows on a re-run attempt of a run this size.
gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=50" \
--jq '.jobs[] | select(.name | startswith("retrace (")) | select(.conclusion == "failure" or .conclusion == "canceled" or .conclusion == "timed_out" or .conclusion == "action_required") | .name' \
> "${RUNNER_TEMP}/cleanup-jobs.txt"
gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \
--jq '.artifacts[] | select(.name | startswith("MobileGL-trace-fixture-") or startswith("MobileGL-android-retrace-result-") or startswith("trace-fixture-") or startswith("retrace-result-")) | [.id, .name] | @tsv' \
> "${RUNNER_TEMP}/cleanup-artifacts.txt"
declare -A failed_cases=()
while IFS= read -r job_name; do
case_name="${job_name#retrace (*, }"
case_name="${case_name%)}"
failed_cases["${case_name}"]=1
done < "${RUNNER_TEMP}/cleanup-jobs.txt"
if ((${#failed_cases[@]})); then
echo "Retaining fixtures and results for failed retrace case(s):"
printf ' %s\n' "${!failed_cases[@]}"
else
echo "All retrace jobs succeeded; nothing needs to be retained."
fi
deleted=0
retained=0
while IFS=$'\t' read -r artifact_id artifact_name; do
keep=0
if [[ "${artifact_name}" == MobileGL-trace-fixture-* ]]; then
case_name="${artifact_name#MobileGL-trace-fixture-}"
if [[ -v "failed_cases[${case_name}]" ]]; then
keep=1
fi
elif [[ "${artifact_name}" == MobileGL-android-retrace-result-* ]]; then
# The result artifact carries mobilegl.log, retrace.log, logcat,
# the emulator log and the actual/diff images - the only record of
# why a retrace failed. Its name ends in -<backend>-<case>, so a
# suffix match on the case name keeps both backends' results for a
# case that failed on either of them, which is what a comparison
# needs. The match is anchored at the end, so a case name that is a
# prefix of a longer one does not retain the longer one's results.
for case_name in "${!failed_cases[@]}"; do
if [[ "${artifact_name}" == *-"${case_name}" ]]; then
keep=1
break
fi
done
fi
if ((keep)); then
echo "Retaining ${artifact_name} (${artifact_id}) for failed retrace."
((retained += 1))
continue
fi
echo "Deleting ${artifact_name} (${artifact_id})"
gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}"
((deleted += 1))
done < "${RUNNER_TEMP}/cleanup-artifacts.txt"
echo "Deleted ${deleted} intermediate Android artifact(s); retained ${retained} failed-retrace fixture(s)."
python3 android-plugin/retrace_artifact_cleanup.py \
--downloads retrace-artifacts/monolith-control \
--downloads retrace-artifacts/inproc-acceptance \
--downloads retrace-artifacts/spawn-acceptance
.github/workflows/test.yml
+305 -435
View File
@@ -19,7 +19,7 @@ env:
MOBILEGL_GPU_HANG_BUDGET_MS: "0"
# ONE LIVE RUN PER BRANCH. The disaggregation branch pushes (often docs-only) commits minutes apart
# and each push fans out to ~330 retrace legs here and ~275 jobs in apk.yml; with no group, runs
# and each push fans out to about 30 replay jobs here and the emulator shards in apk.yml; with no group, runs
# sat queued for 10+ hours behind their own successors. A manual dispatch gets a group of its own,
# so a push never cancels a manually started run; dev queues rather
# than cancels. No paths-ignore here, unlike apk.yml: this workflow hosts the doc citation lint.
@@ -2668,12 +2668,15 @@ jobs:
runs-on: ubuntu-26.04
env:
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
# How many jobs replay one (lane, transport, backend) cell of the Linux retrace lanes. Each
# job replays its share of the cell's cases one after another; a cell with little to
# replay gets fewer jobs (tools/trace_replay/trace_cases.py, shard_matrix).
RETRACE_SHARDS_PER_CELL: "4"
outputs:
matrix: ${{ steps.trace-cases.outputs.matrix }}
names: ${{ steps.trace-cases.outputs.names }}
verify-matrix: ${{ steps.trace-cases.outputs.verify-matrix }}
split-matrix: ${{ steps.trace-cases.outputs.split-matrix }}
split-spawn-matrix: ${{ steps.trace-cases.outputs.split-spawn-matrix }}
shards: ${{ steps.trace-cases.outputs.shards }}
verify-shards: ${{ steps.trace-cases.outputs.verify-shards }}
split-shards: ${{ steps.trace-cases.outputs.split-shards }}
split-spawn-shards: ${{ steps.trace-cases.outputs.split-spawn-shards }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
@@ -2702,29 +2705,49 @@ jobs:
id: trace-cases
run: |
python3 tools/trace_replay/trace_cases.py --self-test-transport-matrices
matrix_format=github-test-matrix
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then matrix_format=github-test-transport-matrix; fi
echo "matrix=$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}")" >> "$GITHUB_OUTPUT"
echo "names=$(python3 tools/trace_replay/trace_cases.py --ci --format names)" >> "$GITHUB_OUTPUT"
# The subset the verify build retraces ("verify": true in trace_cases.json). It is a
# SUBSET of the matrix above, so retrace-verify needs no fixtures of its own.
echo "verify-matrix=$(python3 tools/trace_replay/trace_cases.py --ci --format github-verify-matrix)" >> "$GITHUB_OUTPUT"
# The split subset: {backend, case, transport} over inproc AND spawn, both backends, for
# every CI case that has not said `"split": false`. Still a SUBSET of the matrix above,
# so retrace-split needs no fixtures of its own. P5 shipped it as one case (OpenRA, which
# the phase gate names); P6 widened it to the whole CI matrix, because every bug the
# spawn work found was a question about the SERVER's process rather than about one trace.
# trace_cases.py refuses a `split` case that is not in CI, so the subset stays a subset.
SPLIT_MATRIX=$(python3 tools/trace_replay/trace_cases.py --ci --format github-split-matrix)
# AND IT MUST NOT BE EMPTY. An empty `include` is not an error to GitHub - it skips the
# whole retrace-split job with no red anywhere - so the one way this subset can vanish
# silently is guarded here. trace_cases.py now also rejects an unknown manifest key, which
# was the hole: `"splitt": true` loaded clean and emptied the subset (review N-4).
if [ "$(printf '%s' "${SPLIT_MATRIX}" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["include"]))')" -lt 1 ]; then
echo "::error::the split retrace subset is EMPTY. 'split' is an OPT-OUT as of P6 - a CI case is in this subset unless it carries \"split\": false - so an empty subset means either every case opted out or the CI matrix itself is empty. Either way retrace-split would be skipped with no red, and exit gate E2 names OpenRA."
# The retrace jobs replay shards: several cases of one (lane, transport, backend) cell
# per job. The first self-test proves every shard matrix replays exactly the rows of
# the per-case matrix it is built from; the second proves the shard runner gives each
# case the settings of its lane.
python3 tools/trace_replay/trace_cases.py --self-test-shard-matrices
python3 scripts/ci/retrace_shard.py --self-test
shards() {
python3 tools/trace_replay/trace_cases.py --ci --shards "${RETRACE_SHARDS_PER_CELL}" "$@"
}
shard_count() {
python3 -c 'import json,sys; print(len(json.load(sys.stdin)["include"]))'
}
shard_format=github-test-shard-matrix
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then shard_format=github-test-transport-shard-matrix; fi
# An empty `include` is not an error to GitHub: it skips the job with no red anywhere,
# so each matrix is checked before it is handed over.
require_shards() {
if [ "$(printf '%s' "$2" | shard_count)" -lt 1 ]; then
echo "::error::the $1 shard matrix is empty, so its job would be skipped with no red"
exit 1
fi
}
SHARDS=$(shards --format "${shard_format}")
require_shards retrace "${SHARDS}"
echo "shards=${SHARDS}" >> "$GITHUB_OUTPUT"
# The subset the verify build retraces ("verify": true in trace_cases.json).
VERIFY_SHARDS=$(shards --format github-verify-shard-matrix)
require_shards retrace-verify "${VERIFY_SHARDS}"
echo "verify-shards=${VERIFY_SHARDS}" >> "$GITHUB_OUTPUT"
# The split subset: {backend, case, transport} over inproc and spawn, both backends, for
# every CI case that has not said `"split": false`. It is the whole CI matrix because
# the bugs the spawn arm finds are about the server's process rather than about one
# trace. trace_cases.py refuses a `split` case that is not in CI.
SPLIT_SHARDS=$(shards --format github-split-shard-matrix)
# It must not be empty. An empty `include` is not an error to GitHub: it skips the
# whole retrace-split job with no red anywhere, so that is guarded here.
# trace_cases.py also rejects an unknown manifest key, since a mistyped `split` key
# would otherwise load clean and change the subset.
if [ "$(printf '%s' "${SPLIT_SHARDS}" | shard_count)" -lt 1 ]; then
echo "::error::the split retrace subset is empty. 'split' is an opt-out: a CI case is in this subset unless it carries \"split\": false, so an empty subset means either every case opted out or the CI matrix itself is empty. Either way retrace-split would be skipped with no red."
exit 1
fi
echo "split-matrix=${SPLIT_MATRIX}" >> "$GITHUB_OUTPUT"
echo "split-shards=${SPLIT_SHARDS}" >> "$GITHUB_OUTPUT"
# On the disaggregation branches, the same subset without the inproc rows the
# inproc-acceptance retrace rows replay on the same library, which retrace-split uses when
# it reuses build-linux's library. OpenRA keeps its inproc rows: they carry the
@@ -2733,101 +2756,110 @@ jobs:
# is not written, and retrace-split takes the full subset, as it does elsewhere.
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then
if python3 tools/trace_replay/trace_cases.py --self-test-split-matrix \
&& SPLIT_SPAWN_MATRIX=$(python3 tools/trace_replay/trace_cases.py --ci --format github-split-matrix \
&& SPLIT_SPAWN_SHARDS=$(shards --format github-split-shard-matrix \
--transports spawn --keep-inproc OpenRA) \
&& [ "$(printf '%s' "${SPLIT_SPAWN_MATRIX}" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["include"]))' || echo 0)" -ge 1 ]; then
echo "split-spawn-matrix=${SPLIT_SPAWN_MATRIX}" >> "$GITHUB_OUTPUT"
&& [ "$(printf '%s' "${SPLIT_SPAWN_SHARDS}" | shard_count || echo 0)" -ge 1 ]; then
echo "split-spawn-shards=${SPLIT_SPAWN_SHARDS}" >> "$GITHUB_OUTPUT"
else
echo "::warning::the spawn-only split retrace subset failed its self-test or is empty, so retrace-split replays the full split subset"
fi
fi
echo "retrace shards: $(printf '%s' "${SHARDS}" | shard_count) retrace, $(printf '%s' "${VERIFY_SHARDS}" | shard_count) verify, $(printf '%s' "${SPLIT_SHARDS}" | shard_count) split (full subset)"
# One cache entry holds every Git LFS fixture of the CI cases, and each replay job restores
# that entry itself. This job only makes sure the entry exists. On a hit it downloads
# nothing. On a miss it starts from the newest older bundle, fetches from the mirror the
# cases whose pointers changed, saves the entry under the new key and confirms it is there,
# so that no replay job has to download fixtures itself. The key, the path and the steps are
# .github/scripts/trace-fixture-bundle.sh's; apk.yml uses the same entry. The timeout bounds
# a fixture download that stalls, which would otherwise hold every replay job behind it.
trace-fixtures:
name: trace fixture (${{ matrix.case }})
name: trace fixtures
runs-on: ubuntu-26.04
needs: [trace-cases]
strategy:
fail-fast: false
max-parallel: 4
matrix:
case: ${{ fromJSON(needs.trace-cases.outputs.names) }}
timeout-minutes: 30
permissions:
actions: read
contents: read
outputs:
key: ${{ steps.bundle.outputs.key }}
path: ${{ steps.bundle.outputs.path }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Derive the fixture bundle key
id: bundle
run: bash .github/scripts/trace-fixture-bundle.sh key
- name: Derive trace fixture cache key
id: fixture-key
run: bash .github/scripts/trace-fixture-cache.sh key '${{ matrix.case }}'
- name: Restore trace fixture cache
id: fixture-cache
if: steps.fixture-key.outputs.cacheable == 'true'
- name: Look up the fixture bundle
id: lookup
uses: actions/cache/restore@v5
with:
path: ${{ steps.fixture-key.outputs.paths }}
key: ${{ steps.fixture-key.outputs.key }}
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
lookup-only: true
- name: Verify restored trace fixture
id: fixture-verify
if: steps.fixture-cache.outputs.cache-hit == 'true'
run: |
if bash .github/scripts/trace-fixture-cache.sh verify '${{ matrix.case }}'; then
echo "ok=true" >> "$GITHUB_OUTPUT"
else
echo "ok=false" >> "$GITHUB_OUTPUT"
echo "::warning::Cached fixture for ${{ matrix.case }} failed verification; falling back to the download path"
bash .github/scripts/trace-fixture-cache.sh reset '${{ matrix.case }}'
fi
- name: Restore the newest older bundle
if: steps.lookup.outputs.cache-hit != 'true'
uses: actions/cache/restore@v5
with:
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
restore-keys: ${{ steps.bundle.outputs.restore-prefix }}
- name: Fetch trace fixture
if: steps.fixture-verify.outputs.ok != 'true'
run: bash .github/scripts/fetch-trace-fixture-lfs.sh '${{ matrix.case }}'
- name: Complete the fixture bundle
if: steps.lookup.outputs.cache-hit != 'true'
run: bash .github/scripts/trace-fixture-bundle.sh ensure
- name: Save trace fixture cache
if: steps.fixture-key.outputs.cacheable == 'true' && steps.fixture-cache.outputs.cache-hit != 'true'
# The APK workflow of the same push saves the same entry, and only one save can win.
# The steps below find the entry whichever workflow wrote it.
- name: Save the fixture bundle
if: steps.lookup.outputs.cache-hit != 'true'
continue-on-error: true
uses: actions/cache/save@v5
with:
path: ${{ steps.fixture-key.outputs.paths }}
key: ${{ steps.fixture-key.outputs.key }}
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
- name: Stage trace fixture
run: |
safe_case="$(printf '%s' '${{ matrix.case }}' | sed 's/[^A-Za-z0-9._-]/_/g')"
stage_dir="trace-fixtures/${safe_case}"
mkdir -p "${stage_dir}"
python3 tools/trace_replay/trace_cases.py --format fixture-files --case '${{ matrix.case }}' |
while IFS= read -r file; do
cp "${file}" "${stage_dir}/"
done
# The APK workflow of the same push may be the one whose save won. Its entry is as good,
# once it is listed.
- name: Wait for the fixture bundle entry
if: steps.lookup.outputs.cache-hit != 'true'
env:
GH_TOKEN: ${{ github.token }}
run: bash .github/scripts/trace-fixture-bundle.sh wait '${{ steps.bundle.outputs.key }}'
- name: Upload trace fixture
uses: actions/upload-artifact@v7
- name: Confirm the fixture bundle entry
if: steps.lookup.outputs.cache-hit != 'true'
uses: actions/cache/restore@v5
with:
name: trace-fixture-${{ matrix.case }}
path: trace-fixtures/**
if-no-files-found: error
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
lookup-only: true
fail-on-cache-miss: true
# One job replays a shard: some of the cases of one lane, transport and backend, one after
# another, each through its own ctest entry (scripts/ci/retrace_shard.py). The shard's step
# summary has a row per case, and a failed case has an error annotation of its own.
retrace:
name: retrace (${{ matrix.backend }} / ${{ matrix.transport || 'monolith' }}, ${{ matrix.case }})
name: retrace (${{ matrix.backend }} / ${{ matrix.transport || 'monolith' }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
runs-on: ubuntu-26.04
timeout-minutes: 240
env:
MOBILEGL_TRANSPORT: ${{ matrix.transport || 'monolith' }}
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
MOBILEGL_IPC_RUN_AHEAD: "1"
SHARD_NAMES: ${{ toJSON(matrix.names) }}
needs:
- build-linux
- build-retrace
- trace-cases
- trace-fixtures
if: ${{ !cancelled() && needs.build-linux.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }}
if: ${{ !cancelled() && needs.build-linux.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }}
strategy:
fail-fast: false
max-parallel: 8
matrix: ${{ fromJSON(needs.trace-cases.outputs.matrix) }}
matrix: ${{ fromJSON(needs.trace-cases.outputs.shards) }}
steps:
- name: Checkout repo
@@ -2844,16 +2876,20 @@ jobs:
with:
gpu: "true"
- name: Download trace fixture
uses: actions/download-artifact@v8
# A miss does not fail the job: the entry may have been evicted since trace-fixtures
# confirmed it (a re-run of failed jobs on an older run does not run that job again).
- name: Restore trace fixtures
uses: actions/cache/restore@v5
with:
name: trace-fixture-${{ matrix.case }}
path: trace-fixture-download
path: ${{ needs.trace-fixtures.outputs.path }}
key: ${{ needs.trace-fixtures.outputs.key }}
- name: Install trace fixture
run: |
mkdir -p tools/trace_replay/fixtures
find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;
# Moves this shard's fixtures into place and checks them against their Git LFS pointers.
# A case the restored bundle does not hold is downloaded here, with a warning annotation;
# the timeout bounds a download that stalls.
- name: Install trace fixtures
timeout-minutes: 15
run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
- name: Get CMake
timeout-minutes: 15
@@ -2895,93 +2931,66 @@ jobs:
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
# The per-case settings are retrace_shard.py's "retrace" profile: the Magma fallback and
# the two per-case repair knobs on DirectVulkan, and on the inproc lane the IPC
# configuration the client must log, the split evidence check and the refusal census from
# the library's own logs. A failing case does not stop the shard; the step fails at the
# end if a case failed or a case has no verdict.
- name: Retrace and validate
working-directory: build-retrace/tools/trace_replay
run: |
ulimit -c unlimited
if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then
export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1
fi
if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
&& [ '${{ matrix.case }}' = 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' ]; then
export MOBILEGL_ITRP_FIX=ForceOn
fi
# The blended depth-write quirk auto-enables only on Qualcomm, which no CI
# runner has, so force it on for the OIT case it exists to fix. ForceOn
# bypasses only the vendor gate, so this exercises the real strip on
# lavapipe. The Android AVD lane deliberately leaves it off, keeping the
# unstripped path covered for the same trace.
if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
&& [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then
export MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1
fi
# The inproc rows stand in for retrace-split's dropped inproc rows, so they prove the
# configuration they run with from the client's own log.
if [ "${MOBILEGL_TRANSPORT}" = inproc ]; then
export MGTRACE_EXPECT_IPC='strict=1 role-split-state=1 run-ahead=1'
fi
ctest -V --no-tests=error -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$' --output-junit "${RUNNER_TEMP}/retrace.xml"
# A split row's transport assertions ran on this leg and found their evidence.
if [ "${MOBILEGL_TRANSPORT}" != monolith ]; then
python3 "${GITHUB_WORKSPACE}/scripts/ci/retrace_split_evidence.py" \
'${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt' \
--transport "${MOBILEGL_TRANSPORT}" --case '${{ matrix.case }}' --backend '${{ matrix.backend }}'
fi
# The refusal census, recorded rather than gated. run_trace_case.cmake already REDS the case
# on any Fatal{, so reaching here means the count is zero - but the number and the distinct
# slot names are what MEASUREMENTS wants from every split run, and reading them out of the
# library's own log is the only way to get them (ctest -V's transcript is a false zero).
- name: Refusal census from the library log
if: ${{ always() && matrix.transport == 'inproc' }}
working-directory: build-retrace/tools/trace_replay
run: |
# BOTH ROLES, because under spawn every applier refusal is raised in the SERVER's
# process and a census reading only the client's file reports a confident zero for the
# half it cannot see.
log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.client.log"
server_log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.server.log"
cat "${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt" 2>/dev/null || echo "no split-evidence.txt"
if [ ! -f "${log}" ]; then
echo "no ${log} - the replay wrote no library log"
exit 0
fi
census_files="${log}"
if [ -f "${server_log}" ]; then census_files="${census_files} ${server_log}"; fi
echo "census over: ${census_files}"
echo "Fatal{ lines: $(cat ${census_files} | grep -c 'Fatal{' || true)"
cat ${census_files} | grep -o 'Fatal{[A-Za-z]*, "[^"]*"' | sort | uniq -c | sort -rn | head -20 || true
# The transport line is the CLIENT's: ConfigLoader resolves it there, and under spawn
# the server states its own separately.
grep -m1 "MOBILEGL_TRANSPORT=" "${log}" || echo "no transport line in ${log}"
python3 scripts/ci/retrace_shard.py run --profile retrace \
--backend '${{ matrix.backend }}' --transport "${MOBILEGL_TRANSPORT}" \
--lane '${{ matrix.lane || 'monolith-control' }}' \
--shard '${{ matrix.shard }}/${{ matrix.shards }}' \
--build-dir build-retrace/tools/trace_replay \
--result-dir "${RUNNER_TEMP}/retrace-shard"
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py --excerpts 1 "${RUNNER_TEMP}"
run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit"
- name: Upload core dumps
if: failure()
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-core-dumps-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-${{ matrix.case }}
name: retrace-core-dumps-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }}
path: /tmp/core.*
if-no-files-found: ignore
- name: Upload actual image
# What retrace-summary renders: per case, the images and the replay's output directory.
# The summary embeds the images, so these are short-lived; a failed shard keeps its own
# copy below.
- name: Upload shard results
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-${{ matrix.case }}
path: |
build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/**
build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/**
name: retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }}
path: ${{ runner.temp }}/retrace-shard/results
if-no-files-found: warn
retention-days: 1
overwrite: true
- name: Upload failed shard results
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-failed-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }}
path: |
${{ runner.temp }}/retrace-shard/results
${{ runner.temp }}/retrace-shard/junit
${{ runner.temp }}/retrace-shard/logs
if-no-files-found: warn
overwrite: true
retrace-summary:
name: retrace summary
runs-on: ubuntu-26.04
needs: [retrace]
if: ${{ !cancelled() && needs.retrace.result != 'skipped' }}
# actions: write is for the last step, which deletes the artifacts this job rendered.
permissions:
actions: write
contents: read
env:
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
@@ -3050,25 +3059,48 @@ jobs:
archive: false
if-no-files-found: error
# The summaries embed every image, so the shard result artifacts they were rendered from
# have no reader left and are deleted here. That happens only when every shard passed: a
# failed shard has its own retrace-failed-* artifact, and after a failure the results
# stay for their one day of retention, so that re-running the failed shards renders every
# case again. A failed delete only leaves them to that retention.
- name: Delete the rendered shard results
if: ${{ needs.retrace.result == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
run: |
gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \
--jq '.artifacts[] | select(.name | startswith("retrace-result-")) | [.id, .name] | @tsv' \
> "${RUNNER_TEMP}/rendered-artifacts.txt"
deleted=0
while IFS=$'\t' read -r artifact_id artifact_name; do
echo "Deleting ${artifact_name} (${artifact_id})"
gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}"
deleted=$((deleted + 1))
done < "${RUNNER_TEMP}/rendered-artifacts.txt"
echo "Deleted ${deleted} rendered shard result artifact(s)."
# The trace half of the third CI mode. Same replay, same goldens, but the library underneath is
# the verify build and MOBILEGL_PIPE_VERIFY=1 is in the environment, so every backend read of
# frontend state is checked against a snapshot taken at the verb boundary. Eight cases rather
# than the full lane's 40 (tools/trace_replay/trace_cases.json, "verify": true): the comparator
# is budgeted at 5-10x, and the full sweep is a phase-exit / workflow_dispatch run.
retrace-verify:
name: retrace verify (${{ matrix.backend }}, ${{ matrix.case }})
name: retrace verify (${{ matrix.backend }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
runs-on: ubuntu-26.04
timeout-minutes: 240
env:
SHARD_NAMES: ${{ toJSON(matrix.names) }}
needs:
- build-linux-verify
- build-retrace
- trace-cases
- trace-fixtures
if: ${{ !cancelled() && needs.build-linux-verify.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }}
if: ${{ !cancelled() && needs.build-linux-verify.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }}
strategy:
fail-fast: false
max-parallel: 4
matrix: ${{ fromJSON(needs.trace-cases.outputs.verify-matrix) }}
matrix: ${{ fromJSON(needs.trace-cases.outputs.verify-shards) }}
steps:
- name: Checkout repo
@@ -3085,16 +3117,16 @@ jobs:
with:
gpu: "true"
- name: Download trace fixture
uses: actions/download-artifact@v8
# A miss does not fail the job; the install step downloads what the bundle does not hold.
- name: Restore trace fixtures
uses: actions/cache/restore@v5
with:
name: trace-fixture-${{ matrix.case }}
path: trace-fixture-download
path: ${{ needs.trace-fixtures.outputs.path }}
key: ${{ needs.trace-fixtures.outputs.key }}
- name: Install trace fixture
run: |
mkdir -p tools/trace_replay/fixtures
find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;
- name: Install trace fixtures
timeout-minutes: 15
run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
- name: Get CMake
timeout-minutes: 15
@@ -3151,82 +3183,60 @@ jobs:
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
- name: Retrace and validate (verify)
working-directory: build-retrace/tools/trace_replay
# run_trace_case.cmake turns MOBILEGL_PIPE_VERIFY into three assertions of its own (the
# arming line, no Fatal{PipeVerifyDiffer, no Fatal{UnmigratedPipeInput), so a case that
# somehow ran the wrong library reds here instead of passing on its golden.
# --timeout 10800: the 1800s cases run 5-10x slower with both comparator arms live, which
# is well past ctest's 1500s default.
run: |
ulimit -c unlimited
export MOBILEGL_PIPE_VERIFY=1
if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then
export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1
fi
if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
&& [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then
export MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1
fi
ctest -V --no-tests=error --timeout 10800 \
-R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$'
# The retrace lane's own always-on negative control, on one case so it costs one short trace:
# with a snapshot field corrupted, the SAME replay must fail. Without it, "40 traces, zero
# divergences" would be a statement about a comparator nobody watched.
# Every case replays with MOBILEGL_PIPE_VERIFY=1 (retrace_shard.py's "verify" profile).
# run_trace_case.cmake turns that into three assertions of its own (the arming line, no
# Fatal{PipeVerifyDiffer, no Fatal{UnmigratedPipeInput), so a case that somehow ran the
# wrong library reds here instead of passing on its golden. ctest's timeout is raised to
# 10800 s: the long cases run 5-10x slower with both comparator arms live, well past the
# 1500 s default.
#
# The rerun replays into the SAME case directory, so the verified run's images are put aside
# first and restored before the verdict: "Upload actual image" below runs `if: always()` and
# would otherwise ship the deliberately corrupted run's output under the name of the good one.
# The restore happens whichever way the control goes, which is why the ctest exit status is
# captured rather than tested inline.
- name: Negative control (corrupted snapshot)
if: ${{ matrix.case == 'OpenRA' && matrix.backend == 'DirectGLES' }}
working-directory: build-retrace/tools/trace_replay
# The shard that holds OpenRA on DirectGLES also runs the lane's negative control right
# after that case: with a snapshot field corrupted, the same replay must fail, and for the
# comparator's reason (scripts/ci/retrace_verify_reason.py). Without it, zero divergences
# would be a statement about a comparator nobody watched. The control has a verdict row of
# its own, and its output is discarded so that only the verified run is uploaded.
- name: Retrace and validate (verify)
run: |
export MOBILEGL_PIPE_VERIFY=1
export MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters
GOOD_OUTPUT="${RUNNER_TEMP}/openra-verified-output"
rm -rf "${GOOD_OUTPUT}"
if [ -d OpenRA ]; then
cp -a OpenRA "${GOOD_OUTPUT}"
fi
set +e
ctest -V --no-tests=error --timeout 10800 \
-R '^MobileGLTraceReplay\.OpenRA\.DirectGLES$' 2>&1 | tee "${RUNNER_TEMP}/verify-control.txt"
control_rc=${PIPESTATUS[0]}
set -e
if [ -d "${GOOD_OUTPUT}" ]; then
rm -rf OpenRA
mv "${GOOD_OUTPUT}" OpenRA
echo "restored the verified run's OpenRA output over the corrupted rerun's"
fi
if [ "${control_rc}" -eq 0 ]; then
echo "::error::MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left the OpenRA retrace GREEN, so the comparator is not comparing and the whole verify retrace lane proves nothing."
exit 1
fi
# Its red must carry the comparator's divergence on the corrupted field
# (scripts/ci/retrace_verify_reason.py, which test_run_trace_case.py runs on canned output).
python3 "${GITHUB_WORKSPACE}/scripts/ci/retrace_verify_reason.py" "${RUNNER_TEMP}/verify-control.txt"
echo "the corrupted field turned the retrace red, as it must (ctest exit ${control_rc})"
python3 scripts/ci/retrace_shard.py run --profile verify \
--backend '${{ matrix.backend }}' --lane verify \
--shard '${{ matrix.shard }}/${{ matrix.shards }}' \
--build-dir build-retrace/tools/trace_replay \
--result-dir "${RUNNER_TEMP}/retrace-shard"
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit"
- name: Upload core dumps
if: failure()
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-verify-core-dumps-${{ matrix.backend }}-${{ matrix.case }}
name: retrace-verify-core-dumps-${{ matrix.backend }}-shard${{ matrix.shard }}
path: /tmp/core.*
if-no-files-found: ignore
- name: Upload actual image
# Kept for every shard, green or not: per case, the verified images and the replay's
# output directory, whose library log has the comparator's arming line.
- name: Upload shard results
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: retrace-verify-result-${{ matrix.backend }}-${{ matrix.case }}
path: |
build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/**
build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/**
name: retrace-verify-result-${{ matrix.backend }}-shard${{ matrix.shard }}
path: ${{ runner.temp }}/retrace-shard/results
if-no-files-found: warn
overwrite: true
- name: Upload failed shard results
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-verify-failed-${{ matrix.backend }}-shard${{ matrix.shard }}
path: |
${{ runner.temp }}/retrace-shard/results
${{ runner.temp }}/retrace-shard/junit
${{ runner.temp }}/retrace-shard/logs
if-no-files-found: warn
overwrite: true
# Trace retrace on the split library: the retrace-verify shape with MOBILEGL_TRANSPORT set
# instead of MOBILEGL_PIPE_VERIFY, over {backend, case, transport} with inproc and spawn.
@@ -3237,7 +3247,7 @@ jobs:
# guards are the nm check for MG_Remote, run_trace_case.cmake's transport assertions (and, on
# OpenRA, its wire-volume assertion), and the per-leg check of the evidence file they leave.
retrace-split:
name: retrace split (${{ matrix.backend }} / ${{ matrix.transport }}, ${{ matrix.case }})
name: retrace split (${{ matrix.backend }} / ${{ matrix.transport }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
runs-on: ubuntu-26.04
timeout-minutes: 240
# The setup action selects lavapipe on GitHub-hosted runners.
@@ -3248,19 +3258,20 @@ jobs:
- build-retrace
- trace-cases
- trace-fixtures
if: ${{ !cancelled() && needs.build-linux-split.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && (needs.build-guard.outputs.verdict != 'identical' || needs.build-linux.result == 'success') }}
if: ${{ !cancelled() && needs.build-linux-split.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' && (needs.build-guard.outputs.verdict != 'identical' || needs.build-linux.result == 'success') }}
env:
SPLIT_REUSE: ${{ needs.build-guard.outputs.verdict == 'identical' && 'true' || 'false' }}
SPLIT_LIB: ${{ needs.build-guard.outputs.verdict == 'identical' && 'mobilegl-linux-lib' || 'mobilegl-linux-lib-split' }}
SPLIT_LIB_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }}
SPLIT_BUILD_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }}
SHARD_NAMES: ${{ toJSON(matrix.names) }}
strategy:
fail-fast: false
max-parallel: 8
# With build-linux reused, the inproc rows (except OpenRA's) are replayed by the retrace job's
# inproc-acceptance rows on the same library, strict and with the dual block and run-ahead
# on; the spawn rows below prove per leg that those are also the defaults they run with.
matrix: ${{ fromJSON(needs.build-guard.outputs.verdict == 'identical' && needs.trace-cases.outputs.split-spawn-matrix || needs.trace-cases.outputs.split-matrix) }}
# on; the spawn rows below prove per case that those are also the defaults they run with.
# A shard holds one transport, so OpenRA's inproc rows are shards of their own.
matrix: ${{ fromJSON(needs.build-guard.outputs.verdict == 'identical' && needs.trace-cases.outputs.split-spawn-shards || needs.trace-cases.outputs.split-shards) }}
steps:
- name: Checkout repo
@@ -3288,16 +3299,16 @@ jobs:
with:
gpu: "true"
- name: Download trace fixture
uses: actions/download-artifact@v8
# A miss does not fail the job; the install step downloads what the bundle does not hold.
- name: Restore trace fixtures
uses: actions/cache/restore@v5
with:
name: trace-fixture-${{ matrix.case }}
path: trace-fixture-download
path: ${{ needs.trace-fixtures.outputs.path }}
key: ${{ needs.trace-fixtures.outputs.key }}
- name: Install trace fixture
run: |
mkdir -p tools/trace_replay/fixtures
find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;
- name: Install trace fixtures
timeout-minutes: 15
run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
- name: Get CMake
timeout-minutes: 15
@@ -3351,12 +3362,11 @@ jobs:
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
# THE SPAWN ARM NEEDS A SECOND BINARY and the inproc arm needs none, which is the whole
# difference between the two at this level: inproc's server role is a thread in the replay
# process, spawn's is a PROCESS the client launches from this image. It ships in
# the split retrace library - libMobileGLServer.so is an executable that wears a
# .so name so an APK will install it somewhere exec-able - and a missing one must be a NAMED
# failure here rather than a LaunchServer refusal 78 jobs deep.
# The spawn arm needs a second binary and the inproc arm needs none: inproc's server role
# is a thread in the replay process, spawn's is a process the client launches from this
# image. It ships in the split retrace library (libMobileGLServer.so is an executable
# with a .so name, so that an APK installs it somewhere it can be executed), and a missing
# one must be a named failure here rather than a LaunchServer refusal in every case.
- name: Place the server image (spawn arm only)
if: ${{ matrix.transport == 'spawn' }}
run: |
@@ -3370,114 +3380,54 @@ jobs:
chmod +x build-linux/libMobileGLServer.so
echo "server image: $(readlink -f build-linux/libMobileGLServer.so)"
# The Magma knobs go to $GITHUB_ENV rather than to one step's shell, so every later step of
# this leg runs the same repair set as the replay.
- name: Magma repair knobs (all later steps)
run: |
# P7 package L: THE SAME MAGMA KNOBS THE MONOLITH RETRACE JOB EXPORTS, ALL OF THEM.
# This job exported none at all, so every DirectVulkan row of the split matrix was
# compared against a monolith row running a different repair set - and a split retrace
# whose control is not comparable cannot say whether a divergence is a transport fact.
# R11F_G11F_B10F fallback is whole-backend; the blended depth-write quirk auto-enables
# only on Qualcomm, which no runner has, so the OIT case forces it exactly as the
# monolith job does (ForceOn bypasses only the vendor gate).
if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then
echo "MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1" >> "${GITHUB_ENV}"
fi
if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
&& [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then
echo "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1" >> "${GITHUB_ENV}"
fi
# THE THREE ITERATIONRP KNOBS, FOR THE ONE CASE THE MONOLITH JOB NAMES.
# The iterationRP repairs are tri-state quirks that default to device auto-detection and
# lavapipe is on no auto list, so without them Program 203 runs unrepaired and misses its
# golden output. The monolith job has exported them since they landed; this job never
# did, so `iterationrp x DirectVulkan` was the one cell of the split matrix whose red
# could not be read as a transport fact - the monolith control it is compared against was
# running a different repair set. Exit gate 1 asks for the two halves to be COMPARABLE,
# and two runs that differ in a quirk knob are not.
if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
&& [ '${{ matrix.case }}' = 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' ]; then
echo "MOBILEGL_ITRP_FIX=ForceOn" >> "${GITHUB_ENV}"
fi
# The per-case settings are retrace_shard.py's "split" profile:
# - the Magma knobs the retrace job's monolith rows run with (the R11F_G11F_B10F fallback,
# the blended depth-write quirk on its one case, the iterationRP repairs on theirs), so a
# split row and its monolith control differ in the transport alone;
# - MOBILEGL_TRANSPORT and the absolute server image path;
# - on OpenRA, the wire-volume counters and the check that draw records crossed the wire;
# - with build-linux reused, the IPC defaults the client must log (strict=0).
# run_trace_case.cmake turns MOBILEGL_TRANSPORT into assertions of its own: the library must
# have resolved the transport in this process and its log must carry no Fatal{, so a case
# that somehow ran the wrong library reds here instead of passing on its golden. After
# ctest the evidence file those assertions leave is checked for every case, and the
# refusal census is printed from the library's own logs of both roles, which is the only
# valid source: the console sink is compiled out of this configuration.
- name: Retrace and validate (${{ matrix.transport }})
working-directory: build-retrace/tools/trace_replay
# run_trace_case.cmake turns MOBILEGL_TRANSPORT into assertions of its own - the library
# must have RESOLVED the transport in this process, and its log must carry no Fatal{ - so a
# case that somehow ran the wrong library reds here instead of passing on its golden. That
# log is also the only valid refusal census: the console sink is compiled out of this
# configuration, so a `ctest -V` transcript reports a false zero for Fatal{ lines.
run: |
ulimit -c unlimited
export MOBILEGL_TRANSPORT='${{ matrix.transport }}'
# Absolute, because the replay's working directory is not this one and LaunchServer
# resolves the image relative to nothing. Harmless on the inproc arm, which reads it
# only to decide it does not need it.
export MOBILEGL_IPC_SERVER_PATH="${GITHUB_WORKSPACE}/build-linux/libMobileGLServer.so"
# OpenRA's legs also count the wire's draw records and the server's draws; that turns on
# the library's per-frame counters, so the other cases keep the plain configuration.
wire=()
if [ '${{ matrix.case }}' = 'OpenRA' ]; then
export MGTRACE_WIRE_EVIDENCE=1
wire=(--require-wire)
fi
if [ "${SPLIT_REUSE}" = "true" ]; then
export MGTRACE_EXPECT_IPC='strict=0 role-split-state=1 run-ahead=1'
export MGTRACE_EXPECT_IPC_REASON="retrace-split dropped its inproc rows because the inproc-acceptance retrace rows replay the same library with strict=1 role-split-state=1 run-ahead=1, which differs from this leg's defaults only in strict while these hold. A default changed: restore the inproc rows (the trace-cases job's split-spawn-matrix without --transports spawn) or update the expectation."
fi
ctest -V --no-tests=error --timeout 10800 \
-R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$'
# The transport assertions ran on this leg and found their evidence.
python3 "${GITHUB_WORKSPACE}/scripts/ci/retrace_split_evidence.py" \
'${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt' \
--transport '${{ matrix.transport }}' --case '${{ matrix.case }}' \
--backend '${{ matrix.backend }}' "${wire[@]}"
reuse=()
if [ "${SPLIT_REUSE}" = "true" ]; then reuse=(--split-reuse); fi
python3 scripts/ci/retrace_shard.py run --profile split \
--backend '${{ matrix.backend }}' --transport '${{ matrix.transport }}' --lane split \
--shard '${{ matrix.shard }}/${{ matrix.shards }}' \
--server-path "${GITHUB_WORKSPACE}/build-linux/libMobileGLServer.so" \
--build-dir build-retrace/tools/trace_replay \
--result-dir "${RUNNER_TEMP}/retrace-shard" "${reuse[@]}"
# The refusal census, recorded rather than gated. run_trace_case.cmake already REDS the case
# on any Fatal{, so reaching here means the count is zero - but the number and the distinct
# slot names are what MEASUREMENTS wants from every split run, and reading them out of the
# library's own log is the only way to get them (ctest -V's transcript is a false zero).
- name: Refusal census from the library log
if: ${{ always() }}
working-directory: build-retrace/tools/trace_replay
run: |
# BOTH ROLES, because under spawn every applier refusal is raised in the SERVER's
# process and a census reading only the client's file reports a confident zero for the
# half it cannot see.
log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.client.log"
server_log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.server.log"
cat "${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt" 2>/dev/null || echo "no split-evidence.txt"
if [ ! -f "${log}" ]; then
echo "no ${log} - the replay wrote no library log"
exit 0
fi
census_files="${log}"
if [ -f "${server_log}" ]; then census_files="${census_files} ${server_log}"; fi
echo "census over: ${census_files}"
echo "Fatal{ lines: $(cat ${census_files} | grep -c 'Fatal{' || true)"
cat ${census_files} | grep -o 'Fatal{[A-Za-z]*, "[^"]*"' | sort | uniq -c | sort -rn | head -20 || true
# The transport line is the CLIENT's: ConfigLoader resolves it there, and under spawn
# the server states its own separately.
grep -m1 "MOBILEGL_TRANSPORT=" "${log}" || echo "no transport line in ${log}"
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit"
- name: Upload core dumps
if: failure()
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-split-core-dumps-${{ matrix.transport }}-${{ matrix.backend }}-${{ matrix.case }}
name: retrace-split-core-dumps-${{ matrix.transport }}-${{ matrix.backend }}-shard${{ matrix.shard }}
path: /tmp/core.*
if-no-files-found: ignore
- name: Upload actual image
if: ${{ always() }}
# Nothing reads a green shard's output, so only a failed shard uploads it.
- name: Upload failed shard results
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-split-result-${{ matrix.transport }}-${{ matrix.backend }}-${{ matrix.case }}
name: retrace-split-failed-${{ matrix.transport }}-${{ matrix.backend }}-shard${{ matrix.shard }}
path: |
build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/**
build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/**
${{ runner.temp }}/retrace-shard/results
${{ runner.temp }}/retrace-shard/junit
${{ runner.temp }}/retrace-shard/logs
if-no-files-found: warn
overwrite: true
# The shipped library, replayed. build-linux-monolith-control's library has no MG_Remote, the
# shape FCL embeds; on the disaggregation branches nothing else replays that shape, so it runs
@@ -3490,8 +3440,8 @@ jobs:
needs:
- build-linux-monolith-control
- build-retrace
- trace-fixtures
if: ${{ !cancelled() && (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && needs.build-linux-monolith-control.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-fixtures.result != 'skipped' }}
- trace-cases
if: ${{ !cancelled() && (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && needs.build-linux-monolith-control.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }}
strategy:
fail-fast: false
matrix:
@@ -3513,16 +3463,11 @@ jobs:
with:
gpu: "true"
- name: Download trace fixture
uses: actions/download-artifact@v8
with:
name: trace-fixture-${{ matrix.case }}
path: trace-fixture-download
- name: Install trace fixture
run: |
mkdir -p tools/trace_replay/fixtures
find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;
# This job's one case keeps its fixtures in Git itself, not in Git LFS, so the checkout
# has them and the fixture bundle is not restored. The check is that they are real files;
# a case with Git LFS fixtures would be downloaded here, with a warning annotation.
- name: Check trace fixture
run: bash .github/scripts/trace-fixture-bundle.sh install '${{ matrix.case }}'
- name: Get CMake
timeout-minutes: 15
@@ -3588,15 +3533,16 @@ jobs:
run: python3 scripts/ci/junit_annotate.py --excerpts 1 "${RUNNER_TEMP}"
- name: Upload core dumps
if: failure()
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-shipped-core-dumps-${{ matrix.backend }}-${{ matrix.case }}
path: /tmp/core.*
if-no-files-found: ignore
# Nothing reads a green leg's output, so only a failed leg uploads it.
- name: Upload actual image
if: ${{ always() }}
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-shipped-result-${{ matrix.backend }}-${{ matrix.case }}
@@ -3605,82 +3551,6 @@ jobs:
build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/**
if-no-files-found: warn
remove-artifact-clutter:
name: remove artifact clutter
runs-on: ubuntu-26.04
# Every retrace job downloads the shared fixtures, so this waits for all of them.
needs:
- retrace-summary
- retrace-verify
- retrace-split
- retrace-shipped
if: ${{ always() }}
permissions:
actions: write
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Delete intermediate Linux retrace artifacts
env:
GH_TOKEN: ${{ github.token }}
run: |
# Read the complete inventories before deleting anything. A failed API
# request must fail this step, not look like "no failed cases" through
# an unchecked process substitution.
# 50 rows per page: the jobs endpoint answers HTTP 502 for 90+ rows on a re-run attempt of a run this size.
gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=50" \
--jq '.jobs[] | select((.name | startswith("retrace (")) or (.name | startswith("retrace verify (")) or (.name | startswith("retrace split (")) or (.name | startswith("retrace shipped library ("))) | select(.conclusion == "failure" or .conclusion == "canceled" or .conclusion == "timed_out" or .conclusion == "action_required") | .name' \
> "${RUNNER_TEMP}/cleanup-jobs.txt"
gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \
--jq '.artifacts[] | select(.name | startswith("trace-fixture-") or startswith("retrace-result-") or startswith("retrace-split-result-") or startswith("retrace-shipped-result-")) | [.id, .name] | @tsv' \
> "${RUNNER_TEMP}/cleanup-artifacts.txt"
declare -A failed_cases=()
while IFS= read -r job_name; do
case_name="${job_name#retrace split (*, }"
case_name="${case_name#retrace shipped library (*, }"
case_name="${case_name#retrace verify (*, }"
case_name="${case_name#retrace (*, }"
case_name="${case_name%)}"
failed_cases["${case_name}"]=1
done < "${RUNNER_TEMP}/cleanup-jobs.txt"
if ((${#failed_cases[@]})); then
echo "Retaining fixtures and results for failed retrace case(s):"
printf ' %s\n' "${!failed_cases[@]}"
else
echo "All retrace jobs succeeded; no fixtures need to be retained."
fi
deleted=0
retained=0
while IFS=$'\t' read -r artifact_id artifact_name; do
keep=0
if [[ "${artifact_name}" == trace-fixture-* ]]; then
case_name="${artifact_name#trace-fixture-}"
if [[ -v "failed_cases[${case_name}]" ]]; then
keep=1
fi
else
for case_name in "${!failed_cases[@]}"; do
if [[ "${artifact_name}" == *-"${case_name}" ]]; then keep=1; break; fi
done
fi
if ((keep)); then
echo "Retaining ${artifact_name} (${artifact_id}) for failed retrace."
((retained += 1))
continue
fi
echo "Deleting ${artifact_name} (${artifact_id})"
gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}"
((deleted += 1))
done < "${RUNNER_TEMP}/cleanup-artifacts.txt"
echo "Deleted ${deleted} intermediate Linux artifact(s); retained ${retained} failed-retrace artifact(s)."
pipe-gates:
name: MGPipe generators and hygiene gates
runs-on: ubuntu-26.04