swung0x48/MobileGL
1
0
Fork 0
mirror of https://github.com/MobileGL-Dev/MobileGL synced 2026-11-16 06:23:23 +09:00
Code Issues Packages Projects Releases Wiki Activity

[CI, Tools] (...): replay trace cases in shards from one fixture cache

Browse Source
This commit is contained in:
swung0x48
2026-11-15 11:47:18 -04:00
parent 222f6209a0
commit 697bcc6d45
20 changed files with 4176 additions and 884 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/trace-fixture-bundle.sh
+304
View File
@@ -0,0 +1,304 @@
#!/usr/bin/env bash
# One cache entry that holds every Git LFS trace fixture of the CI cases.
#
# key derive the actions/cache key, restore prefix and path of the bundle
# ensure after a cache miss: complete the bundle directory, fetching only the
# cases whose files are missing from it or do not match their pointers
# wait <key> [secs] wait until the cache entry <key> is listed for this ref (needs GH_TOKEN)
# install <case>... move the fixtures of these cases from the restored bundle directory into
# the fixture directory, fetch the cases it does not hold, and check every
# file against its pointer
#
# The key is content-addressed on the pointer oids tracked at HEAD, which a plain checkout
# reads without smudging. The bundle directory is the same path for every revision, so
# `ensure` can start from an older bundle. Fixtures stored in Git itself rather than in Git
# LFS (OpenRA) are not in the bundle; `install` checks that they are real files.
set -euo pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=trace-fixture-lib.sh
. "${script_dir}/trace-fixture-lib.sh"
# Bump when the key derivation or the bundle layout changes in a way that must invalidate
# old entries; the content digest alone would not notice a format change.
key_schema="v1"
bundle_dir="trace-fixture-bundle"
fixture_dir="tools/trace_replay/fixtures"
python_bin="${PYTHON:-python3}"
if ! command -v "${python_bin}" >/dev/null 2>&1 && command -v python >/dev/null 2>&1; then
python_bin=python
fi
usage() {
echo "usage: $0 <key|ensure|wait <key> [seconds]|install <trace-case>...>" >&2
exit 2
}
# Writes "name=value" to $GITHUB_OUTPUT when running under Actions, and to stdout as well so
# the script stays runnable off-CI.
emit_output() {
local name="$1"
local value="$2"
if [ -n "${GITHUB_OUTPUT:-}" ]; then
printf '%s=%s\n' "${name}" "${value}" >> "${GITHUB_OUTPUT}"
fi
printf '%s=%s\n' "${name}" "${value}"
}
# "<case><TAB><file>" for every fixture file of every CI case.
ci_fixture_manifest() {
"${python_bin}" tools/trace_replay/trace_cases.py --ci --format fixture-manifest \
--fixture-root "${fixture_dir}" | tr -d '\r'
}
is_lfs_fixture() {
get_lfs_metadata "$1" >/dev/null 2>&1
}
# A fixture that Git stores directly must be a real file, not a pointer left by a wrong
# attribute.
check_plain_fixture() {
local file="$1"
if [ ! -s "${file}" ]; then
echo "fixture file is missing or empty: ${file}" >&2
return 1
fi
if head -n 1 "${file}" | grep -q "version https://git-lfs.github.com/spec/v1"; then
echo "fixture should not be stored as an LFS pointer: ${file}" >&2
return 1
fi
}
# Puts the tracked pointer files back. The smudge filter is skipped so that a clone with Git
# LFS set up does not download the objects here.
restore_pointers() {
GIT_LFS_SKIP_SMUDGE=1 git checkout -- "$@"
}
verify_lfs_fixture() {
local on_disk="$1"
local tracked="$2"
local metadata expected_oid expected_size
metadata="$(get_lfs_metadata "${tracked}")" || return 1
read -r expected_oid expected_size <<< "${metadata}"
verify_fixture_file "${on_disk}" "${tracked}" "${expected_oid}" "${expected_size}"
}
# Moves the given Git LFS fixtures of one case from the bundle directory into the fixture
# directory. Succeeds when every one of them is then in place and matches its pointer.
take_from_bundle() {
local file bundled
local incomplete=0
for file in "$@"; do
bundled="${bundle_dir}/$(basename "${file}")"
# Two cases may share a file; the second finds it already in place.
if [ -f "${bundled}" ]; then
mv -f "${bundled}" "${file}"
fi
if ! verify_lfs_fixture "${file}" "${file}" 2>/dev/null; then
incomplete=1
fi
done
return "${incomplete}"
}
# Downloads the given Git LFS fixtures of one case over their pointers: the mirrors first,
# Git LFS for what no mirror serves. The fetch script verifies what it downloaded.
fetch_case_fixtures() {
local case_name="$1"
shift
local file
for file in "$@"; do
rm -f "${file}" "${file}.tmp"
done
restore_pointers "$@" || return 1
bash "${script_dir}/fetch-trace-fixture-lfs.sh" "${case_name}" "${fixture_dir}"
}
[ "$#" -ge 1 ] || usage
command_name="$1"
shift
case "${command_name}" in
key)
[ "$#" -eq 0 ] || usage
# One "<basename> <oid> <size>" line per bundled file, sorted, each file once.
manifest=""
count=0
while IFS= read -r file; do
[ -n "${file}" ] || continue
if ! metadata="$(get_lfs_metadata "${file}" 2>/dev/null)"; then
continue
fi
read -r expected_oid expected_size <<< "${metadata}"
manifest+="$(basename "${file}") ${expected_oid} ${expected_size}"$'\n'
count=$((count + 1))
done < <(ci_fixture_manifest | cut -f2 | LC_ALL=C sort -u)
if [ "${count}" -eq 0 ]; then
echo "no Git LFS fixture found for the CI trace cases; the bundle would be empty" >&2
exit 1
fi
duplicates="$(printf '%s' "${manifest}" | cut -d' ' -f1 | LC_ALL=C sort | uniq -d)"
if [ -n "${duplicates}" ]; then
echo "two fixture files share a name, so the bundle directory cannot hold both: ${duplicates}" >&2
exit 1
fi
digest="$(printf '%s' "${manifest}" | sha256sum | awk '{ print substr($1, 1, 16) }')"
emit_output "key" "trace-fixture-bundle-${key_schema}-${digest}"
emit_output "restore-prefix" "trace-fixture-bundle-${key_schema}-"
emit_output "path" "${bundle_dir}"
emit_output "files" "${count}"
;;
ensure)
[ "$#" -eq 0 ] || usage
mkdir -p "${bundle_dir}" "${fixture_dir}"
mapfile -t manifest_lines < <(ci_fixture_manifest)
declare -A case_seen=()
case_order=()
for line in "${manifest_lines[@]}"; do
case_name="${line%%$'\t'*}"
if [ -z "${case_seen[${case_name}]:-}" ]; then
case_seen["${case_name}"]=1
case_order+=("${case_name}")
fi
done
fetched=0
reused=0
bundled_files=()
for case_name in "${case_order[@]}"; do
lfs_files=()
for line in "${manifest_lines[@]}"; do
[ "${line%%$'\t'*}" = "${case_name}" ] || continue
file="${line#*$'\t'}"
if is_lfs_fixture "${file}"; then
lfs_files+=("${file}")
else
check_plain_fixture "${file}"
fi
done
[ "${#lfs_files[@]}" -gt 0 ] || continue
# Take what an older bundle already holds, and fetch the case if that is not all of it.
if take_from_bundle "${lfs_files[@]}"; then
reused=$((reused + 1))
else
echo "Fetching trace fixtures of ${case_name}: the bundle does not hold them"
fetch_case_fixtures "${case_name}" "${lfs_files[@]}"
fetched=$((fetched + 1))
fi
bundled_files+=("${lfs_files[@]}")
done
# The bundle is rebuilt from the verified files, so nothing an older bundle carried and
# no case names any more is saved again.
rm -rf "${bundle_dir}"
mkdir -p "${bundle_dir}"
declare -A file_seen=()
saved=0
for file in "${bundled_files[@]}"; do
[ -z "${file_seen[${file}]:-}" ] || continue
file_seen["${file}"]=1
verify_lfs_fixture "${file}" "${file}"
mv -f "${file}" "${bundle_dir}/$(basename "${file}")"
saved=$((saved + 1))
done
if [ "${saved}" -eq 0 ]; then
echo "the trace fixture bundle came out empty" >&2
exit 1
fi
# Leave the working tree as a checkout has it: pointers in the fixture directory.
restore_pointers "${!file_seen[@]}"
echo "Trace fixture bundle: ${saved} file(s), ${reused} case(s) from the restored bundle, ${fetched} case(s) fetched."
emit_output "fetched" "${fetched}"
;;
wait)
[ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage
key="$1"
limit="${2:-300}"
# Another workflow of the same push may be the one saving this key. The replay jobs
# restore the entry, so it has to be listed before this job ends. The caller checks the
# entry itself afterwards; this only waits, and never fails.
#
# The listing is narrowed to the ref of this run. The key is content-addressed, so another
# branch may already hold an entry under it, and that entry cannot be restored from here:
# counting it would end the wait while the entry of this ref is still being uploaded.
query=(-f "key=${key}" -f per_page=100)
if [ -n "${GITHUB_REF:-}" ]; then
query+=(-f "ref=${GITHUB_REF}")
fi
waited=0
refused=0
while :; do
if listed="$(gh api -X GET "repos/${GITHUB_REPOSITORY}/actions/caches" "${query[@]}" \
--jq '[.actions_caches[] | select(.key == "'"${key}"'")] | length' 2>/dev/null)"; then
if [ "${listed:-0}" -ge 1 ]; then
echo "cache entry ${key} is listed after ${waited} s"
break
fi
else
refused=$((refused + 1))
if [ "${refused}" -ge 3 ]; then
echo "the cache listing cannot be read (no token, or no actions: read permission); not waiting"
break
fi
fi
if [ "${waited}" -ge "${limit}" ]; then
echo "cache entry ${key} is not listed after ${waited} s"
break
fi
sleep 10
waited=$((waited + 10))
done
;;
install)
[ "$#" -ge 1 ] || usage
installed=0
fetched=0
for case_name in "$@"; do
mapfile -t files < <(trace_fixture_files "${case_name}" "${fixture_dir}" "${python_bin}")
if [ "${#files[@]}" -eq 0 ]; then
echo "no fixture files declared for trace case: ${case_name}" >&2
exit 1
fi
lfs_files=()
for file in "${files[@]}"; do
if is_lfs_fixture "${file}"; then
lfs_files+=("${file}")
else
check_plain_fixture "${file}"
fi
done
[ "${#lfs_files[@]}" -gt 0 ] || continue
# A bundle that was not restored (the cache entry was evicted, or its download failed)
# or that lacks a file costs this job a download of the case, not the replay.
if ! take_from_bundle "${lfs_files[@]}"; then
echo "::warning::the restored fixture bundle does not hold the trace fixtures of ${case_name}; fetching them"
if ! fetch_case_fixtures "${case_name}" "${lfs_files[@]}"; then
echo "::error::the trace fixtures of ${case_name} are not in the restored bundle and could not be fetched"
exit 1
fi
fetched=$((fetched + 1))
for file in "${lfs_files[@]}"; do
if ! verify_lfs_fixture "${file}" "${file}"; then
echo "::error::trace fixture ${file} of ${case_name} does not match its Git LFS pointer after the fetch"
exit 1
fi
done
fi
installed=$((installed + ${#lfs_files[@]}))
done
# The rest of the bundle belongs to other shards.
rm -rf "${bundle_dir}"
echo "Installed and verified ${installed} Git LFS fixture file(s) for $# trace case(s), ${fetched} case(s) fetched."
;;
*)
usage
;;
esac
.github/scripts/trace-fixture-cache.sh
-117
View File
@@ -1,117 +0,0 @@
#!/usr/bin/env bash
# Cache-side helper for trace fixtures.
#
# key <case> [fixture-dir] derive the actions/cache key and path list
# verify <case> [fixture-dir] check restored fixtures against their pointers
# reset <case> [fixture-dir] drop restored fixtures, leaving the pointers
#
# The cache key is content-addressed on the Git LFS pointer oids tracked at
# HEAD, which are readable from a plain checkout without smudging. Fixture
# content therefore maps 1:1 onto a key: unchanged content hits, changed
# content is a new key and thus a miss, and the download path handles it. The
# key deliberately carries no restore-keys prefix in the workflow - a fixture
# that does not match the pointer exactly must never be restored.
set -euo pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=trace-fixture-lib.sh
. "${script_dir}/trace-fixture-lib.sh"
# Bump when the key derivation changes in a way that must invalidate old
# entries; the content digest alone would not notice a format change.
key_schema="v1"
if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then
echo "usage: $0 <key|verify|reset> <trace-case> [fixture-dir]" >&2
exit 2
fi
command_name="$1"
case_name="$2"
fixture_dir="${3:-tools/trace_replay/fixtures}"
python_bin="${PYTHON:-python3}"
if ! command -v "${python_bin}" >/dev/null 2>&1 && command -v python >/dev/null 2>&1; then
python_bin=python
fi
mapfile -t files < <(trace_fixture_files "${case_name}" "${fixture_dir}" "${python_bin}")
if [ "${#files[@]}" -eq 0 ]; then
echo "no fixture files declared for trace case: ${case_name}" >&2
exit 1
fi
# Writes "name=value" to $GITHUB_OUTPUT when running under Actions, and to
# stdout otherwise so the script stays runnable (and testable) off-CI.
emit_output() {
local name="$1"
local value="$2"
if [ -n "${GITHUB_OUTPUT:-}" ]; then
if [[ "${value}" == *$'\n'* ]]; then
local delimiter="ghadelim_$(date +%s%N)_$$"
{
printf '%s<<%s\n' "${name}" "${delimiter}"
printf '%s\n' "${value}"
printf '%s\n' "${delimiter}"
} >> "${GITHUB_OUTPUT}"
else
printf '%s=%s\n' "${name}" "${value}" >> "${GITHUB_OUTPUT}"
fi
fi
printf '%s=%s\n' "${name}" "${value}"
}
sanitize_case() {
printf '%s' "$1" | sed 's/[^A-Za-z0-9._-]/_/g'
}
case "${command_name}" in
key)
manifest=""
for file in "${files[@]}"; do
# A case whose fixtures are committed directly rather than through Git LFS
# (OpenRA) has no pointer oid to key on, and nothing to download either.
# Report it as uncacheable so the workflow skips the cache entirely.
if ! metadata="$(get_lfs_metadata "${file}" 2>/dev/null)"; then
echo "trace case ${case_name} is not stored in Git LFS; skipping fixture cache" >&2
emit_output "cacheable" "false"
emit_output "key" ""
exit 0
fi
read -r expected_oid expected_size <<< "${metadata}"
manifest+="$(basename "${file}") ${expected_oid} ${expected_size}"$'\n'
done
digest="$(printf '%s' "${manifest}" | sha256sum | awk '{ print substr($1, 1, 16) }')"
safe_case="$(sanitize_case "${case_name}")"
emit_output "cacheable" "true"
emit_output "key" "trace-fixture-${key_schema}-${safe_case}-${digest}"
emit_output "paths" "$(printf '%s\n' "${files[@]}")"
;;
verify)
for file in "${files[@]}"; do
metadata="$(get_lfs_metadata "${file}")"
read -r expected_oid expected_size <<< "${metadata}"
verify_fixture_file "${file}" "${file}" "${expected_oid}" "${expected_size}"
done
echo "Verified ${#files[@]} fixture file(s) for ${case_name} against the tracked Git LFS pointers."
;;
reset)
# Put the working tree back to the pointer files a fresh checkout would
# have, so that a rejected cache entry falls through to exactly the same
# download path a cache miss takes.
for file in "${files[@]}"; do
rm -f "${file}" "${file}.tmp"
done
git checkout -- "${files[@]}"
echo "Reset ${#files[@]} fixture file(s) for ${case_name} to their tracked Git LFS pointers."
;;
*)
echo "unknown command: ${command_name}" >&2
exit 2
;;
esac
.github/scripts/trace-fixture-lib.sh
+1 -1
View File
@@ -2,7 +2,7 @@
# Shared helpers for trace-fixture handling: reading the in-tree Git LFS pointer
# metadata and verifying a fixture file against it. Sourced by
# fetch-trace-fixture-lfs.sh (verify after download) and by
# trace-fixture-cache.sh (cache key derivation and verify after cache restore),
# trace-fixture-bundle.sh (cache key derivation and verify after cache restore),
# so both paths agree on what a valid fixture is.
# Reads the Git LFS pointer tracked at HEAD for a fixture path and prints
.github/workflows/apk.yml
+350 -301
View File
@@ -12,11 +12,28 @@ on:
- feat/disaggregated
- codex/fix-github-ci
- cleanup
# A docs-only push builds nothing this workflow tests (231 emulator legs per run). The Test
# workflow keeps running for docs: it hosts the doc citation lint.
# A push that only touches these paths changes nothing this workflow builds or replays. The
# Test workflow keeps running for docs: it hosts the doc citation lint. The Android build
# compiles none of the unit, integration or benchmark trees, with one exception that keeps
# MobileGL/MG_Benchmark from being listed whole: MG_Util/SelfTest/DriverBenchJni.cpp
# includes MG_Benchmark/Driver/DriverBenchCases.inc.
paths-ignore:
- 'docs/**'
- '**/*.md'
- 'MobileGL/MG_Test/**'
- 'MobileGL/MG_IntegrationTest/**'
- 'MobileGL/MG_Benchmark/CMakeLists.txt'
- 'MobileGL/MG_Benchmark/SanityBench.cpp'
- 'MobileGL/MG_Benchmark/Buffer/**'
- 'MobileGL/MG_Benchmark/Container/**'
- 'MobileGL/MG_Benchmark/Program/**'
- 'MobileGL/MG_Benchmark/ShaderCache/**'
- 'MobileGL/MG_Benchmark/Transpile/**'
- 'tools/cts/**'
- 'tools/device_bench/**'
- '.github/workflows/test.yml'
- '.github/workflows/hygiene.yml'
- '.github/actions/setup-test-runner/**'
workflow_dispatch:
env:
@@ -39,7 +56,10 @@ jobs:
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_COMPRESS: "true"
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 4G
# One build's objects (three native configurations) take about 1 GB. The limit leaves
# half of that again for the objects a push replaces, and is what the cleanup before
# the save trims the entry to.
CCACHE_MAXSIZE: 1500M
CCACHE_NOHASHDIR: "true"
MOBILEGL_CMAKE_COMPILER_LAUNCHER: ccache
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
@@ -47,8 +67,67 @@ jobs:
steps:
- name: Checkout repo
uses: actions/checkout@v6
with:
submodules: recursive
# Only the submodules the two APKs are built from, shallow and in parallel, in place of a
# recursive checkout of every submodule in the tree: the ones the root CMakeLists.txt
# configures, and apitrace with the eight directories its thirdparty/CMakeLists.txt
# requires, for the trace APK. Each row names a submodule and one file the build reads
# from it, and a row whose file is missing afterwards fails the step: without
# 3rdparty/flatbuffers the root CMakeLists.txt would go on and build a library without
# MG_Remote instead of stopping. A submodule the build starts to use needs a row here.
- name: Check out build submodules
timeout-minutes: 15
run: |
top=(
"3rdparty/glslang CMakeLists.txt"
"3rdparty/SPIRV-Cross CMakeLists.txt"
"3rdparty/SPIRV-Reflect CMakeLists.txt"
"3rdparty/VulkanMemoryAllocator CMakeLists.txt"
"3rdparty/Vulkan-Headers CMakeLists.txt"
"3rdparty/Vulkan-Utility-Libraries CMakeLists.txt"
"3rdparty/xxHash build/cmake/CMakeLists.txt"
"3rdparty/asio include/asio.hpp"
"3rdparty/flatbuffers include/flatbuffers/flatbuffers.h"
"include/ska flat_hash_map.hpp"
"3rdparty/apitrace thirdparty/CMakeLists.txt"
)
apitrace=(
"thirdparty/brotli LICENSE"
"thirdparty/gtest LICENSE"
"thirdparty/libbacktrace LICENSE"
"thirdparty/libpng LICENSE"
"thirdparty/snappy COPYING"
"thirdparty/zlib README"
"thirdparty/zstd LICENSE"
"thirdparty/directxmath LICENSE"
)
# update <directory> <row>...: the update is idempotent, so a fetch that failed on
# the network is tried again, three times in all. Then every row's file must exist.
update() {
local dir="$1" paths=() row attempt updated=0 missing=0
shift
for row in "$@"; do
paths+=("${row%% *}")
done
for attempt in 1 2 3; do
if git -C "${dir}" submodule update --init --depth=1 --jobs 8 -- "${paths[@]}"; then
updated=1
break
fi
echo "the submodule update in ${dir} failed (attempt ${attempt} of 3)" >&2
sleep $((attempt * 5))
done
[ "${updated}" -eq 1 ] || return 1
for row in "$@"; do
if [ ! -f "${dir}/${row%% *}/${row#* }" ]; then
echo "::error::${dir}/${row%% *}/${row#* } is missing after the submodule update, so the APK build cannot use ${row%% *}"
missing=1
fi
done
return "${missing}"
}
update . "${top[@]}"
update 3rdparty/apitrace "${apitrace[@]}"
- name: Set artifact metadata
run: |
@@ -82,6 +161,9 @@ jobs:
sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 update
sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 install -y ccache
ccache --version
# The restored entry carries the counters of every earlier build; zero them so the
# statistics printed after the build are this run's.
ccache --zero-stats
- name: Setup Android SDK
id: netsetup84
@@ -183,11 +265,20 @@ jobs:
if: always()
run: ccache --show-stats
# Rewrite one rolling entry per job and branch. The upload stays cumulative - it
# carries every object restored at the top of this run plus the few TUs that
# changed - but Actions cache keys are immutable, so this branch's superseded
# blob is released first; --ref keeps the delete off other branches' entries.
# Running after the build means a failed build leaves the entry untouched.
# A build served from the cache stores nothing, so nothing would ever evict the objects
# of older source states. Trim to CCACHE_MAXSIZE, least recently used first, before the
# entry is saved; the objects this build hit were just touched and stay.
- name: Trim ccache to its size limit
run: |
ccache --cleanup
ccache --show-stats | grep -i 'cache size' || true
du -sh "${CCACHE_DIR}"
# Rewrite one rolling entry per job and branch. The upload carries the objects this
# build used plus what fits under the size limit, but Actions cache keys are
# immutable, so this branch's superseded blob is released first; --ref keeps the
# delete off other branches' entries. Running after the build means a failed build
# leaves the entry untouched.
- name: Release superseded ccache entry
env:
GH_TOKEN: ${{ github.token }}
@@ -252,9 +343,17 @@ jobs:
runs-on: ubuntu-26.04
env:
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
# How much replay one emulator job aims for, in seconds as the duration hints count them.
# A (lane, transport, backend) cell is cut into as many shards as that takes, and its
# cases are spread so the shards of a cell come out even (tools/trace_replay/trace_cases.py,
# shard_matrix). A smaller value gives more and shorter jobs, each of which pays the
# emulator boot again; every case is replayed once per cell whatever the value.
RETRACE_SHARD_SECONDS: "720"
# The most a shard runner may spend on its cases. A shard gets the time in which every
# one of its cases could run to the end of its own time budget, up to this cap.
RETRACE_SHARD_DEADLINE_CAP_SECONDS: "7200"
outputs:
android: ${{ steps.trace-cases.outputs.android }}
names: ${{ steps.trace-cases.outputs.names }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
@@ -263,77 +362,105 @@ jobs:
id: trace-cases
run: |
python3 tools/trace_replay/trace_cases.py --self-test-transport-matrices
matrix_format=github-apk-matrix
# The retrace jobs replay shards: several cases of one (lane, transport, backend) cell
# per emulator boot. The self-test proves every shard matrix replays exactly the rows
# of the per-case matrix it is built from, and trace_cases.py refuses to print one
# that does not.
python3 tools/trace_replay/trace_cases.py --self-test-shard-matrices
matrix_format=github-apk-shard-matrix
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then
matrix_format=github-apk-transport-matrix
matrix_format=github-apk-transport-shard-matrix
fi
echo "android=$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}")" >> "$GITHUB_OUTPUT"
echo "names=$(python3 tools/trace_replay/trace_cases.py --ci --format names)" >> "$GITHUB_OUTPUT"
android="$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}" \
--shard-seconds "${RETRACE_SHARD_SECONDS}")"
# An empty `include` is not an error to GitHub: it skips the retrace job with no red.
shards="$(printf '%s' "${android}" | python3 -c 'import json, sys; print(len(json.load(sys.stdin)["include"]))')"
if [ "${shards}" -lt 1 ]; then
echo "::error::the Android retrace matrix is empty, so the retrace job would be skipped with no red"
exit 1
fi
# The cases of a shard share one job, so a case that hangs spends time the cases after
# it need. retrace_shard_budget.py gives every shard its own deadline_seconds, the
# time in which each of its cases could run to the end of its time budget, capped so
# that a shard whose replays all hang cannot hold a runner for hours, and the
# timeout_minutes of its job. The retrace job reads both from the matrix.
android="$(printf '%s' "${android}" | python3 android-plugin/retrace_shard_budget.py \
--cap-seconds "${RETRACE_SHARD_DEADLINE_CAP_SECONDS}")"
echo "android=${android}" >> "$GITHUB_OUTPUT"
echo "Android retrace: ${shards} shard(s) from ${matrix_format} at ${RETRACE_SHARD_SECONDS} s per shard"
# One cache entry holds every Git LFS fixture of the CI cases, and each retrace job restores
# that entry itself. This job only makes sure the entry exists. On a hit it downloads
# nothing. On a miss it starts from the newest older bundle, fetches from the mirror the
# cases whose pointers changed, saves the entry under the new key and confirms it is there,
# so that no retrace job has to download fixtures itself. The key, the path and the steps
# are .github/scripts/trace-fixture-bundle.sh's, and test.yml has the same job for the same
# entry: the two workflows of one push look the key up at the same time, and on a miss both
# fetch and both save. The save that loses is not an error, since the entry it wanted to
# write is the one the confirming step then finds. The timeout bounds a fixture download
# that stalls, which would otherwise hold every retrace job behind it.
trace-fixtures:
name: trace fixture (${{ matrix.case }})
name: trace fixtures
runs-on: ubuntu-26.04
needs: trace-cases
strategy:
fail-fast: false
max-parallel: 4
matrix:
case: ${{ fromJSON(needs.trace-cases.outputs.names) }}
timeout-minutes: 30
permissions:
actions: read
contents: read
outputs:
key: ${{ steps.bundle.outputs.key }}
path: ${{ steps.bundle.outputs.path }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Derive trace fixture cache key
id: fixture-key
run: bash .github/scripts/trace-fixture-cache.sh key '${{ matrix.case }}'
- name: Derive the fixture bundle key
id: bundle
run: bash .github/scripts/trace-fixture-bundle.sh key
- name: Restore trace fixture cache
id: fixture-cache
if: steps.fixture-key.outputs.cacheable == 'true'
- name: Look up the fixture bundle
id: lookup
uses: actions/cache/restore@v5
with:
path: ${{ steps.fixture-key.outputs.paths }}
key: ${{ steps.fixture-key.outputs.key }}
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
lookup-only: true
- name: Verify restored trace fixture
id: fixture-verify
if: steps.fixture-cache.outputs.cache-hit == 'true'
run: |
if bash .github/scripts/trace-fixture-cache.sh verify '${{ matrix.case }}'; then
echo "ok=true" >> "$GITHUB_OUTPUT"
else
echo "ok=false" >> "$GITHUB_OUTPUT"
echo "::warning::Cached fixture for ${{ matrix.case }} failed verification; falling back to the download path"
bash .github/scripts/trace-fixture-cache.sh reset '${{ matrix.case }}'
fi
- name: Restore the newest older bundle
if: steps.lookup.outputs.cache-hit != 'true'
uses: actions/cache/restore@v5
with:
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
restore-keys: ${{ steps.bundle.outputs.restore-prefix }}
- name: Fetch trace fixture
if: steps.fixture-verify.outputs.ok != 'true'
run: bash .github/scripts/fetch-trace-fixture-lfs.sh '${{ matrix.case }}'
- name: Complete the fixture bundle
if: steps.lookup.outputs.cache-hit != 'true'
run: bash .github/scripts/trace-fixture-bundle.sh ensure
- name: Save trace fixture cache
if: steps.fixture-key.outputs.cacheable == 'true' && steps.fixture-cache.outputs.cache-hit != 'true'
- name: Save the fixture bundle
if: steps.lookup.outputs.cache-hit != 'true'
continue-on-error: true
uses: actions/cache/save@v5
with:
path: ${{ steps.fixture-key.outputs.paths }}
key: ${{ steps.fixture-key.outputs.key }}
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
- name: Stage trace fixture
run: |
safe_case="$(printf '%s' '${{ matrix.case }}' | sed 's/[^A-Za-z0-9._-]/_/g')"
stage_dir="trace-fixtures/${safe_case}"
mkdir -p "${stage_dir}"
python3 tools/trace_replay/trace_cases.py --format fixture-files --case '${{ matrix.case }}' |
while IFS= read -r file; do
cp "${file}" "${stage_dir}/"
done
# The Test workflow of the same push may be the one whose save won. Its entry is as good,
# once it is listed.
- name: Wait for the fixture bundle entry
if: steps.lookup.outputs.cache-hit != 'true'
env:
GH_TOKEN: ${{ github.token }}
run: bash .github/scripts/trace-fixture-bundle.sh wait '${{ steps.bundle.outputs.key }}'
- name: Upload trace fixture
uses: actions/upload-artifact@v7
- name: Confirm the fixture bundle entry
if: steps.lookup.outputs.cache-hit != 'true'
uses: actions/cache/restore@v5
with:
name: MobileGL-trace-fixture-${{ matrix.case }}
path: trace-fixtures/**
if-no-files-found: error
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
lookup-only: true
fail-on-cache-miss: true
android-avd:
name: android avd image
@@ -347,8 +474,33 @@ jobs:
- name: Checkout repo
uses: actions/checkout@v6
# The emulator shard loop and the artifact cleanup, driven by fakes: what each case is
# run with, a verdict for every case, and what stays after a failure. It runs in this
# job because the retrace jobs wait for it and the APK build does not.
- name: Emulator shard runner regression
run: |
python3 android-plugin/test_trace_replay_shard.py
python3 android-plugin/test_retrace_artifact_cleanup.py
# This job only makes sure the cache entry exists; the retrace jobs restore it. A hit
# downloads nothing and skips every step below; a miss builds the AVD, and the post step
# of this action saves it.
- name: Look up Android AVD cache
id: android-avd-cache
uses: actions/cache@v5
with:
lookup-only: true
path: |
${{ env.ANDROID_AVD_HOME }}
${{ env.ANDROID_SDK_ROOT }}/emulator
${{ env.ANDROID_SDK_ROOT }}/platform-tools
${{ env.ANDROID_SDK_ROOT }}/platforms/android-35
${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64
key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v4-${{ hashFiles('android-plugin/run-avd-ci.sh') }}
- name: Setup Android SDK
id: netsetup336
if: steps.android-avd-cache.outputs.cache-hit != 'true'
continue-on-error: true
timeout-minutes: 15
uses: android-actions/setup-android@v4
@@ -367,20 +519,9 @@ jobs:
cmdline-tools-version: 13114758
- name: Accept Android SDK licenses
if: steps.android-avd-cache.outputs.cache-hit != 'true'
run: yes | sdkmanager --licenses >/dev/null
- name: Restore Android AVD cache
id: android-avd-cache
uses: actions/cache@v5
with:
path: |
${{ env.ANDROID_AVD_HOME }}
${{ env.ANDROID_SDK_ROOT }}/emulator
${{ env.ANDROID_SDK_ROOT }}/platform-tools
${{ env.ANDROID_SDK_ROOT }}/platforms/android-35
${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64
key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v4-${{ hashFiles('android-plugin/run-avd-ci.sh') }}
- name: Create AVD
if: steps.android-avd-cache.outputs.cache-hit != 'true'
run: |
@@ -418,16 +559,27 @@ jobs:
fi
du -sh "${ANDROID_AVD_HOME}"
# One job replays a shard: some of the cases of one lane, transport and backend, one after
# another on one emulator boot (android-plugin/trace_replay_shard.py). The shard's step
# summary has a row per case, and a case that did not pass has an error annotation of its own.
retrace:
name: retrace (${{ matrix.backend.name }} / ${{ matrix.transport || 'monolith' }}, ${{ matrix.case.name }})
name: retrace (${{ matrix.backend.name }} / ${{ matrix.transport || 'monolith' }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
runs-on: ubuntu-26.04
needs:
- build
- android-avd
- trace-cases
- trace-fixtures
if: ${{ !cancelled() && needs.build.result == 'success' && needs.android-avd.result == 'success' && needs.trace-cases.result == 'success' }}
timeout-minutes: 75
if: ${{ !cancelled() && needs.build.result == 'success' && needs.android-avd.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }}
# Paired with --deadline-seconds below, and far above what a shard takes when its replays
# end by themselves. Both come from the matrix. The deadline is the smaller of the time in
# which every case of the shard could run to the end of its own time budget
# (trace_replay_shard.py, needed_seconds) and RETRACE_SHARD_DEADLINE_CAP_SECONDS. With
# budgets of up to 1800 s a case the cap is what a shard of several cases gets: about
# three replays can hang to the end of their budgets before a later case is recorded as
# not-run. The minutes between the deadline and this limit are the setup before the
# replays and the uploads after them.
timeout-minutes: ${{ matrix.timeout_minutes }}
env:
MOBILEGL_CI_TRANSPORT: ${{ matrix.transport || 'monolith' }}
MOBILEGL_CI_LANE: ${{ matrix.lane || 'monolith-control' }}
@@ -435,38 +587,53 @@ jobs:
ANDROID_AVD_HOME: ${{ github.workspace }}/.android/avd
ANDROID_HOME: ${{ github.workspace }}/.android/sdk
ANDROID_SDK_ROOT: ${{ github.workspace }}/.android/sdk
# The shard's cases in run order, as the JSON array of case objects the shard runner
# takes: every field of the case in trace_cases.json, with the fixture paths in place.
MOBILEGL_SHARD_CASES: ${{ toJSON(matrix.cases) }}
# Set the repository variable to 1 to cold boot the emulator before every case of a
# shard, should one boot not survive a whole shard of software-rendered replays.
MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE: ${{ vars.MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE || '0' }}
# The emulator renders on the host with a software rasterizer that can keep compiled
# shaders on disk in the runner's home directory, where neither clearing the app's data
# nor restarting the emulator reaches. Off, so every case of a shard compiles its
# shaders as a job with a fresh machine does.
MESA_SHADER_CACHE_DISABLE: "true"
strategy:
fail-fast: false
# The account runs at most 20 jobs at once, shared with Test. These ~230 emulator jobs are
# the longer of the two runs, so they may take the whole pool: what is minimized is when the
# later of APK and Test finishes, not Test alone.
max-parallel: 20
# No max-parallel: the account's limit of 20 concurrent jobs, shared with Test, already
# bounds this matrix, and a cap below the number of shards only makes the round longer.
matrix: ${{ fromJSON(needs.trace-cases.outputs.android) }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
# First, so the steps that run whatever happened (diagnostics, stopping the emulator, the
# result upload) have these even when a step before the emulator launch failed.
- name: Set artifact metadata
run: |
echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
echo "EMULATOR_LOG=${RUNNER_TEMP}/mobilegl-emulator.log" >> "$GITHUB_ENV"
echo "EMULATOR_PID_FILE=${RUNNER_TEMP}/mobilegl-emulator.pid" >> "$GITHUB_ENV"
- name: Set swap space
uses: ./.github/actions/set-swap-space
with:
swap-size-gb: 8
- name: Download trace fixture
uses: actions/download-artifact@v8
# A miss does not fail the job: the entry may have been evicted since trace-fixtures
# confirmed it (a re-run of failed jobs on an older run does not run that job again).
- name: Restore trace fixtures
uses: actions/cache/restore@v5
with:
name: MobileGL-trace-fixture-${{ matrix.case.name }}
path: trace-fixture-download
path: ${{ needs.trace-fixtures.outputs.path }}
key: ${{ needs.trace-fixtures.outputs.key }}
- name: Install trace fixture
run: |
mkdir -p tools/trace_replay/fixtures
find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;
- name: Set artifact metadata
run: |
echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
echo "EMULATOR_LOG=${RUNNER_TEMP}/mobilegl-emulator.log" >> "$GITHUB_ENV"
echo "EMULATOR_PID_FILE=${RUNNER_TEMP}/mobilegl-emulator.pid" >> "$GITHUB_ENV"
# Moves this shard's fixtures into place and checks them against their Git LFS pointers.
# A case the restored bundle does not hold is downloaded here, with a warning annotation;
# the timeout bounds a download that stalls.
- name: Install trace fixtures
timeout-minutes: 15
run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
- name: Restore Android AVD cache
id: android-avd-cache
@@ -543,135 +710,44 @@ jobs:
--pid-file "${EMULATOR_PID_FILE}" \
--boot-timeout 300
# One emulator boot replays every case of the shard in order, each in a fresh app process
# with cleared app data and with the settings its case, backend and transport need. The
# runner keeps going after a failing case, on a cold boot so that case cannot touch the
# next one's verdict, retries a case once on a restarted emulator when the emulator
# itself failed, and exits non-zero when a case failed or a verdict is missing. The
# deadline stops it while the steps below can still upload what it has: the job timeout
# less the setup above and the uploads below. A case it stops or never starts is
# recorded as not-run, which is red.
- name: Retrace and validate
env:
MOBILEGL_ESPRYT_USE_ANGLE: ${{ matrix.backend.name == 'DirectGLES' && '1' || '0' }}
MOBILEGL_TRACE_ANGLE_VARIANT: ${{ matrix.case.name == 'minecraft-1.21.4-fabric-iris-bliss-in-world' && '90a62123d794' || 'ec889e6ea831' }}
MOBILEGL_MAGMA_R11G11B10F_FALLBACK: ${{ matrix.backend.name == 'DirectVulkan' && '1' || '0' }}
MOBILEGL_ITRP_FIX: ${{ matrix.case.name == 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' && 'ForceOn' || 'Auto' }}
# Lets the artifact cleanup tell which job a shard's verdicts came from.
MOBILEGL_SHARD_JOB_ID: ${{ job.check_run_id }}
run: |
apk_file="android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk"
test -f "${apk_file}"
extra_retrace_args=()
if [ "${MOBILEGL_CI_TRANSPORT}" = "inproc" ]; then
extra_retrace_args+=(
--env "MOBILEGL_TRANSPORT=inproc;MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1;MOBILEGL_GPU_HANG_BUDGET_MS=${MOBILEGL_GPU_HANG_BUDGET_MS}"
--require-inproc
)
elif [ "${MOBILEGL_CI_TRANSPORT}" = "spawn" ]; then
# THE SAME CLIENT KNOBS AS inproc, so the two acceptance lanes differ in exactly one
# thing: which PROCESS the server role runs in. They are scrubbed out of the CHILD's
# environment by the launcher and read by the client, which is where they belong.
#
# MOBILEGL_IPC_SERVER_PATH IS DELIBERATELY ABSENT. nativeLibraryDir carries an
# install-time hash, so nothing off the device can spell it; TraceReplayActivity
# resolves it from getApplicationInfo() when the env asks for spawn, and an explicit
# value would override that. --use-pbuffer is NOT optional here: a spawned
# server cannot be handed the client's window, because an ANativeWindow* is a pointer
# into the CLIENT's process and SetWindowHandle is refused on the way across with
# Fatal{UnmigratedSurface, "AndroidNativeWindow:use-MOBILEGL_IPC_SURFACE=server"} (Rule H, permanent;
# a windowed client uses MOBILEGL_IPC_SURFACE=server instead).
extra_retrace_args+=(
--env "MOBILEGL_TRANSPORT=spawn;MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1;MOBILEGL_GPU_HANG_BUDGET_MS=${MOBILEGL_GPU_HANG_BUDGET_MS}"
--use-pbuffer
--require-spawn
)
else
extra_retrace_args+=(--env "MOBILEGL_TRANSPORT=monolith;MOBILEGL_GPU_HANG_BUDGET_MS=${MOBILEGL_GPU_HANG_BUDGET_MS}")
fi
# Bliss needs the newer signed ANGLE variant plus sampler mipmap
# min-filter downgrading on ANGLE llvmpipe.
if [ "${{ matrix.backend.name }}" = "DirectGLES" ] && [ "${{ matrix.case.name }}" = "minecraft-1.21.4-fabric-iris-bliss-in-world" ]; then
extra_retrace_args+=(--avoid-angle-llvmpipe-sampler-mipmap-min-filter)
fi
if [ "${{ matrix.backend.name }}" = "DirectGLES" ] && [ "${{ matrix.case.avoid_angle_llvmpipe_explicit_lod_bias || false }}" = "true" ]; then
extra_retrace_args+=(--avoid-angle-llvmpipe-explicit-lod-bias)
fi
if [ "${{ matrix.case.coherent_as_flush || false }}" = "true" ]; then
extra_retrace_args+=(--coherent-as-flush)
fi
run_retrace() {
timeout "$(( ${{ matrix.case.timeout_seconds }} + 300 ))" sh android-plugin/trace-replay-ci.sh \
--apk-file "${apk_file}" \
--package top.mobilegl.plugin.trace \
--backend "${{ matrix.backend.name }}" \
--result-root android-retrace-result \
--fixture-root android-retrace-fixture \
--case "${{ matrix.case.name }}" \
--trace-archive "${{ matrix.case.trace_archive }}" \
--trace-file "${{ matrix.case.trace_file }}" \
--golden "${{ matrix.case.golden }}" \
--alternate-golden "${{ matrix.case.alternate_golden || '' }}" \
--target-call "${{ matrix.case.target_call }}" \
--width "${{ matrix.case.width }}" \
--height "${{ matrix.case.height }}" \
--ssim-threshold "${{ matrix.case.ssim_threshold || '0.99' }}" \
--crop-x "${{ matrix.case.crop_x }}" \
--crop-y "${{ matrix.case.crop_y }}" \
--crop-width "${{ matrix.case.crop_width }}" \
--crop-height "${{ matrix.case.crop_height }}" \
--timeout-seconds "${{ matrix.case.timeout_seconds }}" \
"${extra_retrace_args[@]}"
}
retrace_status=0
run_retrace || retrace_status=$?
if [ "${retrace_status}" -eq 75 ]; then
echo "::warning::Android emulator infrastructure failed; restarting it and retrying this retrace once."
# Surface-lost is retried rather than failed, so it would otherwise
# be invisible. Report it per job - a healthy run prints nothing and
# a rate spike shows up as a row per affected case.
reason_file="android-retrace-result/infrastructure-failure-reason.txt"
surface_lost_retries=0
if [ -f "${reason_file}" ]; then
surface_lost_retries="$(grep -c 'angle-surface-lost' "${reason_file}" || true)"
fi
if [ "${surface_lost_retries}" -gt 0 ]; then
echo "surface-lost retries: ${surface_lost_retries} (${{ matrix.backend.name }}, ${{ matrix.case.name }})" \
>> "${GITHUB_STEP_SUMMARY}"
fi
# The restart truncates EMULATOR_LOG, and the attempt that lost the
# emulator is the one worth reading - the retry usually only shows
# the wreckage. Keep the first attempt's log before it is clobbered.
if [ -f "${EMULATOR_LOG}" ]; then
cp "${EMULATOR_LOG}" "${EMULATOR_LOG}.first-attempt" || true
fi
sh android-plugin/run-avd-ci.sh stop \
--avd-name "${AVD_NAME}" \
--emulator-log "${EMULATOR_LOG}" \
--pid-file "${EMULATOR_PID_FILE}"
adb kill-server || true
sleep 2
sh android-plugin/run-avd-ci.sh start \
--avd-name "${AVD_NAME}" \
--gpu "${{ matrix.backend.gpu }}" \
--emulator-log "${EMULATOR_LOG}" \
--pid-file "${EMULATOR_PID_FILE}" \
--boot-timeout 300
run_retrace
elif [ "${retrace_status}" -ne 0 ]; then
exit "${retrace_status}"
fi
sh android-plugin/trace-replay-shard.sh \
--apk-file "android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk" \
--backend "${{ matrix.backend.name }}" \
--gpu "${{ matrix.backend.gpu }}" \
--transport "${MOBILEGL_CI_TRANSPORT}" \
--lane "${MOBILEGL_CI_LANE}" \
--cases-env MOBILEGL_SHARD_CASES \
--deadline-seconds "${{ matrix.deadline_seconds }}"
# Runs whatever happened above, without the device: every case of the shard gets its
# lane.json and golden copies for the summary, and a not-run verdict when the runner was
# killed or never started.
- name: Collect retrace summary inputs
if: always()
env:
MOBILEGL_SHARD_JOB_ID: ${{ job.check_run_id }}
run: |
safe_case="$(printf '%s' '${{ matrix.case.name }}' | sed 's/[^A-Za-z0-9._-]/_/g')"
result_dir="android-retrace-result/${safe_case}-${{ matrix.backend.name }}"
mkdir -p "${result_dir}"
python3 - "${result_dir}/lane.json" <<'PY'
import json, os, sys
from pathlib import Path
Path(sys.argv[1]).write_text(json.dumps({"lane": os.environ["MOBILEGL_CI_LANE"],
"transport": os.environ["MOBILEGL_CI_TRANSPORT"]}, indent=2))
PY
if [ -s "${{ matrix.case.golden }}" ]; then
cp "${{ matrix.case.golden }}" "${result_dir}/${safe_case}-${{ matrix.backend.name }}-golden.png"
fi
if [ -n "${{ matrix.case.alternate_golden || '' }}" ] && [ -s "${{ matrix.case.alternate_golden || '' }}" ]; then
cp "${{ matrix.case.alternate_golden || '' }}" "${result_dir}/${safe_case}-${{ matrix.backend.name }}-alternate-golden.png"
fi
sh android-plugin/trace-replay-shard.sh \
--apk-file "android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk" \
--backend "${{ matrix.backend.name }}" \
--gpu "${{ matrix.backend.gpu }}" \
--transport "${MOBILEGL_CI_TRANSPORT}" \
--lane "${MOBILEGL_CI_LANE}" \
--cases-env MOBILEGL_SHARD_CASES \
--finalize
- name: Collect emulator diagnostics
if: always()
@@ -679,12 +755,18 @@ jobs:
mkdir -p android-retrace-result/diagnostics
adb devices -l > android-retrace-result/diagnostics/adb-devices.txt || true
timeout 30 adb logcat -d -t 1000 > android-retrace-result/diagnostics/logcat.txt || true
# What the guest was given; replay time inside the emulator depends on it.
timeout 30 adb shell 'nproc; grep -E "MemTotal|SwapTotal" /proc/meminfo' \
> android-retrace-result/diagnostics/guest-resources.txt 2>&1 || true
cp "${ANDROID_AVD_HOME}/${AVD_NAME}.avd/config.ini" android-retrace-result/diagnostics/avd-config.ini 2>/dev/null || true
# Expected to list nothing: the host rasterizer's on-disk shader cache is disabled.
du -sh "${XDG_CACHE_HOME:-${HOME}/.cache}"/mesa_shader_cache* \
> android-retrace-result/diagnostics/host-shader-cache.txt 2>&1 || true
# The log of the last boot. The shard runner keeps the log of every boot it ended
# as diagnostics/emulator-boot-<n>-*.log before a restart truncates it.
if [ -f "${EMULATOR_LOG}" ]; then
cp "${EMULATOR_LOG}" android-retrace-result/diagnostics/emulator.log
fi
if [ -f "${EMULATOR_LOG}.first-attempt" ]; then
cp "${EMULATOR_LOG}.first-attempt" android-retrace-result/diagnostics/emulator-first-attempt.log
fi
# A vanished emulator looks identical whether the host OOM killer took
# qemu or the renderer faulted. These two say which.
free -h > android-retrace-result/diagnostics/host-memory.txt 2>&1 || true
@@ -704,9 +786,14 @@ jobs:
continue-on-error: true
uses: actions/upload-artifact@v7
with:
name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-${{ matrix.case.name }}
# The summary job downloads by the lane and transport at the front of this name, and
# its renderer reads the backend and the case from the paths below it, so the name
# ends in the shard number and holds no case name.
name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-shard${{ matrix.shard }}
path: android-retrace-result/**
if-no-files-found: warn
# A re-run of a failed job replaces the artifact its first attempt kept.
overwrite: true
# The artifact service sometimes times out (ETIMEDOUT on CreateArtifact, one job in 231 on
# 2026-10-18), which turned a passing case red; one delayed retry absorbs that.
@@ -718,7 +805,7 @@ jobs:
if: always() && steps.upload-retrace-result.outcome == 'failure'
uses: actions/upload-artifact@v7
with:
name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-${{ matrix.case.name }}
name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-shard${{ matrix.shard }}
path: android-retrace-result/**
if-no-files-found: warn
overwrite: true
@@ -727,44 +814,51 @@ jobs:
name: retrace summary
runs-on: ubuntu-26.04
needs: retrace
if: ${{ !cancelled() }}
# Also runs for a canceled run, where it only downloads the results and deletes the
# intermediate artifacts; the rendering steps are skipped then. The checkout and the
# downloads say so themselves, because a step with no condition is skipped once its job is
# canceled.
if: always()
permissions:
actions: write
contents: read
env:
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
steps:
- name: Checkout repo
if: ${{ success() || cancelled() }}
uses: actions/checkout@v6
- name: Set artifact metadata
run: |
echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
- name: Set up Node.js
if: ${{ !cancelled() }}
uses: actions/setup-node@v7
with:
node-version: '22'
- name: Download Android monolith control results
if: ${{ success() || cancelled() }}
uses: actions/download-artifact@v8
with:
pattern: MobileGL-android-retrace-result-monolith-control-monolith-*
path: retrace-artifacts/monolith-control
- name: Download Android inproc acceptance results
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
if: ${{ (success() || cancelled()) && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
uses: actions/download-artifact@v8
with:
pattern: MobileGL-android-retrace-result-inproc-acceptance-inproc-*
path: retrace-artifacts/inproc-acceptance
- name: Download Android spawn acceptance results
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
if: ${{ (success() || cancelled()) && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
uses: actions/download-artifact@v8
with:
pattern: MobileGL-android-retrace-result-spawn-acceptance-spawn-*
path: retrace-artifacts/spawn-acceptance
- name: Render monolith control summary
if: ${{ !cancelled() }}
run: |
node tools/trace_replay/render_retrace_summary.mjs \
--input retrace-artifacts/monolith-control \
@@ -774,7 +868,7 @@ jobs:
--html mobilegl-android-monolith-control.html
- name: Render inproc acceptance summary
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
run: |
node tools/trace_replay/render_retrace_summary.mjs \
--input retrace-artifacts/inproc-acceptance \
@@ -784,11 +878,17 @@ jobs:
--html mobilegl-android-inproc-acceptance.html
- name: Render spawn acceptance summary
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
run: |
node tools/trace_replay/render_retrace_summary.mjs --input retrace-artifacts/spawn-acceptance --output-dir android-retrace-summary/spawn-acceptance --title "MobileGL Android spawn acceptance" --group-label "Android Emulator / spawn acceptance" --html mobilegl-android-spawn-acceptance.html
node tools/trace_replay/render_retrace_summary.mjs \
--input retrace-artifacts/spawn-acceptance \
--output-dir android-retrace-summary/spawn-acceptance \
--title "MobileGL Android spawn acceptance" \
--group-label "Android Emulator / spawn acceptance" \
--html mobilegl-android-spawn-acceptance.html
- name: Upload Android monolith control summary
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v7
with:
name: MobileGL-android-monolith-control-summary-${{ github.sha }}
@@ -797,7 +897,7 @@ jobs:
if-no-files-found: error
- name: Upload Android inproc acceptance summary
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
uses: actions/upload-artifact@v7
with:
name: MobileGL-android-inproc-acceptance-summary-${{ github.sha }}
@@ -806,7 +906,7 @@ jobs:
if-no-files-found: error
- name: Upload Android spawn acceptance summary
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
uses: actions/upload-artifact@v7
with:
name: MobileGL-android-spawn-acceptance-summary-${{ github.sha }}
@@ -814,74 +914,23 @@ jobs:
archive: false
if-no-files-found: error
remove-artifact-clutter:
name: remove artifact clutter
runs-on: ubuntu-26.04
needs: retrace-summary
if: always()
permissions:
actions: write
steps:
# The summaries embed every image they show, so the per-shard results are intermediates
# from here on. This is the last step of the workflow rather than a job of its own, which
# would wait for a runner and start from an empty machine. It keeps what a failure needs,
# read from the shard-status.json of the results downloaded above: the results of every
# shard that ran a case which did not pass somewhere, on all backends and transports, and
# every result when a job failed without saying which cases it ran. Deletes are paced
# below GitHub's limit for mutating requests, and a refused delete is a warning. Skipped
# when a summary step failed, so that re-running this job still finds the results, and
# when a retrace job did not succeed in a run that was not canceled: every result stays
# then, so a shard stopped by its job limit keeps its diagnostics and a re-run of the
# failed jobs renders complete summaries.
- name: Delete intermediate Android retrace artifacts
if: ${{ (success() && needs.retrace.result == 'success') || cancelled() }}
env:
GH_TOKEN: ${{ github.token }}
run: |
# Fail before deletion if either paginated inventory cannot be read.
# 50 rows per page: the jobs endpoint answers HTTP 502 for 90+ rows on a re-run attempt of a run this size.
gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=50" \
--jq '.jobs[] | select(.name | startswith("retrace (")) | select(.conclusion == "failure" or .conclusion == "canceled" or .conclusion == "timed_out" or .conclusion == "action_required") | .name' \
> "${RUNNER_TEMP}/cleanup-jobs.txt"
gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \
--jq '.artifacts[] | select(.name | startswith("MobileGL-trace-fixture-") or startswith("MobileGL-android-retrace-result-") or startswith("trace-fixture-") or startswith("retrace-result-")) | [.id, .name] | @tsv' \
> "${RUNNER_TEMP}/cleanup-artifacts.txt"
declare -A failed_cases=()
while IFS= read -r job_name; do
case_name="${job_name#retrace (*, }"
case_name="${case_name%)}"
failed_cases["${case_name}"]=1
done < "${RUNNER_TEMP}/cleanup-jobs.txt"
if ((${#failed_cases[@]})); then
echo "Retaining fixtures and results for failed retrace case(s):"
printf ' %s\n' "${!failed_cases[@]}"
else
echo "All retrace jobs succeeded; nothing needs to be retained."
fi
deleted=0
retained=0
while IFS=$'\t' read -r artifact_id artifact_name; do
keep=0
if [[ "${artifact_name}" == MobileGL-trace-fixture-* ]]; then
case_name="${artifact_name#MobileGL-trace-fixture-}"
if [[ -v "failed_cases[${case_name}]" ]]; then
keep=1
fi
elif [[ "${artifact_name}" == MobileGL-android-retrace-result-* ]]; then
# The result artifact carries mobilegl.log, retrace.log, logcat,
# the emulator log and the actual/diff images - the only record of
# why a retrace failed. Its name ends in -<backend>-<case>, so a
# suffix match on the case name keeps both backends' results for a
# case that failed on either of them, which is what a comparison
# needs. The match is anchored at the end, so a case name that is a
# prefix of a longer one does not retain the longer one's results.
for case_name in "${!failed_cases[@]}"; do
if [[ "${artifact_name}" == *-"${case_name}" ]]; then
keep=1
break
fi
done
fi
if ((keep)); then
echo "Retaining ${artifact_name} (${artifact_id}) for failed retrace."
((retained += 1))
continue
fi
echo "Deleting ${artifact_name} (${artifact_id})"
gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}"
((deleted += 1))
done < "${RUNNER_TEMP}/cleanup-artifacts.txt"
echo "Deleted ${deleted} intermediate Android artifact(s); retained ${retained} failed-retrace fixture(s)."
python3 android-plugin/retrace_artifact_cleanup.py \
--downloads retrace-artifacts/monolith-control \
--downloads retrace-artifacts/inproc-acceptance \
--downloads retrace-artifacts/spawn-acceptance
.github/workflows/test.yml
+305 -435
View File
@@ -19,7 +19,7 @@ env:
MOBILEGL_GPU_HANG_BUDGET_MS: "0"
# ONE LIVE RUN PER BRANCH. The disaggregation branch pushes (often docs-only) commits minutes apart
# and each push fans out to ~330 retrace legs here and ~275 jobs in apk.yml; with no group, runs
# and each push fans out to about 30 replay jobs here and the emulator shards in apk.yml; with no group, runs
# sat queued for 10+ hours behind their own successors. A manual dispatch gets a group of its own,
# so a push never cancels a manually started run; dev queues rather
# than cancels. No paths-ignore here, unlike apk.yml: this workflow hosts the doc citation lint.
@@ -2668,12 +2668,15 @@ jobs:
runs-on: ubuntu-26.04
env:
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
# How many jobs replay one (lane, transport, backend) cell of the Linux retrace lanes. Each
# job replays its share of the cell's cases one after another; a cell with little to
# replay gets fewer jobs (tools/trace_replay/trace_cases.py, shard_matrix).
RETRACE_SHARDS_PER_CELL: "4"
outputs:
matrix: ${{ steps.trace-cases.outputs.matrix }}
names: ${{ steps.trace-cases.outputs.names }}
verify-matrix: ${{ steps.trace-cases.outputs.verify-matrix }}
split-matrix: ${{ steps.trace-cases.outputs.split-matrix }}
split-spawn-matrix: ${{ steps.trace-cases.outputs.split-spawn-matrix }}
shards: ${{ steps.trace-cases.outputs.shards }}
verify-shards: ${{ steps.trace-cases.outputs.verify-shards }}
split-shards: ${{ steps.trace-cases.outputs.split-shards }}
split-spawn-shards: ${{ steps.trace-cases.outputs.split-spawn-shards }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
@@ -2702,29 +2705,49 @@ jobs:
id: trace-cases
run: |
python3 tools/trace_replay/trace_cases.py --self-test-transport-matrices
matrix_format=github-test-matrix
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then matrix_format=github-test-transport-matrix; fi
echo "matrix=$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}")" >> "$GITHUB_OUTPUT"
echo "names=$(python3 tools/trace_replay/trace_cases.py --ci --format names)" >> "$GITHUB_OUTPUT"
# The subset the verify build retraces ("verify": true in trace_cases.json). It is a
# SUBSET of the matrix above, so retrace-verify needs no fixtures of its own.
echo "verify-matrix=$(python3 tools/trace_replay/trace_cases.py --ci --format github-verify-matrix)" >> "$GITHUB_OUTPUT"
# The split subset: {backend, case, transport} over inproc AND spawn, both backends, for
# every CI case that has not said `"split": false`. Still a SUBSET of the matrix above,
# so retrace-split needs no fixtures of its own. P5 shipped it as one case (OpenRA, which
# the phase gate names); P6 widened it to the whole CI matrix, because every bug the
# spawn work found was a question about the SERVER's process rather than about one trace.
# trace_cases.py refuses a `split` case that is not in CI, so the subset stays a subset.
SPLIT_MATRIX=$(python3 tools/trace_replay/trace_cases.py --ci --format github-split-matrix)
# AND IT MUST NOT BE EMPTY. An empty `include` is not an error to GitHub - it skips the
# whole retrace-split job with no red anywhere - so the one way this subset can vanish
# silently is guarded here. trace_cases.py now also rejects an unknown manifest key, which
# was the hole: `"splitt": true` loaded clean and emptied the subset (review N-4).
if [ "$(printf '%s' "${SPLIT_MATRIX}" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["include"]))')" -lt 1 ]; then
echo "::error::the split retrace subset is EMPTY. 'split' is an OPT-OUT as of P6 - a CI case is in this subset unless it carries \"split\": false - so an empty subset means either every case opted out or the CI matrix itself is empty. Either way retrace-split would be skipped with no red, and exit gate E2 names OpenRA."
# The retrace jobs replay shards: several cases of one (lane, transport, backend) cell
# per job. The first self-test proves every shard matrix replays exactly the rows of
# the per-case matrix it is built from; the second proves the shard runner gives each
# case the settings of its lane.
python3 tools/trace_replay/trace_cases.py --self-test-shard-matrices
python3 scripts/ci/retrace_shard.py --self-test
shards() {
python3 tools/trace_replay/trace_cases.py --ci --shards "${RETRACE_SHARDS_PER_CELL}" "$@"
}
shard_count() {
python3 -c 'import json,sys; print(len(json.load(sys.stdin)["include"]))'
}
shard_format=github-test-shard-matrix
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then shard_format=github-test-transport-shard-matrix; fi
# An empty `include` is not an error to GitHub: it skips the job with no red anywhere,
# so each matrix is checked before it is handed over.
require_shards() {
if [ "$(printf '%s' "$2" | shard_count)" -lt 1 ]; then
echo "::error::the $1 shard matrix is empty, so its job would be skipped with no red"
exit 1
fi
}
SHARDS=$(shards --format "${shard_format}")
require_shards retrace "${SHARDS}"
echo "shards=${SHARDS}" >> "$GITHUB_OUTPUT"
# The subset the verify build retraces ("verify": true in trace_cases.json).
VERIFY_SHARDS=$(shards --format github-verify-shard-matrix)
require_shards retrace-verify "${VERIFY_SHARDS}"
echo "verify-shards=${VERIFY_SHARDS}" >> "$GITHUB_OUTPUT"
# The split subset: {backend, case, transport} over inproc and spawn, both backends, for
# every CI case that has not said `"split": false`. It is the whole CI matrix because
# the bugs the spawn arm finds are about the server's process rather than about one
# trace. trace_cases.py refuses a `split` case that is not in CI.
SPLIT_SHARDS=$(shards --format github-split-shard-matrix)
# It must not be empty. An empty `include` is not an error to GitHub: it skips the
# whole retrace-split job with no red anywhere, so that is guarded here.
# trace_cases.py also rejects an unknown manifest key, since a mistyped `split` key
# would otherwise load clean and change the subset.
if [ "$(printf '%s' "${SPLIT_SHARDS}" | shard_count)" -lt 1 ]; then
echo "::error::the split retrace subset is empty. 'split' is an opt-out: a CI case is in this subset unless it carries \"split\": false, so an empty subset means either every case opted out or the CI matrix itself is empty. Either way retrace-split would be skipped with no red."
exit 1
fi
echo "split-matrix=${SPLIT_MATRIX}" >> "$GITHUB_OUTPUT"
echo "split-shards=${SPLIT_SHARDS}" >> "$GITHUB_OUTPUT"
# On the disaggregation branches, the same subset without the inproc rows the
# inproc-acceptance retrace rows replay on the same library, which retrace-split uses when
# it reuses build-linux's library. OpenRA keeps its inproc rows: they carry the
@@ -2733,101 +2756,110 @@ jobs:
# is not written, and retrace-split takes the full subset, as it does elsewhere.
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then
if python3 tools/trace_replay/trace_cases.py --self-test-split-matrix \
&& SPLIT_SPAWN_MATRIX=$(python3 tools/trace_replay/trace_cases.py --ci --format github-split-matrix \
&& SPLIT_SPAWN_SHARDS=$(shards --format github-split-shard-matrix \
--transports spawn --keep-inproc OpenRA) \
&& [ "$(printf '%s' "${SPLIT_SPAWN_MATRIX}" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["include"]))' || echo 0)" -ge 1 ]; then
echo "split-spawn-matrix=${SPLIT_SPAWN_MATRIX}" >> "$GITHUB_OUTPUT"
&& [ "$(printf '%s' "${SPLIT_SPAWN_SHARDS}" | shard_count || echo 0)" -ge 1 ]; then
echo "split-spawn-shards=${SPLIT_SPAWN_SHARDS}" >> "$GITHUB_OUTPUT"
else
echo "::warning::the spawn-only split retrace subset failed its self-test or is empty, so retrace-split replays the full split subset"
fi
fi
echo "retrace shards: $(printf '%s' "${SHARDS}" | shard_count) retrace, $(printf '%s' "${VERIFY_SHARDS}" | shard_count) verify, $(printf '%s' "${SPLIT_SHARDS}" | shard_count) split (full subset)"
# One cache entry holds every Git LFS fixture of the CI cases, and each replay job restores
# that entry itself. This job only makes sure the entry exists. On a hit it downloads
# nothing. On a miss it starts from the newest older bundle, fetches from the mirror the
# cases whose pointers changed, saves the entry under the new key and confirms it is there,
# so that no replay job has to download fixtures itself. The key, the path and the steps are
# .github/scripts/trace-fixture-bundle.sh's; apk.yml uses the same entry. The timeout bounds
# a fixture download that stalls, which would otherwise hold every replay job behind it.
trace-fixtures:
name: trace fixture (${{ matrix.case }})
name: trace fixtures
runs-on: ubuntu-26.04
needs: [trace-cases]
strategy:
fail-fast: false
max-parallel: 4
matrix:
case: ${{ fromJSON(needs.trace-cases.outputs.names) }}
timeout-minutes: 30
permissions:
actions: read
contents: read
outputs:
key: ${{ steps.bundle.outputs.key }}
path: ${{ steps.bundle.outputs.path }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Derive the fixture bundle key
id: bundle
run: bash .github/scripts/trace-fixture-bundle.sh key
- name: Derive trace fixture cache key
id: fixture-key
run: bash .github/scripts/trace-fixture-cache.sh key '${{ matrix.case }}'
- name: Restore trace fixture cache
id: fixture-cache
if: steps.fixture-key.outputs.cacheable == 'true'
- name: Look up the fixture bundle
id: lookup
uses: actions/cache/restore@v5
with:
path: ${{ steps.fixture-key.outputs.paths }}
key: ${{ steps.fixture-key.outputs.key }}
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
lookup-only: true
- name: Verify restored trace fixture
id: fixture-verify
if: steps.fixture-cache.outputs.cache-hit == 'true'
run: |
if bash .github/scripts/trace-fixture-cache.sh verify '${{ matrix.case }}'; then
echo "ok=true" >> "$GITHUB_OUTPUT"
else
echo "ok=false" >> "$GITHUB_OUTPUT"
echo "::warning::Cached fixture for ${{ matrix.case }} failed verification; falling back to the download path"
bash .github/scripts/trace-fixture-cache.sh reset '${{ matrix.case }}'
fi
- name: Restore the newest older bundle
if: steps.lookup.outputs.cache-hit != 'true'
uses: actions/cache/restore@v5
with:
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
restore-keys: ${{ steps.bundle.outputs.restore-prefix }}
- name: Fetch trace fixture
if: steps.fixture-verify.outputs.ok != 'true'
run: bash .github/scripts/fetch-trace-fixture-lfs.sh '${{ matrix.case }}'
- name: Complete the fixture bundle
if: steps.lookup.outputs.cache-hit != 'true'
run: bash .github/scripts/trace-fixture-bundle.sh ensure
- name: Save trace fixture cache
if: steps.fixture-key.outputs.cacheable == 'true' && steps.fixture-cache.outputs.cache-hit != 'true'
# The APK workflow of the same push saves the same entry, and only one save can win.
# The steps below find the entry whichever workflow wrote it.
- name: Save the fixture bundle
if: steps.lookup.outputs.cache-hit != 'true'
continue-on-error: true
uses: actions/cache/save@v5
with:
path: ${{ steps.fixture-key.outputs.paths }}
key: ${{ steps.fixture-key.outputs.key }}
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
- name: Stage trace fixture
run: |
safe_case="$(printf '%s' '${{ matrix.case }}' | sed 's/[^A-Za-z0-9._-]/_/g')"
stage_dir="trace-fixtures/${safe_case}"
mkdir -p "${stage_dir}"
python3 tools/trace_replay/trace_cases.py --format fixture-files --case '${{ matrix.case }}' |
while IFS= read -r file; do
cp "${file}" "${stage_dir}/"
done
# The APK workflow of the same push may be the one whose save won. Its entry is as good,
# once it is listed.
- name: Wait for the fixture bundle entry
if: steps.lookup.outputs.cache-hit != 'true'
env:
GH_TOKEN: ${{ github.token }}
run: bash .github/scripts/trace-fixture-bundle.sh wait '${{ steps.bundle.outputs.key }}'
- name: Upload trace fixture
uses: actions/upload-artifact@v7
- name: Confirm the fixture bundle entry
if: steps.lookup.outputs.cache-hit != 'true'
uses: actions/cache/restore@v5
with:
name: trace-fixture-${{ matrix.case }}
path: trace-fixtures/**
if-no-files-found: error
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
lookup-only: true
fail-on-cache-miss: true
# One job replays a shard: some of the cases of one lane, transport and backend, one after
# another, each through its own ctest entry (scripts/ci/retrace_shard.py). The shard's step
# summary has a row per case, and a failed case has an error annotation of its own.
retrace:
name: retrace (${{ matrix.backend }} / ${{ matrix.transport || 'monolith' }}, ${{ matrix.case }})
name: retrace (${{ matrix.backend }} / ${{ matrix.transport || 'monolith' }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
runs-on: ubuntu-26.04
timeout-minutes: 240
env:
MOBILEGL_TRANSPORT: ${{ matrix.transport || 'monolith' }}
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
MOBILEGL_IPC_RUN_AHEAD: "1"
SHARD_NAMES: ${{ toJSON(matrix.names) }}
needs:
- build-linux
- build-retrace
- trace-cases
- trace-fixtures
if: ${{ !cancelled() && needs.build-linux.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }}
if: ${{ !cancelled() && needs.build-linux.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }}
strategy:
fail-fast: false
max-parallel: 8
matrix: ${{ fromJSON(needs.trace-cases.outputs.matrix) }}
matrix: ${{ fromJSON(needs.trace-cases.outputs.shards) }}
steps:
- name: Checkout repo
@@ -2844,16 +2876,20 @@ jobs:
with:
gpu: "true"
- name: Download trace fixture
uses: actions/download-artifact@v8
# A miss does not fail the job: the entry may have been evicted since trace-fixtures
# confirmed it (a re-run of failed jobs on an older run does not run that job again).
- name: Restore trace fixtures
uses: actions/cache/restore@v5
with:
name: trace-fixture-${{ matrix.case }}
path: trace-fixture-download
path: ${{ needs.trace-fixtures.outputs.path }}
key: ${{ needs.trace-fixtures.outputs.key }}
- name: Install trace fixture
run: |
mkdir -p tools/trace_replay/fixtures
find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;
# Moves this shard's fixtures into place and checks them against their Git LFS pointers.
# A case the restored bundle does not hold is downloaded here, with a warning annotation;
# the timeout bounds a download that stalls.
- name: Install trace fixtures
timeout-minutes: 15
run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
- name: Get CMake
timeout-minutes: 15
@@ -2895,93 +2931,66 @@ jobs:
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
# The per-case settings are retrace_shard.py's "retrace" profile: the Magma fallback and
# the two per-case repair knobs on DirectVulkan, and on the inproc lane the IPC
# configuration the client must log, the split evidence check and the refusal census from
# the library's own logs. A failing case does not stop the shard; the step fails at the
# end if a case failed or a case has no verdict.
- name: Retrace and validate
working-directory: build-retrace/tools/trace_replay
run: |
ulimit -c unlimited
if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then
export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1
fi
if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
&& [ '${{ matrix.case }}' = 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' ]; then
export MOBILEGL_ITRP_FIX=ForceOn
fi
# The blended depth-write quirk auto-enables only on Qualcomm, which no CI
# runner has, so force it on for the OIT case it exists to fix. ForceOn
# bypasses only the vendor gate, so this exercises the real strip on
# lavapipe. The Android AVD lane deliberately leaves it off, keeping the
# unstripped path covered for the same trace.
if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
&& [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then
export MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1
fi
# The inproc rows stand in for retrace-split's dropped inproc rows, so they prove the
# configuration they run with from the client's own log.
if [ "${MOBILEGL_TRANSPORT}" = inproc ]; then
export MGTRACE_EXPECT_IPC='strict=1 role-split-state=1 run-ahead=1'
fi
ctest -V --no-tests=error -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$' --output-junit "${RUNNER_TEMP}/retrace.xml"
# A split row's transport assertions ran on this leg and found their evidence.
if [ "${MOBILEGL_TRANSPORT}" != monolith ]; then
python3 "${GITHUB_WORKSPACE}/scripts/ci/retrace_split_evidence.py" \
'${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt' \
--transport "${MOBILEGL_TRANSPORT}" --case '${{ matrix.case }}' --backend '${{ matrix.backend }}'
fi
# The refusal census, recorded rather than gated. run_trace_case.cmake already REDS the case
# on any Fatal{, so reaching here means the count is zero - but the number and the distinct
# slot names are what MEASUREMENTS wants from every split run, and reading them out of the
# library's own log is the only way to get them (ctest -V's transcript is a false zero).
- name: Refusal census from the library log
if: ${{ always() && matrix.transport == 'inproc' }}
working-directory: build-retrace/tools/trace_replay
run: |
# BOTH ROLES, because under spawn every applier refusal is raised in the SERVER's
# process and a census reading only the client's file reports a confident zero for the
# half it cannot see.
log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.client.log"
server_log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.server.log"
cat "${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt" 2>/dev/null || echo "no split-evidence.txt"
if [ ! -f "${log}" ]; then
echo "no ${log} - the replay wrote no library log"
exit 0
fi
census_files="${log}"
if [ -f "${server_log}" ]; then census_files="${census_files} ${server_log}"; fi
echo "census over: ${census_files}"
echo "Fatal{ lines: $(cat ${census_files} | grep -c 'Fatal{' || true)"
cat ${census_files} | grep -o 'Fatal{[A-Za-z]*, "[^"]*"' | sort | uniq -c | sort -rn | head -20 || true
# The transport line is the CLIENT's: ConfigLoader resolves it there, and under spawn
# the server states its own separately.
grep -m1 "MOBILEGL_TRANSPORT=" "${log}" || echo "no transport line in ${log}"
python3 scripts/ci/retrace_shard.py run --profile retrace \
--backend '${{ matrix.backend }}' --transport "${MOBILEGL_TRANSPORT}" \
--lane '${{ matrix.lane || 'monolith-control' }}' \
--shard '${{ matrix.shard }}/${{ matrix.shards }}' \
--build-dir build-retrace/tools/trace_replay \
--result-dir "${RUNNER_TEMP}/retrace-shard"
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py --excerpts 1 "${RUNNER_TEMP}"
run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit"
- name: Upload core dumps
if: failure()
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-core-dumps-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-${{ matrix.case }}
name: retrace-core-dumps-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }}
path: /tmp/core.*
if-no-files-found: ignore
- name: Upload actual image
# What retrace-summary renders: per case, the images and the replay's output directory.
# The summary embeds the images, so these are short-lived; a failed shard keeps its own
# copy below.
- name: Upload shard results
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-${{ matrix.case }}
path: |
build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/**
build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/**
name: retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }}
path: ${{ runner.temp }}/retrace-shard/results
if-no-files-found: warn
retention-days: 1
overwrite: true
- name: Upload failed shard results
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-failed-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }}
path: |
${{ runner.temp }}/retrace-shard/results
${{ runner.temp }}/retrace-shard/junit
${{ runner.temp }}/retrace-shard/logs
if-no-files-found: warn
overwrite: true
retrace-summary:
name: retrace summary
runs-on: ubuntu-26.04
needs: [retrace]
if: ${{ !cancelled() && needs.retrace.result != 'skipped' }}
# actions: write is for the last step, which deletes the artifacts this job rendered.
permissions:
actions: write
contents: read
env:
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
@@ -3050,25 +3059,48 @@ jobs:
archive: false
if-no-files-found: error
# The summaries embed every image, so the shard result artifacts they were rendered from
# have no reader left and are deleted here. That happens only when every shard passed: a
# failed shard has its own retrace-failed-* artifact, and after a failure the results
# stay for their one day of retention, so that re-running the failed shards renders every
# case again. A failed delete only leaves them to that retention.
- name: Delete the rendered shard results
if: ${{ needs.retrace.result == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
run: |
gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \
--jq '.artifacts[] | select(.name | startswith("retrace-result-")) | [.id, .name] | @tsv' \
> "${RUNNER_TEMP}/rendered-artifacts.txt"
deleted=0
while IFS=$'\t' read -r artifact_id artifact_name; do
echo "Deleting ${artifact_name} (${artifact_id})"
gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}"
deleted=$((deleted + 1))
done < "${RUNNER_TEMP}/rendered-artifacts.txt"
echo "Deleted ${deleted} rendered shard result artifact(s)."
# The trace half of the third CI mode. Same replay, same goldens, but the library underneath is
# the verify build and MOBILEGL_PIPE_VERIFY=1 is in the environment, so every backend read of
# frontend state is checked against a snapshot taken at the verb boundary. Eight cases rather
# than the full lane's 40 (tools/trace_replay/trace_cases.json, "verify": true): the comparator
# is budgeted at 5-10x, and the full sweep is a phase-exit / workflow_dispatch run.
retrace-verify:
name: retrace verify (${{ matrix.backend }}, ${{ matrix.case }})
name: retrace verify (${{ matrix.backend }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
runs-on: ubuntu-26.04
timeout-minutes: 240
env:
SHARD_NAMES: ${{ toJSON(matrix.names) }}
needs:
- build-linux-verify
- build-retrace
- trace-cases
- trace-fixtures
if: ${{ !cancelled() && needs.build-linux-verify.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }}
if: ${{ !cancelled() && needs.build-linux-verify.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }}
strategy:
fail-fast: false
max-parallel: 4
matrix: ${{ fromJSON(needs.trace-cases.outputs.verify-matrix) }}
matrix: ${{ fromJSON(needs.trace-cases.outputs.verify-shards) }}
steps:
- name: Checkout repo
@@ -3085,16 +3117,16 @@ jobs:
with:
gpu: "true"
- name: Download trace fixture
uses: actions/download-artifact@v8
# A miss does not fail the job; the install step downloads what the bundle does not hold.
- name: Restore trace fixtures
uses: actions/cache/restore@v5
with:
name: trace-fixture-${{ matrix.case }}
path: trace-fixture-download
path: ${{ needs.trace-fixtures.outputs.path }}
key: ${{ needs.trace-fixtures.outputs.key }}
- name: Install trace fixture
run: |
mkdir -p tools/trace_replay/fixtures
find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;
- name: Install trace fixtures
timeout-minutes: 15
run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
- name: Get CMake
timeout-minutes: 15
@@ -3151,82 +3183,60 @@ jobs:
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
- name: Retrace and validate (verify)
working-directory: build-retrace/tools/trace_replay
# run_trace_case.cmake turns MOBILEGL_PIPE_VERIFY into three assertions of its own (the
# arming line, no Fatal{PipeVerifyDiffer, no Fatal{UnmigratedPipeInput), so a case that
# somehow ran the wrong library reds here instead of passing on its golden.
# --timeout 10800: the 1800s cases run 5-10x slower with both comparator arms live, which
# is well past ctest's 1500s default.
run: |
ulimit -c unlimited
export MOBILEGL_PIPE_VERIFY=1
if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then
export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1
fi
if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
&& [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then
export MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1
fi
ctest -V --no-tests=error --timeout 10800 \
-R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$'
# The retrace lane's own always-on negative control, on one case so it costs one short trace:
# with a snapshot field corrupted, the SAME replay must fail. Without it, "40 traces, zero
# divergences" would be a statement about a comparator nobody watched.
# Every case replays with MOBILEGL_PIPE_VERIFY=1 (retrace_shard.py's "verify" profile).
# run_trace_case.cmake turns that into three assertions of its own (the arming line, no
# Fatal{PipeVerifyDiffer, no Fatal{UnmigratedPipeInput), so a case that somehow ran the
# wrong library reds here instead of passing on its golden. ctest's timeout is raised to
# 10800 s: the long cases run 5-10x slower with both comparator arms live, well past the
# 1500 s default.
#
# The rerun replays into the SAME case directory, so the verified run's images are put aside
# first and restored before the verdict: "Upload actual image" below runs `if: always()` and
# would otherwise ship the deliberately corrupted run's output under the name of the good one.
# The restore happens whichever way the control goes, which is why the ctest exit status is
# captured rather than tested inline.
- name: Negative control (corrupted snapshot)
if: ${{ matrix.case == 'OpenRA' && matrix.backend == 'DirectGLES' }}
working-directory: build-retrace/tools/trace_replay
# The shard that holds OpenRA on DirectGLES also runs the lane's negative control right
# after that case: with a snapshot field corrupted, the same replay must fail, and for the
# comparator's reason (scripts/ci/retrace_verify_reason.py). Without it, zero divergences
# would be a statement about a comparator nobody watched. The control has a verdict row of
# its own, and its output is discarded so that only the verified run is uploaded.
- name: Retrace and validate (verify)
run: |
export MOBILEGL_PIPE_VERIFY=1
export MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters
GOOD_OUTPUT="${RUNNER_TEMP}/openra-verified-output"
rm -rf "${GOOD_OUTPUT}"
if [ -d OpenRA ]; then
cp -a OpenRA "${GOOD_OUTPUT}"
fi
set +e
ctest -V --no-tests=error --timeout 10800 \
-R '^MobileGLTraceReplay\.OpenRA\.DirectGLES$' 2>&1 | tee "${RUNNER_TEMP}/verify-control.txt"
control_rc=${PIPESTATUS[0]}
set -e
if [ -d "${GOOD_OUTPUT}" ]; then
rm -rf OpenRA
mv "${GOOD_OUTPUT}" OpenRA
echo "restored the verified run's OpenRA output over the corrupted rerun's"
fi
if [ "${control_rc}" -eq 0 ]; then
echo "::error::MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left the OpenRA retrace GREEN, so the comparator is not comparing and the whole verify retrace lane proves nothing."
exit 1
fi
# Its red must carry the comparator's divergence on the corrupted field
# (scripts/ci/retrace_verify_reason.py, which test_run_trace_case.py runs on canned output).
python3 "${GITHUB_WORKSPACE}/scripts/ci/retrace_verify_reason.py" "${RUNNER_TEMP}/verify-control.txt"
echo "the corrupted field turned the retrace red, as it must (ctest exit ${control_rc})"
python3 scripts/ci/retrace_shard.py run --profile verify \
--backend '${{ matrix.backend }}' --lane verify \
--shard '${{ matrix.shard }}/${{ matrix.shards }}' \
--build-dir build-retrace/tools/trace_replay \
--result-dir "${RUNNER_TEMP}/retrace-shard"
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit"
- name: Upload core dumps
if: failure()
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-verify-core-dumps-${{ matrix.backend }}-${{ matrix.case }}
name: retrace-verify-core-dumps-${{ matrix.backend }}-shard${{ matrix.shard }}
path: /tmp/core.*
if-no-files-found: ignore
- name: Upload actual image
# Kept for every shard, green or not: per case, the verified images and the replay's
# output directory, whose library log has the comparator's arming line.
- name: Upload shard results
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: retrace-verify-result-${{ matrix.backend }}-${{ matrix.case }}
path: |
build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/**
build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/**
name: retrace-verify-result-${{ matrix.backend }}-shard${{ matrix.shard }}
path: ${{ runner.temp }}/retrace-shard/results
if-no-files-found: warn
overwrite: true
- name: Upload failed shard results
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-verify-failed-${{ matrix.backend }}-shard${{ matrix.shard }}
path: |
${{ runner.temp }}/retrace-shard/results
${{ runner.temp }}/retrace-shard/junit
${{ runner.temp }}/retrace-shard/logs
if-no-files-found: warn
overwrite: true
# Trace retrace on the split library: the retrace-verify shape with MOBILEGL_TRANSPORT set
# instead of MOBILEGL_PIPE_VERIFY, over {backend, case, transport} with inproc and spawn.
@@ -3237,7 +3247,7 @@ jobs:
# guards are the nm check for MG_Remote, run_trace_case.cmake's transport assertions (and, on
# OpenRA, its wire-volume assertion), and the per-leg check of the evidence file they leave.
retrace-split:
name: retrace split (${{ matrix.backend }} / ${{ matrix.transport }}, ${{ matrix.case }})
name: retrace split (${{ matrix.backend }} / ${{ matrix.transport }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
runs-on: ubuntu-26.04
timeout-minutes: 240
# The setup action selects lavapipe on GitHub-hosted runners.
@@ -3248,19 +3258,20 @@ jobs:
- build-retrace
- trace-cases
- trace-fixtures
if: ${{ !cancelled() && needs.build-linux-split.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && (needs.build-guard.outputs.verdict != 'identical' || needs.build-linux.result == 'success') }}
if: ${{ !cancelled() && needs.build-linux-split.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' && (needs.build-guard.outputs.verdict != 'identical' || needs.build-linux.result == 'success') }}
env:
SPLIT_REUSE: ${{ needs.build-guard.outputs.verdict == 'identical' && 'true' || 'false' }}
SPLIT_LIB: ${{ needs.build-guard.outputs.verdict == 'identical' && 'mobilegl-linux-lib' || 'mobilegl-linux-lib-split' }}
SPLIT_LIB_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }}
SPLIT_BUILD_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }}
SHARD_NAMES: ${{ toJSON(matrix.names) }}
strategy:
fail-fast: false
max-parallel: 8
# With build-linux reused, the inproc rows (except OpenRA's) are replayed by the retrace job's
# inproc-acceptance rows on the same library, strict and with the dual block and run-ahead
# on; the spawn rows below prove per leg that those are also the defaults they run with.
matrix: ${{ fromJSON(needs.build-guard.outputs.verdict == 'identical' && needs.trace-cases.outputs.split-spawn-matrix || needs.trace-cases.outputs.split-matrix) }}
# on; the spawn rows below prove per case that those are also the defaults they run with.
# A shard holds one transport, so OpenRA's inproc rows are shards of their own.
matrix: ${{ fromJSON(needs.build-guard.outputs.verdict == 'identical' && needs.trace-cases.outputs.split-spawn-shards || needs.trace-cases.outputs.split-shards) }}
steps:
- name: Checkout repo
@@ -3288,16 +3299,16 @@ jobs:
with:
gpu: "true"
- name: Download trace fixture
uses: actions/download-artifact@v8
# A miss does not fail the job; the install step downloads what the bundle does not hold.
- name: Restore trace fixtures
uses: actions/cache/restore@v5
with:
name: trace-fixture-${{ matrix.case }}
path: trace-fixture-download
path: ${{ needs.trace-fixtures.outputs.path }}
key: ${{ needs.trace-fixtures.outputs.key }}
- name: Install trace fixture
run: |
mkdir -p tools/trace_replay/fixtures
find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;
- name: Install trace fixtures
timeout-minutes: 15
run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
- name: Get CMake
timeout-minutes: 15
@@ -3351,12 +3362,11 @@ jobs:
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
# THE SPAWN ARM NEEDS A SECOND BINARY and the inproc arm needs none, which is the whole
# difference between the two at this level: inproc's server role is a thread in the replay
# process, spawn's is a PROCESS the client launches from this image. It ships in
# the split retrace library - libMobileGLServer.so is an executable that wears a
# .so name so an APK will install it somewhere exec-able - and a missing one must be a NAMED
# failure here rather than a LaunchServer refusal 78 jobs deep.
# The spawn arm needs a second binary and the inproc arm needs none: inproc's server role
# is a thread in the replay process, spawn's is a process the client launches from this
# image. It ships in the split retrace library (libMobileGLServer.so is an executable
# with a .so name, so that an APK installs it somewhere it can be executed), and a missing
# one must be a named failure here rather than a LaunchServer refusal in every case.
- name: Place the server image (spawn arm only)
if: ${{ matrix.transport == 'spawn' }}
run: |
@@ -3370,114 +3380,54 @@ jobs:
chmod +x build-linux/libMobileGLServer.so
echo "server image: $(readlink -f build-linux/libMobileGLServer.so)"
# The Magma knobs go to $GITHUB_ENV rather than to one step's shell, so every later step of
# this leg runs the same repair set as the replay.
- name: Magma repair knobs (all later steps)
run: |
# P7 package L: THE SAME MAGMA KNOBS THE MONOLITH RETRACE JOB EXPORTS, ALL OF THEM.
# This job exported none at all, so every DirectVulkan row of the split matrix was
# compared against a monolith row running a different repair set - and a split retrace
# whose control is not comparable cannot say whether a divergence is a transport fact.
# R11F_G11F_B10F fallback is whole-backend; the blended depth-write quirk auto-enables
# only on Qualcomm, which no runner has, so the OIT case forces it exactly as the
# monolith job does (ForceOn bypasses only the vendor gate).
if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then
echo "MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1" >> "${GITHUB_ENV}"
fi
if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
&& [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then
echo "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1" >> "${GITHUB_ENV}"
fi
# THE THREE ITERATIONRP KNOBS, FOR THE ONE CASE THE MONOLITH JOB NAMES.
# The iterationRP repairs are tri-state quirks that default to device auto-detection and
# lavapipe is on no auto list, so without them Program 203 runs unrepaired and misses its
# golden output. The monolith job has exported them since they landed; this job never
# did, so `iterationrp x DirectVulkan` was the one cell of the split matrix whose red
# could not be read as a transport fact - the monolith control it is compared against was
# running a different repair set. Exit gate 1 asks for the two halves to be COMPARABLE,
# and two runs that differ in a quirk knob are not.
if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
&& [ '${{ matrix.case }}' = 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' ]; then
echo "MOBILEGL_ITRP_FIX=ForceOn" >> "${GITHUB_ENV}"
fi
# The per-case settings are retrace_shard.py's "split" profile:
# - the Magma knobs the retrace job's monolith rows run with (the R11F_G11F_B10F fallback,
# the blended depth-write quirk on its one case, the iterationRP repairs on theirs), so a
# split row and its monolith control differ in the transport alone;
# - MOBILEGL_TRANSPORT and the absolute server image path;
# - on OpenRA, the wire-volume counters and the check that draw records crossed the wire;
# - with build-linux reused, the IPC defaults the client must log (strict=0).
# run_trace_case.cmake turns MOBILEGL_TRANSPORT into assertions of its own: the library must
# have resolved the transport in this process and its log must carry no Fatal{, so a case
# that somehow ran the wrong library reds here instead of passing on its golden. After
# ctest the evidence file those assertions leave is checked for every case, and the
# refusal census is printed from the library's own logs of both roles, which is the only
# valid source: the console sink is compiled out of this configuration.
- name: Retrace and validate (${{ matrix.transport }})
working-directory: build-retrace/tools/trace_replay
# run_trace_case.cmake turns MOBILEGL_TRANSPORT into assertions of its own - the library
# must have RESOLVED the transport in this process, and its log must carry no Fatal{ - so a
# case that somehow ran the wrong library reds here instead of passing on its golden. That
# log is also the only valid refusal census: the console sink is compiled out of this
# configuration, so a `ctest -V` transcript reports a false zero for Fatal{ lines.
run: |
ulimit -c unlimited
export MOBILEGL_TRANSPORT='${{ matrix.transport }}'
# Absolute, because the replay's working directory is not this one and LaunchServer
# resolves the image relative to nothing. Harmless on the inproc arm, which reads it
# only to decide it does not need it.
export MOBILEGL_IPC_SERVER_PATH="${GITHUB_WORKSPACE}/build-linux/libMobileGLServer.so"
# OpenRA's legs also count the wire's draw records and the server's draws; that turns on
# the library's per-frame counters, so the other cases keep the plain configuration.
wire=()
if [ '${{ matrix.case }}' = 'OpenRA' ]; then
export MGTRACE_WIRE_EVIDENCE=1
wire=(--require-wire)
fi
if [ "${SPLIT_REUSE}" = "true" ]; then
export MGTRACE_EXPECT_IPC='strict=0 role-split-state=1 run-ahead=1'
export MGTRACE_EXPECT_IPC_REASON="retrace-split dropped its inproc rows because the inproc-acceptance retrace rows replay the same library with strict=1 role-split-state=1 run-ahead=1, which differs from this leg's defaults only in strict while these hold. A default changed: restore the inproc rows (the trace-cases job's split-spawn-matrix without --transports spawn) or update the expectation."
fi
ctest -V --no-tests=error --timeout 10800 \
-R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$'
# The transport assertions ran on this leg and found their evidence.
python3 "${GITHUB_WORKSPACE}/scripts/ci/retrace_split_evidence.py" \
'${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt' \
--transport '${{ matrix.transport }}' --case '${{ matrix.case }}' \
--backend '${{ matrix.backend }}' "${wire[@]}"
reuse=()
if [ "${SPLIT_REUSE}" = "true" ]; then reuse=(--split-reuse); fi
python3 scripts/ci/retrace_shard.py run --profile split \
--backend '${{ matrix.backend }}' --transport '${{ matrix.transport }}' --lane split \
--shard '${{ matrix.shard }}/${{ matrix.shards }}' \
--server-path "${GITHUB_WORKSPACE}/build-linux/libMobileGLServer.so" \
--build-dir build-retrace/tools/trace_replay \
--result-dir "${RUNNER_TEMP}/retrace-shard" "${reuse[@]}"
# The refusal census, recorded rather than gated. run_trace_case.cmake already REDS the case
# on any Fatal{, so reaching here means the count is zero - but the number and the distinct
# slot names are what MEASUREMENTS wants from every split run, and reading them out of the
# library's own log is the only way to get them (ctest -V's transcript is a false zero).
- name: Refusal census from the library log
if: ${{ always() }}
working-directory: build-retrace/tools/trace_replay
run: |
# BOTH ROLES, because under spawn every applier refusal is raised in the SERVER's
# process and a census reading only the client's file reports a confident zero for the
# half it cannot see.
log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.client.log"
server_log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.server.log"
cat "${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt" 2>/dev/null || echo "no split-evidence.txt"
if [ ! -f "${log}" ]; then
echo "no ${log} - the replay wrote no library log"
exit 0
fi
census_files="${log}"
if [ -f "${server_log}" ]; then census_files="${census_files} ${server_log}"; fi
echo "census over: ${census_files}"
echo "Fatal{ lines: $(cat ${census_files} | grep -c 'Fatal{' || true)"
cat ${census_files} | grep -o 'Fatal{[A-Za-z]*, "[^"]*"' | sort | uniq -c | sort -rn | head -20 || true
# The transport line is the CLIENT's: ConfigLoader resolves it there, and under spawn
# the server states its own separately.
grep -m1 "MOBILEGL_TRANSPORT=" "${log}" || echo "no transport line in ${log}"
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit"
- name: Upload core dumps
if: failure()
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-split-core-dumps-${{ matrix.transport }}-${{ matrix.backend }}-${{ matrix.case }}
name: retrace-split-core-dumps-${{ matrix.transport }}-${{ matrix.backend }}-shard${{ matrix.shard }}
path: /tmp/core.*
if-no-files-found: ignore
- name: Upload actual image
if: ${{ always() }}
# Nothing reads a green shard's output, so only a failed shard uploads it.
- name: Upload failed shard results
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-split-result-${{ matrix.transport }}-${{ matrix.backend }}-${{ matrix.case }}
name: retrace-split-failed-${{ matrix.transport }}-${{ matrix.backend }}-shard${{ matrix.shard }}
path: |
build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/**
build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/**
${{ runner.temp }}/retrace-shard/results
${{ runner.temp }}/retrace-shard/junit
${{ runner.temp }}/retrace-shard/logs
if-no-files-found: warn
overwrite: true
# The shipped library, replayed. build-linux-monolith-control's library has no MG_Remote, the
# shape FCL embeds; on the disaggregation branches nothing else replays that shape, so it runs
@@ -3490,8 +3440,8 @@ jobs:
needs:
- build-linux-monolith-control
- build-retrace
- trace-fixtures
if: ${{ !cancelled() && (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && needs.build-linux-monolith-control.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-fixtures.result != 'skipped' }}
- trace-cases
if: ${{ !cancelled() && (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && needs.build-linux-monolith-control.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }}
strategy:
fail-fast: false
matrix:
@@ -3513,16 +3463,11 @@ jobs:
with:
gpu: "true"
- name: Download trace fixture
uses: actions/download-artifact@v8
with:
name: trace-fixture-${{ matrix.case }}
path: trace-fixture-download
- name: Install trace fixture
run: |
mkdir -p tools/trace_replay/fixtures
find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;
# This job's one case keeps its fixtures in Git itself, not in Git LFS, so the checkout
# has them and the fixture bundle is not restored. The check is that they are real files;
# a case with Git LFS fixtures would be downloaded here, with a warning annotation.
- name: Check trace fixture
run: bash .github/scripts/trace-fixture-bundle.sh install '${{ matrix.case }}'
- name: Get CMake
timeout-minutes: 15
@@ -3588,15 +3533,16 @@ jobs:
run: python3 scripts/ci/junit_annotate.py --excerpts 1 "${RUNNER_TEMP}"
- name: Upload core dumps
if: failure()
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-shipped-core-dumps-${{ matrix.backend }}-${{ matrix.case }}
path: /tmp/core.*
if-no-files-found: ignore
# Nothing reads a green leg's output, so only a failed leg uploads it.
- name: Upload actual image
if: ${{ always() }}
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-shipped-result-${{ matrix.backend }}-${{ matrix.case }}
@@ -3605,82 +3551,6 @@ jobs:
build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/**
if-no-files-found: warn
remove-artifact-clutter:
name: remove artifact clutter
runs-on: ubuntu-26.04
# Every retrace job downloads the shared fixtures, so this waits for all of them.
needs:
- retrace-summary
- retrace-verify
- retrace-split
- retrace-shipped
if: ${{ always() }}
permissions:
actions: write
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Delete intermediate Linux retrace artifacts
env:
GH_TOKEN: ${{ github.token }}
run: |
# Read the complete inventories before deleting anything. A failed API
# request must fail this step, not look like "no failed cases" through
# an unchecked process substitution.
# 50 rows per page: the jobs endpoint answers HTTP 502 for 90+ rows on a re-run attempt of a run this size.
gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=50" \
--jq '.jobs[] | select((.name | startswith("retrace (")) or (.name | startswith("retrace verify (")) or (.name | startswith("retrace split (")) or (.name | startswith("retrace shipped library ("))) | select(.conclusion == "failure" or .conclusion == "canceled" or .conclusion == "timed_out" or .conclusion == "action_required") | .name' \
> "${RUNNER_TEMP}/cleanup-jobs.txt"
gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \
--jq '.artifacts[] | select(.name | startswith("trace-fixture-") or startswith("retrace-result-") or startswith("retrace-split-result-") or startswith("retrace-shipped-result-")) | [.id, .name] | @tsv' \
> "${RUNNER_TEMP}/cleanup-artifacts.txt"
declare -A failed_cases=()
while IFS= read -r job_name; do
case_name="${job_name#retrace split (*, }"
case_name="${case_name#retrace shipped library (*, }"
case_name="${case_name#retrace verify (*, }"
case_name="${case_name#retrace (*, }"
case_name="${case_name%)}"
failed_cases["${case_name}"]=1
done < "${RUNNER_TEMP}/cleanup-jobs.txt"
if ((${#failed_cases[@]})); then
echo "Retaining fixtures and results for failed retrace case(s):"
printf ' %s\n' "${!failed_cases[@]}"
else
echo "All retrace jobs succeeded; no fixtures need to be retained."
fi
deleted=0
retained=0
while IFS=$'\t' read -r artifact_id artifact_name; do
keep=0
if [[ "${artifact_name}" == trace-fixture-* ]]; then
case_name="${artifact_name#trace-fixture-}"
if [[ -v "failed_cases[${case_name}]" ]]; then
keep=1
fi
else
for case_name in "${!failed_cases[@]}"; do
if [[ "${artifact_name}" == *-"${case_name}" ]]; then keep=1; break; fi
done
fi
if ((keep)); then
echo "Retaining ${artifact_name} (${artifact_id}) for failed retrace."
((retained += 1))
continue
fi
echo "Deleting ${artifact_name} (${artifact_id})"
gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}"
((deleted += 1))
done < "${RUNNER_TEMP}/cleanup-artifacts.txt"
echo "Deleted ${deleted} intermediate Linux artifact(s); retained ${retained} failed-retrace artifact(s)."
pipe-gates:
name: MGPipe generators and hygiene gates
runs-on: ubuntu-26.04
.gitignore
+2
View File
@@ -29,3 +29,5 @@ __pycache__/
*.py[cod]
/.gradle
/.trace-work/
# The restored trace fixture cache entry (.github/scripts/trace-fixture-bundle.sh)
/trace-fixture-bundle/
android-plugin/TRACE_REPLAY.md
+54
View File
@@ -74,3 +74,57 @@ adb shell am start -a top.mobilegl.plugin.TRACE_REPLAY \
--es ssim_threshold 0.99
adb shell run-as top.mobilegl.plugin.espryt.trace cat files/trace-replay/output/result.json
```
## Emulator shards in CI
The APK workflow replays its cases through `trace-replay-shard.sh`: one emulator boot, one APK
install, then an ordered list of cases for one lane, transport and backend. Each case still runs
through `trace-replay-ci.sh`, with the same arguments, environment and transport proof as a
one-case job.
```sh
sh android-plugin/trace-replay-shard.sh \
--apk-file MobileGL-plugin-trace-release.apk \
--backend DirectVulkan --gpu lavapipe \
--transport inproc --lane inproc-acceptance \
--cases-file cases.json --deadline-seconds 7200
```
The case list is a JSON array of case objects: the `cases` member of a row of
`trace_cases.py --ci --format github-apk-transport-shard-matrix`, which is what the workflow
passes, or the `case` members of `--format github-apk-transport-matrix` rows. `--dry-run` prints
what each case would be run with and touches no device.
The workflow's matrix is that shard matrix: one row per emulator job, holding some of the cases
of one lane, transport and backend. `--shard-seconds` (`RETRACE_SHARD_SECONDS` in `apk.yml`) sets
how much hinted replay time one shard aims for, and `trace_cases.py` refuses to print a shard
matrix whose shards, taken apart, are not exactly the per-case matrix. The fixtures of a shard's
cases must be in `tools/trace_replay/fixtures` before the runner starts; the workflow restores
them from one cache entry (`.github/scripts/trace-fixture-bundle.sh`).
Between cases the runner force-stops the app, checks that no process of the app's uid is left,
clears the app data, removes the case's files from `/data/local/tmp` and from the host, and
checks that the device still answers. It restarts the emulator when it does not, before the
case that follows one that did not pass, and before the single retry of a case that
`trace-replay-ci.sh` charged to the infrastructure (exit 75).
`MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE=1` cold boots before every case.
The cases of a shard share the job's time. `--deadline-seconds` stops the runner that long after
it starts, so that the job can still upload what it has: a case that is running then, or has not
started, is recorded as `not-run`. `retrace_shard_budget.py` reads a shard matrix and adds two
fields to every shard: `deadline_seconds`, the time in which every case of the shard could use
its whole time budget (`timeout_seconds` plus 300 s) when the cases before it used all of
theirs, with one infrastructure retry of the longest budget and a cold boot per case on top,
and `timeout_minutes`, the limit of the shard's job. `--cap-seconds`
(`RETRACE_SHARD_DEADLINE_CAP_SECONDS` in `apk.yml`) bounds the deadline, so a shard whose
replays all hang cannot hold a runner for hours; a capped shard loses verdicts only when several
of its replays hang to the end of their budgets.
A failing case does not stop the shard. Every case ends with `<case>-<backend>/case-status.json`
(`passed`, `failed` or `not-run`), the result root gets `shard-status.json`, the job log gets one
`::error` per case that did not pass, and the exit status is non-zero when a case did not pass or
a verdict is missing. `--finalize` writes the summary inputs and a `not-run` verdict for every
case without one, without a device; the workflow runs it after the replay step whatever happened.
`python3 android-plugin/test_trace_replay_shard.py` drives the runner with a fake adb, replay
script and AVD script.
android-plugin/retrace_artifact_cleanup.py
+235
View File
@@ -0,0 +1,235 @@
#!/usr/bin/env python3
# MobileGL - android-plugin/retrace_artifact_cleanup.py
# Copyright (c) 2025-2026 MobileGL-Dev
# Licensed under the GNU Lesser General Public License v3.0:
# https://www.gnu.org/licenses/gpl-3.0.txt
# https://www.gnu.org/licenses/lgpl-3.0.txt
# SPDX-License-Identifier: LGPL-3.0-only
# End of Source File Header
"""Delete the intermediate artifacts of an APK workflow run, keeping what a failure needs.
The retrace summary embeds every image it shows, so once it is uploaded the per-shard result
artifacts are only worth keeping for cases that did not pass:
- a result artifact stays when its shard-status.json is missing, when its job failed, or when
it holds a case that did not pass in any shard (so both backends and all transports of a
failed case stay comparable);
- every result artifact stays when a job failed without leaving a shard-status.json to say
which cases it was running.
A canceled job retains nothing by itself; a failed verdict inside it still does. The trace
fixtures are not artifacts: the retrace jobs restore them from a cache entry.
"""
import argparse
import json
import os
import shlex
import subprocess
import sys
import threading
import time
from pathlib import Path
RESULT_PREFIXES = ("MobileGL-android-retrace-result-", "retrace-result-")
FAILED_CONCLUSIONS = ("failure", "timed_out", "action_required")
# GitHub's secondary rate limit allows about 180 mutating REST calls a minute. Requests start
# at most this often, whatever the number of workers.
DELETE_INTERVAL_SECONDS = 60.0 / 170.0
DELETE_WORKERS = 3
MAX_REFUSED_DELETES = 5
def pause(seconds):
time.sleep(seconds)
def read_rows(text, columns):
rows = []
for line in text.splitlines():
if line.strip():
fields = line.split("\t")
rows.append(fields + [""] * (columns - len(fields)))
return rows
def shard_statuses(download_roots):
"""Artifact name -> its shard-status.json, for every downloaded result artifact that has one."""
statuses = {}
for root in download_roots:
root = Path(root)
if not root.is_dir():
continue
for directory in root.iterdir():
try:
status = json.loads((directory / "shard-status.json").read_text(encoding="utf-8"))
except (OSError, ValueError):
continue
if isinstance(status, dict) and isinstance(status.get("cases"), list):
statuses[directory.name] = status
return statuses
def plan(artifacts, jobs, statuses):
"""Returns (ids to delete, lines explaining what stays)."""
failed_job_ids = {job_id for job_id, conclusion, _ in jobs if conclusion in FAILED_CONCLUSIONS}
status_job_ids = {str(status.get("job_id") or "") for status in statuses.values()} - {""}
# Failed jobs that left no status file: nothing says which cases they were running.
unexplained = failed_job_ids - status_job_ids
if not status_job_ids and failed_job_ids:
# The statuses carry no job ids to match against. Count instead: each failed job must
# be explained by an artifact that shows a case without a pass.
explaining = sum(1 for status in statuses.values()
if any(record.get("status") != "passed" for record in status["cases"]))
unexplained = set() if explaining >= len(failed_job_ids) else failed_job_ids
failed_cases = set() # did not pass somewhere: every result that holds one stays
for status in statuses.values():
not_passed = [record for record in status["cases"] if record.get("status") != "passed"]
if status_job_ids:
job_failed = str(status.get("job_id") or "") in failed_job_ids
else:
# Without job ids, a status that shows a case without a pass stands for a failed job.
job_failed = bool(failed_job_ids) and bool(not_passed)
for record in not_passed:
if record.get("status") == "failed" or job_failed:
failed_cases.add(record.get("case"))
delete, kept = [], []
for artifact_id, name in artifacts:
if not name.startswith(RESULT_PREFIXES):
continue
reason = ""
status = statuses.get(name)
if unexplained:
reason = f"{len(unexplained)} failed job(s) left no shard status"
elif status is None:
reason = "no shard status was downloaded for it"
elif str(status.get("job_id") or "") in failed_job_ids:
reason = "its job failed"
else:
cases = sorted(failed_cases.intersection(record.get("case") for record in status["cases"]))
if cases:
reason = "holds failed case(s): " + ", ".join(cases)
if reason:
kept.append(f"{name} ({artifact_id}): {reason}")
else:
delete.append((artifact_id, name))
return delete, kept
class Deleter:
"""Runs `command` once per artifact, with the artifact id appended to its last word."""
def __init__(self, command):
self.command = command
self.lock = threading.Lock()
self.next_start = 0.0
self.refused = []
self.deleted = 0
def wait_for_slot(self):
with self.lock:
now = time.monotonic()
start = max(now, self.next_start)
self.next_start = start + DELETE_INTERVAL_SECONDS
if start > now:
pause(start - now)
def delete(self, artifact_id, name):
for attempt in (1, 2):
with self.lock:
if len(self.refused) >= MAX_REFUSED_DELETES:
return
self.wait_for_slot()
done = subprocess.run(self.command[:-1] + [self.command[-1] + artifact_id],
capture_output=True, text=True)
if done.returncode == 0:
with self.lock:
self.deleted += 1
return
if attempt == 1:
# Give a rate limit time to clear before the one retry.
with self.lock:
self.next_start = max(self.next_start, time.monotonic() + 30)
with self.lock:
self.refused.append(f"{name} ({artifact_id}): {' '.join(done.stderr.split())[:200]}")
def run(self, entries):
pending = list(entries)
def work():
while True:
with self.lock:
if not pending:
return
artifact_id, name = pending.pop(0)
self.delete(artifact_id, name)
threads = [threading.Thread(target=work) for _ in range(min(DELETE_WORKERS, len(pending)))]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
def gh_rows(path, query, columns):
done = subprocess.run(["gh", "api", "--paginate", path, "--jq", query], capture_output=True, text=True)
if done.returncode != 0:
raise RuntimeError(f"gh api {path} failed: {done.stderr.strip()}")
return read_rows(done.stdout, columns)
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--downloads", action="append", default=[],
help="Directory holding one sub-directory per downloaded result artifact.")
parser.add_argument("--artifacts-file", help="Tab-separated id and name per line, instead of asking gh.")
parser.add_argument("--jobs-file", help="Tab-separated id, conclusion and name per line, instead of asking gh.")
parser.add_argument("--dry-run", action="store_true", help="Print the plan, delete nothing.")
options = parser.parse_args(argv)
repository = os.environ.get("GITHUB_REPOSITORY", "")
run_id = os.environ.get("GITHUB_RUN_ID", "")
try:
if options.artifacts_file:
artifacts = read_rows(Path(options.artifacts_file).read_text(encoding="utf-8"), 2)
else:
artifacts = gh_rows(f"repos/{repository}/actions/runs/{run_id}/artifacts?per_page=100",
".artifacts[] | [.id, .name] | @tsv", 2)
if options.jobs_file:
jobs = read_rows(Path(options.jobs_file).read_text(encoding="utf-8"), 3)
else:
# 50 rows per page: the jobs endpoint answers HTTP 502 for larger pages of a re-run
# attempt of a run this size.
jobs = gh_rows(f"repos/{repository}/actions/runs/{run_id}/jobs?per_page=50",
'.jobs[] | [.id, .conclusion // "", .name] | @tsv', 3)
except (OSError, RuntimeError) as error:
print(f"::error::Could not read the run's artifacts and jobs; nothing was deleted: {error}")
return 1
delete, kept = plan([(row[0], row[1]) for row in artifacts], [tuple(row[:3]) for row in jobs],
shard_statuses(options.downloads))
for line in kept:
print(f"Retaining {line}")
if options.dry_run:
for artifact_id, name in delete:
print(f"Would delete {name} ({artifact_id})")
return 0
deleter = Deleter(shlex.split(os.environ.get("MOBILEGL_CLEANUP_DELETE_COMMAND") or
f"gh api --method DELETE repos/{repository}/actions/artifacts/"))
started = time.monotonic()
deleter.run(delete)
for line in deleter.refused:
print(f"::warning::Could not delete intermediate artifact {line}")
if len(deleter.refused) >= MAX_REFUSED_DELETES:
print("::warning::Stopped deleting intermediate artifacts after repeated refusals; the rest stay until they expire.")
print(f"Deleted {deleter.deleted} intermediate Android artifact(s) in {time.monotonic() - started:.0f}s; "
f"retained {len(kept)}.")
return 0
if __name__ == "__main__":
sys.exit(main())
android-plugin/retrace_shard_budget.py
+104
View File
@@ -0,0 +1,104 @@
#!/usr/bin/env python3
# MobileGL - android-plugin/retrace_shard_budget.py
# Copyright (c) 2025-2026 MobileGL-Dev
# Licensed under the GNU Lesser General Public License v3.0:
# https://www.gnu.org/licenses/gpl-3.0.txt
# https://www.gnu.org/licenses/lgpl-3.0.txt
# SPDX-License-Identifier: LGPL-3.0-only
# End of Source File Header
"""Give every emulator shard the deadline and the job timeout its own cases need.
The cases of a shard share one job, so one case that hangs spends time the cases after it
need. This reads a shard matrix of trace_cases.py (--format github-apk-shard-matrix or
github-apk-transport-shard-matrix) on standard input, or from --matrix-file, and prints it
with two fields added to every shard: deadline_seconds, which the retrace job passes to
trace-replay-shard.sh, and timeout_minutes, the limit of that job. The deadline is what the
shard needs for every case to use its whole time budget (trace_replay_shard.needed_seconds),
up to --cap-seconds. A capped shard still replays every case; only when several of its
replays hang to the end of their budgets are the cases after them recorded as not-run.
"""
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import trace_replay_shard # noqa: E402 (a sibling file, found through the line above)
def shard_label(row):
backend = row.get("backend")
backend = backend.get("name", "") if isinstance(backend, dict) else str(backend or "")
return f"{backend} / {row.get('transport') or 'monolith'}, shard {row.get('shard')}/{row.get('shards')}"
def shard_needs(matrix):
"""(label, case count, seconds needed) for every shard of the matrix."""
rows = matrix.get("include") if isinstance(matrix, dict) else None
if not isinstance(rows, list) or not rows:
raise ValueError("the matrix has no shard rows under 'include'")
needs = []
for row in rows:
cases = row.get("cases") if isinstance(row, dict) else None
if not isinstance(cases, list) or not cases:
raise ValueError(f"a shard row has no 'cases' list: {str(row)[:200]}")
needs.append((shard_label(row), len(cases), trace_replay_shard.needed_seconds(cases)))
return needs
# Minutes a retrace job has around its shard runner: the setup before the replays and the
# uploads after them.
JOB_OVERHEAD_MINUTES = 15
# A hosted job may run six hours.
HOSTED_JOB_LIMIT_MINUTES = 360
def with_limits(matrix, cap_seconds):
"""The matrix with deadline_seconds and timeout_minutes on every shard, and the report."""
needs = shard_needs(matrix)
rows = []
report = []
for row, (label, cases, seconds) in zip(matrix["include"], needs):
deadline = min(seconds, cap_seconds)
rows.append({**row, "deadline_seconds": deadline,
"timeout_minutes": -(-deadline // 60) + JOB_OVERHEAD_MINUTES})
if seconds > cap_seconds:
report.append(f"retrace shard budget: {label} would need {seconds} s for all of its "
f"{cases} case(s) to use their whole budgets; capped at {cap_seconds} s")
largest = max(needs, key=lambda entry: entry[2])
report.append(f"retrace shard budget: {len(needs)} shard(s), {len(report)} capped at "
f"{cap_seconds} s; the most a shard needs is {largest[2]} s ({largest[0]}, "
f"{largest[1]} case(s))")
return {**matrix, "include": rows}, report
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--cap-seconds", type=int, required=True,
help="The longest deadline a shard is given.")
parser.add_argument("--matrix-file", help="The shard matrix; standard input when not given.")
options = parser.parse_args(argv)
limit = (HOSTED_JOB_LIMIT_MINUTES - JOB_OVERHEAD_MINUTES) * 60
if not 0 < options.cap_seconds <= limit:
print(f"retrace shard budget: --cap-seconds must be between 1 and {limit}", file=sys.stderr)
return 2
if options.matrix_file:
text = Path(options.matrix_file).read_text(encoding="utf-8")
else:
text = sys.stdin.read()
try:
matrix, report = with_limits(json.loads(text), options.cap_seconds)
except (ValueError, KeyError, TypeError) as error:
print(f"retrace shard budget: the shard matrix cannot be read: {error}", file=sys.stderr)
return 2
for line in report:
print(line, file=sys.stderr)
print(json.dumps(matrix, separators=(",", ":")))
return 0
if __name__ == "__main__":
sys.exit(main())
android-plugin/test_retrace_artifact_cleanup.py
+178
View File
@@ -0,0 +1,178 @@
#!/usr/bin/env python3
# MobileGL - android-plugin/test_retrace_artifact_cleanup.py
# Copyright (c) 2025-2026 MobileGL-Dev
# Licensed under the GNU Lesser General Public License v3.0:
# https://www.gnu.org/licenses/gpl-3.0.txt
# https://www.gnu.org/licenses/lgpl-3.0.txt
# SPDX-License-Identifier: LGPL-3.0-only
# End of Source File Header
"""Pin which intermediate APK artifacts the cleanup keeps, and how fast it deletes the rest."""
import contextlib
import importlib.util
import io
import json
import os
import pathlib
import sys
import tempfile
import unittest
from unittest import mock
HERE = pathlib.Path(__file__).resolve().parent
_spec = importlib.util.spec_from_file_location("retrace_artifact_cleanup", HERE / "retrace_artifact_cleanup.py")
cleanup = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(cleanup)
RESULT = "MobileGL-android-retrace-result-"
def status(job_id, **cases):
return {"job_id": job_id, "cases": [{"case": name, "status": verdict} for name, verdict in cases.items()]}
def names(entries):
return sorted(name for _, name in entries)
class PlanTest(unittest.TestCase):
ARTIFACTS = [("1", RESULT + "mono-gles"), ("2", RESULT + "inproc-gles"), ("3", RESULT + "spawn-vk"),
("4", "mobilegl-android-monolith-control.html"), ("5", "MobileGL-plugin-release-abc.apk"),
("6", "MobileGL-plugin-trace-release-abc.apk")]
def test_a_green_run_deletes_every_intermediate_and_nothing_else(self):
statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed"),
RESULT + "inproc-gles": status("12", a="passed", b="passed"),
RESULT + "spawn-vk": status("13", c="passed")}
jobs = [("11", "success", "x"), ("12", "success", "y"), ("13", "success", "z"), ("14", "", "retrace summary")]
delete, kept = cleanup.plan(self.ARTIFACTS, jobs, statuses)
self.assertEqual(kept, [])
self.assertEqual(names(delete), sorted(name for _, name in self.ARTIFACTS[:3]))
def test_a_failed_case_keeps_every_shard_that_ran_it(self):
statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed"),
RESULT + "inproc-gles": status("12", a="passed", b="failed"),
RESULT + "spawn-vk": status("13", c="passed")}
jobs = [("11", "success", "x"), ("12", "failure", "y"), ("13", "success", "z")]
delete, kept = cleanup.plan(self.ARTIFACTS, jobs, statuses)
# Only b's results are worth comparing, so the monolith shard stays for b and the
# spawn shard goes.
self.assertEqual(names(delete), [RESULT + "spawn-vk"])
self.assertEqual(len(kept), 2)
self.assertTrue(any("inproc-gles" in line and "its job failed" in line for line in kept))
self.assertTrue(any("mono-gles" in line and "holds failed case(s): b" in line for line in kept))
def test_cases_a_failed_job_never_reached_are_kept(self):
statuses = {RESULT + "mono-gles": status("11", a="passed", b="not-run"),
RESULT + "spawn-vk": status("13", c="passed")}
jobs = [("11", "timed_out", "x"), ("13", "success", "z")]
delete, kept = cleanup.plan(self.ARTIFACTS[:1] + self.ARTIFACTS[2:], jobs, statuses)
self.assertEqual(names(delete), [RESULT + "spawn-vk"])
self.assertEqual(len(kept), 1)
def test_a_job_that_failed_after_its_cases_passed_keeps_its_results(self):
statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed"),
RESULT + "spawn-vk": status("13", c="passed")}
jobs = [("11", "failure", "upload failed"), ("13", "success", "z")]
delete, kept = cleanup.plan(self.ARTIFACTS[:1] + self.ARTIFACTS[2:], jobs, statuses)
self.assertEqual(names(delete), [RESULT + "spawn-vk"])
self.assertTrue("mono-gles" in kept[0] and "its job failed" in kept[0])
def test_a_canceled_job_keeps_nothing_by_itself(self):
statuses = {RESULT + "mono-gles": status("11", a="passed", b="not-run"),
RESULT + "inproc-gles": status("12", a="failed", b="not-run")}
jobs = [("11", "cancelled", "x"), ("12", "cancelled", "y")]
delete, kept = cleanup.plan(self.ARTIFACTS[:2], jobs, statuses)
# Case a failed in the inproc shard, so both shards that ran it stay.
self.assertEqual(delete, [])
self.assertEqual(len(kept), 2)
only_not_run = {RESULT + "mono-gles": status("11", a="passed", b="not-run")}
delete, kept = cleanup.plan(self.ARTIFACTS[:1], [("11", "cancelled", "x")], only_not_run)
self.assertEqual((names(delete), kept), ([RESULT + "mono-gles"], []))
def test_a_result_without_a_status_is_kept(self):
statuses = {RESULT + "mono-gles": status("11", a="passed")}
delete, kept = cleanup.plan(self.ARTIFACTS[:2], [("11", "success", "x")], statuses)
self.assertEqual(names(delete), [RESULT + "mono-gles"])
self.assertIn("no shard status", kept[0])
def test_a_failed_job_without_a_status_keeps_everything(self):
statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed")}
jobs = [("11", "success", "x"), ("12", "failure", "died before its upload")]
delete, kept = cleanup.plan(self.ARTIFACTS, jobs, statuses)
self.assertEqual(delete, [])
self.assertEqual(len(kept), 3)
def test_statuses_without_job_ids_fall_back_to_counting(self):
artifacts = self.ARTIFACTS[:1] + self.ARTIFACTS[2:]
explained = {RESULT + "mono-gles": status("", a="passed", b="not-run"),
RESULT + "spawn-vk": status("", c="passed")}
delete, _ = cleanup.plan(artifacts, [("12", "failure", "y")], explained)
# The one status with a case that did not pass stands for the one failed job.
self.assertEqual(names(delete), [RESULT + "spawn-vk"])
unexplained = {RESULT + "mono-gles": status("", a="passed", b="passed"),
RESULT + "spawn-vk": status("", c="passed")}
delete, _ = cleanup.plan(artifacts, [("12", "failure", "y")], unexplained)
self.assertEqual(delete, [])
class RunTest(unittest.TestCase):
@staticmethod
def passed(artifacts):
"""A shard status with one passed case for every artifact, each from a job of its own."""
return {name: status(f"9{artifact_id}", a="passed") for artifact_id, name in artifacts}
def run_cleanup(self, artifacts, jobs, statuses, exit_code=0, dry_run=False):
with tempfile.TemporaryDirectory() as directory:
root = pathlib.Path(directory)
for name, value in statuses.items():
(root / "downloads" / "lane" / name).mkdir(parents=True)
(root / "downloads" / "lane" / name / "shard-status.json").write_text(json.dumps(value))
(root / "artifacts.tsv").write_text("".join(f"{i}\t{n}\n" for i, n in artifacts))
(root / "jobs.tsv").write_text("".join("\t".join(job) + "\n" for job in jobs))
# One file per call: the deletes run in parallel processes.
(root / "deleted").mkdir()
script = root / "delete.py"
script.write_text("import pathlib, sys\n"
"(pathlib.Path(sys.argv[1]) / sys.argv[2].replace('/', '_')).write_text('')\n"
f"sys.exit({exit_code})\n")
python = pathlib.Path(sys.executable).as_posix()
command = f'"{python}" "{script.as_posix()}" "{(root / "deleted").as_posix()}" artifacts/'
pauses = []
output = io.StringIO()
with mock.patch.dict(os.environ, {"MOBILEGL_CLEANUP_DELETE_COMMAND": command}), \
mock.patch.object(cleanup, "pause", pauses.append), contextlib.redirect_stdout(output):
code = cleanup.main(["--downloads", str(root / "downloads" / "lane"),
"--artifacts-file", str(root / "artifacts.tsv"),
"--jobs-file", str(root / "jobs.tsv")] + (["--dry-run"] if dry_run else []))
deleted = sorted(entry.name.replace("_", "/") for entry in (root / "deleted").iterdir())
return code, output.getvalue(), deleted, pauses
def test_deletes_are_spaced_under_the_secondary_rate_limit(self):
artifacts = [(str(index), f"{RESULT}shard{index}") for index in range(12)]
code, output, deleted, pauses = self.run_cleanup(artifacts, [], self.passed(artifacts))
self.assertEqual(code, 0)
self.assertEqual(deleted, sorted(f"artifacts/{index}" for index in range(12)))
self.assertIn("Deleted 12 intermediate Android artifact(s)", output)
# Twelve starts are booked at least 11 intervals apart, so the rate stays under 180 a minute.
self.assertLess(60 / cleanup.DELETE_INTERVAL_SECONDS, 180)
self.assertGreater(len(pauses), 0)
def test_refused_deletes_warn_and_leave_the_run_green(self):
artifacts = [(str(index), f"{RESULT}shard{index}") for index in range(8)]
code, output, _, _ = self.run_cleanup(artifacts, [], self.passed(artifacts), exit_code=1)
self.assertEqual(code, 0)
self.assertIn("::warning::Could not delete intermediate artifact", output)
self.assertIn("Stopped deleting", output)
self.assertIn("Deleted 0 intermediate", output)
def test_a_dry_run_deletes_nothing(self):
artifacts = [("1", RESULT + "shard1")]
code, output, deleted, _ = self.run_cleanup(artifacts, [], self.passed(artifacts), dry_run=True)
self.assertEqual((code, deleted), (0, []))
self.assertIn(f"Would delete {RESULT}shard1 (1)", output)
if __name__ == "__main__":
unittest.main()
android-plugin/test_trace_replay_shard.py
+587
View File
@@ -0,0 +1,587 @@
#!/usr/bin/env python3
# MobileGL - android-plugin/test_trace_replay_shard.py
# Copyright (c) 2025-2026 MobileGL-Dev
# Licensed under the GNU Lesser General Public License v3.0:
# https://www.gnu.org/licenses/gpl-3.0.txt
# https://www.gnu.org/licenses/lgpl-3.0.txt
# SPDX-License-Identifier: LGPL-3.0-only
# End of Source File Header
"""Drive the emulator shard runner with a fake adb, replay script and AVD script.
No device is needed. The fakes record every call, so the tests pin what each case is run
with, that every case ends with a verdict, and what a failure, an infrastructure retry, a
timeout or a dead emulator does to the rest of the shard.
"""
import contextlib
import importlib.util
import io
import json
import os
import pathlib
import re
import signal
import subprocess
import sys
import tempfile
import time
import unittest
from unittest import mock
HERE = pathlib.Path(__file__).resolve().parent
def load(name):
spec = importlib.util.spec_from_file_location(name, HERE / f"{name}.py")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
shard = load("trace_replay_shard")
budget = load("retrace_shard_budget")
BLISS = "minecraft-1.21.4-fabric-iris-bliss-in-world"
ITERATIONRP = "minecraft-1.21.4-fabric-iris-iterationrp-in-world"
SPLIT = "MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1"
FAKE_COMMON = r'''
import json, os, pathlib, sys
state = pathlib.Path(os.environ["FAKE_STATE"])
def record(name, entry):
with open(state / name, "a", encoding="utf-8") as log:
log.write(json.dumps(entry) + "\n")
def scenario():
path = state / "scenario.json"
return json.loads(path.read_text()) if path.exists() else {}
def count(name):
path = state / f"count-{name}"
value = int(path.read_text()) if path.exists() else 0
path.write_text(str(value + 1))
return value
'''
FAKE_ADB = FAKE_COMMON + r'''
args = sys.argv[1:]
record("adb.log", args)
plan = scenario()
if args[:1] == ["get-state"]:
print("offline" if count("get-state") < plan.get("offline_get_state_calls", 0) else "device")
elif args[:3] == ["shell", "getprop", "sys.boot_completed"]:
print("1")
elif args[:3] == ["shell", "pm", "path"]:
print("package:/data/app/trace/base.apk")
elif args[:3] == ["shell", "pm", "clear"]:
print("Failed" if plan.get("pm_clear_fails") else "Success")
elif args[:1] == ["install"]:
if count("install") < plan.get("failing_installs", 0):
print("Failure [INSTALL_FAILED]")
sys.exit(1)
print("Success")
elif args[:2] == ["shell", "run-as"] and args[3:] == ["id", "-un"]:
print("u0_a77")
elif args[:3] == ["shell", "ps", "-A"]:
print("USER PID NAME")
print("root 1 init")
if count("ps") < plan.get("ps_calls_with_leftover", 0):
print("u0_a77 4242 libMobileGLServer.so")
'''
FAKE_REPLAY = FAKE_COMMON + r'''
import time
args = sys.argv[1:]
def value(flag):
return args[args.index(flag) + 1]
case, backend, root = value("--case"), value("--backend"), pathlib.Path(value("--result-root"))
record("replay.log", {"case": case, "arguments": args, "environment": {
key: os.environ.get(key) for key in (
"MOBILEGL_ESPRYT_USE_ANGLE", "MOBILEGL_TRACE_ANGLE_VARIANT", "MOBILEGL_MAGMA_R11G11B10F_FALLBACK",
"MOBILEGL_ITRP_FIX", "MOBILEGL_TRACE_SKIP_INSTALL", "MOBILEGL_TRACE_POLL_INTERVAL")}})
(state / "replay.pid").write_text(str(os.getpid()))
steps = scenario().get("replay", {}).get(case, [])
attempt = count("replay-" + case)
step = steps[attempt] if attempt < len(steps) else {}
time.sleep(step.get("sleep", 0))
directory = root / f"{case}-{backend}"
directory.mkdir(parents=True, exist_ok=True)
code = step.get("exit", 0)
if code == 75:
with open(root / "infrastructure-failure-reason.txt", "a") as reasons:
reasons.write(step.get("reason", "device-unavailable") + "\n")
if step.get("result", code in (0, 1)):
(directory / "result.json").write_text(json.dumps(
{"passed": step.get("passed", code == 0), "ssim": 0.999, "message": step.get("message", "retrace completed")}))
transport = "inproc" if "--require-inproc" in args else "spawn" if "--require-spawn" in args else ""
if transport and code == 0 and step.get("proof", True):
(directory / "transport-proof.json").write_text(json.dumps({"passed": True, "required_transport": transport}))
sys.exit(code)
'''
FAKE_AVD = FAKE_COMMON + r'''
args = sys.argv[1:]
record("avd.log", args)
if args[:1] == ["start"]:
exits = scenario().get("avd_start_exits", [])
index = count("avd-start")
sys.exit(exits[index] if index < len(exits) else 0)
'''
def case(name, **extra):
base = {"name": name, "trace_archive": f"fixtures/{name}.tgz", "trace_file": "trace.trace",
"golden": f"fixtures/{name}.png", "target_call": 1234, "width": 854, "height": 480,
"ssim_threshold": 0.99, "crop_x": 0, "crop_y": 0, "crop_width": 0, "crop_height": 0,
"timeout_seconds": 900}
base.update(extra)
return base
class ShardTest(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.root = pathlib.Path(self.temporary.name)
self.state = self.root / "state"
self.state.mkdir()
self.results = self.root / "results"
(self.root / "fixtures").mkdir()
self.apk = self.root / "trace.apk"
self.apk.write_bytes(b"apk")
self.emulator_log = self.root / "emulator.log"
self.emulator_log.write_text("boot\n")
python = pathlib.Path(sys.executable).as_posix()
commands = {}
for name, source in (("adb", FAKE_ADB), ("replay", FAKE_REPLAY), ("avd", FAKE_AVD)):
script = self.root / f"fake_{name}.py"
script.write_text(source)
commands[name] = f'"{python}" "{script.as_posix()}"'
self.environment = {
"FAKE_STATE": str(self.state), "ADB": commands["adb"],
"MOBILEGL_SHARD_REPLAY_COMMAND": commands["replay"], "MOBILEGL_SHARD_AVD_COMMAND": commands["avd"],
"MOBILEGL_GPU_HANG_BUDGET_MS": "0", "AVD_NAME": "mobilegl-ci",
"EMULATOR_LOG": str(self.emulator_log), "EMULATOR_PID_FILE": str(self.root / "emulator.pid"),
"GITHUB_STEP_SUMMARY": str(self.root / "summary.md"),
}
def fixture(self, name):
path = self.root / "fixtures" / name
path.write_bytes(b"png")
return str(path)
def run_shard(self, cases, *extra, backend="DirectVulkan", transport="monolith", scenario=None, env=None):
if scenario is not None:
(self.state / "scenario.json").write_text(json.dumps(scenario))
arguments = ["--apk-file", str(self.apk), "--backend", backend, "--gpu", "lavapipe",
"--transport", transport, "--lane", f"{transport}-lane", "--cases-json", json.dumps(cases),
"--result-root", str(self.results), "--fixture-root", str(self.root / "work"), *extra]
output = io.StringIO()
with mock.patch.dict(os.environ, {**self.environment, **(env or {})}), \
mock.patch.object(shard, "pause", lambda seconds: None), \
contextlib.redirect_stdout(output):
status = shard.main(arguments)
return status, output.getvalue()
def log(self, name):
path = self.state / name
return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else []
def status(self, name, backend="DirectVulkan"):
return json.loads((self.results / f"{name}-{backend}" / "case-status.json").read_text())
def shard_status(self):
return json.loads((self.results / "shard-status.json").read_text())
# ---- what each case is run with ----------------------------------------------------
def planned(self, cases, backend, transport):
status, output = self.run_shard(cases, "--dry-run", backend=backend, transport=transport)
self.assertEqual(status, 0)
return [json.loads(line) for line in output.splitlines()]
def test_plain_case_arguments_per_transport(self):
cases = [case("plain", alternate_golden="fixtures/plain-mali.png")]
head = ["--apk-file", str(self.apk), "--package", "top.mobilegl.plugin.trace", "--backend", "DirectVulkan",
"--result-root", str(self.results), "--fixture-root", str(self.root / "work"), "--case", "plain",
"--trace-archive", "fixtures/plain.tgz", "--trace-file", "trace.trace",
"--golden", "fixtures/plain.png", "--alternate-golden", "fixtures/plain-mali.png",
"--target-call", "1234", "--width", "854", "--height", "480", "--ssim-threshold", "0.99",
"--crop-x", "0", "--crop-y", "0", "--crop-width", "0", "--crop-height", "0",
"--timeout-seconds", "900"]
tails = {
"monolith": ["--env", "MOBILEGL_TRANSPORT=monolith;MOBILEGL_GPU_HANG_BUDGET_MS=0"],
"inproc": ["--env", f"MOBILEGL_TRANSPORT=inproc;{SPLIT};MOBILEGL_GPU_HANG_BUDGET_MS=0", "--require-inproc"],
"spawn": ["--env", f"MOBILEGL_TRANSPORT=spawn;{SPLIT};MOBILEGL_GPU_HANG_BUDGET_MS=0",
"--use-pbuffer", "--require-spawn"],
}
for transport, tail in tails.items():
plan, = self.planned(cases, "DirectVulkan", transport)
self.assertEqual(plan["arguments"], head + tail, transport)
self.assertEqual(plan["timeout"], 1200)
self.assertEqual(plan["environment"], {
"MOBILEGL_ESPRYT_USE_ANGLE": "0", "MOBILEGL_TRACE_ANGLE_VARIANT": "ec889e6ea831",
"MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "1", "MOBILEGL_ITRP_FIX": "Auto"})
def test_case_specific_settings(self):
cases = [case(BLISS), case(ITERATIONRP), case("flush", coherent_as_flush=True),
case("lod", avoid_angle_llvmpipe_explicit_lod_bias=True), case("plain", ssim_threshold=0.985)]
gles = {plan["case"]: plan for plan in self.planned(cases, "DirectGLES", "monolith")}
vulkan = {plan["case"]: plan for plan in self.planned(cases, "DirectVulkan", "monolith")}
def flags(plan):
return [argument for argument in plan["arguments"][plan["arguments"].index("--env") + 2:]]
self.assertEqual(gles[BLISS]["environment"]["MOBILEGL_TRACE_ANGLE_VARIANT"], "90a62123d794")
self.assertEqual(vulkan[BLISS]["environment"]["MOBILEGL_TRACE_ANGLE_VARIANT"], "90a62123d794")
self.assertEqual(gles["plain"]["environment"], {
"MOBILEGL_ESPRYT_USE_ANGLE": "1", "MOBILEGL_TRACE_ANGLE_VARIANT": "ec889e6ea831",
"MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "0", "MOBILEGL_ITRP_FIX": "Auto"})
self.assertEqual(gles[ITERATIONRP]["environment"]["MOBILEGL_ITRP_FIX"], "ForceOn")
self.assertEqual(flags(gles[BLISS]), ["--avoid-angle-llvmpipe-sampler-mipmap-min-filter"])
self.assertEqual(flags(vulkan[BLISS]), [])
self.assertEqual(flags(gles["lod"]), ["--avoid-angle-llvmpipe-explicit-lod-bias"])
self.assertEqual(flags(vulkan["lod"]), [])
self.assertEqual(flags(gles["flush"]), ["--coherent-as-flush"])
self.assertEqual(flags(vulkan["flush"]), ["--coherent-as-flush"])
self.assertEqual(flags(gles["plain"]), [])
arguments = gles["plain"]["arguments"]
self.assertEqual(arguments[arguments.index("--ssim-threshold") + 1], "0.985")
self.assertEqual(arguments[arguments.index("--alternate-golden") + 1], "")
def test_misconfigured_case_lists_run_nothing(self):
for cases in ([case("a"), case("a")], [case("a", ci_backends=["DirectGLES"])], [{"name": "a"}], []):
status, output = self.run_shard(cases)
self.assertEqual(status, 2, cases)
self.assertIn("::error title=Android retrace shard misconfigured::", output)
self.assertEqual(self.log("adb.log"), [])
# ---- the loop ----------------------------------------------------------------------
def test_every_case_runs_in_order_on_one_install(self):
cases = [case("a", golden=self.fixture("a.png"), alternate_golden=self.fixture("a-mali.png")),
case("b", golden=self.fixture("b.png")), case("c")]
status, output = self.run_shard(cases, transport="inproc")
self.assertEqual(status, 0, output)
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b", "c"])
for entry in self.log("replay.log"):
self.assertEqual(entry["environment"]["MOBILEGL_TRACE_SKIP_INSTALL"], "1")
self.assertEqual(entry["environment"]["MOBILEGL_TRACE_POLL_INTERVAL"], "0.5")
adb = self.log("adb.log")
self.assertEqual(sum(1 for call in adb if call[:1] == ["install"]), 1)
self.assertEqual(sum(1 for call in adb if call[:3] == ["shell", "pm", "clear"]), 3)
self.assertEqual(sum(1 for call in adb if call[:3] == ["shell", "rm", "-f"]), 3)
self.assertIn(["shell", "rm", "-f", "/data/local/tmp/mobilegl-b.trace", "/data/local/tmp/mobilegl-b.golden.png",
"/data/local/tmp/mobilegl-b.alternate-golden.png"], adb)
self.assertEqual(self.log("avd.log"), [])
directory = self.results / "a-DirectVulkan"
self.assertEqual(json.loads((directory / "lane.json").read_text()),
{"lane": "inproc-lane", "transport": "inproc"})
self.assertTrue((directory / "a-DirectVulkan-golden.png").is_file())
self.assertTrue((directory / "a-DirectVulkan-alternate-golden.png").is_file())
self.assertFalse((self.results / "c-DirectVulkan" / "c-DirectVulkan-golden.png").exists())
whole = self.shard_status()
self.assertTrue(whole["complete"])
self.assertEqual([(entry["case"], entry["status"]) for entry in whole["cases"]],
[("a", "passed"), ("b", "passed"), ("c", "passed")])
self.assertIn("| b | passed | 1 |", (self.root / "summary.md").read_text())
def test_a_failed_case_does_not_stop_the_shard(self):
status, output = self.run_shard([case("a"), case("b"), case("c")], scenario={
"replay": {"b": [{"exit": 1, "message": "ssim=0.5"}]}})
self.assertEqual(status, 1)
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b", "c"])
self.assertEqual([self.status(name)["status"] for name in "abc"], ["passed", "failed", "passed"])
self.assertIn("::error title=Android retrace failed (DirectVulkan / monolith)::b: exit 1: ssim=0.5", output)
self.assertEqual(output.count("::error"), 1)
self.assertTrue(self.shard_status()["complete"])
self.assertTrue((self.results / "b-DirectVulkan" / "emulator.log").is_file())
def test_the_case_after_one_that_did_not_pass_starts_on_a_cold_boot(self):
status, _ = self.run_shard([case("a"), case("b"), case("c"), case("d")], scenario={
"replay": {"a": [{"exit": 75}], "b": [{"exit": 1}]}})
self.assertEqual(status, 1)
self.assertEqual([self.status(name)["status"] for name in "abcd"], ["passed", "failed", "passed", "passed"])
# One restart for a's infrastructure retry and one before c. A case that passed on
# its retry costs the next case nothing.
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start", "stop", "start"])
self.assertEqual(self.status("b")["notes"], [])
self.assertEqual(self.status("c")["notes"], ["emulator restarted before the case: "
"the case before it, b, did not pass"])
self.assertEqual(self.status("d")["notes"], [])
self.assertEqual(sum(1 for call in self.log("adb.log") if call[:1] == ["install"]), 3)
def test_a_failed_case_without_an_avd_to_restart_does_not_stop_the_shard(self):
status, _ = self.run_shard([case("a"), case("b")], scenario={"replay": {"a": [{"exit": 1}]}},
env={"AVD_NAME": ""})
self.assertEqual(status, 1)
self.assertEqual([self.status(name)["status"] for name in "ab"], ["failed", "passed"])
self.assertEqual(self.log("avd.log"), [])
def test_an_exit_0_without_a_passed_result_or_proof_fails(self):
status, _ = self.run_shard([case("a"), case("b")], transport="spawn", scenario={
"replay": {"a": [{"exit": 0, "result": False}], "b": [{"exit": 0, "proof": False}]}})
self.assertEqual(status, 1)
self.assertIn("without a passed result.json", self.status("a")["reason"])
self.assertIn("without a passed spawn transport proof", self.status("b")["reason"])
def test_an_infrastructure_failure_restarts_and_retries_once(self):
status, output = self.run_shard([case("a"), case("b")], scenario={
"replay": {"a": [{"exit": 75, "reason": "angle-surface-lost", "result": True, "passed": False}]}})
self.assertEqual(status, 0, output)
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "a", "b"])
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start"])
self.assertEqual(self.log("avd.log")[1], [
"start", "--avd-name", "mobilegl-ci", "--emulator-log", str(self.emulator_log),
"--pid-file", str(self.root / "emulator.pid"), "--gpu", "lavapipe", "--boot-timeout", "300"])
self.assertIn(["kill-server"], self.log("adb.log"))
self.assertEqual(sum(1 for call in self.log("adb.log") if call[:1] == ["install"]), 2)
record = self.status("a")
self.assertEqual((record["status"], record["attempts"], record["exit_codes"]), ("passed", 2, [75, 0]))
self.assertEqual(record["surface_lost_retries"], 1)
kept = self.results / "a-DirectVulkan" / "first-attempt"
self.assertTrue((kept / "result.first-attempt.json").is_file())
self.assertFalse((kept / "result.json").exists())
self.assertIn("::warning::Android emulator infrastructure failed", output)
self.assertIn("surface-lost retries: 1 (DirectVulkan, a)", (self.root / "summary.md").read_text())
self.assertTrue((self.results / "diagnostics" / "emulator-boot-1-ended.log").is_file())
self.assertEqual(self.shard_status()["emulator_boots"], 2)
def test_a_second_infrastructure_failure_is_the_verdict(self):
status, _ = self.run_shard([case("a"), case("b")], scenario={
"replay": {"a": [{"exit": 75}, {"exit": 75}]}})
self.assertEqual(status, 1)
self.assertEqual(self.status("a")["exit_codes"], [75, 75])
self.assertEqual(self.status("a")["status"], "failed")
self.assertEqual(self.status("b")["status"], "passed")
# The surface-lost count is this case's own: the reasons of earlier cases do not leak in.
self.assertEqual(self.status("b")["surface_lost_retries"], 0)
def test_the_deadline_cuts_a_running_case_and_names_the_rest(self):
status, output = self.run_shard([case("a"), case("b"), case("c")], "--deadline-seconds", "5", scenario={
"replay": {"b": [{"sleep": 30}]}})
self.assertEqual(status, 1)
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b"])
# b never had its own 1200 s, so the cut is not charged to it as a failure.
self.assertEqual([self.status(name)["status"] for name in "abc"], ["passed", "not-run", "not-run"])
self.assertRegex(self.status("b")["reason"], r"^cut by the shard deadline after \d+ s of a 1200 s budget$")
self.assertEqual(self.status("b")["exit_codes"], [124])
self.assertIn("deadline passed before this case started", self.status("c")["reason"])
self.assertIn("::error title=Android retrace not-run (DirectVulkan / monolith)::b: cut by the shard deadline",
output)
self.assertIn("::error title=Android retrace shard incomplete (DirectVulkan / monolith)::"
"1 verdict(s) for 3 case(s)", output)
self.assertFalse(self.shard_status()["complete"])
# Nothing is booted for a case the deadline keeps from starting.
self.assertEqual(self.log("avd.log"), [])
def test_a_case_that_runs_out_of_its_own_budget_fails(self):
# The deadline is far away, so the one second a has is its own budget.
with mock.patch.object(shard, "REPLAY_GRACE_SECONDS", 0):
status, _ = self.run_shard([case("a", timeout_seconds=1), case("b")], "--deadline-seconds", "600",
scenario={"replay": {"a": [{"sleep": 30}]}})
self.assertEqual(status, 1)
self.assertEqual([self.status(name)["status"] for name in "ab"], ["failed", "passed"])
self.assertEqual(self.status("a")["reason"], "the replay script exceeded its time budget")
self.assertTrue(self.shard_status()["complete"])
def test_an_infrastructure_retry_is_not_started_after_the_deadline(self):
def restart_past_the_deadline(runner, reason):
runner.started -= 1000
return True
with mock.patch.object(shard.Shard, "restart_emulator", restart_past_the_deadline):
status, _ = self.run_shard([case("a"), case("b")], "--deadline-seconds", "600",
scenario={"replay": {"a": [{"exit": 75}]}})
self.assertEqual(status, 1)
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a"])
self.assertEqual([self.status(name)["status"] for name in "ab"], ["not-run", "not-run"])
self.assertEqual(self.status("a")["reason"],
"the shard deadline passed before the infrastructure retry could start")
self.assertEqual(self.status("a")["exit_codes"], [75])
# ---- the time a shard needs --------------------------------------------------------
def test_needed_seconds_lets_every_case_use_its_budget(self):
cases = [case("a", timeout_seconds=1800), case("b", timeout_seconds=900), case("c", timeout_seconds="180")]
# Every case to the end of its budget, the longest once more, and the overhead per case.
self.assertEqual(shard.needed_seconds(cases), (2100 + 1200 + 480) + 2100 + 3 * 180)
self.assertEqual(shard.needed_seconds(cases[2:]), 480 + 480 + 180)
def test_a_deadline_of_needed_seconds_gives_every_hanging_case_its_verdict(self):
cases = [case(name, timeout_seconds=1) for name in "abc"]
# Scaled down: one second per case, and ten for what the fakes take around it.
with mock.patch.object(shard, "REPLAY_GRACE_SECONDS", 0), \
mock.patch.object(shard, "CASE_OVERHEAD_SECONDS", 10):
deadline = shard.needed_seconds(cases)
self.assertEqual(deadline, 3 + 1 + 30)
status, _ = self.run_shard(cases, "--deadline-seconds", str(deadline), scenario={
"replay": {name: [{"sleep": 60}] for name in "abc"}})
self.assertEqual(status, 1)
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b", "c"])
for name in "abc":
self.assertEqual((self.status(name)["status"], self.status(name)["reason"]),
("failed", "the replay script exceeded its time budget"))
self.assertTrue(self.shard_status()["complete"])
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start", "stop", "start"])
def test_every_shard_gets_its_own_deadline_and_job_timeout(self):
matrix = {"include": [
{"backend": {"name": "DirectGLES", "gpu": "software"}, "lane": "spawn-acceptance", "transport": "spawn",
"shard": 1, "shards": 2, "names": ["a", "b"],
"cases": [case("a", timeout_seconds=1800), case("b", timeout_seconds=1800)]},
{"backend": {"name": "DirectVulkan", "gpu": "lavapipe"}, "shard": 2, "shards": 2, "names": ["c"],
"cases": [case("c", timeout_seconds=180)]}]}
matrix_file = self.root / "matrix.json"
matrix_file.write_text(json.dumps(matrix))
def limits(cap, path=matrix_file):
output, report = io.StringIO(), io.StringIO()
with contextlib.redirect_stdout(output), contextlib.redirect_stderr(report):
status = budget.main(["--cap-seconds", str(cap), "--matrix-file", str(path)])
return status, output.getvalue(), report.getvalue()
self.assertEqual(budget.shard_needs(matrix), [("DirectGLES / spawn, shard 1/2", 2, 6660),
("DirectVulkan / monolith, shard 2/2", 1, 1140)])
# Under the cap every shard gets exactly what its cases need.
status, output, report = limits(6660)
self.assertEqual(status, 0, report)
rows = json.loads(output)["include"]
self.assertEqual([(row["deadline_seconds"], row["timeout_minutes"]) for row in rows],
[(6660, 111 + budget.JOB_OVERHEAD_MINUTES), (1140, 19 + budget.JOB_OVERHEAD_MINUTES)])
# Nothing but the two fields is added, and no case is dropped or changed.
self.assertEqual([{key: value for key, value in row.items()
if key not in ("deadline_seconds", "timeout_minutes")} for row in rows],
matrix["include"])
self.assertIn("2 shard(s), 0 capped at 6660 s; the most a shard needs is 6660 s", report)
# Over the cap a shard is cut to it and named; the other shard is untouched.
status, output, report = limits(6000)
self.assertEqual(status, 0, report)
rows = json.loads(output)["include"]
self.assertEqual([(row["deadline_seconds"], row["timeout_minutes"]) for row in rows],
[(6000, 100 + budget.JOB_OVERHEAD_MINUTES), (1140, 19 + budget.JOB_OVERHEAD_MINUTES)])
self.assertIn("DirectGLES / spawn, shard 1/2 would need 6660 s for all of its 2 case(s)", report)
self.assertNotIn("DirectVulkan / monolith, shard 2/2 would need", report)
# A cap that could not fit in a hosted job, and a matrix that cannot be read.
self.assertEqual(limits(0)[0], 2)
self.assertEqual(limits(360 * 60)[0], 2)
for broken in ({"include": []}, {"include": [{"names": ["a"]}]}, []):
matrix_file.write_text(json.dumps(broken))
status, output, _ = limits(7200)
self.assertEqual((status, output), (2, ""), broken)
def test_the_workflow_takes_its_limits_from_the_matrix(self):
text = (HERE.parent / ".github" / "workflows" / "apk.yml").read_text(encoding="utf-8")
retrace = text[text.index("\n retrace:"):text.index("\n retrace-summary:")]
cap, = re.findall(r'^\s+RETRACE_SHARD_DEADLINE_CAP_SECONDS: "(\d+)"\s*$', text, flags=re.MULTILINE)
# The runner's deadline and the job's limit are the two fields the budget script adds.
self.assertIn('--deadline-seconds "${{ matrix.deadline_seconds }}"', retrace)
self.assertEqual(re.findall(r"^ timeout-minutes: (.+?)\s*$", retrace, flags=re.MULTILINE),
["${{ matrix.timeout_minutes }}"])
self.assertIn('--cap-seconds "${RETRACE_SHARD_DEADLINE_CAP_SECONDS}"', text)
# The capped deadline and the job's overhead stay inside the six hours a hosted job may run.
self.assertLessEqual(-(-int(cap) // 60) + budget.JOB_OVERHEAD_MINUTES, budget.HOSTED_JOB_LIMIT_MINUTES)
def test_an_unhealthy_emulator_is_restarted_before_the_next_case(self):
status, output = self.run_shard([case("a"), case("b")], scenario={"offline_get_state_calls": 1})
self.assertEqual(status, 0, output)
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start"])
self.assertIn("emulator restarted before the case: adb state is offline", self.status("a")["notes"])
self.assertEqual(self.status("b")["notes"], [])
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b"])
def test_the_restart_switch_cold_boots_before_every_case_but_the_first(self):
status, _ = self.run_shard([case("a"), case("b"), case("c")],
env={"MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE": "1"})
self.assertEqual(status, 0)
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start", "stop", "start"])
self.assertEqual(sum(1 for call in self.log("adb.log") if call[:1] == ["install"]), 3)
def test_an_emulator_that_does_not_come_back_leaves_named_rows(self):
status, output = self.run_shard([case("a"), case("b"), case("c")], scenario={
"replay": {"a": [{"exit": 75}]}, "avd_start_exits": [2, 2]})
self.assertEqual(status, 1)
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a"])
self.assertEqual([self.status(name)["status"] for name in "abc"], ["failed", "not-run", "not-run"])
self.assertEqual(output.count("::error title=Android retrace"), 4)
for name in "abc":
self.assertTrue((self.results / f"{name}-DirectVulkan" / "lane.json").is_file())
def test_an_apk_that_does_not_install_runs_nothing(self):
status, _ = self.run_shard([case("a"), case("b")], scenario={"failing_installs": 9})
self.assertEqual(status, 1)
self.assertEqual(self.log("replay.log"), [])
self.assertEqual([self.status(name)["status"] for name in "ab"], ["not-run", "not-run"])
def test_processes_left_after_force_stop_are_killed_and_recorded(self):
# Listed on all four looks after the force-stop, gone after the kill.
status, _ = self.run_shard([case("a"), case("b")], scenario={"ps_calls_with_leftover": 4})
self.assertEqual(status, 0)
self.assertIn(["shell", "run-as", "top.mobilegl.plugin.trace", "kill", "-9", "4242"], self.log("adb.log"))
self.assertIn("processes left after force-stop: 4242 libMobileGLServer.so", self.status("a")["notes"])
self.assertEqual(self.status("b")["notes"], [])
self.assertEqual(self.log("avd.log"), [])
def test_a_process_that_is_still_exiting_is_not_reported(self):
status, _ = self.run_shard([case("a")], scenario={"ps_calls_with_leftover": 2})
self.assertEqual(status, 0)
self.assertEqual(self.status("a")["notes"], [])
self.assertNotIn("kill", [call[3] for call in self.log("adb.log") if call[:2] == ["shell", "run-as"]])
def test_processes_that_survive_the_kill_cost_an_emulator_restart(self):
status, _ = self.run_shard([case("a")], scenario={"ps_calls_with_leftover": 10})
self.assertEqual(status, 0)
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start"])
self.assertIn("the emulator was restarted to remove them", self.status("a")["notes"])
def test_a_failed_pm_clear_falls_back_to_a_reinstall(self):
status, _ = self.run_shard([case("a")], scenario={"pm_clear_fails": True})
self.assertEqual(status, 0)
adb = self.log("adb.log")
self.assertIn(["uninstall", "top.mobilegl.plugin.trace"], adb)
self.assertEqual(sum(1 for call in adb if call[:1] == ["install"]), 2)
@unittest.skipUnless(os.name == "posix", "signals and process groups")
def test_a_terminated_runner_takes_the_running_replay_with_it(self):
cases = [case("a"), case("b")]
(self.state / "scenario.json").write_text(json.dumps({"replay": {"a": [{"sleep": 60}]}}))
arguments = ["--apk-file", str(self.apk), "--backend", "DirectVulkan", "--gpu", "lavapipe",
"--transport", "monolith", "--lane", "monolith-lane", "--cases-json", json.dumps(cases),
"--result-root", str(self.results), "--fixture-root", str(self.root / "work")]
runner = subprocess.Popen([sys.executable, str(HERE / "trace_replay_shard.py")] + arguments,
env={**os.environ, **self.environment}, stdout=subprocess.DEVNULL)
deadline = time.monotonic() + 30
while not (self.state / "replay.pid").exists() and time.monotonic() < deadline:
time.sleep(0.05)
replay_pid = int((self.state / "replay.pid").read_text())
runner.send_signal(signal.SIGTERM)
self.assertEqual(runner.wait(timeout=30), 128 + signal.SIGTERM)
for _ in range(100):
try:
os.kill(replay_pid, 0)
except OSError:
break
time.sleep(0.05)
else:
self.fail("the replay outlived the runner")
# The step that runs afterwards names both cases.
status, _ = self.run_shard(cases, "--finalize")
self.assertEqual(status, 0)
self.assertEqual([self.status(name)["status"] for name in "ab"], ["not-run", "not-run"])
def test_finalize_names_every_case_without_a_device(self):
cases = [case("a", golden=self.fixture("a.png")), case("b")]
status, _ = self.run_shard(cases)
self.assertEqual(status, 0)
(self.results / "b-DirectVulkan" / "case-status.json").unlink()
calls_before = len(self.log("adb.log"))
status, _ = self.run_shard(cases + [case("c")], "--finalize")
self.assertEqual(status, 0)
self.assertEqual(len(self.log("adb.log")), calls_before)
self.assertEqual([self.status(name)["status"] for name in "abc"], ["passed", "not-run", "not-run"])
self.assertTrue((self.results / "c-DirectVulkan" / "lane.json").is_file())
self.assertFalse(self.shard_status()["complete"])
self.assertEqual(self.shard_status()["expected_cases"], ["a", "b", "c"])
if __name__ == "__main__":
unittest.main()
android-plugin/trace-replay-ci.sh
+108 -28
View File
@@ -62,6 +62,9 @@ copies benchmark.json (per-frame times plus mean/median/p95) out of the app, and
"passed" only means the replay reached the end of the trace without an error.
Pass --reuse-fixture to skip re-extracting and re-pushing the trace, for repeat
runs of a case whose fixture is already in /data/local/tmp.
Set MOBILEGL_TRACE_POLL_INTERVAL to the seconds between result polls (for example
0.5) to wait on a wall clock: the wait then ends --timeout-seconds plus a fifth after
the app starts, which is about how long that many one-second polls take.
Pass --env "K=V;K=V" (or set MOBILEGL_TRACE_ENV) to hand arbitrary environment
variables to the replay process. They are applied last, immediately before
libMobileGL.so is loaded, so they override every flag above; an entry with no "="
@@ -93,6 +96,21 @@ adb_device_path() {
MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL='*' "${ADB}" "$@"
}
# Milliseconds since the epoch; whole seconds where date has no %N.
now_ms() {
stamp="$(date +%s%3N 2>/dev/null || true)"
case "${stamp}" in
''|*[!0-9]*) stamp="$(($(date +%s) * 1000))" ;;
esac
printf '%s' "${stamp}"
}
# Appends "<phase> <ms>" to phase-timings.txt in the result directory, so a run records where
# its wall time went: push, install, replay, log pulls.
mark_phase() {
printf '%s %s\n' "$1" "$(now_ms)" >> "${result_dir}/phase-timings.txt" 2>/dev/null || true
}
next_arg() {
if [ "$#" -lt 2 ]; then
die "$1 requires a value"
@@ -411,6 +429,81 @@ copy_fixture_to_app() {
fi
}
# Waits for result.json with one poll per second; --timeout-seconds counts the polls.
wait_for_result_by_count() {
# One failed pidof is not an exit: under memory pressure the adb shell call
# itself can fail while the app is alive (seen with the process still in
# pidof and the activity still initializing right after the "exit").
missed_app_polls=0
for _ in $(seq 1 "${timeout_seconds}"); do
if adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then
break
fi
if adb_device_path shell pidof "${package_name}" >/dev/null 2>&1; then
saw_app_process=1
missed_app_polls=0
else
poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
if [ "${poll_adb_state}" != "device" ]; then
printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt"
break
fi
if [ "${saw_app_process}" -eq 1 ]; then
missed_app_polls=$((missed_app_polls + 1))
if [ "${missed_app_polls}" -ge 3 ]; then
app_exited=1
break
fi
fi
fi
sleep 1
done
}
# The same wait on a wall clock, selected by MOBILEGL_TRACE_POLL_INTERVAL (seconds between
# polls, fractions allowed). One adb call per poll asks for result.json and for the app
# process. The budget is --timeout-seconds of wall time plus a fifth: a poll of the
# one-second loop takes about 1.2 s with its two adb calls, so a replay gets the same time
# to finish under either loop. An exit needs three missed polls spread over at least three
# seconds, so a short interval keeps the one-second loop's tolerance for adb calls that fail
# under memory pressure.
wait_for_result_by_clock() {
poll_deadline=$((poll_started + timeout_seconds + timeout_seconds / 5))
missed_app_polls=0
first_missed_at=0
while [ "$(date +%s)" -lt "${poll_deadline}" ]; do
poll_output="$(adb_device_path shell "run-as ${package_name} ls ${app_dir}/output/result.json >/dev/null 2>&1 && echo mobilegl-poll-result; pidof ${package_name} >/dev/null 2>&1 && echo mobilegl-poll-alive; echo mobilegl-poll-end" 2>/dev/null | tr -d '\r' || true)"
case "${poll_output}" in
*mobilegl-poll-result*)
break
;;
*mobilegl-poll-alive*)
saw_app_process=1
missed_app_polls=0
;;
*)
poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
if [ "${poll_adb_state}" != "device" ]; then
printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt"
break
fi
if [ "${saw_app_process}" -eq 1 ]; then
poll_now="$(date +%s)"
if [ "${missed_app_polls}" -eq 0 ]; then
first_missed_at="${poll_now}"
fi
missed_app_polls=$((missed_app_polls + 1))
if [ "${missed_app_polls}" -ge 3 ] && [ "$((poll_now - first_missed_at))" -ge 3 ]; then
app_exited=1
break
fi
fi
;;
esac
sleep "${MOBILEGL_TRACE_POLL_INTERVAL}"
done
}
run_retrace() {
result_dir="${result_root}/${safe_case}-${backend}"
alternate_golden_app_path=""
@@ -427,13 +520,17 @@ run_retrace() {
fi
mkdir -p "${result_dir}"
printf 'script-started %s\n' "${script_started_ms}" > "${result_dir}/phase-timings.txt" 2>/dev/null || true
mark_phase fixture-pushed
# A repeat run of a case the previous invocation already installed and pushed only needs
# the on-device copy back into a fresh app output directory.
# MOBILEGL_TRACE_SKIP_INSTALL=1: the caller installed the APK itself (ColorOS shows an install-confirmation
# dialog whose tap drops adb for a few seconds, so the install has to be done and settled outside a run).
# dialog whose tap drops adb for a few seconds, so the install has to be done and settled outside a run;
# trace-replay-shard.sh installs once per emulator boot).
if [ "${reuse_fixture}" -eq 0 ] && [ "${MOBILEGL_TRACE_SKIP_INSTALL:-0}" != "1" ]; then
"${ADB}" install -r "$(host_path_for_adb "${apk_file}")"
fi
mark_phase apk-installed
copy_fixture_to_app
adb_device_path shell am force-stop "${package_name}"
"${ADB}" logcat -c
@@ -502,39 +599,20 @@ run_retrace() {
--ei crop_width "${crop_width}" \
--ei crop_height "${crop_height}"
adb_device_path shell "$@"
mark_phase app-started
app_exited=0
saw_app_process=0
# One failed pidof is not an exit: under memory pressure the adb shell call
# itself can fail while the app is alive (seen with the process still in
# pidof and the activity still initializing right after the "exit").
missed_app_polls=0
poll_started="$(date +%s)"
for _ in $(seq 1 "${timeout_seconds}"); do
if adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then
break
fi
if adb_device_path shell pidof "${package_name}" >/dev/null 2>&1; then
saw_app_process=1
missed_app_polls=0
else
poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
if [ "${poll_adb_state}" != "device" ]; then
printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt"
break
fi
if [ "${saw_app_process}" -eq 1 ]; then
missed_app_polls=$((missed_app_polls + 1))
if [ "${missed_app_polls}" -ge 3 ]; then
app_exited=1
break
fi
fi
fi
sleep 1
done
if [ -n "${MOBILEGL_TRACE_POLL_INTERVAL:-}" ]; then
wait_for_result_by_clock
else
wait_for_result_by_count
fi
mark_phase wait-ended
collect_run_diagnostics "${result_dir}"
mark_phase diagnostics-collected
if ! adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then
if [ -f "${result_dir}/adb-disconnected.txt" ]; then
echo "trace-replay-ci.sh: device disconnected while waiting for result.json" >&2
@@ -574,6 +652,7 @@ run_retrace() {
copy_app_artifact "${app_dir}/output/benchmark.json" "${result_dir}/benchmark.json"
fi
copy_texture_2d_dumps
mark_phase artifacts-copied
# A replay that wrote result.json but did not pass used to print nothing but
# the JSON, which for a non-zero statusCode says only "retrace failed with
@@ -696,6 +775,7 @@ PY
fi
}
script_started_ms="$(now_ms)"
mkdir -p "${fixture_root}" "${result_root}"
if [ "${reuse_fixture}" -eq 0 ]; then
prepare_fixture
android-plugin/trace-replay-shard.sh
+28
View File
@@ -0,0 +1,28 @@
#!/bin/sh
# MobileGL - android-plugin/trace-replay-shard.sh
# Copyright (c) 2025-2026 MobileGL-Dev
# Licensed under the GNU Lesser General Public License v3.0:
# https://www.gnu.org/licenses/gpl-3.0.txt
# https://www.gnu.org/licenses/lgpl-3.0.txt
# SPDX-License-Identifier: LGPL-3.0-only
# End of Source File Header
# Replays an ordered list of trace cases for one (lane, transport, backend) on one emulator
# boot, each through trace-replay-ci.sh. The loop, the per-case settings and the verdict files
# are in trace_replay_shard.py; `--help` lists the arguments.
#
# sh android-plugin/trace-replay-shard.sh \
# --apk-file FILE --backend DirectGLES|DirectVulkan --gpu GPU \
# --transport monolith|inproc|spawn --lane LANE \
# --cases-env VARIABLE | --cases-file FILE | --cases-json JSON \
# [--deadline-seconds N] [--dry-run | --finalize]
#
# Environment:
# AVD_NAME, EMULATOR_LOG, EMULATOR_PID_FILE the emulator to restart when it stops answering
# MOBILEGL_GPU_HANG_BUDGET_MS forwarded to every replay (required)
# MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE 1 cold boots the emulator before every case
# MOBILEGL_SHARD_POLL_INTERVAL seconds between result polls (default 0.5);
# empty selects trace-replay-ci.sh's 1 s loop
set -eu
exec "${PYTHON:-python3}" "$(dirname "$0")/trace_replay_shard.py" "$@"
android-plugin/trace_replay_shard.py
+792
View File
@@ -0,0 +1,792 @@
#!/usr/bin/env python3
# MobileGL - android-plugin/trace_replay_shard.py
# Copyright (c) 2025-2026 MobileGL-Dev
# Licensed under the GNU Lesser General Public License v3.0:
# https://www.gnu.org/licenses/gpl-3.0.txt
# https://www.gnu.org/licenses/lgpl-3.0.txt
# SPDX-License-Identifier: LGPL-3.0-only
# End of Source File Header
"""Replay an ordered list of trace cases on one emulator boot.
One shard is one (lane, transport, backend). Each case runs through trace-replay-ci.sh with
the per-case and per-transport settings of the APK workflow, in a fresh app process with
cleared app data. A failing case does not stop the shard: the case after it starts on a cold
boot, every case gets a verdict file, and the exit status is non-zero when a case failed or a
verdict is missing.
Layout under --result-root (the directory the workflow uploads):
<case>-<backend>/ what trace-replay-ci.sh writes, plus lane.json, the golden
copies the summary renders, and case-status.json
<case>-<backend>/first-attempt/ files of an attempt that was retried
shard-status.json every case of the shard with its verdict
diagnostics/ emulator logs kept across restarts
"""
import argparse
import json
import os
import re
import shlex
import shutil
import signal
import subprocess
import sys
import time
from pathlib import Path
INFRASTRUCTURE_FAILURE_EXIT_CODE = 75
TIMEOUT_EXIT_CODE = 124
BACKENDS = ("DirectGLES", "DirectVulkan")
TRANSPORTS = ("monolith", "inproc", "spawn")
PASSED, FAILED, NOT_RUN = "passed", "failed", "not-run"
# Cases with settings of their own. Bliss needs the newer packaged ANGLE build and, on
# DirectGLES, sampler mipmap min-filter downgrading on ANGLE llvmpipe; iterationRP needs its
# SPIR-V repairs forced on.
BLISS_CASE = "minecraft-1.21.4-fabric-iris-bliss-in-world"
ITERATIONRP_CASE = "minecraft-1.21.4-fabric-iris-iterationrp-in-world"
DEFAULT_ANGLE_VARIANT = "ec889e6ea831"
BLISS_ANGLE_VARIANT = "90a62123d794"
# The inproc and spawn lanes pass the same client settings, so they differ only in which
# process runs the server role.
SPLIT_CLIENT_SETTINGS = "MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1"
REQUIRED_CASE_KEYS = ("name", "trace_archive", "trace_file", "golden", "target_call", "width", "height",
"crop_x", "crop_y", "crop_width", "crop_height", "timeout_seconds")
# Seconds trace-replay-ci.sh gets beyond the case's own budget for install, push and log pulls.
REPLAY_GRACE_SECONDS = 300
BOOT_TIMEOUT_SECONDS = 300
# Seconds a case may take around its replays: the app reset, the failure diagnostics and the
# cold boot that follows a case that did not pass.
CASE_OVERHEAD_SECONDS = 180
def say(message):
print(message, flush=True)
def pause(seconds):
time.sleep(seconds)
def safe_case(name):
return re.sub(r"[^A-Za-z0-9._-]", "_", name)
def as_text(value):
if value is True:
return "true"
if value is False:
return "false"
return "" if value is None else str(value)
def is_true(value):
return value is True or value == "true"
def load_cases(text, backend):
"""The case objects of the APK matrix, as text ready for the command line."""
try:
# Numbers stay the literal text of the matrix, so 0.985 is passed on as 0.985.
cases = json.loads(text, parse_float=str, parse_int=str)
except json.JSONDecodeError as error:
raise ValueError(f"the case list is not JSON: {error}") from error
if not isinstance(cases, list) or not cases:
raise ValueError("the case list must be a non-empty JSON array of case objects")
names = set()
for index, case in enumerate(cases):
if not isinstance(case, dict):
raise ValueError(f"case {index} is not an object; pass the case objects of the APK matrix")
missing = [key for key in REQUIRED_CASE_KEYS if case.get(key) in (None, "")]
if missing:
raise ValueError(f"case {case.get('name', index)} lacks {', '.join(missing)}")
name = case["name"]
if name in names:
raise ValueError(f"case {name} is listed twice")
names.add(name)
backends = case.get("ci_backends")
if backends is not None and backend not in backends:
raise ValueError(f"case {name} does not run on {backend} (ci_backends: {backends})")
return cases
def needed_seconds(cases):
"""Wall time in which every case of a shard can use its whole time budget.
It covers each case running until its replay script is stopped, one infrastructure retry
that does so again for the longest budget, and CASE_OVERHEAD_SECONDS per case. A shard
deadline at or above this never stops a case that is within its own budget, unless more
than one case needs a retry as long as its first attempt.
"""
budgets = [int(as_text(case["timeout_seconds"])) + REPLAY_GRACE_SECONDS for case in cases]
return sum(budgets) + max(budgets) + CASE_OVERHEAD_SECONDS * len(budgets)
def replay_environment(backend, case_name):
return {
"MOBILEGL_ESPRYT_USE_ANGLE": "1" if backend == "DirectGLES" else "0",
"MOBILEGL_TRACE_ANGLE_VARIANT": BLISS_ANGLE_VARIANT if case_name == BLISS_CASE else DEFAULT_ANGLE_VARIANT,
"MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "1" if backend == "DirectVulkan" else "0",
"MOBILEGL_ITRP_FIX": "ForceOn" if case_name == ITERATIONRP_CASE else "Auto",
}
def transport_arguments(transport, hang_budget_ms):
hang_budget = f"MOBILEGL_GPU_HANG_BUDGET_MS={hang_budget_ms}"
if transport == "inproc":
return ["--env", f"MOBILEGL_TRANSPORT=inproc;{SPLIT_CLIENT_SETTINGS};{hang_budget}", "--require-inproc"]
if transport == "spawn":
# No MOBILEGL_IPC_SERVER_PATH: only the app can spell nativeLibraryDir, and
# TraceReplayActivity resolves it when the environment asks for spawn. The pbuffer is
# required: a spawned server cannot be handed the client's window.
return ["--env", f"MOBILEGL_TRANSPORT=spawn;{SPLIT_CLIENT_SETTINGS};{hang_budget}",
"--use-pbuffer", "--require-spawn"]
return ["--env", f"MOBILEGL_TRANSPORT=monolith;{hang_budget}"]
def replay_arguments(options, case, hang_budget_ms):
name = case["name"]
arguments = [
"--apk-file", options.apk_file,
"--package", options.package,
"--backend", options.backend,
"--result-root", options.result_root,
"--fixture-root", options.fixture_root,
"--case", name,
"--trace-archive", as_text(case["trace_archive"]),
"--trace-file", as_text(case["trace_file"]),
"--golden", as_text(case["golden"]),
"--alternate-golden", as_text(case.get("alternate_golden") or ""),
"--target-call", as_text(case["target_call"]),
"--width", as_text(case["width"]),
"--height", as_text(case["height"]),
"--ssim-threshold", as_text(case.get("ssim_threshold") or "0.99"),
"--crop-x", as_text(case["crop_x"]),
"--crop-y", as_text(case["crop_y"]),
"--crop-width", as_text(case["crop_width"]),
"--crop-height", as_text(case["crop_height"]),
"--timeout-seconds", as_text(case["timeout_seconds"]),
]
arguments += transport_arguments(options.transport, hang_budget_ms)
if options.backend == "DirectGLES" and name == BLISS_CASE:
arguments.append("--avoid-angle-llvmpipe-sampler-mipmap-min-filter")
if options.backend == "DirectGLES" and is_true(case.get("avoid_angle_llvmpipe_explicit_lod_bias")):
arguments.append("--avoid-angle-llvmpipe-explicit-lod-bias")
if is_true(case.get("coherent_as_flush")):
arguments.append("--coherent-as-flush")
return arguments
def read_json(path):
try:
return json.loads(Path(path).read_text(encoding="utf-8"))
except (OSError, ValueError):
return None
def write_json(path, value):
Path(path).write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8")
def one_line(text):
return " ".join(str(text).split())
class Shard:
def __init__(self, options, cases, environ):
self.options = options
self.cases = cases
self.environ = dict(environ)
self.adb_command = shlex.split(environ.get("ADB", "adb"))
directory = Path(__file__).resolve().parent
self.replay_command = shlex.split(environ.get("MOBILEGL_SHARD_REPLAY_COMMAND") or
f"sh {shlex.quote(str(directory / 'trace-replay-ci.sh'))}")
self.avd_command = shlex.split(environ.get("MOBILEGL_SHARD_AVD_COMMAND") or
f"sh {shlex.quote(str(directory / 'run-avd-ci.sh'))}")
self.hang_budget_ms = environ.get("MOBILEGL_GPU_HANG_BUDGET_MS")
self.restart_each_case = environ.get("MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE", "0") == "1"
self.poll_interval = environ.get("MOBILEGL_SHARD_POLL_INTERVAL", "0.5")
self.avd_name = environ.get("AVD_NAME", "")
self.emulator_log = environ.get("EMULATOR_LOG", "")
self.emulator_pid_file = environ.get("EMULATOR_PID_FILE", "")
self.result_root = Path(options.result_root)
self.diagnostics = self.result_root / "diagnostics"
self.started = time.monotonic()
self.boots = 1
self.install_seconds = []
self.records = {}
self.aborted = ""
# The case before the current one when it did not pass; the next case gets a cold boot.
self.unclean_case = ""
# ---- bookkeeping -------------------------------------------------------------------
def result_dir(self, case):
return self.result_root / f"{safe_case(case['name'])}-{self.options.backend}"
def record_for(self, case, status, reason, **extra):
record = {"case": case["name"], "backend": self.options.backend, "transport": self.options.transport,
"lane": self.options.lane, "status": status, "reason": reason}
record.update(extra)
return record
def write_summary_inputs(self, case):
"""lane.json and the golden copies the retrace summary reads next to each result."""
directory = self.result_dir(case)
directory.mkdir(parents=True, exist_ok=True)
write_json(directory / "lane.json", {"lane": self.options.lane, "transport": self.options.transport})
stem = f"{safe_case(case['name'])}-{self.options.backend}"
for key, suffix in (("golden", "golden"), ("alternate_golden", "alternate-golden")):
source = as_text(case.get(key) or "")
if source and os.path.isfile(source) and os.path.getsize(source) > 0:
shutil.copyfile(source, directory / f"{stem}-{suffix}.png")
def save_record(self, case, record):
self.records[case["name"]] = record
self.write_summary_inputs(case)
write_json(self.result_dir(case) / "case-status.json", record)
self.write_shard_status()
def write_shard_status(self, complete=False):
self.result_root.mkdir(parents=True, exist_ok=True)
records = [self.records.get(case["name"]) or
self.record_for(case, NOT_RUN, "the shard has not reached this case") for case in self.cases]
write_json(self.result_root / "shard-status.json", {
"schema": 1, "lane": self.options.lane, "transport": self.options.transport,
"backend": self.options.backend, "complete": complete,
# The Actions job that ran the shard, so the artifact cleanup can tell whose job failed.
"job_id": self.environ.get("MOBILEGL_SHARD_JOB_ID", ""),
"expected_cases": [case["name"] for case in self.cases], "cases": records,
"emulator_boots": self.boots, "install_seconds": self.install_seconds})
def remaining_seconds(self):
if self.options.deadline_seconds <= 0:
return None
return self.options.deadline_seconds - (time.monotonic() - self.started)
def deadline_passed(self):
remaining = self.remaining_seconds()
return remaining is not None and remaining <= 0
# ---- processes ---------------------------------------------------------------------
def run(self, command, timeout=None, capture=False, extra_env=None):
"""Returns (exit status, output). A timeout kills the whole process group and reads as 124."""
environment = dict(self.environ)
environment.update(extra_env or {})
own_group = os.name == "posix"
try:
process = subprocess.Popen(command, env=environment, start_new_session=own_group,
stdout=subprocess.PIPE if capture else None,
stderr=subprocess.STDOUT if capture else None)
except OSError as error:
return 127, str(error)
def kill():
if own_group:
try:
os.killpg(process.pid, signal.SIGKILL)
except OSError:
pass
process.kill()
try:
output, _ = process.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
kill()
output, _ = process.communicate()
return TIMEOUT_EXIT_CODE, (output or b"").decode("utf-8", "replace")
except BaseException:
# The job was canceled or timed out. The child is in a session of its own, so it
# would otherwise keep driving the device behind the steps that still run.
kill()
raise
return process.returncode, (output or b"").decode("utf-8", "replace")
def adb(self, *arguments, timeout=60):
status, output = self.run(self.adb_command + list(arguments), timeout=timeout, capture=True)
return status, output.replace("\r", "")
# ---- emulator ----------------------------------------------------------------------
def can_restart(self):
return bool(self.avd_name and self.options.gpu and self.emulator_log and self.emulator_pid_file)
def emulator_pid(self):
try:
return int(Path(self.emulator_pid_file).read_text().strip())
except (OSError, ValueError):
return None
def emulator_process_alive(self):
pid = self.emulator_pid()
if pid is None:
return True # nothing to check against, for example a real device
try:
os.kill(pid, 0)
except OSError:
return False
return True
def health_problem(self):
"""Empty when the device answers; otherwise what did not."""
if not self.emulator_process_alive():
return "the emulator process is gone"
status, output = self.adb("get-state", timeout=15)
if status != 0 or output.strip() != "device":
return f"adb state is {output.strip() or 'unknown'}"
status, output = self.adb("shell", "getprop", "sys.boot_completed", timeout=15)
if status != 0 or output.strip() != "1":
return "sys.boot_completed is not 1"
# A framework call: boot_completed stays 1 across a system_server restart.
status, output = self.adb("shell", "pm", "path", self.options.package, timeout=30)
if status != 0 or "package:" not in output:
return "the package manager does not list the trace APK"
return ""
def keep_emulator_log(self, label):
if self.emulator_log and os.path.isfile(self.emulator_log):
self.diagnostics.mkdir(parents=True, exist_ok=True)
shutil.copyfile(self.emulator_log, self.diagnostics / f"emulator-boot-{self.boots}-{label}.log")
def prepare_booted_device(self):
# A shard outlives the screen timeout a single replay never reached.
self.adb("shell", "svc", "power", "stayon", "true", timeout=30)
return self.install()
def install(self):
started = time.monotonic()
status, output = self.adb("install", "-r", self.options.apk_file, timeout=300)
self.install_seconds.append(round(time.monotonic() - started, 1))
say(output.strip())
return status == 0
def restart_emulator(self, reason):
"""Stop, cold boot and reinstall. False when the emulator did not come back."""
if not self.can_restart():
say(f"trace-replay-shard: cannot restart the emulator ({reason}): no AVD is configured")
return False
say(f"trace-replay-shard: restarting the emulator: {reason}")
common = ["--avd-name", self.avd_name, "--emulator-log", self.emulator_log,
"--pid-file", self.emulator_pid_file]
for attempt in (1, 2):
# The start truncates the log, and the boot that failed is the one worth reading.
self.keep_emulator_log(f"attempt-{attempt}" if attempt > 1 else "ended")
old_pid = self.emulator_pid()
self.run(self.avd_command + ["stop"] + common, timeout=120)
self.wait_for_exit(old_pid)
self.adb("kill-server", timeout=30)
pause(2)
self.boots += 1
status, _ = self.run(self.avd_command + ["start"] + common +
["--gpu", self.options.gpu, "--boot-timeout", str(BOOT_TIMEOUT_SECONDS)],
timeout=2 * BOOT_TIMEOUT_SECONDS + 60)
if status == 0 and self.prepare_booted_device():
return True
say(f"trace-replay-shard: emulator restart attempt {attempt} failed")
return False
def wait_for_exit(self, pid):
if pid is None or os.name != "posix":
return
for _ in range(60):
try:
os.kill(pid, 0)
except OSError:
return
pause(0.5)
try:
os.kill(pid, signal.SIGKILL)
except OSError:
pass
# ---- app state ---------------------------------------------------------------------
def leftover_processes(self):
"""Processes still running under the app's uid, as 'pid name' strings."""
status, user = self.adb("shell", "run-as", self.options.package, "id", "-un", timeout=30)
user = user.strip()
if status != 0 or not re.fullmatch(r"[A-Za-z0-9_]+", user):
return None
status, listing = self.adb("shell", "ps", "-A", "-o", "USER,PID,NAME", timeout=30)
if status != 0:
return None
found = []
for line in listing.splitlines():
fields = line.split()
if len(fields) >= 3 and fields[0] == user and fields[1].isdigit():
found.append(f"{fields[1]} {fields[2]}")
return found
def settled_leftover_processes(self, polls):
"""leftover_processes once the app's uid is empty, or as it still is after `polls` looks.
A killed process stays listed for a moment, so one look right after the kill would
report every force-stop as incomplete.
"""
found = self.leftover_processes()
for _ in range(polls - 1):
if not found:
break
pause(0.5)
found = self.leftover_processes()
return found
def reset_app_state(self):
"""Fresh process and fresh app data, as a new install gives. Returns notes for the record."""
notes = []
package = self.options.package
self.adb("shell", "am", "force-stop", package, timeout=60)
leftovers = self.settled_leftover_processes(polls=4)
if leftovers is None:
notes.append("the app's processes could not be listed after force-stop")
if leftovers:
# A force-stop is expected to take a spawned server with it; say so when it did not.
notes.append("processes left after force-stop: " + ", ".join(leftovers))
self.adb("shell", "run-as", package, "kill", "-9", *[entry.split()[0] for entry in leftovers],
timeout=30)
if self.settled_leftover_processes(polls=6):
notes.append("the leftover processes could not be killed")
if self.restart_emulator("app processes survived force-stop and kill"):
notes.append("the emulator was restarted to remove them")
status, output = self.adb("shell", "pm", "clear", package, timeout=60)
probe_status, _ = self.adb("shell", "run-as", package, "ls", f"/data/user/0/{package}", timeout=30)
if status != 0 or "Success" not in output or probe_status != 0:
# The state a fresh install leaves, reached the other way.
notes.append("pm clear did not leave a usable app data directory; reinstalled the APK")
self.adb("uninstall", package, timeout=120)
self.install()
return notes
def remove_case_leftovers(self, case):
name = safe_case(case["name"])
base = f"/data/local/tmp/mobilegl-{name}"
self.adb("shell", "rm", "-f", f"{base}.trace", f"{base}.golden.png", f"{base}.alternate-golden.png",
timeout=60)
self.adb("shell", "am", "force-stop", self.options.package, timeout=60)
shutil.rmtree(Path(self.options.fixture_root) / name, ignore_errors=True)
# ---- one case ----------------------------------------------------------------------
def reason_lines(self):
try:
return (self.result_root / "infrastructure-failure-reason.txt").read_text().splitlines()
except OSError:
return []
def run_attempt(self, case):
"""Returns (exit status, seconds, how the shard deadline cut the attempt short or "")."""
case_budget = int(as_text(case["timeout_seconds"])) + REPLAY_GRACE_SECONDS
budget = case_budget
remaining = self.remaining_seconds()
if remaining is not None and remaining < case_budget:
budget = max(remaining, 1.0)
environment = replay_environment(self.options.backend, case["name"])
# The shard installs the APK once per boot.
environment["MOBILEGL_TRACE_SKIP_INSTALL"] = "1"
environment["MOBILEGL_TRACE_POLL_INTERVAL"] = self.poll_interval
started = time.monotonic()
status, _ = self.run(self.replay_command + replay_arguments(self.options, case, self.hang_budget_ms),
timeout=budget, extra_env=environment)
elapsed = round(time.monotonic() - started, 1)
cut = ""
if status == TIMEOUT_EXIT_CODE and budget < case_budget:
# The case never had its own budget, so the timeout says nothing about the case.
cut = f"cut by the shard deadline after {elapsed:.0f} s of a {case_budget} s budget"
return status, elapsed, cut
def keep_first_attempt(self, case):
"""Move the retried attempt's files aside so the retry starts from an empty directory.
result.json is renamed: the summary keys rows on any result.json below a directory
that names the case, and the retry's is the one that counts.
"""
directory = self.result_dir(case)
if not directory.is_dir():
return
kept = directory / "first-attempt"
kept.mkdir(exist_ok=True)
for entry in directory.iterdir():
if entry.is_file():
name = "result.first-attempt.json" if entry.name == "result.json" else entry.name
entry.replace(kept / name)
def verdict(self, case, status):
directory = self.result_dir(case)
if status == 0:
result = read_json(directory / "result.json")
if not isinstance(result, dict) or result.get("passed") is not True:
return FAILED, "the replay exited 0 without a passed result.json"
if self.options.transport != "monolith":
proof = read_json(directory / "transport-proof.json")
if not (isinstance(proof, dict) and proof.get("passed") is True and
proof.get("required_transport") == self.options.transport):
return FAILED, f"the replay exited 0 without a passed {self.options.transport} transport proof"
return PASSED, ""
if status == INFRASTRUCTURE_FAILURE_EXIT_CODE:
return FAILED, "the emulator infrastructure failed again on the retry"
if status == TIMEOUT_EXIT_CODE:
return FAILED, "the replay script exceeded its time budget"
result = read_json(directory / "result.json")
if not isinstance(result, dict):
return FAILED, f"exit {status} without a result.json"
if result.get("passed") is True:
# The picture matched, so what failed is a check that runs after it.
return FAILED, f"exit {status} after a passed result.json: the transport proof or a log check failed"
return FAILED, f"exit {status}: {one_line(result.get('message', ''))}"
def collect_failure_diagnostics(self, case):
directory = self.result_dir(case)
directory.mkdir(parents=True, exist_ok=True)
logcat = directory / "logcat.txt"
if not logcat.is_file() or logcat.stat().st_size == 0:
# trace-replay-ci.sh pulls this itself unless it was killed first.
_, output = self.adb("logcat", "-d", "-t", "20000", timeout=30)
logcat.write_text(output, encoding="utf-8", errors="replace")
app_output = f"/data/user/0/{self.options.package}/files/trace-replay/output"
for name in ("retrace.log", "mobilegl.client.log", "mobilegl.server.log"):
status, output = self.adb("exec-out", "run-as", self.options.package, "cat",
f"{app_output}/{name}", timeout=30)
if status == 0 and output and not output.startswith("cat: "):
(directory / name).write_text(output, encoding="utf-8", errors="replace")
if self.emulator_log and os.path.isfile(self.emulator_log):
shutil.copyfile(self.emulator_log, directory / "emulator.log")
# A vanished emulator looks the same whether the host OOM killer took it or the
# renderer faulted. These two say which.
_, memory = self.run(["free", "-h"], timeout=30, capture=True)
(directory / "host-memory.txt").write_text(memory, encoding="utf-8")
_, dmesg = self.run(["sudo", "-n", "dmesg", "-T"], timeout=30, capture=True)
(directory / "host-dmesg.txt").write_text("\n".join(dmesg.splitlines()[-300:]) + "\n", encoding="utf-8")
def run_case(self, index, case):
name = case["name"]
notes = []
seconds = {}
case_started = time.monotonic()
if self.deadline_passed():
return self.record_for(case, NOT_RUN, "the shard deadline passed before this case started")
restarted_for = ""
if self.restart_each_case and index > 0:
restarted_for = "MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE=1"
elif self.unclean_case and self.can_restart():
# A crashed, hung or killed replay can leave the guest or the host renderer in a
# state no adb query shows, and this case's verdict must not depend on it.
restarted_for = f"the case before it, {self.unclean_case}, did not pass"
else:
restarted_for = self.health_problem()
if restarted_for:
notes.append(f"emulator restarted before the case: {restarted_for}")
if not self.restart_emulator(restarted_for):
self.aborted = "the emulator could not be restarted"
return self.record_for(case, NOT_RUN, self.aborted, notes=notes)
notes += self.reset_app_state()
seconds["reset"] = round(time.monotonic() - case_started, 1)
reasons_before = len(self.reason_lines())
exit_codes = []
replay_seconds = []
status, elapsed, cut = self.run_attempt(case)
exit_codes.append(status)
replay_seconds.append(elapsed)
surface_lost = 0
if status == INFRASTRUCTURE_FAILURE_EXIT_CODE:
say(f"::warning::Android emulator infrastructure failed; restarting it and retrying {name} once.")
new_reasons = self.reason_lines()[reasons_before:]
# A lost surface is retried, so it is counted where a rate spike shows.
surface_lost = sum(1 for line in new_reasons if "angle-surface-lost" in line)
if surface_lost:
self.append_step_summary(f"surface-lost retries: {surface_lost} ({self.options.backend}, {name})\n")
notes.append("infrastructure retry: " + (", ".join(new_reasons) or "no reason recorded"))
self.keep_first_attempt(case)
if not self.restart_emulator(f"infrastructure failure in {name}"):
self.aborted = "the emulator could not be restarted"
elif self.deadline_passed():
cut = "the shard deadline passed before the infrastructure retry could start"
else:
notes += self.reset_app_state()
status, elapsed, cut = self.run_attempt(case)
exit_codes.append(status)
replay_seconds.append(elapsed)
seconds["replay"] = replay_seconds
finish_started = time.monotonic()
if self.aborted:
verdict, reason = FAILED, "the emulator did not come back for the infrastructure retry"
elif cut:
verdict, reason = NOT_RUN, cut
else:
verdict, reason = self.verdict(case, status)
if verdict != PASSED:
self.collect_failure_diagnostics(case)
if not self.aborted:
self.remove_case_leftovers(case)
seconds["finish"] = round(time.monotonic() - finish_started, 1)
seconds["total"] = round(time.monotonic() - case_started, 1)
return self.record_for(case, verdict, reason, attempts=len(exit_codes), exit_codes=exit_codes,
surface_lost_retries=surface_lost, seconds=seconds, notes=notes)
# ---- the shard ---------------------------------------------------------------------
def append_step_summary(self, text):
path = self.environ.get("GITHUB_STEP_SUMMARY")
if path:
with open(path, "a", encoding="utf-8") as summary:
summary.write(text)
def run_all(self):
self.write_shard_status()
say(f"trace-replay-shard: {len(self.cases)} case(s) on {self.options.backend} / "
f"{self.options.transport} ({self.options.lane})")
if not self.prepare_booted_device():
if not self.restart_emulator("the trace APK did not install"):
self.aborted = "the trace APK could not be installed"
for index, case in enumerate(self.cases):
say(f"::group::{case['name']} ({index + 1}/{len(self.cases)})")
if self.aborted:
record = self.record_for(case, NOT_RUN, self.aborted)
else:
try:
record = self.run_case(index, case)
except Exception as error: # one case's bookkeeping must not cost the others their verdicts
record = self.record_for(case, FAILED, f"the shard runner failed: {one_line(error)}")
self.unclean_case = "" if record["status"] == PASSED else case["name"]
self.save_record(case, record)
say(f"trace-replay-shard: {case['name']}: {record['status']}"
+ (f" ({record['reason']})" if record["reason"] else ""))
say("::endgroup::")
return self.finish()
def finish(self):
"""Tally from the files on disk, so a verdict the loop never wrote counts as missing."""
label = f"{self.options.backend} / {self.options.transport}"
rows = []
verdicts = 0
failed = []
for case in self.cases:
record = read_json(self.result_dir(case) / "case-status.json")
if not isinstance(record, dict) or record.get("case") != case["name"]:
record = self.record_for(case, NOT_RUN, "no verdict file was written")
if record["status"] in (PASSED, FAILED):
verdicts += 1
if record["status"] == PASSED:
result = read_json(self.result_dir(case) / "result.json")
if not isinstance(result, dict) or result.get("passed") is not True:
record = dict(record, status=FAILED, reason="the verdict is passed but result.json is not")
if record["status"] != PASSED:
failed.append(record)
result = read_json(self.result_dir(case) / "result.json")
ssim = result.get("ssim", "") if isinstance(result, dict) else ""
seconds = record.get("seconds", {}).get("total", "")
note = one_line("; ".join([record.get("reason", "")] + record.get("notes", [])).strip("; "))
rows.append(f"| {case['name']} | {record['status']} | {record.get('attempts', 0)} | {seconds} | "
f"{ssim} | {note.replace('|', '/')} |")
self.write_shard_status(complete=verdicts == len(self.cases))
summary = [f"### Android retrace: {label} ({self.options.lane})", "",
f"{len(self.cases) - len(failed)} of {len(self.cases)} passed, {self.boots} emulator boot(s).", "",
"| case | status | attempts | seconds | SSIM | note |", "| --- | --- | --- | --- | --- | --- |"]
self.append_step_summary("\n".join(summary + rows) + "\n\n")
for record in failed:
say(f"::error title=Android retrace {record['status']} ({label})::"
f"{record['case']}: {record.get('reason') or record['status']}")
if verdicts != len(self.cases):
say(f"::error title=Android retrace shard incomplete ({label})::"
f"{verdicts} verdict(s) for {len(self.cases)} case(s)")
say(f"trace-replay-shard: {len(self.cases) - len(failed)} passed, {len(failed)} not passed, "
f"{verdicts} verdict(s) for {len(self.cases)} case(s)")
return 0 if not failed and verdicts == len(self.cases) else 1
def finalize(self):
"""Give every case its summary inputs and a verdict file without touching the device.
Runs after the replay step whatever its outcome, so a shard that was killed or never
started still uploads one named row per case.
"""
for case in self.cases:
record = read_json(self.result_dir(case) / "case-status.json")
if isinstance(record, dict) and record.get("case") == case["name"]:
self.records[case["name"]] = record
self.write_summary_inputs(case)
else:
self.save_record(case, self.record_for(case, NOT_RUN, "the shard did not reach this case"))
previous = read_json(self.result_root / "shard-status.json")
if isinstance(previous, dict):
self.boots = previous.get("emulator_boots", self.boots)
self.install_seconds = previous.get("install_seconds", self.install_seconds)
self.write_shard_status(
complete=all(self.records[case["name"]]["status"] in (PASSED, FAILED) for case in self.cases))
return 0
def dry_run(self):
for case in self.cases:
say(json.dumps({
"case": case["name"],
"environment": replay_environment(self.options.backend, case["name"]),
"arguments": replay_arguments(self.options, case, self.hang_budget_ms),
"timeout": int(as_text(case["timeout_seconds"])) + REPLAY_GRACE_SECONDS,
}, sort_keys=True))
return 0
def parse_arguments(argv):
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--apk-file", required=True, help="Trace APK to install once per emulator boot.")
parser.add_argument("--package", default="top.mobilegl.plugin.trace")
parser.add_argument("--backend", required=True, choices=BACKENDS)
parser.add_argument("--transport", required=True, choices=TRANSPORTS)
parser.add_argument("--lane", required=True, help="Lane name written to lane.json, e.g. inproc-acceptance.")
parser.add_argument("--gpu", default="", help="Emulator -gpu mode, needed to restart the emulator.")
source = parser.add_mutually_exclusive_group(required=True)
source.add_argument("--cases-json", help="JSON array of the APK matrix's case objects.")
source.add_argument("--cases-file", help="File holding that JSON array.")
source.add_argument("--cases-env", help="Name of the environment variable holding that JSON array.")
parser.add_argument("--result-root", default="android-retrace-result")
parser.add_argument("--fixture-root", default="android-retrace-fixture")
parser.add_argument("--deadline-seconds", type=int, default=0,
help="Stop starting cases, and cut a running one, this long after the start; both are "
"recorded as not-run. 0: no limit. retrace_shard_budget.py says what a shard needs.")
mode = parser.add_mutually_exclusive_group()
mode.add_argument("--dry-run", action="store_true",
help="Print each case's environment and trace-replay-ci.sh arguments, run nothing.")
mode.add_argument("--finalize", action="store_true",
help="Only write the summary inputs and a not-run verdict for cases without one.")
return parser.parse_args(argv)
def main(argv=None):
options = parse_arguments(argv)
if options.cases_json is not None:
text = options.cases_json
elif options.cases_file is not None:
text = Path(options.cases_file).read_text(encoding="utf-8")
else:
text = os.environ.get(options.cases_env, "")
try:
cases = load_cases(text, options.backend)
except ValueError as error:
say(f"::error title=Android retrace shard misconfigured::{error}")
return 2
shard = Shard(options, cases, os.environ)
if shard.hang_budget_ms is None:
say("::error title=Android retrace shard misconfigured::MOBILEGL_GPU_HANG_BUDGET_MS is not set")
return 2
if options.dry_run:
return shard.dry_run()
if options.finalize:
return shard.finalize()
if not os.path.isfile(options.apk_file):
say(f"::error title=Android retrace shard misconfigured::APK does not exist: {options.apk_file}")
return 2
# A termination request unwinds through Shard.run, which kills the running child first.
signal.signal(signal.SIGTERM, lambda number, frame: sys.exit(128 + number))
return shard.run_all()
if __name__ == "__main__":
sys.exit(main())
docs/ci/ci-cleanup-log.md
+4
View File
@@ -64,6 +64,10 @@ them wrong.
| 2026-10-18 | `Negative control A (corrupted snapshot)` and `Keep lane logs before controls` | Test / `integration-verify-split` | the first re-ran the ten `VerifySplit.ClearThenReadPixelsScenario` entries of both backends with the same knob and accepted any red; the second only kept the lane's logs from being overwritten by that rerun | the in-lane `VerifySplitCorrupted` and `VerifySplitReadCorrupted` entries of both backends, required by name (`Verify-split control entries`); that the ambient entries arm is the per-entry arm proof, and that they run fatal is `VerifySplitArming.PipeVerifyArmingScenario.Armed` (the ambient environment plus its marker and log), which requires `fatal=1` on the arming line |
| 2026-10-18 | unit `PipeInputsTest.{PoisonOmitKnobArmsTheOmission, BadPoisonOmitKnobIsFatalNamingTheKnob}`, and the `MOBILEGL_PIPE_POISON_OMIT` knob: `Features.PipePoisonOmit`, its parser on every verb (`MGPipeValidateForVerb`'s `ParsePoisonOmissionKnob`, compiled into every build) and the server stamp's own parse of it | Test / unit (every poison lane); product code | the knob existed only for negative control B, and the two cases pinned its parser and its `Fatal{PipeVerifyBadKnob}` | the PoisonOmitted and VerifySplitPoisonOmitted entries set the harness marker `MGITEST_POISON_OMIT` to the same pair, and the re-executed child arms it through `MGPipeSetPoisonOmission`, compiled only with `-DMOBILEGL_PIPE_VERIFY=ON` (`Harness/PoisonOmissionArm.h`); `OmittingOneFieldForOneVerbLeavesExactlyThatFieldStale`, `ReadingAnOmittedFieldAbortsNamingTheVerb` and `TheMutationNoticeRefreshesTheValueButNotTheStamp` withhold the stamp test-side and still run in every poison lane; the setter's scope to its verb, not its class, is checked by `OmittingOneFieldForOneVerbLeavesExactlyThatFieldStale` in a forked child, on the verify runtime only |
| 2026-10-18 | the `serial == resets` assertion of `CtWireScenario.ApplierResetCrossesAtThePrimedEdgeInSerialOrder`, and the `MOBILEGL_TEST_SERVER_COUNTERS` knob: `ServerMain`'s `MGPipe server counters:` line on every LogFlush and `ServerVerbSink`'s test-only tallies (`ApplierResets`, `ObjectDeaths`, `ExpectedApplierResetSerial`) | Test / `integration-split`, `integration-split-strict`, `integration-verify-split`, `integration (inproc)`; product code | the knob existed only so the spawn and tcp arms could read the sink's tallies; the serial assertion could not fail, since `OnApplierReset` moves the count and the serial together and any other serial latches `Fatal{ProtocolCorruption, "ApplierReset.ContextSerial"}` | `ar` / `od` on the server role's `MGPipe stats:` line, counted in `ServerVerbSink::OnApplierReset` / `OnObjectDeath`; every CtWire entry that counts them sets `MOBILEGL_PIPE_STATS=1` with period 1 and reads them on every arm, inproc included, over a window closed by a swap and a readback; the unit cases `ApplierResetWireRoundTrip.TheSerialSequenceIsAssertedAndTheServerResetRuns` and `ObjectDeathWireRoundTrip.TheTextureAndFramebufferTwinsReleaseByHandleAndNeverByAStaleGeneration` read the same counters, and the first shows the expected serial moving by having serial 2 accepted |
| 2026-11-15 | per-case jobs `retrace (<backend>, <case>)`, `retrace verify (<backend>, <case>)`, `retrace split (<backend>, <case>, <transport>)` | Test / `retrace`, `retrace-verify`, `retrace-split` | one runner per case spent most of its time on setup; no case, knob, assertion or arm proof is dropped | shard jobs that replay the same rows one after another through `scripts/ci/retrace_shard.py`, one ctest entry and one verdict per case; `trace_cases.py --self-test-shard-matrices` checks that the shards hold exactly the per-case rows |
| 2026-11-15 | per-case emulator jobs `retrace (<backend>, <case>)` of each lane | MobileGL APK / `retrace` | one emulator boot per replay; no case, setting or arm proof is dropped | shard jobs that replay the cases of one lane, transport and backend on one boot through `android-plugin/trace_replay_shard.py`, with app data cleared between cases, a cold boot after a case that did not pass, and one verdict per case |
| 2026-11-15 | per-case `trace fixture (<case>)` jobs, their per-case cache entries and artifacts, `.github/scripts/trace-fixture-cache.sh` | Test and MobileGL APK / `trace-fixtures` | 40 jobs and 40 cache entries per workflow for files that change rarely | one content-addressed cache entry for all Git LFS fixtures (`.github/scripts/trace-fixture-bundle.sh`); a replay job restores it and downloads a case only when the entry does not hold it |
| 2026-11-15 | `remove-artifact-clutter` job | Test and MobileGL APK | it deleted the fixture artifacts and the green result artifacts after the run; the fixture artifacts no longer exist | the delete step at the end of `retrace-summary`; on Linux `retention-days: 1` on the summary inputs as well, `retrace-split` and `retrace-shipped` upload results only for a failed shard (their green results were deleted unread before), and `retrace-verify` keeps its results for every shard as before |
## Kept after investigation
scripts/ci/retrace_shard.py
+689
View File
@@ -0,0 +1,689 @@
#!/usr/bin/env python3
# MobileGL - scripts/ci/retrace_shard.py
# Copyright (c) 2025-2026 MobileGL-Dev
# Licensed under the GNU Lesser General Public License v3.0:
# https://www.gnu.org/licenses/gpl-3.0.txt
# https://www.gnu.org/licenses/lgpl-3.0.txt
# SPDX-License-Identifier: LGPL-3.0-only
# End of Source File Header
"""Replay the cases of one retrace shard, one after another, and give each its own verdict.
A shard is some of the cases of one (lane, transport, backend) cell, replayed by one CI job
(tools/trace_replay/trace_cases.py, shard_matrix). Each case runs through its own ctest entry
with the environment and the checks of its lane (case_plan), in a process group and a driver
cache directory of its own, so no case sees what another left behind. A failing case does not
stop the shard. The shard fails when a case failed, or when the verdicts it recorded are not
exactly the cases it was given.
For every case it prints a log group, appends a row to the step summary and records a verdict
in <result-dir>/results/shard-results.json; a failed case also gets an error annotation.
The replay output the lane uploads is moved to <result-dir>/results/<case>/, the JUnit files
are in <result-dir>/junit/, and the extracted trace is deleted.
Usage:
retrace_shard.py run --profile {retrace,split,verify} --backend B [--transport T] [--lane L]
--build-dir DIR --result-dir DIR [--names-env VAR | --names JSON]
[--split-reuse] [--server-path PATH] [--shard N/M]
retrace_shard.py --self-test
"""
import argparse
import json
import os
import re
import shutil
import signal
import subprocess
import sys
import tempfile
import time
import xml.etree.ElementTree as ET
from pathlib import Path
SCRIPTS = Path(__file__).resolve().parent
ITERATIONRP_CASE = "minecraft-1.21.4-fabric-iris-iterationrp-in-world"
BLENDED_DEPTH_WRITE_CASE = "improved-transparency-minecraft-26.3"
WIRE_EVIDENCE_CASE = "OpenRA"
CONTROL_CASE = "OpenRA"
CONTROL_BACKEND = "DirectGLES"
CONTROL_NAME = "OpenRA (corrupted-snapshot control)"
# The comparator runs the long cases several times slower than a plain replay, and a spawned
# server can take as long, so the verify and split lanes raise ctest's 1500 s default.
LONG_TIMEOUT = ("--timeout", "10800")
SPLIT_REUSE_EXPECT_IPC = "strict=0 role-split-state=1 run-ahead=1"
SPLIT_REUSE_EXPECT_IPC_REASON = (
"retrace-split dropped its inproc rows because the inproc-acceptance retrace rows replay the "
"same library with strict=1 role-split-state=1 run-ahead=1, which differs from this leg's "
"defaults only in strict while these hold. A default changed: restore the inproc rows (the "
"trace-cases job's split-spawn-shards without --transports spawn) or update the expectation.")
# GitHub shows about ten error annotations of a step. The first failures are annotated as they
# happen, so a shard killed later still shows them; the rest wait for the roll-up that names
# every failed case.
IMMEDIATE_ANNOTATIONS = 8
def test_regex(case, backend):
return rf"^MobileGLTraceReplay\.{case}\.{backend}$"
def test_name(case, backend):
return f"MobileGLTraceReplay.{case}.{backend}"
def case_plan(profile, transport, backend, case, *, split_reuse=False, server_path=""):
"""What one case of a lane runs with: the environment it adds, ctest's arguments, its checks.
retrace the monolith-control and inproc-acceptance lanes of the retrace job. The job itself
exports MOBILEGL_TRANSPORT and the three IPC knobs for every case alike.
split the retrace-split job, inproc and spawn.
verify the retrace-verify job; its library has no transport.
"""
if profile not in ("retrace", "split", "verify"):
raise ValueError(f"unknown profile: {profile}")
env = {}
if backend == "DirectVulkan":
env["MOBILEGL_MAGMA_R11G11B10F_FALLBACK"] = "1"
# The blended depth-write quirk enables itself only on Qualcomm, which no runner has,
# so the case it exists for forces it on; ForceOn bypasses only the vendor gate.
if case == BLENDED_DEPTH_WRITE_CASE:
env["MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE"] = "1"
# The iterationRP repairs default to device detection and the software driver is on no
# list, so the one case that needs them forces them. The verify lane does not set it.
if case == ITERATIONRP_CASE and profile != "verify":
env["MOBILEGL_ITRP_FIX"] = "ForceOn"
ctest = ["-V", "--no-tests=error"]
evidence = None
census = False
if profile == "retrace":
if transport == "inproc":
# These rows stand in for retrace-split's dropped inproc rows, so they prove the
# configuration they ran with from the client's own log.
env["MGTRACE_EXPECT_IPC"] = "strict=1 role-split-state=1 run-ahead=1"
census = True
if transport != "monolith":
evidence = []
elif profile == "split":
if transport not in ("inproc", "spawn"):
raise ValueError(f"the split profile runs inproc or spawn, not {transport!r}")
if not server_path.startswith("/"):
# The replay's working directory is not the workspace and the launcher resolves the
# image relative to nothing. The inproc arm reads it only to decide it needs none.
raise ValueError("the split profile needs an absolute --server-path")
env["MOBILEGL_TRANSPORT"] = transport
env["MOBILEGL_IPC_SERVER_PATH"] = server_path
evidence = []
if case == WIRE_EVIDENCE_CASE:
# This case also counts the wire's draw records and the server's draws. That turns
# on the library's per-frame counters, so the others keep the plain configuration.
env["MGTRACE_WIRE_EVIDENCE"] = "1"
evidence = ["--require-wire"]
if split_reuse:
env["MGTRACE_EXPECT_IPC"] = SPLIT_REUSE_EXPECT_IPC
env["MGTRACE_EXPECT_IPC_REASON"] = SPLIT_REUSE_EXPECT_IPC_REASON
ctest += LONG_TIMEOUT
census = True
else:
env["MOBILEGL_PIPE_VERIFY"] = "1"
ctest += LONG_TIMEOUT
ctest += ["-R", test_regex(case, backend)]
return {"env": env, "ctest": ctest, "evidence": evidence, "census": census,
"control": has_control(profile, backend, case)}
def control_plan():
"""The verify lane's negative control: the same replay with one snapshot field corrupted.
It must fail, and for the comparator's reason. It adds only these two variables and raises
no core limit, since the failure is the expected outcome.
"""
return {"env": {"MOBILEGL_PIPE_VERIFY": "1",
"MOBILEGL_PIPE_VERIFY_CORRUPT": "GetRenderStateParameters"},
"ctest": ["-V", "--no-tests=error", *LONG_TIMEOUT,
"-R", test_regex(CONTROL_CASE, CONTROL_BACKEND)]}
def has_control(profile, backend, case):
return profile == "verify" and case == CONTROL_CASE and backend == CONTROL_BACKEND
def expected_results(profile, backend, names):
"""The verdicts a shard owes, in order."""
out = []
for name in names:
out.append(name)
if has_control(profile, backend, name):
out.append(CONTROL_NAME)
return out
def step_ignores_sigpipe():
"""Whether the process that started this script, the step's shell, ignores SIGPIPE."""
try:
with open(f"/proc/{os.getppid()}/status", encoding="ascii") as status:
for line in status:
if line.startswith("SigIgn:"):
return bool(int(line.split()[1], 16) & (1 << (signal.SIGPIPE - 1)))
except (OSError, ValueError, IndexError):
pass
return False
def raise_core_limit():
# What `ulimit -c unlimited` does; the runner's core handler honors each process's limit.
import resource
resource.setrlimit(resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
class Shard:
def __init__(self, args, names, base_env):
self.args = args
self.names = names
self.base_env = dict(base_env)
self.build_dir = Path(args.build_dir).resolve()
self.result_dir = Path(args.result_dir).resolve()
self.results_root = self.result_dir / "results"
self.junit_dir = self.result_dir / "junit"
self.log_dir = self.result_dir / "logs"
self.cache_root = self.result_dir / "driver-cache"
self.records = []
self.failures = 0
self.child = None
self.lane = args.lane or "-"
self.transport = args.transport or "-"
self.where = f"{self.lane} / {self.transport} / {args.backend}"
# --- output -------------------------------------------------------------------------------
def say(self, text=""):
print(text, flush=True)
def summary(self, text):
path = os.environ.get("GITHUB_STEP_SUMMARY")
if path:
with open(path, "a", encoding="utf-8") as file:
file.write(text + "\n")
def annotate(self, title, message):
escaped = message.replace("%", "%25").replace("\r", "").replace("\n", "%0A")
# A property value also escapes the two characters that delimit properties.
title = title.replace("%", "%25").replace(":", "%3A").replace(",", "%2C")
self.say(f"::error title={title}::{escaped}")
def write_results(self):
self.results_root.mkdir(parents=True, exist_ok=True)
payload = {"profile": self.args.profile, "lane": self.args.lane, "transport": self.args.transport,
"backend": self.args.backend, "shard": self.args.shard, "cases": self.names,
"results": self.records}
target = self.results_root / "shard-results.json"
scratch = target.with_suffix(".tmp")
scratch.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
scratch.replace(target)
def record(self, name, status, seconds, reasons, *, ssim="", cores=()):
"""One verdict: "passed", "failed", or "skipped" for a control whose case already failed."""
entry = {"case": name, "status": status, "seconds": round(seconds, 1), "ssim": ssim,
"reasons": list(reasons), "cores": list(cores), "annotated": False}
if status == "failed":
self.failures += 1
entry["annotated"] = self.failures <= IMMEDIATE_ANNOTATIONS
self.records.append(entry)
self.write_results()
note = "; ".join(reasons)
if cores:
note = (note + "; " if note else "") + "core: " + ", ".join(cores)
shown = "**failed**" if status == "failed" else status
self.summary(f"| {name} | {shown} | {seconds:.0f} | {ssim} | {note.replace('|', '/')[:300]} |")
if entry["annotated"]:
self.annotate(f"retrace failed: {self.where} / {name}", note or "failed")
# --- one process ----------------------------------------------------------------------------
def run_ctest(self, plan, *, junit=None, transcript=None, core_limit=True, cache_home=None):
env = dict(self.base_env)
env.update(plan["env"])
if cache_home is not None:
env["XDG_CACHE_HOME"] = str(cache_home)
command = [self.args.ctest, *plan["ctest"]]
if junit is not None:
command += ["--output-junit", str(junit)]
self.say("+ " + " ".join(f"{key}={value!r}" for key, value in sorted(plan["env"].items())))
self.say("+ " + " ".join(command))
options = {"cwd": self.build_dir, "env": env, "start_new_session": True}
if core_limit:
options["preexec_fn"] = raise_core_limit
# Python ignores SIGPIPE itself and by default hands a child the default action. The
# replay gets what a step's own child would get instead, so a write to a closed pipe
# or socket ends the same way as when the step ran ctest directly.
options["restore_signals"] = not step_ignores_sigpipe()
sys.stdout.flush()
if transcript is None:
self.child = subprocess.Popen(command, **options)
status = self.child.wait()
else:
# The transcript is both shown and kept, as `2>&1 | tee` would.
self.child = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options)
with open(transcript, "wb") as file:
for chunk in iter(lambda: self.child.stdout.read1(65536), b""):
file.write(chunk)
sys.stdout.buffer.write(chunk)
sys.stdout.buffer.flush()
status = self.child.wait()
self.reap()
return status
def reap(self, grace=5.0):
"""End whatever the finished replay left running, so it cannot run under the next case.
A spawned server exits by itself when its client is gone, so the group gets a moment
first. True when something had to be killed.
"""
child, self.child = self.child, None
if child is None:
return False
deadline = time.monotonic() + grace
try:
while True:
os.killpg(child.pid, 0)
if time.monotonic() >= deadline:
break
time.sleep(0.25)
os.killpg(child.pid, signal.SIGKILL)
except (ProcessLookupError, PermissionError):
return False
self.say(f"killed the processes ctest (pid {child.pid}) left running")
return True
# --- one case -------------------------------------------------------------------------------
def fresh_cache_home(self):
"""An empty driver cache directory, as on a runner that has replayed nothing yet.
The software drivers keep compiled shaders on disk under the user cache directory. Each
case gets a new one, so a case never starts from shaders another case compiled. On a
hosted runner the default location is emptied as well, in case a process ignores the
redirect; a self-hosted machine's own cache is left alone.
"""
self.cache_root.mkdir(parents=True, exist_ok=True)
cache_home = Path(tempfile.mkdtemp(prefix="case-", dir=self.cache_root))
if self.base_env.get("RUNNER_ENVIRONMENT") != "self-hosted":
default_home = Path(self.base_env.get("HOME", "/nonexistent")) / ".cache"
for stale in default_home.glob("mesa_shader_cache*"):
self.say(f"removing {stale} left by an earlier process")
shutil.rmtree(stale, ignore_errors=True)
return cache_home
def drop_cache_home(self, cache_home):
entries = sum(1 for path in cache_home.rglob("*") if path.is_file())
self.say(f"driver cache: {entries} file(s) written under {cache_home}, removed")
shutil.rmtree(cache_home, ignore_errors=True)
def junit_problem(self, junit, name):
"""Why this JUnit file does not show exactly the one entry that was asked for, or ''."""
if not junit.is_file():
return f"ctest wrote no {junit.name}"
try:
entries = [case.get("name") for case in ET.parse(junit).getroot().iter("testcase")]
except ET.ParseError as error:
return f"{junit.name} does not parse: {error}"
if entries != [name]:
return f"ctest ran {entries or 'no entry'} where exactly {name} was asked for"
return ""
def check_evidence(self, plan, case):
"""Run the lane's evidence check; its error lines are folded into the case's annotation."""
evidence = self.build_dir / case / self.args.backend / "output" / "split-evidence.txt"
command = [sys.executable, str(SCRIPTS / "retrace_split_evidence.py"), str(evidence),
"--transport", self.args.transport, "--case", case, "--backend", self.args.backend,
*plan["evidence"]]
done = subprocess.run(command, cwd=self.build_dir, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, text=True)
text = done.stdout.strip()
self.say(re.sub(r"^::error::", "split evidence: ", text, flags=re.M))
if done.returncode == 0:
return ""
return "split evidence: " + re.sub(r"^::error::", "", text, flags=re.M).replace("\n", " ")[:600]
def census(self, case):
"""The refusal census from the library's own logs, recorded and never a verdict.
Both roles are read: under spawn the applier's refusals are raised in the server's
process. ctest's transcript cannot stand in, since the console sink is compiled out.
"""
output = self.build_dir / case / self.args.backend / "output"
log = output / "mobilegl.client.log"
server_log = output / "mobilegl.server.log"
evidence = output / "split-evidence.txt"
self.say(evidence.read_text(errors="replace").rstrip() if evidence.is_file() else "no split-evidence.txt")
if not log.is_file():
self.say(f"no {log} - the replay wrote no library log")
return
files = [log] + ([server_log] if server_log.is_file() else [])
self.say("census over: " + " ".join(str(path) for path in files))
lines = [line for path in files for line in path.read_text(errors="replace").splitlines()]
self.say(f"Fatal{{ lines: {sum(1 for line in lines if 'Fatal{' in line)}")
counts = {}
for line in lines:
for match in re.findall(r'Fatal\{[A-Za-z]*, "[^"]*"', line):
counts[match] = counts.get(match, 0) + 1
for match, count in sorted(counts.items(), key=lambda item: (-item[1], item[0]))[:20]:
self.say(f"{count:7d} {match}")
transport_line = next((line for line in log.read_text(errors="replace").splitlines()
if "MOBILEGL_TRANSPORT=" in line), None)
self.say(transport_line or f"no transport line in {log}")
def collect(self, case):
"""Move what the lane uploads into the result directory and delete the extracted trace."""
source = self.build_dir / case
target = self.results_root / case
shutil.rmtree(source / self.args.backend / "input", ignore_errors=True)
for relative in ("actual-images", f"{self.args.backend}/output"):
if (source / relative).is_dir():
(target / relative).parent.mkdir(parents=True, exist_ok=True)
shutil.rmtree(target / relative, ignore_errors=True)
shutil.move(str(source / relative), str(target / relative))
shutil.rmtree(source, ignore_errors=True)
def ssim_of(self, case):
try:
result = json.loads((self.build_dir / case / self.args.backend / "output" / "result.json")
.read_text(errors="replace"))
except (OSError, ValueError):
return ""
value = result.get("ssim")
return "" if value is None else str(value)
def run_case(self, case):
plan = case_plan(self.args.profile, self.args.transport, self.args.backend, case,
split_reuse=self.args.split_reuse, server_path=self.args.server_path)
self.say(f"::group::{case} ({self.where})")
started = time.monotonic()
cores_before = set(Path("/tmp").glob("core.*"))
junit = self.junit_dir / f"{case}.xml"
reasons = []
cache_home = self.fresh_cache_home()
try:
try:
status = self.run_ctest(plan, junit=junit, cache_home=cache_home)
except (OSError, subprocess.SubprocessError) as error:
# The case still gets its verdict, and the cases after it still run.
self.child = None
status = None
reasons.append(f"ctest could not be run: {error}")
if status is not None and status > 0:
reasons.append(f"ctest exit {status}")
elif status is not None and status < 0:
reasons.append(f"ctest ended by signal {-status}")
problem = self.junit_problem(junit, test_name(case, self.args.backend))
if problem:
reasons.append(problem)
if plan["evidence"] is not None:
# The transport assertions ran on this case and found their evidence. A failed
# replay is already red, and its missing evidence file would only repeat that.
if status == 0:
problem = self.check_evidence(plan, case)
if problem:
reasons.append(problem)
if plan["census"]:
try:
self.census(case)
except OSError as error:
self.say(f"refusal census skipped: {error}")
finally:
self.drop_cache_home(cache_home)
ssim = self.ssim_of(case)
self.collect(case)
cores = sorted(path.name for path in set(Path("/tmp").glob("core.*")) - cores_before)
seconds = time.monotonic() - started
self.say(f"{case}: {'passed' if not reasons else 'FAILED: ' + '; '.join(reasons)} in {seconds:.0f} s")
self.say("::endgroup::")
self.record(case, "failed" if reasons else "passed", seconds, reasons, ssim=ssim, cores=cores)
return not reasons
def run_control(self, verified_passed):
"""With a snapshot field corrupted the same replay must fail, or the lane proves nothing.
It replays into the case directory the verified run used. That run's output has already
been moved to the result directory, and the corrupted run's output is deleted, so only
the verified run is uploaded under the case's name.
"""
self.say(f"::group::{CONTROL_NAME}")
if not verified_passed:
self.say("not run: the verified replay it is compared with failed")
self.say("::endgroup::")
self.record(CONTROL_NAME, "skipped", 0, ["not run: the verified replay failed"])
return
started = time.monotonic()
reasons = []
transcript = self.log_dir / "verify-control.txt"
cache_home = self.fresh_cache_home()
status = None
try:
status = self.run_ctest(control_plan(), transcript=transcript, core_limit=False,
cache_home=cache_home)
except (OSError, subprocess.SubprocessError) as error:
self.child = None
reasons.append(f"ctest could not be run: {error}")
finally:
self.drop_cache_home(cache_home)
shutil.rmtree(self.build_dir / CONTROL_CASE, ignore_errors=True)
if status is None:
pass
elif status == 0:
reasons.append(
"MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left the OpenRA retrace "
"green, so the comparator is not comparing and the verify retrace lane proves nothing")
else:
# Its red must carry the comparator's divergence on the corrupted field.
done = subprocess.run([sys.executable, str(SCRIPTS / "retrace_verify_reason.py"), str(transcript)],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
text = re.sub(r"^::error::", "", done.stdout.strip(), flags=re.M)
self.say(text)
if done.returncode != 0:
reasons.append(text.replace("\n", " ")[:600])
else:
self.say(f"the corrupted field turned the retrace red, as it must (ctest exit {status})")
seconds = time.monotonic() - started
self.say("::endgroup::")
self.record(CONTROL_NAME, "failed" if reasons else "passed", seconds, reasons)
# --- the shard --------------------------------------------------------------------------------
def run(self):
for directory in (self.results_root, self.junit_dir, self.log_dir):
directory.mkdir(parents=True, exist_ok=True)
expected = expected_results(self.args.profile, self.args.backend, self.names)
self.say(f"retrace shard {self.args.shard or ''} ({self.where}): {len(self.names)} case(s), "
f"{len(expected)} verdict(s) owed")
self.summary(f"### retrace shard {self.args.shard or ''} ({self.where})\n\n"
"| case | result | seconds | SSIM | note |\n|---|---|---|---|---|")
self.write_results()
for case in self.names:
passed = self.run_case(case)
if case_plan(self.args.profile, self.args.transport, self.args.backend, case,
split_reuse=self.args.split_reuse, server_path=self.args.server_path)["control"]:
self.run_control(passed)
return self.finish(expected)
def finish(self, expected):
problems = verdict_problems(expected, self.records)
# Counted again from what ctest wrote, apart from the loop's own bookkeeping.
ran = sum(1 for case in self.names
if not self.junit_problem(self.junit_dir / f"{case}.xml", test_name(case, self.args.backend)))
passed_cases = sum(1 for entry in self.records
if entry["status"] == "passed" and entry["case"] in self.names)
if ran < passed_cases:
problems.append(f"{passed_cases} case(s) recorded as passed but only {ran} JUnit file(s) "
f"show their entry ran")
failed = [entry for entry in self.records if entry["status"] == "failed"]
if failed:
names = ", ".join(entry["case"] for entry in failed)
self.annotate(f"retrace shard {self.args.shard or ''}: {len(failed)} of {len(expected)} failed",
f"{self.where}: {names}")
for entry in failed:
if not entry["annotated"]:
self.annotate(f"retrace failed: {self.where} / {entry['case']}",
"; ".join(entry["reasons"]) or "failed")
for problem in problems:
self.annotate(f"retrace shard {self.args.shard or ''}: verdicts do not match the cases", problem)
passed = sum(1 for entry in self.records if entry["status"] == "passed")
closing = f"{passed} passed, {len(failed)} failed, {len(expected)} owed"
if problems:
closing += "; " + "; ".join(problems)
self.summary(f"\n{closing}\n")
self.say(f"retrace shard ({self.where}): {closing}")
return 1 if failed or problems else 0
def verdict_problems(expected, records):
"""Why the recorded verdicts are not exactly the ones owed; empty when they are."""
recorded = [entry["case"] for entry in records]
problems = []
if len(recorded) != len(expected):
problems.append(f"{len(recorded)} verdict(s) recorded for {len(expected)} owed")
if recorded != expected:
missing = [name for name in expected if name not in recorded]
extra = [name for name in recorded if name not in expected]
if missing:
problems.append("no verdict for " + ", ".join(missing))
if extra:
problems.append("verdict for a case that was not assigned: " + ", ".join(extra))
if not missing and not extra and len(recorded) == len(expected):
problems.append("verdicts are out of order or repeated")
return problems
def run(args):
if args.names is not None:
names = json.loads(args.names)
else:
names = json.loads(os.environ.get(args.names_env) or "null")
if not isinstance(names, list) or not names or not all(isinstance(name, str) and name for name in names):
print(f"::error::retrace shard: no case list (looked in --names and ${args.names_env})")
return 2
if len(set(names)) != len(names):
print(f"::error::retrace shard: the case list repeats a case: {names}")
return 2
if args.profile == "retrace":
# The job exports the transport for this lane; the plan and the library must agree on it.
exported = os.environ.get("MOBILEGL_TRANSPORT", "")
if exported != args.transport:
print(f"::error::retrace shard: --transport is {args.transport!r} but the job exports "
f"MOBILEGL_TRANSPORT={exported!r}")
return 2
try:
case_plan(args.profile, args.transport, args.backend, names[0],
split_reuse=args.split_reuse, server_path=args.server_path)
except ValueError as error:
print(f"::error::retrace shard: {error}")
return 2
shard = Shard(args, names, os.environ)
def stop(signum, _frame):
shard.reap(grace=0)
shard.say(f"retrace shard: stopped by signal {signum} after {len(shard.records)} verdict(s)")
sys.exit(128 + signum)
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
return shard.run()
def self_test():
"""The plan of every lane against the values the lanes are defined by, and the verdict check."""
oit, itrp, other = BLENDED_DEPTH_WRITE_CASE, ITERATIONRP_CASE, "minecraft-1.21.4-in-world"
server = "/w/build-linux/libMobileGLServer.so"
regex = lambda case, backend: ["-R", rf"^MobileGLTraceReplay\.{case}\.{backend}$"]
plain, long = ["-V", "--no-tests=error"], ["-V", "--no-tests=error", "--timeout", "10800"]
vk = {"MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "1"}
expect_strict = {"MGTRACE_EXPECT_IPC": "strict=1 role-split-state=1 run-ahead=1"}
reuse = {"MGTRACE_EXPECT_IPC": SPLIT_REUSE_EXPECT_IPC, "MGTRACE_EXPECT_IPC_REASON": SPLIT_REUSE_EXPECT_IPC_REASON}
split = lambda transport: {"MOBILEGL_TRANSPORT": transport, "MOBILEGL_IPC_SERVER_PATH": server}
table = [
# profile, transport, backend, case, reuse -> env, ctest, evidence, census, control
("retrace", "monolith", "DirectGLES", other, False, {}, plain, None, False, False),
("retrace", "monolith", "DirectVulkan", other, False, vk, plain, None, False, False),
("retrace", "monolith", "DirectVulkan", oit, False,
{**vk, "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE": "1"}, plain, None, False, False),
("retrace", "monolith", "DirectGLES", oit, False, {}, plain, None, False, False),
("retrace", "monolith", "DirectVulkan", itrp, False, {**vk, "MOBILEGL_ITRP_FIX": "ForceOn"},
plain, None, False, False),
("retrace", "inproc", "DirectGLES", other, False, expect_strict, plain, [], True, False),
("retrace", "inproc", "DirectVulkan", itrp, False,
{**vk, "MOBILEGL_ITRP_FIX": "ForceOn", **expect_strict}, plain, [], True, False),
("retrace", "inproc", "DirectGLES", "OpenRA", False, expect_strict, plain, [], True, False),
("split", "spawn", "DirectGLES", other, True, {**split("spawn"), **reuse}, long, [], True, False),
("split", "spawn", "DirectGLES", other, False, split("spawn"), long, [], True, False),
("split", "spawn", "DirectVulkan", oit, True,
{**vk, "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE": "1", **split("spawn"), **reuse},
long, [], True, False),
("split", "spawn", "DirectVulkan", itrp, True,
{**vk, "MOBILEGL_ITRP_FIX": "ForceOn", **split("spawn"), **reuse}, long, [], True, False),
("split", "spawn", "DirectGLES", "OpenRA", True,
{**split("spawn"), "MGTRACE_WIRE_EVIDENCE": "1", **reuse}, long, ["--require-wire"], True, False),
("split", "inproc", "DirectVulkan", "OpenRA", True,
{**vk, **split("inproc"), "MGTRACE_WIRE_EVIDENCE": "1", **reuse}, long, ["--require-wire"], True, False),
("split", "inproc", "DirectGLES", other, False, split("inproc"), long, [], True, False),
("verify", "", "DirectGLES", other, False, {"MOBILEGL_PIPE_VERIFY": "1"}, long, None, False, False),
("verify", "", "DirectGLES", "OpenRA", False, {"MOBILEGL_PIPE_VERIFY": "1"}, long, None, False, True),
("verify", "", "DirectVulkan", "OpenRA", False, {"MOBILEGL_PIPE_VERIFY": "1", **vk}, long, None, False, False),
("verify", "", "DirectVulkan", oit, False,
{"MOBILEGL_PIPE_VERIFY": "1", **vk, "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE": "1"},
long, None, False, False),
("verify", "", "DirectVulkan", itrp, False, {"MOBILEGL_PIPE_VERIFY": "1", **vk}, long, None, False, False),
]
for profile, transport, backend, case, split_reuse, env, ctest, evidence, census, control in table:
plan = case_plan(profile, transport, backend, case, split_reuse=split_reuse, server_path=server)
wanted = {"env": env, "ctest": ctest + regex(case, backend), "evidence": evidence,
"census": census, "control": control}
assert plan == wanted, f"{profile} {transport} {backend} {case}: {plan} != {wanted}"
control = control_plan()
assert control["env"] == {"MOBILEGL_PIPE_VERIFY": "1", "MOBILEGL_PIPE_VERIFY_CORRUPT": "GetRenderStateParameters"}
assert control["ctest"] == long + regex("OpenRA", "DirectGLES")
for bad in (("split", "spawn", ""), ("split", "spawn", "relative/server"), ("split", "monolith", server),
("pull", "monolith", server)):
try:
case_plan(bad[0], bad[1], "DirectGLES", other, server_path=bad[2])
except ValueError:
continue
raise AssertionError(f"case_plan accepted {bad}")
assert expected_results("verify", "DirectGLES", ["a", "OpenRA", "b"]) == ["a", "OpenRA", CONTROL_NAME, "b"]
assert expected_results("verify", "DirectVulkan", ["OpenRA"]) == ["OpenRA"]
assert expected_results("split", "DirectGLES", ["OpenRA"]) == ["OpenRA"]
entry = lambda name: {"case": name, "status": "passed"}
assert verdict_problems(["a", "b"], [entry("a"), entry("b")]) == []
assert verdict_problems(["a", "b"], [entry("a")]), "a case without a verdict passed the check"
assert verdict_problems(["a", "b"], []), "a shard that ran nothing passed the check"
assert verdict_problems(["a"], [entry("a"), entry("a")]), "a repeated verdict passed the check"
assert verdict_problems(["a", "b"], [entry("a"), entry("c")]), "an unassigned case passed the check"
assert verdict_problems(["a", "b"], [entry("b"), entry("a")]), "out-of-order verdicts passed the check"
print(f"retrace_shard: {len(table)} plans, the control plan and the verdict check OK")
return 0
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("command", nargs="?", choices=("run",))
parser.add_argument("--self-test", action="store_true")
parser.add_argument("--profile", choices=("retrace", "split", "verify"))
parser.add_argument("--backend")
parser.add_argument("--transport", default="")
parser.add_argument("--lane", default="")
parser.add_argument("--shard", default="", help="N/M, for the report only")
parser.add_argument("--build-dir", help="the directory ctest runs in")
parser.add_argument("--result-dir", help="where verdicts, replay output and JUnit files go")
parser.add_argument("--names", help="the cases, as a JSON list")
parser.add_argument("--names-env", default="SHARD_NAMES", help="the variable holding the JSON list")
parser.add_argument("--split-reuse", action="store_true",
help="split profile: the library is the retrace job's, so prove the IPC defaults")
parser.add_argument("--server-path", default="", help="split profile: absolute path of the server image")
parser.add_argument("--ctest", default="ctest")
args = parser.parse_args(argv)
if args.self_test:
return self_test()
if args.command != "run" or not (args.profile and args.backend and args.build_dir and args.result_dir):
parser.error("run needs --profile, --backend, --build-dir and --result-dir")
return run(args)
if __name__ == "__main__":
sys.exit(main())
tools/trace_replay/SpawnServerPathTest.java
+2 -1
View File
@@ -48,7 +48,8 @@ public final class SpawnServerPathTest {
public static void main(String[] args) {
// ---- the shapes the lanes really pass -------------------------------------------------
// .github/workflows/apk.yml's spawn acceptance lane, verbatim.
// The spawn acceptance lane of android-plugin/trace_replay_shard.py (transport_arguments()),
// verbatim.
String spawnLane = "MOBILEGL_TRANSPORT=spawn;MOBILEGL_IPC_ROLE_SPLIT_STATE=1;"
+ "MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1";
expect("the spawn acceptance lane still gets its server path appended",
tools/trace_replay/run_android_retrace_local.py
+2 -1
View File
@@ -650,7 +650,8 @@ def transport_proof_args(args):
return {"inproc": ["--require-inproc"], "spawn": ["--require-spawn"]}.get(args.transport, [])
# The knobs .github/workflows/apk.yml hands BOTH acceptance lanes. They are kept here verbatim so
# The knobs android-plugin/trace_replay_shard.py (transport_arguments()) hands both acceptance
# lanes. They are kept here verbatim so
# a local run is the same run: the arm proof below reads the library's own `Config: IPC` line and
# requires strict/role-split-state/run-ahead to be 1, so a local invocation that quietly omitted
# them would fail a gate CI passes - or worse, pass a weaker one.
tools/trace_replay/trace_cases.py
+332
View File
@@ -518,6 +518,311 @@ def self_test_split_matrix(cases=None):
f"kept); all {len(dropped)} dropped inproc rows are inproc-acceptance rows")
# --- Shards: several cases of one cell replayed by one CI job ---------------------------------
#
# A cell is everything a per-case matrix row carries except its case: the backend, and the lane
# and transport where the matrix has them. A shard row keeps the cell's keys unchanged and adds
# the cases it replays, so the job that runs it reads the same fields a per-case leg reads.
# Every shard matrix is built from the per-case matrix it replaces and is refused unless its
# rows, taken apart again, are exactly that matrix.
TRACE_DURATION_HINTS_JSON = Path(__file__).with_name("trace_duration_hints.json")
# Shards per cell on Linux, the one tunable of that layout. A cell with little to replay gets
# fewer: a shard costs about half a minute of setup, so it is not split below
# MIN_SHARD_SECONDS of replay.
DEFAULT_SHARDS_PER_CELL = 4
MIN_SHARD_SECONDS = 90
# Replay seconds one emulator shard aims for, the one tunable of the Android layout. A cell is
# split into as many shards as that takes.
DEFAULT_ANDROID_SHARD_SECONDS = 720
# The hint a row without a transport uses. The verify lane has timings of its own.
HINT_KIND_MONOLITH = "monolith"
HINT_KIND_VERIFY = "verify"
# A shard's case names reach a shell command line, so the characters are restricted.
SHARD_SAFE_NAME = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-")
def load_duration_hints(platform, path=TRACE_DURATION_HINTS_JSON):
"""{case: {backend: {kind: seconds}}} for "linux" or "android"; empty when there is none."""
path = Path(path)
if not path.is_file():
return {}
with path.open("r", encoding="utf-8") as file:
table = json.load(file).get(platform, {}).get("seconds", {})
for case, backends in table.items():
for backend, kinds in backends.items():
for kind, seconds in kinds.items():
if isinstance(seconds, bool) or not isinstance(seconds, (int, float)) or seconds <= 0:
raise ValueError(f"duration hint for {case} {backend} {kind} must be a positive "
f"number: {seconds!r}")
return table
def median(values):
ordered = sorted(values)
if not ordered:
return None
middle = len(ordered) // 2
if len(ordered) % 2:
return ordered[middle]
return (ordered[middle - 1] + ordered[middle]) / 2
def hint_seconds(hints, case, backend, kind):
"""The hinted replay time of one row, or the median of its backend and kind, or of everything."""
try:
return hints[case][backend][kind]
except KeyError:
pass
same_cell = [kinds[kind] for backends in hints.values()
for name, kinds in backends.items() if name == backend and kind in kinds]
everything = [seconds for backends in hints.values() for kinds in backends.values()
for seconds in kinds.values()]
return median(same_cell) or median(everything) or 30
def row_case_name(row):
case = row["case"]
return case["name"] if isinstance(case, dict) else case
def row_backend_name(row):
backend = row["backend"]
return backend["name"] if isinstance(backend, dict) else backend
def row_cell(row):
return {key: value for key, value in row.items() if key != "case"}
def canonical(value):
return json.dumps(value, sort_keys=True, separators=(",", ":"))
def shard_rows_as_cases(shard_rows):
"""Take shard rows apart into the per-case rows they stand for."""
added = ("shard", "shards", "names", "cases", "seconds")
rows = []
for shard in shard_rows:
cell = {key: value for key, value in shard.items() if key not in added}
for case in shard.get("cases", shard["names"]):
rows.append({**cell, "case": case})
return rows
def check_shard_union(rows, shard_rows):
"""Refuse a shard matrix that is not exactly the per-case matrix `rows`, regrouped."""
expected = sorted(canonical(row) for row in rows)
actual = sorted(canonical(row) for row in shard_rows_as_cases(shard_rows))
if expected != actual:
missing = sorted(set(expected) - set(actual))
extra = sorted(set(actual) - set(expected))
raise ValueError(
f"the shard matrix is not the per-case matrix regrouped: {len(expected)} per-case rows, "
f"{len(actual)} rows in the shards; missing {missing[:3]}, unexpected {extra[:3]}")
if len(set(expected)) != len(expected):
raise ValueError("the per-case matrix repeats a row, so a shard would replay it twice")
cells = {}
for shard in shard_rows:
if not shard["names"]:
raise ValueError(f"empty shard: {shard}")
if "cases" in shard and [case["name"] for case in shard["cases"]] != shard["names"]:
raise ValueError(f"a shard's names and cases disagree: {shard['names']}")
cell = canonical({key: value for key, value in shard.items()
if key not in ("shard", "shards", "names", "cases", "seconds")})
cells.setdefault(cell, []).append((shard["shard"], shard["shards"]))
for cell, numbers in cells.items():
count = len(numbers)
if sorted(numbers) != [(index, count) for index in range(1, count + 1)]:
raise ValueError(f"shards of {cell} are not numbered 1..{count} of {count}: {numbers}")
def shard_matrix(rows, hints, *, default_kind=HINT_KIND_MONOLITH, shards=None, shard_seconds=None):
"""Regroup a per-case matrix into shards of one cell each.
Exactly one of `shards` (shards per cell, capped so that none holds less than
MIN_SHARD_SECONDS of hinted replay) and `shard_seconds` (hinted replay per shard) sets the
layout. Cases go longest first into the shard with the least hinted time so far, which is
deterministic and keeps the shards of a cell even; inside a shard they run in matrix order.
"""
if (shards is None) == (shard_seconds is None):
raise ValueError("give exactly one of shards and shard_seconds")
if shards is not None and shards < 1:
raise ValueError(f"shards per cell must be at least 1: {shards}")
if shard_seconds is not None and shard_seconds < 1:
raise ValueError(f"seconds per shard must be at least 1: {shard_seconds}")
cells = {}
for index, row in enumerate(rows):
name = row_case_name(row)
if not name or not set(name) <= SHARD_SAFE_NAME:
raise ValueError(f"case name {name!r} cannot be passed on a command line; allowed "
f"characters are letters, digits, '.', '_' and '-'")
seconds = hint_seconds(hints, name, row_backend_name(row),
row.get("transport", default_kind))
cells.setdefault(canonical(row_cell(row)), []).append((index, row, seconds))
out = []
for members in cells.values():
total = sum(seconds for _, _, seconds in members)
if shards is not None:
count = min(shards, int(total // MIN_SHARD_SECONDS))
else:
count = -(-int(total) // shard_seconds)
count = max(1, min(count, len(members)))
bins = [[] for _ in range(count)]
loads = [0] * count
for member in sorted(members, key=lambda item: (-item[2], item[0])):
target = loads.index(min(loads))
bins[target].append(member)
loads[target] += member[2]
for number, members_of_shard in enumerate(bins, start=1):
members_of_shard.sort(key=lambda item: item[0])
shard = {**row_cell(members_of_shard[0][1]), "shard": number, "shards": count,
"names": [row_case_name(row) for _, row, _ in members_of_shard]}
if isinstance(members_of_shard[0][1]["case"], dict):
shard["cases"] = [row["case"] for _, row, _ in members_of_shard]
shard["seconds"] = round(sum(seconds for _, _, seconds in members_of_shard))
out.append(shard)
check_shard_union(rows, out)
return {"include": out}
def per_case_matrix(format_name, cases, *, transports=SPLIT_TRANSPORTS, keep_inproc=()):
"""The per-case matrix a shard format regroups, and the hint kind of its transport-less rows."""
if format_name == "github-test-shard-matrix":
return github_test_matrix(cases), HINT_KIND_MONOLITH
if format_name == "github-test-transport-shard-matrix":
return github_transport_matrix(cases), HINT_KIND_MONOLITH
if format_name == "github-verify-shard-matrix":
return github_verify_matrix(cases), HINT_KIND_VERIFY
if format_name == "github-split-shard-matrix":
return github_split_matrix(cases, transports, keep_inproc), HINT_KIND_MONOLITH
if format_name == "github-apk-shard-matrix":
return github_apk_matrix(cases), HINT_KIND_MONOLITH
if format_name == "github-apk-transport-shard-matrix":
return github_transport_matrix(cases, apk=True), HINT_KIND_MONOLITH
raise ValueError(f"not a shard format: {format_name}")
LINUX_SHARD_FORMATS = (
"github-test-shard-matrix",
"github-test-transport-shard-matrix",
"github-verify-shard-matrix",
"github-split-shard-matrix",
)
ANDROID_SHARD_FORMATS = (
"github-apk-shard-matrix",
"github-apk-transport-shard-matrix",
)
def github_shard_matrix(format_name, cases, *, shards=DEFAULT_SHARDS_PER_CELL,
shard_seconds=DEFAULT_ANDROID_SHARD_SECONDS,
transports=SPLIT_TRANSPORTS, keep_inproc=(), hints_path=TRACE_DURATION_HINTS_JSON):
"""A shard format: the Linux ones lay out by shards per cell, the Android ones by seconds."""
matrix, default_kind = per_case_matrix(format_name, cases, transports=transports,
keep_inproc=keep_inproc)
if format_name in ANDROID_SHARD_FORMATS:
return shard_matrix(matrix["include"], load_duration_hints("android", hints_path),
default_kind=default_kind, shard_seconds=shard_seconds)
return shard_matrix(matrix["include"], load_duration_hints("linux", hints_path),
default_kind=default_kind, shards=shards)
def self_test_shard_matrices():
"""Every shard format replays exactly the rows of the per-case matrix it replaces."""
cases = ci_trace_cases(load_trace_cases())
narrowed = {"transports": ("spawn",), "keep_inproc": ("OpenRA",)}
layouts = [(name, {}, {"shards": count}) for name in LINUX_SHARD_FORMATS for count in (1, 4, 100)]
layouts += [("github-split-shard-matrix", narrowed, {"shards": count}) for count in (1, 4, 100)]
layouts += [(name, {}, {"shard_seconds": seconds}) for name in ANDROID_SHARD_FORMATS
for seconds in (60, DEFAULT_ANDROID_SHARD_SECONDS, 10 ** 9)]
for name, selection, layout in layouts:
matrix, _ = per_case_matrix(name, cases, **selection)
rows = matrix["include"]
built = github_shard_matrix(name, cases, **selection, **layout)["include"]
again = github_shard_matrix(name, cases, **selection, **layout)["include"]
assert built == again, f"{name} {layout}: the layout is not deterministic"
# Compared here as well as inside shard_matrix, with nothing shared but the JSON form.
replayed = sorted(
canonical({**{key: value for key, value in shard.items()
if key not in ("shard", "shards", "names", "cases", "seconds")},
"case": case})
for shard in built for case in shard.get("cases", shard["names"]))
assert replayed == sorted(canonical(row) for row in rows), \
f"{name} {layout}: the shards do not replay the per-case matrix"
assert sum(len(shard["names"]) for shard in built) == len(rows)
cells = {canonical(row_cell(row)) for row in rows}
assert len({canonical({key: value for key, value in shard.items()
if key not in ("shard", "shards", "names", "cases", "seconds")})
for shard in built}) == len(cells), f"{name} {layout}: a shard spans two cells"
if layout.get("shards") == 1 or layout.get("shard_seconds") == 10 ** 9:
assert len(built) == len(cells), f"{name} {layout}: expected one shard per cell"
seconds = [shard["seconds"] for shard in built]
print(f"shard matrix {name} {selection or ''} {layout}: {len(rows)} rows in {len(built)} "
f"shards over {len(cells)} cells, {min(seconds)}-{max(seconds)} s hinted per shard")
# The check itself must be able to fail: a dropped, a repeated and a moved case.
if len(built) >= 2 and any(len(shard["names"]) >= 2 for shard in built):
index = next(i for i, shard in enumerate(built) if len(shard["names"]) >= 2)
for label, damage in (("dropped", lambda names: names[1:]),
("repeated", lambda names: names + names[:1])):
broken = [dict(shard) for shard in built]
broken[index]["names"] = damage(list(broken[index]["names"]))
if "cases" in broken[index]:
broken[index]["cases"] = damage(list(broken[index]["cases"]))
try:
check_shard_union(rows, broken)
except ValueError:
pass
else:
raise AssertionError(f"{name} {layout}: a {label} case passed the union check")
other = next((i for i, shard in enumerate(built)
if row_backend_name(shard) != row_backend_name(built[index])), None)
if other is not None:
broken = [dict(shard) for shard in built]
for key in ("names", "cases"):
if key in broken[index]:
moved = list(broken[index][key])
broken[other][key] = list(broken[other][key]) + moved[:1]
broken[index][key] = moved[1:]
try:
check_shard_union(rows, broken)
except ValueError:
pass
else:
raise AssertionError(f"{name} {layout}: a case moved to another backend's "
f"shard passed the union check")
# Hints only weigh the packing. With none at all, and with a case no hint names, the rows
# replayed are the same.
rows = github_transport_matrix(cases)["include"]
for hints in ({}, {"no-such-case": {"DirectGLES": {"monolith": 5}}}):
check_shard_union(rows, shard_matrix(rows, hints, shards=4)["include"])
assert hint_seconds({}, "x", "DirectGLES", "monolith") == 30
assert hint_seconds({"a": {"DirectGLES": {"monolith": 10}}, "b": {"DirectGLES": {"monolith": 20}},
"c": {"DirectVulkan": {"monolith": 90}}}, "x", "DirectGLES", "monolith") == 15
for bad in ({"shards": 0}, {"shard_seconds": 0}, {}, {"shards": 4, "shard_seconds": 60}):
try:
shard_matrix(rows, {}, **bad)
except ValueError:
continue
raise AssertionError(f"shard_matrix accepted {bad}")
try:
shard_matrix([{"backend": "DirectGLES", "case": "bad name; rm"}], {}, shards=1)
except ValueError:
pass
else:
raise AssertionError("a case name with shell characters was accepted")
# The verify lane's corrupted-snapshot control replays OpenRA on DirectGLES, so that row has
# to be in the verify matrix for the control to have a shard to run in.
verify_rows = github_verify_matrix(cases)["include"]
assert {"backend": "DirectGLES", "case": "OpenRA"} in verify_rows, \
"the verify matrix has no OpenRA DirectGLES row, so its negative control would not run"
print("shard matrices: union, numbering, determinism and negative controls OK")
def cmake_quote(value):
return '"' + str(value).replace("\\", "/").replace('"', '\\"') + '"'
@@ -570,11 +875,19 @@ def parse_args():
parser.add_argument("--self-test-transport-matrices", action="store_true")
parser.add_argument("--self-test-split-matrix", action="store_true",
help="prove the spawn-only split matrix drops only inproc-acceptance rows")
parser.add_argument("--self-test-shard-matrices", action="store_true",
help="prove every shard matrix replays exactly its per-case matrix")
parser.add_argument("--fixture-root", default="tools/trace_replay/fixtures")
parser.add_argument("--transports", default=",".join(SPLIT_TRANSPORTS),
help="github-split-matrix: the comma-separated split transports to emit")
parser.add_argument("--keep-inproc", default="",
help="github-split-matrix: comma-separated split cases whose inproc rows stay")
parser.add_argument("--shards", type=int, default=DEFAULT_SHARDS_PER_CELL,
help="Linux shard formats: shards per (lane, transport, backend) cell")
parser.add_argument("--shard-seconds", type=int, default=DEFAULT_ANDROID_SHARD_SECONDS,
help="Android shard formats: hinted replay seconds one shard aims for")
parser.add_argument("--duration-hints", default=str(TRACE_DURATION_HINTS_JSON),
help="shard formats: the JSON of replay seconds the packing weighs cases by")
parser.add_argument(
"--format",
choices=(
@@ -586,7 +899,10 @@ def parse_args():
"github-apk",
"github-apk-matrix",
"github-apk-transport-matrix",
*LINUX_SHARD_FORMATS,
*ANDROID_SHARD_FORMATS,
"fixture-files",
"fixture-manifest",
"cmake",
),
default="names",
@@ -602,6 +918,9 @@ def main():
if args.self_test_split_matrix:
self_test_split_matrix()
return 0
if args.self_test_shard_matrices:
self_test_shard_matrices()
return 0
cases = load_trace_cases()
if args.ci:
cases = ci_trace_cases(cases)
@@ -623,6 +942,19 @@ def main():
print(json.dumps(github_apk_matrix(cases), separators=(",", ":")))
elif args.format == "github-apk-transport-matrix":
print(json.dumps(github_transport_matrix(cases, apk=True), separators=(",", ":")))
elif args.format in LINUX_SHARD_FORMATS + ANDROID_SHARD_FORMATS:
transports = tuple(t for t in args.transports.split(",") if t)
keep = tuple(c for c in args.keep_inproc.split(",") if c)
print(json.dumps(
github_shard_matrix(args.format, cases, shards=args.shards,
shard_seconds=args.shard_seconds, transports=transports,
keep_inproc=keep, hints_path=args.duration_hints),
separators=(",", ":")))
elif args.format == "fixture-manifest":
# One "<case><TAB><file>" line per fixture file, for the fixture bundle script.
for case in cases:
for file in fixture_files(case, args.fixture_root):
print(f"{case['name']}\t{file}")
elif args.format == "fixture-files":
if not args.case_name:
print("--case is required for --format fixture-files", file=sys.stderr)
tools/trace_replay/trace_duration_hints.json
+99
View File
@@ -0,0 +1,99 @@
{
"description": [
"Replay seconds per case, backend and transport, used only to pack retrace shards evenly",
"(tools/trace_replay/trace_cases.py, shard_matrix). A missing entry falls back to the median of",
"its backend and transport, so a new case needs no entry here. Nothing asserts these values;",
"a stale number only makes shards uneven. To refresh, copy the seconds column of the shard",
"step summaries of a green run."
],
"linux": {
"source": "Test run 43666299896 (2026-11-15), step 'Retrace and validate' of each one-case job, two gaps filled from run 43700779712; the verify key is the retrace-verify job",
"seconds": {
"OpenRA": {"DirectGLES": {"inproc": 3, "monolith": 3, "spawn": 3, "verify": 3}, "DirectVulkan": {"inproc": 3, "monolith": 3, "spawn": 2, "verify": 4}},
"improved-transparency-minecraft-26.3": {"DirectGLES": {"inproc": 86, "monolith": 51, "spawn": 78, "verify": 81}, "DirectVulkan": {"inproc": 82, "monolith": 104, "spawn": 107, "verify": 85}},
"minecraft-1.17-main-menu-854": {"DirectGLES": {"inproc": 4, "monolith": 6, "spawn": 3, "verify": 3}, "DirectVulkan": {"inproc": 5, "monolith": 5, "spawn": 3, "verify": 3}},
"minecraft-1.20.1-fabric-iris-iterationt-in-world": {"DirectGLES": {"inproc": 59, "monolith": 70, "spawn": 72}, "DirectVulkan": {"inproc": 62, "monolith": 69, "spawn": 68}},
"minecraft-1.21.1-neoforge-create-indirect-in-world-align1024": {"DirectGLES": {"inproc": 9, "monolith": 18, "spawn": 16}, "DirectVulkan": {"inproc": 13, "monolith": 13, "spawn": 8}},
"minecraft-1.21.1-neoforge-create-instancing-in-world": {"DirectGLES": {"inproc": 12, "monolith": 14, "spawn": 14}, "DirectVulkan": {"inproc": 10, "monolith": 14, "spawn": 10}},
"minecraft-1.21.11-main-menu": {"DirectGLES": {"inproc": 5, "monolith": 5, "spawn": 3, "verify": 6}, "DirectVulkan": {"inproc": 7, "monolith": 6, "spawn": 6, "verify": 5}},
"minecraft-1.21.4-fabric-common-mods-in-world": {"DirectGLES": {"inproc": 13, "monolith": 15, "spawn": 15}, "DirectVulkan": {"inproc": 19, "monolith": 17, "spawn": 14}},
"minecraft-1.21.4-fabric-common-mods-inventory": {"DirectGLES": {"inproc": 20, "monolith": 13, "spawn": 11}, "DirectVulkan": {"inproc": 22, "monolith": 21, "spawn": 22}},
"minecraft-1.21.4-fabric-iris-bliss-in-world": {"DirectGLES": {"inproc": 11, "monolith": 23, "spawn": 16}, "DirectVulkan": {"inproc": 17, "monolith": 17, "spawn": 16}},
"minecraft-1.21.4-fabric-iris-bsl-esc-menu-854": {"DirectGLES": {"inproc": 126, "monolith": 127, "spawn": 124}, "DirectVulkan": {"inproc": 130, "monolith": 131, "spawn": 134}},
"minecraft-1.21.4-fabric-iris-bsl-in-world": {"DirectGLES": {"inproc": 15, "monolith": 18, "spawn": 17}, "DirectVulkan": {"inproc": 16, "monolith": 15, "spawn": 17}},
"minecraft-1.21.4-fabric-iris-chocapic-v6-lite-in-world": {"DirectGLES": {"inproc": 10, "monolith": 15, "spawn": 14}, "DirectVulkan": {"inproc": 14, "monolith": 13, "spawn": 15}},
"minecraft-1.21.4-fabric-iris-complementary-reimagined-in-world": {"DirectGLES": {"inproc": 27, "monolith": 35, "spawn": 29}, "DirectVulkan": {"inproc": 41, "monolith": 31, "spawn": 28}},
"minecraft-1.21.4-fabric-iris-complementary-unbound-in-world": {"DirectGLES": {"inproc": 41, "monolith": 33, "spawn": 38}, "DirectVulkan": {"inproc": 26, "monolith": 31, "spawn": 25}},
"minecraft-1.21.4-fabric-iris-derivative-main-d24.4.14-in-world": {"DirectGLES": {"inproc": 23, "monolith": 33, "spawn": 24}, "DirectVulkan": {"inproc": 20, "monolith": 26, "spawn": 27}},
"minecraft-1.21.4-fabric-iris-iterationrp-in-world": {"DirectVulkan": {"inproc": 42, "monolith": 42, "spawn": 41}},
"minecraft-1.21.4-fabric-iris-iterationt-in-world": {"DirectGLES": {"inproc": 26, "monolith": 27, "spawn": 23}, "DirectVulkan": {"inproc": 23, "monolith": 20, "spawn": 19}},
"minecraft-1.21.4-fabric-iris-iterationt-nodsa-in-world": {"DirectGLES": {"inproc": 24, "monolith": 19, "spawn": 19}, "DirectVulkan": {"inproc": 19, "monolith": 13, "spawn": 21}},
"minecraft-1.21.4-fabric-iris-makeup-ultrafast-in-world": {"DirectGLES": {"inproc": 13, "monolith": 9, "spawn": 10}, "DirectVulkan": {"inproc": 11, "monolith": 8, "spawn": 8}},
"minecraft-1.21.4-fabric-iris-mellow-in-world": {"DirectGLES": {"inproc": 7, "monolith": 11, "spawn": 9}, "DirectVulkan": {"inproc": 9, "monolith": 9, "spawn": 7}},
"minecraft-1.21.4-fabric-iris-nostalgia-in-world": {"DirectGLES": {"inproc": 25, "monolith": 21, "spawn": 27}, "DirectVulkan": {"inproc": 24, "monolith": 26, "spawn": 16}},
"minecraft-1.21.4-fabric-iris-photon-v1.1-in-world": {"DirectGLES": {"inproc": 25, "monolith": 34, "spawn": 21}, "DirectVulkan": {"inproc": 24, "monolith": 24, "spawn": 33}},
"minecraft-1.21.4-fabric-iris-photon-v1.3b-in-world": {"DirectGLES": {"inproc": 29, "monolith": 36, "spawn": 29}, "DirectVulkan": {"inproc": 30, "monolith": 27, "spawn": 36}},
"minecraft-1.21.4-fabric-iris-sundial-lite-in-world": {"DirectGLES": {"inproc": 35, "monolith": 27, "spawn": 21}, "DirectVulkan": {"inproc": 31, "monolith": 20, "spawn": 30}},
"minecraft-1.21.4-fabric-iris-super-duper-vanilla-in-world": {"DirectGLES": {"monolith": 22, "spawn": 21}, "DirectVulkan": {"inproc": 14, "monolith": 20, "spawn": 14}},
"minecraft-1.21.4-fabric-journeymap-in-world": {"DirectGLES": {"inproc": 43, "monolith": 66, "spawn": 65}, "DirectVulkan": {"inproc": 41, "monolith": 60, "spawn": 90}},
"minecraft-1.21.4-fabric-journeymap-in-world-normal-world": {"DirectGLES": {"inproc": 10, "monolith": 12, "spawn": 14}, "DirectVulkan": {"inproc": 15, "monolith": 12, "spawn": 14}},
"minecraft-1.21.4-fabric-modernui-inventory": {"DirectGLES": {"inproc": 54, "monolith": 50, "spawn": 67}, "DirectVulkan": {"inproc": 50, "monolith": 66, "spawn": 70}},
"minecraft-1.21.4-fabric-modernui-inventory-normal-world": {"DirectGLES": {"inproc": 15, "monolith": 17, "spawn": 9}, "DirectVulkan": {"inproc": 18, "monolith": 13, "spawn": 10}},
"minecraft-1.21.4-fabric-rei-inventory": {"DirectGLES": {"inproc": 72, "monolith": 63, "spawn": 90}, "DirectVulkan": {"inproc": 79, "monolith": 84, "spawn": 49}},
"minecraft-1.21.4-fabric-rei-inventory-normal-world": {"DirectGLES": {"inproc": 15, "monolith": 22, "spawn": 17}, "DirectVulkan": {"inproc": 20, "monolith": 18, "spawn": 19}},
"minecraft-1.21.4-fabric-sodium-in-world": {"DirectGLES": {"inproc": 19, "monolith": 15, "spawn": 26, "verify": 25}, "DirectVulkan": {"inproc": 29, "monolith": 32, "spawn": 23, "verify": 16}},
"minecraft-1.21.4-fabric-xaero-minimap-in-world": {"DirectGLES": {"inproc": 41, "monolith": 51, "spawn": 64}, "DirectVulkan": {"inproc": 55, "monolith": 69, "spawn": 77}},
"minecraft-1.21.4-fabric-xaero-minimap-in-world-normal-world": {"DirectGLES": {"inproc": 12, "monolith": 13, "spawn": 8}, "DirectVulkan": {"inproc": 9, "monolith": 10, "spawn": 13}},
"minecraft-1.21.4-fabric-xaero-world-map-in-world": {"DirectGLES": {"inproc": 41, "monolith": 40, "spawn": 52}, "DirectVulkan": {"inproc": 42, "monolith": 43, "spawn": 59}},
"minecraft-1.21.4-fabric-xaero-world-map-in-world-normal-world": {"DirectGLES": {"inproc": 6, "monolith": 11, "spawn": 8}, "DirectVulkan": {"inproc": 13, "monolith": 9, "spawn": 10}},
"minecraft-1.21.4-in-world": {"DirectGLES": {"inproc": 17, "monolith": 10, "spawn": 11, "verify": 11}, "DirectVulkan": {"inproc": 10, "monolith": 9, "spawn": 12, "verify": 11}},
"minecraft-1.21.4-main-menu": {"DirectGLES": {"inproc": 7, "monolith": 4, "spawn": 9, "verify": 6}, "DirectVulkan": {"inproc": 5, "monolith": 10, "spawn": 6, "verify": 6}},
"minecraft-1.21.4-startup": {"DirectGLES": {"inproc": 5, "monolith": 3, "spawn": 1, "verify": 3}, "DirectVulkan": {"inproc": 1, "monolith": 4, "spawn": 2, "verify": 2}}
}
},
"android": {
"source": "MobileGL APK run 43666299861 (2026-11-15), step 'Retrace and validate' of each one-case emulator job, APK install included",
"seconds": {
"OpenRA": {"DirectGLES": {"inproc": 28, "monolith": 28, "spawn": 24}, "DirectVulkan": {"inproc": 24, "monolith": 22, "spawn": 16}},
"improved-transparency-minecraft-26.3": {"DirectGLES": {"inproc": 179, "monolith": 170, "spawn": 142}, "DirectVulkan": {"inproc": 169, "monolith": 209, "spawn": 153}},
"minecraft-1.17-main-menu-854": {"DirectGLES": {"inproc": 27, "monolith": 19, "spawn": 22}, "DirectVulkan": {"inproc": 25, "monolith": 25, "spawn": 26}},
"minecraft-1.20.1-fabric-iris-iterationt-in-world": {"DirectGLES": {"inproc": 131, "monolith": 147, "spawn": 153}, "DirectVulkan": {"inproc": 149, "monolith": 111, "spawn": 116}},
"minecraft-1.21.1-neoforge-create-indirect-in-world-align1024": {"DirectGLES": {"inproc": 74, "monolith": 85, "spawn": 73}, "DirectVulkan": {"inproc": 44, "monolith": 31, "spawn": 53}},
"minecraft-1.21.1-neoforge-create-instancing-in-world": {"DirectGLES": {"inproc": 74, "monolith": 56, "spawn": 78}, "DirectVulkan": {"inproc": 42, "monolith": 49, "spawn": 37}},
"minecraft-1.21.11-main-menu": {"DirectGLES": {"inproc": 38, "monolith": 37, "spawn": 30}, "DirectVulkan": {"inproc": 46, "monolith": 25, "spawn": 45}},
"minecraft-1.21.4-fabric-common-mods-in-world": {"DirectGLES": {"inproc": 66, "monolith": 51, "spawn": 61}, "DirectVulkan": {"inproc": 70, "monolith": 62, "spawn": 53}},
"minecraft-1.21.4-fabric-common-mods-inventory": {"DirectGLES": {"inproc": 74, "monolith": 86, "spawn": 41}, "DirectVulkan": {"inproc": 61, "monolith": 55, "spawn": 67}},
"minecraft-1.21.4-fabric-iris-bliss-in-world": {"DirectGLES": {"inproc": 74, "monolith": 70, "spawn": 53}, "DirectVulkan": {"inproc": 53, "monolith": 69, "spawn": 50}},
"minecraft-1.21.4-fabric-iris-bsl-esc-menu-854": {"DirectGLES": {"inproc": 220, "monolith": 171, "spawn": 176}, "DirectVulkan": {"inproc": 138, "monolith": 234, "spawn": 193}},
"minecraft-1.21.4-fabric-iris-bsl-in-world": {"DirectGLES": {"inproc": 60, "monolith": 42, "spawn": 48}, "DirectVulkan": {"inproc": 56, "monolith": 48, "spawn": 60}},
"minecraft-1.21.4-fabric-iris-chocapic-v6-lite-in-world": {"DirectGLES": {"inproc": 62, "monolith": 54, "spawn": 50}, "DirectVulkan": {"inproc": 51, "monolith": 42, "spawn": 36}},
"minecraft-1.21.4-fabric-iris-complementary-reimagined-in-world": {"DirectGLES": {"inproc": 93, "monolith": 94, "spawn": 110}, "DirectVulkan": {"inproc": 92, "monolith": 112, "spawn": 88}},
"minecraft-1.21.4-fabric-iris-complementary-unbound-in-world": {"DirectGLES": {"inproc": 99, "monolith": 97, "spawn": 99}, "DirectVulkan": {"inproc": 98, "monolith": 107, "spawn": 115}},
"minecraft-1.21.4-fabric-iris-derivative-main-d24.4.14-in-world": {"DirectGLES": {"inproc": 67, "monolith": 81, "spawn": 60}, "DirectVulkan": {"inproc": 77, "monolith": 76, "spawn": 47}},
"minecraft-1.21.4-fabric-iris-iterationrp-in-world": {"DirectVulkan": {"inproc": 102, "monolith": 108, "spawn": 94}},
"minecraft-1.21.4-fabric-iris-iterationt-in-world": {"DirectGLES": {"inproc": 40, "monolith": 57, "spawn": 48}, "DirectVulkan": {"inproc": 66, "monolith": 56, "spawn": 61}},
"minecraft-1.21.4-fabric-iris-iterationt-nodsa-in-world": {"DirectGLES": {"inproc": 68, "monolith": 65, "spawn": 39}, "DirectVulkan": {"inproc": 43, "monolith": 70, "spawn": 66}},
"minecraft-1.21.4-fabric-iris-makeup-ultrafast-in-world": {"DirectGLES": {"inproc": 32, "monolith": 47, "spawn": 46}, "DirectVulkan": {"inproc": 39, "monolith": 42, "spawn": 43}},
"minecraft-1.21.4-fabric-iris-mellow-in-world": {"DirectGLES": {"inproc": 47, "monolith": 37, "spawn": 42}, "DirectVulkan": {"inproc": 46, "monolith": 40, "spawn": 40}},
"minecraft-1.21.4-fabric-iris-nostalgia-in-world": {"DirectGLES": {"inproc": 92, "monolith": 51, "spawn": 87}, "DirectVulkan": {"inproc": 60, "monolith": 55, "spawn": 83}},
"minecraft-1.21.4-fabric-iris-photon-v1.1-in-world": {"DirectGLES": {"inproc": 107, "monolith": 105, "spawn": 79}, "DirectVulkan": {"inproc": 59, "monolith": 100, "spawn": 96}},
"minecraft-1.21.4-fabric-iris-photon-v1.3b-in-world": {"DirectGLES": {"inproc": 127, "monolith": 114, "spawn": 60}, "DirectVulkan": {"inproc": 94, "monolith": 106, "spawn": 81}},
"minecraft-1.21.4-fabric-iris-sundial-lite-in-world": {"DirectGLES": {"inproc": 80, "monolith": 77, "spawn": 77}, "DirectVulkan": {"inproc": 76, "monolith": 77, "spawn": 91}},
"minecraft-1.21.4-fabric-iris-super-duper-vanilla-in-world": {"DirectGLES": {"inproc": 71, "monolith": 63, "spawn": 48}, "DirectVulkan": {"inproc": 53, "monolith": 67, "spawn": 49}},
"minecraft-1.21.4-fabric-journeymap-in-world": {"DirectGLES": {"inproc": 139, "monolith": 241, "spawn": 172}, "DirectVulkan": {"inproc": 180, "monolith": 257}},
"minecraft-1.21.4-fabric-journeymap-in-world-normal-world": {"DirectGLES": {"inproc": 56, "monolith": 37, "spawn": 67}, "DirectVulkan": {"inproc": 51, "monolith": 53, "spawn": 54}},
"minecraft-1.21.4-fabric-modernui-inventory": {"DirectGLES": {"inproc": 207, "monolith": 248, "spawn": 105}, "DirectVulkan": {"inproc": 303, "monolith": 324, "spawn": 195}},
"minecraft-1.21.4-fabric-modernui-inventory-normal-world": {"DirectGLES": {"inproc": 56, "monolith": 58, "spawn": 53}, "DirectVulkan": {"inproc": 60, "monolith": 58, "spawn": 81}},
"minecraft-1.21.4-fabric-rei-inventory": {"DirectGLES": {"inproc": 328, "monolith": 337, "spawn": 144}, "DirectVulkan": {"inproc": 325, "monolith": 197, "spawn": 199}},
"minecraft-1.21.4-fabric-rei-inventory-normal-world": {"DirectGLES": {"inproc": 79, "monolith": 62, "spawn": 46}, "DirectVulkan": {"inproc": 82, "monolith": 51, "spawn": 66}},
"minecraft-1.21.4-fabric-sodium-in-world": {"DirectGLES": {"inproc": 178, "monolith": 175, "spawn": 91}, "DirectVulkan": {"inproc": 166, "monolith": 136, "spawn": 72}},
"minecraft-1.21.4-fabric-xaero-minimap-in-world": {"DirectGLES": {"inproc": 231, "monolith": 254, "spawn": 99}, "DirectVulkan": {"inproc": 278, "monolith": 225, "spawn": 127}},
"minecraft-1.21.4-fabric-xaero-minimap-in-world-normal-world": {"DirectGLES": {"inproc": 61, "monolith": 47, "spawn": 46}, "DirectVulkan": {"inproc": 61, "monolith": 44, "spawn": 42}},
"minecraft-1.21.4-fabric-xaero-world-map-in-world": {"DirectGLES": {"inproc": 313, "monolith": 223, "spawn": 104}, "DirectVulkan": {"inproc": 276, "monolith": 277, "spawn": 121}},
"minecraft-1.21.4-fabric-xaero-world-map-in-world-normal-world": {"DirectGLES": {"inproc": 62, "monolith": 47, "spawn": 41}, "DirectVulkan": {"inproc": 52, "monolith": 40, "spawn": 69}},
"minecraft-1.21.4-in-world": {"DirectGLES": {"inproc": 44, "monolith": 59, "spawn": 43}, "DirectVulkan": {"inproc": 53, "monolith": 59, "spawn": 51}},
"minecraft-1.21.4-main-menu": {"DirectGLES": {"inproc": 54, "monolith": 48, "spawn": 35}, "DirectVulkan": {"inproc": 58, "monolith": 67, "spawn": 46}},
"minecraft-1.21.4-startup": {"DirectGLES": {"inproc": 22, "monolith": 19, "spawn": 19}, "DirectVulkan": {"inproc": 18, "monolith": 14, "spawn": 14}}
}
}
}