diff --git a/.github/scripts/trace-fixture-bundle.sh b/.github/scripts/trace-fixture-bundle.sh new file mode 100644 index 00000000..d086aa7c --- /dev/null +++ b/.github/scripts/trace-fixture-bundle.sh @@ -0,0 +1,304 @@ +#!/usr/bin/env bash +# One cache entry that holds every Git LFS trace fixture of the CI cases. +# +# key derive the actions/cache key, restore prefix and path of the bundle +# ensure after a cache miss: complete the bundle directory, fetching only the +# cases whose files are missing from it or do not match their pointers +# wait [secs] wait until the cache entry is listed for this ref (needs GH_TOKEN) +# install ... move the fixtures of these cases from the restored bundle directory into +# the fixture directory, fetch the cases it does not hold, and check every +# file against its pointer +# +# The key is content-addressed on the pointer oids tracked at HEAD, which a plain checkout +# reads without smudging. The bundle directory is the same path for every revision, so +# `ensure` can start from an older bundle. Fixtures stored in Git itself rather than in Git +# LFS (OpenRA) are not in the bundle; `install` checks that they are real files. +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=trace-fixture-lib.sh +. "${script_dir}/trace-fixture-lib.sh" + +# Bump when the key derivation or the bundle layout changes in a way that must invalidate +# old entries; the content digest alone would not notice a format change. +key_schema="v1" +bundle_dir="trace-fixture-bundle" +fixture_dir="tools/trace_replay/fixtures" +python_bin="${PYTHON:-python3}" + +if ! command -v "${python_bin}" >/dev/null 2>&1 && command -v python >/dev/null 2>&1; then + python_bin=python +fi + +usage() { + echo "usage: $0 [seconds]|install ...>" >&2 + exit 2 +} + +# Writes "name=value" to $GITHUB_OUTPUT when running under Actions, and to stdout as well so +# the script stays runnable off-CI. +emit_output() { + local name="$1" + local value="$2" + if [ -n "${GITHUB_OUTPUT:-}" ]; then + printf '%s=%s\n' "${name}" "${value}" >> "${GITHUB_OUTPUT}" + fi + printf '%s=%s\n' "${name}" "${value}" +} + +# "" for every fixture file of every CI case. +ci_fixture_manifest() { + "${python_bin}" tools/trace_replay/trace_cases.py --ci --format fixture-manifest \ + --fixture-root "${fixture_dir}" | tr -d '\r' +} + +is_lfs_fixture() { + get_lfs_metadata "$1" >/dev/null 2>&1 +} + +# A fixture that Git stores directly must be a real file, not a pointer left by a wrong +# attribute. +check_plain_fixture() { + local file="$1" + if [ ! -s "${file}" ]; then + echo "fixture file is missing or empty: ${file}" >&2 + return 1 + fi + if head -n 1 "${file}" | grep -q "version https://git-lfs.github.com/spec/v1"; then + echo "fixture should not be stored as an LFS pointer: ${file}" >&2 + return 1 + fi +} + +# Puts the tracked pointer files back. The smudge filter is skipped so that a clone with Git +# LFS set up does not download the objects here. +restore_pointers() { + GIT_LFS_SKIP_SMUDGE=1 git checkout -- "$@" +} + +verify_lfs_fixture() { + local on_disk="$1" + local tracked="$2" + local metadata expected_oid expected_size + metadata="$(get_lfs_metadata "${tracked}")" || return 1 + read -r expected_oid expected_size <<< "${metadata}" + verify_fixture_file "${on_disk}" "${tracked}" "${expected_oid}" "${expected_size}" +} + +# Moves the given Git LFS fixtures of one case from the bundle directory into the fixture +# directory. Succeeds when every one of them is then in place and matches its pointer. +take_from_bundle() { + local file bundled + local incomplete=0 + for file in "$@"; do + bundled="${bundle_dir}/$(basename "${file}")" + # Two cases may share a file; the second finds it already in place. + if [ -f "${bundled}" ]; then + mv -f "${bundled}" "${file}" + fi + if ! verify_lfs_fixture "${file}" "${file}" 2>/dev/null; then + incomplete=1 + fi + done + return "${incomplete}" +} + +# Downloads the given Git LFS fixtures of one case over their pointers: the mirrors first, +# Git LFS for what no mirror serves. The fetch script verifies what it downloaded. +fetch_case_fixtures() { + local case_name="$1" + shift + local file + for file in "$@"; do + rm -f "${file}" "${file}.tmp" + done + restore_pointers "$@" || return 1 + bash "${script_dir}/fetch-trace-fixture-lfs.sh" "${case_name}" "${fixture_dir}" +} + +[ "$#" -ge 1 ] || usage +command_name="$1" +shift + +case "${command_name}" in + key) + [ "$#" -eq 0 ] || usage + # One " " line per bundled file, sorted, each file once. + manifest="" + count=0 + while IFS= read -r file; do + [ -n "${file}" ] || continue + if ! metadata="$(get_lfs_metadata "${file}" 2>/dev/null)"; then + continue + fi + read -r expected_oid expected_size <<< "${metadata}" + manifest+="$(basename "${file}") ${expected_oid} ${expected_size}"$'\n' + count=$((count + 1)) + done < <(ci_fixture_manifest | cut -f2 | LC_ALL=C sort -u) + if [ "${count}" -eq 0 ]; then + echo "no Git LFS fixture found for the CI trace cases; the bundle would be empty" >&2 + exit 1 + fi + duplicates="$(printf '%s' "${manifest}" | cut -d' ' -f1 | LC_ALL=C sort | uniq -d)" + if [ -n "${duplicates}" ]; then + echo "two fixture files share a name, so the bundle directory cannot hold both: ${duplicates}" >&2 + exit 1 + fi + digest="$(printf '%s' "${manifest}" | sha256sum | awk '{ print substr($1, 1, 16) }')" + emit_output "key" "trace-fixture-bundle-${key_schema}-${digest}" + emit_output "restore-prefix" "trace-fixture-bundle-${key_schema}-" + emit_output "path" "${bundle_dir}" + emit_output "files" "${count}" + ;; + + ensure) + [ "$#" -eq 0 ] || usage + mkdir -p "${bundle_dir}" "${fixture_dir}" + mapfile -t manifest_lines < <(ci_fixture_manifest) + declare -A case_seen=() + case_order=() + for line in "${manifest_lines[@]}"; do + case_name="${line%%$'\t'*}" + if [ -z "${case_seen[${case_name}]:-}" ]; then + case_seen["${case_name}"]=1 + case_order+=("${case_name}") + fi + done + + fetched=0 + reused=0 + bundled_files=() + for case_name in "${case_order[@]}"; do + lfs_files=() + for line in "${manifest_lines[@]}"; do + [ "${line%%$'\t'*}" = "${case_name}" ] || continue + file="${line#*$'\t'}" + if is_lfs_fixture "${file}"; then + lfs_files+=("${file}") + else + check_plain_fixture "${file}" + fi + done + [ "${#lfs_files[@]}" -gt 0 ] || continue + + # Take what an older bundle already holds, and fetch the case if that is not all of it. + if take_from_bundle "${lfs_files[@]}"; then + reused=$((reused + 1)) + else + echo "Fetching trace fixtures of ${case_name}: the bundle does not hold them" + fetch_case_fixtures "${case_name}" "${lfs_files[@]}" + fetched=$((fetched + 1)) + fi + bundled_files+=("${lfs_files[@]}") + done + + # The bundle is rebuilt from the verified files, so nothing an older bundle carried and + # no case names any more is saved again. + rm -rf "${bundle_dir}" + mkdir -p "${bundle_dir}" + declare -A file_seen=() + saved=0 + for file in "${bundled_files[@]}"; do + [ -z "${file_seen[${file}]:-}" ] || continue + file_seen["${file}"]=1 + verify_lfs_fixture "${file}" "${file}" + mv -f "${file}" "${bundle_dir}/$(basename "${file}")" + saved=$((saved + 1)) + done + if [ "${saved}" -eq 0 ]; then + echo "the trace fixture bundle came out empty" >&2 + exit 1 + fi + # Leave the working tree as a checkout has it: pointers in the fixture directory. + restore_pointers "${!file_seen[@]}" + echo "Trace fixture bundle: ${saved} file(s), ${reused} case(s) from the restored bundle, ${fetched} case(s) fetched." + emit_output "fetched" "${fetched}" + ;; + + wait) + [ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage + key="$1" + limit="${2:-300}" + # Another workflow of the same push may be the one saving this key. The replay jobs + # restore the entry, so it has to be listed before this job ends. The caller checks the + # entry itself afterwards; this only waits, and never fails. + # + # The listing is narrowed to the ref of this run. The key is content-addressed, so another + # branch may already hold an entry under it, and that entry cannot be restored from here: + # counting it would end the wait while the entry of this ref is still being uploaded. + query=(-f "key=${key}" -f per_page=100) + if [ -n "${GITHUB_REF:-}" ]; then + query+=(-f "ref=${GITHUB_REF}") + fi + waited=0 + refused=0 + while :; do + if listed="$(gh api -X GET "repos/${GITHUB_REPOSITORY}/actions/caches" "${query[@]}" \ + --jq '[.actions_caches[] | select(.key == "'"${key}"'")] | length' 2>/dev/null)"; then + if [ "${listed:-0}" -ge 1 ]; then + echo "cache entry ${key} is listed after ${waited} s" + break + fi + else + refused=$((refused + 1)) + if [ "${refused}" -ge 3 ]; then + echo "the cache listing cannot be read (no token, or no actions: read permission); not waiting" + break + fi + fi + if [ "${waited}" -ge "${limit}" ]; then + echo "cache entry ${key} is not listed after ${waited} s" + break + fi + sleep 10 + waited=$((waited + 10)) + done + ;; + + install) + [ "$#" -ge 1 ] || usage + installed=0 + fetched=0 + for case_name in "$@"; do + mapfile -t files < <(trace_fixture_files "${case_name}" "${fixture_dir}" "${python_bin}") + if [ "${#files[@]}" -eq 0 ]; then + echo "no fixture files declared for trace case: ${case_name}" >&2 + exit 1 + fi + lfs_files=() + for file in "${files[@]}"; do + if is_lfs_fixture "${file}"; then + lfs_files+=("${file}") + else + check_plain_fixture "${file}" + fi + done + [ "${#lfs_files[@]}" -gt 0 ] || continue + + # A bundle that was not restored (the cache entry was evicted, or its download failed) + # or that lacks a file costs this job a download of the case, not the replay. + if ! take_from_bundle "${lfs_files[@]}"; then + echo "::warning::the restored fixture bundle does not hold the trace fixtures of ${case_name}; fetching them" + if ! fetch_case_fixtures "${case_name}" "${lfs_files[@]}"; then + echo "::error::the trace fixtures of ${case_name} are not in the restored bundle and could not be fetched" + exit 1 + fi + fetched=$((fetched + 1)) + for file in "${lfs_files[@]}"; do + if ! verify_lfs_fixture "${file}" "${file}"; then + echo "::error::trace fixture ${file} of ${case_name} does not match its Git LFS pointer after the fetch" + exit 1 + fi + done + fi + installed=$((installed + ${#lfs_files[@]})) + done + # The rest of the bundle belongs to other shards. + rm -rf "${bundle_dir}" + echo "Installed and verified ${installed} Git LFS fixture file(s) for $# trace case(s), ${fetched} case(s) fetched." + ;; + + *) + usage + ;; +esac diff --git a/.github/scripts/trace-fixture-cache.sh b/.github/scripts/trace-fixture-cache.sh deleted file mode 100644 index 11b525bd..00000000 --- a/.github/scripts/trace-fixture-cache.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bash -# Cache-side helper for trace fixtures. -# -# key [fixture-dir] derive the actions/cache key and path list -# verify [fixture-dir] check restored fixtures against their pointers -# reset [fixture-dir] drop restored fixtures, leaving the pointers -# -# The cache key is content-addressed on the Git LFS pointer oids tracked at -# HEAD, which are readable from a plain checkout without smudging. Fixture -# content therefore maps 1:1 onto a key: unchanged content hits, changed -# content is a new key and thus a miss, and the download path handles it. The -# key deliberately carries no restore-keys prefix in the workflow - a fixture -# that does not match the pointer exactly must never be restored. -set -euo pipefail - -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=trace-fixture-lib.sh -. "${script_dir}/trace-fixture-lib.sh" - -# Bump when the key derivation changes in a way that must invalidate old -# entries; the content digest alone would not notice a format change. -key_schema="v1" - -if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then - echo "usage: $0 [fixture-dir]" >&2 - exit 2 -fi - -command_name="$1" -case_name="$2" -fixture_dir="${3:-tools/trace_replay/fixtures}" -python_bin="${PYTHON:-python3}" - -if ! command -v "${python_bin}" >/dev/null 2>&1 && command -v python >/dev/null 2>&1; then - python_bin=python -fi - -mapfile -t files < <(trace_fixture_files "${case_name}" "${fixture_dir}" "${python_bin}") -if [ "${#files[@]}" -eq 0 ]; then - echo "no fixture files declared for trace case: ${case_name}" >&2 - exit 1 -fi - -# Writes "name=value" to $GITHUB_OUTPUT when running under Actions, and to -# stdout otherwise so the script stays runnable (and testable) off-CI. -emit_output() { - local name="$1" - local value="$2" - if [ -n "${GITHUB_OUTPUT:-}" ]; then - if [[ "${value}" == *$'\n'* ]]; then - local delimiter="ghadelim_$(date +%s%N)_$$" - { - printf '%s<<%s\n' "${name}" "${delimiter}" - printf '%s\n' "${value}" - printf '%s\n' "${delimiter}" - } >> "${GITHUB_OUTPUT}" - else - printf '%s=%s\n' "${name}" "${value}" >> "${GITHUB_OUTPUT}" - fi - fi - printf '%s=%s\n' "${name}" "${value}" -} - -sanitize_case() { - printf '%s' "$1" | sed 's/[^A-Za-z0-9._-]/_/g' -} - -case "${command_name}" in - key) - manifest="" - for file in "${files[@]}"; do - # A case whose fixtures are committed directly rather than through Git LFS - # (OpenRA) has no pointer oid to key on, and nothing to download either. - # Report it as uncacheable so the workflow skips the cache entirely. - if ! metadata="$(get_lfs_metadata "${file}" 2>/dev/null)"; then - echo "trace case ${case_name} is not stored in Git LFS; skipping fixture cache" >&2 - emit_output "cacheable" "false" - emit_output "key" "" - exit 0 - fi - read -r expected_oid expected_size <<< "${metadata}" - manifest+="$(basename "${file}") ${expected_oid} ${expected_size}"$'\n' - done - - digest="$(printf '%s' "${manifest}" | sha256sum | awk '{ print substr($1, 1, 16) }')" - safe_case="$(sanitize_case "${case_name}")" - - emit_output "cacheable" "true" - emit_output "key" "trace-fixture-${key_schema}-${safe_case}-${digest}" - emit_output "paths" "$(printf '%s\n' "${files[@]}")" - ;; - - verify) - for file in "${files[@]}"; do - metadata="$(get_lfs_metadata "${file}")" - read -r expected_oid expected_size <<< "${metadata}" - verify_fixture_file "${file}" "${file}" "${expected_oid}" "${expected_size}" - done - echo "Verified ${#files[@]} fixture file(s) for ${case_name} against the tracked Git LFS pointers." - ;; - - reset) - # Put the working tree back to the pointer files a fresh checkout would - # have, so that a rejected cache entry falls through to exactly the same - # download path a cache miss takes. - for file in "${files[@]}"; do - rm -f "${file}" "${file}.tmp" - done - git checkout -- "${files[@]}" - echo "Reset ${#files[@]} fixture file(s) for ${case_name} to their tracked Git LFS pointers." - ;; - - *) - echo "unknown command: ${command_name}" >&2 - exit 2 - ;; -esac diff --git a/.github/scripts/trace-fixture-lib.sh b/.github/scripts/trace-fixture-lib.sh index 0e3b971b..925af97c 100644 --- a/.github/scripts/trace-fixture-lib.sh +++ b/.github/scripts/trace-fixture-lib.sh @@ -2,7 +2,7 @@ # Shared helpers for trace-fixture handling: reading the in-tree Git LFS pointer # metadata and verifying a fixture file against it. Sourced by # fetch-trace-fixture-lfs.sh (verify after download) and by -# trace-fixture-cache.sh (cache key derivation and verify after cache restore), +# trace-fixture-bundle.sh (cache key derivation and verify after cache restore), # so both paths agree on what a valid fixture is. # Reads the Git LFS pointer tracked at HEAD for a fixture path and prints diff --git a/.github/workflows/apk.yml b/.github/workflows/apk.yml index ccefc7c8..e358792d 100644 --- a/.github/workflows/apk.yml +++ b/.github/workflows/apk.yml @@ -12,11 +12,28 @@ on: - feat/disaggregated - codex/fix-github-ci - cleanup - # A docs-only push builds nothing this workflow tests (231 emulator legs per run). The Test - # workflow keeps running for docs: it hosts the doc citation lint. + # A push that only touches these paths changes nothing this workflow builds or replays. The + # Test workflow keeps running for docs: it hosts the doc citation lint. The Android build + # compiles none of the unit, integration or benchmark trees, with one exception that keeps + # MobileGL/MG_Benchmark from being listed whole: MG_Util/SelfTest/DriverBenchJni.cpp + # includes MG_Benchmark/Driver/DriverBenchCases.inc. paths-ignore: - 'docs/**' - '**/*.md' + - 'MobileGL/MG_Test/**' + - 'MobileGL/MG_IntegrationTest/**' + - 'MobileGL/MG_Benchmark/CMakeLists.txt' + - 'MobileGL/MG_Benchmark/SanityBench.cpp' + - 'MobileGL/MG_Benchmark/Buffer/**' + - 'MobileGL/MG_Benchmark/Container/**' + - 'MobileGL/MG_Benchmark/Program/**' + - 'MobileGL/MG_Benchmark/ShaderCache/**' + - 'MobileGL/MG_Benchmark/Transpile/**' + - 'tools/cts/**' + - 'tools/device_bench/**' + - '.github/workflows/test.yml' + - '.github/workflows/hygiene.yml' + - '.github/actions/setup-test-runner/**' workflow_dispatch: env: @@ -39,7 +56,10 @@ jobs: CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "true" CCACHE_DIR: ${{ github.workspace }}/.ccache - CCACHE_MAXSIZE: 4G + # One build's objects (three native configurations) take about 1 GB. The limit leaves + # half of that again for the objects a push replaces, and is what the cleanup before + # the save trims the entry to. + CCACHE_MAXSIZE: 1500M CCACHE_NOHASHDIR: "true" MOBILEGL_CMAKE_COMPILER_LAUNCHER: ccache MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }} @@ -47,8 +67,67 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v6 - with: - submodules: recursive + + # Only the submodules the two APKs are built from, shallow and in parallel, in place of a + # recursive checkout of every submodule in the tree: the ones the root CMakeLists.txt + # configures, and apitrace with the eight directories its thirdparty/CMakeLists.txt + # requires, for the trace APK. Each row names a submodule and one file the build reads + # from it, and a row whose file is missing afterwards fails the step: without + # 3rdparty/flatbuffers the root CMakeLists.txt would go on and build a library without + # MG_Remote instead of stopping. A submodule the build starts to use needs a row here. + - name: Check out build submodules + timeout-minutes: 15 + run: | + top=( + "3rdparty/glslang CMakeLists.txt" + "3rdparty/SPIRV-Cross CMakeLists.txt" + "3rdparty/SPIRV-Reflect CMakeLists.txt" + "3rdparty/VulkanMemoryAllocator CMakeLists.txt" + "3rdparty/Vulkan-Headers CMakeLists.txt" + "3rdparty/Vulkan-Utility-Libraries CMakeLists.txt" + "3rdparty/xxHash build/cmake/CMakeLists.txt" + "3rdparty/asio include/asio.hpp" + "3rdparty/flatbuffers include/flatbuffers/flatbuffers.h" + "include/ska flat_hash_map.hpp" + "3rdparty/apitrace thirdparty/CMakeLists.txt" + ) + apitrace=( + "thirdparty/brotli LICENSE" + "thirdparty/gtest LICENSE" + "thirdparty/libbacktrace LICENSE" + "thirdparty/libpng LICENSE" + "thirdparty/snappy COPYING" + "thirdparty/zlib README" + "thirdparty/zstd LICENSE" + "thirdparty/directxmath LICENSE" + ) + # update ...: the update is idempotent, so a fetch that failed on + # the network is tried again, three times in all. Then every row's file must exist. + update() { + local dir="$1" paths=() row attempt updated=0 missing=0 + shift + for row in "$@"; do + paths+=("${row%% *}") + done + for attempt in 1 2 3; do + if git -C "${dir}" submodule update --init --depth=1 --jobs 8 -- "${paths[@]}"; then + updated=1 + break + fi + echo "the submodule update in ${dir} failed (attempt ${attempt} of 3)" >&2 + sleep $((attempt * 5)) + done + [ "${updated}" -eq 1 ] || return 1 + for row in "$@"; do + if [ ! -f "${dir}/${row%% *}/${row#* }" ]; then + echo "::error::${dir}/${row%% *}/${row#* } is missing after the submodule update, so the APK build cannot use ${row%% *}" + missing=1 + fi + done + return "${missing}" + } + update . "${top[@]}" + update 3rdparty/apitrace "${apitrace[@]}" - name: Set artifact metadata run: | @@ -82,6 +161,9 @@ jobs: sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 update sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 install -y ccache ccache --version + # The restored entry carries the counters of every earlier build; zero them so the + # statistics printed after the build are this run's. + ccache --zero-stats - name: Setup Android SDK id: netsetup84 @@ -183,11 +265,20 @@ jobs: if: always() run: ccache --show-stats - # Rewrite one rolling entry per job and branch. The upload stays cumulative - it - # carries every object restored at the top of this run plus the few TUs that - # changed - but Actions cache keys are immutable, so this branch's superseded - # blob is released first; --ref keeps the delete off other branches' entries. - # Running after the build means a failed build leaves the entry untouched. + # A build served from the cache stores nothing, so nothing would ever evict the objects + # of older source states. Trim to CCACHE_MAXSIZE, least recently used first, before the + # entry is saved; the objects this build hit were just touched and stay. + - name: Trim ccache to its size limit + run: | + ccache --cleanup + ccache --show-stats | grep -i 'cache size' || true + du -sh "${CCACHE_DIR}" + + # Rewrite one rolling entry per job and branch. The upload carries the objects this + # build used plus what fits under the size limit, but Actions cache keys are + # immutable, so this branch's superseded blob is released first; --ref keeps the + # delete off other branches' entries. Running after the build means a failed build + # leaves the entry untouched. - name: Release superseded ccache entry env: GH_TOKEN: ${{ github.token }} @@ -252,9 +343,17 @@ jobs: runs-on: ubuntu-26.04 env: MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }} + # How much replay one emulator job aims for, in seconds as the duration hints count them. + # A (lane, transport, backend) cell is cut into as many shards as that takes, and its + # cases are spread so the shards of a cell come out even (tools/trace_replay/trace_cases.py, + # shard_matrix). A smaller value gives more and shorter jobs, each of which pays the + # emulator boot again; every case is replayed once per cell whatever the value. + RETRACE_SHARD_SECONDS: "720" + # The most a shard runner may spend on its cases. A shard gets the time in which every + # one of its cases could run to the end of its own time budget, up to this cap. + RETRACE_SHARD_DEADLINE_CAP_SECONDS: "7200" outputs: android: ${{ steps.trace-cases.outputs.android }} - names: ${{ steps.trace-cases.outputs.names }} steps: - name: Checkout repo uses: actions/checkout@v6 @@ -263,77 +362,105 @@ jobs: id: trace-cases run: | python3 tools/trace_replay/trace_cases.py --self-test-transport-matrices - matrix_format=github-apk-matrix + # The retrace jobs replay shards: several cases of one (lane, transport, backend) cell + # per emulator boot. The self-test proves every shard matrix replays exactly the rows + # of the per-case matrix it is built from, and trace_cases.py refuses to print one + # that does not. + python3 tools/trace_replay/trace_cases.py --self-test-shard-matrices + matrix_format=github-apk-shard-matrix if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then - matrix_format=github-apk-transport-matrix + matrix_format=github-apk-transport-shard-matrix fi - echo "android=$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}")" >> "$GITHUB_OUTPUT" - echo "names=$(python3 tools/trace_replay/trace_cases.py --ci --format names)" >> "$GITHUB_OUTPUT" + android="$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}" \ + --shard-seconds "${RETRACE_SHARD_SECONDS}")" + # An empty `include` is not an error to GitHub: it skips the retrace job with no red. + shards="$(printf '%s' "${android}" | python3 -c 'import json, sys; print(len(json.load(sys.stdin)["include"]))')" + if [ "${shards}" -lt 1 ]; then + echo "::error::the Android retrace matrix is empty, so the retrace job would be skipped with no red" + exit 1 + fi + # The cases of a shard share one job, so a case that hangs spends time the cases after + # it need. retrace_shard_budget.py gives every shard its own deadline_seconds, the + # time in which each of its cases could run to the end of its time budget, capped so + # that a shard whose replays all hang cannot hold a runner for hours, and the + # timeout_minutes of its job. The retrace job reads both from the matrix. + android="$(printf '%s' "${android}" | python3 android-plugin/retrace_shard_budget.py \ + --cap-seconds "${RETRACE_SHARD_DEADLINE_CAP_SECONDS}")" + echo "android=${android}" >> "$GITHUB_OUTPUT" + echo "Android retrace: ${shards} shard(s) from ${matrix_format} at ${RETRACE_SHARD_SECONDS} s per shard" + # One cache entry holds every Git LFS fixture of the CI cases, and each retrace job restores + # that entry itself. This job only makes sure the entry exists. On a hit it downloads + # nothing. On a miss it starts from the newest older bundle, fetches from the mirror the + # cases whose pointers changed, saves the entry under the new key and confirms it is there, + # so that no retrace job has to download fixtures itself. The key, the path and the steps + # are .github/scripts/trace-fixture-bundle.sh's, and test.yml has the same job for the same + # entry: the two workflows of one push look the key up at the same time, and on a miss both + # fetch and both save. The save that loses is not an error, since the entry it wanted to + # write is the one the confirming step then finds. The timeout bounds a fixture download + # that stalls, which would otherwise hold every retrace job behind it. trace-fixtures: - name: trace fixture (${{ matrix.case }}) + name: trace fixtures runs-on: ubuntu-26.04 - needs: trace-cases - strategy: - fail-fast: false - max-parallel: 4 - matrix: - case: ${{ fromJSON(needs.trace-cases.outputs.names) }} + timeout-minutes: 30 + permissions: + actions: read + contents: read + outputs: + key: ${{ steps.bundle.outputs.key }} + path: ${{ steps.bundle.outputs.path }} steps: - name: Checkout repo uses: actions/checkout@v6 - - name: Derive trace fixture cache key - id: fixture-key - run: bash .github/scripts/trace-fixture-cache.sh key '${{ matrix.case }}' + - name: Derive the fixture bundle key + id: bundle + run: bash .github/scripts/trace-fixture-bundle.sh key - - name: Restore trace fixture cache - id: fixture-cache - if: steps.fixture-key.outputs.cacheable == 'true' + - name: Look up the fixture bundle + id: lookup uses: actions/cache/restore@v5 with: - path: ${{ steps.fixture-key.outputs.paths }} - key: ${{ steps.fixture-key.outputs.key }} + path: ${{ steps.bundle.outputs.path }} + key: ${{ steps.bundle.outputs.key }} + lookup-only: true - - name: Verify restored trace fixture - id: fixture-verify - if: steps.fixture-cache.outputs.cache-hit == 'true' - run: | - if bash .github/scripts/trace-fixture-cache.sh verify '${{ matrix.case }}'; then - echo "ok=true" >> "$GITHUB_OUTPUT" - else - echo "ok=false" >> "$GITHUB_OUTPUT" - echo "::warning::Cached fixture for ${{ matrix.case }} failed verification; falling back to the download path" - bash .github/scripts/trace-fixture-cache.sh reset '${{ matrix.case }}' - fi + - name: Restore the newest older bundle + if: steps.lookup.outputs.cache-hit != 'true' + uses: actions/cache/restore@v5 + with: + path: ${{ steps.bundle.outputs.path }} + key: ${{ steps.bundle.outputs.key }} + restore-keys: ${{ steps.bundle.outputs.restore-prefix }} - - name: Fetch trace fixture - if: steps.fixture-verify.outputs.ok != 'true' - run: bash .github/scripts/fetch-trace-fixture-lfs.sh '${{ matrix.case }}' + - name: Complete the fixture bundle + if: steps.lookup.outputs.cache-hit != 'true' + run: bash .github/scripts/trace-fixture-bundle.sh ensure - - name: Save trace fixture cache - if: steps.fixture-key.outputs.cacheable == 'true' && steps.fixture-cache.outputs.cache-hit != 'true' + - name: Save the fixture bundle + if: steps.lookup.outputs.cache-hit != 'true' + continue-on-error: true uses: actions/cache/save@v5 with: - path: ${{ steps.fixture-key.outputs.paths }} - key: ${{ steps.fixture-key.outputs.key }} + path: ${{ steps.bundle.outputs.path }} + key: ${{ steps.bundle.outputs.key }} - - name: Stage trace fixture - run: | - safe_case="$(printf '%s' '${{ matrix.case }}' | sed 's/[^A-Za-z0-9._-]/_/g')" - stage_dir="trace-fixtures/${safe_case}" - mkdir -p "${stage_dir}" - python3 tools/trace_replay/trace_cases.py --format fixture-files --case '${{ matrix.case }}' | - while IFS= read -r file; do - cp "${file}" "${stage_dir}/" - done + # The Test workflow of the same push may be the one whose save won. Its entry is as good, + # once it is listed. + - name: Wait for the fixture bundle entry + if: steps.lookup.outputs.cache-hit != 'true' + env: + GH_TOKEN: ${{ github.token }} + run: bash .github/scripts/trace-fixture-bundle.sh wait '${{ steps.bundle.outputs.key }}' - - name: Upload trace fixture - uses: actions/upload-artifact@v7 + - name: Confirm the fixture bundle entry + if: steps.lookup.outputs.cache-hit != 'true' + uses: actions/cache/restore@v5 with: - name: MobileGL-trace-fixture-${{ matrix.case }} - path: trace-fixtures/** - if-no-files-found: error + path: ${{ steps.bundle.outputs.path }} + key: ${{ steps.bundle.outputs.key }} + lookup-only: true + fail-on-cache-miss: true android-avd: name: android avd image @@ -347,8 +474,33 @@ jobs: - name: Checkout repo uses: actions/checkout@v6 + # The emulator shard loop and the artifact cleanup, driven by fakes: what each case is + # run with, a verdict for every case, and what stays after a failure. It runs in this + # job because the retrace jobs wait for it and the APK build does not. + - name: Emulator shard runner regression + run: | + python3 android-plugin/test_trace_replay_shard.py + python3 android-plugin/test_retrace_artifact_cleanup.py + + # This job only makes sure the cache entry exists; the retrace jobs restore it. A hit + # downloads nothing and skips every step below; a miss builds the AVD, and the post step + # of this action saves it. + - name: Look up Android AVD cache + id: android-avd-cache + uses: actions/cache@v5 + with: + lookup-only: true + path: | + ${{ env.ANDROID_AVD_HOME }} + ${{ env.ANDROID_SDK_ROOT }}/emulator + ${{ env.ANDROID_SDK_ROOT }}/platform-tools + ${{ env.ANDROID_SDK_ROOT }}/platforms/android-35 + ${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64 + key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v4-${{ hashFiles('android-plugin/run-avd-ci.sh') }} + - name: Setup Android SDK id: netsetup336 + if: steps.android-avd-cache.outputs.cache-hit != 'true' continue-on-error: true timeout-minutes: 15 uses: android-actions/setup-android@v4 @@ -367,20 +519,9 @@ jobs: cmdline-tools-version: 13114758 - name: Accept Android SDK licenses + if: steps.android-avd-cache.outputs.cache-hit != 'true' run: yes | sdkmanager --licenses >/dev/null - - name: Restore Android AVD cache - id: android-avd-cache - uses: actions/cache@v5 - with: - path: | - ${{ env.ANDROID_AVD_HOME }} - ${{ env.ANDROID_SDK_ROOT }}/emulator - ${{ env.ANDROID_SDK_ROOT }}/platform-tools - ${{ env.ANDROID_SDK_ROOT }}/platforms/android-35 - ${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64 - key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v4-${{ hashFiles('android-plugin/run-avd-ci.sh') }} - - name: Create AVD if: steps.android-avd-cache.outputs.cache-hit != 'true' run: | @@ -418,16 +559,27 @@ jobs: fi du -sh "${ANDROID_AVD_HOME}" + # One job replays a shard: some of the cases of one lane, transport and backend, one after + # another on one emulator boot (android-plugin/trace_replay_shard.py). The shard's step + # summary has a row per case, and a case that did not pass has an error annotation of its own. retrace: - name: retrace (${{ matrix.backend.name }} / ${{ matrix.transport || 'monolith' }}, ${{ matrix.case.name }}) + name: retrace (${{ matrix.backend.name }} / ${{ matrix.transport || 'monolith' }}, shard ${{ matrix.shard }}/${{ matrix.shards }}) runs-on: ubuntu-26.04 needs: - build - android-avd - trace-cases - trace-fixtures - if: ${{ !cancelled() && needs.build.result == 'success' && needs.android-avd.result == 'success' && needs.trace-cases.result == 'success' }} - timeout-minutes: 75 + if: ${{ !cancelled() && needs.build.result == 'success' && needs.android-avd.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }} + # Paired with --deadline-seconds below, and far above what a shard takes when its replays + # end by themselves. Both come from the matrix. The deadline is the smaller of the time in + # which every case of the shard could run to the end of its own time budget + # (trace_replay_shard.py, needed_seconds) and RETRACE_SHARD_DEADLINE_CAP_SECONDS. With + # budgets of up to 1800 s a case the cap is what a shard of several cases gets: about + # three replays can hang to the end of their budgets before a later case is recorded as + # not-run. The minutes between the deadline and this limit are the setup before the + # replays and the uploads after them. + timeout-minutes: ${{ matrix.timeout_minutes }} env: MOBILEGL_CI_TRANSPORT: ${{ matrix.transport || 'monolith' }} MOBILEGL_CI_LANE: ${{ matrix.lane || 'monolith-control' }} @@ -435,38 +587,53 @@ jobs: ANDROID_AVD_HOME: ${{ github.workspace }}/.android/avd ANDROID_HOME: ${{ github.workspace }}/.android/sdk ANDROID_SDK_ROOT: ${{ github.workspace }}/.android/sdk + # The shard's cases in run order, as the JSON array of case objects the shard runner + # takes: every field of the case in trace_cases.json, with the fixture paths in place. + MOBILEGL_SHARD_CASES: ${{ toJSON(matrix.cases) }} + # Set the repository variable to 1 to cold boot the emulator before every case of a + # shard, should one boot not survive a whole shard of software-rendered replays. + MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE: ${{ vars.MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE || '0' }} + # The emulator renders on the host with a software rasterizer that can keep compiled + # shaders on disk in the runner's home directory, where neither clearing the app's data + # nor restarting the emulator reaches. Off, so every case of a shard compiles its + # shaders as a job with a fresh machine does. + MESA_SHADER_CACHE_DISABLE: "true" strategy: fail-fast: false - # The account runs at most 20 jobs at once, shared with Test. These ~230 emulator jobs are - # the longer of the two runs, so they may take the whole pool: what is minimized is when the - # later of APK and Test finishes, not Test alone. - max-parallel: 20 + # No max-parallel: the account's limit of 20 concurrent jobs, shared with Test, already + # bounds this matrix, and a cap below the number of shards only makes the round longer. matrix: ${{ fromJSON(needs.trace-cases.outputs.android) }} steps: - name: Checkout repo uses: actions/checkout@v6 + # First, so the steps that run whatever happened (diagnostics, stopping the emulator, the + # result upload) have these even when a step before the emulator launch failed. + - name: Set artifact metadata + run: | + echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV" + echo "EMULATOR_LOG=${RUNNER_TEMP}/mobilegl-emulator.log" >> "$GITHUB_ENV" + echo "EMULATOR_PID_FILE=${RUNNER_TEMP}/mobilegl-emulator.pid" >> "$GITHUB_ENV" + - name: Set swap space uses: ./.github/actions/set-swap-space with: swap-size-gb: 8 - - name: Download trace fixture - uses: actions/download-artifact@v8 + # A miss does not fail the job: the entry may have been evicted since trace-fixtures + # confirmed it (a re-run of failed jobs on an older run does not run that job again). + - name: Restore trace fixtures + uses: actions/cache/restore@v5 with: - name: MobileGL-trace-fixture-${{ matrix.case.name }} - path: trace-fixture-download + path: ${{ needs.trace-fixtures.outputs.path }} + key: ${{ needs.trace-fixtures.outputs.key }} - - name: Install trace fixture - run: | - mkdir -p tools/trace_replay/fixtures - find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \; - - - name: Set artifact metadata - run: | - echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV" - echo "EMULATOR_LOG=${RUNNER_TEMP}/mobilegl-emulator.log" >> "$GITHUB_ENV" - echo "EMULATOR_PID_FILE=${RUNNER_TEMP}/mobilegl-emulator.pid" >> "$GITHUB_ENV" + # Moves this shard's fixtures into place and checks them against their Git LFS pointers. + # A case the restored bundle does not hold is downloaded here, with a warning annotation; + # the timeout bounds a download that stalls. + - name: Install trace fixtures + timeout-minutes: 15 + run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }} - name: Restore Android AVD cache id: android-avd-cache @@ -543,135 +710,44 @@ jobs: --pid-file "${EMULATOR_PID_FILE}" \ --boot-timeout 300 + # One emulator boot replays every case of the shard in order, each in a fresh app process + # with cleared app data and with the settings its case, backend and transport need. The + # runner keeps going after a failing case, on a cold boot so that case cannot touch the + # next one's verdict, retries a case once on a restarted emulator when the emulator + # itself failed, and exits non-zero when a case failed or a verdict is missing. The + # deadline stops it while the steps below can still upload what it has: the job timeout + # less the setup above and the uploads below. A case it stops or never starts is + # recorded as not-run, which is red. - name: Retrace and validate env: - MOBILEGL_ESPRYT_USE_ANGLE: ${{ matrix.backend.name == 'DirectGLES' && '1' || '0' }} - MOBILEGL_TRACE_ANGLE_VARIANT: ${{ matrix.case.name == 'minecraft-1.21.4-fabric-iris-bliss-in-world' && '90a62123d794' || 'ec889e6ea831' }} - MOBILEGL_MAGMA_R11G11B10F_FALLBACK: ${{ matrix.backend.name == 'DirectVulkan' && '1' || '0' }} - MOBILEGL_ITRP_FIX: ${{ matrix.case.name == 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' && 'ForceOn' || 'Auto' }} + # Lets the artifact cleanup tell which job a shard's verdicts came from. + MOBILEGL_SHARD_JOB_ID: ${{ job.check_run_id }} run: | - apk_file="android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk" - test -f "${apk_file}" - extra_retrace_args=() - if [ "${MOBILEGL_CI_TRANSPORT}" = "inproc" ]; then - extra_retrace_args+=( - --env "MOBILEGL_TRANSPORT=inproc;MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1;MOBILEGL_GPU_HANG_BUDGET_MS=${MOBILEGL_GPU_HANG_BUDGET_MS}" - --require-inproc - ) - elif [ "${MOBILEGL_CI_TRANSPORT}" = "spawn" ]; then - # THE SAME CLIENT KNOBS AS inproc, so the two acceptance lanes differ in exactly one - # thing: which PROCESS the server role runs in. They are scrubbed out of the CHILD's - # environment by the launcher and read by the client, which is where they belong. - # - # MOBILEGL_IPC_SERVER_PATH IS DELIBERATELY ABSENT. nativeLibraryDir carries an - # install-time hash, so nothing off the device can spell it; TraceReplayActivity - # resolves it from getApplicationInfo() when the env asks for spawn, and an explicit - # value would override that. --use-pbuffer is NOT optional here: a spawned - # server cannot be handed the client's window, because an ANativeWindow* is a pointer - # into the CLIENT's process and SetWindowHandle is refused on the way across with - # Fatal{UnmigratedSurface, "AndroidNativeWindow:use-MOBILEGL_IPC_SURFACE=server"} (Rule H, permanent; - # a windowed client uses MOBILEGL_IPC_SURFACE=server instead). - extra_retrace_args+=( - --env "MOBILEGL_TRANSPORT=spawn;MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1;MOBILEGL_GPU_HANG_BUDGET_MS=${MOBILEGL_GPU_HANG_BUDGET_MS}" - --use-pbuffer - --require-spawn - ) - else - extra_retrace_args+=(--env "MOBILEGL_TRANSPORT=monolith;MOBILEGL_GPU_HANG_BUDGET_MS=${MOBILEGL_GPU_HANG_BUDGET_MS}") - fi - # Bliss needs the newer signed ANGLE variant plus sampler mipmap - # min-filter downgrading on ANGLE llvmpipe. - if [ "${{ matrix.backend.name }}" = "DirectGLES" ] && [ "${{ matrix.case.name }}" = "minecraft-1.21.4-fabric-iris-bliss-in-world" ]; then - extra_retrace_args+=(--avoid-angle-llvmpipe-sampler-mipmap-min-filter) - fi - if [ "${{ matrix.backend.name }}" = "DirectGLES" ] && [ "${{ matrix.case.avoid_angle_llvmpipe_explicit_lod_bias || false }}" = "true" ]; then - extra_retrace_args+=(--avoid-angle-llvmpipe-explicit-lod-bias) - fi - if [ "${{ matrix.case.coherent_as_flush || false }}" = "true" ]; then - extra_retrace_args+=(--coherent-as-flush) - fi - - run_retrace() { - timeout "$(( ${{ matrix.case.timeout_seconds }} + 300 ))" sh android-plugin/trace-replay-ci.sh \ - --apk-file "${apk_file}" \ - --package top.mobilegl.plugin.trace \ - --backend "${{ matrix.backend.name }}" \ - --result-root android-retrace-result \ - --fixture-root android-retrace-fixture \ - --case "${{ matrix.case.name }}" \ - --trace-archive "${{ matrix.case.trace_archive }}" \ - --trace-file "${{ matrix.case.trace_file }}" \ - --golden "${{ matrix.case.golden }}" \ - --alternate-golden "${{ matrix.case.alternate_golden || '' }}" \ - --target-call "${{ matrix.case.target_call }}" \ - --width "${{ matrix.case.width }}" \ - --height "${{ matrix.case.height }}" \ - --ssim-threshold "${{ matrix.case.ssim_threshold || '0.99' }}" \ - --crop-x "${{ matrix.case.crop_x }}" \ - --crop-y "${{ matrix.case.crop_y }}" \ - --crop-width "${{ matrix.case.crop_width }}" \ - --crop-height "${{ matrix.case.crop_height }}" \ - --timeout-seconds "${{ matrix.case.timeout_seconds }}" \ - "${extra_retrace_args[@]}" - } - - retrace_status=0 - run_retrace || retrace_status=$? - if [ "${retrace_status}" -eq 75 ]; then - echo "::warning::Android emulator infrastructure failed; restarting it and retrying this retrace once." - # Surface-lost is retried rather than failed, so it would otherwise - # be invisible. Report it per job - a healthy run prints nothing and - # a rate spike shows up as a row per affected case. - reason_file="android-retrace-result/infrastructure-failure-reason.txt" - surface_lost_retries=0 - if [ -f "${reason_file}" ]; then - surface_lost_retries="$(grep -c 'angle-surface-lost' "${reason_file}" || true)" - fi - if [ "${surface_lost_retries}" -gt 0 ]; then - echo "surface-lost retries: ${surface_lost_retries} (${{ matrix.backend.name }}, ${{ matrix.case.name }})" \ - >> "${GITHUB_STEP_SUMMARY}" - fi - # The restart truncates EMULATOR_LOG, and the attempt that lost the - # emulator is the one worth reading - the retry usually only shows - # the wreckage. Keep the first attempt's log before it is clobbered. - if [ -f "${EMULATOR_LOG}" ]; then - cp "${EMULATOR_LOG}" "${EMULATOR_LOG}.first-attempt" || true - fi - sh android-plugin/run-avd-ci.sh stop \ - --avd-name "${AVD_NAME}" \ - --emulator-log "${EMULATOR_LOG}" \ - --pid-file "${EMULATOR_PID_FILE}" - adb kill-server || true - sleep 2 - sh android-plugin/run-avd-ci.sh start \ - --avd-name "${AVD_NAME}" \ - --gpu "${{ matrix.backend.gpu }}" \ - --emulator-log "${EMULATOR_LOG}" \ - --pid-file "${EMULATOR_PID_FILE}" \ - --boot-timeout 300 - run_retrace - elif [ "${retrace_status}" -ne 0 ]; then - exit "${retrace_status}" - fi + sh android-plugin/trace-replay-shard.sh \ + --apk-file "android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk" \ + --backend "${{ matrix.backend.name }}" \ + --gpu "${{ matrix.backend.gpu }}" \ + --transport "${MOBILEGL_CI_TRANSPORT}" \ + --lane "${MOBILEGL_CI_LANE}" \ + --cases-env MOBILEGL_SHARD_CASES \ + --deadline-seconds "${{ matrix.deadline_seconds }}" + # Runs whatever happened above, without the device: every case of the shard gets its + # lane.json and golden copies for the summary, and a not-run verdict when the runner was + # killed or never started. - name: Collect retrace summary inputs if: always() + env: + MOBILEGL_SHARD_JOB_ID: ${{ job.check_run_id }} run: | - safe_case="$(printf '%s' '${{ matrix.case.name }}' | sed 's/[^A-Za-z0-9._-]/_/g')" - result_dir="android-retrace-result/${safe_case}-${{ matrix.backend.name }}" - mkdir -p "${result_dir}" - python3 - "${result_dir}/lane.json" <<'PY' - import json, os, sys - from pathlib import Path - Path(sys.argv[1]).write_text(json.dumps({"lane": os.environ["MOBILEGL_CI_LANE"], - "transport": os.environ["MOBILEGL_CI_TRANSPORT"]}, indent=2)) - PY - if [ -s "${{ matrix.case.golden }}" ]; then - cp "${{ matrix.case.golden }}" "${result_dir}/${safe_case}-${{ matrix.backend.name }}-golden.png" - fi - if [ -n "${{ matrix.case.alternate_golden || '' }}" ] && [ -s "${{ matrix.case.alternate_golden || '' }}" ]; then - cp "${{ matrix.case.alternate_golden || '' }}" "${result_dir}/${safe_case}-${{ matrix.backend.name }}-alternate-golden.png" - fi + sh android-plugin/trace-replay-shard.sh \ + --apk-file "android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk" \ + --backend "${{ matrix.backend.name }}" \ + --gpu "${{ matrix.backend.gpu }}" \ + --transport "${MOBILEGL_CI_TRANSPORT}" \ + --lane "${MOBILEGL_CI_LANE}" \ + --cases-env MOBILEGL_SHARD_CASES \ + --finalize - name: Collect emulator diagnostics if: always() @@ -679,12 +755,18 @@ jobs: mkdir -p android-retrace-result/diagnostics adb devices -l > android-retrace-result/diagnostics/adb-devices.txt || true timeout 30 adb logcat -d -t 1000 > android-retrace-result/diagnostics/logcat.txt || true + # What the guest was given; replay time inside the emulator depends on it. + timeout 30 adb shell 'nproc; grep -E "MemTotal|SwapTotal" /proc/meminfo' \ + > android-retrace-result/diagnostics/guest-resources.txt 2>&1 || true + cp "${ANDROID_AVD_HOME}/${AVD_NAME}.avd/config.ini" android-retrace-result/diagnostics/avd-config.ini 2>/dev/null || true + # Expected to list nothing: the host rasterizer's on-disk shader cache is disabled. + du -sh "${XDG_CACHE_HOME:-${HOME}/.cache}"/mesa_shader_cache* \ + > android-retrace-result/diagnostics/host-shader-cache.txt 2>&1 || true + # The log of the last boot. The shard runner keeps the log of every boot it ended + # as diagnostics/emulator-boot--*.log before a restart truncates it. if [ -f "${EMULATOR_LOG}" ]; then cp "${EMULATOR_LOG}" android-retrace-result/diagnostics/emulator.log fi - if [ -f "${EMULATOR_LOG}.first-attempt" ]; then - cp "${EMULATOR_LOG}.first-attempt" android-retrace-result/diagnostics/emulator-first-attempt.log - fi # A vanished emulator looks identical whether the host OOM killer took # qemu or the renderer faulted. These two say which. free -h > android-retrace-result/diagnostics/host-memory.txt 2>&1 || true @@ -704,9 +786,14 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v7 with: - name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-${{ matrix.case.name }} + # The summary job downloads by the lane and transport at the front of this name, and + # its renderer reads the backend and the case from the paths below it, so the name + # ends in the shard number and holds no case name. + name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-shard${{ matrix.shard }} path: android-retrace-result/** if-no-files-found: warn + # A re-run of a failed job replaces the artifact its first attempt kept. + overwrite: true # The artifact service sometimes times out (ETIMEDOUT on CreateArtifact, one job in 231 on # 2026-10-18), which turned a passing case red; one delayed retry absorbs that. @@ -718,7 +805,7 @@ jobs: if: always() && steps.upload-retrace-result.outcome == 'failure' uses: actions/upload-artifact@v7 with: - name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-${{ matrix.case.name }} + name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-shard${{ matrix.shard }} path: android-retrace-result/** if-no-files-found: warn overwrite: true @@ -727,44 +814,51 @@ jobs: name: retrace summary runs-on: ubuntu-26.04 needs: retrace - if: ${{ !cancelled() }} + # Also runs for a canceled run, where it only downloads the results and deletes the + # intermediate artifacts; the rendering steps are skipped then. The checkout and the + # downloads say so themselves, because a step with no condition is skipped once its job is + # canceled. + if: always() + permissions: + actions: write + contents: read env: MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }} steps: - name: Checkout repo + if: ${{ success() || cancelled() }} uses: actions/checkout@v6 - - name: Set artifact metadata - run: | - echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV" - - name: Set up Node.js + if: ${{ !cancelled() }} uses: actions/setup-node@v7 with: node-version: '22' - name: Download Android monolith control results + if: ${{ success() || cancelled() }} uses: actions/download-artifact@v8 with: pattern: MobileGL-android-retrace-result-monolith-control-monolith-* path: retrace-artifacts/monolith-control - name: Download Android inproc acceptance results - if: env.MOBILEGL_CI_DISAGGREGATED == 'ON' + if: ${{ (success() || cancelled()) && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }} uses: actions/download-artifact@v8 with: pattern: MobileGL-android-retrace-result-inproc-acceptance-inproc-* path: retrace-artifacts/inproc-acceptance - name: Download Android spawn acceptance results - if: env.MOBILEGL_CI_DISAGGREGATED == 'ON' + if: ${{ (success() || cancelled()) && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }} uses: actions/download-artifact@v8 with: pattern: MobileGL-android-retrace-result-spawn-acceptance-spawn-* path: retrace-artifacts/spawn-acceptance - name: Render monolith control summary + if: ${{ !cancelled() }} run: | node tools/trace_replay/render_retrace_summary.mjs \ --input retrace-artifacts/monolith-control \ @@ -774,7 +868,7 @@ jobs: --html mobilegl-android-monolith-control.html - name: Render inproc acceptance summary - if: env.MOBILEGL_CI_DISAGGREGATED == 'ON' + if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }} run: | node tools/trace_replay/render_retrace_summary.mjs \ --input retrace-artifacts/inproc-acceptance \ @@ -784,11 +878,17 @@ jobs: --html mobilegl-android-inproc-acceptance.html - name: Render spawn acceptance summary - if: env.MOBILEGL_CI_DISAGGREGATED == 'ON' + if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }} run: | - node tools/trace_replay/render_retrace_summary.mjs --input retrace-artifacts/spawn-acceptance --output-dir android-retrace-summary/spawn-acceptance --title "MobileGL Android spawn acceptance" --group-label "Android Emulator / spawn acceptance" --html mobilegl-android-spawn-acceptance.html + node tools/trace_replay/render_retrace_summary.mjs \ + --input retrace-artifacts/spawn-acceptance \ + --output-dir android-retrace-summary/spawn-acceptance \ + --title "MobileGL Android spawn acceptance" \ + --group-label "Android Emulator / spawn acceptance" \ + --html mobilegl-android-spawn-acceptance.html - name: Upload Android monolith control summary + if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 with: name: MobileGL-android-monolith-control-summary-${{ github.sha }} @@ -797,7 +897,7 @@ jobs: if-no-files-found: error - name: Upload Android inproc acceptance summary - if: env.MOBILEGL_CI_DISAGGREGATED == 'ON' + if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }} uses: actions/upload-artifact@v7 with: name: MobileGL-android-inproc-acceptance-summary-${{ github.sha }} @@ -806,7 +906,7 @@ jobs: if-no-files-found: error - name: Upload Android spawn acceptance summary - if: env.MOBILEGL_CI_DISAGGREGATED == 'ON' + if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }} uses: actions/upload-artifact@v7 with: name: MobileGL-android-spawn-acceptance-summary-${{ github.sha }} @@ -814,74 +914,23 @@ jobs: archive: false if-no-files-found: error - remove-artifact-clutter: - name: remove artifact clutter - runs-on: ubuntu-26.04 - needs: retrace-summary - if: always() - permissions: - actions: write - steps: + # The summaries embed every image they show, so the per-shard results are intermediates + # from here on. This is the last step of the workflow rather than a job of its own, which + # would wait for a runner and start from an empty machine. It keeps what a failure needs, + # read from the shard-status.json of the results downloaded above: the results of every + # shard that ran a case which did not pass somewhere, on all backends and transports, and + # every result when a job failed without saying which cases it ran. Deletes are paced + # below GitHub's limit for mutating requests, and a refused delete is a warning. Skipped + # when a summary step failed, so that re-running this job still finds the results, and + # when a retrace job did not succeed in a run that was not canceled: every result stays + # then, so a shard stopped by its job limit keeps its diagnostics and a re-run of the + # failed jobs renders complete summaries. - name: Delete intermediate Android retrace artifacts + if: ${{ (success() && needs.retrace.result == 'success') || cancelled() }} env: GH_TOKEN: ${{ github.token }} run: | - # Fail before deletion if either paginated inventory cannot be read. - # 50 rows per page: the jobs endpoint answers HTTP 502 for 90+ rows on a re-run attempt of a run this size. - gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=50" \ - --jq '.jobs[] | select(.name | startswith("retrace (")) | select(.conclusion == "failure" or .conclusion == "canceled" or .conclusion == "timed_out" or .conclusion == "action_required") | .name' \ - > "${RUNNER_TEMP}/cleanup-jobs.txt" - gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \ - --jq '.artifacts[] | select(.name | startswith("MobileGL-trace-fixture-") or startswith("MobileGL-android-retrace-result-") or startswith("trace-fixture-") or startswith("retrace-result-")) | [.id, .name] | @tsv' \ - > "${RUNNER_TEMP}/cleanup-artifacts.txt" - declare -A failed_cases=() - while IFS= read -r job_name; do - case_name="${job_name#retrace (*, }" - case_name="${case_name%)}" - failed_cases["${case_name}"]=1 - done < "${RUNNER_TEMP}/cleanup-jobs.txt" - - if ((${#failed_cases[@]})); then - echo "Retaining fixtures and results for failed retrace case(s):" - printf ' %s\n' "${!failed_cases[@]}" - else - echo "All retrace jobs succeeded; nothing needs to be retained." - fi - - deleted=0 - retained=0 - while IFS=$'\t' read -r artifact_id artifact_name; do - keep=0 - if [[ "${artifact_name}" == MobileGL-trace-fixture-* ]]; then - case_name="${artifact_name#MobileGL-trace-fixture-}" - if [[ -v "failed_cases[${case_name}]" ]]; then - keep=1 - fi - elif [[ "${artifact_name}" == MobileGL-android-retrace-result-* ]]; then - # The result artifact carries mobilegl.log, retrace.log, logcat, - # the emulator log and the actual/diff images - the only record of - # why a retrace failed. Its name ends in --, so a - # suffix match on the case name keeps both backends' results for a - # case that failed on either of them, which is what a comparison - # needs. The match is anchored at the end, so a case name that is a - # prefix of a longer one does not retain the longer one's results. - for case_name in "${!failed_cases[@]}"; do - if [[ "${artifact_name}" == *-"${case_name}" ]]; then - keep=1 - break - fi - done - fi - - if ((keep)); then - echo "Retaining ${artifact_name} (${artifact_id}) for failed retrace." - ((retained += 1)) - continue - fi - - echo "Deleting ${artifact_name} (${artifact_id})" - gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}" - ((deleted += 1)) - done < "${RUNNER_TEMP}/cleanup-artifacts.txt" - - echo "Deleted ${deleted} intermediate Android artifact(s); retained ${retained} failed-retrace fixture(s)." + python3 android-plugin/retrace_artifact_cleanup.py \ + --downloads retrace-artifacts/monolith-control \ + --downloads retrace-artifacts/inproc-acceptance \ + --downloads retrace-artifacts/spawn-acceptance diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 79451080..87733d5d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ env: MOBILEGL_GPU_HANG_BUDGET_MS: "0" # ONE LIVE RUN PER BRANCH. The disaggregation branch pushes (often docs-only) commits minutes apart -# and each push fans out to ~330 retrace legs here and ~275 jobs in apk.yml; with no group, runs +# and each push fans out to about 30 replay jobs here and the emulator shards in apk.yml; with no group, runs # sat queued for 10+ hours behind their own successors. A manual dispatch gets a group of its own, # so a push never cancels a manually started run; dev queues rather # than cancels. No paths-ignore here, unlike apk.yml: this workflow hosts the doc citation lint. @@ -2668,12 +2668,15 @@ jobs: runs-on: ubuntu-26.04 env: MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }} + # How many jobs replay one (lane, transport, backend) cell of the Linux retrace lanes. Each + # job replays its share of the cell's cases one after another; a cell with little to + # replay gets fewer jobs (tools/trace_replay/trace_cases.py, shard_matrix). + RETRACE_SHARDS_PER_CELL: "4" outputs: - matrix: ${{ steps.trace-cases.outputs.matrix }} - names: ${{ steps.trace-cases.outputs.names }} - verify-matrix: ${{ steps.trace-cases.outputs.verify-matrix }} - split-matrix: ${{ steps.trace-cases.outputs.split-matrix }} - split-spawn-matrix: ${{ steps.trace-cases.outputs.split-spawn-matrix }} + shards: ${{ steps.trace-cases.outputs.shards }} + verify-shards: ${{ steps.trace-cases.outputs.verify-shards }} + split-shards: ${{ steps.trace-cases.outputs.split-shards }} + split-spawn-shards: ${{ steps.trace-cases.outputs.split-spawn-shards }} steps: - name: Checkout repo uses: actions/checkout@v6 @@ -2702,29 +2705,49 @@ jobs: id: trace-cases run: | python3 tools/trace_replay/trace_cases.py --self-test-transport-matrices - matrix_format=github-test-matrix - if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then matrix_format=github-test-transport-matrix; fi - echo "matrix=$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}")" >> "$GITHUB_OUTPUT" - echo "names=$(python3 tools/trace_replay/trace_cases.py --ci --format names)" >> "$GITHUB_OUTPUT" - # The subset the verify build retraces ("verify": true in trace_cases.json). It is a - # SUBSET of the matrix above, so retrace-verify needs no fixtures of its own. - echo "verify-matrix=$(python3 tools/trace_replay/trace_cases.py --ci --format github-verify-matrix)" >> "$GITHUB_OUTPUT" - # The split subset: {backend, case, transport} over inproc AND spawn, both backends, for - # every CI case that has not said `"split": false`. Still a SUBSET of the matrix above, - # so retrace-split needs no fixtures of its own. P5 shipped it as one case (OpenRA, which - # the phase gate names); P6 widened it to the whole CI matrix, because every bug the - # spawn work found was a question about the SERVER's process rather than about one trace. - # trace_cases.py refuses a `split` case that is not in CI, so the subset stays a subset. - SPLIT_MATRIX=$(python3 tools/trace_replay/trace_cases.py --ci --format github-split-matrix) - # AND IT MUST NOT BE EMPTY. An empty `include` is not an error to GitHub - it skips the - # whole retrace-split job with no red anywhere - so the one way this subset can vanish - # silently is guarded here. trace_cases.py now also rejects an unknown manifest key, which - # was the hole: `"splitt": true` loaded clean and emptied the subset (review N-4). - if [ "$(printf '%s' "${SPLIT_MATRIX}" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["include"]))')" -lt 1 ]; then - echo "::error::the split retrace subset is EMPTY. 'split' is an OPT-OUT as of P6 - a CI case is in this subset unless it carries \"split\": false - so an empty subset means either every case opted out or the CI matrix itself is empty. Either way retrace-split would be skipped with no red, and exit gate E2 names OpenRA." + # The retrace jobs replay shards: several cases of one (lane, transport, backend) cell + # per job. The first self-test proves every shard matrix replays exactly the rows of + # the per-case matrix it is built from; the second proves the shard runner gives each + # case the settings of its lane. + python3 tools/trace_replay/trace_cases.py --self-test-shard-matrices + python3 scripts/ci/retrace_shard.py --self-test + shards() { + python3 tools/trace_replay/trace_cases.py --ci --shards "${RETRACE_SHARDS_PER_CELL}" "$@" + } + shard_count() { + python3 -c 'import json,sys; print(len(json.load(sys.stdin)["include"]))' + } + shard_format=github-test-shard-matrix + if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then shard_format=github-test-transport-shard-matrix; fi + # An empty `include` is not an error to GitHub: it skips the job with no red anywhere, + # so each matrix is checked before it is handed over. + require_shards() { + if [ "$(printf '%s' "$2" | shard_count)" -lt 1 ]; then + echo "::error::the $1 shard matrix is empty, so its job would be skipped with no red" + exit 1 + fi + } + SHARDS=$(shards --format "${shard_format}") + require_shards retrace "${SHARDS}" + echo "shards=${SHARDS}" >> "$GITHUB_OUTPUT" + # The subset the verify build retraces ("verify": true in trace_cases.json). + VERIFY_SHARDS=$(shards --format github-verify-shard-matrix) + require_shards retrace-verify "${VERIFY_SHARDS}" + echo "verify-shards=${VERIFY_SHARDS}" >> "$GITHUB_OUTPUT" + # The split subset: {backend, case, transport} over inproc and spawn, both backends, for + # every CI case that has not said `"split": false`. It is the whole CI matrix because + # the bugs the spawn arm finds are about the server's process rather than about one + # trace. trace_cases.py refuses a `split` case that is not in CI. + SPLIT_SHARDS=$(shards --format github-split-shard-matrix) + # It must not be empty. An empty `include` is not an error to GitHub: it skips the + # whole retrace-split job with no red anywhere, so that is guarded here. + # trace_cases.py also rejects an unknown manifest key, since a mistyped `split` key + # would otherwise load clean and change the subset. + if [ "$(printf '%s' "${SPLIT_SHARDS}" | shard_count)" -lt 1 ]; then + echo "::error::the split retrace subset is empty. 'split' is an opt-out: a CI case is in this subset unless it carries \"split\": false, so an empty subset means either every case opted out or the CI matrix itself is empty. Either way retrace-split would be skipped with no red." exit 1 fi - echo "split-matrix=${SPLIT_MATRIX}" >> "$GITHUB_OUTPUT" + echo "split-shards=${SPLIT_SHARDS}" >> "$GITHUB_OUTPUT" # On the disaggregation branches, the same subset without the inproc rows the # inproc-acceptance retrace rows replay on the same library, which retrace-split uses when # it reuses build-linux's library. OpenRA keeps its inproc rows: they carry the @@ -2733,101 +2756,110 @@ jobs: # is not written, and retrace-split takes the full subset, as it does elsewhere. if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then if python3 tools/trace_replay/trace_cases.py --self-test-split-matrix \ - && SPLIT_SPAWN_MATRIX=$(python3 tools/trace_replay/trace_cases.py --ci --format github-split-matrix \ + && SPLIT_SPAWN_SHARDS=$(shards --format github-split-shard-matrix \ --transports spawn --keep-inproc OpenRA) \ - && [ "$(printf '%s' "${SPLIT_SPAWN_MATRIX}" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["include"]))' || echo 0)" -ge 1 ]; then - echo "split-spawn-matrix=${SPLIT_SPAWN_MATRIX}" >> "$GITHUB_OUTPUT" + && [ "$(printf '%s' "${SPLIT_SPAWN_SHARDS}" | shard_count || echo 0)" -ge 1 ]; then + echo "split-spawn-shards=${SPLIT_SPAWN_SHARDS}" >> "$GITHUB_OUTPUT" else echo "::warning::the spawn-only split retrace subset failed its self-test or is empty, so retrace-split replays the full split subset" fi fi + echo "retrace shards: $(printf '%s' "${SHARDS}" | shard_count) retrace, $(printf '%s' "${VERIFY_SHARDS}" | shard_count) verify, $(printf '%s' "${SPLIT_SHARDS}" | shard_count) split (full subset)" + # One cache entry holds every Git LFS fixture of the CI cases, and each replay job restores + # that entry itself. This job only makes sure the entry exists. On a hit it downloads + # nothing. On a miss it starts from the newest older bundle, fetches from the mirror the + # cases whose pointers changed, saves the entry under the new key and confirms it is there, + # so that no replay job has to download fixtures itself. The key, the path and the steps are + # .github/scripts/trace-fixture-bundle.sh's; apk.yml uses the same entry. The timeout bounds + # a fixture download that stalls, which would otherwise hold every replay job behind it. trace-fixtures: - name: trace fixture (${{ matrix.case }}) + name: trace fixtures runs-on: ubuntu-26.04 - needs: [trace-cases] - strategy: - fail-fast: false - max-parallel: 4 - matrix: - case: ${{ fromJSON(needs.trace-cases.outputs.names) }} + timeout-minutes: 30 + permissions: + actions: read + contents: read + outputs: + key: ${{ steps.bundle.outputs.key }} + path: ${{ steps.bundle.outputs.path }} steps: - name: Checkout repo uses: actions/checkout@v6 - - name: Configure runner environment - uses: ./.github/actions/setup-test-runner + - name: Derive the fixture bundle key + id: bundle + run: bash .github/scripts/trace-fixture-bundle.sh key - - name: Derive trace fixture cache key - id: fixture-key - run: bash .github/scripts/trace-fixture-cache.sh key '${{ matrix.case }}' - - - name: Restore trace fixture cache - id: fixture-cache - if: steps.fixture-key.outputs.cacheable == 'true' + - name: Look up the fixture bundle + id: lookup uses: actions/cache/restore@v5 with: - path: ${{ steps.fixture-key.outputs.paths }} - key: ${{ steps.fixture-key.outputs.key }} + path: ${{ steps.bundle.outputs.path }} + key: ${{ steps.bundle.outputs.key }} + lookup-only: true - - name: Verify restored trace fixture - id: fixture-verify - if: steps.fixture-cache.outputs.cache-hit == 'true' - run: | - if bash .github/scripts/trace-fixture-cache.sh verify '${{ matrix.case }}'; then - echo "ok=true" >> "$GITHUB_OUTPUT" - else - echo "ok=false" >> "$GITHUB_OUTPUT" - echo "::warning::Cached fixture for ${{ matrix.case }} failed verification; falling back to the download path" - bash .github/scripts/trace-fixture-cache.sh reset '${{ matrix.case }}' - fi + - name: Restore the newest older bundle + if: steps.lookup.outputs.cache-hit != 'true' + uses: actions/cache/restore@v5 + with: + path: ${{ steps.bundle.outputs.path }} + key: ${{ steps.bundle.outputs.key }} + restore-keys: ${{ steps.bundle.outputs.restore-prefix }} - - name: Fetch trace fixture - if: steps.fixture-verify.outputs.ok != 'true' - run: bash .github/scripts/fetch-trace-fixture-lfs.sh '${{ matrix.case }}' + - name: Complete the fixture bundle + if: steps.lookup.outputs.cache-hit != 'true' + run: bash .github/scripts/trace-fixture-bundle.sh ensure - - name: Save trace fixture cache - if: steps.fixture-key.outputs.cacheable == 'true' && steps.fixture-cache.outputs.cache-hit != 'true' + # The APK workflow of the same push saves the same entry, and only one save can win. + # The steps below find the entry whichever workflow wrote it. + - name: Save the fixture bundle + if: steps.lookup.outputs.cache-hit != 'true' + continue-on-error: true uses: actions/cache/save@v5 with: - path: ${{ steps.fixture-key.outputs.paths }} - key: ${{ steps.fixture-key.outputs.key }} + path: ${{ steps.bundle.outputs.path }} + key: ${{ steps.bundle.outputs.key }} - - name: Stage trace fixture - run: | - safe_case="$(printf '%s' '${{ matrix.case }}' | sed 's/[^A-Za-z0-9._-]/_/g')" - stage_dir="trace-fixtures/${safe_case}" - mkdir -p "${stage_dir}" - python3 tools/trace_replay/trace_cases.py --format fixture-files --case '${{ matrix.case }}' | - while IFS= read -r file; do - cp "${file}" "${stage_dir}/" - done + # The APK workflow of the same push may be the one whose save won. Its entry is as good, + # once it is listed. + - name: Wait for the fixture bundle entry + if: steps.lookup.outputs.cache-hit != 'true' + env: + GH_TOKEN: ${{ github.token }} + run: bash .github/scripts/trace-fixture-bundle.sh wait '${{ steps.bundle.outputs.key }}' - - name: Upload trace fixture - uses: actions/upload-artifact@v7 + - name: Confirm the fixture bundle entry + if: steps.lookup.outputs.cache-hit != 'true' + uses: actions/cache/restore@v5 with: - name: trace-fixture-${{ matrix.case }} - path: trace-fixtures/** - if-no-files-found: error + path: ${{ steps.bundle.outputs.path }} + key: ${{ steps.bundle.outputs.key }} + lookup-only: true + fail-on-cache-miss: true + # One job replays a shard: some of the cases of one lane, transport and backend, one after + # another, each through its own ctest entry (scripts/ci/retrace_shard.py). The shard's step + # summary has a row per case, and a failed case has an error annotation of its own. retrace: - name: retrace (${{ matrix.backend }} / ${{ matrix.transport || 'monolith' }}, ${{ matrix.case }}) + name: retrace (${{ matrix.backend }} / ${{ matrix.transport || 'monolith' }}, shard ${{ matrix.shard }}/${{ matrix.shards }}) runs-on: ubuntu-26.04 + timeout-minutes: 240 env: MOBILEGL_TRANSPORT: ${{ matrix.transport || 'monolith' }} MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" MOBILEGL_IPC_RUN_AHEAD: "1" + SHARD_NAMES: ${{ toJSON(matrix.names) }} needs: - build-linux - build-retrace - trace-cases - trace-fixtures - if: ${{ !cancelled() && needs.build-linux.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }} + if: ${{ !cancelled() && needs.build-linux.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }} strategy: fail-fast: false - max-parallel: 8 - matrix: ${{ fromJSON(needs.trace-cases.outputs.matrix) }} + matrix: ${{ fromJSON(needs.trace-cases.outputs.shards) }} steps: - name: Checkout repo @@ -2844,16 +2876,20 @@ jobs: with: gpu: "true" - - name: Download trace fixture - uses: actions/download-artifact@v8 + # A miss does not fail the job: the entry may have been evicted since trace-fixtures + # confirmed it (a re-run of failed jobs on an older run does not run that job again). + - name: Restore trace fixtures + uses: actions/cache/restore@v5 with: - name: trace-fixture-${{ matrix.case }} - path: trace-fixture-download + path: ${{ needs.trace-fixtures.outputs.path }} + key: ${{ needs.trace-fixtures.outputs.key }} - - name: Install trace fixture - run: | - mkdir -p tools/trace_replay/fixtures - find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \; + # Moves this shard's fixtures into place and checks them against their Git LFS pointers. + # A case the restored bundle does not hold is downloaded here, with a warning annotation; + # the timeout bounds a download that stalls. + - name: Install trace fixtures + timeout-minutes: 15 + run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }} - name: Get CMake timeout-minutes: 15 @@ -2895,93 +2931,66 @@ jobs: if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi done + # The per-case settings are retrace_shard.py's "retrace" profile: the Magma fallback and + # the two per-case repair knobs on DirectVulkan, and on the inproc lane the IPC + # configuration the client must log, the split evidence check and the refusal census from + # the library's own logs. A failing case does not stop the shard; the step fails at the + # end if a case failed or a case has no verdict. - name: Retrace and validate - working-directory: build-retrace/tools/trace_replay run: | - ulimit -c unlimited - if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then - export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1 - fi - if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \ - && [ '${{ matrix.case }}' = 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' ]; then - export MOBILEGL_ITRP_FIX=ForceOn - fi - # The blended depth-write quirk auto-enables only on Qualcomm, which no CI - # runner has, so force it on for the OIT case it exists to fix. ForceOn - # bypasses only the vendor gate, so this exercises the real strip on - # lavapipe. The Android AVD lane deliberately leaves it off, keeping the - # unstripped path covered for the same trace. - if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \ - && [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then - export MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1 - fi - # The inproc rows stand in for retrace-split's dropped inproc rows, so they prove the - # configuration they run with from the client's own log. - if [ "${MOBILEGL_TRANSPORT}" = inproc ]; then - export MGTRACE_EXPECT_IPC='strict=1 role-split-state=1 run-ahead=1' - fi - ctest -V --no-tests=error -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$' --output-junit "${RUNNER_TEMP}/retrace.xml" - # A split row's transport assertions ran on this leg and found their evidence. - if [ "${MOBILEGL_TRANSPORT}" != monolith ]; then - python3 "${GITHUB_WORKSPACE}/scripts/ci/retrace_split_evidence.py" \ - '${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt' \ - --transport "${MOBILEGL_TRANSPORT}" --case '${{ matrix.case }}' --backend '${{ matrix.backend }}' - fi - - # The refusal census, recorded rather than gated. run_trace_case.cmake already REDS the case - # on any Fatal{, so reaching here means the count is zero - but the number and the distinct - # slot names are what MEASUREMENTS wants from every split run, and reading them out of the - # library's own log is the only way to get them (ctest -V's transcript is a false zero). - - name: Refusal census from the library log - if: ${{ always() && matrix.transport == 'inproc' }} - working-directory: build-retrace/tools/trace_replay - run: | - # BOTH ROLES, because under spawn every applier refusal is raised in the SERVER's - # process and a census reading only the client's file reports a confident zero for the - # half it cannot see. - log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.client.log" - server_log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.server.log" - cat "${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt" 2>/dev/null || echo "no split-evidence.txt" - if [ ! -f "${log}" ]; then - echo "no ${log} - the replay wrote no library log" - exit 0 - fi - census_files="${log}" - if [ -f "${server_log}" ]; then census_files="${census_files} ${server_log}"; fi - echo "census over: ${census_files}" - echo "Fatal{ lines: $(cat ${census_files} | grep -c 'Fatal{' || true)" - cat ${census_files} | grep -o 'Fatal{[A-Za-z]*, "[^"]*"' | sort | uniq -c | sort -rn | head -20 || true - # The transport line is the CLIENT's: ConfigLoader resolves it there, and under spawn - # the server states its own separately. - grep -m1 "MOBILEGL_TRANSPORT=" "${log}" || echo "no transport line in ${log}" + python3 scripts/ci/retrace_shard.py run --profile retrace \ + --backend '${{ matrix.backend }}' --transport "${MOBILEGL_TRANSPORT}" \ + --lane '${{ matrix.lane || 'monolith-control' }}' \ + --shard '${{ matrix.shard }}/${{ matrix.shards }}' \ + --build-dir build-retrace/tools/trace_replay \ + --result-dir "${RUNNER_TEMP}/retrace-shard" - name: Annotate failed entries if: failure() - run: python3 scripts/ci/junit_annotate.py --excerpts 1 "${RUNNER_TEMP}" + run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit" - name: Upload core dumps - if: failure() + if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v7 with: - name: retrace-core-dumps-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-${{ matrix.case }} + name: retrace-core-dumps-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }} path: /tmp/core.* if-no-files-found: ignore - - name: Upload actual image + # What retrace-summary renders: per case, the images and the replay's output directory. + # The summary embeds the images, so these are short-lived; a failed shard keeps its own + # copy below. + - name: Upload shard results if: ${{ always() }} uses: actions/upload-artifact@v7 with: - name: retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-${{ matrix.case }} - path: | - build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/** - build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/** + name: retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }} + path: ${{ runner.temp }}/retrace-shard/results if-no-files-found: warn + retention-days: 1 + overwrite: true + + - name: Upload failed shard results + if: ${{ failure() || cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: retrace-failed-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }} + path: | + ${{ runner.temp }}/retrace-shard/results + ${{ runner.temp }}/retrace-shard/junit + ${{ runner.temp }}/retrace-shard/logs + if-no-files-found: warn + overwrite: true retrace-summary: name: retrace summary runs-on: ubuntu-26.04 needs: [retrace] if: ${{ !cancelled() && needs.retrace.result != 'skipped' }} + # actions: write is for the last step, which deletes the artifacts this job rendered. + permissions: + actions: write + contents: read env: MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }} @@ -3050,25 +3059,48 @@ jobs: archive: false if-no-files-found: error + # The summaries embed every image, so the shard result artifacts they were rendered from + # have no reader left and are deleted here. That happens only when every shard passed: a + # failed shard has its own retrace-failed-* artifact, and after a failure the results + # stay for their one day of retention, so that re-running the failed shards renders every + # case again. A failed delete only leaves them to that retention. + - name: Delete the rendered shard results + if: ${{ needs.retrace.result == 'success' }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \ + --jq '.artifacts[] | select(.name | startswith("retrace-result-")) | [.id, .name] | @tsv' \ + > "${RUNNER_TEMP}/rendered-artifacts.txt" + deleted=0 + while IFS=$'\t' read -r artifact_id artifact_name; do + echo "Deleting ${artifact_name} (${artifact_id})" + gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}" + deleted=$((deleted + 1)) + done < "${RUNNER_TEMP}/rendered-artifacts.txt" + echo "Deleted ${deleted} rendered shard result artifact(s)." + # The trace half of the third CI mode. Same replay, same goldens, but the library underneath is # the verify build and MOBILEGL_PIPE_VERIFY=1 is in the environment, so every backend read of # frontend state is checked against a snapshot taken at the verb boundary. Eight cases rather # than the full lane's 40 (tools/trace_replay/trace_cases.json, "verify": true): the comparator # is budgeted at 5-10x, and the full sweep is a phase-exit / workflow_dispatch run. retrace-verify: - name: retrace verify (${{ matrix.backend }}, ${{ matrix.case }}) + name: retrace verify (${{ matrix.backend }}, shard ${{ matrix.shard }}/${{ matrix.shards }}) runs-on: ubuntu-26.04 timeout-minutes: 240 + env: + SHARD_NAMES: ${{ toJSON(matrix.names) }} needs: - build-linux-verify - build-retrace - trace-cases - trace-fixtures - if: ${{ !cancelled() && needs.build-linux-verify.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }} + if: ${{ !cancelled() && needs.build-linux-verify.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }} strategy: fail-fast: false - max-parallel: 4 - matrix: ${{ fromJSON(needs.trace-cases.outputs.verify-matrix) }} + matrix: ${{ fromJSON(needs.trace-cases.outputs.verify-shards) }} steps: - name: Checkout repo @@ -3085,16 +3117,16 @@ jobs: with: gpu: "true" - - name: Download trace fixture - uses: actions/download-artifact@v8 + # A miss does not fail the job; the install step downloads what the bundle does not hold. + - name: Restore trace fixtures + uses: actions/cache/restore@v5 with: - name: trace-fixture-${{ matrix.case }} - path: trace-fixture-download + path: ${{ needs.trace-fixtures.outputs.path }} + key: ${{ needs.trace-fixtures.outputs.key }} - - name: Install trace fixture - run: | - mkdir -p tools/trace_replay/fixtures - find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \; + - name: Install trace fixtures + timeout-minutes: 15 + run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }} - name: Get CMake timeout-minutes: 15 @@ -3151,82 +3183,60 @@ jobs: if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi done - - name: Retrace and validate (verify) - working-directory: build-retrace/tools/trace_replay - # run_trace_case.cmake turns MOBILEGL_PIPE_VERIFY into three assertions of its own (the - # arming line, no Fatal{PipeVerifyDiffer, no Fatal{UnmigratedPipeInput), so a case that - # somehow ran the wrong library reds here instead of passing on its golden. - # --timeout 10800: the 1800s cases run 5-10x slower with both comparator arms live, which - # is well past ctest's 1500s default. - run: | - ulimit -c unlimited - export MOBILEGL_PIPE_VERIFY=1 - if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then - export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1 - fi - if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \ - && [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then - export MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1 - fi - ctest -V --no-tests=error --timeout 10800 \ - -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$' - - # The retrace lane's own always-on negative control, on one case so it costs one short trace: - # with a snapshot field corrupted, the SAME replay must fail. Without it, "40 traces, zero - # divergences" would be a statement about a comparator nobody watched. + # Every case replays with MOBILEGL_PIPE_VERIFY=1 (retrace_shard.py's "verify" profile). + # run_trace_case.cmake turns that into three assertions of its own (the arming line, no + # Fatal{PipeVerifyDiffer, no Fatal{UnmigratedPipeInput), so a case that somehow ran the + # wrong library reds here instead of passing on its golden. ctest's timeout is raised to + # 10800 s: the long cases run 5-10x slower with both comparator arms live, well past the + # 1500 s default. # - # The rerun replays into the SAME case directory, so the verified run's images are put aside - # first and restored before the verdict: "Upload actual image" below runs `if: always()` and - # would otherwise ship the deliberately corrupted run's output under the name of the good one. - # The restore happens whichever way the control goes, which is why the ctest exit status is - # captured rather than tested inline. - - name: Negative control (corrupted snapshot) - if: ${{ matrix.case == 'OpenRA' && matrix.backend == 'DirectGLES' }} - working-directory: build-retrace/tools/trace_replay + # The shard that holds OpenRA on DirectGLES also runs the lane's negative control right + # after that case: with a snapshot field corrupted, the same replay must fail, and for the + # comparator's reason (scripts/ci/retrace_verify_reason.py). Without it, zero divergences + # would be a statement about a comparator nobody watched. The control has a verdict row of + # its own, and its output is discarded so that only the verified run is uploaded. + - name: Retrace and validate (verify) run: | - export MOBILEGL_PIPE_VERIFY=1 - export MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters - GOOD_OUTPUT="${RUNNER_TEMP}/openra-verified-output" - rm -rf "${GOOD_OUTPUT}" - if [ -d OpenRA ]; then - cp -a OpenRA "${GOOD_OUTPUT}" - fi - set +e - ctest -V --no-tests=error --timeout 10800 \ - -R '^MobileGLTraceReplay\.OpenRA\.DirectGLES$' 2>&1 | tee "${RUNNER_TEMP}/verify-control.txt" - control_rc=${PIPESTATUS[0]} - set -e - if [ -d "${GOOD_OUTPUT}" ]; then - rm -rf OpenRA - mv "${GOOD_OUTPUT}" OpenRA - echo "restored the verified run's OpenRA output over the corrupted rerun's" - fi - if [ "${control_rc}" -eq 0 ]; then - echo "::error::MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left the OpenRA retrace GREEN, so the comparator is not comparing and the whole verify retrace lane proves nothing." - exit 1 - fi - # Its red must carry the comparator's divergence on the corrupted field - # (scripts/ci/retrace_verify_reason.py, which test_run_trace_case.py runs on canned output). - python3 "${GITHUB_WORKSPACE}/scripts/ci/retrace_verify_reason.py" "${RUNNER_TEMP}/verify-control.txt" - echo "the corrupted field turned the retrace red, as it must (ctest exit ${control_rc})" + python3 scripts/ci/retrace_shard.py run --profile verify \ + --backend '${{ matrix.backend }}' --lane verify \ + --shard '${{ matrix.shard }}/${{ matrix.shards }}' \ + --build-dir build-retrace/tools/trace_replay \ + --result-dir "${RUNNER_TEMP}/retrace-shard" + + - name: Annotate failed entries + if: failure() + run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit" - name: Upload core dumps - if: failure() + if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v7 with: - name: retrace-verify-core-dumps-${{ matrix.backend }}-${{ matrix.case }} + name: retrace-verify-core-dumps-${{ matrix.backend }}-shard${{ matrix.shard }} path: /tmp/core.* if-no-files-found: ignore - - name: Upload actual image + # Kept for every shard, green or not: per case, the verified images and the replay's + # output directory, whose library log has the comparator's arming line. + - name: Upload shard results if: ${{ always() }} uses: actions/upload-artifact@v7 with: - name: retrace-verify-result-${{ matrix.backend }}-${{ matrix.case }} - path: | - build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/** - build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/** + name: retrace-verify-result-${{ matrix.backend }}-shard${{ matrix.shard }} + path: ${{ runner.temp }}/retrace-shard/results if-no-files-found: warn + overwrite: true + + - name: Upload failed shard results + if: ${{ failure() || cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: retrace-verify-failed-${{ matrix.backend }}-shard${{ matrix.shard }} + path: | + ${{ runner.temp }}/retrace-shard/results + ${{ runner.temp }}/retrace-shard/junit + ${{ runner.temp }}/retrace-shard/logs + if-no-files-found: warn + overwrite: true # Trace retrace on the split library: the retrace-verify shape with MOBILEGL_TRANSPORT set # instead of MOBILEGL_PIPE_VERIFY, over {backend, case, transport} with inproc and spawn. @@ -3237,7 +3247,7 @@ jobs: # guards are the nm check for MG_Remote, run_trace_case.cmake's transport assertions (and, on # OpenRA, its wire-volume assertion), and the per-leg check of the evidence file they leave. retrace-split: - name: retrace split (${{ matrix.backend }} / ${{ matrix.transport }}, ${{ matrix.case }}) + name: retrace split (${{ matrix.backend }} / ${{ matrix.transport }}, shard ${{ matrix.shard }}/${{ matrix.shards }}) runs-on: ubuntu-26.04 timeout-minutes: 240 # The setup action selects lavapipe on GitHub-hosted runners. @@ -3248,19 +3258,20 @@ jobs: - build-retrace - trace-cases - trace-fixtures - if: ${{ !cancelled() && needs.build-linux-split.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && (needs.build-guard.outputs.verdict != 'identical' || needs.build-linux.result == 'success') }} + if: ${{ !cancelled() && needs.build-linux-split.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' && (needs.build-guard.outputs.verdict != 'identical' || needs.build-linux.result == 'success') }} env: SPLIT_REUSE: ${{ needs.build-guard.outputs.verdict == 'identical' && 'true' || 'false' }} SPLIT_LIB: ${{ needs.build-guard.outputs.verdict == 'identical' && 'mobilegl-linux-lib' || 'mobilegl-linux-lib-split' }} SPLIT_LIB_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }} SPLIT_BUILD_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }} + SHARD_NAMES: ${{ toJSON(matrix.names) }} strategy: fail-fast: false - max-parallel: 8 # With build-linux reused, the inproc rows (except OpenRA's) are replayed by the retrace job's # inproc-acceptance rows on the same library, strict and with the dual block and run-ahead - # on; the spawn rows below prove per leg that those are also the defaults they run with. - matrix: ${{ fromJSON(needs.build-guard.outputs.verdict == 'identical' && needs.trace-cases.outputs.split-spawn-matrix || needs.trace-cases.outputs.split-matrix) }} + # on; the spawn rows below prove per case that those are also the defaults they run with. + # A shard holds one transport, so OpenRA's inproc rows are shards of their own. + matrix: ${{ fromJSON(needs.build-guard.outputs.verdict == 'identical' && needs.trace-cases.outputs.split-spawn-shards || needs.trace-cases.outputs.split-shards) }} steps: - name: Checkout repo @@ -3288,16 +3299,16 @@ jobs: with: gpu: "true" - - name: Download trace fixture - uses: actions/download-artifact@v8 + # A miss does not fail the job; the install step downloads what the bundle does not hold. + - name: Restore trace fixtures + uses: actions/cache/restore@v5 with: - name: trace-fixture-${{ matrix.case }} - path: trace-fixture-download + path: ${{ needs.trace-fixtures.outputs.path }} + key: ${{ needs.trace-fixtures.outputs.key }} - - name: Install trace fixture - run: | - mkdir -p tools/trace_replay/fixtures - find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \; + - name: Install trace fixtures + timeout-minutes: 15 + run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }} - name: Get CMake timeout-minutes: 15 @@ -3351,12 +3362,11 @@ jobs: if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi done - # THE SPAWN ARM NEEDS A SECOND BINARY and the inproc arm needs none, which is the whole - # difference between the two at this level: inproc's server role is a thread in the replay - # process, spawn's is a PROCESS the client launches from this image. It ships in - # the split retrace library - libMobileGLServer.so is an executable that wears a - # .so name so an APK will install it somewhere exec-able - and a missing one must be a NAMED - # failure here rather than a LaunchServer refusal 78 jobs deep. + # The spawn arm needs a second binary and the inproc arm needs none: inproc's server role + # is a thread in the replay process, spawn's is a process the client launches from this + # image. It ships in the split retrace library (libMobileGLServer.so is an executable + # with a .so name, so that an APK installs it somewhere it can be executed), and a missing + # one must be a named failure here rather than a LaunchServer refusal in every case. - name: Place the server image (spawn arm only) if: ${{ matrix.transport == 'spawn' }} run: | @@ -3370,114 +3380,54 @@ jobs: chmod +x build-linux/libMobileGLServer.so echo "server image: $(readlink -f build-linux/libMobileGLServer.so)" - # The Magma knobs go to $GITHUB_ENV rather than to one step's shell, so every later step of - # this leg runs the same repair set as the replay. - - name: Magma repair knobs (all later steps) - run: | - # P7 package L: THE SAME MAGMA KNOBS THE MONOLITH RETRACE JOB EXPORTS, ALL OF THEM. - # This job exported none at all, so every DirectVulkan row of the split matrix was - # compared against a monolith row running a different repair set - and a split retrace - # whose control is not comparable cannot say whether a divergence is a transport fact. - # R11F_G11F_B10F fallback is whole-backend; the blended depth-write quirk auto-enables - # only on Qualcomm, which no runner has, so the OIT case forces it exactly as the - # monolith job does (ForceOn bypasses only the vendor gate). - if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then - echo "MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1" >> "${GITHUB_ENV}" - fi - if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \ - && [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then - echo "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1" >> "${GITHUB_ENV}" - fi - # THE THREE ITERATIONRP KNOBS, FOR THE ONE CASE THE MONOLITH JOB NAMES. - # The iterationRP repairs are tri-state quirks that default to device auto-detection and - # lavapipe is on no auto list, so without them Program 203 runs unrepaired and misses its - # golden output. The monolith job has exported them since they landed; this job never - # did, so `iterationrp x DirectVulkan` was the one cell of the split matrix whose red - # could not be read as a transport fact - the monolith control it is compared against was - # running a different repair set. Exit gate 1 asks for the two halves to be COMPARABLE, - # and two runs that differ in a quirk knob are not. - if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \ - && [ '${{ matrix.case }}' = 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' ]; then - echo "MOBILEGL_ITRP_FIX=ForceOn" >> "${GITHUB_ENV}" - fi - + # The per-case settings are retrace_shard.py's "split" profile: + # - the Magma knobs the retrace job's monolith rows run with (the R11F_G11F_B10F fallback, + # the blended depth-write quirk on its one case, the iterationRP repairs on theirs), so a + # split row and its monolith control differ in the transport alone; + # - MOBILEGL_TRANSPORT and the absolute server image path; + # - on OpenRA, the wire-volume counters and the check that draw records crossed the wire; + # - with build-linux reused, the IPC defaults the client must log (strict=0). + # run_trace_case.cmake turns MOBILEGL_TRANSPORT into assertions of its own: the library must + # have resolved the transport in this process and its log must carry no Fatal{, so a case + # that somehow ran the wrong library reds here instead of passing on its golden. After + # ctest the evidence file those assertions leave is checked for every case, and the + # refusal census is printed from the library's own logs of both roles, which is the only + # valid source: the console sink is compiled out of this configuration. - name: Retrace and validate (${{ matrix.transport }}) - working-directory: build-retrace/tools/trace_replay - # run_trace_case.cmake turns MOBILEGL_TRANSPORT into assertions of its own - the library - # must have RESOLVED the transport in this process, and its log must carry no Fatal{ - so a - # case that somehow ran the wrong library reds here instead of passing on its golden. That - # log is also the only valid refusal census: the console sink is compiled out of this - # configuration, so a `ctest -V` transcript reports a false zero for Fatal{ lines. run: | - ulimit -c unlimited - export MOBILEGL_TRANSPORT='${{ matrix.transport }}' - # Absolute, because the replay's working directory is not this one and LaunchServer - # resolves the image relative to nothing. Harmless on the inproc arm, which reads it - # only to decide it does not need it. - export MOBILEGL_IPC_SERVER_PATH="${GITHUB_WORKSPACE}/build-linux/libMobileGLServer.so" - # OpenRA's legs also count the wire's draw records and the server's draws; that turns on - # the library's per-frame counters, so the other cases keep the plain configuration. - wire=() - if [ '${{ matrix.case }}' = 'OpenRA' ]; then - export MGTRACE_WIRE_EVIDENCE=1 - wire=(--require-wire) - fi - if [ "${SPLIT_REUSE}" = "true" ]; then - export MGTRACE_EXPECT_IPC='strict=0 role-split-state=1 run-ahead=1' - export MGTRACE_EXPECT_IPC_REASON="retrace-split dropped its inproc rows because the inproc-acceptance retrace rows replay the same library with strict=1 role-split-state=1 run-ahead=1, which differs from this leg's defaults only in strict while these hold. A default changed: restore the inproc rows (the trace-cases job's split-spawn-matrix without --transports spawn) or update the expectation." - fi - ctest -V --no-tests=error --timeout 10800 \ - -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$' - # The transport assertions ran on this leg and found their evidence. - python3 "${GITHUB_WORKSPACE}/scripts/ci/retrace_split_evidence.py" \ - '${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt' \ - --transport '${{ matrix.transport }}' --case '${{ matrix.case }}' \ - --backend '${{ matrix.backend }}' "${wire[@]}" + reuse=() + if [ "${SPLIT_REUSE}" = "true" ]; then reuse=(--split-reuse); fi + python3 scripts/ci/retrace_shard.py run --profile split \ + --backend '${{ matrix.backend }}' --transport '${{ matrix.transport }}' --lane split \ + --shard '${{ matrix.shard }}/${{ matrix.shards }}' \ + --server-path "${GITHUB_WORKSPACE}/build-linux/libMobileGLServer.so" \ + --build-dir build-retrace/tools/trace_replay \ + --result-dir "${RUNNER_TEMP}/retrace-shard" "${reuse[@]}" - # The refusal census, recorded rather than gated. run_trace_case.cmake already REDS the case - # on any Fatal{, so reaching here means the count is zero - but the number and the distinct - # slot names are what MEASUREMENTS wants from every split run, and reading them out of the - # library's own log is the only way to get them (ctest -V's transcript is a false zero). - - name: Refusal census from the library log - if: ${{ always() }} - working-directory: build-retrace/tools/trace_replay - run: | - # BOTH ROLES, because under spawn every applier refusal is raised in the SERVER's - # process and a census reading only the client's file reports a confident zero for the - # half it cannot see. - log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.client.log" - server_log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.server.log" - cat "${{ matrix.case }}/${{ matrix.backend }}/output/split-evidence.txt" 2>/dev/null || echo "no split-evidence.txt" - if [ ! -f "${log}" ]; then - echo "no ${log} - the replay wrote no library log" - exit 0 - fi - census_files="${log}" - if [ -f "${server_log}" ]; then census_files="${census_files} ${server_log}"; fi - echo "census over: ${census_files}" - echo "Fatal{ lines: $(cat ${census_files} | grep -c 'Fatal{' || true)" - cat ${census_files} | grep -o 'Fatal{[A-Za-z]*, "[^"]*"' | sort | uniq -c | sort -rn | head -20 || true - # The transport line is the CLIENT's: ConfigLoader resolves it there, and under spawn - # the server states its own separately. - grep -m1 "MOBILEGL_TRANSPORT=" "${log}" || echo "no transport line in ${log}" + - name: Annotate failed entries + if: failure() + run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit" - name: Upload core dumps - if: failure() + if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v7 with: - name: retrace-split-core-dumps-${{ matrix.transport }}-${{ matrix.backend }}-${{ matrix.case }} + name: retrace-split-core-dumps-${{ matrix.transport }}-${{ matrix.backend }}-shard${{ matrix.shard }} path: /tmp/core.* if-no-files-found: ignore - - name: Upload actual image - if: ${{ always() }} + # Nothing reads a green shard's output, so only a failed shard uploads it. + - name: Upload failed shard results + if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v7 with: - name: retrace-split-result-${{ matrix.transport }}-${{ matrix.backend }}-${{ matrix.case }} + name: retrace-split-failed-${{ matrix.transport }}-${{ matrix.backend }}-shard${{ matrix.shard }} path: | - build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/** - build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/** + ${{ runner.temp }}/retrace-shard/results + ${{ runner.temp }}/retrace-shard/junit + ${{ runner.temp }}/retrace-shard/logs if-no-files-found: warn + overwrite: true # The shipped library, replayed. build-linux-monolith-control's library has no MG_Remote, the # shape FCL embeds; on the disaggregation branches nothing else replays that shape, so it runs @@ -3490,8 +3440,8 @@ jobs: needs: - build-linux-monolith-control - build-retrace - - trace-fixtures - if: ${{ !cancelled() && (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && needs.build-linux-monolith-control.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-fixtures.result != 'skipped' }} + - trace-cases + if: ${{ !cancelled() && (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && needs.build-linux-monolith-control.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }} strategy: fail-fast: false matrix: @@ -3513,16 +3463,11 @@ jobs: with: gpu: "true" - - name: Download trace fixture - uses: actions/download-artifact@v8 - with: - name: trace-fixture-${{ matrix.case }} - path: trace-fixture-download - - - name: Install trace fixture - run: | - mkdir -p tools/trace_replay/fixtures - find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \; + # This job's one case keeps its fixtures in Git itself, not in Git LFS, so the checkout + # has them and the fixture bundle is not restored. The check is that they are real files; + # a case with Git LFS fixtures would be downloaded here, with a warning annotation. + - name: Check trace fixture + run: bash .github/scripts/trace-fixture-bundle.sh install '${{ matrix.case }}' - name: Get CMake timeout-minutes: 15 @@ -3588,15 +3533,16 @@ jobs: run: python3 scripts/ci/junit_annotate.py --excerpts 1 "${RUNNER_TEMP}" - name: Upload core dumps - if: failure() + if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v7 with: name: retrace-shipped-core-dumps-${{ matrix.backend }}-${{ matrix.case }} path: /tmp/core.* if-no-files-found: ignore + # Nothing reads a green leg's output, so only a failed leg uploads it. - name: Upload actual image - if: ${{ always() }} + if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v7 with: name: retrace-shipped-result-${{ matrix.backend }}-${{ matrix.case }} @@ -3605,82 +3551,6 @@ jobs: build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/** if-no-files-found: warn - remove-artifact-clutter: - name: remove artifact clutter - runs-on: ubuntu-26.04 - # Every retrace job downloads the shared fixtures, so this waits for all of them. - needs: - - retrace-summary - - retrace-verify - - retrace-split - - retrace-shipped - if: ${{ always() }} - permissions: - actions: write - steps: - - name: Checkout repo - uses: actions/checkout@v6 - - name: Configure runner environment - uses: ./.github/actions/setup-test-runner - - - name: Delete intermediate Linux retrace artifacts - env: - GH_TOKEN: ${{ github.token }} - run: | - # Read the complete inventories before deleting anything. A failed API - # request must fail this step, not look like "no failed cases" through - # an unchecked process substitution. - # 50 rows per page: the jobs endpoint answers HTTP 502 for 90+ rows on a re-run attempt of a run this size. - gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=50" \ - --jq '.jobs[] | select((.name | startswith("retrace (")) or (.name | startswith("retrace verify (")) or (.name | startswith("retrace split (")) or (.name | startswith("retrace shipped library ("))) | select(.conclusion == "failure" or .conclusion == "canceled" or .conclusion == "timed_out" or .conclusion == "action_required") | .name' \ - > "${RUNNER_TEMP}/cleanup-jobs.txt" - gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \ - --jq '.artifacts[] | select(.name | startswith("trace-fixture-") or startswith("retrace-result-") or startswith("retrace-split-result-") or startswith("retrace-shipped-result-")) | [.id, .name] | @tsv' \ - > "${RUNNER_TEMP}/cleanup-artifacts.txt" - declare -A failed_cases=() - while IFS= read -r job_name; do - case_name="${job_name#retrace split (*, }" - case_name="${case_name#retrace shipped library (*, }" - case_name="${case_name#retrace verify (*, }" - case_name="${case_name#retrace (*, }" - case_name="${case_name%)}" - failed_cases["${case_name}"]=1 - done < "${RUNNER_TEMP}/cleanup-jobs.txt" - - if ((${#failed_cases[@]})); then - echo "Retaining fixtures and results for failed retrace case(s):" - printf ' %s\n' "${!failed_cases[@]}" - else - echo "All retrace jobs succeeded; no fixtures need to be retained." - fi - - deleted=0 - retained=0 - while IFS=$'\t' read -r artifact_id artifact_name; do - keep=0 - if [[ "${artifact_name}" == trace-fixture-* ]]; then - case_name="${artifact_name#trace-fixture-}" - if [[ -v "failed_cases[${case_name}]" ]]; then - keep=1 - fi - else - for case_name in "${!failed_cases[@]}"; do - if [[ "${artifact_name}" == *-"${case_name}" ]]; then keep=1; break; fi - done - fi - if ((keep)); then - echo "Retaining ${artifact_name} (${artifact_id}) for failed retrace." - ((retained += 1)) - continue - fi - - echo "Deleting ${artifact_name} (${artifact_id})" - gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}" - ((deleted += 1)) - done < "${RUNNER_TEMP}/cleanup-artifacts.txt" - - echo "Deleted ${deleted} intermediate Linux artifact(s); retained ${retained} failed-retrace artifact(s)." - pipe-gates: name: MGPipe generators and hygiene gates runs-on: ubuntu-26.04 diff --git a/.gitignore b/.gitignore index 56ec4bbd..8f31b4be 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,5 @@ __pycache__/ *.py[cod] /.gradle /.trace-work/ +# The restored trace fixture cache entry (.github/scripts/trace-fixture-bundle.sh) +/trace-fixture-bundle/ diff --git a/android-plugin/TRACE_REPLAY.md b/android-plugin/TRACE_REPLAY.md index 5fa25313..698bbdca 100644 --- a/android-plugin/TRACE_REPLAY.md +++ b/android-plugin/TRACE_REPLAY.md @@ -74,3 +74,57 @@ adb shell am start -a top.mobilegl.plugin.TRACE_REPLAY \ --es ssim_threshold 0.99 adb shell run-as top.mobilegl.plugin.espryt.trace cat files/trace-replay/output/result.json ``` + +## Emulator shards in CI + +The APK workflow replays its cases through `trace-replay-shard.sh`: one emulator boot, one APK +install, then an ordered list of cases for one lane, transport and backend. Each case still runs +through `trace-replay-ci.sh`, with the same arguments, environment and transport proof as a +one-case job. + +```sh +sh android-plugin/trace-replay-shard.sh \ + --apk-file MobileGL-plugin-trace-release.apk \ + --backend DirectVulkan --gpu lavapipe \ + --transport inproc --lane inproc-acceptance \ + --cases-file cases.json --deadline-seconds 7200 +``` + +The case list is a JSON array of case objects: the `cases` member of a row of +`trace_cases.py --ci --format github-apk-transport-shard-matrix`, which is what the workflow +passes, or the `case` members of `--format github-apk-transport-matrix` rows. `--dry-run` prints +what each case would be run with and touches no device. + +The workflow's matrix is that shard matrix: one row per emulator job, holding some of the cases +of one lane, transport and backend. `--shard-seconds` (`RETRACE_SHARD_SECONDS` in `apk.yml`) sets +how much hinted replay time one shard aims for, and `trace_cases.py` refuses to print a shard +matrix whose shards, taken apart, are not exactly the per-case matrix. The fixtures of a shard's +cases must be in `tools/trace_replay/fixtures` before the runner starts; the workflow restores +them from one cache entry (`.github/scripts/trace-fixture-bundle.sh`). + +Between cases the runner force-stops the app, checks that no process of the app's uid is left, +clears the app data, removes the case's files from `/data/local/tmp` and from the host, and +checks that the device still answers. It restarts the emulator when it does not, before the +case that follows one that did not pass, and before the single retry of a case that +`trace-replay-ci.sh` charged to the infrastructure (exit 75). +`MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE=1` cold boots before every case. + +The cases of a shard share the job's time. `--deadline-seconds` stops the runner that long after +it starts, so that the job can still upload what it has: a case that is running then, or has not +started, is recorded as `not-run`. `retrace_shard_budget.py` reads a shard matrix and adds two +fields to every shard: `deadline_seconds`, the time in which every case of the shard could use +its whole time budget (`timeout_seconds` plus 300 s) when the cases before it used all of +theirs, with one infrastructure retry of the longest budget and a cold boot per case on top, +and `timeout_minutes`, the limit of the shard's job. `--cap-seconds` +(`RETRACE_SHARD_DEADLINE_CAP_SECONDS` in `apk.yml`) bounds the deadline, so a shard whose +replays all hang cannot hold a runner for hours; a capped shard loses verdicts only when several +of its replays hang to the end of their budgets. + +A failing case does not stop the shard. Every case ends with `-/case-status.json` +(`passed`, `failed` or `not-run`), the result root gets `shard-status.json`, the job log gets one +`::error` per case that did not pass, and the exit status is non-zero when a case did not pass or +a verdict is missing. `--finalize` writes the summary inputs and a `not-run` verdict for every +case without one, without a device; the workflow runs it after the replay step whatever happened. + +`python3 android-plugin/test_trace_replay_shard.py` drives the runner with a fake adb, replay +script and AVD script. diff --git a/android-plugin/retrace_artifact_cleanup.py b/android-plugin/retrace_artifact_cleanup.py new file mode 100644 index 00000000..6e108bb2 --- /dev/null +++ b/android-plugin/retrace_artifact_cleanup.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +# MobileGL - android-plugin/retrace_artifact_cleanup.py +# Copyright (c) 2025-2026 MobileGL-Dev +# Licensed under the GNU Lesser General Public License v3.0: +# https://www.gnu.org/licenses/gpl-3.0.txt +# https://www.gnu.org/licenses/lgpl-3.0.txt +# SPDX-License-Identifier: LGPL-3.0-only +# End of Source File Header + +"""Delete the intermediate artifacts of an APK workflow run, keeping what a failure needs. + +The retrace summary embeds every image it shows, so once it is uploaded the per-shard result +artifacts are only worth keeping for cases that did not pass: + +- a result artifact stays when its shard-status.json is missing, when its job failed, or when + it holds a case that did not pass in any shard (so both backends and all transports of a + failed case stay comparable); +- every result artifact stays when a job failed without leaving a shard-status.json to say + which cases it was running. + +A canceled job retains nothing by itself; a failed verdict inside it still does. The trace +fixtures are not artifacts: the retrace jobs restore them from a cache entry. +""" + +import argparse +import json +import os +import shlex +import subprocess +import sys +import threading +import time +from pathlib import Path + +RESULT_PREFIXES = ("MobileGL-android-retrace-result-", "retrace-result-") +FAILED_CONCLUSIONS = ("failure", "timed_out", "action_required") +# GitHub's secondary rate limit allows about 180 mutating REST calls a minute. Requests start +# at most this often, whatever the number of workers. +DELETE_INTERVAL_SECONDS = 60.0 / 170.0 +DELETE_WORKERS = 3 +MAX_REFUSED_DELETES = 5 + + +def pause(seconds): + time.sleep(seconds) + + +def read_rows(text, columns): + rows = [] + for line in text.splitlines(): + if line.strip(): + fields = line.split("\t") + rows.append(fields + [""] * (columns - len(fields))) + return rows + + +def shard_statuses(download_roots): + """Artifact name -> its shard-status.json, for every downloaded result artifact that has one.""" + statuses = {} + for root in download_roots: + root = Path(root) + if not root.is_dir(): + continue + for directory in root.iterdir(): + try: + status = json.loads((directory / "shard-status.json").read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if isinstance(status, dict) and isinstance(status.get("cases"), list): + statuses[directory.name] = status + return statuses + + +def plan(artifacts, jobs, statuses): + """Returns (ids to delete, lines explaining what stays).""" + failed_job_ids = {job_id for job_id, conclusion, _ in jobs if conclusion in FAILED_CONCLUSIONS} + status_job_ids = {str(status.get("job_id") or "") for status in statuses.values()} - {""} + # Failed jobs that left no status file: nothing says which cases they were running. + unexplained = failed_job_ids - status_job_ids + if not status_job_ids and failed_job_ids: + # The statuses carry no job ids to match against. Count instead: each failed job must + # be explained by an artifact that shows a case without a pass. + explaining = sum(1 for status in statuses.values() + if any(record.get("status") != "passed" for record in status["cases"])) + unexplained = set() if explaining >= len(failed_job_ids) else failed_job_ids + + failed_cases = set() # did not pass somewhere: every result that holds one stays + for status in statuses.values(): + not_passed = [record for record in status["cases"] if record.get("status") != "passed"] + if status_job_ids: + job_failed = str(status.get("job_id") or "") in failed_job_ids + else: + # Without job ids, a status that shows a case without a pass stands for a failed job. + job_failed = bool(failed_job_ids) and bool(not_passed) + for record in not_passed: + if record.get("status") == "failed" or job_failed: + failed_cases.add(record.get("case")) + + delete, kept = [], [] + for artifact_id, name in artifacts: + if not name.startswith(RESULT_PREFIXES): + continue + reason = "" + status = statuses.get(name) + if unexplained: + reason = f"{len(unexplained)} failed job(s) left no shard status" + elif status is None: + reason = "no shard status was downloaded for it" + elif str(status.get("job_id") or "") in failed_job_ids: + reason = "its job failed" + else: + cases = sorted(failed_cases.intersection(record.get("case") for record in status["cases"])) + if cases: + reason = "holds failed case(s): " + ", ".join(cases) + if reason: + kept.append(f"{name} ({artifact_id}): {reason}") + else: + delete.append((artifact_id, name)) + return delete, kept + + +class Deleter: + """Runs `command` once per artifact, with the artifact id appended to its last word.""" + + def __init__(self, command): + self.command = command + self.lock = threading.Lock() + self.next_start = 0.0 + self.refused = [] + self.deleted = 0 + + def wait_for_slot(self): + with self.lock: + now = time.monotonic() + start = max(now, self.next_start) + self.next_start = start + DELETE_INTERVAL_SECONDS + if start > now: + pause(start - now) + + def delete(self, artifact_id, name): + for attempt in (1, 2): + with self.lock: + if len(self.refused) >= MAX_REFUSED_DELETES: + return + self.wait_for_slot() + done = subprocess.run(self.command[:-1] + [self.command[-1] + artifact_id], + capture_output=True, text=True) + if done.returncode == 0: + with self.lock: + self.deleted += 1 + return + if attempt == 1: + # Give a rate limit time to clear before the one retry. + with self.lock: + self.next_start = max(self.next_start, time.monotonic() + 30) + with self.lock: + self.refused.append(f"{name} ({artifact_id}): {' '.join(done.stderr.split())[:200]}") + + def run(self, entries): + pending = list(entries) + + def work(): + while True: + with self.lock: + if not pending: + return + artifact_id, name = pending.pop(0) + self.delete(artifact_id, name) + + threads = [threading.Thread(target=work) for _ in range(min(DELETE_WORKERS, len(pending)))] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + +def gh_rows(path, query, columns): + done = subprocess.run(["gh", "api", "--paginate", path, "--jq", query], capture_output=True, text=True) + if done.returncode != 0: + raise RuntimeError(f"gh api {path} failed: {done.stderr.strip()}") + return read_rows(done.stdout, columns) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--downloads", action="append", default=[], + help="Directory holding one sub-directory per downloaded result artifact.") + parser.add_argument("--artifacts-file", help="Tab-separated id and name per line, instead of asking gh.") + parser.add_argument("--jobs-file", help="Tab-separated id, conclusion and name per line, instead of asking gh.") + parser.add_argument("--dry-run", action="store_true", help="Print the plan, delete nothing.") + options = parser.parse_args(argv) + + repository = os.environ.get("GITHUB_REPOSITORY", "") + run_id = os.environ.get("GITHUB_RUN_ID", "") + try: + if options.artifacts_file: + artifacts = read_rows(Path(options.artifacts_file).read_text(encoding="utf-8"), 2) + else: + artifacts = gh_rows(f"repos/{repository}/actions/runs/{run_id}/artifacts?per_page=100", + ".artifacts[] | [.id, .name] | @tsv", 2) + if options.jobs_file: + jobs = read_rows(Path(options.jobs_file).read_text(encoding="utf-8"), 3) + else: + # 50 rows per page: the jobs endpoint answers HTTP 502 for larger pages of a re-run + # attempt of a run this size. + jobs = gh_rows(f"repos/{repository}/actions/runs/{run_id}/jobs?per_page=50", + '.jobs[] | [.id, .conclusion // "", .name] | @tsv', 3) + except (OSError, RuntimeError) as error: + print(f"::error::Could not read the run's artifacts and jobs; nothing was deleted: {error}") + return 1 + + delete, kept = plan([(row[0], row[1]) for row in artifacts], [tuple(row[:3]) for row in jobs], + shard_statuses(options.downloads)) + for line in kept: + print(f"Retaining {line}") + if options.dry_run: + for artifact_id, name in delete: + print(f"Would delete {name} ({artifact_id})") + return 0 + + deleter = Deleter(shlex.split(os.environ.get("MOBILEGL_CLEANUP_DELETE_COMMAND") or + f"gh api --method DELETE repos/{repository}/actions/artifacts/")) + started = time.monotonic() + deleter.run(delete) + for line in deleter.refused: + print(f"::warning::Could not delete intermediate artifact {line}") + if len(deleter.refused) >= MAX_REFUSED_DELETES: + print("::warning::Stopped deleting intermediate artifacts after repeated refusals; the rest stay until they expire.") + print(f"Deleted {deleter.deleted} intermediate Android artifact(s) in {time.monotonic() - started:.0f}s; " + f"retained {len(kept)}.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/android-plugin/retrace_shard_budget.py b/android-plugin/retrace_shard_budget.py new file mode 100644 index 00000000..7e42ef0d --- /dev/null +++ b/android-plugin/retrace_shard_budget.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# MobileGL - android-plugin/retrace_shard_budget.py +# Copyright (c) 2025-2026 MobileGL-Dev +# Licensed under the GNU Lesser General Public License v3.0: +# https://www.gnu.org/licenses/gpl-3.0.txt +# https://www.gnu.org/licenses/lgpl-3.0.txt +# SPDX-License-Identifier: LGPL-3.0-only +# End of Source File Header + +"""Give every emulator shard the deadline and the job timeout its own cases need. + +The cases of a shard share one job, so one case that hangs spends time the cases after it +need. This reads a shard matrix of trace_cases.py (--format github-apk-shard-matrix or +github-apk-transport-shard-matrix) on standard input, or from --matrix-file, and prints it +with two fields added to every shard: deadline_seconds, which the retrace job passes to +trace-replay-shard.sh, and timeout_minutes, the limit of that job. The deadline is what the +shard needs for every case to use its whole time budget (trace_replay_shard.needed_seconds), +up to --cap-seconds. A capped shard still replays every case; only when several of its +replays hang to the end of their budgets are the cases after them recorded as not-run. +""" + +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import trace_replay_shard # noqa: E402 (a sibling file, found through the line above) + + +def shard_label(row): + backend = row.get("backend") + backend = backend.get("name", "") if isinstance(backend, dict) else str(backend or "") + return f"{backend} / {row.get('transport') or 'monolith'}, shard {row.get('shard')}/{row.get('shards')}" + + +def shard_needs(matrix): + """(label, case count, seconds needed) for every shard of the matrix.""" + rows = matrix.get("include") if isinstance(matrix, dict) else None + if not isinstance(rows, list) or not rows: + raise ValueError("the matrix has no shard rows under 'include'") + needs = [] + for row in rows: + cases = row.get("cases") if isinstance(row, dict) else None + if not isinstance(cases, list) or not cases: + raise ValueError(f"a shard row has no 'cases' list: {str(row)[:200]}") + needs.append((shard_label(row), len(cases), trace_replay_shard.needed_seconds(cases))) + return needs + + +# Minutes a retrace job has around its shard runner: the setup before the replays and the +# uploads after them. +JOB_OVERHEAD_MINUTES = 15 + +# A hosted job may run six hours. +HOSTED_JOB_LIMIT_MINUTES = 360 + + +def with_limits(matrix, cap_seconds): + """The matrix with deadline_seconds and timeout_minutes on every shard, and the report.""" + needs = shard_needs(matrix) + rows = [] + report = [] + for row, (label, cases, seconds) in zip(matrix["include"], needs): + deadline = min(seconds, cap_seconds) + rows.append({**row, "deadline_seconds": deadline, + "timeout_minutes": -(-deadline // 60) + JOB_OVERHEAD_MINUTES}) + if seconds > cap_seconds: + report.append(f"retrace shard budget: {label} would need {seconds} s for all of its " + f"{cases} case(s) to use their whole budgets; capped at {cap_seconds} s") + largest = max(needs, key=lambda entry: entry[2]) + report.append(f"retrace shard budget: {len(needs)} shard(s), {len(report)} capped at " + f"{cap_seconds} s; the most a shard needs is {largest[2]} s ({largest[0]}, " + f"{largest[1]} case(s))") + return {**matrix, "include": rows}, report + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--cap-seconds", type=int, required=True, + help="The longest deadline a shard is given.") + parser.add_argument("--matrix-file", help="The shard matrix; standard input when not given.") + options = parser.parse_args(argv) + limit = (HOSTED_JOB_LIMIT_MINUTES - JOB_OVERHEAD_MINUTES) * 60 + if not 0 < options.cap_seconds <= limit: + print(f"retrace shard budget: --cap-seconds must be between 1 and {limit}", file=sys.stderr) + return 2 + if options.matrix_file: + text = Path(options.matrix_file).read_text(encoding="utf-8") + else: + text = sys.stdin.read() + try: + matrix, report = with_limits(json.loads(text), options.cap_seconds) + except (ValueError, KeyError, TypeError) as error: + print(f"retrace shard budget: the shard matrix cannot be read: {error}", file=sys.stderr) + return 2 + for line in report: + print(line, file=sys.stderr) + print(json.dumps(matrix, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/android-plugin/test_retrace_artifact_cleanup.py b/android-plugin/test_retrace_artifact_cleanup.py new file mode 100644 index 00000000..4b6f9268 --- /dev/null +++ b/android-plugin/test_retrace_artifact_cleanup.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +# MobileGL - android-plugin/test_retrace_artifact_cleanup.py +# Copyright (c) 2025-2026 MobileGL-Dev +# Licensed under the GNU Lesser General Public License v3.0: +# https://www.gnu.org/licenses/gpl-3.0.txt +# https://www.gnu.org/licenses/lgpl-3.0.txt +# SPDX-License-Identifier: LGPL-3.0-only +# End of Source File Header + +"""Pin which intermediate APK artifacts the cleanup keeps, and how fast it deletes the rest.""" + +import contextlib +import importlib.util +import io +import json +import os +import pathlib +import sys +import tempfile +import unittest +from unittest import mock + +HERE = pathlib.Path(__file__).resolve().parent +_spec = importlib.util.spec_from_file_location("retrace_artifact_cleanup", HERE / "retrace_artifact_cleanup.py") +cleanup = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(cleanup) + +RESULT = "MobileGL-android-retrace-result-" + + +def status(job_id, **cases): + return {"job_id": job_id, "cases": [{"case": name, "status": verdict} for name, verdict in cases.items()]} + + +def names(entries): + return sorted(name for _, name in entries) + + +class PlanTest(unittest.TestCase): + ARTIFACTS = [("1", RESULT + "mono-gles"), ("2", RESULT + "inproc-gles"), ("3", RESULT + "spawn-vk"), + ("4", "mobilegl-android-monolith-control.html"), ("5", "MobileGL-plugin-release-abc.apk"), + ("6", "MobileGL-plugin-trace-release-abc.apk")] + + def test_a_green_run_deletes_every_intermediate_and_nothing_else(self): + statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed"), + RESULT + "inproc-gles": status("12", a="passed", b="passed"), + RESULT + "spawn-vk": status("13", c="passed")} + jobs = [("11", "success", "x"), ("12", "success", "y"), ("13", "success", "z"), ("14", "", "retrace summary")] + delete, kept = cleanup.plan(self.ARTIFACTS, jobs, statuses) + self.assertEqual(kept, []) + self.assertEqual(names(delete), sorted(name for _, name in self.ARTIFACTS[:3])) + + def test_a_failed_case_keeps_every_shard_that_ran_it(self): + statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed"), + RESULT + "inproc-gles": status("12", a="passed", b="failed"), + RESULT + "spawn-vk": status("13", c="passed")} + jobs = [("11", "success", "x"), ("12", "failure", "y"), ("13", "success", "z")] + delete, kept = cleanup.plan(self.ARTIFACTS, jobs, statuses) + # Only b's results are worth comparing, so the monolith shard stays for b and the + # spawn shard goes. + self.assertEqual(names(delete), [RESULT + "spawn-vk"]) + self.assertEqual(len(kept), 2) + self.assertTrue(any("inproc-gles" in line and "its job failed" in line for line in kept)) + self.assertTrue(any("mono-gles" in line and "holds failed case(s): b" in line for line in kept)) + + def test_cases_a_failed_job_never_reached_are_kept(self): + statuses = {RESULT + "mono-gles": status("11", a="passed", b="not-run"), + RESULT + "spawn-vk": status("13", c="passed")} + jobs = [("11", "timed_out", "x"), ("13", "success", "z")] + delete, kept = cleanup.plan(self.ARTIFACTS[:1] + self.ARTIFACTS[2:], jobs, statuses) + self.assertEqual(names(delete), [RESULT + "spawn-vk"]) + self.assertEqual(len(kept), 1) + + def test_a_job_that_failed_after_its_cases_passed_keeps_its_results(self): + statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed"), + RESULT + "spawn-vk": status("13", c="passed")} + jobs = [("11", "failure", "upload failed"), ("13", "success", "z")] + delete, kept = cleanup.plan(self.ARTIFACTS[:1] + self.ARTIFACTS[2:], jobs, statuses) + self.assertEqual(names(delete), [RESULT + "spawn-vk"]) + self.assertTrue("mono-gles" in kept[0] and "its job failed" in kept[0]) + + def test_a_canceled_job_keeps_nothing_by_itself(self): + statuses = {RESULT + "mono-gles": status("11", a="passed", b="not-run"), + RESULT + "inproc-gles": status("12", a="failed", b="not-run")} + jobs = [("11", "cancelled", "x"), ("12", "cancelled", "y")] + delete, kept = cleanup.plan(self.ARTIFACTS[:2], jobs, statuses) + # Case a failed in the inproc shard, so both shards that ran it stay. + self.assertEqual(delete, []) + self.assertEqual(len(kept), 2) + only_not_run = {RESULT + "mono-gles": status("11", a="passed", b="not-run")} + delete, kept = cleanup.plan(self.ARTIFACTS[:1], [("11", "cancelled", "x")], only_not_run) + self.assertEqual((names(delete), kept), ([RESULT + "mono-gles"], [])) + + def test_a_result_without_a_status_is_kept(self): + statuses = {RESULT + "mono-gles": status("11", a="passed")} + delete, kept = cleanup.plan(self.ARTIFACTS[:2], [("11", "success", "x")], statuses) + self.assertEqual(names(delete), [RESULT + "mono-gles"]) + self.assertIn("no shard status", kept[0]) + + def test_a_failed_job_without_a_status_keeps_everything(self): + statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed")} + jobs = [("11", "success", "x"), ("12", "failure", "died before its upload")] + delete, kept = cleanup.plan(self.ARTIFACTS, jobs, statuses) + self.assertEqual(delete, []) + self.assertEqual(len(kept), 3) + + def test_statuses_without_job_ids_fall_back_to_counting(self): + artifacts = self.ARTIFACTS[:1] + self.ARTIFACTS[2:] + explained = {RESULT + "mono-gles": status("", a="passed", b="not-run"), + RESULT + "spawn-vk": status("", c="passed")} + delete, _ = cleanup.plan(artifacts, [("12", "failure", "y")], explained) + # The one status with a case that did not pass stands for the one failed job. + self.assertEqual(names(delete), [RESULT + "spawn-vk"]) + unexplained = {RESULT + "mono-gles": status("", a="passed", b="passed"), + RESULT + "spawn-vk": status("", c="passed")} + delete, _ = cleanup.plan(artifacts, [("12", "failure", "y")], unexplained) + self.assertEqual(delete, []) + + +class RunTest(unittest.TestCase): + @staticmethod + def passed(artifacts): + """A shard status with one passed case for every artifact, each from a job of its own.""" + return {name: status(f"9{artifact_id}", a="passed") for artifact_id, name in artifacts} + + def run_cleanup(self, artifacts, jobs, statuses, exit_code=0, dry_run=False): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + for name, value in statuses.items(): + (root / "downloads" / "lane" / name).mkdir(parents=True) + (root / "downloads" / "lane" / name / "shard-status.json").write_text(json.dumps(value)) + (root / "artifacts.tsv").write_text("".join(f"{i}\t{n}\n" for i, n in artifacts)) + (root / "jobs.tsv").write_text("".join("\t".join(job) + "\n" for job in jobs)) + # One file per call: the deletes run in parallel processes. + (root / "deleted").mkdir() + script = root / "delete.py" + script.write_text("import pathlib, sys\n" + "(pathlib.Path(sys.argv[1]) / sys.argv[2].replace('/', '_')).write_text('')\n" + f"sys.exit({exit_code})\n") + python = pathlib.Path(sys.executable).as_posix() + command = f'"{python}" "{script.as_posix()}" "{(root / "deleted").as_posix()}" artifacts/' + pauses = [] + output = io.StringIO() + with mock.patch.dict(os.environ, {"MOBILEGL_CLEANUP_DELETE_COMMAND": command}), \ + mock.patch.object(cleanup, "pause", pauses.append), contextlib.redirect_stdout(output): + code = cleanup.main(["--downloads", str(root / "downloads" / "lane"), + "--artifacts-file", str(root / "artifacts.tsv"), + "--jobs-file", str(root / "jobs.tsv")] + (["--dry-run"] if dry_run else [])) + deleted = sorted(entry.name.replace("_", "/") for entry in (root / "deleted").iterdir()) + return code, output.getvalue(), deleted, pauses + + def test_deletes_are_spaced_under_the_secondary_rate_limit(self): + artifacts = [(str(index), f"{RESULT}shard{index}") for index in range(12)] + code, output, deleted, pauses = self.run_cleanup(artifacts, [], self.passed(artifacts)) + self.assertEqual(code, 0) + self.assertEqual(deleted, sorted(f"artifacts/{index}" for index in range(12))) + self.assertIn("Deleted 12 intermediate Android artifact(s)", output) + # Twelve starts are booked at least 11 intervals apart, so the rate stays under 180 a minute. + self.assertLess(60 / cleanup.DELETE_INTERVAL_SECONDS, 180) + self.assertGreater(len(pauses), 0) + + def test_refused_deletes_warn_and_leave_the_run_green(self): + artifacts = [(str(index), f"{RESULT}shard{index}") for index in range(8)] + code, output, _, _ = self.run_cleanup(artifacts, [], self.passed(artifacts), exit_code=1) + self.assertEqual(code, 0) + self.assertIn("::warning::Could not delete intermediate artifact", output) + self.assertIn("Stopped deleting", output) + self.assertIn("Deleted 0 intermediate", output) + + def test_a_dry_run_deletes_nothing(self): + artifacts = [("1", RESULT + "shard1")] + code, output, deleted, _ = self.run_cleanup(artifacts, [], self.passed(artifacts), dry_run=True) + self.assertEqual((code, deleted), (0, [])) + self.assertIn(f"Would delete {RESULT}shard1 (1)", output) + + +if __name__ == "__main__": + unittest.main() diff --git a/android-plugin/test_trace_replay_shard.py b/android-plugin/test_trace_replay_shard.py new file mode 100644 index 00000000..6ca2b231 --- /dev/null +++ b/android-plugin/test_trace_replay_shard.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python3 +# MobileGL - android-plugin/test_trace_replay_shard.py +# Copyright (c) 2025-2026 MobileGL-Dev +# Licensed under the GNU Lesser General Public License v3.0: +# https://www.gnu.org/licenses/gpl-3.0.txt +# https://www.gnu.org/licenses/lgpl-3.0.txt +# SPDX-License-Identifier: LGPL-3.0-only +# End of Source File Header + +"""Drive the emulator shard runner with a fake adb, replay script and AVD script. + +No device is needed. The fakes record every call, so the tests pin what each case is run +with, that every case ends with a verdict, and what a failure, an infrastructure retry, a +timeout or a dead emulator does to the rest of the shard. +""" + +import contextlib +import importlib.util +import io +import json +import os +import pathlib +import re +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from unittest import mock + +HERE = pathlib.Path(__file__).resolve().parent + + +def load(name): + spec = importlib.util.spec_from_file_location(name, HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +shard = load("trace_replay_shard") +budget = load("retrace_shard_budget") + +BLISS = "minecraft-1.21.4-fabric-iris-bliss-in-world" +ITERATIONRP = "minecraft-1.21.4-fabric-iris-iterationrp-in-world" +SPLIT = "MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1" + +FAKE_COMMON = r''' +import json, os, pathlib, sys +state = pathlib.Path(os.environ["FAKE_STATE"]) +def record(name, entry): + with open(state / name, "a", encoding="utf-8") as log: + log.write(json.dumps(entry) + "\n") +def scenario(): + path = state / "scenario.json" + return json.loads(path.read_text()) if path.exists() else {} +def count(name): + path = state / f"count-{name}" + value = int(path.read_text()) if path.exists() else 0 + path.write_text(str(value + 1)) + return value +''' + +FAKE_ADB = FAKE_COMMON + r''' +args = sys.argv[1:] +record("adb.log", args) +plan = scenario() +if args[:1] == ["get-state"]: + print("offline" if count("get-state") < plan.get("offline_get_state_calls", 0) else "device") +elif args[:3] == ["shell", "getprop", "sys.boot_completed"]: + print("1") +elif args[:3] == ["shell", "pm", "path"]: + print("package:/data/app/trace/base.apk") +elif args[:3] == ["shell", "pm", "clear"]: + print("Failed" if plan.get("pm_clear_fails") else "Success") +elif args[:1] == ["install"]: + if count("install") < plan.get("failing_installs", 0): + print("Failure [INSTALL_FAILED]") + sys.exit(1) + print("Success") +elif args[:2] == ["shell", "run-as"] and args[3:] == ["id", "-un"]: + print("u0_a77") +elif args[:3] == ["shell", "ps", "-A"]: + print("USER PID NAME") + print("root 1 init") + if count("ps") < plan.get("ps_calls_with_leftover", 0): + print("u0_a77 4242 libMobileGLServer.so") +''' + +FAKE_REPLAY = FAKE_COMMON + r''' +import time +args = sys.argv[1:] +def value(flag): + return args[args.index(flag) + 1] +case, backend, root = value("--case"), value("--backend"), pathlib.Path(value("--result-root")) +record("replay.log", {"case": case, "arguments": args, "environment": { + key: os.environ.get(key) for key in ( + "MOBILEGL_ESPRYT_USE_ANGLE", "MOBILEGL_TRACE_ANGLE_VARIANT", "MOBILEGL_MAGMA_R11G11B10F_FALLBACK", + "MOBILEGL_ITRP_FIX", "MOBILEGL_TRACE_SKIP_INSTALL", "MOBILEGL_TRACE_POLL_INTERVAL")}}) +(state / "replay.pid").write_text(str(os.getpid())) +steps = scenario().get("replay", {}).get(case, []) +attempt = count("replay-" + case) +step = steps[attempt] if attempt < len(steps) else {} +time.sleep(step.get("sleep", 0)) +directory = root / f"{case}-{backend}" +directory.mkdir(parents=True, exist_ok=True) +code = step.get("exit", 0) +if code == 75: + with open(root / "infrastructure-failure-reason.txt", "a") as reasons: + reasons.write(step.get("reason", "device-unavailable") + "\n") +if step.get("result", code in (0, 1)): + (directory / "result.json").write_text(json.dumps( + {"passed": step.get("passed", code == 0), "ssim": 0.999, "message": step.get("message", "retrace completed")})) +transport = "inproc" if "--require-inproc" in args else "spawn" if "--require-spawn" in args else "" +if transport and code == 0 and step.get("proof", True): + (directory / "transport-proof.json").write_text(json.dumps({"passed": True, "required_transport": transport})) +sys.exit(code) +''' + +FAKE_AVD = FAKE_COMMON + r''' +args = sys.argv[1:] +record("avd.log", args) +if args[:1] == ["start"]: + exits = scenario().get("avd_start_exits", []) + index = count("avd-start") + sys.exit(exits[index] if index < len(exits) else 0) +''' + + +def case(name, **extra): + base = {"name": name, "trace_archive": f"fixtures/{name}.tgz", "trace_file": "trace.trace", + "golden": f"fixtures/{name}.png", "target_call": 1234, "width": 854, "height": 480, + "ssim_threshold": 0.99, "crop_x": 0, "crop_y": 0, "crop_width": 0, "crop_height": 0, + "timeout_seconds": 900} + base.update(extra) + return base + + +class ShardTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = pathlib.Path(self.temporary.name) + self.state = self.root / "state" + self.state.mkdir() + self.results = self.root / "results" + (self.root / "fixtures").mkdir() + self.apk = self.root / "trace.apk" + self.apk.write_bytes(b"apk") + self.emulator_log = self.root / "emulator.log" + self.emulator_log.write_text("boot\n") + python = pathlib.Path(sys.executable).as_posix() + commands = {} + for name, source in (("adb", FAKE_ADB), ("replay", FAKE_REPLAY), ("avd", FAKE_AVD)): + script = self.root / f"fake_{name}.py" + script.write_text(source) + commands[name] = f'"{python}" "{script.as_posix()}"' + self.environment = { + "FAKE_STATE": str(self.state), "ADB": commands["adb"], + "MOBILEGL_SHARD_REPLAY_COMMAND": commands["replay"], "MOBILEGL_SHARD_AVD_COMMAND": commands["avd"], + "MOBILEGL_GPU_HANG_BUDGET_MS": "0", "AVD_NAME": "mobilegl-ci", + "EMULATOR_LOG": str(self.emulator_log), "EMULATOR_PID_FILE": str(self.root / "emulator.pid"), + "GITHUB_STEP_SUMMARY": str(self.root / "summary.md"), + } + + def fixture(self, name): + path = self.root / "fixtures" / name + path.write_bytes(b"png") + return str(path) + + def run_shard(self, cases, *extra, backend="DirectVulkan", transport="monolith", scenario=None, env=None): + if scenario is not None: + (self.state / "scenario.json").write_text(json.dumps(scenario)) + arguments = ["--apk-file", str(self.apk), "--backend", backend, "--gpu", "lavapipe", + "--transport", transport, "--lane", f"{transport}-lane", "--cases-json", json.dumps(cases), + "--result-root", str(self.results), "--fixture-root", str(self.root / "work"), *extra] + output = io.StringIO() + with mock.patch.dict(os.environ, {**self.environment, **(env or {})}), \ + mock.patch.object(shard, "pause", lambda seconds: None), \ + contextlib.redirect_stdout(output): + status = shard.main(arguments) + return status, output.getvalue() + + def log(self, name): + path = self.state / name + return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] + + def status(self, name, backend="DirectVulkan"): + return json.loads((self.results / f"{name}-{backend}" / "case-status.json").read_text()) + + def shard_status(self): + return json.loads((self.results / "shard-status.json").read_text()) + + # ---- what each case is run with ---------------------------------------------------- + + def planned(self, cases, backend, transport): + status, output = self.run_shard(cases, "--dry-run", backend=backend, transport=transport) + self.assertEqual(status, 0) + return [json.loads(line) for line in output.splitlines()] + + def test_plain_case_arguments_per_transport(self): + cases = [case("plain", alternate_golden="fixtures/plain-mali.png")] + head = ["--apk-file", str(self.apk), "--package", "top.mobilegl.plugin.trace", "--backend", "DirectVulkan", + "--result-root", str(self.results), "--fixture-root", str(self.root / "work"), "--case", "plain", + "--trace-archive", "fixtures/plain.tgz", "--trace-file", "trace.trace", + "--golden", "fixtures/plain.png", "--alternate-golden", "fixtures/plain-mali.png", + "--target-call", "1234", "--width", "854", "--height", "480", "--ssim-threshold", "0.99", + "--crop-x", "0", "--crop-y", "0", "--crop-width", "0", "--crop-height", "0", + "--timeout-seconds", "900"] + tails = { + "monolith": ["--env", "MOBILEGL_TRANSPORT=monolith;MOBILEGL_GPU_HANG_BUDGET_MS=0"], + "inproc": ["--env", f"MOBILEGL_TRANSPORT=inproc;{SPLIT};MOBILEGL_GPU_HANG_BUDGET_MS=0", "--require-inproc"], + "spawn": ["--env", f"MOBILEGL_TRANSPORT=spawn;{SPLIT};MOBILEGL_GPU_HANG_BUDGET_MS=0", + "--use-pbuffer", "--require-spawn"], + } + for transport, tail in tails.items(): + plan, = self.planned(cases, "DirectVulkan", transport) + self.assertEqual(plan["arguments"], head + tail, transport) + self.assertEqual(plan["timeout"], 1200) + self.assertEqual(plan["environment"], { + "MOBILEGL_ESPRYT_USE_ANGLE": "0", "MOBILEGL_TRACE_ANGLE_VARIANT": "ec889e6ea831", + "MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "1", "MOBILEGL_ITRP_FIX": "Auto"}) + + def test_case_specific_settings(self): + cases = [case(BLISS), case(ITERATIONRP), case("flush", coherent_as_flush=True), + case("lod", avoid_angle_llvmpipe_explicit_lod_bias=True), case("plain", ssim_threshold=0.985)] + gles = {plan["case"]: plan for plan in self.planned(cases, "DirectGLES", "monolith")} + vulkan = {plan["case"]: plan for plan in self.planned(cases, "DirectVulkan", "monolith")} + + def flags(plan): + return [argument for argument in plan["arguments"][plan["arguments"].index("--env") + 2:]] + + self.assertEqual(gles[BLISS]["environment"]["MOBILEGL_TRACE_ANGLE_VARIANT"], "90a62123d794") + self.assertEqual(vulkan[BLISS]["environment"]["MOBILEGL_TRACE_ANGLE_VARIANT"], "90a62123d794") + self.assertEqual(gles["plain"]["environment"], { + "MOBILEGL_ESPRYT_USE_ANGLE": "1", "MOBILEGL_TRACE_ANGLE_VARIANT": "ec889e6ea831", + "MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "0", "MOBILEGL_ITRP_FIX": "Auto"}) + self.assertEqual(gles[ITERATIONRP]["environment"]["MOBILEGL_ITRP_FIX"], "ForceOn") + self.assertEqual(flags(gles[BLISS]), ["--avoid-angle-llvmpipe-sampler-mipmap-min-filter"]) + self.assertEqual(flags(vulkan[BLISS]), []) + self.assertEqual(flags(gles["lod"]), ["--avoid-angle-llvmpipe-explicit-lod-bias"]) + self.assertEqual(flags(vulkan["lod"]), []) + self.assertEqual(flags(gles["flush"]), ["--coherent-as-flush"]) + self.assertEqual(flags(vulkan["flush"]), ["--coherent-as-flush"]) + self.assertEqual(flags(gles["plain"]), []) + arguments = gles["plain"]["arguments"] + self.assertEqual(arguments[arguments.index("--ssim-threshold") + 1], "0.985") + self.assertEqual(arguments[arguments.index("--alternate-golden") + 1], "") + + def test_misconfigured_case_lists_run_nothing(self): + for cases in ([case("a"), case("a")], [case("a", ci_backends=["DirectGLES"])], [{"name": "a"}], []): + status, output = self.run_shard(cases) + self.assertEqual(status, 2, cases) + self.assertIn("::error title=Android retrace shard misconfigured::", output) + self.assertEqual(self.log("adb.log"), []) + + # ---- the loop ---------------------------------------------------------------------- + + def test_every_case_runs_in_order_on_one_install(self): + cases = [case("a", golden=self.fixture("a.png"), alternate_golden=self.fixture("a-mali.png")), + case("b", golden=self.fixture("b.png")), case("c")] + status, output = self.run_shard(cases, transport="inproc") + self.assertEqual(status, 0, output) + self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b", "c"]) + for entry in self.log("replay.log"): + self.assertEqual(entry["environment"]["MOBILEGL_TRACE_SKIP_INSTALL"], "1") + self.assertEqual(entry["environment"]["MOBILEGL_TRACE_POLL_INTERVAL"], "0.5") + adb = self.log("adb.log") + self.assertEqual(sum(1 for call in adb if call[:1] == ["install"]), 1) + self.assertEqual(sum(1 for call in adb if call[:3] == ["shell", "pm", "clear"]), 3) + self.assertEqual(sum(1 for call in adb if call[:3] == ["shell", "rm", "-f"]), 3) + self.assertIn(["shell", "rm", "-f", "/data/local/tmp/mobilegl-b.trace", "/data/local/tmp/mobilegl-b.golden.png", + "/data/local/tmp/mobilegl-b.alternate-golden.png"], adb) + self.assertEqual(self.log("avd.log"), []) + directory = self.results / "a-DirectVulkan" + self.assertEqual(json.loads((directory / "lane.json").read_text()), + {"lane": "inproc-lane", "transport": "inproc"}) + self.assertTrue((directory / "a-DirectVulkan-golden.png").is_file()) + self.assertTrue((directory / "a-DirectVulkan-alternate-golden.png").is_file()) + self.assertFalse((self.results / "c-DirectVulkan" / "c-DirectVulkan-golden.png").exists()) + whole = self.shard_status() + self.assertTrue(whole["complete"]) + self.assertEqual([(entry["case"], entry["status"]) for entry in whole["cases"]], + [("a", "passed"), ("b", "passed"), ("c", "passed")]) + self.assertIn("| b | passed | 1 |", (self.root / "summary.md").read_text()) + + def test_a_failed_case_does_not_stop_the_shard(self): + status, output = self.run_shard([case("a"), case("b"), case("c")], scenario={ + "replay": {"b": [{"exit": 1, "message": "ssim=0.5"}]}}) + self.assertEqual(status, 1) + self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b", "c"]) + self.assertEqual([self.status(name)["status"] for name in "abc"], ["passed", "failed", "passed"]) + self.assertIn("::error title=Android retrace failed (DirectVulkan / monolith)::b: exit 1: ssim=0.5", output) + self.assertEqual(output.count("::error"), 1) + self.assertTrue(self.shard_status()["complete"]) + self.assertTrue((self.results / "b-DirectVulkan" / "emulator.log").is_file()) + + def test_the_case_after_one_that_did_not_pass_starts_on_a_cold_boot(self): + status, _ = self.run_shard([case("a"), case("b"), case("c"), case("d")], scenario={ + "replay": {"a": [{"exit": 75}], "b": [{"exit": 1}]}}) + self.assertEqual(status, 1) + self.assertEqual([self.status(name)["status"] for name in "abcd"], ["passed", "failed", "passed", "passed"]) + # One restart for a's infrastructure retry and one before c. A case that passed on + # its retry costs the next case nothing. + self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start", "stop", "start"]) + self.assertEqual(self.status("b")["notes"], []) + self.assertEqual(self.status("c")["notes"], ["emulator restarted before the case: " + "the case before it, b, did not pass"]) + self.assertEqual(self.status("d")["notes"], []) + self.assertEqual(sum(1 for call in self.log("adb.log") if call[:1] == ["install"]), 3) + + def test_a_failed_case_without_an_avd_to_restart_does_not_stop_the_shard(self): + status, _ = self.run_shard([case("a"), case("b")], scenario={"replay": {"a": [{"exit": 1}]}}, + env={"AVD_NAME": ""}) + self.assertEqual(status, 1) + self.assertEqual([self.status(name)["status"] for name in "ab"], ["failed", "passed"]) + self.assertEqual(self.log("avd.log"), []) + + def test_an_exit_0_without_a_passed_result_or_proof_fails(self): + status, _ = self.run_shard([case("a"), case("b")], transport="spawn", scenario={ + "replay": {"a": [{"exit": 0, "result": False}], "b": [{"exit": 0, "proof": False}]}}) + self.assertEqual(status, 1) + self.assertIn("without a passed result.json", self.status("a")["reason"]) + self.assertIn("without a passed spawn transport proof", self.status("b")["reason"]) + + def test_an_infrastructure_failure_restarts_and_retries_once(self): + status, output = self.run_shard([case("a"), case("b")], scenario={ + "replay": {"a": [{"exit": 75, "reason": "angle-surface-lost", "result": True, "passed": False}]}}) + self.assertEqual(status, 0, output) + self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "a", "b"]) + self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start"]) + self.assertEqual(self.log("avd.log")[1], [ + "start", "--avd-name", "mobilegl-ci", "--emulator-log", str(self.emulator_log), + "--pid-file", str(self.root / "emulator.pid"), "--gpu", "lavapipe", "--boot-timeout", "300"]) + self.assertIn(["kill-server"], self.log("adb.log")) + self.assertEqual(sum(1 for call in self.log("adb.log") if call[:1] == ["install"]), 2) + record = self.status("a") + self.assertEqual((record["status"], record["attempts"], record["exit_codes"]), ("passed", 2, [75, 0])) + self.assertEqual(record["surface_lost_retries"], 1) + kept = self.results / "a-DirectVulkan" / "first-attempt" + self.assertTrue((kept / "result.first-attempt.json").is_file()) + self.assertFalse((kept / "result.json").exists()) + self.assertIn("::warning::Android emulator infrastructure failed", output) + self.assertIn("surface-lost retries: 1 (DirectVulkan, a)", (self.root / "summary.md").read_text()) + self.assertTrue((self.results / "diagnostics" / "emulator-boot-1-ended.log").is_file()) + self.assertEqual(self.shard_status()["emulator_boots"], 2) + + def test_a_second_infrastructure_failure_is_the_verdict(self): + status, _ = self.run_shard([case("a"), case("b")], scenario={ + "replay": {"a": [{"exit": 75}, {"exit": 75}]}}) + self.assertEqual(status, 1) + self.assertEqual(self.status("a")["exit_codes"], [75, 75]) + self.assertEqual(self.status("a")["status"], "failed") + self.assertEqual(self.status("b")["status"], "passed") + # The surface-lost count is this case's own: the reasons of earlier cases do not leak in. + self.assertEqual(self.status("b")["surface_lost_retries"], 0) + + def test_the_deadline_cuts_a_running_case_and_names_the_rest(self): + status, output = self.run_shard([case("a"), case("b"), case("c")], "--deadline-seconds", "5", scenario={ + "replay": {"b": [{"sleep": 30}]}}) + self.assertEqual(status, 1) + self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b"]) + # b never had its own 1200 s, so the cut is not charged to it as a failure. + self.assertEqual([self.status(name)["status"] for name in "abc"], ["passed", "not-run", "not-run"]) + self.assertRegex(self.status("b")["reason"], r"^cut by the shard deadline after \d+ s of a 1200 s budget$") + self.assertEqual(self.status("b")["exit_codes"], [124]) + self.assertIn("deadline passed before this case started", self.status("c")["reason"]) + self.assertIn("::error title=Android retrace not-run (DirectVulkan / monolith)::b: cut by the shard deadline", + output) + self.assertIn("::error title=Android retrace shard incomplete (DirectVulkan / monolith)::" + "1 verdict(s) for 3 case(s)", output) + self.assertFalse(self.shard_status()["complete"]) + # Nothing is booted for a case the deadline keeps from starting. + self.assertEqual(self.log("avd.log"), []) + + def test_a_case_that_runs_out_of_its_own_budget_fails(self): + # The deadline is far away, so the one second a has is its own budget. + with mock.patch.object(shard, "REPLAY_GRACE_SECONDS", 0): + status, _ = self.run_shard([case("a", timeout_seconds=1), case("b")], "--deadline-seconds", "600", + scenario={"replay": {"a": [{"sleep": 30}]}}) + self.assertEqual(status, 1) + self.assertEqual([self.status(name)["status"] for name in "ab"], ["failed", "passed"]) + self.assertEqual(self.status("a")["reason"], "the replay script exceeded its time budget") + self.assertTrue(self.shard_status()["complete"]) + + def test_an_infrastructure_retry_is_not_started_after_the_deadline(self): + def restart_past_the_deadline(runner, reason): + runner.started -= 1000 + return True + + with mock.patch.object(shard.Shard, "restart_emulator", restart_past_the_deadline): + status, _ = self.run_shard([case("a"), case("b")], "--deadline-seconds", "600", + scenario={"replay": {"a": [{"exit": 75}]}}) + self.assertEqual(status, 1) + self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a"]) + self.assertEqual([self.status(name)["status"] for name in "ab"], ["not-run", "not-run"]) + self.assertEqual(self.status("a")["reason"], + "the shard deadline passed before the infrastructure retry could start") + self.assertEqual(self.status("a")["exit_codes"], [75]) + + # ---- the time a shard needs -------------------------------------------------------- + + def test_needed_seconds_lets_every_case_use_its_budget(self): + cases = [case("a", timeout_seconds=1800), case("b", timeout_seconds=900), case("c", timeout_seconds="180")] + # Every case to the end of its budget, the longest once more, and the overhead per case. + self.assertEqual(shard.needed_seconds(cases), (2100 + 1200 + 480) + 2100 + 3 * 180) + self.assertEqual(shard.needed_seconds(cases[2:]), 480 + 480 + 180) + + def test_a_deadline_of_needed_seconds_gives_every_hanging_case_its_verdict(self): + cases = [case(name, timeout_seconds=1) for name in "abc"] + # Scaled down: one second per case, and ten for what the fakes take around it. + with mock.patch.object(shard, "REPLAY_GRACE_SECONDS", 0), \ + mock.patch.object(shard, "CASE_OVERHEAD_SECONDS", 10): + deadline = shard.needed_seconds(cases) + self.assertEqual(deadline, 3 + 1 + 30) + status, _ = self.run_shard(cases, "--deadline-seconds", str(deadline), scenario={ + "replay": {name: [{"sleep": 60}] for name in "abc"}}) + self.assertEqual(status, 1) + self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b", "c"]) + for name in "abc": + self.assertEqual((self.status(name)["status"], self.status(name)["reason"]), + ("failed", "the replay script exceeded its time budget")) + self.assertTrue(self.shard_status()["complete"]) + self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start", "stop", "start"]) + + def test_every_shard_gets_its_own_deadline_and_job_timeout(self): + matrix = {"include": [ + {"backend": {"name": "DirectGLES", "gpu": "software"}, "lane": "spawn-acceptance", "transport": "spawn", + "shard": 1, "shards": 2, "names": ["a", "b"], + "cases": [case("a", timeout_seconds=1800), case("b", timeout_seconds=1800)]}, + {"backend": {"name": "DirectVulkan", "gpu": "lavapipe"}, "shard": 2, "shards": 2, "names": ["c"], + "cases": [case("c", timeout_seconds=180)]}]} + matrix_file = self.root / "matrix.json" + matrix_file.write_text(json.dumps(matrix)) + + def limits(cap, path=matrix_file): + output, report = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(report): + status = budget.main(["--cap-seconds", str(cap), "--matrix-file", str(path)]) + return status, output.getvalue(), report.getvalue() + + self.assertEqual(budget.shard_needs(matrix), [("DirectGLES / spawn, shard 1/2", 2, 6660), + ("DirectVulkan / monolith, shard 2/2", 1, 1140)]) + # Under the cap every shard gets exactly what its cases need. + status, output, report = limits(6660) + self.assertEqual(status, 0, report) + rows = json.loads(output)["include"] + self.assertEqual([(row["deadline_seconds"], row["timeout_minutes"]) for row in rows], + [(6660, 111 + budget.JOB_OVERHEAD_MINUTES), (1140, 19 + budget.JOB_OVERHEAD_MINUTES)]) + # Nothing but the two fields is added, and no case is dropped or changed. + self.assertEqual([{key: value for key, value in row.items() + if key not in ("deadline_seconds", "timeout_minutes")} for row in rows], + matrix["include"]) + self.assertIn("2 shard(s), 0 capped at 6660 s; the most a shard needs is 6660 s", report) + # Over the cap a shard is cut to it and named; the other shard is untouched. + status, output, report = limits(6000) + self.assertEqual(status, 0, report) + rows = json.loads(output)["include"] + self.assertEqual([(row["deadline_seconds"], row["timeout_minutes"]) for row in rows], + [(6000, 100 + budget.JOB_OVERHEAD_MINUTES), (1140, 19 + budget.JOB_OVERHEAD_MINUTES)]) + self.assertIn("DirectGLES / spawn, shard 1/2 would need 6660 s for all of its 2 case(s)", report) + self.assertNotIn("DirectVulkan / monolith, shard 2/2 would need", report) + # A cap that could not fit in a hosted job, and a matrix that cannot be read. + self.assertEqual(limits(0)[0], 2) + self.assertEqual(limits(360 * 60)[0], 2) + for broken in ({"include": []}, {"include": [{"names": ["a"]}]}, []): + matrix_file.write_text(json.dumps(broken)) + status, output, _ = limits(7200) + self.assertEqual((status, output), (2, ""), broken) + + def test_the_workflow_takes_its_limits_from_the_matrix(self): + text = (HERE.parent / ".github" / "workflows" / "apk.yml").read_text(encoding="utf-8") + retrace = text[text.index("\n retrace:"):text.index("\n retrace-summary:")] + cap, = re.findall(r'^\s+RETRACE_SHARD_DEADLINE_CAP_SECONDS: "(\d+)"\s*$', text, flags=re.MULTILINE) + # The runner's deadline and the job's limit are the two fields the budget script adds. + self.assertIn('--deadline-seconds "${{ matrix.deadline_seconds }}"', retrace) + self.assertEqual(re.findall(r"^ timeout-minutes: (.+?)\s*$", retrace, flags=re.MULTILINE), + ["${{ matrix.timeout_minutes }}"]) + self.assertIn('--cap-seconds "${RETRACE_SHARD_DEADLINE_CAP_SECONDS}"', text) + # The capped deadline and the job's overhead stay inside the six hours a hosted job may run. + self.assertLessEqual(-(-int(cap) // 60) + budget.JOB_OVERHEAD_MINUTES, budget.HOSTED_JOB_LIMIT_MINUTES) + + def test_an_unhealthy_emulator_is_restarted_before_the_next_case(self): + status, output = self.run_shard([case("a"), case("b")], scenario={"offline_get_state_calls": 1}) + self.assertEqual(status, 0, output) + self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start"]) + self.assertIn("emulator restarted before the case: adb state is offline", self.status("a")["notes"]) + self.assertEqual(self.status("b")["notes"], []) + self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b"]) + + def test_the_restart_switch_cold_boots_before_every_case_but_the_first(self): + status, _ = self.run_shard([case("a"), case("b"), case("c")], + env={"MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE": "1"}) + self.assertEqual(status, 0) + self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start", "stop", "start"]) + self.assertEqual(sum(1 for call in self.log("adb.log") if call[:1] == ["install"]), 3) + + def test_an_emulator_that_does_not_come_back_leaves_named_rows(self): + status, output = self.run_shard([case("a"), case("b"), case("c")], scenario={ + "replay": {"a": [{"exit": 75}]}, "avd_start_exits": [2, 2]}) + self.assertEqual(status, 1) + self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a"]) + self.assertEqual([self.status(name)["status"] for name in "abc"], ["failed", "not-run", "not-run"]) + self.assertEqual(output.count("::error title=Android retrace"), 4) + for name in "abc": + self.assertTrue((self.results / f"{name}-DirectVulkan" / "lane.json").is_file()) + + def test_an_apk_that_does_not_install_runs_nothing(self): + status, _ = self.run_shard([case("a"), case("b")], scenario={"failing_installs": 9}) + self.assertEqual(status, 1) + self.assertEqual(self.log("replay.log"), []) + self.assertEqual([self.status(name)["status"] for name in "ab"], ["not-run", "not-run"]) + + def test_processes_left_after_force_stop_are_killed_and_recorded(self): + # Listed on all four looks after the force-stop, gone after the kill. + status, _ = self.run_shard([case("a"), case("b")], scenario={"ps_calls_with_leftover": 4}) + self.assertEqual(status, 0) + self.assertIn(["shell", "run-as", "top.mobilegl.plugin.trace", "kill", "-9", "4242"], self.log("adb.log")) + self.assertIn("processes left after force-stop: 4242 libMobileGLServer.so", self.status("a")["notes"]) + self.assertEqual(self.status("b")["notes"], []) + self.assertEqual(self.log("avd.log"), []) + + def test_a_process_that_is_still_exiting_is_not_reported(self): + status, _ = self.run_shard([case("a")], scenario={"ps_calls_with_leftover": 2}) + self.assertEqual(status, 0) + self.assertEqual(self.status("a")["notes"], []) + self.assertNotIn("kill", [call[3] for call in self.log("adb.log") if call[:2] == ["shell", "run-as"]]) + + def test_processes_that_survive_the_kill_cost_an_emulator_restart(self): + status, _ = self.run_shard([case("a")], scenario={"ps_calls_with_leftover": 10}) + self.assertEqual(status, 0) + self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start"]) + self.assertIn("the emulator was restarted to remove them", self.status("a")["notes"]) + + def test_a_failed_pm_clear_falls_back_to_a_reinstall(self): + status, _ = self.run_shard([case("a")], scenario={"pm_clear_fails": True}) + self.assertEqual(status, 0) + adb = self.log("adb.log") + self.assertIn(["uninstall", "top.mobilegl.plugin.trace"], adb) + self.assertEqual(sum(1 for call in adb if call[:1] == ["install"]), 2) + + @unittest.skipUnless(os.name == "posix", "signals and process groups") + def test_a_terminated_runner_takes_the_running_replay_with_it(self): + cases = [case("a"), case("b")] + (self.state / "scenario.json").write_text(json.dumps({"replay": {"a": [{"sleep": 60}]}})) + arguments = ["--apk-file", str(self.apk), "--backend", "DirectVulkan", "--gpu", "lavapipe", + "--transport", "monolith", "--lane", "monolith-lane", "--cases-json", json.dumps(cases), + "--result-root", str(self.results), "--fixture-root", str(self.root / "work")] + runner = subprocess.Popen([sys.executable, str(HERE / "trace_replay_shard.py")] + arguments, + env={**os.environ, **self.environment}, stdout=subprocess.DEVNULL) + deadline = time.monotonic() + 30 + while not (self.state / "replay.pid").exists() and time.monotonic() < deadline: + time.sleep(0.05) + replay_pid = int((self.state / "replay.pid").read_text()) + runner.send_signal(signal.SIGTERM) + self.assertEqual(runner.wait(timeout=30), 128 + signal.SIGTERM) + for _ in range(100): + try: + os.kill(replay_pid, 0) + except OSError: + break + time.sleep(0.05) + else: + self.fail("the replay outlived the runner") + # The step that runs afterwards names both cases. + status, _ = self.run_shard(cases, "--finalize") + self.assertEqual(status, 0) + self.assertEqual([self.status(name)["status"] for name in "ab"], ["not-run", "not-run"]) + + def test_finalize_names_every_case_without_a_device(self): + cases = [case("a", golden=self.fixture("a.png")), case("b")] + status, _ = self.run_shard(cases) + self.assertEqual(status, 0) + (self.results / "b-DirectVulkan" / "case-status.json").unlink() + calls_before = len(self.log("adb.log")) + status, _ = self.run_shard(cases + [case("c")], "--finalize") + self.assertEqual(status, 0) + self.assertEqual(len(self.log("adb.log")), calls_before) + self.assertEqual([self.status(name)["status"] for name in "abc"], ["passed", "not-run", "not-run"]) + self.assertTrue((self.results / "c-DirectVulkan" / "lane.json").is_file()) + self.assertFalse(self.shard_status()["complete"]) + self.assertEqual(self.shard_status()["expected_cases"], ["a", "b", "c"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/android-plugin/trace-replay-ci.sh b/android-plugin/trace-replay-ci.sh index f5c751e4..a21d26fa 100644 --- a/android-plugin/trace-replay-ci.sh +++ b/android-plugin/trace-replay-ci.sh @@ -62,6 +62,9 @@ copies benchmark.json (per-frame times plus mean/median/p95) out of the app, and "passed" only means the replay reached the end of the trace without an error. Pass --reuse-fixture to skip re-extracting and re-pushing the trace, for repeat runs of a case whose fixture is already in /data/local/tmp. +Set MOBILEGL_TRACE_POLL_INTERVAL to the seconds between result polls (for example +0.5) to wait on a wall clock: the wait then ends --timeout-seconds plus a fifth after +the app starts, which is about how long that many one-second polls take. Pass --env "K=V;K=V" (or set MOBILEGL_TRACE_ENV) to hand arbitrary environment variables to the replay process. They are applied last, immediately before libMobileGL.so is loaded, so they override every flag above; an entry with no "=" @@ -93,6 +96,21 @@ adb_device_path() { MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL='*' "${ADB}" "$@" } +# Milliseconds since the epoch; whole seconds where date has no %N. +now_ms() { + stamp="$(date +%s%3N 2>/dev/null || true)" + case "${stamp}" in + ''|*[!0-9]*) stamp="$(($(date +%s) * 1000))" ;; + esac + printf '%s' "${stamp}" +} + +# Appends " " to phase-timings.txt in the result directory, so a run records where +# its wall time went: push, install, replay, log pulls. +mark_phase() { + printf '%s %s\n' "$1" "$(now_ms)" >> "${result_dir}/phase-timings.txt" 2>/dev/null || true +} + next_arg() { if [ "$#" -lt 2 ]; then die "$1 requires a value" @@ -411,6 +429,81 @@ copy_fixture_to_app() { fi } +# Waits for result.json with one poll per second; --timeout-seconds counts the polls. +wait_for_result_by_count() { + # One failed pidof is not an exit: under memory pressure the adb shell call + # itself can fail while the app is alive (seen with the process still in + # pidof and the activity still initializing right after the "exit"). + missed_app_polls=0 + for _ in $(seq 1 "${timeout_seconds}"); do + if adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then + break + fi + if adb_device_path shell pidof "${package_name}" >/dev/null 2>&1; then + saw_app_process=1 + missed_app_polls=0 + else + poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)" + if [ "${poll_adb_state}" != "device" ]; then + printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt" + break + fi + if [ "${saw_app_process}" -eq 1 ]; then + missed_app_polls=$((missed_app_polls + 1)) + if [ "${missed_app_polls}" -ge 3 ]; then + app_exited=1 + break + fi + fi + fi + sleep 1 + done +} + +# The same wait on a wall clock, selected by MOBILEGL_TRACE_POLL_INTERVAL (seconds between +# polls, fractions allowed). One adb call per poll asks for result.json and for the app +# process. The budget is --timeout-seconds of wall time plus a fifth: a poll of the +# one-second loop takes about 1.2 s with its two adb calls, so a replay gets the same time +# to finish under either loop. An exit needs three missed polls spread over at least three +# seconds, so a short interval keeps the one-second loop's tolerance for adb calls that fail +# under memory pressure. +wait_for_result_by_clock() { + poll_deadline=$((poll_started + timeout_seconds + timeout_seconds / 5)) + missed_app_polls=0 + first_missed_at=0 + while [ "$(date +%s)" -lt "${poll_deadline}" ]; do + poll_output="$(adb_device_path shell "run-as ${package_name} ls ${app_dir}/output/result.json >/dev/null 2>&1 && echo mobilegl-poll-result; pidof ${package_name} >/dev/null 2>&1 && echo mobilegl-poll-alive; echo mobilegl-poll-end" 2>/dev/null | tr -d '\r' || true)" + case "${poll_output}" in + *mobilegl-poll-result*) + break + ;; + *mobilegl-poll-alive*) + saw_app_process=1 + missed_app_polls=0 + ;; + *) + poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)" + if [ "${poll_adb_state}" != "device" ]; then + printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt" + break + fi + if [ "${saw_app_process}" -eq 1 ]; then + poll_now="$(date +%s)" + if [ "${missed_app_polls}" -eq 0 ]; then + first_missed_at="${poll_now}" + fi + missed_app_polls=$((missed_app_polls + 1)) + if [ "${missed_app_polls}" -ge 3 ] && [ "$((poll_now - first_missed_at))" -ge 3 ]; then + app_exited=1 + break + fi + fi + ;; + esac + sleep "${MOBILEGL_TRACE_POLL_INTERVAL}" + done +} + run_retrace() { result_dir="${result_root}/${safe_case}-${backend}" alternate_golden_app_path="" @@ -427,13 +520,17 @@ run_retrace() { fi mkdir -p "${result_dir}" + printf 'script-started %s\n' "${script_started_ms}" > "${result_dir}/phase-timings.txt" 2>/dev/null || true + mark_phase fixture-pushed # A repeat run of a case the previous invocation already installed and pushed only needs # the on-device copy back into a fresh app output directory. # MOBILEGL_TRACE_SKIP_INSTALL=1: the caller installed the APK itself (ColorOS shows an install-confirmation - # dialog whose tap drops adb for a few seconds, so the install has to be done and settled outside a run). + # dialog whose tap drops adb for a few seconds, so the install has to be done and settled outside a run; + # trace-replay-shard.sh installs once per emulator boot). if [ "${reuse_fixture}" -eq 0 ] && [ "${MOBILEGL_TRACE_SKIP_INSTALL:-0}" != "1" ]; then "${ADB}" install -r "$(host_path_for_adb "${apk_file}")" fi + mark_phase apk-installed copy_fixture_to_app adb_device_path shell am force-stop "${package_name}" "${ADB}" logcat -c @@ -502,39 +599,20 @@ run_retrace() { --ei crop_width "${crop_width}" \ --ei crop_height "${crop_height}" adb_device_path shell "$@" + mark_phase app-started app_exited=0 saw_app_process=0 - # One failed pidof is not an exit: under memory pressure the adb shell call - # itself can fail while the app is alive (seen with the process still in - # pidof and the activity still initializing right after the "exit"). - missed_app_polls=0 poll_started="$(date +%s)" - for _ in $(seq 1 "${timeout_seconds}"); do - if adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then - break - fi - if adb_device_path shell pidof "${package_name}" >/dev/null 2>&1; then - saw_app_process=1 - missed_app_polls=0 - else - poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)" - if [ "${poll_adb_state}" != "device" ]; then - printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt" - break - fi - if [ "${saw_app_process}" -eq 1 ]; then - missed_app_polls=$((missed_app_polls + 1)) - if [ "${missed_app_polls}" -ge 3 ]; then - app_exited=1 - break - fi - fi - fi - sleep 1 - done + if [ -n "${MOBILEGL_TRACE_POLL_INTERVAL:-}" ]; then + wait_for_result_by_clock + else + wait_for_result_by_count + fi + mark_phase wait-ended collect_run_diagnostics "${result_dir}" + mark_phase diagnostics-collected if ! adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then if [ -f "${result_dir}/adb-disconnected.txt" ]; then echo "trace-replay-ci.sh: device disconnected while waiting for result.json" >&2 @@ -574,6 +652,7 @@ run_retrace() { copy_app_artifact "${app_dir}/output/benchmark.json" "${result_dir}/benchmark.json" fi copy_texture_2d_dumps + mark_phase artifacts-copied # A replay that wrote result.json but did not pass used to print nothing but # the JSON, which for a non-zero statusCode says only "retrace failed with @@ -696,6 +775,7 @@ PY fi } +script_started_ms="$(now_ms)" mkdir -p "${fixture_root}" "${result_root}" if [ "${reuse_fixture}" -eq 0 ]; then prepare_fixture diff --git a/android-plugin/trace-replay-shard.sh b/android-plugin/trace-replay-shard.sh new file mode 100644 index 00000000..384f1670 --- /dev/null +++ b/android-plugin/trace-replay-shard.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# MobileGL - android-plugin/trace-replay-shard.sh +# Copyright (c) 2025-2026 MobileGL-Dev +# Licensed under the GNU Lesser General Public License v3.0: +# https://www.gnu.org/licenses/gpl-3.0.txt +# https://www.gnu.org/licenses/lgpl-3.0.txt +# SPDX-License-Identifier: LGPL-3.0-only +# End of Source File Header + +# Replays an ordered list of trace cases for one (lane, transport, backend) on one emulator +# boot, each through trace-replay-ci.sh. The loop, the per-case settings and the verdict files +# are in trace_replay_shard.py; `--help` lists the arguments. +# +# sh android-plugin/trace-replay-shard.sh \ +# --apk-file FILE --backend DirectGLES|DirectVulkan --gpu GPU \ +# --transport monolith|inproc|spawn --lane LANE \ +# --cases-env VARIABLE | --cases-file FILE | --cases-json JSON \ +# [--deadline-seconds N] [--dry-run | --finalize] +# +# Environment: +# AVD_NAME, EMULATOR_LOG, EMULATOR_PID_FILE the emulator to restart when it stops answering +# MOBILEGL_GPU_HANG_BUDGET_MS forwarded to every replay (required) +# MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE 1 cold boots the emulator before every case +# MOBILEGL_SHARD_POLL_INTERVAL seconds between result polls (default 0.5); +# empty selects trace-replay-ci.sh's 1 s loop +set -eu + +exec "${PYTHON:-python3}" "$(dirname "$0")/trace_replay_shard.py" "$@" diff --git a/android-plugin/trace_replay_shard.py b/android-plugin/trace_replay_shard.py new file mode 100644 index 00000000..b5ec4b9c --- /dev/null +++ b/android-plugin/trace_replay_shard.py @@ -0,0 +1,792 @@ +#!/usr/bin/env python3 +# MobileGL - android-plugin/trace_replay_shard.py +# Copyright (c) 2025-2026 MobileGL-Dev +# Licensed under the GNU Lesser General Public License v3.0: +# https://www.gnu.org/licenses/gpl-3.0.txt +# https://www.gnu.org/licenses/lgpl-3.0.txt +# SPDX-License-Identifier: LGPL-3.0-only +# End of Source File Header + +"""Replay an ordered list of trace cases on one emulator boot. + +One shard is one (lane, transport, backend). Each case runs through trace-replay-ci.sh with +the per-case and per-transport settings of the APK workflow, in a fresh app process with +cleared app data. A failing case does not stop the shard: the case after it starts on a cold +boot, every case gets a verdict file, and the exit status is non-zero when a case failed or a +verdict is missing. + +Layout under --result-root (the directory the workflow uploads): + + -/ what trace-replay-ci.sh writes, plus lane.json, the golden + copies the summary renders, and case-status.json + -/first-attempt/ files of an attempt that was retried + shard-status.json every case of the shard with its verdict + diagnostics/ emulator logs kept across restarts +""" + +import argparse +import json +import os +import re +import shlex +import shutil +import signal +import subprocess +import sys +import time +from pathlib import Path + +INFRASTRUCTURE_FAILURE_EXIT_CODE = 75 +TIMEOUT_EXIT_CODE = 124 +BACKENDS = ("DirectGLES", "DirectVulkan") +TRANSPORTS = ("monolith", "inproc", "spawn") +PASSED, FAILED, NOT_RUN = "passed", "failed", "not-run" + +# Cases with settings of their own. Bliss needs the newer packaged ANGLE build and, on +# DirectGLES, sampler mipmap min-filter downgrading on ANGLE llvmpipe; iterationRP needs its +# SPIR-V repairs forced on. +BLISS_CASE = "minecraft-1.21.4-fabric-iris-bliss-in-world" +ITERATIONRP_CASE = "minecraft-1.21.4-fabric-iris-iterationrp-in-world" +DEFAULT_ANGLE_VARIANT = "ec889e6ea831" +BLISS_ANGLE_VARIANT = "90a62123d794" + +# The inproc and spawn lanes pass the same client settings, so they differ only in which +# process runs the server role. +SPLIT_CLIENT_SETTINGS = "MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1" + +REQUIRED_CASE_KEYS = ("name", "trace_archive", "trace_file", "golden", "target_call", "width", "height", + "crop_x", "crop_y", "crop_width", "crop_height", "timeout_seconds") +# Seconds trace-replay-ci.sh gets beyond the case's own budget for install, push and log pulls. +REPLAY_GRACE_SECONDS = 300 +BOOT_TIMEOUT_SECONDS = 300 +# Seconds a case may take around its replays: the app reset, the failure diagnostics and the +# cold boot that follows a case that did not pass. +CASE_OVERHEAD_SECONDS = 180 + + +def say(message): + print(message, flush=True) + + +def pause(seconds): + time.sleep(seconds) + + +def safe_case(name): + return re.sub(r"[^A-Za-z0-9._-]", "_", name) + + +def as_text(value): + if value is True: + return "true" + if value is False: + return "false" + return "" if value is None else str(value) + + +def is_true(value): + return value is True or value == "true" + + +def load_cases(text, backend): + """The case objects of the APK matrix, as text ready for the command line.""" + try: + # Numbers stay the literal text of the matrix, so 0.985 is passed on as 0.985. + cases = json.loads(text, parse_float=str, parse_int=str) + except json.JSONDecodeError as error: + raise ValueError(f"the case list is not JSON: {error}") from error + if not isinstance(cases, list) or not cases: + raise ValueError("the case list must be a non-empty JSON array of case objects") + names = set() + for index, case in enumerate(cases): + if not isinstance(case, dict): + raise ValueError(f"case {index} is not an object; pass the case objects of the APK matrix") + missing = [key for key in REQUIRED_CASE_KEYS if case.get(key) in (None, "")] + if missing: + raise ValueError(f"case {case.get('name', index)} lacks {', '.join(missing)}") + name = case["name"] + if name in names: + raise ValueError(f"case {name} is listed twice") + names.add(name) + backends = case.get("ci_backends") + if backends is not None and backend not in backends: + raise ValueError(f"case {name} does not run on {backend} (ci_backends: {backends})") + return cases + + +def needed_seconds(cases): + """Wall time in which every case of a shard can use its whole time budget. + + It covers each case running until its replay script is stopped, one infrastructure retry + that does so again for the longest budget, and CASE_OVERHEAD_SECONDS per case. A shard + deadline at or above this never stops a case that is within its own budget, unless more + than one case needs a retry as long as its first attempt. + """ + budgets = [int(as_text(case["timeout_seconds"])) + REPLAY_GRACE_SECONDS for case in cases] + return sum(budgets) + max(budgets) + CASE_OVERHEAD_SECONDS * len(budgets) + + +def replay_environment(backend, case_name): + return { + "MOBILEGL_ESPRYT_USE_ANGLE": "1" if backend == "DirectGLES" else "0", + "MOBILEGL_TRACE_ANGLE_VARIANT": BLISS_ANGLE_VARIANT if case_name == BLISS_CASE else DEFAULT_ANGLE_VARIANT, + "MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "1" if backend == "DirectVulkan" else "0", + "MOBILEGL_ITRP_FIX": "ForceOn" if case_name == ITERATIONRP_CASE else "Auto", + } + + +def transport_arguments(transport, hang_budget_ms): + hang_budget = f"MOBILEGL_GPU_HANG_BUDGET_MS={hang_budget_ms}" + if transport == "inproc": + return ["--env", f"MOBILEGL_TRANSPORT=inproc;{SPLIT_CLIENT_SETTINGS};{hang_budget}", "--require-inproc"] + if transport == "spawn": + # No MOBILEGL_IPC_SERVER_PATH: only the app can spell nativeLibraryDir, and + # TraceReplayActivity resolves it when the environment asks for spawn. The pbuffer is + # required: a spawned server cannot be handed the client's window. + return ["--env", f"MOBILEGL_TRANSPORT=spawn;{SPLIT_CLIENT_SETTINGS};{hang_budget}", + "--use-pbuffer", "--require-spawn"] + return ["--env", f"MOBILEGL_TRANSPORT=monolith;{hang_budget}"] + + +def replay_arguments(options, case, hang_budget_ms): + name = case["name"] + arguments = [ + "--apk-file", options.apk_file, + "--package", options.package, + "--backend", options.backend, + "--result-root", options.result_root, + "--fixture-root", options.fixture_root, + "--case", name, + "--trace-archive", as_text(case["trace_archive"]), + "--trace-file", as_text(case["trace_file"]), + "--golden", as_text(case["golden"]), + "--alternate-golden", as_text(case.get("alternate_golden") or ""), + "--target-call", as_text(case["target_call"]), + "--width", as_text(case["width"]), + "--height", as_text(case["height"]), + "--ssim-threshold", as_text(case.get("ssim_threshold") or "0.99"), + "--crop-x", as_text(case["crop_x"]), + "--crop-y", as_text(case["crop_y"]), + "--crop-width", as_text(case["crop_width"]), + "--crop-height", as_text(case["crop_height"]), + "--timeout-seconds", as_text(case["timeout_seconds"]), + ] + arguments += transport_arguments(options.transport, hang_budget_ms) + if options.backend == "DirectGLES" and name == BLISS_CASE: + arguments.append("--avoid-angle-llvmpipe-sampler-mipmap-min-filter") + if options.backend == "DirectGLES" and is_true(case.get("avoid_angle_llvmpipe_explicit_lod_bias")): + arguments.append("--avoid-angle-llvmpipe-explicit-lod-bias") + if is_true(case.get("coherent_as_flush")): + arguments.append("--coherent-as-flush") + return arguments + + +def read_json(path): + try: + return json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + + +def write_json(path, value): + Path(path).write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + + +def one_line(text): + return " ".join(str(text).split()) + + +class Shard: + def __init__(self, options, cases, environ): + self.options = options + self.cases = cases + self.environ = dict(environ) + self.adb_command = shlex.split(environ.get("ADB", "adb")) + directory = Path(__file__).resolve().parent + self.replay_command = shlex.split(environ.get("MOBILEGL_SHARD_REPLAY_COMMAND") or + f"sh {shlex.quote(str(directory / 'trace-replay-ci.sh'))}") + self.avd_command = shlex.split(environ.get("MOBILEGL_SHARD_AVD_COMMAND") or + f"sh {shlex.quote(str(directory / 'run-avd-ci.sh'))}") + self.hang_budget_ms = environ.get("MOBILEGL_GPU_HANG_BUDGET_MS") + self.restart_each_case = environ.get("MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE", "0") == "1" + self.poll_interval = environ.get("MOBILEGL_SHARD_POLL_INTERVAL", "0.5") + self.avd_name = environ.get("AVD_NAME", "") + self.emulator_log = environ.get("EMULATOR_LOG", "") + self.emulator_pid_file = environ.get("EMULATOR_PID_FILE", "") + self.result_root = Path(options.result_root) + self.diagnostics = self.result_root / "diagnostics" + self.started = time.monotonic() + self.boots = 1 + self.install_seconds = [] + self.records = {} + self.aborted = "" + # The case before the current one when it did not pass; the next case gets a cold boot. + self.unclean_case = "" + + # ---- bookkeeping ------------------------------------------------------------------- + + def result_dir(self, case): + return self.result_root / f"{safe_case(case['name'])}-{self.options.backend}" + + def record_for(self, case, status, reason, **extra): + record = {"case": case["name"], "backend": self.options.backend, "transport": self.options.transport, + "lane": self.options.lane, "status": status, "reason": reason} + record.update(extra) + return record + + def write_summary_inputs(self, case): + """lane.json and the golden copies the retrace summary reads next to each result.""" + directory = self.result_dir(case) + directory.mkdir(parents=True, exist_ok=True) + write_json(directory / "lane.json", {"lane": self.options.lane, "transport": self.options.transport}) + stem = f"{safe_case(case['name'])}-{self.options.backend}" + for key, suffix in (("golden", "golden"), ("alternate_golden", "alternate-golden")): + source = as_text(case.get(key) or "") + if source and os.path.isfile(source) and os.path.getsize(source) > 0: + shutil.copyfile(source, directory / f"{stem}-{suffix}.png") + + def save_record(self, case, record): + self.records[case["name"]] = record + self.write_summary_inputs(case) + write_json(self.result_dir(case) / "case-status.json", record) + self.write_shard_status() + + def write_shard_status(self, complete=False): + self.result_root.mkdir(parents=True, exist_ok=True) + records = [self.records.get(case["name"]) or + self.record_for(case, NOT_RUN, "the shard has not reached this case") for case in self.cases] + write_json(self.result_root / "shard-status.json", { + "schema": 1, "lane": self.options.lane, "transport": self.options.transport, + "backend": self.options.backend, "complete": complete, + # The Actions job that ran the shard, so the artifact cleanup can tell whose job failed. + "job_id": self.environ.get("MOBILEGL_SHARD_JOB_ID", ""), + "expected_cases": [case["name"] for case in self.cases], "cases": records, + "emulator_boots": self.boots, "install_seconds": self.install_seconds}) + + def remaining_seconds(self): + if self.options.deadline_seconds <= 0: + return None + return self.options.deadline_seconds - (time.monotonic() - self.started) + + def deadline_passed(self): + remaining = self.remaining_seconds() + return remaining is not None and remaining <= 0 + + # ---- processes --------------------------------------------------------------------- + + def run(self, command, timeout=None, capture=False, extra_env=None): + """Returns (exit status, output). A timeout kills the whole process group and reads as 124.""" + environment = dict(self.environ) + environment.update(extra_env or {}) + own_group = os.name == "posix" + try: + process = subprocess.Popen(command, env=environment, start_new_session=own_group, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.STDOUT if capture else None) + except OSError as error: + return 127, str(error) + def kill(): + if own_group: + try: + os.killpg(process.pid, signal.SIGKILL) + except OSError: + pass + process.kill() + + try: + output, _ = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + kill() + output, _ = process.communicate() + return TIMEOUT_EXIT_CODE, (output or b"").decode("utf-8", "replace") + except BaseException: + # The job was canceled or timed out. The child is in a session of its own, so it + # would otherwise keep driving the device behind the steps that still run. + kill() + raise + return process.returncode, (output or b"").decode("utf-8", "replace") + + def adb(self, *arguments, timeout=60): + status, output = self.run(self.adb_command + list(arguments), timeout=timeout, capture=True) + return status, output.replace("\r", "") + + # ---- emulator ---------------------------------------------------------------------- + + def can_restart(self): + return bool(self.avd_name and self.options.gpu and self.emulator_log and self.emulator_pid_file) + + def emulator_pid(self): + try: + return int(Path(self.emulator_pid_file).read_text().strip()) + except (OSError, ValueError): + return None + + def emulator_process_alive(self): + pid = self.emulator_pid() + if pid is None: + return True # nothing to check against, for example a real device + try: + os.kill(pid, 0) + except OSError: + return False + return True + + def health_problem(self): + """Empty when the device answers; otherwise what did not.""" + if not self.emulator_process_alive(): + return "the emulator process is gone" + status, output = self.adb("get-state", timeout=15) + if status != 0 or output.strip() != "device": + return f"adb state is {output.strip() or 'unknown'}" + status, output = self.adb("shell", "getprop", "sys.boot_completed", timeout=15) + if status != 0 or output.strip() != "1": + return "sys.boot_completed is not 1" + # A framework call: boot_completed stays 1 across a system_server restart. + status, output = self.adb("shell", "pm", "path", self.options.package, timeout=30) + if status != 0 or "package:" not in output: + return "the package manager does not list the trace APK" + return "" + + def keep_emulator_log(self, label): + if self.emulator_log and os.path.isfile(self.emulator_log): + self.diagnostics.mkdir(parents=True, exist_ok=True) + shutil.copyfile(self.emulator_log, self.diagnostics / f"emulator-boot-{self.boots}-{label}.log") + + def prepare_booted_device(self): + # A shard outlives the screen timeout a single replay never reached. + self.adb("shell", "svc", "power", "stayon", "true", timeout=30) + return self.install() + + def install(self): + started = time.monotonic() + status, output = self.adb("install", "-r", self.options.apk_file, timeout=300) + self.install_seconds.append(round(time.monotonic() - started, 1)) + say(output.strip()) + return status == 0 + + def restart_emulator(self, reason): + """Stop, cold boot and reinstall. False when the emulator did not come back.""" + if not self.can_restart(): + say(f"trace-replay-shard: cannot restart the emulator ({reason}): no AVD is configured") + return False + say(f"trace-replay-shard: restarting the emulator: {reason}") + common = ["--avd-name", self.avd_name, "--emulator-log", self.emulator_log, + "--pid-file", self.emulator_pid_file] + for attempt in (1, 2): + # The start truncates the log, and the boot that failed is the one worth reading. + self.keep_emulator_log(f"attempt-{attempt}" if attempt > 1 else "ended") + old_pid = self.emulator_pid() + self.run(self.avd_command + ["stop"] + common, timeout=120) + self.wait_for_exit(old_pid) + self.adb("kill-server", timeout=30) + pause(2) + self.boots += 1 + status, _ = self.run(self.avd_command + ["start"] + common + + ["--gpu", self.options.gpu, "--boot-timeout", str(BOOT_TIMEOUT_SECONDS)], + timeout=2 * BOOT_TIMEOUT_SECONDS + 60) + if status == 0 and self.prepare_booted_device(): + return True + say(f"trace-replay-shard: emulator restart attempt {attempt} failed") + return False + + def wait_for_exit(self, pid): + if pid is None or os.name != "posix": + return + for _ in range(60): + try: + os.kill(pid, 0) + except OSError: + return + pause(0.5) + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + + # ---- app state --------------------------------------------------------------------- + + def leftover_processes(self): + """Processes still running under the app's uid, as 'pid name' strings.""" + status, user = self.adb("shell", "run-as", self.options.package, "id", "-un", timeout=30) + user = user.strip() + if status != 0 or not re.fullmatch(r"[A-Za-z0-9_]+", user): + return None + status, listing = self.adb("shell", "ps", "-A", "-o", "USER,PID,NAME", timeout=30) + if status != 0: + return None + found = [] + for line in listing.splitlines(): + fields = line.split() + if len(fields) >= 3 and fields[0] == user and fields[1].isdigit(): + found.append(f"{fields[1]} {fields[2]}") + return found + + def settled_leftover_processes(self, polls): + """leftover_processes once the app's uid is empty, or as it still is after `polls` looks. + + A killed process stays listed for a moment, so one look right after the kill would + report every force-stop as incomplete. + """ + found = self.leftover_processes() + for _ in range(polls - 1): + if not found: + break + pause(0.5) + found = self.leftover_processes() + return found + + def reset_app_state(self): + """Fresh process and fresh app data, as a new install gives. Returns notes for the record.""" + notes = [] + package = self.options.package + self.adb("shell", "am", "force-stop", package, timeout=60) + leftovers = self.settled_leftover_processes(polls=4) + if leftovers is None: + notes.append("the app's processes could not be listed after force-stop") + if leftovers: + # A force-stop is expected to take a spawned server with it; say so when it did not. + notes.append("processes left after force-stop: " + ", ".join(leftovers)) + self.adb("shell", "run-as", package, "kill", "-9", *[entry.split()[0] for entry in leftovers], + timeout=30) + if self.settled_leftover_processes(polls=6): + notes.append("the leftover processes could not be killed") + if self.restart_emulator("app processes survived force-stop and kill"): + notes.append("the emulator was restarted to remove them") + status, output = self.adb("shell", "pm", "clear", package, timeout=60) + probe_status, _ = self.adb("shell", "run-as", package, "ls", f"/data/user/0/{package}", timeout=30) + if status != 0 or "Success" not in output or probe_status != 0: + # The state a fresh install leaves, reached the other way. + notes.append("pm clear did not leave a usable app data directory; reinstalled the APK") + self.adb("uninstall", package, timeout=120) + self.install() + return notes + + def remove_case_leftovers(self, case): + name = safe_case(case["name"]) + base = f"/data/local/tmp/mobilegl-{name}" + self.adb("shell", "rm", "-f", f"{base}.trace", f"{base}.golden.png", f"{base}.alternate-golden.png", + timeout=60) + self.adb("shell", "am", "force-stop", self.options.package, timeout=60) + shutil.rmtree(Path(self.options.fixture_root) / name, ignore_errors=True) + + # ---- one case ---------------------------------------------------------------------- + + def reason_lines(self): + try: + return (self.result_root / "infrastructure-failure-reason.txt").read_text().splitlines() + except OSError: + return [] + + def run_attempt(self, case): + """Returns (exit status, seconds, how the shard deadline cut the attempt short or "").""" + case_budget = int(as_text(case["timeout_seconds"])) + REPLAY_GRACE_SECONDS + budget = case_budget + remaining = self.remaining_seconds() + if remaining is not None and remaining < case_budget: + budget = max(remaining, 1.0) + environment = replay_environment(self.options.backend, case["name"]) + # The shard installs the APK once per boot. + environment["MOBILEGL_TRACE_SKIP_INSTALL"] = "1" + environment["MOBILEGL_TRACE_POLL_INTERVAL"] = self.poll_interval + started = time.monotonic() + status, _ = self.run(self.replay_command + replay_arguments(self.options, case, self.hang_budget_ms), + timeout=budget, extra_env=environment) + elapsed = round(time.monotonic() - started, 1) + cut = "" + if status == TIMEOUT_EXIT_CODE and budget < case_budget: + # The case never had its own budget, so the timeout says nothing about the case. + cut = f"cut by the shard deadline after {elapsed:.0f} s of a {case_budget} s budget" + return status, elapsed, cut + + def keep_first_attempt(self, case): + """Move the retried attempt's files aside so the retry starts from an empty directory. + + result.json is renamed: the summary keys rows on any result.json below a directory + that names the case, and the retry's is the one that counts. + """ + directory = self.result_dir(case) + if not directory.is_dir(): + return + kept = directory / "first-attempt" + kept.mkdir(exist_ok=True) + for entry in directory.iterdir(): + if entry.is_file(): + name = "result.first-attempt.json" if entry.name == "result.json" else entry.name + entry.replace(kept / name) + + def verdict(self, case, status): + directory = self.result_dir(case) + if status == 0: + result = read_json(directory / "result.json") + if not isinstance(result, dict) or result.get("passed") is not True: + return FAILED, "the replay exited 0 without a passed result.json" + if self.options.transport != "monolith": + proof = read_json(directory / "transport-proof.json") + if not (isinstance(proof, dict) and proof.get("passed") is True and + proof.get("required_transport") == self.options.transport): + return FAILED, f"the replay exited 0 without a passed {self.options.transport} transport proof" + return PASSED, "" + if status == INFRASTRUCTURE_FAILURE_EXIT_CODE: + return FAILED, "the emulator infrastructure failed again on the retry" + if status == TIMEOUT_EXIT_CODE: + return FAILED, "the replay script exceeded its time budget" + result = read_json(directory / "result.json") + if not isinstance(result, dict): + return FAILED, f"exit {status} without a result.json" + if result.get("passed") is True: + # The picture matched, so what failed is a check that runs after it. + return FAILED, f"exit {status} after a passed result.json: the transport proof or a log check failed" + return FAILED, f"exit {status}: {one_line(result.get('message', ''))}" + + def collect_failure_diagnostics(self, case): + directory = self.result_dir(case) + directory.mkdir(parents=True, exist_ok=True) + logcat = directory / "logcat.txt" + if not logcat.is_file() or logcat.stat().st_size == 0: + # trace-replay-ci.sh pulls this itself unless it was killed first. + _, output = self.adb("logcat", "-d", "-t", "20000", timeout=30) + logcat.write_text(output, encoding="utf-8", errors="replace") + app_output = f"/data/user/0/{self.options.package}/files/trace-replay/output" + for name in ("retrace.log", "mobilegl.client.log", "mobilegl.server.log"): + status, output = self.adb("exec-out", "run-as", self.options.package, "cat", + f"{app_output}/{name}", timeout=30) + if status == 0 and output and not output.startswith("cat: "): + (directory / name).write_text(output, encoding="utf-8", errors="replace") + if self.emulator_log and os.path.isfile(self.emulator_log): + shutil.copyfile(self.emulator_log, directory / "emulator.log") + # A vanished emulator looks the same whether the host OOM killer took it or the + # renderer faulted. These two say which. + _, memory = self.run(["free", "-h"], timeout=30, capture=True) + (directory / "host-memory.txt").write_text(memory, encoding="utf-8") + _, dmesg = self.run(["sudo", "-n", "dmesg", "-T"], timeout=30, capture=True) + (directory / "host-dmesg.txt").write_text("\n".join(dmesg.splitlines()[-300:]) + "\n", encoding="utf-8") + + def run_case(self, index, case): + name = case["name"] + notes = [] + seconds = {} + case_started = time.monotonic() + + if self.deadline_passed(): + return self.record_for(case, NOT_RUN, "the shard deadline passed before this case started") + + restarted_for = "" + if self.restart_each_case and index > 0: + restarted_for = "MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE=1" + elif self.unclean_case and self.can_restart(): + # A crashed, hung or killed replay can leave the guest or the host renderer in a + # state no adb query shows, and this case's verdict must not depend on it. + restarted_for = f"the case before it, {self.unclean_case}, did not pass" + else: + restarted_for = self.health_problem() + if restarted_for: + notes.append(f"emulator restarted before the case: {restarted_for}") + if not self.restart_emulator(restarted_for): + self.aborted = "the emulator could not be restarted" + return self.record_for(case, NOT_RUN, self.aborted, notes=notes) + + notes += self.reset_app_state() + seconds["reset"] = round(time.monotonic() - case_started, 1) + + reasons_before = len(self.reason_lines()) + exit_codes = [] + replay_seconds = [] + status, elapsed, cut = self.run_attempt(case) + exit_codes.append(status) + replay_seconds.append(elapsed) + surface_lost = 0 + if status == INFRASTRUCTURE_FAILURE_EXIT_CODE: + say(f"::warning::Android emulator infrastructure failed; restarting it and retrying {name} once.") + new_reasons = self.reason_lines()[reasons_before:] + # A lost surface is retried, so it is counted where a rate spike shows. + surface_lost = sum(1 for line in new_reasons if "angle-surface-lost" in line) + if surface_lost: + self.append_step_summary(f"surface-lost retries: {surface_lost} ({self.options.backend}, {name})\n") + notes.append("infrastructure retry: " + (", ".join(new_reasons) or "no reason recorded")) + self.keep_first_attempt(case) + if not self.restart_emulator(f"infrastructure failure in {name}"): + self.aborted = "the emulator could not be restarted" + elif self.deadline_passed(): + cut = "the shard deadline passed before the infrastructure retry could start" + else: + notes += self.reset_app_state() + status, elapsed, cut = self.run_attempt(case) + exit_codes.append(status) + replay_seconds.append(elapsed) + seconds["replay"] = replay_seconds + + finish_started = time.monotonic() + if self.aborted: + verdict, reason = FAILED, "the emulator did not come back for the infrastructure retry" + elif cut: + verdict, reason = NOT_RUN, cut + else: + verdict, reason = self.verdict(case, status) + if verdict != PASSED: + self.collect_failure_diagnostics(case) + if not self.aborted: + self.remove_case_leftovers(case) + seconds["finish"] = round(time.monotonic() - finish_started, 1) + seconds["total"] = round(time.monotonic() - case_started, 1) + return self.record_for(case, verdict, reason, attempts=len(exit_codes), exit_codes=exit_codes, + surface_lost_retries=surface_lost, seconds=seconds, notes=notes) + + # ---- the shard --------------------------------------------------------------------- + + def append_step_summary(self, text): + path = self.environ.get("GITHUB_STEP_SUMMARY") + if path: + with open(path, "a", encoding="utf-8") as summary: + summary.write(text) + + def run_all(self): + self.write_shard_status() + say(f"trace-replay-shard: {len(self.cases)} case(s) on {self.options.backend} / " + f"{self.options.transport} ({self.options.lane})") + if not self.prepare_booted_device(): + if not self.restart_emulator("the trace APK did not install"): + self.aborted = "the trace APK could not be installed" + for index, case in enumerate(self.cases): + say(f"::group::{case['name']} ({index + 1}/{len(self.cases)})") + if self.aborted: + record = self.record_for(case, NOT_RUN, self.aborted) + else: + try: + record = self.run_case(index, case) + except Exception as error: # one case's bookkeeping must not cost the others their verdicts + record = self.record_for(case, FAILED, f"the shard runner failed: {one_line(error)}") + self.unclean_case = "" if record["status"] == PASSED else case["name"] + self.save_record(case, record) + say(f"trace-replay-shard: {case['name']}: {record['status']}" + + (f" ({record['reason']})" if record["reason"] else "")) + say("::endgroup::") + return self.finish() + + def finish(self): + """Tally from the files on disk, so a verdict the loop never wrote counts as missing.""" + label = f"{self.options.backend} / {self.options.transport}" + rows = [] + verdicts = 0 + failed = [] + for case in self.cases: + record = read_json(self.result_dir(case) / "case-status.json") + if not isinstance(record, dict) or record.get("case") != case["name"]: + record = self.record_for(case, NOT_RUN, "no verdict file was written") + if record["status"] in (PASSED, FAILED): + verdicts += 1 + if record["status"] == PASSED: + result = read_json(self.result_dir(case) / "result.json") + if not isinstance(result, dict) or result.get("passed") is not True: + record = dict(record, status=FAILED, reason="the verdict is passed but result.json is not") + if record["status"] != PASSED: + failed.append(record) + result = read_json(self.result_dir(case) / "result.json") + ssim = result.get("ssim", "") if isinstance(result, dict) else "" + seconds = record.get("seconds", {}).get("total", "") + note = one_line("; ".join([record.get("reason", "")] + record.get("notes", [])).strip("; ")) + rows.append(f"| {case['name']} | {record['status']} | {record.get('attempts', 0)} | {seconds} | " + f"{ssim} | {note.replace('|', '/')} |") + self.write_shard_status(complete=verdicts == len(self.cases)) + + summary = [f"### Android retrace: {label} ({self.options.lane})", "", + f"{len(self.cases) - len(failed)} of {len(self.cases)} passed, {self.boots} emulator boot(s).", "", + "| case | status | attempts | seconds | SSIM | note |", "| --- | --- | --- | --- | --- | --- |"] + self.append_step_summary("\n".join(summary + rows) + "\n\n") + for record in failed: + say(f"::error title=Android retrace {record['status']} ({label})::" + f"{record['case']}: {record.get('reason') or record['status']}") + if verdicts != len(self.cases): + say(f"::error title=Android retrace shard incomplete ({label})::" + f"{verdicts} verdict(s) for {len(self.cases)} case(s)") + say(f"trace-replay-shard: {len(self.cases) - len(failed)} passed, {len(failed)} not passed, " + f"{verdicts} verdict(s) for {len(self.cases)} case(s)") + return 0 if not failed and verdicts == len(self.cases) else 1 + + def finalize(self): + """Give every case its summary inputs and a verdict file without touching the device. + + Runs after the replay step whatever its outcome, so a shard that was killed or never + started still uploads one named row per case. + """ + for case in self.cases: + record = read_json(self.result_dir(case) / "case-status.json") + if isinstance(record, dict) and record.get("case") == case["name"]: + self.records[case["name"]] = record + self.write_summary_inputs(case) + else: + self.save_record(case, self.record_for(case, NOT_RUN, "the shard did not reach this case")) + previous = read_json(self.result_root / "shard-status.json") + if isinstance(previous, dict): + self.boots = previous.get("emulator_boots", self.boots) + self.install_seconds = previous.get("install_seconds", self.install_seconds) + self.write_shard_status( + complete=all(self.records[case["name"]]["status"] in (PASSED, FAILED) for case in self.cases)) + return 0 + + def dry_run(self): + for case in self.cases: + say(json.dumps({ + "case": case["name"], + "environment": replay_environment(self.options.backend, case["name"]), + "arguments": replay_arguments(self.options, case, self.hang_budget_ms), + "timeout": int(as_text(case["timeout_seconds"])) + REPLAY_GRACE_SECONDS, + }, sort_keys=True)) + return 0 + + +def parse_arguments(argv): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--apk-file", required=True, help="Trace APK to install once per emulator boot.") + parser.add_argument("--package", default="top.mobilegl.plugin.trace") + parser.add_argument("--backend", required=True, choices=BACKENDS) + parser.add_argument("--transport", required=True, choices=TRANSPORTS) + parser.add_argument("--lane", required=True, help="Lane name written to lane.json, e.g. inproc-acceptance.") + parser.add_argument("--gpu", default="", help="Emulator -gpu mode, needed to restart the emulator.") + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--cases-json", help="JSON array of the APK matrix's case objects.") + source.add_argument("--cases-file", help="File holding that JSON array.") + source.add_argument("--cases-env", help="Name of the environment variable holding that JSON array.") + parser.add_argument("--result-root", default="android-retrace-result") + parser.add_argument("--fixture-root", default="android-retrace-fixture") + parser.add_argument("--deadline-seconds", type=int, default=0, + help="Stop starting cases, and cut a running one, this long after the start; both are " + "recorded as not-run. 0: no limit. retrace_shard_budget.py says what a shard needs.") + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--dry-run", action="store_true", + help="Print each case's environment and trace-replay-ci.sh arguments, run nothing.") + mode.add_argument("--finalize", action="store_true", + help="Only write the summary inputs and a not-run verdict for cases without one.") + return parser.parse_args(argv) + + +def main(argv=None): + options = parse_arguments(argv) + if options.cases_json is not None: + text = options.cases_json + elif options.cases_file is not None: + text = Path(options.cases_file).read_text(encoding="utf-8") + else: + text = os.environ.get(options.cases_env, "") + try: + cases = load_cases(text, options.backend) + except ValueError as error: + say(f"::error title=Android retrace shard misconfigured::{error}") + return 2 + shard = Shard(options, cases, os.environ) + if shard.hang_budget_ms is None: + say("::error title=Android retrace shard misconfigured::MOBILEGL_GPU_HANG_BUDGET_MS is not set") + return 2 + if options.dry_run: + return shard.dry_run() + if options.finalize: + return shard.finalize() + if not os.path.isfile(options.apk_file): + say(f"::error title=Android retrace shard misconfigured::APK does not exist: {options.apk_file}") + return 2 + # A termination request unwinds through Shard.run, which kills the running child first. + signal.signal(signal.SIGTERM, lambda number, frame: sys.exit(128 + number)) + return shard.run_all() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/ci/ci-cleanup-log.md b/docs/ci/ci-cleanup-log.md index 71d86aff..adc1ace0 100644 --- a/docs/ci/ci-cleanup-log.md +++ b/docs/ci/ci-cleanup-log.md @@ -64,6 +64,10 @@ them wrong. | 2026-10-18 | `Negative control A (corrupted snapshot)` and `Keep lane logs before controls` | Test / `integration-verify-split` | the first re-ran the ten `VerifySplit.ClearThenReadPixelsScenario` entries of both backends with the same knob and accepted any red; the second only kept the lane's logs from being overwritten by that rerun | the in-lane `VerifySplitCorrupted` and `VerifySplitReadCorrupted` entries of both backends, required by name (`Verify-split control entries`); that the ambient entries arm is the per-entry arm proof, and that they run fatal is `VerifySplitArming.PipeVerifyArmingScenario.Armed` (the ambient environment plus its marker and log), which requires `fatal=1` on the arming line | | 2026-10-18 | unit `PipeInputsTest.{PoisonOmitKnobArmsTheOmission, BadPoisonOmitKnobIsFatalNamingTheKnob}`, and the `MOBILEGL_PIPE_POISON_OMIT` knob: `Features.PipePoisonOmit`, its parser on every verb (`MGPipeValidateForVerb`'s `ParsePoisonOmissionKnob`, compiled into every build) and the server stamp's own parse of it | Test / unit (every poison lane); product code | the knob existed only for negative control B, and the two cases pinned its parser and its `Fatal{PipeVerifyBadKnob}` | the PoisonOmitted and VerifySplitPoisonOmitted entries set the harness marker `MGITEST_POISON_OMIT` to the same pair, and the re-executed child arms it through `MGPipeSetPoisonOmission`, compiled only with `-DMOBILEGL_PIPE_VERIFY=ON` (`Harness/PoisonOmissionArm.h`); `OmittingOneFieldForOneVerbLeavesExactlyThatFieldStale`, `ReadingAnOmittedFieldAbortsNamingTheVerb` and `TheMutationNoticeRefreshesTheValueButNotTheStamp` withhold the stamp test-side and still run in every poison lane; the setter's scope to its verb, not its class, is checked by `OmittingOneFieldForOneVerbLeavesExactlyThatFieldStale` in a forked child, on the verify runtime only | | 2026-10-18 | the `serial == resets` assertion of `CtWireScenario.ApplierResetCrossesAtThePrimedEdgeInSerialOrder`, and the `MOBILEGL_TEST_SERVER_COUNTERS` knob: `ServerMain`'s `MGPipe server counters:` line on every LogFlush and `ServerVerbSink`'s test-only tallies (`ApplierResets`, `ObjectDeaths`, `ExpectedApplierResetSerial`) | Test / `integration-split`, `integration-split-strict`, `integration-verify-split`, `integration (inproc)`; product code | the knob existed only so the spawn and tcp arms could read the sink's tallies; the serial assertion could not fail, since `OnApplierReset` moves the count and the serial together and any other serial latches `Fatal{ProtocolCorruption, "ApplierReset.ContextSerial"}` | `ar` / `od` on the server role's `MGPipe stats:` line, counted in `ServerVerbSink::OnApplierReset` / `OnObjectDeath`; every CtWire entry that counts them sets `MOBILEGL_PIPE_STATS=1` with period 1 and reads them on every arm, inproc included, over a window closed by a swap and a readback; the unit cases `ApplierResetWireRoundTrip.TheSerialSequenceIsAssertedAndTheServerResetRuns` and `ObjectDeathWireRoundTrip.TheTextureAndFramebufferTwinsReleaseByHandleAndNeverByAStaleGeneration` read the same counters, and the first shows the expected serial moving by having serial 2 accepted | +| 2026-11-15 | per-case jobs `retrace (, )`, `retrace verify (, )`, `retrace split (, , )` | Test / `retrace`, `retrace-verify`, `retrace-split` | one runner per case spent most of its time on setup; no case, knob, assertion or arm proof is dropped | shard jobs that replay the same rows one after another through `scripts/ci/retrace_shard.py`, one ctest entry and one verdict per case; `trace_cases.py --self-test-shard-matrices` checks that the shards hold exactly the per-case rows | +| 2026-11-15 | per-case emulator jobs `retrace (, )` of each lane | MobileGL APK / `retrace` | one emulator boot per replay; no case, setting or arm proof is dropped | shard jobs that replay the cases of one lane, transport and backend on one boot through `android-plugin/trace_replay_shard.py`, with app data cleared between cases, a cold boot after a case that did not pass, and one verdict per case | +| 2026-11-15 | per-case `trace fixture ()` jobs, their per-case cache entries and artifacts, `.github/scripts/trace-fixture-cache.sh` | Test and MobileGL APK / `trace-fixtures` | 40 jobs and 40 cache entries per workflow for files that change rarely | one content-addressed cache entry for all Git LFS fixtures (`.github/scripts/trace-fixture-bundle.sh`); a replay job restores it and downloads a case only when the entry does not hold it | +| 2026-11-15 | `remove-artifact-clutter` job | Test and MobileGL APK | it deleted the fixture artifacts and the green result artifacts after the run; the fixture artifacts no longer exist | the delete step at the end of `retrace-summary`; on Linux `retention-days: 1` on the summary inputs as well, `retrace-split` and `retrace-shipped` upload results only for a failed shard (their green results were deleted unread before), and `retrace-verify` keeps its results for every shard as before | ## Kept after investigation diff --git a/scripts/ci/retrace_shard.py b/scripts/ci/retrace_shard.py new file mode 100644 index 00000000..131db833 --- /dev/null +++ b/scripts/ci/retrace_shard.py @@ -0,0 +1,689 @@ +#!/usr/bin/env python3 +# MobileGL - scripts/ci/retrace_shard.py +# Copyright (c) 2025-2026 MobileGL-Dev +# Licensed under the GNU Lesser General Public License v3.0: +# https://www.gnu.org/licenses/gpl-3.0.txt +# https://www.gnu.org/licenses/lgpl-3.0.txt +# SPDX-License-Identifier: LGPL-3.0-only +# End of Source File Header + +"""Replay the cases of one retrace shard, one after another, and give each its own verdict. + +A shard is some of the cases of one (lane, transport, backend) cell, replayed by one CI job +(tools/trace_replay/trace_cases.py, shard_matrix). Each case runs through its own ctest entry +with the environment and the checks of its lane (case_plan), in a process group and a driver +cache directory of its own, so no case sees what another left behind. A failing case does not +stop the shard. The shard fails when a case failed, or when the verdicts it recorded are not +exactly the cases it was given. + +For every case it prints a log group, appends a row to the step summary and records a verdict +in /results/shard-results.json; a failed case also gets an error annotation. +The replay output the lane uploads is moved to /results//, the JUnit files +are in /junit/, and the extracted trace is deleted. + +Usage: + retrace_shard.py run --profile {retrace,split,verify} --backend B [--transport T] [--lane L] + --build-dir DIR --result-dir DIR [--names-env VAR | --names JSON] + [--split-reuse] [--server-path PATH] [--shard N/M] + retrace_shard.py --self-test +""" +import argparse +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import xml.etree.ElementTree as ET +from pathlib import Path + +SCRIPTS = Path(__file__).resolve().parent +ITERATIONRP_CASE = "minecraft-1.21.4-fabric-iris-iterationrp-in-world" +BLENDED_DEPTH_WRITE_CASE = "improved-transparency-minecraft-26.3" +WIRE_EVIDENCE_CASE = "OpenRA" +CONTROL_CASE = "OpenRA" +CONTROL_BACKEND = "DirectGLES" +CONTROL_NAME = "OpenRA (corrupted-snapshot control)" +# The comparator runs the long cases several times slower than a plain replay, and a spawned +# server can take as long, so the verify and split lanes raise ctest's 1500 s default. +LONG_TIMEOUT = ("--timeout", "10800") +SPLIT_REUSE_EXPECT_IPC = "strict=0 role-split-state=1 run-ahead=1" +SPLIT_REUSE_EXPECT_IPC_REASON = ( + "retrace-split dropped its inproc rows because the inproc-acceptance retrace rows replay the " + "same library with strict=1 role-split-state=1 run-ahead=1, which differs from this leg's " + "defaults only in strict while these hold. A default changed: restore the inproc rows (the " + "trace-cases job's split-spawn-shards without --transports spawn) or update the expectation.") +# GitHub shows about ten error annotations of a step. The first failures are annotated as they +# happen, so a shard killed later still shows them; the rest wait for the roll-up that names +# every failed case. +IMMEDIATE_ANNOTATIONS = 8 + + +def test_regex(case, backend): + return rf"^MobileGLTraceReplay\.{case}\.{backend}$" + + +def test_name(case, backend): + return f"MobileGLTraceReplay.{case}.{backend}" + + +def case_plan(profile, transport, backend, case, *, split_reuse=False, server_path=""): + """What one case of a lane runs with: the environment it adds, ctest's arguments, its checks. + + retrace the monolith-control and inproc-acceptance lanes of the retrace job. The job itself + exports MOBILEGL_TRANSPORT and the three IPC knobs for every case alike. + split the retrace-split job, inproc and spawn. + verify the retrace-verify job; its library has no transport. + """ + if profile not in ("retrace", "split", "verify"): + raise ValueError(f"unknown profile: {profile}") + env = {} + if backend == "DirectVulkan": + env["MOBILEGL_MAGMA_R11G11B10F_FALLBACK"] = "1" + # The blended depth-write quirk enables itself only on Qualcomm, which no runner has, + # so the case it exists for forces it on; ForceOn bypasses only the vendor gate. + if case == BLENDED_DEPTH_WRITE_CASE: + env["MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE"] = "1" + # The iterationRP repairs default to device detection and the software driver is on no + # list, so the one case that needs them forces them. The verify lane does not set it. + if case == ITERATIONRP_CASE and profile != "verify": + env["MOBILEGL_ITRP_FIX"] = "ForceOn" + ctest = ["-V", "--no-tests=error"] + evidence = None + census = False + if profile == "retrace": + if transport == "inproc": + # These rows stand in for retrace-split's dropped inproc rows, so they prove the + # configuration they ran with from the client's own log. + env["MGTRACE_EXPECT_IPC"] = "strict=1 role-split-state=1 run-ahead=1" + census = True + if transport != "monolith": + evidence = [] + elif profile == "split": + if transport not in ("inproc", "spawn"): + raise ValueError(f"the split profile runs inproc or spawn, not {transport!r}") + if not server_path.startswith("/"): + # The replay's working directory is not the workspace and the launcher resolves the + # image relative to nothing. The inproc arm reads it only to decide it needs none. + raise ValueError("the split profile needs an absolute --server-path") + env["MOBILEGL_TRANSPORT"] = transport + env["MOBILEGL_IPC_SERVER_PATH"] = server_path + evidence = [] + if case == WIRE_EVIDENCE_CASE: + # This case also counts the wire's draw records and the server's draws. That turns + # on the library's per-frame counters, so the others keep the plain configuration. + env["MGTRACE_WIRE_EVIDENCE"] = "1" + evidence = ["--require-wire"] + if split_reuse: + env["MGTRACE_EXPECT_IPC"] = SPLIT_REUSE_EXPECT_IPC + env["MGTRACE_EXPECT_IPC_REASON"] = SPLIT_REUSE_EXPECT_IPC_REASON + ctest += LONG_TIMEOUT + census = True + else: + env["MOBILEGL_PIPE_VERIFY"] = "1" + ctest += LONG_TIMEOUT + ctest += ["-R", test_regex(case, backend)] + return {"env": env, "ctest": ctest, "evidence": evidence, "census": census, + "control": has_control(profile, backend, case)} + + +def control_plan(): + """The verify lane's negative control: the same replay with one snapshot field corrupted. + + It must fail, and for the comparator's reason. It adds only these two variables and raises + no core limit, since the failure is the expected outcome. + """ + return {"env": {"MOBILEGL_PIPE_VERIFY": "1", + "MOBILEGL_PIPE_VERIFY_CORRUPT": "GetRenderStateParameters"}, + "ctest": ["-V", "--no-tests=error", *LONG_TIMEOUT, + "-R", test_regex(CONTROL_CASE, CONTROL_BACKEND)]} + + +def has_control(profile, backend, case): + return profile == "verify" and case == CONTROL_CASE and backend == CONTROL_BACKEND + + +def expected_results(profile, backend, names): + """The verdicts a shard owes, in order.""" + out = [] + for name in names: + out.append(name) + if has_control(profile, backend, name): + out.append(CONTROL_NAME) + return out + + +def step_ignores_sigpipe(): + """Whether the process that started this script, the step's shell, ignores SIGPIPE.""" + try: + with open(f"/proc/{os.getppid()}/status", encoding="ascii") as status: + for line in status: + if line.startswith("SigIgn:"): + return bool(int(line.split()[1], 16) & (1 << (signal.SIGPIPE - 1))) + except (OSError, ValueError, IndexError): + pass + return False + + +def raise_core_limit(): + # What `ulimit -c unlimited` does; the runner's core handler honors each process's limit. + import resource + resource.setrlimit(resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY)) + + +class Shard: + def __init__(self, args, names, base_env): + self.args = args + self.names = names + self.base_env = dict(base_env) + self.build_dir = Path(args.build_dir).resolve() + self.result_dir = Path(args.result_dir).resolve() + self.results_root = self.result_dir / "results" + self.junit_dir = self.result_dir / "junit" + self.log_dir = self.result_dir / "logs" + self.cache_root = self.result_dir / "driver-cache" + self.records = [] + self.failures = 0 + self.child = None + self.lane = args.lane or "-" + self.transport = args.transport or "-" + self.where = f"{self.lane} / {self.transport} / {args.backend}" + + # --- output ------------------------------------------------------------------------------- + + def say(self, text=""): + print(text, flush=True) + + def summary(self, text): + path = os.environ.get("GITHUB_STEP_SUMMARY") + if path: + with open(path, "a", encoding="utf-8") as file: + file.write(text + "\n") + + def annotate(self, title, message): + escaped = message.replace("%", "%25").replace("\r", "").replace("\n", "%0A") + # A property value also escapes the two characters that delimit properties. + title = title.replace("%", "%25").replace(":", "%3A").replace(",", "%2C") + self.say(f"::error title={title}::{escaped}") + + def write_results(self): + self.results_root.mkdir(parents=True, exist_ok=True) + payload = {"profile": self.args.profile, "lane": self.args.lane, "transport": self.args.transport, + "backend": self.args.backend, "shard": self.args.shard, "cases": self.names, + "results": self.records} + target = self.results_root / "shard-results.json" + scratch = target.with_suffix(".tmp") + scratch.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + scratch.replace(target) + + def record(self, name, status, seconds, reasons, *, ssim="", cores=()): + """One verdict: "passed", "failed", or "skipped" for a control whose case already failed.""" + entry = {"case": name, "status": status, "seconds": round(seconds, 1), "ssim": ssim, + "reasons": list(reasons), "cores": list(cores), "annotated": False} + if status == "failed": + self.failures += 1 + entry["annotated"] = self.failures <= IMMEDIATE_ANNOTATIONS + self.records.append(entry) + self.write_results() + note = "; ".join(reasons) + if cores: + note = (note + "; " if note else "") + "core: " + ", ".join(cores) + shown = "**failed**" if status == "failed" else status + self.summary(f"| {name} | {shown} | {seconds:.0f} | {ssim} | {note.replace('|', '/')[:300]} |") + if entry["annotated"]: + self.annotate(f"retrace failed: {self.where} / {name}", note or "failed") + + # --- one process ---------------------------------------------------------------------------- + + def run_ctest(self, plan, *, junit=None, transcript=None, core_limit=True, cache_home=None): + env = dict(self.base_env) + env.update(plan["env"]) + if cache_home is not None: + env["XDG_CACHE_HOME"] = str(cache_home) + command = [self.args.ctest, *plan["ctest"]] + if junit is not None: + command += ["--output-junit", str(junit)] + self.say("+ " + " ".join(f"{key}={value!r}" for key, value in sorted(plan["env"].items()))) + self.say("+ " + " ".join(command)) + options = {"cwd": self.build_dir, "env": env, "start_new_session": True} + if core_limit: + options["preexec_fn"] = raise_core_limit + # Python ignores SIGPIPE itself and by default hands a child the default action. The + # replay gets what a step's own child would get instead, so a write to a closed pipe + # or socket ends the same way as when the step ran ctest directly. + options["restore_signals"] = not step_ignores_sigpipe() + sys.stdout.flush() + if transcript is None: + self.child = subprocess.Popen(command, **options) + status = self.child.wait() + else: + # The transcript is both shown and kept, as `2>&1 | tee` would. + self.child = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options) + with open(transcript, "wb") as file: + for chunk in iter(lambda: self.child.stdout.read1(65536), b""): + file.write(chunk) + sys.stdout.buffer.write(chunk) + sys.stdout.buffer.flush() + status = self.child.wait() + self.reap() + return status + + def reap(self, grace=5.0): + """End whatever the finished replay left running, so it cannot run under the next case. + + A spawned server exits by itself when its client is gone, so the group gets a moment + first. True when something had to be killed. + """ + child, self.child = self.child, None + if child is None: + return False + deadline = time.monotonic() + grace + try: + while True: + os.killpg(child.pid, 0) + if time.monotonic() >= deadline: + break + time.sleep(0.25) + os.killpg(child.pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + return False + self.say(f"killed the processes ctest (pid {child.pid}) left running") + return True + + # --- one case ------------------------------------------------------------------------------- + + def fresh_cache_home(self): + """An empty driver cache directory, as on a runner that has replayed nothing yet. + + The software drivers keep compiled shaders on disk under the user cache directory. Each + case gets a new one, so a case never starts from shaders another case compiled. On a + hosted runner the default location is emptied as well, in case a process ignores the + redirect; a self-hosted machine's own cache is left alone. + """ + self.cache_root.mkdir(parents=True, exist_ok=True) + cache_home = Path(tempfile.mkdtemp(prefix="case-", dir=self.cache_root)) + if self.base_env.get("RUNNER_ENVIRONMENT") != "self-hosted": + default_home = Path(self.base_env.get("HOME", "/nonexistent")) / ".cache" + for stale in default_home.glob("mesa_shader_cache*"): + self.say(f"removing {stale} left by an earlier process") + shutil.rmtree(stale, ignore_errors=True) + return cache_home + + def drop_cache_home(self, cache_home): + entries = sum(1 for path in cache_home.rglob("*") if path.is_file()) + self.say(f"driver cache: {entries} file(s) written under {cache_home}, removed") + shutil.rmtree(cache_home, ignore_errors=True) + + def junit_problem(self, junit, name): + """Why this JUnit file does not show exactly the one entry that was asked for, or ''.""" + if not junit.is_file(): + return f"ctest wrote no {junit.name}" + try: + entries = [case.get("name") for case in ET.parse(junit).getroot().iter("testcase")] + except ET.ParseError as error: + return f"{junit.name} does not parse: {error}" + if entries != [name]: + return f"ctest ran {entries or 'no entry'} where exactly {name} was asked for" + return "" + + def check_evidence(self, plan, case): + """Run the lane's evidence check; its error lines are folded into the case's annotation.""" + evidence = self.build_dir / case / self.args.backend / "output" / "split-evidence.txt" + command = [sys.executable, str(SCRIPTS / "retrace_split_evidence.py"), str(evidence), + "--transport", self.args.transport, "--case", case, "--backend", self.args.backend, + *plan["evidence"]] + done = subprocess.run(command, cwd=self.build_dir, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True) + text = done.stdout.strip() + self.say(re.sub(r"^::error::", "split evidence: ", text, flags=re.M)) + if done.returncode == 0: + return "" + return "split evidence: " + re.sub(r"^::error::", "", text, flags=re.M).replace("\n", " ")[:600] + + def census(self, case): + """The refusal census from the library's own logs, recorded and never a verdict. + + Both roles are read: under spawn the applier's refusals are raised in the server's + process. ctest's transcript cannot stand in, since the console sink is compiled out. + """ + output = self.build_dir / case / self.args.backend / "output" + log = output / "mobilegl.client.log" + server_log = output / "mobilegl.server.log" + evidence = output / "split-evidence.txt" + self.say(evidence.read_text(errors="replace").rstrip() if evidence.is_file() else "no split-evidence.txt") + if not log.is_file(): + self.say(f"no {log} - the replay wrote no library log") + return + files = [log] + ([server_log] if server_log.is_file() else []) + self.say("census over: " + " ".join(str(path) for path in files)) + lines = [line for path in files for line in path.read_text(errors="replace").splitlines()] + self.say(f"Fatal{{ lines: {sum(1 for line in lines if 'Fatal{' in line)}") + counts = {} + for line in lines: + for match in re.findall(r'Fatal\{[A-Za-z]*, "[^"]*"', line): + counts[match] = counts.get(match, 0) + 1 + for match, count in sorted(counts.items(), key=lambda item: (-item[1], item[0]))[:20]: + self.say(f"{count:7d} {match}") + transport_line = next((line for line in log.read_text(errors="replace").splitlines() + if "MOBILEGL_TRANSPORT=" in line), None) + self.say(transport_line or f"no transport line in {log}") + + def collect(self, case): + """Move what the lane uploads into the result directory and delete the extracted trace.""" + source = self.build_dir / case + target = self.results_root / case + shutil.rmtree(source / self.args.backend / "input", ignore_errors=True) + for relative in ("actual-images", f"{self.args.backend}/output"): + if (source / relative).is_dir(): + (target / relative).parent.mkdir(parents=True, exist_ok=True) + shutil.rmtree(target / relative, ignore_errors=True) + shutil.move(str(source / relative), str(target / relative)) + shutil.rmtree(source, ignore_errors=True) + + def ssim_of(self, case): + try: + result = json.loads((self.build_dir / case / self.args.backend / "output" / "result.json") + .read_text(errors="replace")) + except (OSError, ValueError): + return "" + value = result.get("ssim") + return "" if value is None else str(value) + + def run_case(self, case): + plan = case_plan(self.args.profile, self.args.transport, self.args.backend, case, + split_reuse=self.args.split_reuse, server_path=self.args.server_path) + self.say(f"::group::{case} ({self.where})") + started = time.monotonic() + cores_before = set(Path("/tmp").glob("core.*")) + junit = self.junit_dir / f"{case}.xml" + reasons = [] + cache_home = self.fresh_cache_home() + try: + try: + status = self.run_ctest(plan, junit=junit, cache_home=cache_home) + except (OSError, subprocess.SubprocessError) as error: + # The case still gets its verdict, and the cases after it still run. + self.child = None + status = None + reasons.append(f"ctest could not be run: {error}") + if status is not None and status > 0: + reasons.append(f"ctest exit {status}") + elif status is not None and status < 0: + reasons.append(f"ctest ended by signal {-status}") + problem = self.junit_problem(junit, test_name(case, self.args.backend)) + if problem: + reasons.append(problem) + if plan["evidence"] is not None: + # The transport assertions ran on this case and found their evidence. A failed + # replay is already red, and its missing evidence file would only repeat that. + if status == 0: + problem = self.check_evidence(plan, case) + if problem: + reasons.append(problem) + if plan["census"]: + try: + self.census(case) + except OSError as error: + self.say(f"refusal census skipped: {error}") + finally: + self.drop_cache_home(cache_home) + ssim = self.ssim_of(case) + self.collect(case) + cores = sorted(path.name for path in set(Path("/tmp").glob("core.*")) - cores_before) + seconds = time.monotonic() - started + self.say(f"{case}: {'passed' if not reasons else 'FAILED: ' + '; '.join(reasons)} in {seconds:.0f} s") + self.say("::endgroup::") + self.record(case, "failed" if reasons else "passed", seconds, reasons, ssim=ssim, cores=cores) + return not reasons + + def run_control(self, verified_passed): + """With a snapshot field corrupted the same replay must fail, or the lane proves nothing. + + It replays into the case directory the verified run used. That run's output has already + been moved to the result directory, and the corrupted run's output is deleted, so only + the verified run is uploaded under the case's name. + """ + self.say(f"::group::{CONTROL_NAME}") + if not verified_passed: + self.say("not run: the verified replay it is compared with failed") + self.say("::endgroup::") + self.record(CONTROL_NAME, "skipped", 0, ["not run: the verified replay failed"]) + return + started = time.monotonic() + reasons = [] + transcript = self.log_dir / "verify-control.txt" + cache_home = self.fresh_cache_home() + status = None + try: + status = self.run_ctest(control_plan(), transcript=transcript, core_limit=False, + cache_home=cache_home) + except (OSError, subprocess.SubprocessError) as error: + self.child = None + reasons.append(f"ctest could not be run: {error}") + finally: + self.drop_cache_home(cache_home) + shutil.rmtree(self.build_dir / CONTROL_CASE, ignore_errors=True) + if status is None: + pass + elif status == 0: + reasons.append( + "MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left the OpenRA retrace " + "green, so the comparator is not comparing and the verify retrace lane proves nothing") + else: + # Its red must carry the comparator's divergence on the corrupted field. + done = subprocess.run([sys.executable, str(SCRIPTS / "retrace_verify_reason.py"), str(transcript)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + text = re.sub(r"^::error::", "", done.stdout.strip(), flags=re.M) + self.say(text) + if done.returncode != 0: + reasons.append(text.replace("\n", " ")[:600]) + else: + self.say(f"the corrupted field turned the retrace red, as it must (ctest exit {status})") + seconds = time.monotonic() - started + self.say("::endgroup::") + self.record(CONTROL_NAME, "failed" if reasons else "passed", seconds, reasons) + + # --- the shard -------------------------------------------------------------------------------- + + def run(self): + for directory in (self.results_root, self.junit_dir, self.log_dir): + directory.mkdir(parents=True, exist_ok=True) + expected = expected_results(self.args.profile, self.args.backend, self.names) + self.say(f"retrace shard {self.args.shard or ''} ({self.where}): {len(self.names)} case(s), " + f"{len(expected)} verdict(s) owed") + self.summary(f"### retrace shard {self.args.shard or ''} ({self.where})\n\n" + "| case | result | seconds | SSIM | note |\n|---|---|---|---|---|") + self.write_results() + for case in self.names: + passed = self.run_case(case) + if case_plan(self.args.profile, self.args.transport, self.args.backend, case, + split_reuse=self.args.split_reuse, server_path=self.args.server_path)["control"]: + self.run_control(passed) + return self.finish(expected) + + def finish(self, expected): + problems = verdict_problems(expected, self.records) + # Counted again from what ctest wrote, apart from the loop's own bookkeeping. + ran = sum(1 for case in self.names + if not self.junit_problem(self.junit_dir / f"{case}.xml", test_name(case, self.args.backend))) + passed_cases = sum(1 for entry in self.records + if entry["status"] == "passed" and entry["case"] in self.names) + if ran < passed_cases: + problems.append(f"{passed_cases} case(s) recorded as passed but only {ran} JUnit file(s) " + f"show their entry ran") + failed = [entry for entry in self.records if entry["status"] == "failed"] + if failed: + names = ", ".join(entry["case"] for entry in failed) + self.annotate(f"retrace shard {self.args.shard or ''}: {len(failed)} of {len(expected)} failed", + f"{self.where}: {names}") + for entry in failed: + if not entry["annotated"]: + self.annotate(f"retrace failed: {self.where} / {entry['case']}", + "; ".join(entry["reasons"]) or "failed") + for problem in problems: + self.annotate(f"retrace shard {self.args.shard or ''}: verdicts do not match the cases", problem) + passed = sum(1 for entry in self.records if entry["status"] == "passed") + closing = f"{passed} passed, {len(failed)} failed, {len(expected)} owed" + if problems: + closing += "; " + "; ".join(problems) + self.summary(f"\n{closing}\n") + self.say(f"retrace shard ({self.where}): {closing}") + return 1 if failed or problems else 0 + + +def verdict_problems(expected, records): + """Why the recorded verdicts are not exactly the ones owed; empty when they are.""" + recorded = [entry["case"] for entry in records] + problems = [] + if len(recorded) != len(expected): + problems.append(f"{len(recorded)} verdict(s) recorded for {len(expected)} owed") + if recorded != expected: + missing = [name for name in expected if name not in recorded] + extra = [name for name in recorded if name not in expected] + if missing: + problems.append("no verdict for " + ", ".join(missing)) + if extra: + problems.append("verdict for a case that was not assigned: " + ", ".join(extra)) + if not missing and not extra and len(recorded) == len(expected): + problems.append("verdicts are out of order or repeated") + return problems + + +def run(args): + if args.names is not None: + names = json.loads(args.names) + else: + names = json.loads(os.environ.get(args.names_env) or "null") + if not isinstance(names, list) or not names or not all(isinstance(name, str) and name for name in names): + print(f"::error::retrace shard: no case list (looked in --names and ${args.names_env})") + return 2 + if len(set(names)) != len(names): + print(f"::error::retrace shard: the case list repeats a case: {names}") + return 2 + if args.profile == "retrace": + # The job exports the transport for this lane; the plan and the library must agree on it. + exported = os.environ.get("MOBILEGL_TRANSPORT", "") + if exported != args.transport: + print(f"::error::retrace shard: --transport is {args.transport!r} but the job exports " + f"MOBILEGL_TRANSPORT={exported!r}") + return 2 + try: + case_plan(args.profile, args.transport, args.backend, names[0], + split_reuse=args.split_reuse, server_path=args.server_path) + except ValueError as error: + print(f"::error::retrace shard: {error}") + return 2 + shard = Shard(args, names, os.environ) + + def stop(signum, _frame): + shard.reap(grace=0) + shard.say(f"retrace shard: stopped by signal {signum} after {len(shard.records)} verdict(s)") + sys.exit(128 + signum) + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + return shard.run() + + +def self_test(): + """The plan of every lane against the values the lanes are defined by, and the verdict check.""" + oit, itrp, other = BLENDED_DEPTH_WRITE_CASE, ITERATIONRP_CASE, "minecraft-1.21.4-in-world" + server = "/w/build-linux/libMobileGLServer.so" + regex = lambda case, backend: ["-R", rf"^MobileGLTraceReplay\.{case}\.{backend}$"] + plain, long = ["-V", "--no-tests=error"], ["-V", "--no-tests=error", "--timeout", "10800"] + vk = {"MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "1"} + expect_strict = {"MGTRACE_EXPECT_IPC": "strict=1 role-split-state=1 run-ahead=1"} + reuse = {"MGTRACE_EXPECT_IPC": SPLIT_REUSE_EXPECT_IPC, "MGTRACE_EXPECT_IPC_REASON": SPLIT_REUSE_EXPECT_IPC_REASON} + split = lambda transport: {"MOBILEGL_TRANSPORT": transport, "MOBILEGL_IPC_SERVER_PATH": server} + table = [ + # profile, transport, backend, case, reuse -> env, ctest, evidence, census, control + ("retrace", "monolith", "DirectGLES", other, False, {}, plain, None, False, False), + ("retrace", "monolith", "DirectVulkan", other, False, vk, plain, None, False, False), + ("retrace", "monolith", "DirectVulkan", oit, False, + {**vk, "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE": "1"}, plain, None, False, False), + ("retrace", "monolith", "DirectGLES", oit, False, {}, plain, None, False, False), + ("retrace", "monolith", "DirectVulkan", itrp, False, {**vk, "MOBILEGL_ITRP_FIX": "ForceOn"}, + plain, None, False, False), + ("retrace", "inproc", "DirectGLES", other, False, expect_strict, plain, [], True, False), + ("retrace", "inproc", "DirectVulkan", itrp, False, + {**vk, "MOBILEGL_ITRP_FIX": "ForceOn", **expect_strict}, plain, [], True, False), + ("retrace", "inproc", "DirectGLES", "OpenRA", False, expect_strict, plain, [], True, False), + ("split", "spawn", "DirectGLES", other, True, {**split("spawn"), **reuse}, long, [], True, False), + ("split", "spawn", "DirectGLES", other, False, split("spawn"), long, [], True, False), + ("split", "spawn", "DirectVulkan", oit, True, + {**vk, "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE": "1", **split("spawn"), **reuse}, + long, [], True, False), + ("split", "spawn", "DirectVulkan", itrp, True, + {**vk, "MOBILEGL_ITRP_FIX": "ForceOn", **split("spawn"), **reuse}, long, [], True, False), + ("split", "spawn", "DirectGLES", "OpenRA", True, + {**split("spawn"), "MGTRACE_WIRE_EVIDENCE": "1", **reuse}, long, ["--require-wire"], True, False), + ("split", "inproc", "DirectVulkan", "OpenRA", True, + {**vk, **split("inproc"), "MGTRACE_WIRE_EVIDENCE": "1", **reuse}, long, ["--require-wire"], True, False), + ("split", "inproc", "DirectGLES", other, False, split("inproc"), long, [], True, False), + ("verify", "", "DirectGLES", other, False, {"MOBILEGL_PIPE_VERIFY": "1"}, long, None, False, False), + ("verify", "", "DirectGLES", "OpenRA", False, {"MOBILEGL_PIPE_VERIFY": "1"}, long, None, False, True), + ("verify", "", "DirectVulkan", "OpenRA", False, {"MOBILEGL_PIPE_VERIFY": "1", **vk}, long, None, False, False), + ("verify", "", "DirectVulkan", oit, False, + {"MOBILEGL_PIPE_VERIFY": "1", **vk, "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE": "1"}, + long, None, False, False), + ("verify", "", "DirectVulkan", itrp, False, {"MOBILEGL_PIPE_VERIFY": "1", **vk}, long, None, False, False), + ] + for profile, transport, backend, case, split_reuse, env, ctest, evidence, census, control in table: + plan = case_plan(profile, transport, backend, case, split_reuse=split_reuse, server_path=server) + wanted = {"env": env, "ctest": ctest + regex(case, backend), "evidence": evidence, + "census": census, "control": control} + assert plan == wanted, f"{profile} {transport} {backend} {case}: {plan} != {wanted}" + control = control_plan() + assert control["env"] == {"MOBILEGL_PIPE_VERIFY": "1", "MOBILEGL_PIPE_VERIFY_CORRUPT": "GetRenderStateParameters"} + assert control["ctest"] == long + regex("OpenRA", "DirectGLES") + for bad in (("split", "spawn", ""), ("split", "spawn", "relative/server"), ("split", "monolith", server), + ("pull", "monolith", server)): + try: + case_plan(bad[0], bad[1], "DirectGLES", other, server_path=bad[2]) + except ValueError: + continue + raise AssertionError(f"case_plan accepted {bad}") + assert expected_results("verify", "DirectGLES", ["a", "OpenRA", "b"]) == ["a", "OpenRA", CONTROL_NAME, "b"] + assert expected_results("verify", "DirectVulkan", ["OpenRA"]) == ["OpenRA"] + assert expected_results("split", "DirectGLES", ["OpenRA"]) == ["OpenRA"] + entry = lambda name: {"case": name, "status": "passed"} + assert verdict_problems(["a", "b"], [entry("a"), entry("b")]) == [] + assert verdict_problems(["a", "b"], [entry("a")]), "a case without a verdict passed the check" + assert verdict_problems(["a", "b"], []), "a shard that ran nothing passed the check" + assert verdict_problems(["a"], [entry("a"), entry("a")]), "a repeated verdict passed the check" + assert verdict_problems(["a", "b"], [entry("a"), entry("c")]), "an unassigned case passed the check" + assert verdict_problems(["a", "b"], [entry("b"), entry("a")]), "out-of-order verdicts passed the check" + print(f"retrace_shard: {len(table)} plans, the control plan and the verdict check OK") + return 0 + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("command", nargs="?", choices=("run",)) + parser.add_argument("--self-test", action="store_true") + parser.add_argument("--profile", choices=("retrace", "split", "verify")) + parser.add_argument("--backend") + parser.add_argument("--transport", default="") + parser.add_argument("--lane", default="") + parser.add_argument("--shard", default="", help="N/M, for the report only") + parser.add_argument("--build-dir", help="the directory ctest runs in") + parser.add_argument("--result-dir", help="where verdicts, replay output and JUnit files go") + parser.add_argument("--names", help="the cases, as a JSON list") + parser.add_argument("--names-env", default="SHARD_NAMES", help="the variable holding the JSON list") + parser.add_argument("--split-reuse", action="store_true", + help="split profile: the library is the retrace job's, so prove the IPC defaults") + parser.add_argument("--server-path", default="", help="split profile: absolute path of the server image") + parser.add_argument("--ctest", default="ctest") + args = parser.parse_args(argv) + if args.self_test: + return self_test() + if args.command != "run" or not (args.profile and args.backend and args.build_dir and args.result_dir): + parser.error("run needs --profile, --backend, --build-dir and --result-dir") + return run(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/trace_replay/SpawnServerPathTest.java b/tools/trace_replay/SpawnServerPathTest.java index d783afe7..46170e7b 100644 --- a/tools/trace_replay/SpawnServerPathTest.java +++ b/tools/trace_replay/SpawnServerPathTest.java @@ -48,7 +48,8 @@ public final class SpawnServerPathTest { public static void main(String[] args) { // ---- the shapes the lanes really pass ------------------------------------------------- - // .github/workflows/apk.yml's spawn acceptance lane, verbatim. + // The spawn acceptance lane of android-plugin/trace_replay_shard.py (transport_arguments()), + // verbatim. String spawnLane = "MOBILEGL_TRANSPORT=spawn;MOBILEGL_IPC_ROLE_SPLIT_STATE=1;" + "MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1"; expect("the spawn acceptance lane still gets its server path appended", diff --git a/tools/trace_replay/run_android_retrace_local.py b/tools/trace_replay/run_android_retrace_local.py index e771f759..5e2d4783 100644 --- a/tools/trace_replay/run_android_retrace_local.py +++ b/tools/trace_replay/run_android_retrace_local.py @@ -650,7 +650,8 @@ def transport_proof_args(args): return {"inproc": ["--require-inproc"], "spawn": ["--require-spawn"]}.get(args.transport, []) -# The knobs .github/workflows/apk.yml hands BOTH acceptance lanes. They are kept here verbatim so +# The knobs android-plugin/trace_replay_shard.py (transport_arguments()) hands both acceptance +# lanes. They are kept here verbatim so # a local run is the same run: the arm proof below reads the library's own `Config: IPC` line and # requires strict/role-split-state/run-ahead to be 1, so a local invocation that quietly omitted # them would fail a gate CI passes - or worse, pass a weaker one. diff --git a/tools/trace_replay/trace_cases.py b/tools/trace_replay/trace_cases.py index 31a92bc4..9c963a96 100644 --- a/tools/trace_replay/trace_cases.py +++ b/tools/trace_replay/trace_cases.py @@ -518,6 +518,311 @@ def self_test_split_matrix(cases=None): f"kept); all {len(dropped)} dropped inproc rows are inproc-acceptance rows") +# --- Shards: several cases of one cell replayed by one CI job --------------------------------- +# +# A cell is everything a per-case matrix row carries except its case: the backend, and the lane +# and transport where the matrix has them. A shard row keeps the cell's keys unchanged and adds +# the cases it replays, so the job that runs it reads the same fields a per-case leg reads. +# Every shard matrix is built from the per-case matrix it replaces and is refused unless its +# rows, taken apart again, are exactly that matrix. + +TRACE_DURATION_HINTS_JSON = Path(__file__).with_name("trace_duration_hints.json") + +# Shards per cell on Linux, the one tunable of that layout. A cell with little to replay gets +# fewer: a shard costs about half a minute of setup, so it is not split below +# MIN_SHARD_SECONDS of replay. +DEFAULT_SHARDS_PER_CELL = 4 +MIN_SHARD_SECONDS = 90 + +# Replay seconds one emulator shard aims for, the one tunable of the Android layout. A cell is +# split into as many shards as that takes. +DEFAULT_ANDROID_SHARD_SECONDS = 720 + +# The hint a row without a transport uses. The verify lane has timings of its own. +HINT_KIND_MONOLITH = "monolith" +HINT_KIND_VERIFY = "verify" + +# A shard's case names reach a shell command line, so the characters are restricted. +SHARD_SAFE_NAME = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-") + + +def load_duration_hints(platform, path=TRACE_DURATION_HINTS_JSON): + """{case: {backend: {kind: seconds}}} for "linux" or "android"; empty when there is none.""" + path = Path(path) + if not path.is_file(): + return {} + with path.open("r", encoding="utf-8") as file: + table = json.load(file).get(platform, {}).get("seconds", {}) + for case, backends in table.items(): + for backend, kinds in backends.items(): + for kind, seconds in kinds.items(): + if isinstance(seconds, bool) or not isinstance(seconds, (int, float)) or seconds <= 0: + raise ValueError(f"duration hint for {case} {backend} {kind} must be a positive " + f"number: {seconds!r}") + return table + + +def median(values): + ordered = sorted(values) + if not ordered: + return None + middle = len(ordered) // 2 + if len(ordered) % 2: + return ordered[middle] + return (ordered[middle - 1] + ordered[middle]) / 2 + + +def hint_seconds(hints, case, backend, kind): + """The hinted replay time of one row, or the median of its backend and kind, or of everything.""" + try: + return hints[case][backend][kind] + except KeyError: + pass + same_cell = [kinds[kind] for backends in hints.values() + for name, kinds in backends.items() if name == backend and kind in kinds] + everything = [seconds for backends in hints.values() for kinds in backends.values() + for seconds in kinds.values()] + return median(same_cell) or median(everything) or 30 + + +def row_case_name(row): + case = row["case"] + return case["name"] if isinstance(case, dict) else case + + +def row_backend_name(row): + backend = row["backend"] + return backend["name"] if isinstance(backend, dict) else backend + + +def row_cell(row): + return {key: value for key, value in row.items() if key != "case"} + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":")) + + +def shard_rows_as_cases(shard_rows): + """Take shard rows apart into the per-case rows they stand for.""" + added = ("shard", "shards", "names", "cases", "seconds") + rows = [] + for shard in shard_rows: + cell = {key: value for key, value in shard.items() if key not in added} + for case in shard.get("cases", shard["names"]): + rows.append({**cell, "case": case}) + return rows + + +def check_shard_union(rows, shard_rows): + """Refuse a shard matrix that is not exactly the per-case matrix `rows`, regrouped.""" + expected = sorted(canonical(row) for row in rows) + actual = sorted(canonical(row) for row in shard_rows_as_cases(shard_rows)) + if expected != actual: + missing = sorted(set(expected) - set(actual)) + extra = sorted(set(actual) - set(expected)) + raise ValueError( + f"the shard matrix is not the per-case matrix regrouped: {len(expected)} per-case rows, " + f"{len(actual)} rows in the shards; missing {missing[:3]}, unexpected {extra[:3]}") + if len(set(expected)) != len(expected): + raise ValueError("the per-case matrix repeats a row, so a shard would replay it twice") + cells = {} + for shard in shard_rows: + if not shard["names"]: + raise ValueError(f"empty shard: {shard}") + if "cases" in shard and [case["name"] for case in shard["cases"]] != shard["names"]: + raise ValueError(f"a shard's names and cases disagree: {shard['names']}") + cell = canonical({key: value for key, value in shard.items() + if key not in ("shard", "shards", "names", "cases", "seconds")}) + cells.setdefault(cell, []).append((shard["shard"], shard["shards"])) + for cell, numbers in cells.items(): + count = len(numbers) + if sorted(numbers) != [(index, count) for index in range(1, count + 1)]: + raise ValueError(f"shards of {cell} are not numbered 1..{count} of {count}: {numbers}") + + +def shard_matrix(rows, hints, *, default_kind=HINT_KIND_MONOLITH, shards=None, shard_seconds=None): + """Regroup a per-case matrix into shards of one cell each. + + Exactly one of `shards` (shards per cell, capped so that none holds less than + MIN_SHARD_SECONDS of hinted replay) and `shard_seconds` (hinted replay per shard) sets the + layout. Cases go longest first into the shard with the least hinted time so far, which is + deterministic and keeps the shards of a cell even; inside a shard they run in matrix order. + """ + if (shards is None) == (shard_seconds is None): + raise ValueError("give exactly one of shards and shard_seconds") + if shards is not None and shards < 1: + raise ValueError(f"shards per cell must be at least 1: {shards}") + if shard_seconds is not None and shard_seconds < 1: + raise ValueError(f"seconds per shard must be at least 1: {shard_seconds}") + cells = {} + for index, row in enumerate(rows): + name = row_case_name(row) + if not name or not set(name) <= SHARD_SAFE_NAME: + raise ValueError(f"case name {name!r} cannot be passed on a command line; allowed " + f"characters are letters, digits, '.', '_' and '-'") + seconds = hint_seconds(hints, name, row_backend_name(row), + row.get("transport", default_kind)) + cells.setdefault(canonical(row_cell(row)), []).append((index, row, seconds)) + out = [] + for members in cells.values(): + total = sum(seconds for _, _, seconds in members) + if shards is not None: + count = min(shards, int(total // MIN_SHARD_SECONDS)) + else: + count = -(-int(total) // shard_seconds) + count = max(1, min(count, len(members))) + bins = [[] for _ in range(count)] + loads = [0] * count + for member in sorted(members, key=lambda item: (-item[2], item[0])): + target = loads.index(min(loads)) + bins[target].append(member) + loads[target] += member[2] + for number, members_of_shard in enumerate(bins, start=1): + members_of_shard.sort(key=lambda item: item[0]) + shard = {**row_cell(members_of_shard[0][1]), "shard": number, "shards": count, + "names": [row_case_name(row) for _, row, _ in members_of_shard]} + if isinstance(members_of_shard[0][1]["case"], dict): + shard["cases"] = [row["case"] for _, row, _ in members_of_shard] + shard["seconds"] = round(sum(seconds for _, _, seconds in members_of_shard)) + out.append(shard) + check_shard_union(rows, out) + return {"include": out} + + +def per_case_matrix(format_name, cases, *, transports=SPLIT_TRANSPORTS, keep_inproc=()): + """The per-case matrix a shard format regroups, and the hint kind of its transport-less rows.""" + if format_name == "github-test-shard-matrix": + return github_test_matrix(cases), HINT_KIND_MONOLITH + if format_name == "github-test-transport-shard-matrix": + return github_transport_matrix(cases), HINT_KIND_MONOLITH + if format_name == "github-verify-shard-matrix": + return github_verify_matrix(cases), HINT_KIND_VERIFY + if format_name == "github-split-shard-matrix": + return github_split_matrix(cases, transports, keep_inproc), HINT_KIND_MONOLITH + if format_name == "github-apk-shard-matrix": + return github_apk_matrix(cases), HINT_KIND_MONOLITH + if format_name == "github-apk-transport-shard-matrix": + return github_transport_matrix(cases, apk=True), HINT_KIND_MONOLITH + raise ValueError(f"not a shard format: {format_name}") + + +LINUX_SHARD_FORMATS = ( + "github-test-shard-matrix", + "github-test-transport-shard-matrix", + "github-verify-shard-matrix", + "github-split-shard-matrix", +) +ANDROID_SHARD_FORMATS = ( + "github-apk-shard-matrix", + "github-apk-transport-shard-matrix", +) + + +def github_shard_matrix(format_name, cases, *, shards=DEFAULT_SHARDS_PER_CELL, + shard_seconds=DEFAULT_ANDROID_SHARD_SECONDS, + transports=SPLIT_TRANSPORTS, keep_inproc=(), hints_path=TRACE_DURATION_HINTS_JSON): + """A shard format: the Linux ones lay out by shards per cell, the Android ones by seconds.""" + matrix, default_kind = per_case_matrix(format_name, cases, transports=transports, + keep_inproc=keep_inproc) + if format_name in ANDROID_SHARD_FORMATS: + return shard_matrix(matrix["include"], load_duration_hints("android", hints_path), + default_kind=default_kind, shard_seconds=shard_seconds) + return shard_matrix(matrix["include"], load_duration_hints("linux", hints_path), + default_kind=default_kind, shards=shards) + + +def self_test_shard_matrices(): + """Every shard format replays exactly the rows of the per-case matrix it replaces.""" + cases = ci_trace_cases(load_trace_cases()) + narrowed = {"transports": ("spawn",), "keep_inproc": ("OpenRA",)} + layouts = [(name, {}, {"shards": count}) for name in LINUX_SHARD_FORMATS for count in (1, 4, 100)] + layouts += [("github-split-shard-matrix", narrowed, {"shards": count}) for count in (1, 4, 100)] + layouts += [(name, {}, {"shard_seconds": seconds}) for name in ANDROID_SHARD_FORMATS + for seconds in (60, DEFAULT_ANDROID_SHARD_SECONDS, 10 ** 9)] + for name, selection, layout in layouts: + matrix, _ = per_case_matrix(name, cases, **selection) + rows = matrix["include"] + built = github_shard_matrix(name, cases, **selection, **layout)["include"] + again = github_shard_matrix(name, cases, **selection, **layout)["include"] + assert built == again, f"{name} {layout}: the layout is not deterministic" + # Compared here as well as inside shard_matrix, with nothing shared but the JSON form. + replayed = sorted( + canonical({**{key: value for key, value in shard.items() + if key not in ("shard", "shards", "names", "cases", "seconds")}, + "case": case}) + for shard in built for case in shard.get("cases", shard["names"])) + assert replayed == sorted(canonical(row) for row in rows), \ + f"{name} {layout}: the shards do not replay the per-case matrix" + assert sum(len(shard["names"]) for shard in built) == len(rows) + cells = {canonical(row_cell(row)) for row in rows} + assert len({canonical({key: value for key, value in shard.items() + if key not in ("shard", "shards", "names", "cases", "seconds")}) + for shard in built}) == len(cells), f"{name} {layout}: a shard spans two cells" + if layout.get("shards") == 1 or layout.get("shard_seconds") == 10 ** 9: + assert len(built) == len(cells), f"{name} {layout}: expected one shard per cell" + seconds = [shard["seconds"] for shard in built] + print(f"shard matrix {name} {selection or ''} {layout}: {len(rows)} rows in {len(built)} " + f"shards over {len(cells)} cells, {min(seconds)}-{max(seconds)} s hinted per shard") + # The check itself must be able to fail: a dropped, a repeated and a moved case. + if len(built) >= 2 and any(len(shard["names"]) >= 2 for shard in built): + index = next(i for i, shard in enumerate(built) if len(shard["names"]) >= 2) + for label, damage in (("dropped", lambda names: names[1:]), + ("repeated", lambda names: names + names[:1])): + broken = [dict(shard) for shard in built] + broken[index]["names"] = damage(list(broken[index]["names"])) + if "cases" in broken[index]: + broken[index]["cases"] = damage(list(broken[index]["cases"])) + try: + check_shard_union(rows, broken) + except ValueError: + pass + else: + raise AssertionError(f"{name} {layout}: a {label} case passed the union check") + other = next((i for i, shard in enumerate(built) + if row_backend_name(shard) != row_backend_name(built[index])), None) + if other is not None: + broken = [dict(shard) for shard in built] + for key in ("names", "cases"): + if key in broken[index]: + moved = list(broken[index][key]) + broken[other][key] = list(broken[other][key]) + moved[:1] + broken[index][key] = moved[1:] + try: + check_shard_union(rows, broken) + except ValueError: + pass + else: + raise AssertionError(f"{name} {layout}: a case moved to another backend's " + f"shard passed the union check") + # Hints only weigh the packing. With none at all, and with a case no hint names, the rows + # replayed are the same. + rows = github_transport_matrix(cases)["include"] + for hints in ({}, {"no-such-case": {"DirectGLES": {"monolith": 5}}}): + check_shard_union(rows, shard_matrix(rows, hints, shards=4)["include"]) + assert hint_seconds({}, "x", "DirectGLES", "monolith") == 30 + assert hint_seconds({"a": {"DirectGLES": {"monolith": 10}}, "b": {"DirectGLES": {"monolith": 20}}, + "c": {"DirectVulkan": {"monolith": 90}}}, "x", "DirectGLES", "monolith") == 15 + for bad in ({"shards": 0}, {"shard_seconds": 0}, {}, {"shards": 4, "shard_seconds": 60}): + try: + shard_matrix(rows, {}, **bad) + except ValueError: + continue + raise AssertionError(f"shard_matrix accepted {bad}") + try: + shard_matrix([{"backend": "DirectGLES", "case": "bad name; rm"}], {}, shards=1) + except ValueError: + pass + else: + raise AssertionError("a case name with shell characters was accepted") + # The verify lane's corrupted-snapshot control replays OpenRA on DirectGLES, so that row has + # to be in the verify matrix for the control to have a shard to run in. + verify_rows = github_verify_matrix(cases)["include"] + assert {"backend": "DirectGLES", "case": "OpenRA"} in verify_rows, \ + "the verify matrix has no OpenRA DirectGLES row, so its negative control would not run" + print("shard matrices: union, numbering, determinism and negative controls OK") + + def cmake_quote(value): return '"' + str(value).replace("\\", "/").replace('"', '\\"') + '"' @@ -570,11 +875,19 @@ def parse_args(): parser.add_argument("--self-test-transport-matrices", action="store_true") parser.add_argument("--self-test-split-matrix", action="store_true", help="prove the spawn-only split matrix drops only inproc-acceptance rows") + parser.add_argument("--self-test-shard-matrices", action="store_true", + help="prove every shard matrix replays exactly its per-case matrix") parser.add_argument("--fixture-root", default="tools/trace_replay/fixtures") parser.add_argument("--transports", default=",".join(SPLIT_TRANSPORTS), help="github-split-matrix: the comma-separated split transports to emit") parser.add_argument("--keep-inproc", default="", help="github-split-matrix: comma-separated split cases whose inproc rows stay") + parser.add_argument("--shards", type=int, default=DEFAULT_SHARDS_PER_CELL, + help="Linux shard formats: shards per (lane, transport, backend) cell") + parser.add_argument("--shard-seconds", type=int, default=DEFAULT_ANDROID_SHARD_SECONDS, + help="Android shard formats: hinted replay seconds one shard aims for") + parser.add_argument("--duration-hints", default=str(TRACE_DURATION_HINTS_JSON), + help="shard formats: the JSON of replay seconds the packing weighs cases by") parser.add_argument( "--format", choices=( @@ -586,7 +899,10 @@ def parse_args(): "github-apk", "github-apk-matrix", "github-apk-transport-matrix", + *LINUX_SHARD_FORMATS, + *ANDROID_SHARD_FORMATS, "fixture-files", + "fixture-manifest", "cmake", ), default="names", @@ -602,6 +918,9 @@ def main(): if args.self_test_split_matrix: self_test_split_matrix() return 0 + if args.self_test_shard_matrices: + self_test_shard_matrices() + return 0 cases = load_trace_cases() if args.ci: cases = ci_trace_cases(cases) @@ -623,6 +942,19 @@ def main(): print(json.dumps(github_apk_matrix(cases), separators=(",", ":"))) elif args.format == "github-apk-transport-matrix": print(json.dumps(github_transport_matrix(cases, apk=True), separators=(",", ":"))) + elif args.format in LINUX_SHARD_FORMATS + ANDROID_SHARD_FORMATS: + transports = tuple(t for t in args.transports.split(",") if t) + keep = tuple(c for c in args.keep_inproc.split(",") if c) + print(json.dumps( + github_shard_matrix(args.format, cases, shards=args.shards, + shard_seconds=args.shard_seconds, transports=transports, + keep_inproc=keep, hints_path=args.duration_hints), + separators=(",", ":"))) + elif args.format == "fixture-manifest": + # One "" line per fixture file, for the fixture bundle script. + for case in cases: + for file in fixture_files(case, args.fixture_root): + print(f"{case['name']}\t{file}") elif args.format == "fixture-files": if not args.case_name: print("--case is required for --format fixture-files", file=sys.stderr) diff --git a/tools/trace_replay/trace_duration_hints.json b/tools/trace_replay/trace_duration_hints.json new file mode 100644 index 00000000..41bb9596 --- /dev/null +++ b/tools/trace_replay/trace_duration_hints.json @@ -0,0 +1,99 @@ +{ + "description": [ + "Replay seconds per case, backend and transport, used only to pack retrace shards evenly", + "(tools/trace_replay/trace_cases.py, shard_matrix). A missing entry falls back to the median of", + "its backend and transport, so a new case needs no entry here. Nothing asserts these values;", + "a stale number only makes shards uneven. To refresh, copy the seconds column of the shard", + "step summaries of a green run." + ], + "linux": { + "source": "Test run 43666299896 (2026-11-15), step 'Retrace and validate' of each one-case job, two gaps filled from run 43700779712; the verify key is the retrace-verify job", + "seconds": { + "OpenRA": {"DirectGLES": {"inproc": 3, "monolith": 3, "spawn": 3, "verify": 3}, "DirectVulkan": {"inproc": 3, "monolith": 3, "spawn": 2, "verify": 4}}, + "improved-transparency-minecraft-26.3": {"DirectGLES": {"inproc": 86, "monolith": 51, "spawn": 78, "verify": 81}, "DirectVulkan": {"inproc": 82, "monolith": 104, "spawn": 107, "verify": 85}}, + "minecraft-1.17-main-menu-854": {"DirectGLES": {"inproc": 4, "monolith": 6, "spawn": 3, "verify": 3}, "DirectVulkan": {"inproc": 5, "monolith": 5, "spawn": 3, "verify": 3}}, + "minecraft-1.20.1-fabric-iris-iterationt-in-world": {"DirectGLES": {"inproc": 59, "monolith": 70, "spawn": 72}, "DirectVulkan": {"inproc": 62, "monolith": 69, "spawn": 68}}, + "minecraft-1.21.1-neoforge-create-indirect-in-world-align1024": {"DirectGLES": {"inproc": 9, "monolith": 18, "spawn": 16}, "DirectVulkan": {"inproc": 13, "monolith": 13, "spawn": 8}}, + "minecraft-1.21.1-neoforge-create-instancing-in-world": {"DirectGLES": {"inproc": 12, "monolith": 14, "spawn": 14}, "DirectVulkan": {"inproc": 10, "monolith": 14, "spawn": 10}}, + "minecraft-1.21.11-main-menu": {"DirectGLES": {"inproc": 5, "monolith": 5, "spawn": 3, "verify": 6}, "DirectVulkan": {"inproc": 7, "monolith": 6, "spawn": 6, "verify": 5}}, + "minecraft-1.21.4-fabric-common-mods-in-world": {"DirectGLES": {"inproc": 13, "monolith": 15, "spawn": 15}, "DirectVulkan": {"inproc": 19, "monolith": 17, "spawn": 14}}, + "minecraft-1.21.4-fabric-common-mods-inventory": {"DirectGLES": {"inproc": 20, "monolith": 13, "spawn": 11}, "DirectVulkan": {"inproc": 22, "monolith": 21, "spawn": 22}}, + "minecraft-1.21.4-fabric-iris-bliss-in-world": {"DirectGLES": {"inproc": 11, "monolith": 23, "spawn": 16}, "DirectVulkan": {"inproc": 17, "monolith": 17, "spawn": 16}}, + "minecraft-1.21.4-fabric-iris-bsl-esc-menu-854": {"DirectGLES": {"inproc": 126, "monolith": 127, "spawn": 124}, "DirectVulkan": {"inproc": 130, "monolith": 131, "spawn": 134}}, + "minecraft-1.21.4-fabric-iris-bsl-in-world": {"DirectGLES": {"inproc": 15, "monolith": 18, "spawn": 17}, "DirectVulkan": {"inproc": 16, "monolith": 15, "spawn": 17}}, + "minecraft-1.21.4-fabric-iris-chocapic-v6-lite-in-world": {"DirectGLES": {"inproc": 10, "monolith": 15, "spawn": 14}, "DirectVulkan": {"inproc": 14, "monolith": 13, "spawn": 15}}, + "minecraft-1.21.4-fabric-iris-complementary-reimagined-in-world": {"DirectGLES": {"inproc": 27, "monolith": 35, "spawn": 29}, "DirectVulkan": {"inproc": 41, "monolith": 31, "spawn": 28}}, + "minecraft-1.21.4-fabric-iris-complementary-unbound-in-world": {"DirectGLES": {"inproc": 41, "monolith": 33, "spawn": 38}, "DirectVulkan": {"inproc": 26, "monolith": 31, "spawn": 25}}, + "minecraft-1.21.4-fabric-iris-derivative-main-d24.4.14-in-world": {"DirectGLES": {"inproc": 23, "monolith": 33, "spawn": 24}, "DirectVulkan": {"inproc": 20, "monolith": 26, "spawn": 27}}, + "minecraft-1.21.4-fabric-iris-iterationrp-in-world": {"DirectVulkan": {"inproc": 42, "monolith": 42, "spawn": 41}}, + "minecraft-1.21.4-fabric-iris-iterationt-in-world": {"DirectGLES": {"inproc": 26, "monolith": 27, "spawn": 23}, "DirectVulkan": {"inproc": 23, "monolith": 20, "spawn": 19}}, + "minecraft-1.21.4-fabric-iris-iterationt-nodsa-in-world": {"DirectGLES": {"inproc": 24, "monolith": 19, "spawn": 19}, "DirectVulkan": {"inproc": 19, "monolith": 13, "spawn": 21}}, + "minecraft-1.21.4-fabric-iris-makeup-ultrafast-in-world": {"DirectGLES": {"inproc": 13, "monolith": 9, "spawn": 10}, "DirectVulkan": {"inproc": 11, "monolith": 8, "spawn": 8}}, + "minecraft-1.21.4-fabric-iris-mellow-in-world": {"DirectGLES": {"inproc": 7, "monolith": 11, "spawn": 9}, "DirectVulkan": {"inproc": 9, "monolith": 9, "spawn": 7}}, + "minecraft-1.21.4-fabric-iris-nostalgia-in-world": {"DirectGLES": {"inproc": 25, "monolith": 21, "spawn": 27}, "DirectVulkan": {"inproc": 24, "monolith": 26, "spawn": 16}}, + "minecraft-1.21.4-fabric-iris-photon-v1.1-in-world": {"DirectGLES": {"inproc": 25, "monolith": 34, "spawn": 21}, "DirectVulkan": {"inproc": 24, "monolith": 24, "spawn": 33}}, + "minecraft-1.21.4-fabric-iris-photon-v1.3b-in-world": {"DirectGLES": {"inproc": 29, "monolith": 36, "spawn": 29}, "DirectVulkan": {"inproc": 30, "monolith": 27, "spawn": 36}}, + "minecraft-1.21.4-fabric-iris-sundial-lite-in-world": {"DirectGLES": {"inproc": 35, "monolith": 27, "spawn": 21}, "DirectVulkan": {"inproc": 31, "monolith": 20, "spawn": 30}}, + "minecraft-1.21.4-fabric-iris-super-duper-vanilla-in-world": {"DirectGLES": {"monolith": 22, "spawn": 21}, "DirectVulkan": {"inproc": 14, "monolith": 20, "spawn": 14}}, + "minecraft-1.21.4-fabric-journeymap-in-world": {"DirectGLES": {"inproc": 43, "monolith": 66, "spawn": 65}, "DirectVulkan": {"inproc": 41, "monolith": 60, "spawn": 90}}, + "minecraft-1.21.4-fabric-journeymap-in-world-normal-world": {"DirectGLES": {"inproc": 10, "monolith": 12, "spawn": 14}, "DirectVulkan": {"inproc": 15, "monolith": 12, "spawn": 14}}, + "minecraft-1.21.4-fabric-modernui-inventory": {"DirectGLES": {"inproc": 54, "monolith": 50, "spawn": 67}, "DirectVulkan": {"inproc": 50, "monolith": 66, "spawn": 70}}, + "minecraft-1.21.4-fabric-modernui-inventory-normal-world": {"DirectGLES": {"inproc": 15, "monolith": 17, "spawn": 9}, "DirectVulkan": {"inproc": 18, "monolith": 13, "spawn": 10}}, + "minecraft-1.21.4-fabric-rei-inventory": {"DirectGLES": {"inproc": 72, "monolith": 63, "spawn": 90}, "DirectVulkan": {"inproc": 79, "monolith": 84, "spawn": 49}}, + "minecraft-1.21.4-fabric-rei-inventory-normal-world": {"DirectGLES": {"inproc": 15, "monolith": 22, "spawn": 17}, "DirectVulkan": {"inproc": 20, "monolith": 18, "spawn": 19}}, + "minecraft-1.21.4-fabric-sodium-in-world": {"DirectGLES": {"inproc": 19, "monolith": 15, "spawn": 26, "verify": 25}, "DirectVulkan": {"inproc": 29, "monolith": 32, "spawn": 23, "verify": 16}}, + "minecraft-1.21.4-fabric-xaero-minimap-in-world": {"DirectGLES": {"inproc": 41, "monolith": 51, "spawn": 64}, "DirectVulkan": {"inproc": 55, "monolith": 69, "spawn": 77}}, + "minecraft-1.21.4-fabric-xaero-minimap-in-world-normal-world": {"DirectGLES": {"inproc": 12, "monolith": 13, "spawn": 8}, "DirectVulkan": {"inproc": 9, "monolith": 10, "spawn": 13}}, + "minecraft-1.21.4-fabric-xaero-world-map-in-world": {"DirectGLES": {"inproc": 41, "monolith": 40, "spawn": 52}, "DirectVulkan": {"inproc": 42, "monolith": 43, "spawn": 59}}, + "minecraft-1.21.4-fabric-xaero-world-map-in-world-normal-world": {"DirectGLES": {"inproc": 6, "monolith": 11, "spawn": 8}, "DirectVulkan": {"inproc": 13, "monolith": 9, "spawn": 10}}, + "minecraft-1.21.4-in-world": {"DirectGLES": {"inproc": 17, "monolith": 10, "spawn": 11, "verify": 11}, "DirectVulkan": {"inproc": 10, "monolith": 9, "spawn": 12, "verify": 11}}, + "minecraft-1.21.4-main-menu": {"DirectGLES": {"inproc": 7, "monolith": 4, "spawn": 9, "verify": 6}, "DirectVulkan": {"inproc": 5, "monolith": 10, "spawn": 6, "verify": 6}}, + "minecraft-1.21.4-startup": {"DirectGLES": {"inproc": 5, "monolith": 3, "spawn": 1, "verify": 3}, "DirectVulkan": {"inproc": 1, "monolith": 4, "spawn": 2, "verify": 2}} + } + }, + "android": { + "source": "MobileGL APK run 43666299861 (2026-11-15), step 'Retrace and validate' of each one-case emulator job, APK install included", + "seconds": { + "OpenRA": {"DirectGLES": {"inproc": 28, "monolith": 28, "spawn": 24}, "DirectVulkan": {"inproc": 24, "monolith": 22, "spawn": 16}}, + "improved-transparency-minecraft-26.3": {"DirectGLES": {"inproc": 179, "monolith": 170, "spawn": 142}, "DirectVulkan": {"inproc": 169, "monolith": 209, "spawn": 153}}, + "minecraft-1.17-main-menu-854": {"DirectGLES": {"inproc": 27, "monolith": 19, "spawn": 22}, "DirectVulkan": {"inproc": 25, "monolith": 25, "spawn": 26}}, + "minecraft-1.20.1-fabric-iris-iterationt-in-world": {"DirectGLES": {"inproc": 131, "monolith": 147, "spawn": 153}, "DirectVulkan": {"inproc": 149, "monolith": 111, "spawn": 116}}, + "minecraft-1.21.1-neoforge-create-indirect-in-world-align1024": {"DirectGLES": {"inproc": 74, "monolith": 85, "spawn": 73}, "DirectVulkan": {"inproc": 44, "monolith": 31, "spawn": 53}}, + "minecraft-1.21.1-neoforge-create-instancing-in-world": {"DirectGLES": {"inproc": 74, "monolith": 56, "spawn": 78}, "DirectVulkan": {"inproc": 42, "monolith": 49, "spawn": 37}}, + "minecraft-1.21.11-main-menu": {"DirectGLES": {"inproc": 38, "monolith": 37, "spawn": 30}, "DirectVulkan": {"inproc": 46, "monolith": 25, "spawn": 45}}, + "minecraft-1.21.4-fabric-common-mods-in-world": {"DirectGLES": {"inproc": 66, "monolith": 51, "spawn": 61}, "DirectVulkan": {"inproc": 70, "monolith": 62, "spawn": 53}}, + "minecraft-1.21.4-fabric-common-mods-inventory": {"DirectGLES": {"inproc": 74, "monolith": 86, "spawn": 41}, "DirectVulkan": {"inproc": 61, "monolith": 55, "spawn": 67}}, + "minecraft-1.21.4-fabric-iris-bliss-in-world": {"DirectGLES": {"inproc": 74, "monolith": 70, "spawn": 53}, "DirectVulkan": {"inproc": 53, "monolith": 69, "spawn": 50}}, + "minecraft-1.21.4-fabric-iris-bsl-esc-menu-854": {"DirectGLES": {"inproc": 220, "monolith": 171, "spawn": 176}, "DirectVulkan": {"inproc": 138, "monolith": 234, "spawn": 193}}, + "minecraft-1.21.4-fabric-iris-bsl-in-world": {"DirectGLES": {"inproc": 60, "monolith": 42, "spawn": 48}, "DirectVulkan": {"inproc": 56, "monolith": 48, "spawn": 60}}, + "minecraft-1.21.4-fabric-iris-chocapic-v6-lite-in-world": {"DirectGLES": {"inproc": 62, "monolith": 54, "spawn": 50}, "DirectVulkan": {"inproc": 51, "monolith": 42, "spawn": 36}}, + "minecraft-1.21.4-fabric-iris-complementary-reimagined-in-world": {"DirectGLES": {"inproc": 93, "monolith": 94, "spawn": 110}, "DirectVulkan": {"inproc": 92, "monolith": 112, "spawn": 88}}, + "minecraft-1.21.4-fabric-iris-complementary-unbound-in-world": {"DirectGLES": {"inproc": 99, "monolith": 97, "spawn": 99}, "DirectVulkan": {"inproc": 98, "monolith": 107, "spawn": 115}}, + "minecraft-1.21.4-fabric-iris-derivative-main-d24.4.14-in-world": {"DirectGLES": {"inproc": 67, "monolith": 81, "spawn": 60}, "DirectVulkan": {"inproc": 77, "monolith": 76, "spawn": 47}}, + "minecraft-1.21.4-fabric-iris-iterationrp-in-world": {"DirectVulkan": {"inproc": 102, "monolith": 108, "spawn": 94}}, + "minecraft-1.21.4-fabric-iris-iterationt-in-world": {"DirectGLES": {"inproc": 40, "monolith": 57, "spawn": 48}, "DirectVulkan": {"inproc": 66, "monolith": 56, "spawn": 61}}, + "minecraft-1.21.4-fabric-iris-iterationt-nodsa-in-world": {"DirectGLES": {"inproc": 68, "monolith": 65, "spawn": 39}, "DirectVulkan": {"inproc": 43, "monolith": 70, "spawn": 66}}, + "minecraft-1.21.4-fabric-iris-makeup-ultrafast-in-world": {"DirectGLES": {"inproc": 32, "monolith": 47, "spawn": 46}, "DirectVulkan": {"inproc": 39, "monolith": 42, "spawn": 43}}, + "minecraft-1.21.4-fabric-iris-mellow-in-world": {"DirectGLES": {"inproc": 47, "monolith": 37, "spawn": 42}, "DirectVulkan": {"inproc": 46, "monolith": 40, "spawn": 40}}, + "minecraft-1.21.4-fabric-iris-nostalgia-in-world": {"DirectGLES": {"inproc": 92, "monolith": 51, "spawn": 87}, "DirectVulkan": {"inproc": 60, "monolith": 55, "spawn": 83}}, + "minecraft-1.21.4-fabric-iris-photon-v1.1-in-world": {"DirectGLES": {"inproc": 107, "monolith": 105, "spawn": 79}, "DirectVulkan": {"inproc": 59, "monolith": 100, "spawn": 96}}, + "minecraft-1.21.4-fabric-iris-photon-v1.3b-in-world": {"DirectGLES": {"inproc": 127, "monolith": 114, "spawn": 60}, "DirectVulkan": {"inproc": 94, "monolith": 106, "spawn": 81}}, + "minecraft-1.21.4-fabric-iris-sundial-lite-in-world": {"DirectGLES": {"inproc": 80, "monolith": 77, "spawn": 77}, "DirectVulkan": {"inproc": 76, "monolith": 77, "spawn": 91}}, + "minecraft-1.21.4-fabric-iris-super-duper-vanilla-in-world": {"DirectGLES": {"inproc": 71, "monolith": 63, "spawn": 48}, "DirectVulkan": {"inproc": 53, "monolith": 67, "spawn": 49}}, + "minecraft-1.21.4-fabric-journeymap-in-world": {"DirectGLES": {"inproc": 139, "monolith": 241, "spawn": 172}, "DirectVulkan": {"inproc": 180, "monolith": 257}}, + "minecraft-1.21.4-fabric-journeymap-in-world-normal-world": {"DirectGLES": {"inproc": 56, "monolith": 37, "spawn": 67}, "DirectVulkan": {"inproc": 51, "monolith": 53, "spawn": 54}}, + "minecraft-1.21.4-fabric-modernui-inventory": {"DirectGLES": {"inproc": 207, "monolith": 248, "spawn": 105}, "DirectVulkan": {"inproc": 303, "monolith": 324, "spawn": 195}}, + "minecraft-1.21.4-fabric-modernui-inventory-normal-world": {"DirectGLES": {"inproc": 56, "monolith": 58, "spawn": 53}, "DirectVulkan": {"inproc": 60, "monolith": 58, "spawn": 81}}, + "minecraft-1.21.4-fabric-rei-inventory": {"DirectGLES": {"inproc": 328, "monolith": 337, "spawn": 144}, "DirectVulkan": {"inproc": 325, "monolith": 197, "spawn": 199}}, + "minecraft-1.21.4-fabric-rei-inventory-normal-world": {"DirectGLES": {"inproc": 79, "monolith": 62, "spawn": 46}, "DirectVulkan": {"inproc": 82, "monolith": 51, "spawn": 66}}, + "minecraft-1.21.4-fabric-sodium-in-world": {"DirectGLES": {"inproc": 178, "monolith": 175, "spawn": 91}, "DirectVulkan": {"inproc": 166, "monolith": 136, "spawn": 72}}, + "minecraft-1.21.4-fabric-xaero-minimap-in-world": {"DirectGLES": {"inproc": 231, "monolith": 254, "spawn": 99}, "DirectVulkan": {"inproc": 278, "monolith": 225, "spawn": 127}}, + "minecraft-1.21.4-fabric-xaero-minimap-in-world-normal-world": {"DirectGLES": {"inproc": 61, "monolith": 47, "spawn": 46}, "DirectVulkan": {"inproc": 61, "monolith": 44, "spawn": 42}}, + "minecraft-1.21.4-fabric-xaero-world-map-in-world": {"DirectGLES": {"inproc": 313, "monolith": 223, "spawn": 104}, "DirectVulkan": {"inproc": 276, "monolith": 277, "spawn": 121}}, + "minecraft-1.21.4-fabric-xaero-world-map-in-world-normal-world": {"DirectGLES": {"inproc": 62, "monolith": 47, "spawn": 41}, "DirectVulkan": {"inproc": 52, "monolith": 40, "spawn": 69}}, + "minecraft-1.21.4-in-world": {"DirectGLES": {"inproc": 44, "monolith": 59, "spawn": 43}, "DirectVulkan": {"inproc": 53, "monolith": 59, "spawn": 51}}, + "minecraft-1.21.4-main-menu": {"DirectGLES": {"inproc": 54, "monolith": 48, "spawn": 35}, "DirectVulkan": {"inproc": 58, "monolith": 67, "spawn": 46}}, + "minecraft-1.21.4-startup": {"DirectGLES": {"inproc": 22, "monolith": 19, "spawn": 19}, "DirectVulkan": {"inproc": 18, "monolith": 14, "spawn": 14}} + } + } +}