[CI] (.github/workflows): run the validation branch in the split CI shape

- Android inproc and all three Linux transports still replay every case
- A manual run with android_full_matrix replays the full emulator matrix

.github/scripts/trace-fixture-bundle.sh

@@ -0,0 +1,304 @@
+#!/usr/bin/env bash
+# One cache entry that holds every Git LFS trace fixture of the CI cases.
+#
+#   key                derive the actions/cache key, restore prefix and path of the bundle
+#   ensure             after a cache miss: complete the bundle directory, fetching only the
+#                      cases whose files are missing from it or do not match their pointers
+#   wait <key> [secs]  wait until the cache entry <key> is listed for this ref (needs GH_TOKEN)
+#   install <case>...  move the fixtures of these cases from the restored bundle directory into
+#                      the fixture directory, fetch the cases it does not hold, and check every
+#                      file against its pointer
+#
+# The key is content-addressed on the pointer oids tracked at HEAD, which a plain checkout
+# reads without smudging. The bundle directory is the same path for every revision, so
+# `ensure` can start from an older bundle. Fixtures stored in Git itself rather than in Git
+# LFS (OpenRA) are not in the bundle; `install` checks that they are real files.
+set -euo pipefail
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=trace-fixture-lib.sh
+. "${script_dir}/trace-fixture-lib.sh"
+
+# Bump when the key derivation or the bundle layout changes in a way that must invalidate
+# old entries; the content digest alone would not notice a format change.
+key_schema="v1"
+bundle_dir="trace-fixture-bundle"
+fixture_dir="tools/trace_replay/fixtures"
+python_bin="${PYTHON:-python3}"
+
+if ! command -v "${python_bin}" >/dev/null 2>&1 && command -v python >/dev/null 2>&1; then
+  python_bin=python
+fi
+
+usage() {
+  echo "usage: $0 <key|ensure|wait <key> [seconds]|install <trace-case>...>" >&2
+  exit 2
+}
+
+# Writes "name=value" to $GITHUB_OUTPUT when running under Actions, and to stdout as well so
+# the script stays runnable off-CI.
+emit_output() {
+  local name="$1"
+  local value="$2"
+  if [ -n "${GITHUB_OUTPUT:-}" ]; then
+    printf '%s=%s\n' "${name}" "${value}" >> "${GITHUB_OUTPUT}"
+  fi
+  printf '%s=%s\n' "${name}" "${value}"
+}
+
+# "<case><TAB><file>" for every fixture file of every CI case.
+ci_fixture_manifest() {
+  "${python_bin}" tools/trace_replay/trace_cases.py --ci --format fixture-manifest \
+    --fixture-root "${fixture_dir}" | tr -d '\r'
+}
+
+is_lfs_fixture() {
+  get_lfs_metadata "$1" >/dev/null 2>&1
+}
+
+# A fixture that Git stores directly must be a real file, not a pointer left by a wrong
+# attribute.
+check_plain_fixture() {
+  local file="$1"
+  if [ ! -s "${file}" ]; then
+    echo "fixture file is missing or empty: ${file}" >&2
+    return 1
+  fi
+  if head -n 1 "${file}" | grep -q "version https://git-lfs.github.com/spec/v1"; then
+    echo "fixture should not be stored as an LFS pointer: ${file}" >&2
+    return 1
+  fi
+}
+
+# Puts the tracked pointer files back. The smudge filter is skipped so that a clone with Git
+# LFS set up does not download the objects here.
+restore_pointers() {
+  GIT_LFS_SKIP_SMUDGE=1 git checkout -- "$@"
+}
+
+verify_lfs_fixture() {
+  local on_disk="$1"
+  local tracked="$2"
+  local metadata expected_oid expected_size
+  metadata="$(get_lfs_metadata "${tracked}")" || return 1
+  read -r expected_oid expected_size <<< "${metadata}"
+  verify_fixture_file "${on_disk}" "${tracked}" "${expected_oid}" "${expected_size}"
+}
+
+# Moves the given Git LFS fixtures of one case from the bundle directory into the fixture
+# directory. Succeeds when every one of them is then in place and matches its pointer.
+take_from_bundle() {
+  local file bundled
+  local incomplete=0
+  for file in "$@"; do
+    bundled="${bundle_dir}/$(basename "${file}")"
+    # Two cases may share a file; the second finds it already in place.
+    if [ -f "${bundled}" ]; then
+      mv -f "${bundled}" "${file}"
+    fi
+    if ! verify_lfs_fixture "${file}" "${file}" 2>/dev/null; then
+      incomplete=1
+    fi
+  done
+  return "${incomplete}"
+}
+
+# Downloads the given Git LFS fixtures of one case over their pointers: the mirrors first,
+# Git LFS for what no mirror serves. The fetch script verifies what it downloaded.
+fetch_case_fixtures() {
+  local case_name="$1"
+  shift
+  local file
+  for file in "$@"; do
+    rm -f "${file}" "${file}.tmp"
+  done
+  restore_pointers "$@" || return 1
+  bash "${script_dir}/fetch-trace-fixture-lfs.sh" "${case_name}" "${fixture_dir}"
+}
+
+[ "$#" -ge 1 ] || usage
+command_name="$1"
+shift
+
+case "${command_name}" in
+  key)
+    [ "$#" -eq 0 ] || usage
+    # One "<basename> <oid> <size>" line per bundled file, sorted, each file once.
+    manifest=""
+    count=0
+    while IFS= read -r file; do
+      [ -n "${file}" ] || continue
+      if ! metadata="$(get_lfs_metadata "${file}" 2>/dev/null)"; then
+        continue
+      fi
+      read -r expected_oid expected_size <<< "${metadata}"
+      manifest+="$(basename "${file}") ${expected_oid} ${expected_size}"$'\n'
+      count=$((count + 1))
+    done < <(ci_fixture_manifest | cut -f2 | LC_ALL=C sort -u)
+    if [ "${count}" -eq 0 ]; then
+      echo "no Git LFS fixture found for the CI trace cases; the bundle would be empty" >&2
+      exit 1
+    fi
+    duplicates="$(printf '%s' "${manifest}" | cut -d' ' -f1 | LC_ALL=C sort | uniq -d)"
+    if [ -n "${duplicates}" ]; then
+      echo "two fixture files share a name, so the bundle directory cannot hold both: ${duplicates}" >&2
+      exit 1
+    fi
+    digest="$(printf '%s' "${manifest}" | sha256sum | awk '{ print substr($1, 1, 16) }')"
+    emit_output "key" "trace-fixture-bundle-${key_schema}-${digest}"
+    emit_output "restore-prefix" "trace-fixture-bundle-${key_schema}-"
+    emit_output "path" "${bundle_dir}"
+    emit_output "files" "${count}"
+    ;;
+
+  ensure)
+    [ "$#" -eq 0 ] || usage
+    mkdir -p "${bundle_dir}" "${fixture_dir}"
+    mapfile -t manifest_lines < <(ci_fixture_manifest)
+    declare -A case_seen=()
+    case_order=()
+    for line in "${manifest_lines[@]}"; do
+      case_name="${line%%$'\t'*}"
+      if [ -z "${case_seen[${case_name}]:-}" ]; then
+        case_seen["${case_name}"]=1
+        case_order+=("${case_name}")
+      fi
+    done
+
+    fetched=0
+    reused=0
+    bundled_files=()
+    for case_name in "${case_order[@]}"; do
+      lfs_files=()
+      for line in "${manifest_lines[@]}"; do
+        [ "${line%%$'\t'*}" = "${case_name}" ] || continue
+        file="${line#*$'\t'}"
+        if is_lfs_fixture "${file}"; then
+          lfs_files+=("${file}")
+        else
+          check_plain_fixture "${file}"
+        fi
+      done
+      [ "${#lfs_files[@]}" -gt 0 ] || continue
+
+      # Take what an older bundle already holds, and fetch the case if that is not all of it.
+      if take_from_bundle "${lfs_files[@]}"; then
+        reused=$((reused + 1))
+      else
+        echo "Fetching trace fixtures of ${case_name}: the bundle does not hold them"
+        fetch_case_fixtures "${case_name}" "${lfs_files[@]}"
+        fetched=$((fetched + 1))
+      fi
+      bundled_files+=("${lfs_files[@]}")
+    done
+
+    # The bundle is rebuilt from the verified files, so nothing an older bundle carried and
+    # no case names any more is saved again.
+    rm -rf "${bundle_dir}"
+    mkdir -p "${bundle_dir}"
+    declare -A file_seen=()
+    saved=0
+    for file in "${bundled_files[@]}"; do
+      [ -z "${file_seen[${file}]:-}" ] || continue
+      file_seen["${file}"]=1
+      verify_lfs_fixture "${file}" "${file}"
+      mv -f "${file}" "${bundle_dir}/$(basename "${file}")"
+      saved=$((saved + 1))
+    done
+    if [ "${saved}" -eq 0 ]; then
+      echo "the trace fixture bundle came out empty" >&2
+      exit 1
+    fi
+    # Leave the working tree as a checkout has it: pointers in the fixture directory.
+    restore_pointers "${!file_seen[@]}"
+    echo "Trace fixture bundle: ${saved} file(s), ${reused} case(s) from the restored bundle, ${fetched} case(s) fetched."
+    emit_output "fetched" "${fetched}"
+    ;;
+
+  wait)
+    [ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage
+    key="$1"
+    limit="${2:-300}"
+    # Another workflow of the same push may be the one saving this key. The replay jobs
+    # restore the entry, so it has to be listed before this job ends. The caller checks the
+    # entry itself afterwards; this only waits, and never fails.
+    #
+    # The listing is narrowed to the ref of this run. The key is content-addressed, so another
+    # branch may already hold an entry under it, and that entry cannot be restored from here:
+    # counting it would end the wait while the entry of this ref is still being uploaded.
+    query=(-f "key=${key}" -f per_page=100)
+    if [ -n "${GITHUB_REF:-}" ]; then
+      query+=(-f "ref=${GITHUB_REF}")
+    fi
+    waited=0
+    refused=0
+    while :; do
+      if listed="$(gh api -X GET "repos/${GITHUB_REPOSITORY}/actions/caches" "${query[@]}" \
+            --jq '[.actions_caches[] | select(.key == "'"${key}"'")] | length' 2>/dev/null)"; then
+        if [ "${listed:-0}" -ge 1 ]; then
+          echo "cache entry ${key} is listed after ${waited} s"
+          break
+        fi
+      else
+        refused=$((refused + 1))
+        if [ "${refused}" -ge 3 ]; then
+          echo "the cache listing cannot be read (no token, or no actions: read permission); not waiting"
+          break
+        fi
+      fi
+      if [ "${waited}" -ge "${limit}" ]; then
+        echo "cache entry ${key} is not listed after ${waited} s"
+        break
+      fi
+      sleep 10
+      waited=$((waited + 10))
+    done
+    ;;
+
+  install)
+    [ "$#" -ge 1 ] || usage
+    installed=0
+    fetched=0
+    for case_name in "$@"; do
+      mapfile -t files < <(trace_fixture_files "${case_name}" "${fixture_dir}" "${python_bin}")
+      if [ "${#files[@]}" -eq 0 ]; then
+        echo "no fixture files declared for trace case: ${case_name}" >&2
+        exit 1
+      fi
+      lfs_files=()
+      for file in "${files[@]}"; do
+        if is_lfs_fixture "${file}"; then
+          lfs_files+=("${file}")
+        else
+          check_plain_fixture "${file}"
+        fi
+      done
+      [ "${#lfs_files[@]}" -gt 0 ] || continue
+
+      # A bundle that was not restored (the cache entry was evicted, or its download failed)
+      # or that lacks a file costs this job a download of the case, not the replay.
+      if ! take_from_bundle "${lfs_files[@]}"; then
+        echo "::warning::the restored fixture bundle does not hold the trace fixtures of ${case_name}; fetching them"
+        if ! fetch_case_fixtures "${case_name}" "${lfs_files[@]}"; then
+          echo "::error::the trace fixtures of ${case_name} are not in the restored bundle and could not be fetched"
+          exit 1
+        fi
+        fetched=$((fetched + 1))
+        for file in "${lfs_files[@]}"; do
+          if ! verify_lfs_fixture "${file}" "${file}"; then
+            echo "::error::trace fixture ${file} of ${case_name} does not match its Git LFS pointer after the fetch"
+            exit 1
+          fi
+        done
+      fi
+      installed=$((installed + ${#lfs_files[@]}))
+    done
+    # The rest of the bundle belongs to other shards.
+    rm -rf "${bundle_dir}"
+    echo "Installed and verified ${installed} Git LFS fixture file(s) for $# trace case(s), ${fetched} case(s) fetched."
+    ;;
+
+  *)
+    usage
+    ;;
+esac

.github/scripts/trace-fixture-cache.sh

@@ -1,117 +0,0 @@
-#!/usr/bin/env bash
-# Cache-side helper for trace fixtures.
-#
-#   key <case> [fixture-dir]     derive the actions/cache key and path list
-#   verify <case> [fixture-dir]  check restored fixtures against their pointers
-#   reset <case> [fixture-dir]   drop restored fixtures, leaving the pointers
-#
-# The cache key is content-addressed on the Git LFS pointer oids tracked at
-# HEAD, which are readable from a plain checkout without smudging. Fixture
-# content therefore maps 1:1 onto a key: unchanged content hits, changed
-# content is a new key and thus a miss, and the download path handles it. The
-# key deliberately carries no restore-keys prefix in the workflow - a fixture
-# that does not match the pointer exactly must never be restored.
-set -euo pipefail
-
-script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=trace-fixture-lib.sh
-. "${script_dir}/trace-fixture-lib.sh"
-
-# Bump when the key derivation changes in a way that must invalidate old
-# entries; the content digest alone would not notice a format change.
-key_schema="v1"
-
-if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then
-  echo "usage: $0 <key|verify|reset> <trace-case> [fixture-dir]" >&2
-  exit 2
-fi
-
-command_name="$1"
-case_name="$2"
-fixture_dir="${3:-tools/trace_replay/fixtures}"
-python_bin="${PYTHON:-python3}"
-
-if ! command -v "${python_bin}" >/dev/null 2>&1 && command -v python >/dev/null 2>&1; then
-  python_bin=python
-fi
-
-mapfile -t files < <(trace_fixture_files "${case_name}" "${fixture_dir}" "${python_bin}")
-if [ "${#files[@]}" -eq 0 ]; then
-  echo "no fixture files declared for trace case: ${case_name}" >&2
-  exit 1
-fi
-
-# Writes "name=value" to $GITHUB_OUTPUT when running under Actions, and to
-# stdout otherwise so the script stays runnable (and testable) off-CI.
-emit_output() {
-  local name="$1"
-  local value="$2"
-  if [ -n "${GITHUB_OUTPUT:-}" ]; then
-    if [[ "${value}" == *$'\n'* ]]; then
-      local delimiter="ghadelim_$(date +%s%N)_$$"
-      {
-        printf '%s<<%s\n' "${name}" "${delimiter}"
-        printf '%s\n' "${value}"
-        printf '%s\n' "${delimiter}"
-      } >> "${GITHUB_OUTPUT}"
-    else
-      printf '%s=%s\n' "${name}" "${value}" >> "${GITHUB_OUTPUT}"
-    fi
-  fi
-  printf '%s=%s\n' "${name}" "${value}"
-}
-
-sanitize_case() {
-  printf '%s' "$1" | sed 's/[^A-Za-z0-9._-]/_/g'
-}
-
-case "${command_name}" in
-  key)
-    manifest=""
-    for file in "${files[@]}"; do
-      # A case whose fixtures are committed directly rather than through Git LFS
-      # (OpenRA) has no pointer oid to key on, and nothing to download either.
-      # Report it as uncacheable so the workflow skips the cache entirely.
-      if ! metadata="$(get_lfs_metadata "${file}" 2>/dev/null)"; then
-        echo "trace case ${case_name} is not stored in Git LFS; skipping fixture cache" >&2
-        emit_output "cacheable" "false"
-        emit_output "key" ""
-        exit 0
-      fi
-      read -r expected_oid expected_size <<< "${metadata}"
-      manifest+="$(basename "${file}") ${expected_oid} ${expected_size}"$'\n'
-    done
-
-    digest="$(printf '%s' "${manifest}" | sha256sum | awk '{ print substr($1, 1, 16) }')"
-    safe_case="$(sanitize_case "${case_name}")"
-
-    emit_output "cacheable" "true"
-    emit_output "key" "trace-fixture-${key_schema}-${safe_case}-${digest}"
-    emit_output "paths" "$(printf '%s\n' "${files[@]}")"
-    ;;
-
-  verify)
-    for file in "${files[@]}"; do
-      metadata="$(get_lfs_metadata "${file}")"
-      read -r expected_oid expected_size <<< "${metadata}"
-      verify_fixture_file "${file}" "${file}" "${expected_oid}" "${expected_size}"
-    done
-    echo "Verified ${#files[@]} fixture file(s) for ${case_name} against the tracked Git LFS pointers."
-    ;;
-
-  reset)
-    # Put the working tree back to the pointer files a fresh checkout would
-    # have, so that a rejected cache entry falls through to exactly the same
-    # download path a cache miss takes.
-    for file in "${files[@]}"; do
-      rm -f "${file}" "${file}.tmp"
-    done
-    git checkout -- "${files[@]}"
-    echo "Reset ${#files[@]} fixture file(s) for ${case_name} to their tracked Git LFS pointers."
-    ;;
-
-  *)
-    echo "unknown command: ${command_name}" >&2
-    exit 2
-    ;;
-esac

.github/scripts/trace-fixture-lib.sh

@@ -2,7 +2,7 @@
 # Shared helpers for trace-fixture handling: reading the in-tree Git LFS pointer
 # metadata and verifying a fixture file against it. Sourced by
 # fetch-trace-fixture-lfs.sh (verify after download) and by
-# trace-fixture-cache.sh (cache key derivation and verify after cache restore),
+# trace-fixture-bundle.sh (cache key derivation and verify after cache restore),
 # so both paths agree on what a valid fixture is.
 
 # Reads the Git LFS pointer tracked at HEAD for a fixture path and prints

.github/workflows/apk.yml

@@ -11,17 +11,46 @@ on:
       # someone remembering to dispatch the workflow by hand.
       - feat/disaggregated
       - codex/fix-github-ci
+      - ci/round-30min
       - cleanup
-    # A docs-only push builds nothing this workflow tests (231 emulator legs per run). The Test
+    # A push that only touches these paths changes nothing this workflow builds or replays. The
-    # workflow keeps running for docs: it hosts the doc citation lint.
+    # Test workflow keeps running for docs: it hosts the doc citation lint. The Android build
+    # compiles none of the unit, integration or benchmark trees, with one exception that keeps
+    # MobileGL/MG_Benchmark from being listed whole: MG_Util/SelfTest/DriverBenchJni.cpp
+    # includes MG_Benchmark/Driver/DriverBenchCases.inc.
     paths-ignore:
       - 'docs/**'
       - '**/*.md'
+      - 'MobileGL/MG_Test/**'
+      - 'MobileGL/MG_IntegrationTest/**'
+      - 'MobileGL/MG_Benchmark/CMakeLists.txt'
+      - 'MobileGL/MG_Benchmark/SanityBench.cpp'
+      - 'MobileGL/MG_Benchmark/Buffer/**'
+      - 'MobileGL/MG_Benchmark/Container/**'
+      - 'MobileGL/MG_Benchmark/Program/**'
+      - 'MobileGL/MG_Benchmark/ShaderCache/**'
+      - 'MobileGL/MG_Benchmark/Transpile/**'
+      - 'tools/cts/**'
+      - 'tools/device_bench/**'
+      - '.github/workflows/test.yml'
+      - '.github/workflows/hygiene.yml'
+      - '.github/actions/setup-test-runner/**'
   workflow_dispatch:
+    inputs:
+      android_full_matrix:
+        description: Replay every trace case on the monolith and spawn emulator lanes too
+        type: boolean
+        default: false
 
 env:
   # Software-rendered CI submissions may exceed the interactive GPU hang budget.
   MOBILEGL_GPU_HANG_BUDGET_MS: "0"
+  # The emulator replays every trace case on inproc, the transport the launcher ships. On
+  # monolith and spawn it replays the android_smoke cases of trace_cases.json, because
+  # test.yml replays every case on all three transports on Linux. A manual run with
+  # android_full_matrix replays every case on all three emulator lanes. A push has no inputs,
+  # so it gets the reduced lanes.
+  MOBILEGL_ANDROID_LANES: ${{ inputs.android_full_matrix && 'full' || 'inproc-full' }}
 
 # ONE LIVE RUN PER BRANCH, as in test.yml: a superseded push is canceled rather than left queued
 # behind its successor; a manual dispatch gets a group of its own; dev queues rather than cancels.
@@ -39,16 +68,26 @@ jobs:
       CCACHE_BASEDIR: ${{ github.workspace }}
       CCACHE_COMPRESS: "true"
       CCACHE_DIR: ${{ github.workspace }}/.ccache
-      CCACHE_MAXSIZE: 4G
+      # One build's objects (three native configurations) take about 1 GB. The limit leaves
+      # half of that again for the objects a push replaces, and is what the cleanup before
+      # the save trims the entry to.
+      CCACHE_MAXSIZE: 1500M
       CCACHE_NOHASHDIR: "true"
       MOBILEGL_CMAKE_COMPILER_LAUNCHER: ccache
-      MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
+      MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'ci/round-30min' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
 
     steps:
       - name: Checkout repo
         uses: actions/checkout@v6
-        with:
+
-          submodules: recursive
+      # Only the submodules the two APKs are built from, shallow and in parallel, in place of a
+      # recursive checkout of every submodule in the tree: the ones the root CMakeLists.txt
+      # configures, and apitrace with the directories its thirdparty/CMakeLists.txt requires,
+      # for the trace APK. The step fails when a file the build reads from one of them did
+      # not arrive (scripts/ci/checkout_submodules.sh).
+      - name: Check out build submodules
+        timeout-minutes: 15
+        run: bash scripts/ci/checkout_submodules.sh retrace
 
       - name: Set artifact metadata
         run: |
@@ -82,6 +121,9 @@ jobs:
           sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 update
           sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 install -y ccache
           ccache --version
+          # The restored entry carries the counters of every earlier build; zero them so the
+          # statistics printed after the build are this run's.
+          ccache --zero-stats
 
       - name: Setup Android SDK
         id: netsetup84
@@ -183,11 +225,20 @@ jobs:
         if: always()
         run: ccache --show-stats
 
-      # Rewrite one rolling entry per job and branch. The upload stays cumulative - it
+      # A build served from the cache stores nothing, so nothing would ever evict the objects
-      # carries every object restored at the top of this run plus the few TUs that
+      # of older source states. Trim to CCACHE_MAXSIZE, least recently used first, before the
-      # changed - but Actions cache keys are immutable, so this branch's superseded
+      # entry is saved; the objects this build hit were just touched and stay.
-      # blob is released first; --ref keeps the delete off other branches' entries.
+      - name: Trim ccache to its size limit
-      # Running after the build means a failed build leaves the entry untouched.
+        run: |
+          ccache --cleanup
+          ccache --show-stats | grep -i 'cache size' || true
+          du -sh "${CCACHE_DIR}"
+
+      # Rewrite one rolling entry per job and branch. The upload carries the objects this
+      # build used plus what fits under the size limit, but Actions cache keys are
+      # immutable, so this branch's superseded blob is released first; --ref keeps the
+      # delete off other branches' entries. Running after the build means a failed build
+      # leaves the entry untouched.
       - name: Release superseded ccache entry
         env:
           GH_TOKEN: ${{ github.token }}
@@ -251,10 +302,18 @@ jobs:
     name: trace case matrix
     runs-on: ubuntu-26.04
     env:
-      MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
+      MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'ci/round-30min' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
+      # How much replay one emulator job aims for, in seconds as the duration hints count them.
+      # A (lane, transport, backend) cell is cut into as many shards as that takes, and its
+      # cases are spread so the shards of a cell come out even (tools/trace_replay/trace_cases.py,
+      # shard_matrix). A smaller value gives more and shorter jobs, each of which pays the
+      # emulator boot again; every case is replayed once per cell whatever the value.
+      RETRACE_SHARD_SECONDS: "720"
+      # The most a shard runner may spend on its cases. A shard gets the time in which every
+      # one of its cases could run to the end of its own time budget, up to this cap.
+      RETRACE_SHARD_DEADLINE_CAP_SECONDS: "7200"
     outputs:
       android: ${{ steps.trace-cases.outputs.android }}
-      names: ${{ steps.trace-cases.outputs.names }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v6
@@ -263,77 +322,109 @@ jobs:
         id: trace-cases
         run: |
           python3 tools/trace_replay/trace_cases.py --self-test-transport-matrices
-          matrix_format=github-apk-matrix
+          # The retrace jobs replay shards: several cases of one (lane, transport, backend) cell
+          # per emulator boot. The self-test proves every shard matrix replays exactly the rows
+          # of the per-case matrix it is built from, and trace_cases.py refuses to print one
+          # that does not.
+          python3 tools/trace_replay/trace_cases.py --self-test-shard-matrices
+          matrix_format=github-apk-shard-matrix
           if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then
-            matrix_format=github-apk-transport-matrix
+            matrix_format=github-apk-transport-shard-matrix
           fi
-          echo "android=$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}")" >> "$GITHUB_OUTPUT"
+          android="$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}" \
-          echo "names=$(python3 tools/trace_replay/trace_cases.py --ci --format names)" >> "$GITHUB_OUTPUT"
+            --shard-seconds "${RETRACE_SHARD_SECONDS}" --android-lanes "${MOBILEGL_ANDROID_LANES}")"
+          # An empty `include` is not an error to GitHub: it skips the retrace job with no red.
+          shards="$(printf '%s' "${android}" | python3 -c 'import json, sys; print(len(json.load(sys.stdin)["include"]))')"
+          if [ "${shards}" -lt 1 ]; then
+            echo "::error::the Android retrace matrix is empty, so the retrace job would be skipped with no red"
+            exit 1
+          fi
+          # The cases of a shard share one job, so a case that hangs spends time the cases after
+          # it need. retrace_shard_budget.py gives every shard its own deadline_seconds, the
+          # time in which each of its cases could run to the end of its time budget, capped so
+          # that a shard whose replays all hang cannot hold a runner for hours, and the
+          # timeout_minutes of its job. The retrace job reads both from the matrix.
+          android="$(printf '%s' "${android}" | python3 android-plugin/retrace_shard_budget.py \
+            --cap-seconds "${RETRACE_SHARD_DEADLINE_CAP_SECONDS}")"
+          echo "android=${android}" >> "$GITHUB_OUTPUT"
+          lanes=""
+          if [ "${matrix_format}" = "github-apk-transport-shard-matrix" ]; then
+            lanes=" (${MOBILEGL_ANDROID_LANES} lanes)"
+          fi
+          echo "Android retrace: ${shards} shard(s) from ${matrix_format}${lanes} at ${RETRACE_SHARD_SECONDS} s per shard"
 
+  # One cache entry holds every Git LFS fixture of the CI cases, and each retrace job restores
+  # that entry itself. This job only makes sure the entry exists. On a hit it downloads
+  # nothing. On a miss it starts from the newest older bundle, fetches from the mirror the
+  # cases whose pointers changed, saves the entry under the new key and confirms it is there,
+  # so that no retrace job has to download fixtures itself. The key, the path and the steps
+  # are .github/scripts/trace-fixture-bundle.sh's, and test.yml has the same job for the same
+  # entry: the two workflows of one push look the key up at the same time, and on a miss both
+  # fetch and both save. The save that loses is not an error, since the entry it wanted to
+  # write is the one the confirming step then finds. The timeout bounds a fixture download
+  # that stalls, which would otherwise hold every retrace job behind it.
   trace-fixtures:
-    name: trace fixture (${{ matrix.case }})
+    name: trace fixtures
     runs-on: ubuntu-26.04
-    needs: trace-cases
+    timeout-minutes: 30
-    strategy:
+    permissions:
-      fail-fast: false
+      actions: read
-      max-parallel: 4
+      contents: read
-      matrix:
+    outputs:
-        case: ${{ fromJSON(needs.trace-cases.outputs.names) }}
+      key: ${{ steps.bundle.outputs.key }}
+      path: ${{ steps.bundle.outputs.path }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v6
 
-      - name: Derive trace fixture cache key
+      - name: Derive the fixture bundle key
-        id: fixture-key
+        id: bundle
-        run: bash .github/scripts/trace-fixture-cache.sh key '${{ matrix.case }}'
+        run: bash .github/scripts/trace-fixture-bundle.sh key
 
-      - name: Restore trace fixture cache
+      - name: Look up the fixture bundle
-        id: fixture-cache
+        id: lookup
-        if: steps.fixture-key.outputs.cacheable == 'true'
         uses: actions/cache/restore@v5
         with:
-          path: ${{ steps.fixture-key.outputs.paths }}
+          path: ${{ steps.bundle.outputs.path }}
-          key: ${{ steps.fixture-key.outputs.key }}
+          key: ${{ steps.bundle.outputs.key }}
+          lookup-only: true
 
-      - name: Verify restored trace fixture
+      - name: Restore the newest older bundle
-        id: fixture-verify
+        if: steps.lookup.outputs.cache-hit != 'true'
-        if: steps.fixture-cache.outputs.cache-hit == 'true'
+        uses: actions/cache/restore@v5
-        run: |
+        with:
-          if bash .github/scripts/trace-fixture-cache.sh verify '${{ matrix.case }}'; then
+          path: ${{ steps.bundle.outputs.path }}
-            echo "ok=true" >> "$GITHUB_OUTPUT"
+          key: ${{ steps.bundle.outputs.key }}
-          else
+          restore-keys: ${{ steps.bundle.outputs.restore-prefix }}
-            echo "ok=false" >> "$GITHUB_OUTPUT"
-            echo "::warning::Cached fixture for ${{ matrix.case }} failed verification; falling back to the download path"
-            bash .github/scripts/trace-fixture-cache.sh reset '${{ matrix.case }}'
-          fi
 
-      - name: Fetch trace fixture
+      - name: Complete the fixture bundle
-        if: steps.fixture-verify.outputs.ok != 'true'
+        if: steps.lookup.outputs.cache-hit != 'true'
-        run: bash .github/scripts/fetch-trace-fixture-lfs.sh '${{ matrix.case }}'
+        run: bash .github/scripts/trace-fixture-bundle.sh ensure
 
-      - name: Save trace fixture cache
+      - name: Save the fixture bundle
-        if: steps.fixture-key.outputs.cacheable == 'true' && steps.fixture-cache.outputs.cache-hit != 'true'
+        if: steps.lookup.outputs.cache-hit != 'true'
+        continue-on-error: true
         uses: actions/cache/save@v5
         with:
-          path: ${{ steps.fixture-key.outputs.paths }}
+          path: ${{ steps.bundle.outputs.path }}
-          key: ${{ steps.fixture-key.outputs.key }}
+          key: ${{ steps.bundle.outputs.key }}
 
-      - name: Stage trace fixture
+      # The Test workflow of the same push may be the one whose save won. Its entry is as good,
-        run: |
+      # once it is listed.
-          safe_case="$(printf '%s' '${{ matrix.case }}' | sed 's/[^A-Za-z0-9._-]/_/g')"
+      - name: Wait for the fixture bundle entry
-          stage_dir="trace-fixtures/${safe_case}"
+        if: steps.lookup.outputs.cache-hit != 'true'
-          mkdir -p "${stage_dir}"
+        env:
-          python3 tools/trace_replay/trace_cases.py --format fixture-files --case '${{ matrix.case }}' |
+          GH_TOKEN: ${{ github.token }}
-            while IFS= read -r file; do
+        run: bash .github/scripts/trace-fixture-bundle.sh wait '${{ steps.bundle.outputs.key }}'
-              cp "${file}" "${stage_dir}/"
-            done
 
-      - name: Upload trace fixture
+      - name: Confirm the fixture bundle entry
-        uses: actions/upload-artifact@v7
+        if: steps.lookup.outputs.cache-hit != 'true'
+        uses: actions/cache/restore@v5
         with:
-          name: MobileGL-trace-fixture-${{ matrix.case }}
+          path: ${{ steps.bundle.outputs.path }}
-          path: trace-fixtures/**
+          key: ${{ steps.bundle.outputs.key }}
-          if-no-files-found: error
+          lookup-only: true
+          fail-on-cache-miss: true
 
   android-avd:
     name: android avd image
@@ -347,8 +438,33 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v6
 
+      # The emulator shard loop and the artifact cleanup, driven by fakes: what each case is
+      # run with, a verdict for every case, and what stays after a failure. It runs in this
+      # job because the retrace jobs wait for it and the APK build does not.
+      - name: Emulator shard runner regression
+        run: |
+          python3 android-plugin/test_trace_replay_shard.py
+          python3 android-plugin/test_retrace_artifact_cleanup.py
+
+      # This job only makes sure the cache entry exists; the retrace jobs restore it. A hit
+      # downloads nothing and skips every step below; a miss builds the AVD, and the post step
+      # of this action saves it.
+      - name: Look up Android AVD cache
+        id: android-avd-cache
+        uses: actions/cache@v5
+        with:
+          lookup-only: true
+          path: |
+            ${{ env.ANDROID_AVD_HOME }}
+            ${{ env.ANDROID_SDK_ROOT }}/emulator
+            ${{ env.ANDROID_SDK_ROOT }}/platform-tools
+            ${{ env.ANDROID_SDK_ROOT }}/platforms/android-35
+            ${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64
+          key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v4-${{ hashFiles('android-plugin/run-avd-ci.sh') }}
+
       - name: Setup Android SDK
         id: netsetup336
+        if: steps.android-avd-cache.outputs.cache-hit != 'true'
         continue-on-error: true
         timeout-minutes: 15
         uses: android-actions/setup-android@v4
@@ -367,20 +483,9 @@ jobs:
           cmdline-tools-version: 13114758
 
       - name: Accept Android SDK licenses
+        if: steps.android-avd-cache.outputs.cache-hit != 'true'
         run: yes | sdkmanager --licenses >/dev/null
 
-      - name: Restore Android AVD cache
-        id: android-avd-cache
-        uses: actions/cache@v5
-        with:
-          path: |
-            ${{ env.ANDROID_AVD_HOME }}
-            ${{ env.ANDROID_SDK_ROOT }}/emulator
-            ${{ env.ANDROID_SDK_ROOT }}/platform-tools
-            ${{ env.ANDROID_SDK_ROOT }}/platforms/android-35
-            ${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64
-          key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v4-${{ hashFiles('android-plugin/run-avd-ci.sh') }}
-
       - name: Create AVD
         if: steps.android-avd-cache.outputs.cache-hit != 'true'
         run: |
@@ -418,16 +523,27 @@ jobs:
           fi
           du -sh "${ANDROID_AVD_HOME}"
 
+  # One job replays a shard: some of the cases of one lane, transport and backend, one after
+  # another on one emulator boot (android-plugin/trace_replay_shard.py). The shard's step
+  # summary has a row per case, and a case that did not pass has an error annotation of its own.
   retrace:
-    name: retrace (${{ matrix.backend.name }} / ${{ matrix.transport || 'monolith' }}, ${{ matrix.case.name }})
+    name: retrace (${{ matrix.backend.name }} / ${{ matrix.transport || 'monolith' }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
     runs-on: ubuntu-26.04
     needs:
       - build
       - android-avd
       - trace-cases
       - trace-fixtures
-    if: ${{ !cancelled() && needs.build.result == 'success' && needs.android-avd.result == 'success' && needs.trace-cases.result == 'success' }}
+    if: ${{ !cancelled() && needs.build.result == 'success' && needs.android-avd.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }}
-    timeout-minutes: 75
+    # Paired with --deadline-seconds below, and far above what a shard takes when its replays
+    # end by themselves. Both come from the matrix. The deadline is the smaller of the time in
+    # which every case of the shard could run to the end of its own time budget
+    # (trace_replay_shard.py, needed_seconds) and RETRACE_SHARD_DEADLINE_CAP_SECONDS. With
+    # budgets of up to 1800 s a case the cap is what a shard of several cases gets: about
+    # three replays can hang to the end of their budgets before a later case is recorded as
+    # not-run. The minutes between the deadline and this limit are the setup before the
+    # replays and the uploads after them.
+    timeout-minutes: ${{ matrix.timeout_minutes }}
     env:
       MOBILEGL_CI_TRANSPORT: ${{ matrix.transport || 'monolith' }}
       MOBILEGL_CI_LANE: ${{ matrix.lane || 'monolith-control' }}
@@ -435,38 +551,53 @@ jobs:
       ANDROID_AVD_HOME: ${{ github.workspace }}/.android/avd
       ANDROID_HOME: ${{ github.workspace }}/.android/sdk
       ANDROID_SDK_ROOT: ${{ github.workspace }}/.android/sdk
+      # The shard's cases in run order, as the JSON array of case objects the shard runner
+      # takes: every field of the case in trace_cases.json, with the fixture paths in place.
+      MOBILEGL_SHARD_CASES: ${{ toJSON(matrix.cases) }}
+      # Set the repository variable to 1 to cold boot the emulator before every case of a
+      # shard, should one boot not survive a whole shard of software-rendered replays.
+      MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE: ${{ vars.MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE || '0' }}
+      # The emulator renders on the host with a software rasterizer that can keep compiled
+      # shaders on disk in the runner's home directory, where neither clearing the app's data
+      # nor restarting the emulator reaches. Off, so every case of a shard compiles its
+      # shaders as a job with a fresh machine does.
+      MESA_SHADER_CACHE_DISABLE: "true"
     strategy:
       fail-fast: false
-      # The account runs at most 20 jobs at once, shared with Test. These ~230 emulator jobs are
+      # No max-parallel: the account's limit of 20 concurrent jobs, shared with Test, already
-      # the longer of the two runs, so they may take the whole pool: what is minimized is when the
+      # bounds this matrix, and a cap below the number of shards only makes the round longer.
-      # later of APK and Test finishes, not Test alone.
-      max-parallel: 20
       matrix: ${{ fromJSON(needs.trace-cases.outputs.android) }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v6
 
+      # First, so the steps that run whatever happened (diagnostics, stopping the emulator, the
+      # result upload) have these even when a step before the emulator launch failed.
+      - name: Set artifact metadata
+        run: |
+          echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
+          echo "EMULATOR_LOG=${RUNNER_TEMP}/mobilegl-emulator.log" >> "$GITHUB_ENV"
+          echo "EMULATOR_PID_FILE=${RUNNER_TEMP}/mobilegl-emulator.pid" >> "$GITHUB_ENV"
+
       - name: Set swap space
         uses: ./.github/actions/set-swap-space
         with:
           swap-size-gb: 8
 
-      - name: Download trace fixture
+      # A miss does not fail the job: the entry may have been evicted since trace-fixtures
-        uses: actions/download-artifact@v8
+      # confirmed it (a re-run of failed jobs on an older run does not run that job again).
+      - name: Restore trace fixtures
+        uses: actions/cache/restore@v5
         with:
-          name: MobileGL-trace-fixture-${{ matrix.case.name }}
+          path: ${{ needs.trace-fixtures.outputs.path }}
-          path: trace-fixture-download
+          key: ${{ needs.trace-fixtures.outputs.key }}
 
-      - name: Install trace fixture
+      # Moves this shard's fixtures into place and checks them against their Git LFS pointers.
-        run: |
+      # A case the restored bundle does not hold is downloaded here, with a warning annotation;
-          mkdir -p tools/trace_replay/fixtures
+      # the timeout bounds a download that stalls.
-          find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;
+      - name: Install trace fixtures
-
+        timeout-minutes: 15
-      - name: Set artifact metadata
+        run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
-        run: |
-          echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
-          echo "EMULATOR_LOG=${RUNNER_TEMP}/mobilegl-emulator.log" >> "$GITHUB_ENV"
-          echo "EMULATOR_PID_FILE=${RUNNER_TEMP}/mobilegl-emulator.pid" >> "$GITHUB_ENV"
 
       - name: Restore Android AVD cache
         id: android-avd-cache
@@ -543,135 +674,44 @@ jobs:
             --pid-file "${EMULATOR_PID_FILE}" \
             --boot-timeout 300
 
+      # One emulator boot replays every case of the shard in order, each in a fresh app process
+      # with cleared app data and with the settings its case, backend and transport need. The
+      # runner keeps going after a failing case, on a cold boot so that case cannot touch the
+      # next one's verdict, retries a case once on a restarted emulator when the emulator
+      # itself failed, and exits non-zero when a case failed or a verdict is missing. The
+      # deadline stops it while the steps below can still upload what it has: the job timeout
+      # less the setup above and the uploads below. A case it stops or never starts is
+      # recorded as not-run, which is red.
       - name: Retrace and validate
         env:
-          MOBILEGL_ESPRYT_USE_ANGLE: ${{ matrix.backend.name == 'DirectGLES' && '1' || '0' }}
+          # Lets the artifact cleanup tell which job a shard's verdicts came from.
-          MOBILEGL_TRACE_ANGLE_VARIANT: ${{ matrix.case.name == 'minecraft-1.21.4-fabric-iris-bliss-in-world' && '90a62123d794' || 'ec889e6ea831' }}
+          MOBILEGL_SHARD_JOB_ID: ${{ job.check_run_id }}
-          MOBILEGL_MAGMA_R11G11B10F_FALLBACK: ${{ matrix.backend.name == 'DirectVulkan' && '1' || '0' }}
-          MOBILEGL_ITRP_FIX: ${{ matrix.case.name == 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' && 'ForceOn' || 'Auto' }}
         run: |
-          apk_file="android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk"
+          sh android-plugin/trace-replay-shard.sh \
-          test -f "${apk_file}"
+            --apk-file "android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk" \
-          extra_retrace_args=()
+            --backend "${{ matrix.backend.name }}" \
-          if [ "${MOBILEGL_CI_TRANSPORT}" = "inproc" ]; then
+            --gpu "${{ matrix.backend.gpu }}" \
-            extra_retrace_args+=(
+            --transport "${MOBILEGL_CI_TRANSPORT}" \
-              --env "MOBILEGL_TRANSPORT=inproc;MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1;MOBILEGL_GPU_HANG_BUDGET_MS=${MOBILEGL_GPU_HANG_BUDGET_MS}"
+            --lane "${MOBILEGL_CI_LANE}" \
-              --require-inproc
+            --cases-env MOBILEGL_SHARD_CASES \
-            )
+            --deadline-seconds "${{ matrix.deadline_seconds }}"
-          elif [ "${MOBILEGL_CI_TRANSPORT}" = "spawn" ]; then
-            # THE SAME CLIENT KNOBS AS inproc, so the two acceptance lanes differ in exactly one
-            # thing: which PROCESS the server role runs in. They are scrubbed out of the CHILD's
-            # environment by the launcher and read by the client, which is where they belong.
-            #
-            # MOBILEGL_IPC_SERVER_PATH IS DELIBERATELY ABSENT. nativeLibraryDir carries an
-            # install-time hash, so nothing off the device can spell it; TraceReplayActivity
-            # resolves it from getApplicationInfo() when the env asks for spawn, and an explicit
-            # value would override that. --use-pbuffer is NOT optional here: a spawned
-            # server cannot be handed the client's window, because an ANativeWindow* is a pointer
-            # into the CLIENT's process and SetWindowHandle is refused on the way across with
-            # Fatal{UnmigratedSurface, "AndroidNativeWindow:use-MOBILEGL_IPC_SURFACE=server"} (Rule H, permanent;
-            # a windowed client uses MOBILEGL_IPC_SURFACE=server instead).
-            extra_retrace_args+=(
-              --env "MOBILEGL_TRANSPORT=spawn;MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1;MOBILEGL_GPU_HANG_BUDGET_MS=${MOBILEGL_GPU_HANG_BUDGET_MS}"
-              --use-pbuffer
-              --require-spawn
-            )
-          else
-            extra_retrace_args+=(--env "MOBILEGL_TRANSPORT=monolith;MOBILEGL_GPU_HANG_BUDGET_MS=${MOBILEGL_GPU_HANG_BUDGET_MS}")
-          fi
-          # Bliss needs the newer signed ANGLE variant plus sampler mipmap
-          # min-filter downgrading on ANGLE llvmpipe.
-          if [ "${{ matrix.backend.name }}" = "DirectGLES" ] && [ "${{ matrix.case.name }}" = "minecraft-1.21.4-fabric-iris-bliss-in-world" ]; then
-            extra_retrace_args+=(--avoid-angle-llvmpipe-sampler-mipmap-min-filter)
-          fi
-          if [ "${{ matrix.backend.name }}" = "DirectGLES" ] && [ "${{ matrix.case.avoid_angle_llvmpipe_explicit_lod_bias || false }}" = "true" ]; then
-            extra_retrace_args+=(--avoid-angle-llvmpipe-explicit-lod-bias)
-          fi
-          if [ "${{ matrix.case.coherent_as_flush || false }}" = "true" ]; then
-            extra_retrace_args+=(--coherent-as-flush)
-          fi
-
-          run_retrace() {
-            timeout "$(( ${{ matrix.case.timeout_seconds }} + 300 ))" sh android-plugin/trace-replay-ci.sh \
-              --apk-file "${apk_file}" \
-              --package top.mobilegl.plugin.trace \
-              --backend "${{ matrix.backend.name }}" \
-              --result-root android-retrace-result \
-              --fixture-root android-retrace-fixture \
-              --case "${{ matrix.case.name }}" \
-              --trace-archive "${{ matrix.case.trace_archive }}" \
-              --trace-file "${{ matrix.case.trace_file }}" \
-              --golden "${{ matrix.case.golden }}" \
-              --alternate-golden "${{ matrix.case.alternate_golden || '' }}" \
-              --target-call "${{ matrix.case.target_call }}" \
-              --width "${{ matrix.case.width }}" \
-              --height "${{ matrix.case.height }}" \
-              --ssim-threshold "${{ matrix.case.ssim_threshold || '0.99' }}" \
-              --crop-x "${{ matrix.case.crop_x }}" \
-              --crop-y "${{ matrix.case.crop_y }}" \
-              --crop-width "${{ matrix.case.crop_width }}" \
-              --crop-height "${{ matrix.case.crop_height }}" \
-              --timeout-seconds "${{ matrix.case.timeout_seconds }}" \
-              "${extra_retrace_args[@]}"
-          }
-
-          retrace_status=0
-          run_retrace || retrace_status=$?
-          if [ "${retrace_status}" -eq 75 ]; then
-            echo "::warning::Android emulator infrastructure failed; restarting it and retrying this retrace once."
-            # Surface-lost is retried rather than failed, so it would otherwise
-            # be invisible. Report it per job - a healthy run prints nothing and
-            # a rate spike shows up as a row per affected case.
-            reason_file="android-retrace-result/infrastructure-failure-reason.txt"
-            surface_lost_retries=0
-            if [ -f "${reason_file}" ]; then
-              surface_lost_retries="$(grep -c 'angle-surface-lost' "${reason_file}" || true)"
-            fi
-            if [ "${surface_lost_retries}" -gt 0 ]; then
-              echo "surface-lost retries: ${surface_lost_retries} (${{ matrix.backend.name }}, ${{ matrix.case.name }})" \
-                >> "${GITHUB_STEP_SUMMARY}"
-            fi
-            # The restart truncates EMULATOR_LOG, and the attempt that lost the
-            # emulator is the one worth reading - the retry usually only shows
-            # the wreckage. Keep the first attempt's log before it is clobbered.
-            if [ -f "${EMULATOR_LOG}" ]; then
-              cp "${EMULATOR_LOG}" "${EMULATOR_LOG}.first-attempt" || true
-            fi
-            sh android-plugin/run-avd-ci.sh stop \
-              --avd-name "${AVD_NAME}" \
-              --emulator-log "${EMULATOR_LOG}" \
-              --pid-file "${EMULATOR_PID_FILE}"
-            adb kill-server || true
-            sleep 2
-            sh android-plugin/run-avd-ci.sh start \
-              --avd-name "${AVD_NAME}" \
-              --gpu "${{ matrix.backend.gpu }}" \
-              --emulator-log "${EMULATOR_LOG}" \
-              --pid-file "${EMULATOR_PID_FILE}" \
-              --boot-timeout 300
-            run_retrace
-          elif [ "${retrace_status}" -ne 0 ]; then
-            exit "${retrace_status}"
-          fi
 
+      # Runs whatever happened above, without the device: every case of the shard gets its
+      # lane.json and golden copies for the summary, and a not-run verdict when the runner was
+      # killed or never started.
       - name: Collect retrace summary inputs
         if: always()
+        env:
+          MOBILEGL_SHARD_JOB_ID: ${{ job.check_run_id }}
         run: |
-          safe_case="$(printf '%s' '${{ matrix.case.name }}' | sed 's/[^A-Za-z0-9._-]/_/g')"
+          sh android-plugin/trace-replay-shard.sh \
-          result_dir="android-retrace-result/${safe_case}-${{ matrix.backend.name }}"
+            --apk-file "android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk" \
-          mkdir -p "${result_dir}"
+            --backend "${{ matrix.backend.name }}" \
-          python3 - "${result_dir}/lane.json" <<'PY'
+            --gpu "${{ matrix.backend.gpu }}" \
-          import json, os, sys
+            --transport "${MOBILEGL_CI_TRANSPORT}" \
-          from pathlib import Path
+            --lane "${MOBILEGL_CI_LANE}" \
-          Path(sys.argv[1]).write_text(json.dumps({"lane": os.environ["MOBILEGL_CI_LANE"],
+            --cases-env MOBILEGL_SHARD_CASES \
-              "transport": os.environ["MOBILEGL_CI_TRANSPORT"]}, indent=2))
+            --finalize
-          PY
-          if [ -s "${{ matrix.case.golden }}" ]; then
-            cp "${{ matrix.case.golden }}" "${result_dir}/${safe_case}-${{ matrix.backend.name }}-golden.png"
-          fi
-          if [ -n "${{ matrix.case.alternate_golden || '' }}" ] && [ -s "${{ matrix.case.alternate_golden || '' }}" ]; then
-            cp "${{ matrix.case.alternate_golden || '' }}" "${result_dir}/${safe_case}-${{ matrix.backend.name }}-alternate-golden.png"
-          fi
 
       - name: Collect emulator diagnostics
         if: always()
@@ -679,12 +719,18 @@ jobs:
           mkdir -p android-retrace-result/diagnostics
           adb devices -l > android-retrace-result/diagnostics/adb-devices.txt || true
           timeout 30 adb logcat -d -t 1000 > android-retrace-result/diagnostics/logcat.txt || true
+          # What the guest was given; replay time inside the emulator depends on it.
+          timeout 30 adb shell 'nproc; grep -E "MemTotal|SwapTotal" /proc/meminfo' \
+            > android-retrace-result/diagnostics/guest-resources.txt 2>&1 || true
+          cp "${ANDROID_AVD_HOME}/${AVD_NAME}.avd/config.ini" android-retrace-result/diagnostics/avd-config.ini 2>/dev/null || true
+          # Expected to list nothing: the host rasterizer's on-disk shader cache is disabled.
+          du -sh "${XDG_CACHE_HOME:-${HOME}/.cache}"/mesa_shader_cache* \
+            > android-retrace-result/diagnostics/host-shader-cache.txt 2>&1 || true
+          # The log of the last boot. The shard runner keeps the log of every boot it ended
+          # as diagnostics/emulator-boot-<n>-*.log before a restart truncates it.
           if [ -f "${EMULATOR_LOG}" ]; then
             cp "${EMULATOR_LOG}" android-retrace-result/diagnostics/emulator.log
           fi
-          if [ -f "${EMULATOR_LOG}.first-attempt" ]; then
-            cp "${EMULATOR_LOG}.first-attempt" android-retrace-result/diagnostics/emulator-first-attempt.log
-          fi
           # A vanished emulator looks identical whether the host OOM killer took
           # qemu or the renderer faulted. These two say which.
           free -h > android-retrace-result/diagnostics/host-memory.txt 2>&1 || true
@@ -704,9 +750,14 @@ jobs:
         continue-on-error: true
         uses: actions/upload-artifact@v7
         with:
-          name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-${{ matrix.case.name }}
+          # The summary job downloads by the lane and transport at the front of this name, and
+          # its renderer reads the backend and the case from the paths below it, so the name
+          # ends in the shard number and holds no case name.
+          name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-shard${{ matrix.shard }}
           path: android-retrace-result/**
           if-no-files-found: warn
+          # A re-run of a failed job replaces the artifact its first attempt kept.
+          overwrite: true
 
       # The artifact service sometimes times out (ETIMEDOUT on CreateArtifact, one job in 231 on
       # 2026-10-18), which turned a passing case red; one delayed retry absorbs that.
@@ -718,7 +769,7 @@ jobs:
         if: always() && steps.upload-retrace-result.outcome == 'failure'
         uses: actions/upload-artifact@v7
         with:
-          name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-${{ matrix.case.name }}
+          name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-shard${{ matrix.shard }}
           path: android-retrace-result/**
           if-no-files-found: warn
           overwrite: true
@@ -727,46 +778,67 @@ jobs:
     name: retrace summary
     runs-on: ubuntu-26.04
     needs: retrace
-    if: ${{ !cancelled() }}
+    # Also runs for a canceled run, where it only downloads the results and deletes the
+    # intermediate artifacts; the rendering steps are skipped then. The checkout and the
+    # downloads say so themselves, because a step with no condition is skipped once its job is
+    # canceled.
+    if: always()
+    permissions:
+      actions: write
+      contents: read
     env:
-      MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
+      MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'ci/round-30min' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
 
     steps:
       - name: Checkout repo
+        if: ${{ success() || cancelled() }}
         uses: actions/checkout@v6
 
-      - name: Set artifact metadata
-        run: |
-          echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
-
       - name: Set up Node.js
+        if: ${{ !cancelled() }}
         uses: actions/setup-node@v7
         with:
           node-version: '22'
 
       - name: Download Android monolith control results
+        if: ${{ success() || cancelled() }}
         uses: actions/download-artifact@v8
         with:
           pattern: MobileGL-android-retrace-result-monolith-control-monolith-*
           path: retrace-artifacts/monolith-control
 
       - name: Download Android inproc acceptance results
-        if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
+        if: ${{ (success() || cancelled()) && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
         uses: actions/download-artifact@v8
         with:
           pattern: MobileGL-android-retrace-result-inproc-acceptance-inproc-*
           path: retrace-artifacts/inproc-acceptance
 
       - name: Download Android spawn acceptance results
-        if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
+        if: ${{ (success() || cancelled()) && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
         uses: actions/download-artifact@v8
         with:
           pattern: MobileGL-android-retrace-result-spawn-acceptance-spawn-*
           path: retrace-artifacts/spawn-acceptance
 
+      # The reduced monolith and spawn lanes replay the android_smoke cases only, so their
+      # summaries list those cases and no others. A branch outside the split shape replays
+      # every case on monolith, its only lane.
+      - name: Select the cases of the reduced lanes
+        if: ${{ !cancelled() }}
+        run: |
+          reduced_lane_cases=""
+          if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ] && [ "${MOBILEGL_ANDROID_LANES}" = "inproc-full" ]; then
+            reduced_lane_cases="$(python3 tools/trace_replay/trace_cases.py --format android-smoke-names)"
+            test -n "${reduced_lane_cases}"
+          fi
+          echo "REDUCED_LANE_CASES=${reduced_lane_cases}" >> "$GITHUB_ENV"
+
       - name: Render monolith control summary
+        if: ${{ !cancelled() }}
         run: |
           node tools/trace_replay/render_retrace_summary.mjs \
+            --cases "${REDUCED_LANE_CASES}" \
             --input retrace-artifacts/monolith-control \
             --output-dir android-retrace-summary/monolith-control \
             --title "MobileGL Android monolith control" \
@@ -774,7 +846,7 @@ jobs:
             --html mobilegl-android-monolith-control.html
 
       - name: Render inproc acceptance summary
-        if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
+        if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
         run: |
           node tools/trace_replay/render_retrace_summary.mjs \
             --input retrace-artifacts/inproc-acceptance \
@@ -784,11 +856,18 @@ jobs:
             --html mobilegl-android-inproc-acceptance.html
 
       - name: Render spawn acceptance summary
-        if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
+        if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
         run: |
-          node tools/trace_replay/render_retrace_summary.mjs             --input retrace-artifacts/spawn-acceptance             --output-dir android-retrace-summary/spawn-acceptance             --title "MobileGL Android spawn acceptance"             --group-label "Android Emulator / spawn acceptance"             --html mobilegl-android-spawn-acceptance.html
+          node tools/trace_replay/render_retrace_summary.mjs \
+            --cases "${REDUCED_LANE_CASES}" \
+            --input retrace-artifacts/spawn-acceptance \
+            --output-dir android-retrace-summary/spawn-acceptance \
+            --title "MobileGL Android spawn acceptance" \
+            --group-label "Android Emulator / spawn acceptance" \
+            --html mobilegl-android-spawn-acceptance.html
 
       - name: Upload Android monolith control summary
+        if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v7
         with:
           name: MobileGL-android-monolith-control-summary-${{ github.sha }}
@@ -797,7 +876,7 @@ jobs:
           if-no-files-found: error
 
       - name: Upload Android inproc acceptance summary
-        if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
+        if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
         uses: actions/upload-artifact@v7
         with:
           name: MobileGL-android-inproc-acceptance-summary-${{ github.sha }}
@@ -806,7 +885,7 @@ jobs:
           if-no-files-found: error
 
       - name: Upload Android spawn acceptance summary
-        if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
+        if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
         uses: actions/upload-artifact@v7
         with:
           name: MobileGL-android-spawn-acceptance-summary-${{ github.sha }}
@@ -814,74 +893,23 @@ jobs:
           archive: false
           if-no-files-found: error
 
-  remove-artifact-clutter:
+      # The summaries embed every image they show, so the per-shard results are intermediates
-    name: remove artifact clutter
+      # from here on. This is the last step of the workflow rather than a job of its own, which
-    runs-on: ubuntu-26.04
+      # would wait for a runner and start from an empty machine. It keeps what a failure needs,
-    needs: retrace-summary
+      # read from the shard-status.json of the results downloaded above: the results of every
-    if: always()
+      # shard that ran a case which did not pass somewhere, on all backends and transports, and
-    permissions:
+      # every result when a job failed without saying which cases it ran. Deletes are paced
-      actions: write
+      # below GitHub's limit for mutating requests, and a refused delete is a warning. Skipped
-    steps:
+      # when a summary step failed, so that re-running this job still finds the results, and
+      # when a retrace job did not succeed in a run that was not canceled: every result stays
+      # then, so a shard stopped by its job limit keeps its diagnostics and a re-run of the
+      # failed jobs renders complete summaries.
       - name: Delete intermediate Android retrace artifacts
+        if: ${{ (success() && needs.retrace.result == 'success') || cancelled() }}
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          # Fail before deletion if either paginated inventory cannot be read.
+          python3 android-plugin/retrace_artifact_cleanup.py \
-          # 50 rows per page: the jobs endpoint answers HTTP 502 for 90+ rows on a re-run attempt of a run this size.
+            --downloads retrace-artifacts/monolith-control \
-          gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=50" \
+            --downloads retrace-artifacts/inproc-acceptance \
-            --jq '.jobs[] | select(.name | startswith("retrace (")) | select(.conclusion == "failure" or .conclusion == "canceled" or .conclusion == "timed_out" or .conclusion == "action_required") | .name' \
+            --downloads retrace-artifacts/spawn-acceptance
-            > "${RUNNER_TEMP}/cleanup-jobs.txt"
-          gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \
-            --jq '.artifacts[] | select(.name | startswith("MobileGL-trace-fixture-") or startswith("MobileGL-android-retrace-result-") or startswith("trace-fixture-") or startswith("retrace-result-")) | [.id, .name] | @tsv' \
-            > "${RUNNER_TEMP}/cleanup-artifacts.txt"
-          declare -A failed_cases=()
-          while IFS= read -r job_name; do
-            case_name="${job_name#retrace (*, }"
-            case_name="${case_name%)}"
-            failed_cases["${case_name}"]=1
-          done < "${RUNNER_TEMP}/cleanup-jobs.txt"
-
-          if ((${#failed_cases[@]})); then
-            echo "Retaining fixtures and results for failed retrace case(s):"
-            printf '  %s\n' "${!failed_cases[@]}"
-          else
-            echo "All retrace jobs succeeded; nothing needs to be retained."
-          fi
-
-          deleted=0
-          retained=0
-          while IFS=$'\t' read -r artifact_id artifact_name; do
-            keep=0
-            if [[ "${artifact_name}" == MobileGL-trace-fixture-* ]]; then
-              case_name="${artifact_name#MobileGL-trace-fixture-}"
-              if [[ -v "failed_cases[${case_name}]" ]]; then
-                keep=1
-              fi
-            elif [[ "${artifact_name}" == MobileGL-android-retrace-result-* ]]; then
-              # The result artifact carries mobilegl.log, retrace.log, logcat,
-              # the emulator log and the actual/diff images - the only record of
-              # why a retrace failed. Its name ends in -<backend>-<case>, so a
-              # suffix match on the case name keeps both backends' results for a
-              # case that failed on either of them, which is what a comparison
-              # needs. The match is anchored at the end, so a case name that is a
-              # prefix of a longer one does not retain the longer one's results.
-              for case_name in "${!failed_cases[@]}"; do
-                if [[ "${artifact_name}" == *-"${case_name}" ]]; then
-                  keep=1
-                  break
-                fi
-              done
-            fi
-
-            if ((keep)); then
-              echo "Retaining ${artifact_name} (${artifact_id}) for failed retrace."
-              ((retained += 1))
-              continue
-            fi
-
-            echo "Deleting ${artifact_name} (${artifact_id})"
-            gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}"
-            ((deleted += 1))
-          done < "${RUNNER_TEMP}/cleanup-artifacts.txt"
-
-          echo "Deleted ${deleted} intermediate Android artifact(s); retained ${retained} failed-retrace fixture(s)."

.gitignore

@@ -29,3 +29,5 @@ __pycache__/
 *.py[cod]
 /.gradle
 /.trace-work/
+# The restored trace fixture cache entry (.github/scripts/trace-fixture-bundle.sh)
+/trace-fixture-bundle/

android-plugin/TRACE_REPLAY.md

@@ -74,3 +74,67 @@ adb shell am start -a top.mobilegl.plugin.TRACE_REPLAY \
   --es ssim_threshold 0.99
 adb shell run-as top.mobilegl.plugin.espryt.trace cat files/trace-replay/output/result.json
 ```
+
+## Emulator shards in CI
+
+The APK workflow replays its cases through `trace-replay-shard.sh`: one emulator boot, one APK
+install, then an ordered list of cases for one lane, transport and backend. Each case still runs
+through `trace-replay-ci.sh`, with the same arguments, environment and transport proof as a
+one-case job.
+
+```sh
+sh android-plugin/trace-replay-shard.sh \
+  --apk-file MobileGL-plugin-trace-release.apk \
+  --backend DirectVulkan --gpu lavapipe \
+  --transport inproc --lane inproc-acceptance \
+  --cases-file cases.json --deadline-seconds 7200
+```
+
+The case list is a JSON array of case objects: the `cases` member of a row of
+`trace_cases.py --ci --format github-apk-transport-shard-matrix`, which is what the workflow
+passes, or the `case` members of `--format github-apk-transport-matrix` rows. `--dry-run` prints
+what each case would be run with and touches no device.
+
+The workflow's matrix is that shard matrix: one row per emulator job, holding some of the cases
+of one lane, transport and backend. `--shard-seconds` (`RETRACE_SHARD_SECONDS` in `apk.yml`) sets
+how much hinted replay time one shard aims for, and `trace_cases.py` refuses to print a shard
+matrix whose shards, taken apart, are not exactly the per-case matrix. The fixtures of a shard's
+cases must be in `tools/trace_replay/fixtures` before the runner starts; the workflow restores
+them from one cache entry (`.github/scripts/trace-fixture-bundle.sh`).
+
+Unless a manual run sets `android_full_matrix`, the workflow passes `--android-lanes
+inproc-full`: the inproc lane, the transport the launcher ships, replays every CI case, and the
+monolith and spawn lanes replay the cases `trace_cases.json` lists under `android_smoke`. The
+Linux retrace jobs of `test.yml` replay every case on all three transports, so those two
+emulator lanes keep the cases that prove the Android side of each arm: the server started from
+the APK's native library directory, the pbuffer surface of the spawn lane, the monolith build
+of the same commit as a control, and every per-case replay setting at least once. A manual run
+with `android_full_matrix` replays every case on all three lanes. A branch that runs only the
+monolith lane replays every case on it.
+
+Between cases the runner force-stops the app, checks that no process of the app's uid is left,
+clears the app data, removes the case's files from `/data/local/tmp` and from the host, and
+checks that the device still answers. It restarts the emulator when it does not, before the
+case that follows one that did not pass, and before the single retry of a case that
+`trace-replay-ci.sh` charged to the infrastructure (exit 75).
+`MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE=1` cold boots before every case.
+
+The cases of a shard share the job's time. `--deadline-seconds` stops the runner that long after
+it starts, so that the job can still upload what it has: a case that is running then, or has not
+started, is recorded as `not-run`. `retrace_shard_budget.py` reads a shard matrix and adds two
+fields to every shard: `deadline_seconds`, the time in which every case of the shard could use
+its whole time budget (`timeout_seconds` plus 300 s) when the cases before it used all of
+theirs, with one infrastructure retry of the longest budget and a cold boot per case on top,
+and `timeout_minutes`, the limit of the shard's job. `--cap-seconds`
+(`RETRACE_SHARD_DEADLINE_CAP_SECONDS` in `apk.yml`) bounds the deadline, so a shard whose
+replays all hang cannot hold a runner for hours; a capped shard loses verdicts only when several
+of its replays hang to the end of their budgets.
+
+A failing case does not stop the shard. Every case ends with `<case>-<backend>/case-status.json`
+(`passed`, `failed` or `not-run`), the result root gets `shard-status.json`, the job log gets one
+`::error` per case that did not pass, and the exit status is non-zero when a case did not pass or
+a verdict is missing. `--finalize` writes the summary inputs and a `not-run` verdict for every
+case without one, without a device; the workflow runs it after the replay step whatever happened.
+
+`python3 android-plugin/test_trace_replay_shard.py` drives the runner with a fake adb, replay
+script and AVD script.

android-plugin/retrace_artifact_cleanup.py

@@ -0,0 +1,235 @@
+#!/usr/bin/env python3
+# MobileGL - android-plugin/retrace_artifact_cleanup.py
+# Copyright (c) 2025-2026 MobileGL-Dev
+# Licensed under the GNU Lesser General Public License v3.0:
+#   https://www.gnu.org/licenses/gpl-3.0.txt
+#   https://www.gnu.org/licenses/lgpl-3.0.txt
+# SPDX-License-Identifier: LGPL-3.0-only
+# End of Source File Header
+
+"""Delete the intermediate artifacts of an APK workflow run, keeping what a failure needs.
+
+The retrace summary embeds every image it shows, so once it is uploaded the per-shard result
+artifacts are only worth keeping for cases that did not pass:
+
+- a result artifact stays when its shard-status.json is missing, when its job failed, or when
+  it holds a case that did not pass in any shard (so both backends and all transports of a
+  failed case stay comparable);
+- every result artifact stays when a job failed without leaving a shard-status.json to say
+  which cases it was running.
+
+A canceled job retains nothing by itself; a failed verdict inside it still does. The trace
+fixtures are not artifacts: the retrace jobs restore them from a cache entry.
+"""
+
+import argparse
+import json
+import os
+import shlex
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+
+RESULT_PREFIXES = ("MobileGL-android-retrace-result-", "retrace-result-")
+FAILED_CONCLUSIONS = ("failure", "timed_out", "action_required")
+# GitHub's secondary rate limit allows about 180 mutating REST calls a minute. Requests start
+# at most this often, whatever the number of workers.
+DELETE_INTERVAL_SECONDS = 60.0 / 170.0
+DELETE_WORKERS = 3
+MAX_REFUSED_DELETES = 5
+
+
+def pause(seconds):
+    time.sleep(seconds)
+
+
+def read_rows(text, columns):
+    rows = []
+    for line in text.splitlines():
+        if line.strip():
+            fields = line.split("\t")
+            rows.append(fields + [""] * (columns - len(fields)))
+    return rows
+
+
+def shard_statuses(download_roots):
+    """Artifact name -> its shard-status.json, for every downloaded result artifact that has one."""
+    statuses = {}
+    for root in download_roots:
+        root = Path(root)
+        if not root.is_dir():
+            continue
+        for directory in root.iterdir():
+            try:
+                status = json.loads((directory / "shard-status.json").read_text(encoding="utf-8"))
+            except (OSError, ValueError):
+                continue
+            if isinstance(status, dict) and isinstance(status.get("cases"), list):
+                statuses[directory.name] = status
+    return statuses
+
+
+def plan(artifacts, jobs, statuses):
+    """Returns (ids to delete, lines explaining what stays)."""
+    failed_job_ids = {job_id for job_id, conclusion, _ in jobs if conclusion in FAILED_CONCLUSIONS}
+    status_job_ids = {str(status.get("job_id") or "") for status in statuses.values()} - {""}
+    # Failed jobs that left no status file: nothing says which cases they were running.
+    unexplained = failed_job_ids - status_job_ids
+    if not status_job_ids and failed_job_ids:
+        # The statuses carry no job ids to match against. Count instead: each failed job must
+        # be explained by an artifact that shows a case without a pass.
+        explaining = sum(1 for status in statuses.values()
+                         if any(record.get("status") != "passed" for record in status["cases"]))
+        unexplained = set() if explaining >= len(failed_job_ids) else failed_job_ids
+
+    failed_cases = set()  # did not pass somewhere: every result that holds one stays
+    for status in statuses.values():
+        not_passed = [record for record in status["cases"] if record.get("status") != "passed"]
+        if status_job_ids:
+            job_failed = str(status.get("job_id") or "") in failed_job_ids
+        else:
+            # Without job ids, a status that shows a case without a pass stands for a failed job.
+            job_failed = bool(failed_job_ids) and bool(not_passed)
+        for record in not_passed:
+            if record.get("status") == "failed" or job_failed:
+                failed_cases.add(record.get("case"))
+
+    delete, kept = [], []
+    for artifact_id, name in artifacts:
+        if not name.startswith(RESULT_PREFIXES):
+            continue
+        reason = ""
+        status = statuses.get(name)
+        if unexplained:
+            reason = f"{len(unexplained)} failed job(s) left no shard status"
+        elif status is None:
+            reason = "no shard status was downloaded for it"
+        elif str(status.get("job_id") or "") in failed_job_ids:
+            reason = "its job failed"
+        else:
+            cases = sorted(failed_cases.intersection(record.get("case") for record in status["cases"]))
+            if cases:
+                reason = "holds failed case(s): " + ", ".join(cases)
+        if reason:
+            kept.append(f"{name} ({artifact_id}): {reason}")
+        else:
+            delete.append((artifact_id, name))
+    return delete, kept
+
+
+class Deleter:
+    """Runs `command` once per artifact, with the artifact id appended to its last word."""
+
+    def __init__(self, command):
+        self.command = command
+        self.lock = threading.Lock()
+        self.next_start = 0.0
+        self.refused = []
+        self.deleted = 0
+
+    def wait_for_slot(self):
+        with self.lock:
+            now = time.monotonic()
+            start = max(now, self.next_start)
+            self.next_start = start + DELETE_INTERVAL_SECONDS
+        if start > now:
+            pause(start - now)
+
+    def delete(self, artifact_id, name):
+        for attempt in (1, 2):
+            with self.lock:
+                if len(self.refused) >= MAX_REFUSED_DELETES:
+                    return
+            self.wait_for_slot()
+            done = subprocess.run(self.command[:-1] + [self.command[-1] + artifact_id],
+                                  capture_output=True, text=True)
+            if done.returncode == 0:
+                with self.lock:
+                    self.deleted += 1
+                return
+            if attempt == 1:
+                # Give a rate limit time to clear before the one retry.
+                with self.lock:
+                    self.next_start = max(self.next_start, time.monotonic() + 30)
+        with self.lock:
+            self.refused.append(f"{name} ({artifact_id}): {' '.join(done.stderr.split())[:200]}")
+
+    def run(self, entries):
+        pending = list(entries)
+
+        def work():
+            while True:
+                with self.lock:
+                    if not pending:
+                        return
+                    artifact_id, name = pending.pop(0)
+                self.delete(artifact_id, name)
+
+        threads = [threading.Thread(target=work) for _ in range(min(DELETE_WORKERS, len(pending)))]
+        for thread in threads:
+            thread.start()
+        for thread in threads:
+            thread.join()
+
+
+def gh_rows(path, query, columns):
+    done = subprocess.run(["gh", "api", "--paginate", path, "--jq", query], capture_output=True, text=True)
+    if done.returncode != 0:
+        raise RuntimeError(f"gh api {path} failed: {done.stderr.strip()}")
+    return read_rows(done.stdout, columns)
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    parser.add_argument("--downloads", action="append", default=[],
+                        help="Directory holding one sub-directory per downloaded result artifact.")
+    parser.add_argument("--artifacts-file", help="Tab-separated id and name per line, instead of asking gh.")
+    parser.add_argument("--jobs-file", help="Tab-separated id, conclusion and name per line, instead of asking gh.")
+    parser.add_argument("--dry-run", action="store_true", help="Print the plan, delete nothing.")
+    options = parser.parse_args(argv)
+
+    repository = os.environ.get("GITHUB_REPOSITORY", "")
+    run_id = os.environ.get("GITHUB_RUN_ID", "")
+    try:
+        if options.artifacts_file:
+            artifacts = read_rows(Path(options.artifacts_file).read_text(encoding="utf-8"), 2)
+        else:
+            artifacts = gh_rows(f"repos/{repository}/actions/runs/{run_id}/artifacts?per_page=100",
+                                ".artifacts[] | [.id, .name] | @tsv", 2)
+        if options.jobs_file:
+            jobs = read_rows(Path(options.jobs_file).read_text(encoding="utf-8"), 3)
+        else:
+            # 50 rows per page: the jobs endpoint answers HTTP 502 for larger pages of a re-run
+            # attempt of a run this size.
+            jobs = gh_rows(f"repos/{repository}/actions/runs/{run_id}/jobs?per_page=50",
+                           '.jobs[] | [.id, .conclusion // "", .name] | @tsv', 3)
+    except (OSError, RuntimeError) as error:
+        print(f"::error::Could not read the run's artifacts and jobs; nothing was deleted: {error}")
+        return 1
+
+    delete, kept = plan([(row[0], row[1]) for row in artifacts], [tuple(row[:3]) for row in jobs],
+                        shard_statuses(options.downloads))
+    for line in kept:
+        print(f"Retaining {line}")
+    if options.dry_run:
+        for artifact_id, name in delete:
+            print(f"Would delete {name} ({artifact_id})")
+        return 0
+
+    deleter = Deleter(shlex.split(os.environ.get("MOBILEGL_CLEANUP_DELETE_COMMAND") or
+                                  f"gh api --method DELETE repos/{repository}/actions/artifacts/"))
+    started = time.monotonic()
+    deleter.run(delete)
+    for line in deleter.refused:
+        print(f"::warning::Could not delete intermediate artifact {line}")
+    if len(deleter.refused) >= MAX_REFUSED_DELETES:
+        print("::warning::Stopped deleting intermediate artifacts after repeated refusals; the rest stay until they expire.")
+    print(f"Deleted {deleter.deleted} intermediate Android artifact(s) in {time.monotonic() - started:.0f}s; "
+          f"retained {len(kept)}.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

android-plugin/retrace_shard_budget.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+# MobileGL - android-plugin/retrace_shard_budget.py
+# Copyright (c) 2025-2026 MobileGL-Dev
+# Licensed under the GNU Lesser General Public License v3.0:
+#   https://www.gnu.org/licenses/gpl-3.0.txt
+#   https://www.gnu.org/licenses/lgpl-3.0.txt
+# SPDX-License-Identifier: LGPL-3.0-only
+# End of Source File Header
+
+"""Give every emulator shard the deadline and the job timeout its own cases need.
+
+The cases of a shard share one job, so one case that hangs spends time the cases after it
+need. This reads a shard matrix of trace_cases.py (--format github-apk-shard-matrix or
+github-apk-transport-shard-matrix) on standard input, or from --matrix-file, and prints it
+with two fields added to every shard: deadline_seconds, which the retrace job passes to
+trace-replay-shard.sh, and timeout_minutes, the limit of that job. The deadline is what the
+shard needs for every case to use its whole time budget (trace_replay_shard.needed_seconds),
+up to --cap-seconds. A capped shard still replays every case; only when several of its
+replays hang to the end of their budgets are the cases after them recorded as not-run.
+"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import trace_replay_shard  # noqa: E402  (a sibling file, found through the line above)
+
+
+def shard_label(row):
+    backend = row.get("backend")
+    backend = backend.get("name", "") if isinstance(backend, dict) else str(backend or "")
+    return f"{backend} / {row.get('transport') or 'monolith'}, shard {row.get('shard')}/{row.get('shards')}"
+
+
+def shard_needs(matrix):
+    """(label, case count, seconds needed) for every shard of the matrix."""
+    rows = matrix.get("include") if isinstance(matrix, dict) else None
+    if not isinstance(rows, list) or not rows:
+        raise ValueError("the matrix has no shard rows under 'include'")
+    needs = []
+    for row in rows:
+        cases = row.get("cases") if isinstance(row, dict) else None
+        if not isinstance(cases, list) or not cases:
+            raise ValueError(f"a shard row has no 'cases' list: {str(row)[:200]}")
+        needs.append((shard_label(row), len(cases), trace_replay_shard.needed_seconds(cases)))
+    return needs
+
+
+# Minutes a retrace job has around its shard runner: the setup before the replays and the
+# uploads after them.
+JOB_OVERHEAD_MINUTES = 15
+
+# A hosted job may run six hours.
+HOSTED_JOB_LIMIT_MINUTES = 360
+
+
+def with_limits(matrix, cap_seconds):
+    """The matrix with deadline_seconds and timeout_minutes on every shard, and the report."""
+    needs = shard_needs(matrix)
+    rows = []
+    report = []
+    for row, (label, cases, seconds) in zip(matrix["include"], needs):
+        deadline = min(seconds, cap_seconds)
+        rows.append({**row, "deadline_seconds": deadline,
+                     "timeout_minutes": -(-deadline // 60) + JOB_OVERHEAD_MINUTES})
+        if seconds > cap_seconds:
+            report.append(f"retrace shard budget: {label} would need {seconds} s for all of its "
+                          f"{cases} case(s) to use their whole budgets; capped at {cap_seconds} s")
+    largest = max(needs, key=lambda entry: entry[2])
+    report.append(f"retrace shard budget: {len(needs)} shard(s), {len(report)} capped at "
+                  f"{cap_seconds} s; the most a shard needs is {largest[2]} s ({largest[0]}, "
+                  f"{largest[1]} case(s))")
+    return {**matrix, "include": rows}, report
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    parser.add_argument("--cap-seconds", type=int, required=True,
+                        help="The longest deadline a shard is given.")
+    parser.add_argument("--matrix-file", help="The shard matrix; standard input when not given.")
+    options = parser.parse_args(argv)
+    limit = (HOSTED_JOB_LIMIT_MINUTES - JOB_OVERHEAD_MINUTES) * 60
+    if not 0 < options.cap_seconds <= limit:
+        print(f"retrace shard budget: --cap-seconds must be between 1 and {limit}", file=sys.stderr)
+        return 2
+    if options.matrix_file:
+        text = Path(options.matrix_file).read_text(encoding="utf-8")
+    else:
+        text = sys.stdin.read()
+    try:
+        matrix, report = with_limits(json.loads(text), options.cap_seconds)
+    except (ValueError, KeyError, TypeError) as error:
+        print(f"retrace shard budget: the shard matrix cannot be read: {error}", file=sys.stderr)
+        return 2
+    for line in report:
+        print(line, file=sys.stderr)
+    print(json.dumps(matrix, separators=(",", ":")))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

android-plugin/test_retrace_artifact_cleanup.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+# MobileGL - android-plugin/test_retrace_artifact_cleanup.py
+# Copyright (c) 2025-2026 MobileGL-Dev
+# Licensed under the GNU Lesser General Public License v3.0:
+#   https://www.gnu.org/licenses/gpl-3.0.txt
+#   https://www.gnu.org/licenses/lgpl-3.0.txt
+# SPDX-License-Identifier: LGPL-3.0-only
+# End of Source File Header
+
+"""Pin which intermediate APK artifacts the cleanup keeps, and how fast it deletes the rest."""
+
+import contextlib
+import importlib.util
+import io
+import json
+import os
+import pathlib
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+HERE = pathlib.Path(__file__).resolve().parent
+_spec = importlib.util.spec_from_file_location("retrace_artifact_cleanup", HERE / "retrace_artifact_cleanup.py")
+cleanup = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(cleanup)
+
+RESULT = "MobileGL-android-retrace-result-"
+
+
+def status(job_id, **cases):
+    return {"job_id": job_id, "cases": [{"case": name, "status": verdict} for name, verdict in cases.items()]}
+
+
+def names(entries):
+    return sorted(name for _, name in entries)
+
+
+class PlanTest(unittest.TestCase):
+    ARTIFACTS = [("1", RESULT + "mono-gles"), ("2", RESULT + "inproc-gles"), ("3", RESULT + "spawn-vk"),
+                 ("4", "mobilegl-android-monolith-control.html"), ("5", "MobileGL-plugin-release-abc.apk"),
+                 ("6", "MobileGL-plugin-trace-release-abc.apk")]
+
+    def test_a_green_run_deletes_every_intermediate_and_nothing_else(self):
+        statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed"),
+                    RESULT + "inproc-gles": status("12", a="passed", b="passed"),
+                    RESULT + "spawn-vk": status("13", c="passed")}
+        jobs = [("11", "success", "x"), ("12", "success", "y"), ("13", "success", "z"), ("14", "", "retrace summary")]
+        delete, kept = cleanup.plan(self.ARTIFACTS, jobs, statuses)
+        self.assertEqual(kept, [])
+        self.assertEqual(names(delete), sorted(name for _, name in self.ARTIFACTS[:3]))
+
+    def test_a_failed_case_keeps_every_shard_that_ran_it(self):
+        statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed"),
+                    RESULT + "inproc-gles": status("12", a="passed", b="failed"),
+                    RESULT + "spawn-vk": status("13", c="passed")}
+        jobs = [("11", "success", "x"), ("12", "failure", "y"), ("13", "success", "z")]
+        delete, kept = cleanup.plan(self.ARTIFACTS, jobs, statuses)
+        # Only b's results are worth comparing, so the monolith shard stays for b and the
+        # spawn shard goes.
+        self.assertEqual(names(delete), [RESULT + "spawn-vk"])
+        self.assertEqual(len(kept), 2)
+        self.assertTrue(any("inproc-gles" in line and "its job failed" in line for line in kept))
+        self.assertTrue(any("mono-gles" in line and "holds failed case(s): b" in line for line in kept))
+
+    def test_cases_a_failed_job_never_reached_are_kept(self):
+        statuses = {RESULT + "mono-gles": status("11", a="passed", b="not-run"),
+                    RESULT + "spawn-vk": status("13", c="passed")}
+        jobs = [("11", "timed_out", "x"), ("13", "success", "z")]
+        delete, kept = cleanup.plan(self.ARTIFACTS[:1] + self.ARTIFACTS[2:], jobs, statuses)
+        self.assertEqual(names(delete), [RESULT + "spawn-vk"])
+        self.assertEqual(len(kept), 1)
+
+    def test_a_job_that_failed_after_its_cases_passed_keeps_its_results(self):
+        statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed"),
+                    RESULT + "spawn-vk": status("13", c="passed")}
+        jobs = [("11", "failure", "upload failed"), ("13", "success", "z")]
+        delete, kept = cleanup.plan(self.ARTIFACTS[:1] + self.ARTIFACTS[2:], jobs, statuses)
+        self.assertEqual(names(delete), [RESULT + "spawn-vk"])
+        self.assertTrue("mono-gles" in kept[0] and "its job failed" in kept[0])
+
+    def test_a_canceled_job_keeps_nothing_by_itself(self):
+        statuses = {RESULT + "mono-gles": status("11", a="passed", b="not-run"),
+                    RESULT + "inproc-gles": status("12", a="failed", b="not-run")}
+        jobs = [("11", "cancelled", "x"), ("12", "cancelled", "y")]
+        delete, kept = cleanup.plan(self.ARTIFACTS[:2], jobs, statuses)
+        # Case a failed in the inproc shard, so both shards that ran it stay.
+        self.assertEqual(delete, [])
+        self.assertEqual(len(kept), 2)
+        only_not_run = {RESULT + "mono-gles": status("11", a="passed", b="not-run")}
+        delete, kept = cleanup.plan(self.ARTIFACTS[:1], [("11", "cancelled", "x")], only_not_run)
+        self.assertEqual((names(delete), kept), ([RESULT + "mono-gles"], []))
+
+    def test_a_result_without_a_status_is_kept(self):
+        statuses = {RESULT + "mono-gles": status("11", a="passed")}
+        delete, kept = cleanup.plan(self.ARTIFACTS[:2], [("11", "success", "x")], statuses)
+        self.assertEqual(names(delete), [RESULT + "mono-gles"])
+        self.assertIn("no shard status", kept[0])
+
+    def test_a_failed_job_without_a_status_keeps_everything(self):
+        statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed")}
+        jobs = [("11", "success", "x"), ("12", "failure", "died before its upload")]
+        delete, kept = cleanup.plan(self.ARTIFACTS, jobs, statuses)
+        self.assertEqual(delete, [])
+        self.assertEqual(len(kept), 3)
+
+    def test_statuses_without_job_ids_fall_back_to_counting(self):
+        artifacts = self.ARTIFACTS[:1] + self.ARTIFACTS[2:]
+        explained = {RESULT + "mono-gles": status("", a="passed", b="not-run"),
+                     RESULT + "spawn-vk": status("", c="passed")}
+        delete, _ = cleanup.plan(artifacts, [("12", "failure", "y")], explained)
+        # The one status with a case that did not pass stands for the one failed job.
+        self.assertEqual(names(delete), [RESULT + "spawn-vk"])
+        unexplained = {RESULT + "mono-gles": status("", a="passed", b="passed"),
+                       RESULT + "spawn-vk": status("", c="passed")}
+        delete, _ = cleanup.plan(artifacts, [("12", "failure", "y")], unexplained)
+        self.assertEqual(delete, [])
+
+
+class RunTest(unittest.TestCase):
+    @staticmethod
+    def passed(artifacts):
+        """A shard status with one passed case for every artifact, each from a job of its own."""
+        return {name: status(f"9{artifact_id}", a="passed") for artifact_id, name in artifacts}
+
+    def run_cleanup(self, artifacts, jobs, statuses, exit_code=0, dry_run=False):
+        with tempfile.TemporaryDirectory() as directory:
+            root = pathlib.Path(directory)
+            for name, value in statuses.items():
+                (root / "downloads" / "lane" / name).mkdir(parents=True)
+                (root / "downloads" / "lane" / name / "shard-status.json").write_text(json.dumps(value))
+            (root / "artifacts.tsv").write_text("".join(f"{i}\t{n}\n" for i, n in artifacts))
+            (root / "jobs.tsv").write_text("".join("\t".join(job) + "\n" for job in jobs))
+            # One file per call: the deletes run in parallel processes.
+            (root / "deleted").mkdir()
+            script = root / "delete.py"
+            script.write_text("import pathlib, sys\n"
+                              "(pathlib.Path(sys.argv[1]) / sys.argv[2].replace('/', '_')).write_text('')\n"
+                              f"sys.exit({exit_code})\n")
+            python = pathlib.Path(sys.executable).as_posix()
+            command = f'"{python}" "{script.as_posix()}" "{(root / "deleted").as_posix()}" artifacts/'
+            pauses = []
+            output = io.StringIO()
+            with mock.patch.dict(os.environ, {"MOBILEGL_CLEANUP_DELETE_COMMAND": command}), \
+                    mock.patch.object(cleanup, "pause", pauses.append), contextlib.redirect_stdout(output):
+                code = cleanup.main(["--downloads", str(root / "downloads" / "lane"),
+                                     "--artifacts-file", str(root / "artifacts.tsv"),
+                                     "--jobs-file", str(root / "jobs.tsv")] + (["--dry-run"] if dry_run else []))
+            deleted = sorted(entry.name.replace("_", "/") for entry in (root / "deleted").iterdir())
+            return code, output.getvalue(), deleted, pauses
+
+    def test_deletes_are_spaced_under_the_secondary_rate_limit(self):
+        artifacts = [(str(index), f"{RESULT}shard{index}") for index in range(12)]
+        code, output, deleted, pauses = self.run_cleanup(artifacts, [], self.passed(artifacts))
+        self.assertEqual(code, 0)
+        self.assertEqual(deleted, sorted(f"artifacts/{index}" for index in range(12)))
+        self.assertIn("Deleted 12 intermediate Android artifact(s)", output)
+        # Twelve starts are booked at least 11 intervals apart, so the rate stays under 180 a minute.
+        self.assertLess(60 / cleanup.DELETE_INTERVAL_SECONDS, 180)
+        self.assertGreater(len(pauses), 0)
+
+    def test_refused_deletes_warn_and_leave_the_run_green(self):
+        artifacts = [(str(index), f"{RESULT}shard{index}") for index in range(8)]
+        code, output, _, _ = self.run_cleanup(artifacts, [], self.passed(artifacts), exit_code=1)
+        self.assertEqual(code, 0)
+        self.assertIn("::warning::Could not delete intermediate artifact", output)
+        self.assertIn("Stopped deleting", output)
+        self.assertIn("Deleted 0 intermediate", output)
+
+    def test_a_dry_run_deletes_nothing(self):
+        artifacts = [("1", RESULT + "shard1")]
+        code, output, deleted, _ = self.run_cleanup(artifacts, [], self.passed(artifacts), dry_run=True)
+        self.assertEqual((code, deleted), (0, []))
+        self.assertIn(f"Would delete {RESULT}shard1 (1)", output)
+
+
+if __name__ == "__main__":
+    unittest.main()

android-plugin/test_trace_replay_shard.py

@@ -0,0 +1,587 @@
+#!/usr/bin/env python3
+# MobileGL - android-plugin/test_trace_replay_shard.py
+# Copyright (c) 2025-2026 MobileGL-Dev
+# Licensed under the GNU Lesser General Public License v3.0:
+#   https://www.gnu.org/licenses/gpl-3.0.txt
+#   https://www.gnu.org/licenses/lgpl-3.0.txt
+# SPDX-License-Identifier: LGPL-3.0-only
+# End of Source File Header
+
+"""Drive the emulator shard runner with a fake adb, replay script and AVD script.
+
+No device is needed. The fakes record every call, so the tests pin what each case is run
+with, that every case ends with a verdict, and what a failure, an infrastructure retry, a
+timeout or a dead emulator does to the rest of the shard.
+"""
+
+import contextlib
+import importlib.util
+import io
+import json
+import os
+import pathlib
+import re
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+from unittest import mock
+
+HERE = pathlib.Path(__file__).resolve().parent
+
+
+def load(name):
+    spec = importlib.util.spec_from_file_location(name, HERE / f"{name}.py")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+shard = load("trace_replay_shard")
+budget = load("retrace_shard_budget")
+
+BLISS = "minecraft-1.21.4-fabric-iris-bliss-in-world"
+ITERATIONRP = "minecraft-1.21.4-fabric-iris-iterationrp-in-world"
+SPLIT = "MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1"
+
+FAKE_COMMON = r'''
+import json, os, pathlib, sys
+state = pathlib.Path(os.environ["FAKE_STATE"])
+def record(name, entry):
+    with open(state / name, "a", encoding="utf-8") as log:
+        log.write(json.dumps(entry) + "\n")
+def scenario():
+    path = state / "scenario.json"
+    return json.loads(path.read_text()) if path.exists() else {}
+def count(name):
+    path = state / f"count-{name}"
+    value = int(path.read_text()) if path.exists() else 0
+    path.write_text(str(value + 1))
+    return value
+'''
+
+FAKE_ADB = FAKE_COMMON + r'''
+args = sys.argv[1:]
+record("adb.log", args)
+plan = scenario()
+if args[:1] == ["get-state"]:
+    print("offline" if count("get-state") < plan.get("offline_get_state_calls", 0) else "device")
+elif args[:3] == ["shell", "getprop", "sys.boot_completed"]:
+    print("1")
+elif args[:3] == ["shell", "pm", "path"]:
+    print("package:/data/app/trace/base.apk")
+elif args[:3] == ["shell", "pm", "clear"]:
+    print("Failed" if plan.get("pm_clear_fails") else "Success")
+elif args[:1] == ["install"]:
+    if count("install") < plan.get("failing_installs", 0):
+        print("Failure [INSTALL_FAILED]")
+        sys.exit(1)
+    print("Success")
+elif args[:2] == ["shell", "run-as"] and args[3:] == ["id", "-un"]:
+    print("u0_a77")
+elif args[:3] == ["shell", "ps", "-A"]:
+    print("USER PID NAME")
+    print("root 1 init")
+    if count("ps") < plan.get("ps_calls_with_leftover", 0):
+        print("u0_a77 4242 libMobileGLServer.so")
+'''
+
+FAKE_REPLAY = FAKE_COMMON + r'''
+import time
+args = sys.argv[1:]
+def value(flag):
+    return args[args.index(flag) + 1]
+case, backend, root = value("--case"), value("--backend"), pathlib.Path(value("--result-root"))
+record("replay.log", {"case": case, "arguments": args, "environment": {
+    key: os.environ.get(key) for key in (
+        "MOBILEGL_ESPRYT_USE_ANGLE", "MOBILEGL_TRACE_ANGLE_VARIANT", "MOBILEGL_MAGMA_R11G11B10F_FALLBACK",
+        "MOBILEGL_ITRP_FIX", "MOBILEGL_TRACE_SKIP_INSTALL", "MOBILEGL_TRACE_POLL_INTERVAL")}})
+(state / "replay.pid").write_text(str(os.getpid()))
+steps = scenario().get("replay", {}).get(case, [])
+attempt = count("replay-" + case)
+step = steps[attempt] if attempt < len(steps) else {}
+time.sleep(step.get("sleep", 0))
+directory = root / f"{case}-{backend}"
+directory.mkdir(parents=True, exist_ok=True)
+code = step.get("exit", 0)
+if code == 75:
+    with open(root / "infrastructure-failure-reason.txt", "a") as reasons:
+        reasons.write(step.get("reason", "device-unavailable") + "\n")
+if step.get("result", code in (0, 1)):
+    (directory / "result.json").write_text(json.dumps(
+        {"passed": step.get("passed", code == 0), "ssim": 0.999, "message": step.get("message", "retrace completed")}))
+transport = "inproc" if "--require-inproc" in args else "spawn" if "--require-spawn" in args else ""
+if transport and code == 0 and step.get("proof", True):
+    (directory / "transport-proof.json").write_text(json.dumps({"passed": True, "required_transport": transport}))
+sys.exit(code)
+'''
+
+FAKE_AVD = FAKE_COMMON + r'''
+args = sys.argv[1:]
+record("avd.log", args)
+if args[:1] == ["start"]:
+    exits = scenario().get("avd_start_exits", [])
+    index = count("avd-start")
+    sys.exit(exits[index] if index < len(exits) else 0)
+'''
+
+
+def case(name, **extra):
+    base = {"name": name, "trace_archive": f"fixtures/{name}.tgz", "trace_file": "trace.trace",
+            "golden": f"fixtures/{name}.png", "target_call": 1234, "width": 854, "height": 480,
+            "ssim_threshold": 0.99, "crop_x": 0, "crop_y": 0, "crop_width": 0, "crop_height": 0,
+            "timeout_seconds": 900}
+    base.update(extra)
+    return base
+
+
+class ShardTest(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temporary.cleanup)
+        self.root = pathlib.Path(self.temporary.name)
+        self.state = self.root / "state"
+        self.state.mkdir()
+        self.results = self.root / "results"
+        (self.root / "fixtures").mkdir()
+        self.apk = self.root / "trace.apk"
+        self.apk.write_bytes(b"apk")
+        self.emulator_log = self.root / "emulator.log"
+        self.emulator_log.write_text("boot\n")
+        python = pathlib.Path(sys.executable).as_posix()
+        commands = {}
+        for name, source in (("adb", FAKE_ADB), ("replay", FAKE_REPLAY), ("avd", FAKE_AVD)):
+            script = self.root / f"fake_{name}.py"
+            script.write_text(source)
+            commands[name] = f'"{python}" "{script.as_posix()}"'
+        self.environment = {
+            "FAKE_STATE": str(self.state), "ADB": commands["adb"],
+            "MOBILEGL_SHARD_REPLAY_COMMAND": commands["replay"], "MOBILEGL_SHARD_AVD_COMMAND": commands["avd"],
+            "MOBILEGL_GPU_HANG_BUDGET_MS": "0", "AVD_NAME": "mobilegl-ci",
+            "EMULATOR_LOG": str(self.emulator_log), "EMULATOR_PID_FILE": str(self.root / "emulator.pid"),
+            "GITHUB_STEP_SUMMARY": str(self.root / "summary.md"),
+        }
+
+    def fixture(self, name):
+        path = self.root / "fixtures" / name
+        path.write_bytes(b"png")
+        return str(path)
+
+    def run_shard(self, cases, *extra, backend="DirectVulkan", transport="monolith", scenario=None, env=None):
+        if scenario is not None:
+            (self.state / "scenario.json").write_text(json.dumps(scenario))
+        arguments = ["--apk-file", str(self.apk), "--backend", backend, "--gpu", "lavapipe",
+                     "--transport", transport, "--lane", f"{transport}-lane", "--cases-json", json.dumps(cases),
+                     "--result-root", str(self.results), "--fixture-root", str(self.root / "work"), *extra]
+        output = io.StringIO()
+        with mock.patch.dict(os.environ, {**self.environment, **(env or {})}), \
+                mock.patch.object(shard, "pause", lambda seconds: None), \
+                contextlib.redirect_stdout(output):
+            status = shard.main(arguments)
+        return status, output.getvalue()
+
+    def log(self, name):
+        path = self.state / name
+        return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else []
+
+    def status(self, name, backend="DirectVulkan"):
+        return json.loads((self.results / f"{name}-{backend}" / "case-status.json").read_text())
+
+    def shard_status(self):
+        return json.loads((self.results / "shard-status.json").read_text())
+
+    # ---- what each case is run with ----------------------------------------------------
+
+    def planned(self, cases, backend, transport):
+        status, output = self.run_shard(cases, "--dry-run", backend=backend, transport=transport)
+        self.assertEqual(status, 0)
+        return [json.loads(line) for line in output.splitlines()]
+
+    def test_plain_case_arguments_per_transport(self):
+        cases = [case("plain", alternate_golden="fixtures/plain-mali.png")]
+        head = ["--apk-file", str(self.apk), "--package", "top.mobilegl.plugin.trace", "--backend", "DirectVulkan",
+                "--result-root", str(self.results), "--fixture-root", str(self.root / "work"), "--case", "plain",
+                "--trace-archive", "fixtures/plain.tgz", "--trace-file", "trace.trace",
+                "--golden", "fixtures/plain.png", "--alternate-golden", "fixtures/plain-mali.png",
+                "--target-call", "1234", "--width", "854", "--height", "480", "--ssim-threshold", "0.99",
+                "--crop-x", "0", "--crop-y", "0", "--crop-width", "0", "--crop-height", "0",
+                "--timeout-seconds", "900"]
+        tails = {
+            "monolith": ["--env", "MOBILEGL_TRANSPORT=monolith;MOBILEGL_GPU_HANG_BUDGET_MS=0"],
+            "inproc": ["--env", f"MOBILEGL_TRANSPORT=inproc;{SPLIT};MOBILEGL_GPU_HANG_BUDGET_MS=0", "--require-inproc"],
+            "spawn": ["--env", f"MOBILEGL_TRANSPORT=spawn;{SPLIT};MOBILEGL_GPU_HANG_BUDGET_MS=0",
+                      "--use-pbuffer", "--require-spawn"],
+        }
+        for transport, tail in tails.items():
+            plan, = self.planned(cases, "DirectVulkan", transport)
+            self.assertEqual(plan["arguments"], head + tail, transport)
+            self.assertEqual(plan["timeout"], 1200)
+            self.assertEqual(plan["environment"], {
+                "MOBILEGL_ESPRYT_USE_ANGLE": "0", "MOBILEGL_TRACE_ANGLE_VARIANT": "ec889e6ea831",
+                "MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "1", "MOBILEGL_ITRP_FIX": "Auto"})
+
+    def test_case_specific_settings(self):
+        cases = [case(BLISS), case(ITERATIONRP), case("flush", coherent_as_flush=True),
+                 case("lod", avoid_angle_llvmpipe_explicit_lod_bias=True), case("plain", ssim_threshold=0.985)]
+        gles = {plan["case"]: plan for plan in self.planned(cases, "DirectGLES", "monolith")}
+        vulkan = {plan["case"]: plan for plan in self.planned(cases, "DirectVulkan", "monolith")}
+
+        def flags(plan):
+            return [argument for argument in plan["arguments"][plan["arguments"].index("--env") + 2:]]
+
+        self.assertEqual(gles[BLISS]["environment"]["MOBILEGL_TRACE_ANGLE_VARIANT"], "90a62123d794")
+        self.assertEqual(vulkan[BLISS]["environment"]["MOBILEGL_TRACE_ANGLE_VARIANT"], "90a62123d794")
+        self.assertEqual(gles["plain"]["environment"], {
+            "MOBILEGL_ESPRYT_USE_ANGLE": "1", "MOBILEGL_TRACE_ANGLE_VARIANT": "ec889e6ea831",
+            "MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "0", "MOBILEGL_ITRP_FIX": "Auto"})
+        self.assertEqual(gles[ITERATIONRP]["environment"]["MOBILEGL_ITRP_FIX"], "ForceOn")
+        self.assertEqual(flags(gles[BLISS]), ["--avoid-angle-llvmpipe-sampler-mipmap-min-filter"])
+        self.assertEqual(flags(vulkan[BLISS]), [])
+        self.assertEqual(flags(gles["lod"]), ["--avoid-angle-llvmpipe-explicit-lod-bias"])
+        self.assertEqual(flags(vulkan["lod"]), [])
+        self.assertEqual(flags(gles["flush"]), ["--coherent-as-flush"])
+        self.assertEqual(flags(vulkan["flush"]), ["--coherent-as-flush"])
+        self.assertEqual(flags(gles["plain"]), [])
+        arguments = gles["plain"]["arguments"]
+        self.assertEqual(arguments[arguments.index("--ssim-threshold") + 1], "0.985")
+        self.assertEqual(arguments[arguments.index("--alternate-golden") + 1], "")
+
+    def test_misconfigured_case_lists_run_nothing(self):
+        for cases in ([case("a"), case("a")], [case("a", ci_backends=["DirectGLES"])], [{"name": "a"}], []):
+            status, output = self.run_shard(cases)
+            self.assertEqual(status, 2, cases)
+            self.assertIn("::error title=Android retrace shard misconfigured::", output)
+        self.assertEqual(self.log("adb.log"), [])
+
+    # ---- the loop ----------------------------------------------------------------------
+
+    def test_every_case_runs_in_order_on_one_install(self):
+        cases = [case("a", golden=self.fixture("a.png"), alternate_golden=self.fixture("a-mali.png")),
+                 case("b", golden=self.fixture("b.png")), case("c")]
+        status, output = self.run_shard(cases, transport="inproc")
+        self.assertEqual(status, 0, output)
+        self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b", "c"])
+        for entry in self.log("replay.log"):
+            self.assertEqual(entry["environment"]["MOBILEGL_TRACE_SKIP_INSTALL"], "1")
+            self.assertEqual(entry["environment"]["MOBILEGL_TRACE_POLL_INTERVAL"], "0.5")
+        adb = self.log("adb.log")
+        self.assertEqual(sum(1 for call in adb if call[:1] == ["install"]), 1)
+        self.assertEqual(sum(1 for call in adb if call[:3] == ["shell", "pm", "clear"]), 3)
+        self.assertEqual(sum(1 for call in adb if call[:3] == ["shell", "rm", "-f"]), 3)
+        self.assertIn(["shell", "rm", "-f", "/data/local/tmp/mobilegl-b.trace", "/data/local/tmp/mobilegl-b.golden.png",
+                       "/data/local/tmp/mobilegl-b.alternate-golden.png"], adb)
+        self.assertEqual(self.log("avd.log"), [])
+        directory = self.results / "a-DirectVulkan"
+        self.assertEqual(json.loads((directory / "lane.json").read_text()),
+                         {"lane": "inproc-lane", "transport": "inproc"})
+        self.assertTrue((directory / "a-DirectVulkan-golden.png").is_file())
+        self.assertTrue((directory / "a-DirectVulkan-alternate-golden.png").is_file())
+        self.assertFalse((self.results / "c-DirectVulkan" / "c-DirectVulkan-golden.png").exists())
+        whole = self.shard_status()
+        self.assertTrue(whole["complete"])
+        self.assertEqual([(entry["case"], entry["status"]) for entry in whole["cases"]],
+                         [("a", "passed"), ("b", "passed"), ("c", "passed")])
+        self.assertIn("| b | passed | 1 |", (self.root / "summary.md").read_text())
+
+    def test_a_failed_case_does_not_stop_the_shard(self):
+        status, output = self.run_shard([case("a"), case("b"), case("c")], scenario={
+            "replay": {"b": [{"exit": 1, "message": "ssim=0.5"}]}})
+        self.assertEqual(status, 1)
+        self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b", "c"])
+        self.assertEqual([self.status(name)["status"] for name in "abc"], ["passed", "failed", "passed"])
+        self.assertIn("::error title=Android retrace failed (DirectVulkan / monolith)::b: exit 1: ssim=0.5", output)
+        self.assertEqual(output.count("::error"), 1)
+        self.assertTrue(self.shard_status()["complete"])
+        self.assertTrue((self.results / "b-DirectVulkan" / "emulator.log").is_file())
+
+    def test_the_case_after_one_that_did_not_pass_starts_on_a_cold_boot(self):
+        status, _ = self.run_shard([case("a"), case("b"), case("c"), case("d")], scenario={
+            "replay": {"a": [{"exit": 75}], "b": [{"exit": 1}]}})
+        self.assertEqual(status, 1)
+        self.assertEqual([self.status(name)["status"] for name in "abcd"], ["passed", "failed", "passed", "passed"])
+        # One restart for a's infrastructure retry and one before c. A case that passed on
+        # its retry costs the next case nothing.
+        self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start", "stop", "start"])
+        self.assertEqual(self.status("b")["notes"], [])
+        self.assertEqual(self.status("c")["notes"], ["emulator restarted before the case: "
+                                                    "the case before it, b, did not pass"])
+        self.assertEqual(self.status("d")["notes"], [])
+        self.assertEqual(sum(1 for call in self.log("adb.log") if call[:1] == ["install"]), 3)
+
+    def test_a_failed_case_without_an_avd_to_restart_does_not_stop_the_shard(self):
+        status, _ = self.run_shard([case("a"), case("b")], scenario={"replay": {"a": [{"exit": 1}]}},
+                                   env={"AVD_NAME": ""})
+        self.assertEqual(status, 1)
+        self.assertEqual([self.status(name)["status"] for name in "ab"], ["failed", "passed"])
+        self.assertEqual(self.log("avd.log"), [])
+
+    def test_an_exit_0_without_a_passed_result_or_proof_fails(self):
+        status, _ = self.run_shard([case("a"), case("b")], transport="spawn", scenario={
+            "replay": {"a": [{"exit": 0, "result": False}], "b": [{"exit": 0, "proof": False}]}})
+        self.assertEqual(status, 1)
+        self.assertIn("without a passed result.json", self.status("a")["reason"])
+        self.assertIn("without a passed spawn transport proof", self.status("b")["reason"])
+
+    def test_an_infrastructure_failure_restarts_and_retries_once(self):
+        status, output = self.run_shard([case("a"), case("b")], scenario={
+            "replay": {"a": [{"exit": 75, "reason": "angle-surface-lost", "result": True, "passed": False}]}})
+        self.assertEqual(status, 0, output)
+        self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "a", "b"])
+        self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start"])
+        self.assertEqual(self.log("avd.log")[1], [
+            "start", "--avd-name", "mobilegl-ci", "--emulator-log", str(self.emulator_log),
+            "--pid-file", str(self.root / "emulator.pid"), "--gpu", "lavapipe", "--boot-timeout", "300"])
+        self.assertIn(["kill-server"], self.log("adb.log"))
+        self.assertEqual(sum(1 for call in self.log("adb.log") if call[:1] == ["install"]), 2)
+        record = self.status("a")
+        self.assertEqual((record["status"], record["attempts"], record["exit_codes"]), ("passed", 2, [75, 0]))
+        self.assertEqual(record["surface_lost_retries"], 1)
+        kept = self.results / "a-DirectVulkan" / "first-attempt"
+        self.assertTrue((kept / "result.first-attempt.json").is_file())
+        self.assertFalse((kept / "result.json").exists())
+        self.assertIn("::warning::Android emulator infrastructure failed", output)
+        self.assertIn("surface-lost retries: 1 (DirectVulkan, a)", (self.root / "summary.md").read_text())
+        self.assertTrue((self.results / "diagnostics" / "emulator-boot-1-ended.log").is_file())
+        self.assertEqual(self.shard_status()["emulator_boots"], 2)
+
+    def test_a_second_infrastructure_failure_is_the_verdict(self):
+        status, _ = self.run_shard([case("a"), case("b")], scenario={
+            "replay": {"a": [{"exit": 75}, {"exit": 75}]}})
+        self.assertEqual(status, 1)
+        self.assertEqual(self.status("a")["exit_codes"], [75, 75])
+        self.assertEqual(self.status("a")["status"], "failed")
+        self.assertEqual(self.status("b")["status"], "passed")
+        # The surface-lost count is this case's own: the reasons of earlier cases do not leak in.
+        self.assertEqual(self.status("b")["surface_lost_retries"], 0)
+
+    def test_the_deadline_cuts_a_running_case_and_names_the_rest(self):
+        status, output = self.run_shard([case("a"), case("b"), case("c")], "--deadline-seconds", "5", scenario={
+            "replay": {"b": [{"sleep": 30}]}})
+        self.assertEqual(status, 1)
+        self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b"])
+        # b never had its own 1200 s, so the cut is not charged to it as a failure.
+        self.assertEqual([self.status(name)["status"] for name in "abc"], ["passed", "not-run", "not-run"])
+        self.assertRegex(self.status("b")["reason"], r"^cut by the shard deadline after \d+ s of a 1200 s budget$")
+        self.assertEqual(self.status("b")["exit_codes"], [124])
+        self.assertIn("deadline passed before this case started", self.status("c")["reason"])
+        self.assertIn("::error title=Android retrace not-run (DirectVulkan / monolith)::b: cut by the shard deadline",
+                      output)
+        self.assertIn("::error title=Android retrace shard incomplete (DirectVulkan / monolith)::"
+                      "1 verdict(s) for 3 case(s)", output)
+        self.assertFalse(self.shard_status()["complete"])
+        # Nothing is booted for a case the deadline keeps from starting.
+        self.assertEqual(self.log("avd.log"), [])
+
+    def test_a_case_that_runs_out_of_its_own_budget_fails(self):
+        # The deadline is far away, so the one second a has is its own budget.
+        with mock.patch.object(shard, "REPLAY_GRACE_SECONDS", 0):
+            status, _ = self.run_shard([case("a", timeout_seconds=1), case("b")], "--deadline-seconds", "600",
+                                       scenario={"replay": {"a": [{"sleep": 30}]}})
+        self.assertEqual(status, 1)
+        self.assertEqual([self.status(name)["status"] for name in "ab"], ["failed", "passed"])
+        self.assertEqual(self.status("a")["reason"], "the replay script exceeded its time budget")
+        self.assertTrue(self.shard_status()["complete"])
+
+    def test_an_infrastructure_retry_is_not_started_after_the_deadline(self):
+        def restart_past_the_deadline(runner, reason):
+            runner.started -= 1000
+            return True
+
+        with mock.patch.object(shard.Shard, "restart_emulator", restart_past_the_deadline):
+            status, _ = self.run_shard([case("a"), case("b")], "--deadline-seconds", "600",
+                                       scenario={"replay": {"a": [{"exit": 75}]}})
+        self.assertEqual(status, 1)
+        self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a"])
+        self.assertEqual([self.status(name)["status"] for name in "ab"], ["not-run", "not-run"])
+        self.assertEqual(self.status("a")["reason"],
+                         "the shard deadline passed before the infrastructure retry could start")
+        self.assertEqual(self.status("a")["exit_codes"], [75])
+
+    # ---- the time a shard needs --------------------------------------------------------
+
+    def test_needed_seconds_lets_every_case_use_its_budget(self):
+        cases = [case("a", timeout_seconds=1800), case("b", timeout_seconds=900), case("c", timeout_seconds="180")]
+        # Every case to the end of its budget, the longest once more, and the overhead per case.
+        self.assertEqual(shard.needed_seconds(cases), (2100 + 1200 + 480) + 2100 + 3 * 180)
+        self.assertEqual(shard.needed_seconds(cases[2:]), 480 + 480 + 180)
+
+    def test_a_deadline_of_needed_seconds_gives_every_hanging_case_its_verdict(self):
+        cases = [case(name, timeout_seconds=1) for name in "abc"]
+        # Scaled down: one second per case, and ten for what the fakes take around it.
+        with mock.patch.object(shard, "REPLAY_GRACE_SECONDS", 0), \
+                mock.patch.object(shard, "CASE_OVERHEAD_SECONDS", 10):
+            deadline = shard.needed_seconds(cases)
+            self.assertEqual(deadline, 3 + 1 + 30)
+            status, _ = self.run_shard(cases, "--deadline-seconds", str(deadline), scenario={
+                "replay": {name: [{"sleep": 60}] for name in "abc"}})
+        self.assertEqual(status, 1)
+        self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b", "c"])
+        for name in "abc":
+            self.assertEqual((self.status(name)["status"], self.status(name)["reason"]),
+                             ("failed", "the replay script exceeded its time budget"))
+        self.assertTrue(self.shard_status()["complete"])
+        self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start", "stop", "start"])
+
+    def test_every_shard_gets_its_own_deadline_and_job_timeout(self):
+        matrix = {"include": [
+            {"backend": {"name": "DirectGLES", "gpu": "software"}, "lane": "spawn-acceptance", "transport": "spawn",
+             "shard": 1, "shards": 2, "names": ["a", "b"],
+             "cases": [case("a", timeout_seconds=1800), case("b", timeout_seconds=1800)]},
+            {"backend": {"name": "DirectVulkan", "gpu": "lavapipe"}, "shard": 2, "shards": 2, "names": ["c"],
+             "cases": [case("c", timeout_seconds=180)]}]}
+        matrix_file = self.root / "matrix.json"
+        matrix_file.write_text(json.dumps(matrix))
+
+        def limits(cap, path=matrix_file):
+            output, report = io.StringIO(), io.StringIO()
+            with contextlib.redirect_stdout(output), contextlib.redirect_stderr(report):
+                status = budget.main(["--cap-seconds", str(cap), "--matrix-file", str(path)])
+            return status, output.getvalue(), report.getvalue()
+
+        self.assertEqual(budget.shard_needs(matrix), [("DirectGLES / spawn, shard 1/2", 2, 6660),
+                                                      ("DirectVulkan / monolith, shard 2/2", 1, 1140)])
+        # Under the cap every shard gets exactly what its cases need.
+        status, output, report = limits(6660)
+        self.assertEqual(status, 0, report)
+        rows = json.loads(output)["include"]
+        self.assertEqual([(row["deadline_seconds"], row["timeout_minutes"]) for row in rows],
+                         [(6660, 111 + budget.JOB_OVERHEAD_MINUTES), (1140, 19 + budget.JOB_OVERHEAD_MINUTES)])
+        # Nothing but the two fields is added, and no case is dropped or changed.
+        self.assertEqual([{key: value for key, value in row.items()
+                           if key not in ("deadline_seconds", "timeout_minutes")} for row in rows],
+                         matrix["include"])
+        self.assertIn("2 shard(s), 0 capped at 6660 s; the most a shard needs is 6660 s", report)
+        # Over the cap a shard is cut to it and named; the other shard is untouched.
+        status, output, report = limits(6000)
+        self.assertEqual(status, 0, report)
+        rows = json.loads(output)["include"]
+        self.assertEqual([(row["deadline_seconds"], row["timeout_minutes"]) for row in rows],
+                         [(6000, 100 + budget.JOB_OVERHEAD_MINUTES), (1140, 19 + budget.JOB_OVERHEAD_MINUTES)])
+        self.assertIn("DirectGLES / spawn, shard 1/2 would need 6660 s for all of its 2 case(s)", report)
+        self.assertNotIn("DirectVulkan / monolith, shard 2/2 would need", report)
+        # A cap that could not fit in a hosted job, and a matrix that cannot be read.
+        self.assertEqual(limits(0)[0], 2)
+        self.assertEqual(limits(360 * 60)[0], 2)
+        for broken in ({"include": []}, {"include": [{"names": ["a"]}]}, []):
+            matrix_file.write_text(json.dumps(broken))
+            status, output, _ = limits(7200)
+            self.assertEqual((status, output), (2, ""), broken)
+
+    def test_the_workflow_takes_its_limits_from_the_matrix(self):
+        text = (HERE.parent / ".github" / "workflows" / "apk.yml").read_text(encoding="utf-8")
+        retrace = text[text.index("\n  retrace:"):text.index("\n  retrace-summary:")]
+        cap, = re.findall(r'^\s+RETRACE_SHARD_DEADLINE_CAP_SECONDS: "(\d+)"\s*$', text, flags=re.MULTILINE)
+        # The runner's deadline and the job's limit are the two fields the budget script adds.
+        self.assertIn('--deadline-seconds "${{ matrix.deadline_seconds }}"', retrace)
+        self.assertEqual(re.findall(r"^    timeout-minutes: (.+?)\s*$", retrace, flags=re.MULTILINE),
+                         ["${{ matrix.timeout_minutes }}"])
+        self.assertIn('--cap-seconds "${RETRACE_SHARD_DEADLINE_CAP_SECONDS}"', text)
+        # The capped deadline and the job's overhead stay inside the six hours a hosted job may run.
+        self.assertLessEqual(-(-int(cap) // 60) + budget.JOB_OVERHEAD_MINUTES, budget.HOSTED_JOB_LIMIT_MINUTES)
+
+    def test_an_unhealthy_emulator_is_restarted_before_the_next_case(self):
+        status, output = self.run_shard([case("a"), case("b")], scenario={"offline_get_state_calls": 1})
+        self.assertEqual(status, 0, output)
+        self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start"])
+        self.assertIn("emulator restarted before the case: adb state is offline", self.status("a")["notes"])
+        self.assertEqual(self.status("b")["notes"], [])
+        self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b"])
+
+    def test_the_restart_switch_cold_boots_before_every_case_but_the_first(self):
+        status, _ = self.run_shard([case("a"), case("b"), case("c")],
+                                   env={"MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE": "1"})
+        self.assertEqual(status, 0)
+        self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start", "stop", "start"])
+        self.assertEqual(sum(1 for call in self.log("adb.log") if call[:1] == ["install"]), 3)
+
+    def test_an_emulator_that_does_not_come_back_leaves_named_rows(self):
+        status, output = self.run_shard([case("a"), case("b"), case("c")], scenario={
+            "replay": {"a": [{"exit": 75}]}, "avd_start_exits": [2, 2]})
+        self.assertEqual(status, 1)
+        self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a"])
+        self.assertEqual([self.status(name)["status"] for name in "abc"], ["failed", "not-run", "not-run"])
+        self.assertEqual(output.count("::error title=Android retrace"), 4)
+        for name in "abc":
+            self.assertTrue((self.results / f"{name}-DirectVulkan" / "lane.json").is_file())
+
+    def test_an_apk_that_does_not_install_runs_nothing(self):
+        status, _ = self.run_shard([case("a"), case("b")], scenario={"failing_installs": 9})
+        self.assertEqual(status, 1)
+        self.assertEqual(self.log("replay.log"), [])
+        self.assertEqual([self.status(name)["status"] for name in "ab"], ["not-run", "not-run"])
+
+    def test_processes_left_after_force_stop_are_killed_and_recorded(self):
+        # Listed on all four looks after the force-stop, gone after the kill.
+        status, _ = self.run_shard([case("a"), case("b")], scenario={"ps_calls_with_leftover": 4})
+        self.assertEqual(status, 0)
+        self.assertIn(["shell", "run-as", "top.mobilegl.plugin.trace", "kill", "-9", "4242"], self.log("adb.log"))
+        self.assertIn("processes left after force-stop: 4242 libMobileGLServer.so", self.status("a")["notes"])
+        self.assertEqual(self.status("b")["notes"], [])
+        self.assertEqual(self.log("avd.log"), [])
+
+    def test_a_process_that_is_still_exiting_is_not_reported(self):
+        status, _ = self.run_shard([case("a")], scenario={"ps_calls_with_leftover": 2})
+        self.assertEqual(status, 0)
+        self.assertEqual(self.status("a")["notes"], [])
+        self.assertNotIn("kill", [call[3] for call in self.log("adb.log") if call[:2] == ["shell", "run-as"]])
+
+    def test_processes_that_survive_the_kill_cost_an_emulator_restart(self):
+        status, _ = self.run_shard([case("a")], scenario={"ps_calls_with_leftover": 10})
+        self.assertEqual(status, 0)
+        self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start"])
+        self.assertIn("the emulator was restarted to remove them", self.status("a")["notes"])
+
+    def test_a_failed_pm_clear_falls_back_to_a_reinstall(self):
+        status, _ = self.run_shard([case("a")], scenario={"pm_clear_fails": True})
+        self.assertEqual(status, 0)
+        adb = self.log("adb.log")
+        self.assertIn(["uninstall", "top.mobilegl.plugin.trace"], adb)
+        self.assertEqual(sum(1 for call in adb if call[:1] == ["install"]), 2)
+
+    @unittest.skipUnless(os.name == "posix", "signals and process groups")
+    def test_a_terminated_runner_takes_the_running_replay_with_it(self):
+        cases = [case("a"), case("b")]
+        (self.state / "scenario.json").write_text(json.dumps({"replay": {"a": [{"sleep": 60}]}}))
+        arguments = ["--apk-file", str(self.apk), "--backend", "DirectVulkan", "--gpu", "lavapipe",
+                     "--transport", "monolith", "--lane", "monolith-lane", "--cases-json", json.dumps(cases),
+                     "--result-root", str(self.results), "--fixture-root", str(self.root / "work")]
+        runner = subprocess.Popen([sys.executable, str(HERE / "trace_replay_shard.py")] + arguments,
+                                  env={**os.environ, **self.environment}, stdout=subprocess.DEVNULL)
+        deadline = time.monotonic() + 30
+        while not (self.state / "replay.pid").exists() and time.monotonic() < deadline:
+            time.sleep(0.05)
+        replay_pid = int((self.state / "replay.pid").read_text())
+        runner.send_signal(signal.SIGTERM)
+        self.assertEqual(runner.wait(timeout=30), 128 + signal.SIGTERM)
+        for _ in range(100):
+            try:
+                os.kill(replay_pid, 0)
+            except OSError:
+                break
+            time.sleep(0.05)
+        else:
+            self.fail("the replay outlived the runner")
+        # The step that runs afterwards names both cases.
+        status, _ = self.run_shard(cases, "--finalize")
+        self.assertEqual(status, 0)
+        self.assertEqual([self.status(name)["status"] for name in "ab"], ["not-run", "not-run"])
+
+    def test_finalize_names_every_case_without_a_device(self):
+        cases = [case("a", golden=self.fixture("a.png")), case("b")]
+        status, _ = self.run_shard(cases)
+        self.assertEqual(status, 0)
+        (self.results / "b-DirectVulkan" / "case-status.json").unlink()
+        calls_before = len(self.log("adb.log"))
+        status, _ = self.run_shard(cases + [case("c")], "--finalize")
+        self.assertEqual(status, 0)
+        self.assertEqual(len(self.log("adb.log")), calls_before)
+        self.assertEqual([self.status(name)["status"] for name in "abc"], ["passed", "not-run", "not-run"])
+        self.assertTrue((self.results / "c-DirectVulkan" / "lane.json").is_file())
+        self.assertFalse(self.shard_status()["complete"])
+        self.assertEqual(self.shard_status()["expected_cases"], ["a", "b", "c"])
+
+
+if __name__ == "__main__":
+    unittest.main()

android-plugin/trace-replay-ci.sh

@@ -62,6 +62,9 @@ copies benchmark.json (per-frame times plus mean/median/p95) out of the app, and
 "passed" only means the replay reached the end of the trace without an error.
 Pass --reuse-fixture to skip re-extracting and re-pushing the trace, for repeat
 runs of a case whose fixture is already in /data/local/tmp.
+Set MOBILEGL_TRACE_POLL_INTERVAL to the seconds between result polls (for example
+0.5) to wait on a wall clock: the wait then ends --timeout-seconds plus a fifth after
+the app starts, which is about how long that many one-second polls take.
 Pass --env "K=V;K=V" (or set MOBILEGL_TRACE_ENV) to hand arbitrary environment
 variables to the replay process. They are applied last, immediately before
 libMobileGL.so is loaded, so they override every flag above; an entry with no "="
@@ -93,6 +96,21 @@ adb_device_path() {
   MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL='*' "${ADB}" "$@"
 }
 
+# Milliseconds since the epoch; whole seconds where date has no %N.
+now_ms() {
+  stamp="$(date +%s%3N 2>/dev/null || true)"
+  case "${stamp}" in
+    ''|*[!0-9]*) stamp="$(($(date +%s) * 1000))" ;;
+  esac
+  printf '%s' "${stamp}"
+}
+
+# Appends "<phase> <ms>" to phase-timings.txt in the result directory, so a run records where
+# its wall time went: push, install, replay, log pulls.
+mark_phase() {
+  printf '%s %s\n' "$1" "$(now_ms)" >> "${result_dir}/phase-timings.txt" 2>/dev/null || true
+}
+
 next_arg() {
   if [ "$#" -lt 2 ]; then
     die "$1 requires a value"
@@ -411,6 +429,81 @@ copy_fixture_to_app() {
   fi
 }
 
+# Waits for result.json with one poll per second; --timeout-seconds counts the polls.
+wait_for_result_by_count() {
+  # One failed pidof is not an exit: under memory pressure the adb shell call
+  # itself can fail while the app is alive (seen with the process still in
+  # pidof and the activity still initializing right after the "exit").
+  missed_app_polls=0
+  for _ in $(seq 1 "${timeout_seconds}"); do
+    if adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then
+      break
+    fi
+    if adb_device_path shell pidof "${package_name}" >/dev/null 2>&1; then
+      saw_app_process=1
+      missed_app_polls=0
+    else
+      poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
+      if [ "${poll_adb_state}" != "device" ]; then
+        printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt"
+        break
+      fi
+      if [ "${saw_app_process}" -eq 1 ]; then
+        missed_app_polls=$((missed_app_polls + 1))
+        if [ "${missed_app_polls}" -ge 3 ]; then
+          app_exited=1
+          break
+        fi
+      fi
+    fi
+    sleep 1
+  done
+}
+
+# The same wait on a wall clock, selected by MOBILEGL_TRACE_POLL_INTERVAL (seconds between
+# polls, fractions allowed). One adb call per poll asks for result.json and for the app
+# process. The budget is --timeout-seconds of wall time plus a fifth: a poll of the
+# one-second loop takes about 1.2 s with its two adb calls, so a replay gets the same time
+# to finish under either loop. An exit needs three missed polls spread over at least three
+# seconds, so a short interval keeps the one-second loop's tolerance for adb calls that fail
+# under memory pressure.
+wait_for_result_by_clock() {
+  poll_deadline=$((poll_started + timeout_seconds + timeout_seconds / 5))
+  missed_app_polls=0
+  first_missed_at=0
+  while [ "$(date +%s)" -lt "${poll_deadline}" ]; do
+    poll_output="$(adb_device_path shell "run-as ${package_name} ls ${app_dir}/output/result.json >/dev/null 2>&1 && echo mobilegl-poll-result; pidof ${package_name} >/dev/null 2>&1 && echo mobilegl-poll-alive; echo mobilegl-poll-end" 2>/dev/null | tr -d '\r' || true)"
+    case "${poll_output}" in
+      *mobilegl-poll-result*)
+        break
+        ;;
+      *mobilegl-poll-alive*)
+        saw_app_process=1
+        missed_app_polls=0
+        ;;
+      *)
+        poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
+        if [ "${poll_adb_state}" != "device" ]; then
+          printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt"
+          break
+        fi
+        if [ "${saw_app_process}" -eq 1 ]; then
+          poll_now="$(date +%s)"
+          if [ "${missed_app_polls}" -eq 0 ]; then
+            first_missed_at="${poll_now}"
+          fi
+          missed_app_polls=$((missed_app_polls + 1))
+          if [ "${missed_app_polls}" -ge 3 ] && [ "$((poll_now - first_missed_at))" -ge 3 ]; then
+            app_exited=1
+            break
+          fi
+        fi
+        ;;
+    esac
+    sleep "${MOBILEGL_TRACE_POLL_INTERVAL}"
+  done
+}
+
 run_retrace() {
   result_dir="${result_root}/${safe_case}-${backend}"
   alternate_golden_app_path=""
@@ -427,13 +520,17 @@ run_retrace() {
   fi
 
   mkdir -p "${result_dir}"
+  printf 'script-started %s\n' "${script_started_ms}" > "${result_dir}/phase-timings.txt" 2>/dev/null || true
+  mark_phase fixture-pushed
   # A repeat run of a case the previous invocation already installed and pushed only needs
   # the on-device copy back into a fresh app output directory.
   # MOBILEGL_TRACE_SKIP_INSTALL=1: the caller installed the APK itself (ColorOS shows an install-confirmation
-  # dialog whose tap drops adb for a few seconds, so the install has to be done and settled outside a run).
+  # dialog whose tap drops adb for a few seconds, so the install has to be done and settled outside a run;
+  # trace-replay-shard.sh installs once per emulator boot).
   if [ "${reuse_fixture}" -eq 0 ] && [ "${MOBILEGL_TRACE_SKIP_INSTALL:-0}" != "1" ]; then
     "${ADB}" install -r "$(host_path_for_adb "${apk_file}")"
   fi
+  mark_phase apk-installed
   copy_fixture_to_app
   adb_device_path shell am force-stop "${package_name}"
   "${ADB}" logcat -c
@@ -502,39 +599,20 @@ run_retrace() {
     --ei crop_width "${crop_width}" \
     --ei crop_height "${crop_height}"
   adb_device_path shell "$@"
+  mark_phase app-started
 
   app_exited=0
   saw_app_process=0
-  # One failed pidof is not an exit: under memory pressure the adb shell call
-  # itself can fail while the app is alive (seen with the process still in
-  # pidof and the activity still initializing right after the "exit").
-  missed_app_polls=0
   poll_started="$(date +%s)"
-  for _ in $(seq 1 "${timeout_seconds}"); do
+  if [ -n "${MOBILEGL_TRACE_POLL_INTERVAL:-}" ]; then
-    if adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then
+    wait_for_result_by_clock
-      break
+  else
-    fi
+    wait_for_result_by_count
-    if adb_device_path shell pidof "${package_name}" >/dev/null 2>&1; then
+  fi
-      saw_app_process=1
+  mark_phase wait-ended
-      missed_app_polls=0
-    else
-      poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
-      if [ "${poll_adb_state}" != "device" ]; then
-        printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt"
-        break
-      fi
-      if [ "${saw_app_process}" -eq 1 ]; then
-        missed_app_polls=$((missed_app_polls + 1))
-        if [ "${missed_app_polls}" -ge 3 ]; then
-          app_exited=1
-          break
-        fi
-      fi
-    fi
-    sleep 1
-  done
 
   collect_run_diagnostics "${result_dir}"
+  mark_phase diagnostics-collected
   if ! adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then
     if [ -f "${result_dir}/adb-disconnected.txt" ]; then
       echo "trace-replay-ci.sh: device disconnected while waiting for result.json" >&2
@@ -574,6 +652,7 @@ run_retrace() {
     copy_app_artifact "${app_dir}/output/benchmark.json" "${result_dir}/benchmark.json"
   fi
   copy_texture_2d_dumps
+  mark_phase artifacts-copied
 
   # A replay that wrote result.json but did not pass used to print nothing but
   # the JSON, which for a non-zero statusCode says only "retrace failed with
@@ -696,6 +775,7 @@ PY
   fi
 }
 
+script_started_ms="$(now_ms)"
 mkdir -p "${fixture_root}" "${result_root}"
 if [ "${reuse_fixture}" -eq 0 ]; then
   prepare_fixture

android-plugin/trace-replay-shard.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+# MobileGL - android-plugin/trace-replay-shard.sh
+# Copyright (c) 2025-2026 MobileGL-Dev
+# Licensed under the GNU Lesser General Public License v3.0:
+#   https://www.gnu.org/licenses/gpl-3.0.txt
+#   https://www.gnu.org/licenses/lgpl-3.0.txt
+# SPDX-License-Identifier: LGPL-3.0-only
+# End of Source File Header
+
+# Replays an ordered list of trace cases for one (lane, transport, backend) on one emulator
+# boot, each through trace-replay-ci.sh. The loop, the per-case settings and the verdict files
+# are in trace_replay_shard.py; `--help` lists the arguments.
+#
+#   sh android-plugin/trace-replay-shard.sh \
+#     --apk-file FILE --backend DirectGLES|DirectVulkan --gpu GPU \
+#     --transport monolith|inproc|spawn --lane LANE \
+#     --cases-env VARIABLE | --cases-file FILE | --cases-json JSON \
+#     [--deadline-seconds N] [--dry-run | --finalize]
+#
+# Environment:
+#   AVD_NAME, EMULATOR_LOG, EMULATOR_PID_FILE   the emulator to restart when it stops answering
+#   MOBILEGL_GPU_HANG_BUDGET_MS                 forwarded to every replay (required)
+#   MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE   1 cold boots the emulator before every case
+#   MOBILEGL_SHARD_POLL_INTERVAL                seconds between result polls (default 0.5);
+#                                               empty selects trace-replay-ci.sh's 1 s loop
+set -eu
+
+exec "${PYTHON:-python3}" "$(dirname "$0")/trace_replay_shard.py" "$@"

android-plugin/trace_replay_shard.py

@@ -0,0 +1,792 @@
+#!/usr/bin/env python3
+# MobileGL - android-plugin/trace_replay_shard.py
+# Copyright (c) 2025-2026 MobileGL-Dev
+# Licensed under the GNU Lesser General Public License v3.0:
+#   https://www.gnu.org/licenses/gpl-3.0.txt
+#   https://www.gnu.org/licenses/lgpl-3.0.txt
+# SPDX-License-Identifier: LGPL-3.0-only
+# End of Source File Header
+
+"""Replay an ordered list of trace cases on one emulator boot.
+
+One shard is one (lane, transport, backend). Each case runs through trace-replay-ci.sh with
+the per-case and per-transport settings of the APK workflow, in a fresh app process with
+cleared app data. A failing case does not stop the shard: the case after it starts on a cold
+boot, every case gets a verdict file, and the exit status is non-zero when a case failed or a
+verdict is missing.
+
+Layout under --result-root (the directory the workflow uploads):
+
+  <case>-<backend>/           what trace-replay-ci.sh writes, plus lane.json, the golden
+                              copies the summary renders, and case-status.json
+  <case>-<backend>/first-attempt/   files of an attempt that was retried
+  shard-status.json           every case of the shard with its verdict
+  diagnostics/                emulator logs kept across restarts
+"""
+
+import argparse
+import json
+import os
+import re
+import shlex
+import shutil
+import signal
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+INFRASTRUCTURE_FAILURE_EXIT_CODE = 75
+TIMEOUT_EXIT_CODE = 124
+BACKENDS = ("DirectGLES", "DirectVulkan")
+TRANSPORTS = ("monolith", "inproc", "spawn")
+PASSED, FAILED, NOT_RUN = "passed", "failed", "not-run"
+
+# Cases with settings of their own. Bliss needs the newer packaged ANGLE build and, on
+# DirectGLES, sampler mipmap min-filter downgrading on ANGLE llvmpipe; iterationRP needs its
+# SPIR-V repairs forced on.
+BLISS_CASE = "minecraft-1.21.4-fabric-iris-bliss-in-world"
+ITERATIONRP_CASE = "minecraft-1.21.4-fabric-iris-iterationrp-in-world"
+DEFAULT_ANGLE_VARIANT = "ec889e6ea831"
+BLISS_ANGLE_VARIANT = "90a62123d794"
+
+# The inproc and spawn lanes pass the same client settings, so they differ only in which
+# process runs the server role.
+SPLIT_CLIENT_SETTINGS = "MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1"
+
+REQUIRED_CASE_KEYS = ("name", "trace_archive", "trace_file", "golden", "target_call", "width", "height",
+                      "crop_x", "crop_y", "crop_width", "crop_height", "timeout_seconds")
+# Seconds trace-replay-ci.sh gets beyond the case's own budget for install, push and log pulls.
+REPLAY_GRACE_SECONDS = 300
+BOOT_TIMEOUT_SECONDS = 300
+# Seconds a case may take around its replays: the app reset, the failure diagnostics and the
+# cold boot that follows a case that did not pass.
+CASE_OVERHEAD_SECONDS = 180
+
+
+def say(message):
+    print(message, flush=True)
+
+
+def pause(seconds):
+    time.sleep(seconds)
+
+
+def safe_case(name):
+    return re.sub(r"[^A-Za-z0-9._-]", "_", name)
+
+
+def as_text(value):
+    if value is True:
+        return "true"
+    if value is False:
+        return "false"
+    return "" if value is None else str(value)
+
+
+def is_true(value):
+    return value is True or value == "true"
+
+
+def load_cases(text, backend):
+    """The case objects of the APK matrix, as text ready for the command line."""
+    try:
+        # Numbers stay the literal text of the matrix, so 0.985 is passed on as 0.985.
+        cases = json.loads(text, parse_float=str, parse_int=str)
+    except json.JSONDecodeError as error:
+        raise ValueError(f"the case list is not JSON: {error}") from error
+    if not isinstance(cases, list) or not cases:
+        raise ValueError("the case list must be a non-empty JSON array of case objects")
+    names = set()
+    for index, case in enumerate(cases):
+        if not isinstance(case, dict):
+            raise ValueError(f"case {index} is not an object; pass the case objects of the APK matrix")
+        missing = [key for key in REQUIRED_CASE_KEYS if case.get(key) in (None, "")]
+        if missing:
+            raise ValueError(f"case {case.get('name', index)} lacks {', '.join(missing)}")
+        name = case["name"]
+        if name in names:
+            raise ValueError(f"case {name} is listed twice")
+        names.add(name)
+        backends = case.get("ci_backends")
+        if backends is not None and backend not in backends:
+            raise ValueError(f"case {name} does not run on {backend} (ci_backends: {backends})")
+    return cases
+
+
+def needed_seconds(cases):
+    """Wall time in which every case of a shard can use its whole time budget.
+
+    It covers each case running until its replay script is stopped, one infrastructure retry
+    that does so again for the longest budget, and CASE_OVERHEAD_SECONDS per case. A shard
+    deadline at or above this never stops a case that is within its own budget, unless more
+    than one case needs a retry as long as its first attempt.
+    """
+    budgets = [int(as_text(case["timeout_seconds"])) + REPLAY_GRACE_SECONDS for case in cases]
+    return sum(budgets) + max(budgets) + CASE_OVERHEAD_SECONDS * len(budgets)
+
+
+def replay_environment(backend, case_name):
+    return {
+        "MOBILEGL_ESPRYT_USE_ANGLE": "1" if backend == "DirectGLES" else "0",
+        "MOBILEGL_TRACE_ANGLE_VARIANT": BLISS_ANGLE_VARIANT if case_name == BLISS_CASE else DEFAULT_ANGLE_VARIANT,
+        "MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "1" if backend == "DirectVulkan" else "0",
+        "MOBILEGL_ITRP_FIX": "ForceOn" if case_name == ITERATIONRP_CASE else "Auto",
+    }
+
+
+def transport_arguments(transport, hang_budget_ms):
+    hang_budget = f"MOBILEGL_GPU_HANG_BUDGET_MS={hang_budget_ms}"
+    if transport == "inproc":
+        return ["--env", f"MOBILEGL_TRANSPORT=inproc;{SPLIT_CLIENT_SETTINGS};{hang_budget}", "--require-inproc"]
+    if transport == "spawn":
+        # No MOBILEGL_IPC_SERVER_PATH: only the app can spell nativeLibraryDir, and
+        # TraceReplayActivity resolves it when the environment asks for spawn. The pbuffer is
+        # required: a spawned server cannot be handed the client's window.
+        return ["--env", f"MOBILEGL_TRANSPORT=spawn;{SPLIT_CLIENT_SETTINGS};{hang_budget}",
+                "--use-pbuffer", "--require-spawn"]
+    return ["--env", f"MOBILEGL_TRANSPORT=monolith;{hang_budget}"]
+
+
+def replay_arguments(options, case, hang_budget_ms):
+    name = case["name"]
+    arguments = [
+        "--apk-file", options.apk_file,
+        "--package", options.package,
+        "--backend", options.backend,
+        "--result-root", options.result_root,
+        "--fixture-root", options.fixture_root,
+        "--case", name,
+        "--trace-archive", as_text(case["trace_archive"]),
+        "--trace-file", as_text(case["trace_file"]),
+        "--golden", as_text(case["golden"]),
+        "--alternate-golden", as_text(case.get("alternate_golden") or ""),
+        "--target-call", as_text(case["target_call"]),
+        "--width", as_text(case["width"]),
+        "--height", as_text(case["height"]),
+        "--ssim-threshold", as_text(case.get("ssim_threshold") or "0.99"),
+        "--crop-x", as_text(case["crop_x"]),
+        "--crop-y", as_text(case["crop_y"]),
+        "--crop-width", as_text(case["crop_width"]),
+        "--crop-height", as_text(case["crop_height"]),
+        "--timeout-seconds", as_text(case["timeout_seconds"]),
+    ]
+    arguments += transport_arguments(options.transport, hang_budget_ms)
+    if options.backend == "DirectGLES" and name == BLISS_CASE:
+        arguments.append("--avoid-angle-llvmpipe-sampler-mipmap-min-filter")
+    if options.backend == "DirectGLES" and is_true(case.get("avoid_angle_llvmpipe_explicit_lod_bias")):
+        arguments.append("--avoid-angle-llvmpipe-explicit-lod-bias")
+    if is_true(case.get("coherent_as_flush")):
+        arguments.append("--coherent-as-flush")
+    return arguments
+
+
+def read_json(path):
+    try:
+        return json.loads(Path(path).read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+
+
+def write_json(path, value):
+    Path(path).write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8")
+
+
+def one_line(text):
+    return " ".join(str(text).split())
+
+
+class Shard:
+    def __init__(self, options, cases, environ):
+        self.options = options
+        self.cases = cases
+        self.environ = dict(environ)
+        self.adb_command = shlex.split(environ.get("ADB", "adb"))
+        directory = Path(__file__).resolve().parent
+        self.replay_command = shlex.split(environ.get("MOBILEGL_SHARD_REPLAY_COMMAND") or
+                                          f"sh {shlex.quote(str(directory / 'trace-replay-ci.sh'))}")
+        self.avd_command = shlex.split(environ.get("MOBILEGL_SHARD_AVD_COMMAND") or
+                                       f"sh {shlex.quote(str(directory / 'run-avd-ci.sh'))}")
+        self.hang_budget_ms = environ.get("MOBILEGL_GPU_HANG_BUDGET_MS")
+        self.restart_each_case = environ.get("MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE", "0") == "1"
+        self.poll_interval = environ.get("MOBILEGL_SHARD_POLL_INTERVAL", "0.5")
+        self.avd_name = environ.get("AVD_NAME", "")
+        self.emulator_log = environ.get("EMULATOR_LOG", "")
+        self.emulator_pid_file = environ.get("EMULATOR_PID_FILE", "")
+        self.result_root = Path(options.result_root)
+        self.diagnostics = self.result_root / "diagnostics"
+        self.started = time.monotonic()
+        self.boots = 1
+        self.install_seconds = []
+        self.records = {}
+        self.aborted = ""
+        # The case before the current one when it did not pass; the next case gets a cold boot.
+        self.unclean_case = ""
+
+    # ---- bookkeeping -------------------------------------------------------------------
+
+    def result_dir(self, case):
+        return self.result_root / f"{safe_case(case['name'])}-{self.options.backend}"
+
+    def record_for(self, case, status, reason, **extra):
+        record = {"case": case["name"], "backend": self.options.backend, "transport": self.options.transport,
+                  "lane": self.options.lane, "status": status, "reason": reason}
+        record.update(extra)
+        return record
+
+    def write_summary_inputs(self, case):
+        """lane.json and the golden copies the retrace summary reads next to each result."""
+        directory = self.result_dir(case)
+        directory.mkdir(parents=True, exist_ok=True)
+        write_json(directory / "lane.json", {"lane": self.options.lane, "transport": self.options.transport})
+        stem = f"{safe_case(case['name'])}-{self.options.backend}"
+        for key, suffix in (("golden", "golden"), ("alternate_golden", "alternate-golden")):
+            source = as_text(case.get(key) or "")
+            if source and os.path.isfile(source) and os.path.getsize(source) > 0:
+                shutil.copyfile(source, directory / f"{stem}-{suffix}.png")
+
+    def save_record(self, case, record):
+        self.records[case["name"]] = record
+        self.write_summary_inputs(case)
+        write_json(self.result_dir(case) / "case-status.json", record)
+        self.write_shard_status()
+
+    def write_shard_status(self, complete=False):
+        self.result_root.mkdir(parents=True, exist_ok=True)
+        records = [self.records.get(case["name"]) or
+                   self.record_for(case, NOT_RUN, "the shard has not reached this case") for case in self.cases]
+        write_json(self.result_root / "shard-status.json", {
+            "schema": 1, "lane": self.options.lane, "transport": self.options.transport,
+            "backend": self.options.backend, "complete": complete,
+            # The Actions job that ran the shard, so the artifact cleanup can tell whose job failed.
+            "job_id": self.environ.get("MOBILEGL_SHARD_JOB_ID", ""),
+            "expected_cases": [case["name"] for case in self.cases], "cases": records,
+            "emulator_boots": self.boots, "install_seconds": self.install_seconds})
+
+    def remaining_seconds(self):
+        if self.options.deadline_seconds <= 0:
+            return None
+        return self.options.deadline_seconds - (time.monotonic() - self.started)
+
+    def deadline_passed(self):
+        remaining = self.remaining_seconds()
+        return remaining is not None and remaining <= 0
+
+    # ---- processes ---------------------------------------------------------------------
+
+    def run(self, command, timeout=None, capture=False, extra_env=None):
+        """Returns (exit status, output). A timeout kills the whole process group and reads as 124."""
+        environment = dict(self.environ)
+        environment.update(extra_env or {})
+        own_group = os.name == "posix"
+        try:
+            process = subprocess.Popen(command, env=environment, start_new_session=own_group,
+                                       stdout=subprocess.PIPE if capture else None,
+                                       stderr=subprocess.STDOUT if capture else None)
+        except OSError as error:
+            return 127, str(error)
+        def kill():
+            if own_group:
+                try:
+                    os.killpg(process.pid, signal.SIGKILL)
+                except OSError:
+                    pass
+            process.kill()
+
+        try:
+            output, _ = process.communicate(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            kill()
+            output, _ = process.communicate()
+            return TIMEOUT_EXIT_CODE, (output or b"").decode("utf-8", "replace")
+        except BaseException:
+            # The job was canceled or timed out. The child is in a session of its own, so it
+            # would otherwise keep driving the device behind the steps that still run.
+            kill()
+            raise
+        return process.returncode, (output or b"").decode("utf-8", "replace")
+
+    def adb(self, *arguments, timeout=60):
+        status, output = self.run(self.adb_command + list(arguments), timeout=timeout, capture=True)
+        return status, output.replace("\r", "")
+
+    # ---- emulator ----------------------------------------------------------------------
+
+    def can_restart(self):
+        return bool(self.avd_name and self.options.gpu and self.emulator_log and self.emulator_pid_file)
+
+    def emulator_pid(self):
+        try:
+            return int(Path(self.emulator_pid_file).read_text().strip())
+        except (OSError, ValueError):
+            return None
+
+    def emulator_process_alive(self):
+        pid = self.emulator_pid()
+        if pid is None:
+            return True  # nothing to check against, for example a real device
+        try:
+            os.kill(pid, 0)
+        except OSError:
+            return False
+        return True
+
+    def health_problem(self):
+        """Empty when the device answers; otherwise what did not."""
+        if not self.emulator_process_alive():
+            return "the emulator process is gone"
+        status, output = self.adb("get-state", timeout=15)
+        if status != 0 or output.strip() != "device":
+            return f"adb state is {output.strip() or 'unknown'}"
+        status, output = self.adb("shell", "getprop", "sys.boot_completed", timeout=15)
+        if status != 0 or output.strip() != "1":
+            return "sys.boot_completed is not 1"
+        # A framework call: boot_completed stays 1 across a system_server restart.
+        status, output = self.adb("shell", "pm", "path", self.options.package, timeout=30)
+        if status != 0 or "package:" not in output:
+            return "the package manager does not list the trace APK"
+        return ""
+
+    def keep_emulator_log(self, label):
+        if self.emulator_log and os.path.isfile(self.emulator_log):
+            self.diagnostics.mkdir(parents=True, exist_ok=True)
+            shutil.copyfile(self.emulator_log, self.diagnostics / f"emulator-boot-{self.boots}-{label}.log")
+
+    def prepare_booted_device(self):
+        # A shard outlives the screen timeout a single replay never reached.
+        self.adb("shell", "svc", "power", "stayon", "true", timeout=30)
+        return self.install()
+
+    def install(self):
+        started = time.monotonic()
+        status, output = self.adb("install", "-r", self.options.apk_file, timeout=300)
+        self.install_seconds.append(round(time.monotonic() - started, 1))
+        say(output.strip())
+        return status == 0
+
+    def restart_emulator(self, reason):
+        """Stop, cold boot and reinstall. False when the emulator did not come back."""
+        if not self.can_restart():
+            say(f"trace-replay-shard: cannot restart the emulator ({reason}): no AVD is configured")
+            return False
+        say(f"trace-replay-shard: restarting the emulator: {reason}")
+        common = ["--avd-name", self.avd_name, "--emulator-log", self.emulator_log,
+                  "--pid-file", self.emulator_pid_file]
+        for attempt in (1, 2):
+            # The start truncates the log, and the boot that failed is the one worth reading.
+            self.keep_emulator_log(f"attempt-{attempt}" if attempt > 1 else "ended")
+            old_pid = self.emulator_pid()
+            self.run(self.avd_command + ["stop"] + common, timeout=120)
+            self.wait_for_exit(old_pid)
+            self.adb("kill-server", timeout=30)
+            pause(2)
+            self.boots += 1
+            status, _ = self.run(self.avd_command + ["start"] + common +
+                                 ["--gpu", self.options.gpu, "--boot-timeout", str(BOOT_TIMEOUT_SECONDS)],
+                                 timeout=2 * BOOT_TIMEOUT_SECONDS + 60)
+            if status == 0 and self.prepare_booted_device():
+                return True
+            say(f"trace-replay-shard: emulator restart attempt {attempt} failed")
+        return False
+
+    def wait_for_exit(self, pid):
+        if pid is None or os.name != "posix":
+            return
+        for _ in range(60):
+            try:
+                os.kill(pid, 0)
+            except OSError:
+                return
+            pause(0.5)
+        try:
+            os.kill(pid, signal.SIGKILL)
+        except OSError:
+            pass
+
+    # ---- app state ---------------------------------------------------------------------
+
+    def leftover_processes(self):
+        """Processes still running under the app's uid, as 'pid name' strings."""
+        status, user = self.adb("shell", "run-as", self.options.package, "id", "-un", timeout=30)
+        user = user.strip()
+        if status != 0 or not re.fullmatch(r"[A-Za-z0-9_]+", user):
+            return None
+        status, listing = self.adb("shell", "ps", "-A", "-o", "USER,PID,NAME", timeout=30)
+        if status != 0:
+            return None
+        found = []
+        for line in listing.splitlines():
+            fields = line.split()
+            if len(fields) >= 3 and fields[0] == user and fields[1].isdigit():
+                found.append(f"{fields[1]} {fields[2]}")
+        return found
+
+    def settled_leftover_processes(self, polls):
+        """leftover_processes once the app's uid is empty, or as it still is after `polls` looks.
+
+        A killed process stays listed for a moment, so one look right after the kill would
+        report every force-stop as incomplete.
+        """
+        found = self.leftover_processes()
+        for _ in range(polls - 1):
+            if not found:
+                break
+            pause(0.5)
+            found = self.leftover_processes()
+        return found
+
+    def reset_app_state(self):
+        """Fresh process and fresh app data, as a new install gives. Returns notes for the record."""
+        notes = []
+        package = self.options.package
+        self.adb("shell", "am", "force-stop", package, timeout=60)
+        leftovers = self.settled_leftover_processes(polls=4)
+        if leftovers is None:
+            notes.append("the app's processes could not be listed after force-stop")
+        if leftovers:
+            # A force-stop is expected to take a spawned server with it; say so when it did not.
+            notes.append("processes left after force-stop: " + ", ".join(leftovers))
+            self.adb("shell", "run-as", package, "kill", "-9", *[entry.split()[0] for entry in leftovers],
+                     timeout=30)
+            if self.settled_leftover_processes(polls=6):
+                notes.append("the leftover processes could not be killed")
+                if self.restart_emulator("app processes survived force-stop and kill"):
+                    notes.append("the emulator was restarted to remove them")
+        status, output = self.adb("shell", "pm", "clear", package, timeout=60)
+        probe_status, _ = self.adb("shell", "run-as", package, "ls", f"/data/user/0/{package}", timeout=30)
+        if status != 0 or "Success" not in output or probe_status != 0:
+            # The state a fresh install leaves, reached the other way.
+            notes.append("pm clear did not leave a usable app data directory; reinstalled the APK")
+            self.adb("uninstall", package, timeout=120)
+            self.install()
+        return notes
+
+    def remove_case_leftovers(self, case):
+        name = safe_case(case["name"])
+        base = f"/data/local/tmp/mobilegl-{name}"
+        self.adb("shell", "rm", "-f", f"{base}.trace", f"{base}.golden.png", f"{base}.alternate-golden.png",
+                 timeout=60)
+        self.adb("shell", "am", "force-stop", self.options.package, timeout=60)
+        shutil.rmtree(Path(self.options.fixture_root) / name, ignore_errors=True)
+
+    # ---- one case ----------------------------------------------------------------------
+
+    def reason_lines(self):
+        try:
+            return (self.result_root / "infrastructure-failure-reason.txt").read_text().splitlines()
+        except OSError:
+            return []
+
+    def run_attempt(self, case):
+        """Returns (exit status, seconds, how the shard deadline cut the attempt short or "")."""
+        case_budget = int(as_text(case["timeout_seconds"])) + REPLAY_GRACE_SECONDS
+        budget = case_budget
+        remaining = self.remaining_seconds()
+        if remaining is not None and remaining < case_budget:
+            budget = max(remaining, 1.0)
+        environment = replay_environment(self.options.backend, case["name"])
+        # The shard installs the APK once per boot.
+        environment["MOBILEGL_TRACE_SKIP_INSTALL"] = "1"
+        environment["MOBILEGL_TRACE_POLL_INTERVAL"] = self.poll_interval
+        started = time.monotonic()
+        status, _ = self.run(self.replay_command + replay_arguments(self.options, case, self.hang_budget_ms),
+                             timeout=budget, extra_env=environment)
+        elapsed = round(time.monotonic() - started, 1)
+        cut = ""
+        if status == TIMEOUT_EXIT_CODE and budget < case_budget:
+            # The case never had its own budget, so the timeout says nothing about the case.
+            cut = f"cut by the shard deadline after {elapsed:.0f} s of a {case_budget} s budget"
+        return status, elapsed, cut
+
+    def keep_first_attempt(self, case):
+        """Move the retried attempt's files aside so the retry starts from an empty directory.
+
+        result.json is renamed: the summary keys rows on any result.json below a directory
+        that names the case, and the retry's is the one that counts.
+        """
+        directory = self.result_dir(case)
+        if not directory.is_dir():
+            return
+        kept = directory / "first-attempt"
+        kept.mkdir(exist_ok=True)
+        for entry in directory.iterdir():
+            if entry.is_file():
+                name = "result.first-attempt.json" if entry.name == "result.json" else entry.name
+                entry.replace(kept / name)
+
+    def verdict(self, case, status):
+        directory = self.result_dir(case)
+        if status == 0:
+            result = read_json(directory / "result.json")
+            if not isinstance(result, dict) or result.get("passed") is not True:
+                return FAILED, "the replay exited 0 without a passed result.json"
+            if self.options.transport != "monolith":
+                proof = read_json(directory / "transport-proof.json")
+                if not (isinstance(proof, dict) and proof.get("passed") is True and
+                        proof.get("required_transport") == self.options.transport):
+                    return FAILED, f"the replay exited 0 without a passed {self.options.transport} transport proof"
+            return PASSED, ""
+        if status == INFRASTRUCTURE_FAILURE_EXIT_CODE:
+            return FAILED, "the emulator infrastructure failed again on the retry"
+        if status == TIMEOUT_EXIT_CODE:
+            return FAILED, "the replay script exceeded its time budget"
+        result = read_json(directory / "result.json")
+        if not isinstance(result, dict):
+            return FAILED, f"exit {status} without a result.json"
+        if result.get("passed") is True:
+            # The picture matched, so what failed is a check that runs after it.
+            return FAILED, f"exit {status} after a passed result.json: the transport proof or a log check failed"
+        return FAILED, f"exit {status}: {one_line(result.get('message', ''))}"
+
+    def collect_failure_diagnostics(self, case):
+        directory = self.result_dir(case)
+        directory.mkdir(parents=True, exist_ok=True)
+        logcat = directory / "logcat.txt"
+        if not logcat.is_file() or logcat.stat().st_size == 0:
+            # trace-replay-ci.sh pulls this itself unless it was killed first.
+            _, output = self.adb("logcat", "-d", "-t", "20000", timeout=30)
+            logcat.write_text(output, encoding="utf-8", errors="replace")
+            app_output = f"/data/user/0/{self.options.package}/files/trace-replay/output"
+            for name in ("retrace.log", "mobilegl.client.log", "mobilegl.server.log"):
+                status, output = self.adb("exec-out", "run-as", self.options.package, "cat",
+                                          f"{app_output}/{name}", timeout=30)
+                if status == 0 and output and not output.startswith("cat: "):
+                    (directory / name).write_text(output, encoding="utf-8", errors="replace")
+        if self.emulator_log and os.path.isfile(self.emulator_log):
+            shutil.copyfile(self.emulator_log, directory / "emulator.log")
+        # A vanished emulator looks the same whether the host OOM killer took it or the
+        # renderer faulted. These two say which.
+        _, memory = self.run(["free", "-h"], timeout=30, capture=True)
+        (directory / "host-memory.txt").write_text(memory, encoding="utf-8")
+        _, dmesg = self.run(["sudo", "-n", "dmesg", "-T"], timeout=30, capture=True)
+        (directory / "host-dmesg.txt").write_text("\n".join(dmesg.splitlines()[-300:]) + "\n", encoding="utf-8")
+
+    def run_case(self, index, case):
+        name = case["name"]
+        notes = []
+        seconds = {}
+        case_started = time.monotonic()
+
+        if self.deadline_passed():
+            return self.record_for(case, NOT_RUN, "the shard deadline passed before this case started")
+
+        restarted_for = ""
+        if self.restart_each_case and index > 0:
+            restarted_for = "MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE=1"
+        elif self.unclean_case and self.can_restart():
+            # A crashed, hung or killed replay can leave the guest or the host renderer in a
+            # state no adb query shows, and this case's verdict must not depend on it.
+            restarted_for = f"the case before it, {self.unclean_case}, did not pass"
+        else:
+            restarted_for = self.health_problem()
+        if restarted_for:
+            notes.append(f"emulator restarted before the case: {restarted_for}")
+            if not self.restart_emulator(restarted_for):
+                self.aborted = "the emulator could not be restarted"
+                return self.record_for(case, NOT_RUN, self.aborted, notes=notes)
+
+        notes += self.reset_app_state()
+        seconds["reset"] = round(time.monotonic() - case_started, 1)
+
+        reasons_before = len(self.reason_lines())
+        exit_codes = []
+        replay_seconds = []
+        status, elapsed, cut = self.run_attempt(case)
+        exit_codes.append(status)
+        replay_seconds.append(elapsed)
+        surface_lost = 0
+        if status == INFRASTRUCTURE_FAILURE_EXIT_CODE:
+            say(f"::warning::Android emulator infrastructure failed; restarting it and retrying {name} once.")
+            new_reasons = self.reason_lines()[reasons_before:]
+            # A lost surface is retried, so it is counted where a rate spike shows.
+            surface_lost = sum(1 for line in new_reasons if "angle-surface-lost" in line)
+            if surface_lost:
+                self.append_step_summary(f"surface-lost retries: {surface_lost} ({self.options.backend}, {name})\n")
+            notes.append("infrastructure retry: " + (", ".join(new_reasons) or "no reason recorded"))
+            self.keep_first_attempt(case)
+            if not self.restart_emulator(f"infrastructure failure in {name}"):
+                self.aborted = "the emulator could not be restarted"
+            elif self.deadline_passed():
+                cut = "the shard deadline passed before the infrastructure retry could start"
+            else:
+                notes += self.reset_app_state()
+                status, elapsed, cut = self.run_attempt(case)
+                exit_codes.append(status)
+                replay_seconds.append(elapsed)
+        seconds["replay"] = replay_seconds
+
+        finish_started = time.monotonic()
+        if self.aborted:
+            verdict, reason = FAILED, "the emulator did not come back for the infrastructure retry"
+        elif cut:
+            verdict, reason = NOT_RUN, cut
+        else:
+            verdict, reason = self.verdict(case, status)
+        if verdict != PASSED:
+            self.collect_failure_diagnostics(case)
+        if not self.aborted:
+            self.remove_case_leftovers(case)
+        seconds["finish"] = round(time.monotonic() - finish_started, 1)
+        seconds["total"] = round(time.monotonic() - case_started, 1)
+        return self.record_for(case, verdict, reason, attempts=len(exit_codes), exit_codes=exit_codes,
+                               surface_lost_retries=surface_lost, seconds=seconds, notes=notes)
+
+    # ---- the shard ---------------------------------------------------------------------
+
+    def append_step_summary(self, text):
+        path = self.environ.get("GITHUB_STEP_SUMMARY")
+        if path:
+            with open(path, "a", encoding="utf-8") as summary:
+                summary.write(text)
+
+    def run_all(self):
+        self.write_shard_status()
+        say(f"trace-replay-shard: {len(self.cases)} case(s) on {self.options.backend} / "
+            f"{self.options.transport} ({self.options.lane})")
+        if not self.prepare_booted_device():
+            if not self.restart_emulator("the trace APK did not install"):
+                self.aborted = "the trace APK could not be installed"
+        for index, case in enumerate(self.cases):
+            say(f"::group::{case['name']} ({index + 1}/{len(self.cases)})")
+            if self.aborted:
+                record = self.record_for(case, NOT_RUN, self.aborted)
+            else:
+                try:
+                    record = self.run_case(index, case)
+                except Exception as error:  # one case's bookkeeping must not cost the others their verdicts
+                    record = self.record_for(case, FAILED, f"the shard runner failed: {one_line(error)}")
+            self.unclean_case = "" if record["status"] == PASSED else case["name"]
+            self.save_record(case, record)
+            say(f"trace-replay-shard: {case['name']}: {record['status']}"
+                + (f" ({record['reason']})" if record["reason"] else ""))
+            say("::endgroup::")
+        return self.finish()
+
+    def finish(self):
+        """Tally from the files on disk, so a verdict the loop never wrote counts as missing."""
+        label = f"{self.options.backend} / {self.options.transport}"
+        rows = []
+        verdicts = 0
+        failed = []
+        for case in self.cases:
+            record = read_json(self.result_dir(case) / "case-status.json")
+            if not isinstance(record, dict) or record.get("case") != case["name"]:
+                record = self.record_for(case, NOT_RUN, "no verdict file was written")
+            if record["status"] in (PASSED, FAILED):
+                verdicts += 1
+            if record["status"] == PASSED:
+                result = read_json(self.result_dir(case) / "result.json")
+                if not isinstance(result, dict) or result.get("passed") is not True:
+                    record = dict(record, status=FAILED, reason="the verdict is passed but result.json is not")
+            if record["status"] != PASSED:
+                failed.append(record)
+            result = read_json(self.result_dir(case) / "result.json")
+            ssim = result.get("ssim", "") if isinstance(result, dict) else ""
+            seconds = record.get("seconds", {}).get("total", "")
+            note = one_line("; ".join([record.get("reason", "")] + record.get("notes", [])).strip("; "))
+            rows.append(f"| {case['name']} | {record['status']} | {record.get('attempts', 0)} | {seconds} | "
+                        f"{ssim} | {note.replace('|', '/')} |")
+        self.write_shard_status(complete=verdicts == len(self.cases))
+
+        summary = [f"### Android retrace: {label} ({self.options.lane})", "",
+                   f"{len(self.cases) - len(failed)} of {len(self.cases)} passed, {self.boots} emulator boot(s).", "",
+                   "| case | status | attempts | seconds | SSIM | note |", "| --- | --- | --- | --- | --- | --- |"]
+        self.append_step_summary("\n".join(summary + rows) + "\n\n")
+        for record in failed:
+            say(f"::error title=Android retrace {record['status']} ({label})::"
+                f"{record['case']}: {record.get('reason') or record['status']}")
+        if verdicts != len(self.cases):
+            say(f"::error title=Android retrace shard incomplete ({label})::"
+                f"{verdicts} verdict(s) for {len(self.cases)} case(s)")
+        say(f"trace-replay-shard: {len(self.cases) - len(failed)} passed, {len(failed)} not passed, "
+            f"{verdicts} verdict(s) for {len(self.cases)} case(s)")
+        return 0 if not failed and verdicts == len(self.cases) else 1
+
+    def finalize(self):
+        """Give every case its summary inputs and a verdict file without touching the device.
+
+        Runs after the replay step whatever its outcome, so a shard that was killed or never
+        started still uploads one named row per case.
+        """
+        for case in self.cases:
+            record = read_json(self.result_dir(case) / "case-status.json")
+            if isinstance(record, dict) and record.get("case") == case["name"]:
+                self.records[case["name"]] = record
+                self.write_summary_inputs(case)
+            else:
+                self.save_record(case, self.record_for(case, NOT_RUN, "the shard did not reach this case"))
+        previous = read_json(self.result_root / "shard-status.json")
+        if isinstance(previous, dict):
+            self.boots = previous.get("emulator_boots", self.boots)
+            self.install_seconds = previous.get("install_seconds", self.install_seconds)
+        self.write_shard_status(
+            complete=all(self.records[case["name"]]["status"] in (PASSED, FAILED) for case in self.cases))
+        return 0
+
+    def dry_run(self):
+        for case in self.cases:
+            say(json.dumps({
+                "case": case["name"],
+                "environment": replay_environment(self.options.backend, case["name"]),
+                "arguments": replay_arguments(self.options, case, self.hang_budget_ms),
+                "timeout": int(as_text(case["timeout_seconds"])) + REPLAY_GRACE_SECONDS,
+            }, sort_keys=True))
+        return 0
+
+
+def parse_arguments(argv):
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    parser.add_argument("--apk-file", required=True, help="Trace APK to install once per emulator boot.")
+    parser.add_argument("--package", default="top.mobilegl.plugin.trace")
+    parser.add_argument("--backend", required=True, choices=BACKENDS)
+    parser.add_argument("--transport", required=True, choices=TRANSPORTS)
+    parser.add_argument("--lane", required=True, help="Lane name written to lane.json, e.g. inproc-acceptance.")
+    parser.add_argument("--gpu", default="", help="Emulator -gpu mode, needed to restart the emulator.")
+    source = parser.add_mutually_exclusive_group(required=True)
+    source.add_argument("--cases-json", help="JSON array of the APK matrix's case objects.")
+    source.add_argument("--cases-file", help="File holding that JSON array.")
+    source.add_argument("--cases-env", help="Name of the environment variable holding that JSON array.")
+    parser.add_argument("--result-root", default="android-retrace-result")
+    parser.add_argument("--fixture-root", default="android-retrace-fixture")
+    parser.add_argument("--deadline-seconds", type=int, default=0,
+                        help="Stop starting cases, and cut a running one, this long after the start; both are "
+                             "recorded as not-run. 0: no limit. retrace_shard_budget.py says what a shard needs.")
+    mode = parser.add_mutually_exclusive_group()
+    mode.add_argument("--dry-run", action="store_true",
+                      help="Print each case's environment and trace-replay-ci.sh arguments, run nothing.")
+    mode.add_argument("--finalize", action="store_true",
+                      help="Only write the summary inputs and a not-run verdict for cases without one.")
+    return parser.parse_args(argv)
+
+
+def main(argv=None):
+    options = parse_arguments(argv)
+    if options.cases_json is not None:
+        text = options.cases_json
+    elif options.cases_file is not None:
+        text = Path(options.cases_file).read_text(encoding="utf-8")
+    else:
+        text = os.environ.get(options.cases_env, "")
+    try:
+        cases = load_cases(text, options.backend)
+    except ValueError as error:
+        say(f"::error title=Android retrace shard misconfigured::{error}")
+        return 2
+    shard = Shard(options, cases, os.environ)
+    if shard.hang_budget_ms is None:
+        say("::error title=Android retrace shard misconfigured::MOBILEGL_GPU_HANG_BUDGET_MS is not set")
+        return 2
+    if options.dry_run:
+        return shard.dry_run()
+    if options.finalize:
+        return shard.finalize()
+    if not os.path.isfile(options.apk_file):
+        say(f"::error title=Android retrace shard misconfigured::APK does not exist: {options.apk_file}")
+        return 2
+    # A termination request unwinds through Shard.run, which kills the running child first.
+    signal.signal(signal.SIGTERM, lambda number, frame: sys.exit(128 + number))
+    return shard.run_all()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

docs/ci/ci-cleanup-log.md

@@ -64,6 +64,12 @@ them wrong.
 | 2026-10-18 | `Negative control A (corrupted snapshot)` and `Keep lane logs before controls` | Test / `integration-verify-split` | the first re-ran the ten `VerifySplit.ClearThenReadPixelsScenario` entries of both backends with the same knob and accepted any red; the second only kept the lane's logs from being overwritten by that rerun | the in-lane `VerifySplitCorrupted` and `VerifySplitReadCorrupted` entries of both backends, required by name (`Verify-split control entries`); that the ambient entries arm is the per-entry arm proof, and that they run fatal is `VerifySplitArming.PipeVerifyArmingScenario.Armed` (the ambient environment plus its marker and log), which requires `fatal=1` on the arming line |
 | 2026-10-18 | unit `PipeInputsTest.{PoisonOmitKnobArmsTheOmission, BadPoisonOmitKnobIsFatalNamingTheKnob}`, and the `MOBILEGL_PIPE_POISON_OMIT` knob: `Features.PipePoisonOmit`, its parser on every verb (`MGPipeValidateForVerb`'s `ParsePoisonOmissionKnob`, compiled into every build) and the server stamp's own parse of it | Test / unit (every poison lane); product code | the knob existed only for negative control B, and the two cases pinned its parser and its `Fatal{PipeVerifyBadKnob}` | the PoisonOmitted and VerifySplitPoisonOmitted entries set the harness marker `MGITEST_POISON_OMIT` to the same pair, and the re-executed child arms it through `MGPipeSetPoisonOmission`, compiled only with `-DMOBILEGL_PIPE_VERIFY=ON` (`Harness/PoisonOmissionArm.h`); `OmittingOneFieldForOneVerbLeavesExactlyThatFieldStale`, `ReadingAnOmittedFieldAbortsNamingTheVerb` and `TheMutationNoticeRefreshesTheValueButNotTheStamp` withhold the stamp test-side and still run in every poison lane; the setter's scope to its verb, not its class, is checked by `OmittingOneFieldForOneVerbLeavesExactlyThatFieldStale` in a forked child, on the verify runtime only |
 | 2026-10-18 | the `serial == resets` assertion of `CtWireScenario.ApplierResetCrossesAtThePrimedEdgeInSerialOrder`, and the `MOBILEGL_TEST_SERVER_COUNTERS` knob: `ServerMain`'s `MGPipe server counters:` line on every LogFlush and `ServerVerbSink`'s test-only tallies (`ApplierResets`, `ObjectDeaths`, `ExpectedApplierResetSerial`) | Test / `integration-split`, `integration-split-strict`, `integration-verify-split`, `integration (inproc)`; product code | the knob existed only so the spawn and tcp arms could read the sink's tallies; the serial assertion could not fail, since `OnApplierReset` moves the count and the serial together and any other serial latches `Fatal{ProtocolCorruption, "ApplierReset.ContextSerial"}` | `ar` / `od` on the server role's `MGPipe stats:` line, counted in `ServerVerbSink::OnApplierReset` / `OnObjectDeath`; every CtWire entry that counts them sets `MOBILEGL_PIPE_STATS=1` with period 1 and reads them on every arm, inproc included, over a window closed by a swap and a readback; the unit cases `ApplierResetWireRoundTrip.TheSerialSequenceIsAssertedAndTheServerResetRuns` and `ObjectDeathWireRoundTrip.TheTextureAndFramebufferTwinsReleaseByHandleAndNeverByAStaleGeneration` read the same counters, and the first shows the expected serial moving by having serial 2 accepted |
+| 2026-11-15 | per-case jobs `retrace (<backend>, <case>)`, `retrace verify (<backend>, <case>)`, `retrace split (<backend>, <case>, <transport>)` | Test / `retrace`, `retrace-verify`, `retrace-split` | one runner per case spent most of its time on setup; no case, knob, assertion or arm proof is dropped | shard jobs that replay the same rows one after another through `scripts/ci/retrace_shard.py`, one ctest entry and one verdict per case; `trace_cases.py --self-test-shard-matrices` checks that the shards hold exactly the per-case rows |
+| 2026-11-15 | per-case emulator jobs `retrace (<backend>, <case>)` of each lane | MobileGL APK / `retrace` | one emulator boot per replay; no case, setting or arm proof is dropped | shard jobs that replay the cases of one lane, transport and backend on one boot through `android-plugin/trace_replay_shard.py`, with app data cleared between cases, a cold boot after a case that did not pass, and one verdict per case |
+| 2026-11-15 | per-case `trace fixture (<case>)` jobs, their per-case cache entries and artifacts, `.github/scripts/trace-fixture-cache.sh` | Test and MobileGL APK / `trace-fixtures` | 40 jobs and 40 cache entries per workflow for files that change rarely | one content-addressed cache entry for all Git LFS fixtures (`.github/scripts/trace-fixture-bundle.sh`); a replay job restores it and downloads a case only when the entry does not hold it |
+| 2026-11-15 | `remove-artifact-clutter` job | Test and MobileGL APK | it deleted the fixture artifacts and the green result artifacts after the run; the fixture artifacts no longer exist | the delete step at the end of `retrace-summary`; on Linux `retention-days: 1` on the summary inputs as well, `retrace-split` and `retrace-shipped` upload results only for a failed shard (their green results were deleted unread before), and `retrace-verify` keeps its results for every shard as before |
+| 2026-11-15 | emulator replays of the cases outside `android_smoke` on the monolith control and spawn acceptance lanes (56 of 79 rows per lane; the 12 smoke cases and the whole inproc lane stay) | MobileGL APK / `retrace` | the 237 emulator replays of a push held the 20 runner slots for most of an hour, and the launcher ships inproc; what goes is the same-commit Android monolith control and the Android spawn evidence (server exec from the native library directory, the pbuffer surface) for those 56 rows | Test replays every case on monolith, inproc and spawn on Linux; the Android inproc lane replays every case; `workflow_dispatch` with `android_full_matrix` runs the full emulator matrix; `trace_cases.py --self-test-transport-matrices` checks that the reduced lanes are the full matrix less exactly those rows |
+| 2026-11-15 | the three separate ctest invocations of `Split lanes (inproc, spawn, TCP)` (`-L integration-split`, `-L integration-spawn`, `-L '^integration-tcp$'`) | Test / `integration-split` | every TCP entry holds one resource lock, so the TCP invocation ran one entry at a time while the other cores were idle; the three selections share no entry | one invocation of the union of the three selections: each arm is still discovered with its own label regex, `lane_selection.py union` writes the exact names and fails when an arm selects nothing, and `junit_tally.py --select-from/--write-selected` carves each arm's JUnit out of the run under its old stem and fails when a selected name is missing or reported twice; the per-arm tallies, the entries required by name, the run-ahead proof and the skip-census rows are unchanged |
 
 ## Kept after investigation
 

scripts/ci/checkout_submodules.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+# MobileGL - scripts/ci/checkout_submodules.sh
+# Copyright (c) 2025-2026 MobileGL-Dev
+# Licensed under the GNU Lesser General Public License v3.0:
+#   https://www.gnu.org/licenses/gpl-3.0.txt
+#   https://www.gnu.org/licenses/lgpl-3.0.txt
+# SPDX-License-Identifier: LGPL-3.0-only
+# End of Source File Header
+
+# The submodules a CI build reads, checked out shallow and in parallel, in place of a
+# recursive checkout of every submodule in the tree.
+#   library  what the root CMakeLists.txt configures for the library, its tests and benchmarks.
+#   retrace  the library set plus the trace replay tool's: apitrace and the eight directories
+#            its thirdparty/CMakeLists.txt requires.
+# Each row names a submodule and one file the build reads from it. A row whose file is missing
+# after the update fails the script: the configure would otherwise go on without it, and for
+# 3rdparty/flatbuffers the root CMakeLists.txt then builds a library without MG_Remote instead
+# of stopping. A submodule the build starts to use needs a row here.
+#
+# usage: checkout_submodules.sh <library|retrace>      (from the repository root)
+#        checkout_submodules.sh --list <library|retrace>   prints the rows, checks out nothing
+set -euo pipefail
+
+usage="usage: checkout_submodules.sh [--list] <library|retrace>"
+
+list_only=false
+if [ "${1:-}" = --list ]; then
+  list_only=true
+  shift
+fi
+set_name="${1:?${usage}}"
+
+# <submodule path> <file the build reads from it>
+library=(
+  "3rdparty/glslang CMakeLists.txt"
+  "3rdparty/SPIRV-Cross CMakeLists.txt"
+  "3rdparty/SPIRV-Reflect CMakeLists.txt"
+  "3rdparty/VulkanMemoryAllocator CMakeLists.txt"
+  "3rdparty/Vulkan-Headers CMakeLists.txt"
+  "3rdparty/Vulkan-Utility-Libraries CMakeLists.txt"
+  "3rdparty/xxHash build/cmake/CMakeLists.txt"
+  "3rdparty/asio include/asio.hpp"
+  "3rdparty/flatbuffers include/flatbuffers/flatbuffers.h"
+  "include/ska flat_hash_map.hpp"
+)
+retrace=(
+  "3rdparty/apitrace thirdparty/CMakeLists.txt"
+)
+# Inside 3rdparty/apitrace: the files its thirdparty/CMakeLists.txt tests before it configures.
+apitrace=(
+  "thirdparty/brotli LICENSE"
+  "thirdparty/gtest LICENSE"
+  "thirdparty/libbacktrace LICENSE"
+  "thirdparty/libpng LICENSE"
+  "thirdparty/snappy COPYING"
+  "thirdparty/zlib README"
+  "thirdparty/zstd LICENSE"
+  "thirdparty/directxmath LICENSE"
+)
+
+case "${set_name}" in
+  library) top=("${library[@]}"); nested=() ;;
+  retrace) top=("${library[@]}" "${retrace[@]}"); nested=("${apitrace[@]}") ;;
+  *)
+    echo "checkout_submodules.sh: unknown set '${set_name}' (library or retrace)" >&2
+    echo "${usage}" >&2
+    exit 2
+    ;;
+esac
+
+if [ "${list_only}" = true ]; then
+  printf '%s\n' "${top[@]}"
+  if [ "${#nested[@]}" -gt 0 ]; then
+    printf '3rdparty/apitrace/%s\n' "${nested[@]}"
+  fi
+  exit 0
+fi
+
+# update <directory> <row>...: one shallow, parallel update of the rows' submodules. The update
+# is idempotent, so a fetch that failed on the network is tried again, three times in all.
+update() {
+  local dir="$1"
+  shift
+  local paths=() row attempt
+  for row in "$@"; do
+    paths+=("${row%% *}")
+  done
+  for attempt in 1 2 3; do
+    if git -C "${dir}" submodule update --init --depth=1 --jobs 8 -- "${paths[@]}"; then
+      return 0
+    fi
+    echo "checkout_submodules.sh: the submodule update in ${dir} failed (attempt ${attempt} of 3)" >&2
+    if [ "${attempt}" -lt 3 ]; then
+      sleep $((attempt * 5))
+    fi
+  done
+  return 1
+}
+
+# verify <directory> <row>...: every row's file exists; prints each missing one and returns 1.
+verify() {
+  local dir="$1" missing=0 row path file
+  shift
+  for row in "$@"; do
+    path="${row%% *}"
+    file="${row#* }"
+    if [ ! -f "${dir}/${path}/${file}" ]; then
+      echo "::error::checkout_submodules.sh: ${dir}/${path}/${file} is missing after the submodule update, so the '${set_name}' build cannot use ${path}"
+      missing=1
+    fi
+  done
+  return "${missing}"
+}
+
+update . "${top[@]}"
+verify . "${top[@]}"
+if [ "${#nested[@]}" -gt 0 ]; then
+  update 3rdparty/apitrace "${nested[@]}"
+  verify 3rdparty/apitrace "${nested[@]}"
+fi
+echo "checkout_submodules.sh: the '${set_name}' set is checked out ($(( ${#top[@]} + ${#nested[@]} )) submodules)"

scripts/ci/configure_build.sh

@@ -25,12 +25,17 @@ variant="${1:?usage: configure_build.sh <primary|split> <build dir>}"
 build_dir="${2:?usage: configure_build.sh <primary|split> <build dir>}"
 shift 2
 
+# ENABLE_PCH is glslang's option, on by default. ccache cannot cache a compile that uses a
+# precompiled header, so with it on every glslang translation unit is compiled again in every
+# run. It is in the common part so that both lines stay the same configuration for
+# build_fingerprint.py.
 common=(
   -S . -B "${build_dir}" -G Ninja
   -DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}"
   -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}"
   -DCMAKE_C_COMPILER_LAUNCHER=ccache
   -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+  -DENABLE_PCH=OFF
 )
 
 case "${variant}" in

scripts/ci/lane_selection.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+# MobileGL - scripts/ci/lane_selection.py
+# Copyright (c) 2025-2026 MobileGL-Dev
+# Licensed under the GNU Lesser General Public License v3.0:
+#   https://www.gnu.org/licenses/gpl-3.0.txt
+#   https://www.gnu.org/licenses/lgpl-3.0.txt
+# SPDX-License-Identifier: LGPL-3.0-only
+# End of Source File Header
+
+"""One ctest run for several lanes: the exact names of every lane, as one --tests-from-file list.
+
+A lane is a `ctest -L <label regex>` selection. Running the lanes of one job as one ctest
+invocation lets a lane that holds a resource lock (the TCP arm runs one entry at a time) share
+the run with the others. The lanes keep their own selections: each is discovered with its own
+label regex, this script writes the union of the discovered names, ctest runs that list, and
+junit_tally.py --select-from/--write-selected carves each lane's JUnit back out, which fails
+when a lane's name is missing from the run or reported twice.
+
+  lane_selection.py union OUT.txt LANE=DISCOVERY.json [LANE=DISCOVERY.json ...]
+  lane_selection.py --self-test
+
+DISCOVERY.json is `ctest -L <regex> --show-only=json-v1`. union fails when a lane selects
+nothing (what --no-tests=error says for a lane run on its own), names an entry twice, or holds a
+name a tests-from-file line cannot carry. It prints each lane's count and the size of the union.
+"""
+import json
+from pathlib import Path
+import sys
+import tempfile
+
+
+def lane_names(label, path):
+    """The names one discovery selects, in order; raises ValueError when the lane is unusable."""
+    names = [test['name'] for test in json.loads(Path(path).read_text()).get('tests', [])]
+    if not names:
+        raise ValueError(f'lane {label}: {path} selects no test')
+    if len(names) != len(set(names)):
+        twice = sorted({name for name in names if names.count(name) > 1})
+        raise ValueError(f'lane {label}: {path} names {len(twice)} entr(ies) twice, e.g. {twice[0]}')
+    # ctest reads the list one name per line and trims each line.
+    bad = [name for name in names if name != name.strip() or '\n' in name or '\r' in name]
+    if bad:
+        raise ValueError(f'lane {label}: {len(bad)} name(s) cannot be written one per line, e.g. {bad[0]!r}')
+    return names
+
+
+def union(lanes):
+    """(sorted union of names, {label: count}, number of names more than one lane selects)."""
+    counts, seen, shared = {}, set(), set()
+    for label, path in lanes:
+        if label in counts:
+            raise ValueError(f'lane {label} is given twice')
+        names = lane_names(label, path)
+        counts[label] = len(names)
+        shared |= seen & set(names)
+        seen |= set(names)
+    return sorted(seen), counts, len(shared)
+
+
+def parse_lanes(arguments):
+    lanes = []
+    for argument in arguments:
+        label, separator, path = argument.partition('=')
+        if not separator or not label or not path:
+            raise ValueError(f'expected LANE=DISCOVERY.json, got {argument!r}')
+        lanes.append((label, path))
+    if not lanes:
+        raise ValueError('no lane given')
+    return lanes
+
+
+def run(argv):
+    if len(argv) < 3 or argv[0] != 'union':
+        print(__doc__, file=sys.stderr)
+        return 2
+    try:
+        names, counts, shared = union(parse_lanes(argv[2:]))
+    except (ValueError, OSError, json.JSONDecodeError) as error:
+        print(f'::error::lane_selection.py: {error}')
+        return 1
+    Path(argv[1]).write_text(''.join(name + '\n' for name in names))
+    summary = ', '.join(f'{label} {count}' for label, count in counts.items())
+    print(f'lane_selection.py: {len(names)} entr(ies) in one run ({summary}; {shared} in more than one lane)')
+    return 0
+
+
+def self_test():
+    import contextlib
+    import io
+
+    def quiet(argv):
+        # The negative controls print ::error:: lines on purpose; keep them off a green run.
+        with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
+            return run(argv)
+
+    with tempfile.TemporaryDirectory() as tmp:
+        tmp = Path(tmp)
+
+        def discovery(name, names):
+            path = tmp / name
+            path.write_text(json.dumps({'tests': [{'name': n} for n in names]}))
+            return str(path)
+
+        out = tmp / 'tests.txt'
+        split = discovery('split.json', ['DirectGLES.Split.A', 'DirectGLES.Split.B'])
+        spawn = discovery('spawn.json', ['DirectGLES.Spawn.A', 'SpawnLane.Control'])
+        tcp = discovery('tcp.json', ['TcpServer.Start', 'DirectGLES.Tcp.A', 'TcpServer.Stop'])
+        assert quiet(['union', str(out), f'split={split}', f'spawn={spawn}', f'tcp={tcp}']) == 0
+        assert out.read_text().splitlines() == sorted(
+            ['DirectGLES.Split.A', 'DirectGLES.Split.B', 'DirectGLES.Spawn.A', 'SpawnLane.Control',
+             'TcpServer.Start', 'DirectGLES.Tcp.A', 'TcpServer.Stop']), 'the union is every lane, each name once'
+        # A name two lanes select is written once; each lane still carves its own copy.
+        shared = discovery('shared.json', ['DirectGLES.Split.A', 'Shared.Fixture'])
+        assert quiet(['union', str(out), f'split={split}', f'other={shared}']) == 0
+        assert out.read_text().splitlines() == ['DirectGLES.Split.A', 'DirectGLES.Split.B', 'Shared.Fixture']
+        out.unlink()
+        controls = {
+            'a lane that selects nothing': [f'split={split}', f"spawn={discovery('empty.json', [])}"],
+            'a lane that names an entry twice': [f"split={discovery('twice.json', ['A', 'A'])}"],
+            'a name with a trailing blank': [f"split={discovery('blank.json', ['A '])}"],
+            'a name with a line break': [f"split={discovery('break.json', ['A' + chr(10) + 'B'])}"],
+            'a lane given twice': [f'split={split}', f'split={spawn}'],
+            'a missing discovery file': [f"split={tmp / 'absent.json'}"],
+            'an argument without a lane name': [split],
+        }
+        for label, arguments in controls.items():
+            assert quiet(['union', str(out)] + arguments) == 1, f'{label} must fail'
+            assert not out.exists(), f'{label} must not leave a list behind'
+        assert quiet(['union', str(out)]) == 2, 'no lane is a usage error'
+        assert quiet(['nonsense', str(out), f'split={split}']) == 2, 'an unknown command is a usage error'
+    print('lane_selection self-test: ok')
+    return 0
+
+
+if __name__ == '__main__':
+    if sys.argv[1:] == ['--self-test']:
+        sys.exit(self_test())
+    sys.exit(run(sys.argv[1:]))

scripts/ci/retrace_shard.py

@@ -0,0 +1,689 @@
+#!/usr/bin/env python3
+# MobileGL - scripts/ci/retrace_shard.py
+# Copyright (c) 2025-2026 MobileGL-Dev
+# Licensed under the GNU Lesser General Public License v3.0:
+#   https://www.gnu.org/licenses/gpl-3.0.txt
+#   https://www.gnu.org/licenses/lgpl-3.0.txt
+# SPDX-License-Identifier: LGPL-3.0-only
+# End of Source File Header
+
+"""Replay the cases of one retrace shard, one after another, and give each its own verdict.
+
+A shard is some of the cases of one (lane, transport, backend) cell, replayed by one CI job
+(tools/trace_replay/trace_cases.py, shard_matrix). Each case runs through its own ctest entry
+with the environment and the checks of its lane (case_plan), in a process group and a driver
+cache directory of its own, so no case sees what another left behind. A failing case does not
+stop the shard. The shard fails when a case failed, or when the verdicts it recorded are not
+exactly the cases it was given.
+
+For every case it prints a log group, appends a row to the step summary and records a verdict
+in <result-dir>/results/shard-results.json; a failed case also gets an error annotation.
+The replay output the lane uploads is moved to <result-dir>/results/<case>/, the JUnit files
+are in <result-dir>/junit/, and the extracted trace is deleted.
+
+Usage:
+  retrace_shard.py run --profile {retrace,split,verify} --backend B [--transport T] [--lane L]
+                       --build-dir DIR --result-dir DIR [--names-env VAR | --names JSON]
+                       [--split-reuse] [--server-path PATH] [--shard N/M]
+  retrace_shard.py --self-test
+"""
+import argparse
+import json
+import os
+import re
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+SCRIPTS = Path(__file__).resolve().parent
+ITERATIONRP_CASE = "minecraft-1.21.4-fabric-iris-iterationrp-in-world"
+BLENDED_DEPTH_WRITE_CASE = "improved-transparency-minecraft-26.3"
+WIRE_EVIDENCE_CASE = "OpenRA"
+CONTROL_CASE = "OpenRA"
+CONTROL_BACKEND = "DirectGLES"
+CONTROL_NAME = "OpenRA (corrupted-snapshot control)"
+# The comparator runs the long cases several times slower than a plain replay, and a spawned
+# server can take as long, so the verify and split lanes raise ctest's 1500 s default.
+LONG_TIMEOUT = ("--timeout", "10800")
+SPLIT_REUSE_EXPECT_IPC = "strict=0 role-split-state=1 run-ahead=1"
+SPLIT_REUSE_EXPECT_IPC_REASON = (
+    "retrace-split dropped its inproc rows because the inproc-acceptance retrace rows replay the "
+    "same library with strict=1 role-split-state=1 run-ahead=1, which differs from this leg's "
+    "defaults only in strict while these hold. A default changed: restore the inproc rows (the "
+    "trace-cases job's split-spawn-shards without --transports spawn) or update the expectation.")
+# GitHub shows about ten error annotations of a step. The first failures are annotated as they
+# happen, so a shard killed later still shows them; the rest wait for the roll-up that names
+# every failed case.
+IMMEDIATE_ANNOTATIONS = 8
+
+
+def test_regex(case, backend):
+    return rf"^MobileGLTraceReplay\.{case}\.{backend}$"
+
+
+def test_name(case, backend):
+    return f"MobileGLTraceReplay.{case}.{backend}"
+
+
+def case_plan(profile, transport, backend, case, *, split_reuse=False, server_path=""):
+    """What one case of a lane runs with: the environment it adds, ctest's arguments, its checks.
+
+    retrace  the monolith-control and inproc-acceptance lanes of the retrace job. The job itself
+             exports MOBILEGL_TRANSPORT and the three IPC knobs for every case alike.
+    split    the retrace-split job, inproc and spawn.
+    verify   the retrace-verify job; its library has no transport.
+    """
+    if profile not in ("retrace", "split", "verify"):
+        raise ValueError(f"unknown profile: {profile}")
+    env = {}
+    if backend == "DirectVulkan":
+        env["MOBILEGL_MAGMA_R11G11B10F_FALLBACK"] = "1"
+        # The blended depth-write quirk enables itself only on Qualcomm, which no runner has,
+        # so the case it exists for forces it on; ForceOn bypasses only the vendor gate.
+        if case == BLENDED_DEPTH_WRITE_CASE:
+            env["MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE"] = "1"
+        # The iterationRP repairs default to device detection and the software driver is on no
+        # list, so the one case that needs them forces them. The verify lane does not set it.
+        if case == ITERATIONRP_CASE and profile != "verify":
+            env["MOBILEGL_ITRP_FIX"] = "ForceOn"
+    ctest = ["-V", "--no-tests=error"]
+    evidence = None
+    census = False
+    if profile == "retrace":
+        if transport == "inproc":
+            # These rows stand in for retrace-split's dropped inproc rows, so they prove the
+            # configuration they ran with from the client's own log.
+            env["MGTRACE_EXPECT_IPC"] = "strict=1 role-split-state=1 run-ahead=1"
+            census = True
+        if transport != "monolith":
+            evidence = []
+    elif profile == "split":
+        if transport not in ("inproc", "spawn"):
+            raise ValueError(f"the split profile runs inproc or spawn, not {transport!r}")
+        if not server_path.startswith("/"):
+            # The replay's working directory is not the workspace and the launcher resolves the
+            # image relative to nothing. The inproc arm reads it only to decide it needs none.
+            raise ValueError("the split profile needs an absolute --server-path")
+        env["MOBILEGL_TRANSPORT"] = transport
+        env["MOBILEGL_IPC_SERVER_PATH"] = server_path
+        evidence = []
+        if case == WIRE_EVIDENCE_CASE:
+            # This case also counts the wire's draw records and the server's draws. That turns
+            # on the library's per-frame counters, so the others keep the plain configuration.
+            env["MGTRACE_WIRE_EVIDENCE"] = "1"
+            evidence = ["--require-wire"]
+        if split_reuse:
+            env["MGTRACE_EXPECT_IPC"] = SPLIT_REUSE_EXPECT_IPC
+            env["MGTRACE_EXPECT_IPC_REASON"] = SPLIT_REUSE_EXPECT_IPC_REASON
+        ctest += LONG_TIMEOUT
+        census = True
+    else:
+        env["MOBILEGL_PIPE_VERIFY"] = "1"
+        ctest += LONG_TIMEOUT
+    ctest += ["-R", test_regex(case, backend)]
+    return {"env": env, "ctest": ctest, "evidence": evidence, "census": census,
+            "control": has_control(profile, backend, case)}
+
+
+def control_plan():
+    """The verify lane's negative control: the same replay with one snapshot field corrupted.
+
+    It must fail, and for the comparator's reason. It adds only these two variables and raises
+    no core limit, since the failure is the expected outcome.
+    """
+    return {"env": {"MOBILEGL_PIPE_VERIFY": "1",
+                    "MOBILEGL_PIPE_VERIFY_CORRUPT": "GetRenderStateParameters"},
+            "ctest": ["-V", "--no-tests=error", *LONG_TIMEOUT,
+                      "-R", test_regex(CONTROL_CASE, CONTROL_BACKEND)]}
+
+
+def has_control(profile, backend, case):
+    return profile == "verify" and case == CONTROL_CASE and backend == CONTROL_BACKEND
+
+
+def expected_results(profile, backend, names):
+    """The verdicts a shard owes, in order."""
+    out = []
+    for name in names:
+        out.append(name)
+        if has_control(profile, backend, name):
+            out.append(CONTROL_NAME)
+    return out
+
+
+def step_ignores_sigpipe():
+    """Whether the process that started this script, the step's shell, ignores SIGPIPE."""
+    try:
+        with open(f"/proc/{os.getppid()}/status", encoding="ascii") as status:
+            for line in status:
+                if line.startswith("SigIgn:"):
+                    return bool(int(line.split()[1], 16) & (1 << (signal.SIGPIPE - 1)))
+    except (OSError, ValueError, IndexError):
+        pass
+    return False
+
+
+def raise_core_limit():
+    # What `ulimit -c unlimited` does; the runner's core handler honors each process's limit.
+    import resource
+    resource.setrlimit(resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
+
+
+class Shard:
+    def __init__(self, args, names, base_env):
+        self.args = args
+        self.names = names
+        self.base_env = dict(base_env)
+        self.build_dir = Path(args.build_dir).resolve()
+        self.result_dir = Path(args.result_dir).resolve()
+        self.results_root = self.result_dir / "results"
+        self.junit_dir = self.result_dir / "junit"
+        self.log_dir = self.result_dir / "logs"
+        self.cache_root = self.result_dir / "driver-cache"
+        self.records = []
+        self.failures = 0
+        self.child = None
+        self.lane = args.lane or "-"
+        self.transport = args.transport or "-"
+        self.where = f"{self.lane} / {self.transport} / {args.backend}"
+
+    # --- output -------------------------------------------------------------------------------
+
+    def say(self, text=""):
+        print(text, flush=True)
+
+    def summary(self, text):
+        path = os.environ.get("GITHUB_STEP_SUMMARY")
+        if path:
+            with open(path, "a", encoding="utf-8") as file:
+                file.write(text + "\n")
+
+    def annotate(self, title, message):
+        escaped = message.replace("%", "%25").replace("\r", "").replace("\n", "%0A")
+        # A property value also escapes the two characters that delimit properties.
+        title = title.replace("%", "%25").replace(":", "%3A").replace(",", "%2C")
+        self.say(f"::error title={title}::{escaped}")
+
+    def write_results(self):
+        self.results_root.mkdir(parents=True, exist_ok=True)
+        payload = {"profile": self.args.profile, "lane": self.args.lane, "transport": self.args.transport,
+                   "backend": self.args.backend, "shard": self.args.shard, "cases": self.names,
+                   "results": self.records}
+        target = self.results_root / "shard-results.json"
+        scratch = target.with_suffix(".tmp")
+        scratch.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+        scratch.replace(target)
+
+    def record(self, name, status, seconds, reasons, *, ssim="", cores=()):
+        """One verdict: "passed", "failed", or "skipped" for a control whose case already failed."""
+        entry = {"case": name, "status": status, "seconds": round(seconds, 1), "ssim": ssim,
+                 "reasons": list(reasons), "cores": list(cores), "annotated": False}
+        if status == "failed":
+            self.failures += 1
+            entry["annotated"] = self.failures <= IMMEDIATE_ANNOTATIONS
+        self.records.append(entry)
+        self.write_results()
+        note = "; ".join(reasons)
+        if cores:
+            note = (note + "; " if note else "") + "core: " + ", ".join(cores)
+        shown = "**failed**" if status == "failed" else status
+        self.summary(f"| {name} | {shown} | {seconds:.0f} | {ssim} | {note.replace('|', '/')[:300]} |")
+        if entry["annotated"]:
+            self.annotate(f"retrace failed: {self.where} / {name}", note or "failed")
+
+    # --- one process ----------------------------------------------------------------------------
+
+    def run_ctest(self, plan, *, junit=None, transcript=None, core_limit=True, cache_home=None):
+        env = dict(self.base_env)
+        env.update(plan["env"])
+        if cache_home is not None:
+            env["XDG_CACHE_HOME"] = str(cache_home)
+        command = [self.args.ctest, *plan["ctest"]]
+        if junit is not None:
+            command += ["--output-junit", str(junit)]
+        self.say("+ " + " ".join(f"{key}={value!r}" for key, value in sorted(plan["env"].items())))
+        self.say("+ " + " ".join(command))
+        options = {"cwd": self.build_dir, "env": env, "start_new_session": True}
+        if core_limit:
+            options["preexec_fn"] = raise_core_limit
+        # Python ignores SIGPIPE itself and by default hands a child the default action. The
+        # replay gets what a step's own child would get instead, so a write to a closed pipe
+        # or socket ends the same way as when the step ran ctest directly.
+        options["restore_signals"] = not step_ignores_sigpipe()
+        sys.stdout.flush()
+        if transcript is None:
+            self.child = subprocess.Popen(command, **options)
+            status = self.child.wait()
+        else:
+            # The transcript is both shown and kept, as `2>&1 | tee` would.
+            self.child = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options)
+            with open(transcript, "wb") as file:
+                for chunk in iter(lambda: self.child.stdout.read1(65536), b""):
+                    file.write(chunk)
+                    sys.stdout.buffer.write(chunk)
+                    sys.stdout.buffer.flush()
+            status = self.child.wait()
+        self.reap()
+        return status
+
+    def reap(self, grace=5.0):
+        """End whatever the finished replay left running, so it cannot run under the next case.
+
+        A spawned server exits by itself when its client is gone, so the group gets a moment
+        first. True when something had to be killed.
+        """
+        child, self.child = self.child, None
+        if child is None:
+            return False
+        deadline = time.monotonic() + grace
+        try:
+            while True:
+                os.killpg(child.pid, 0)
+                if time.monotonic() >= deadline:
+                    break
+                time.sleep(0.25)
+            os.killpg(child.pid, signal.SIGKILL)
+        except (ProcessLookupError, PermissionError):
+            return False
+        self.say(f"killed the processes ctest (pid {child.pid}) left running")
+        return True
+
+    # --- one case -------------------------------------------------------------------------------
+
+    def fresh_cache_home(self):
+        """An empty driver cache directory, as on a runner that has replayed nothing yet.
+
+        The software drivers keep compiled shaders on disk under the user cache directory. Each
+        case gets a new one, so a case never starts from shaders another case compiled. On a
+        hosted runner the default location is emptied as well, in case a process ignores the
+        redirect; a self-hosted machine's own cache is left alone.
+        """
+        self.cache_root.mkdir(parents=True, exist_ok=True)
+        cache_home = Path(tempfile.mkdtemp(prefix="case-", dir=self.cache_root))
+        if self.base_env.get("RUNNER_ENVIRONMENT") != "self-hosted":
+            default_home = Path(self.base_env.get("HOME", "/nonexistent")) / ".cache"
+            for stale in default_home.glob("mesa_shader_cache*"):
+                self.say(f"removing {stale} left by an earlier process")
+                shutil.rmtree(stale, ignore_errors=True)
+        return cache_home
+
+    def drop_cache_home(self, cache_home):
+        entries = sum(1 for path in cache_home.rglob("*") if path.is_file())
+        self.say(f"driver cache: {entries} file(s) written under {cache_home}, removed")
+        shutil.rmtree(cache_home, ignore_errors=True)
+
+    def junit_problem(self, junit, name):
+        """Why this JUnit file does not show exactly the one entry that was asked for, or ''."""
+        if not junit.is_file():
+            return f"ctest wrote no {junit.name}"
+        try:
+            entries = [case.get("name") for case in ET.parse(junit).getroot().iter("testcase")]
+        except ET.ParseError as error:
+            return f"{junit.name} does not parse: {error}"
+        if entries != [name]:
+            return f"ctest ran {entries or 'no entry'} where exactly {name} was asked for"
+        return ""
+
+    def check_evidence(self, plan, case):
+        """Run the lane's evidence check; its error lines are folded into the case's annotation."""
+        evidence = self.build_dir / case / self.args.backend / "output" / "split-evidence.txt"
+        command = [sys.executable, str(SCRIPTS / "retrace_split_evidence.py"), str(evidence),
+                   "--transport", self.args.transport, "--case", case, "--backend", self.args.backend,
+                   *plan["evidence"]]
+        done = subprocess.run(command, cwd=self.build_dir, stdout=subprocess.PIPE,
+                              stderr=subprocess.STDOUT, text=True)
+        text = done.stdout.strip()
+        self.say(re.sub(r"^::error::", "split evidence: ", text, flags=re.M))
+        if done.returncode == 0:
+            return ""
+        return "split evidence: " + re.sub(r"^::error::", "", text, flags=re.M).replace("\n", " ")[:600]
+
+    def census(self, case):
+        """The refusal census from the library's own logs, recorded and never a verdict.
+
+        Both roles are read: under spawn the applier's refusals are raised in the server's
+        process. ctest's transcript cannot stand in, since the console sink is compiled out.
+        """
+        output = self.build_dir / case / self.args.backend / "output"
+        log = output / "mobilegl.client.log"
+        server_log = output / "mobilegl.server.log"
+        evidence = output / "split-evidence.txt"
+        self.say(evidence.read_text(errors="replace").rstrip() if evidence.is_file() else "no split-evidence.txt")
+        if not log.is_file():
+            self.say(f"no {log} - the replay wrote no library log")
+            return
+        files = [log] + ([server_log] if server_log.is_file() else [])
+        self.say("census over: " + " ".join(str(path) for path in files))
+        lines = [line for path in files for line in path.read_text(errors="replace").splitlines()]
+        self.say(f"Fatal{{ lines: {sum(1 for line in lines if 'Fatal{' in line)}")
+        counts = {}
+        for line in lines:
+            for match in re.findall(r'Fatal\{[A-Za-z]*, "[^"]*"', line):
+                counts[match] = counts.get(match, 0) + 1
+        for match, count in sorted(counts.items(), key=lambda item: (-item[1], item[0]))[:20]:
+            self.say(f"{count:7d} {match}")
+        transport_line = next((line for line in log.read_text(errors="replace").splitlines()
+                               if "MOBILEGL_TRANSPORT=" in line), None)
+        self.say(transport_line or f"no transport line in {log}")
+
+    def collect(self, case):
+        """Move what the lane uploads into the result directory and delete the extracted trace."""
+        source = self.build_dir / case
+        target = self.results_root / case
+        shutil.rmtree(source / self.args.backend / "input", ignore_errors=True)
+        for relative in ("actual-images", f"{self.args.backend}/output"):
+            if (source / relative).is_dir():
+                (target / relative).parent.mkdir(parents=True, exist_ok=True)
+                shutil.rmtree(target / relative, ignore_errors=True)
+                shutil.move(str(source / relative), str(target / relative))
+        shutil.rmtree(source, ignore_errors=True)
+
+    def ssim_of(self, case):
+        try:
+            result = json.loads((self.build_dir / case / self.args.backend / "output" / "result.json")
+                                .read_text(errors="replace"))
+        except (OSError, ValueError):
+            return ""
+        value = result.get("ssim")
+        return "" if value is None else str(value)
+
+    def run_case(self, case):
+        plan = case_plan(self.args.profile, self.args.transport, self.args.backend, case,
+                         split_reuse=self.args.split_reuse, server_path=self.args.server_path)
+        self.say(f"::group::{case} ({self.where})")
+        started = time.monotonic()
+        cores_before = set(Path("/tmp").glob("core.*"))
+        junit = self.junit_dir / f"{case}.xml"
+        reasons = []
+        cache_home = self.fresh_cache_home()
+        try:
+            try:
+                status = self.run_ctest(plan, junit=junit, cache_home=cache_home)
+            except (OSError, subprocess.SubprocessError) as error:
+                # The case still gets its verdict, and the cases after it still run.
+                self.child = None
+                status = None
+                reasons.append(f"ctest could not be run: {error}")
+            if status is not None and status > 0:
+                reasons.append(f"ctest exit {status}")
+            elif status is not None and status < 0:
+                reasons.append(f"ctest ended by signal {-status}")
+            problem = self.junit_problem(junit, test_name(case, self.args.backend))
+            if problem:
+                reasons.append(problem)
+            if plan["evidence"] is not None:
+                # The transport assertions ran on this case and found their evidence. A failed
+                # replay is already red, and its missing evidence file would only repeat that.
+                if status == 0:
+                    problem = self.check_evidence(plan, case)
+                    if problem:
+                        reasons.append(problem)
+            if plan["census"]:
+                try:
+                    self.census(case)
+                except OSError as error:
+                    self.say(f"refusal census skipped: {error}")
+        finally:
+            self.drop_cache_home(cache_home)
+        ssim = self.ssim_of(case)
+        self.collect(case)
+        cores = sorted(path.name for path in set(Path("/tmp").glob("core.*")) - cores_before)
+        seconds = time.monotonic() - started
+        self.say(f"{case}: {'passed' if not reasons else 'FAILED: ' + '; '.join(reasons)} in {seconds:.0f} s")
+        self.say("::endgroup::")
+        self.record(case, "failed" if reasons else "passed", seconds, reasons, ssim=ssim, cores=cores)
+        return not reasons
+
+    def run_control(self, verified_passed):
+        """With a snapshot field corrupted the same replay must fail, or the lane proves nothing.
+
+        It replays into the case directory the verified run used. That run's output has already
+        been moved to the result directory, and the corrupted run's output is deleted, so only
+        the verified run is uploaded under the case's name.
+        """
+        self.say(f"::group::{CONTROL_NAME}")
+        if not verified_passed:
+            self.say("not run: the verified replay it is compared with failed")
+            self.say("::endgroup::")
+            self.record(CONTROL_NAME, "skipped", 0, ["not run: the verified replay failed"])
+            return
+        started = time.monotonic()
+        reasons = []
+        transcript = self.log_dir / "verify-control.txt"
+        cache_home = self.fresh_cache_home()
+        status = None
+        try:
+            status = self.run_ctest(control_plan(), transcript=transcript, core_limit=False,
+                                    cache_home=cache_home)
+        except (OSError, subprocess.SubprocessError) as error:
+            self.child = None
+            reasons.append(f"ctest could not be run: {error}")
+        finally:
+            self.drop_cache_home(cache_home)
+        shutil.rmtree(self.build_dir / CONTROL_CASE, ignore_errors=True)
+        if status is None:
+            pass
+        elif status == 0:
+            reasons.append(
+                "MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left the OpenRA retrace "
+                "green, so the comparator is not comparing and the verify retrace lane proves nothing")
+        else:
+            # Its red must carry the comparator's divergence on the corrupted field.
+            done = subprocess.run([sys.executable, str(SCRIPTS / "retrace_verify_reason.py"), str(transcript)],
+                                  stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+            text = re.sub(r"^::error::", "", done.stdout.strip(), flags=re.M)
+            self.say(text)
+            if done.returncode != 0:
+                reasons.append(text.replace("\n", " ")[:600])
+            else:
+                self.say(f"the corrupted field turned the retrace red, as it must (ctest exit {status})")
+        seconds = time.monotonic() - started
+        self.say("::endgroup::")
+        self.record(CONTROL_NAME, "failed" if reasons else "passed", seconds, reasons)
+
+    # --- the shard --------------------------------------------------------------------------------
+
+    def run(self):
+        for directory in (self.results_root, self.junit_dir, self.log_dir):
+            directory.mkdir(parents=True, exist_ok=True)
+        expected = expected_results(self.args.profile, self.args.backend, self.names)
+        self.say(f"retrace shard {self.args.shard or ''} ({self.where}): {len(self.names)} case(s), "
+                 f"{len(expected)} verdict(s) owed")
+        self.summary(f"### retrace shard {self.args.shard or ''} ({self.where})\n\n"
+                     "| case | result | seconds | SSIM | note |\n|---|---|---|---|---|")
+        self.write_results()
+        for case in self.names:
+            passed = self.run_case(case)
+            if case_plan(self.args.profile, self.args.transport, self.args.backend, case,
+                         split_reuse=self.args.split_reuse, server_path=self.args.server_path)["control"]:
+                self.run_control(passed)
+        return self.finish(expected)
+
+    def finish(self, expected):
+        problems = verdict_problems(expected, self.records)
+        # Counted again from what ctest wrote, apart from the loop's own bookkeeping.
+        ran = sum(1 for case in self.names
+                  if not self.junit_problem(self.junit_dir / f"{case}.xml", test_name(case, self.args.backend)))
+        passed_cases = sum(1 for entry in self.records
+                           if entry["status"] == "passed" and entry["case"] in self.names)
+        if ran < passed_cases:
+            problems.append(f"{passed_cases} case(s) recorded as passed but only {ran} JUnit file(s) "
+                            f"show their entry ran")
+        failed = [entry for entry in self.records if entry["status"] == "failed"]
+        if failed:
+            names = ", ".join(entry["case"] for entry in failed)
+            self.annotate(f"retrace shard {self.args.shard or ''}: {len(failed)} of {len(expected)} failed",
+                          f"{self.where}: {names}")
+            for entry in failed:
+                if not entry["annotated"]:
+                    self.annotate(f"retrace failed: {self.where} / {entry['case']}",
+                                  "; ".join(entry["reasons"]) or "failed")
+        for problem in problems:
+            self.annotate(f"retrace shard {self.args.shard or ''}: verdicts do not match the cases", problem)
+        passed = sum(1 for entry in self.records if entry["status"] == "passed")
+        closing = f"{passed} passed, {len(failed)} failed, {len(expected)} owed"
+        if problems:
+            closing += "; " + "; ".join(problems)
+        self.summary(f"\n{closing}\n")
+        self.say(f"retrace shard ({self.where}): {closing}")
+        return 1 if failed or problems else 0
+
+
+def verdict_problems(expected, records):
+    """Why the recorded verdicts are not exactly the ones owed; empty when they are."""
+    recorded = [entry["case"] for entry in records]
+    problems = []
+    if len(recorded) != len(expected):
+        problems.append(f"{len(recorded)} verdict(s) recorded for {len(expected)} owed")
+    if recorded != expected:
+        missing = [name for name in expected if name not in recorded]
+        extra = [name for name in recorded if name not in expected]
+        if missing:
+            problems.append("no verdict for " + ", ".join(missing))
+        if extra:
+            problems.append("verdict for a case that was not assigned: " + ", ".join(extra))
+        if not missing and not extra and len(recorded) == len(expected):
+            problems.append("verdicts are out of order or repeated")
+    return problems
+
+
+def run(args):
+    if args.names is not None:
+        names = json.loads(args.names)
+    else:
+        names = json.loads(os.environ.get(args.names_env) or "null")
+    if not isinstance(names, list) or not names or not all(isinstance(name, str) and name for name in names):
+        print(f"::error::retrace shard: no case list (looked in --names and ${args.names_env})")
+        return 2
+    if len(set(names)) != len(names):
+        print(f"::error::retrace shard: the case list repeats a case: {names}")
+        return 2
+    if args.profile == "retrace":
+        # The job exports the transport for this lane; the plan and the library must agree on it.
+        exported = os.environ.get("MOBILEGL_TRANSPORT", "")
+        if exported != args.transport:
+            print(f"::error::retrace shard: --transport is {args.transport!r} but the job exports "
+                  f"MOBILEGL_TRANSPORT={exported!r}")
+            return 2
+    try:
+        case_plan(args.profile, args.transport, args.backend, names[0],
+                  split_reuse=args.split_reuse, server_path=args.server_path)
+    except ValueError as error:
+        print(f"::error::retrace shard: {error}")
+        return 2
+    shard = Shard(args, names, os.environ)
+
+    def stop(signum, _frame):
+        shard.reap(grace=0)
+        shard.say(f"retrace shard: stopped by signal {signum} after {len(shard.records)} verdict(s)")
+        sys.exit(128 + signum)
+
+    signal.signal(signal.SIGTERM, stop)
+    signal.signal(signal.SIGINT, stop)
+    return shard.run()
+
+
+def self_test():
+    """The plan of every lane against the values the lanes are defined by, and the verdict check."""
+    oit, itrp, other = BLENDED_DEPTH_WRITE_CASE, ITERATIONRP_CASE, "minecraft-1.21.4-in-world"
+    server = "/w/build-linux/libMobileGLServer.so"
+    regex = lambda case, backend: ["-R", rf"^MobileGLTraceReplay\.{case}\.{backend}$"]
+    plain, long = ["-V", "--no-tests=error"], ["-V", "--no-tests=error", "--timeout", "10800"]
+    vk = {"MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "1"}
+    expect_strict = {"MGTRACE_EXPECT_IPC": "strict=1 role-split-state=1 run-ahead=1"}
+    reuse = {"MGTRACE_EXPECT_IPC": SPLIT_REUSE_EXPECT_IPC, "MGTRACE_EXPECT_IPC_REASON": SPLIT_REUSE_EXPECT_IPC_REASON}
+    split = lambda transport: {"MOBILEGL_TRANSPORT": transport, "MOBILEGL_IPC_SERVER_PATH": server}
+    table = [
+        # profile, transport, backend, case, reuse -> env, ctest, evidence, census, control
+        ("retrace", "monolith", "DirectGLES", other, False, {}, plain, None, False, False),
+        ("retrace", "monolith", "DirectVulkan", other, False, vk, plain, None, False, False),
+        ("retrace", "monolith", "DirectVulkan", oit, False,
+         {**vk, "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE": "1"}, plain, None, False, False),
+        ("retrace", "monolith", "DirectGLES", oit, False, {}, plain, None, False, False),
+        ("retrace", "monolith", "DirectVulkan", itrp, False, {**vk, "MOBILEGL_ITRP_FIX": "ForceOn"},
+         plain, None, False, False),
+        ("retrace", "inproc", "DirectGLES", other, False, expect_strict, plain, [], True, False),
+        ("retrace", "inproc", "DirectVulkan", itrp, False,
+         {**vk, "MOBILEGL_ITRP_FIX": "ForceOn", **expect_strict}, plain, [], True, False),
+        ("retrace", "inproc", "DirectGLES", "OpenRA", False, expect_strict, plain, [], True, False),
+        ("split", "spawn", "DirectGLES", other, True, {**split("spawn"), **reuse}, long, [], True, False),
+        ("split", "spawn", "DirectGLES", other, False, split("spawn"), long, [], True, False),
+        ("split", "spawn", "DirectVulkan", oit, True,
+         {**vk, "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE": "1", **split("spawn"), **reuse},
+         long, [], True, False),
+        ("split", "spawn", "DirectVulkan", itrp, True,
+         {**vk, "MOBILEGL_ITRP_FIX": "ForceOn", **split("spawn"), **reuse}, long, [], True, False),
+        ("split", "spawn", "DirectGLES", "OpenRA", True,
+         {**split("spawn"), "MGTRACE_WIRE_EVIDENCE": "1", **reuse}, long, ["--require-wire"], True, False),
+        ("split", "inproc", "DirectVulkan", "OpenRA", True,
+         {**vk, **split("inproc"), "MGTRACE_WIRE_EVIDENCE": "1", **reuse}, long, ["--require-wire"], True, False),
+        ("split", "inproc", "DirectGLES", other, False, split("inproc"), long, [], True, False),
+        ("verify", "", "DirectGLES", other, False, {"MOBILEGL_PIPE_VERIFY": "1"}, long, None, False, False),
+        ("verify", "", "DirectGLES", "OpenRA", False, {"MOBILEGL_PIPE_VERIFY": "1"}, long, None, False, True),
+        ("verify", "", "DirectVulkan", "OpenRA", False, {"MOBILEGL_PIPE_VERIFY": "1", **vk}, long, None, False, False),
+        ("verify", "", "DirectVulkan", oit, False,
+         {"MOBILEGL_PIPE_VERIFY": "1", **vk, "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE": "1"},
+         long, None, False, False),
+        ("verify", "", "DirectVulkan", itrp, False, {"MOBILEGL_PIPE_VERIFY": "1", **vk}, long, None, False, False),
+    ]
+    for profile, transport, backend, case, split_reuse, env, ctest, evidence, census, control in table:
+        plan = case_plan(profile, transport, backend, case, split_reuse=split_reuse, server_path=server)
+        wanted = {"env": env, "ctest": ctest + regex(case, backend), "evidence": evidence,
+                  "census": census, "control": control}
+        assert plan == wanted, f"{profile} {transport} {backend} {case}: {plan} != {wanted}"
+    control = control_plan()
+    assert control["env"] == {"MOBILEGL_PIPE_VERIFY": "1", "MOBILEGL_PIPE_VERIFY_CORRUPT": "GetRenderStateParameters"}
+    assert control["ctest"] == long + regex("OpenRA", "DirectGLES")
+    for bad in (("split", "spawn", ""), ("split", "spawn", "relative/server"), ("split", "monolith", server),
+                ("pull", "monolith", server)):
+        try:
+            case_plan(bad[0], bad[1], "DirectGLES", other, server_path=bad[2])
+        except ValueError:
+            continue
+        raise AssertionError(f"case_plan accepted {bad}")
+    assert expected_results("verify", "DirectGLES", ["a", "OpenRA", "b"]) == ["a", "OpenRA", CONTROL_NAME, "b"]
+    assert expected_results("verify", "DirectVulkan", ["OpenRA"]) == ["OpenRA"]
+    assert expected_results("split", "DirectGLES", ["OpenRA"]) == ["OpenRA"]
+    entry = lambda name: {"case": name, "status": "passed"}
+    assert verdict_problems(["a", "b"], [entry("a"), entry("b")]) == []
+    assert verdict_problems(["a", "b"], [entry("a")]), "a case without a verdict passed the check"
+    assert verdict_problems(["a", "b"], []), "a shard that ran nothing passed the check"
+    assert verdict_problems(["a"], [entry("a"), entry("a")]), "a repeated verdict passed the check"
+    assert verdict_problems(["a", "b"], [entry("a"), entry("c")]), "an unassigned case passed the check"
+    assert verdict_problems(["a", "b"], [entry("b"), entry("a")]), "out-of-order verdicts passed the check"
+    print(f"retrace_shard: {len(table)} plans, the control plan and the verdict check OK")
+    return 0
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("command", nargs="?", choices=("run",))
+    parser.add_argument("--self-test", action="store_true")
+    parser.add_argument("--profile", choices=("retrace", "split", "verify"))
+    parser.add_argument("--backend")
+    parser.add_argument("--transport", default="")
+    parser.add_argument("--lane", default="")
+    parser.add_argument("--shard", default="", help="N/M, for the report only")
+    parser.add_argument("--build-dir", help="the directory ctest runs in")
+    parser.add_argument("--result-dir", help="where verdicts, replay output and JUnit files go")
+    parser.add_argument("--names", help="the cases, as a JSON list")
+    parser.add_argument("--names-env", default="SHARD_NAMES", help="the variable holding the JSON list")
+    parser.add_argument("--split-reuse", action="store_true",
+                        help="split profile: the library is the retrace job's, so prove the IPC defaults")
+    parser.add_argument("--server-path", default="", help="split profile: absolute path of the server image")
+    parser.add_argument("--ctest", default="ctest")
+    args = parser.parse_args(argv)
+    if args.self_test:
+        return self_test()
+    if args.command != "run" or not (args.profile and args.backend and args.build_dir and args.result_dir):
+        parser.error("run needs --profile, --backend, --build-dir and --result-dir")
+    return run(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/trace_replay/SpawnServerPathTest.java

@@ -48,7 +48,8 @@ public final class SpawnServerPathTest {
 
     public static void main(String[] args) {
         // ---- the shapes the lanes really pass -------------------------------------------------
-        // .github/workflows/apk.yml's spawn acceptance lane, verbatim.
+        // The spawn acceptance lane of android-plugin/trace_replay_shard.py (transport_arguments()),
+        // verbatim.
         String spawnLane = "MOBILEGL_TRANSPORT=spawn;MOBILEGL_IPC_ROLE_SPLIT_STATE=1;"
                 + "MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1";
         expect("the spawn acceptance lane still gets its server path appended",

tools/trace_replay/render_retrace_summary.mjs

@@ -16,14 +16,15 @@ const BACKENDS = ["DirectGLES", "DirectVulkan"];
 const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
 const DEFAULT_FIXTURE_ROOT = path.join(SCRIPT_DIR, "fixtures");
 const TRACE_CASES = JSON.parse(readFileSync(path.join(SCRIPT_DIR, "trace_cases.json"), "utf8"));
-const CASES = TRACE_CASES.cases.map((traceCase) => traceCase.name);
+// Every case of the manifest, or the ones --cases names when a lane replays a subset.
+let CASES = TRACE_CASES.cases.map((traceCase) => traceCase.name);
 
 function usage() {
   console.error(`Usage:
   node tools/trace_replay/render_retrace_summary.mjs \\
     --input DIR [--input DIR ...] \\
     --output-dir DIR \\
-    [--title TITLE] [--group-label LABEL] [--html NAME]`);
+    [--title TITLE] [--group-label LABEL] [--html NAME] [--cases NAME,NAME,...]`);
 }
 
 function parseArgs(argv) {
@@ -52,6 +53,13 @@ function parseArgs(argv) {
     } else if (arg === "--html" && value) {
       args.htmlName = value;
       i += 1;
+    } else if (arg === "--cases" && value !== undefined) {
+      // An empty list keeps every case, so a caller can pass the option unconditionally.
+      const names = value.split(",").filter((name) => name);
+      const unknown = names.filter((name) => !CASES.includes(name));
+      if (unknown.length) throw new Error(`--cases names unknown trace case(s): ${unknown.join(", ")}`);
+      if (names.length) CASES = CASES.filter((name) => names.includes(name));
+      i += 1;
     } else if (arg === "--help" || arg === "-h") {
       usage();
       process.exit(0);

tools/trace_replay/run_android_retrace_local.py

@@ -650,7 +650,8 @@ def transport_proof_args(args):
     return {"inproc": ["--require-inproc"], "spawn": ["--require-spawn"]}.get(args.transport, [])
 
 
-# The knobs .github/workflows/apk.yml hands BOTH acceptance lanes. They are kept here verbatim so
+# The knobs android-plugin/trace_replay_shard.py (transport_arguments()) hands both acceptance
+# lanes. They are kept here verbatim so
 # a local run is the same run: the arm proof below reads the library's own `Config: IPC` line and
 # requires strict/role-split-state/run-ahead to be 1, so a local invocation that quietly omitted
 # them would fail a gate CI passes - or worse, pass a weaker one.

tools/trace_replay/trace_cases.json

@@ -1,4 +1,18 @@
 {
+  "android_smoke": [
+    "OpenRA",
+    "minecraft-1.21.4-startup",
+    "minecraft-1.21.4-main-menu",
+    "minecraft-1.21.4-fabric-sodium-in-world",
+    "minecraft-1.21.4-fabric-iris-bsl-in-world",
+    "minecraft-1.21.4-fabric-iris-complementary-reimagined-in-world",
+    "minecraft-1.21.4-fabric-iris-bliss-in-world",
+    "minecraft-1.21.4-fabric-iris-sundial-lite-in-world",
+    "minecraft-1.21.1-neoforge-create-indirect-in-world-align1024",
+    "minecraft-1.21.1-neoforge-create-instancing-in-world",
+    "improved-transparency-minecraft-26.3",
+    "minecraft-1.21.4-fabric-iris-iterationrp-in-world"
+  ],
   "defaults": {
     "trace_file": "trace.trace",
     "width": 854,

tools/trace_replay/trace_cases.py

@@ -14,6 +14,9 @@ from pathlib import Path
 
 
 TRACE_CASES_JSON = Path(__file__).with_name("trace_cases.json")
+
+# The top-level keys of the manifest. An unknown one is refused, as an unknown case key is.
+KNOWN_MANIFEST_KEYS = frozenset({"defaults", "cases", "android_smoke"})
 CI_BACKENDS = ("DirectGLES", "DirectVulkan")
 
 # Every key a case or the defaults block may carry. An UNKNOWN key is a hard error rather than a
@@ -127,7 +130,26 @@ def load_trace_case_manifest(path=TRACE_CASES_JSON):
         merged["split"] = split
         validate_backend_overrides(merged)
         cases.append(merged)
-    return {"defaults": defaults, "cases": cases}
+    unknown_top = sorted(set(manifest) - KNOWN_MANIFEST_KEYS)
+    if unknown_top:
+        raise ValueError(f"unknown top-level key(s) in the manifest: {', '.join(unknown_top)}")
+    # "android_smoke" names the cases the emulator replays on the monolith and spawn lanes
+    # when those lanes are reduced. A name that is not a CI case would shrink the subset with
+    # no red, so it is refused here.
+    smoke = manifest.get("android_smoke", [])
+    if not isinstance(smoke, list) or not all(isinstance(name, str) for name in smoke):
+        raise ValueError("android_smoke must be a list of case names")
+    if len(set(smoke)) != len(smoke):
+        raise ValueError("android_smoke repeats a case name")
+    ci_names = {case["name"] for case in cases if case.get("ci", True)}
+    not_ci = [name for name in smoke if name not in ci_names]
+    if not_ci:
+        raise ValueError(f"android_smoke names case(s) that are not CI cases: {', '.join(not_ci)}")
+    return {"defaults": defaults, "cases": cases, "android_smoke": smoke}
+
+
+def android_smoke_names(path=TRACE_CASES_JSON):
+    return load_trace_case_manifest(path)["android_smoke"]
 
 
 def validate_backend_overrides(case):
@@ -400,11 +422,21 @@ def github_apk_matrix(cases):
     }
 
 
-def github_transport_matrix(cases, *, apk=False):
+# The Android lanes of the transport matrix. "full" replays every case on every transport.
+# "inproc-full" replays every case on inproc, the transport the launcher ships, and only the
+# manifest's android_smoke cases on monolith and spawn: the Linux retrace jobs replay every
+# case on all three transports, so those two emulator lanes keep the cases that prove the
+# Android side of each arm.
+ANDROID_LANES = ("full", "inproc-full")
+
+
+def github_transport_matrix(cases, *, apk=False, android_smoke=None):
     """Run the entire original matrix as control and as acceptance, once per transport.
 
     The transports use the same D/P artifact and each case's existing backend
     list. The legacy split:true subset does not limit this main acceptance matrix.
+    With `android_smoke`, an APK matrix keeps every case on inproc and only the named
+    cases on monolith and spawn.
     Nested APK case/backend metadata is deliberately left untouched.
 
     THE SPAWN LANE IS APK-ONLY, and that is a statement about where the evidence is missing
@@ -423,12 +455,19 @@ def github_transport_matrix(cases, *, apk=False):
     build = github_apk_matrix if apk else github_test_matrix
     if not cases:
         raise ValueError("transport matrix needs at least one trace case")
+    reduced = cases
+    if android_smoke is not None:
+        if not apk:
+            raise ValueError("only the APK transport matrix has reduced lanes")
+        reduced = [case for case in cases if case["name"] in android_smoke]
+        if not reduced:
+            raise ValueError("the reduced Android lanes are empty: no case is in android_smoke")
     lanes = [
-        (cases, "monolith-control", "monolith"),
+        (reduced, "monolith-control", "monolith"),
         (cases, "inproc-acceptance", "inproc"),
     ]
     if apk:
-        lanes.append((cases, "spawn-acceptance", "spawn"))
+        lanes.append((reduced, "spawn-acceptance", "spawn"))
     return {"include": [
         {**row, "lane": lane, "transport": transport}
         for selected, lane, transport in lanes
@@ -481,6 +520,29 @@ def self_test_transport_matrices():
         print(f"transport matrix {'APK' if apk else 'Linux'}: {len(controls)} unchanged controls, "
               f"{len(acceptance)} inproc acceptance entries, {len(spawned)} spawn acceptance "
               f"entries; exact metadata and negative controls OK")
+    # The reduced Android lanes are the full matrix less the monolith and spawn rows of the
+    # cases outside android_smoke: nothing else is dropped, changed or added.
+    smoke = android_smoke_names()
+    assert smoke, "android_smoke is empty, so the reduced monolith and spawn lanes would be too"
+    full = github_transport_matrix(cases, apk=True)["include"]
+    reduced = github_transport_matrix(cases, apk=True, android_smoke=smoke)["include"]
+    kept = [row for row in full if row["transport"] == "inproc" or row["case"]["name"] in smoke]
+    assert reduced == kept, "the reduced Android lanes are not the full matrix filtered"
+    for transport in ("monolith", "spawn"):
+        names = {row["case"]["name"] for row in reduced if row["transport"] == transport}
+        assert names == set(smoke), f"the reduced {transport} lane is not the smoke cases: {names}"
+    full_inproc = [row for row in full if row["transport"] == "inproc"]
+    assert [row for row in reduced if row["transport"] == "inproc"] == full_inproc, \
+        "the inproc lane was reduced"
+    for bad in ({"apk": False, "android_smoke": smoke},
+                {"apk": True, "android_smoke": ["no-such-case"]}):
+        try:
+            github_transport_matrix(cases, **bad)
+        except ValueError:
+            continue
+        raise AssertionError(f"github_transport_matrix accepted {bad}")
+    print(f"transport matrix APK reduced: {len(reduced)} of {len(full)} rows; inproc keeps all "
+          f"{len(full_inproc)}, monolith and spawn keep the {len(smoke)} smoke cases")
 
 
 def self_test_split_matrix(cases=None):
@@ -518,6 +580,318 @@ def self_test_split_matrix(cases=None):
           f"kept); all {len(dropped)} dropped inproc rows are inproc-acceptance rows")
 
 
+# --- Shards: several cases of one cell replayed by one CI job ---------------------------------
+#
+# A cell is everything a per-case matrix row carries except its case: the backend, and the lane
+# and transport where the matrix has them. A shard row keeps the cell's keys unchanged and adds
+# the cases it replays, so the job that runs it reads the same fields a per-case leg reads.
+# Every shard matrix is built from the per-case matrix it replaces and is refused unless its
+# rows, taken apart again, are exactly that matrix.
+
+TRACE_DURATION_HINTS_JSON = Path(__file__).with_name("trace_duration_hints.json")
+
+# Shards per cell on Linux, the one tunable of that layout. A cell with little to replay gets
+# fewer: a shard costs about half a minute of setup, so it is not split below
+# MIN_SHARD_SECONDS of replay.
+DEFAULT_SHARDS_PER_CELL = 4
+MIN_SHARD_SECONDS = 90
+
+# Replay seconds one emulator shard aims for, the one tunable of the Android layout. A cell is
+# split into as many shards as that takes.
+DEFAULT_ANDROID_SHARD_SECONDS = 720
+
+# The hint a row without a transport uses. The verify lane has timings of its own.
+HINT_KIND_MONOLITH = "monolith"
+HINT_KIND_VERIFY = "verify"
+
+# A shard's case names reach a shell command line, so the characters are restricted.
+SHARD_SAFE_NAME = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-")
+
+
+def load_duration_hints(platform, path=TRACE_DURATION_HINTS_JSON):
+    """{case: {backend: {kind: seconds}}} for "linux" or "android"; empty when there is none."""
+    path = Path(path)
+    if not path.is_file():
+        return {}
+    with path.open("r", encoding="utf-8") as file:
+        table = json.load(file).get(platform, {}).get("seconds", {})
+    for case, backends in table.items():
+        for backend, kinds in backends.items():
+            for kind, seconds in kinds.items():
+                if isinstance(seconds, bool) or not isinstance(seconds, (int, float)) or seconds <= 0:
+                    raise ValueError(f"duration hint for {case} {backend} {kind} must be a positive "
+                                     f"number: {seconds!r}")
+    return table
+
+
+def median(values):
+    ordered = sorted(values)
+    if not ordered:
+        return None
+    middle = len(ordered) // 2
+    if len(ordered) % 2:
+        return ordered[middle]
+    return (ordered[middle - 1] + ordered[middle]) / 2
+
+
+def hint_seconds(hints, case, backend, kind):
+    """The hinted replay time of one row, or the median of its backend and kind, or of everything."""
+    try:
+        return hints[case][backend][kind]
+    except KeyError:
+        pass
+    same_cell = [kinds[kind] for backends in hints.values()
+                 for name, kinds in backends.items() if name == backend and kind in kinds]
+    everything = [seconds for backends in hints.values() for kinds in backends.values()
+                  for seconds in kinds.values()]
+    return median(same_cell) or median(everything) or 30
+
+
+def row_case_name(row):
+    case = row["case"]
+    return case["name"] if isinstance(case, dict) else case
+
+
+def row_backend_name(row):
+    backend = row["backend"]
+    return backend["name"] if isinstance(backend, dict) else backend
+
+
+def row_cell(row):
+    return {key: value for key, value in row.items() if key != "case"}
+
+
+def canonical(value):
+    return json.dumps(value, sort_keys=True, separators=(",", ":"))
+
+
+def shard_rows_as_cases(shard_rows):
+    """Take shard rows apart into the per-case rows they stand for."""
+    added = ("shard", "shards", "names", "cases", "seconds")
+    rows = []
+    for shard in shard_rows:
+        cell = {key: value for key, value in shard.items() if key not in added}
+        for case in shard.get("cases", shard["names"]):
+            rows.append({**cell, "case": case})
+    return rows
+
+
+def check_shard_union(rows, shard_rows):
+    """Refuse a shard matrix that is not exactly the per-case matrix `rows`, regrouped."""
+    expected = sorted(canonical(row) for row in rows)
+    actual = sorted(canonical(row) for row in shard_rows_as_cases(shard_rows))
+    if expected != actual:
+        missing = sorted(set(expected) - set(actual))
+        extra = sorted(set(actual) - set(expected))
+        raise ValueError(
+            f"the shard matrix is not the per-case matrix regrouped: {len(expected)} per-case rows, "
+            f"{len(actual)} rows in the shards; missing {missing[:3]}, unexpected {extra[:3]}")
+    if len(set(expected)) != len(expected):
+        raise ValueError("the per-case matrix repeats a row, so a shard would replay it twice")
+    cells = {}
+    for shard in shard_rows:
+        if not shard["names"]:
+            raise ValueError(f"empty shard: {shard}")
+        if "cases" in shard and [case["name"] for case in shard["cases"]] != shard["names"]:
+            raise ValueError(f"a shard's names and cases disagree: {shard['names']}")
+        cell = canonical({key: value for key, value in shard.items()
+                          if key not in ("shard", "shards", "names", "cases", "seconds")})
+        cells.setdefault(cell, []).append((shard["shard"], shard["shards"]))
+    for cell, numbers in cells.items():
+        count = len(numbers)
+        if sorted(numbers) != [(index, count) for index in range(1, count + 1)]:
+            raise ValueError(f"shards of {cell} are not numbered 1..{count} of {count}: {numbers}")
+
+
+def shard_matrix(rows, hints, *, default_kind=HINT_KIND_MONOLITH, shards=None, shard_seconds=None):
+    """Regroup a per-case matrix into shards of one cell each.
+
+    Exactly one of `shards` (shards per cell, capped so that none holds less than
+    MIN_SHARD_SECONDS of hinted replay) and `shard_seconds` (hinted replay per shard) sets the
+    layout. Cases go longest first into the shard with the least hinted time so far, which is
+    deterministic and keeps the shards of a cell even; inside a shard they run in matrix order.
+    """
+    if (shards is None) == (shard_seconds is None):
+        raise ValueError("give exactly one of shards and shard_seconds")
+    if shards is not None and shards < 1:
+        raise ValueError(f"shards per cell must be at least 1: {shards}")
+    if shard_seconds is not None and shard_seconds < 1:
+        raise ValueError(f"seconds per shard must be at least 1: {shard_seconds}")
+    cells = {}
+    for index, row in enumerate(rows):
+        name = row_case_name(row)
+        if not name or not set(name) <= SHARD_SAFE_NAME:
+            raise ValueError(f"case name {name!r} cannot be passed on a command line; allowed "
+                             f"characters are letters, digits, '.', '_' and '-'")
+        seconds = hint_seconds(hints, name, row_backend_name(row),
+                               row.get("transport", default_kind))
+        cells.setdefault(canonical(row_cell(row)), []).append((index, row, seconds))
+    out = []
+    for members in cells.values():
+        total = sum(seconds for _, _, seconds in members)
+        if shards is not None:
+            count = min(shards, int(total // MIN_SHARD_SECONDS))
+        else:
+            count = -(-int(total) // shard_seconds)
+        count = max(1, min(count, len(members)))
+        bins = [[] for _ in range(count)]
+        loads = [0] * count
+        for member in sorted(members, key=lambda item: (-item[2], item[0])):
+            target = loads.index(min(loads))
+            bins[target].append(member)
+            loads[target] += member[2]
+        for number, members_of_shard in enumerate(bins, start=1):
+            members_of_shard.sort(key=lambda item: item[0])
+            shard = {**row_cell(members_of_shard[0][1]), "shard": number, "shards": count,
+                     "names": [row_case_name(row) for _, row, _ in members_of_shard]}
+            if isinstance(members_of_shard[0][1]["case"], dict):
+                shard["cases"] = [row["case"] for _, row, _ in members_of_shard]
+            shard["seconds"] = round(sum(seconds for _, _, seconds in members_of_shard))
+            out.append(shard)
+    check_shard_union(rows, out)
+    return {"include": out}
+
+
+def per_case_matrix(format_name, cases, *, transports=SPLIT_TRANSPORTS, keep_inproc=(),
+                    android_smoke=None):
+    """The per-case matrix a shard format regroups, and the hint kind of its transport-less rows."""
+    if format_name == "github-test-shard-matrix":
+        return github_test_matrix(cases), HINT_KIND_MONOLITH
+    if format_name == "github-test-transport-shard-matrix":
+        return github_transport_matrix(cases), HINT_KIND_MONOLITH
+    if format_name == "github-verify-shard-matrix":
+        return github_verify_matrix(cases), HINT_KIND_VERIFY
+    if format_name == "github-split-shard-matrix":
+        return github_split_matrix(cases, transports, keep_inproc), HINT_KIND_MONOLITH
+    if format_name == "github-apk-shard-matrix":
+        return github_apk_matrix(cases), HINT_KIND_MONOLITH
+    if format_name == "github-apk-transport-shard-matrix":
+        return (github_transport_matrix(cases, apk=True, android_smoke=android_smoke),
+                HINT_KIND_MONOLITH)
+    raise ValueError(f"not a shard format: {format_name}")
+
+
+LINUX_SHARD_FORMATS = (
+    "github-test-shard-matrix",
+    "github-test-transport-shard-matrix",
+    "github-verify-shard-matrix",
+    "github-split-shard-matrix",
+)
+ANDROID_SHARD_FORMATS = (
+    "github-apk-shard-matrix",
+    "github-apk-transport-shard-matrix",
+)
+
+
+def github_shard_matrix(format_name, cases, *, shards=DEFAULT_SHARDS_PER_CELL,
+                        shard_seconds=DEFAULT_ANDROID_SHARD_SECONDS,
+                        transports=SPLIT_TRANSPORTS, keep_inproc=(), hints_path=TRACE_DURATION_HINTS_JSON,
+                        android_smoke=None):
+    """A shard format: the Linux ones lay out by shards per cell, the Android ones by seconds."""
+    matrix, default_kind = per_case_matrix(format_name, cases, transports=transports,
+                                           keep_inproc=keep_inproc, android_smoke=android_smoke)
+    if format_name in ANDROID_SHARD_FORMATS:
+        return shard_matrix(matrix["include"], load_duration_hints("android", hints_path),
+                            default_kind=default_kind, shard_seconds=shard_seconds)
+    return shard_matrix(matrix["include"], load_duration_hints("linux", hints_path),
+                        default_kind=default_kind, shards=shards)
+
+
+def self_test_shard_matrices():
+    """Every shard format replays exactly the rows of the per-case matrix it replaces."""
+    cases = ci_trace_cases(load_trace_cases())
+    narrowed = {"transports": ("spawn",), "keep_inproc": ("OpenRA",)}
+    layouts = [(name, {}, {"shards": count}) for name in LINUX_SHARD_FORMATS for count in (1, 4, 100)]
+    layouts += [("github-split-shard-matrix", narrowed, {"shards": count}) for count in (1, 4, 100)]
+    layouts += [(name, {}, {"shard_seconds": seconds}) for name in ANDROID_SHARD_FORMATS
+                for seconds in (60, DEFAULT_ANDROID_SHARD_SECONDS, 10 ** 9)]
+    layouts += [("github-apk-transport-shard-matrix", {"android_smoke": android_smoke_names()},
+                 {"shard_seconds": seconds}) for seconds in (60, DEFAULT_ANDROID_SHARD_SECONDS, 10 ** 9)]
+    for name, selection, layout in layouts:
+        matrix, _ = per_case_matrix(name, cases, **selection)
+        rows = matrix["include"]
+        built = github_shard_matrix(name, cases, **selection, **layout)["include"]
+        again = github_shard_matrix(name, cases, **selection, **layout)["include"]
+        assert built == again, f"{name} {layout}: the layout is not deterministic"
+        # Compared here as well as inside shard_matrix, with nothing shared but the JSON form.
+        replayed = sorted(
+            canonical({**{key: value for key, value in shard.items()
+                          if key not in ("shard", "shards", "names", "cases", "seconds")},
+                       "case": case})
+            for shard in built for case in shard.get("cases", shard["names"]))
+        assert replayed == sorted(canonical(row) for row in rows), \
+            f"{name} {layout}: the shards do not replay the per-case matrix"
+        assert sum(len(shard["names"]) for shard in built) == len(rows)
+        cells = {canonical(row_cell(row)) for row in rows}
+        assert len({canonical({key: value for key, value in shard.items()
+                               if key not in ("shard", "shards", "names", "cases", "seconds")})
+                    for shard in built}) == len(cells), f"{name} {layout}: a shard spans two cells"
+        if layout.get("shards") == 1 or layout.get("shard_seconds") == 10 ** 9:
+            assert len(built) == len(cells), f"{name} {layout}: expected one shard per cell"
+        seconds = [shard["seconds"] for shard in built]
+        shown = {key: (f"{len(value)} smoke cases" if key == "android_smoke" else value)
+                 for key, value in selection.items()}
+        print(f"shard matrix {name} {shown or ''} {layout}: {len(rows)} rows in {len(built)} "
+              f"shards over {len(cells)} cells, {min(seconds)}-{max(seconds)} s hinted per shard")
+        # The check itself must be able to fail: a dropped, a repeated and a moved case.
+        if len(built) >= 2 and any(len(shard["names"]) >= 2 for shard in built):
+            index = next(i for i, shard in enumerate(built) if len(shard["names"]) >= 2)
+            for label, damage in (("dropped", lambda names: names[1:]),
+                                  ("repeated", lambda names: names + names[:1])):
+                broken = [dict(shard) for shard in built]
+                broken[index]["names"] = damage(list(broken[index]["names"]))
+                if "cases" in broken[index]:
+                    broken[index]["cases"] = damage(list(broken[index]["cases"]))
+                try:
+                    check_shard_union(rows, broken)
+                except ValueError:
+                    pass
+                else:
+                    raise AssertionError(f"{name} {layout}: a {label} case passed the union check")
+            other = next((i for i, shard in enumerate(built)
+                          if row_backend_name(shard) != row_backend_name(built[index])), None)
+            if other is not None:
+                broken = [dict(shard) for shard in built]
+                for key in ("names", "cases"):
+                    if key in broken[index]:
+                        moved = list(broken[index][key])
+                        broken[other][key] = list(broken[other][key]) + moved[:1]
+                        broken[index][key] = moved[1:]
+                try:
+                    check_shard_union(rows, broken)
+                except ValueError:
+                    pass
+                else:
+                    raise AssertionError(f"{name} {layout}: a case moved to another backend's "
+                                         f"shard passed the union check")
+    # Hints only weigh the packing. With none at all, and with a case no hint names, the rows
+    # replayed are the same.
+    rows = github_transport_matrix(cases)["include"]
+    for hints in ({}, {"no-such-case": {"DirectGLES": {"monolith": 5}}}):
+        check_shard_union(rows, shard_matrix(rows, hints, shards=4)["include"])
+    assert hint_seconds({}, "x", "DirectGLES", "monolith") == 30
+    assert hint_seconds({"a": {"DirectGLES": {"monolith": 10}}, "b": {"DirectGLES": {"monolith": 20}},
+                         "c": {"DirectVulkan": {"monolith": 90}}}, "x", "DirectGLES", "monolith") == 15
+    for bad in ({"shards": 0}, {"shard_seconds": 0}, {}, {"shards": 4, "shard_seconds": 60}):
+        try:
+            shard_matrix(rows, {}, **bad)
+        except ValueError:
+            continue
+        raise AssertionError(f"shard_matrix accepted {bad}")
+    try:
+        shard_matrix([{"backend": "DirectGLES", "case": "bad name; rm"}], {}, shards=1)
+    except ValueError:
+        pass
+    else:
+        raise AssertionError("a case name with shell characters was accepted")
+    # The verify lane's corrupted-snapshot control replays OpenRA on DirectGLES, so that row has
+    # to be in the verify matrix for the control to have a shard to run in.
+    verify_rows = github_verify_matrix(cases)["include"]
+    assert {"backend": "DirectGLES", "case": "OpenRA"} in verify_rows, \
+        "the verify matrix has no OpenRA DirectGLES row, so its negative control would not run"
+    print("shard matrices: union, numbering, determinism and negative controls OK")
+
+
 def cmake_quote(value):
     return '"' + str(value).replace("\\", "/").replace('"', '\\"') + '"'
 
@@ -570,15 +944,27 @@ def parse_args():
     parser.add_argument("--self-test-transport-matrices", action="store_true")
     parser.add_argument("--self-test-split-matrix", action="store_true",
                         help="prove the spawn-only split matrix drops only inproc-acceptance rows")
+    parser.add_argument("--self-test-shard-matrices", action="store_true",
+                        help="prove every shard matrix replays exactly its per-case matrix")
     parser.add_argument("--fixture-root", default="tools/trace_replay/fixtures")
     parser.add_argument("--transports", default=",".join(SPLIT_TRANSPORTS),
                         help="github-split-matrix: the comma-separated split transports to emit")
     parser.add_argument("--keep-inproc", default="",
                         help="github-split-matrix: comma-separated split cases whose inproc rows stay")
+    parser.add_argument("--shards", type=int, default=DEFAULT_SHARDS_PER_CELL,
+                        help="Linux shard formats: shards per (lane, transport, backend) cell")
+    parser.add_argument("--shard-seconds", type=int, default=DEFAULT_ANDROID_SHARD_SECONDS,
+                        help="Android shard formats: hinted replay seconds one shard aims for")
+    parser.add_argument("--android-lanes", choices=ANDROID_LANES, default="full",
+                        help="APK transport formats: inproc-full keeps every case on inproc and "
+                             "the manifest's android_smoke cases on monolith and spawn")
+    parser.add_argument("--duration-hints", default=str(TRACE_DURATION_HINTS_JSON),
+                        help="shard formats: the JSON of replay seconds the packing weighs cases by")
     parser.add_argument(
         "--format",
         choices=(
             "names",
+            "android-smoke-names",
             "github-test-matrix",
             "github-test-transport-matrix",
             "github-verify-matrix",
@@ -586,7 +972,10 @@ def parse_args():
             "github-apk",
             "github-apk-matrix",
             "github-apk-transport-matrix",
+            *LINUX_SHARD_FORMATS,
+            *ANDROID_SHARD_FORMATS,
             "fixture-files",
+            "fixture-manifest",
             "cmake",
         ),
         default="names",
@@ -602,11 +991,17 @@ def main():
     if args.self_test_split_matrix:
         self_test_split_matrix()
         return 0
+    if args.self_test_shard_matrices:
+        self_test_shard_matrices()
+        return 0
     cases = load_trace_cases()
     if args.ci:
         cases = ci_trace_cases(cases)
+    android_smoke = android_smoke_names() if args.android_lanes == "inproc-full" else None
     if args.format == "names":
         print(json.dumps([case["name"] for case in cases], separators=(",", ":")))
+    elif args.format == "android-smoke-names":
+        print(",".join(android_smoke_names()))
     elif args.format == "github-test-matrix":
         print(json.dumps(github_test_matrix(cases), separators=(",", ":")))
     elif args.format == "github-test-transport-matrix":
@@ -622,7 +1017,22 @@ def main():
     elif args.format == "github-apk-matrix":
         print(json.dumps(github_apk_matrix(cases), separators=(",", ":")))
     elif args.format == "github-apk-transport-matrix":
-        print(json.dumps(github_transport_matrix(cases, apk=True), separators=(",", ":")))
+        print(json.dumps(github_transport_matrix(cases, apk=True, android_smoke=android_smoke),
+                         separators=(",", ":")))
+    elif args.format in LINUX_SHARD_FORMATS + ANDROID_SHARD_FORMATS:
+        transports = tuple(t for t in args.transports.split(",") if t)
+        keep = tuple(c for c in args.keep_inproc.split(",") if c)
+        print(json.dumps(
+            github_shard_matrix(args.format, cases, shards=args.shards,
+                                shard_seconds=args.shard_seconds, transports=transports,
+                                keep_inproc=keep, hints_path=args.duration_hints,
+                                android_smoke=android_smoke),
+            separators=(",", ":")))
+    elif args.format == "fixture-manifest":
+        # One "<case><TAB><file>" line per fixture file, for the fixture bundle script.
+        for case in cases:
+            for file in fixture_files(case, args.fixture_root):
+                print(f"{case['name']}\t{file}")
     elif args.format == "fixture-files":
         if not args.case_name:
             print("--case is required for --format fixture-files", file=sys.stderr)

tools/trace_replay/trace_duration_hints.json

@@ -0,0 +1,99 @@
+{
+  "description": [
+    "Replay seconds per case, backend and transport, used only to pack retrace shards evenly",
+    "(tools/trace_replay/trace_cases.py, shard_matrix). A missing entry falls back to the median of",
+    "its backend and transport, so a new case needs no entry here. Nothing asserts these values;",
+    "a stale number only makes shards uneven. To refresh, copy the seconds column of the shard",
+    "step summaries of a green run."
+  ],
+  "linux": {
+    "source": "Test run 43666299896 (2026-11-15), step 'Retrace and validate' of each one-case job, two gaps filled from run 43700779712; the verify key is the retrace-verify job",
+    "seconds": {
+      "OpenRA": {"DirectGLES": {"inproc": 3, "monolith": 3, "spawn": 3, "verify": 3}, "DirectVulkan": {"inproc": 3, "monolith": 3, "spawn": 2, "verify": 4}},
+      "improved-transparency-minecraft-26.3": {"DirectGLES": {"inproc": 86, "monolith": 51, "spawn": 78, "verify": 81}, "DirectVulkan": {"inproc": 82, "monolith": 104, "spawn": 107, "verify": 85}},
+      "minecraft-1.17-main-menu-854": {"DirectGLES": {"inproc": 4, "monolith": 6, "spawn": 3, "verify": 3}, "DirectVulkan": {"inproc": 5, "monolith": 5, "spawn": 3, "verify": 3}},
+      "minecraft-1.20.1-fabric-iris-iterationt-in-world": {"DirectGLES": {"inproc": 59, "monolith": 70, "spawn": 72}, "DirectVulkan": {"inproc": 62, "monolith": 69, "spawn": 68}},
+      "minecraft-1.21.1-neoforge-create-indirect-in-world-align1024": {"DirectGLES": {"inproc": 9, "monolith": 18, "spawn": 16}, "DirectVulkan": {"inproc": 13, "monolith": 13, "spawn": 8}},
+      "minecraft-1.21.1-neoforge-create-instancing-in-world": {"DirectGLES": {"inproc": 12, "monolith": 14, "spawn": 14}, "DirectVulkan": {"inproc": 10, "monolith": 14, "spawn": 10}},
+      "minecraft-1.21.11-main-menu": {"DirectGLES": {"inproc": 5, "monolith": 5, "spawn": 3, "verify": 6}, "DirectVulkan": {"inproc": 7, "monolith": 6, "spawn": 6, "verify": 5}},
+      "minecraft-1.21.4-fabric-common-mods-in-world": {"DirectGLES": {"inproc": 13, "monolith": 15, "spawn": 15}, "DirectVulkan": {"inproc": 19, "monolith": 17, "spawn": 14}},
+      "minecraft-1.21.4-fabric-common-mods-inventory": {"DirectGLES": {"inproc": 20, "monolith": 13, "spawn": 11}, "DirectVulkan": {"inproc": 22, "monolith": 21, "spawn": 22}},
+      "minecraft-1.21.4-fabric-iris-bliss-in-world": {"DirectGLES": {"inproc": 11, "monolith": 23, "spawn": 16}, "DirectVulkan": {"inproc": 17, "monolith": 17, "spawn": 16}},
+      "minecraft-1.21.4-fabric-iris-bsl-esc-menu-854": {"DirectGLES": {"inproc": 126, "monolith": 127, "spawn": 124}, "DirectVulkan": {"inproc": 130, "monolith": 131, "spawn": 134}},
+      "minecraft-1.21.4-fabric-iris-bsl-in-world": {"DirectGLES": {"inproc": 15, "monolith": 18, "spawn": 17}, "DirectVulkan": {"inproc": 16, "monolith": 15, "spawn": 17}},
+      "minecraft-1.21.4-fabric-iris-chocapic-v6-lite-in-world": {"DirectGLES": {"inproc": 10, "monolith": 15, "spawn": 14}, "DirectVulkan": {"inproc": 14, "monolith": 13, "spawn": 15}},
+      "minecraft-1.21.4-fabric-iris-complementary-reimagined-in-world": {"DirectGLES": {"inproc": 27, "monolith": 35, "spawn": 29}, "DirectVulkan": {"inproc": 41, "monolith": 31, "spawn": 28}},
+      "minecraft-1.21.4-fabric-iris-complementary-unbound-in-world": {"DirectGLES": {"inproc": 41, "monolith": 33, "spawn": 38}, "DirectVulkan": {"inproc": 26, "monolith": 31, "spawn": 25}},
+      "minecraft-1.21.4-fabric-iris-derivative-main-d24.4.14-in-world": {"DirectGLES": {"inproc": 23, "monolith": 33, "spawn": 24}, "DirectVulkan": {"inproc": 20, "monolith": 26, "spawn": 27}},
+      "minecraft-1.21.4-fabric-iris-iterationrp-in-world": {"DirectVulkan": {"inproc": 42, "monolith": 42, "spawn": 41}},
+      "minecraft-1.21.4-fabric-iris-iterationt-in-world": {"DirectGLES": {"inproc": 26, "monolith": 27, "spawn": 23}, "DirectVulkan": {"inproc": 23, "monolith": 20, "spawn": 19}},
+      "minecraft-1.21.4-fabric-iris-iterationt-nodsa-in-world": {"DirectGLES": {"inproc": 24, "monolith": 19, "spawn": 19}, "DirectVulkan": {"inproc": 19, "monolith": 13, "spawn": 21}},
+      "minecraft-1.21.4-fabric-iris-makeup-ultrafast-in-world": {"DirectGLES": {"inproc": 13, "monolith": 9, "spawn": 10}, "DirectVulkan": {"inproc": 11, "monolith": 8, "spawn": 8}},
+      "minecraft-1.21.4-fabric-iris-mellow-in-world": {"DirectGLES": {"inproc": 7, "monolith": 11, "spawn": 9}, "DirectVulkan": {"inproc": 9, "monolith": 9, "spawn": 7}},
+      "minecraft-1.21.4-fabric-iris-nostalgia-in-world": {"DirectGLES": {"inproc": 25, "monolith": 21, "spawn": 27}, "DirectVulkan": {"inproc": 24, "monolith": 26, "spawn": 16}},
+      "minecraft-1.21.4-fabric-iris-photon-v1.1-in-world": {"DirectGLES": {"inproc": 25, "monolith": 34, "spawn": 21}, "DirectVulkan": {"inproc": 24, "monolith": 24, "spawn": 33}},
+      "minecraft-1.21.4-fabric-iris-photon-v1.3b-in-world": {"DirectGLES": {"inproc": 29, "monolith": 36, "spawn": 29}, "DirectVulkan": {"inproc": 30, "monolith": 27, "spawn": 36}},
+      "minecraft-1.21.4-fabric-iris-sundial-lite-in-world": {"DirectGLES": {"inproc": 35, "monolith": 27, "spawn": 21}, "DirectVulkan": {"inproc": 31, "monolith": 20, "spawn": 30}},
+      "minecraft-1.21.4-fabric-iris-super-duper-vanilla-in-world": {"DirectGLES": {"monolith": 22, "spawn": 21}, "DirectVulkan": {"inproc": 14, "monolith": 20, "spawn": 14}},
+      "minecraft-1.21.4-fabric-journeymap-in-world": {"DirectGLES": {"inproc": 43, "monolith": 66, "spawn": 65}, "DirectVulkan": {"inproc": 41, "monolith": 60, "spawn": 90}},
+      "minecraft-1.21.4-fabric-journeymap-in-world-normal-world": {"DirectGLES": {"inproc": 10, "monolith": 12, "spawn": 14}, "DirectVulkan": {"inproc": 15, "monolith": 12, "spawn": 14}},
+      "minecraft-1.21.4-fabric-modernui-inventory": {"DirectGLES": {"inproc": 54, "monolith": 50, "spawn": 67}, "DirectVulkan": {"inproc": 50, "monolith": 66, "spawn": 70}},
+      "minecraft-1.21.4-fabric-modernui-inventory-normal-world": {"DirectGLES": {"inproc": 15, "monolith": 17, "spawn": 9}, "DirectVulkan": {"inproc": 18, "monolith": 13, "spawn": 10}},
+      "minecraft-1.21.4-fabric-rei-inventory": {"DirectGLES": {"inproc": 72, "monolith": 63, "spawn": 90}, "DirectVulkan": {"inproc": 79, "monolith": 84, "spawn": 49}},
+      "minecraft-1.21.4-fabric-rei-inventory-normal-world": {"DirectGLES": {"inproc": 15, "monolith": 22, "spawn": 17}, "DirectVulkan": {"inproc": 20, "monolith": 18, "spawn": 19}},
+      "minecraft-1.21.4-fabric-sodium-in-world": {"DirectGLES": {"inproc": 19, "monolith": 15, "spawn": 26, "verify": 25}, "DirectVulkan": {"inproc": 29, "monolith": 32, "spawn": 23, "verify": 16}},
+      "minecraft-1.21.4-fabric-xaero-minimap-in-world": {"DirectGLES": {"inproc": 41, "monolith": 51, "spawn": 64}, "DirectVulkan": {"inproc": 55, "monolith": 69, "spawn": 77}},
+      "minecraft-1.21.4-fabric-xaero-minimap-in-world-normal-world": {"DirectGLES": {"inproc": 12, "monolith": 13, "spawn": 8}, "DirectVulkan": {"inproc": 9, "monolith": 10, "spawn": 13}},
+      "minecraft-1.21.4-fabric-xaero-world-map-in-world": {"DirectGLES": {"inproc": 41, "monolith": 40, "spawn": 52}, "DirectVulkan": {"inproc": 42, "monolith": 43, "spawn": 59}},
+      "minecraft-1.21.4-fabric-xaero-world-map-in-world-normal-world": {"DirectGLES": {"inproc": 6, "monolith": 11, "spawn": 8}, "DirectVulkan": {"inproc": 13, "monolith": 9, "spawn": 10}},
+      "minecraft-1.21.4-in-world": {"DirectGLES": {"inproc": 17, "monolith": 10, "spawn": 11, "verify": 11}, "DirectVulkan": {"inproc": 10, "monolith": 9, "spawn": 12, "verify": 11}},
+      "minecraft-1.21.4-main-menu": {"DirectGLES": {"inproc": 7, "monolith": 4, "spawn": 9, "verify": 6}, "DirectVulkan": {"inproc": 5, "monolith": 10, "spawn": 6, "verify": 6}},
+      "minecraft-1.21.4-startup": {"DirectGLES": {"inproc": 5, "monolith": 3, "spawn": 1, "verify": 3}, "DirectVulkan": {"inproc": 1, "monolith": 4, "spawn": 2, "verify": 2}}
+    }
+  },
+  "android": {
+    "source": "MobileGL APK run 43666299861 (2026-11-15), step 'Retrace and validate' of each one-case emulator job, APK install included",
+    "seconds": {
+      "OpenRA": {"DirectGLES": {"inproc": 28, "monolith": 28, "spawn": 24}, "DirectVulkan": {"inproc": 24, "monolith": 22, "spawn": 16}},
+      "improved-transparency-minecraft-26.3": {"DirectGLES": {"inproc": 179, "monolith": 170, "spawn": 142}, "DirectVulkan": {"inproc": 169, "monolith": 209, "spawn": 153}},
+      "minecraft-1.17-main-menu-854": {"DirectGLES": {"inproc": 27, "monolith": 19, "spawn": 22}, "DirectVulkan": {"inproc": 25, "monolith": 25, "spawn": 26}},
+      "minecraft-1.20.1-fabric-iris-iterationt-in-world": {"DirectGLES": {"inproc": 131, "monolith": 147, "spawn": 153}, "DirectVulkan": {"inproc": 149, "monolith": 111, "spawn": 116}},
+      "minecraft-1.21.1-neoforge-create-indirect-in-world-align1024": {"DirectGLES": {"inproc": 74, "monolith": 85, "spawn": 73}, "DirectVulkan": {"inproc": 44, "monolith": 31, "spawn": 53}},
+      "minecraft-1.21.1-neoforge-create-instancing-in-world": {"DirectGLES": {"inproc": 74, "monolith": 56, "spawn": 78}, "DirectVulkan": {"inproc": 42, "monolith": 49, "spawn": 37}},
+      "minecraft-1.21.11-main-menu": {"DirectGLES": {"inproc": 38, "monolith": 37, "spawn": 30}, "DirectVulkan": {"inproc": 46, "monolith": 25, "spawn": 45}},
+      "minecraft-1.21.4-fabric-common-mods-in-world": {"DirectGLES": {"inproc": 66, "monolith": 51, "spawn": 61}, "DirectVulkan": {"inproc": 70, "monolith": 62, "spawn": 53}},
+      "minecraft-1.21.4-fabric-common-mods-inventory": {"DirectGLES": {"inproc": 74, "monolith": 86, "spawn": 41}, "DirectVulkan": {"inproc": 61, "monolith": 55, "spawn": 67}},
+      "minecraft-1.21.4-fabric-iris-bliss-in-world": {"DirectGLES": {"inproc": 74, "monolith": 70, "spawn": 53}, "DirectVulkan": {"inproc": 53, "monolith": 69, "spawn": 50}},
+      "minecraft-1.21.4-fabric-iris-bsl-esc-menu-854": {"DirectGLES": {"inproc": 220, "monolith": 171, "spawn": 176}, "DirectVulkan": {"inproc": 138, "monolith": 234, "spawn": 193}},
+      "minecraft-1.21.4-fabric-iris-bsl-in-world": {"DirectGLES": {"inproc": 60, "monolith": 42, "spawn": 48}, "DirectVulkan": {"inproc": 56, "monolith": 48, "spawn": 60}},
+      "minecraft-1.21.4-fabric-iris-chocapic-v6-lite-in-world": {"DirectGLES": {"inproc": 62, "monolith": 54, "spawn": 50}, "DirectVulkan": {"inproc": 51, "monolith": 42, "spawn": 36}},
+      "minecraft-1.21.4-fabric-iris-complementary-reimagined-in-world": {"DirectGLES": {"inproc": 93, "monolith": 94, "spawn": 110}, "DirectVulkan": {"inproc": 92, "monolith": 112, "spawn": 88}},
+      "minecraft-1.21.4-fabric-iris-complementary-unbound-in-world": {"DirectGLES": {"inproc": 99, "monolith": 97, "spawn": 99}, "DirectVulkan": {"inproc": 98, "monolith": 107, "spawn": 115}},
+      "minecraft-1.21.4-fabric-iris-derivative-main-d24.4.14-in-world": {"DirectGLES": {"inproc": 67, "monolith": 81, "spawn": 60}, "DirectVulkan": {"inproc": 77, "monolith": 76, "spawn": 47}},
+      "minecraft-1.21.4-fabric-iris-iterationrp-in-world": {"DirectVulkan": {"inproc": 102, "monolith": 108, "spawn": 94}},
+      "minecraft-1.21.4-fabric-iris-iterationt-in-world": {"DirectGLES": {"inproc": 40, "monolith": 57, "spawn": 48}, "DirectVulkan": {"inproc": 66, "monolith": 56, "spawn": 61}},
+      "minecraft-1.21.4-fabric-iris-iterationt-nodsa-in-world": {"DirectGLES": {"inproc": 68, "monolith": 65, "spawn": 39}, "DirectVulkan": {"inproc": 43, "monolith": 70, "spawn": 66}},
+      "minecraft-1.21.4-fabric-iris-makeup-ultrafast-in-world": {"DirectGLES": {"inproc": 32, "monolith": 47, "spawn": 46}, "DirectVulkan": {"inproc": 39, "monolith": 42, "spawn": 43}},
+      "minecraft-1.21.4-fabric-iris-mellow-in-world": {"DirectGLES": {"inproc": 47, "monolith": 37, "spawn": 42}, "DirectVulkan": {"inproc": 46, "monolith": 40, "spawn": 40}},
+      "minecraft-1.21.4-fabric-iris-nostalgia-in-world": {"DirectGLES": {"inproc": 92, "monolith": 51, "spawn": 87}, "DirectVulkan": {"inproc": 60, "monolith": 55, "spawn": 83}},
+      "minecraft-1.21.4-fabric-iris-photon-v1.1-in-world": {"DirectGLES": {"inproc": 107, "monolith": 105, "spawn": 79}, "DirectVulkan": {"inproc": 59, "monolith": 100, "spawn": 96}},
+      "minecraft-1.21.4-fabric-iris-photon-v1.3b-in-world": {"DirectGLES": {"inproc": 127, "monolith": 114, "spawn": 60}, "DirectVulkan": {"inproc": 94, "monolith": 106, "spawn": 81}},
+      "minecraft-1.21.4-fabric-iris-sundial-lite-in-world": {"DirectGLES": {"inproc": 80, "monolith": 77, "spawn": 77}, "DirectVulkan": {"inproc": 76, "monolith": 77, "spawn": 91}},
+      "minecraft-1.21.4-fabric-iris-super-duper-vanilla-in-world": {"DirectGLES": {"inproc": 71, "monolith": 63, "spawn": 48}, "DirectVulkan": {"inproc": 53, "monolith": 67, "spawn": 49}},
+      "minecraft-1.21.4-fabric-journeymap-in-world": {"DirectGLES": {"inproc": 139, "monolith": 241, "spawn": 172}, "DirectVulkan": {"inproc": 180, "monolith": 257}},
+      "minecraft-1.21.4-fabric-journeymap-in-world-normal-world": {"DirectGLES": {"inproc": 56, "monolith": 37, "spawn": 67}, "DirectVulkan": {"inproc": 51, "monolith": 53, "spawn": 54}},
+      "minecraft-1.21.4-fabric-modernui-inventory": {"DirectGLES": {"inproc": 207, "monolith": 248, "spawn": 105}, "DirectVulkan": {"inproc": 303, "monolith": 324, "spawn": 195}},
+      "minecraft-1.21.4-fabric-modernui-inventory-normal-world": {"DirectGLES": {"inproc": 56, "monolith": 58, "spawn": 53}, "DirectVulkan": {"inproc": 60, "monolith": 58, "spawn": 81}},
+      "minecraft-1.21.4-fabric-rei-inventory": {"DirectGLES": {"inproc": 328, "monolith": 337, "spawn": 144}, "DirectVulkan": {"inproc": 325, "monolith": 197, "spawn": 199}},
+      "minecraft-1.21.4-fabric-rei-inventory-normal-world": {"DirectGLES": {"inproc": 79, "monolith": 62, "spawn": 46}, "DirectVulkan": {"inproc": 82, "monolith": 51, "spawn": 66}},
+      "minecraft-1.21.4-fabric-sodium-in-world": {"DirectGLES": {"inproc": 178, "monolith": 175, "spawn": 91}, "DirectVulkan": {"inproc": 166, "monolith": 136, "spawn": 72}},
+      "minecraft-1.21.4-fabric-xaero-minimap-in-world": {"DirectGLES": {"inproc": 231, "monolith": 254, "spawn": 99}, "DirectVulkan": {"inproc": 278, "monolith": 225, "spawn": 127}},
+      "minecraft-1.21.4-fabric-xaero-minimap-in-world-normal-world": {"DirectGLES": {"inproc": 61, "monolith": 47, "spawn": 46}, "DirectVulkan": {"inproc": 61, "monolith": 44, "spawn": 42}},
+      "minecraft-1.21.4-fabric-xaero-world-map-in-world": {"DirectGLES": {"inproc": 313, "monolith": 223, "spawn": 104}, "DirectVulkan": {"inproc": 276, "monolith": 277, "spawn": 121}},
+      "minecraft-1.21.4-fabric-xaero-world-map-in-world-normal-world": {"DirectGLES": {"inproc": 62, "monolith": 47, "spawn": 41}, "DirectVulkan": {"inproc": 52, "monolith": 40, "spawn": 69}},
+      "minecraft-1.21.4-in-world": {"DirectGLES": {"inproc": 44, "monolith": 59, "spawn": 43}, "DirectVulkan": {"inproc": 53, "monolith": 59, "spawn": 51}},
+      "minecraft-1.21.4-main-menu": {"DirectGLES": {"inproc": 54, "monolith": 48, "spawn": 35}, "DirectVulkan": {"inproc": 58, "monolith": 67, "spawn": 46}},
+      "minecraft-1.21.4-startup": {"DirectGLES": {"inproc": 22, "monolith": 19, "spawn": 19}, "DirectVulkan": {"inproc": 18, "monolith": 14, "spawn": 14}}
+    }
+  }
+}