mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-11-16 06:23:23 +09:00
Compare commits
20
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b48145ced5 | ||
|
|
25c984927d | ||
|
|
8165793e0d | ||
|
|
697bcc6d45 | ||
|
|
222f6209a0 | ||
|
|
b2248b240f | ||
|
|
8db74d69ae | ||
|
|
04c8773d77 | ||
|
|
8497eee197 | ||
|
|
13e9d0df5c | ||
|
|
4256a66d09 | ||
|
|
cefb70d1a7 | ||
|
|
cdf43a4a59 | ||
|
|
e8dc4e2320 | ||
|
|
6182704c03 | ||
|
|
d3ec9b8179 | ||
|
|
05dfff9100 | ||
|
|
6c59b0391c | ||
|
|
d6fd101d2a | ||
|
|
259acb2c57 |
@@ -0,0 +1,29 @@
|
||||
name: Set swap space
|
||||
description: Replace the runner's swap with a swap file of the given size
|
||||
inputs:
|
||||
swap-size-gb:
|
||||
description: Size of the swap file in GiB
|
||||
required: true
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
# Hosted images differ: some ship /swapfile, some ship no swap at all. Turn off whatever is
|
||||
# active, then create the file at the path the image used, or /swapfile when there was none.
|
||||
- shell: bash
|
||||
env:
|
||||
SWAP_SIZE_GB: ${{ inputs.swap-size-gb }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
active="$(swapon --show=NAME --noheadings)"
|
||||
swap_file="$(printf '%s\n' "$active" | grep '^/' | tail -n 1 || true)"
|
||||
swap_file="${swap_file:-/swapfile}"
|
||||
if [ -n "$active" ]; then
|
||||
printf '%s\n' "$active" | xargs -r -n1 sudo swapoff
|
||||
fi
|
||||
sudo rm -f "$swap_file"
|
||||
sudo fallocate -l "${SWAP_SIZE_GB}G" "$swap_file"
|
||||
sudo chmod 600 "$swap_file"
|
||||
sudo mkswap "$swap_file"
|
||||
sudo swapon "$swap_file"
|
||||
free -h
|
||||
swapon --show
|
||||
@@ -0,0 +1,304 @@
|
||||
#!/usr/bin/env bash
|
||||
# One cache entry that holds every Git LFS trace fixture of the CI cases.
|
||||
#
|
||||
# key derive the actions/cache key, restore prefix and path of the bundle
|
||||
# ensure after a cache miss: complete the bundle directory, fetching only the
|
||||
# cases whose files are missing from it or do not match their pointers
|
||||
# wait <key> [secs] wait until the cache entry <key> is listed for this ref (needs GH_TOKEN)
|
||||
# install <case>... move the fixtures of these cases from the restored bundle directory into
|
||||
# the fixture directory, fetch the cases it does not hold, and check every
|
||||
# file against its pointer
|
||||
#
|
||||
# The key is content-addressed on the pointer oids tracked at HEAD, which a plain checkout
|
||||
# reads without smudging. The bundle directory is the same path for every revision, so
|
||||
# `ensure` can start from an older bundle. Fixtures stored in Git itself rather than in Git
|
||||
# LFS (OpenRA) are not in the bundle; `install` checks that they are real files.
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=trace-fixture-lib.sh
|
||||
. "${script_dir}/trace-fixture-lib.sh"
|
||||
|
||||
# Bump when the key derivation or the bundle layout changes in a way that must invalidate
|
||||
# old entries; the content digest alone would not notice a format change.
|
||||
key_schema="v1"
|
||||
bundle_dir="trace-fixture-bundle"
|
||||
fixture_dir="tools/trace_replay/fixtures"
|
||||
python_bin="${PYTHON:-python3}"
|
||||
|
||||
if ! command -v "${python_bin}" >/dev/null 2>&1 && command -v python >/dev/null 2>&1; then
|
||||
python_bin=python
|
||||
fi
|
||||
|
||||
usage() {
|
||||
echo "usage: $0 <key|ensure|wait <key> [seconds]|install <trace-case>...>" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
# Writes "name=value" to $GITHUB_OUTPUT when running under Actions, and to stdout as well so
|
||||
# the script stays runnable off-CI.
|
||||
emit_output() {
|
||||
local name="$1"
|
||||
local value="$2"
|
||||
if [ -n "${GITHUB_OUTPUT:-}" ]; then
|
||||
printf '%s=%s\n' "${name}" "${value}" >> "${GITHUB_OUTPUT}"
|
||||
fi
|
||||
printf '%s=%s\n' "${name}" "${value}"
|
||||
}
|
||||
|
||||
# "<case><TAB><file>" for every fixture file of every CI case.
|
||||
ci_fixture_manifest() {
|
||||
"${python_bin}" tools/trace_replay/trace_cases.py --ci --format fixture-manifest \
|
||||
--fixture-root "${fixture_dir}" | tr -d '\r'
|
||||
}
|
||||
|
||||
is_lfs_fixture() {
|
||||
get_lfs_metadata "$1" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# A fixture that Git stores directly must be a real file, not a pointer left by a wrong
|
||||
# attribute.
|
||||
check_plain_fixture() {
|
||||
local file="$1"
|
||||
if [ ! -s "${file}" ]; then
|
||||
echo "fixture file is missing or empty: ${file}" >&2
|
||||
return 1
|
||||
fi
|
||||
if head -n 1 "${file}" | grep -q "version https://git-lfs.github.com/spec/v1"; then
|
||||
echo "fixture should not be stored as an LFS pointer: ${file}" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Puts the tracked pointer files back. The smudge filter is skipped so that a clone with Git
|
||||
# LFS set up does not download the objects here.
|
||||
restore_pointers() {
|
||||
GIT_LFS_SKIP_SMUDGE=1 git checkout -- "$@"
|
||||
}
|
||||
|
||||
verify_lfs_fixture() {
|
||||
local on_disk="$1"
|
||||
local tracked="$2"
|
||||
local metadata expected_oid expected_size
|
||||
metadata="$(get_lfs_metadata "${tracked}")" || return 1
|
||||
read -r expected_oid expected_size <<< "${metadata}"
|
||||
verify_fixture_file "${on_disk}" "${tracked}" "${expected_oid}" "${expected_size}"
|
||||
}
|
||||
|
||||
# Moves the given Git LFS fixtures of one case from the bundle directory into the fixture
|
||||
# directory. Succeeds when every one of them is then in place and matches its pointer.
|
||||
take_from_bundle() {
|
||||
local file bundled
|
||||
local incomplete=0
|
||||
for file in "$@"; do
|
||||
bundled="${bundle_dir}/$(basename "${file}")"
|
||||
# Two cases may share a file; the second finds it already in place.
|
||||
if [ -f "${bundled}" ]; then
|
||||
mv -f "${bundled}" "${file}"
|
||||
fi
|
||||
if ! verify_lfs_fixture "${file}" "${file}" 2>/dev/null; then
|
||||
incomplete=1
|
||||
fi
|
||||
done
|
||||
return "${incomplete}"
|
||||
}
|
||||
|
||||
# Downloads the given Git LFS fixtures of one case over their pointers: the mirrors first,
|
||||
# Git LFS for what no mirror serves. The fetch script verifies what it downloaded.
|
||||
fetch_case_fixtures() {
|
||||
local case_name="$1"
|
||||
shift
|
||||
local file
|
||||
for file in "$@"; do
|
||||
rm -f "${file}" "${file}.tmp"
|
||||
done
|
||||
restore_pointers "$@" || return 1
|
||||
bash "${script_dir}/fetch-trace-fixture-lfs.sh" "${case_name}" "${fixture_dir}"
|
||||
}
|
||||
|
||||
[ "$#" -ge 1 ] || usage
|
||||
command_name="$1"
|
||||
shift
|
||||
|
||||
case "${command_name}" in
|
||||
key)
|
||||
[ "$#" -eq 0 ] || usage
|
||||
# One "<basename> <oid> <size>" line per bundled file, sorted, each file once.
|
||||
manifest=""
|
||||
count=0
|
||||
while IFS= read -r file; do
|
||||
[ -n "${file}" ] || continue
|
||||
if ! metadata="$(get_lfs_metadata "${file}" 2>/dev/null)"; then
|
||||
continue
|
||||
fi
|
||||
read -r expected_oid expected_size <<< "${metadata}"
|
||||
manifest+="$(basename "${file}") ${expected_oid} ${expected_size}"$'\n'
|
||||
count=$((count + 1))
|
||||
done < <(ci_fixture_manifest | cut -f2 | LC_ALL=C sort -u)
|
||||
if [ "${count}" -eq 0 ]; then
|
||||
echo "no Git LFS fixture found for the CI trace cases; the bundle would be empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
duplicates="$(printf '%s' "${manifest}" | cut -d' ' -f1 | LC_ALL=C sort | uniq -d)"
|
||||
if [ -n "${duplicates}" ]; then
|
||||
echo "two fixture files share a name, so the bundle directory cannot hold both: ${duplicates}" >&2
|
||||
exit 1
|
||||
fi
|
||||
digest="$(printf '%s' "${manifest}" | sha256sum | awk '{ print substr($1, 1, 16) }')"
|
||||
emit_output "key" "trace-fixture-bundle-${key_schema}-${digest}"
|
||||
emit_output "restore-prefix" "trace-fixture-bundle-${key_schema}-"
|
||||
emit_output "path" "${bundle_dir}"
|
||||
emit_output "files" "${count}"
|
||||
;;
|
||||
|
||||
ensure)
|
||||
[ "$#" -eq 0 ] || usage
|
||||
mkdir -p "${bundle_dir}" "${fixture_dir}"
|
||||
mapfile -t manifest_lines < <(ci_fixture_manifest)
|
||||
declare -A case_seen=()
|
||||
case_order=()
|
||||
for line in "${manifest_lines[@]}"; do
|
||||
case_name="${line%%$'\t'*}"
|
||||
if [ -z "${case_seen[${case_name}]:-}" ]; then
|
||||
case_seen["${case_name}"]=1
|
||||
case_order+=("${case_name}")
|
||||
fi
|
||||
done
|
||||
|
||||
fetched=0
|
||||
reused=0
|
||||
bundled_files=()
|
||||
for case_name in "${case_order[@]}"; do
|
||||
lfs_files=()
|
||||
for line in "${manifest_lines[@]}"; do
|
||||
[ "${line%%$'\t'*}" = "${case_name}" ] || continue
|
||||
file="${line#*$'\t'}"
|
||||
if is_lfs_fixture "${file}"; then
|
||||
lfs_files+=("${file}")
|
||||
else
|
||||
check_plain_fixture "${file}"
|
||||
fi
|
||||
done
|
||||
[ "${#lfs_files[@]}" -gt 0 ] || continue
|
||||
|
||||
# Take what an older bundle already holds, and fetch the case if that is not all of it.
|
||||
if take_from_bundle "${lfs_files[@]}"; then
|
||||
reused=$((reused + 1))
|
||||
else
|
||||
echo "Fetching trace fixtures of ${case_name}: the bundle does not hold them"
|
||||
fetch_case_fixtures "${case_name}" "${lfs_files[@]}"
|
||||
fetched=$((fetched + 1))
|
||||
fi
|
||||
bundled_files+=("${lfs_files[@]}")
|
||||
done
|
||||
|
||||
# The bundle is rebuilt from the verified files, so nothing an older bundle carried and
|
||||
# no case names any more is saved again.
|
||||
rm -rf "${bundle_dir}"
|
||||
mkdir -p "${bundle_dir}"
|
||||
declare -A file_seen=()
|
||||
saved=0
|
||||
for file in "${bundled_files[@]}"; do
|
||||
[ -z "${file_seen[${file}]:-}" ] || continue
|
||||
file_seen["${file}"]=1
|
||||
verify_lfs_fixture "${file}" "${file}"
|
||||
mv -f "${file}" "${bundle_dir}/$(basename "${file}")"
|
||||
saved=$((saved + 1))
|
||||
done
|
||||
if [ "${saved}" -eq 0 ]; then
|
||||
echo "the trace fixture bundle came out empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Leave the working tree as a checkout has it: pointers in the fixture directory.
|
||||
restore_pointers "${!file_seen[@]}"
|
||||
echo "Trace fixture bundle: ${saved} file(s), ${reused} case(s) from the restored bundle, ${fetched} case(s) fetched."
|
||||
emit_output "fetched" "${fetched}"
|
||||
;;
|
||||
|
||||
wait)
|
||||
[ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage
|
||||
key="$1"
|
||||
limit="${2:-300}"
|
||||
# Another workflow of the same push may be the one saving this key. The replay jobs
|
||||
# restore the entry, so it has to be listed before this job ends. The caller checks the
|
||||
# entry itself afterwards; this only waits, and never fails.
|
||||
#
|
||||
# The listing is narrowed to the ref of this run. The key is content-addressed, so another
|
||||
# branch may already hold an entry under it, and that entry cannot be restored from here:
|
||||
# counting it would end the wait while the entry of this ref is still being uploaded.
|
||||
query=(-f "key=${key}" -f per_page=100)
|
||||
if [ -n "${GITHUB_REF:-}" ]; then
|
||||
query+=(-f "ref=${GITHUB_REF}")
|
||||
fi
|
||||
waited=0
|
||||
refused=0
|
||||
while :; do
|
||||
if listed="$(gh api -X GET "repos/${GITHUB_REPOSITORY}/actions/caches" "${query[@]}" \
|
||||
--jq '[.actions_caches[] | select(.key == "'"${key}"'")] | length' 2>/dev/null)"; then
|
||||
if [ "${listed:-0}" -ge 1 ]; then
|
||||
echo "cache entry ${key} is listed after ${waited} s"
|
||||
break
|
||||
fi
|
||||
else
|
||||
refused=$((refused + 1))
|
||||
if [ "${refused}" -ge 3 ]; then
|
||||
echo "the cache listing cannot be read (no token, or no actions: read permission); not waiting"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
if [ "${waited}" -ge "${limit}" ]; then
|
||||
echo "cache entry ${key} is not listed after ${waited} s"
|
||||
break
|
||||
fi
|
||||
sleep 10
|
||||
waited=$((waited + 10))
|
||||
done
|
||||
;;
|
||||
|
||||
install)
|
||||
[ "$#" -ge 1 ] || usage
|
||||
installed=0
|
||||
fetched=0
|
||||
for case_name in "$@"; do
|
||||
mapfile -t files < <(trace_fixture_files "${case_name}" "${fixture_dir}" "${python_bin}")
|
||||
if [ "${#files[@]}" -eq 0 ]; then
|
||||
echo "no fixture files declared for trace case: ${case_name}" >&2
|
||||
exit 1
|
||||
fi
|
||||
lfs_files=()
|
||||
for file in "${files[@]}"; do
|
||||
if is_lfs_fixture "${file}"; then
|
||||
lfs_files+=("${file}")
|
||||
else
|
||||
check_plain_fixture "${file}"
|
||||
fi
|
||||
done
|
||||
[ "${#lfs_files[@]}" -gt 0 ] || continue
|
||||
|
||||
# A bundle that was not restored (the cache entry was evicted, or its download failed)
|
||||
# or that lacks a file costs this job a download of the case, not the replay.
|
||||
if ! take_from_bundle "${lfs_files[@]}"; then
|
||||
echo "::warning::the restored fixture bundle does not hold the trace fixtures of ${case_name}; fetching them"
|
||||
if ! fetch_case_fixtures "${case_name}" "${lfs_files[@]}"; then
|
||||
echo "::error::the trace fixtures of ${case_name} are not in the restored bundle and could not be fetched"
|
||||
exit 1
|
||||
fi
|
||||
fetched=$((fetched + 1))
|
||||
for file in "${lfs_files[@]}"; do
|
||||
if ! verify_lfs_fixture "${file}" "${file}"; then
|
||||
echo "::error::trace fixture ${file} of ${case_name} does not match its Git LFS pointer after the fetch"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
installed=$((installed + ${#lfs_files[@]}))
|
||||
done
|
||||
# The rest of the bundle belongs to other shards.
|
||||
rm -rf "${bundle_dir}"
|
||||
echo "Installed and verified ${installed} Git LFS fixture file(s) for $# trace case(s), ${fetched} case(s) fetched."
|
||||
;;
|
||||
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
@@ -1,117 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Cache-side helper for trace fixtures.
|
||||
#
|
||||
# key <case> [fixture-dir] derive the actions/cache key and path list
|
||||
# verify <case> [fixture-dir] check restored fixtures against their pointers
|
||||
# reset <case> [fixture-dir] drop restored fixtures, leaving the pointers
|
||||
#
|
||||
# The cache key is content-addressed on the Git LFS pointer oids tracked at
|
||||
# HEAD, which are readable from a plain checkout without smudging. Fixture
|
||||
# content therefore maps 1:1 onto a key: unchanged content hits, changed
|
||||
# content is a new key and thus a miss, and the download path handles it. The
|
||||
# key deliberately carries no restore-keys prefix in the workflow - a fixture
|
||||
# that does not match the pointer exactly must never be restored.
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=trace-fixture-lib.sh
|
||||
. "${script_dir}/trace-fixture-lib.sh"
|
||||
|
||||
# Bump when the key derivation changes in a way that must invalidate old
|
||||
# entries; the content digest alone would not notice a format change.
|
||||
key_schema="v1"
|
||||
|
||||
if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then
|
||||
echo "usage: $0 <key|verify|reset> <trace-case> [fixture-dir]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
command_name="$1"
|
||||
case_name="$2"
|
||||
fixture_dir="${3:-tools/trace_replay/fixtures}"
|
||||
python_bin="${PYTHON:-python3}"
|
||||
|
||||
if ! command -v "${python_bin}" >/dev/null 2>&1 && command -v python >/dev/null 2>&1; then
|
||||
python_bin=python
|
||||
fi
|
||||
|
||||
mapfile -t files < <(trace_fixture_files "${case_name}" "${fixture_dir}" "${python_bin}")
|
||||
if [ "${#files[@]}" -eq 0 ]; then
|
||||
echo "no fixture files declared for trace case: ${case_name}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Writes "name=value" to $GITHUB_OUTPUT when running under Actions, and to
|
||||
# stdout otherwise so the script stays runnable (and testable) off-CI.
|
||||
emit_output() {
|
||||
local name="$1"
|
||||
local value="$2"
|
||||
if [ -n "${GITHUB_OUTPUT:-}" ]; then
|
||||
if [[ "${value}" == *$'\n'* ]]; then
|
||||
local delimiter="ghadelim_$(date +%s%N)_$$"
|
||||
{
|
||||
printf '%s<<%s\n' "${name}" "${delimiter}"
|
||||
printf '%s\n' "${value}"
|
||||
printf '%s\n' "${delimiter}"
|
||||
} >> "${GITHUB_OUTPUT}"
|
||||
else
|
||||
printf '%s=%s\n' "${name}" "${value}" >> "${GITHUB_OUTPUT}"
|
||||
fi
|
||||
fi
|
||||
printf '%s=%s\n' "${name}" "${value}"
|
||||
}
|
||||
|
||||
sanitize_case() {
|
||||
printf '%s' "$1" | sed 's/[^A-Za-z0-9._-]/_/g'
|
||||
}
|
||||
|
||||
case "${command_name}" in
|
||||
key)
|
||||
manifest=""
|
||||
for file in "${files[@]}"; do
|
||||
# A case whose fixtures are committed directly rather than through Git LFS
|
||||
# (OpenRA) has no pointer oid to key on, and nothing to download either.
|
||||
# Report it as uncacheable so the workflow skips the cache entirely.
|
||||
if ! metadata="$(get_lfs_metadata "${file}" 2>/dev/null)"; then
|
||||
echo "trace case ${case_name} is not stored in Git LFS; skipping fixture cache" >&2
|
||||
emit_output "cacheable" "false"
|
||||
emit_output "key" ""
|
||||
exit 0
|
||||
fi
|
||||
read -r expected_oid expected_size <<< "${metadata}"
|
||||
manifest+="$(basename "${file}") ${expected_oid} ${expected_size}"$'\n'
|
||||
done
|
||||
|
||||
digest="$(printf '%s' "${manifest}" | sha256sum | awk '{ print substr($1, 1, 16) }')"
|
||||
safe_case="$(sanitize_case "${case_name}")"
|
||||
|
||||
emit_output "cacheable" "true"
|
||||
emit_output "key" "trace-fixture-${key_schema}-${safe_case}-${digest}"
|
||||
emit_output "paths" "$(printf '%s\n' "${files[@]}")"
|
||||
;;
|
||||
|
||||
verify)
|
||||
for file in "${files[@]}"; do
|
||||
metadata="$(get_lfs_metadata "${file}")"
|
||||
read -r expected_oid expected_size <<< "${metadata}"
|
||||
verify_fixture_file "${file}" "${file}" "${expected_oid}" "${expected_size}"
|
||||
done
|
||||
echo "Verified ${#files[@]} fixture file(s) for ${case_name} against the tracked Git LFS pointers."
|
||||
;;
|
||||
|
||||
reset)
|
||||
# Put the working tree back to the pointer files a fresh checkout would
|
||||
# have, so that a rejected cache entry falls through to exactly the same
|
||||
# download path a cache miss takes.
|
||||
for file in "${files[@]}"; do
|
||||
rm -f "${file}" "${file}.tmp"
|
||||
done
|
||||
git checkout -- "${files[@]}"
|
||||
echo "Reset ${#files[@]} fixture file(s) for ${case_name} to their tracked Git LFS pointers."
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "unknown command: ${command_name}" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
@@ -2,7 +2,7 @@
|
||||
# Shared helpers for trace-fixture handling: reading the in-tree Git LFS pointer
|
||||
# metadata and verifying a fixture file against it. Sourced by
|
||||
# fetch-trace-fixture-lfs.sh (verify after download) and by
|
||||
# trace-fixture-cache.sh (cache key derivation and verify after cache restore),
|
||||
# trace-fixture-bundle.sh (cache key derivation and verify after cache restore),
|
||||
# so both paths agree on what a valid fixture is.
|
||||
|
||||
# Reads the Git LFS pointer tracked at HEAD for a fixture path and prints
|
||||
|
||||
+334
-306
@@ -11,17 +11,46 @@ on:
|
||||
# someone remembering to dispatch the workflow by hand.
|
||||
- feat/disaggregated
|
||||
- codex/fix-github-ci
|
||||
- ci/round-30min
|
||||
- cleanup
|
||||
# A docs-only push builds nothing this workflow tests (231 emulator legs per run). The Test
|
||||
# workflow keeps running for docs: it hosts the doc citation lint.
|
||||
# A push that only touches these paths changes nothing this workflow builds or replays. The
|
||||
# Test workflow keeps running for docs: it hosts the doc citation lint. The Android build
|
||||
# compiles none of the unit, integration or benchmark trees, with one exception that keeps
|
||||
# MobileGL/MG_Benchmark from being listed whole: MG_Util/SelfTest/DriverBenchJni.cpp
|
||||
# includes MG_Benchmark/Driver/DriverBenchCases.inc.
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
- '**/*.md'
|
||||
- 'MobileGL/MG_Test/**'
|
||||
- 'MobileGL/MG_IntegrationTest/**'
|
||||
- 'MobileGL/MG_Benchmark/CMakeLists.txt'
|
||||
- 'MobileGL/MG_Benchmark/SanityBench.cpp'
|
||||
- 'MobileGL/MG_Benchmark/Buffer/**'
|
||||
- 'MobileGL/MG_Benchmark/Container/**'
|
||||
- 'MobileGL/MG_Benchmark/Program/**'
|
||||
- 'MobileGL/MG_Benchmark/ShaderCache/**'
|
||||
- 'MobileGL/MG_Benchmark/Transpile/**'
|
||||
- 'tools/cts/**'
|
||||
- 'tools/device_bench/**'
|
||||
- '.github/workflows/test.yml'
|
||||
- '.github/workflows/hygiene.yml'
|
||||
- '.github/actions/setup-test-runner/**'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
android_full_matrix:
|
||||
description: Replay every trace case on the monolith and spawn emulator lanes too
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
env:
|
||||
# Software-rendered CI submissions may exceed the interactive GPU hang budget.
|
||||
MOBILEGL_GPU_HANG_BUDGET_MS: "0"
|
||||
# The emulator replays every trace case on inproc, the transport the launcher ships. On
|
||||
# monolith and spawn it replays the android_smoke cases of trace_cases.json, because
|
||||
# test.yml replays every case on all three transports on Linux. A manual run with
|
||||
# android_full_matrix replays every case on all three emulator lanes. A push has no inputs,
|
||||
# so it gets the reduced lanes.
|
||||
MOBILEGL_ANDROID_LANES: ${{ inputs.android_full_matrix && 'full' || 'inproc-full' }}
|
||||
|
||||
# ONE LIVE RUN PER BRANCH, as in test.yml: a superseded push is canceled rather than left queued
|
||||
# behind its successor; a manual dispatch gets a group of its own; dev queues rather than cancels.
|
||||
@@ -39,16 +68,26 @@ jobs:
|
||||
CCACHE_BASEDIR: ${{ github.workspace }}
|
||||
CCACHE_COMPRESS: "true"
|
||||
CCACHE_DIR: ${{ github.workspace }}/.ccache
|
||||
CCACHE_MAXSIZE: 4G
|
||||
# One build's objects (three native configurations) take about 1 GB. The limit leaves
|
||||
# half of that again for the objects a push replaces, and is what the cleanup before
|
||||
# the save trims the entry to.
|
||||
CCACHE_MAXSIZE: 1500M
|
||||
CCACHE_NOHASHDIR: "true"
|
||||
MOBILEGL_CMAKE_COMPILER_LAUNCHER: ccache
|
||||
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
|
||||
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'ci/round-30min' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
|
||||
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
# Only the submodules the two APKs are built from, shallow and in parallel, in place of a
|
||||
# recursive checkout of every submodule in the tree: the ones the root CMakeLists.txt
|
||||
# configures, and apitrace with the directories its thirdparty/CMakeLists.txt requires,
|
||||
# for the trace APK. The step fails when a file the build reads from one of them did
|
||||
# not arrive (scripts/ci/checkout_submodules.sh).
|
||||
- name: Check out build submodules
|
||||
timeout-minutes: 15
|
||||
run: bash scripts/ci/checkout_submodules.sh retrace
|
||||
|
||||
- name: Set artifact metadata
|
||||
run: |
|
||||
@@ -82,6 +121,9 @@ jobs:
|
||||
sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 update
|
||||
sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 install -y ccache
|
||||
ccache --version
|
||||
# The restored entry carries the counters of every earlier build; zero them so the
|
||||
# statistics printed after the build are this run's.
|
||||
ccache --zero-stats
|
||||
|
||||
- name: Setup Android SDK
|
||||
id: netsetup84
|
||||
@@ -183,11 +225,20 @@ jobs:
|
||||
if: always()
|
||||
run: ccache --show-stats
|
||||
|
||||
# Rewrite one rolling entry per job and branch. The upload stays cumulative - it
|
||||
# carries every object restored at the top of this run plus the few TUs that
|
||||
# changed - but Actions cache keys are immutable, so this branch's superseded
|
||||
# blob is released first; --ref keeps the delete off other branches' entries.
|
||||
# Running after the build means a failed build leaves the entry untouched.
|
||||
# A build served from the cache stores nothing, so nothing would ever evict the objects
|
||||
# of older source states. Trim to CCACHE_MAXSIZE, least recently used first, before the
|
||||
# entry is saved; the objects this build hit were just touched and stay.
|
||||
- name: Trim ccache to its size limit
|
||||
run: |
|
||||
ccache --cleanup
|
||||
ccache --show-stats | grep -i 'cache size' || true
|
||||
du -sh "${CCACHE_DIR}"
|
||||
|
||||
# Rewrite one rolling entry per job and branch. The upload carries the objects this
|
||||
# build used plus what fits under the size limit, but Actions cache keys are
|
||||
# immutable, so this branch's superseded blob is released first; --ref keeps the
|
||||
# delete off other branches' entries. Running after the build means a failed build
|
||||
# leaves the entry untouched.
|
||||
- name: Release superseded ccache entry
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
@@ -251,10 +302,18 @@ jobs:
|
||||
name: trace case matrix
|
||||
runs-on: ubuntu-26.04
|
||||
env:
|
||||
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
|
||||
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'ci/round-30min' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
|
||||
# How much replay one emulator job aims for, in seconds as the duration hints count them.
|
||||
# A (lane, transport, backend) cell is cut into as many shards as that takes, and its
|
||||
# cases are spread so the shards of a cell come out even (tools/trace_replay/trace_cases.py,
|
||||
# shard_matrix). A smaller value gives more and shorter jobs, each of which pays the
|
||||
# emulator boot again; every case is replayed once per cell whatever the value.
|
||||
RETRACE_SHARD_SECONDS: "720"
|
||||
# The most a shard runner may spend on its cases. A shard gets the time in which every
|
||||
# one of its cases could run to the end of its own time budget, up to this cap.
|
||||
RETRACE_SHARD_DEADLINE_CAP_SECONDS: "7200"
|
||||
outputs:
|
||||
android: ${{ steps.trace-cases.outputs.android }}
|
||||
names: ${{ steps.trace-cases.outputs.names }}
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v6
|
||||
@@ -263,77 +322,109 @@ jobs:
|
||||
id: trace-cases
|
||||
run: |
|
||||
python3 tools/trace_replay/trace_cases.py --self-test-transport-matrices
|
||||
matrix_format=github-apk-matrix
|
||||
# The retrace jobs replay shards: several cases of one (lane, transport, backend) cell
|
||||
# per emulator boot. The self-test proves every shard matrix replays exactly the rows
|
||||
# of the per-case matrix it is built from, and trace_cases.py refuses to print one
|
||||
# that does not.
|
||||
python3 tools/trace_replay/trace_cases.py --self-test-shard-matrices
|
||||
matrix_format=github-apk-shard-matrix
|
||||
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then
|
||||
matrix_format=github-apk-transport-matrix
|
||||
matrix_format=github-apk-transport-shard-matrix
|
||||
fi
|
||||
echo "android=$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}")" >> "$GITHUB_OUTPUT"
|
||||
echo "names=$(python3 tools/trace_replay/trace_cases.py --ci --format names)" >> "$GITHUB_OUTPUT"
|
||||
android="$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}" \
|
||||
--shard-seconds "${RETRACE_SHARD_SECONDS}" --android-lanes "${MOBILEGL_ANDROID_LANES}")"
|
||||
# An empty `include` is not an error to GitHub: it skips the retrace job with no red.
|
||||
shards="$(printf '%s' "${android}" | python3 -c 'import json, sys; print(len(json.load(sys.stdin)["include"]))')"
|
||||
if [ "${shards}" -lt 1 ]; then
|
||||
echo "::error::the Android retrace matrix is empty, so the retrace job would be skipped with no red"
|
||||
exit 1
|
||||
fi
|
||||
# The cases of a shard share one job, so a case that hangs spends time the cases after
|
||||
# it need. retrace_shard_budget.py gives every shard its own deadline_seconds, the
|
||||
# time in which each of its cases could run to the end of its time budget, capped so
|
||||
# that a shard whose replays all hang cannot hold a runner for hours, and the
|
||||
# timeout_minutes of its job. The retrace job reads both from the matrix.
|
||||
android="$(printf '%s' "${android}" | python3 android-plugin/retrace_shard_budget.py \
|
||||
--cap-seconds "${RETRACE_SHARD_DEADLINE_CAP_SECONDS}")"
|
||||
echo "android=${android}" >> "$GITHUB_OUTPUT"
|
||||
lanes=""
|
||||
if [ "${matrix_format}" = "github-apk-transport-shard-matrix" ]; then
|
||||
lanes=" (${MOBILEGL_ANDROID_LANES} lanes)"
|
||||
fi
|
||||
echo "Android retrace: ${shards} shard(s) from ${matrix_format}${lanes} at ${RETRACE_SHARD_SECONDS} s per shard"
|
||||
|
||||
# One cache entry holds every Git LFS fixture of the CI cases, and each retrace job restores
|
||||
# that entry itself. This job only makes sure the entry exists. On a hit it downloads
|
||||
# nothing. On a miss it starts from the newest older bundle, fetches from the mirror the
|
||||
# cases whose pointers changed, saves the entry under the new key and confirms it is there,
|
||||
# so that no retrace job has to download fixtures itself. The key, the path and the steps
|
||||
# are .github/scripts/trace-fixture-bundle.sh's, and test.yml has the same job for the same
|
||||
# entry: the two workflows of one push look the key up at the same time, and on a miss both
|
||||
# fetch and both save. The save that loses is not an error, since the entry it wanted to
|
||||
# write is the one the confirming step then finds. The timeout bounds a fixture download
|
||||
# that stalls, which would otherwise hold every retrace job behind it.
|
||||
trace-fixtures:
|
||||
name: trace fixture (${{ matrix.case }})
|
||||
name: trace fixtures
|
||||
runs-on: ubuntu-26.04
|
||||
needs: trace-cases
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 4
|
||||
matrix:
|
||||
case: ${{ fromJSON(needs.trace-cases.outputs.names) }}
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
outputs:
|
||||
key: ${{ steps.bundle.outputs.key }}
|
||||
path: ${{ steps.bundle.outputs.path }}
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Derive trace fixture cache key
|
||||
id: fixture-key
|
||||
run: bash .github/scripts/trace-fixture-cache.sh key '${{ matrix.case }}'
|
||||
- name: Derive the fixture bundle key
|
||||
id: bundle
|
||||
run: bash .github/scripts/trace-fixture-bundle.sh key
|
||||
|
||||
- name: Restore trace fixture cache
|
||||
id: fixture-cache
|
||||
if: steps.fixture-key.outputs.cacheable == 'true'
|
||||
- name: Look up the fixture bundle
|
||||
id: lookup
|
||||
uses: actions/cache/restore@v5
|
||||
with:
|
||||
path: ${{ steps.fixture-key.outputs.paths }}
|
||||
key: ${{ steps.fixture-key.outputs.key }}
|
||||
path: ${{ steps.bundle.outputs.path }}
|
||||
key: ${{ steps.bundle.outputs.key }}
|
||||
lookup-only: true
|
||||
|
||||
- name: Verify restored trace fixture
|
||||
id: fixture-verify
|
||||
if: steps.fixture-cache.outputs.cache-hit == 'true'
|
||||
run: |
|
||||
if bash .github/scripts/trace-fixture-cache.sh verify '${{ matrix.case }}'; then
|
||||
echo "ok=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "ok=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::warning::Cached fixture for ${{ matrix.case }} failed verification; falling back to the download path"
|
||||
bash .github/scripts/trace-fixture-cache.sh reset '${{ matrix.case }}'
|
||||
fi
|
||||
- name: Restore the newest older bundle
|
||||
if: steps.lookup.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/restore@v5
|
||||
with:
|
||||
path: ${{ steps.bundle.outputs.path }}
|
||||
key: ${{ steps.bundle.outputs.key }}
|
||||
restore-keys: ${{ steps.bundle.outputs.restore-prefix }}
|
||||
|
||||
- name: Fetch trace fixture
|
||||
if: steps.fixture-verify.outputs.ok != 'true'
|
||||
run: bash .github/scripts/fetch-trace-fixture-lfs.sh '${{ matrix.case }}'
|
||||
- name: Complete the fixture bundle
|
||||
if: steps.lookup.outputs.cache-hit != 'true'
|
||||
run: bash .github/scripts/trace-fixture-bundle.sh ensure
|
||||
|
||||
- name: Save trace fixture cache
|
||||
if: steps.fixture-key.outputs.cacheable == 'true' && steps.fixture-cache.outputs.cache-hit != 'true'
|
||||
- name: Save the fixture bundle
|
||||
if: steps.lookup.outputs.cache-hit != 'true'
|
||||
continue-on-error: true
|
||||
uses: actions/cache/save@v5
|
||||
with:
|
||||
path: ${{ steps.fixture-key.outputs.paths }}
|
||||
key: ${{ steps.fixture-key.outputs.key }}
|
||||
path: ${{ steps.bundle.outputs.path }}
|
||||
key: ${{ steps.bundle.outputs.key }}
|
||||
|
||||
- name: Stage trace fixture
|
||||
run: |
|
||||
safe_case="$(printf '%s' '${{ matrix.case }}' | sed 's/[^A-Za-z0-9._-]/_/g')"
|
||||
stage_dir="trace-fixtures/${safe_case}"
|
||||
mkdir -p "${stage_dir}"
|
||||
python3 tools/trace_replay/trace_cases.py --format fixture-files --case '${{ matrix.case }}' |
|
||||
while IFS= read -r file; do
|
||||
cp "${file}" "${stage_dir}/"
|
||||
done
|
||||
# The Test workflow of the same push may be the one whose save won. Its entry is as good,
|
||||
# once it is listed.
|
||||
- name: Wait for the fixture bundle entry
|
||||
if: steps.lookup.outputs.cache-hit != 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: bash .github/scripts/trace-fixture-bundle.sh wait '${{ steps.bundle.outputs.key }}'
|
||||
|
||||
- name: Upload trace fixture
|
||||
uses: actions/upload-artifact@v7
|
||||
- name: Confirm the fixture bundle entry
|
||||
if: steps.lookup.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/restore@v5
|
||||
with:
|
||||
name: MobileGL-trace-fixture-${{ matrix.case }}
|
||||
path: trace-fixtures/**
|
||||
if-no-files-found: error
|
||||
path: ${{ steps.bundle.outputs.path }}
|
||||
key: ${{ steps.bundle.outputs.key }}
|
||||
lookup-only: true
|
||||
fail-on-cache-miss: true
|
||||
|
||||
android-avd:
|
||||
name: android avd image
|
||||
@@ -347,8 +438,33 @@ jobs:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v6
|
||||
|
||||
# The emulator shard loop and the artifact cleanup, driven by fakes: what each case is
|
||||
# run with, a verdict for every case, and what stays after a failure. It runs in this
|
||||
# job because the retrace jobs wait for it and the APK build does not.
|
||||
- name: Emulator shard runner regression
|
||||
run: |
|
||||
python3 android-plugin/test_trace_replay_shard.py
|
||||
python3 android-plugin/test_retrace_artifact_cleanup.py
|
||||
|
||||
# This job only makes sure the cache entry exists; the retrace jobs restore it. A hit
|
||||
# downloads nothing and skips every step below; a miss builds the AVD, and the post step
|
||||
# of this action saves it.
|
||||
- name: Look up Android AVD cache
|
||||
id: android-avd-cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
lookup-only: true
|
||||
path: |
|
||||
${{ env.ANDROID_AVD_HOME }}
|
||||
${{ env.ANDROID_SDK_ROOT }}/emulator
|
||||
${{ env.ANDROID_SDK_ROOT }}/platform-tools
|
||||
${{ env.ANDROID_SDK_ROOT }}/platforms/android-35
|
||||
${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64
|
||||
key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v4-${{ hashFiles('android-plugin/run-avd-ci.sh') }}
|
||||
|
||||
- name: Setup Android SDK
|
||||
id: netsetup336
|
||||
if: steps.android-avd-cache.outputs.cache-hit != 'true'
|
||||
continue-on-error: true
|
||||
timeout-minutes: 15
|
||||
uses: android-actions/setup-android@v4
|
||||
@@ -367,20 +483,9 @@ jobs:
|
||||
cmdline-tools-version: 13114758
|
||||
|
||||
- name: Accept Android SDK licenses
|
||||
if: steps.android-avd-cache.outputs.cache-hit != 'true'
|
||||
run: yes | sdkmanager --licenses >/dev/null
|
||||
|
||||
- name: Restore Android AVD cache
|
||||
id: android-avd-cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
${{ env.ANDROID_AVD_HOME }}
|
||||
${{ env.ANDROID_SDK_ROOT }}/emulator
|
||||
${{ env.ANDROID_SDK_ROOT }}/platform-tools
|
||||
${{ env.ANDROID_SDK_ROOT }}/platforms/android-35
|
||||
${{ env.ANDROID_SDK_ROOT }}/system-images/android-35/google_apis/x86_64
|
||||
key: ${{ runner.os }}-mobilegl-avd-api35-google_apis-x86_64-pixel_6-v4-${{ hashFiles('android-plugin/run-avd-ci.sh') }}
|
||||
|
||||
- name: Create AVD
|
||||
if: steps.android-avd-cache.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
@@ -418,16 +523,27 @@ jobs:
|
||||
fi
|
||||
du -sh "${ANDROID_AVD_HOME}"
|
||||
|
||||
# One job replays a shard: some of the cases of one lane, transport and backend, one after
|
||||
# another on one emulator boot (android-plugin/trace_replay_shard.py). The shard's step
|
||||
# summary has a row per case, and a case that did not pass has an error annotation of its own.
|
||||
retrace:
|
||||
name: retrace (${{ matrix.backend.name }} / ${{ matrix.transport || 'monolith' }}, ${{ matrix.case.name }})
|
||||
name: retrace (${{ matrix.backend.name }} / ${{ matrix.transport || 'monolith' }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
|
||||
runs-on: ubuntu-26.04
|
||||
needs:
|
||||
- build
|
||||
- android-avd
|
||||
- trace-cases
|
||||
- trace-fixtures
|
||||
if: ${{ !cancelled() && needs.build.result == 'success' && needs.android-avd.result == 'success' && needs.trace-cases.result == 'success' }}
|
||||
timeout-minutes: 75
|
||||
if: ${{ !cancelled() && needs.build.result == 'success' && needs.android-avd.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }}
|
||||
# Paired with --deadline-seconds below, and far above what a shard takes when its replays
|
||||
# end by themselves. Both come from the matrix. The deadline is the smaller of the time in
|
||||
# which every case of the shard could run to the end of its own time budget
|
||||
# (trace_replay_shard.py, needed_seconds) and RETRACE_SHARD_DEADLINE_CAP_SECONDS. With
|
||||
# budgets of up to 1800 s a case the cap is what a shard of several cases gets: about
|
||||
# three replays can hang to the end of their budgets before a later case is recorded as
|
||||
# not-run. The minutes between the deadline and this limit are the setup before the
|
||||
# replays and the uploads after them.
|
||||
timeout-minutes: ${{ matrix.timeout_minutes }}
|
||||
env:
|
||||
MOBILEGL_CI_TRANSPORT: ${{ matrix.transport || 'monolith' }}
|
||||
MOBILEGL_CI_LANE: ${{ matrix.lane || 'monolith-control' }}
|
||||
@@ -435,39 +551,54 @@ jobs:
|
||||
ANDROID_AVD_HOME: ${{ github.workspace }}/.android/avd
|
||||
ANDROID_HOME: ${{ github.workspace }}/.android/sdk
|
||||
ANDROID_SDK_ROOT: ${{ github.workspace }}/.android/sdk
|
||||
# The shard's cases in run order, as the JSON array of case objects the shard runner
|
||||
# takes: every field of the case in trace_cases.json, with the fixture paths in place.
|
||||
MOBILEGL_SHARD_CASES: ${{ toJSON(matrix.cases) }}
|
||||
# Set the repository variable to 1 to cold boot the emulator before every case of a
|
||||
# shard, should one boot not survive a whole shard of software-rendered replays.
|
||||
MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE: ${{ vars.MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE || '0' }}
|
||||
# The emulator renders on the host with a software rasterizer that can keep compiled
|
||||
# shaders on disk in the runner's home directory, where neither clearing the app's data
|
||||
# nor restarting the emulator reaches. Off, so every case of a shard compiles its
|
||||
# shaders as a job with a fresh machine does.
|
||||
MESA_SHADER_CACHE_DISABLE: "true"
|
||||
strategy:
|
||||
fail-fast: false
|
||||
# The account runs at most 20 jobs at once, shared with Test. These ~230 emulator jobs are
|
||||
# the longer of the two runs, so they may take the whole pool: what is minimized is when the
|
||||
# later of APK and Test finishes, not Test alone.
|
||||
max-parallel: 20
|
||||
# No max-parallel: the account's limit of 20 concurrent jobs, shared with Test, already
|
||||
# bounds this matrix, and a cap below the number of shards only makes the round longer.
|
||||
matrix: ${{ fromJSON(needs.trace-cases.outputs.android) }}
|
||||
steps:
|
||||
- name: Set swap space
|
||||
uses: pierotofy/set-swap-space@v1.0
|
||||
with:
|
||||
swap-size-gb: 8
|
||||
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download trace fixture
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: MobileGL-trace-fixture-${{ matrix.case.name }}
|
||||
path: trace-fixture-download
|
||||
|
||||
- name: Install trace fixture
|
||||
run: |
|
||||
mkdir -p tools/trace_replay/fixtures
|
||||
find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;
|
||||
|
||||
# First, so the steps that run whatever happened (diagnostics, stopping the emulator, the
|
||||
# result upload) have these even when a step before the emulator launch failed.
|
||||
- name: Set artifact metadata
|
||||
run: |
|
||||
echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
|
||||
echo "EMULATOR_LOG=${RUNNER_TEMP}/mobilegl-emulator.log" >> "$GITHUB_ENV"
|
||||
echo "EMULATOR_PID_FILE=${RUNNER_TEMP}/mobilegl-emulator.pid" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Set swap space
|
||||
uses: ./.github/actions/set-swap-space
|
||||
with:
|
||||
swap-size-gb: 8
|
||||
|
||||
# A miss does not fail the job: the entry may have been evicted since trace-fixtures
|
||||
# confirmed it (a re-run of failed jobs on an older run does not run that job again).
|
||||
- name: Restore trace fixtures
|
||||
uses: actions/cache/restore@v5
|
||||
with:
|
||||
path: ${{ needs.trace-fixtures.outputs.path }}
|
||||
key: ${{ needs.trace-fixtures.outputs.key }}
|
||||
|
||||
# Moves this shard's fixtures into place and checks them against their Git LFS pointers.
|
||||
# A case the restored bundle does not hold is downloaded here, with a warning annotation;
|
||||
# the timeout bounds a download that stalls.
|
||||
- name: Install trace fixtures
|
||||
timeout-minutes: 15
|
||||
run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
|
||||
|
||||
- name: Restore Android AVD cache
|
||||
id: android-avd-cache
|
||||
uses: actions/cache/restore@v5
|
||||
@@ -543,135 +674,44 @@ jobs:
|
||||
--pid-file "${EMULATOR_PID_FILE}" \
|
||||
--boot-timeout 300
|
||||
|
||||
# One emulator boot replays every case of the shard in order, each in a fresh app process
|
||||
# with cleared app data and with the settings its case, backend and transport need. The
|
||||
# runner keeps going after a failing case, on a cold boot so that case cannot touch the
|
||||
# next one's verdict, retries a case once on a restarted emulator when the emulator
|
||||
# itself failed, and exits non-zero when a case failed or a verdict is missing. The
|
||||
# deadline stops it while the steps below can still upload what it has: the job timeout
|
||||
# less the setup above and the uploads below. A case it stops or never starts is
|
||||
# recorded as not-run, which is red.
|
||||
- name: Retrace and validate
|
||||
env:
|
||||
MOBILEGL_ESPRYT_USE_ANGLE: ${{ matrix.backend.name == 'DirectGLES' && '1' || '0' }}
|
||||
MOBILEGL_TRACE_ANGLE_VARIANT: ${{ matrix.case.name == 'minecraft-1.21.4-fabric-iris-bliss-in-world' && '90a62123d794' || 'ec889e6ea831' }}
|
||||
MOBILEGL_MAGMA_R11G11B10F_FALLBACK: ${{ matrix.backend.name == 'DirectVulkan' && '1' || '0' }}
|
||||
MOBILEGL_ITRP_FIX: ${{ matrix.case.name == 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' && 'ForceOn' || 'Auto' }}
|
||||
# Lets the artifact cleanup tell which job a shard's verdicts came from.
|
||||
MOBILEGL_SHARD_JOB_ID: ${{ job.check_run_id }}
|
||||
run: |
|
||||
apk_file="android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk"
|
||||
test -f "${apk_file}"
|
||||
extra_retrace_args=()
|
||||
if [ "${MOBILEGL_CI_TRANSPORT}" = "inproc" ]; then
|
||||
extra_retrace_args+=(
|
||||
--env "MOBILEGL_TRANSPORT=inproc;MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1;MOBILEGL_GPU_HANG_BUDGET_MS=${MOBILEGL_GPU_HANG_BUDGET_MS}"
|
||||
--require-inproc
|
||||
)
|
||||
elif [ "${MOBILEGL_CI_TRANSPORT}" = "spawn" ]; then
|
||||
# THE SAME CLIENT KNOBS AS inproc, so the two acceptance lanes differ in exactly one
|
||||
# thing: which PROCESS the server role runs in. They are scrubbed out of the CHILD's
|
||||
# environment by the launcher and read by the client, which is where they belong.
|
||||
#
|
||||
# MOBILEGL_IPC_SERVER_PATH IS DELIBERATELY ABSENT. nativeLibraryDir carries an
|
||||
# install-time hash, so nothing off the device can spell it; TraceReplayActivity
|
||||
# resolves it from getApplicationInfo() when the env asks for spawn, and an explicit
|
||||
# value would override that. --use-pbuffer is NOT optional here: a spawned
|
||||
# server cannot be handed the client's window, because an ANativeWindow* is a pointer
|
||||
# into the CLIENT's process and SetWindowHandle is refused on the way across with
|
||||
# Fatal{UnmigratedSurface, "AndroidNativeWindow:use-MOBILEGL_IPC_SURFACE=server"} (Rule H, permanent;
|
||||
# a windowed client uses MOBILEGL_IPC_SURFACE=server instead).
|
||||
extra_retrace_args+=(
|
||||
--env "MOBILEGL_TRANSPORT=spawn;MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1;MOBILEGL_GPU_HANG_BUDGET_MS=${MOBILEGL_GPU_HANG_BUDGET_MS}"
|
||||
--use-pbuffer
|
||||
--require-spawn
|
||||
)
|
||||
else
|
||||
extra_retrace_args+=(--env "MOBILEGL_TRANSPORT=monolith;MOBILEGL_GPU_HANG_BUDGET_MS=${MOBILEGL_GPU_HANG_BUDGET_MS}")
|
||||
fi
|
||||
# Bliss needs the newer signed ANGLE variant plus sampler mipmap
|
||||
# min-filter downgrading on ANGLE llvmpipe.
|
||||
if [ "${{ matrix.backend.name }}" = "DirectGLES" ] && [ "${{ matrix.case.name }}" = "minecraft-1.21.4-fabric-iris-bliss-in-world" ]; then
|
||||
extra_retrace_args+=(--avoid-angle-llvmpipe-sampler-mipmap-min-filter)
|
||||
fi
|
||||
if [ "${{ matrix.backend.name }}" = "DirectGLES" ] && [ "${{ matrix.case.avoid_angle_llvmpipe_explicit_lod_bias || false }}" = "true" ]; then
|
||||
extra_retrace_args+=(--avoid-angle-llvmpipe-explicit-lod-bias)
|
||||
fi
|
||||
if [ "${{ matrix.case.coherent_as_flush || false }}" = "true" ]; then
|
||||
extra_retrace_args+=(--coherent-as-flush)
|
||||
fi
|
||||
|
||||
run_retrace() {
|
||||
timeout "$(( ${{ matrix.case.timeout_seconds }} + 300 ))" sh android-plugin/trace-replay-ci.sh \
|
||||
--apk-file "${apk_file}" \
|
||||
--package top.mobilegl.plugin.trace \
|
||||
--backend "${{ matrix.backend.name }}" \
|
||||
--result-root android-retrace-result \
|
||||
--fixture-root android-retrace-fixture \
|
||||
--case "${{ matrix.case.name }}" \
|
||||
--trace-archive "${{ matrix.case.trace_archive }}" \
|
||||
--trace-file "${{ matrix.case.trace_file }}" \
|
||||
--golden "${{ matrix.case.golden }}" \
|
||||
--alternate-golden "${{ matrix.case.alternate_golden || '' }}" \
|
||||
--target-call "${{ matrix.case.target_call }}" \
|
||||
--width "${{ matrix.case.width }}" \
|
||||
--height "${{ matrix.case.height }}" \
|
||||
--ssim-threshold "${{ matrix.case.ssim_threshold || '0.99' }}" \
|
||||
--crop-x "${{ matrix.case.crop_x }}" \
|
||||
--crop-y "${{ matrix.case.crop_y }}" \
|
||||
--crop-width "${{ matrix.case.crop_width }}" \
|
||||
--crop-height "${{ matrix.case.crop_height }}" \
|
||||
--timeout-seconds "${{ matrix.case.timeout_seconds }}" \
|
||||
"${extra_retrace_args[@]}"
|
||||
}
|
||||
|
||||
retrace_status=0
|
||||
run_retrace || retrace_status=$?
|
||||
if [ "${retrace_status}" -eq 75 ]; then
|
||||
echo "::warning::Android emulator infrastructure failed; restarting it and retrying this retrace once."
|
||||
# Surface-lost is retried rather than failed, so it would otherwise
|
||||
# be invisible. Report it per job - a healthy run prints nothing and
|
||||
# a rate spike shows up as a row per affected case.
|
||||
reason_file="android-retrace-result/infrastructure-failure-reason.txt"
|
||||
surface_lost_retries=0
|
||||
if [ -f "${reason_file}" ]; then
|
||||
surface_lost_retries="$(grep -c 'angle-surface-lost' "${reason_file}" || true)"
|
||||
fi
|
||||
if [ "${surface_lost_retries}" -gt 0 ]; then
|
||||
echo "surface-lost retries: ${surface_lost_retries} (${{ matrix.backend.name }}, ${{ matrix.case.name }})" \
|
||||
>> "${GITHUB_STEP_SUMMARY}"
|
||||
fi
|
||||
# The restart truncates EMULATOR_LOG, and the attempt that lost the
|
||||
# emulator is the one worth reading - the retry usually only shows
|
||||
# the wreckage. Keep the first attempt's log before it is clobbered.
|
||||
if [ -f "${EMULATOR_LOG}" ]; then
|
||||
cp "${EMULATOR_LOG}" "${EMULATOR_LOG}.first-attempt" || true
|
||||
fi
|
||||
sh android-plugin/run-avd-ci.sh stop \
|
||||
--avd-name "${AVD_NAME}" \
|
||||
--emulator-log "${EMULATOR_LOG}" \
|
||||
--pid-file "${EMULATOR_PID_FILE}"
|
||||
adb kill-server || true
|
||||
sleep 2
|
||||
sh android-plugin/run-avd-ci.sh start \
|
||||
--avd-name "${AVD_NAME}" \
|
||||
--gpu "${{ matrix.backend.gpu }}" \
|
||||
--emulator-log "${EMULATOR_LOG}" \
|
||||
--pid-file "${EMULATOR_PID_FILE}" \
|
||||
--boot-timeout 300
|
||||
run_retrace
|
||||
elif [ "${retrace_status}" -ne 0 ]; then
|
||||
exit "${retrace_status}"
|
||||
fi
|
||||
sh android-plugin/trace-replay-shard.sh \
|
||||
--apk-file "android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk" \
|
||||
--backend "${{ matrix.backend.name }}" \
|
||||
--gpu "${{ matrix.backend.gpu }}" \
|
||||
--transport "${MOBILEGL_CI_TRANSPORT}" \
|
||||
--lane "${MOBILEGL_CI_LANE}" \
|
||||
--cases-env MOBILEGL_SHARD_CASES \
|
||||
--deadline-seconds "${{ matrix.deadline_seconds }}"
|
||||
|
||||
# Runs whatever happened above, without the device: every case of the shard gets its
|
||||
# lane.json and golden copies for the summary, and a not-run verdict when the runner was
|
||||
# killed or never started.
|
||||
- name: Collect retrace summary inputs
|
||||
if: always()
|
||||
env:
|
||||
MOBILEGL_SHARD_JOB_ID: ${{ job.check_run_id }}
|
||||
run: |
|
||||
safe_case="$(printf '%s' '${{ matrix.case.name }}' | sed 's/[^A-Za-z0-9._-]/_/g')"
|
||||
result_dir="android-retrace-result/${safe_case}-${{ matrix.backend.name }}"
|
||||
mkdir -p "${result_dir}"
|
||||
python3 - "${result_dir}/lane.json" <<'PY'
|
||||
import json, os, sys
|
||||
from pathlib import Path
|
||||
Path(sys.argv[1]).write_text(json.dumps({"lane": os.environ["MOBILEGL_CI_LANE"],
|
||||
"transport": os.environ["MOBILEGL_CI_TRANSPORT"]}, indent=2))
|
||||
PY
|
||||
if [ -s "${{ matrix.case.golden }}" ]; then
|
||||
cp "${{ matrix.case.golden }}" "${result_dir}/${safe_case}-${{ matrix.backend.name }}-golden.png"
|
||||
fi
|
||||
if [ -n "${{ matrix.case.alternate_golden || '' }}" ] && [ -s "${{ matrix.case.alternate_golden || '' }}" ]; then
|
||||
cp "${{ matrix.case.alternate_golden || '' }}" "${result_dir}/${safe_case}-${{ matrix.backend.name }}-alternate-golden.png"
|
||||
fi
|
||||
sh android-plugin/trace-replay-shard.sh \
|
||||
--apk-file "android-retrace-apks/MobileGL-plugin-trace-release-${GITHUB_SHA}.apk" \
|
||||
--backend "${{ matrix.backend.name }}" \
|
||||
--gpu "${{ matrix.backend.gpu }}" \
|
||||
--transport "${MOBILEGL_CI_TRANSPORT}" \
|
||||
--lane "${MOBILEGL_CI_LANE}" \
|
||||
--cases-env MOBILEGL_SHARD_CASES \
|
||||
--finalize
|
||||
|
||||
- name: Collect emulator diagnostics
|
||||
if: always()
|
||||
@@ -679,12 +719,18 @@ jobs:
|
||||
mkdir -p android-retrace-result/diagnostics
|
||||
adb devices -l > android-retrace-result/diagnostics/adb-devices.txt || true
|
||||
timeout 30 adb logcat -d -t 1000 > android-retrace-result/diagnostics/logcat.txt || true
|
||||
# What the guest was given; replay time inside the emulator depends on it.
|
||||
timeout 30 adb shell 'nproc; grep -E "MemTotal|SwapTotal" /proc/meminfo' \
|
||||
> android-retrace-result/diagnostics/guest-resources.txt 2>&1 || true
|
||||
cp "${ANDROID_AVD_HOME}/${AVD_NAME}.avd/config.ini" android-retrace-result/diagnostics/avd-config.ini 2>/dev/null || true
|
||||
# Expected to list nothing: the host rasterizer's on-disk shader cache is disabled.
|
||||
du -sh "${XDG_CACHE_HOME:-${HOME}/.cache}"/mesa_shader_cache* \
|
||||
> android-retrace-result/diagnostics/host-shader-cache.txt 2>&1 || true
|
||||
# The log of the last boot. The shard runner keeps the log of every boot it ended
|
||||
# as diagnostics/emulator-boot-<n>-*.log before a restart truncates it.
|
||||
if [ -f "${EMULATOR_LOG}" ]; then
|
||||
cp "${EMULATOR_LOG}" android-retrace-result/diagnostics/emulator.log
|
||||
fi
|
||||
if [ -f "${EMULATOR_LOG}.first-attempt" ]; then
|
||||
cp "${EMULATOR_LOG}.first-attempt" android-retrace-result/diagnostics/emulator-first-attempt.log
|
||||
fi
|
||||
# A vanished emulator looks identical whether the host OOM killer took
|
||||
# qemu or the renderer faulted. These two say which.
|
||||
free -h > android-retrace-result/diagnostics/host-memory.txt 2>&1 || true
|
||||
@@ -704,9 +750,14 @@ jobs:
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-${{ matrix.case.name }}
|
||||
# The summary job downloads by the lane and transport at the front of this name, and
|
||||
# its renderer reads the backend and the case from the paths below it, so the name
|
||||
# ends in the shard number and holds no case name.
|
||||
name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-shard${{ matrix.shard }}
|
||||
path: android-retrace-result/**
|
||||
if-no-files-found: warn
|
||||
# A re-run of a failed job replaces the artifact its first attempt kept.
|
||||
overwrite: true
|
||||
|
||||
# The artifact service sometimes times out (ETIMEDOUT on CreateArtifact, one job in 231 on
|
||||
# 2026-10-18), which turned a passing case red; one delayed retry absorbs that.
|
||||
@@ -718,7 +769,7 @@ jobs:
|
||||
if: always() && steps.upload-retrace-result.outcome == 'failure'
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-${{ matrix.case.name }}
|
||||
name: MobileGL-android-retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ env.date_today }}-${{ github.sha }}-${{ matrix.backend.name }}-shard${{ matrix.shard }}
|
||||
path: android-retrace-result/**
|
||||
if-no-files-found: warn
|
||||
overwrite: true
|
||||
@@ -727,46 +778,67 @@ jobs:
|
||||
name: retrace summary
|
||||
runs-on: ubuntu-26.04
|
||||
needs: retrace
|
||||
if: ${{ !cancelled() }}
|
||||
# Also runs for a canceled run, where it only downloads the results and deletes the
|
||||
# intermediate artifacts; the rendering steps are skipped then. The checkout and the
|
||||
# downloads say so themselves, because a step with no condition is skipped once its job is
|
||||
# canceled.
|
||||
if: always()
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
env:
|
||||
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
|
||||
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'ci/round-30min' || github.ref_name == 'cleanup') && 'ON' || 'OFF' }}
|
||||
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
if: ${{ success() || cancelled() }}
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set artifact metadata
|
||||
run: |
|
||||
echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Set up Node.js
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
- name: Download Android monolith control results
|
||||
if: ${{ success() || cancelled() }}
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: MobileGL-android-retrace-result-monolith-control-monolith-*
|
||||
path: retrace-artifacts/monolith-control
|
||||
|
||||
- name: Download Android inproc acceptance results
|
||||
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
|
||||
if: ${{ (success() || cancelled()) && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: MobileGL-android-retrace-result-inproc-acceptance-inproc-*
|
||||
path: retrace-artifacts/inproc-acceptance
|
||||
|
||||
- name: Download Android spawn acceptance results
|
||||
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
|
||||
if: ${{ (success() || cancelled()) && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: MobileGL-android-retrace-result-spawn-acceptance-spawn-*
|
||||
path: retrace-artifacts/spawn-acceptance
|
||||
|
||||
# The reduced monolith and spawn lanes replay the android_smoke cases only, so their
|
||||
# summaries list those cases and no others. A branch outside the split shape replays
|
||||
# every case on monolith, its only lane.
|
||||
- name: Select the cases of the reduced lanes
|
||||
if: ${{ !cancelled() }}
|
||||
run: |
|
||||
reduced_lane_cases=""
|
||||
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ] && [ "${MOBILEGL_ANDROID_LANES}" = "inproc-full" ]; then
|
||||
reduced_lane_cases="$(python3 tools/trace_replay/trace_cases.py --format android-smoke-names)"
|
||||
test -n "${reduced_lane_cases}"
|
||||
fi
|
||||
echo "REDUCED_LANE_CASES=${reduced_lane_cases}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Render monolith control summary
|
||||
if: ${{ !cancelled() }}
|
||||
run: |
|
||||
node tools/trace_replay/render_retrace_summary.mjs \
|
||||
--cases "${REDUCED_LANE_CASES}" \
|
||||
--input retrace-artifacts/monolith-control \
|
||||
--output-dir android-retrace-summary/monolith-control \
|
||||
--title "MobileGL Android monolith control" \
|
||||
@@ -774,7 +846,7 @@ jobs:
|
||||
--html mobilegl-android-monolith-control.html
|
||||
|
||||
- name: Render inproc acceptance summary
|
||||
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
|
||||
if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
|
||||
run: |
|
||||
node tools/trace_replay/render_retrace_summary.mjs \
|
||||
--input retrace-artifacts/inproc-acceptance \
|
||||
@@ -784,11 +856,18 @@ jobs:
|
||||
--html mobilegl-android-inproc-acceptance.html
|
||||
|
||||
- name: Render spawn acceptance summary
|
||||
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
|
||||
if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
|
||||
run: |
|
||||
node tools/trace_replay/render_retrace_summary.mjs --input retrace-artifacts/spawn-acceptance --output-dir android-retrace-summary/spawn-acceptance --title "MobileGL Android spawn acceptance" --group-label "Android Emulator / spawn acceptance" --html mobilegl-android-spawn-acceptance.html
|
||||
node tools/trace_replay/render_retrace_summary.mjs \
|
||||
--cases "${REDUCED_LANE_CASES}" \
|
||||
--input retrace-artifacts/spawn-acceptance \
|
||||
--output-dir android-retrace-summary/spawn-acceptance \
|
||||
--title "MobileGL Android spawn acceptance" \
|
||||
--group-label "Android Emulator / spawn acceptance" \
|
||||
--html mobilegl-android-spawn-acceptance.html
|
||||
|
||||
- name: Upload Android monolith control summary
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: MobileGL-android-monolith-control-summary-${{ github.sha }}
|
||||
@@ -797,7 +876,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Upload Android inproc acceptance summary
|
||||
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
|
||||
if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: MobileGL-android-inproc-acceptance-summary-${{ github.sha }}
|
||||
@@ -806,7 +885,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Upload Android spawn acceptance summary
|
||||
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
|
||||
if: ${{ !cancelled() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: MobileGL-android-spawn-acceptance-summary-${{ github.sha }}
|
||||
@@ -814,74 +893,23 @@ jobs:
|
||||
archive: false
|
||||
if-no-files-found: error
|
||||
|
||||
remove-artifact-clutter:
|
||||
name: remove artifact clutter
|
||||
runs-on: ubuntu-26.04
|
||||
needs: retrace-summary
|
||||
if: always()
|
||||
permissions:
|
||||
actions: write
|
||||
steps:
|
||||
# The summaries embed every image they show, so the per-shard results are intermediates
|
||||
# from here on. This is the last step of the workflow rather than a job of its own, which
|
||||
# would wait for a runner and start from an empty machine. It keeps what a failure needs,
|
||||
# read from the shard-status.json of the results downloaded above: the results of every
|
||||
# shard that ran a case which did not pass somewhere, on all backends and transports, and
|
||||
# every result when a job failed without saying which cases it ran. Deletes are paced
|
||||
# below GitHub's limit for mutating requests, and a refused delete is a warning. Skipped
|
||||
# when a summary step failed, so that re-running this job still finds the results, and
|
||||
# when a retrace job did not succeed in a run that was not canceled: every result stays
|
||||
# then, so a shard stopped by its job limit keeps its diagnostics and a re-run of the
|
||||
# failed jobs renders complete summaries.
|
||||
- name: Delete intermediate Android retrace artifacts
|
||||
if: ${{ (success() && needs.retrace.result == 'success') || cancelled() }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
# Fail before deletion if either paginated inventory cannot be read.
|
||||
# 50 rows per page: the jobs endpoint answers HTTP 502 for 90+ rows on a re-run attempt of a run this size.
|
||||
gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=50" \
|
||||
--jq '.jobs[] | select(.name | startswith("retrace (")) | select(.conclusion == "failure" or .conclusion == "canceled" or .conclusion == "timed_out" or .conclusion == "action_required") | .name' \
|
||||
> "${RUNNER_TEMP}/cleanup-jobs.txt"
|
||||
gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \
|
||||
--jq '.artifacts[] | select(.name | startswith("MobileGL-trace-fixture-") or startswith("MobileGL-android-retrace-result-") or startswith("trace-fixture-") or startswith("retrace-result-")) | [.id, .name] | @tsv' \
|
||||
> "${RUNNER_TEMP}/cleanup-artifacts.txt"
|
||||
declare -A failed_cases=()
|
||||
while IFS= read -r job_name; do
|
||||
case_name="${job_name#retrace (*, }"
|
||||
case_name="${case_name%)}"
|
||||
failed_cases["${case_name}"]=1
|
||||
done < "${RUNNER_TEMP}/cleanup-jobs.txt"
|
||||
|
||||
if ((${#failed_cases[@]})); then
|
||||
echo "Retaining fixtures and results for failed retrace case(s):"
|
||||
printf ' %s\n' "${!failed_cases[@]}"
|
||||
else
|
||||
echo "All retrace jobs succeeded; nothing needs to be retained."
|
||||
fi
|
||||
|
||||
deleted=0
|
||||
retained=0
|
||||
while IFS=$'\t' read -r artifact_id artifact_name; do
|
||||
keep=0
|
||||
if [[ "${artifact_name}" == MobileGL-trace-fixture-* ]]; then
|
||||
case_name="${artifact_name#MobileGL-trace-fixture-}"
|
||||
if [[ -v "failed_cases[${case_name}]" ]]; then
|
||||
keep=1
|
||||
fi
|
||||
elif [[ "${artifact_name}" == MobileGL-android-retrace-result-* ]]; then
|
||||
# The result artifact carries mobilegl.log, retrace.log, logcat,
|
||||
# the emulator log and the actual/diff images - the only record of
|
||||
# why a retrace failed. Its name ends in -<backend>-<case>, so a
|
||||
# suffix match on the case name keeps both backends' results for a
|
||||
# case that failed on either of them, which is what a comparison
|
||||
# needs. The match is anchored at the end, so a case name that is a
|
||||
# prefix of a longer one does not retain the longer one's results.
|
||||
for case_name in "${!failed_cases[@]}"; do
|
||||
if [[ "${artifact_name}" == *-"${case_name}" ]]; then
|
||||
keep=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if ((keep)); then
|
||||
echo "Retaining ${artifact_name} (${artifact_id}) for failed retrace."
|
||||
((retained += 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Deleting ${artifact_name} (${artifact_id})"
|
||||
gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}"
|
||||
((deleted += 1))
|
||||
done < "${RUNNER_TEMP}/cleanup-artifacts.txt"
|
||||
|
||||
echo "Deleted ${deleted} intermediate Android artifact(s); retained ${retained} failed-retrace fixture(s)."
|
||||
python3 android-plugin/retrace_artifact_cleanup.py \
|
||||
--downloads retrace-artifacts/monolith-control \
|
||||
--downloads retrace-artifacts/inproc-acceptance \
|
||||
--downloads retrace-artifacts/spawn-acceptance
|
||||
|
||||
+551
-522
File diff suppressed because it is too large
Load Diff
@@ -29,3 +29,5 @@ __pycache__/
|
||||
*.py[cod]
|
||||
/.gradle
|
||||
/.trace-work/
|
||||
# The restored trace fixture cache entry (.github/scripts/trace-fixture-bundle.sh)
|
||||
/trace-fixture-bundle/
|
||||
|
||||
+16
-14
@@ -14,20 +14,20 @@ option(MOBILEGL_ENABLE_TRACY "Enable tracy for profiling"
|
||||
option(MOBILEGL_BUILD_TRACE_REPLAY "Build desktop apitrace replay runner" OFF)
|
||||
option(MOBILEGL_TRACE_ANGLE_VARIANTS "Enable signed trace-APK ANGLE variant loading" OFF)
|
||||
option(MOBILEGL_IOS "Build MobileGL for iOS instead of macOS when APPLE is set" OFF)
|
||||
# The disaggregated (two-process) shape. OFF is the shipping default and OFF
|
||||
# must stay byte-comparable to a tree without MG_Remote at all: nothing under
|
||||
# MobileGL/MG_Remote/ is compiled, no include path is added, and no library is
|
||||
# linked, so `nm --defined-only libMobileGL.so | grep -i MG_Remote` is empty.
|
||||
# That emptiness is one of the two byte-level equalities the plan's validation
|
||||
# gates keep (section 10.3).
|
||||
option(MOBILEGL_BUILD_DISAGGREGATED "Build the MG_Remote transport layer (two-process shape)" OFF)
|
||||
# The CI / debugging shape (ARCHITECTURE.md §16): both roles in ONE process, talking over the
|
||||
# same SEG_CMD ring and the same G3 codec a spawned server would use. It IMPLIES
|
||||
# MOBILEGL_BUILD_DISAGGREGATED (below) and additionally admits the role-isolation shims that
|
||||
# only make sense when the two roles share an address space. It is a SUPERSET, never a
|
||||
# substitute: MOBILEGL_TRANSPORT=inproc is what selects the shape at run time, and this option
|
||||
# only decides whether the shims are compiled in.
|
||||
option(MOBILEGL_BUILD_DISAGGREGATED_INPROC "Compile the in-process (one-process, two-role) split shims; implies MOBILEGL_BUILD_DISAGGREGATED" OFF)
|
||||
# The disaggregated (two-process) shape, ON by default: the split is the development default and
|
||||
# the shipping library configures with -DMOBILEGL_BUILD_DISAGGREGATED=OFF. That OFF build must
|
||||
# stay byte-comparable to a tree without MG_Remote at all: nothing under MobileGL/MG_Remote/ is
|
||||
# compiled, no include path is added, and no library is linked, so
|
||||
# `nm --defined-only libMobileGL.so | grep -i MG_Remote` is empty. That emptiness is one of the
|
||||
# two byte-level equalities the plan's validation gates keep (section 10.3).
|
||||
option(MOBILEGL_BUILD_DISAGGREGATED "Build the MG_Remote transport layer (two-process shape)" ON)
|
||||
# The in-process split shape (ARCHITECTURE.md §16), ON by default with the option above: both
|
||||
# roles in ONE process, talking over the same SEG_CMD ring and the same G3 codec a spawned server
|
||||
# would use. It IMPLIES MOBILEGL_BUILD_DISAGGREGATED (below) and additionally admits the
|
||||
# role-isolation shims that only make sense when the two roles share an address space. It is a
|
||||
# SUPERSET, never a substitute: MOBILEGL_TRANSPORT=inproc is what selects the shape at run time,
|
||||
# and this option only decides whether the shims are compiled in.
|
||||
option(MOBILEGL_BUILD_DISAGGREGATED_INPROC "Compile the in-process (one-process, two-role) split shims; implies MOBILEGL_BUILD_DISAGGREGATED" ON)
|
||||
option(MOBILEGL_BUILD_SERVER_SPIKE "Build the P0 spike-A MobileGLServer delivery-chain executable (Android only)" OFF)
|
||||
option(MOBILEGL_BUILD_EXTMEM_PROBE "Package tools/spikes/extmem_probe as libMobileGLExtmemProbe.so for the trace APK's exec hook (P11 A3, Android only)" OFF)
|
||||
option(MOBILEGL_EXTMEM_PROBE_AS_SERVER "Spike only (P11 B2 step 1): also package the probe as libMobileGLServer.so, the real server as libMobileGLServerReal.so" OFF)
|
||||
@@ -444,6 +444,7 @@ set(SOURCE_FILES
|
||||
MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/LegalizeResourceArrayIndexPass.cpp
|
||||
MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenAtomicCounterBlockPass.cpp
|
||||
MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DemotePointSizePass.cpp
|
||||
MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateClipDistancePass.cpp
|
||||
|
||||
MobileGL/MG_Util/BackendLoaders/OpenGL/Loader.cpp
|
||||
MobileGL/MG_Util/BackendLoaders/Vulkan/Loader.cpp
|
||||
@@ -451,6 +452,7 @@ set(SOURCE_FILES
|
||||
MobileGL/MG_Util/SelfTest/DriverBugProbes.cpp
|
||||
MobileGL/MG_Util/SelfTest/PersistentBufferOrderingProbe.cpp
|
||||
MobileGL/MG_Util/SelfTest/DriverPost.cpp
|
||||
MobileGL/MG_Util/SelfTest/DriverPostClipDistance.cpp
|
||||
MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.cpp
|
||||
MobileGL/MG_Util/SelfTest/PrimitivesGeneratedNoXfbProbe.cpp
|
||||
|
||||
|
||||
+27
-10
@@ -214,16 +214,16 @@ namespace MobileGL {
|
||||
Int MaxComputeImageUniforms = 8;
|
||||
Int MaxDrawBuffers = 8;
|
||||
Int MaxColorAttachments = 8;
|
||||
// GL_MAX_CLIP_DISTANCES. Zero is a legal answer here, not a placeholder, and a
|
||||
// backend that cannot host a clip distance MUST report it: advertising eight the
|
||||
// backend will refuse does not make gl_ClipDistance work, it only moves the failure
|
||||
// from an honest "unsupported" at query time to a backend shader-compile error the
|
||||
// frontend never surfaces, after which every draw with that program silently renders
|
||||
// nothing. DirectGLES fills it from GL_EXT_clip_cull_distance, DirectVulkan from the
|
||||
// shaderClipDistance device feature. The DEFAULT stays at the GL 4.3 core minimum
|
||||
// because it describes the no-backend case (standalone shader compiles, unit tests),
|
||||
// where there is no device to be honest about and BuildTBuiltInResource still has to
|
||||
// hand glslang a workable gl_MaxClipDistances.
|
||||
// GL_MAX_CLIP_DISTANCES. What the backend can actually CLIP with, natively or through
|
||||
// EmulatesClipDistance below - never a number the backend would then refuse:
|
||||
// advertising eight that no stage can honor only moves the failure from query time to
|
||||
// a backend shader-compile error the frontend never surfaces. DirectGLES fills it from
|
||||
// GL_EXT_clip_cull_distance, DirectVulkan from the shaderClipDistance device feature,
|
||||
// and both answer 8 when they emulate. Zero stays a legal answer for a backend that
|
||||
// neither hosts nor emulates the built-in (MOBILEGL_CLIP_DISTANCE_EMULATION=0 on such
|
||||
// a device). The DEFAULT is the GL 4.3 core minimum because it describes the
|
||||
// no-backend case (standalone shader compiles, unit tests), where
|
||||
// BuildTBuiltInResource still has to hand glslang a workable gl_MaxClipDistances.
|
||||
Int MaxClipDistances = 8;
|
||||
// GL_MAX_CULL_DISTANCES and GL_MAX_COMBINED_CLIP_AND_CULL_DISTANCES, under exactly
|
||||
// the contract stated for MaxClipDistances above: ZERO IS A LEGAL ANSWER and a
|
||||
@@ -349,6 +349,23 @@ namespace MobileGL {
|
||||
Bool SupportsTessellationPointSize = true;
|
||||
// The geometry-stage twin (OpCapability GeometryPointSize).
|
||||
Bool SupportsGeometryPointSize = true;
|
||||
// The backend cannot clip with gl_ClipDistance natively - no
|
||||
// GL_EXT_clip_cull_distance, an extension the POST clip probe caught not clipping or
|
||||
// not honoring the per-distance enables, or shaderClipDistance == VK_FALSE - so
|
||||
// ProgramSpirvTask lowers the built-in to varyings plus a fragment discard
|
||||
// (ShaderCompiler::EmulateClipDistanceForProgram) and the source pipeline declares the
|
||||
// reserved enables uniform the draw path feeds. MOBILEGL_CLIP_DISTANCE_EMULATION
|
||||
// overrides the detection in either direction at backend init. Defaults FALSE: true
|
||||
// ARMS a rewrite, so the no-backend answer is the one that leaves modules untouched.
|
||||
Bool EmulatesClipDistance = false;
|
||||
// The backend HOSTS gl_ClipDistance but has no per-distance enable state: Vulkan
|
||||
// clips by every distance a shader writes, so GL_CLIP_DISTANCEi would mean nothing.
|
||||
// The producing stage then writes `enabled(i) ? d[i] : 1.0` back into the built-in
|
||||
// (EmulateClipDistanceOptions::nativeBuiltin) from the same reserved enables uniform
|
||||
// the emulation uses. DirectVulkan sets it whenever it clips natively; DirectGLES
|
||||
// never does - ES drivers it keeps native passed the POST probe, which checks the
|
||||
// driver honors GL_CLIP_DISTANCEi_EXT itself. Exclusive with EmulatesClipDistance.
|
||||
Bool MasksNativeClipDistances = false;
|
||||
Uint64 MaxShaderStorageBlockSize = 128 * 1024 * 1024;
|
||||
Uint32 SubgroupSize = 0;
|
||||
Uint32 SubgroupSupportedStages = 0;
|
||||
|
||||
+9
-1
@@ -14,7 +14,7 @@ namespace MobileGL::MG_Config {
|
||||
inline const String ProjectName = "MobileGL";
|
||||
inline const String CoreName = "MobileGL Core";
|
||||
inline const String CoreVendor = "MobileGL-Dev (BZLZHH, Swung0x48, Tungsten)";
|
||||
inline const Version CoreVersion = {26, 10, 0, "-dev", VersionType::Development};
|
||||
inline const Version CoreVersion = {26, 11, 0, "-dev", VersionType::Development};
|
||||
inline const VersionStringFormatAttrib DefaultVersionStringFormatAttrib = {2, 2, 0, true, true};
|
||||
inline const Uint64 CacheVersion = 0;
|
||||
|
||||
@@ -237,6 +237,14 @@ namespace MobileGL::MG_Config {
|
||||
// negative control). Cross-backend by design: the demotion runs in the shared
|
||||
// phase-B chain, so one switch covers both. See DemotePointSizePass.
|
||||
QuirkOverride PointSizeDemotion = QuirkOverride::Auto;
|
||||
// MOBILEGL_CLIP_DISTANCE_EMULATION: lower gl_ClipDistance to ordinary varyings plus a
|
||||
// fragment discard (ShaderCompiler::EmulateClipDistanceForProgram) on a backend that
|
||||
// cannot clip natively - no GL_EXT_clip_cull_distance or a driver the POST clip probe
|
||||
// caught misbehaving (DirectGLES), shaderClipDistance == VK_FALSE (DirectVulkan).
|
||||
// Auto arms it exactly there. ForceOn exercises it on a healthy driver (the pinned
|
||||
// integration lane); ForceOff restores the native-or-nothing answer, which reports
|
||||
// GL_MAX_CLIP_DISTANCES = 0 on a device without native support.
|
||||
QuirkOverride ClipDistanceEmulation = QuirkOverride::Auto;
|
||||
// MOBILEGL_COHERENT_AS_FLUSH: app-compat for engines (e.g. Flywheel) that write
|
||||
// GPU-read data through persistent GL_MAP_FLUSH_EXPLICIT_BIT maps they never
|
||||
// flush. Persistent FLUSH_EXPLICIT map requests are rewritten to coherent
|
||||
|
||||
@@ -402,6 +402,7 @@ namespace MobileGL::MG_ConfigLoader {
|
||||
features.EsprytAtomicSubgroupScratch = QueryEnvQuirkOverride("MOBILEGL_ESPRYT_ATOMIC_SUBGROUP_SCRATCH");
|
||||
features.EsprytInvalidImagePlaceholder = QueryEnvQuirkOverride("MOBILEGL_ESPRYT_INVALID_IMAGE_PLACEHOLDER");
|
||||
features.PointSizeDemotion = QueryEnvQuirkOverride("MOBILEGL_POINT_SIZE_DEMOTION");
|
||||
features.ClipDistanceEmulation = QueryEnvQuirkOverride("MOBILEGL_CLIP_DISTANCE_EMULATION");
|
||||
features.CoherentAsFlush = QueryEnvFlag("MOBILEGL_COHERENT_AS_FLUSH");
|
||||
features.EsprytDisableUboRing = QueryEnvFlag("MOBILEGL_ESPRYT_DISABLE_UBO_RING");
|
||||
features.EsprytDisableUnpackRing = QueryEnvFlag("MOBILEGL_ESPRYT_DISABLE_UNPACK_RING");
|
||||
|
||||
@@ -1996,12 +1996,18 @@ namespace MobileGL::MG_Backend::DirectGLES {
|
||||
}
|
||||
m_dynamicParameters.MaxDrawBuffers = m_GLESCapabilities.MaxDrawBuffers;
|
||||
m_dynamicParameters.MaxColorAttachments = m_GLESCapabilities.MaxColorAttachments;
|
||||
m_dynamicParameters.MaxClipDistances = m_GLESCapabilities.MaxClipDistances;
|
||||
// Eight when emulated - the GL 3.0+ minimum, and what EmulateClipDistancePass carries -
|
||||
// else the driver's own number (0 without the extension and with the emulation forced
|
||||
// off: the native-or-nothing answer).
|
||||
m_dynamicParameters.EmulatesClipDistance = m_GLESCapabilities.EmulatesClipDistance;
|
||||
m_dynamicParameters.MaxClipDistances =
|
||||
m_GLESCapabilities.EmulatesClipDistance ? 8 : m_GLESCapabilities.MaxClipDistances;
|
||||
// The loader already gated both on GL_EXT_clip_cull_distance and left 0 without it, which
|
||||
// is the answer that keeps glslang from accepting a gl_CullDistance the ESSL compiler
|
||||
// would reject.
|
||||
m_dynamicParameters.MaxCullDistances = m_GLESCapabilities.MaxCullDistances;
|
||||
m_dynamicParameters.MaxCombinedClipAndCullDistances = m_GLESCapabilities.MaxCombinedClipAndCullDistances;
|
||||
m_dynamicParameters.MaxCombinedClipAndCullDistances =
|
||||
std::max(m_GLESCapabilities.MaxCombinedClipAndCullDistances, m_dynamicParameters.MaxClipDistances);
|
||||
m_dynamicParameters.MaxViewports = m_GLESCapabilities.MaxViewports;
|
||||
// Whatever the driver said about which vertex supplies gl_Layer, and GL_UNDEFINED_VERTEX
|
||||
// for gl_ViewportIndex on every driver without GL_OES_viewport_array - which is both test
|
||||
|
||||
@@ -514,7 +514,7 @@ namespace MobileGL::MG_Backend::DirectGLES {
|
||||
}
|
||||
}
|
||||
|
||||
if (tailSpanDirty && g_GLESCapabilities.SupportsClipDistance) {
|
||||
if (tailSpanDirty && g_GLESCapabilities.SupportsClipDistance && !g_GLESCapabilities.EmulatesClipDistance) {
|
||||
// gl_ClipDistance clipping is per-distance enable state in GL, and ES reaches it
|
||||
// only through GL_EXT_clip_cull_distance. That extension reuses the desktop enum
|
||||
// values for CLIP_DISTANCE0_EXT..7_EXT, but the token is absent from the ES
|
||||
|
||||
@@ -1117,6 +1117,28 @@ namespace MobileGL::MG_Backend::DirectVulkan {
|
||||
}
|
||||
}
|
||||
|
||||
// Without the shaderClipDistance feature the built-in is emulated instead
|
||||
// (EmulateClipDistancePass: varyings plus a fragment discard, which needs nothing from the
|
||||
// device), so the frontend can still answer the GL 3.0+ minimum of eight. Every Mali device
|
||||
// measured so far reports shaderClipDistance = VK_FALSE. MOBILEGL_CLIP_DISTANCE_EMULATION=1
|
||||
// arms it on a device that has the feature (the pinned integration lane); =0 restores the
|
||||
// plain zero.
|
||||
static Bool DecideClipDistanceEmulation(const Bool deviceHasClipDistance) {
|
||||
switch (MG_Config::Features.ClipDistanceEmulation) {
|
||||
case MG_Config::QuirkOverride::ForceOn:
|
||||
MGLOG_I("DirectVulkan: MOBILEGL_CLIP_DISTANCE_EMULATION=1 - emulating gl_ClipDistance even though "
|
||||
"the device %s shaderClipDistance",
|
||||
deviceHasClipDistance ? "has" : "lacks");
|
||||
return true;
|
||||
case MG_Config::QuirkOverride::ForceOff:
|
||||
MGLOG_I("DirectVulkan: MOBILEGL_CLIP_DISTANCE_EMULATION=0 - no gl_ClipDistance emulation");
|
||||
return false;
|
||||
case MG_Config::QuirkOverride::Auto:
|
||||
default:
|
||||
return !deviceHasClipDistance;
|
||||
}
|
||||
}
|
||||
|
||||
void BackendObject_DirectVulkan::UpdateDynamicBackendParameters() {
|
||||
const auto mapShaderStages = [](Uint32 vkStages) {
|
||||
Uint32 glStages = 0;
|
||||
@@ -1343,8 +1365,21 @@ namespace MobileGL::MG_Backend::DirectVulkan {
|
||||
// by every device, but declaring ClipDistance in a module needs the shaderClipDistance
|
||||
// FEATURE, which VulkanRenderer enables exactly where the physical device has it. Without
|
||||
// it the limit describes a capacity no shader may use, so report none.
|
||||
const Bool emulateClipDistance = DecideClipDistanceEmulation(m_vulkanCaps.SupportsShaderClipDistance);
|
||||
m_dynamicParameters.EmulatesClipDistance = emulateClipDistance;
|
||||
// Hosted natively, but Vulkan has no per-distance enable state: apply GL_CLIP_DISTANCEi
|
||||
// in the producing stage instead (EmulateClipDistanceOptions::nativeBuiltin).
|
||||
m_dynamicParameters.MasksNativeClipDistances = !emulateClipDistance && m_vulkanCaps.SupportsShaderClipDistance;
|
||||
m_dynamicParameters.MaxClipDistances =
|
||||
m_vulkanCaps.SupportsShaderClipDistance ? std::max(m_vulkanCaps.MaxClipDistances, 0) : 0;
|
||||
emulateClipDistance
|
||||
? 8
|
||||
: (m_vulkanCaps.SupportsShaderClipDistance ? std::max(m_vulkanCaps.MaxClipDistances, 0) : 0);
|
||||
MGLOG_I("DirectVulkan: gl_ClipDistance %s (GL_MAX_CLIP_DISTANCES = %d)",
|
||||
emulateClipDistance
|
||||
? "EMULATED (varyings + fragment discard)"
|
||||
: (m_dynamicParameters.MasksNativeClipDistances ? "native, GL_CLIP_DISTANCEi applied in the shader"
|
||||
: "unsupported"),
|
||||
m_dynamicParameters.MaxClipDistances);
|
||||
// The cull pair, gated on its own feature. shaderCullDistance is separate from
|
||||
// shaderClipDistance and VulkanRenderer enables it independently, so it gets its own
|
||||
// gate rather than riding on the clip one.
|
||||
@@ -1357,7 +1392,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
|
||||
(m_vulkanCaps.SupportsShaderClipDistance || m_vulkanCaps.SupportsShaderCullDistance)
|
||||
? std::max({m_vulkanCaps.MaxCombinedClipAndCullDistances, m_dynamicParameters.MaxClipDistances,
|
||||
m_dynamicParameters.MaxCullDistances})
|
||||
: 0;
|
||||
: m_dynamicParameters.MaxClipDistances;
|
||||
m_dynamicParameters.MaxViewports = m_vulkanCaps.MaxViewports;
|
||||
// Assigned explicitly rather than left to the struct's defaults, like every other
|
||||
// parameter here, so a second fill cannot inherit a stale value. GL_UNDEFINED_VERTEX is
|
||||
|
||||
@@ -132,6 +132,9 @@ namespace MobileGL::MG_Backend::DirectVulkan {
|
||||
private:
|
||||
VkTextureManager& m_manager;
|
||||
};
|
||||
// True inside a PendingClearHold: the image being resolved is an attachment of the pass
|
||||
// being prepared, which takes pending clears as its load op.
|
||||
Bool PendingClearsHeld() const { return m_pendingClearHold != 0; }
|
||||
// A whole-subresource clear that was recorded but has not run yet (VulkanRenderer::DeferWireClear).
|
||||
struct WirePendingClear {
|
||||
Uint32 level = 0, layer = 0, layers = 1;
|
||||
|
||||
@@ -620,7 +620,35 @@ namespace MobileGL::MG_Backend::DirectVulkan {
|
||||
Uint32 level = 0, layer = 0, layers = 1;
|
||||
};
|
||||
Vector<WireFoldClear> m_wireFoldClears;
|
||||
// Attachments of the pass being prepared whose content is undefined: a default-framebuffer
|
||||
// image first used since its present. Their load op becomes DONT_CARE (WireClearVariant).
|
||||
Vector<Uint32> m_wireDiscardLoads;
|
||||
static Bool WireImageCanDeferClear(const WireImage& image);
|
||||
// A whole-image clear of the default framebuffer, held for the next pass that draws into
|
||||
// the window (its load op) the way WirePendingClear holds a texture's. Any other use of the
|
||||
// window images, and the present, run it first (MaterializeDefaultWirePendingClear).
|
||||
struct DefaultWirePendingClear {
|
||||
VkSwapchainKHR swapchain = VK_NULL_HANDLE;
|
||||
Uint32 swapchainImageIndex = 0;
|
||||
Bool color = false;
|
||||
VkImageAspectFlags depthStencil = 0;
|
||||
ClearAttachmentPayload colorPayload{};
|
||||
// colorPayload against the window format and GL_FRAMEBUFFER_SRGB as they were at the
|
||||
// glClear (PreCompensateSrgbClearColor): the standalone clear a present or another
|
||||
// use of the window records later must not read that state again, after the verb.
|
||||
ClearAttachmentPayload colorPayloadCompensated{};
|
||||
ClearAttachmentPayload depthStencilPayload{};
|
||||
Bool Pending() const { return color || depthStencil != 0; }
|
||||
};
|
||||
DefaultWirePendingClear m_defaultWirePendingClear;
|
||||
GLbitfield DeferDefaultWireClear(const MG_Pipe::MGPFramebufferState& fbo, const ClearAttachmentPayload& payload,
|
||||
GLint drawbuffer);
|
||||
void MaterializeDefaultWirePendingClear();
|
||||
void FoldDefaultWirePendingClear(RenderPassEntry& begun, Uint64& variantKey, VkClearValue* merged);
|
||||
// Records a clear of `area` of one wire image in a single-attachment pass of its own.
|
||||
// `colorPreCompensated`: the payload already went through PreCompensateSrgbClearColor.
|
||||
void RecordWireImageClear(WireImage& image, VkImageAspectFlags aspect, const ClearAttachmentPayload& payload,
|
||||
const VkRect2D& area, Bool textureSurface, Bool colorPreCompensated = false);
|
||||
static void NoteWirePendingClear(const WireImage& image, VkImageAspectFlags aspects, const VkClearValue& value);
|
||||
VkRenderPass WireClearVariant(RenderPassEntry& begun, Uint64 variantKey);
|
||||
Bool BeginWireDrawPassFolding(FrameContext::FrameData& frame, RenderPassEntry& begun, Uint64 variantKey,
|
||||
|
||||
@@ -16,6 +16,15 @@ namespace MobileGL::MG_Backend::DirectVulkan {
|
||||
// Present has its own submit path. Tag its last wire pass before that
|
||||
// submission, rather than retaining it until a draw in the next frame.
|
||||
RetireWireDrawPass();
|
||||
// A window color clear no pass drew after still has to reach the presented image. Its
|
||||
// depth/stencil half is dropped: a swap leaves those undefined (see the content flags
|
||||
// reset below). A frame that cannot be presented drops the whole clear.
|
||||
if (m_swapchainObject.GetHandle() == VK_NULL_HANDLE || m_presentSuspended) {
|
||||
m_defaultWirePendingClear = {};
|
||||
} else {
|
||||
m_defaultWirePendingClear.depthStencil = 0;
|
||||
MaterializeDefaultWirePendingClear();
|
||||
}
|
||||
#endif
|
||||
if (m_swapchainObject.GetHandle() == VK_NULL_HANDLE || m_presentSuspended) {
|
||||
// No usable swapchain: the window was zero-area at initialization, or
|
||||
|
||||
@@ -9,6 +9,14 @@
|
||||
#include "VulkanRendererInternal.h"
|
||||
|
||||
namespace MobileGL::MG_Backend::DirectVulkan {
|
||||
namespace {
|
||||
// A wire pass variant key holds two clear bits per attachment from bit 0 and, from this
|
||||
// bit on, one bit per attachment whose content is undefined (WireClearVariant).
|
||||
constexpr Uint32 kWireDiscardBit = 32;
|
||||
static_assert(2 * (MG_Pipe::kMGPipeMaxColorAttachments + 1) <= kWireDiscardBit,
|
||||
"the clear bits of a wire pass variant key overlap its discard bits");
|
||||
} // namespace
|
||||
|
||||
// GL_EXT_fragment_shading_rate's pipeline rate and combiner ops for the draw about to be
|
||||
// recorded. The packed rate is log2(width) << 2 | log2(height), which is exactly the extent
|
||||
// VK_KHR_fragment_shading_rate takes, and the combiner op indices follow
|
||||
@@ -445,6 +453,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
|
||||
WireDrawScratch& scratch, Bool passContinues, Bool& continuing) {
|
||||
continuing = true;
|
||||
m_wireFoldClears.clear();
|
||||
m_wireDiscardLoads.clear();
|
||||
if (passContinues) {
|
||||
auto& store = MG_Record::ServerStagedTexture();
|
||||
if (!m_wirePassMarksValid || m_wirePassMarksClearGeneration != store.GpuDirtyClearGeneration()) {
|
||||
@@ -536,6 +545,13 @@ namespace MobileGL::MG_Backend::DirectVulkan {
|
||||
if (!image.isDefault && surface.Kind == MG_Pipe::kMGPipeSurfaceKindTexture)
|
||||
scratch.gpuWriteMarks.push_back({image.storage, image.level, image.layer, image.layers});
|
||||
if (image.isDefault) {
|
||||
// A present leaves the default framebuffer undefined (VulkanRenderer::Present):
|
||||
// the first pass after it has nothing to load.
|
||||
const Bool defined =
|
||||
aspect == VK_IMAGE_ASPECT_COLOR_BIT
|
||||
? m_swapchainObject.IsImageContentDefined(image.swapchainImageIndex)
|
||||
: m_swapchainObject.IsDepthStencilContentDefined(image.swapchainImageIndex);
|
||||
if (!defined) m_wireDiscardLoads.push_back(static_cast<Uint32>(attachments.size() - 1));
|
||||
if (aspect == VK_IMAGE_ASPECT_COLOR_BIT)
|
||||
m_swapchainObject.SetImageContentDefined(image.swapchainImageIndex, true);
|
||||
else
|
||||
@@ -1361,9 +1377,18 @@ namespace MobileGL::MG_Backend::DirectVulkan {
|
||||
if (entry.first == variantKey) return entry.second;
|
||||
const Uint32 attachmentCount = static_cast<Uint32>(begun.wireAttachments.size());
|
||||
auto descriptions = begun.wireAttachments;
|
||||
// Bits 2a and 2a+1 clear attachment a (color or depth, and stencil); bit kWireDiscardBit + a
|
||||
// marks its content undefined, so whatever is not cleared is not loaded either.
|
||||
for (Uint32 a = 0; a < attachmentCount; ++a) {
|
||||
if (variantKey >> (2 * a) & 1) descriptions[a].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
|
||||
if (variantKey >> (2 * a + 1) & 1) descriptions[a].stencilLoadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
|
||||
const Bool discard = variantKey >> (kWireDiscardBit + a) & 1;
|
||||
if (variantKey >> (2 * a) & 1)
|
||||
descriptions[a].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
|
||||
else if (discard)
|
||||
descriptions[a].loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
|
||||
if (variantKey >> (2 * a + 1) & 1)
|
||||
descriptions[a].stencilLoadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
|
||||
else if (discard)
|
||||
descriptions[a].stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
|
||||
}
|
||||
VkSubpassDescription subpass{};
|
||||
subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS;
|
||||
@@ -1382,6 +1407,142 @@ namespace MobileGL::MG_Backend::DirectVulkan {
|
||||
return variant;
|
||||
}
|
||||
|
||||
// The default framebuffer's half of the deferred clears: a whole-window glClear waits for the
|
||||
// next pass that draws into the window, which loads it with LOAD_OP_CLEAR
|
||||
// (BeginWireDrawPassFolding) instead of storing a cleared window and loading it back. A pass
|
||||
// open on the window ends first, since the in-pass clear does not take the window
|
||||
// (MagmaInPassClearDefault); a present drops the depth/stencil half (Present). Returns the
|
||||
// buffers it did not take.
|
||||
GLbitfield VulkanRenderer::DeferDefaultWireClear(const MG_Pipe::MGPFramebufferState& fbo,
|
||||
const ClearAttachmentPayload& payload, GLint drawbuffer) {
|
||||
GLbitfield remaining = payload.mask & (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
|
||||
if (!remaining || !fbo.IsDefault || drawbuffer >= 0 || m_wirePreparationDepth != 0) return remaining;
|
||||
if (GLScissorTestClips()) return remaining;
|
||||
GLbitfield handled = 0;
|
||||
Bool color = false;
|
||||
VkImageAspectFlags depthStencil = 0;
|
||||
if (remaining & GL_COLOR_BUFFER_BIT) {
|
||||
const Int32 slot = fbo.DrawBuffers[0];
|
||||
const auto mask = MG_Pipe::gPipeInputs->GetColorMaskIndexed(0);
|
||||
const Bool none = slot < 0 || (!mask.r() && !mask.g() && !mask.b() && !mask.a());
|
||||
if (none)
|
||||
handled |= GL_COLOR_BUFFER_BIT;
|
||||
else if (mask.r() && mask.g() && mask.b() && mask.a() &&
|
||||
payload.colorEncoding == ClearColorEncoding::Float) {
|
||||
handled |= GL_COLOR_BUFFER_BIT;
|
||||
color = true;
|
||||
}
|
||||
}
|
||||
// A window without a depth/stencil buffer clears nothing there.
|
||||
const VkFormat depthStencilFormat = m_swapchainObject.GetDepthStencilFormat();
|
||||
const Bool hasDepth = depthStencilFormat != VK_FORMAT_UNDEFINED;
|
||||
const Bool packed = vkuFormatIsDepthAndStencil(depthStencilFormat);
|
||||
if (remaining & GL_DEPTH_BUFFER_BIT) {
|
||||
handled |= GL_DEPTH_BUFFER_BIT;
|
||||
if (hasDepth && MG_Pipe::gPipeInputs->GetDepthMask()) depthStencil |= VK_IMAGE_ASPECT_DEPTH_BIT;
|
||||
}
|
||||
if (remaining & GL_STENCIL_BUFFER_BIT) {
|
||||
const Uint32 mask = MG_Pipe::gPipeInputs->GetStencilState(StencilFace::Front).WriteMask & 255u;
|
||||
if (mask == 0 || !packed)
|
||||
handled |= GL_STENCIL_BUFFER_BIT;
|
||||
else if (mask == 255u) {
|
||||
handled |= GL_STENCIL_BUFFER_BIT;
|
||||
depthStencil |= VK_IMAGE_ASPECT_STENCIL_BIT;
|
||||
}
|
||||
}
|
||||
if (!color && depthStencil == 0) return remaining & ~handled;
|
||||
if (VkRenderPassManager::GetActiveRenderPass() != nullptr && m_wireDrawPass && m_wireDrawPassKey.isDefault)
|
||||
RetireWireDrawPass();
|
||||
// The index the next drawing pass on the window will address (WireDraw.cpp's pass key).
|
||||
const Uint32 index = DefaultFramebufferWriteIndex(true);
|
||||
auto& pending = m_defaultWirePendingClear;
|
||||
if (pending.Pending() &&
|
||||
(pending.swapchain != m_swapchainObject.GetHandle() || pending.swapchainImageIndex != index))
|
||||
MaterializeDefaultWirePendingClear();
|
||||
pending.swapchain = m_swapchainObject.GetHandle();
|
||||
pending.swapchainImageIndex = index;
|
||||
if (color) {
|
||||
pending.color = true;
|
||||
pending.colorPayload = payload;
|
||||
pending.colorPayloadCompensated = payload;
|
||||
PreCompensateSrgbClearColor(pending.colorPayloadCompensated, m_swapchainObject.GetSurfaceFormat().format);
|
||||
m_swapchainObject.SetImageContentDefined(index, true);
|
||||
}
|
||||
if (depthStencil & VK_IMAGE_ASPECT_DEPTH_BIT) pending.depthStencilPayload.depth = payload.depth;
|
||||
if (depthStencil & VK_IMAGE_ASPECT_STENCIL_BIT) pending.depthStencilPayload.stencil = payload.stencil;
|
||||
if (depthStencil) {
|
||||
pending.depthStencil |= depthStencil;
|
||||
m_swapchainObject.SetDepthStencilContentDefined(index, true);
|
||||
}
|
||||
return remaining & ~handled;
|
||||
}
|
||||
|
||||
void VulkanRenderer::MaterializeDefaultWirePendingClear() {
|
||||
if (!m_defaultWirePendingClear.Pending()) return;
|
||||
const DefaultWirePendingClear pending = m_defaultWirePendingClear;
|
||||
m_defaultWirePendingClear = {};
|
||||
// A recreated swapchain took the images the clear was for.
|
||||
if (pending.swapchain != m_swapchainObject.GetHandle() ||
|
||||
pending.swapchainImageIndex >= m_swapchainObject.GetImageCount())
|
||||
return;
|
||||
const VkExtent2D extent = m_swapchainObject.GetExtent();
|
||||
const VkRect2D area{{0, 0}, extent};
|
||||
WireImage image{};
|
||||
image.isDefault = true;
|
||||
image.extent = extent;
|
||||
image.swapchainImageIndex = pending.swapchainImageIndex;
|
||||
if (pending.color) {
|
||||
image.aspect = VK_IMAGE_ASPECT_COLOR_BIT;
|
||||
image.image = m_swapchainObject.GetImage(pending.swapchainImageIndex);
|
||||
image.format = m_swapchainObject.GetSurfaceFormat().format;
|
||||
image.layout = m_swapchainObject.GetImageLayout(pending.swapchainImageIndex);
|
||||
RecordWireImageClear(image, VK_IMAGE_ASPECT_COLOR_BIT, pending.colorPayloadCompensated, area, false,
|
||||
/*colorPreCompensated=*/true);
|
||||
}
|
||||
if (pending.depthStencil && m_swapchainObject.GetDepthStencilImage(pending.swapchainImageIndex)) {
|
||||
image.aspect = pending.depthStencil;
|
||||
image.image = m_swapchainObject.GetDepthStencilImage(pending.swapchainImageIndex);
|
||||
image.format = m_swapchainObject.GetDepthStencilFormat();
|
||||
image.layout = m_swapchainObject.GetDepthStencilImageLayout(pending.swapchainImageIndex);
|
||||
RecordWireImageClear(image, pending.depthStencil, pending.depthStencilPayload, area, false);
|
||||
}
|
||||
}
|
||||
|
||||
// A pending window clear (DeferDefaultWireClear) becomes the load op of the window pass that
|
||||
// begins: its attachments are matched by image, since a continued pass key is not walked again.
|
||||
// A clear the caller already put in `variantKey` is newer and wins.
|
||||
void VulkanRenderer::FoldDefaultWirePendingClear(RenderPassEntry& begun, Uint64& variantKey, VkClearValue* merged) {
|
||||
auto& pending = m_defaultWirePendingClear;
|
||||
if (!pending.Pending() || !m_wireDrawPassKey.isDefault || m_wireDrawPassKey.swapchain != pending.swapchain ||
|
||||
m_wireDrawPassKey.swapchainImageIndex != pending.swapchainImageIndex)
|
||||
return;
|
||||
const Uint32 attachmentCount =
|
||||
static_cast<Uint32>(std::min<SizeT>(begun.wireAttachments.size(), m_wireDrawPassKey.attachments.size()));
|
||||
const VkImage colorImage = m_swapchainObject.GetImage(pending.swapchainImageIndex);
|
||||
const VkImage depthStencilImage = m_swapchainObject.GetDepthStencilImage(pending.swapchainImageIndex);
|
||||
for (Uint32 a = 0; a < attachmentCount && a < MG_Pipe::kMGPipeMaxColorAttachments + 1; ++a) {
|
||||
const auto& attachment = m_wireDrawPassKey.attachments[a];
|
||||
if (pending.color && attachment.image == colorImage) {
|
||||
if (!(variantKey >> (2 * a) & 1)) {
|
||||
variantKey |= 1ull << (2 * a);
|
||||
merged[a].color = MakeVkClearColorValue(pending.colorPayload, false);
|
||||
}
|
||||
pending.color = false;
|
||||
} else if (pending.depthStencil && attachment.image == depthStencilImage) {
|
||||
if ((pending.depthStencil & VK_IMAGE_ASPECT_DEPTH_BIT) && !(variantKey >> (2 * a) & 1)) {
|
||||
variantKey |= 1ull << (2 * a);
|
||||
merged[a].depthStencil.depth = pending.depthStencilPayload.depth;
|
||||
}
|
||||
if ((pending.depthStencil & VK_IMAGE_ASPECT_STENCIL_BIT) && !(variantKey >> (2 * a + 1) & 1)) {
|
||||
variantKey |= 1ull << (2 * a + 1);
|
||||
merged[a].depthStencil.stencil = pending.depthStencilPayload.stencil;
|
||||
}
|
||||
pending.depthStencil = 0;
|
||||
}
|
||||
}
|
||||
// An aspect the pass does not attach stays pending: none of its draws write it.
|
||||
}
|
||||
|
||||
// P15: EVERY DRAW PASS BEGINS HERE. The pending clears of its attachments
|
||||
// (m_wireFoldClears, gathered by PrepareWireDrawPass) become their load op and leave the
|
||||
// images; a clear the caller already put in `variantKey` (its own glClear, newer than any
|
||||
@@ -1426,6 +1587,10 @@ namespace MobileGL::MG_Backend::DirectVulkan {
|
||||
}
|
||||
}
|
||||
m_wireFoldClears.clear();
|
||||
FoldDefaultWirePendingClear(begun, variantKey, merged);
|
||||
for (const Uint32 a : m_wireDiscardLoads)
|
||||
if (a < attachmentCount) variantKey |= 1ull << (kWireDiscardBit + a);
|
||||
m_wireDiscardLoads.clear();
|
||||
if (variantKey == 0) return VkRenderPassManager::BeginRenderPass(frame.commandBuffer, begun);
|
||||
return VkRenderPassManager::BeginRenderPass(frame.commandBuffer, begun, WireClearVariant(begun, variantKey),
|
||||
merged, attachmentCount);
|
||||
|
||||
@@ -658,6 +658,10 @@ namespace MobileGL::MG_Backend::DirectVulkan {
|
||||
WireImage out{};
|
||||
out.aspect = aspect;
|
||||
if (fbo.IsDefault) {
|
||||
// Only an attachment of the pass being prepared takes a pending window clear as its
|
||||
// load op; every other use runs it first.
|
||||
if (m_defaultWirePendingClear.Pending() && !m_textureManager->PendingClearsHeld())
|
||||
MaterializeDefaultWirePendingClear();
|
||||
out.isDefault = true;
|
||||
out.extent = m_swapchainObject.GetExtent();
|
||||
out.swapchainImageIndex =
|
||||
@@ -1109,6 +1113,87 @@ namespace MobileGL::MG_Backend::DirectVulkan {
|
||||
return remaining & ~handled;
|
||||
}
|
||||
|
||||
void VulkanRenderer::RecordWireImageClear(WireImage& image, VkImageAspectFlags aspect,
|
||||
const ClearAttachmentPayload& payload, const VkRect2D& area,
|
||||
Bool textureSurface, Bool colorPreCompensated) {
|
||||
const Bool color = aspect == VK_IMAGE_ASPECT_COLOR_BIT;
|
||||
const auto layout =
|
||||
color ? VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL : VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
|
||||
TransitionWireImage(image, layout);
|
||||
VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
|
||||
viewInfo.image = image.image;
|
||||
viewInfo.viewType = image.viewType;
|
||||
viewInfo.format = image.format;
|
||||
viewInfo.subresourceRange = {
|
||||
color ? VK_IMAGE_ASPECT_COLOR_BIT
|
||||
: (image.format == VK_FORMAT_D24_UNORM_S8_UINT || image.format == VK_FORMAT_D32_SFLOAT_S8_UINT
|
||||
? VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT
|
||||
: image.aspect),
|
||||
image.level, 1, image.layer, image.layers};
|
||||
VkImageView view = VK_NULL_HANDLE;
|
||||
VK_VERIFY(vkCreateImageView(m_device, &viewInfo, nullptr, &view));
|
||||
VkAttachmentDescription attachment{};
|
||||
attachment.format = image.format;
|
||||
attachment.samples = image.samples;
|
||||
attachment.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
|
||||
attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
|
||||
attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
|
||||
attachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_STORE;
|
||||
attachment.initialLayout = layout;
|
||||
attachment.finalLayout = layout;
|
||||
VkAttachmentReference ref{0, layout};
|
||||
VkSubpassDescription subpass{};
|
||||
subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS;
|
||||
subpass.colorAttachmentCount = color ? 1 : 0;
|
||||
subpass.pColorAttachments = color ? &ref : nullptr;
|
||||
subpass.pDepthStencilAttachment = color ? nullptr : &ref;
|
||||
VkRenderPassCreateInfo passInfo{VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO};
|
||||
passInfo.attachmentCount = 1;
|
||||
passInfo.pAttachments = &attachment;
|
||||
passInfo.subpassCount = 1;
|
||||
passInfo.pSubpasses = &subpass;
|
||||
VkRenderPass pass = VK_NULL_HANDLE;
|
||||
VK_VERIFY(vkCreateRenderPass(m_device, &passInfo, nullptr, &pass));
|
||||
VkFramebufferCreateInfo fbInfo{VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO};
|
||||
fbInfo.renderPass = pass;
|
||||
fbInfo.attachmentCount = 1;
|
||||
fbInfo.pAttachments = &view;
|
||||
fbInfo.width = image.extent.width;
|
||||
fbInfo.height = image.extent.height;
|
||||
fbInfo.layers = image.layers;
|
||||
VkFramebuffer fb = VK_NULL_HANDLE;
|
||||
VK_VERIFY(vkCreateFramebuffer(m_device, &fbInfo, nullptr, &fb));
|
||||
VkRenderPassBeginInfo begin{VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO};
|
||||
begin.renderPass = pass;
|
||||
begin.framebuffer = fb;
|
||||
begin.renderArea = area;
|
||||
auto clearPayload = payload;
|
||||
if (color && !colorPreCompensated) PreCompensateSrgbClearColor(clearPayload, image.format);
|
||||
VkClearAttachment clear{};
|
||||
clear.aspectMask = aspect;
|
||||
clear.clearValue.color = MakeVkClearColorValue(clearPayload, false);
|
||||
if (!color) clear.clearValue.depthStencil = {payload.depth, payload.stencil};
|
||||
VkClearRect rect{area, 0, image.layers};
|
||||
auto& frame = m_frameContext.GetCurrent();
|
||||
EndActiveRenderPassOn(frame.commandBuffer);
|
||||
vkCmdBeginRenderPass(frame.commandBuffer, &begin, VK_SUBPASS_CONTENTS_INLINE);
|
||||
vkCmdClearAttachments(frame.commandBuffer, 1, &clear, 1, &rect);
|
||||
vkCmdEndRenderPass(frame.commandBuffer);
|
||||
WireRetiredObjects retired{};
|
||||
retired.framebuffers.push_back(fb);
|
||||
retired.renderPasses.push_back(pass);
|
||||
retired.imageViews.push_back(view);
|
||||
RetireWireObjects(Move(retired));
|
||||
if (!image.isDefault && textureSurface)
|
||||
m_textureManager->MarkWireTextureGpuWritten(image.storage, image.level, image.layer, image.layers);
|
||||
if (image.isDefault) {
|
||||
if (color)
|
||||
m_swapchainObject.SetImageContentDefined(image.swapchainImageIndex, true);
|
||||
else
|
||||
m_swapchainObject.SetDepthStencilContentDefined(image.swapchainImageIndex, true);
|
||||
}
|
||||
}
|
||||
|
||||
void VulkanRenderer::ClearWireFramebuffer(const MG_Pipe::MGPFramebufferState& fbo,
|
||||
const ClearAttachmentPayload& clearRequest, GLint drawbuffer) {
|
||||
if (MG_Pipe::gPipeInputs->IsCapabilityEnabled(CapabilityInput::RasterizerDiscard)) return;
|
||||
@@ -1126,6 +1211,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
|
||||
payload.mask = ClearWireFramebufferInPass(fbo, payload, drawbuffer);
|
||||
if (payload.mask) payload.mask = ClearWireAttachmentsOfOpenPass(fbo, payload, drawbuffer);
|
||||
if (payload.mask) payload.mask = DeferWireClear(fbo, payload, drawbuffer);
|
||||
if (payload.mask) payload.mask = DeferDefaultWireClear(fbo, payload, drawbuffer);
|
||||
if (payload.mask) payload.mask = ClearWireFramebufferInPass(fbo, payload, drawbuffer);
|
||||
if (!payload.mask) return;
|
||||
RetireWireDrawPass();
|
||||
@@ -1177,82 +1263,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
|
||||
m_swapchainObject.GetPreTransform(), &area.offset, &area.extent))
|
||||
MagmaWireFatal("clear-surface-transform");
|
||||
}
|
||||
const Bool color = aspect == VK_IMAGE_ASPECT_COLOR_BIT;
|
||||
const auto layout =
|
||||
color ? VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL : VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
|
||||
TransitionWireImage(image, layout);
|
||||
VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
|
||||
viewInfo.image = image.image;
|
||||
viewInfo.viewType = image.viewType;
|
||||
viewInfo.format = image.format;
|
||||
viewInfo.subresourceRange = {
|
||||
color ? VK_IMAGE_ASPECT_COLOR_BIT
|
||||
: (image.format == VK_FORMAT_D24_UNORM_S8_UINT || image.format == VK_FORMAT_D32_SFLOAT_S8_UINT
|
||||
? VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT
|
||||
: image.aspect),
|
||||
image.level, 1, image.layer, image.layers};
|
||||
VkImageView view = VK_NULL_HANDLE;
|
||||
VK_VERIFY(vkCreateImageView(m_device, &viewInfo, nullptr, &view));
|
||||
VkAttachmentDescription attachment{};
|
||||
attachment.format = image.format;
|
||||
attachment.samples = image.samples;
|
||||
attachment.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
|
||||
attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
|
||||
attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
|
||||
attachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_STORE;
|
||||
attachment.initialLayout = layout;
|
||||
attachment.finalLayout = layout;
|
||||
VkAttachmentReference ref{0, layout};
|
||||
VkSubpassDescription subpass{};
|
||||
subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS;
|
||||
subpass.colorAttachmentCount = color ? 1 : 0;
|
||||
subpass.pColorAttachments = color ? &ref : nullptr;
|
||||
subpass.pDepthStencilAttachment = color ? nullptr : &ref;
|
||||
VkRenderPassCreateInfo passInfo{VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO};
|
||||
passInfo.attachmentCount = 1;
|
||||
passInfo.pAttachments = &attachment;
|
||||
passInfo.subpassCount = 1;
|
||||
passInfo.pSubpasses = &subpass;
|
||||
VkRenderPass pass = VK_NULL_HANDLE;
|
||||
VK_VERIFY(vkCreateRenderPass(m_device, &passInfo, nullptr, &pass));
|
||||
VkFramebufferCreateInfo fbInfo{VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO};
|
||||
fbInfo.renderPass = pass;
|
||||
fbInfo.attachmentCount = 1;
|
||||
fbInfo.pAttachments = &view;
|
||||
fbInfo.width = image.extent.width;
|
||||
fbInfo.height = image.extent.height;
|
||||
fbInfo.layers = image.layers;
|
||||
VkFramebuffer fb = VK_NULL_HANDLE;
|
||||
VK_VERIFY(vkCreateFramebuffer(m_device, &fbInfo, nullptr, &fb));
|
||||
VkRenderPassBeginInfo begin{VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO};
|
||||
begin.renderPass = pass;
|
||||
begin.framebuffer = fb;
|
||||
begin.renderArea = area;
|
||||
auto clearPayload = payload;
|
||||
if (color) PreCompensateSrgbClearColor(clearPayload, image.format);
|
||||
VkClearAttachment clear{};
|
||||
clear.aspectMask = aspect;
|
||||
clear.clearValue.color = MakeVkClearColorValue(clearPayload, false);
|
||||
if (!color) clear.clearValue.depthStencil = {payload.depth, payload.stencil};
|
||||
VkClearRect rect{area, 0, image.layers};
|
||||
auto& frame = m_frameContext.GetCurrent();
|
||||
EndActiveRenderPassOn(frame.commandBuffer);
|
||||
vkCmdBeginRenderPass(frame.commandBuffer, &begin, VK_SUBPASS_CONTENTS_INLINE);
|
||||
vkCmdClearAttachments(frame.commandBuffer, 1, &clear, 1, &rect);
|
||||
vkCmdEndRenderPass(frame.commandBuffer);
|
||||
WireRetiredObjects retired{};
|
||||
retired.framebuffers.push_back(fb);
|
||||
retired.renderPasses.push_back(pass);
|
||||
retired.imageViews.push_back(view);
|
||||
RetireWireObjects(Move(retired));
|
||||
if (!image.isDefault && surface.Kind == MG_Pipe::kMGPipeSurfaceKindTexture)
|
||||
m_textureManager->MarkWireTextureGpuWritten(image.storage, image.level, image.layer, image.layers);
|
||||
if (image.isDefault) {
|
||||
if (color)
|
||||
m_swapchainObject.SetImageContentDefined(image.swapchainImageIndex, true);
|
||||
else
|
||||
m_swapchainObject.SetDepthStencilContentDefined(image.swapchainImageIndex, true);
|
||||
}
|
||||
RecordWireImageClear(image, aspect, payload, area, surface.Kind == MG_Pipe::kMGPipeSurfaceKindTexture);
|
||||
};
|
||||
if (payload.mask & GL_COLOR_BUFFER_BIT) {
|
||||
for (Uint32 i = 0; i < MG_Pipe::kMGPipeMaxColorAttachments; ++i) {
|
||||
|
||||
@@ -108,6 +108,20 @@ namespace MobileGL::MG_Impl::GLImpl {
|
||||
program->WriteReservedNumSamples(static_cast<Int>(std::max<GLint>(ResolveDrawFramebufferSampleCount(), 1)));
|
||||
}
|
||||
|
||||
// The GL_CLIP_DISTANCE0..7 enables, for a program whose clip distances are emulated (see
|
||||
// EmulateClipDistancePass): the producing stage selects 1.0 for every disabled distance, so
|
||||
// the mask is a property of the render state at THIS draw, not of the program. Same cost
|
||||
// model as the sample count above.
|
||||
static void PublishClipDistanceEnables(const SharedPtr<MG_State::GLState::ProgramObject>& program) {
|
||||
if (!program || !program->UsesReservedClipDistanceEnables()) return;
|
||||
Uint32 enables = 0;
|
||||
for (Uint i = 0; i < 8; ++i) {
|
||||
const auto cap = static_cast<CapabilityInput>(static_cast<Uint>(CapabilityInput::ClipDistance0) + i);
|
||||
if (MG_State::pGLContext->IsCapabilityEnabled(cap)) enables |= 1u << i;
|
||||
}
|
||||
program->WriteReservedClipDistanceEnables(enables);
|
||||
}
|
||||
|
||||
// The one funnel every drawing command passes through. Order is load-bearing: validate first
|
||||
// (a rejected draw must leave state alone), then publish the sample count - which reads the
|
||||
// DRAW FRAMEBUFFER binding, so it has to run after the caller's framebuffer state is settled
|
||||
@@ -133,6 +147,7 @@ namespace MobileGL::MG_Impl::GLImpl {
|
||||
const auto& currentProgram = MG_State::pGLContext->GetProgramForDraw();
|
||||
if (!ValidateResolvedProgramForDraw(currentProgram, functionName)) return false;
|
||||
PublishDrawFramebufferSampleCount(currentProgram);
|
||||
PublishClipDistanceEnables(currentProgram);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -2197,7 +2197,7 @@ namespace MobileGL::MG_Impl::GLImpl {
|
||||
|
||||
// Allocate in TextureObject
|
||||
if (isProxy) {
|
||||
MGLOG_D("%s: isProxy = true, not allocating", __func__);
|
||||
TextureImpl::pProxyTextureManager->RecordProxyLevel(textureUploadTarget, level, {width, height, depth});
|
||||
} else {
|
||||
DiscardMipmapChainOnBaseRespecification(textureMipmapObject, textureUploadTarget, level);
|
||||
textureMipmapObject->AllocateStorage(textureUploadTarget, level, {{width, height, depth}, internalBytes});
|
||||
@@ -2345,7 +2345,7 @@ namespace MobileGL::MG_Impl::GLImpl {
|
||||
|
||||
// Allocate in TextureObject
|
||||
if (isProxy) {
|
||||
MGLOG_D("%s: isProxy = true, not allocating", __func__);
|
||||
TextureImpl::pProxyTextureManager->RecordProxyLevel(textureUploadTarget, level, {width, height, 1});
|
||||
} else {
|
||||
MGLOG_D("%s: Allocating %d bytes at mip %d", __func__, internalBytes, level);
|
||||
DiscardMipmapChainOnBaseRespecification(textureMipmapObject, textureUploadTarget, level);
|
||||
@@ -2460,7 +2460,9 @@ namespace MobileGL::MG_Impl::GLImpl {
|
||||
MOBILEGL_ASSERT(nullptr != static_cast<MG_State::GLState::TextureObjectMipmap*>(textureObject.get()),
|
||||
"Texture object here should always be an object with mipmap");
|
||||
auto textureMipmapObject = static_cast<MG_State::GLState::TextureObjectMipmap*>(textureObject.get());
|
||||
if (!isProxy) {
|
||||
if (isProxy) {
|
||||
TextureImpl::pProxyTextureManager->RecordProxyLevel(textureUploadTarget, level, {width, 1, 1});
|
||||
} else {
|
||||
DiscardMipmapChainOnBaseRespecification(textureMipmapObject, textureUploadTarget, level);
|
||||
textureMipmapObject->AllocateStorage(textureUploadTarget, level, {{width, 1, 1}, internalBytes});
|
||||
// After AllocateStorage, which clears the tag. No block-compressed format has a 1D
|
||||
@@ -2905,7 +2907,12 @@ namespace MobileGL::MG_Impl::GLImpl {
|
||||
case GL_TEXTURE_HEIGHT:
|
||||
case GL_TEXTURE_DEPTH: {
|
||||
const SizeT axis = pname == GL_TEXTURE_WIDTH ? 0 : (pname == GL_TEXTURE_HEIGHT ? 1 : 2);
|
||||
if (storageType == TextureStorageType::Mipmap && mipmapObject != nullptr) {
|
||||
// A proxy level never took storage; its size lives with the proxy manager.
|
||||
const auto proxySize =
|
||||
isProxy ? TextureImpl::pProxyTextureManager->GetProxyLevelTexelSize(uploadTarget, level) : std::nullopt;
|
||||
if (proxySize) {
|
||||
value = (*proxySize)[axis];
|
||||
} else if (storageType == TextureStorageType::Mipmap && mipmapObject != nullptr) {
|
||||
value = mipmapObject->GetMipmapTexelSize(uploadTarget, level)[axis];
|
||||
} else if (storageType == TextureStorageType::Buffer) {
|
||||
// A buffer texture is one-dimensional.
|
||||
@@ -3988,10 +3995,13 @@ namespace MobileGL::MG_Impl::GLImpl {
|
||||
MG_Util::ConvertGLEnumToTextureInternalFormat(internalformat);
|
||||
textureObject->SetInternalFormat(textureInternalFormat);
|
||||
|
||||
// A proxy records the format and nothing else - it must never take storage, and it must never
|
||||
// be tagged compressed, or GL_TEXTURE_COMPRESSED_IMAGE_SIZE on a proxy would stop being
|
||||
// INVALID_OPERATION.
|
||||
if (isProxy) return;
|
||||
// A proxy records the format and the level size and nothing else - it must never take
|
||||
// storage, and it must never be tagged compressed, or GL_TEXTURE_COMPRESSED_IMAGE_SIZE on a
|
||||
// proxy would stop being INVALID_OPERATION.
|
||||
if (isProxy) {
|
||||
TextureImpl::pProxyTextureManager->RecordProxyLevel(textureUploadTarget, level, {width, height, depth});
|
||||
return;
|
||||
}
|
||||
|
||||
const SizeT internalBpp =
|
||||
MG_Util::GetInternalBytesPerPixel(textureInternalFormat, TexturePixelDataType::UnsignedByte);
|
||||
@@ -4072,10 +4082,13 @@ namespace MobileGL::MG_Impl::GLImpl {
|
||||
MG_Util::ConvertGLEnumToTextureInternalFormat(internalformat);
|
||||
textureObject->SetInternalFormat(textureInternalFormat);
|
||||
|
||||
// A proxy records the format and nothing else - it must never take storage, and it must never
|
||||
// be tagged compressed, or GL_TEXTURE_COMPRESSED_IMAGE_SIZE on a proxy would stop being
|
||||
// INVALID_OPERATION.
|
||||
if (isProxy) return;
|
||||
// A proxy records the format and the level size and nothing else - it must never take
|
||||
// storage, and it must never be tagged compressed, or GL_TEXTURE_COMPRESSED_IMAGE_SIZE on a
|
||||
// proxy would stop being INVALID_OPERATION.
|
||||
if (isProxy) {
|
||||
TextureImpl::pProxyTextureManager->RecordProxyLevel(textureUploadTarget, level, {width, height, 1});
|
||||
return;
|
||||
}
|
||||
|
||||
const SizeT internalBpp =
|
||||
MG_Util::GetInternalBytesPerPixel(textureInternalFormat, TexturePixelDataType::UnsignedByte);
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
// End of Source File Header
|
||||
|
||||
#include "ProxyTexture.h"
|
||||
#include <MG_Backend/BackendObjects.h>
|
||||
#include <MG_State/GLState/TextureState/TextureObject1D.h>
|
||||
#include <MG_State/GLState/TextureState/TextureObject2D.h>
|
||||
#include <MG_State/GLState/TextureState/TextureObject2DCube.h>
|
||||
@@ -87,4 +88,51 @@ namespace MobileGL::MG_Impl::GLImpl::TextureImpl {
|
||||
static SharedPtr<MG_State::GLState::ITextureObject> nullTextureObject = nullptr;
|
||||
return nullTextureObject;
|
||||
}
|
||||
|
||||
namespace {
|
||||
// Whether the implementation can hold a level of this size on this proxy target. Every
|
||||
// dimension of a mipmapped image shrinks with the level; array layers do not.
|
||||
Bool ProxyLevelFits(TextureUploadTarget target, GLint level, IntVec3 size) {
|
||||
MG_Backend::DynamicBackendParameters limits{};
|
||||
if (MG_Backend::pActiveBackendObject) limits = MG_Backend::pActiveBackendObject->GetDynamicParameters();
|
||||
const auto levelLimit = [level](Int limit) { return level < 31 ? std::max(limit >> level, 1) : 1; };
|
||||
const Int planar = levelLimit(limits.MaxTextureSize);
|
||||
const Int cube = levelLimit(limits.MaxCubeMapTextureSize);
|
||||
const Int volume = levelLimit(limits.Max3DTextureSize);
|
||||
const Int layers = limits.MaxArrayTextureLayers;
|
||||
switch (target) {
|
||||
case TextureUploadTarget::ProxyTexture1D:
|
||||
return size.x() <= planar;
|
||||
case TextureUploadTarget::ProxyTexture1DArray:
|
||||
return size.x() <= planar && size.y() <= layers;
|
||||
case TextureUploadTarget::ProxyTexture2DArray:
|
||||
return size.x() <= planar && size.y() <= planar && size.z() <= layers;
|
||||
case TextureUploadTarget::ProxyTexture3D:
|
||||
return size.x() <= volume && size.y() <= volume && size.z() <= volume;
|
||||
case TextureUploadTarget::ProxyCubeMap:
|
||||
return size.x() <= cube && size.y() <= cube;
|
||||
case TextureUploadTarget::ProxyCubeMapArray:
|
||||
return size.x() <= cube && size.y() <= cube && size.z() <= layers;
|
||||
case TextureUploadTarget::ProxyTextureRectangle:
|
||||
return level == 0 && size.x() <= limits.MaxTextureSize && size.y() <= limits.MaxTextureSize;
|
||||
default:
|
||||
return size.x() <= planar && size.y() <= planar;
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void ProxyTextureManager::RecordProxyLevel(TextureUploadTarget target, GLint level, IntVec3 size) {
|
||||
if (level < 0) return;
|
||||
auto& levels = m_proxyLevelSizes[target];
|
||||
if (levels.size() <= static_cast<SizeT>(level)) levels.resize(static_cast<SizeT>(level) + 1);
|
||||
levels[level] = ProxyLevelFits(target, level, size) ? size : IntVec3{0, 0, 0};
|
||||
}
|
||||
|
||||
Optional<IntVec3> ProxyTextureManager::GetProxyLevelTexelSize(TextureUploadTarget target, GLint level) const {
|
||||
auto it = m_proxyLevelSizes.find(target);
|
||||
if (it == m_proxyLevelSizes.end() || level < 0 || it->second.size() <= static_cast<SizeT>(level)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return it->second[level];
|
||||
}
|
||||
} // namespace MobileGL::MG_Impl::GLImpl::TextureImpl
|
||||
|
||||
@@ -19,8 +19,15 @@ namespace MobileGL::MG_Impl::GLImpl::TextureImpl {
|
||||
TextureUploadTarget target);
|
||||
const SharedPtr<MG_State::GLState::ITextureObject>& GetProxyTextureObject(TextureUploadTarget target);
|
||||
|
||||
// The image a proxy glTexImage* or glCompressedTexImage* describes, kept here because a proxy
|
||||
// never takes storage. A request past the implementation limits records all zeros (GL 4.6
|
||||
// core 8.22), which is how glGetTexLevelParameter tells the application it cannot be had.
|
||||
void RecordProxyLevel(TextureUploadTarget target, GLint level, IntVec3 size);
|
||||
Optional<IntVec3> GetProxyLevelTexelSize(TextureUploadTarget target, GLint level) const;
|
||||
|
||||
private:
|
||||
UnorderedMap<TextureUploadTarget, SharedPtr<MG_State::GLState::ITextureObject>> m_proxyTexturesMap;
|
||||
UnorderedMap<TextureUploadTarget, Vector<Optional<IntVec3>>> m_proxyLevelSizes;
|
||||
};
|
||||
|
||||
extern UniquePtr<ProxyTextureManager>& pProxyTextureManager;
|
||||
|
||||
@@ -2602,8 +2602,20 @@ namespace MobileGL::MG_Pipe {
|
||||
// boundary reads unpack state, and the staged-repack upload path does not even issue
|
||||
// glPixelStorei (ARCHITECTURE.md 4.6 D5).
|
||||
Uint64 EmitPixelPackState(GLContext& ctx) {
|
||||
MGPPixelPackState pack{};
|
||||
pack.Pack = ctx.GetPixelStoreParameters(false);
|
||||
// Field by field into zeroed bytes: the two bytes of padding after the Bools would
|
||||
// otherwise carry whatever the context's heap block held, onto the wire and into the
|
||||
// recorder's hash.
|
||||
MGPPixelPackState pack;
|
||||
std::memset(&pack, 0, sizeof(pack));
|
||||
const PixelStoreParameters live = ctx.GetPixelStoreParameters(false);
|
||||
pack.Pack.SwapBytes = live.SwapBytes;
|
||||
pack.Pack.LSBFirst = live.LSBFirst;
|
||||
pack.Pack.RowLength = live.RowLength;
|
||||
pack.Pack.ImageHeight = live.ImageHeight;
|
||||
pack.Pack.SkipPixels = live.SkipPixels;
|
||||
pack.Pack.SkipRows = live.SkipRows;
|
||||
pack.Pack.SkipImages = live.SkipImages;
|
||||
pack.Pack.Alignment = live.Alignment;
|
||||
MGPipeRouteSetPixelPackState(pack);
|
||||
return sizeof(MGPPixelPackState);
|
||||
}
|
||||
|
||||
@@ -7,9 +7,11 @@ Configuration-specific controls are selected separately and must pass without sk
|
||||
## Unresolved behavior
|
||||
|
||||
- `ClipDistanceScenario.ADisabledClipDistanceRemovesNothing` and
|
||||
`TheEnablesAreIndependentPerDistance`: per-distance enable semantics are missing
|
||||
on the tested GLES and Vulkan paths. Neither backend supplies a passing control
|
||||
on local llvmpipe/lavapipe. The strict list names the GLES and Magma Fm entries.
|
||||
`TheEnablesAreIndependentPerDistance`: resolved on both backends. Espryt on
|
||||
llvmpipe emulates clip distances (its native probe catches llvmpipe clipping by
|
||||
disabled distances); Magma on lavapipe applies GL_CLIP_DISTANCEi in the shader
|
||||
(Vulkan has no per-distance enable state). The entries stay listed only so a
|
||||
stack that ever skips them again keeps a documented reason.
|
||||
- `PartialUploadOverGpuContentScenario.OneDimensionalArraySubImageKeepsTheGpuClearedLayer`:
|
||||
the Vulkan path cannot attach the 1D array layer. GLES passes, which does not
|
||||
establish Vulkan coverage.
|
||||
|
||||
@@ -26,16 +26,15 @@
|
||||
// identical distances and the enable turned off must leave both sides painted, which is what
|
||||
// says the pixels below are being removed by clipping and not by something else.
|
||||
//
|
||||
// HONEST LIMIT OF THIS FILE IN CI. Of the four cases, only EnableIsObservableThroughIsEnabled is
|
||||
// falsifiable on the software rasterizers every automated lane runs on. llvmpipe and lavapipe
|
||||
// clip by EVERY declared gl_ClipDistance regardless of the enables, so
|
||||
// AnEnabledClipDistanceRemovesTheNegativeHalf goes green there against the broken tree as well,
|
||||
// and the two cases that need real per-distance semantics skip (see
|
||||
// DriverHonorsPerDistanceEnables). What actually pins the behavior is Adreno, through
|
||||
// KHR-GLxx.clip_distance.functional - whose "without dynamic redeclaration" variants declare all
|
||||
// gl_MaxClipDistances slots and enable only the first N, i.e. exactly the subset semantics these
|
||||
// skipped cases assert. Read a green CI run here as "the state survives the frontend", not as
|
||||
// "clipping is correct"; the second claim is a device claim.
|
||||
// WHAT CI CAN AND CANNOT FALSIFY. llvmpipe's ES driver clips by EVERY written gl_ClipDistance
|
||||
// regardless of the enables, which the DirectGLES loader's native clip probe catches - so on
|
||||
// that lane Espryt EMULATES clip distances (EmulateClipDistancePass: varyings, a fragment
|
||||
// discard and a per-draw enables uniform), and the per-distance cases below run and pass
|
||||
// there. That is the emulation every Mali device takes. lavapipe hosts shaderClipDistance, so
|
||||
// Magma clips natively; Vulkan has no per-distance enable state, so Magma applies GL_CLIP_DISTANCEi in
|
||||
// the producing stage (BackendCaps::MasksNativeClipDistances) and the per-distance cases run
|
||||
// green there too. The native ES path - Adreno, which honors GL_CLIP_DISTANCEi_EXT - is pinned on
|
||||
// the device, through KHR-GLxx.clip_distance.functional.
|
||||
//
|
||||
// Every case disables all eight distances on entry rather than assuming they start off:
|
||||
// XfbAfterClipDistanceScenario deliberately leaves one enabled for the rest of the process, and
|
||||
@@ -83,13 +82,33 @@ void main() {
|
||||
const char* const kFragmentSource = R"(#version 400 core
|
||||
out vec4 fragColor;
|
||||
void main() { fragColor = vec4(0.0, 1.0, 0.0, 1.0); }
|
||||
)";
|
||||
|
||||
// The field report's shape (Mali-G615 / Espryt, where GL_MAX_CLIP_DISTANCES used to be 0
|
||||
// and this did not even compile): a #version 330 core stage writing all eight distances
|
||||
// from uniform planes, implicitly sized by use.
|
||||
const char* const kEightPlaneVertexSource = R"(#version 330 core
|
||||
uniform vec4 u_ClipPlane[8];
|
||||
void main() {
|
||||
vec2 positions[3] = vec2[3](vec2(-1.0, -1.0), vec2(3.0, -1.0), vec2(-1.0, 3.0));
|
||||
vec4 eyePos = vec4(positions[gl_VertexID], 0.0, 1.0);
|
||||
gl_Position = eyePos;
|
||||
gl_ClipDistance[0] = dot(u_ClipPlane[0], eyePos);
|
||||
gl_ClipDistance[1] = dot(u_ClipPlane[1], eyePos);
|
||||
gl_ClipDistance[2] = dot(u_ClipPlane[2], eyePos);
|
||||
gl_ClipDistance[3] = dot(u_ClipPlane[3], eyePos);
|
||||
gl_ClipDistance[4] = dot(u_ClipPlane[4], eyePos);
|
||||
gl_ClipDistance[5] = dot(u_ClipPlane[5], eyePos);
|
||||
gl_ClipDistance[6] = dot(u_ClipPlane[6], eyePos);
|
||||
gl_ClipDistance[7] = dot(u_ClipPlane[7], eyePos);
|
||||
}
|
||||
)";
|
||||
|
||||
class ClipDistanceScenario : public ScenarioTest {
|
||||
protected:
|
||||
GLuint BuildProgram() {
|
||||
GLuint BuildProgram(const char* vertexSource = kVertexSource) {
|
||||
const GLuint vs = glCreateShader(GL_VERTEX_SHADER);
|
||||
glShaderSource(vs, 1, &kVertexSource, nullptr);
|
||||
glShaderSource(vs, 1, &vertexSource, nullptr);
|
||||
glCompileShader(vs);
|
||||
GLint compiled = 0;
|
||||
glGetShaderiv(vs, GL_COMPILE_STATUS, &compiled);
|
||||
@@ -178,11 +197,13 @@ void main() { fragColor = vec4(0.0, 1.0, 0.0, 1.0); }
|
||||
// always. DirectVulkan therefore clips by a disabled distance.
|
||||
// - llvmpipe's ES driver behaves like Vulkan here.
|
||||
//
|
||||
// Emulating GL's semantics on those two would mean forcing the disabled slots to a
|
||||
// non-negative value inside the shader, which makes the enable mask part of the
|
||||
// pipeline key - a feature, not a fix, and deliberately not attempted here. The
|
||||
// cases that need the real semantics gate on this probe and say so when they skip,
|
||||
// rather than being deleted or silently weakened.
|
||||
// EmulateClipDistancePass gives those semantics to every backend that EMULATES clip
|
||||
// distances (the disabled slots are forced to +1 from a per-draw uniform, so nothing
|
||||
// joins the pipeline key), and the DirectGLES loader arms that emulation on a driver
|
||||
// whose native clip distances ignore the enables. Magma's native path applies them in
|
||||
// the producing stage (MasksNativeClipDistances). The cases that need the real
|
||||
// semantics still gate on this probe and say so if a stack ever skips, rather than
|
||||
// being deleted or silently weakened.
|
||||
bool DriverHonorsPerDistanceEnables(GLuint program, GLuint vao) const {
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
glDisable(static_cast<GLenum>(GL_CLIP_DISTANCE0 + i));
|
||||
@@ -403,4 +424,91 @@ void main() { fragColor = vec4(0.0, 1.0, 0.0, 1.0); }
|
||||
gl.EndFrame();
|
||||
}
|
||||
|
||||
// GL 3.0+ requires eight clip distances, and the field report was a stack answering zero: a
|
||||
// #version 330 core stage writing gl_ClipDistance[0..7] did not compile. Every backend now
|
||||
// either hosts eight or emulates them (EmulateClipDistancePass), so this case requires eight
|
||||
// and does not skip. Distance 2 clips the left half, the other seven are positive, and all
|
||||
// eight are enabled - which clips identically whether or not the stack honors per-distance
|
||||
// enables. The per-distance half (distance 5 negative on the right but DISABLED) only runs
|
||||
// where the stack honors them, which every emulating backend does by construction.
|
||||
TEST_F(ClipDistanceScenario, EightDistancesFromUniformPlanesClipLikeTheReport) {
|
||||
if (!Ready()) return;
|
||||
GLint maxClipDistances = 0;
|
||||
glGetIntegerv(GL_MAX_CLIP_DISTANCES, &maxClipDistances);
|
||||
ASSERT_GE(maxClipDistances, 8) << "GL 3.0+ requires GL_MAX_CLIP_DISTANCES >= 8";
|
||||
HeadlessGL& gl = Gl();
|
||||
const int width = gl.Width();
|
||||
const int height = gl.Height();
|
||||
ASSERT_GE(width, 8);
|
||||
ASSERT_GE(height, 8);
|
||||
|
||||
GLuint vao = 0;
|
||||
glGenVertexArrays(1, &vao);
|
||||
const GLuint program = BuildProgram(kEightPlaneVertexSource);
|
||||
ASSERT_NE(program, 0u) << "the eight-plane gl_ClipDistance program did not build: " << BuildLog();
|
||||
const GLint planes = glGetUniformLocation(program, "u_ClipPlane");
|
||||
ASSERT_GE(planes, 0);
|
||||
|
||||
BindDefaultFramebuffer();
|
||||
glViewport(0, 0, width, height);
|
||||
glDisable(GL_SCISSOR_TEST);
|
||||
glDisable(GL_DEPTH_TEST);
|
||||
glDisable(GL_CULL_FACE);
|
||||
glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
|
||||
|
||||
// w = 1 everywhere, so (0,0,0,1) is a distance of +1 and keeps everything.
|
||||
GLfloat values[8 * 4];
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
values[i * 4 + 0] = 0.0f;
|
||||
values[i * 4 + 1] = 0.0f;
|
||||
values[i * 4 + 2] = 0.0f;
|
||||
values[i * 4 + 3] = 1.0f;
|
||||
}
|
||||
values[2 * 4 + 0] = 1.0f; // distance 2 = x: negative on the left half
|
||||
values[2 * 4 + 3] = 0.0f;
|
||||
values[5 * 4 + 0] = -1.0f; // distance 5 = -x: negative on the right half
|
||||
values[5 * 4 + 3] = 0.0f;
|
||||
glUseProgram(program);
|
||||
glUniform4fv(planes, 8, values);
|
||||
|
||||
DisableEveryClipDistance();
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
if (i != 5) glEnable(static_cast<GLenum>(GL_CLIP_DISTANCE0 + i));
|
||||
}
|
||||
// First with distance 5 enabled too: both halves clipped, by different distances.
|
||||
glEnable(static_cast<GLenum>(GL_CLIP_DISTANCE0 + 5));
|
||||
DrawClippedTriangle(program, vao);
|
||||
unsigned char left[4] = {0, 0, 0, 0};
|
||||
unsigned char right[4] = {0, 0, 0, 0};
|
||||
PixelAt(width / 4, height / 2, left);
|
||||
PixelAt(width - 1 - width / 4, height / 2, right);
|
||||
EXPECT_EQ(FirstGLError(), 0u);
|
||||
EXPECT_TRUE(IsRed(left)) << "distance 2 is negative on the left and enabled";
|
||||
EXPECT_TRUE(IsRed(right)) << "distance 5 is negative on the right and enabled";
|
||||
|
||||
// Then distance 5 disabled: on a stack with per-distance enables the right half returns.
|
||||
glDisable(static_cast<GLenum>(GL_CLIP_DISTANCE0 + 5));
|
||||
DrawClippedTriangle(program, vao);
|
||||
PixelAt(width / 4, height / 2, left);
|
||||
PixelAt(width - 1 - width / 4, height / 2, right);
|
||||
EXPECT_EQ(FirstGLError(), 0u);
|
||||
EXPECT_TRUE(IsRed(left)) << "distance 2 is still enabled";
|
||||
const bool rightKept = IsGreen(right);
|
||||
|
||||
DisableEveryClipDistance();
|
||||
const GLuint control = BuildProgram();
|
||||
const bool honorsEnables = control != 0 && DriverHonorsPerDistanceEnables(control, vao);
|
||||
if (control != 0) glDeleteProgram(control);
|
||||
if (honorsEnables) {
|
||||
EXPECT_TRUE(rightKept) << "distance 5 is negative on the right but GL_CLIP_DISTANCE5 is disabled, so the "
|
||||
"right half must survive";
|
||||
}
|
||||
|
||||
glUseProgram(0);
|
||||
glBindVertexArray(0);
|
||||
glDeleteProgram(program);
|
||||
glDeleteVertexArrays(1, &vao);
|
||||
gl.EndFrame();
|
||||
}
|
||||
|
||||
} // namespace MGITest
|
||||
|
||||
@@ -357,4 +357,35 @@ void main() { o0 = texture(uPalette, vec2(0.5, 0.5)); }
|
||||
glDeleteProgram(paletteProgram);
|
||||
}
|
||||
|
||||
// A depth clear of the DEFAULT framebuffer while a pass is open on it (a game clearing depth
|
||||
// before its hand or GUI): the in-pass clear does not take the window, so the clear ends the
|
||||
// pass and becomes the load op of the next one, which the farther draw must see.
|
||||
TEST_F(InPassClearScenario, ADefaultFramebufferDepthClearBetweenTestedDrawsDecidesTheSecondDraw) {
|
||||
if (!Ready()) return;
|
||||
HeadlessGL& gl = Gl();
|
||||
const int width = gl.Width(), height = gl.Height();
|
||||
Build();
|
||||
for (int frame = 0; frame < 2; ++frame) {
|
||||
BindDefaultFramebuffer();
|
||||
glViewport(0, 0, width, height);
|
||||
glDisable(GL_SCISSOR_TEST);
|
||||
glDisable(GL_BLEND);
|
||||
glDepthMask(GL_TRUE);
|
||||
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
|
||||
glClearDepth(1.0);
|
||||
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
|
||||
glEnable(GL_DEPTH_TEST);
|
||||
glDepthFunc(GL_LESS);
|
||||
Quad(1.0f, 0.0f, 0.0f, 1.0f, -0.5f);
|
||||
glClear(GL_DEPTH_BUFFER_BIT);
|
||||
Quad(0.0f, 1.0f, 0.0f, 1.0f, 0.0f);
|
||||
glDisable(GL_DEPTH_TEST);
|
||||
EXPECT_EQ(FirstGLError(), 0u);
|
||||
const Image image = ReadPixels(width, height);
|
||||
EXPECT_TRUE(Near(image.At(width / 2, height / 2), 0, 255, 0, 255))
|
||||
<< "frame " << frame << ": the depth clear between the draws let the farther draw pass";
|
||||
gl.EndFrame();
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace MGITest
|
||||
|
||||
@@ -389,7 +389,7 @@
|
||||
F(FragmentInterpolationOffsetBits) F(SupportsWideLines) \
|
||||
F(SupportsDistinctDepthStencilAttachments) F(PerLayerFramebufferAttachmentTargets) \
|
||||
F(SupportsShaderFloat64) F(SupportsFloat64VertexAttributes) F(SupportsTessellationPointSize) \
|
||||
F(SupportsGeometryPointSize) F(MaxShaderStorageBlockSize) F(SubgroupSize) \
|
||||
F(SupportsGeometryPointSize) F(EmulatesClipDistance) F(MasksNativeClipDistances) F(MaxShaderStorageBlockSize) F(SubgroupSize) \
|
||||
F(SubgroupSupportedStages) F(SubgroupSupportedFeatures) F(SamplePositions) \
|
||||
F(SamplePositionCountMask) F(SubgroupQuadOperationsInAllStages) \
|
||||
F(GpuVendor) F(SupportsFragmentShadingRate) F(SupportsPrimitiveFragmentShadingRate) \
|
||||
|
||||
@@ -660,6 +660,8 @@ inline constexpr WireLayoutMember kMGPipeWireLayoutMembers[] = {
|
||||
{"DynamicBackendParameters.SupportsFloat64VertexAttributes", offsetof(DynamicBackendParameters, SupportsFloat64VertexAttributes), sizeof(((DynamicBackendParameters*)nullptr)->SupportsFloat64VertexAttributes)},
|
||||
{"DynamicBackendParameters.SupportsTessellationPointSize", offsetof(DynamicBackendParameters, SupportsTessellationPointSize), sizeof(((DynamicBackendParameters*)nullptr)->SupportsTessellationPointSize)},
|
||||
{"DynamicBackendParameters.SupportsGeometryPointSize", offsetof(DynamicBackendParameters, SupportsGeometryPointSize), sizeof(((DynamicBackendParameters*)nullptr)->SupportsGeometryPointSize)},
|
||||
{"DynamicBackendParameters.EmulatesClipDistance", offsetof(DynamicBackendParameters, EmulatesClipDistance), sizeof(((DynamicBackendParameters*)nullptr)->EmulatesClipDistance)},
|
||||
{"DynamicBackendParameters.MasksNativeClipDistances", offsetof(DynamicBackendParameters, MasksNativeClipDistances), sizeof(((DynamicBackendParameters*)nullptr)->MasksNativeClipDistances)},
|
||||
{"DynamicBackendParameters.MaxShaderStorageBlockSize", offsetof(DynamicBackendParameters, MaxShaderStorageBlockSize), sizeof(((DynamicBackendParameters*)nullptr)->MaxShaderStorageBlockSize)},
|
||||
{"DynamicBackendParameters.SubgroupSize", offsetof(DynamicBackendParameters, SubgroupSize), sizeof(((DynamicBackendParameters*)nullptr)->SubgroupSize)},
|
||||
{"DynamicBackendParameters.SubgroupSupportedStages", offsetof(DynamicBackendParameters, SubgroupSupportedStages), sizeof(((DynamicBackendParameters*)nullptr)->SubgroupSupportedStages)},
|
||||
|
||||
@@ -337,6 +337,12 @@ namespace MobileGL::MG_State::GLState {
|
||||
// gate has to be answerable without joining the SPIR-V job, or every draw of every
|
||||
// program would pay a join to discover it has nothing to write.
|
||||
Bool usesReservedNumSamples = false;
|
||||
// A pre-rasterization stage of this program declared the reserved clip-distance enables
|
||||
// uniform (ShaderTranspiler::CLIP_DISTANCE_ENABLES_UNIFORM_NAME), which the source
|
||||
// pipeline does only on a backend that emulates gl_ClipDistance, so the draw path owes it
|
||||
// the GL_CLIP_DISTANCE0..7 enables before every draw. Phase A for the reason
|
||||
// usesReservedNumSamples is.
|
||||
Bool usesReservedClipDistanceEnables = false;
|
||||
// A shader of this program declared an `es` #version (ShaderTranspiler::FindEsOrigin).
|
||||
// ESSL fixes an image uniform's unit at its binding qualifier, so glUniform* on one is
|
||||
// INVALID_OPERATION for such a program and legal for a desktop one.
|
||||
@@ -415,6 +421,8 @@ namespace MobileGL::MG_State::GLState {
|
||||
// uniformOffsets, because the member has no GL location at all: the link task keeps
|
||||
// it out of the GL-visible uniform index space so no application can see or write it.
|
||||
Uint reservedNumSamplesOffset = kInvalidUniformOffset;
|
||||
// The same, for the reserved clip-distance enables uniform.
|
||||
Uint reservedClipDistanceEnablesOffset = kInvalidUniformOffset;
|
||||
// False for a program whose SPIR-V was never produced (phase B canceled at
|
||||
// teardown or by a relink) or whose optimizer run failed. GL has no way to
|
||||
// retract a LINK_STATUS it already reported true, so such a program stays
|
||||
@@ -443,6 +451,11 @@ namespace MobileGL::MG_State::GLState {
|
||||
// deliberately keeps answering "gl_PointSize": demotion happens after phase A,
|
||||
// so every query keeps the truthful GL spelling.
|
||||
Bool pointSizeDemoted = false;
|
||||
// Whether these modules had gl_ClipDistance lowered to varyings plus a fragment discard
|
||||
// (ShaderCompiler::EmulateClipDistanceForProgram) because the backend cannot clip
|
||||
// natively. Recorded for the same reason pointSizeDemoted is: the final bytes no longer
|
||||
// declare the capability that armed it.
|
||||
Bool clipDistanceEmulated = false;
|
||||
};
|
||||
|
||||
// ---- the archive field tables (ARCHITECTURE.md §7): ONE table per type, serving both directions ----
|
||||
@@ -579,6 +592,7 @@ namespace MobileGL::MG_State::GLState {
|
||||
v("uniformBlocksWithoutBinding", a.uniformBlocksWithoutBinding);
|
||||
v("activeUniformCount", a.activeUniformCount);
|
||||
v("usesReservedNumSamples", a.usesReservedNumSamples);
|
||||
v("usesReservedClipDistanceEnables", a.usesReservedClipDistanceEnables);
|
||||
v("esOrigin", a.esOrigin);
|
||||
v("maxUniformLocation", a.maxUniformLocation);
|
||||
v("uniformNameMaxLength", a.uniformNameMaxLength);
|
||||
@@ -614,10 +628,12 @@ namespace MobileGL::MG_State::GLState {
|
||||
v("uniformOffsets", a.uniformOffsets);
|
||||
v("globalUboScratch", a.globalUboScratch);
|
||||
v("reservedNumSamplesOffset", a.reservedNumSamplesOffset);
|
||||
v("reservedClipDistanceEnablesOffset", a.reservedClipDistanceEnablesOffset);
|
||||
v("spirvStatus", a.spirvStatus);
|
||||
v("nativeFloat64", a.nativeFloat64);
|
||||
v("pointSizeDemoted", a.pointSizeDemoted);
|
||||
} // 8 fields
|
||||
v("clipDistanceEmulated", a.clipDistanceEmulated);
|
||||
} // 10 fields
|
||||
|
||||
// ---- trip wires ----
|
||||
// TypeFacts is a POD on every ABI: 13 Bool + 3 bytes of padding + 7 x 4-byte scalars.
|
||||
@@ -635,7 +651,7 @@ namespace MobileGL::MG_State::GLState {
|
||||
// P7 OQ-8 added storageBlocks, one more Vector, in the disaggregated build ONLY. The pull
|
||||
// number below is the one G1 measures and it has not moved.
|
||||
#define MGL_LINKARTIFACTS_SIZE 1080
|
||||
#define MGL_SPIRVARTIFACTS_SIZE 88
|
||||
#define MGL_SPIRVARTIFACTS_SIZE 96
|
||||
#elif defined(_LIBCPP_VERSION) && (SIZE_MAX == UINT64_MAX) && defined(MGL_ARTIFACT_SIZES_LIBCXX_PINNED)
|
||||
// The integrator pins these from the NDK build (brief C.4); until then this branch is inert.
|
||||
#endif
|
||||
|
||||
@@ -1135,7 +1135,9 @@ namespace MobileGL::MG_State::GLState {
|
||||
for (const ProgramObject::XfbVarying& varying : artifacts.xfbVaryings) {
|
||||
if (varying.name == "gl_PointSize") {
|
||||
spirvHandoff.captureRequestsPointSize = true;
|
||||
break;
|
||||
}
|
||||
if (varying.name.rfind("gl_ClipDistance", 0) == 0) {
|
||||
spirvHandoff.captureRequestsClipDistance = true;
|
||||
}
|
||||
}
|
||||
// Phase B pairs this with its own SpirvArtifacts to insert the completed front end.
|
||||
@@ -1501,6 +1503,21 @@ namespace MobileGL::MG_State::GLState {
|
||||
};
|
||||
for (Int i = 0; i < tProgramUniformCount; i++) {
|
||||
const auto& uniform = artifacts.program->getUniform(i);
|
||||
// The clip-distance enables InjectClipDistanceEnablesShim declared: MobileGL's own
|
||||
// per-draw input to the emulation, hidden from the GL surface on the same terms as
|
||||
// the gl_NumSamples stand-in below. Checked BEFORE the dead-uniform filter on
|
||||
// purpose: no application code reads it - only EmulateClipDistancePass does, after
|
||||
// this reflection ran - so glslang reports it with stages == 0, and filtering it as
|
||||
// dead would leave the draw path never writing it (every distance then selects 1.0
|
||||
// and nothing clips, with no error anywhere).
|
||||
if (isGlobalUboMember(uniform) &&
|
||||
uniform.name == MG_Util::ShaderTranspiler::CLIP_DISTANCE_ENABLES_UNIFORM_NAME) {
|
||||
artifacts.usesReservedClipDistanceEnables = true;
|
||||
MGLOG_D("ProgramObject %u: Reflection - reserved clip-distance enables '%s' hidden from the GL "
|
||||
"uniform surface",
|
||||
in.externalIndex, uniform.name.c_str());
|
||||
continue;
|
||||
}
|
||||
if (isGlobalUboMember(uniform) && uniform.stages == 0) {
|
||||
MGLOG_D("ProgramObject %u: Reflection - dead default-block uniform '%s' filtered from the GL "
|
||||
"surface",
|
||||
|
||||
@@ -131,6 +131,10 @@ namespace MobileGL::MG_State::GLState {
|
||||
// this bit. No new L1 key material: the key already covers
|
||||
// `requestedXfbVaryings`, of which this is a function.
|
||||
Bool captureRequestsPointSize = false;
|
||||
// The same derived bit for gl_ClipDistance (any element): the clip-distance
|
||||
// emulation declines such a program, since the capture wants the unmasked value at a
|
||||
// varying the emulation never creates. Also a function of `requestedXfbVaryings`.
|
||||
Bool captureRequestsClipDistance = false;
|
||||
|
||||
// L1 shader-translation memo key for this program's SPIR-V (see
|
||||
// MG_Util/ShaderTranspiler/TranslationCache.h). Built HERE, at the tail of phase
|
||||
|
||||
@@ -799,6 +799,27 @@ namespace MobileGL::MG_State::GLState {
|
||||
MarkUBOContentDirtyRange(offset, static_cast<Uint>(sizeof(Int)));
|
||||
return true;
|
||||
}
|
||||
// ---- the reserved clip-distance enables (ShaderTranspiler::CLIP_DISTANCE_ENABLES_UNIFORM_NAME) ----
|
||||
//
|
||||
// Same contract as the gl_NumSamples pair above: phase-A gate, value-identical writes
|
||||
// dropped without a content-version bump, false when there is nowhere to put it.
|
||||
Bool UsesReservedClipDistanceEnables() const { return Artifacts().usesReservedClipDistanceEnables; }
|
||||
Bool WriteReservedClipDistanceEnables(Uint32 enables) {
|
||||
if (!UsesReservedClipDistanceEnables()) return false;
|
||||
SpirvArtifacts& spirv = Spirv();
|
||||
const Uint offset = spirv.reservedClipDistanceEnablesOffset;
|
||||
if (offset == kInvalidUniformOffset) return false;
|
||||
if (static_cast<SizeT>(offset) + sizeof(Uint32) > spirv.globalUboScratch.size()) return false;
|
||||
|
||||
Uint8* const slot = spirv.globalUboScratch.data() + offset;
|
||||
Uint32 current = 0;
|
||||
Memcpy(¤t, slot, sizeof(Uint32));
|
||||
if (current == enables) return true;
|
||||
Memcpy(slot, &enables, sizeof(Uint32));
|
||||
MarkUBOContentDirtyRange(offset, static_cast<Uint>(sizeof(Uint32)));
|
||||
return true;
|
||||
}
|
||||
Bool ClipDistanceEmulated() const { return Spirv().clipDistanceEmulated; }
|
||||
// ---- glUniform* inside the phase-A -> phase-B window ----
|
||||
//
|
||||
// True while the program is fully linked and fully queryable but its uniform shadow's
|
||||
|
||||
@@ -33,6 +33,12 @@ namespace MobileGL::MG_State::GLState {
|
||||
}
|
||||
std::atomic_flag g_pointSizeDeclineReported;
|
||||
std::atomic_flag g_pointSizeOptimizerFailureReported;
|
||||
std::atomic_flag g_clipDistanceDeclineReported;
|
||||
std::atomic_flag g_clipDistanceOptimizerFailureReported;
|
||||
std::atomic_flag g_clipDistanceEmulationReported;
|
||||
// Varying locations the clip carriers may reach. 32 vec4s is what both backends' target
|
||||
// drivers expose (128 output components); a program past it keeps the decline.
|
||||
constexpr Uint32 kClipDistanceCarrierLocationBudget = 32;
|
||||
} // namespace
|
||||
|
||||
void ProgramSpirvTask::DeferLog(String line, const Int level) {
|
||||
@@ -154,8 +160,12 @@ namespace MobileGL::MG_State::GLState {
|
||||
m_phaseA->in.env != nullptr && m_phaseA->in.env->DemotesTessellationPointSize();
|
||||
const Bool demoteGeometryPointSize =
|
||||
m_phaseA->in.env != nullptr && m_phaseA->in.env->DemotesGeometryPointSize();
|
||||
// Part of the frontend fingerprint (CompileEnv), so the L1 key already separates the two.
|
||||
const Bool emulateClipDistance = m_phaseA->in.env != nullptr && m_phaseA->in.env->EmulatesClipDistance();
|
||||
const Bool maskNativeClipDistance = m_phaseA->in.env != nullptr && m_phaseA->in.env->MasksNativeClipDistances();
|
||||
GenerateSpirv(handoff, externalIndex, deferOutputValidationForDirectVulkan, enableSpirvValidation,
|
||||
nativeFloat64, demoteTessellationPointSize, demoteGeometryPointSize);
|
||||
nativeFloat64, demoteTessellationPointSize, demoteGeometryPointSize, emulateClipDistance,
|
||||
maskNativeClipDistance);
|
||||
// GlslangToSpv was the only consumer of the parsed ASTs; everything after this point
|
||||
// works on the SPIR-V and on the TProgram's own self-contained reflection pool. Drop
|
||||
// them here rather than at the end of the body, which is ~87% of this node's runtime
|
||||
@@ -214,7 +224,8 @@ namespace MobileGL::MG_State::GLState {
|
||||
void ProgramSpirvTask::GenerateSpirv(const ProgramLinkTask::SpirvHandoff& handoff, const Uint externalIndex,
|
||||
const Bool deferOutputValidationForDirectVulkan,
|
||||
const Bool enableSpirvValidation, const Bool nativeFloat64,
|
||||
const Bool demoteTessellationPointSize, const Bool demoteGeometryPointSize) {
|
||||
const Bool demoteTessellationPointSize, const Bool demoteGeometryPointSize,
|
||||
const Bool emulateClipDistance, const Bool maskNativeClipDistance) {
|
||||
/* As we passed first stage compilation/linking,
|
||||
* we'll assume all the operations here should
|
||||
* pass. We may be able to employ some optimizations
|
||||
@@ -269,6 +280,51 @@ namespace MobileGL::MG_State::GLState {
|
||||
}
|
||||
artifacts.nativeFloat64 = keepFloat64;
|
||||
|
||||
// The clip-distance emulation, program-wide and BEFORE the sanitize chain, on glslang's
|
||||
// raw output. It cannot run after it: a stage whose only default-block uniform is the
|
||||
// reserved enables member (nothing in the application reads it - only this pass does)
|
||||
// has its whole MGL_GLOBAL_UBO deleted by ADCE, and the producer would then fall back
|
||||
// to clipping by every distance. Running first adds the read that keeps it alive. The
|
||||
// point-size demotion below places its carrier past these carriers.
|
||||
artifacts.clipDistanceEmulated = false;
|
||||
if (emulateClipDistance || maskNativeClipDistance) {
|
||||
ShaderCompiler::ClipDistanceEmulationOutcome outcome;
|
||||
if (!ShaderCompiler::EmulateClipDistanceForProgram(
|
||||
artifacts.generatedSpirv, handoff.shaderTypes, handoff.captureRequestsClipDistance,
|
||||
kClipDistanceCarrierLocationBudget, outcome, !deferOutputValidationForDirectVulkan,
|
||||
enableSpirvValidation, maskNativeClipDistance)) {
|
||||
DeferLog(std::format("ProgramObject {}: clip-distance emulation failed in the optimizer; the "
|
||||
"program keeps gl_ClipDistance, which this backend cannot host",
|
||||
externalIndex),
|
||||
FirstTimeWarnLevel(g_clipDistanceOptimizerFailureReported));
|
||||
} else if (outcome.emulated && maskNativeClipDistance) {
|
||||
// Native masking: the built-in is still hosted, only the enables are applied in
|
||||
// the shader. Not "emulated" for the artifact (nothing reads it differently).
|
||||
const Bool first = !g_clipDistanceEmulationReported.test_and_set(std::memory_order_relaxed);
|
||||
DeferLog(std::format("ProgramObject {}: GL_CLIP_DISTANCEi enables applied in the shader to {} "
|
||||
"native gl_ClipDistance output(s){}",
|
||||
externalIndex, outcome.distanceCount,
|
||||
outcome.enablesFound ? "" : "; no enables uniform, every distance clips"),
|
||||
first ? MOBILEGL_LOG_LEVEL_INFO : MOBILEGL_LOG_LEVEL_DEBUG);
|
||||
} else if (outcome.emulated) {
|
||||
artifacts.clipDistanceEmulated = true;
|
||||
// THE ARMING SIGNAL, INFO and latched: the integration lane that pins
|
||||
// MOBILEGL_CLIP_DISTANCE_EMULATION=1 asserts on this line, because a healthy
|
||||
// driver renders the same pixels whether the emulation ran or not.
|
||||
const Bool first = !g_clipDistanceEmulationReported.test_and_set(std::memory_order_relaxed);
|
||||
DeferLog(std::format("ProgramObject {}: gl_ClipDistance emulated ({} distance(s) carried to the "
|
||||
"fragment stage as varyings and clipped by discard{})",
|
||||
externalIndex, outcome.distanceCount,
|
||||
outcome.enablesFound ? "" : "; no enables uniform, every distance clips"),
|
||||
first ? MOBILEGL_LOG_LEVEL_INFO : MOBILEGL_LOG_LEVEL_DEBUG);
|
||||
} else if (!outcome.declineDetail.empty()) {
|
||||
DeferLog(std::format("ProgramObject {}: clip-distance emulation declined ({}); the program keeps "
|
||||
"gl_ClipDistance, which this backend cannot host",
|
||||
externalIndex, outcome.declineDetail),
|
||||
FirstTimeWarnLevel(g_clipDistanceDeclineReported));
|
||||
}
|
||||
}
|
||||
|
||||
// Linked SPIR-V generated, sanitize and optimize it
|
||||
Bool allOptimized = true;
|
||||
{
|
||||
@@ -348,6 +404,7 @@ namespace MobileGL::MG_State::GLState {
|
||||
artifacts.uniformOffsets.clear();
|
||||
artifacts.globalUboScratch.clear();
|
||||
artifacts.reservedNumSamplesOffset = ProgramObject::kInvalidUniformOffset;
|
||||
artifacts.reservedClipDistanceEnablesOffset = ProgramObject::kInvalidUniformOffset;
|
||||
// kInvalidUniformOffset marks locations that end up without global-UBO backing
|
||||
// (e.g. the optimizer eliminated every use of the uniform); the fallback pass
|
||||
// below gives those locations tail storage so glUniform* always has a target.
|
||||
@@ -386,6 +443,10 @@ namespace MobileGL::MG_State::GLState {
|
||||
// precisely so no application can address it - so the lookup below would find
|
||||
// nothing and log it as unbacked. Only the fragment stage declares it, and
|
||||
// every stage's copy sits at the same offset in the one shared global UBO.
|
||||
if (name == CLIP_DISTANCE_ENABLES_UNIFORM_NAME) {
|
||||
artifacts.reservedClipDistanceEnablesOffset = offset;
|
||||
continue;
|
||||
}
|
||||
if (name == NUM_SAMPLES_UNIFORM_NAME) {
|
||||
artifacts.reservedNumSamplesOffset = offset;
|
||||
MGLOG_D("ProgramObject %u: BuildGlobalUboRouting - reserved gl_NumSamples stand-in '%s' "
|
||||
|
||||
@@ -67,7 +67,8 @@ namespace MobileGL::MG_State::GLState {
|
||||
|
||||
void GenerateSpirv(const ProgramLinkTask::SpirvHandoff& handoff, Uint externalIndex,
|
||||
Bool deferOutputValidationForDirectVulkan, Bool enableSpirvValidation, Bool nativeFloat64,
|
||||
Bool demoteTessellationPointSize, Bool demoteGeometryPointSize);
|
||||
Bool demoteTessellationPointSize, Bool demoteGeometryPointSize, Bool emulateClipDistance,
|
||||
Bool maskNativeClipDistance);
|
||||
void BuildGlobalUboRouting(const ProgramLinkTask::SpirvHandoff& handoff, Uint externalIndex);
|
||||
|
||||
// Worker-side MGLOG replacement, replayed by the join on the GL thread. Same reason as
|
||||
|
||||
@@ -45,6 +45,7 @@ enable_testing()
|
||||
add_executable(
|
||||
SanityTest
|
||||
SanityTest.cpp
|
||||
SanityClipDistanceTest.cpp
|
||||
)
|
||||
target_link_libraries(
|
||||
SanityTest
|
||||
|
||||
@@ -531,9 +531,9 @@ TEST(ProgramArtifactsCodec, TheTablesVisitEveryMemberExceptTheLiveProgram) {
|
||||
// P7 OQ-8: 59 members, 58 visited. The 59th is still the live TProgram; the 58th is
|
||||
// storageBlocks, which follows the record arm's guard (ProgramArtifacts.h). esOrigin then
|
||||
// made it 60 and 59.
|
||||
EXPECT_EQ(ProgramArtifactsVisitedFieldCount<LinkArtifacts>(), 59u);
|
||||
EXPECT_EQ(ProgramArtifactsVisitedFieldCount<LinkArtifacts>(), 60u);
|
||||
EXPECT_EQ(ProgramArtifactsVisitedFieldCount<StorageBlockReflection>(), 3u);
|
||||
EXPECT_EQ(ProgramArtifactsVisitedFieldCount<SpirvArtifacts>(), 8u);
|
||||
EXPECT_EQ(ProgramArtifactsVisitedFieldCount<SpirvArtifacts>(), 10u);
|
||||
EXPECT_EQ(ProgramArtifactsVisitedFieldCount<ResourceReflection>(), 14u);
|
||||
EXPECT_EQ(ProgramArtifactsVisitedFieldCount<XfbVarying>(), 11u);
|
||||
EXPECT_EQ(ProgramArtifactsVisitedFieldCount<TypeFacts>(), 20u);
|
||||
|
||||
@@ -84,8 +84,8 @@ namespace {
|
||||
// P7 OQ-8 added LinkArtifacts::storageBlocks and its own three-field table; since P13 W5
|
||||
// the member follows the record arm (ProgramArtifacts.h), which every library builds.
|
||||
EXPECT_EQ(CountFields<StorageBlockReflection>(), 3u);
|
||||
EXPECT_EQ(CountFields<LinkArtifacts>(), 59u);
|
||||
EXPECT_EQ(CountFields<SpirvArtifacts>(), 8u);
|
||||
EXPECT_EQ(CountFields<LinkArtifacts>(), 60u);
|
||||
EXPECT_EQ(CountFields<SpirvArtifacts>(), 10u);
|
||||
}
|
||||
|
||||
// A const walk hands the visitor const references, a non-const walk mutable ones: the one
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
// MobileGL - MobileGL/MG_Test/SanityClipDistanceTest.cpp
|
||||
// Copyright (c) 2026 MobileGL-Dev
|
||||
// Licensed under the GNU Lesser General Public License v3.0:
|
||||
// https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
// https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
// SPDX-License-Identifier: LGPL-3.0-only
|
||||
// End of Source File Header
|
||||
//
|
||||
// The clip and cull distance limits each backend advertises: native where the device clips,
|
||||
// the emulated eight where it does not, and cull only where it is hosted.
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <MG_Backend/DirectGLES/BackendObject_DirectGLES.h>
|
||||
#include <MG_Backend/DirectVulkan/BackendObject_DirectVulkan.h>
|
||||
#include <MG_Util/BackendLoaders/OpenGL/Loader.h>
|
||||
|
||||
// maxClipDistances is a LIMIT every Vulkan device reports; declaring ClipDistance in a module
|
||||
// needs the shaderClipDistance FEATURE, which is separate and which VulkanRenderer enables only
|
||||
// where the physical device has it. Without the feature the device's limit is never forwarded:
|
||||
// the backend EMULATES the built-in instead (EmulateClipDistancePass), which is what lets it
|
||||
// answer GL 3.0's minimum of eight on every Mali device. With the feature it keeps its real
|
||||
// number and the native path.
|
||||
TEST(DirectVulkanSanity, GatesClipDistancesOnTheShaderClipDistanceFeature) {
|
||||
using namespace MobileGL;
|
||||
|
||||
MG_Backend::DirectVulkan::BackendObject_DirectVulkan backend;
|
||||
MG_External::VulkanCapabilities caps;
|
||||
caps.MaxClipDistances = 8;
|
||||
|
||||
caps.SupportsShaderClipDistance = false;
|
||||
caps.MaxClipDistances = 0; // what a Mali device reports alongside the missing feature
|
||||
backend.ApplyVulkanCapabilitiesForTesting(caps);
|
||||
EXPECT_TRUE(backend.GetDynamicParameters().EmulatesClipDistance);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxClipDistances, 8);
|
||||
|
||||
EXPECT_FALSE(backend.GetDynamicParameters().MasksNativeClipDistances);
|
||||
|
||||
caps.SupportsShaderClipDistance = true;
|
||||
caps.MaxClipDistances = 8;
|
||||
backend.ApplyVulkanCapabilitiesForTesting(caps);
|
||||
EXPECT_FALSE(backend.GetDynamicParameters().EmulatesClipDistance);
|
||||
// Vulkan has no per-distance enable state, so the native path masks in the shader.
|
||||
EXPECT_TRUE(backend.GetDynamicParameters().MasksNativeClipDistances);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxClipDistances, 8);
|
||||
}
|
||||
|
||||
// The cull half of the same contract. shaderCullDistance is a SEPARATE feature from
|
||||
// shaderClipDistance - VulkanRenderer enables each independently - so it gets its own gate, and
|
||||
// the combined limit is gated on either being present because GL 4.6 core 11.1.3.10 makes it at
|
||||
// least as large as both halves. These three used to be literal 8s inside BuildTBuiltInResource
|
||||
// with no device consulted at all, which let glslang accept a gl_CullDistance write that then
|
||||
// discarded every primitive it touched.
|
||||
TEST(DirectVulkanSanity, GatesCullDistancesOnTheShaderCullDistanceFeature) {
|
||||
using namespace MobileGL;
|
||||
|
||||
MG_Backend::DirectVulkan::BackendObject_DirectVulkan backend;
|
||||
MG_External::VulkanCapabilities caps;
|
||||
caps.MaxClipDistances = 8;
|
||||
caps.MaxCullDistances = 8;
|
||||
caps.MaxCombinedClipAndCullDistances = 8;
|
||||
|
||||
// Neither feature: cull stays zero (a per-primitive cull has no per-fragment substitute),
|
||||
// while clip is emulated, so the combined limit describes the emulated eight.
|
||||
caps.SupportsShaderClipDistance = false;
|
||||
caps.SupportsShaderCullDistance = false;
|
||||
backend.ApplyVulkanCapabilitiesForTesting(caps);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCullDistances, 0);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCombinedClipAndCullDistances, 8);
|
||||
|
||||
// Clip only: cull stays zero, and the combined limit still describes the clip capacity.
|
||||
caps.SupportsShaderClipDistance = true;
|
||||
backend.ApplyVulkanCapabilitiesForTesting(caps);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCullDistances, 0);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCombinedClipAndCullDistances, 8);
|
||||
|
||||
caps.SupportsShaderCullDistance = true;
|
||||
backend.ApplyVulkanCapabilitiesForTesting(caps);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCullDistances, 8);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCombinedClipAndCullDistances, 8);
|
||||
}
|
||||
|
||||
// DirectGLES reaches clip AND cull distances only through GL_EXT_clip_cull_distance, so the
|
||||
// loader leaves all three at zero without it and the backend forwards that verbatim. Zero is the
|
||||
// answer that stops a gl_CullDistance shader from reaching an ESSL compiler that would reject it.
|
||||
TEST(DirectGLESSanity, ForwardsTheProbedClipAndCullDistanceLimits) {
|
||||
using namespace MobileGL;
|
||||
|
||||
MG_Backend::DirectGLES::BackendObject_DirectGLES backend;
|
||||
MG_External::GLESCapabilities caps;
|
||||
backend.ApplyGLESCapabilitiesForTesting(caps);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxClipDistances, 0);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCullDistances, 0);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCombinedClipAndCullDistances, 0);
|
||||
|
||||
caps.SupportsClipDistance = true;
|
||||
caps.MaxClipDistances = 8;
|
||||
caps.MaxCullDistances = 8;
|
||||
caps.MaxCombinedClipAndCullDistances = 8;
|
||||
backend.ApplyGLESCapabilitiesForTesting(caps);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxClipDistances, 8);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCullDistances, 8);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCombinedClipAndCullDistances, 8);
|
||||
EXPECT_FALSE(backend.GetDynamicParameters().EmulatesClipDistance);
|
||||
}
|
||||
|
||||
// The loader's verdict that the driver cannot clip natively (no extension, or the POST clip probe
|
||||
// caught it misbehaving) turns the clip limit into the emulated eight - GL 3.0's minimum - while
|
||||
// cull keeps the driver's own answer: a per-primitive cull has no per-fragment substitute.
|
||||
TEST(DirectGLESSanity, AnEmulatingLoaderVerdictAdvertisesEightClipDistancesAndNoCull) {
|
||||
using namespace MobileGL;
|
||||
|
||||
MG_Backend::DirectGLES::BackendObject_DirectGLES backend;
|
||||
MG_External::GLESCapabilities caps; // no GL_EXT_clip_cull_distance, as on every Mali driver
|
||||
caps.EmulatesClipDistance = true;
|
||||
backend.ApplyGLESCapabilitiesForTesting(caps);
|
||||
EXPECT_TRUE(backend.GetDynamicParameters().EmulatesClipDistance);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxClipDistances, 8);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCullDistances, 0);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCombinedClipAndCullDistances, 8);
|
||||
}
|
||||
@@ -696,84 +696,6 @@ TEST(DirectGLESSanity, PreservesHostPerStageImageUniformLimits) {
|
||||
EXPECT_EQ(params.MaxComputeImageUniforms, 5);
|
||||
}
|
||||
|
||||
// maxClipDistances is a LIMIT every Vulkan device reports; declaring ClipDistance in a module
|
||||
// needs the shaderClipDistance FEATURE, which is separate and which VulkanRenderer enables only
|
||||
// where the physical device has it. Forwarding the limit without the feature advertises eight
|
||||
// clip planes no shader may use - the same shape as the image-uniform limits above, and the same
|
||||
// shape as the GL_EXT_clip_cull_distance lie on DirectGLES. Not a blanket zero: a device WITH the
|
||||
// feature keeps its real number.
|
||||
TEST(DirectVulkanSanity, GatesClipDistancesOnTheShaderClipDistanceFeature) {
|
||||
using namespace MobileGL;
|
||||
|
||||
MG_Backend::DirectVulkan::BackendObject_DirectVulkan backend;
|
||||
MG_External::VulkanCapabilities caps;
|
||||
caps.MaxClipDistances = 8;
|
||||
|
||||
caps.SupportsShaderClipDistance = false;
|
||||
backend.ApplyVulkanCapabilitiesForTesting(caps);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxClipDistances, 0);
|
||||
|
||||
caps.SupportsShaderClipDistance = true;
|
||||
backend.ApplyVulkanCapabilitiesForTesting(caps);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxClipDistances, 8);
|
||||
}
|
||||
|
||||
// The cull half of the same contract. shaderCullDistance is a SEPARATE feature from
|
||||
// shaderClipDistance - VulkanRenderer enables each independently - so it gets its own gate, and
|
||||
// the combined limit is gated on either being present because GL 4.6 core 11.1.3.10 makes it at
|
||||
// least as large as both halves. These three used to be literal 8s inside BuildTBuiltInResource
|
||||
// with no device consulted at all, which let glslang accept a gl_CullDistance write that then
|
||||
// discarded every primitive it touched.
|
||||
TEST(DirectVulkanSanity, GatesCullDistancesOnTheShaderCullDistanceFeature) {
|
||||
using namespace MobileGL;
|
||||
|
||||
MG_Backend::DirectVulkan::BackendObject_DirectVulkan backend;
|
||||
MG_External::VulkanCapabilities caps;
|
||||
caps.MaxClipDistances = 8;
|
||||
caps.MaxCullDistances = 8;
|
||||
caps.MaxCombinedClipAndCullDistances = 8;
|
||||
|
||||
caps.SupportsShaderClipDistance = false;
|
||||
caps.SupportsShaderCullDistance = false;
|
||||
backend.ApplyVulkanCapabilitiesForTesting(caps);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCullDistances, 0);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCombinedClipAndCullDistances, 0);
|
||||
|
||||
// Clip only: cull stays zero, and the combined limit still describes the clip capacity.
|
||||
caps.SupportsShaderClipDistance = true;
|
||||
backend.ApplyVulkanCapabilitiesForTesting(caps);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCullDistances, 0);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCombinedClipAndCullDistances, 8);
|
||||
|
||||
caps.SupportsShaderCullDistance = true;
|
||||
backend.ApplyVulkanCapabilitiesForTesting(caps);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCullDistances, 8);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCombinedClipAndCullDistances, 8);
|
||||
}
|
||||
|
||||
// DirectGLES reaches clip AND cull distances only through GL_EXT_clip_cull_distance, so the
|
||||
// loader leaves all three at zero without it and the backend forwards that verbatim. Zero is the
|
||||
// answer that stops a gl_CullDistance shader from reaching an ESSL compiler that would reject it.
|
||||
TEST(DirectGLESSanity, ForwardsTheProbedClipAndCullDistanceLimits) {
|
||||
using namespace MobileGL;
|
||||
|
||||
MG_Backend::DirectGLES::BackendObject_DirectGLES backend;
|
||||
MG_External::GLESCapabilities caps;
|
||||
backend.ApplyGLESCapabilitiesForTesting(caps);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxClipDistances, 0);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCullDistances, 0);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCombinedClipAndCullDistances, 0);
|
||||
|
||||
caps.SupportsClipDistance = true;
|
||||
caps.MaxClipDistances = 8;
|
||||
caps.MaxCullDistances = 8;
|
||||
caps.MaxCombinedClipAndCullDistances = 8;
|
||||
backend.ApplyGLESCapabilitiesForTesting(caps);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxClipDistances, 8);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCullDistances, 8);
|
||||
EXPECT_EQ(backend.GetDynamicParameters().MaxCombinedClipAndCullDistances, 8);
|
||||
}
|
||||
|
||||
// GL_LAYER_PROVOKING_VERTEX / GL_VIEWPORT_INDEX_PROVOKING_VERTEX were a hard-coded
|
||||
// GL_LAST_VERTEX_CONVENTION for both backends, derived from nothing, and wrong on both test
|
||||
// devices in opposite directions. DirectGLES now forwards what its loader resolved; DirectVulkan
|
||||
|
||||
@@ -31,6 +31,7 @@ add_executable(
|
||||
WidenImageFormatsTest.cpp
|
||||
GlslangCaptureTest.cpp
|
||||
DemotePointSizeTest.cpp
|
||||
EmulateClipDistanceTest.cpp
|
||||
FragCoordYFlipTest.cpp
|
||||
PixelCenterIntegerTest.cpp
|
||||
ZeroLayerOutputTest.cpp
|
||||
|
||||
@@ -0,0 +1,441 @@
|
||||
// MobileGL - MobileGL/MG_Test/ShaderTranspiler/EmulateClipDistanceTest.cpp
|
||||
// Copyright (c) 2025-2026 MobileGL-Dev
|
||||
// Licensed under the GNU Lesser General Public License v3.0:
|
||||
// https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
// https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
// SPDX-License-Identifier: LGPL-3.0-only
|
||||
// End of Source File Header
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "Includes.h"
|
||||
#include "Init.h"
|
||||
#include <MG_Util/ShaderTranspiler/CompileEnv.h>
|
||||
#include <MG_Util/ShaderTranspiler/ShaderCompiler.h>
|
||||
#include <MG_Util/ShaderTranspiler/ShaderSourceProcessor.h>
|
||||
#include <MG_Util/ShaderTranspiler/SpvcSession.h>
|
||||
#include <MG_Util/ShaderTranspiler/Types.h>
|
||||
|
||||
#include "spirv-tools/libspirv.hpp"
|
||||
|
||||
using namespace MobileGL;
|
||||
using MobileGL::MG_Util::ShaderTranspiler::CompileEnv;
|
||||
using MobileGL::MG_Util::ShaderTranspiler::SessionUsageBit;
|
||||
using MobileGL::MG_Util::ShaderTranspiler::ShaderCompiler;
|
||||
using MobileGL::MG_Util::ShaderTranspiler::SpvcSession;
|
||||
|
||||
namespace {
|
||||
// The exact bytes ProgramSpirvTask hands the emulation: linked, generated, sanitized.
|
||||
Vector<Vector<Uint32>> CompileProgramToSpirv(const Vector<Pair<GLenum, const char*>>& stages,
|
||||
Vector<GLenum>& types) {
|
||||
using namespace MG_Util::ShaderTranspiler;
|
||||
Vector<SharedPtr<glslang::TShader>> shaders;
|
||||
types.clear();
|
||||
for (const auto& [stage, source] : stages) {
|
||||
ShaderAttrib shaderAttrib{.shaderType = stage, .sourceStr = source};
|
||||
auto shaderResult = ShaderCompiler::CompileShader(shaderAttrib);
|
||||
EXPECT_TRUE(shaderResult) << (shaderResult ? String{} : shaderResult.error().log);
|
||||
if (!shaderResult) return {};
|
||||
shaders.push_back(shaderResult.value());
|
||||
types.push_back(stage);
|
||||
}
|
||||
ProgramAttrib programAttrib{.shaders = shaders};
|
||||
auto programResult = ShaderCompiler::LinkProgram(programAttrib);
|
||||
EXPECT_TRUE(programResult) << (programResult ? String{} : programResult.error().log);
|
||||
if (!programResult) return {};
|
||||
ProgramBinaryAttrib binaryAttrib{.shaderTypes = types, .program = *programResult.value()};
|
||||
auto binaryResult = ShaderCompiler::GetSpirvBinaryFromProgram(binaryAttrib);
|
||||
EXPECT_TRUE(binaryResult) << (binaryResult ? String{} : binaryResult.error().log);
|
||||
if (!binaryResult) return {};
|
||||
// RAW glslang output: ProgramSpirvTask runs the emulation BEFORE the sanitize chain.
|
||||
return Move(binaryResult.value());
|
||||
}
|
||||
|
||||
// What ProgramSpirvTask does right after the emulation, so every validity / transpile
|
||||
// assertion is made against the bytes the backends really receive.
|
||||
void SanitizeAll(Vector<Vector<Uint32>>& modules) {
|
||||
for (auto& module : modules) {
|
||||
EXPECT_TRUE(ShaderCompiler::SanitizeAndOptimizeBinary(module, module, true, true));
|
||||
}
|
||||
}
|
||||
|
||||
String Disassemble(const Vector<Uint32>& spirv) {
|
||||
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_1);
|
||||
String text;
|
||||
EXPECT_TRUE(tools.Disassemble(spirv, &text));
|
||||
return text;
|
||||
}
|
||||
|
||||
String Transpile(const Vector<Uint32>& spirv) {
|
||||
SpvcSession session(spirv, SessionUsageBit::Transpile);
|
||||
auto essl = ShaderCompiler::DecompileShader(session);
|
||||
EXPECT_TRUE(essl) << (essl ? String{} : essl.error().log);
|
||||
return essl ? essl.value() : String{};
|
||||
}
|
||||
|
||||
Bool Validates(const Vector<Uint32>& spirv) {
|
||||
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_1);
|
||||
String messages;
|
||||
tools.SetMessageConsumer([&](spv_message_level_t, const char*, const spv_position_t&, const char* message) {
|
||||
messages += message;
|
||||
messages += '\n';
|
||||
});
|
||||
const Bool ok = tools.Validate(spirv);
|
||||
EXPECT_TRUE(ok) << messages;
|
||||
return ok;
|
||||
}
|
||||
|
||||
Bool Contains(const String& haystack, const char* needle) {
|
||||
return haystack.find(needle) != String::npos;
|
||||
}
|
||||
|
||||
// The shape from the field report (Mali-G615, Espryt): a #version 330 core vertex stage
|
||||
// writing all eight distances from uniform planes. The enables uniform is what
|
||||
// InjectClipDistanceEnablesShim adds on an emulating backend; spelled out here because a
|
||||
// unit test has no backend to arm the shim.
|
||||
const char* kEightPlaneVertex = R"(#version 330 core
|
||||
uniform uint mg_ClipDistanceEnables;
|
||||
uniform vec4 u_ClipPlane[8];
|
||||
uniform mat4 u_ModelView;
|
||||
in vec4 a_Position;
|
||||
void main() {
|
||||
vec4 eyePos = u_ModelView * a_Position;
|
||||
gl_Position = eyePos;
|
||||
gl_ClipDistance[0] = dot(u_ClipPlane[0], eyePos);
|
||||
gl_ClipDistance[1] = dot(u_ClipPlane[1], eyePos);
|
||||
gl_ClipDistance[2] = dot(u_ClipPlane[2], eyePos);
|
||||
gl_ClipDistance[3] = dot(u_ClipPlane[3], eyePos);
|
||||
gl_ClipDistance[4] = dot(u_ClipPlane[4], eyePos);
|
||||
gl_ClipDistance[5] = dot(u_ClipPlane[5], eyePos);
|
||||
gl_ClipDistance[6] = dot(u_ClipPlane[6], eyePos);
|
||||
gl_ClipDistance[7] = dot(u_ClipPlane[7], eyePos);
|
||||
}
|
||||
)";
|
||||
|
||||
const char* kPlainFragment = R"(#version 330 core
|
||||
out vec4 o_color;
|
||||
void main() { o_color = vec4(0.0, 1.0, 0.0, 1.0); }
|
||||
)";
|
||||
|
||||
constexpr Uint32 kBudget = 32;
|
||||
} // namespace
|
||||
|
||||
TEST(EmulateClipDistance, TheReportedEightPlaneVertexStageClipsByDiscard) {
|
||||
Vector<GLenum> types;
|
||||
auto modules =
|
||||
CompileProgramToSpirv({{GL_VERTEX_SHADER, kEightPlaneVertex}, {GL_FRAGMENT_SHADER, kPlainFragment}}, types);
|
||||
ASSERT_EQ(modules.size(), 2u);
|
||||
ASSERT_TRUE(Contains(Disassemble(modules[0]), "OpCapability ClipDistance"));
|
||||
|
||||
ShaderCompiler::ClipDistanceEmulationOutcome outcome;
|
||||
ASSERT_TRUE(ShaderCompiler::EmulateClipDistanceForProgram(modules, types, false, kBudget, outcome, true, true));
|
||||
SanitizeAll(modules);
|
||||
EXPECT_TRUE(outcome.emulated) << outcome.declineDetail;
|
||||
EXPECT_EQ(outcome.distanceCount, 8u);
|
||||
EXPECT_TRUE(outcome.enablesFound);
|
||||
|
||||
for (const auto& module : modules) {
|
||||
EXPECT_TRUE(Validates(module));
|
||||
EXPECT_FALSE(Contains(Disassemble(module), "OpCapability ClipDistance"));
|
||||
}
|
||||
const String vertexAsm = Disassemble(modules[0]);
|
||||
EXPECT_TRUE(Contains(vertexAsm, "mg_ClipDistanceEmu0"));
|
||||
EXPECT_TRUE(Contains(vertexAsm, "mg_ClipDistanceEmu1"));
|
||||
EXPECT_TRUE(Contains(vertexAsm, "OpBitwiseAnd"));
|
||||
const String fragmentAsm = Disassemble(modules[1]);
|
||||
EXPECT_TRUE(Contains(fragmentAsm, "OpKill"));
|
||||
EXPECT_TRUE(Contains(fragmentAsm, "mg_ClipDistanceEmu1"));
|
||||
|
||||
// The ES hop must no longer need GL_EXT_clip_cull_distance anywhere.
|
||||
const String vertexEssl = Transpile(modules[0]);
|
||||
const String fragmentEssl = Transpile(modules[1]);
|
||||
EXPECT_FALSE(Contains(vertexEssl, "gl_ClipDistance")) << vertexEssl;
|
||||
EXPECT_FALSE(Contains(vertexEssl, "clip_cull_distance")) << vertexEssl;
|
||||
EXPECT_TRUE(Contains(vertexEssl, "mg_ClipDistanceEmu0")) << vertexEssl;
|
||||
EXPECT_TRUE(Contains(fragmentEssl, "discard")) << fragmentEssl;
|
||||
EXPECT_FALSE(Contains(fragmentEssl, "clip_cull_distance")) << fragmentEssl;
|
||||
}
|
||||
|
||||
TEST(EmulateClipDistance, OneDistanceUsesOneCarrierAndNoEnablesMeansAllEnabled) {
|
||||
const char* vertex = R"(#version 330 core
|
||||
void main() {
|
||||
gl_Position = vec4(float(gl_VertexID), 0.0, 0.0, 1.0);
|
||||
gl_ClipDistance[0] = gl_Position.x;
|
||||
}
|
||||
)";
|
||||
Vector<GLenum> types;
|
||||
auto modules = CompileProgramToSpirv({{GL_VERTEX_SHADER, vertex}, {GL_FRAGMENT_SHADER, kPlainFragment}}, types);
|
||||
ASSERT_EQ(modules.size(), 2u);
|
||||
ShaderCompiler::ClipDistanceEmulationOutcome outcome;
|
||||
ASSERT_TRUE(ShaderCompiler::EmulateClipDistanceForProgram(modules, types, false, kBudget, outcome, true, true));
|
||||
SanitizeAll(modules);
|
||||
EXPECT_TRUE(outcome.emulated) << outcome.declineDetail;
|
||||
EXPECT_EQ(outcome.distanceCount, 1u);
|
||||
EXPECT_FALSE(outcome.enablesFound);
|
||||
for (const auto& module : modules)
|
||||
EXPECT_TRUE(Validates(module));
|
||||
const String vertexAsm = Disassemble(modules[0]);
|
||||
EXPECT_TRUE(Contains(vertexAsm, "mg_ClipDistanceEmu0"));
|
||||
EXPECT_FALSE(Contains(vertexAsm, "mg_ClipDistanceEmu1"));
|
||||
}
|
||||
|
||||
TEST(EmulateClipDistance, AProgramThatNeverClipsIsUntouched) {
|
||||
const char* vertex = R"(#version 330 core
|
||||
void main() { gl_Position = vec4(float(gl_VertexID), 0.0, 0.0, 1.0); }
|
||||
)";
|
||||
Vector<GLenum> types;
|
||||
auto modules = CompileProgramToSpirv({{GL_VERTEX_SHADER, vertex}, {GL_FRAGMENT_SHADER, kPlainFragment}}, types);
|
||||
ASSERT_EQ(modules.size(), 2u);
|
||||
const auto before = modules;
|
||||
ShaderCompiler::ClipDistanceEmulationOutcome outcome;
|
||||
ASSERT_TRUE(ShaderCompiler::EmulateClipDistanceForProgram(modules, types, false, kBudget, outcome, true, true));
|
||||
EXPECT_FALSE(outcome.emulated);
|
||||
EXPECT_TRUE(outcome.declineDetail.empty());
|
||||
EXPECT_EQ(modules, before);
|
||||
}
|
||||
|
||||
TEST(EmulateClipDistance, AGeometryStageProducesBeforeEveryEmitAndTheVertexWriteIsRedirected) {
|
||||
const char* vertex = R"(#version 330 core
|
||||
uniform uint mg_ClipDistanceEnables;
|
||||
void main() {
|
||||
gl_Position = vec4(float(gl_VertexID), 0.0, 0.0, 1.0);
|
||||
gl_ClipDistance[0] = -1.0;
|
||||
}
|
||||
)";
|
||||
const char* geometry = R"(#version 330 core
|
||||
uniform uint mg_ClipDistanceEnables;
|
||||
layout(triangles) in;
|
||||
layout(triangle_strip, max_vertices = 3) out;
|
||||
void main() {
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
gl_Position = gl_in[i].gl_Position;
|
||||
gl_ClipDistance[0] = gl_in[i].gl_Position.x;
|
||||
gl_ClipDistance[1] = 1.0;
|
||||
EmitVertex();
|
||||
}
|
||||
EndPrimitive();
|
||||
}
|
||||
)";
|
||||
Vector<GLenum> types;
|
||||
auto modules = CompileProgramToSpirv(
|
||||
{{GL_VERTEX_SHADER, vertex}, {GL_GEOMETRY_SHADER, geometry}, {GL_FRAGMENT_SHADER, kPlainFragment}}, types);
|
||||
ASSERT_EQ(modules.size(), 3u);
|
||||
ShaderCompiler::ClipDistanceEmulationOutcome outcome;
|
||||
ASSERT_TRUE(ShaderCompiler::EmulateClipDistanceForProgram(modules, types, false, kBudget, outcome, true, true));
|
||||
SanitizeAll(modules);
|
||||
EXPECT_TRUE(outcome.emulated) << outcome.declineDetail;
|
||||
EXPECT_EQ(outcome.distanceCount, 2u);
|
||||
for (const auto& module : modules) {
|
||||
EXPECT_TRUE(Validates(module));
|
||||
EXPECT_FALSE(Contains(Disassemble(module), "OpCapability ClipDistance"));
|
||||
}
|
||||
// The vertex stage is not the producer: redirected, no carriers.
|
||||
EXPECT_FALSE(Contains(Disassemble(modules[0]), "mg_ClipDistanceEmu0"));
|
||||
EXPECT_TRUE(Contains(Disassemble(modules[1]), "mg_ClipDistanceEmu0"));
|
||||
}
|
||||
|
||||
TEST(EmulateClipDistance, AFragmentStageReadOfGlClipDistanceIsServedFromTheCarriers) {
|
||||
const char* vertex = R"(#version 330 core
|
||||
void main() {
|
||||
gl_Position = vec4(float(gl_VertexID), 0.0, 0.0, 1.0);
|
||||
gl_ClipDistance[0] = 0.5;
|
||||
}
|
||||
)";
|
||||
const char* fragment = R"(#version 330 core
|
||||
out vec4 o_color;
|
||||
void main() { o_color = vec4(gl_ClipDistance[0], 1.0, 0.0, 1.0); }
|
||||
)";
|
||||
Vector<GLenum> types;
|
||||
auto modules = CompileProgramToSpirv({{GL_VERTEX_SHADER, vertex}, {GL_FRAGMENT_SHADER, fragment}}, types);
|
||||
ASSERT_EQ(modules.size(), 2u);
|
||||
ShaderCompiler::ClipDistanceEmulationOutcome outcome;
|
||||
ASSERT_TRUE(ShaderCompiler::EmulateClipDistanceForProgram(modules, types, false, kBudget, outcome, true, true));
|
||||
SanitizeAll(modules);
|
||||
EXPECT_TRUE(outcome.emulated) << outcome.declineDetail;
|
||||
for (const auto& module : modules) {
|
||||
EXPECT_TRUE(Validates(module));
|
||||
EXPECT_FALSE(Contains(Disassemble(module), "OpCapability ClipDistance"));
|
||||
}
|
||||
const String fragmentEssl = Transpile(modules[1]);
|
||||
EXPECT_FALSE(Contains(fragmentEssl, "gl_ClipDistance")) << fragmentEssl;
|
||||
EXPECT_TRUE(Contains(fragmentEssl, "mg_ClipDistanceShadow")) << fragmentEssl;
|
||||
}
|
||||
|
||||
TEST(EmulateClipDistance, AGlInReadDeclinesAndLeavesEveryModuleAlone) {
|
||||
const char* vertex = R"(#version 330 core
|
||||
void main() {
|
||||
gl_Position = vec4(float(gl_VertexID), 0.0, 0.0, 1.0);
|
||||
gl_ClipDistance[0] = 1.0;
|
||||
}
|
||||
)";
|
||||
const char* geometry = R"(#version 330 core
|
||||
layout(points) in;
|
||||
layout(points, max_vertices = 1) out;
|
||||
void main() {
|
||||
gl_Position = gl_in[0].gl_Position + vec4(gl_in[0].gl_ClipDistance[0]);
|
||||
EmitVertex();
|
||||
}
|
||||
)";
|
||||
Vector<GLenum> types;
|
||||
auto modules = CompileProgramToSpirv(
|
||||
{{GL_VERTEX_SHADER, vertex}, {GL_GEOMETRY_SHADER, geometry}, {GL_FRAGMENT_SHADER, kPlainFragment}}, types);
|
||||
ASSERT_EQ(modules.size(), 3u);
|
||||
const auto before = modules;
|
||||
ShaderCompiler::ClipDistanceEmulationOutcome outcome;
|
||||
ASSERT_TRUE(ShaderCompiler::EmulateClipDistanceForProgram(modules, types, false, kBudget, outcome, true, true));
|
||||
EXPECT_FALSE(outcome.emulated);
|
||||
EXPECT_NE(outcome.declineDetail.find("gl_in"), String::npos) << outcome.declineDetail;
|
||||
EXPECT_EQ(modules, before);
|
||||
}
|
||||
|
||||
TEST(EmulateClipDistance, ACaptureOfGlClipDistanceDeclines) {
|
||||
Vector<GLenum> types;
|
||||
auto modules =
|
||||
CompileProgramToSpirv({{GL_VERTEX_SHADER, kEightPlaneVertex}, {GL_FRAGMENT_SHADER, kPlainFragment}}, types);
|
||||
ASSERT_EQ(modules.size(), 2u);
|
||||
const auto before = modules;
|
||||
ShaderCompiler::ClipDistanceEmulationOutcome outcome;
|
||||
ASSERT_TRUE(ShaderCompiler::EmulateClipDistanceForProgram(modules, types, true, kBudget, outcome, true, true));
|
||||
EXPECT_FALSE(outcome.emulated);
|
||||
EXPECT_FALSE(outcome.declineDetail.empty());
|
||||
EXPECT_EQ(modules, before);
|
||||
}
|
||||
|
||||
// End to end through the source pipeline: the shim's uniform must survive glslang and the
|
||||
// sanitize chain so the producer finds it. Without this the draw path's mask has nowhere to
|
||||
// land and every distance silently clips (the all-enabled fallback).
|
||||
TEST(EmulateClipDistance, TheShimmedEnablesUniformReachesTheProducer) {
|
||||
CompileEnv emulating{};
|
||||
emulating.backend = BackendType::DirectGLES;
|
||||
emulating.params.EmulatesClipDistance = true;
|
||||
String vertex = R"(#version 400 core
|
||||
out float gl_ClipDistance[2];
|
||||
void main() {
|
||||
vec2 positions[3] = vec2[3](vec2(-1.0, -1.0), vec2(3.0, -1.0), vec2(-1.0, 3.0));
|
||||
vec2 p = positions[gl_VertexID];
|
||||
gl_Position = vec4(p, 0.0, 1.0);
|
||||
gl_ClipDistance[0] = p.x;
|
||||
gl_ClipDistance[1] = p.y;
|
||||
}
|
||||
)";
|
||||
String fragment = "#version 400 core\nout vec4 o;\nvoid main() { o = vec4(1.0); }\n";
|
||||
MG_Util::ShaderTranspiler::PreprocessShaderSource(ShaderStage::Vertex, vertex, emulating);
|
||||
MG_Util::ShaderTranspiler::PreprocessShaderSource(ShaderStage::Fragment, fragment, emulating);
|
||||
ASSERT_TRUE(Contains(vertex, MG_Util::ShaderTranspiler::CLIP_DISTANCE_ENABLES_UNIFORM_NAME)) << vertex;
|
||||
Vector<GLenum> types;
|
||||
auto modules =
|
||||
CompileProgramToSpirv({{GL_VERTEX_SHADER, vertex.c_str()}, {GL_FRAGMENT_SHADER, fragment.c_str()}}, types);
|
||||
ASSERT_EQ(modules.size(), 2u);
|
||||
EXPECT_TRUE(Contains(Disassemble(modules[0]), "mg_ClipDistanceEnables")) << Disassemble(modules[0]);
|
||||
ShaderCompiler::ClipDistanceEmulationOutcome outcome;
|
||||
ASSERT_TRUE(ShaderCompiler::EmulateClipDistanceForProgram(modules, types, false, kBudget, outcome, true, true));
|
||||
EXPECT_TRUE(outcome.emulated) << outcome.declineDetail;
|
||||
EXPECT_TRUE(outcome.enablesFound) << vertex;
|
||||
// And the read the producer added keeps the member alive through the sanitize chain.
|
||||
SanitizeAll(modules);
|
||||
for (const auto& module : modules)
|
||||
EXPECT_TRUE(Validates(module));
|
||||
EXPECT_TRUE(Contains(Disassemble(modules[0]), "mg_ClipDistanceEnables"));
|
||||
}
|
||||
|
||||
// Native masking (Vulkan with shaderClipDistance): the producer keeps the built-in and the
|
||||
// capability, writes `enabled(i) ? d[i] : 1.0` back into it, and nothing else changes - no
|
||||
// carriers, no discard, the fragment stage byte-identical.
|
||||
TEST(EmulateClipDistance, NativeMaskingWritesTheMaskedDistancesBackIntoTheBuiltIn) {
|
||||
Vector<GLenum> types;
|
||||
auto modules =
|
||||
CompileProgramToSpirv({{GL_VERTEX_SHADER, kEightPlaneVertex}, {GL_FRAGMENT_SHADER, kPlainFragment}}, types);
|
||||
ASSERT_EQ(modules.size(), 2u);
|
||||
const auto fragmentBefore = modules[1];
|
||||
ShaderCompiler::ClipDistanceEmulationOutcome outcome;
|
||||
ASSERT_TRUE(ShaderCompiler::EmulateClipDistanceForProgram(modules, types, false, kBudget, outcome, true, true,
|
||||
/*nativeMask=*/true));
|
||||
EXPECT_TRUE(outcome.emulated) << outcome.declineDetail;
|
||||
EXPECT_EQ(outcome.distanceCount, 8u);
|
||||
EXPECT_TRUE(outcome.enablesFound);
|
||||
EXPECT_EQ(modules[1], fragmentBefore);
|
||||
const String vertexRaw = Disassemble(modules[0]);
|
||||
EXPECT_TRUE(Contains(vertexRaw, "OpCapability ClipDistance"));
|
||||
EXPECT_TRUE(Contains(vertexRaw, "OpBitwiseAnd"));
|
||||
EXPECT_FALSE(Contains(vertexRaw, "mg_ClipDistanceEmu0"));
|
||||
SanitizeAll(modules);
|
||||
for (const auto& module : modules)
|
||||
EXPECT_TRUE(Validates(module));
|
||||
EXPECT_FALSE(Contains(Disassemble(modules[1]), "OpKill"));
|
||||
}
|
||||
|
||||
// Native masking leaves gl_in[] reads native: a geometry stage that reads the vertex stage's
|
||||
// distances and re-emits them is the producer, and the vertex stage is untouched.
|
||||
TEST(EmulateClipDistance, NativeMaskingKeepsGlInReadsAndMasksOnlyTheProducer) {
|
||||
const char* vertex = R"(#version 330 core
|
||||
void main() {
|
||||
gl_Position = vec4(float(gl_VertexID), 0.0, 0.0, 1.0);
|
||||
gl_ClipDistance[0] = gl_Position.x;
|
||||
}
|
||||
)";
|
||||
const char* geometry = R"(#version 330 core
|
||||
uniform uint mg_ClipDistanceEnables;
|
||||
layout(triangles) in;
|
||||
layout(triangle_strip, max_vertices = 3) out;
|
||||
void main() {
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
gl_Position = gl_in[i].gl_Position;
|
||||
gl_ClipDistance[0] = gl_in[i].gl_ClipDistance[0];
|
||||
EmitVertex();
|
||||
}
|
||||
}
|
||||
)";
|
||||
Vector<GLenum> types;
|
||||
auto modules = CompileProgramToSpirv(
|
||||
{{GL_VERTEX_SHADER, vertex}, {GL_GEOMETRY_SHADER, geometry}, {GL_FRAGMENT_SHADER, kPlainFragment}}, types);
|
||||
ASSERT_EQ(modules.size(), 3u);
|
||||
const auto vertexBefore = modules[0];
|
||||
ShaderCompiler::ClipDistanceEmulationOutcome outcome;
|
||||
ASSERT_TRUE(ShaderCompiler::EmulateClipDistanceForProgram(modules, types, false, kBudget, outcome, true, true,
|
||||
/*nativeMask=*/true));
|
||||
EXPECT_TRUE(outcome.emulated) << outcome.declineDetail;
|
||||
EXPECT_TRUE(outcome.enablesFound);
|
||||
EXPECT_EQ(modules[0], vertexBefore);
|
||||
SanitizeAll(modules);
|
||||
for (const auto& module : modules)
|
||||
EXPECT_TRUE(Validates(module));
|
||||
}
|
||||
|
||||
TEST(EmulateClipDistance, TheEnablesShimArmsOnlyOnAnEmulatingBackendAndOnlyWhereItIsNamed) {
|
||||
CompileEnv emulating{};
|
||||
emulating.backend = BackendType::DirectGLES;
|
||||
emulating.params.EmulatesClipDistance = true;
|
||||
CompileEnv native = emulating;
|
||||
native.params.EmulatesClipDistance = false;
|
||||
|
||||
const String vertex = "#version 330 core\nvoid main() { gl_Position = vec4(0.0); gl_ClipDistance[0] = 1.0; }\n";
|
||||
const String fragment = "#version 330 core\nout vec4 o; void main() { o = vec4(1.0); }\n";
|
||||
const String nonClipping = "#version 330 core\nvoid main() { gl_Position = vec4(0.0); }\n";
|
||||
|
||||
String source = vertex;
|
||||
MG_Util::ShaderTranspiler::PreprocessShaderSource(ShaderStage::Vertex, source, emulating);
|
||||
EXPECT_TRUE(Contains(source, MG_Util::ShaderTranspiler::CLIP_DISTANCE_ENABLES_UNIFORM_NAME)) << source;
|
||||
|
||||
source = vertex;
|
||||
MG_Util::ShaderTranspiler::PreprocessShaderSource(ShaderStage::Vertex, source, native);
|
||||
EXPECT_FALSE(Contains(source, MG_Util::ShaderTranspiler::CLIP_DISTANCE_ENABLES_UNIFORM_NAME)) << source;
|
||||
|
||||
source = fragment;
|
||||
MG_Util::ShaderTranspiler::PreprocessShaderSource(ShaderStage::Fragment, source, emulating);
|
||||
EXPECT_FALSE(Contains(source, MG_Util::ShaderTranspiler::CLIP_DISTANCE_ENABLES_UNIFORM_NAME)) << source;
|
||||
|
||||
source = nonClipping;
|
||||
MG_Util::ShaderTranspiler::PreprocessShaderSource(ShaderStage::Vertex, source, emulating);
|
||||
EXPECT_FALSE(Contains(source, MG_Util::ShaderTranspiler::CLIP_DISTANCE_ENABLES_UNIFORM_NAME)) << source;
|
||||
|
||||
// A native backend that masks in the shader (Magma) needs the uniform too.
|
||||
CompileEnv masking = native;
|
||||
masking.backend = BackendType::DirectVulkan;
|
||||
masking.params.MasksNativeClipDistances = true;
|
||||
source = vertex;
|
||||
MG_Util::ShaderTranspiler::PreprocessShaderSource(ShaderStage::Vertex, source, masking);
|
||||
EXPECT_TRUE(Contains(source, MG_Util::ShaderTranspiler::CLIP_DISTANCE_ENABLES_UNIFORM_NAME)) << source;
|
||||
}
|
||||
@@ -1204,18 +1204,58 @@ TEST_F(TextureTest, TextureParameterRejectsProxyTargetsThatGetTexLevelParameterS
|
||||
ExpectSingleGlError(GL_INVALID_ENUM);
|
||||
|
||||
// The level query keeps its proxy support - that is the entire point of a proxy texture, and the
|
||||
// predicate deliberately does not gate it. Only the ERROR is asserted, not the width: MobileGL
|
||||
// does not currently report a proxy level's dimensions back (it answers 0), which is a separate
|
||||
// pre-existing gap in GetTexLevelParameter and not something this predicate decides. What
|
||||
// matters here is that the query is not turned into GL_INVALID_ENUM alongside the setters.
|
||||
// predicate deliberately does not gate it: the query answers the size the proxy was given
|
||||
// instead of turning into GL_INVALID_ENUM alongside the setters.
|
||||
GLint proxyWidth = -1;
|
||||
MG_Impl::GLImpl::GetTexLevelParameteriv(GL_PROXY_TEXTURE_2D, 0, GL_TEXTURE_WIDTH, &proxyWidth);
|
||||
EXPECT_EQ(MG_Impl::GLImpl::GetError(), GL_NO_ERROR)
|
||||
<< "the proxy target must still be accepted by GetTexLevelParameter";
|
||||
EXPECT_EQ(proxyWidth, 4) << "a proxy level reports the width it was given";
|
||||
|
||||
DrainPendingGlErrors();
|
||||
}
|
||||
|
||||
// GL 4.6 core 8.22: a proxy level reports the size it was given while the implementation can hold
|
||||
// it, and all zeros - with no error - once a dimension passes the limit at that level. Applications
|
||||
// probe GL_MAX_TEXTURE_SIZE this way, halving from the largest size until the width comes back.
|
||||
TEST_F(TextureTest, ProxyTextureLevelReportsRequestedSizeOrZerosPastTheLimit) {
|
||||
GLint maxTextureSize = 0;
|
||||
MG_Impl::GLImpl::GetIntegerv(GL_MAX_TEXTURE_SIZE, &maxTextureSize);
|
||||
ASSERT_GT(maxTextureSize, 1);
|
||||
const auto levelSize = [](GLenum target, GLint level, GLenum pname) {
|
||||
GLint value = -1;
|
||||
MG_Impl::GLImpl::GetTexLevelParameteriv(target, level, pname, &value);
|
||||
return value;
|
||||
};
|
||||
|
||||
MG_Impl::GLImpl::TexImage2D(GL_PROXY_TEXTURE_2D, 0, GL_RGBA8, maxTextureSize, 16, 0, GL_RGBA, GL_UNSIGNED_BYTE,
|
||||
nullptr);
|
||||
EXPECT_EQ(levelSize(GL_PROXY_TEXTURE_2D, 0, GL_TEXTURE_WIDTH), maxTextureSize);
|
||||
EXPECT_EQ(levelSize(GL_PROXY_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT), 16);
|
||||
|
||||
MG_Impl::GLImpl::TexImage2D(GL_PROXY_TEXTURE_2D, 0, GL_RGBA8, maxTextureSize * 2, 16, 0, GL_RGBA, GL_UNSIGNED_BYTE,
|
||||
nullptr);
|
||||
EXPECT_EQ(levelSize(GL_PROXY_TEXTURE_2D, 0, GL_TEXTURE_WIDTH), 0) << "past GL_MAX_TEXTURE_SIZE";
|
||||
EXPECT_EQ(levelSize(GL_PROXY_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT), 0) << "every dimension is cleared";
|
||||
|
||||
// The limit halves with each level.
|
||||
MG_Impl::GLImpl::TexImage2D(GL_PROXY_TEXTURE_2D, 1, GL_RGBA8, maxTextureSize, 1, 0, GL_RGBA, GL_UNSIGNED_BYTE,
|
||||
nullptr);
|
||||
EXPECT_EQ(levelSize(GL_PROXY_TEXTURE_2D, 1, GL_TEXTURE_WIDTH), 0);
|
||||
MG_Impl::GLImpl::TexImage2D(GL_PROXY_TEXTURE_2D, 1, GL_RGBA8, maxTextureSize / 2, 1, 0, GL_RGBA, GL_UNSIGNED_BYTE,
|
||||
nullptr);
|
||||
EXPECT_EQ(levelSize(GL_PROXY_TEXTURE_2D, 1, GL_TEXTURE_WIDTH), maxTextureSize / 2);
|
||||
|
||||
MG_Impl::GLImpl::TexImage1D(GL_PROXY_TEXTURE_1D, 0, GL_RGBA8, 64, 0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr);
|
||||
EXPECT_EQ(levelSize(GL_PROXY_TEXTURE_1D, 0, GL_TEXTURE_WIDTH), 64);
|
||||
MG_Impl::GLImpl::TexImage3D(GL_PROXY_TEXTURE_3D, 0, GL_RGBA8, 8, 4, 2, 0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr);
|
||||
EXPECT_EQ(levelSize(GL_PROXY_TEXTURE_3D, 0, GL_TEXTURE_WIDTH), 8);
|
||||
EXPECT_EQ(levelSize(GL_PROXY_TEXTURE_3D, 0, GL_TEXTURE_HEIGHT), 4);
|
||||
EXPECT_EQ(levelSize(GL_PROXY_TEXTURE_3D, 0, GL_TEXTURE_DEPTH), 2);
|
||||
|
||||
EXPECT_EQ(MG_Impl::GLImpl::GetError(), GL_NO_ERROR) << "an unsupported proxy size is not an error";
|
||||
}
|
||||
|
||||
TEST_F(TextureTest, GenThenBindCreatesObjectForUnsizedPackedBgraSubImageUpload) {
|
||||
GLuint texture = 0;
|
||||
MG_Impl::GLImpl::GenTextures(1, &texture);
|
||||
|
||||
@@ -1059,6 +1059,43 @@ namespace MobileGL::MG_Util::BackendLoader {
|
||||
caps.MaxFragmentShadingRateAttachmentTexelHeight);
|
||||
}
|
||||
|
||||
// Native clip distances or MobileGL's emulation. Native only where the driver both advertises
|
||||
// GL_EXT_clip_cull_distance AND passes the probe (its own control first): an extension that is
|
||||
// listed but does not clip, or clips by disabled distances, is a wrong answer the application
|
||||
// sees, and the emulation answers it correctly. The probe is skipped entirely without the
|
||||
// extension - there is nothing to measure.
|
||||
static void DecideClipDistanceEmulation(MG_External::GLESCapabilities& caps,
|
||||
const MG_External::GLESFunctionsTable& glesFuncs) {
|
||||
if (caps.SupportsClipDistance && caps.MaxClipDistances > 0) {
|
||||
const SelfTest::NativeClipDistanceMeasurement& clip = SelfTest::NativeClipDistance(glesFuncs);
|
||||
caps.NativeClipDistanceClips = clip.ran && clip.clips;
|
||||
caps.NativeClipDistanceHonorsEnables = clip.ran && clip.honorsEnables;
|
||||
const String verdict = !clip.ran ? "inconclusive (" + clip.failure + ")"
|
||||
: !clip.clips ? "an enabled negative distance did NOT clip"
|
||||
: !clip.honorsEnables ? "clips, but by DISABLED distances too"
|
||||
: "clips, and honors the per-distance enables";
|
||||
MGLOG_I(" Native gl_ClipDistance probe: %s", verdict.c_str());
|
||||
}
|
||||
caps.EmulatesClipDistance = !(caps.SupportsClipDistance && caps.MaxClipDistances > 0 &&
|
||||
caps.NativeClipDistanceClips && caps.NativeClipDistanceHonorsEnables);
|
||||
switch (MG_Config::Features.ClipDistanceEmulation) {
|
||||
case MG_Config::QuirkOverride::ForceOn:
|
||||
caps.EmulatesClipDistance = true;
|
||||
MGLOG_I(" gl_ClipDistance: emulation forced ON by MOBILEGL_CLIP_DISTANCE_EMULATION");
|
||||
break;
|
||||
case MG_Config::QuirkOverride::ForceOff:
|
||||
caps.EmulatesClipDistance = false;
|
||||
MGLOG_I(" gl_ClipDistance: emulation forced OFF by MOBILEGL_CLIP_DISTANCE_EMULATION");
|
||||
break;
|
||||
case MG_Config::QuirkOverride::Auto:
|
||||
default:
|
||||
break;
|
||||
}
|
||||
MGLOG_I(" gl_ClipDistance: %s", caps.EmulatesClipDistance
|
||||
? "EMULATED (varyings + fragment discard; 8 distances advertised)"
|
||||
: "native (GL_EXT_clip_cull_distance)");
|
||||
}
|
||||
|
||||
Bool FillInGLESCapabilities(MG_External::GLESCapabilities& caps, const MG_External::GLESFunctionsTable& glesFuncs) {
|
||||
if (!glesFuncs.glGetString || !glesFuncs.glGetIntegerv) {
|
||||
MGLOG_E("Required GLES functions are not loaded, cannot query capabilities");
|
||||
@@ -1997,6 +2034,7 @@ namespace MobileGL::MG_Util::BackendLoader {
|
||||
MGLOG_I(" GL_MAX_COMPUTE_IMAGE_UNIFORMS: %d", caps.MaxComputeImageUniforms);
|
||||
MGLOG_I(" GL_MAX_DRAW_BUFFERS: %d", caps.MaxDrawBuffers);
|
||||
MGLOG_I(" GL_MAX_COLOR_ATTACHMENTS: %d", caps.MaxColorAttachments);
|
||||
DecideClipDistanceEmulation(caps, glesFuncs);
|
||||
// Worth spelling the reason out for the same reason the per-stage storage block counts
|
||||
// are: a zero here is what stops an application's gl_ClipDistance from ever clipping, and
|
||||
// reading it back from an artifact is the difference between "MobileGL dropped my draw"
|
||||
|
||||
@@ -1274,6 +1274,19 @@ namespace MobileGL {
|
||||
// compile and the per-distance enables have nowhere to go - clipping silently never
|
||||
// happens, which is exactly what KHR-GLxx.clip_distance.functional catches.
|
||||
Bool SupportsClipDistance = false;
|
||||
// What SelfTest::ProbeNativeClipDistance measured - only run where the extension is
|
||||
// advertised: an enabled negative distance removes its fragments, and a disabled one
|
||||
// removes nothing.
|
||||
Bool NativeClipDistanceClips = false;
|
||||
Bool NativeClipDistanceHonorsEnables = false;
|
||||
// DirectGLES lowers gl_ClipDistance to varyings plus a fragment discard
|
||||
// (ShaderCompiler::EmulateClipDistanceForProgram) instead of handing it to the
|
||||
// driver: no extension, no distances behind it, or a probe that caught the driver
|
||||
// not clipping or not honoring the enables. MOBILEGL_CLIP_DISTANCE_EMULATION
|
||||
// overrides. While set, the CLIP_DISTANCEi enables are NOT forwarded to the driver -
|
||||
// no emulated module writes the built-in, and an enabled distance nobody writes is
|
||||
// undefined clipping.
|
||||
Bool EmulatesClipDistance = false;
|
||||
// GL_OES_viewport_array is present: the driver knows gl_ViewportIndex in ESSL - and
|
||||
// only then. ESSL has no core spelling for it at ANY version, while SPIRV-Cross prints
|
||||
// the identifier bare and requests nothing for it (contrast gl_Layer, which it backs
|
||||
|
||||
@@ -410,8 +410,12 @@ namespace MobileGL {
|
||||
return TextureUploadTarget::Texture2D;
|
||||
case GL_TEXTURE_3D:
|
||||
return TextureUploadTarget::Texture3D;
|
||||
case GL_PROXY_TEXTURE_1D:
|
||||
return TextureUploadTarget::ProxyTexture1D;
|
||||
case GL_PROXY_TEXTURE_2D:
|
||||
return TextureUploadTarget::ProxyTexture2D;
|
||||
case GL_PROXY_TEXTURE_3D:
|
||||
return TextureUploadTarget::ProxyTexture3D;
|
||||
case GL_TEXTURE_1D_ARRAY:
|
||||
return TextureUploadTarget::Texture1DArray;
|
||||
case GL_PROXY_TEXTURE_1D_ARRAY:
|
||||
|
||||
@@ -317,8 +317,12 @@ namespace MobileGL {
|
||||
return GL_TEXTURE_1D;
|
||||
case TextureUploadTarget::Texture3D:
|
||||
return GL_TEXTURE_3D;
|
||||
case TextureUploadTarget::ProxyTexture1D:
|
||||
return GL_PROXY_TEXTURE_1D;
|
||||
case TextureUploadTarget::ProxyTexture2D:
|
||||
return GL_PROXY_TEXTURE_2D;
|
||||
case TextureUploadTarget::ProxyTexture3D:
|
||||
return GL_PROXY_TEXTURE_3D;
|
||||
case TextureUploadTarget::Texture1DArray:
|
||||
return GL_TEXTURE_1D_ARRAY;
|
||||
case TextureUploadTarget::ProxyTexture1DArray:
|
||||
|
||||
@@ -2066,6 +2066,168 @@ namespace MobileGL::MG_Util::SelfTest {
|
||||
return measurement;
|
||||
}
|
||||
|
||||
namespace {
|
||||
// ===================== NATIVE CLIP DISTANCE =====================
|
||||
|
||||
constexpr const char* kClipDistanceProbeName = "clip distance";
|
||||
// Four pixels across: pixel 0 is centred at NDC x = -0.75, pixel 3 at +0.75, so the
|
||||
// distance gl_ClipDistance[0] = x is clearly negative on one and positive on the other.
|
||||
constexpr GLsizei kClipDistanceProbeWidth = 4;
|
||||
// GL_CLIP_DISTANCE0_EXT / 1_EXT: the desktop values, absent from the ES headers.
|
||||
constexpr GLenum kClipDistance0 = 0x3000;
|
||||
constexpr GLenum kClipDistance1 = 0x3001;
|
||||
|
||||
const char* const kClipDistanceProbeVertexSource =
|
||||
"#version 300 es\n"
|
||||
"#extension GL_EXT_clip_cull_distance : require\n"
|
||||
"precision highp float;\n"
|
||||
"void main() {\n"
|
||||
" vec2 mg_p = vec2((gl_VertexID == 1) ? 3.0 : -1.0, (gl_VertexID == 2) ? 3.0 : -1.0);\n"
|
||||
" gl_Position = vec4(mg_p, 0.0, 1.0);\n"
|
||||
" gl_ClipDistance[0] = mg_p.x;\n"
|
||||
" gl_ClipDistance[1] = 1.0;\n"
|
||||
"}\n";
|
||||
// The control: the same stage with a distance that is positive everywhere.
|
||||
const char* const kClipDistanceProbeControlVertexSource =
|
||||
"#version 300 es\n"
|
||||
"#extension GL_EXT_clip_cull_distance : require\n"
|
||||
"precision highp float;\n"
|
||||
"void main() {\n"
|
||||
" vec2 mg_p = vec2((gl_VertexID == 1) ? 3.0 : -1.0, (gl_VertexID == 2) ? 3.0 : -1.0);\n"
|
||||
" gl_Position = vec4(mg_p, 0.0, 1.0);\n"
|
||||
" gl_ClipDistance[0] = 1.0;\n"
|
||||
" gl_ClipDistance[1] = 1.0;\n"
|
||||
"}\n";
|
||||
const char* const kClipDistanceProbeFragmentSource =
|
||||
"#version 300 es\n"
|
||||
"precision mediump float;\n"
|
||||
"layout(location = 0) out vec4 mg_probeColor;\n"
|
||||
"void main() { mg_probeColor = vec4(0.0, 1.0, 0.0, 1.0); }\n";
|
||||
|
||||
Bool IsProbeGreen(const GLubyte* pixel) {
|
||||
return pixel[0] < 64 && pixel[1] > 192;
|
||||
}
|
||||
|
||||
// One draw into the bound 4x1 target, cleared red first. Returns false when the draw or
|
||||
// the read-back raised an error (inconclusive).
|
||||
Bool DrawClipDistanceProbe(const GLESFunctionsTable& gl, Bool enableDistance0, Bool& leftPainted,
|
||||
Bool& rightPainted) {
|
||||
if (enableDistance0) {
|
||||
gl.glEnable(kClipDistance0);
|
||||
} else {
|
||||
gl.glDisable(kClipDistance0);
|
||||
}
|
||||
gl.glDisable(kClipDistance1);
|
||||
gl.glClearColor(1.0f, 0.0f, 0.0f, 1.0f);
|
||||
gl.glClear(GL_COLOR_BUFFER_BIT);
|
||||
Drain(gl);
|
||||
gl.glDrawArrays(GL_TRIANGLES, 0, 3);
|
||||
if (gl.glGetError() != GL_NO_ERROR) return false;
|
||||
GLubyte row[kClipDistanceProbeWidth * 4] = {};
|
||||
gl.glReadPixels(0, 0, kClipDistanceProbeWidth, 1, GL_RGBA, GL_UNSIGNED_BYTE, row);
|
||||
if (gl.glGetError() != GL_NO_ERROR) return false;
|
||||
leftPainted = IsProbeGreen(row);
|
||||
rightPainted = IsProbeGreen(row + (kClipDistanceProbeWidth - 1) * 4);
|
||||
return true;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
NativeClipDistanceMeasurement ProbeNativeClipDistance(const GLESFunctionsTable& gl) {
|
||||
NativeClipDistanceMeasurement measurement;
|
||||
if (!HasIoBlockProbeEntryPoints(gl)) {
|
||||
measurement.failure = "a probe entry point is missing";
|
||||
return measurement;
|
||||
}
|
||||
|
||||
SavedState saved;
|
||||
Save(gl, saved);
|
||||
GLboolean savedColorMask[4] = {GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE};
|
||||
const Bool canMaskColor = gl.glColorMask != nullptr && gl.glGetBooleanv != nullptr;
|
||||
if (canMaskColor) {
|
||||
gl.glGetBooleanv(GL_COLOR_WRITEMASK, savedColorMask);
|
||||
gl.glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
|
||||
}
|
||||
GLuint vao = 0;
|
||||
gl.glGenVertexArrays(1, &vao);
|
||||
gl.glBindVertexArray(vao);
|
||||
PrepareForProbeDraw(gl);
|
||||
|
||||
Vector<StageSource> stages;
|
||||
stages.push_back({GL_VERTEX_SHADER, kClipDistanceProbeVertexSource, "vertex"});
|
||||
stages.push_back({GL_FRAGMENT_SHADER, kClipDistanceProbeFragmentSource, "fragment"});
|
||||
const ProgramBuild build = BuildProgram(gl, stages, kClipDistanceProbeName);
|
||||
Vector<StageSource> controlStages;
|
||||
controlStages.push_back({GL_VERTEX_SHADER, kClipDistanceProbeControlVertexSource, "control vertex"});
|
||||
controlStages.push_back({GL_FRAGMENT_SHADER, kClipDistanceProbeFragmentSource, "fragment"});
|
||||
const ProgramBuild control = BuildProgram(gl, controlStages, kClipDistanceProbeName);
|
||||
if (!build.compiled || !control.compiled) {
|
||||
measurement.failure = "the probe's gl_ClipDistance vertex stage did not compile";
|
||||
} else if (!build.linked || !control.linked) {
|
||||
measurement.failure =
|
||||
"the probe program did not link: " + FirstLine(!build.linked ? build.infoLog : control.infoLog);
|
||||
} else {
|
||||
GLuint renderbuffer = 0;
|
||||
GLuint framebuffer = 0;
|
||||
gl.glGenRenderbuffers(1, &renderbuffer);
|
||||
gl.glBindRenderbuffer(GL_RENDERBUFFER, renderbuffer);
|
||||
gl.glRenderbufferStorage(GL_RENDERBUFFER, GL_RGBA8, kClipDistanceProbeWidth, 1);
|
||||
gl.glGenFramebuffers(1, &framebuffer);
|
||||
gl.glBindFramebuffer(GL_FRAMEBUFFER, framebuffer);
|
||||
gl.glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, renderbuffer);
|
||||
if (gl.glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) {
|
||||
measurement.failure = "the probe framebuffer is incomplete";
|
||||
} else {
|
||||
gl.glViewport(0, 0, kClipDistanceProbeWidth, 1);
|
||||
Bool left = false;
|
||||
Bool right = false;
|
||||
// THE CONTROL first: a positive distance, enabled, so both halves must paint.
|
||||
gl.glUseProgram(control.program);
|
||||
const Bool controlRan = DrawClipDistanceProbe(gl, true, left, right);
|
||||
gl.glUseProgram(build.program);
|
||||
if (!controlRan || !left || !right) {
|
||||
measurement.failure = "the control draw (a positive distance, enabled) did not paint the target";
|
||||
} else if (!DrawClipDistanceProbe(gl, true, left, right) || !right) {
|
||||
measurement.failure = "the clipped draw raised an error or lost its kept half";
|
||||
} else {
|
||||
measurement.clips = !left;
|
||||
Bool leftAfter = false;
|
||||
Bool rightAfter = false;
|
||||
if (DrawClipDistanceProbe(gl, false, leftAfter, rightAfter) && rightAfter) {
|
||||
measurement.ran = true;
|
||||
measurement.honorsEnables = leftAfter;
|
||||
} else {
|
||||
measurement.failure = "the re-disabled draw raised an error or lost its kept half";
|
||||
}
|
||||
}
|
||||
}
|
||||
gl.glDisable(kClipDistance0);
|
||||
gl.glDisable(kClipDistance1);
|
||||
if (framebuffer != 0) gl.glDeleteFramebuffers(1, &framebuffer);
|
||||
if (renderbuffer != 0) gl.glDeleteRenderbuffers(1, &renderbuffer);
|
||||
}
|
||||
if (build.program != 0) gl.glDeleteProgram(build.program);
|
||||
if (control.program != 0) gl.glDeleteProgram(control.program);
|
||||
|
||||
if (vao != 0) {
|
||||
gl.glBindVertexArray(0);
|
||||
gl.glDeleteVertexArrays(1, &vao);
|
||||
}
|
||||
if (canMaskColor) {
|
||||
gl.glColorMask(savedColorMask[0], savedColorMask[1], savedColorMask[2], savedColorMask[3]);
|
||||
}
|
||||
Restore(gl, saved);
|
||||
Drain(gl);
|
||||
if (!measurement.failure.empty()) {
|
||||
MGLOG_I("[driver-bug] %s probe: %s", kClipDistanceProbeName, measurement.failure.c_str());
|
||||
}
|
||||
return measurement;
|
||||
}
|
||||
|
||||
const NativeClipDistanceMeasurement& NativeClipDistance(const GLESFunctionsTable& gl) {
|
||||
static const NativeClipDistanceMeasurement measurement = ProbeNativeClipDistance(gl);
|
||||
return measurement;
|
||||
}
|
||||
|
||||
namespace {
|
||||
// ===================== PACKED16 COPY-IMAGE FIELD ORDER =====================
|
||||
|
||||
|
||||
@@ -95,6 +95,38 @@ namespace MobileGL::MG_Util::SelfTest {
|
||||
// ProbeLocatedIoBlocksLosePayload(), evaluated at most once per process.
|
||||
const LocatedIoBlockMeasurement& LocatedIoBlocksLosePayload(const MG_External::GLESFunctionsTable& gl);
|
||||
|
||||
// What the native clip-distance probe measured. Not a driver-bug finding by itself: the
|
||||
// loader turns it into the DirectGLES choice between the driver's gl_ClipDistance and
|
||||
// MobileGL's emulation (EmulateClipDistancePass), and the POST "Clip distances" row reports
|
||||
// that choice.
|
||||
struct NativeClipDistanceMeasurement {
|
||||
// The probe built its program and drew it; every field below is meaningless otherwise.
|
||||
Bool ran = false;
|
||||
// GL_CLIP_DISTANCE0_EXT enabled, distance negative on the left half: the left half was
|
||||
// removed and the right half painted.
|
||||
Bool clips = false;
|
||||
// GL_CLIP_DISTANCE0_EXT disabled, same distances: nothing was removed.
|
||||
Bool honorsEnables = false;
|
||||
// The driver's first info-log line (or a short reason) when the probe could not run.
|
||||
String failure;
|
||||
};
|
||||
|
||||
// Draws full-viewport triangles under GL_EXT_clip_cull_distance. THE CONTROL first: a
|
||||
// program whose gl_ClipDistance[0] is +1 everywhere, with distance 0 enabled - both halves
|
||||
// must paint whatever the driver does with enables, or the extension's draw itself is broken
|
||||
// and the probe has nothing to say. Then the SUBJECT, writing gl_ClipDistance[0] = x: with
|
||||
// distance 0 enabled the left half must vanish, and with it disabled nothing may vanish
|
||||
// (the llvmpipe ES driver clips by every written distance regardless of the enables, a wrong
|
||||
// answer GL applications see).
|
||||
//
|
||||
// Requires the extension; the caller only runs it where the extension string names it.
|
||||
// Restores every piece of GL state it touches and leaves both distances disabled - the
|
||||
// state DirectGLES's enable forwarding assumes a fresh context is in.
|
||||
NativeClipDistanceMeasurement ProbeNativeClipDistance(const MG_External::GLESFunctionsTable& gl);
|
||||
|
||||
// ProbeNativeClipDistance(), evaluated at most once per process.
|
||||
const NativeClipDistanceMeasurement& NativeClipDistance(const MG_External::GLESFunctionsTable& gl);
|
||||
|
||||
// Blits one layer of an RGBA8 2D array onto another array's layer 1 and reports whether the
|
||||
// copy landed where it was asked to. Returns true only when the destination layer is ignored
|
||||
// while the control lands correctly.
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
// End of Source File Header
|
||||
|
||||
#include "DriverPost.h"
|
||||
#include "DriverPostClipDistance.h"
|
||||
#include "DriverPostIterationRPWitness.h"
|
||||
#include "DriverPostIterationRPWitnessSpv.h"
|
||||
#include "PrimitivesGeneratedNoXfbProbe.h"
|
||||
@@ -340,6 +341,22 @@ namespace MobileGL::MG_Util::SelfTest {
|
||||
}
|
||||
}
|
||||
|
||||
void EmitClipDistanceRows(ReportBuilder& builder, Vector<ClipDistancePostRow> rows) {
|
||||
for (ClipDistancePostRow& row : rows) {
|
||||
switch (row.verdict) {
|
||||
case ClipDistancePostRow::Verdict::Pass:
|
||||
builder.Pass(Move(row.name), Move(row.detail));
|
||||
break;
|
||||
case ClipDistancePostRow::Verdict::Warn:
|
||||
builder.Warn(Move(row.name), Move(row.detail));
|
||||
break;
|
||||
case ClipDistancePostRow::Verdict::FailOptional:
|
||||
builder.FailOptional(Move(row.name), Move(row.detail));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void EvaluateGlesChecklist(ReportBuilder& builder, const MG_External::GLESCapabilities& caps,
|
||||
const MG_External::GLESFunctionsTable& glesFuncs) {
|
||||
const Int major = caps.GLESVersion.Major;
|
||||
@@ -388,6 +405,7 @@ namespace MobileGL::MG_Util::SelfTest {
|
||||
"hard-fails, and a second fragment output cannot be produced any other "
|
||||
"way");
|
||||
}
|
||||
EmitClipDistanceRows(builder, GlesClipDistancePostRows(caps, glesFuncs));
|
||||
|
||||
if (es31) {
|
||||
GLint maxVertexSsboBlocks = 0;
|
||||
@@ -2776,6 +2794,10 @@ namespace MobileGL::MG_Util::SelfTest {
|
||||
builder.FailOptional("dualSrcBlend", "unsupported; a draw using a GL_SRC1_* blend factor hard-fails, and a "
|
||||
"second fragment output cannot be produced any other way");
|
||||
}
|
||||
EmitClipDistanceRows(builder, VulkanClipDistancePostRows(features.shaderClipDistance == VK_TRUE,
|
||||
features.shaderCullDistance == VK_TRUE,
|
||||
properties.limits.maxClipDistances,
|
||||
properties.limits.maxCullDistances));
|
||||
// The Magma counterpart of the GLES "Buffer textures" row, so the two sections can be
|
||||
// read side by side. Vulkan has no optional-feature bit here: a uniform texel buffer is
|
||||
// core, and maxTexelBufferElements has a spec floor of 65536 - exactly the GL 3.1 floor
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostClipDistance.cpp
|
||||
// Copyright (c) 2026 MobileGL-Dev
|
||||
// Licensed under the GNU Lesser General Public License v3.0:
|
||||
// https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
// https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
// SPDX-License-Identifier: LGPL-3.0-only
|
||||
// End of Source File Header
|
||||
|
||||
#include "DriverPostClipDistance.h"
|
||||
#include "DriverBugProbes.h"
|
||||
#include <Config.h>
|
||||
#include <format>
|
||||
|
||||
namespace MobileGL::MG_Util::SelfTest {
|
||||
namespace {
|
||||
using Verdict = ClipDistancePostRow::Verdict;
|
||||
|
||||
// What the emulation still cannot do, shared by both backends' WARN rows.
|
||||
constexpr const char* kClipDistanceEmulationResidue =
|
||||
"MobileGL emulates 8 distances: gl_ClipDistance becomes two ordinary varyings and the "
|
||||
"fragment stage discards where a distance is negative, honoring GL_CLIP_DISTANCEi. It costs "
|
||||
"two varying locations and early depth/stencil for programs that clip; transform-feedback "
|
||||
"capture of gl_ClipDistance and gl_in[].gl_ClipDistance reads are not emulated";
|
||||
|
||||
// Culling is per primitive, which a per-fragment discard cannot reproduce.
|
||||
constexpr const char* kNoCullDistance =
|
||||
"GL_MAX_CULL_DISTANCES = 0. Culling is per primitive, which a per-fragment discard cannot "
|
||||
"reproduce, so a gl_CullDistance shader fails to compile";
|
||||
|
||||
String GlesEmulationReason(const MG_External::GLESCapabilities& caps, const String& probeFailure) {
|
||||
if (!caps.SupportsClipDistance) return "no GL_EXT_clip_cull_distance";
|
||||
if (caps.MaxClipDistances <= 0) return "GL_EXT_clip_cull_distance reports no distances";
|
||||
if (!probeFailure.empty()) {
|
||||
return "GL_EXT_clip_cull_distance is listed, but the native probe was inconclusive (" + probeFailure +
|
||||
")";
|
||||
}
|
||||
if (!caps.NativeClipDistanceClips) {
|
||||
return "GL_EXT_clip_cull_distance is listed, but in the probe an enabled negative distance did not "
|
||||
"clip";
|
||||
}
|
||||
if (!caps.NativeClipDistanceHonorsEnables) return "the driver clips by DISABLED distances too (probe)";
|
||||
return "forced by MOBILEGL_CLIP_DISTANCE_EMULATION=1";
|
||||
}
|
||||
} // namespace
|
||||
|
||||
Vector<ClipDistancePostRow> GlesClipDistancePostRows(const MG_External::GLESCapabilities& caps,
|
||||
const MG_External::GLESFunctionsTable& glesFuncs) {
|
||||
Vector<ClipDistancePostRow> rows;
|
||||
// Only meaningful where the loader ran it (extension present); cached per process.
|
||||
String probeFailure;
|
||||
if (caps.SupportsClipDistance && caps.MaxClipDistances > 0) {
|
||||
const NativeClipDistanceMeasurement& probe = NativeClipDistance(glesFuncs);
|
||||
if (!probe.ran) probeFailure = probe.failure;
|
||||
}
|
||||
const Bool native = caps.SupportsClipDistance && caps.MaxClipDistances > 0 && caps.NativeClipDistanceClips &&
|
||||
caps.NativeClipDistanceHonorsEnables;
|
||||
if (!caps.EmulatesClipDistance && native) {
|
||||
rows.push_back({Verdict::Pass, "Clip distances",
|
||||
std::format("native via GL_EXT_clip_cull_distance ({} distances); probe: an enabled "
|
||||
"distance clips and a disabled one does not",
|
||||
caps.MaxClipDistances)});
|
||||
} else if (caps.EmulatesClipDistance) {
|
||||
rows.push_back(
|
||||
{Verdict::Warn, "Clip distances",
|
||||
std::format("{}; {}", GlesEmulationReason(caps, probeFailure), kClipDistanceEmulationResidue)});
|
||||
} else {
|
||||
rows.push_back({Verdict::FailOptional, "Clip distances",
|
||||
caps.SupportsClipDistance
|
||||
? "GL_EXT_clip_cull_distance is listed but misbehaves in the probe, and the emulation "
|
||||
"is disabled (MOBILEGL_CLIP_DISTANCE_EMULATION=0)"
|
||||
: "no GL_EXT_clip_cull_distance and the emulation is disabled "
|
||||
"(MOBILEGL_CLIP_DISTANCE_EMULATION=0): GL_MAX_CLIP_DISTANCES = 0, so a "
|
||||
"gl_ClipDistance shader fails to compile"});
|
||||
}
|
||||
if (caps.SupportsClipDistance && caps.MaxCullDistances > 0) {
|
||||
rows.push_back({Verdict::Pass, "Cull distances",
|
||||
std::format("native via GL_EXT_clip_cull_distance ({} distances)", caps.MaxCullDistances)});
|
||||
} else {
|
||||
rows.push_back({Verdict::FailOptional, "Cull distances",
|
||||
std::format("no GL_EXT_clip_cull_distance: {}", kNoCullDistance)});
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
Vector<ClipDistancePostRow> VulkanClipDistancePostRows(const Bool hasClipDistance, const Bool hasCullDistance,
|
||||
const Uint32 maxClipDistances,
|
||||
const Uint32 maxCullDistances) {
|
||||
Vector<ClipDistancePostRow> rows;
|
||||
Bool emulate = !hasClipDistance;
|
||||
if (MG_Config::Features.ClipDistanceEmulation == MG_Config::QuirkOverride::ForceOn) emulate = true;
|
||||
if (MG_Config::Features.ClipDistanceEmulation == MG_Config::QuirkOverride::ForceOff) emulate = false;
|
||||
if (!emulate && hasClipDistance) {
|
||||
rows.push_back({Verdict::Pass, "shaderClipDistance",
|
||||
std::format("native ({} distances); Vulkan has no per-distance enable state, so MobileGL "
|
||||
"applies GL_CLIP_DISTANCEi in the last pre-rasterization stage (a disabled "
|
||||
"distance is written as +1)",
|
||||
maxClipDistances)});
|
||||
} else if (emulate) {
|
||||
rows.push_back({Verdict::Warn, "shaderClipDistance",
|
||||
std::format("{}; {}",
|
||||
hasClipDistance ? "forced by MOBILEGL_CLIP_DISTANCE_EMULATION=1"
|
||||
: "unsupported by the device",
|
||||
kClipDistanceEmulationResidue)});
|
||||
} else {
|
||||
rows.push_back({Verdict::FailOptional, "shaderClipDistance",
|
||||
"unsupported, and the emulation is disabled (MOBILEGL_CLIP_DISTANCE_EMULATION=0): "
|
||||
"GL_MAX_CLIP_DISTANCES = 0, so a gl_ClipDistance shader fails to compile"});
|
||||
}
|
||||
if (hasCullDistance) {
|
||||
rows.push_back(
|
||||
{Verdict::Pass, "shaderCullDistance", std::format("native ({} distances)", maxCullDistances)});
|
||||
} else {
|
||||
rows.push_back(
|
||||
{Verdict::FailOptional, "shaderCullDistance", std::format("unsupported: {}", kNoCullDistance)});
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
} // namespace MobileGL::MG_Util::SelfTest
|
||||
@@ -0,0 +1,40 @@
|
||||
// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostClipDistance.h
|
||||
// Copyright (c) 2026 MobileGL-Dev
|
||||
// Licensed under the GNU Lesser General Public License v3.0:
|
||||
// https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
// https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
// SPDX-License-Identifier: LGPL-3.0-only
|
||||
// End of Source File Header
|
||||
//
|
||||
// The POST rows for clip and cull distances on both backends. GL_MAX_CLIP_DISTANCES >= 8 is a
|
||||
// GL 3.0 requirement: PASS where the backend clips natively, WARN where MobileGL emulates the
|
||||
// built-in, FAIL where it does neither.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <Includes.h>
|
||||
#include <MG_Util/BackendLoaders/OpenGL/Loader.h>
|
||||
|
||||
namespace MobileGL::MG_Util::SelfTest {
|
||||
struct ClipDistancePostRow {
|
||||
enum class Verdict {
|
||||
Pass,
|
||||
Warn,
|
||||
FailOptional
|
||||
};
|
||||
Verdict verdict = Verdict::Pass;
|
||||
String name;
|
||||
String detail;
|
||||
};
|
||||
|
||||
// "Clip distances" and "Cull distances" for the GLES backend, from the loader's decision and
|
||||
// its cached native probe.
|
||||
Vector<ClipDistancePostRow> GlesClipDistancePostRows(const MG_External::GLESCapabilities& caps,
|
||||
const MG_External::GLESFunctionsTable& glesFuncs);
|
||||
|
||||
// "shaderClipDistance" and "shaderCullDistance" for the Vulkan backend, mirroring
|
||||
// BackendObject_DirectVulkan's decision: emulate exactly when the feature is absent, unless
|
||||
// MOBILEGL_CLIP_DISTANCE_EMULATION says otherwise.
|
||||
Vector<ClipDistancePostRow> VulkanClipDistancePostRows(Bool hasClipDistance, Bool hasCullDistance,
|
||||
Uint32 maxClipDistances, Uint32 maxCullDistances);
|
||||
} // namespace MobileGL::MG_Util::SelfTest
|
||||
@@ -75,6 +75,11 @@ namespace MobileGL::MG_Util::ShaderTranspiler {
|
||||
// and expands gl_MaxClipDistances from the same number, so it decides both whether a
|
||||
// shader compiles at all and what a module that reads the constant generates.
|
||||
HashValue(state, env.params.MaxClipDistances);
|
||||
// Unlike the point-size demotion bits this IS frontend material: when it is set the
|
||||
// source pipeline declares the reserved enables uniform in every pre-rasterization
|
||||
// stage that names gl_ClipDistance, so glslang parses a different source.
|
||||
HashValue(state, env.params.EmulatesClipDistance);
|
||||
HashValue(state, env.params.MasksNativeClipDistances);
|
||||
// The cull-distance pair, added when the GL 4.6 API-surface wave made them env-derived:
|
||||
// they were bare literals (8/8) in BuildTBuiltInResource while no backend had ever been
|
||||
// asked whether it can host a cull distance. Exactly the MaxClipDistances class - glslang
|
||||
|
||||
@@ -165,6 +165,16 @@ namespace MobileGL::MG_Util::ShaderTranspiler {
|
||||
// fingerprint: glslang produces the same thing either way.
|
||||
Bool DemotesTessellationPointSize() const { return HasBackend() && !params.SupportsTessellationPointSize; }
|
||||
Bool DemotesGeometryPointSize() const { return HasBackend() && !params.SupportsGeometryPointSize; }
|
||||
// The backend cannot clip with gl_ClipDistance natively and emulates it
|
||||
// (BackendCaps::EmulatesClipDistance). Part of the frontend fingerprint, not a separate
|
||||
// L1 bit: it changes the parsed source (the reserved enables uniform), not only phase B.
|
||||
Bool EmulatesClipDistance() const { return HasBackend() && params.EmulatesClipDistance; }
|
||||
// BackendCaps::MasksNativeClipDistances, same fingerprint reasoning.
|
||||
Bool MasksNativeClipDistances() const {
|
||||
return HasBackend() && params.MasksNativeClipDistances && !params.EmulatesClipDistance;
|
||||
}
|
||||
// Either way the program needs the reserved enables uniform.
|
||||
Bool NeedsClipDistanceEnables() const { return EmulatesClipDistance() || MasksNativeClipDistances(); }
|
||||
// Matches the historical rule exactly: with no active backend every extension counts
|
||||
// as advertised, because the frontend then has nothing to gate against.
|
||||
Bool IsExtensionAdvertised(GLExtension extension) const {
|
||||
|
||||
@@ -61,6 +61,7 @@
|
||||
#include "SpirvPasses/LegalizeResourceArrayIndexPass.h"
|
||||
#include "SpirvPasses/FlattenAtomicCounterBlockPass.h"
|
||||
#include "SpirvPasses/DemotePointSizePass.h"
|
||||
#include "SpirvPasses/EmulateClipDistancePass.h"
|
||||
#include "spirv-tools/libspirv.h"
|
||||
#include "spirv-tools/optimizer.hpp"
|
||||
#include "source/opt/build_module.h"
|
||||
@@ -1215,6 +1216,140 @@ namespace MobileGL {
|
||||
return true;
|
||||
}
|
||||
|
||||
Bool ShaderCompiler::EmulateClipDistanceForProgram(
|
||||
Vector<Vector<Uint32>>& modules, const Vector<GLenum>& shaderTypes,
|
||||
const Bool captureRequestsClipDistance, const Uint32 maxVaryingLocations,
|
||||
ClipDistanceEmulationOutcome& outcome, const bool validateOutput, const bool enableSpirvValidation,
|
||||
const bool nativeMask) {
|
||||
outcome = {};
|
||||
// Pipeline order: VS, TCS, TES, GS, FS.
|
||||
Int stageIndex[5] = {-1, -1, -1, -1, -1};
|
||||
for (SizeT i = 0; i < shaderTypes.size() && i < modules.size(); ++i) {
|
||||
switch (shaderTypes[i]) {
|
||||
case GL_VERTEX_SHADER:
|
||||
stageIndex[0] = static_cast<Int>(i);
|
||||
break;
|
||||
case GL_TESS_CONTROL_SHADER:
|
||||
stageIndex[1] = static_cast<Int>(i);
|
||||
break;
|
||||
case GL_TESS_EVALUATION_SHADER:
|
||||
stageIndex[2] = static_cast<Int>(i);
|
||||
break;
|
||||
case GL_GEOMETRY_SHADER:
|
||||
stageIndex[3] = static_cast<Int>(i);
|
||||
break;
|
||||
case GL_FRAGMENT_SHADER:
|
||||
stageIndex[4] = static_cast<Int>(i);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
ClipDistanceModuleProbe probes[5];
|
||||
Bool anyAccess = false;
|
||||
Uint32 carrierLocation = 0;
|
||||
for (Int stage = 0; stage < 5; ++stage) {
|
||||
if (stageIndex[stage] < 0) continue;
|
||||
const Vector<Uint32>& module = modules[stageIndex[stage]];
|
||||
probes[stage] = EmulateClipDistancePass::ProbeModule(module);
|
||||
if (!probes[stage].parsed) return true; // broken for other reasons; owns its failure
|
||||
anyAccess |= probes[stage].declaresClipDistanceCapability;
|
||||
const PointSizeModuleProbe locations = ProbePointSizeModule(module);
|
||||
carrierLocation = std::max(carrierLocation, locations.locationSlotEnd);
|
||||
}
|
||||
if (!anyAccess) return true;
|
||||
if (captureRequestsClipDistance) {
|
||||
outcome.declineDetail = "transform feedback captures gl_ClipDistance";
|
||||
return true;
|
||||
}
|
||||
|
||||
// The producer: the last pre-rasterization stage present, and only when it
|
||||
// actually writes the built-in - an earlier stage's writes are dead in GL unless
|
||||
// a later stage reads gl_in[].gl_ClipDistance, which the pass declines.
|
||||
Int producer = -1;
|
||||
for (Int stage = 3; stage >= 0; --stage) {
|
||||
if (stage == 1) continue; // a control stage is never last
|
||||
if (stageIndex[stage] >= 0) {
|
||||
producer = stage;
|
||||
break;
|
||||
}
|
||||
}
|
||||
Uint32 distanceCount = 0;
|
||||
if (producer >= 0 && probes[producer].declaresClipDistanceCapability) {
|
||||
distanceCount = probes[producer].outputDistanceCount;
|
||||
}
|
||||
if (distanceCount > EmulateClipDistancePass::kMaxCarriers * 4) {
|
||||
outcome.declineDetail =
|
||||
std::format("{} clip distances, more than the eight the carriers hold", distanceCount);
|
||||
return true;
|
||||
}
|
||||
const Uint32 carrierCount = nativeMask ? 0 : (distanceCount + 3) / 4;
|
||||
if (carrierCount > 0 && carrierLocation + carrierCount > maxVaryingLocations) {
|
||||
outcome.declineDetail = std::format(
|
||||
"the program's varyings already reach location {}, leaving no room for {} clip carrier(s) "
|
||||
"under the {}-location budget",
|
||||
carrierLocation, carrierCount, maxVaryingLocations);
|
||||
return true;
|
||||
}
|
||||
|
||||
Vector<Vector<Uint32>> rewritten(modules.size());
|
||||
Bool rewrote[5] = {false, false, false, false, false};
|
||||
Bool enablesFound = false;
|
||||
for (Int stage = 0; stage < 5; ++stage) {
|
||||
const Int moduleIndex = stageIndex[stage];
|
||||
if (moduleIndex < 0) continue;
|
||||
EmulateClipDistanceOptions options;
|
||||
options.location = carrierLocation;
|
||||
// Native masking touches the producer alone: every other stage hosts the
|
||||
// built-in, and only the producer's values reach the clipper.
|
||||
if (nativeMask && !(stage == producer && distanceCount > 0)) continue;
|
||||
options.nativeBuiltin = nativeMask;
|
||||
if (stage == 4) {
|
||||
options.role = EmulateClipDistanceOptions::Role::Consumer;
|
||||
options.distanceCount = distanceCount;
|
||||
if (distanceCount == 0 && !probes[stage].declaresClipDistanceCapability) continue;
|
||||
} else if (stage == producer && distanceCount > 0) {
|
||||
options.role = EmulateClipDistanceOptions::Role::Producer;
|
||||
options.enablesMemberName = CLIP_DISTANCE_ENABLES_UNIFORM_NAME;
|
||||
} else {
|
||||
options.role = EmulateClipDistanceOptions::Role::Redirect;
|
||||
if (!probes[stage].declaresClipDistanceCapability) continue;
|
||||
}
|
||||
|
||||
EmulateClipDistanceReport report;
|
||||
spvtools::Optimizer optimizer(SPV_ENV_VULKAN_1_1);
|
||||
optimizer.RegisterPass(EmulateClipDistancePass::CreateEmulateClipDistancePass(options, &report));
|
||||
if (!RunOptimizerChecked("EmulateClipDistanceForProgram", optimizer, modules[moduleIndex],
|
||||
rewritten[moduleIndex], validateOutput, enableSpirvValidation)) {
|
||||
return false; // modules untouched: nothing was committed
|
||||
}
|
||||
if (report.declined) {
|
||||
outcome.declineDetail = Move(report.declineReason);
|
||||
return true;
|
||||
}
|
||||
if (options.role == EmulateClipDistanceOptions::Role::Producer) {
|
||||
if (report.distanceCount != distanceCount) {
|
||||
outcome.declineDetail =
|
||||
std::format("the producing stage carries {} clip distances where {} were planned",
|
||||
report.distanceCount, distanceCount);
|
||||
return true;
|
||||
}
|
||||
enablesFound = report.enablesFound;
|
||||
}
|
||||
rewrote[stage] = report.rewrote;
|
||||
}
|
||||
|
||||
for (Int stage = 0; stage < 5; ++stage) {
|
||||
if (!rewrote[stage]) continue;
|
||||
modules[stageIndex[stage]] = Move(rewritten[stageIndex[stage]]);
|
||||
}
|
||||
outcome.emulated = true;
|
||||
outcome.distanceCount = distanceCount;
|
||||
outcome.enablesFound = enablesFound;
|
||||
return true;
|
||||
}
|
||||
|
||||
Bool ShaderCompiler::ModuleDeclaresFloat64(const Vector<Uint32>& spirv) {
|
||||
if (spirv.empty()) {
|
||||
// Same reasoning as ModuleDeclaresBufferTextureSampler: a stage that produced
|
||||
|
||||
@@ -650,6 +650,44 @@ namespace MobileGL {
|
||||
Bool demoteGeometry, Bool captureRequestsPointSize, PointSizeDemotionOutcome& outcome,
|
||||
bool validateOutput = true, bool enableSpirvValidation = false);
|
||||
|
||||
// What the program-scoped clip-distance emulation left behind. `emulated` false
|
||||
// with an empty detail means the program never touches gl_ClipDistance; false
|
||||
// WITH a detail is a shape the pass cannot express (modules byte-identical, so the
|
||||
// program keeps the built-in and the backend's own refusal of it).
|
||||
struct ClipDistanceEmulationOutcome {
|
||||
Bool emulated = false;
|
||||
// How many distances the producing stage carries to the fragment stage.
|
||||
Uint32 distanceCount = 0;
|
||||
// False: the producer had no enables uniform, so every distance clips.
|
||||
Bool enablesFound = false;
|
||||
String declineDetail;
|
||||
};
|
||||
|
||||
// Emulates gl_ClipDistance across a WHOLE program on a backend that cannot clip
|
||||
// natively (BackendCaps::EmulatesClipDistance): every stage's accesses move onto
|
||||
// a Private shadow, the last pre-rasterization stage carries
|
||||
// `enabled(i) ? d[i] : 1.0` to the fragment stage in ceil(N/4) vec4 varyings
|
||||
// placed past every location the program uses, and the fragment stage discards
|
||||
// where a carried distance is negative. See EmulateClipDistancePass for the
|
||||
// per-module rewrite. Runs BEFORE the sanitize chain (ProgramSpirvTask): the
|
||||
// reserved enables uniform is unread until this pass reads it, and ADCE would
|
||||
// otherwise delete a global UBO that holds nothing else. Atomic: every stage is
|
||||
// rewritten or none is.
|
||||
// `captureRequestsClipDistance` declines (a capture would need the unmasked
|
||||
// value at a varying the pass does not create); `maxVaryingLocations` is the
|
||||
// carrier budget. Returns false only when the optimizer itself failed.
|
||||
//
|
||||
// `nativeMask`: the backend HOSTS gl_ClipDistance but has no per-distance enable
|
||||
// state (Vulkan). Only the producing stage is rewritten - it writes the masked
|
||||
// values back into the built-in (EmulateClipDistanceOptions::nativeBuiltin) - and
|
||||
// no carrier, location or fragment discard is involved.
|
||||
static Bool EmulateClipDistanceForProgram(Vector<Vector<Uint32>>& modules,
|
||||
const Vector<GLenum>& shaderTypes,
|
||||
Bool captureRequestsClipDistance, Uint32 maxVaryingLocations,
|
||||
ClipDistanceEmulationOutcome& outcome,
|
||||
bool validateOutput = true,
|
||||
bool enableSpirvValidation = false, bool nativeMask = false);
|
||||
|
||||
// True when the module still declares a 64-bit float type. After
|
||||
// SanitizeAndOptimizeBinary that can only mean DemoteFloat64Pass declined the
|
||||
// module (see its header for the two operations that make it decline), which is
|
||||
|
||||
@@ -1880,6 +1880,28 @@ namespace {
|
||||
source.insert(afterVersion.Get(source), shim);
|
||||
}
|
||||
|
||||
// On a backend that emulates gl_ClipDistance, the stage that ends up producing the clip
|
||||
// carriers needs GL's per-distance enables, and the only thing a module can read per draw
|
||||
// without a pipeline key is a uniform. Declared in every pre-rasterization stage that names
|
||||
// the built-in, because which of them is LAST is a link-time fact this per-shader pass cannot
|
||||
// know; the stages that are not the producer simply never read it, and glslang folds the one
|
||||
// name into the shared MGL_GLOBAL_UBO for every stage of the program alike.
|
||||
void InjectClipDistanceEnablesShim(MobileGL::ShaderStage stage, MobileGL::String& source,
|
||||
AfterVersionAnchor& afterVersion) {
|
||||
if (stage != MobileGL::ShaderStage::Vertex && stage != MobileGL::ShaderStage::TessControl &&
|
||||
stage != MobileGL::ShaderStage::TessEval && stage != MobileGL::ShaderStage::Geometry) {
|
||||
return;
|
||||
}
|
||||
if (source.find("gl_ClipDistance") == MobileGL::String::npos) return;
|
||||
const MobileGL::String masked = MaskCommentsAndQuotedText(source);
|
||||
if (!MaskedSourceHasIdentifier(masked, "gl_ClipDistance")) return;
|
||||
if (MaskedSourceHasIdentifier(masked,
|
||||
MobileGL::MG_Util::ShaderTranspiler::CLIP_DISTANCE_ENABLES_UNIFORM_NAME)) {
|
||||
return;
|
||||
}
|
||||
source.insert(afterVersion.Get(source), "uniform uint mg_ClipDistanceEnables;\n");
|
||||
}
|
||||
|
||||
// The uniforms `text` declares with the external sampler type, and (into `others`) the names it
|
||||
// declares with any other type - the same name can be both in a source whose #if branches pick
|
||||
// one (a compositor's shader generator does exactly that).
|
||||
@@ -2020,6 +2042,7 @@ namespace MobileGL {
|
||||
ModernizeLegacyGLSL(stage, source, afterVersion);
|
||||
InjectDepthRangeBuiltinShim(stage, source, afterVersion);
|
||||
InjectNumSamplesBuiltinShim(stage, originalLanguage, source, afterVersion);
|
||||
if (env.NeedsClipDistanceEnables()) InjectClipDistanceEnablesShim(stage, source, afterVersion);
|
||||
}
|
||||
|
||||
Vector<String> ExternalSamplerUniformNames(const String& preprocessedSource) {
|
||||
|
||||
@@ -0,0 +1,814 @@
|
||||
// MobileGL - MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateClipDistancePass.cpp
|
||||
// Copyright (c) 2025-2026 MobileGL-Dev
|
||||
// Licensed under the GNU Lesser General Public License v3.0:
|
||||
// https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
// https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
// SPDX-License-Identifier: LGPL-3.0-only
|
||||
// End of Source File Header
|
||||
|
||||
#include "EmulateClipDistancePass.h"
|
||||
|
||||
#include "spirv.hpp"
|
||||
#include "source/opt/basic_block.h"
|
||||
#include "source/opt/build_module.h"
|
||||
#include "source/opt/constants.h"
|
||||
#include "source/opt/def_use_manager.h"
|
||||
#include "source/opt/function.h"
|
||||
#include "source/opt/instruction.h"
|
||||
#include "source/opt/ir_builder.h"
|
||||
#include "source/opt/ir_context.h"
|
||||
#include "source/opt/module.h"
|
||||
#include "source/opt/type_manager.h"
|
||||
#include "source/opt/types.h"
|
||||
#include "source/util/make_unique.h"
|
||||
#include "source/util/string_utils.h"
|
||||
|
||||
#include <format>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
namespace MobileGL {
|
||||
namespace MG_Util {
|
||||
namespace ShaderTranspiler {
|
||||
namespace {
|
||||
using spvtools::opt::BasicBlock;
|
||||
using spvtools::opt::Function;
|
||||
using spvtools::opt::Instruction;
|
||||
using spvtools::opt::InstructionBuilder;
|
||||
using spvtools::opt::IRContext;
|
||||
using spvtools::opt::Operand;
|
||||
namespace analysis = spvtools::opt::analysis;
|
||||
|
||||
const IRContext::Analysis kBuilderAnalyses = static_cast<IRContext::Analysis>(
|
||||
IRContext::kAnalysisDefUse | IRContext::kAnalysisInstrToBlockMapping);
|
||||
|
||||
Instruction* EntryPoint(IRContext* ctx) {
|
||||
for (Instruction& ep : ctx->module()->entry_points()) {
|
||||
return &ep;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
Function* EntryFunction(IRContext* ctx, Instruction* entryPoint) {
|
||||
const uint32_t id = entryPoint->GetSingleWordInOperand(1);
|
||||
for (auto& function : *ctx->module()) {
|
||||
if (function.result_id() == id) return &function;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
uint32_t PointeeOf(IRContext* ctx, uint32_t pointerTypeId) {
|
||||
Instruction* ptrType = ctx->get_def_use_mgr()->GetDef(pointerTypeId);
|
||||
if (ptrType == nullptr || ptrType->opcode() != spv::Op::OpTypePointer) return 0;
|
||||
return ptrType->GetSingleWordInOperand(1);
|
||||
}
|
||||
|
||||
bool PlainConstantValue(IRContext* ctx, uint32_t id, uint32_t& outValue) {
|
||||
Instruction* def = ctx->get_def_use_mgr()->GetDef(id);
|
||||
if (def == nullptr || def->opcode() != spv::Op::OpConstant) return false;
|
||||
if (def->NumInOperands() != 1) return false;
|
||||
outValue = def->GetSingleWordInOperand(0);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Length of an OpTypeArray of 32-bit float with a plain constant length, else 0.
|
||||
uint32_t FloatArrayLength(IRContext* ctx, uint32_t typeId) {
|
||||
Instruction* t = ctx->get_def_use_mgr()->GetDef(typeId);
|
||||
if (t == nullptr || t->opcode() != spv::Op::OpTypeArray) return 0;
|
||||
Instruction* element = ctx->get_def_use_mgr()->GetDef(t->GetSingleWordInOperand(0));
|
||||
if (element == nullptr || element->opcode() != spv::Op::OpTypeFloat ||
|
||||
element->GetSingleWordInOperand(0) != 32) {
|
||||
return 0;
|
||||
}
|
||||
uint32_t length = 0;
|
||||
if (!PlainConstantValue(ctx, t->GetSingleWordInOperand(1), length)) return 0;
|
||||
return length;
|
||||
}
|
||||
|
||||
void AddNameFor(IRContext* ctx, uint32_t id, const String& name) {
|
||||
std::vector<Operand> operands;
|
||||
operands.push_back({SPV_OPERAND_TYPE_ID, {id}});
|
||||
operands.push_back({SPV_OPERAND_TYPE_LITERAL_STRING, spvtools::utils::MakeVector(name)});
|
||||
ctx->AddDebug2Inst(spvtools::MakeUnique<Instruction>(ctx, spv::Op::OpName, 0, 0, operands));
|
||||
}
|
||||
|
||||
uint32_t AddGlobalVariable(IRContext* ctx, uint32_t pointeeTypeId, spv::StorageClass storage,
|
||||
const String& name) {
|
||||
const uint32_t ptrTypeId = ctx->get_type_mgr()->FindPointerToType(pointeeTypeId, storage);
|
||||
if (ptrTypeId == 0) return 0;
|
||||
const uint32_t varId = ctx->TakeNextId();
|
||||
auto var = spvtools::MakeUnique<Instruction>(
|
||||
ctx, spv::Op::OpVariable, ptrTypeId, varId,
|
||||
std::initializer_list<Operand>{
|
||||
{SPV_OPERAND_TYPE_STORAGE_CLASS, {static_cast<uint32_t>(storage)}}});
|
||||
Instruction* varInst = var.get();
|
||||
ctx->AddGlobalValue(std::move(var));
|
||||
ctx->get_def_use_mgr()->AnalyzeInstDefUse(varInst);
|
||||
AddNameFor(ctx, varId, name);
|
||||
return varId;
|
||||
}
|
||||
|
||||
void AddLocationDecoration(IRContext* ctx, uint32_t id, uint32_t location) {
|
||||
ctx->AddAnnotationInst(spvtools::MakeUnique<Instruction>(
|
||||
ctx, spv::Op::OpDecorate, 0, 0,
|
||||
std::initializer_list<Operand>{
|
||||
{SPV_OPERAND_TYPE_ID, {id}},
|
||||
{SPV_OPERAND_TYPE_DECORATION, {static_cast<uint32_t>(spv::Decoration::Location)}},
|
||||
{SPV_OPERAND_TYPE_LITERAL_INTEGER, {location}}}));
|
||||
}
|
||||
|
||||
void RemoveFromInterface(Instruction* entryPoint, uint32_t id) {
|
||||
Instruction::OperandList kept;
|
||||
for (uint32_t i = 0; i < entryPoint->NumInOperands(); ++i) {
|
||||
const Operand& operand = entryPoint->GetInOperand(i);
|
||||
if (i >= 3 && operand.type == SPV_OPERAND_TYPE_ID && operand.words.size() == 1 &&
|
||||
operand.words[0] == id) {
|
||||
continue;
|
||||
}
|
||||
kept.push_back(operand);
|
||||
}
|
||||
entryPoint->SetInOperands(Move(kept));
|
||||
}
|
||||
|
||||
// Whether every use of a pointer into the built-in is one the retype below can
|
||||
// follow: loads, stores (as the POINTER), copies, and further access chains.
|
||||
bool PointerUsesAreRetypable(IRContext* ctx, Instruction* pointer, String& reason) {
|
||||
bool ok = true;
|
||||
ctx->get_def_use_mgr()->ForEachUse(pointer, [&](Instruction* user, uint32_t operandIndex) {
|
||||
if (!ok) return;
|
||||
switch (user->opcode()) {
|
||||
case spv::Op::OpName:
|
||||
case spv::Op::OpDecorate:
|
||||
case spv::Op::OpEntryPoint:
|
||||
return;
|
||||
case spv::Op::OpLoad:
|
||||
case spv::Op::OpCopyMemory:
|
||||
return;
|
||||
case spv::Op::OpStore:
|
||||
// Operand 0 of OpStore is the pointer; anything else would be
|
||||
// storing the pointer itself, which logical addressing forbids.
|
||||
if (operandIndex != 0) {
|
||||
ok = false;
|
||||
reason = "a pointer into gl_ClipDistance stored as a value";
|
||||
}
|
||||
return;
|
||||
case spv::Op::OpAccessChain:
|
||||
case spv::Op::OpInBoundsAccessChain:
|
||||
// Base operand: the instruction's first id after result type/id.
|
||||
if (operandIndex != 2) {
|
||||
ok = false;
|
||||
reason = "a pointer into gl_ClipDistance used as an access-chain index";
|
||||
return;
|
||||
}
|
||||
if (!PointerUsesAreRetypable(ctx, user, reason)) ok = false;
|
||||
return;
|
||||
default:
|
||||
ok = false;
|
||||
reason = std::format("a pointer into gl_ClipDistance reaching SPIR-V opcode {}",
|
||||
static_cast<uint32_t>(user->opcode()));
|
||||
return;
|
||||
}
|
||||
});
|
||||
return ok;
|
||||
}
|
||||
|
||||
// Re-points every access chain hanging off `pointer` (which now lives in Private
|
||||
// storage) at a Private pointer type of the same pointee.
|
||||
void RetypeChainUsers(IRContext* ctx, Instruction* pointer) {
|
||||
std::vector<Instruction*> chains;
|
||||
ctx->get_def_use_mgr()->ForEachUser(pointer, [&](Instruction* user) {
|
||||
if ((user->opcode() == spv::Op::OpAccessChain ||
|
||||
user->opcode() == spv::Op::OpInBoundsAccessChain) &&
|
||||
user->GetSingleWordInOperand(0) == pointer->result_id()) {
|
||||
chains.push_back(user);
|
||||
}
|
||||
});
|
||||
for (Instruction* chain : chains) {
|
||||
const uint32_t pointee = PointeeOf(ctx, chain->type_id());
|
||||
chain->SetResultType(
|
||||
ctx->get_type_mgr()->FindPointerToType(pointee, spv::StorageClass::Private));
|
||||
ctx->get_def_use_mgr()->AnalyzeInstUse(chain);
|
||||
RetypeChainUsers(ctx, chain);
|
||||
}
|
||||
}
|
||||
|
||||
// A scalar constant of the given 32-bit integer type.
|
||||
uint32_t IntConstantOfType(IRContext* ctx, uint32_t intTypeId, uint32_t value) {
|
||||
const analysis::Type* type = ctx->get_type_mgr()->GetType(intTypeId);
|
||||
const analysis::Integer* integer = type != nullptr ? type->AsInteger() : nullptr;
|
||||
if (integer != nullptr && integer->IsSigned()) {
|
||||
return ctx->get_constant_mgr()->GetSIntConstId(static_cast<int32_t>(value));
|
||||
}
|
||||
return ctx->get_constant_mgr()->GetUIntConstId(value);
|
||||
}
|
||||
|
||||
bool IsInt32Type(IRContext* ctx, uint32_t typeId) {
|
||||
const analysis::Type* type = ctx->get_type_mgr()->GetType(typeId);
|
||||
const analysis::Integer* integer = type != nullptr ? type->AsInteger() : nullptr;
|
||||
return integer != nullptr && integer->width() == 32;
|
||||
}
|
||||
|
||||
struct ClipSites {
|
||||
// struct type id -> member index of its BuiltIn ClipDistance member
|
||||
std::vector<std::pair<uint32_t, uint32_t>> members;
|
||||
std::unordered_set<uint32_t> standaloneVars;
|
||||
Instruction* capability = nullptr;
|
||||
};
|
||||
|
||||
ClipSites FindClipSites(IRContext* ctx) {
|
||||
ClipSites sites;
|
||||
for (Instruction& ann : ctx->annotations()) {
|
||||
if (ann.opcode() == spv::Op::OpMemberDecorate && ann.NumInOperands() >= 4 &&
|
||||
static_cast<spv::Decoration>(ann.GetSingleWordInOperand(2)) == spv::Decoration::BuiltIn &&
|
||||
static_cast<spv::BuiltIn>(ann.GetSingleWordInOperand(3)) == spv::BuiltIn::ClipDistance) {
|
||||
sites.members.emplace_back(ann.GetSingleWordInOperand(0), ann.GetSingleWordInOperand(1));
|
||||
} else if (ann.opcode() == spv::Op::OpDecorate && ann.NumInOperands() >= 3 &&
|
||||
static_cast<spv::Decoration>(ann.GetSingleWordInOperand(1)) ==
|
||||
spv::Decoration::BuiltIn &&
|
||||
static_cast<spv::BuiltIn>(ann.GetSingleWordInOperand(2)) ==
|
||||
spv::BuiltIn::ClipDistance) {
|
||||
Instruction* var = ctx->get_def_use_mgr()->GetDef(ann.GetSingleWordInOperand(0));
|
||||
if (var != nullptr && var->opcode() == spv::Op::OpVariable) {
|
||||
sites.standaloneVars.insert(var->result_id());
|
||||
}
|
||||
}
|
||||
}
|
||||
for (Instruction& capability : ctx->module()->capabilities()) {
|
||||
if (capability.NumInOperands() >= 1 &&
|
||||
static_cast<spv::Capability>(capability.GetSingleWordInOperand(0)) ==
|
||||
spv::Capability::ClipDistance) {
|
||||
sites.capability = &capability;
|
||||
}
|
||||
}
|
||||
return sites;
|
||||
}
|
||||
|
||||
bool ClipMemberOf(const ClipSites& sites, uint32_t structId, uint32_t& member) {
|
||||
for (const auto& [id, index] : sites.members) {
|
||||
if (id == structId) {
|
||||
member = index;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Every redirected access must agree on one gl_ClipDistance array type.
|
||||
bool NoteShadowType(uint32_t& shadowTypeId, uint32_t typeId) {
|
||||
if (shadowTypeId == 0) {
|
||||
shadowTypeId = typeId;
|
||||
return true;
|
||||
}
|
||||
return shadowTypeId == typeId;
|
||||
}
|
||||
|
||||
// A chain that stops short of the gl_PerVertex member (a whole block or gl_in[i]
|
||||
// pointer) is harmless unless something then reaches the clip member through it or
|
||||
// copies the whole struct.
|
||||
bool ShortChainTouchesClipMember(IRContext* ctx, Instruction* chain, uint32_t clipMember) {
|
||||
bool touches = false;
|
||||
ctx->get_def_use_mgr()->ForEachUser(chain, [&](Instruction* next) {
|
||||
if (next->opcode() == spv::Op::OpAccessChain ||
|
||||
next->opcode() == spv::Op::OpInBoundsAccessChain) {
|
||||
uint32_t member = 0;
|
||||
if (next->NumInOperands() < 2 ||
|
||||
!PlainConstantValue(ctx, next->GetSingleWordInOperand(1), member) ||
|
||||
member == clipMember) {
|
||||
touches = true;
|
||||
}
|
||||
} else if (next->opcode() != spv::Op::OpName) {
|
||||
touches = true; // whole-struct load/store/copy
|
||||
}
|
||||
});
|
||||
return touches;
|
||||
}
|
||||
|
||||
struct BlockVariable {
|
||||
Instruction* var = nullptr;
|
||||
spv::StorageClass storage = spv::StorageClass::Output;
|
||||
bool arrayed = false;
|
||||
uint32_t clipMember = 0;
|
||||
uint32_t memberTypeId = 0;
|
||||
};
|
||||
|
||||
// Classifies every use of a gl_PerVertex block variable: chains onto the clip member
|
||||
// of a non-arrayed block are collected, anything else that reaches it declines.
|
||||
bool CollectBlockClipChains(IRContext* ctx, const BlockVariable& block, uint32_t& shadowTypeId,
|
||||
std::vector<Instruction*>& memberChains, String& reason) {
|
||||
const uint32_t memberOperand = block.arrayed ? 2u : 1u; // [vertex,] member, ...
|
||||
bool declined = false;
|
||||
ctx->get_def_use_mgr()->ForEachUser(block.var, [&](Instruction* user) {
|
||||
if (declined) return;
|
||||
switch (user->opcode()) {
|
||||
case spv::Op::OpEntryPoint:
|
||||
case spv::Op::OpName:
|
||||
case spv::Op::OpDecorate:
|
||||
return;
|
||||
case spv::Op::OpAccessChain:
|
||||
case spv::Op::OpInBoundsAccessChain: {
|
||||
if (user->NumInOperands() <= memberOperand) {
|
||||
if (ShortChainTouchesClipMember(ctx, user, block.clipMember)) {
|
||||
declined = true;
|
||||
reason = "a split access chain or whole-gl_PerVertex copy reaching gl_ClipDistance";
|
||||
}
|
||||
return;
|
||||
}
|
||||
uint32_t member = 0;
|
||||
if (!PlainConstantValue(ctx, user->GetSingleWordInOperand(memberOperand), member)) {
|
||||
declined = true;
|
||||
reason = "a non-constant gl_PerVertex member index";
|
||||
return;
|
||||
}
|
||||
if (member != block.clipMember) return; // another member
|
||||
if (block.arrayed) {
|
||||
declined = true;
|
||||
reason = block.storage == spv::StorageClass::Input ? "a gl_in[].gl_ClipDistance read"
|
||||
: "a gl_out[].gl_ClipDistance write";
|
||||
return;
|
||||
}
|
||||
if (!NoteShadowType(shadowTypeId, block.memberTypeId)) {
|
||||
declined = true;
|
||||
reason = "two differently sized gl_ClipDistance arrays";
|
||||
return;
|
||||
}
|
||||
if (!PointerUsesAreRetypable(ctx, user, reason)) {
|
||||
declined = true;
|
||||
return;
|
||||
}
|
||||
memberChains.push_back(user);
|
||||
return;
|
||||
}
|
||||
case spv::Op::OpLoad:
|
||||
case spv::Op::OpStore:
|
||||
case spv::Op::OpCopyMemory:
|
||||
declined = true;
|
||||
reason = "a whole-aggregate load/store/copy of the gl_PerVertex interface";
|
||||
return;
|
||||
default:
|
||||
declined = true;
|
||||
reason = std::format("SPIR-V opcode {} reaching the gl_PerVertex interface",
|
||||
static_cast<uint32_t>(user->opcode()));
|
||||
return;
|
||||
}
|
||||
});
|
||||
return !declined;
|
||||
}
|
||||
|
||||
// The clip member type of the stage's gl_PerVertex OUTPUT block, or 0. A producer with
|
||||
// nothing redirected may still have declared the length the driver planned with.
|
||||
uint32_t DeclaredOutputClipType(IRContext* ctx, const ClipSites& sites) {
|
||||
for (Instruction& inst : ctx->module()->types_values()) {
|
||||
if (inst.opcode() != spv::Op::OpVariable ||
|
||||
static_cast<spv::StorageClass>(inst.GetSingleWordInOperand(0)) !=
|
||||
spv::StorageClass::Output) {
|
||||
continue;
|
||||
}
|
||||
Instruction* type = ctx->get_def_use_mgr()->GetDef(PointeeOf(ctx, inst.type_id()));
|
||||
uint32_t member = 0;
|
||||
if (type != nullptr && type->opcode() == spv::Op::OpTypeStruct &&
|
||||
ClipMemberOf(sites, type->result_id(), member)) {
|
||||
return type->GetSingleWordInOperand(member);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct EnablesUniform {
|
||||
uint32_t varId = 0;
|
||||
uint32_t memberIndex = 0;
|
||||
uint32_t typeId = 0;
|
||||
};
|
||||
|
||||
// The producer's enables uniform: a 32-bit integer member, by name, of a
|
||||
// Uniform-storage struct.
|
||||
EnablesUniform FindEnablesUniform(IRContext* ctx, const String& memberName) {
|
||||
EnablesUniform found;
|
||||
for (Instruction& debug : ctx->module()->debugs2()) {
|
||||
if (debug.opcode() != spv::Op::OpMemberName || debug.NumInOperands() < 3) continue;
|
||||
if (debug.GetOperand(2).AsString() != memberName) continue;
|
||||
const uint32_t structId = debug.GetSingleWordInOperand(0);
|
||||
const uint32_t member = debug.GetSingleWordInOperand(1);
|
||||
Instruction* structType = ctx->get_def_use_mgr()->GetDef(structId);
|
||||
if (structType == nullptr || structType->opcode() != spv::Op::OpTypeStruct ||
|
||||
member >= structType->NumInOperands()) {
|
||||
continue;
|
||||
}
|
||||
const uint32_t memberType = structType->GetSingleWordInOperand(member);
|
||||
if (!IsInt32Type(ctx, memberType)) continue;
|
||||
for (Instruction& var : ctx->module()->types_values()) {
|
||||
if (var.opcode() == spv::Op::OpVariable &&
|
||||
static_cast<spv::StorageClass>(var.GetSingleWordInOperand(0)) ==
|
||||
spv::StorageClass::Uniform &&
|
||||
PointeeOf(ctx, var.type_id()) == structId) {
|
||||
return {var.result_id(), member, memberType};
|
||||
}
|
||||
}
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
// Where the producer writes its distances out: before each vertex a geometry stage
|
||||
// emits, else at the entry function's returns.
|
||||
std::vector<Instruction*> FlushSites(IRContext* ctx, Instruction* entryPoint, bool isGeometry) {
|
||||
std::vector<Instruction*> sites;
|
||||
if (isGeometry) {
|
||||
for (auto& function : *ctx->module()) {
|
||||
function.ForEachInst([&](Instruction* inst) {
|
||||
if (inst->opcode() == spv::Op::OpEmitVertex ||
|
||||
inst->opcode() == spv::Op::OpEmitStreamVertex) {
|
||||
sites.push_back(inst);
|
||||
}
|
||||
});
|
||||
}
|
||||
} else if (Function* entry = EntryFunction(ctx, entryPoint)) {
|
||||
entry->ForEachInst([&](Instruction* inst) {
|
||||
if (inst->opcode() == spv::Op::OpReturn || inst->opcode() == spv::Op::OpReturnValue) {
|
||||
sites.push_back(inst);
|
||||
}
|
||||
});
|
||||
}
|
||||
return sites;
|
||||
}
|
||||
|
||||
struct ScalarIds {
|
||||
uint32_t floatType = 0;
|
||||
uint32_t vec4Type = 0;
|
||||
uint32_t boolType = 0;
|
||||
uint32_t one = 0;
|
||||
uint32_t zero = 0;
|
||||
uint32_t ptrPrivateFloat = 0;
|
||||
};
|
||||
|
||||
ScalarIds MakeScalarIds(IRContext* ctx) {
|
||||
auto* typeMgr = ctx->get_type_mgr();
|
||||
ScalarIds ids;
|
||||
analysis::Float floatTy(32);
|
||||
ids.floatType = typeMgr->GetTypeInstruction(&floatTy);
|
||||
analysis::Vector vec4Ty(typeMgr->GetRegisteredType(&floatTy), 4);
|
||||
ids.vec4Type = typeMgr->GetTypeInstruction(&vec4Ty);
|
||||
analysis::Bool boolTy;
|
||||
ids.boolType = typeMgr->GetTypeInstruction(&boolTy);
|
||||
ids.one = ctx->get_constant_mgr()->GetFloatConstId(1.0f);
|
||||
ids.zero = ctx->get_constant_mgr()->GetFloatConstId(0.0f);
|
||||
ids.ptrPrivateFloat = typeMgr->FindPointerToType(ids.floatType, spv::StorageClass::Private);
|
||||
return ids;
|
||||
}
|
||||
|
||||
// Fragment stage: load the carriers at the top of the entry function, serve the
|
||||
// shadow's gl_ClipDistance reads from them, and kill on any negative distance.
|
||||
void EmitCarrierDiscard(IRContext* ctx, Instruction* entryPoint, const ScalarIds& ids,
|
||||
Uint32 consumedCount, Uint32 location, uint32_t shadowId,
|
||||
uint32_t shadowLength) {
|
||||
Function* entry = EntryFunction(ctx, entryPoint);
|
||||
if (entry == nullptr || entry->begin() == entry->end()) return;
|
||||
auto* constMgr = ctx->get_constant_mgr();
|
||||
const Uint32 carrierCount = (consumedCount + 3) / 4;
|
||||
uint32_t carriers[EmulateClipDistancePass::kMaxCarriers] = {0, 0};
|
||||
for (Uint32 k = 0; k < carrierCount; ++k) {
|
||||
carriers[k] = AddGlobalVariable(ctx, ids.vec4Type, spv::StorageClass::Input,
|
||||
EmulateClipDistancePass::CarrierName(k));
|
||||
AddLocationDecoration(ctx, carriers[k], location + k);
|
||||
entryPoint->AddOperand({SPV_OPERAND_TYPE_ID, {carriers[k]}});
|
||||
}
|
||||
BasicBlock* head = &*entry->begin();
|
||||
auto split = head->begin();
|
||||
while (split != head->end() && split->opcode() == spv::Op::OpVariable)
|
||||
++split;
|
||||
BasicBlock* rest = head->SplitBasicBlock(ctx, ctx->TakeNextId(), split);
|
||||
|
||||
InstructionBuilder b(ctx, head, kBuilderAnalyses);
|
||||
uint32_t loaded[EmulateClipDistancePass::kMaxCarriers] = {0, 0};
|
||||
for (Uint32 k = 0; k < carrierCount; ++k) {
|
||||
loaded[k] = b.AddLoad(ids.vec4Type, carriers[k])->result_id();
|
||||
}
|
||||
uint32_t anyNegative = 0;
|
||||
for (Uint32 i = 0; i < consumedCount; ++i) {
|
||||
const uint32_t component =
|
||||
b.AddCompositeExtract(ids.floatType, loaded[i / 4], {i % 4})->result_id();
|
||||
if (shadowId != 0 && i < shadowLength) {
|
||||
Instruction* ptr =
|
||||
b.AddAccessChain(ids.ptrPrivateFloat, shadowId, {constMgr->GetUIntConstId(i)});
|
||||
b.AddStore(ptr->result_id(), component);
|
||||
}
|
||||
const uint32_t negative =
|
||||
b.AddBinaryOp(ids.boolType, spv::Op::OpFOrdLessThan, component, ids.zero)->result_id();
|
||||
anyNegative =
|
||||
anyNegative == 0
|
||||
? negative
|
||||
: b.AddBinaryOp(ids.boolType, spv::Op::OpLogicalOr, anyNegative, negative)->result_id();
|
||||
}
|
||||
const uint32_t killLabel = ctx->TakeNextId();
|
||||
b.AddConditionalBranch(anyNegative, killLabel, rest->id(), rest->id());
|
||||
|
||||
auto killBlock = spvtools::MakeUnique<BasicBlock>(spvtools::MakeUnique<Instruction>(
|
||||
ctx, spv::Op::OpLabel, 0, killLabel, std::initializer_list<Operand>{}));
|
||||
killBlock->AddInstruction(spvtools::MakeUnique<Instruction>(ctx, spv::Op::OpKill, 0, 0,
|
||||
std::initializer_list<Operand>{}));
|
||||
entry->InsertBasicBlockAfter(std::move(killBlock), head);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
ClipDistanceModuleProbe EmulateClipDistancePass::ProbeModule(const Vector<Uint32>& spirv) {
|
||||
ClipDistanceModuleProbe probe;
|
||||
if (spirv.empty()) {
|
||||
probe.parsed = true;
|
||||
return probe;
|
||||
}
|
||||
std::unique_ptr<IRContext> ctx = spvtools::BuildModule(
|
||||
SPV_ENV_VULKAN_1_1, [](spv_message_level_t, const char*, const spv_position_t&, const char*) {},
|
||||
spirv.data(), spirv.size());
|
||||
if (!ctx) return probe;
|
||||
probe.parsed = true;
|
||||
const ClipSites sites = FindClipSites(ctx.get());
|
||||
probe.declaresClipDistanceCapability = sites.capability != nullptr;
|
||||
for (Instruction& inst : ctx->module()->types_values()) {
|
||||
if (inst.opcode() != spv::Op::OpVariable) continue;
|
||||
if (static_cast<spv::StorageClass>(inst.GetSingleWordInOperand(0)) != spv::StorageClass::Output) {
|
||||
continue;
|
||||
}
|
||||
const uint32_t pointee = PointeeOf(ctx.get(), inst.type_id());
|
||||
if (sites.standaloneVars.count(inst.result_id()) != 0) {
|
||||
probe.outputDistanceCount = FloatArrayLength(ctx.get(), pointee);
|
||||
break;
|
||||
}
|
||||
Instruction* type = ctx->get_def_use_mgr()->GetDef(pointee);
|
||||
uint32_t member = 0;
|
||||
if (type != nullptr && type->opcode() == spv::Op::OpTypeStruct &&
|
||||
ClipMemberOf(sites, type->result_id(), member)) {
|
||||
probe.outputDistanceCount = FloatArrayLength(ctx.get(), type->GetSingleWordInOperand(member));
|
||||
break;
|
||||
}
|
||||
}
|
||||
return probe;
|
||||
}
|
||||
|
||||
spvtools::opt::Pass::Status EmulateClipDistancePass::Process() {
|
||||
IRContext* ctx = context();
|
||||
auto* defUse = ctx->get_def_use_mgr();
|
||||
auto* typeMgr = ctx->get_type_mgr();
|
||||
auto* constMgr = ctx->get_constant_mgr();
|
||||
using Role = EmulateClipDistanceOptions::Role;
|
||||
|
||||
Instruction* entryPoint = EntryPoint(ctx);
|
||||
if (entryPoint == nullptr) return Status::SuccessWithoutChange;
|
||||
const auto model = static_cast<spv::ExecutionModel>(entryPoint->GetSingleWordInOperand(0));
|
||||
const bool isFragment = model == spv::ExecutionModel::Fragment;
|
||||
const bool isGeometry = model == spv::ExecutionModel::Geometry;
|
||||
const bool isPreRaster = model == spv::ExecutionModel::Vertex ||
|
||||
model == spv::ExecutionModel::TessellationControl ||
|
||||
model == spv::ExecutionModel::TessellationEvaluation || isGeometry;
|
||||
|
||||
const auto decline = [&](String reason) {
|
||||
if (m_report != nullptr) {
|
||||
m_report->declined = true;
|
||||
m_report->declineReason = Move(reason);
|
||||
}
|
||||
return Status::SuccessWithoutChange;
|
||||
};
|
||||
if (m_options.role == Role::Consumer && !isFragment)
|
||||
return decline("consumer role on a non-fragment stage");
|
||||
if (m_options.role != Role::Consumer && !isPreRaster) {
|
||||
return decline("producer/redirect role on a non-pre-rasterization stage");
|
||||
}
|
||||
if (m_options.role == Role::Producer && model == spv::ExecutionModel::TessellationControl) {
|
||||
return decline("a control stage cannot be the last pre-rasterization stage");
|
||||
}
|
||||
const bool native = m_options.nativeBuiltin;
|
||||
if (native && m_options.role != Role::Producer) return decline("native masking outside the producer");
|
||||
|
||||
const ClipSites sites = FindClipSites(ctx);
|
||||
|
||||
// ---- Phase 1: ANALYSIS ONLY. A decline leaves the module byte-identical. ----
|
||||
std::vector<Instruction*> memberChains; // [clipMember, rest...] on a non-arrayed block
|
||||
std::vector<Instruction*> standaloneVars; // standalone ClipDistance variables
|
||||
// Native masking: where the masked values are written back to.
|
||||
Instruction* nativeBlockVar = nullptr;
|
||||
uint32_t nativeBlockMember = 0;
|
||||
Instruction* nativeStandaloneVar = nullptr;
|
||||
uint32_t shadowTypeId = 0;
|
||||
String reason;
|
||||
|
||||
for (Instruction& inst : ctx->module()->types_values()) {
|
||||
if (inst.opcode() != spv::Op::OpVariable) continue;
|
||||
const auto storage = static_cast<spv::StorageClass>(inst.GetSingleWordInOperand(0));
|
||||
if (storage != spv::StorageClass::Input && storage != spv::StorageClass::Output) continue;
|
||||
// Native masking leaves every INPUT alone: gl_in[] reads are hosted natively.
|
||||
if (native && storage == spv::StorageClass::Input) continue;
|
||||
const uint32_t pointeeId = PointeeOf(ctx, inst.type_id());
|
||||
|
||||
if (sites.standaloneVars.count(inst.result_id()) != 0) {
|
||||
if (native) nativeStandaloneVar = &inst;
|
||||
if (FloatArrayLength(ctx, pointeeId) == 0) {
|
||||
return decline("a standalone gl_ClipDistance that is not a sized float array");
|
||||
}
|
||||
// A pre-rasterization stage's standalone INPUT would be gl_in data,
|
||||
// which a non-arrayed variable cannot be; only the fragment stage reads
|
||||
// gl_ClipDistance this way.
|
||||
if (storage == spv::StorageClass::Input && !isFragment) {
|
||||
return decline("a standalone gl_ClipDistance input outside the fragment stage");
|
||||
}
|
||||
if (!NoteShadowType(shadowTypeId, pointeeId)) {
|
||||
return decline("two differently sized gl_ClipDistance arrays");
|
||||
}
|
||||
if (!PointerUsesAreRetypable(ctx, &inst, reason)) return decline(Move(reason));
|
||||
standaloneVars.push_back(&inst);
|
||||
continue;
|
||||
}
|
||||
|
||||
Instruction* pointee = defUse->GetDef(pointeeId);
|
||||
if (pointee == nullptr) continue;
|
||||
bool arrayed = false;
|
||||
if (pointee->opcode() == spv::Op::OpTypeArray) {
|
||||
arrayed = true;
|
||||
pointee = defUse->GetDef(pointee->GetSingleWordInOperand(0));
|
||||
if (pointee == nullptr) continue;
|
||||
}
|
||||
BlockVariable block{&inst, storage, arrayed, 0, 0};
|
||||
if (pointee->opcode() != spv::Op::OpTypeStruct ||
|
||||
!ClipMemberOf(sites, pointee->result_id(), block.clipMember)) {
|
||||
continue;
|
||||
}
|
||||
block.memberTypeId = pointee->GetSingleWordInOperand(block.clipMember);
|
||||
if (native && !arrayed && storage == spv::StorageClass::Output) {
|
||||
nativeBlockVar = &inst;
|
||||
nativeBlockMember = block.clipMember;
|
||||
}
|
||||
if (!CollectBlockClipChains(ctx, block, shadowTypeId, memberChains, reason)) {
|
||||
return decline(Move(reason));
|
||||
}
|
||||
}
|
||||
|
||||
const bool hasAccesses = !memberChains.empty() || !standaloneVars.empty();
|
||||
if (shadowTypeId == 0 && m_options.role == Role::Producer) {
|
||||
shadowTypeId = DeclaredOutputClipType(ctx, sites);
|
||||
}
|
||||
const uint32_t shadowLength = shadowTypeId != 0 ? FloatArrayLength(ctx, shadowTypeId) : 0;
|
||||
if (shadowTypeId != 0 && shadowLength == 0)
|
||||
return decline("gl_ClipDistance is not a sized float array");
|
||||
|
||||
Uint32 producedCount = 0;
|
||||
if (m_options.role == Role::Producer) {
|
||||
producedCount = std::min<Uint32>(shadowLength, kMaxCarriers * 4);
|
||||
if (shadowLength > kMaxCarriers * 4) return decline("more than eight clip distances");
|
||||
}
|
||||
const Uint32 consumedCount =
|
||||
m_options.role == Role::Consumer ? std::min<Uint32>(m_options.distanceCount, kMaxCarriers * 4) : 0;
|
||||
|
||||
const EnablesUniform enables = producedCount > 0 && !m_options.enablesMemberName.empty()
|
||||
? FindEnablesUniform(ctx, m_options.enablesMemberName)
|
||||
: EnablesUniform{};
|
||||
const uint32_t enablesVarId = enables.varId;
|
||||
const uint32_t enablesTypeId = enables.typeId;
|
||||
|
||||
if (!hasAccesses && sites.capability == nullptr && producedCount == 0 && consumedCount == 0) {
|
||||
return Status::SuccessWithoutChange;
|
||||
}
|
||||
if (native && producedCount > 0 && nativeBlockVar == nullptr && nativeStandaloneVar == nullptr) {
|
||||
return decline("no gl_ClipDistance output to write the masked distances back into");
|
||||
}
|
||||
|
||||
// ---- Phase 2: MUTATION. Nothing below may decline. ----
|
||||
const bool interfaceListsAllGlobals = ctx->module()->version() >= 0x00010400u;
|
||||
uint32_t shadowId = 0;
|
||||
if (shadowTypeId != 0 && (hasAccesses || producedCount > 0)) {
|
||||
shadowId = AddGlobalVariable(ctx, shadowTypeId, spv::StorageClass::Private, kShadowName);
|
||||
if (interfaceListsAllGlobals) entryPoint->AddOperand({SPV_OPERAND_TYPE_ID, {shadowId}});
|
||||
}
|
||||
|
||||
for (Instruction* chain : memberChains) {
|
||||
if (chain->NumInOperands() == 2) {
|
||||
// The chain IS the whole array: every use now addresses the shadow.
|
||||
ctx->ReplaceAllUsesWith(chain->result_id(), shadowId);
|
||||
ctx->KillInst(chain);
|
||||
continue;
|
||||
}
|
||||
Instruction::OperandList operands;
|
||||
operands.push_back({SPV_OPERAND_TYPE_ID, {shadowId}});
|
||||
for (uint32_t i = 2; i < chain->NumInOperands(); ++i) {
|
||||
operands.push_back(chain->GetInOperand(i));
|
||||
}
|
||||
chain->SetInOperands(Move(operands));
|
||||
chain->SetResultType(
|
||||
typeMgr->FindPointerToType(PointeeOf(ctx, chain->type_id()), spv::StorageClass::Private));
|
||||
defUse->AnalyzeInstUse(chain);
|
||||
RetypeChainUsers(ctx, chain);
|
||||
}
|
||||
for (Instruction* var : standaloneVars) {
|
||||
const uint32_t varId = var->result_id();
|
||||
if (native) {
|
||||
// The variable stays the built-in the flush writes; only the shader's
|
||||
// own accesses move to the shadow.
|
||||
ctx->ReplaceAllUsesWithPredicate(varId, shadowId, [](Instruction* user) {
|
||||
return user->opcode() != spv::Op::OpDecorate && user->opcode() != spv::Op::OpName &&
|
||||
user->opcode() != spv::Op::OpEntryPoint;
|
||||
});
|
||||
continue;
|
||||
}
|
||||
ctx->KillNamesAndDecorates(varId);
|
||||
RemoveFromInterface(entryPoint, varId);
|
||||
ctx->ReplaceAllUsesWith(varId, shadowId);
|
||||
ctx->KillInst(var);
|
||||
}
|
||||
if (shadowId != 0) RetypeChainUsers(ctx, defUse->GetDef(shadowId));
|
||||
if (sites.capability != nullptr && !native) ctx->KillInst(sites.capability);
|
||||
|
||||
const ScalarIds ids = MakeScalarIds(ctx);
|
||||
const uint32_t floatTypeId = ids.floatType;
|
||||
const uint32_t vec4TypeId = ids.vec4Type;
|
||||
const uint32_t boolTypeId = ids.boolType;
|
||||
const uint32_t oneId = ids.one;
|
||||
|
||||
// ---- producer: write the carriers ----
|
||||
if (producedCount > 0) {
|
||||
const Uint32 carrierCount = native ? 0 : (producedCount + 3) / 4;
|
||||
uint32_t carriers[kMaxCarriers] = {0, 0};
|
||||
const uint32_t ptrOutputFloat = typeMgr->FindPointerToType(floatTypeId, spv::StorageClass::Output);
|
||||
const uint32_t nativeMemberConst =
|
||||
nativeBlockVar != nullptr ? constMgr->GetSIntConstId(static_cast<int32_t>(nativeBlockMember))
|
||||
: 0;
|
||||
for (Uint32 k = 0; k < carrierCount; ++k) {
|
||||
carriers[k] = AddGlobalVariable(ctx, vec4TypeId, spv::StorageClass::Output, CarrierName(k));
|
||||
AddLocationDecoration(ctx, carriers[k], m_options.location + k);
|
||||
entryPoint->AddOperand({SPV_OPERAND_TYPE_ID, {carriers[k]}});
|
||||
}
|
||||
uint32_t ptrUniformEnables = 0;
|
||||
uint32_t enablesMemberConst = 0;
|
||||
if (enablesVarId != 0) {
|
||||
ptrUniformEnables = typeMgr->FindPointerToType(enablesTypeId, spv::StorageClass::Uniform);
|
||||
enablesMemberConst = constMgr->GetSIntConstId(static_cast<int32_t>(enables.memberIndex));
|
||||
if (interfaceListsAllGlobals) {
|
||||
bool listed = false;
|
||||
for (uint32_t i = 3; i < entryPoint->NumInOperands(); ++i) {
|
||||
if (entryPoint->GetSingleWordInOperand(i) == enablesVarId) listed = true;
|
||||
}
|
||||
if (!listed) entryPoint->AddOperand({SPV_OPERAND_TYPE_ID, {enablesVarId}});
|
||||
}
|
||||
}
|
||||
|
||||
for (Instruction* site : FlushSites(ctx, entryPoint, isGeometry)) {
|
||||
InstructionBuilder b(ctx, site, kBuilderAnalyses);
|
||||
uint32_t maskId = 0;
|
||||
if (enablesVarId != 0) {
|
||||
Instruction* ptr = b.AddAccessChain(ptrUniformEnables, enablesVarId, {enablesMemberConst});
|
||||
maskId = b.AddLoad(enablesTypeId, ptr->result_id())->result_id();
|
||||
}
|
||||
std::vector<uint32_t> values;
|
||||
for (Uint32 i = 0; i < producedCount; ++i) {
|
||||
Instruction* ptr =
|
||||
b.AddAccessChain(ids.ptrPrivateFloat, shadowId, {constMgr->GetUIntConstId(i)});
|
||||
uint32_t value = b.AddLoad(floatTypeId, ptr->result_id())->result_id();
|
||||
if (maskId != 0) {
|
||||
const uint32_t bit = b.AddBinaryOp(enablesTypeId, spv::Op::OpBitwiseAnd, maskId,
|
||||
IntConstantOfType(ctx, enablesTypeId, 1u << i))
|
||||
->result_id();
|
||||
const uint32_t on = b.AddBinaryOp(boolTypeId, spv::Op::OpINotEqual, bit,
|
||||
IntConstantOfType(ctx, enablesTypeId, 0u))
|
||||
->result_id();
|
||||
value = b.AddSelect(floatTypeId, on, value, oneId)->result_id();
|
||||
}
|
||||
values.push_back(value);
|
||||
}
|
||||
if (native) {
|
||||
for (Uint32 i = 0; i < producedCount; ++i) {
|
||||
const uint32_t index = constMgr->GetUIntConstId(i);
|
||||
Instruction* ptr =
|
||||
nativeBlockVar != nullptr
|
||||
? b.AddAccessChain(ptrOutputFloat, nativeBlockVar->result_id(),
|
||||
{nativeMemberConst, index})
|
||||
: b.AddAccessChain(ptrOutputFloat, nativeStandaloneVar->result_id(), {index});
|
||||
b.AddStore(ptr->result_id(), values[i]);
|
||||
}
|
||||
}
|
||||
while (values.size() % 4 != 0)
|
||||
values.push_back(oneId);
|
||||
for (Uint32 k = 0; k < carrierCount; ++k) {
|
||||
const uint32_t composite =
|
||||
b.AddCompositeConstruct(vec4TypeId, {values[k * 4], values[k * 4 + 1],
|
||||
values[k * 4 + 2], values[k * 4 + 3]})
|
||||
->result_id();
|
||||
b.AddStore(carriers[k], composite);
|
||||
}
|
||||
}
|
||||
if (m_report != nullptr) {
|
||||
m_report->distanceCount = producedCount;
|
||||
m_report->enablesFound = enablesVarId != 0;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- consumer: read the carriers, discard, serve gl_ClipDistance reads ----
|
||||
if (m_options.role == Role::Consumer && consumedCount > 0) {
|
||||
EmitCarrierDiscard(ctx, entryPoint, ids, consumedCount, m_options.location, shadowId, shadowLength);
|
||||
}
|
||||
|
||||
if (m_report != nullptr) m_report->rewrote = true;
|
||||
ctx->InvalidateAnalysesExceptFor(IRContext::kAnalysisNone);
|
||||
return Status::SuccessWithChange;
|
||||
}
|
||||
|
||||
spvtools::Optimizer::PassToken EmulateClipDistancePass::CreateEmulateClipDistancePass(
|
||||
EmulateClipDistanceOptions options, EmulateClipDistanceReport* report) {
|
||||
return spvtools::Optimizer::PassToken(MakeUnique<EmulateClipDistancePass>(Move(options), report));
|
||||
}
|
||||
} // namespace ShaderTranspiler
|
||||
} // namespace MG_Util
|
||||
} // namespace MobileGL
|
||||
@@ -0,0 +1,140 @@
|
||||
// MobileGL - MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateClipDistancePass.h
|
||||
// Copyright (c) 2025-2026 MobileGL-Dev
|
||||
// Licensed under the GNU Lesser General Public License v3.0:
|
||||
// https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
// https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
// SPDX-License-Identifier: LGPL-3.0-only
|
||||
// End of Source File Header
|
||||
|
||||
#pragma once
|
||||
#include "source/opt/pass.h"
|
||||
#include "spirv-tools/optimizer.hpp"
|
||||
|
||||
#include <Includes.h>
|
||||
|
||||
namespace MobileGL {
|
||||
namespace MG_Util {
|
||||
namespace ShaderTranspiler {
|
||||
// Emulates gl_ClipDistance on a backend that cannot host the built-in at all: an ES
|
||||
// driver without GL_EXT_clip_cull_distance, or a Vulkan device with
|
||||
// shaderClipDistance == VK_FALSE (every Mali GPU measured so far is both). GL 3.0+
|
||||
// requires GL_MAX_CLIP_DISTANCES >= 8, so a frontend that reports GL 4.6 cannot
|
||||
// honestly answer 0 there, and answering 8 without this pass makes every clipping
|
||||
// draw either fail to compile or render unclipped.
|
||||
//
|
||||
// WHAT THE EMULATION IS. Clip distance d is linear in clip space, so its
|
||||
// perspective-correct interpolant at a fragment is exactly the distance the
|
||||
// rasterizer's clipper would have evaluated there; discarding the fragments where it
|
||||
// is negative removes the same pixels hardware clipping removes. So:
|
||||
// - every access to the ClipDistance built-in, in every stage, is redirected onto
|
||||
// a Private float[N] shadow (mg_ClipDistanceShadow) and the ClipDistance
|
||||
// capability is stripped - the gl_PerVertex member stays declared but unaccessed,
|
||||
// which is the shape glslang emits for every program that never clips;
|
||||
// - the LAST pre-rasterization stage (the PRODUCER) writes
|
||||
// `enabled(i) ? shadow[i] : 1.0` into ceil(N/4) ordinary vec4 varyings
|
||||
// (mg_ClipDistanceEmu0/1) at every return of its entry function - before every
|
||||
// EmitVertex in a geometry stage, which is when one vertex's outputs latch.
|
||||
// enabled(i) is bit i of the reserved default-block uniform the source pipeline
|
||||
// declared (CLIP_DISTANCE_ENABLES_UNIFORM_NAME), which the draw path refreshes
|
||||
// from GL_CLIP_DISTANCE0..7 before every draw. Applying the enables HERE keeps
|
||||
// the varyings free of pipeline-key state and makes a disabled distance - even
|
||||
// one the shader writes negative - clip nothing, as GL requires;
|
||||
// - the fragment stage (the CONSUMER) reads the carriers before anything else and
|
||||
// executes OpKill when any carried distance is negative. Its own gl_ClipDistance
|
||||
// reads are served from the same carriers (GLSL leaves the disabled elements
|
||||
// undefined there, so the 1.0 they carry is a legal value).
|
||||
// - every other stage that touches the built-in (a vertex stage ahead of a
|
||||
// geometry stage, say) only has its accesses redirected: in GL those writes are
|
||||
// dead unless a later stage reads gl_in[].gl_ClipDistance.
|
||||
//
|
||||
// WHAT IT COSTS: two varying locations past every location the program uses, and a
|
||||
// discard in the fragment stage of every clipping program (early depth/stencil is
|
||||
// lost for those programs). Cull distance is NOT emulated: culling is per primitive,
|
||||
// which a per-fragment discard cannot reproduce.
|
||||
//
|
||||
// WHAT DECLINES (the module is left byte-identical and the report says why):
|
||||
// gl_in[].gl_ClipDistance reads and gl_out[].gl_ClipDistance writes (arrayed
|
||||
// per-vertex access in tessellation/geometry stages), a non-constant gl_PerVertex
|
||||
// member index, whole-gl_PerVertex loads/stores/copies, and a pointer into the
|
||||
// built-in that escapes into an opcode the pass cannot retype (a function argument).
|
||||
//
|
||||
// NATIVE MASKING (EmulateClipDistanceOptions::nativeBuiltin) is the same producer
|
||||
// rewrite for a backend that hosts the built-in but cannot switch distances off: the
|
||||
// masked values go back into gl_ClipDistance instead of into carriers.
|
||||
//
|
||||
// One module per run; the program-wide contract (one shared location, which stage
|
||||
// is the producer, every stage rewritten or none) is owned by
|
||||
// ShaderCompiler::EmulateClipDistanceForProgram, the only caller.
|
||||
struct EmulateClipDistanceOptions {
|
||||
enum class Role : Uint8 {
|
||||
// Redirect the accesses and strip the capability; no carriers.
|
||||
Redirect,
|
||||
// The last pre-rasterization stage: redirect, then write the carriers.
|
||||
Producer,
|
||||
// The fragment stage: read the carriers, discard, serve gl_ClipDistance reads.
|
||||
Consumer,
|
||||
};
|
||||
Role role = Role::Redirect;
|
||||
// The first carrier location; carrier k sits at location + k.
|
||||
Uint32 location = 0;
|
||||
// Consumer only: how many distances the producer carries (0 = none, in which
|
||||
// case the stage only has its own reads redirected).
|
||||
Uint32 distanceCount = 0;
|
||||
// Producer only: the default-block uniform member holding the enable bits. When
|
||||
// the module has no such member every distance counts as enabled.
|
||||
String enablesMemberName;
|
||||
// Producer only: the backend HOSTS the built-in (a Vulkan device with
|
||||
// shaderClipDistance) but has no per-distance enable state - Vulkan clips by
|
||||
// every distance a shader writes. Instead of carriers, the producer writes
|
||||
// `enabled(i) ? d[i] : 1.0` back into gl_ClipDistance itself at the same flush
|
||||
// points, so hardware clipping stays and a disabled distance clips nothing. The
|
||||
// capability is kept, nothing else in the program is touched, and gl_in /
|
||||
// fragment-stage reads stay native.
|
||||
Bool nativeBuiltin = false;
|
||||
};
|
||||
|
||||
struct EmulateClipDistanceReport {
|
||||
Bool declined = false;
|
||||
String declineReason;
|
||||
Bool rewrote = false;
|
||||
// Producer: how many distances it now carries.
|
||||
Uint32 distanceCount = 0;
|
||||
// Producer: whether the enables uniform was found (false = all-enabled fallback).
|
||||
Bool enablesFound = false;
|
||||
};
|
||||
|
||||
// What one module looks like to the program-wide driver, from a single parse.
|
||||
struct ClipDistanceModuleProbe {
|
||||
Bool parsed = false;
|
||||
// glslang declares the capability on first ACCESS, so this is the trigger.
|
||||
Bool declaresClipDistanceCapability = false;
|
||||
// The length of the module's non-arrayed ClipDistance OUTPUT (block member or
|
||||
// standalone variable), or 0 when it declares none.
|
||||
Uint32 outputDistanceCount = 0;
|
||||
};
|
||||
|
||||
class EmulateClipDistancePass : public spvtools::opt::Pass {
|
||||
public:
|
||||
static constexpr const char* kShadowName = "mg_ClipDistanceShadow";
|
||||
static constexpr Uint32 kMaxCarriers = 2;
|
||||
static const char* CarrierName(Uint32 index) {
|
||||
return index == 0 ? "mg_ClipDistanceEmu0" : "mg_ClipDistanceEmu1";
|
||||
}
|
||||
|
||||
EmulateClipDistancePass(EmulateClipDistanceOptions options, EmulateClipDistanceReport* report)
|
||||
: m_options(Move(options)), m_report(report) {}
|
||||
const char* name() const override { return "mobilegl-emulate-clip-distance"; }
|
||||
Status Process() override;
|
||||
|
||||
static ClipDistanceModuleProbe ProbeModule(const Vector<Uint32>& spirv);
|
||||
|
||||
static spvtools::Optimizer::PassToken CreateEmulateClipDistancePass(EmulateClipDistanceOptions options,
|
||||
EmulateClipDistanceReport* report);
|
||||
|
||||
private:
|
||||
EmulateClipDistanceOptions m_options;
|
||||
EmulateClipDistanceReport* m_report;
|
||||
};
|
||||
} // namespace ShaderTranspiler
|
||||
} // namespace MG_Util
|
||||
} // namespace MobileGL
|
||||
@@ -28,6 +28,14 @@ namespace MobileGL {
|
||||
// uses gl_NumSamples the link task will still hide its uniform. That is the same
|
||||
// bargain every mg_-prefixed rewrite in this pipeline strikes.
|
||||
inline const char* NUM_SAMPLES_UNIFORM_NAME = "mg_NumSamples";
|
||||
// The default-block uniform InjectClipDistanceEnablesShim declares, on a backend that
|
||||
// emulates gl_ClipDistance (BackendCaps::EmulatesClipDistance), in every
|
||||
// pre-rasterization stage that names the built-in: bit i is GL_CLIP_DISTANCE0 + i's
|
||||
// enable. EmulateClipDistancePass reads it where the producing stage writes its
|
||||
// carriers; the draw path refreshes it from the render state before every draw, and
|
||||
// the link task hides it from the GL uniform surface exactly as it hides
|
||||
// NUM_SAMPLES_UNIFORM_NAME. Reserved on the same terms.
|
||||
inline const char* CLIP_DISTANCE_ENABLES_UNIFORM_NAME = "mg_ClipDistanceEnables";
|
||||
// glslang's Vulkan-relaxed parse rewrites every atomic_uint into a member of a
|
||||
// synthesized storage block named "<this>_<GL atomic-counter binding>"
|
||||
// (ParseContextBase::growAtomicCounterBlock). That block IS the GL atomic counter
|
||||
|
||||
@@ -74,3 +74,67 @@ adb shell am start -a top.mobilegl.plugin.TRACE_REPLAY \
|
||||
--es ssim_threshold 0.99
|
||||
adb shell run-as top.mobilegl.plugin.espryt.trace cat files/trace-replay/output/result.json
|
||||
```
|
||||
|
||||
## Emulator shards in CI
|
||||
|
||||
The APK workflow replays its cases through `trace-replay-shard.sh`: one emulator boot, one APK
|
||||
install, then an ordered list of cases for one lane, transport and backend. Each case still runs
|
||||
through `trace-replay-ci.sh`, with the same arguments, environment and transport proof as a
|
||||
one-case job.
|
||||
|
||||
```sh
|
||||
sh android-plugin/trace-replay-shard.sh \
|
||||
--apk-file MobileGL-plugin-trace-release.apk \
|
||||
--backend DirectVulkan --gpu lavapipe \
|
||||
--transport inproc --lane inproc-acceptance \
|
||||
--cases-file cases.json --deadline-seconds 7200
|
||||
```
|
||||
|
||||
The case list is a JSON array of case objects: the `cases` member of a row of
|
||||
`trace_cases.py --ci --format github-apk-transport-shard-matrix`, which is what the workflow
|
||||
passes, or the `case` members of `--format github-apk-transport-matrix` rows. `--dry-run` prints
|
||||
what each case would be run with and touches no device.
|
||||
|
||||
The workflow's matrix is that shard matrix: one row per emulator job, holding some of the cases
|
||||
of one lane, transport and backend. `--shard-seconds` (`RETRACE_SHARD_SECONDS` in `apk.yml`) sets
|
||||
how much hinted replay time one shard aims for, and `trace_cases.py` refuses to print a shard
|
||||
matrix whose shards, taken apart, are not exactly the per-case matrix. The fixtures of a shard's
|
||||
cases must be in `tools/trace_replay/fixtures` before the runner starts; the workflow restores
|
||||
them from one cache entry (`.github/scripts/trace-fixture-bundle.sh`).
|
||||
|
||||
Unless a manual run sets `android_full_matrix`, the workflow passes `--android-lanes
|
||||
inproc-full`: the inproc lane, the transport the launcher ships, replays every CI case, and the
|
||||
monolith and spawn lanes replay the cases `trace_cases.json` lists under `android_smoke`. The
|
||||
Linux retrace jobs of `test.yml` replay every case on all three transports, so those two
|
||||
emulator lanes keep the cases that prove the Android side of each arm: the server started from
|
||||
the APK's native library directory, the pbuffer surface of the spawn lane, the monolith build
|
||||
of the same commit as a control, and every per-case replay setting at least once. A manual run
|
||||
with `android_full_matrix` replays every case on all three lanes. A branch that runs only the
|
||||
monolith lane replays every case on it.
|
||||
|
||||
Between cases the runner force-stops the app, checks that no process of the app's uid is left,
|
||||
clears the app data, removes the case's files from `/data/local/tmp` and from the host, and
|
||||
checks that the device still answers. It restarts the emulator when it does not, before the
|
||||
case that follows one that did not pass, and before the single retry of a case that
|
||||
`trace-replay-ci.sh` charged to the infrastructure (exit 75).
|
||||
`MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE=1` cold boots before every case.
|
||||
|
||||
The cases of a shard share the job's time. `--deadline-seconds` stops the runner that long after
|
||||
it starts, so that the job can still upload what it has: a case that is running then, or has not
|
||||
started, is recorded as `not-run`. `retrace_shard_budget.py` reads a shard matrix and adds two
|
||||
fields to every shard: `deadline_seconds`, the time in which every case of the shard could use
|
||||
its whole time budget (`timeout_seconds` plus 300 s) when the cases before it used all of
|
||||
theirs, with one infrastructure retry of the longest budget and a cold boot per case on top,
|
||||
and `timeout_minutes`, the limit of the shard's job. `--cap-seconds`
|
||||
(`RETRACE_SHARD_DEADLINE_CAP_SECONDS` in `apk.yml`) bounds the deadline, so a shard whose
|
||||
replays all hang cannot hold a runner for hours; a capped shard loses verdicts only when several
|
||||
of its replays hang to the end of their budgets.
|
||||
|
||||
A failing case does not stop the shard. Every case ends with `<case>-<backend>/case-status.json`
|
||||
(`passed`, `failed` or `not-run`), the result root gets `shard-status.json`, the job log gets one
|
||||
`::error` per case that did not pass, and the exit status is non-zero when a case did not pass or
|
||||
a verdict is missing. `--finalize` writes the summary inputs and a `not-run` verdict for every
|
||||
case without one, without a device; the workflow runs it after the replay step whatever happened.
|
||||
|
||||
`python3 android-plugin/test_trace_replay_shard.py` drives the runner with a fake adb, replay
|
||||
script and AVD script.
|
||||
|
||||
@@ -80,7 +80,7 @@ val releaseSigningReady = signingStoreFile.exists()
|
||||
val debuggableRelease = (findProperty("mobilegl.debuggableRelease") ?: "false").toString().toBoolean()
|
||||
|
||||
val mobileGlVersionMajor = 26
|
||||
val mobileGlVersionMinor = 10
|
||||
val mobileGlVersionMinor = 11
|
||||
val mobileGlGitShortHash = runGit("rev-parse", "--short=7", "HEAD") ?: "nogit"
|
||||
val mobileGlMonthlyRevision = runGit(
|
||||
"rev-list",
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
#!/usr/bin/env python3
|
||||
# MobileGL - android-plugin/retrace_artifact_cleanup.py
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
"""Delete the intermediate artifacts of an APK workflow run, keeping what a failure needs.
|
||||
|
||||
The retrace summary embeds every image it shows, so once it is uploaded the per-shard result
|
||||
artifacts are only worth keeping for cases that did not pass:
|
||||
|
||||
- a result artifact stays when its shard-status.json is missing, when its job failed, or when
|
||||
it holds a case that did not pass in any shard (so both backends and all transports of a
|
||||
failed case stay comparable);
|
||||
- every result artifact stays when a job failed without leaving a shard-status.json to say
|
||||
which cases it was running.
|
||||
|
||||
A canceled job retains nothing by itself; a failed verdict inside it still does. The trace
|
||||
fixtures are not artifacts: the retrace jobs restore them from a cache entry.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
RESULT_PREFIXES = ("MobileGL-android-retrace-result-", "retrace-result-")
|
||||
FAILED_CONCLUSIONS = ("failure", "timed_out", "action_required")
|
||||
# GitHub's secondary rate limit allows about 180 mutating REST calls a minute. Requests start
|
||||
# at most this often, whatever the number of workers.
|
||||
DELETE_INTERVAL_SECONDS = 60.0 / 170.0
|
||||
DELETE_WORKERS = 3
|
||||
MAX_REFUSED_DELETES = 5
|
||||
|
||||
|
||||
def pause(seconds):
|
||||
time.sleep(seconds)
|
||||
|
||||
|
||||
def read_rows(text, columns):
|
||||
rows = []
|
||||
for line in text.splitlines():
|
||||
if line.strip():
|
||||
fields = line.split("\t")
|
||||
rows.append(fields + [""] * (columns - len(fields)))
|
||||
return rows
|
||||
|
||||
|
||||
def shard_statuses(download_roots):
|
||||
"""Artifact name -> its shard-status.json, for every downloaded result artifact that has one."""
|
||||
statuses = {}
|
||||
for root in download_roots:
|
||||
root = Path(root)
|
||||
if not root.is_dir():
|
||||
continue
|
||||
for directory in root.iterdir():
|
||||
try:
|
||||
status = json.loads((directory / "shard-status.json").read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if isinstance(status, dict) and isinstance(status.get("cases"), list):
|
||||
statuses[directory.name] = status
|
||||
return statuses
|
||||
|
||||
|
||||
def plan(artifacts, jobs, statuses):
|
||||
"""Returns (ids to delete, lines explaining what stays)."""
|
||||
failed_job_ids = {job_id for job_id, conclusion, _ in jobs if conclusion in FAILED_CONCLUSIONS}
|
||||
status_job_ids = {str(status.get("job_id") or "") for status in statuses.values()} - {""}
|
||||
# Failed jobs that left no status file: nothing says which cases they were running.
|
||||
unexplained = failed_job_ids - status_job_ids
|
||||
if not status_job_ids and failed_job_ids:
|
||||
# The statuses carry no job ids to match against. Count instead: each failed job must
|
||||
# be explained by an artifact that shows a case without a pass.
|
||||
explaining = sum(1 for status in statuses.values()
|
||||
if any(record.get("status") != "passed" for record in status["cases"]))
|
||||
unexplained = set() if explaining >= len(failed_job_ids) else failed_job_ids
|
||||
|
||||
failed_cases = set() # did not pass somewhere: every result that holds one stays
|
||||
for status in statuses.values():
|
||||
not_passed = [record for record in status["cases"] if record.get("status") != "passed"]
|
||||
if status_job_ids:
|
||||
job_failed = str(status.get("job_id") or "") in failed_job_ids
|
||||
else:
|
||||
# Without job ids, a status that shows a case without a pass stands for a failed job.
|
||||
job_failed = bool(failed_job_ids) and bool(not_passed)
|
||||
for record in not_passed:
|
||||
if record.get("status") == "failed" or job_failed:
|
||||
failed_cases.add(record.get("case"))
|
||||
|
||||
delete, kept = [], []
|
||||
for artifact_id, name in artifacts:
|
||||
if not name.startswith(RESULT_PREFIXES):
|
||||
continue
|
||||
reason = ""
|
||||
status = statuses.get(name)
|
||||
if unexplained:
|
||||
reason = f"{len(unexplained)} failed job(s) left no shard status"
|
||||
elif status is None:
|
||||
reason = "no shard status was downloaded for it"
|
||||
elif str(status.get("job_id") or "") in failed_job_ids:
|
||||
reason = "its job failed"
|
||||
else:
|
||||
cases = sorted(failed_cases.intersection(record.get("case") for record in status["cases"]))
|
||||
if cases:
|
||||
reason = "holds failed case(s): " + ", ".join(cases)
|
||||
if reason:
|
||||
kept.append(f"{name} ({artifact_id}): {reason}")
|
||||
else:
|
||||
delete.append((artifact_id, name))
|
||||
return delete, kept
|
||||
|
||||
|
||||
class Deleter:
|
||||
"""Runs `command` once per artifact, with the artifact id appended to its last word."""
|
||||
|
||||
def __init__(self, command):
|
||||
self.command = command
|
||||
self.lock = threading.Lock()
|
||||
self.next_start = 0.0
|
||||
self.refused = []
|
||||
self.deleted = 0
|
||||
|
||||
def wait_for_slot(self):
|
||||
with self.lock:
|
||||
now = time.monotonic()
|
||||
start = max(now, self.next_start)
|
||||
self.next_start = start + DELETE_INTERVAL_SECONDS
|
||||
if start > now:
|
||||
pause(start - now)
|
||||
|
||||
def delete(self, artifact_id, name):
|
||||
for attempt in (1, 2):
|
||||
with self.lock:
|
||||
if len(self.refused) >= MAX_REFUSED_DELETES:
|
||||
return
|
||||
self.wait_for_slot()
|
||||
done = subprocess.run(self.command[:-1] + [self.command[-1] + artifact_id],
|
||||
capture_output=True, text=True)
|
||||
if done.returncode == 0:
|
||||
with self.lock:
|
||||
self.deleted += 1
|
||||
return
|
||||
if attempt == 1:
|
||||
# Give a rate limit time to clear before the one retry.
|
||||
with self.lock:
|
||||
self.next_start = max(self.next_start, time.monotonic() + 30)
|
||||
with self.lock:
|
||||
self.refused.append(f"{name} ({artifact_id}): {' '.join(done.stderr.split())[:200]}")
|
||||
|
||||
def run(self, entries):
|
||||
pending = list(entries)
|
||||
|
||||
def work():
|
||||
while True:
|
||||
with self.lock:
|
||||
if not pending:
|
||||
return
|
||||
artifact_id, name = pending.pop(0)
|
||||
self.delete(artifact_id, name)
|
||||
|
||||
threads = [threading.Thread(target=work) for _ in range(min(DELETE_WORKERS, len(pending)))]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
|
||||
def gh_rows(path, query, columns):
|
||||
done = subprocess.run(["gh", "api", "--paginate", path, "--jq", query], capture_output=True, text=True)
|
||||
if done.returncode != 0:
|
||||
raise RuntimeError(f"gh api {path} failed: {done.stderr.strip()}")
|
||||
return read_rows(done.stdout, columns)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--downloads", action="append", default=[],
|
||||
help="Directory holding one sub-directory per downloaded result artifact.")
|
||||
parser.add_argument("--artifacts-file", help="Tab-separated id and name per line, instead of asking gh.")
|
||||
parser.add_argument("--jobs-file", help="Tab-separated id, conclusion and name per line, instead of asking gh.")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Print the plan, delete nothing.")
|
||||
options = parser.parse_args(argv)
|
||||
|
||||
repository = os.environ.get("GITHUB_REPOSITORY", "")
|
||||
run_id = os.environ.get("GITHUB_RUN_ID", "")
|
||||
try:
|
||||
if options.artifacts_file:
|
||||
artifacts = read_rows(Path(options.artifacts_file).read_text(encoding="utf-8"), 2)
|
||||
else:
|
||||
artifacts = gh_rows(f"repos/{repository}/actions/runs/{run_id}/artifacts?per_page=100",
|
||||
".artifacts[] | [.id, .name] | @tsv", 2)
|
||||
if options.jobs_file:
|
||||
jobs = read_rows(Path(options.jobs_file).read_text(encoding="utf-8"), 3)
|
||||
else:
|
||||
# 50 rows per page: the jobs endpoint answers HTTP 502 for larger pages of a re-run
|
||||
# attempt of a run this size.
|
||||
jobs = gh_rows(f"repos/{repository}/actions/runs/{run_id}/jobs?per_page=50",
|
||||
'.jobs[] | [.id, .conclusion // "", .name] | @tsv', 3)
|
||||
except (OSError, RuntimeError) as error:
|
||||
print(f"::error::Could not read the run's artifacts and jobs; nothing was deleted: {error}")
|
||||
return 1
|
||||
|
||||
delete, kept = plan([(row[0], row[1]) for row in artifacts], [tuple(row[:3]) for row in jobs],
|
||||
shard_statuses(options.downloads))
|
||||
for line in kept:
|
||||
print(f"Retaining {line}")
|
||||
if options.dry_run:
|
||||
for artifact_id, name in delete:
|
||||
print(f"Would delete {name} ({artifact_id})")
|
||||
return 0
|
||||
|
||||
deleter = Deleter(shlex.split(os.environ.get("MOBILEGL_CLEANUP_DELETE_COMMAND") or
|
||||
f"gh api --method DELETE repos/{repository}/actions/artifacts/"))
|
||||
started = time.monotonic()
|
||||
deleter.run(delete)
|
||||
for line in deleter.refused:
|
||||
print(f"::warning::Could not delete intermediate artifact {line}")
|
||||
if len(deleter.refused) >= MAX_REFUSED_DELETES:
|
||||
print("::warning::Stopped deleting intermediate artifacts after repeated refusals; the rest stay until they expire.")
|
||||
print(f"Deleted {deleter.deleted} intermediate Android artifact(s) in {time.monotonic() - started:.0f}s; "
|
||||
f"retained {len(kept)}.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
# MobileGL - android-plugin/retrace_shard_budget.py
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
"""Give every emulator shard the deadline and the job timeout its own cases need.
|
||||
|
||||
The cases of a shard share one job, so one case that hangs spends time the cases after it
|
||||
need. This reads a shard matrix of trace_cases.py (--format github-apk-shard-matrix or
|
||||
github-apk-transport-shard-matrix) on standard input, or from --matrix-file, and prints it
|
||||
with two fields added to every shard: deadline_seconds, which the retrace job passes to
|
||||
trace-replay-shard.sh, and timeout_minutes, the limit of that job. The deadline is what the
|
||||
shard needs for every case to use its whole time budget (trace_replay_shard.needed_seconds),
|
||||
up to --cap-seconds. A capped shard still replays every case; only when several of its
|
||||
replays hang to the end of their budgets are the cases after them recorded as not-run.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import trace_replay_shard # noqa: E402 (a sibling file, found through the line above)
|
||||
|
||||
|
||||
def shard_label(row):
|
||||
backend = row.get("backend")
|
||||
backend = backend.get("name", "") if isinstance(backend, dict) else str(backend or "")
|
||||
return f"{backend} / {row.get('transport') or 'monolith'}, shard {row.get('shard')}/{row.get('shards')}"
|
||||
|
||||
|
||||
def shard_needs(matrix):
|
||||
"""(label, case count, seconds needed) for every shard of the matrix."""
|
||||
rows = matrix.get("include") if isinstance(matrix, dict) else None
|
||||
if not isinstance(rows, list) or not rows:
|
||||
raise ValueError("the matrix has no shard rows under 'include'")
|
||||
needs = []
|
||||
for row in rows:
|
||||
cases = row.get("cases") if isinstance(row, dict) else None
|
||||
if not isinstance(cases, list) or not cases:
|
||||
raise ValueError(f"a shard row has no 'cases' list: {str(row)[:200]}")
|
||||
needs.append((shard_label(row), len(cases), trace_replay_shard.needed_seconds(cases)))
|
||||
return needs
|
||||
|
||||
|
||||
# Minutes a retrace job has around its shard runner: the setup before the replays and the
|
||||
# uploads after them.
|
||||
JOB_OVERHEAD_MINUTES = 15
|
||||
|
||||
# A hosted job may run six hours.
|
||||
HOSTED_JOB_LIMIT_MINUTES = 360
|
||||
|
||||
|
||||
def with_limits(matrix, cap_seconds):
|
||||
"""The matrix with deadline_seconds and timeout_minutes on every shard, and the report."""
|
||||
needs = shard_needs(matrix)
|
||||
rows = []
|
||||
report = []
|
||||
for row, (label, cases, seconds) in zip(matrix["include"], needs):
|
||||
deadline = min(seconds, cap_seconds)
|
||||
rows.append({**row, "deadline_seconds": deadline,
|
||||
"timeout_minutes": -(-deadline // 60) + JOB_OVERHEAD_MINUTES})
|
||||
if seconds > cap_seconds:
|
||||
report.append(f"retrace shard budget: {label} would need {seconds} s for all of its "
|
||||
f"{cases} case(s) to use their whole budgets; capped at {cap_seconds} s")
|
||||
largest = max(needs, key=lambda entry: entry[2])
|
||||
report.append(f"retrace shard budget: {len(needs)} shard(s), {len(report)} capped at "
|
||||
f"{cap_seconds} s; the most a shard needs is {largest[2]} s ({largest[0]}, "
|
||||
f"{largest[1]} case(s))")
|
||||
return {**matrix, "include": rows}, report
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--cap-seconds", type=int, required=True,
|
||||
help="The longest deadline a shard is given.")
|
||||
parser.add_argument("--matrix-file", help="The shard matrix; standard input when not given.")
|
||||
options = parser.parse_args(argv)
|
||||
limit = (HOSTED_JOB_LIMIT_MINUTES - JOB_OVERHEAD_MINUTES) * 60
|
||||
if not 0 < options.cap_seconds <= limit:
|
||||
print(f"retrace shard budget: --cap-seconds must be between 1 and {limit}", file=sys.stderr)
|
||||
return 2
|
||||
if options.matrix_file:
|
||||
text = Path(options.matrix_file).read_text(encoding="utf-8")
|
||||
else:
|
||||
text = sys.stdin.read()
|
||||
try:
|
||||
matrix, report = with_limits(json.loads(text), options.cap_seconds)
|
||||
except (ValueError, KeyError, TypeError) as error:
|
||||
print(f"retrace shard budget: the shard matrix cannot be read: {error}", file=sys.stderr)
|
||||
return 2
|
||||
for line in report:
|
||||
print(line, file=sys.stderr)
|
||||
print(json.dumps(matrix, separators=(",", ":")))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env python3
|
||||
# MobileGL - android-plugin/test_retrace_artifact_cleanup.py
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
"""Pin which intermediate APK artifacts the cleanup keeps, and how fast it deletes the rest."""
|
||||
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
HERE = pathlib.Path(__file__).resolve().parent
|
||||
_spec = importlib.util.spec_from_file_location("retrace_artifact_cleanup", HERE / "retrace_artifact_cleanup.py")
|
||||
cleanup = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(cleanup)
|
||||
|
||||
RESULT = "MobileGL-android-retrace-result-"
|
||||
|
||||
|
||||
def status(job_id, **cases):
|
||||
return {"job_id": job_id, "cases": [{"case": name, "status": verdict} for name, verdict in cases.items()]}
|
||||
|
||||
|
||||
def names(entries):
|
||||
return sorted(name for _, name in entries)
|
||||
|
||||
|
||||
class PlanTest(unittest.TestCase):
|
||||
ARTIFACTS = [("1", RESULT + "mono-gles"), ("2", RESULT + "inproc-gles"), ("3", RESULT + "spawn-vk"),
|
||||
("4", "mobilegl-android-monolith-control.html"), ("5", "MobileGL-plugin-release-abc.apk"),
|
||||
("6", "MobileGL-plugin-trace-release-abc.apk")]
|
||||
|
||||
def test_a_green_run_deletes_every_intermediate_and_nothing_else(self):
|
||||
statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed"),
|
||||
RESULT + "inproc-gles": status("12", a="passed", b="passed"),
|
||||
RESULT + "spawn-vk": status("13", c="passed")}
|
||||
jobs = [("11", "success", "x"), ("12", "success", "y"), ("13", "success", "z"), ("14", "", "retrace summary")]
|
||||
delete, kept = cleanup.plan(self.ARTIFACTS, jobs, statuses)
|
||||
self.assertEqual(kept, [])
|
||||
self.assertEqual(names(delete), sorted(name for _, name in self.ARTIFACTS[:3]))
|
||||
|
||||
def test_a_failed_case_keeps_every_shard_that_ran_it(self):
|
||||
statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed"),
|
||||
RESULT + "inproc-gles": status("12", a="passed", b="failed"),
|
||||
RESULT + "spawn-vk": status("13", c="passed")}
|
||||
jobs = [("11", "success", "x"), ("12", "failure", "y"), ("13", "success", "z")]
|
||||
delete, kept = cleanup.plan(self.ARTIFACTS, jobs, statuses)
|
||||
# Only b's results are worth comparing, so the monolith shard stays for b and the
|
||||
# spawn shard goes.
|
||||
self.assertEqual(names(delete), [RESULT + "spawn-vk"])
|
||||
self.assertEqual(len(kept), 2)
|
||||
self.assertTrue(any("inproc-gles" in line and "its job failed" in line for line in kept))
|
||||
self.assertTrue(any("mono-gles" in line and "holds failed case(s): b" in line for line in kept))
|
||||
|
||||
def test_cases_a_failed_job_never_reached_are_kept(self):
|
||||
statuses = {RESULT + "mono-gles": status("11", a="passed", b="not-run"),
|
||||
RESULT + "spawn-vk": status("13", c="passed")}
|
||||
jobs = [("11", "timed_out", "x"), ("13", "success", "z")]
|
||||
delete, kept = cleanup.plan(self.ARTIFACTS[:1] + self.ARTIFACTS[2:], jobs, statuses)
|
||||
self.assertEqual(names(delete), [RESULT + "spawn-vk"])
|
||||
self.assertEqual(len(kept), 1)
|
||||
|
||||
def test_a_job_that_failed_after_its_cases_passed_keeps_its_results(self):
|
||||
statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed"),
|
||||
RESULT + "spawn-vk": status("13", c="passed")}
|
||||
jobs = [("11", "failure", "upload failed"), ("13", "success", "z")]
|
||||
delete, kept = cleanup.plan(self.ARTIFACTS[:1] + self.ARTIFACTS[2:], jobs, statuses)
|
||||
self.assertEqual(names(delete), [RESULT + "spawn-vk"])
|
||||
self.assertTrue("mono-gles" in kept[0] and "its job failed" in kept[0])
|
||||
|
||||
def test_a_canceled_job_keeps_nothing_by_itself(self):
|
||||
statuses = {RESULT + "mono-gles": status("11", a="passed", b="not-run"),
|
||||
RESULT + "inproc-gles": status("12", a="failed", b="not-run")}
|
||||
jobs = [("11", "cancelled", "x"), ("12", "cancelled", "y")]
|
||||
delete, kept = cleanup.plan(self.ARTIFACTS[:2], jobs, statuses)
|
||||
# Case a failed in the inproc shard, so both shards that ran it stay.
|
||||
self.assertEqual(delete, [])
|
||||
self.assertEqual(len(kept), 2)
|
||||
only_not_run = {RESULT + "mono-gles": status("11", a="passed", b="not-run")}
|
||||
delete, kept = cleanup.plan(self.ARTIFACTS[:1], [("11", "cancelled", "x")], only_not_run)
|
||||
self.assertEqual((names(delete), kept), ([RESULT + "mono-gles"], []))
|
||||
|
||||
def test_a_result_without_a_status_is_kept(self):
|
||||
statuses = {RESULT + "mono-gles": status("11", a="passed")}
|
||||
delete, kept = cleanup.plan(self.ARTIFACTS[:2], [("11", "success", "x")], statuses)
|
||||
self.assertEqual(names(delete), [RESULT + "mono-gles"])
|
||||
self.assertIn("no shard status", kept[0])
|
||||
|
||||
def test_a_failed_job_without_a_status_keeps_everything(self):
|
||||
statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed")}
|
||||
jobs = [("11", "success", "x"), ("12", "failure", "died before its upload")]
|
||||
delete, kept = cleanup.plan(self.ARTIFACTS, jobs, statuses)
|
||||
self.assertEqual(delete, [])
|
||||
self.assertEqual(len(kept), 3)
|
||||
|
||||
def test_statuses_without_job_ids_fall_back_to_counting(self):
|
||||
artifacts = self.ARTIFACTS[:1] + self.ARTIFACTS[2:]
|
||||
explained = {RESULT + "mono-gles": status("", a="passed", b="not-run"),
|
||||
RESULT + "spawn-vk": status("", c="passed")}
|
||||
delete, _ = cleanup.plan(artifacts, [("12", "failure", "y")], explained)
|
||||
# The one status with a case that did not pass stands for the one failed job.
|
||||
self.assertEqual(names(delete), [RESULT + "spawn-vk"])
|
||||
unexplained = {RESULT + "mono-gles": status("", a="passed", b="passed"),
|
||||
RESULT + "spawn-vk": status("", c="passed")}
|
||||
delete, _ = cleanup.plan(artifacts, [("12", "failure", "y")], unexplained)
|
||||
self.assertEqual(delete, [])
|
||||
|
||||
|
||||
class RunTest(unittest.TestCase):
|
||||
@staticmethod
|
||||
def passed(artifacts):
|
||||
"""A shard status with one passed case for every artifact, each from a job of its own."""
|
||||
return {name: status(f"9{artifact_id}", a="passed") for artifact_id, name in artifacts}
|
||||
|
||||
def run_cleanup(self, artifacts, jobs, statuses, exit_code=0, dry_run=False):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = pathlib.Path(directory)
|
||||
for name, value in statuses.items():
|
||||
(root / "downloads" / "lane" / name).mkdir(parents=True)
|
||||
(root / "downloads" / "lane" / name / "shard-status.json").write_text(json.dumps(value))
|
||||
(root / "artifacts.tsv").write_text("".join(f"{i}\t{n}\n" for i, n in artifacts))
|
||||
(root / "jobs.tsv").write_text("".join("\t".join(job) + "\n" for job in jobs))
|
||||
# One file per call: the deletes run in parallel processes.
|
||||
(root / "deleted").mkdir()
|
||||
script = root / "delete.py"
|
||||
script.write_text("import pathlib, sys\n"
|
||||
"(pathlib.Path(sys.argv[1]) / sys.argv[2].replace('/', '_')).write_text('')\n"
|
||||
f"sys.exit({exit_code})\n")
|
||||
python = pathlib.Path(sys.executable).as_posix()
|
||||
command = f'"{python}" "{script.as_posix()}" "{(root / "deleted").as_posix()}" artifacts/'
|
||||
pauses = []
|
||||
output = io.StringIO()
|
||||
with mock.patch.dict(os.environ, {"MOBILEGL_CLEANUP_DELETE_COMMAND": command}), \
|
||||
mock.patch.object(cleanup, "pause", pauses.append), contextlib.redirect_stdout(output):
|
||||
code = cleanup.main(["--downloads", str(root / "downloads" / "lane"),
|
||||
"--artifacts-file", str(root / "artifacts.tsv"),
|
||||
"--jobs-file", str(root / "jobs.tsv")] + (["--dry-run"] if dry_run else []))
|
||||
deleted = sorted(entry.name.replace("_", "/") for entry in (root / "deleted").iterdir())
|
||||
return code, output.getvalue(), deleted, pauses
|
||||
|
||||
def test_deletes_are_spaced_under_the_secondary_rate_limit(self):
|
||||
artifacts = [(str(index), f"{RESULT}shard{index}") for index in range(12)]
|
||||
code, output, deleted, pauses = self.run_cleanup(artifacts, [], self.passed(artifacts))
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(deleted, sorted(f"artifacts/{index}" for index in range(12)))
|
||||
self.assertIn("Deleted 12 intermediate Android artifact(s)", output)
|
||||
# Twelve starts are booked at least 11 intervals apart, so the rate stays under 180 a minute.
|
||||
self.assertLess(60 / cleanup.DELETE_INTERVAL_SECONDS, 180)
|
||||
self.assertGreater(len(pauses), 0)
|
||||
|
||||
def test_refused_deletes_warn_and_leave_the_run_green(self):
|
||||
artifacts = [(str(index), f"{RESULT}shard{index}") for index in range(8)]
|
||||
code, output, _, _ = self.run_cleanup(artifacts, [], self.passed(artifacts), exit_code=1)
|
||||
self.assertEqual(code, 0)
|
||||
self.assertIn("::warning::Could not delete intermediate artifact", output)
|
||||
self.assertIn("Stopped deleting", output)
|
||||
self.assertIn("Deleted 0 intermediate", output)
|
||||
|
||||
def test_a_dry_run_deletes_nothing(self):
|
||||
artifacts = [("1", RESULT + "shard1")]
|
||||
code, output, deleted, _ = self.run_cleanup(artifacts, [], self.passed(artifacts), dry_run=True)
|
||||
self.assertEqual((code, deleted), (0, []))
|
||||
self.assertIn(f"Would delete {RESULT}shard1 (1)", output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,587 @@
|
||||
#!/usr/bin/env python3
|
||||
# MobileGL - android-plugin/test_trace_replay_shard.py
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
"""Drive the emulator shard runner with a fake adb, replay script and AVD script.
|
||||
|
||||
No device is needed. The fakes record every call, so the tests pin what each case is run
|
||||
with, that every case ends with a verdict, and what a failure, an infrastructure retry, a
|
||||
timeout or a dead emulator does to the rest of the shard.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
HERE = pathlib.Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def load(name):
|
||||
spec = importlib.util.spec_from_file_location(name, HERE / f"{name}.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
shard = load("trace_replay_shard")
|
||||
budget = load("retrace_shard_budget")
|
||||
|
||||
BLISS = "minecraft-1.21.4-fabric-iris-bliss-in-world"
|
||||
ITERATIONRP = "minecraft-1.21.4-fabric-iris-iterationrp-in-world"
|
||||
SPLIT = "MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1"
|
||||
|
||||
FAKE_COMMON = r'''
|
||||
import json, os, pathlib, sys
|
||||
state = pathlib.Path(os.environ["FAKE_STATE"])
|
||||
def record(name, entry):
|
||||
with open(state / name, "a", encoding="utf-8") as log:
|
||||
log.write(json.dumps(entry) + "\n")
|
||||
def scenario():
|
||||
path = state / "scenario.json"
|
||||
return json.loads(path.read_text()) if path.exists() else {}
|
||||
def count(name):
|
||||
path = state / f"count-{name}"
|
||||
value = int(path.read_text()) if path.exists() else 0
|
||||
path.write_text(str(value + 1))
|
||||
return value
|
||||
'''
|
||||
|
||||
FAKE_ADB = FAKE_COMMON + r'''
|
||||
args = sys.argv[1:]
|
||||
record("adb.log", args)
|
||||
plan = scenario()
|
||||
if args[:1] == ["get-state"]:
|
||||
print("offline" if count("get-state") < plan.get("offline_get_state_calls", 0) else "device")
|
||||
elif args[:3] == ["shell", "getprop", "sys.boot_completed"]:
|
||||
print("1")
|
||||
elif args[:3] == ["shell", "pm", "path"]:
|
||||
print("package:/data/app/trace/base.apk")
|
||||
elif args[:3] == ["shell", "pm", "clear"]:
|
||||
print("Failed" if plan.get("pm_clear_fails") else "Success")
|
||||
elif args[:1] == ["install"]:
|
||||
if count("install") < plan.get("failing_installs", 0):
|
||||
print("Failure [INSTALL_FAILED]")
|
||||
sys.exit(1)
|
||||
print("Success")
|
||||
elif args[:2] == ["shell", "run-as"] and args[3:] == ["id", "-un"]:
|
||||
print("u0_a77")
|
||||
elif args[:3] == ["shell", "ps", "-A"]:
|
||||
print("USER PID NAME")
|
||||
print("root 1 init")
|
||||
if count("ps") < plan.get("ps_calls_with_leftover", 0):
|
||||
print("u0_a77 4242 libMobileGLServer.so")
|
||||
'''
|
||||
|
||||
FAKE_REPLAY = FAKE_COMMON + r'''
|
||||
import time
|
||||
args = sys.argv[1:]
|
||||
def value(flag):
|
||||
return args[args.index(flag) + 1]
|
||||
case, backend, root = value("--case"), value("--backend"), pathlib.Path(value("--result-root"))
|
||||
record("replay.log", {"case": case, "arguments": args, "environment": {
|
||||
key: os.environ.get(key) for key in (
|
||||
"MOBILEGL_ESPRYT_USE_ANGLE", "MOBILEGL_TRACE_ANGLE_VARIANT", "MOBILEGL_MAGMA_R11G11B10F_FALLBACK",
|
||||
"MOBILEGL_ITRP_FIX", "MOBILEGL_TRACE_SKIP_INSTALL", "MOBILEGL_TRACE_POLL_INTERVAL")}})
|
||||
(state / "replay.pid").write_text(str(os.getpid()))
|
||||
steps = scenario().get("replay", {}).get(case, [])
|
||||
attempt = count("replay-" + case)
|
||||
step = steps[attempt] if attempt < len(steps) else {}
|
||||
time.sleep(step.get("sleep", 0))
|
||||
directory = root / f"{case}-{backend}"
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
code = step.get("exit", 0)
|
||||
if code == 75:
|
||||
with open(root / "infrastructure-failure-reason.txt", "a") as reasons:
|
||||
reasons.write(step.get("reason", "device-unavailable") + "\n")
|
||||
if step.get("result", code in (0, 1)):
|
||||
(directory / "result.json").write_text(json.dumps(
|
||||
{"passed": step.get("passed", code == 0), "ssim": 0.999, "message": step.get("message", "retrace completed")}))
|
||||
transport = "inproc" if "--require-inproc" in args else "spawn" if "--require-spawn" in args else ""
|
||||
if transport and code == 0 and step.get("proof", True):
|
||||
(directory / "transport-proof.json").write_text(json.dumps({"passed": True, "required_transport": transport}))
|
||||
sys.exit(code)
|
||||
'''
|
||||
|
||||
FAKE_AVD = FAKE_COMMON + r'''
|
||||
args = sys.argv[1:]
|
||||
record("avd.log", args)
|
||||
if args[:1] == ["start"]:
|
||||
exits = scenario().get("avd_start_exits", [])
|
||||
index = count("avd-start")
|
||||
sys.exit(exits[index] if index < len(exits) else 0)
|
||||
'''
|
||||
|
||||
|
||||
def case(name, **extra):
|
||||
base = {"name": name, "trace_archive": f"fixtures/{name}.tgz", "trace_file": "trace.trace",
|
||||
"golden": f"fixtures/{name}.png", "target_call": 1234, "width": 854, "height": 480,
|
||||
"ssim_threshold": 0.99, "crop_x": 0, "crop_y": 0, "crop_width": 0, "crop_height": 0,
|
||||
"timeout_seconds": 900}
|
||||
base.update(extra)
|
||||
return base
|
||||
|
||||
|
||||
class ShardTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temporary.cleanup)
|
||||
self.root = pathlib.Path(self.temporary.name)
|
||||
self.state = self.root / "state"
|
||||
self.state.mkdir()
|
||||
self.results = self.root / "results"
|
||||
(self.root / "fixtures").mkdir()
|
||||
self.apk = self.root / "trace.apk"
|
||||
self.apk.write_bytes(b"apk")
|
||||
self.emulator_log = self.root / "emulator.log"
|
||||
self.emulator_log.write_text("boot\n")
|
||||
python = pathlib.Path(sys.executable).as_posix()
|
||||
commands = {}
|
||||
for name, source in (("adb", FAKE_ADB), ("replay", FAKE_REPLAY), ("avd", FAKE_AVD)):
|
||||
script = self.root / f"fake_{name}.py"
|
||||
script.write_text(source)
|
||||
commands[name] = f'"{python}" "{script.as_posix()}"'
|
||||
self.environment = {
|
||||
"FAKE_STATE": str(self.state), "ADB": commands["adb"],
|
||||
"MOBILEGL_SHARD_REPLAY_COMMAND": commands["replay"], "MOBILEGL_SHARD_AVD_COMMAND": commands["avd"],
|
||||
"MOBILEGL_GPU_HANG_BUDGET_MS": "0", "AVD_NAME": "mobilegl-ci",
|
||||
"EMULATOR_LOG": str(self.emulator_log), "EMULATOR_PID_FILE": str(self.root / "emulator.pid"),
|
||||
"GITHUB_STEP_SUMMARY": str(self.root / "summary.md"),
|
||||
}
|
||||
|
||||
def fixture(self, name):
|
||||
path = self.root / "fixtures" / name
|
||||
path.write_bytes(b"png")
|
||||
return str(path)
|
||||
|
||||
def run_shard(self, cases, *extra, backend="DirectVulkan", transport="monolith", scenario=None, env=None):
|
||||
if scenario is not None:
|
||||
(self.state / "scenario.json").write_text(json.dumps(scenario))
|
||||
arguments = ["--apk-file", str(self.apk), "--backend", backend, "--gpu", "lavapipe",
|
||||
"--transport", transport, "--lane", f"{transport}-lane", "--cases-json", json.dumps(cases),
|
||||
"--result-root", str(self.results), "--fixture-root", str(self.root / "work"), *extra]
|
||||
output = io.StringIO()
|
||||
with mock.patch.dict(os.environ, {**self.environment, **(env or {})}), \
|
||||
mock.patch.object(shard, "pause", lambda seconds: None), \
|
||||
contextlib.redirect_stdout(output):
|
||||
status = shard.main(arguments)
|
||||
return status, output.getvalue()
|
||||
|
||||
def log(self, name):
|
||||
path = self.state / name
|
||||
return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else []
|
||||
|
||||
def status(self, name, backend="DirectVulkan"):
|
||||
return json.loads((self.results / f"{name}-{backend}" / "case-status.json").read_text())
|
||||
|
||||
def shard_status(self):
|
||||
return json.loads((self.results / "shard-status.json").read_text())
|
||||
|
||||
# ---- what each case is run with ----------------------------------------------------
|
||||
|
||||
def planned(self, cases, backend, transport):
|
||||
status, output = self.run_shard(cases, "--dry-run", backend=backend, transport=transport)
|
||||
self.assertEqual(status, 0)
|
||||
return [json.loads(line) for line in output.splitlines()]
|
||||
|
||||
def test_plain_case_arguments_per_transport(self):
|
||||
cases = [case("plain", alternate_golden="fixtures/plain-mali.png")]
|
||||
head = ["--apk-file", str(self.apk), "--package", "top.mobilegl.plugin.trace", "--backend", "DirectVulkan",
|
||||
"--result-root", str(self.results), "--fixture-root", str(self.root / "work"), "--case", "plain",
|
||||
"--trace-archive", "fixtures/plain.tgz", "--trace-file", "trace.trace",
|
||||
"--golden", "fixtures/plain.png", "--alternate-golden", "fixtures/plain-mali.png",
|
||||
"--target-call", "1234", "--width", "854", "--height", "480", "--ssim-threshold", "0.99",
|
||||
"--crop-x", "0", "--crop-y", "0", "--crop-width", "0", "--crop-height", "0",
|
||||
"--timeout-seconds", "900"]
|
||||
tails = {
|
||||
"monolith": ["--env", "MOBILEGL_TRANSPORT=monolith;MOBILEGL_GPU_HANG_BUDGET_MS=0"],
|
||||
"inproc": ["--env", f"MOBILEGL_TRANSPORT=inproc;{SPLIT};MOBILEGL_GPU_HANG_BUDGET_MS=0", "--require-inproc"],
|
||||
"spawn": ["--env", f"MOBILEGL_TRANSPORT=spawn;{SPLIT};MOBILEGL_GPU_HANG_BUDGET_MS=0",
|
||||
"--use-pbuffer", "--require-spawn"],
|
||||
}
|
||||
for transport, tail in tails.items():
|
||||
plan, = self.planned(cases, "DirectVulkan", transport)
|
||||
self.assertEqual(plan["arguments"], head + tail, transport)
|
||||
self.assertEqual(plan["timeout"], 1200)
|
||||
self.assertEqual(plan["environment"], {
|
||||
"MOBILEGL_ESPRYT_USE_ANGLE": "0", "MOBILEGL_TRACE_ANGLE_VARIANT": "ec889e6ea831",
|
||||
"MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "1", "MOBILEGL_ITRP_FIX": "Auto"})
|
||||
|
||||
def test_case_specific_settings(self):
|
||||
cases = [case(BLISS), case(ITERATIONRP), case("flush", coherent_as_flush=True),
|
||||
case("lod", avoid_angle_llvmpipe_explicit_lod_bias=True), case("plain", ssim_threshold=0.985)]
|
||||
gles = {plan["case"]: plan for plan in self.planned(cases, "DirectGLES", "monolith")}
|
||||
vulkan = {plan["case"]: plan for plan in self.planned(cases, "DirectVulkan", "monolith")}
|
||||
|
||||
def flags(plan):
|
||||
return [argument for argument in plan["arguments"][plan["arguments"].index("--env") + 2:]]
|
||||
|
||||
self.assertEqual(gles[BLISS]["environment"]["MOBILEGL_TRACE_ANGLE_VARIANT"], "90a62123d794")
|
||||
self.assertEqual(vulkan[BLISS]["environment"]["MOBILEGL_TRACE_ANGLE_VARIANT"], "90a62123d794")
|
||||
self.assertEqual(gles["plain"]["environment"], {
|
||||
"MOBILEGL_ESPRYT_USE_ANGLE": "1", "MOBILEGL_TRACE_ANGLE_VARIANT": "ec889e6ea831",
|
||||
"MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "0", "MOBILEGL_ITRP_FIX": "Auto"})
|
||||
self.assertEqual(gles[ITERATIONRP]["environment"]["MOBILEGL_ITRP_FIX"], "ForceOn")
|
||||
self.assertEqual(flags(gles[BLISS]), ["--avoid-angle-llvmpipe-sampler-mipmap-min-filter"])
|
||||
self.assertEqual(flags(vulkan[BLISS]), [])
|
||||
self.assertEqual(flags(gles["lod"]), ["--avoid-angle-llvmpipe-explicit-lod-bias"])
|
||||
self.assertEqual(flags(vulkan["lod"]), [])
|
||||
self.assertEqual(flags(gles["flush"]), ["--coherent-as-flush"])
|
||||
self.assertEqual(flags(vulkan["flush"]), ["--coherent-as-flush"])
|
||||
self.assertEqual(flags(gles["plain"]), [])
|
||||
arguments = gles["plain"]["arguments"]
|
||||
self.assertEqual(arguments[arguments.index("--ssim-threshold") + 1], "0.985")
|
||||
self.assertEqual(arguments[arguments.index("--alternate-golden") + 1], "")
|
||||
|
||||
def test_misconfigured_case_lists_run_nothing(self):
|
||||
for cases in ([case("a"), case("a")], [case("a", ci_backends=["DirectGLES"])], [{"name": "a"}], []):
|
||||
status, output = self.run_shard(cases)
|
||||
self.assertEqual(status, 2, cases)
|
||||
self.assertIn("::error title=Android retrace shard misconfigured::", output)
|
||||
self.assertEqual(self.log("adb.log"), [])
|
||||
|
||||
# ---- the loop ----------------------------------------------------------------------
|
||||
|
||||
def test_every_case_runs_in_order_on_one_install(self):
|
||||
cases = [case("a", golden=self.fixture("a.png"), alternate_golden=self.fixture("a-mali.png")),
|
||||
case("b", golden=self.fixture("b.png")), case("c")]
|
||||
status, output = self.run_shard(cases, transport="inproc")
|
||||
self.assertEqual(status, 0, output)
|
||||
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b", "c"])
|
||||
for entry in self.log("replay.log"):
|
||||
self.assertEqual(entry["environment"]["MOBILEGL_TRACE_SKIP_INSTALL"], "1")
|
||||
self.assertEqual(entry["environment"]["MOBILEGL_TRACE_POLL_INTERVAL"], "0.5")
|
||||
adb = self.log("adb.log")
|
||||
self.assertEqual(sum(1 for call in adb if call[:1] == ["install"]), 1)
|
||||
self.assertEqual(sum(1 for call in adb if call[:3] == ["shell", "pm", "clear"]), 3)
|
||||
self.assertEqual(sum(1 for call in adb if call[:3] == ["shell", "rm", "-f"]), 3)
|
||||
self.assertIn(["shell", "rm", "-f", "/data/local/tmp/mobilegl-b.trace", "/data/local/tmp/mobilegl-b.golden.png",
|
||||
"/data/local/tmp/mobilegl-b.alternate-golden.png"], adb)
|
||||
self.assertEqual(self.log("avd.log"), [])
|
||||
directory = self.results / "a-DirectVulkan"
|
||||
self.assertEqual(json.loads((directory / "lane.json").read_text()),
|
||||
{"lane": "inproc-lane", "transport": "inproc"})
|
||||
self.assertTrue((directory / "a-DirectVulkan-golden.png").is_file())
|
||||
self.assertTrue((directory / "a-DirectVulkan-alternate-golden.png").is_file())
|
||||
self.assertFalse((self.results / "c-DirectVulkan" / "c-DirectVulkan-golden.png").exists())
|
||||
whole = self.shard_status()
|
||||
self.assertTrue(whole["complete"])
|
||||
self.assertEqual([(entry["case"], entry["status"]) for entry in whole["cases"]],
|
||||
[("a", "passed"), ("b", "passed"), ("c", "passed")])
|
||||
self.assertIn("| b | passed | 1 |", (self.root / "summary.md").read_text())
|
||||
|
||||
def test_a_failed_case_does_not_stop_the_shard(self):
|
||||
status, output = self.run_shard([case("a"), case("b"), case("c")], scenario={
|
||||
"replay": {"b": [{"exit": 1, "message": "ssim=0.5"}]}})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b", "c"])
|
||||
self.assertEqual([self.status(name)["status"] for name in "abc"], ["passed", "failed", "passed"])
|
||||
self.assertIn("::error title=Android retrace failed (DirectVulkan / monolith)::b: exit 1: ssim=0.5", output)
|
||||
self.assertEqual(output.count("::error"), 1)
|
||||
self.assertTrue(self.shard_status()["complete"])
|
||||
self.assertTrue((self.results / "b-DirectVulkan" / "emulator.log").is_file())
|
||||
|
||||
def test_the_case_after_one_that_did_not_pass_starts_on_a_cold_boot(self):
|
||||
status, _ = self.run_shard([case("a"), case("b"), case("c"), case("d")], scenario={
|
||||
"replay": {"a": [{"exit": 75}], "b": [{"exit": 1}]}})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual([self.status(name)["status"] for name in "abcd"], ["passed", "failed", "passed", "passed"])
|
||||
# One restart for a's infrastructure retry and one before c. A case that passed on
|
||||
# its retry costs the next case nothing.
|
||||
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start", "stop", "start"])
|
||||
self.assertEqual(self.status("b")["notes"], [])
|
||||
self.assertEqual(self.status("c")["notes"], ["emulator restarted before the case: "
|
||||
"the case before it, b, did not pass"])
|
||||
self.assertEqual(self.status("d")["notes"], [])
|
||||
self.assertEqual(sum(1 for call in self.log("adb.log") if call[:1] == ["install"]), 3)
|
||||
|
||||
def test_a_failed_case_without_an_avd_to_restart_does_not_stop_the_shard(self):
|
||||
status, _ = self.run_shard([case("a"), case("b")], scenario={"replay": {"a": [{"exit": 1}]}},
|
||||
env={"AVD_NAME": ""})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual([self.status(name)["status"] for name in "ab"], ["failed", "passed"])
|
||||
self.assertEqual(self.log("avd.log"), [])
|
||||
|
||||
def test_an_exit_0_without_a_passed_result_or_proof_fails(self):
|
||||
status, _ = self.run_shard([case("a"), case("b")], transport="spawn", scenario={
|
||||
"replay": {"a": [{"exit": 0, "result": False}], "b": [{"exit": 0, "proof": False}]}})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertIn("without a passed result.json", self.status("a")["reason"])
|
||||
self.assertIn("without a passed spawn transport proof", self.status("b")["reason"])
|
||||
|
||||
def test_an_infrastructure_failure_restarts_and_retries_once(self):
|
||||
status, output = self.run_shard([case("a"), case("b")], scenario={
|
||||
"replay": {"a": [{"exit": 75, "reason": "angle-surface-lost", "result": True, "passed": False}]}})
|
||||
self.assertEqual(status, 0, output)
|
||||
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "a", "b"])
|
||||
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start"])
|
||||
self.assertEqual(self.log("avd.log")[1], [
|
||||
"start", "--avd-name", "mobilegl-ci", "--emulator-log", str(self.emulator_log),
|
||||
"--pid-file", str(self.root / "emulator.pid"), "--gpu", "lavapipe", "--boot-timeout", "300"])
|
||||
self.assertIn(["kill-server"], self.log("adb.log"))
|
||||
self.assertEqual(sum(1 for call in self.log("adb.log") if call[:1] == ["install"]), 2)
|
||||
record = self.status("a")
|
||||
self.assertEqual((record["status"], record["attempts"], record["exit_codes"]), ("passed", 2, [75, 0]))
|
||||
self.assertEqual(record["surface_lost_retries"], 1)
|
||||
kept = self.results / "a-DirectVulkan" / "first-attempt"
|
||||
self.assertTrue((kept / "result.first-attempt.json").is_file())
|
||||
self.assertFalse((kept / "result.json").exists())
|
||||
self.assertIn("::warning::Android emulator infrastructure failed", output)
|
||||
self.assertIn("surface-lost retries: 1 (DirectVulkan, a)", (self.root / "summary.md").read_text())
|
||||
self.assertTrue((self.results / "diagnostics" / "emulator-boot-1-ended.log").is_file())
|
||||
self.assertEqual(self.shard_status()["emulator_boots"], 2)
|
||||
|
||||
def test_a_second_infrastructure_failure_is_the_verdict(self):
|
||||
status, _ = self.run_shard([case("a"), case("b")], scenario={
|
||||
"replay": {"a": [{"exit": 75}, {"exit": 75}]}})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual(self.status("a")["exit_codes"], [75, 75])
|
||||
self.assertEqual(self.status("a")["status"], "failed")
|
||||
self.assertEqual(self.status("b")["status"], "passed")
|
||||
# The surface-lost count is this case's own: the reasons of earlier cases do not leak in.
|
||||
self.assertEqual(self.status("b")["surface_lost_retries"], 0)
|
||||
|
||||
def test_the_deadline_cuts_a_running_case_and_names_the_rest(self):
|
||||
status, output = self.run_shard([case("a"), case("b"), case("c")], "--deadline-seconds", "5", scenario={
|
||||
"replay": {"b": [{"sleep": 30}]}})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b"])
|
||||
# b never had its own 1200 s, so the cut is not charged to it as a failure.
|
||||
self.assertEqual([self.status(name)["status"] for name in "abc"], ["passed", "not-run", "not-run"])
|
||||
self.assertRegex(self.status("b")["reason"], r"^cut by the shard deadline after \d+ s of a 1200 s budget$")
|
||||
self.assertEqual(self.status("b")["exit_codes"], [124])
|
||||
self.assertIn("deadline passed before this case started", self.status("c")["reason"])
|
||||
self.assertIn("::error title=Android retrace not-run (DirectVulkan / monolith)::b: cut by the shard deadline",
|
||||
output)
|
||||
self.assertIn("::error title=Android retrace shard incomplete (DirectVulkan / monolith)::"
|
||||
"1 verdict(s) for 3 case(s)", output)
|
||||
self.assertFalse(self.shard_status()["complete"])
|
||||
# Nothing is booted for a case the deadline keeps from starting.
|
||||
self.assertEqual(self.log("avd.log"), [])
|
||||
|
||||
def test_a_case_that_runs_out_of_its_own_budget_fails(self):
|
||||
# The deadline is far away, so the one second a has is its own budget.
|
||||
with mock.patch.object(shard, "REPLAY_GRACE_SECONDS", 0):
|
||||
status, _ = self.run_shard([case("a", timeout_seconds=1), case("b")], "--deadline-seconds", "600",
|
||||
scenario={"replay": {"a": [{"sleep": 30}]}})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual([self.status(name)["status"] for name in "ab"], ["failed", "passed"])
|
||||
self.assertEqual(self.status("a")["reason"], "the replay script exceeded its time budget")
|
||||
self.assertTrue(self.shard_status()["complete"])
|
||||
|
||||
def test_an_infrastructure_retry_is_not_started_after_the_deadline(self):
|
||||
def restart_past_the_deadline(runner, reason):
|
||||
runner.started -= 1000
|
||||
return True
|
||||
|
||||
with mock.patch.object(shard.Shard, "restart_emulator", restart_past_the_deadline):
|
||||
status, _ = self.run_shard([case("a"), case("b")], "--deadline-seconds", "600",
|
||||
scenario={"replay": {"a": [{"exit": 75}]}})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a"])
|
||||
self.assertEqual([self.status(name)["status"] for name in "ab"], ["not-run", "not-run"])
|
||||
self.assertEqual(self.status("a")["reason"],
|
||||
"the shard deadline passed before the infrastructure retry could start")
|
||||
self.assertEqual(self.status("a")["exit_codes"], [75])
|
||||
|
||||
# ---- the time a shard needs --------------------------------------------------------
|
||||
|
||||
def test_needed_seconds_lets_every_case_use_its_budget(self):
|
||||
cases = [case("a", timeout_seconds=1800), case("b", timeout_seconds=900), case("c", timeout_seconds="180")]
|
||||
# Every case to the end of its budget, the longest once more, and the overhead per case.
|
||||
self.assertEqual(shard.needed_seconds(cases), (2100 + 1200 + 480) + 2100 + 3 * 180)
|
||||
self.assertEqual(shard.needed_seconds(cases[2:]), 480 + 480 + 180)
|
||||
|
||||
def test_a_deadline_of_needed_seconds_gives_every_hanging_case_its_verdict(self):
|
||||
cases = [case(name, timeout_seconds=1) for name in "abc"]
|
||||
# Scaled down: one second per case, and ten for what the fakes take around it.
|
||||
with mock.patch.object(shard, "REPLAY_GRACE_SECONDS", 0), \
|
||||
mock.patch.object(shard, "CASE_OVERHEAD_SECONDS", 10):
|
||||
deadline = shard.needed_seconds(cases)
|
||||
self.assertEqual(deadline, 3 + 1 + 30)
|
||||
status, _ = self.run_shard(cases, "--deadline-seconds", str(deadline), scenario={
|
||||
"replay": {name: [{"sleep": 60}] for name in "abc"}})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b", "c"])
|
||||
for name in "abc":
|
||||
self.assertEqual((self.status(name)["status"], self.status(name)["reason"]),
|
||||
("failed", "the replay script exceeded its time budget"))
|
||||
self.assertTrue(self.shard_status()["complete"])
|
||||
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start", "stop", "start"])
|
||||
|
||||
def test_every_shard_gets_its_own_deadline_and_job_timeout(self):
|
||||
matrix = {"include": [
|
||||
{"backend": {"name": "DirectGLES", "gpu": "software"}, "lane": "spawn-acceptance", "transport": "spawn",
|
||||
"shard": 1, "shards": 2, "names": ["a", "b"],
|
||||
"cases": [case("a", timeout_seconds=1800), case("b", timeout_seconds=1800)]},
|
||||
{"backend": {"name": "DirectVulkan", "gpu": "lavapipe"}, "shard": 2, "shards": 2, "names": ["c"],
|
||||
"cases": [case("c", timeout_seconds=180)]}]}
|
||||
matrix_file = self.root / "matrix.json"
|
||||
matrix_file.write_text(json.dumps(matrix))
|
||||
|
||||
def limits(cap, path=matrix_file):
|
||||
output, report = io.StringIO(), io.StringIO()
|
||||
with contextlib.redirect_stdout(output), contextlib.redirect_stderr(report):
|
||||
status = budget.main(["--cap-seconds", str(cap), "--matrix-file", str(path)])
|
||||
return status, output.getvalue(), report.getvalue()
|
||||
|
||||
self.assertEqual(budget.shard_needs(matrix), [("DirectGLES / spawn, shard 1/2", 2, 6660),
|
||||
("DirectVulkan / monolith, shard 2/2", 1, 1140)])
|
||||
# Under the cap every shard gets exactly what its cases need.
|
||||
status, output, report = limits(6660)
|
||||
self.assertEqual(status, 0, report)
|
||||
rows = json.loads(output)["include"]
|
||||
self.assertEqual([(row["deadline_seconds"], row["timeout_minutes"]) for row in rows],
|
||||
[(6660, 111 + budget.JOB_OVERHEAD_MINUTES), (1140, 19 + budget.JOB_OVERHEAD_MINUTES)])
|
||||
# Nothing but the two fields is added, and no case is dropped or changed.
|
||||
self.assertEqual([{key: value for key, value in row.items()
|
||||
if key not in ("deadline_seconds", "timeout_minutes")} for row in rows],
|
||||
matrix["include"])
|
||||
self.assertIn("2 shard(s), 0 capped at 6660 s; the most a shard needs is 6660 s", report)
|
||||
# Over the cap a shard is cut to it and named; the other shard is untouched.
|
||||
status, output, report = limits(6000)
|
||||
self.assertEqual(status, 0, report)
|
||||
rows = json.loads(output)["include"]
|
||||
self.assertEqual([(row["deadline_seconds"], row["timeout_minutes"]) for row in rows],
|
||||
[(6000, 100 + budget.JOB_OVERHEAD_MINUTES), (1140, 19 + budget.JOB_OVERHEAD_MINUTES)])
|
||||
self.assertIn("DirectGLES / spawn, shard 1/2 would need 6660 s for all of its 2 case(s)", report)
|
||||
self.assertNotIn("DirectVulkan / monolith, shard 2/2 would need", report)
|
||||
# A cap that could not fit in a hosted job, and a matrix that cannot be read.
|
||||
self.assertEqual(limits(0)[0], 2)
|
||||
self.assertEqual(limits(360 * 60)[0], 2)
|
||||
for broken in ({"include": []}, {"include": [{"names": ["a"]}]}, []):
|
||||
matrix_file.write_text(json.dumps(broken))
|
||||
status, output, _ = limits(7200)
|
||||
self.assertEqual((status, output), (2, ""), broken)
|
||||
|
||||
def test_the_workflow_takes_its_limits_from_the_matrix(self):
|
||||
text = (HERE.parent / ".github" / "workflows" / "apk.yml").read_text(encoding="utf-8")
|
||||
retrace = text[text.index("\n retrace:"):text.index("\n retrace-summary:")]
|
||||
cap, = re.findall(r'^\s+RETRACE_SHARD_DEADLINE_CAP_SECONDS: "(\d+)"\s*$', text, flags=re.MULTILINE)
|
||||
# The runner's deadline and the job's limit are the two fields the budget script adds.
|
||||
self.assertIn('--deadline-seconds "${{ matrix.deadline_seconds }}"', retrace)
|
||||
self.assertEqual(re.findall(r"^ timeout-minutes: (.+?)\s*$", retrace, flags=re.MULTILINE),
|
||||
["${{ matrix.timeout_minutes }}"])
|
||||
self.assertIn('--cap-seconds "${RETRACE_SHARD_DEADLINE_CAP_SECONDS}"', text)
|
||||
# The capped deadline and the job's overhead stay inside the six hours a hosted job may run.
|
||||
self.assertLessEqual(-(-int(cap) // 60) + budget.JOB_OVERHEAD_MINUTES, budget.HOSTED_JOB_LIMIT_MINUTES)
|
||||
|
||||
def test_an_unhealthy_emulator_is_restarted_before_the_next_case(self):
|
||||
status, output = self.run_shard([case("a"), case("b")], scenario={"offline_get_state_calls": 1})
|
||||
self.assertEqual(status, 0, output)
|
||||
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start"])
|
||||
self.assertIn("emulator restarted before the case: adb state is offline", self.status("a")["notes"])
|
||||
self.assertEqual(self.status("b")["notes"], [])
|
||||
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b"])
|
||||
|
||||
def test_the_restart_switch_cold_boots_before_every_case_but_the_first(self):
|
||||
status, _ = self.run_shard([case("a"), case("b"), case("c")],
|
||||
env={"MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE": "1"})
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start", "stop", "start"])
|
||||
self.assertEqual(sum(1 for call in self.log("adb.log") if call[:1] == ["install"]), 3)
|
||||
|
||||
def test_an_emulator_that_does_not_come_back_leaves_named_rows(self):
|
||||
status, output = self.run_shard([case("a"), case("b"), case("c")], scenario={
|
||||
"replay": {"a": [{"exit": 75}]}, "avd_start_exits": [2, 2]})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a"])
|
||||
self.assertEqual([self.status(name)["status"] for name in "abc"], ["failed", "not-run", "not-run"])
|
||||
self.assertEqual(output.count("::error title=Android retrace"), 4)
|
||||
for name in "abc":
|
||||
self.assertTrue((self.results / f"{name}-DirectVulkan" / "lane.json").is_file())
|
||||
|
||||
def test_an_apk_that_does_not_install_runs_nothing(self):
|
||||
status, _ = self.run_shard([case("a"), case("b")], scenario={"failing_installs": 9})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual(self.log("replay.log"), [])
|
||||
self.assertEqual([self.status(name)["status"] for name in "ab"], ["not-run", "not-run"])
|
||||
|
||||
def test_processes_left_after_force_stop_are_killed_and_recorded(self):
|
||||
# Listed on all four looks after the force-stop, gone after the kill.
|
||||
status, _ = self.run_shard([case("a"), case("b")], scenario={"ps_calls_with_leftover": 4})
|
||||
self.assertEqual(status, 0)
|
||||
self.assertIn(["shell", "run-as", "top.mobilegl.plugin.trace", "kill", "-9", "4242"], self.log("adb.log"))
|
||||
self.assertIn("processes left after force-stop: 4242 libMobileGLServer.so", self.status("a")["notes"])
|
||||
self.assertEqual(self.status("b")["notes"], [])
|
||||
self.assertEqual(self.log("avd.log"), [])
|
||||
|
||||
def test_a_process_that_is_still_exiting_is_not_reported(self):
|
||||
status, _ = self.run_shard([case("a")], scenario={"ps_calls_with_leftover": 2})
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(self.status("a")["notes"], [])
|
||||
self.assertNotIn("kill", [call[3] for call in self.log("adb.log") if call[:2] == ["shell", "run-as"]])
|
||||
|
||||
def test_processes_that_survive_the_kill_cost_an_emulator_restart(self):
|
||||
status, _ = self.run_shard([case("a")], scenario={"ps_calls_with_leftover": 10})
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start"])
|
||||
self.assertIn("the emulator was restarted to remove them", self.status("a")["notes"])
|
||||
|
||||
def test_a_failed_pm_clear_falls_back_to_a_reinstall(self):
|
||||
status, _ = self.run_shard([case("a")], scenario={"pm_clear_fails": True})
|
||||
self.assertEqual(status, 0)
|
||||
adb = self.log("adb.log")
|
||||
self.assertIn(["uninstall", "top.mobilegl.plugin.trace"], adb)
|
||||
self.assertEqual(sum(1 for call in adb if call[:1] == ["install"]), 2)
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "signals and process groups")
|
||||
def test_a_terminated_runner_takes_the_running_replay_with_it(self):
|
||||
cases = [case("a"), case("b")]
|
||||
(self.state / "scenario.json").write_text(json.dumps({"replay": {"a": [{"sleep": 60}]}}))
|
||||
arguments = ["--apk-file", str(self.apk), "--backend", "DirectVulkan", "--gpu", "lavapipe",
|
||||
"--transport", "monolith", "--lane", "monolith-lane", "--cases-json", json.dumps(cases),
|
||||
"--result-root", str(self.results), "--fixture-root", str(self.root / "work")]
|
||||
runner = subprocess.Popen([sys.executable, str(HERE / "trace_replay_shard.py")] + arguments,
|
||||
env={**os.environ, **self.environment}, stdout=subprocess.DEVNULL)
|
||||
deadline = time.monotonic() + 30
|
||||
while not (self.state / "replay.pid").exists() and time.monotonic() < deadline:
|
||||
time.sleep(0.05)
|
||||
replay_pid = int((self.state / "replay.pid").read_text())
|
||||
runner.send_signal(signal.SIGTERM)
|
||||
self.assertEqual(runner.wait(timeout=30), 128 + signal.SIGTERM)
|
||||
for _ in range(100):
|
||||
try:
|
||||
os.kill(replay_pid, 0)
|
||||
except OSError:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
else:
|
||||
self.fail("the replay outlived the runner")
|
||||
# The step that runs afterwards names both cases.
|
||||
status, _ = self.run_shard(cases, "--finalize")
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual([self.status(name)["status"] for name in "ab"], ["not-run", "not-run"])
|
||||
|
||||
def test_finalize_names_every_case_without_a_device(self):
|
||||
cases = [case("a", golden=self.fixture("a.png")), case("b")]
|
||||
status, _ = self.run_shard(cases)
|
||||
self.assertEqual(status, 0)
|
||||
(self.results / "b-DirectVulkan" / "case-status.json").unlink()
|
||||
calls_before = len(self.log("adb.log"))
|
||||
status, _ = self.run_shard(cases + [case("c")], "--finalize")
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(len(self.log("adb.log")), calls_before)
|
||||
self.assertEqual([self.status(name)["status"] for name in "abc"], ["passed", "not-run", "not-run"])
|
||||
self.assertTrue((self.results / "c-DirectVulkan" / "lane.json").is_file())
|
||||
self.assertFalse(self.shard_status()["complete"])
|
||||
self.assertEqual(self.shard_status()["expected_cases"], ["a", "b", "c"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -62,6 +62,9 @@ copies benchmark.json (per-frame times plus mean/median/p95) out of the app, and
|
||||
"passed" only means the replay reached the end of the trace without an error.
|
||||
Pass --reuse-fixture to skip re-extracting and re-pushing the trace, for repeat
|
||||
runs of a case whose fixture is already in /data/local/tmp.
|
||||
Set MOBILEGL_TRACE_POLL_INTERVAL to the seconds between result polls (for example
|
||||
0.5) to wait on a wall clock: the wait then ends --timeout-seconds plus a fifth after
|
||||
the app starts, which is about how long that many one-second polls take.
|
||||
Pass --env "K=V;K=V" (or set MOBILEGL_TRACE_ENV) to hand arbitrary environment
|
||||
variables to the replay process. They are applied last, immediately before
|
||||
libMobileGL.so is loaded, so they override every flag above; an entry with no "="
|
||||
@@ -93,6 +96,21 @@ adb_device_path() {
|
||||
MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL='*' "${ADB}" "$@"
|
||||
}
|
||||
|
||||
# Milliseconds since the epoch; whole seconds where date has no %N.
|
||||
now_ms() {
|
||||
stamp="$(date +%s%3N 2>/dev/null || true)"
|
||||
case "${stamp}" in
|
||||
''|*[!0-9]*) stamp="$(($(date +%s) * 1000))" ;;
|
||||
esac
|
||||
printf '%s' "${stamp}"
|
||||
}
|
||||
|
||||
# Appends "<phase> <ms>" to phase-timings.txt in the result directory, so a run records where
|
||||
# its wall time went: push, install, replay, log pulls.
|
||||
mark_phase() {
|
||||
printf '%s %s\n' "$1" "$(now_ms)" >> "${result_dir}/phase-timings.txt" 2>/dev/null || true
|
||||
}
|
||||
|
||||
next_arg() {
|
||||
if [ "$#" -lt 2 ]; then
|
||||
die "$1 requires a value"
|
||||
@@ -411,6 +429,81 @@ copy_fixture_to_app() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Waits for result.json with one poll per second; --timeout-seconds counts the polls.
|
||||
wait_for_result_by_count() {
|
||||
# One failed pidof is not an exit: under memory pressure the adb shell call
|
||||
# itself can fail while the app is alive (seen with the process still in
|
||||
# pidof and the activity still initializing right after the "exit").
|
||||
missed_app_polls=0
|
||||
for _ in $(seq 1 "${timeout_seconds}"); do
|
||||
if adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
if adb_device_path shell pidof "${package_name}" >/dev/null 2>&1; then
|
||||
saw_app_process=1
|
||||
missed_app_polls=0
|
||||
else
|
||||
poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
|
||||
if [ "${poll_adb_state}" != "device" ]; then
|
||||
printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt"
|
||||
break
|
||||
fi
|
||||
if [ "${saw_app_process}" -eq 1 ]; then
|
||||
missed_app_polls=$((missed_app_polls + 1))
|
||||
if [ "${missed_app_polls}" -ge 3 ]; then
|
||||
app_exited=1
|
||||
break
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
# The same wait on a wall clock, selected by MOBILEGL_TRACE_POLL_INTERVAL (seconds between
|
||||
# polls, fractions allowed). One adb call per poll asks for result.json and for the app
|
||||
# process. The budget is --timeout-seconds of wall time plus a fifth: a poll of the
|
||||
# one-second loop takes about 1.2 s with its two adb calls, so a replay gets the same time
|
||||
# to finish under either loop. An exit needs three missed polls spread over at least three
|
||||
# seconds, so a short interval keeps the one-second loop's tolerance for adb calls that fail
|
||||
# under memory pressure.
|
||||
wait_for_result_by_clock() {
|
||||
poll_deadline=$((poll_started + timeout_seconds + timeout_seconds / 5))
|
||||
missed_app_polls=0
|
||||
first_missed_at=0
|
||||
while [ "$(date +%s)" -lt "${poll_deadline}" ]; do
|
||||
poll_output="$(adb_device_path shell "run-as ${package_name} ls ${app_dir}/output/result.json >/dev/null 2>&1 && echo mobilegl-poll-result; pidof ${package_name} >/dev/null 2>&1 && echo mobilegl-poll-alive; echo mobilegl-poll-end" 2>/dev/null | tr -d '\r' || true)"
|
||||
case "${poll_output}" in
|
||||
*mobilegl-poll-result*)
|
||||
break
|
||||
;;
|
||||
*mobilegl-poll-alive*)
|
||||
saw_app_process=1
|
||||
missed_app_polls=0
|
||||
;;
|
||||
*)
|
||||
poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
|
||||
if [ "${poll_adb_state}" != "device" ]; then
|
||||
printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt"
|
||||
break
|
||||
fi
|
||||
if [ "${saw_app_process}" -eq 1 ]; then
|
||||
poll_now="$(date +%s)"
|
||||
if [ "${missed_app_polls}" -eq 0 ]; then
|
||||
first_missed_at="${poll_now}"
|
||||
fi
|
||||
missed_app_polls=$((missed_app_polls + 1))
|
||||
if [ "${missed_app_polls}" -ge 3 ] && [ "$((poll_now - first_missed_at))" -ge 3 ]; then
|
||||
app_exited=1
|
||||
break
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
sleep "${MOBILEGL_TRACE_POLL_INTERVAL}"
|
||||
done
|
||||
}
|
||||
|
||||
run_retrace() {
|
||||
result_dir="${result_root}/${safe_case}-${backend}"
|
||||
alternate_golden_app_path=""
|
||||
@@ -427,13 +520,17 @@ run_retrace() {
|
||||
fi
|
||||
|
||||
mkdir -p "${result_dir}"
|
||||
printf 'script-started %s\n' "${script_started_ms}" > "${result_dir}/phase-timings.txt" 2>/dev/null || true
|
||||
mark_phase fixture-pushed
|
||||
# A repeat run of a case the previous invocation already installed and pushed only needs
|
||||
# the on-device copy back into a fresh app output directory.
|
||||
# MOBILEGL_TRACE_SKIP_INSTALL=1: the caller installed the APK itself (ColorOS shows an install-confirmation
|
||||
# dialog whose tap drops adb for a few seconds, so the install has to be done and settled outside a run).
|
||||
# dialog whose tap drops adb for a few seconds, so the install has to be done and settled outside a run;
|
||||
# trace-replay-shard.sh installs once per emulator boot).
|
||||
if [ "${reuse_fixture}" -eq 0 ] && [ "${MOBILEGL_TRACE_SKIP_INSTALL:-0}" != "1" ]; then
|
||||
"${ADB}" install -r "$(host_path_for_adb "${apk_file}")"
|
||||
fi
|
||||
mark_phase apk-installed
|
||||
copy_fixture_to_app
|
||||
adb_device_path shell am force-stop "${package_name}"
|
||||
"${ADB}" logcat -c
|
||||
@@ -502,39 +599,20 @@ run_retrace() {
|
||||
--ei crop_width "${crop_width}" \
|
||||
--ei crop_height "${crop_height}"
|
||||
adb_device_path shell "$@"
|
||||
mark_phase app-started
|
||||
|
||||
app_exited=0
|
||||
saw_app_process=0
|
||||
# One failed pidof is not an exit: under memory pressure the adb shell call
|
||||
# itself can fail while the app is alive (seen with the process still in
|
||||
# pidof and the activity still initializing right after the "exit").
|
||||
missed_app_polls=0
|
||||
poll_started="$(date +%s)"
|
||||
for _ in $(seq 1 "${timeout_seconds}"); do
|
||||
if adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
if adb_device_path shell pidof "${package_name}" >/dev/null 2>&1; then
|
||||
saw_app_process=1
|
||||
missed_app_polls=0
|
||||
else
|
||||
poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
|
||||
if [ "${poll_adb_state}" != "device" ]; then
|
||||
printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt"
|
||||
break
|
||||
fi
|
||||
if [ "${saw_app_process}" -eq 1 ]; then
|
||||
missed_app_polls=$((missed_app_polls + 1))
|
||||
if [ "${missed_app_polls}" -ge 3 ]; then
|
||||
app_exited=1
|
||||
break
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ -n "${MOBILEGL_TRACE_POLL_INTERVAL:-}" ]; then
|
||||
wait_for_result_by_clock
|
||||
else
|
||||
wait_for_result_by_count
|
||||
fi
|
||||
mark_phase wait-ended
|
||||
|
||||
collect_run_diagnostics "${result_dir}"
|
||||
mark_phase diagnostics-collected
|
||||
if ! adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then
|
||||
if [ -f "${result_dir}/adb-disconnected.txt" ]; then
|
||||
echo "trace-replay-ci.sh: device disconnected while waiting for result.json" >&2
|
||||
@@ -574,6 +652,7 @@ run_retrace() {
|
||||
copy_app_artifact "${app_dir}/output/benchmark.json" "${result_dir}/benchmark.json"
|
||||
fi
|
||||
copy_texture_2d_dumps
|
||||
mark_phase artifacts-copied
|
||||
|
||||
# A replay that wrote result.json but did not pass used to print nothing but
|
||||
# the JSON, which for a non-zero statusCode says only "retrace failed with
|
||||
@@ -696,6 +775,7 @@ PY
|
||||
fi
|
||||
}
|
||||
|
||||
script_started_ms="$(now_ms)"
|
||||
mkdir -p "${fixture_root}" "${result_root}"
|
||||
if [ "${reuse_fixture}" -eq 0 ]; then
|
||||
prepare_fixture
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
# MobileGL - android-plugin/trace-replay-shard.sh
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
# Replays an ordered list of trace cases for one (lane, transport, backend) on one emulator
|
||||
# boot, each through trace-replay-ci.sh. The loop, the per-case settings and the verdict files
|
||||
# are in trace_replay_shard.py; `--help` lists the arguments.
|
||||
#
|
||||
# sh android-plugin/trace-replay-shard.sh \
|
||||
# --apk-file FILE --backend DirectGLES|DirectVulkan --gpu GPU \
|
||||
# --transport monolith|inproc|spawn --lane LANE \
|
||||
# --cases-env VARIABLE | --cases-file FILE | --cases-json JSON \
|
||||
# [--deadline-seconds N] [--dry-run | --finalize]
|
||||
#
|
||||
# Environment:
|
||||
# AVD_NAME, EMULATOR_LOG, EMULATOR_PID_FILE the emulator to restart when it stops answering
|
||||
# MOBILEGL_GPU_HANG_BUDGET_MS forwarded to every replay (required)
|
||||
# MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE 1 cold boots the emulator before every case
|
||||
# MOBILEGL_SHARD_POLL_INTERVAL seconds between result polls (default 0.5);
|
||||
# empty selects trace-replay-ci.sh's 1 s loop
|
||||
set -eu
|
||||
|
||||
exec "${PYTHON:-python3}" "$(dirname "$0")/trace_replay_shard.py" "$@"
|
||||
@@ -0,0 +1,792 @@
|
||||
#!/usr/bin/env python3
|
||||
# MobileGL - android-plugin/trace_replay_shard.py
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
"""Replay an ordered list of trace cases on one emulator boot.
|
||||
|
||||
One shard is one (lane, transport, backend). Each case runs through trace-replay-ci.sh with
|
||||
the per-case and per-transport settings of the APK workflow, in a fresh app process with
|
||||
cleared app data. A failing case does not stop the shard: the case after it starts on a cold
|
||||
boot, every case gets a verdict file, and the exit status is non-zero when a case failed or a
|
||||
verdict is missing.
|
||||
|
||||
Layout under --result-root (the directory the workflow uploads):
|
||||
|
||||
<case>-<backend>/ what trace-replay-ci.sh writes, plus lane.json, the golden
|
||||
copies the summary renders, and case-status.json
|
||||
<case>-<backend>/first-attempt/ files of an attempt that was retried
|
||||
shard-status.json every case of the shard with its verdict
|
||||
diagnostics/ emulator logs kept across restarts
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
INFRASTRUCTURE_FAILURE_EXIT_CODE = 75
|
||||
TIMEOUT_EXIT_CODE = 124
|
||||
BACKENDS = ("DirectGLES", "DirectVulkan")
|
||||
TRANSPORTS = ("monolith", "inproc", "spawn")
|
||||
PASSED, FAILED, NOT_RUN = "passed", "failed", "not-run"
|
||||
|
||||
# Cases with settings of their own. Bliss needs the newer packaged ANGLE build and, on
|
||||
# DirectGLES, sampler mipmap min-filter downgrading on ANGLE llvmpipe; iterationRP needs its
|
||||
# SPIR-V repairs forced on.
|
||||
BLISS_CASE = "minecraft-1.21.4-fabric-iris-bliss-in-world"
|
||||
ITERATIONRP_CASE = "minecraft-1.21.4-fabric-iris-iterationrp-in-world"
|
||||
DEFAULT_ANGLE_VARIANT = "ec889e6ea831"
|
||||
BLISS_ANGLE_VARIANT = "90a62123d794"
|
||||
|
||||
# The inproc and spawn lanes pass the same client settings, so they differ only in which
|
||||
# process runs the server role.
|
||||
SPLIT_CLIENT_SETTINGS = "MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1"
|
||||
|
||||
REQUIRED_CASE_KEYS = ("name", "trace_archive", "trace_file", "golden", "target_call", "width", "height",
|
||||
"crop_x", "crop_y", "crop_width", "crop_height", "timeout_seconds")
|
||||
# Seconds trace-replay-ci.sh gets beyond the case's own budget for install, push and log pulls.
|
||||
REPLAY_GRACE_SECONDS = 300
|
||||
BOOT_TIMEOUT_SECONDS = 300
|
||||
# Seconds a case may take around its replays: the app reset, the failure diagnostics and the
|
||||
# cold boot that follows a case that did not pass.
|
||||
CASE_OVERHEAD_SECONDS = 180
|
||||
|
||||
|
||||
def say(message):
|
||||
print(message, flush=True)
|
||||
|
||||
|
||||
def pause(seconds):
|
||||
time.sleep(seconds)
|
||||
|
||||
|
||||
def safe_case(name):
|
||||
return re.sub(r"[^A-Za-z0-9._-]", "_", name)
|
||||
|
||||
|
||||
def as_text(value):
|
||||
if value is True:
|
||||
return "true"
|
||||
if value is False:
|
||||
return "false"
|
||||
return "" if value is None else str(value)
|
||||
|
||||
|
||||
def is_true(value):
|
||||
return value is True or value == "true"
|
||||
|
||||
|
||||
def load_cases(text, backend):
|
||||
"""The case objects of the APK matrix, as text ready for the command line."""
|
||||
try:
|
||||
# Numbers stay the literal text of the matrix, so 0.985 is passed on as 0.985.
|
||||
cases = json.loads(text, parse_float=str, parse_int=str)
|
||||
except json.JSONDecodeError as error:
|
||||
raise ValueError(f"the case list is not JSON: {error}") from error
|
||||
if not isinstance(cases, list) or not cases:
|
||||
raise ValueError("the case list must be a non-empty JSON array of case objects")
|
||||
names = set()
|
||||
for index, case in enumerate(cases):
|
||||
if not isinstance(case, dict):
|
||||
raise ValueError(f"case {index} is not an object; pass the case objects of the APK matrix")
|
||||
missing = [key for key in REQUIRED_CASE_KEYS if case.get(key) in (None, "")]
|
||||
if missing:
|
||||
raise ValueError(f"case {case.get('name', index)} lacks {', '.join(missing)}")
|
||||
name = case["name"]
|
||||
if name in names:
|
||||
raise ValueError(f"case {name} is listed twice")
|
||||
names.add(name)
|
||||
backends = case.get("ci_backends")
|
||||
if backends is not None and backend not in backends:
|
||||
raise ValueError(f"case {name} does not run on {backend} (ci_backends: {backends})")
|
||||
return cases
|
||||
|
||||
|
||||
def needed_seconds(cases):
|
||||
"""Wall time in which every case of a shard can use its whole time budget.
|
||||
|
||||
It covers each case running until its replay script is stopped, one infrastructure retry
|
||||
that does so again for the longest budget, and CASE_OVERHEAD_SECONDS per case. A shard
|
||||
deadline at or above this never stops a case that is within its own budget, unless more
|
||||
than one case needs a retry as long as its first attempt.
|
||||
"""
|
||||
budgets = [int(as_text(case["timeout_seconds"])) + REPLAY_GRACE_SECONDS for case in cases]
|
||||
return sum(budgets) + max(budgets) + CASE_OVERHEAD_SECONDS * len(budgets)
|
||||
|
||||
|
||||
def replay_environment(backend, case_name):
|
||||
return {
|
||||
"MOBILEGL_ESPRYT_USE_ANGLE": "1" if backend == "DirectGLES" else "0",
|
||||
"MOBILEGL_TRACE_ANGLE_VARIANT": BLISS_ANGLE_VARIANT if case_name == BLISS_CASE else DEFAULT_ANGLE_VARIANT,
|
||||
"MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "1" if backend == "DirectVulkan" else "0",
|
||||
"MOBILEGL_ITRP_FIX": "ForceOn" if case_name == ITERATIONRP_CASE else "Auto",
|
||||
}
|
||||
|
||||
|
||||
def transport_arguments(transport, hang_budget_ms):
|
||||
hang_budget = f"MOBILEGL_GPU_HANG_BUDGET_MS={hang_budget_ms}"
|
||||
if transport == "inproc":
|
||||
return ["--env", f"MOBILEGL_TRANSPORT=inproc;{SPLIT_CLIENT_SETTINGS};{hang_budget}", "--require-inproc"]
|
||||
if transport == "spawn":
|
||||
# No MOBILEGL_IPC_SERVER_PATH: only the app can spell nativeLibraryDir, and
|
||||
# TraceReplayActivity resolves it when the environment asks for spawn. The pbuffer is
|
||||
# required: a spawned server cannot be handed the client's window.
|
||||
return ["--env", f"MOBILEGL_TRANSPORT=spawn;{SPLIT_CLIENT_SETTINGS};{hang_budget}",
|
||||
"--use-pbuffer", "--require-spawn"]
|
||||
return ["--env", f"MOBILEGL_TRANSPORT=monolith;{hang_budget}"]
|
||||
|
||||
|
||||
def replay_arguments(options, case, hang_budget_ms):
|
||||
name = case["name"]
|
||||
arguments = [
|
||||
"--apk-file", options.apk_file,
|
||||
"--package", options.package,
|
||||
"--backend", options.backend,
|
||||
"--result-root", options.result_root,
|
||||
"--fixture-root", options.fixture_root,
|
||||
"--case", name,
|
||||
"--trace-archive", as_text(case["trace_archive"]),
|
||||
"--trace-file", as_text(case["trace_file"]),
|
||||
"--golden", as_text(case["golden"]),
|
||||
"--alternate-golden", as_text(case.get("alternate_golden") or ""),
|
||||
"--target-call", as_text(case["target_call"]),
|
||||
"--width", as_text(case["width"]),
|
||||
"--height", as_text(case["height"]),
|
||||
"--ssim-threshold", as_text(case.get("ssim_threshold") or "0.99"),
|
||||
"--crop-x", as_text(case["crop_x"]),
|
||||
"--crop-y", as_text(case["crop_y"]),
|
||||
"--crop-width", as_text(case["crop_width"]),
|
||||
"--crop-height", as_text(case["crop_height"]),
|
||||
"--timeout-seconds", as_text(case["timeout_seconds"]),
|
||||
]
|
||||
arguments += transport_arguments(options.transport, hang_budget_ms)
|
||||
if options.backend == "DirectGLES" and name == BLISS_CASE:
|
||||
arguments.append("--avoid-angle-llvmpipe-sampler-mipmap-min-filter")
|
||||
if options.backend == "DirectGLES" and is_true(case.get("avoid_angle_llvmpipe_explicit_lod_bias")):
|
||||
arguments.append("--avoid-angle-llvmpipe-explicit-lod-bias")
|
||||
if is_true(case.get("coherent_as_flush")):
|
||||
arguments.append("--coherent-as-flush")
|
||||
return arguments
|
||||
|
||||
|
||||
def read_json(path):
|
||||
try:
|
||||
return json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def write_json(path, value):
|
||||
Path(path).write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def one_line(text):
|
||||
return " ".join(str(text).split())
|
||||
|
||||
|
||||
class Shard:
|
||||
def __init__(self, options, cases, environ):
|
||||
self.options = options
|
||||
self.cases = cases
|
||||
self.environ = dict(environ)
|
||||
self.adb_command = shlex.split(environ.get("ADB", "adb"))
|
||||
directory = Path(__file__).resolve().parent
|
||||
self.replay_command = shlex.split(environ.get("MOBILEGL_SHARD_REPLAY_COMMAND") or
|
||||
f"sh {shlex.quote(str(directory / 'trace-replay-ci.sh'))}")
|
||||
self.avd_command = shlex.split(environ.get("MOBILEGL_SHARD_AVD_COMMAND") or
|
||||
f"sh {shlex.quote(str(directory / 'run-avd-ci.sh'))}")
|
||||
self.hang_budget_ms = environ.get("MOBILEGL_GPU_HANG_BUDGET_MS")
|
||||
self.restart_each_case = environ.get("MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE", "0") == "1"
|
||||
self.poll_interval = environ.get("MOBILEGL_SHARD_POLL_INTERVAL", "0.5")
|
||||
self.avd_name = environ.get("AVD_NAME", "")
|
||||
self.emulator_log = environ.get("EMULATOR_LOG", "")
|
||||
self.emulator_pid_file = environ.get("EMULATOR_PID_FILE", "")
|
||||
self.result_root = Path(options.result_root)
|
||||
self.diagnostics = self.result_root / "diagnostics"
|
||||
self.started = time.monotonic()
|
||||
self.boots = 1
|
||||
self.install_seconds = []
|
||||
self.records = {}
|
||||
self.aborted = ""
|
||||
# The case before the current one when it did not pass; the next case gets a cold boot.
|
||||
self.unclean_case = ""
|
||||
|
||||
# ---- bookkeeping -------------------------------------------------------------------
|
||||
|
||||
def result_dir(self, case):
|
||||
return self.result_root / f"{safe_case(case['name'])}-{self.options.backend}"
|
||||
|
||||
def record_for(self, case, status, reason, **extra):
|
||||
record = {"case": case["name"], "backend": self.options.backend, "transport": self.options.transport,
|
||||
"lane": self.options.lane, "status": status, "reason": reason}
|
||||
record.update(extra)
|
||||
return record
|
||||
|
||||
def write_summary_inputs(self, case):
|
||||
"""lane.json and the golden copies the retrace summary reads next to each result."""
|
||||
directory = self.result_dir(case)
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
write_json(directory / "lane.json", {"lane": self.options.lane, "transport": self.options.transport})
|
||||
stem = f"{safe_case(case['name'])}-{self.options.backend}"
|
||||
for key, suffix in (("golden", "golden"), ("alternate_golden", "alternate-golden")):
|
||||
source = as_text(case.get(key) or "")
|
||||
if source and os.path.isfile(source) and os.path.getsize(source) > 0:
|
||||
shutil.copyfile(source, directory / f"{stem}-{suffix}.png")
|
||||
|
||||
def save_record(self, case, record):
|
||||
self.records[case["name"]] = record
|
||||
self.write_summary_inputs(case)
|
||||
write_json(self.result_dir(case) / "case-status.json", record)
|
||||
self.write_shard_status()
|
||||
|
||||
def write_shard_status(self, complete=False):
|
||||
self.result_root.mkdir(parents=True, exist_ok=True)
|
||||
records = [self.records.get(case["name"]) or
|
||||
self.record_for(case, NOT_RUN, "the shard has not reached this case") for case in self.cases]
|
||||
write_json(self.result_root / "shard-status.json", {
|
||||
"schema": 1, "lane": self.options.lane, "transport": self.options.transport,
|
||||
"backend": self.options.backend, "complete": complete,
|
||||
# The Actions job that ran the shard, so the artifact cleanup can tell whose job failed.
|
||||
"job_id": self.environ.get("MOBILEGL_SHARD_JOB_ID", ""),
|
||||
"expected_cases": [case["name"] for case in self.cases], "cases": records,
|
||||
"emulator_boots": self.boots, "install_seconds": self.install_seconds})
|
||||
|
||||
def remaining_seconds(self):
|
||||
if self.options.deadline_seconds <= 0:
|
||||
return None
|
||||
return self.options.deadline_seconds - (time.monotonic() - self.started)
|
||||
|
||||
def deadline_passed(self):
|
||||
remaining = self.remaining_seconds()
|
||||
return remaining is not None and remaining <= 0
|
||||
|
||||
# ---- processes ---------------------------------------------------------------------
|
||||
|
||||
def run(self, command, timeout=None, capture=False, extra_env=None):
|
||||
"""Returns (exit status, output). A timeout kills the whole process group and reads as 124."""
|
||||
environment = dict(self.environ)
|
||||
environment.update(extra_env or {})
|
||||
own_group = os.name == "posix"
|
||||
try:
|
||||
process = subprocess.Popen(command, env=environment, start_new_session=own_group,
|
||||
stdout=subprocess.PIPE if capture else None,
|
||||
stderr=subprocess.STDOUT if capture else None)
|
||||
except OSError as error:
|
||||
return 127, str(error)
|
||||
def kill():
|
||||
if own_group:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except OSError:
|
||||
pass
|
||||
process.kill()
|
||||
|
||||
try:
|
||||
output, _ = process.communicate(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
kill()
|
||||
output, _ = process.communicate()
|
||||
return TIMEOUT_EXIT_CODE, (output or b"").decode("utf-8", "replace")
|
||||
except BaseException:
|
||||
# The job was canceled or timed out. The child is in a session of its own, so it
|
||||
# would otherwise keep driving the device behind the steps that still run.
|
||||
kill()
|
||||
raise
|
||||
return process.returncode, (output or b"").decode("utf-8", "replace")
|
||||
|
||||
def adb(self, *arguments, timeout=60):
|
||||
status, output = self.run(self.adb_command + list(arguments), timeout=timeout, capture=True)
|
||||
return status, output.replace("\r", "")
|
||||
|
||||
# ---- emulator ----------------------------------------------------------------------
|
||||
|
||||
def can_restart(self):
|
||||
return bool(self.avd_name and self.options.gpu and self.emulator_log and self.emulator_pid_file)
|
||||
|
||||
def emulator_pid(self):
|
||||
try:
|
||||
return int(Path(self.emulator_pid_file).read_text().strip())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
def emulator_process_alive(self):
|
||||
pid = self.emulator_pid()
|
||||
if pid is None:
|
||||
return True # nothing to check against, for example a real device
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def health_problem(self):
|
||||
"""Empty when the device answers; otherwise what did not."""
|
||||
if not self.emulator_process_alive():
|
||||
return "the emulator process is gone"
|
||||
status, output = self.adb("get-state", timeout=15)
|
||||
if status != 0 or output.strip() != "device":
|
||||
return f"adb state is {output.strip() or 'unknown'}"
|
||||
status, output = self.adb("shell", "getprop", "sys.boot_completed", timeout=15)
|
||||
if status != 0 or output.strip() != "1":
|
||||
return "sys.boot_completed is not 1"
|
||||
# A framework call: boot_completed stays 1 across a system_server restart.
|
||||
status, output = self.adb("shell", "pm", "path", self.options.package, timeout=30)
|
||||
if status != 0 or "package:" not in output:
|
||||
return "the package manager does not list the trace APK"
|
||||
return ""
|
||||
|
||||
def keep_emulator_log(self, label):
|
||||
if self.emulator_log and os.path.isfile(self.emulator_log):
|
||||
self.diagnostics.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(self.emulator_log, self.diagnostics / f"emulator-boot-{self.boots}-{label}.log")
|
||||
|
||||
def prepare_booted_device(self):
|
||||
# A shard outlives the screen timeout a single replay never reached.
|
||||
self.adb("shell", "svc", "power", "stayon", "true", timeout=30)
|
||||
return self.install()
|
||||
|
||||
def install(self):
|
||||
started = time.monotonic()
|
||||
status, output = self.adb("install", "-r", self.options.apk_file, timeout=300)
|
||||
self.install_seconds.append(round(time.monotonic() - started, 1))
|
||||
say(output.strip())
|
||||
return status == 0
|
||||
|
||||
def restart_emulator(self, reason):
|
||||
"""Stop, cold boot and reinstall. False when the emulator did not come back."""
|
||||
if not self.can_restart():
|
||||
say(f"trace-replay-shard: cannot restart the emulator ({reason}): no AVD is configured")
|
||||
return False
|
||||
say(f"trace-replay-shard: restarting the emulator: {reason}")
|
||||
common = ["--avd-name", self.avd_name, "--emulator-log", self.emulator_log,
|
||||
"--pid-file", self.emulator_pid_file]
|
||||
for attempt in (1, 2):
|
||||
# The start truncates the log, and the boot that failed is the one worth reading.
|
||||
self.keep_emulator_log(f"attempt-{attempt}" if attempt > 1 else "ended")
|
||||
old_pid = self.emulator_pid()
|
||||
self.run(self.avd_command + ["stop"] + common, timeout=120)
|
||||
self.wait_for_exit(old_pid)
|
||||
self.adb("kill-server", timeout=30)
|
||||
pause(2)
|
||||
self.boots += 1
|
||||
status, _ = self.run(self.avd_command + ["start"] + common +
|
||||
["--gpu", self.options.gpu, "--boot-timeout", str(BOOT_TIMEOUT_SECONDS)],
|
||||
timeout=2 * BOOT_TIMEOUT_SECONDS + 60)
|
||||
if status == 0 and self.prepare_booted_device():
|
||||
return True
|
||||
say(f"trace-replay-shard: emulator restart attempt {attempt} failed")
|
||||
return False
|
||||
|
||||
def wait_for_exit(self, pid):
|
||||
if pid is None or os.name != "posix":
|
||||
return
|
||||
for _ in range(60):
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except OSError:
|
||||
return
|
||||
pause(0.5)
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# ---- app state ---------------------------------------------------------------------
|
||||
|
||||
def leftover_processes(self):
|
||||
"""Processes still running under the app's uid, as 'pid name' strings."""
|
||||
status, user = self.adb("shell", "run-as", self.options.package, "id", "-un", timeout=30)
|
||||
user = user.strip()
|
||||
if status != 0 or not re.fullmatch(r"[A-Za-z0-9_]+", user):
|
||||
return None
|
||||
status, listing = self.adb("shell", "ps", "-A", "-o", "USER,PID,NAME", timeout=30)
|
||||
if status != 0:
|
||||
return None
|
||||
found = []
|
||||
for line in listing.splitlines():
|
||||
fields = line.split()
|
||||
if len(fields) >= 3 and fields[0] == user and fields[1].isdigit():
|
||||
found.append(f"{fields[1]} {fields[2]}")
|
||||
return found
|
||||
|
||||
def settled_leftover_processes(self, polls):
|
||||
"""leftover_processes once the app's uid is empty, or as it still is after `polls` looks.
|
||||
|
||||
A killed process stays listed for a moment, so one look right after the kill would
|
||||
report every force-stop as incomplete.
|
||||
"""
|
||||
found = self.leftover_processes()
|
||||
for _ in range(polls - 1):
|
||||
if not found:
|
||||
break
|
||||
pause(0.5)
|
||||
found = self.leftover_processes()
|
||||
return found
|
||||
|
||||
def reset_app_state(self):
|
||||
"""Fresh process and fresh app data, as a new install gives. Returns notes for the record."""
|
||||
notes = []
|
||||
package = self.options.package
|
||||
self.adb("shell", "am", "force-stop", package, timeout=60)
|
||||
leftovers = self.settled_leftover_processes(polls=4)
|
||||
if leftovers is None:
|
||||
notes.append("the app's processes could not be listed after force-stop")
|
||||
if leftovers:
|
||||
# A force-stop is expected to take a spawned server with it; say so when it did not.
|
||||
notes.append("processes left after force-stop: " + ", ".join(leftovers))
|
||||
self.adb("shell", "run-as", package, "kill", "-9", *[entry.split()[0] for entry in leftovers],
|
||||
timeout=30)
|
||||
if self.settled_leftover_processes(polls=6):
|
||||
notes.append("the leftover processes could not be killed")
|
||||
if self.restart_emulator("app processes survived force-stop and kill"):
|
||||
notes.append("the emulator was restarted to remove them")
|
||||
status, output = self.adb("shell", "pm", "clear", package, timeout=60)
|
||||
probe_status, _ = self.adb("shell", "run-as", package, "ls", f"/data/user/0/{package}", timeout=30)
|
||||
if status != 0 or "Success" not in output or probe_status != 0:
|
||||
# The state a fresh install leaves, reached the other way.
|
||||
notes.append("pm clear did not leave a usable app data directory; reinstalled the APK")
|
||||
self.adb("uninstall", package, timeout=120)
|
||||
self.install()
|
||||
return notes
|
||||
|
||||
def remove_case_leftovers(self, case):
|
||||
name = safe_case(case["name"])
|
||||
base = f"/data/local/tmp/mobilegl-{name}"
|
||||
self.adb("shell", "rm", "-f", f"{base}.trace", f"{base}.golden.png", f"{base}.alternate-golden.png",
|
||||
timeout=60)
|
||||
self.adb("shell", "am", "force-stop", self.options.package, timeout=60)
|
||||
shutil.rmtree(Path(self.options.fixture_root) / name, ignore_errors=True)
|
||||
|
||||
# ---- one case ----------------------------------------------------------------------
|
||||
|
||||
def reason_lines(self):
|
||||
try:
|
||||
return (self.result_root / "infrastructure-failure-reason.txt").read_text().splitlines()
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
def run_attempt(self, case):
|
||||
"""Returns (exit status, seconds, how the shard deadline cut the attempt short or "")."""
|
||||
case_budget = int(as_text(case["timeout_seconds"])) + REPLAY_GRACE_SECONDS
|
||||
budget = case_budget
|
||||
remaining = self.remaining_seconds()
|
||||
if remaining is not None and remaining < case_budget:
|
||||
budget = max(remaining, 1.0)
|
||||
environment = replay_environment(self.options.backend, case["name"])
|
||||
# The shard installs the APK once per boot.
|
||||
environment["MOBILEGL_TRACE_SKIP_INSTALL"] = "1"
|
||||
environment["MOBILEGL_TRACE_POLL_INTERVAL"] = self.poll_interval
|
||||
started = time.monotonic()
|
||||
status, _ = self.run(self.replay_command + replay_arguments(self.options, case, self.hang_budget_ms),
|
||||
timeout=budget, extra_env=environment)
|
||||
elapsed = round(time.monotonic() - started, 1)
|
||||
cut = ""
|
||||
if status == TIMEOUT_EXIT_CODE and budget < case_budget:
|
||||
# The case never had its own budget, so the timeout says nothing about the case.
|
||||
cut = f"cut by the shard deadline after {elapsed:.0f} s of a {case_budget} s budget"
|
||||
return status, elapsed, cut
|
||||
|
||||
def keep_first_attempt(self, case):
|
||||
"""Move the retried attempt's files aside so the retry starts from an empty directory.
|
||||
|
||||
result.json is renamed: the summary keys rows on any result.json below a directory
|
||||
that names the case, and the retry's is the one that counts.
|
||||
"""
|
||||
directory = self.result_dir(case)
|
||||
if not directory.is_dir():
|
||||
return
|
||||
kept = directory / "first-attempt"
|
||||
kept.mkdir(exist_ok=True)
|
||||
for entry in directory.iterdir():
|
||||
if entry.is_file():
|
||||
name = "result.first-attempt.json" if entry.name == "result.json" else entry.name
|
||||
entry.replace(kept / name)
|
||||
|
||||
def verdict(self, case, status):
|
||||
directory = self.result_dir(case)
|
||||
if status == 0:
|
||||
result = read_json(directory / "result.json")
|
||||
if not isinstance(result, dict) or result.get("passed") is not True:
|
||||
return FAILED, "the replay exited 0 without a passed result.json"
|
||||
if self.options.transport != "monolith":
|
||||
proof = read_json(directory / "transport-proof.json")
|
||||
if not (isinstance(proof, dict) and proof.get("passed") is True and
|
||||
proof.get("required_transport") == self.options.transport):
|
||||
return FAILED, f"the replay exited 0 without a passed {self.options.transport} transport proof"
|
||||
return PASSED, ""
|
||||
if status == INFRASTRUCTURE_FAILURE_EXIT_CODE:
|
||||
return FAILED, "the emulator infrastructure failed again on the retry"
|
||||
if status == TIMEOUT_EXIT_CODE:
|
||||
return FAILED, "the replay script exceeded its time budget"
|
||||
result = read_json(directory / "result.json")
|
||||
if not isinstance(result, dict):
|
||||
return FAILED, f"exit {status} without a result.json"
|
||||
if result.get("passed") is True:
|
||||
# The picture matched, so what failed is a check that runs after it.
|
||||
return FAILED, f"exit {status} after a passed result.json: the transport proof or a log check failed"
|
||||
return FAILED, f"exit {status}: {one_line(result.get('message', ''))}"
|
||||
|
||||
def collect_failure_diagnostics(self, case):
|
||||
directory = self.result_dir(case)
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
logcat = directory / "logcat.txt"
|
||||
if not logcat.is_file() or logcat.stat().st_size == 0:
|
||||
# trace-replay-ci.sh pulls this itself unless it was killed first.
|
||||
_, output = self.adb("logcat", "-d", "-t", "20000", timeout=30)
|
||||
logcat.write_text(output, encoding="utf-8", errors="replace")
|
||||
app_output = f"/data/user/0/{self.options.package}/files/trace-replay/output"
|
||||
for name in ("retrace.log", "mobilegl.client.log", "mobilegl.server.log"):
|
||||
status, output = self.adb("exec-out", "run-as", self.options.package, "cat",
|
||||
f"{app_output}/{name}", timeout=30)
|
||||
if status == 0 and output and not output.startswith("cat: "):
|
||||
(directory / name).write_text(output, encoding="utf-8", errors="replace")
|
||||
if self.emulator_log and os.path.isfile(self.emulator_log):
|
||||
shutil.copyfile(self.emulator_log, directory / "emulator.log")
|
||||
# A vanished emulator looks the same whether the host OOM killer took it or the
|
||||
# renderer faulted. These two say which.
|
||||
_, memory = self.run(["free", "-h"], timeout=30, capture=True)
|
||||
(directory / "host-memory.txt").write_text(memory, encoding="utf-8")
|
||||
_, dmesg = self.run(["sudo", "-n", "dmesg", "-T"], timeout=30, capture=True)
|
||||
(directory / "host-dmesg.txt").write_text("\n".join(dmesg.splitlines()[-300:]) + "\n", encoding="utf-8")
|
||||
|
||||
def run_case(self, index, case):
|
||||
name = case["name"]
|
||||
notes = []
|
||||
seconds = {}
|
||||
case_started = time.monotonic()
|
||||
|
||||
if self.deadline_passed():
|
||||
return self.record_for(case, NOT_RUN, "the shard deadline passed before this case started")
|
||||
|
||||
restarted_for = ""
|
||||
if self.restart_each_case and index > 0:
|
||||
restarted_for = "MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE=1"
|
||||
elif self.unclean_case and self.can_restart():
|
||||
# A crashed, hung or killed replay can leave the guest or the host renderer in a
|
||||
# state no adb query shows, and this case's verdict must not depend on it.
|
||||
restarted_for = f"the case before it, {self.unclean_case}, did not pass"
|
||||
else:
|
||||
restarted_for = self.health_problem()
|
||||
if restarted_for:
|
||||
notes.append(f"emulator restarted before the case: {restarted_for}")
|
||||
if not self.restart_emulator(restarted_for):
|
||||
self.aborted = "the emulator could not be restarted"
|
||||
return self.record_for(case, NOT_RUN, self.aborted, notes=notes)
|
||||
|
||||
notes += self.reset_app_state()
|
||||
seconds["reset"] = round(time.monotonic() - case_started, 1)
|
||||
|
||||
reasons_before = len(self.reason_lines())
|
||||
exit_codes = []
|
||||
replay_seconds = []
|
||||
status, elapsed, cut = self.run_attempt(case)
|
||||
exit_codes.append(status)
|
||||
replay_seconds.append(elapsed)
|
||||
surface_lost = 0
|
||||
if status == INFRASTRUCTURE_FAILURE_EXIT_CODE:
|
||||
say(f"::warning::Android emulator infrastructure failed; restarting it and retrying {name} once.")
|
||||
new_reasons = self.reason_lines()[reasons_before:]
|
||||
# A lost surface is retried, so it is counted where a rate spike shows.
|
||||
surface_lost = sum(1 for line in new_reasons if "angle-surface-lost" in line)
|
||||
if surface_lost:
|
||||
self.append_step_summary(f"surface-lost retries: {surface_lost} ({self.options.backend}, {name})\n")
|
||||
notes.append("infrastructure retry: " + (", ".join(new_reasons) or "no reason recorded"))
|
||||
self.keep_first_attempt(case)
|
||||
if not self.restart_emulator(f"infrastructure failure in {name}"):
|
||||
self.aborted = "the emulator could not be restarted"
|
||||
elif self.deadline_passed():
|
||||
cut = "the shard deadline passed before the infrastructure retry could start"
|
||||
else:
|
||||
notes += self.reset_app_state()
|
||||
status, elapsed, cut = self.run_attempt(case)
|
||||
exit_codes.append(status)
|
||||
replay_seconds.append(elapsed)
|
||||
seconds["replay"] = replay_seconds
|
||||
|
||||
finish_started = time.monotonic()
|
||||
if self.aborted:
|
||||
verdict, reason = FAILED, "the emulator did not come back for the infrastructure retry"
|
||||
elif cut:
|
||||
verdict, reason = NOT_RUN, cut
|
||||
else:
|
||||
verdict, reason = self.verdict(case, status)
|
||||
if verdict != PASSED:
|
||||
self.collect_failure_diagnostics(case)
|
||||
if not self.aborted:
|
||||
self.remove_case_leftovers(case)
|
||||
seconds["finish"] = round(time.monotonic() - finish_started, 1)
|
||||
seconds["total"] = round(time.monotonic() - case_started, 1)
|
||||
return self.record_for(case, verdict, reason, attempts=len(exit_codes), exit_codes=exit_codes,
|
||||
surface_lost_retries=surface_lost, seconds=seconds, notes=notes)
|
||||
|
||||
# ---- the shard ---------------------------------------------------------------------
|
||||
|
||||
def append_step_summary(self, text):
|
||||
path = self.environ.get("GITHUB_STEP_SUMMARY")
|
||||
if path:
|
||||
with open(path, "a", encoding="utf-8") as summary:
|
||||
summary.write(text)
|
||||
|
||||
def run_all(self):
|
||||
self.write_shard_status()
|
||||
say(f"trace-replay-shard: {len(self.cases)} case(s) on {self.options.backend} / "
|
||||
f"{self.options.transport} ({self.options.lane})")
|
||||
if not self.prepare_booted_device():
|
||||
if not self.restart_emulator("the trace APK did not install"):
|
||||
self.aborted = "the trace APK could not be installed"
|
||||
for index, case in enumerate(self.cases):
|
||||
say(f"::group::{case['name']} ({index + 1}/{len(self.cases)})")
|
||||
if self.aborted:
|
||||
record = self.record_for(case, NOT_RUN, self.aborted)
|
||||
else:
|
||||
try:
|
||||
record = self.run_case(index, case)
|
||||
except Exception as error: # one case's bookkeeping must not cost the others their verdicts
|
||||
record = self.record_for(case, FAILED, f"the shard runner failed: {one_line(error)}")
|
||||
self.unclean_case = "" if record["status"] == PASSED else case["name"]
|
||||
self.save_record(case, record)
|
||||
say(f"trace-replay-shard: {case['name']}: {record['status']}"
|
||||
+ (f" ({record['reason']})" if record["reason"] else ""))
|
||||
say("::endgroup::")
|
||||
return self.finish()
|
||||
|
||||
def finish(self):
|
||||
"""Tally from the files on disk, so a verdict the loop never wrote counts as missing."""
|
||||
label = f"{self.options.backend} / {self.options.transport}"
|
||||
rows = []
|
||||
verdicts = 0
|
||||
failed = []
|
||||
for case in self.cases:
|
||||
record = read_json(self.result_dir(case) / "case-status.json")
|
||||
if not isinstance(record, dict) or record.get("case") != case["name"]:
|
||||
record = self.record_for(case, NOT_RUN, "no verdict file was written")
|
||||
if record["status"] in (PASSED, FAILED):
|
||||
verdicts += 1
|
||||
if record["status"] == PASSED:
|
||||
result = read_json(self.result_dir(case) / "result.json")
|
||||
if not isinstance(result, dict) or result.get("passed") is not True:
|
||||
record = dict(record, status=FAILED, reason="the verdict is passed but result.json is not")
|
||||
if record["status"] != PASSED:
|
||||
failed.append(record)
|
||||
result = read_json(self.result_dir(case) / "result.json")
|
||||
ssim = result.get("ssim", "") if isinstance(result, dict) else ""
|
||||
seconds = record.get("seconds", {}).get("total", "")
|
||||
note = one_line("; ".join([record.get("reason", "")] + record.get("notes", [])).strip("; "))
|
||||
rows.append(f"| {case['name']} | {record['status']} | {record.get('attempts', 0)} | {seconds} | "
|
||||
f"{ssim} | {note.replace('|', '/')} |")
|
||||
self.write_shard_status(complete=verdicts == len(self.cases))
|
||||
|
||||
summary = [f"### Android retrace: {label} ({self.options.lane})", "",
|
||||
f"{len(self.cases) - len(failed)} of {len(self.cases)} passed, {self.boots} emulator boot(s).", "",
|
||||
"| case | status | attempts | seconds | SSIM | note |", "| --- | --- | --- | --- | --- | --- |"]
|
||||
self.append_step_summary("\n".join(summary + rows) + "\n\n")
|
||||
for record in failed:
|
||||
say(f"::error title=Android retrace {record['status']} ({label})::"
|
||||
f"{record['case']}: {record.get('reason') or record['status']}")
|
||||
if verdicts != len(self.cases):
|
||||
say(f"::error title=Android retrace shard incomplete ({label})::"
|
||||
f"{verdicts} verdict(s) for {len(self.cases)} case(s)")
|
||||
say(f"trace-replay-shard: {len(self.cases) - len(failed)} passed, {len(failed)} not passed, "
|
||||
f"{verdicts} verdict(s) for {len(self.cases)} case(s)")
|
||||
return 0 if not failed and verdicts == len(self.cases) else 1
|
||||
|
||||
def finalize(self):
|
||||
"""Give every case its summary inputs and a verdict file without touching the device.
|
||||
|
||||
Runs after the replay step whatever its outcome, so a shard that was killed or never
|
||||
started still uploads one named row per case.
|
||||
"""
|
||||
for case in self.cases:
|
||||
record = read_json(self.result_dir(case) / "case-status.json")
|
||||
if isinstance(record, dict) and record.get("case") == case["name"]:
|
||||
self.records[case["name"]] = record
|
||||
self.write_summary_inputs(case)
|
||||
else:
|
||||
self.save_record(case, self.record_for(case, NOT_RUN, "the shard did not reach this case"))
|
||||
previous = read_json(self.result_root / "shard-status.json")
|
||||
if isinstance(previous, dict):
|
||||
self.boots = previous.get("emulator_boots", self.boots)
|
||||
self.install_seconds = previous.get("install_seconds", self.install_seconds)
|
||||
self.write_shard_status(
|
||||
complete=all(self.records[case["name"]]["status"] in (PASSED, FAILED) for case in self.cases))
|
||||
return 0
|
||||
|
||||
def dry_run(self):
|
||||
for case in self.cases:
|
||||
say(json.dumps({
|
||||
"case": case["name"],
|
||||
"environment": replay_environment(self.options.backend, case["name"]),
|
||||
"arguments": replay_arguments(self.options, case, self.hang_budget_ms),
|
||||
"timeout": int(as_text(case["timeout_seconds"])) + REPLAY_GRACE_SECONDS,
|
||||
}, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def parse_arguments(argv):
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--apk-file", required=True, help="Trace APK to install once per emulator boot.")
|
||||
parser.add_argument("--package", default="top.mobilegl.plugin.trace")
|
||||
parser.add_argument("--backend", required=True, choices=BACKENDS)
|
||||
parser.add_argument("--transport", required=True, choices=TRANSPORTS)
|
||||
parser.add_argument("--lane", required=True, help="Lane name written to lane.json, e.g. inproc-acceptance.")
|
||||
parser.add_argument("--gpu", default="", help="Emulator -gpu mode, needed to restart the emulator.")
|
||||
source = parser.add_mutually_exclusive_group(required=True)
|
||||
source.add_argument("--cases-json", help="JSON array of the APK matrix's case objects.")
|
||||
source.add_argument("--cases-file", help="File holding that JSON array.")
|
||||
source.add_argument("--cases-env", help="Name of the environment variable holding that JSON array.")
|
||||
parser.add_argument("--result-root", default="android-retrace-result")
|
||||
parser.add_argument("--fixture-root", default="android-retrace-fixture")
|
||||
parser.add_argument("--deadline-seconds", type=int, default=0,
|
||||
help="Stop starting cases, and cut a running one, this long after the start; both are "
|
||||
"recorded as not-run. 0: no limit. retrace_shard_budget.py says what a shard needs.")
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
mode.add_argument("--dry-run", action="store_true",
|
||||
help="Print each case's environment and trace-replay-ci.sh arguments, run nothing.")
|
||||
mode.add_argument("--finalize", action="store_true",
|
||||
help="Only write the summary inputs and a not-run verdict for cases without one.")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
options = parse_arguments(argv)
|
||||
if options.cases_json is not None:
|
||||
text = options.cases_json
|
||||
elif options.cases_file is not None:
|
||||
text = Path(options.cases_file).read_text(encoding="utf-8")
|
||||
else:
|
||||
text = os.environ.get(options.cases_env, "")
|
||||
try:
|
||||
cases = load_cases(text, options.backend)
|
||||
except ValueError as error:
|
||||
say(f"::error title=Android retrace shard misconfigured::{error}")
|
||||
return 2
|
||||
shard = Shard(options, cases, os.environ)
|
||||
if shard.hang_budget_ms is None:
|
||||
say("::error title=Android retrace shard misconfigured::MOBILEGL_GPU_HANG_BUDGET_MS is not set")
|
||||
return 2
|
||||
if options.dry_run:
|
||||
return shard.dry_run()
|
||||
if options.finalize:
|
||||
return shard.finalize()
|
||||
if not os.path.isfile(options.apk_file):
|
||||
say(f"::error title=Android retrace shard misconfigured::APK does not exist: {options.apk_file}")
|
||||
return 2
|
||||
# A termination request unwinds through Shard.run, which kills the running child first.
|
||||
signal.signal(signal.SIGTERM, lambda number, frame: sys.exit(128 + number))
|
||||
return shard.run_all()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -18,9 +18,9 @@ MobileGL/MG_Remote/ only with MOBILEGL_BUILD_DISAGGREGATED: CONTRACT-*.
|
||||
SurfaceControlFrame, ServerDisplay, InProcessServer, DisplayServerJni
|
||||
```
|
||||
|
||||
- `MOBILEGL_BUILD_DISAGGREGATED` (default OFF) adds `MG_Remote/**`; when OFF, `MG_Config::Transport` is a `constexpr Monolith`. `MOBILEGL_BUILD_DISAGGREGATED_INPROC` implies the former and adds role isolation: the two roles are separated by the apply thread and the control mailbox, rather than by adding TLS to the 1494 `pGLContext->` read points.
|
||||
- `MOBILEGL_BUILD_DISAGGREGATED` (default ON) adds `MG_Remote/**`; when OFF, `MG_Config::Transport` is a `constexpr Monolith`. `MOBILEGL_BUILD_DISAGGREGATED_INPROC` implies the former and adds role isolation: the two roles are separated by the apply thread and the control mailbox, rather than by adding TLS to the 1494 `pGLContext->` read points.
|
||||
- `MOBILEGL_TRANSPORT = monolith | inproc | spawn` is the **topology** (G1-relevant); the control plane and data plane are selected by the two variables in §11.9. **A split build with `MOBILEGL_TRANSPORT` unset is the monolith control arm**.
|
||||
- Three build flavors (since P13): default (the monolith record arm, with `MG_Remote` not compiled, the shipping shape), verify, and split; Android has two APK flavors (monolith / split). The pull build and the `MOBILEGL_PIPE_PUSH` CMake option are deleted in P13.
|
||||
- Three build flavors (since P13): default (the split shape, `MG_Remote` compiled, the in-process shims), verify, and monolith (the shipping shape: configure with `-DMOBILEGL_BUILD_DISAGGREGATED=OFF`, `MG_Remote` not compiled); Android has two APK flavors (monolith / split). The pull build and the `MOBILEGL_PIPE_PUSH` CMake option are deleted in P13.
|
||||
- Modules (P13 W7): `SOURCE_FILES` is split into seven OBJECT libraries according to the link ratchet's PARTITION, and `MobileGL` / `MobileGL_s` are assembled from them; in a split build `MobileGL_server_linkcheck` links only the SERVER and SHARED modules with `--no-undefined`, so a frontend reference outside the baseline fails at build time (`scripts/ci/server_link_check.py`). The record arm's server-side code is in `MG_Backend/Record/` (`RecordVerbSink`, `StagedShadow`, `StagedTextureStore`), and client verb emission is in `MG_Impl/Pipe/Verb/`, no longer depending on `MG_Remote`.
|
||||
- Test wiring: the ctest `ENVIRONMENT` is built with `mgl_itest_join_environment(...)`; every split / spawn / tcp entry has its own log path, and logs are split per role (`<base>.client.log` / `<base>.server.log`); the spawn and tcp lanes match the split name set (`scripts/ci/spawn_lane_parity.py`), and each entry records its arm proof.
|
||||
- CI (`.github/workflows/test.yml`, `apk.yml`): `pipe-gates` (G1–G8, the generator self-test, the symbol report, dirty-surface, field ownership, G5, the documentation-reference lint), `flatc-check`, the include closure, the build / integration / retrace lanes of each flavor, APK + AVD. The `feat/disaggregated` triggers in both workflows are temporary and are removed before merging into `dev`.
|
||||
@@ -40,7 +40,7 @@ Every lane in `MG_IntegrationTest/CMakeLists.txt` must be able to go red for the
|
||||
|
||||
## Appendix A: switches
|
||||
|
||||
CMake: `MOBILEGL_BUILD_DISAGGREGATED` (OFF), `MOBILEGL_BUILD_DISAGGREGATED_INPROC` (OFF, implies the former), `MOBILEGL_PIPE_VERIFY` (OFF, never ships), `MOBILEGL_BUILD_SERVER_SPIKE` (Android spike only), `MOBILEGL_FLATC_EXECUTABLE` (serves `flatc-check` only). `MOBILEGL_PIPE_PUSH` and `MOBILEGL_PIPE_LEGACY_MEMOS` are deleted in P13 (an OFF in an old cache only warns and is ignored).
|
||||
CMake: `MOBILEGL_BUILD_DISAGGREGATED` (ON), `MOBILEGL_BUILD_DISAGGREGATED_INPROC` (ON, implies the former), `MOBILEGL_PIPE_VERIFY` (OFF, never ships), `MOBILEGL_BUILD_SERVER_SPIKE` (Android spike only), `MOBILEGL_FLATC_EXECUTABLE` (serves `flatc-check` only). `MOBILEGL_PIPE_PUSH` and `MOBILEGL_PIPE_LEGACY_MEMOS` are deleted in P13 (an OFF in an old cache only warns and is ignored).
|
||||
|
||||
At runtime, MGPipe (`MobileGL/Config.h`, `ConfigLoader.cpp`):
|
||||
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
---
|
||||
name: mali-gpu-counters
|
||||
description: Capture and compare Mali GPU hardware counters (fragment/vertex cycles, tiles, ZS, bandwidth, tile write elimination) on a rooted Android phone through perfetto and the vendor's libgpudataproducer, to find where one renderer spends more GPU time than another. Use when a GPU-bound workload is slower on Magma (DirectVulkan) than on the device's GLES driver, or slower after a change, and you need to know which part of the GPU pipeline the gap is in.
|
||||
---
|
||||
|
||||
# Mali GPU performance counters
|
||||
|
||||
Use this to answer "where does Magma spend the GPU time that the system GLES driver (or the previous build) doesn't?" on a Mali phone. Two captures, each normalized per frame, show which pipeline stage is responsible: vertex/tiling, fragment shading, depth testing, or memory bandwidth. Do this before guessing at shader or pass-structure fixes.
|
||||
|
||||
The workflow was built and verified on a MediaTek MT6893 (Mali-G77 MC9, r32 driver, TrebleDroid). The vendor library path and counter IDs can differ on other devices; step 2 shows how to list them.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- A rooted phone (`adb root` or `su`). Check that `/vendor/lib64/libgpudataproducer.so` exists. This is the vendor library behind Android GPU Inspector's counters.
|
||||
- The workload must be **GPU-bound**. Check `/sys/kernel/ged/hal/gpu_utilization` on MediaTek (first field ≥ 90%). Otherwise counter differences say nothing about FPS.
|
||||
- Fixed clocks, so cycles are comparable across runs. On MT6893:
|
||||
- GPU: `echo 836000 > /proc/gpufreq/gpufreq_opp_freq`.
|
||||
- CPU: `echo 7 1 > /proc/ppm/policy_status`, then `echo <cluster> <kHz>` into `/proc/ppm/policy/userlimit_{min,max}_cpu_freq`.
|
||||
- The sysfs `scaling_*_freq` files are read-only there.
|
||||
- Restore everything afterwards.
|
||||
- Host: the NDK, `protoc`, and a venv with `pip install perfetto` (TraceProcessor downloads its shell binary on first use).
|
||||
|
||||
## 1. Start the counter producer
|
||||
|
||||
`gpuprod.c` dlopens the vendor library and calls its exported `start()`, which registers the perfetto data source `gpu.counters`.
|
||||
|
||||
```sh
|
||||
$NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android30-clang -O2 -o gpuprod gpuprod.c
|
||||
adb push gpuprod /data/local/tmp/ && adb shell chmod 755 /data/local/tmp/gpuprod
|
||||
adb shell 'su -c "getenforce"' # note the mode to restore later
|
||||
adb shell 'su -c "setenforce 0; nohup /data/local/tmp/gpuprod > /data/local/tmp/gpuprod.log 2>&1 &"'
|
||||
adb shell perfetto --query | grep gpu.counters # must list /data/local/tmp/gpuprod
|
||||
```
|
||||
|
||||
It needs SELinux permissive. **Afterwards**, kill the producer (`su -c "kill $(pidof gpuprod)"`) and restore the mode (`setenforce 1` if it was Enforcing).
|
||||
|
||||
## 2. List the counters this GPU offers
|
||||
|
||||
A capture without `counter_ids` records nothing, and the IDs are device-specific. Read them from the producer's descriptor:
|
||||
|
||||
```sh
|
||||
adb exec-out perfetto --query-raw > qraw.bin
|
||||
protoc --decode_raw < qraw.bin > qraw.txt
|
||||
python3 - <<'P'
|
||||
import re
|
||||
s = open('qraw.txt').read()
|
||||
for i, n in re.findall(r'\n\s*1: (\d+)\n\s*2: "([^"\n]{2,90})"\n\s*3: "', s):
|
||||
print(i, n)
|
||||
P
|
||||
```
|
||||
|
||||
IDs below 65535 are raw hardware counters. IDs 65535 and up are derived expressions (percentages, rates). Prefer raw counters: they add up over time, while ratios don't.
|
||||
|
||||
On the Mali-G77 these raw IDs were the useful set:
|
||||
|
||||
| ID | Counter | Use |
|
||||
|---|---|---|
|
||||
| 6 | GPU active cycles | total GPU cost |
|
||||
| 196 / 214 | Fragment / non-fragment active cycles | which side the gap is on |
|
||||
| 68 | Tiler active cycles | vertex + binning cost |
|
||||
| 160 / 175 | Output external read / write beats | DRAM bandwidth |
|
||||
| 210 | Tiles | how many tile passes the frame runs (≈ passes × screen tiles) |
|
||||
| 211 | Constant tiles killed | tile write-backs eliminated because the tile was unchanged |
|
||||
| 254 | Tile buffer write beats to L2 | tile write-back traffic |
|
||||
| 203 / 202 | Rasterized / partial rasterized quads | fragment workload size |
|
||||
| 204 / 206 | Early ZS tested / killed quads | early depth test working |
|
||||
| 207 / 208 | Late ZS tested / killed quads | shaders forced to late ZS (discard, depth writes, clears via shader) |
|
||||
| 212 | FPK occluder quads | forward pixel kill |
|
||||
| 234 | Texture quads | quads that sample textures |
|
||||
| 219 / 220 / 221 / 222 | FMA / CVT / SFU / message instructions | shader instruction mix |
|
||||
| 242 / 243 | 32-bit / 16-bit interpolation active | fp32 vs fp16 varyings |
|
||||
| 209 | Warps using more than 32 registers | register pressure |
|
||||
| 70 / 75 | Triangle / visible primitives | geometry identity check |
|
||||
| 85 / 101 | IDVS position / varying shading requests | vertex shading |
|
||||
|
||||
`counters.cfg` captures this set: 5 s at a 20 ms sampling period.
|
||||
|
||||
## 3. Capture each renderer
|
||||
|
||||
Get each renderer into the same steady scene: same camera, frozen world time, same resolution. Then capture while it runs:
|
||||
|
||||
```sh
|
||||
adb push counters.cfg /data/local/tmp/
|
||||
adb shell 'cat /data/local/tmp/counters.cfg | perfetto --txt -c - -o /data/misc/perfetto-traces/cnt.pftrace'
|
||||
adb pull /data/misc/perfetto-traces/cnt.pftrace A.pftrace
|
||||
```
|
||||
|
||||
- Record each run's FPS during the capture (for example, the on-screen FPS counter in a screenshot taken right after).
|
||||
- Change only the renderer between captures.
|
||||
- Cool the phone between runs and keep the clocks locked.
|
||||
|
||||
## 4. Compare per frame
|
||||
|
||||
```sh
|
||||
python compare.py A.pftrace <fpsA> B.pftrace <fpsB>
|
||||
```
|
||||
|
||||
It sums each counter over the capture, converts to per second, divides by FPS, and prints A/frame, B/frame and B/A, sorted by A's per-frame cost. Many values come out fractional, because the producer averages per-core counters across shader cores. Only the ratios matter.
|
||||
|
||||
## 5. Read the result
|
||||
|
||||
Work top-down:
|
||||
|
||||
1. **GPU active cycles B/A**: the size of the gap. It should roughly match the FPS ratio when GPU-bound.
|
||||
2. **Fragment vs non-fragment active cycles.** Whichever ratio is above 1 holds the gap. Equal vertex counters (non-fragment, tiler, primitives, IDVS) mean the geometry and vertex shading are identical, so look only at fragment work.
|
||||
3. **Tiles.** B/A ≈ 1 + k means B renders about k extra full-screen passes per frame. Typical causes:
|
||||
- a standalone clear pass;
|
||||
- a pass split by a mid-frame upload or barrier;
|
||||
- a blit, copy or clear outside a render pass (Mali runs those as fragment jobs too).
|
||||
|
||||
Find them with a per-frame Vulkan command census (see [Census of Magma's Vulkan commands](#census-of-magmas-vulkan-commands)).
|
||||
4. **Rasterized quads, late ZS, texture quads.**
|
||||
- With equal tiles: more rasterized quads mean extra full-screen draws inside a pass. For example, `vkCmdClearAttachments` mid-pass costs about one screen of quads each time.
|
||||
- High late-ZS counts mean fragments lost the early depth test: shaders with `discard`, `gl_FragDepth` or `gl_SampleMask` writes, or clears implemented as shader draws.
|
||||
- More early-ZS kills with equal visible primitives often mean the same thing seen from the other side.
|
||||
5. **External write beats vs tile buffer write beats.**
|
||||
- If B's extra external writes ≈ extra tile write beats × shader-core count, the difference is tile write-back.
|
||||
- Check `Constant tiles killed`: the GLES driver eliminates write-back of unchanged tiles. A Vulkan path with a much lower count is losing that elimination.
|
||||
- In a static benchmark scene this is a big effect. With a moving camera it is a small one.
|
||||
6. **Instruction mix and interpolation.** More FMA or CVT instructions with equal quads point at shader code quality. Much more 32-bit than 16-bit interpolation points at precision: for example, ESSL `mediump` lost when a backend recompiles the shader as desktop GLSL.
|
||||
|
||||
## Census of Magma's Vulkan commands
|
||||
|
||||
When the tile count says Magma runs extra passes, find out which ones. Use a temporary, never-committed build:
|
||||
|
||||
- **Wrappers:** write a header that declares counting wrappers and maps each `vkCmd*` entry point to its wrapper with a function-like macro (`#define vkCmdBeginRenderPass(...) Census_BeginRenderPass(__VA_ARGS__)`). Function-like macros leave function-pointer assignments such as `fns.vkCmdBeginRenderPass = vkCmdBeginRenderPass;` alone.
|
||||
- Include `VK_USE_PLATFORM_ANDROID_KHR` before `<vulkan/vulkan.h>` in that header.
|
||||
- Map at least render pass begin, render pass creation (to record each pass's load/store ops), blit, copy, clear (including `vkCmdClearAttachments`), resolve, dispatch, barrier, submit and present.
|
||||
- **Force-include:** apply it only to the DirectVulkan sources:
|
||||
|
||||
```cmake
|
||||
file(GLOB _census ${CMAKE_CURRENT_SOURCE_DIR}/MobileGL/MG_Backend/DirectVulkan/*.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/MobileGL/MG_Backend/DirectVulkan/Renderer/*.cpp)
|
||||
list(FILTER _census EXCLUDE REGEX "Census") # the wrapper definitions must call the real entry points
|
||||
set_source_files_properties(${_census} PROPERTIES COMPILE_OPTIONS "-include;<path>/Census.h")
|
||||
```
|
||||
|
||||
- **Recording:** in each wrapper, key the count by kind, the caller's offset in the library (`__builtin_return_address(0)` minus `dladdr`'s `dli_fbase`) and details: render area plus load/store ops for passes, area for clears and blits. In the present wrapper, every few seconds, log the counts divided by the number of frames to logcat.
|
||||
- **Symbolize** the offsets against the unstripped library: `llvm-addr2line -f -C -i -e build/intermediates/cxx/RelWithDebInfo/*/obj/arm64-v8a/libMobileGL.so 0x<offset>`. The library under `stripped_native_libs` has no symbols. If a wrapper's caller is a shared helper, log `__builtin_return_address(1..5)` inside that helper; arm64 Android keeps frame pointers.
|
||||
|
||||
## Example: Magma vs the system driver
|
||||
|
||||
Minecraft 1.12.2 with OptiFine at 2400×1080 on a Mali-G77, clocks locked, SFPEW over each backend:
|
||||
|
||||
| Finding | Counters | Cause | Fix |
|
||||
|---|---|---|---|
|
||||
| Fragment active cycles 1.33×, tiles 1.46× | rasterized quads, late ZS and tiles all up; vertex counters equal | one extra full-screen depth-only pass per frame (census: `vkCmdBeginRenderPass` with only `D24S8` LOAD/STORE, plus a full-screen `vkCmdClearAttachments`) | a window clear with a window pass open now ends the pass and folds into the next one (`VulkanRenderer::DeferDefaultWireClear`) |
|
||||
| Extra external writes 1.7× | tile buffer write beats ≈ 23×, constant tiles killed 0.07× | the GLES driver eliminates the write-back of unchanged tiles in a static scene; Magma's attachments get none | still open |
|
||||
| Fragment cycles still 1.18× with equal tiles | rasterized quads 1.26×, late ZS 1.44× | the game's two in-pass depth clears per frame, recorded as full-screen `vkCmdClearAttachments` | under investigation |
|
||||
|
||||
Separately, SFPEW hands MobileGL ESSL with `precision mediump float;` inside `#ifdef GL_ES`. The source processor rewrites ESSL to desktop GLSL with `GL_ES` undefined, so that guard silently turned the shaders fp32. 16-bit interpolation counters and the RelaxedPrecision decorations in the SPIR-V show this kind of loss.
|
||||
|
||||
## Pitfalls
|
||||
|
||||
- **Unlocked clocks.** Cycle counts are only comparable at the same GPU frequency, and FPS also depends on the CPU clock when the workload is not fully GPU-bound.
|
||||
- **Not checking the build.** Before comparing builds, verify that each run actually ran the build you think it did. `adb install -r` of an APK with a lower versionCode fails with `INSTALL_FAILED_VERSION_DOWNGRADE`, and the run silently uses whatever was installed.
|
||||
- **Different scenes.** Counters from different scenes or camera angles are not comparable at all. Freeze world time and camera position between runs.
|
||||
- **Leaving the phone modified.** The producer and permissive SELinux stay in place until you undo them. Restore both, and the clock locks, when you are done.
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Per-frame comparison of two Mali counter captures.
|
||||
|
||||
usage: python compare.py A.pftrace FPS_A B.pftrace FPS_B
|
||||
Needs `pip install perfetto`.
|
||||
"""
|
||||
import sys
|
||||
|
||||
from perfetto.trace_processor import TraceProcessor
|
||||
|
||||
|
||||
def per_second(path):
|
||||
tp = TraceProcessor(trace=path)
|
||||
rows = tp.query(
|
||||
"select t.name, sum(c.value) s, (max(c.ts) - min(c.ts)) / 1e9 dur "
|
||||
"from gpu_counter_track t join counter c on c.track_id = t.id group by t.id")
|
||||
out = {r.name: r.s / r.dur for r in rows if r.dur > 0}
|
||||
tp.close()
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 5:
|
||||
sys.exit(__doc__)
|
||||
a, fps_a, b, fps_b = sys.argv[1], float(sys.argv[2]), sys.argv[3], float(sys.argv[4])
|
||||
A, B = per_second(a), per_second(b)
|
||||
print(f"{'counter':52s} {'A/frame':>14s} {'B/frame':>14s} {'B/A':>6s}")
|
||||
for name in sorted(A, key=lambda k: -A[k] / fps_a):
|
||||
pa, pb = A[name] / fps_a, B.get(name, 0.0) / fps_b
|
||||
ratio = f"{pb / pa:6.2f}" if pa else " n/a"
|
||||
print(f"{name:52s} {pa:14.0f} {pb:14.0f} {ratio}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,50 @@
|
||||
# Mali-G77 raw counters (IDs from step 2 of SKILL.md); 5 s at a 20 ms period.
|
||||
buffers { size_kb: 131072 fill_policy: DISCARD }
|
||||
data_sources { config { name: "gpu.counters" gpu_counter_config { counter_period_ns: 20000000
|
||||
counter_ids: 6
|
||||
counter_ids: 10
|
||||
counter_ids: 18
|
||||
counter_ids: 196
|
||||
counter_ids: 214
|
||||
counter_ids: 68
|
||||
counter_ids: 160
|
||||
counter_ids: 175
|
||||
counter_ids: 218
|
||||
counter_ids: 219
|
||||
counter_ids: 220
|
||||
counter_ids: 221
|
||||
counter_ids: 222
|
||||
counter_ids: 245
|
||||
counter_ids: 201
|
||||
counter_ids: 216
|
||||
counter_ids: 203
|
||||
counter_ids: 202
|
||||
counter_ids: 204
|
||||
counter_ids: 206
|
||||
counter_ids: 207
|
||||
counter_ids: 208
|
||||
counter_ids: 212
|
||||
counter_ids: 210
|
||||
counter_ids: 211
|
||||
counter_ids: 254
|
||||
counter_ids: 250
|
||||
counter_ids: 251
|
||||
counter_ids: 231
|
||||
counter_ids: 234
|
||||
counter_ids: 241
|
||||
counter_ids: 242
|
||||
counter_ids: 243
|
||||
counter_ids: 247
|
||||
counter_ids: 248
|
||||
counter_ids: 249
|
||||
counter_ids: 70
|
||||
counter_ids: 75
|
||||
counter_ids: 85
|
||||
counter_ids: 101
|
||||
counter_ids: 209
|
||||
counter_ids: 8
|
||||
counter_ids: 16
|
||||
counter_ids: 226
|
||||
counter_ids: 223
|
||||
} } }
|
||||
duration_ms: 5000
|
||||
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
* Starts the vendor Mali counter producer so perfetto can record the
|
||||
* "gpu.counters" data source. Run as root, with SELinux permissive.
|
||||
*
|
||||
* Build: aarch64-linux-android30-clang -O2 -o gpuprod gpuprod.c
|
||||
*/
|
||||
#include <dlfcn.h>
|
||||
#include <stdio.h>
|
||||
#include <unistd.h>
|
||||
|
||||
int main(void) {
|
||||
void* lib = dlopen("/vendor/lib64/libgpudataproducer.so", RTLD_NOW);
|
||||
if (!lib) {
|
||||
fprintf(stderr, "dlopen: %s\n", dlerror());
|
||||
return 1;
|
||||
}
|
||||
void (*start)(void) = (void (*)(void))dlsym(lib, "start");
|
||||
if (!start) {
|
||||
fprintf(stderr, "libgpudataproducer.so exports no start()\n");
|
||||
return 1;
|
||||
}
|
||||
fprintf(stderr, "starting producer\n");
|
||||
start();
|
||||
for (;;)
|
||||
pause();
|
||||
}
|
||||
@@ -64,6 +64,12 @@ them wrong.
|
||||
| 2026-10-18 | `Negative control A (corrupted snapshot)` and `Keep lane logs before controls` | Test / `integration-verify-split` | the first re-ran the ten `VerifySplit.ClearThenReadPixelsScenario` entries of both backends with the same knob and accepted any red; the second only kept the lane's logs from being overwritten by that rerun | the in-lane `VerifySplitCorrupted` and `VerifySplitReadCorrupted` entries of both backends, required by name (`Verify-split control entries`); that the ambient entries arm is the per-entry arm proof, and that they run fatal is `VerifySplitArming.PipeVerifyArmingScenario.Armed` (the ambient environment plus its marker and log), which requires `fatal=1` on the arming line |
|
||||
| 2026-10-18 | unit `PipeInputsTest.{PoisonOmitKnobArmsTheOmission, BadPoisonOmitKnobIsFatalNamingTheKnob}`, and the `MOBILEGL_PIPE_POISON_OMIT` knob: `Features.PipePoisonOmit`, its parser on every verb (`MGPipeValidateForVerb`'s `ParsePoisonOmissionKnob`, compiled into every build) and the server stamp's own parse of it | Test / unit (every poison lane); product code | the knob existed only for negative control B, and the two cases pinned its parser and its `Fatal{PipeVerifyBadKnob}` | the PoisonOmitted and VerifySplitPoisonOmitted entries set the harness marker `MGITEST_POISON_OMIT` to the same pair, and the re-executed child arms it through `MGPipeSetPoisonOmission`, compiled only with `-DMOBILEGL_PIPE_VERIFY=ON` (`Harness/PoisonOmissionArm.h`); `OmittingOneFieldForOneVerbLeavesExactlyThatFieldStale`, `ReadingAnOmittedFieldAbortsNamingTheVerb` and `TheMutationNoticeRefreshesTheValueButNotTheStamp` withhold the stamp test-side and still run in every poison lane; the setter's scope to its verb, not its class, is checked by `OmittingOneFieldForOneVerbLeavesExactlyThatFieldStale` in a forked child, on the verify runtime only |
|
||||
| 2026-10-18 | the `serial == resets` assertion of `CtWireScenario.ApplierResetCrossesAtThePrimedEdgeInSerialOrder`, and the `MOBILEGL_TEST_SERVER_COUNTERS` knob: `ServerMain`'s `MGPipe server counters:` line on every LogFlush and `ServerVerbSink`'s test-only tallies (`ApplierResets`, `ObjectDeaths`, `ExpectedApplierResetSerial`) | Test / `integration-split`, `integration-split-strict`, `integration-verify-split`, `integration (inproc)`; product code | the knob existed only so the spawn and tcp arms could read the sink's tallies; the serial assertion could not fail, since `OnApplierReset` moves the count and the serial together and any other serial latches `Fatal{ProtocolCorruption, "ApplierReset.ContextSerial"}` | `ar` / `od` on the server role's `MGPipe stats:` line, counted in `ServerVerbSink::OnApplierReset` / `OnObjectDeath`; every CtWire entry that counts them sets `MOBILEGL_PIPE_STATS=1` with period 1 and reads them on every arm, inproc included, over a window closed by a swap and a readback; the unit cases `ApplierResetWireRoundTrip.TheSerialSequenceIsAssertedAndTheServerResetRuns` and `ObjectDeathWireRoundTrip.TheTextureAndFramebufferTwinsReleaseByHandleAndNeverByAStaleGeneration` read the same counters, and the first shows the expected serial moving by having serial 2 accepted |
|
||||
| 2026-11-15 | per-case jobs `retrace (<backend>, <case>)`, `retrace verify (<backend>, <case>)`, `retrace split (<backend>, <case>, <transport>)` | Test / `retrace`, `retrace-verify`, `retrace-split` | one runner per case spent most of its time on setup; no case, knob, assertion or arm proof is dropped | shard jobs that replay the same rows one after another through `scripts/ci/retrace_shard.py`, one ctest entry and one verdict per case; `trace_cases.py --self-test-shard-matrices` checks that the shards hold exactly the per-case rows |
|
||||
| 2026-11-15 | per-case emulator jobs `retrace (<backend>, <case>)` of each lane | MobileGL APK / `retrace` | one emulator boot per replay; no case, setting or arm proof is dropped | shard jobs that replay the cases of one lane, transport and backend on one boot through `android-plugin/trace_replay_shard.py`, with app data cleared between cases, a cold boot after a case that did not pass, and one verdict per case |
|
||||
| 2026-11-15 | per-case `trace fixture (<case>)` jobs, their per-case cache entries and artifacts, `.github/scripts/trace-fixture-cache.sh` | Test and MobileGL APK / `trace-fixtures` | 40 jobs and 40 cache entries per workflow for files that change rarely | one content-addressed cache entry for all Git LFS fixtures (`.github/scripts/trace-fixture-bundle.sh`); a replay job restores it and downloads a case only when the entry does not hold it |
|
||||
| 2026-11-15 | `remove-artifact-clutter` job | Test and MobileGL APK | it deleted the fixture artifacts and the green result artifacts after the run; the fixture artifacts no longer exist | the delete step at the end of `retrace-summary`; on Linux `retention-days: 1` on the summary inputs as well, `retrace-split` and `retrace-shipped` upload results only for a failed shard (their green results were deleted unread before), and `retrace-verify` keeps its results for every shard as before |
|
||||
| 2026-11-15 | emulator replays of the cases outside `android_smoke` on the monolith control and spawn acceptance lanes (56 of 79 rows per lane; the 12 smoke cases and the whole inproc lane stay) | MobileGL APK / `retrace` | the 237 emulator replays of a push held the 20 runner slots for most of an hour, and the launcher ships inproc; what goes is the same-commit Android monolith control and the Android spawn evidence (server exec from the native library directory, the pbuffer surface) for those 56 rows | Test replays every case on monolith, inproc and spawn on Linux; the Android inproc lane replays every case; `workflow_dispatch` with `android_full_matrix` runs the full emulator matrix; `trace_cases.py --self-test-transport-matrices` checks that the reduced lanes are the full matrix less exactly those rows |
|
||||
| 2026-11-15 | the three separate ctest invocations of `Split lanes (inproc, spawn, TCP)` (`-L integration-split`, `-L integration-spawn`, `-L '^integration-tcp$'`) | Test / `integration-split` | every TCP entry holds one resource lock, so the TCP invocation ran one entry at a time while the other cores were idle; the three selections share no entry | one invocation of the union of the three selections: each arm is still discovered with its own label regex, `lane_selection.py union` writes the exact names and fails when an arm selects nothing, and `junit_tally.py --select-from/--write-selected` carves each arm's JUnit out of the run under its old stem and fails when a selected name is missing or reported twice; the per-arm tallies, the entries required by name, the run-ahead proof and the skip-census rows are unchanged |
|
||||
|
||||
## Kept after investigation
|
||||
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
#!/bin/bash
|
||||
# MobileGL - scripts/ci/checkout_submodules.sh
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
# The submodules a CI build reads, checked out shallow and in parallel, in place of a
|
||||
# recursive checkout of every submodule in the tree.
|
||||
# library what the root CMakeLists.txt configures for the library, its tests and benchmarks.
|
||||
# retrace the library set plus the trace replay tool's: apitrace and the eight directories
|
||||
# its thirdparty/CMakeLists.txt requires.
|
||||
# Each row names a submodule and one file the build reads from it. A row whose file is missing
|
||||
# after the update fails the script: the configure would otherwise go on without it, and for
|
||||
# 3rdparty/flatbuffers the root CMakeLists.txt then builds a library without MG_Remote instead
|
||||
# of stopping. A submodule the build starts to use needs a row here.
|
||||
#
|
||||
# usage: checkout_submodules.sh <library|retrace> (from the repository root)
|
||||
# checkout_submodules.sh --list <library|retrace> prints the rows, checks out nothing
|
||||
set -euo pipefail
|
||||
|
||||
usage="usage: checkout_submodules.sh [--list] <library|retrace>"
|
||||
|
||||
list_only=false
|
||||
if [ "${1:-}" = --list ]; then
|
||||
list_only=true
|
||||
shift
|
||||
fi
|
||||
set_name="${1:?${usage}}"
|
||||
|
||||
# <submodule path> <file the build reads from it>
|
||||
library=(
|
||||
"3rdparty/glslang CMakeLists.txt"
|
||||
"3rdparty/SPIRV-Cross CMakeLists.txt"
|
||||
"3rdparty/SPIRV-Reflect CMakeLists.txt"
|
||||
"3rdparty/VulkanMemoryAllocator CMakeLists.txt"
|
||||
"3rdparty/Vulkan-Headers CMakeLists.txt"
|
||||
"3rdparty/Vulkan-Utility-Libraries CMakeLists.txt"
|
||||
"3rdparty/xxHash build/cmake/CMakeLists.txt"
|
||||
"3rdparty/asio include/asio.hpp"
|
||||
"3rdparty/flatbuffers include/flatbuffers/flatbuffers.h"
|
||||
"include/ska flat_hash_map.hpp"
|
||||
)
|
||||
retrace=(
|
||||
"3rdparty/apitrace thirdparty/CMakeLists.txt"
|
||||
)
|
||||
# Inside 3rdparty/apitrace: the files its thirdparty/CMakeLists.txt tests before it configures.
|
||||
apitrace=(
|
||||
"thirdparty/brotli LICENSE"
|
||||
"thirdparty/gtest LICENSE"
|
||||
"thirdparty/libbacktrace LICENSE"
|
||||
"thirdparty/libpng LICENSE"
|
||||
"thirdparty/snappy COPYING"
|
||||
"thirdparty/zlib README"
|
||||
"thirdparty/zstd LICENSE"
|
||||
"thirdparty/directxmath LICENSE"
|
||||
)
|
||||
|
||||
case "${set_name}" in
|
||||
library) top=("${library[@]}"); nested=() ;;
|
||||
retrace) top=("${library[@]}" "${retrace[@]}"); nested=("${apitrace[@]}") ;;
|
||||
*)
|
||||
echo "checkout_submodules.sh: unknown set '${set_name}' (library or retrace)" >&2
|
||||
echo "${usage}" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ "${list_only}" = true ]; then
|
||||
printf '%s\n' "${top[@]}"
|
||||
if [ "${#nested[@]}" -gt 0 ]; then
|
||||
printf '3rdparty/apitrace/%s\n' "${nested[@]}"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# update <directory> <row>...: one shallow, parallel update of the rows' submodules. The update
|
||||
# is idempotent, so a fetch that failed on the network is tried again, three times in all.
|
||||
update() {
|
||||
local dir="$1"
|
||||
shift
|
||||
local paths=() row attempt
|
||||
for row in "$@"; do
|
||||
paths+=("${row%% *}")
|
||||
done
|
||||
for attempt in 1 2 3; do
|
||||
if git -C "${dir}" submodule update --init --depth=1 --jobs 8 -- "${paths[@]}"; then
|
||||
return 0
|
||||
fi
|
||||
echo "checkout_submodules.sh: the submodule update in ${dir} failed (attempt ${attempt} of 3)" >&2
|
||||
if [ "${attempt}" -lt 3 ]; then
|
||||
sleep $((attempt * 5))
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# verify <directory> <row>...: every row's file exists; prints each missing one and returns 1.
|
||||
verify() {
|
||||
local dir="$1" missing=0 row path file
|
||||
shift
|
||||
for row in "$@"; do
|
||||
path="${row%% *}"
|
||||
file="${row#* }"
|
||||
if [ ! -f "${dir}/${path}/${file}" ]; then
|
||||
echo "::error::checkout_submodules.sh: ${dir}/${path}/${file} is missing after the submodule update, so the '${set_name}' build cannot use ${path}"
|
||||
missing=1
|
||||
fi
|
||||
done
|
||||
return "${missing}"
|
||||
}
|
||||
|
||||
update . "${top[@]}"
|
||||
verify . "${top[@]}"
|
||||
if [ "${#nested[@]}" -gt 0 ]; then
|
||||
update 3rdparty/apitrace "${nested[@]}"
|
||||
verify 3rdparty/apitrace "${nested[@]}"
|
||||
fi
|
||||
echo "checkout_submodules.sh: the '${set_name}' set is checked out ($(( ${#top[@]} + ${#nested[@]} )) submodules)"
|
||||
@@ -25,12 +25,17 @@ variant="${1:?usage: configure_build.sh <primary|split> <build dir>}"
|
||||
build_dir="${2:?usage: configure_build.sh <primary|split> <build dir>}"
|
||||
shift 2
|
||||
|
||||
# ENABLE_PCH is glslang's option, on by default. ccache cannot cache a compile that uses a
|
||||
# precompiled header, so with it on every glslang translation unit is compiled again in every
|
||||
# run. It is in the common part so that both lines stay the same configuration for
|
||||
# build_fingerprint.py.
|
||||
common=(
|
||||
-S . -B "${build_dir}" -G Ninja
|
||||
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}"
|
||||
-DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}"
|
||||
-DCMAKE_C_COMPILER_LAUNCHER=ccache
|
||||
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
|
||||
-DENABLE_PCH=OFF
|
||||
)
|
||||
|
||||
case "${variant}" in
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
# MobileGL - scripts/ci/lane_selection.py
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
"""One ctest run for several lanes: the exact names of every lane, as one --tests-from-file list.
|
||||
|
||||
A lane is a `ctest -L <label regex>` selection. Running the lanes of one job as one ctest
|
||||
invocation lets a lane that holds a resource lock (the TCP arm runs one entry at a time) share
|
||||
the run with the others. The lanes keep their own selections: each is discovered with its own
|
||||
label regex, this script writes the union of the discovered names, ctest runs that list, and
|
||||
junit_tally.py --select-from/--write-selected carves each lane's JUnit back out, which fails
|
||||
when a lane's name is missing from the run or reported twice.
|
||||
|
||||
lane_selection.py union OUT.txt LANE=DISCOVERY.json [LANE=DISCOVERY.json ...]
|
||||
lane_selection.py --self-test
|
||||
|
||||
DISCOVERY.json is `ctest -L <regex> --show-only=json-v1`. union fails when a lane selects
|
||||
nothing (what --no-tests=error says for a lane run on its own), names an entry twice, or holds a
|
||||
name a tests-from-file line cannot carry. It prints each lane's count and the size of the union.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
|
||||
def lane_names(label, path):
|
||||
"""The names one discovery selects, in order; raises ValueError when the lane is unusable."""
|
||||
names = [test['name'] for test in json.loads(Path(path).read_text()).get('tests', [])]
|
||||
if not names:
|
||||
raise ValueError(f'lane {label}: {path} selects no test')
|
||||
if len(names) != len(set(names)):
|
||||
twice = sorted({name for name in names if names.count(name) > 1})
|
||||
raise ValueError(f'lane {label}: {path} names {len(twice)} entr(ies) twice, e.g. {twice[0]}')
|
||||
# ctest reads the list one name per line and trims each line.
|
||||
bad = [name for name in names if name != name.strip() or '\n' in name or '\r' in name]
|
||||
if bad:
|
||||
raise ValueError(f'lane {label}: {len(bad)} name(s) cannot be written one per line, e.g. {bad[0]!r}')
|
||||
return names
|
||||
|
||||
|
||||
def union(lanes):
|
||||
"""(sorted union of names, {label: count}, number of names more than one lane selects)."""
|
||||
counts, seen, shared = {}, set(), set()
|
||||
for label, path in lanes:
|
||||
if label in counts:
|
||||
raise ValueError(f'lane {label} is given twice')
|
||||
names = lane_names(label, path)
|
||||
counts[label] = len(names)
|
||||
shared |= seen & set(names)
|
||||
seen |= set(names)
|
||||
return sorted(seen), counts, len(shared)
|
||||
|
||||
|
||||
def parse_lanes(arguments):
|
||||
lanes = []
|
||||
for argument in arguments:
|
||||
label, separator, path = argument.partition('=')
|
||||
if not separator or not label or not path:
|
||||
raise ValueError(f'expected LANE=DISCOVERY.json, got {argument!r}')
|
||||
lanes.append((label, path))
|
||||
if not lanes:
|
||||
raise ValueError('no lane given')
|
||||
return lanes
|
||||
|
||||
|
||||
def run(argv):
|
||||
if len(argv) < 3 or argv[0] != 'union':
|
||||
print(__doc__, file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
names, counts, shared = union(parse_lanes(argv[2:]))
|
||||
except (ValueError, OSError, json.JSONDecodeError) as error:
|
||||
print(f'::error::lane_selection.py: {error}')
|
||||
return 1
|
||||
Path(argv[1]).write_text(''.join(name + '\n' for name in names))
|
||||
summary = ', '.join(f'{label} {count}' for label, count in counts.items())
|
||||
print(f'lane_selection.py: {len(names)} entr(ies) in one run ({summary}; {shared} in more than one lane)')
|
||||
return 0
|
||||
|
||||
|
||||
def self_test():
|
||||
import contextlib
|
||||
import io
|
||||
|
||||
def quiet(argv):
|
||||
# The negative controls print ::error:: lines on purpose; keep them off a green run.
|
||||
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
||||
return run(argv)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp = Path(tmp)
|
||||
|
||||
def discovery(name, names):
|
||||
path = tmp / name
|
||||
path.write_text(json.dumps({'tests': [{'name': n} for n in names]}))
|
||||
return str(path)
|
||||
|
||||
out = tmp / 'tests.txt'
|
||||
split = discovery('split.json', ['DirectGLES.Split.A', 'DirectGLES.Split.B'])
|
||||
spawn = discovery('spawn.json', ['DirectGLES.Spawn.A', 'SpawnLane.Control'])
|
||||
tcp = discovery('tcp.json', ['TcpServer.Start', 'DirectGLES.Tcp.A', 'TcpServer.Stop'])
|
||||
assert quiet(['union', str(out), f'split={split}', f'spawn={spawn}', f'tcp={tcp}']) == 0
|
||||
assert out.read_text().splitlines() == sorted(
|
||||
['DirectGLES.Split.A', 'DirectGLES.Split.B', 'DirectGLES.Spawn.A', 'SpawnLane.Control',
|
||||
'TcpServer.Start', 'DirectGLES.Tcp.A', 'TcpServer.Stop']), 'the union is every lane, each name once'
|
||||
# A name two lanes select is written once; each lane still carves its own copy.
|
||||
shared = discovery('shared.json', ['DirectGLES.Split.A', 'Shared.Fixture'])
|
||||
assert quiet(['union', str(out), f'split={split}', f'other={shared}']) == 0
|
||||
assert out.read_text().splitlines() == ['DirectGLES.Split.A', 'DirectGLES.Split.B', 'Shared.Fixture']
|
||||
out.unlink()
|
||||
controls = {
|
||||
'a lane that selects nothing': [f'split={split}', f"spawn={discovery('empty.json', [])}"],
|
||||
'a lane that names an entry twice': [f"split={discovery('twice.json', ['A', 'A'])}"],
|
||||
'a name with a trailing blank': [f"split={discovery('blank.json', ['A '])}"],
|
||||
'a name with a line break': [f"split={discovery('break.json', ['A' + chr(10) + 'B'])}"],
|
||||
'a lane given twice': [f'split={split}', f'split={spawn}'],
|
||||
'a missing discovery file': [f"split={tmp / 'absent.json'}"],
|
||||
'an argument without a lane name': [split],
|
||||
}
|
||||
for label, arguments in controls.items():
|
||||
assert quiet(['union', str(out)] + arguments) == 1, f'{label} must fail'
|
||||
assert not out.exists(), f'{label} must not leave a list behind'
|
||||
assert quiet(['union', str(out)]) == 2, 'no lane is a usage error'
|
||||
assert quiet(['nonsense', str(out), f'split={split}']) == 2, 'an unknown command is a usage error'
|
||||
print('lane_selection self-test: ok')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if sys.argv[1:] == ['--self-test']:
|
||||
sys.exit(self_test())
|
||||
sys.exit(run(sys.argv[1:]))
|
||||
@@ -0,0 +1,689 @@
|
||||
#!/usr/bin/env python3
|
||||
# MobileGL - scripts/ci/retrace_shard.py
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
"""Replay the cases of one retrace shard, one after another, and give each its own verdict.
|
||||
|
||||
A shard is some of the cases of one (lane, transport, backend) cell, replayed by one CI job
|
||||
(tools/trace_replay/trace_cases.py, shard_matrix). Each case runs through its own ctest entry
|
||||
with the environment and the checks of its lane (case_plan), in a process group and a driver
|
||||
cache directory of its own, so no case sees what another left behind. A failing case does not
|
||||
stop the shard. The shard fails when a case failed, or when the verdicts it recorded are not
|
||||
exactly the cases it was given.
|
||||
|
||||
For every case it prints a log group, appends a row to the step summary and records a verdict
|
||||
in <result-dir>/results/shard-results.json; a failed case also gets an error annotation.
|
||||
The replay output the lane uploads is moved to <result-dir>/results/<case>/, the JUnit files
|
||||
are in <result-dir>/junit/, and the extracted trace is deleted.
|
||||
|
||||
Usage:
|
||||
retrace_shard.py run --profile {retrace,split,verify} --backend B [--transport T] [--lane L]
|
||||
--build-dir DIR --result-dir DIR [--names-env VAR | --names JSON]
|
||||
[--split-reuse] [--server-path PATH] [--shard N/M]
|
||||
retrace_shard.py --self-test
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPTS = Path(__file__).resolve().parent
|
||||
ITERATIONRP_CASE = "minecraft-1.21.4-fabric-iris-iterationrp-in-world"
|
||||
BLENDED_DEPTH_WRITE_CASE = "improved-transparency-minecraft-26.3"
|
||||
WIRE_EVIDENCE_CASE = "OpenRA"
|
||||
CONTROL_CASE = "OpenRA"
|
||||
CONTROL_BACKEND = "DirectGLES"
|
||||
CONTROL_NAME = "OpenRA (corrupted-snapshot control)"
|
||||
# The comparator runs the long cases several times slower than a plain replay, and a spawned
|
||||
# server can take as long, so the verify and split lanes raise ctest's 1500 s default.
|
||||
LONG_TIMEOUT = ("--timeout", "10800")
|
||||
SPLIT_REUSE_EXPECT_IPC = "strict=0 role-split-state=1 run-ahead=1"
|
||||
SPLIT_REUSE_EXPECT_IPC_REASON = (
|
||||
"retrace-split dropped its inproc rows because the inproc-acceptance retrace rows replay the "
|
||||
"same library with strict=1 role-split-state=1 run-ahead=1, which differs from this leg's "
|
||||
"defaults only in strict while these hold. A default changed: restore the inproc rows (the "
|
||||
"trace-cases job's split-spawn-shards without --transports spawn) or update the expectation.")
|
||||
# GitHub shows about ten error annotations of a step. The first failures are annotated as they
|
||||
# happen, so a shard killed later still shows them; the rest wait for the roll-up that names
|
||||
# every failed case.
|
||||
IMMEDIATE_ANNOTATIONS = 8
|
||||
|
||||
|
||||
def test_regex(case, backend):
|
||||
return rf"^MobileGLTraceReplay\.{case}\.{backend}$"
|
||||
|
||||
|
||||
def test_name(case, backend):
|
||||
return f"MobileGLTraceReplay.{case}.{backend}"
|
||||
|
||||
|
||||
def case_plan(profile, transport, backend, case, *, split_reuse=False, server_path=""):
|
||||
"""What one case of a lane runs with: the environment it adds, ctest's arguments, its checks.
|
||||
|
||||
retrace the monolith-control and inproc-acceptance lanes of the retrace job. The job itself
|
||||
exports MOBILEGL_TRANSPORT and the three IPC knobs for every case alike.
|
||||
split the retrace-split job, inproc and spawn.
|
||||
verify the retrace-verify job; its library has no transport.
|
||||
"""
|
||||
if profile not in ("retrace", "split", "verify"):
|
||||
raise ValueError(f"unknown profile: {profile}")
|
||||
env = {}
|
||||
if backend == "DirectVulkan":
|
||||
env["MOBILEGL_MAGMA_R11G11B10F_FALLBACK"] = "1"
|
||||
# The blended depth-write quirk enables itself only on Qualcomm, which no runner has,
|
||||
# so the case it exists for forces it on; ForceOn bypasses only the vendor gate.
|
||||
if case == BLENDED_DEPTH_WRITE_CASE:
|
||||
env["MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE"] = "1"
|
||||
# The iterationRP repairs default to device detection and the software driver is on no
|
||||
# list, so the one case that needs them forces them. The verify lane does not set it.
|
||||
if case == ITERATIONRP_CASE and profile != "verify":
|
||||
env["MOBILEGL_ITRP_FIX"] = "ForceOn"
|
||||
ctest = ["-V", "--no-tests=error"]
|
||||
evidence = None
|
||||
census = False
|
||||
if profile == "retrace":
|
||||
if transport == "inproc":
|
||||
# These rows stand in for retrace-split's dropped inproc rows, so they prove the
|
||||
# configuration they ran with from the client's own log.
|
||||
env["MGTRACE_EXPECT_IPC"] = "strict=1 role-split-state=1 run-ahead=1"
|
||||
census = True
|
||||
if transport != "monolith":
|
||||
evidence = []
|
||||
elif profile == "split":
|
||||
if transport not in ("inproc", "spawn"):
|
||||
raise ValueError(f"the split profile runs inproc or spawn, not {transport!r}")
|
||||
if not server_path.startswith("/"):
|
||||
# The replay's working directory is not the workspace and the launcher resolves the
|
||||
# image relative to nothing. The inproc arm reads it only to decide it needs none.
|
||||
raise ValueError("the split profile needs an absolute --server-path")
|
||||
env["MOBILEGL_TRANSPORT"] = transport
|
||||
env["MOBILEGL_IPC_SERVER_PATH"] = server_path
|
||||
evidence = []
|
||||
if case == WIRE_EVIDENCE_CASE:
|
||||
# This case also counts the wire's draw records and the server's draws. That turns
|
||||
# on the library's per-frame counters, so the others keep the plain configuration.
|
||||
env["MGTRACE_WIRE_EVIDENCE"] = "1"
|
||||
evidence = ["--require-wire"]
|
||||
if split_reuse:
|
||||
env["MGTRACE_EXPECT_IPC"] = SPLIT_REUSE_EXPECT_IPC
|
||||
env["MGTRACE_EXPECT_IPC_REASON"] = SPLIT_REUSE_EXPECT_IPC_REASON
|
||||
ctest += LONG_TIMEOUT
|
||||
census = True
|
||||
else:
|
||||
env["MOBILEGL_PIPE_VERIFY"] = "1"
|
||||
ctest += LONG_TIMEOUT
|
||||
ctest += ["-R", test_regex(case, backend)]
|
||||
return {"env": env, "ctest": ctest, "evidence": evidence, "census": census,
|
||||
"control": has_control(profile, backend, case)}
|
||||
|
||||
|
||||
def control_plan():
|
||||
"""The verify lane's negative control: the same replay with one snapshot field corrupted.
|
||||
|
||||
It must fail, and for the comparator's reason. It adds only these two variables and raises
|
||||
no core limit, since the failure is the expected outcome.
|
||||
"""
|
||||
return {"env": {"MOBILEGL_PIPE_VERIFY": "1",
|
||||
"MOBILEGL_PIPE_VERIFY_CORRUPT": "GetRenderStateParameters"},
|
||||
"ctest": ["-V", "--no-tests=error", *LONG_TIMEOUT,
|
||||
"-R", test_regex(CONTROL_CASE, CONTROL_BACKEND)]}
|
||||
|
||||
|
||||
def has_control(profile, backend, case):
|
||||
return profile == "verify" and case == CONTROL_CASE and backend == CONTROL_BACKEND
|
||||
|
||||
|
||||
def expected_results(profile, backend, names):
|
||||
"""The verdicts a shard owes, in order."""
|
||||
out = []
|
||||
for name in names:
|
||||
out.append(name)
|
||||
if has_control(profile, backend, name):
|
||||
out.append(CONTROL_NAME)
|
||||
return out
|
||||
|
||||
|
||||
def step_ignores_sigpipe():
|
||||
"""Whether the process that started this script, the step's shell, ignores SIGPIPE."""
|
||||
try:
|
||||
with open(f"/proc/{os.getppid()}/status", encoding="ascii") as status:
|
||||
for line in status:
|
||||
if line.startswith("SigIgn:"):
|
||||
return bool(int(line.split()[1], 16) & (1 << (signal.SIGPIPE - 1)))
|
||||
except (OSError, ValueError, IndexError):
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def raise_core_limit():
|
||||
# What `ulimit -c unlimited` does; the runner's core handler honors each process's limit.
|
||||
import resource
|
||||
resource.setrlimit(resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
|
||||
|
||||
|
||||
class Shard:
|
||||
def __init__(self, args, names, base_env):
|
||||
self.args = args
|
||||
self.names = names
|
||||
self.base_env = dict(base_env)
|
||||
self.build_dir = Path(args.build_dir).resolve()
|
||||
self.result_dir = Path(args.result_dir).resolve()
|
||||
self.results_root = self.result_dir / "results"
|
||||
self.junit_dir = self.result_dir / "junit"
|
||||
self.log_dir = self.result_dir / "logs"
|
||||
self.cache_root = self.result_dir / "driver-cache"
|
||||
self.records = []
|
||||
self.failures = 0
|
||||
self.child = None
|
||||
self.lane = args.lane or "-"
|
||||
self.transport = args.transport or "-"
|
||||
self.where = f"{self.lane} / {self.transport} / {args.backend}"
|
||||
|
||||
# --- output -------------------------------------------------------------------------------
|
||||
|
||||
def say(self, text=""):
|
||||
print(text, flush=True)
|
||||
|
||||
def summary(self, text):
|
||||
path = os.environ.get("GITHUB_STEP_SUMMARY")
|
||||
if path:
|
||||
with open(path, "a", encoding="utf-8") as file:
|
||||
file.write(text + "\n")
|
||||
|
||||
def annotate(self, title, message):
|
||||
escaped = message.replace("%", "%25").replace("\r", "").replace("\n", "%0A")
|
||||
# A property value also escapes the two characters that delimit properties.
|
||||
title = title.replace("%", "%25").replace(":", "%3A").replace(",", "%2C")
|
||||
self.say(f"::error title={title}::{escaped}")
|
||||
|
||||
def write_results(self):
|
||||
self.results_root.mkdir(parents=True, exist_ok=True)
|
||||
payload = {"profile": self.args.profile, "lane": self.args.lane, "transport": self.args.transport,
|
||||
"backend": self.args.backend, "shard": self.args.shard, "cases": self.names,
|
||||
"results": self.records}
|
||||
target = self.results_root / "shard-results.json"
|
||||
scratch = target.with_suffix(".tmp")
|
||||
scratch.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
|
||||
scratch.replace(target)
|
||||
|
||||
def record(self, name, status, seconds, reasons, *, ssim="", cores=()):
|
||||
"""One verdict: "passed", "failed", or "skipped" for a control whose case already failed."""
|
||||
entry = {"case": name, "status": status, "seconds": round(seconds, 1), "ssim": ssim,
|
||||
"reasons": list(reasons), "cores": list(cores), "annotated": False}
|
||||
if status == "failed":
|
||||
self.failures += 1
|
||||
entry["annotated"] = self.failures <= IMMEDIATE_ANNOTATIONS
|
||||
self.records.append(entry)
|
||||
self.write_results()
|
||||
note = "; ".join(reasons)
|
||||
if cores:
|
||||
note = (note + "; " if note else "") + "core: " + ", ".join(cores)
|
||||
shown = "**failed**" if status == "failed" else status
|
||||
self.summary(f"| {name} | {shown} | {seconds:.0f} | {ssim} | {note.replace('|', '/')[:300]} |")
|
||||
if entry["annotated"]:
|
||||
self.annotate(f"retrace failed: {self.where} / {name}", note or "failed")
|
||||
|
||||
# --- one process ----------------------------------------------------------------------------
|
||||
|
||||
def run_ctest(self, plan, *, junit=None, transcript=None, core_limit=True, cache_home=None):
|
||||
env = dict(self.base_env)
|
||||
env.update(plan["env"])
|
||||
if cache_home is not None:
|
||||
env["XDG_CACHE_HOME"] = str(cache_home)
|
||||
command = [self.args.ctest, *plan["ctest"]]
|
||||
if junit is not None:
|
||||
command += ["--output-junit", str(junit)]
|
||||
self.say("+ " + " ".join(f"{key}={value!r}" for key, value in sorted(plan["env"].items())))
|
||||
self.say("+ " + " ".join(command))
|
||||
options = {"cwd": self.build_dir, "env": env, "start_new_session": True}
|
||||
if core_limit:
|
||||
options["preexec_fn"] = raise_core_limit
|
||||
# Python ignores SIGPIPE itself and by default hands a child the default action. The
|
||||
# replay gets what a step's own child would get instead, so a write to a closed pipe
|
||||
# or socket ends the same way as when the step ran ctest directly.
|
||||
options["restore_signals"] = not step_ignores_sigpipe()
|
||||
sys.stdout.flush()
|
||||
if transcript is None:
|
||||
self.child = subprocess.Popen(command, **options)
|
||||
status = self.child.wait()
|
||||
else:
|
||||
# The transcript is both shown and kept, as `2>&1 | tee` would.
|
||||
self.child = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options)
|
||||
with open(transcript, "wb") as file:
|
||||
for chunk in iter(lambda: self.child.stdout.read1(65536), b""):
|
||||
file.write(chunk)
|
||||
sys.stdout.buffer.write(chunk)
|
||||
sys.stdout.buffer.flush()
|
||||
status = self.child.wait()
|
||||
self.reap()
|
||||
return status
|
||||
|
||||
def reap(self, grace=5.0):
|
||||
"""End whatever the finished replay left running, so it cannot run under the next case.
|
||||
|
||||
A spawned server exits by itself when its client is gone, so the group gets a moment
|
||||
first. True when something had to be killed.
|
||||
"""
|
||||
child, self.child = self.child, None
|
||||
if child is None:
|
||||
return False
|
||||
deadline = time.monotonic() + grace
|
||||
try:
|
||||
while True:
|
||||
os.killpg(child.pid, 0)
|
||||
if time.monotonic() >= deadline:
|
||||
break
|
||||
time.sleep(0.25)
|
||||
os.killpg(child.pid, signal.SIGKILL)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
return False
|
||||
self.say(f"killed the processes ctest (pid {child.pid}) left running")
|
||||
return True
|
||||
|
||||
# --- one case -------------------------------------------------------------------------------
|
||||
|
||||
def fresh_cache_home(self):
|
||||
"""An empty driver cache directory, as on a runner that has replayed nothing yet.
|
||||
|
||||
The software drivers keep compiled shaders on disk under the user cache directory. Each
|
||||
case gets a new one, so a case never starts from shaders another case compiled. On a
|
||||
hosted runner the default location is emptied as well, in case a process ignores the
|
||||
redirect; a self-hosted machine's own cache is left alone.
|
||||
"""
|
||||
self.cache_root.mkdir(parents=True, exist_ok=True)
|
||||
cache_home = Path(tempfile.mkdtemp(prefix="case-", dir=self.cache_root))
|
||||
if self.base_env.get("RUNNER_ENVIRONMENT") != "self-hosted":
|
||||
default_home = Path(self.base_env.get("HOME", "/nonexistent")) / ".cache"
|
||||
for stale in default_home.glob("mesa_shader_cache*"):
|
||||
self.say(f"removing {stale} left by an earlier process")
|
||||
shutil.rmtree(stale, ignore_errors=True)
|
||||
return cache_home
|
||||
|
||||
def drop_cache_home(self, cache_home):
|
||||
entries = sum(1 for path in cache_home.rglob("*") if path.is_file())
|
||||
self.say(f"driver cache: {entries} file(s) written under {cache_home}, removed")
|
||||
shutil.rmtree(cache_home, ignore_errors=True)
|
||||
|
||||
def junit_problem(self, junit, name):
|
||||
"""Why this JUnit file does not show exactly the one entry that was asked for, or ''."""
|
||||
if not junit.is_file():
|
||||
return f"ctest wrote no {junit.name}"
|
||||
try:
|
||||
entries = [case.get("name") for case in ET.parse(junit).getroot().iter("testcase")]
|
||||
except ET.ParseError as error:
|
||||
return f"{junit.name} does not parse: {error}"
|
||||
if entries != [name]:
|
||||
return f"ctest ran {entries or 'no entry'} where exactly {name} was asked for"
|
||||
return ""
|
||||
|
||||
def check_evidence(self, plan, case):
|
||||
"""Run the lane's evidence check; its error lines are folded into the case's annotation."""
|
||||
evidence = self.build_dir / case / self.args.backend / "output" / "split-evidence.txt"
|
||||
command = [sys.executable, str(SCRIPTS / "retrace_split_evidence.py"), str(evidence),
|
||||
"--transport", self.args.transport, "--case", case, "--backend", self.args.backend,
|
||||
*plan["evidence"]]
|
||||
done = subprocess.run(command, cwd=self.build_dir, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, text=True)
|
||||
text = done.stdout.strip()
|
||||
self.say(re.sub(r"^::error::", "split evidence: ", text, flags=re.M))
|
||||
if done.returncode == 0:
|
||||
return ""
|
||||
return "split evidence: " + re.sub(r"^::error::", "", text, flags=re.M).replace("\n", " ")[:600]
|
||||
|
||||
def census(self, case):
|
||||
"""The refusal census from the library's own logs, recorded and never a verdict.
|
||||
|
||||
Both roles are read: under spawn the applier's refusals are raised in the server's
|
||||
process. ctest's transcript cannot stand in, since the console sink is compiled out.
|
||||
"""
|
||||
output = self.build_dir / case / self.args.backend / "output"
|
||||
log = output / "mobilegl.client.log"
|
||||
server_log = output / "mobilegl.server.log"
|
||||
evidence = output / "split-evidence.txt"
|
||||
self.say(evidence.read_text(errors="replace").rstrip() if evidence.is_file() else "no split-evidence.txt")
|
||||
if not log.is_file():
|
||||
self.say(f"no {log} - the replay wrote no library log")
|
||||
return
|
||||
files = [log] + ([server_log] if server_log.is_file() else [])
|
||||
self.say("census over: " + " ".join(str(path) for path in files))
|
||||
lines = [line for path in files for line in path.read_text(errors="replace").splitlines()]
|
||||
self.say(f"Fatal{{ lines: {sum(1 for line in lines if 'Fatal{' in line)}")
|
||||
counts = {}
|
||||
for line in lines:
|
||||
for match in re.findall(r'Fatal\{[A-Za-z]*, "[^"]*"', line):
|
||||
counts[match] = counts.get(match, 0) + 1
|
||||
for match, count in sorted(counts.items(), key=lambda item: (-item[1], item[0]))[:20]:
|
||||
self.say(f"{count:7d} {match}")
|
||||
transport_line = next((line for line in log.read_text(errors="replace").splitlines()
|
||||
if "MOBILEGL_TRANSPORT=" in line), None)
|
||||
self.say(transport_line or f"no transport line in {log}")
|
||||
|
||||
def collect(self, case):
|
||||
"""Move what the lane uploads into the result directory and delete the extracted trace."""
|
||||
source = self.build_dir / case
|
||||
target = self.results_root / case
|
||||
shutil.rmtree(source / self.args.backend / "input", ignore_errors=True)
|
||||
for relative in ("actual-images", f"{self.args.backend}/output"):
|
||||
if (source / relative).is_dir():
|
||||
(target / relative).parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.rmtree(target / relative, ignore_errors=True)
|
||||
shutil.move(str(source / relative), str(target / relative))
|
||||
shutil.rmtree(source, ignore_errors=True)
|
||||
|
||||
def ssim_of(self, case):
|
||||
try:
|
||||
result = json.loads((self.build_dir / case / self.args.backend / "output" / "result.json")
|
||||
.read_text(errors="replace"))
|
||||
except (OSError, ValueError):
|
||||
return ""
|
||||
value = result.get("ssim")
|
||||
return "" if value is None else str(value)
|
||||
|
||||
def run_case(self, case):
|
||||
plan = case_plan(self.args.profile, self.args.transport, self.args.backend, case,
|
||||
split_reuse=self.args.split_reuse, server_path=self.args.server_path)
|
||||
self.say(f"::group::{case} ({self.where})")
|
||||
started = time.monotonic()
|
||||
cores_before = set(Path("/tmp").glob("core.*"))
|
||||
junit = self.junit_dir / f"{case}.xml"
|
||||
reasons = []
|
||||
cache_home = self.fresh_cache_home()
|
||||
try:
|
||||
try:
|
||||
status = self.run_ctest(plan, junit=junit, cache_home=cache_home)
|
||||
except (OSError, subprocess.SubprocessError) as error:
|
||||
# The case still gets its verdict, and the cases after it still run.
|
||||
self.child = None
|
||||
status = None
|
||||
reasons.append(f"ctest could not be run: {error}")
|
||||
if status is not None and status > 0:
|
||||
reasons.append(f"ctest exit {status}")
|
||||
elif status is not None and status < 0:
|
||||
reasons.append(f"ctest ended by signal {-status}")
|
||||
problem = self.junit_problem(junit, test_name(case, self.args.backend))
|
||||
if problem:
|
||||
reasons.append(problem)
|
||||
if plan["evidence"] is not None:
|
||||
# The transport assertions ran on this case and found their evidence. A failed
|
||||
# replay is already red, and its missing evidence file would only repeat that.
|
||||
if status == 0:
|
||||
problem = self.check_evidence(plan, case)
|
||||
if problem:
|
||||
reasons.append(problem)
|
||||
if plan["census"]:
|
||||
try:
|
||||
self.census(case)
|
||||
except OSError as error:
|
||||
self.say(f"refusal census skipped: {error}")
|
||||
finally:
|
||||
self.drop_cache_home(cache_home)
|
||||
ssim = self.ssim_of(case)
|
||||
self.collect(case)
|
||||
cores = sorted(path.name for path in set(Path("/tmp").glob("core.*")) - cores_before)
|
||||
seconds = time.monotonic() - started
|
||||
self.say(f"{case}: {'passed' if not reasons else 'FAILED: ' + '; '.join(reasons)} in {seconds:.0f} s")
|
||||
self.say("::endgroup::")
|
||||
self.record(case, "failed" if reasons else "passed", seconds, reasons, ssim=ssim, cores=cores)
|
||||
return not reasons
|
||||
|
||||
def run_control(self, verified_passed):
|
||||
"""With a snapshot field corrupted the same replay must fail, or the lane proves nothing.
|
||||
|
||||
It replays into the case directory the verified run used. That run's output has already
|
||||
been moved to the result directory, and the corrupted run's output is deleted, so only
|
||||
the verified run is uploaded under the case's name.
|
||||
"""
|
||||
self.say(f"::group::{CONTROL_NAME}")
|
||||
if not verified_passed:
|
||||
self.say("not run: the verified replay it is compared with failed")
|
||||
self.say("::endgroup::")
|
||||
self.record(CONTROL_NAME, "skipped", 0, ["not run: the verified replay failed"])
|
||||
return
|
||||
started = time.monotonic()
|
||||
reasons = []
|
||||
transcript = self.log_dir / "verify-control.txt"
|
||||
cache_home = self.fresh_cache_home()
|
||||
status = None
|
||||
try:
|
||||
status = self.run_ctest(control_plan(), transcript=transcript, core_limit=False,
|
||||
cache_home=cache_home)
|
||||
except (OSError, subprocess.SubprocessError) as error:
|
||||
self.child = None
|
||||
reasons.append(f"ctest could not be run: {error}")
|
||||
finally:
|
||||
self.drop_cache_home(cache_home)
|
||||
shutil.rmtree(self.build_dir / CONTROL_CASE, ignore_errors=True)
|
||||
if status is None:
|
||||
pass
|
||||
elif status == 0:
|
||||
reasons.append(
|
||||
"MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left the OpenRA retrace "
|
||||
"green, so the comparator is not comparing and the verify retrace lane proves nothing")
|
||||
else:
|
||||
# Its red must carry the comparator's divergence on the corrupted field.
|
||||
done = subprocess.run([sys.executable, str(SCRIPTS / "retrace_verify_reason.py"), str(transcript)],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
text = re.sub(r"^::error::", "", done.stdout.strip(), flags=re.M)
|
||||
self.say(text)
|
||||
if done.returncode != 0:
|
||||
reasons.append(text.replace("\n", " ")[:600])
|
||||
else:
|
||||
self.say(f"the corrupted field turned the retrace red, as it must (ctest exit {status})")
|
||||
seconds = time.monotonic() - started
|
||||
self.say("::endgroup::")
|
||||
self.record(CONTROL_NAME, "failed" if reasons else "passed", seconds, reasons)
|
||||
|
||||
# --- the shard --------------------------------------------------------------------------------
|
||||
|
||||
def run(self):
|
||||
for directory in (self.results_root, self.junit_dir, self.log_dir):
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
expected = expected_results(self.args.profile, self.args.backend, self.names)
|
||||
self.say(f"retrace shard {self.args.shard or ''} ({self.where}): {len(self.names)} case(s), "
|
||||
f"{len(expected)} verdict(s) owed")
|
||||
self.summary(f"### retrace shard {self.args.shard or ''} ({self.where})\n\n"
|
||||
"| case | result | seconds | SSIM | note |\n|---|---|---|---|---|")
|
||||
self.write_results()
|
||||
for case in self.names:
|
||||
passed = self.run_case(case)
|
||||
if case_plan(self.args.profile, self.args.transport, self.args.backend, case,
|
||||
split_reuse=self.args.split_reuse, server_path=self.args.server_path)["control"]:
|
||||
self.run_control(passed)
|
||||
return self.finish(expected)
|
||||
|
||||
def finish(self, expected):
|
||||
problems = verdict_problems(expected, self.records)
|
||||
# Counted again from what ctest wrote, apart from the loop's own bookkeeping.
|
||||
ran = sum(1 for case in self.names
|
||||
if not self.junit_problem(self.junit_dir / f"{case}.xml", test_name(case, self.args.backend)))
|
||||
passed_cases = sum(1 for entry in self.records
|
||||
if entry["status"] == "passed" and entry["case"] in self.names)
|
||||
if ran < passed_cases:
|
||||
problems.append(f"{passed_cases} case(s) recorded as passed but only {ran} JUnit file(s) "
|
||||
f"show their entry ran")
|
||||
failed = [entry for entry in self.records if entry["status"] == "failed"]
|
||||
if failed:
|
||||
names = ", ".join(entry["case"] for entry in failed)
|
||||
self.annotate(f"retrace shard {self.args.shard or ''}: {len(failed)} of {len(expected)} failed",
|
||||
f"{self.where}: {names}")
|
||||
for entry in failed:
|
||||
if not entry["annotated"]:
|
||||
self.annotate(f"retrace failed: {self.where} / {entry['case']}",
|
||||
"; ".join(entry["reasons"]) or "failed")
|
||||
for problem in problems:
|
||||
self.annotate(f"retrace shard {self.args.shard or ''}: verdicts do not match the cases", problem)
|
||||
passed = sum(1 for entry in self.records if entry["status"] == "passed")
|
||||
closing = f"{passed} passed, {len(failed)} failed, {len(expected)} owed"
|
||||
if problems:
|
||||
closing += "; " + "; ".join(problems)
|
||||
self.summary(f"\n{closing}\n")
|
||||
self.say(f"retrace shard ({self.where}): {closing}")
|
||||
return 1 if failed or problems else 0
|
||||
|
||||
|
||||
def verdict_problems(expected, records):
|
||||
"""Why the recorded verdicts are not exactly the ones owed; empty when they are."""
|
||||
recorded = [entry["case"] for entry in records]
|
||||
problems = []
|
||||
if len(recorded) != len(expected):
|
||||
problems.append(f"{len(recorded)} verdict(s) recorded for {len(expected)} owed")
|
||||
if recorded != expected:
|
||||
missing = [name for name in expected if name not in recorded]
|
||||
extra = [name for name in recorded if name not in expected]
|
||||
if missing:
|
||||
problems.append("no verdict for " + ", ".join(missing))
|
||||
if extra:
|
||||
problems.append("verdict for a case that was not assigned: " + ", ".join(extra))
|
||||
if not missing and not extra and len(recorded) == len(expected):
|
||||
problems.append("verdicts are out of order or repeated")
|
||||
return problems
|
||||
|
||||
|
||||
def run(args):
|
||||
if args.names is not None:
|
||||
names = json.loads(args.names)
|
||||
else:
|
||||
names = json.loads(os.environ.get(args.names_env) or "null")
|
||||
if not isinstance(names, list) or not names or not all(isinstance(name, str) and name for name in names):
|
||||
print(f"::error::retrace shard: no case list (looked in --names and ${args.names_env})")
|
||||
return 2
|
||||
if len(set(names)) != len(names):
|
||||
print(f"::error::retrace shard: the case list repeats a case: {names}")
|
||||
return 2
|
||||
if args.profile == "retrace":
|
||||
# The job exports the transport for this lane; the plan and the library must agree on it.
|
||||
exported = os.environ.get("MOBILEGL_TRANSPORT", "")
|
||||
if exported != args.transport:
|
||||
print(f"::error::retrace shard: --transport is {args.transport!r} but the job exports "
|
||||
f"MOBILEGL_TRANSPORT={exported!r}")
|
||||
return 2
|
||||
try:
|
||||
case_plan(args.profile, args.transport, args.backend, names[0],
|
||||
split_reuse=args.split_reuse, server_path=args.server_path)
|
||||
except ValueError as error:
|
||||
print(f"::error::retrace shard: {error}")
|
||||
return 2
|
||||
shard = Shard(args, names, os.environ)
|
||||
|
||||
def stop(signum, _frame):
|
||||
shard.reap(grace=0)
|
||||
shard.say(f"retrace shard: stopped by signal {signum} after {len(shard.records)} verdict(s)")
|
||||
sys.exit(128 + signum)
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
return shard.run()
|
||||
|
||||
|
||||
def self_test():
|
||||
"""The plan of every lane against the values the lanes are defined by, and the verdict check."""
|
||||
oit, itrp, other = BLENDED_DEPTH_WRITE_CASE, ITERATIONRP_CASE, "minecraft-1.21.4-in-world"
|
||||
server = "/w/build-linux/libMobileGLServer.so"
|
||||
regex = lambda case, backend: ["-R", rf"^MobileGLTraceReplay\.{case}\.{backend}$"]
|
||||
plain, long = ["-V", "--no-tests=error"], ["-V", "--no-tests=error", "--timeout", "10800"]
|
||||
vk = {"MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "1"}
|
||||
expect_strict = {"MGTRACE_EXPECT_IPC": "strict=1 role-split-state=1 run-ahead=1"}
|
||||
reuse = {"MGTRACE_EXPECT_IPC": SPLIT_REUSE_EXPECT_IPC, "MGTRACE_EXPECT_IPC_REASON": SPLIT_REUSE_EXPECT_IPC_REASON}
|
||||
split = lambda transport: {"MOBILEGL_TRANSPORT": transport, "MOBILEGL_IPC_SERVER_PATH": server}
|
||||
table = [
|
||||
# profile, transport, backend, case, reuse -> env, ctest, evidence, census, control
|
||||
("retrace", "monolith", "DirectGLES", other, False, {}, plain, None, False, False),
|
||||
("retrace", "monolith", "DirectVulkan", other, False, vk, plain, None, False, False),
|
||||
("retrace", "monolith", "DirectVulkan", oit, False,
|
||||
{**vk, "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE": "1"}, plain, None, False, False),
|
||||
("retrace", "monolith", "DirectGLES", oit, False, {}, plain, None, False, False),
|
||||
("retrace", "monolith", "DirectVulkan", itrp, False, {**vk, "MOBILEGL_ITRP_FIX": "ForceOn"},
|
||||
plain, None, False, False),
|
||||
("retrace", "inproc", "DirectGLES", other, False, expect_strict, plain, [], True, False),
|
||||
("retrace", "inproc", "DirectVulkan", itrp, False,
|
||||
{**vk, "MOBILEGL_ITRP_FIX": "ForceOn", **expect_strict}, plain, [], True, False),
|
||||
("retrace", "inproc", "DirectGLES", "OpenRA", False, expect_strict, plain, [], True, False),
|
||||
("split", "spawn", "DirectGLES", other, True, {**split("spawn"), **reuse}, long, [], True, False),
|
||||
("split", "spawn", "DirectGLES", other, False, split("spawn"), long, [], True, False),
|
||||
("split", "spawn", "DirectVulkan", oit, True,
|
||||
{**vk, "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE": "1", **split("spawn"), **reuse},
|
||||
long, [], True, False),
|
||||
("split", "spawn", "DirectVulkan", itrp, True,
|
||||
{**vk, "MOBILEGL_ITRP_FIX": "ForceOn", **split("spawn"), **reuse}, long, [], True, False),
|
||||
("split", "spawn", "DirectGLES", "OpenRA", True,
|
||||
{**split("spawn"), "MGTRACE_WIRE_EVIDENCE": "1", **reuse}, long, ["--require-wire"], True, False),
|
||||
("split", "inproc", "DirectVulkan", "OpenRA", True,
|
||||
{**vk, **split("inproc"), "MGTRACE_WIRE_EVIDENCE": "1", **reuse}, long, ["--require-wire"], True, False),
|
||||
("split", "inproc", "DirectGLES", other, False, split("inproc"), long, [], True, False),
|
||||
("verify", "", "DirectGLES", other, False, {"MOBILEGL_PIPE_VERIFY": "1"}, long, None, False, False),
|
||||
("verify", "", "DirectGLES", "OpenRA", False, {"MOBILEGL_PIPE_VERIFY": "1"}, long, None, False, True),
|
||||
("verify", "", "DirectVulkan", "OpenRA", False, {"MOBILEGL_PIPE_VERIFY": "1", **vk}, long, None, False, False),
|
||||
("verify", "", "DirectVulkan", oit, False,
|
||||
{"MOBILEGL_PIPE_VERIFY": "1", **vk, "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE": "1"},
|
||||
long, None, False, False),
|
||||
("verify", "", "DirectVulkan", itrp, False, {"MOBILEGL_PIPE_VERIFY": "1", **vk}, long, None, False, False),
|
||||
]
|
||||
for profile, transport, backend, case, split_reuse, env, ctest, evidence, census, control in table:
|
||||
plan = case_plan(profile, transport, backend, case, split_reuse=split_reuse, server_path=server)
|
||||
wanted = {"env": env, "ctest": ctest + regex(case, backend), "evidence": evidence,
|
||||
"census": census, "control": control}
|
||||
assert plan == wanted, f"{profile} {transport} {backend} {case}: {plan} != {wanted}"
|
||||
control = control_plan()
|
||||
assert control["env"] == {"MOBILEGL_PIPE_VERIFY": "1", "MOBILEGL_PIPE_VERIFY_CORRUPT": "GetRenderStateParameters"}
|
||||
assert control["ctest"] == long + regex("OpenRA", "DirectGLES")
|
||||
for bad in (("split", "spawn", ""), ("split", "spawn", "relative/server"), ("split", "monolith", server),
|
||||
("pull", "monolith", server)):
|
||||
try:
|
||||
case_plan(bad[0], bad[1], "DirectGLES", other, server_path=bad[2])
|
||||
except ValueError:
|
||||
continue
|
||||
raise AssertionError(f"case_plan accepted {bad}")
|
||||
assert expected_results("verify", "DirectGLES", ["a", "OpenRA", "b"]) == ["a", "OpenRA", CONTROL_NAME, "b"]
|
||||
assert expected_results("verify", "DirectVulkan", ["OpenRA"]) == ["OpenRA"]
|
||||
assert expected_results("split", "DirectGLES", ["OpenRA"]) == ["OpenRA"]
|
||||
entry = lambda name: {"case": name, "status": "passed"}
|
||||
assert verdict_problems(["a", "b"], [entry("a"), entry("b")]) == []
|
||||
assert verdict_problems(["a", "b"], [entry("a")]), "a case without a verdict passed the check"
|
||||
assert verdict_problems(["a", "b"], []), "a shard that ran nothing passed the check"
|
||||
assert verdict_problems(["a"], [entry("a"), entry("a")]), "a repeated verdict passed the check"
|
||||
assert verdict_problems(["a", "b"], [entry("a"), entry("c")]), "an unassigned case passed the check"
|
||||
assert verdict_problems(["a", "b"], [entry("b"), entry("a")]), "out-of-order verdicts passed the check"
|
||||
print(f"retrace_shard: {len(table)} plans, the control plan and the verdict check OK")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("command", nargs="?", choices=("run",))
|
||||
parser.add_argument("--self-test", action="store_true")
|
||||
parser.add_argument("--profile", choices=("retrace", "split", "verify"))
|
||||
parser.add_argument("--backend")
|
||||
parser.add_argument("--transport", default="")
|
||||
parser.add_argument("--lane", default="")
|
||||
parser.add_argument("--shard", default="", help="N/M, for the report only")
|
||||
parser.add_argument("--build-dir", help="the directory ctest runs in")
|
||||
parser.add_argument("--result-dir", help="where verdicts, replay output and JUnit files go")
|
||||
parser.add_argument("--names", help="the cases, as a JSON list")
|
||||
parser.add_argument("--names-env", default="SHARD_NAMES", help="the variable holding the JSON list")
|
||||
parser.add_argument("--split-reuse", action="store_true",
|
||||
help="split profile: the library is the retrace job's, so prove the IPC defaults")
|
||||
parser.add_argument("--server-path", default="", help="split profile: absolute path of the server image")
|
||||
parser.add_argument("--ctest", default="ctest")
|
||||
args = parser.parse_args(argv)
|
||||
if args.self_test:
|
||||
return self_test()
|
||||
if args.command != "run" or not (args.profile and args.backend and args.build_dir and args.result_dir):
|
||||
parser.error("run needs --profile, --backend, --build-dir and --result-dir")
|
||||
return run(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -48,7 +48,8 @@ public final class SpawnServerPathTest {
|
||||
|
||||
public static void main(String[] args) {
|
||||
// ---- the shapes the lanes really pass -------------------------------------------------
|
||||
// .github/workflows/apk.yml's spawn acceptance lane, verbatim.
|
||||
// The spawn acceptance lane of android-plugin/trace_replay_shard.py (transport_arguments()),
|
||||
// verbatim.
|
||||
String spawnLane = "MOBILEGL_TRANSPORT=spawn;MOBILEGL_IPC_ROLE_SPLIT_STATE=1;"
|
||||
+ "MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1";
|
||||
expect("the spawn acceptance lane still gets its server path appended",
|
||||
|
||||
@@ -16,14 +16,15 @@ const BACKENDS = ["DirectGLES", "DirectVulkan"];
|
||||
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
const DEFAULT_FIXTURE_ROOT = path.join(SCRIPT_DIR, "fixtures");
|
||||
const TRACE_CASES = JSON.parse(readFileSync(path.join(SCRIPT_DIR, "trace_cases.json"), "utf8"));
|
||||
const CASES = TRACE_CASES.cases.map((traceCase) => traceCase.name);
|
||||
// Every case of the manifest, or the ones --cases names when a lane replays a subset.
|
||||
let CASES = TRACE_CASES.cases.map((traceCase) => traceCase.name);
|
||||
|
||||
function usage() {
|
||||
console.error(`Usage:
|
||||
node tools/trace_replay/render_retrace_summary.mjs \\
|
||||
--input DIR [--input DIR ...] \\
|
||||
--output-dir DIR \\
|
||||
[--title TITLE] [--group-label LABEL] [--html NAME]`);
|
||||
[--title TITLE] [--group-label LABEL] [--html NAME] [--cases NAME,NAME,...]`);
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
@@ -52,6 +53,13 @@ function parseArgs(argv) {
|
||||
} else if (arg === "--html" && value) {
|
||||
args.htmlName = value;
|
||||
i += 1;
|
||||
} else if (arg === "--cases" && value !== undefined) {
|
||||
// An empty list keeps every case, so a caller can pass the option unconditionally.
|
||||
const names = value.split(",").filter((name) => name);
|
||||
const unknown = names.filter((name) => !CASES.includes(name));
|
||||
if (unknown.length) throw new Error(`--cases names unknown trace case(s): ${unknown.join(", ")}`);
|
||||
if (names.length) CASES = CASES.filter((name) => names.includes(name));
|
||||
i += 1;
|
||||
} else if (arg === "--help" || arg === "-h") {
|
||||
usage();
|
||||
process.exit(0);
|
||||
|
||||
@@ -650,7 +650,8 @@ def transport_proof_args(args):
|
||||
return {"inproc": ["--require-inproc"], "spawn": ["--require-spawn"]}.get(args.transport, [])
|
||||
|
||||
|
||||
# The knobs .github/workflows/apk.yml hands BOTH acceptance lanes. They are kept here verbatim so
|
||||
# The knobs android-plugin/trace_replay_shard.py (transport_arguments()) hands both acceptance
|
||||
# lanes. They are kept here verbatim so
|
||||
# a local run is the same run: the arm proof below reads the library's own `Config: IPC` line and
|
||||
# requires strict/role-split-state/run-ahead to be 1, so a local invocation that quietly omitted
|
||||
# them would fail a gate CI passes - or worse, pass a weaker one.
|
||||
|
||||
@@ -1,4 +1,18 @@
|
||||
{
|
||||
"android_smoke": [
|
||||
"OpenRA",
|
||||
"minecraft-1.21.4-startup",
|
||||
"minecraft-1.21.4-main-menu",
|
||||
"minecraft-1.21.4-fabric-sodium-in-world",
|
||||
"minecraft-1.21.4-fabric-iris-bsl-in-world",
|
||||
"minecraft-1.21.4-fabric-iris-complementary-reimagined-in-world",
|
||||
"minecraft-1.21.4-fabric-iris-bliss-in-world",
|
||||
"minecraft-1.21.4-fabric-iris-sundial-lite-in-world",
|
||||
"minecraft-1.21.1-neoforge-create-indirect-in-world-align1024",
|
||||
"minecraft-1.21.1-neoforge-create-instancing-in-world",
|
||||
"improved-transparency-minecraft-26.3",
|
||||
"minecraft-1.21.4-fabric-iris-iterationrp-in-world"
|
||||
],
|
||||
"defaults": {
|
||||
"trace_file": "trace.trace",
|
||||
"width": 854,
|
||||
|
||||
@@ -14,6 +14,9 @@ from pathlib import Path
|
||||
|
||||
|
||||
TRACE_CASES_JSON = Path(__file__).with_name("trace_cases.json")
|
||||
|
||||
# The top-level keys of the manifest. An unknown one is refused, as an unknown case key is.
|
||||
KNOWN_MANIFEST_KEYS = frozenset({"defaults", "cases", "android_smoke"})
|
||||
CI_BACKENDS = ("DirectGLES", "DirectVulkan")
|
||||
|
||||
# Every key a case or the defaults block may carry. An UNKNOWN key is a hard error rather than a
|
||||
@@ -127,7 +130,26 @@ def load_trace_case_manifest(path=TRACE_CASES_JSON):
|
||||
merged["split"] = split
|
||||
validate_backend_overrides(merged)
|
||||
cases.append(merged)
|
||||
return {"defaults": defaults, "cases": cases}
|
||||
unknown_top = sorted(set(manifest) - KNOWN_MANIFEST_KEYS)
|
||||
if unknown_top:
|
||||
raise ValueError(f"unknown top-level key(s) in the manifest: {', '.join(unknown_top)}")
|
||||
# "android_smoke" names the cases the emulator replays on the monolith and spawn lanes
|
||||
# when those lanes are reduced. A name that is not a CI case would shrink the subset with
|
||||
# no red, so it is refused here.
|
||||
smoke = manifest.get("android_smoke", [])
|
||||
if not isinstance(smoke, list) or not all(isinstance(name, str) for name in smoke):
|
||||
raise ValueError("android_smoke must be a list of case names")
|
||||
if len(set(smoke)) != len(smoke):
|
||||
raise ValueError("android_smoke repeats a case name")
|
||||
ci_names = {case["name"] for case in cases if case.get("ci", True)}
|
||||
not_ci = [name for name in smoke if name not in ci_names]
|
||||
if not_ci:
|
||||
raise ValueError(f"android_smoke names case(s) that are not CI cases: {', '.join(not_ci)}")
|
||||
return {"defaults": defaults, "cases": cases, "android_smoke": smoke}
|
||||
|
||||
|
||||
def android_smoke_names(path=TRACE_CASES_JSON):
|
||||
return load_trace_case_manifest(path)["android_smoke"]
|
||||
|
||||
|
||||
def validate_backend_overrides(case):
|
||||
@@ -400,11 +422,21 @@ def github_apk_matrix(cases):
|
||||
}
|
||||
|
||||
|
||||
def github_transport_matrix(cases, *, apk=False):
|
||||
# The Android lanes of the transport matrix. "full" replays every case on every transport.
|
||||
# "inproc-full" replays every case on inproc, the transport the launcher ships, and only the
|
||||
# manifest's android_smoke cases on monolith and spawn: the Linux retrace jobs replay every
|
||||
# case on all three transports, so those two emulator lanes keep the cases that prove the
|
||||
# Android side of each arm.
|
||||
ANDROID_LANES = ("full", "inproc-full")
|
||||
|
||||
|
||||
def github_transport_matrix(cases, *, apk=False, android_smoke=None):
|
||||
"""Run the entire original matrix as control and as acceptance, once per transport.
|
||||
|
||||
The transports use the same D/P artifact and each case's existing backend
|
||||
list. The legacy split:true subset does not limit this main acceptance matrix.
|
||||
With `android_smoke`, an APK matrix keeps every case on inproc and only the named
|
||||
cases on monolith and spawn.
|
||||
Nested APK case/backend metadata is deliberately left untouched.
|
||||
|
||||
THE SPAWN LANE IS APK-ONLY, and that is a statement about where the evidence is missing
|
||||
@@ -423,12 +455,19 @@ def github_transport_matrix(cases, *, apk=False):
|
||||
build = github_apk_matrix if apk else github_test_matrix
|
||||
if not cases:
|
||||
raise ValueError("transport matrix needs at least one trace case")
|
||||
reduced = cases
|
||||
if android_smoke is not None:
|
||||
if not apk:
|
||||
raise ValueError("only the APK transport matrix has reduced lanes")
|
||||
reduced = [case for case in cases if case["name"] in android_smoke]
|
||||
if not reduced:
|
||||
raise ValueError("the reduced Android lanes are empty: no case is in android_smoke")
|
||||
lanes = [
|
||||
(cases, "monolith-control", "monolith"),
|
||||
(reduced, "monolith-control", "monolith"),
|
||||
(cases, "inproc-acceptance", "inproc"),
|
||||
]
|
||||
if apk:
|
||||
lanes.append((cases, "spawn-acceptance", "spawn"))
|
||||
lanes.append((reduced, "spawn-acceptance", "spawn"))
|
||||
return {"include": [
|
||||
{**row, "lane": lane, "transport": transport}
|
||||
for selected, lane, transport in lanes
|
||||
@@ -481,6 +520,29 @@ def self_test_transport_matrices():
|
||||
print(f"transport matrix {'APK' if apk else 'Linux'}: {len(controls)} unchanged controls, "
|
||||
f"{len(acceptance)} inproc acceptance entries, {len(spawned)} spawn acceptance "
|
||||
f"entries; exact metadata and negative controls OK")
|
||||
# The reduced Android lanes are the full matrix less the monolith and spawn rows of the
|
||||
# cases outside android_smoke: nothing else is dropped, changed or added.
|
||||
smoke = android_smoke_names()
|
||||
assert smoke, "android_smoke is empty, so the reduced monolith and spawn lanes would be too"
|
||||
full = github_transport_matrix(cases, apk=True)["include"]
|
||||
reduced = github_transport_matrix(cases, apk=True, android_smoke=smoke)["include"]
|
||||
kept = [row for row in full if row["transport"] == "inproc" or row["case"]["name"] in smoke]
|
||||
assert reduced == kept, "the reduced Android lanes are not the full matrix filtered"
|
||||
for transport in ("monolith", "spawn"):
|
||||
names = {row["case"]["name"] for row in reduced if row["transport"] == transport}
|
||||
assert names == set(smoke), f"the reduced {transport} lane is not the smoke cases: {names}"
|
||||
full_inproc = [row for row in full if row["transport"] == "inproc"]
|
||||
assert [row for row in reduced if row["transport"] == "inproc"] == full_inproc, \
|
||||
"the inproc lane was reduced"
|
||||
for bad in ({"apk": False, "android_smoke": smoke},
|
||||
{"apk": True, "android_smoke": ["no-such-case"]}):
|
||||
try:
|
||||
github_transport_matrix(cases, **bad)
|
||||
except ValueError:
|
||||
continue
|
||||
raise AssertionError(f"github_transport_matrix accepted {bad}")
|
||||
print(f"transport matrix APK reduced: {len(reduced)} of {len(full)} rows; inproc keeps all "
|
||||
f"{len(full_inproc)}, monolith and spawn keep the {len(smoke)} smoke cases")
|
||||
|
||||
|
||||
def self_test_split_matrix(cases=None):
|
||||
@@ -518,6 +580,318 @@ def self_test_split_matrix(cases=None):
|
||||
f"kept); all {len(dropped)} dropped inproc rows are inproc-acceptance rows")
|
||||
|
||||
|
||||
# --- Shards: several cases of one cell replayed by one CI job ---------------------------------
|
||||
#
|
||||
# A cell is everything a per-case matrix row carries except its case: the backend, and the lane
|
||||
# and transport where the matrix has them. A shard row keeps the cell's keys unchanged and adds
|
||||
# the cases it replays, so the job that runs it reads the same fields a per-case leg reads.
|
||||
# Every shard matrix is built from the per-case matrix it replaces and is refused unless its
|
||||
# rows, taken apart again, are exactly that matrix.
|
||||
|
||||
TRACE_DURATION_HINTS_JSON = Path(__file__).with_name("trace_duration_hints.json")
|
||||
|
||||
# Shards per cell on Linux, the one tunable of that layout. A cell with little to replay gets
|
||||
# fewer: a shard costs about half a minute of setup, so it is not split below
|
||||
# MIN_SHARD_SECONDS of replay.
|
||||
DEFAULT_SHARDS_PER_CELL = 4
|
||||
MIN_SHARD_SECONDS = 90
|
||||
|
||||
# Replay seconds one emulator shard aims for, the one tunable of the Android layout. A cell is
|
||||
# split into as many shards as that takes.
|
||||
DEFAULT_ANDROID_SHARD_SECONDS = 720
|
||||
|
||||
# The hint a row without a transport uses. The verify lane has timings of its own.
|
||||
HINT_KIND_MONOLITH = "monolith"
|
||||
HINT_KIND_VERIFY = "verify"
|
||||
|
||||
# A shard's case names reach a shell command line, so the characters are restricted.
|
||||
SHARD_SAFE_NAME = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-")
|
||||
|
||||
|
||||
def load_duration_hints(platform, path=TRACE_DURATION_HINTS_JSON):
|
||||
"""{case: {backend: {kind: seconds}}} for "linux" or "android"; empty when there is none."""
|
||||
path = Path(path)
|
||||
if not path.is_file():
|
||||
return {}
|
||||
with path.open("r", encoding="utf-8") as file:
|
||||
table = json.load(file).get(platform, {}).get("seconds", {})
|
||||
for case, backends in table.items():
|
||||
for backend, kinds in backends.items():
|
||||
for kind, seconds in kinds.items():
|
||||
if isinstance(seconds, bool) or not isinstance(seconds, (int, float)) or seconds <= 0:
|
||||
raise ValueError(f"duration hint for {case} {backend} {kind} must be a positive "
|
||||
f"number: {seconds!r}")
|
||||
return table
|
||||
|
||||
|
||||
def median(values):
|
||||
ordered = sorted(values)
|
||||
if not ordered:
|
||||
return None
|
||||
middle = len(ordered) // 2
|
||||
if len(ordered) % 2:
|
||||
return ordered[middle]
|
||||
return (ordered[middle - 1] + ordered[middle]) / 2
|
||||
|
||||
|
||||
def hint_seconds(hints, case, backend, kind):
|
||||
"""The hinted replay time of one row, or the median of its backend and kind, or of everything."""
|
||||
try:
|
||||
return hints[case][backend][kind]
|
||||
except KeyError:
|
||||
pass
|
||||
same_cell = [kinds[kind] for backends in hints.values()
|
||||
for name, kinds in backends.items() if name == backend and kind in kinds]
|
||||
everything = [seconds for backends in hints.values() for kinds in backends.values()
|
||||
for seconds in kinds.values()]
|
||||
return median(same_cell) or median(everything) or 30
|
||||
|
||||
|
||||
def row_case_name(row):
|
||||
case = row["case"]
|
||||
return case["name"] if isinstance(case, dict) else case
|
||||
|
||||
|
||||
def row_backend_name(row):
|
||||
backend = row["backend"]
|
||||
return backend["name"] if isinstance(backend, dict) else backend
|
||||
|
||||
|
||||
def row_cell(row):
|
||||
return {key: value for key, value in row.items() if key != "case"}
|
||||
|
||||
|
||||
def canonical(value):
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def shard_rows_as_cases(shard_rows):
|
||||
"""Take shard rows apart into the per-case rows they stand for."""
|
||||
added = ("shard", "shards", "names", "cases", "seconds")
|
||||
rows = []
|
||||
for shard in shard_rows:
|
||||
cell = {key: value for key, value in shard.items() if key not in added}
|
||||
for case in shard.get("cases", shard["names"]):
|
||||
rows.append({**cell, "case": case})
|
||||
return rows
|
||||
|
||||
|
||||
def check_shard_union(rows, shard_rows):
|
||||
"""Refuse a shard matrix that is not exactly the per-case matrix `rows`, regrouped."""
|
||||
expected = sorted(canonical(row) for row in rows)
|
||||
actual = sorted(canonical(row) for row in shard_rows_as_cases(shard_rows))
|
||||
if expected != actual:
|
||||
missing = sorted(set(expected) - set(actual))
|
||||
extra = sorted(set(actual) - set(expected))
|
||||
raise ValueError(
|
||||
f"the shard matrix is not the per-case matrix regrouped: {len(expected)} per-case rows, "
|
||||
f"{len(actual)} rows in the shards; missing {missing[:3]}, unexpected {extra[:3]}")
|
||||
if len(set(expected)) != len(expected):
|
||||
raise ValueError("the per-case matrix repeats a row, so a shard would replay it twice")
|
||||
cells = {}
|
||||
for shard in shard_rows:
|
||||
if not shard["names"]:
|
||||
raise ValueError(f"empty shard: {shard}")
|
||||
if "cases" in shard and [case["name"] for case in shard["cases"]] != shard["names"]:
|
||||
raise ValueError(f"a shard's names and cases disagree: {shard['names']}")
|
||||
cell = canonical({key: value for key, value in shard.items()
|
||||
if key not in ("shard", "shards", "names", "cases", "seconds")})
|
||||
cells.setdefault(cell, []).append((shard["shard"], shard["shards"]))
|
||||
for cell, numbers in cells.items():
|
||||
count = len(numbers)
|
||||
if sorted(numbers) != [(index, count) for index in range(1, count + 1)]:
|
||||
raise ValueError(f"shards of {cell} are not numbered 1..{count} of {count}: {numbers}")
|
||||
|
||||
|
||||
def shard_matrix(rows, hints, *, default_kind=HINT_KIND_MONOLITH, shards=None, shard_seconds=None):
|
||||
"""Regroup a per-case matrix into shards of one cell each.
|
||||
|
||||
Exactly one of `shards` (shards per cell, capped so that none holds less than
|
||||
MIN_SHARD_SECONDS of hinted replay) and `shard_seconds` (hinted replay per shard) sets the
|
||||
layout. Cases go longest first into the shard with the least hinted time so far, which is
|
||||
deterministic and keeps the shards of a cell even; inside a shard they run in matrix order.
|
||||
"""
|
||||
if (shards is None) == (shard_seconds is None):
|
||||
raise ValueError("give exactly one of shards and shard_seconds")
|
||||
if shards is not None and shards < 1:
|
||||
raise ValueError(f"shards per cell must be at least 1: {shards}")
|
||||
if shard_seconds is not None and shard_seconds < 1:
|
||||
raise ValueError(f"seconds per shard must be at least 1: {shard_seconds}")
|
||||
cells = {}
|
||||
for index, row in enumerate(rows):
|
||||
name = row_case_name(row)
|
||||
if not name or not set(name) <= SHARD_SAFE_NAME:
|
||||
raise ValueError(f"case name {name!r} cannot be passed on a command line; allowed "
|
||||
f"characters are letters, digits, '.', '_' and '-'")
|
||||
seconds = hint_seconds(hints, name, row_backend_name(row),
|
||||
row.get("transport", default_kind))
|
||||
cells.setdefault(canonical(row_cell(row)), []).append((index, row, seconds))
|
||||
out = []
|
||||
for members in cells.values():
|
||||
total = sum(seconds for _, _, seconds in members)
|
||||
if shards is not None:
|
||||
count = min(shards, int(total // MIN_SHARD_SECONDS))
|
||||
else:
|
||||
count = -(-int(total) // shard_seconds)
|
||||
count = max(1, min(count, len(members)))
|
||||
bins = [[] for _ in range(count)]
|
||||
loads = [0] * count
|
||||
for member in sorted(members, key=lambda item: (-item[2], item[0])):
|
||||
target = loads.index(min(loads))
|
||||
bins[target].append(member)
|
||||
loads[target] += member[2]
|
||||
for number, members_of_shard in enumerate(bins, start=1):
|
||||
members_of_shard.sort(key=lambda item: item[0])
|
||||
shard = {**row_cell(members_of_shard[0][1]), "shard": number, "shards": count,
|
||||
"names": [row_case_name(row) for _, row, _ in members_of_shard]}
|
||||
if isinstance(members_of_shard[0][1]["case"], dict):
|
||||
shard["cases"] = [row["case"] for _, row, _ in members_of_shard]
|
||||
shard["seconds"] = round(sum(seconds for _, _, seconds in members_of_shard))
|
||||
out.append(shard)
|
||||
check_shard_union(rows, out)
|
||||
return {"include": out}
|
||||
|
||||
|
||||
def per_case_matrix(format_name, cases, *, transports=SPLIT_TRANSPORTS, keep_inproc=(),
|
||||
android_smoke=None):
|
||||
"""The per-case matrix a shard format regroups, and the hint kind of its transport-less rows."""
|
||||
if format_name == "github-test-shard-matrix":
|
||||
return github_test_matrix(cases), HINT_KIND_MONOLITH
|
||||
if format_name == "github-test-transport-shard-matrix":
|
||||
return github_transport_matrix(cases), HINT_KIND_MONOLITH
|
||||
if format_name == "github-verify-shard-matrix":
|
||||
return github_verify_matrix(cases), HINT_KIND_VERIFY
|
||||
if format_name == "github-split-shard-matrix":
|
||||
return github_split_matrix(cases, transports, keep_inproc), HINT_KIND_MONOLITH
|
||||
if format_name == "github-apk-shard-matrix":
|
||||
return github_apk_matrix(cases), HINT_KIND_MONOLITH
|
||||
if format_name == "github-apk-transport-shard-matrix":
|
||||
return (github_transport_matrix(cases, apk=True, android_smoke=android_smoke),
|
||||
HINT_KIND_MONOLITH)
|
||||
raise ValueError(f"not a shard format: {format_name}")
|
||||
|
||||
|
||||
LINUX_SHARD_FORMATS = (
|
||||
"github-test-shard-matrix",
|
||||
"github-test-transport-shard-matrix",
|
||||
"github-verify-shard-matrix",
|
||||
"github-split-shard-matrix",
|
||||
)
|
||||
ANDROID_SHARD_FORMATS = (
|
||||
"github-apk-shard-matrix",
|
||||
"github-apk-transport-shard-matrix",
|
||||
)
|
||||
|
||||
|
||||
def github_shard_matrix(format_name, cases, *, shards=DEFAULT_SHARDS_PER_CELL,
|
||||
shard_seconds=DEFAULT_ANDROID_SHARD_SECONDS,
|
||||
transports=SPLIT_TRANSPORTS, keep_inproc=(), hints_path=TRACE_DURATION_HINTS_JSON,
|
||||
android_smoke=None):
|
||||
"""A shard format: the Linux ones lay out by shards per cell, the Android ones by seconds."""
|
||||
matrix, default_kind = per_case_matrix(format_name, cases, transports=transports,
|
||||
keep_inproc=keep_inproc, android_smoke=android_smoke)
|
||||
if format_name in ANDROID_SHARD_FORMATS:
|
||||
return shard_matrix(matrix["include"], load_duration_hints("android", hints_path),
|
||||
default_kind=default_kind, shard_seconds=shard_seconds)
|
||||
return shard_matrix(matrix["include"], load_duration_hints("linux", hints_path),
|
||||
default_kind=default_kind, shards=shards)
|
||||
|
||||
|
||||
def self_test_shard_matrices():
|
||||
"""Every shard format replays exactly the rows of the per-case matrix it replaces."""
|
||||
cases = ci_trace_cases(load_trace_cases())
|
||||
narrowed = {"transports": ("spawn",), "keep_inproc": ("OpenRA",)}
|
||||
layouts = [(name, {}, {"shards": count}) for name in LINUX_SHARD_FORMATS for count in (1, 4, 100)]
|
||||
layouts += [("github-split-shard-matrix", narrowed, {"shards": count}) for count in (1, 4, 100)]
|
||||
layouts += [(name, {}, {"shard_seconds": seconds}) for name in ANDROID_SHARD_FORMATS
|
||||
for seconds in (60, DEFAULT_ANDROID_SHARD_SECONDS, 10 ** 9)]
|
||||
layouts += [("github-apk-transport-shard-matrix", {"android_smoke": android_smoke_names()},
|
||||
{"shard_seconds": seconds}) for seconds in (60, DEFAULT_ANDROID_SHARD_SECONDS, 10 ** 9)]
|
||||
for name, selection, layout in layouts:
|
||||
matrix, _ = per_case_matrix(name, cases, **selection)
|
||||
rows = matrix["include"]
|
||||
built = github_shard_matrix(name, cases, **selection, **layout)["include"]
|
||||
again = github_shard_matrix(name, cases, **selection, **layout)["include"]
|
||||
assert built == again, f"{name} {layout}: the layout is not deterministic"
|
||||
# Compared here as well as inside shard_matrix, with nothing shared but the JSON form.
|
||||
replayed = sorted(
|
||||
canonical({**{key: value for key, value in shard.items()
|
||||
if key not in ("shard", "shards", "names", "cases", "seconds")},
|
||||
"case": case})
|
||||
for shard in built for case in shard.get("cases", shard["names"]))
|
||||
assert replayed == sorted(canonical(row) for row in rows), \
|
||||
f"{name} {layout}: the shards do not replay the per-case matrix"
|
||||
assert sum(len(shard["names"]) for shard in built) == len(rows)
|
||||
cells = {canonical(row_cell(row)) for row in rows}
|
||||
assert len({canonical({key: value for key, value in shard.items()
|
||||
if key not in ("shard", "shards", "names", "cases", "seconds")})
|
||||
for shard in built}) == len(cells), f"{name} {layout}: a shard spans two cells"
|
||||
if layout.get("shards") == 1 or layout.get("shard_seconds") == 10 ** 9:
|
||||
assert len(built) == len(cells), f"{name} {layout}: expected one shard per cell"
|
||||
seconds = [shard["seconds"] for shard in built]
|
||||
shown = {key: (f"{len(value)} smoke cases" if key == "android_smoke" else value)
|
||||
for key, value in selection.items()}
|
||||
print(f"shard matrix {name} {shown or ''} {layout}: {len(rows)} rows in {len(built)} "
|
||||
f"shards over {len(cells)} cells, {min(seconds)}-{max(seconds)} s hinted per shard")
|
||||
# The check itself must be able to fail: a dropped, a repeated and a moved case.
|
||||
if len(built) >= 2 and any(len(shard["names"]) >= 2 for shard in built):
|
||||
index = next(i for i, shard in enumerate(built) if len(shard["names"]) >= 2)
|
||||
for label, damage in (("dropped", lambda names: names[1:]),
|
||||
("repeated", lambda names: names + names[:1])):
|
||||
broken = [dict(shard) for shard in built]
|
||||
broken[index]["names"] = damage(list(broken[index]["names"]))
|
||||
if "cases" in broken[index]:
|
||||
broken[index]["cases"] = damage(list(broken[index]["cases"]))
|
||||
try:
|
||||
check_shard_union(rows, broken)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError(f"{name} {layout}: a {label} case passed the union check")
|
||||
other = next((i for i, shard in enumerate(built)
|
||||
if row_backend_name(shard) != row_backend_name(built[index])), None)
|
||||
if other is not None:
|
||||
broken = [dict(shard) for shard in built]
|
||||
for key in ("names", "cases"):
|
||||
if key in broken[index]:
|
||||
moved = list(broken[index][key])
|
||||
broken[other][key] = list(broken[other][key]) + moved[:1]
|
||||
broken[index][key] = moved[1:]
|
||||
try:
|
||||
check_shard_union(rows, broken)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError(f"{name} {layout}: a case moved to another backend's "
|
||||
f"shard passed the union check")
|
||||
# Hints only weigh the packing. With none at all, and with a case no hint names, the rows
|
||||
# replayed are the same.
|
||||
rows = github_transport_matrix(cases)["include"]
|
||||
for hints in ({}, {"no-such-case": {"DirectGLES": {"monolith": 5}}}):
|
||||
check_shard_union(rows, shard_matrix(rows, hints, shards=4)["include"])
|
||||
assert hint_seconds({}, "x", "DirectGLES", "monolith") == 30
|
||||
assert hint_seconds({"a": {"DirectGLES": {"monolith": 10}}, "b": {"DirectGLES": {"monolith": 20}},
|
||||
"c": {"DirectVulkan": {"monolith": 90}}}, "x", "DirectGLES", "monolith") == 15
|
||||
for bad in ({"shards": 0}, {"shard_seconds": 0}, {}, {"shards": 4, "shard_seconds": 60}):
|
||||
try:
|
||||
shard_matrix(rows, {}, **bad)
|
||||
except ValueError:
|
||||
continue
|
||||
raise AssertionError(f"shard_matrix accepted {bad}")
|
||||
try:
|
||||
shard_matrix([{"backend": "DirectGLES", "case": "bad name; rm"}], {}, shards=1)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("a case name with shell characters was accepted")
|
||||
# The verify lane's corrupted-snapshot control replays OpenRA on DirectGLES, so that row has
|
||||
# to be in the verify matrix for the control to have a shard to run in.
|
||||
verify_rows = github_verify_matrix(cases)["include"]
|
||||
assert {"backend": "DirectGLES", "case": "OpenRA"} in verify_rows, \
|
||||
"the verify matrix has no OpenRA DirectGLES row, so its negative control would not run"
|
||||
print("shard matrices: union, numbering, determinism and negative controls OK")
|
||||
|
||||
|
||||
def cmake_quote(value):
|
||||
return '"' + str(value).replace("\\", "/").replace('"', '\\"') + '"'
|
||||
|
||||
@@ -570,15 +944,27 @@ def parse_args():
|
||||
parser.add_argument("--self-test-transport-matrices", action="store_true")
|
||||
parser.add_argument("--self-test-split-matrix", action="store_true",
|
||||
help="prove the spawn-only split matrix drops only inproc-acceptance rows")
|
||||
parser.add_argument("--self-test-shard-matrices", action="store_true",
|
||||
help="prove every shard matrix replays exactly its per-case matrix")
|
||||
parser.add_argument("--fixture-root", default="tools/trace_replay/fixtures")
|
||||
parser.add_argument("--transports", default=",".join(SPLIT_TRANSPORTS),
|
||||
help="github-split-matrix: the comma-separated split transports to emit")
|
||||
parser.add_argument("--keep-inproc", default="",
|
||||
help="github-split-matrix: comma-separated split cases whose inproc rows stay")
|
||||
parser.add_argument("--shards", type=int, default=DEFAULT_SHARDS_PER_CELL,
|
||||
help="Linux shard formats: shards per (lane, transport, backend) cell")
|
||||
parser.add_argument("--shard-seconds", type=int, default=DEFAULT_ANDROID_SHARD_SECONDS,
|
||||
help="Android shard formats: hinted replay seconds one shard aims for")
|
||||
parser.add_argument("--android-lanes", choices=ANDROID_LANES, default="full",
|
||||
help="APK transport formats: inproc-full keeps every case on inproc and "
|
||||
"the manifest's android_smoke cases on monolith and spawn")
|
||||
parser.add_argument("--duration-hints", default=str(TRACE_DURATION_HINTS_JSON),
|
||||
help="shard formats: the JSON of replay seconds the packing weighs cases by")
|
||||
parser.add_argument(
|
||||
"--format",
|
||||
choices=(
|
||||
"names",
|
||||
"android-smoke-names",
|
||||
"github-test-matrix",
|
||||
"github-test-transport-matrix",
|
||||
"github-verify-matrix",
|
||||
@@ -586,7 +972,10 @@ def parse_args():
|
||||
"github-apk",
|
||||
"github-apk-matrix",
|
||||
"github-apk-transport-matrix",
|
||||
*LINUX_SHARD_FORMATS,
|
||||
*ANDROID_SHARD_FORMATS,
|
||||
"fixture-files",
|
||||
"fixture-manifest",
|
||||
"cmake",
|
||||
),
|
||||
default="names",
|
||||
@@ -602,11 +991,17 @@ def main():
|
||||
if args.self_test_split_matrix:
|
||||
self_test_split_matrix()
|
||||
return 0
|
||||
if args.self_test_shard_matrices:
|
||||
self_test_shard_matrices()
|
||||
return 0
|
||||
cases = load_trace_cases()
|
||||
if args.ci:
|
||||
cases = ci_trace_cases(cases)
|
||||
android_smoke = android_smoke_names() if args.android_lanes == "inproc-full" else None
|
||||
if args.format == "names":
|
||||
print(json.dumps([case["name"] for case in cases], separators=(",", ":")))
|
||||
elif args.format == "android-smoke-names":
|
||||
print(",".join(android_smoke_names()))
|
||||
elif args.format == "github-test-matrix":
|
||||
print(json.dumps(github_test_matrix(cases), separators=(",", ":")))
|
||||
elif args.format == "github-test-transport-matrix":
|
||||
@@ -622,7 +1017,22 @@ def main():
|
||||
elif args.format == "github-apk-matrix":
|
||||
print(json.dumps(github_apk_matrix(cases), separators=(",", ":")))
|
||||
elif args.format == "github-apk-transport-matrix":
|
||||
print(json.dumps(github_transport_matrix(cases, apk=True), separators=(",", ":")))
|
||||
print(json.dumps(github_transport_matrix(cases, apk=True, android_smoke=android_smoke),
|
||||
separators=(",", ":")))
|
||||
elif args.format in LINUX_SHARD_FORMATS + ANDROID_SHARD_FORMATS:
|
||||
transports = tuple(t for t in args.transports.split(",") if t)
|
||||
keep = tuple(c for c in args.keep_inproc.split(",") if c)
|
||||
print(json.dumps(
|
||||
github_shard_matrix(args.format, cases, shards=args.shards,
|
||||
shard_seconds=args.shard_seconds, transports=transports,
|
||||
keep_inproc=keep, hints_path=args.duration_hints,
|
||||
android_smoke=android_smoke),
|
||||
separators=(",", ":")))
|
||||
elif args.format == "fixture-manifest":
|
||||
# One "<case><TAB><file>" line per fixture file, for the fixture bundle script.
|
||||
for case in cases:
|
||||
for file in fixture_files(case, args.fixture_root):
|
||||
print(f"{case['name']}\t{file}")
|
||||
elif args.format == "fixture-files":
|
||||
if not args.case_name:
|
||||
print("--case is required for --format fixture-files", file=sys.stderr)
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
{
|
||||
"description": [
|
||||
"Replay seconds per case, backend and transport, used only to pack retrace shards evenly",
|
||||
"(tools/trace_replay/trace_cases.py, shard_matrix). A missing entry falls back to the median of",
|
||||
"its backend and transport, so a new case needs no entry here. Nothing asserts these values;",
|
||||
"a stale number only makes shards uneven. To refresh, copy the seconds column of the shard",
|
||||
"step summaries of a green run."
|
||||
],
|
||||
"linux": {
|
||||
"source": "Test run 43666299896 (2026-11-15), step 'Retrace and validate' of each one-case job, two gaps filled from run 43700779712; the verify key is the retrace-verify job",
|
||||
"seconds": {
|
||||
"OpenRA": {"DirectGLES": {"inproc": 3, "monolith": 3, "spawn": 3, "verify": 3}, "DirectVulkan": {"inproc": 3, "monolith": 3, "spawn": 2, "verify": 4}},
|
||||
"improved-transparency-minecraft-26.3": {"DirectGLES": {"inproc": 86, "monolith": 51, "spawn": 78, "verify": 81}, "DirectVulkan": {"inproc": 82, "monolith": 104, "spawn": 107, "verify": 85}},
|
||||
"minecraft-1.17-main-menu-854": {"DirectGLES": {"inproc": 4, "monolith": 6, "spawn": 3, "verify": 3}, "DirectVulkan": {"inproc": 5, "monolith": 5, "spawn": 3, "verify": 3}},
|
||||
"minecraft-1.20.1-fabric-iris-iterationt-in-world": {"DirectGLES": {"inproc": 59, "monolith": 70, "spawn": 72}, "DirectVulkan": {"inproc": 62, "monolith": 69, "spawn": 68}},
|
||||
"minecraft-1.21.1-neoforge-create-indirect-in-world-align1024": {"DirectGLES": {"inproc": 9, "monolith": 18, "spawn": 16}, "DirectVulkan": {"inproc": 13, "monolith": 13, "spawn": 8}},
|
||||
"minecraft-1.21.1-neoforge-create-instancing-in-world": {"DirectGLES": {"inproc": 12, "monolith": 14, "spawn": 14}, "DirectVulkan": {"inproc": 10, "monolith": 14, "spawn": 10}},
|
||||
"minecraft-1.21.11-main-menu": {"DirectGLES": {"inproc": 5, "monolith": 5, "spawn": 3, "verify": 6}, "DirectVulkan": {"inproc": 7, "monolith": 6, "spawn": 6, "verify": 5}},
|
||||
"minecraft-1.21.4-fabric-common-mods-in-world": {"DirectGLES": {"inproc": 13, "monolith": 15, "spawn": 15}, "DirectVulkan": {"inproc": 19, "monolith": 17, "spawn": 14}},
|
||||
"minecraft-1.21.4-fabric-common-mods-inventory": {"DirectGLES": {"inproc": 20, "monolith": 13, "spawn": 11}, "DirectVulkan": {"inproc": 22, "monolith": 21, "spawn": 22}},
|
||||
"minecraft-1.21.4-fabric-iris-bliss-in-world": {"DirectGLES": {"inproc": 11, "monolith": 23, "spawn": 16}, "DirectVulkan": {"inproc": 17, "monolith": 17, "spawn": 16}},
|
||||
"minecraft-1.21.4-fabric-iris-bsl-esc-menu-854": {"DirectGLES": {"inproc": 126, "monolith": 127, "spawn": 124}, "DirectVulkan": {"inproc": 130, "monolith": 131, "spawn": 134}},
|
||||
"minecraft-1.21.4-fabric-iris-bsl-in-world": {"DirectGLES": {"inproc": 15, "monolith": 18, "spawn": 17}, "DirectVulkan": {"inproc": 16, "monolith": 15, "spawn": 17}},
|
||||
"minecraft-1.21.4-fabric-iris-chocapic-v6-lite-in-world": {"DirectGLES": {"inproc": 10, "monolith": 15, "spawn": 14}, "DirectVulkan": {"inproc": 14, "monolith": 13, "spawn": 15}},
|
||||
"minecraft-1.21.4-fabric-iris-complementary-reimagined-in-world": {"DirectGLES": {"inproc": 27, "monolith": 35, "spawn": 29}, "DirectVulkan": {"inproc": 41, "monolith": 31, "spawn": 28}},
|
||||
"minecraft-1.21.4-fabric-iris-complementary-unbound-in-world": {"DirectGLES": {"inproc": 41, "monolith": 33, "spawn": 38}, "DirectVulkan": {"inproc": 26, "monolith": 31, "spawn": 25}},
|
||||
"minecraft-1.21.4-fabric-iris-derivative-main-d24.4.14-in-world": {"DirectGLES": {"inproc": 23, "monolith": 33, "spawn": 24}, "DirectVulkan": {"inproc": 20, "monolith": 26, "spawn": 27}},
|
||||
"minecraft-1.21.4-fabric-iris-iterationrp-in-world": {"DirectVulkan": {"inproc": 42, "monolith": 42, "spawn": 41}},
|
||||
"minecraft-1.21.4-fabric-iris-iterationt-in-world": {"DirectGLES": {"inproc": 26, "monolith": 27, "spawn": 23}, "DirectVulkan": {"inproc": 23, "monolith": 20, "spawn": 19}},
|
||||
"minecraft-1.21.4-fabric-iris-iterationt-nodsa-in-world": {"DirectGLES": {"inproc": 24, "monolith": 19, "spawn": 19}, "DirectVulkan": {"inproc": 19, "monolith": 13, "spawn": 21}},
|
||||
"minecraft-1.21.4-fabric-iris-makeup-ultrafast-in-world": {"DirectGLES": {"inproc": 13, "monolith": 9, "spawn": 10}, "DirectVulkan": {"inproc": 11, "monolith": 8, "spawn": 8}},
|
||||
"minecraft-1.21.4-fabric-iris-mellow-in-world": {"DirectGLES": {"inproc": 7, "monolith": 11, "spawn": 9}, "DirectVulkan": {"inproc": 9, "monolith": 9, "spawn": 7}},
|
||||
"minecraft-1.21.4-fabric-iris-nostalgia-in-world": {"DirectGLES": {"inproc": 25, "monolith": 21, "spawn": 27}, "DirectVulkan": {"inproc": 24, "monolith": 26, "spawn": 16}},
|
||||
"minecraft-1.21.4-fabric-iris-photon-v1.1-in-world": {"DirectGLES": {"inproc": 25, "monolith": 34, "spawn": 21}, "DirectVulkan": {"inproc": 24, "monolith": 24, "spawn": 33}},
|
||||
"minecraft-1.21.4-fabric-iris-photon-v1.3b-in-world": {"DirectGLES": {"inproc": 29, "monolith": 36, "spawn": 29}, "DirectVulkan": {"inproc": 30, "monolith": 27, "spawn": 36}},
|
||||
"minecraft-1.21.4-fabric-iris-sundial-lite-in-world": {"DirectGLES": {"inproc": 35, "monolith": 27, "spawn": 21}, "DirectVulkan": {"inproc": 31, "monolith": 20, "spawn": 30}},
|
||||
"minecraft-1.21.4-fabric-iris-super-duper-vanilla-in-world": {"DirectGLES": {"monolith": 22, "spawn": 21}, "DirectVulkan": {"inproc": 14, "monolith": 20, "spawn": 14}},
|
||||
"minecraft-1.21.4-fabric-journeymap-in-world": {"DirectGLES": {"inproc": 43, "monolith": 66, "spawn": 65}, "DirectVulkan": {"inproc": 41, "monolith": 60, "spawn": 90}},
|
||||
"minecraft-1.21.4-fabric-journeymap-in-world-normal-world": {"DirectGLES": {"inproc": 10, "monolith": 12, "spawn": 14}, "DirectVulkan": {"inproc": 15, "monolith": 12, "spawn": 14}},
|
||||
"minecraft-1.21.4-fabric-modernui-inventory": {"DirectGLES": {"inproc": 54, "monolith": 50, "spawn": 67}, "DirectVulkan": {"inproc": 50, "monolith": 66, "spawn": 70}},
|
||||
"minecraft-1.21.4-fabric-modernui-inventory-normal-world": {"DirectGLES": {"inproc": 15, "monolith": 17, "spawn": 9}, "DirectVulkan": {"inproc": 18, "monolith": 13, "spawn": 10}},
|
||||
"minecraft-1.21.4-fabric-rei-inventory": {"DirectGLES": {"inproc": 72, "monolith": 63, "spawn": 90}, "DirectVulkan": {"inproc": 79, "monolith": 84, "spawn": 49}},
|
||||
"minecraft-1.21.4-fabric-rei-inventory-normal-world": {"DirectGLES": {"inproc": 15, "monolith": 22, "spawn": 17}, "DirectVulkan": {"inproc": 20, "monolith": 18, "spawn": 19}},
|
||||
"minecraft-1.21.4-fabric-sodium-in-world": {"DirectGLES": {"inproc": 19, "monolith": 15, "spawn": 26, "verify": 25}, "DirectVulkan": {"inproc": 29, "monolith": 32, "spawn": 23, "verify": 16}},
|
||||
"minecraft-1.21.4-fabric-xaero-minimap-in-world": {"DirectGLES": {"inproc": 41, "monolith": 51, "spawn": 64}, "DirectVulkan": {"inproc": 55, "monolith": 69, "spawn": 77}},
|
||||
"minecraft-1.21.4-fabric-xaero-minimap-in-world-normal-world": {"DirectGLES": {"inproc": 12, "monolith": 13, "spawn": 8}, "DirectVulkan": {"inproc": 9, "monolith": 10, "spawn": 13}},
|
||||
"minecraft-1.21.4-fabric-xaero-world-map-in-world": {"DirectGLES": {"inproc": 41, "monolith": 40, "spawn": 52}, "DirectVulkan": {"inproc": 42, "monolith": 43, "spawn": 59}},
|
||||
"minecraft-1.21.4-fabric-xaero-world-map-in-world-normal-world": {"DirectGLES": {"inproc": 6, "monolith": 11, "spawn": 8}, "DirectVulkan": {"inproc": 13, "monolith": 9, "spawn": 10}},
|
||||
"minecraft-1.21.4-in-world": {"DirectGLES": {"inproc": 17, "monolith": 10, "spawn": 11, "verify": 11}, "DirectVulkan": {"inproc": 10, "monolith": 9, "spawn": 12, "verify": 11}},
|
||||
"minecraft-1.21.4-main-menu": {"DirectGLES": {"inproc": 7, "monolith": 4, "spawn": 9, "verify": 6}, "DirectVulkan": {"inproc": 5, "monolith": 10, "spawn": 6, "verify": 6}},
|
||||
"minecraft-1.21.4-startup": {"DirectGLES": {"inproc": 5, "monolith": 3, "spawn": 1, "verify": 3}, "DirectVulkan": {"inproc": 1, "monolith": 4, "spawn": 2, "verify": 2}}
|
||||
}
|
||||
},
|
||||
"android": {
|
||||
"source": "MobileGL APK run 43666299861 (2026-11-15), step 'Retrace and validate' of each one-case emulator job, APK install included",
|
||||
"seconds": {
|
||||
"OpenRA": {"DirectGLES": {"inproc": 28, "monolith": 28, "spawn": 24}, "DirectVulkan": {"inproc": 24, "monolith": 22, "spawn": 16}},
|
||||
"improved-transparency-minecraft-26.3": {"DirectGLES": {"inproc": 179, "monolith": 170, "spawn": 142}, "DirectVulkan": {"inproc": 169, "monolith": 209, "spawn": 153}},
|
||||
"minecraft-1.17-main-menu-854": {"DirectGLES": {"inproc": 27, "monolith": 19, "spawn": 22}, "DirectVulkan": {"inproc": 25, "monolith": 25, "spawn": 26}},
|
||||
"minecraft-1.20.1-fabric-iris-iterationt-in-world": {"DirectGLES": {"inproc": 131, "monolith": 147, "spawn": 153}, "DirectVulkan": {"inproc": 149, "monolith": 111, "spawn": 116}},
|
||||
"minecraft-1.21.1-neoforge-create-indirect-in-world-align1024": {"DirectGLES": {"inproc": 74, "monolith": 85, "spawn": 73}, "DirectVulkan": {"inproc": 44, "monolith": 31, "spawn": 53}},
|
||||
"minecraft-1.21.1-neoforge-create-instancing-in-world": {"DirectGLES": {"inproc": 74, "monolith": 56, "spawn": 78}, "DirectVulkan": {"inproc": 42, "monolith": 49, "spawn": 37}},
|
||||
"minecraft-1.21.11-main-menu": {"DirectGLES": {"inproc": 38, "monolith": 37, "spawn": 30}, "DirectVulkan": {"inproc": 46, "monolith": 25, "spawn": 45}},
|
||||
"minecraft-1.21.4-fabric-common-mods-in-world": {"DirectGLES": {"inproc": 66, "monolith": 51, "spawn": 61}, "DirectVulkan": {"inproc": 70, "monolith": 62, "spawn": 53}},
|
||||
"minecraft-1.21.4-fabric-common-mods-inventory": {"DirectGLES": {"inproc": 74, "monolith": 86, "spawn": 41}, "DirectVulkan": {"inproc": 61, "monolith": 55, "spawn": 67}},
|
||||
"minecraft-1.21.4-fabric-iris-bliss-in-world": {"DirectGLES": {"inproc": 74, "monolith": 70, "spawn": 53}, "DirectVulkan": {"inproc": 53, "monolith": 69, "spawn": 50}},
|
||||
"minecraft-1.21.4-fabric-iris-bsl-esc-menu-854": {"DirectGLES": {"inproc": 220, "monolith": 171, "spawn": 176}, "DirectVulkan": {"inproc": 138, "monolith": 234, "spawn": 193}},
|
||||
"minecraft-1.21.4-fabric-iris-bsl-in-world": {"DirectGLES": {"inproc": 60, "monolith": 42, "spawn": 48}, "DirectVulkan": {"inproc": 56, "monolith": 48, "spawn": 60}},
|
||||
"minecraft-1.21.4-fabric-iris-chocapic-v6-lite-in-world": {"DirectGLES": {"inproc": 62, "monolith": 54, "spawn": 50}, "DirectVulkan": {"inproc": 51, "monolith": 42, "spawn": 36}},
|
||||
"minecraft-1.21.4-fabric-iris-complementary-reimagined-in-world": {"DirectGLES": {"inproc": 93, "monolith": 94, "spawn": 110}, "DirectVulkan": {"inproc": 92, "monolith": 112, "spawn": 88}},
|
||||
"minecraft-1.21.4-fabric-iris-complementary-unbound-in-world": {"DirectGLES": {"inproc": 99, "monolith": 97, "spawn": 99}, "DirectVulkan": {"inproc": 98, "monolith": 107, "spawn": 115}},
|
||||
"minecraft-1.21.4-fabric-iris-derivative-main-d24.4.14-in-world": {"DirectGLES": {"inproc": 67, "monolith": 81, "spawn": 60}, "DirectVulkan": {"inproc": 77, "monolith": 76, "spawn": 47}},
|
||||
"minecraft-1.21.4-fabric-iris-iterationrp-in-world": {"DirectVulkan": {"inproc": 102, "monolith": 108, "spawn": 94}},
|
||||
"minecraft-1.21.4-fabric-iris-iterationt-in-world": {"DirectGLES": {"inproc": 40, "monolith": 57, "spawn": 48}, "DirectVulkan": {"inproc": 66, "monolith": 56, "spawn": 61}},
|
||||
"minecraft-1.21.4-fabric-iris-iterationt-nodsa-in-world": {"DirectGLES": {"inproc": 68, "monolith": 65, "spawn": 39}, "DirectVulkan": {"inproc": 43, "monolith": 70, "spawn": 66}},
|
||||
"minecraft-1.21.4-fabric-iris-makeup-ultrafast-in-world": {"DirectGLES": {"inproc": 32, "monolith": 47, "spawn": 46}, "DirectVulkan": {"inproc": 39, "monolith": 42, "spawn": 43}},
|
||||
"minecraft-1.21.4-fabric-iris-mellow-in-world": {"DirectGLES": {"inproc": 47, "monolith": 37, "spawn": 42}, "DirectVulkan": {"inproc": 46, "monolith": 40, "spawn": 40}},
|
||||
"minecraft-1.21.4-fabric-iris-nostalgia-in-world": {"DirectGLES": {"inproc": 92, "monolith": 51, "spawn": 87}, "DirectVulkan": {"inproc": 60, "monolith": 55, "spawn": 83}},
|
||||
"minecraft-1.21.4-fabric-iris-photon-v1.1-in-world": {"DirectGLES": {"inproc": 107, "monolith": 105, "spawn": 79}, "DirectVulkan": {"inproc": 59, "monolith": 100, "spawn": 96}},
|
||||
"minecraft-1.21.4-fabric-iris-photon-v1.3b-in-world": {"DirectGLES": {"inproc": 127, "monolith": 114, "spawn": 60}, "DirectVulkan": {"inproc": 94, "monolith": 106, "spawn": 81}},
|
||||
"minecraft-1.21.4-fabric-iris-sundial-lite-in-world": {"DirectGLES": {"inproc": 80, "monolith": 77, "spawn": 77}, "DirectVulkan": {"inproc": 76, "monolith": 77, "spawn": 91}},
|
||||
"minecraft-1.21.4-fabric-iris-super-duper-vanilla-in-world": {"DirectGLES": {"inproc": 71, "monolith": 63, "spawn": 48}, "DirectVulkan": {"inproc": 53, "monolith": 67, "spawn": 49}},
|
||||
"minecraft-1.21.4-fabric-journeymap-in-world": {"DirectGLES": {"inproc": 139, "monolith": 241, "spawn": 172}, "DirectVulkan": {"inproc": 180, "monolith": 257}},
|
||||
"minecraft-1.21.4-fabric-journeymap-in-world-normal-world": {"DirectGLES": {"inproc": 56, "monolith": 37, "spawn": 67}, "DirectVulkan": {"inproc": 51, "monolith": 53, "spawn": 54}},
|
||||
"minecraft-1.21.4-fabric-modernui-inventory": {"DirectGLES": {"inproc": 207, "monolith": 248, "spawn": 105}, "DirectVulkan": {"inproc": 303, "monolith": 324, "spawn": 195}},
|
||||
"minecraft-1.21.4-fabric-modernui-inventory-normal-world": {"DirectGLES": {"inproc": 56, "monolith": 58, "spawn": 53}, "DirectVulkan": {"inproc": 60, "monolith": 58, "spawn": 81}},
|
||||
"minecraft-1.21.4-fabric-rei-inventory": {"DirectGLES": {"inproc": 328, "monolith": 337, "spawn": 144}, "DirectVulkan": {"inproc": 325, "monolith": 197, "spawn": 199}},
|
||||
"minecraft-1.21.4-fabric-rei-inventory-normal-world": {"DirectGLES": {"inproc": 79, "monolith": 62, "spawn": 46}, "DirectVulkan": {"inproc": 82, "monolith": 51, "spawn": 66}},
|
||||
"minecraft-1.21.4-fabric-sodium-in-world": {"DirectGLES": {"inproc": 178, "monolith": 175, "spawn": 91}, "DirectVulkan": {"inproc": 166, "monolith": 136, "spawn": 72}},
|
||||
"minecraft-1.21.4-fabric-xaero-minimap-in-world": {"DirectGLES": {"inproc": 231, "monolith": 254, "spawn": 99}, "DirectVulkan": {"inproc": 278, "monolith": 225, "spawn": 127}},
|
||||
"minecraft-1.21.4-fabric-xaero-minimap-in-world-normal-world": {"DirectGLES": {"inproc": 61, "monolith": 47, "spawn": 46}, "DirectVulkan": {"inproc": 61, "monolith": 44, "spawn": 42}},
|
||||
"minecraft-1.21.4-fabric-xaero-world-map-in-world": {"DirectGLES": {"inproc": 313, "monolith": 223, "spawn": 104}, "DirectVulkan": {"inproc": 276, "monolith": 277, "spawn": 121}},
|
||||
"minecraft-1.21.4-fabric-xaero-world-map-in-world-normal-world": {"DirectGLES": {"inproc": 62, "monolith": 47, "spawn": 41}, "DirectVulkan": {"inproc": 52, "monolith": 40, "spawn": 69}},
|
||||
"minecraft-1.21.4-in-world": {"DirectGLES": {"inproc": 44, "monolith": 59, "spawn": 43}, "DirectVulkan": {"inproc": 53, "monolith": 59, "spawn": 51}},
|
||||
"minecraft-1.21.4-main-menu": {"DirectGLES": {"inproc": 54, "monolith": 48, "spawn": 35}, "DirectVulkan": {"inproc": 58, "monolith": 67, "spawn": 46}},
|
||||
"minecraft-1.21.4-startup": {"DirectGLES": {"inproc": 22, "monolith": 19, "spawn": 19}, "DirectVulkan": {"inproc": 18, "monolith": 14, "spawn": 14}}
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -68,7 +68,7 @@ frame. It exists for sharing one GPU-owning server between processes, sandboxes
|
||||
|
||||
- `GL_RENDERER` starts with `Espryt (MobileGL Core)` or `Magma (MobileGL Core)`.
|
||||
- `GL_VENDOR` contains `MobileGL-Dev`.
|
||||
- `GL_VERSION` looks like `4.6 MobileGL 26.10.0-dev..., Direct (GLES) Backend, GIT@<hash>` (or
|
||||
- `GL_VERSION` looks like `4.6.0 MobileGL 26.11-dev, Direct (GLES) Backend, GIT@<hash>` (or
|
||||
`Direct (Vulkan)`).
|
||||
- In Minecraft, the F3 screen shows these strings.
|
||||
- The log says `Config: Active backend type set to DirectGLES` (or `DirectVulkan`). Logs go to
|
||||
|
||||
@@ -75,7 +75,7 @@ latency.
|
||||
The plugin's launcher environment includes `MOBILEGL_TRANSPORT=inproc` **only if** the APK was built
|
||||
with `-Pmobilegl.buildDisaggregated=ON` or `-Pmobilegl.buildDisaggregatedInproc=ON` passed explicitly
|
||||
(or the matching `MOBILEGL_BUILD_DISAGGREGATED` / `MOBILEGL_BUILD_DISAGGREGATED_INPROC` environment
|
||||
variable set to `ON`; the CI build of `feat/disaggregated` does this). A plain `gradle :app:assemblePluginRelease`
|
||||
variable set to `ON`; the CI builds of `dev` and `feat/disaggregated` do this, so their APKs default to `inproc` and carry the render server). A plain `gradle :app:assemblePluginRelease`
|
||||
still builds the split layer into the library, but the launcher environment then omits the transport
|
||||
and the plugin runs monolith. Either rebuild with the property, or add
|
||||
`MOBILEGL_TRANSPORT=inproc` in the launcher's environment editor.
|
||||
|
||||
Reference in New Issue
Block a user