swung0x48/MobileGL
1
0
Fork 0
mirror of https://github.com/MobileGL-Dev/MobileGL synced 2026-11-07 01:53:24 +09:00
Code Issues Packages Projects Releases Wiki Activity
Files
4eee6ae251ca295850ff5034bac3e82728cfa36f
MobileGL/.github/workflows/test.yml
T
Workflow config file is invalid. Please check your config file: interpolate runs-on for job "build-linux": Cannot parse non-string type invalid as JSON

3686 lines
192 KiB
YAML
Raw Blame History

name: Test
on:
push:
branches:
- dev
- Feat/Backend-Direct-GLES
- Feat/Backend-Direct-Vulkan
# TEMPORARY, remove before merging the MGPipe work into dev: the disaggregation
# branch runs the full lane on every push so a phase's landing is not gated on
# someone remembering to dispatch the workflow by hand.
- feat/disaggregated
- codex/fix-github-ci
workflow_dispatch:
env:
# Software-rendered CI submissions may exceed the interactive GPU hang budget.
MOBILEGL_GPU_HANG_BUDGET_MS: "0"
# ONE LIVE RUN PER BRANCH. The disaggregation branch pushes (often docs-only) commits minutes apart
# and each push fans out to ~330 retrace legs here and ~275 jobs in apk.yml; with no group, runs
# sat queued for 10+ hours behind their own successors. A manual dispatch gets a group of its own,
# so a push never cancels a manually started run; dev queues rather
# than cancels. No paths-ignore here, unlike apk.yml: this workflow hosts the doc citation lint.
concurrency:
group: ${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.ref }}
cancel-in-progress: ${{ github.event_name == 'push' && github.ref != 'refs/heads/dev' }}
jobs:
select-runner:
name: Select Test runner
runs-on: ubuntu-latest
permissions:
contents: read
outputs:
runner: ${{ steps.select.outputs.runner }}
pool: ${{ steps.select.outputs.pool }}
steps:
- uses: actions/checkout@v6
- name: Test runner selection
run: python3 scripts/ci/test_select_test_runner.py
- name: Check preferred runner availability
id: select
env:
RUNNER_STATUS_TOKEN: ${{ secrets.RUNNER_STATUS_TOKEN }}
run: python3 scripts/ci/select_test_runner.py
build-linux:
needs: select-runner
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
permissions:
actions: write
contents: read
env:
BUILD_DIR: build-linux
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci') && 'ON' || 'OFF' }}
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_COMPRESS: "true"
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 4G
CCACHE_NOHASHDIR: "true"
steps:
- name: Set swap space
if: runner.environment != 'self-hosted'
uses: pierotofy/set-swap-space@v1.0
with:
swap-size-gb: 32
- name: Checkout repo
uses: actions/checkout@v6
with:
submodules: recursive
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
if: runner.environment != 'self-hosted'
uses: lukka/get-cmake@v4.3.3
- name: Restore ccache
uses: actions/cache/restore@v5
with:
path: .ccache
key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1
restore-keys: |
${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-
- name: Prepare Vulkan SDK
if: runner.environment != 'self-hosted'
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Update glslang external sources
working-directory: 3rdparty/glslang
run: python update_glslang_sources.py
- name: Install build dependencies
if: runner.environment != 'self-hosted'
run: |
sudo apt-get update
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
- name: Show installed toolchain
run: |
ccache --version
"${MOBILEGL_CI_CC}" --version
"${MOBILEGL_CI_CXX}" --version
"${MOBILEGL_CI_LLD}" --version
dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true
- name: Configure CMake
run: |
if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
BUILD_TYPE=Debug
else
BUILD_TYPE=Release
fi
cmake -S . -B "${BUILD_DIR}" -G Ninja \
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \
-DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
-DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
-DMOBILEGL_BUILD_TEST=ON \
-DMOBILEGL_BUILD_BENCHMARK=ON \
-DMOBILEGL_BUILD_INTEGRATION_TEST=ON \
-DMOBILEGL_ITEST_VK_ICD="${VK_ICD_FILENAMES}" \
-DMOBILEGL_BUILD_TRACE_REPLAY=OFF \
-DMOBILEGL_BUILD_DISAGGREGATED="${MOBILEGL_CI_DISAGGREGATED}" \
-DMOBILEGL_BUILD_DISAGGREGATED_INPROC="${MOBILEGL_CI_DISAGGREGATED}" \
-DMOBILEGL_PIPE_PUSH="${MOBILEGL_CI_DISAGGREGATED}" \
-DMOBILEGL_PIPE_VERIFY=OFF \
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON \
-DBENCHMARK_ENABLE_TESTING=OFF \
-DCMAKE_POLICY_VERSION_MINIMUM=3.5
- name: Build
run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)"
- name: Primary runtime mode proof
run: |
python3 scripts/ci/runtime_mode_proof.py self-test
mode=monolith
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then mode=disaggregated; fi
python3 scripts/ci/runtime_mode_proof.py build --build-dir "${BUILD_DIR}" \
--mode "${mode}" --output "${BUILD_DIR}/runtime-mode.json"
- name: Show ccache stats
if: ${{ always() && needs.select-runner.result == 'success' }}
run: ccache --show-stats
# Rewrite one rolling entry per job on the default branch. The upload stays
# cumulative - it carries every object restored at the top of this run plus
# the few TUs that actually changed - but Actions cache keys are immutable,
# so the superseded blob has to be released before the same key can be
# re-uploaded. Running after the build means a failed build leaves the
# existing entry untouched. The other trigger branches restore this entry
# rather than each writing one of their own.
- name: Release superseded ccache entry
if: github.ref_name == github.event.repository.default_branch
env:
GH_TOKEN: ${{ github.token }}
CACHE_KEY: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1
run: gh cache delete "${CACHE_KEY}" || true
- name: Save ccache
if: github.ref_name == github.event.repository.default_branch
continue-on-error: true
uses: actions/cache/save@v5
with:
path: .ccache
key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1
- name: Package Linux runtime
run: |
mkdir -p ci-artifacts
mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort)
tar \
--exclude='*/CMakeFiles' \
--exclude='*.o' \
--exclude='*.a' \
--exclude='*.ninja*' \
--exclude='build.ninja' \
--exclude='cmake_install.cmake' \
-czf ci-artifacts/mobilegl-linux-runtime.tgz \
"${BUILD_DIR}/runtime-mode.json" \
"${BUILD_DIR}/CTestTestfile.cmake" \
"${BUILD_DIR}/MobileGL/MG_Test" \
"${BUILD_DIR}/MobileGL/MG_Benchmark" \
"${BUILD_DIR}/MobileGL/MG_IntegrationTest" \
"${SHARED_LIBS[@]}"
- name: Upload Linux runtime
uses: actions/upload-artifact@v7
with:
name: mobilegl-linux-runtime
path: ci-artifacts/mobilegl-linux-runtime.tgz
if-no-files-found: error
# The pull-library negative control must not consume the primary artifact:
# on disaggregation branches that artifact now contains the real transport.
build-linux-monolith-control:
name: build explicit pull runtime control
needs: select-runner
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
env:
BUILD_DIR: build-monolith-control
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_COMPRESS: "true"
CCACHE_NOHASHDIR: "true"
steps:
- name: Set swap space
if: runner.environment != 'self-hosted'
uses: pierotofy/set-swap-space@v1.0
with:
swap-size-gb: 32
- name: Checkout repo
uses: actions/checkout@v6
with:
submodules: recursive
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
if: runner.environment != 'self-hosted'
uses: lukka/get-cmake@v4.3.3
- name: Restore pull compiler cache
uses: actions/cache/restore@v5
with:
path: .ccache
key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-build-linux-ccache-v1
- name: Prepare Vulkan SDK
if: runner.environment != 'self-hosted'
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Update glslang external sources
working-directory: 3rdparty/glslang
run: python update_glslang_sources.py
- name: Install build dependencies
if: runner.environment != 'self-hosted'
run: |
sudo apt-get update
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
- name: Configure explicit pull control
run: |
cmake -S . -B "${BUILD_DIR}" -G Ninja \
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \
-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_BUILD_TYPE=Release -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
-DMOBILEGL_BUILD_TEST=OFF -DMOBILEGL_BUILD_BENCHMARK=OFF \
-DMOBILEGL_BUILD_INTEGRATION_TEST=OFF -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \
-DMOBILEGL_BUILD_DISAGGREGATED=OFF -DMOBILEGL_BUILD_DISAGGREGATED_INPROC=OFF \
-DMOBILEGL_PIPE_PUSH=OFF -DMOBILEGL_PIPE_VERIFY=OFF \
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5
- name: Build and identify pull control
run: |
cmake --build "${BUILD_DIR}" --target MobileGL --parallel "$(nproc)"
python3 scripts/ci/runtime_mode_proof.py build --build-dir "${BUILD_DIR}" \
--mode monolith --output "${BUILD_DIR}/runtime-mode.json"
- name: Package pull control
run: |
tar -czf mobilegl-linux-monolith-control.tgz \
"${BUILD_DIR}/libMobileGL.so" "${BUILD_DIR}/runtime-mode.json"
- name: Upload pull control
uses: actions/upload-artifact@v7
with:
name: mobilegl-linux-monolith-control
path: mobilegl-linux-monolith-control.tgz
if-no-files-found: error
test:
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
needs: [select-runner, build-linux]
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
if: runner.environment != 'self-hosted'
uses: lukka/get-cmake@v4.3.3
- name: Install runtime dependencies
if: runner.environment != 'self-hosted'
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1 libgles2 libgl1-mesa-dri mesa-vulkan-drivers
- name: Download Linux runtime
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-runtime
path: .
- name: Unpack Linux runtime
run: tar -xzf mobilegl-linux-runtime.tgz
- name: Normalize CTest command paths
run: |
python - <<'PY'
from pathlib import Path
import re
for path in Path('build-linux').rglob('CTestTestfile.cmake'):
text = path.read_text()
text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text)
path.write_text(text)
PY
- name: Test
working-directory: build-linux
run: |
ulimit -c unlimited
if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi
if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
ctest -V -L unit --no-tests=error
else
ctest --output-on-failure -L unit --no-tests=error
fi
- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v7
with:
name: unit-core-dumps
path: /tmp/core.*
if-no-files-found: ignore
integration:
name: integration (${{ matrix.transport }})
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
needs: [select-runner, build-linux]
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON((github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci') && '[{"lane":"monolith-control","transport":"monolith"},{"lane":"inproc-acceptance","transport":"inproc"}]' || '[{"lane":"monolith-control","transport":"monolith"}]') }}
env:
MOBILEGL_TRANSPORT: ${{ matrix.transport }}
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
MOBILEGL_IPC_RUN_AHEAD: "1"
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
if: runner.environment != 'self-hosted'
uses: lukka/get-cmake@v4.3.3
- name: Install runtime dependencies
if: runner.environment != 'self-hosted'
# Same set as the benchmark job, for the same reason: the scenarios bring
# up real headless EGL (llvmpipe) and Vulkan (lavapipe) contexts, and
# libegl-mesa0 - the EGL vendor library behind glvnd's libegl1 dispatch -
# only arrives as a Recommends.
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers
- name: Download Linux runtime
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-runtime
path: .
- name: Unpack Linux runtime
run: tar -xzf mobilegl-linux-runtime.tgz
- name: Normalize CTest command paths
run: |
python - <<'PY'
from pathlib import Path
import re
for path in Path('build-linux').rglob('CTestTestfile.cmake'):
text = path.read_text()
text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text)
path.write_text(text)
PY
- name: Integration scenarios
working-directory: build-linux
# REQUIRE_GPU makes a driverless runner FAIL instead of skipping every
# scenario - an all-skip run is otherwise indistinguishable from a pass,
# which is how a five-month-old draw-dropping bug survived unseen until
# this lane existed.
#
# Configure CMake with the selected runner's ICD; each test's ENVIRONMENT
# property takes precedence over ambient job variables.
#
# Cores are armed so that any crash - the harness pre-flight child's
# included - leaves /tmp/core.*, which the failure-only step below ships
# as an artifact. Analyzing a downloaded core against the runtime
# artifact's binary in an ubuntu-24.04 userspace reproduces the exact
# crash stack without burning a CI round on an in-workflow debugger.
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH: "1"
MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS: "1"
MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER: "1"
run: |
ulimit -c unlimited
if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi
# Second, filtered pass: with the range-invalidating map flush disabled,
# the buffer scenarios run on the upload ring's staged-copy tier - which
# the default pass never reaches (the map tier absorbs every flush on
# Mesa), so without this the Mali fallback tier would have zero CI
# coverage. The flag is NOT baked into the ctest ENVIRONMENT properties,
# so an inline env reaches the test processes (unlike the ICD pin above).
if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
ctest -V -L integration-gpu --no-tests=error --output-junit "${RUNNER_TEMP}/integration.xml"
MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 ctest -V -L integration-gpu \
-R 'Buffer|Readback|Atomic|Ssbo|Arena' --no-tests=error --output-junit "${RUNNER_TEMP}/integration-staged-copy.xml"
else
ctest --output-on-failure -L integration-gpu --no-tests=error --output-junit "${RUNNER_TEMP}/integration.xml"
MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 ctest --output-on-failure -L integration-gpu \
-R 'Buffer|Readback|Atomic|Ssbo|Arena' --no-tests=error --output-junit "${RUNNER_TEMP}/integration-staged-copy.xml"
fi
# Both backend roles must execute inproc in the primary artifact.
- name: Inproc execution proof (both backends)
if: matrix.transport == 'inproc'
working-directory: build-linux
run: |
ctest -L integration-p5f-rsp --show-only=json-v1 > "${RUNNER_TEMP}/primary-inproc-lane.json"
ctest -L integration-p5f-rsp --no-tests=error --output-on-failure \
--output-junit "${RUNNER_TEMP}/primary-inproc.xml"
python3 ../scripts/ci/runtime_mode_proof.py logs \
--discovery "${RUNNER_TEMP}/primary-inproc-lane.json" \
--junit "${RUNNER_TEMP}/primary-inproc.xml" --output "${RUNNER_TEMP}/primary-inproc-proof.json"
# The run log needs a signed-in reader; annotations do not. Name every failed ctest entry,
# with the lines of its output that say why, where anyone reading the run can see them.
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}"
- name: Upload integration lane evidence
if: ${{ always() && needs.select-runner.result == 'success' }}
uses: actions/upload-artifact@v7
with:
name: integration-evidence-${{ matrix.lane }}-${{ matrix.transport }}
path: |
${{ runner.temp }}/integration*.xml
${{ runner.temp }}/primary-inproc*
build-linux/runtime-mode.json
build-linux/MobileGL/MG_IntegrationTest/p5f-rsp-*.log
if-no-files-found: warn
- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v7
with:
name: integration-core-dumps-${{ matrix.lane }}-${{ matrix.transport }}
path: /tmp/core.*
if-no-files-found: ignore
# THE THIRD CI MODE (ARCHITECTURE.md 13.2-(2)): the same library, built with the PipeInputs
# comparator compiled in, running the integration suite and a trace subset with two state models
# in one address space. It is a second build rather than a flag on the first because
# MOBILEGL_PIPE_VERIFY is a compile-time option - the snapshot, the entry compare and the
# compare-at-read hook do not exist in the shipped library, and are never meant to.
build-linux-verify:
needs: select-runner
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
timeout-minutes: 120
permissions:
actions: write
contents: read
env:
BUILD_DIR: build-verify
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_COMPRESS: "true"
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 4G
CCACHE_NOHASHDIR: "true"
steps:
- name: Set swap space
if: runner.environment != 'self-hosted'
uses: pierotofy/set-swap-space@v1.0
with:
swap-size-gb: 32
- name: Checkout repo
uses: actions/checkout@v6
with:
submodules: recursive
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
if: runner.environment != 'self-hosted'
uses: lukka/get-cmake@v4.3.3
- name: Restore ccache
uses: actions/cache/restore@v5
with:
path: .ccache
key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1
restore-keys: |
${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-
- name: Prepare Vulkan SDK
if: runner.environment != 'self-hosted'
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Update glslang external sources
working-directory: 3rdparty/glslang
run: python update_glslang_sources.py
- name: Install build dependencies
if: runner.environment != 'self-hosted'
run: |
sudo apt-get update
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
- name: Show installed toolchain
run: |
ccache --version
"${MOBILEGL_CI_CC}" --version
"${MOBILEGL_CI_CXX}" --version
"${MOBILEGL_CI_LLD}" --version
dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true
- name: Configure CMake
# Release/INFO like the shipped build on purpose. The poison arms in this configuration
# through MOBILEGL_PIPE_VERIFY (PipeInputs.h derives MOBILEGL_PIPE_POISON from it), so this
# job needs neither a Debug log level nor MOBILEGL_BUILD_DISAGGREGATED - and a Debug build
# would compare a different library from the one the other lanes measure. (build-linux
# switches to Debug under ACTIONS_STEP_DEBUG; this job deliberately does not - a Debug
# build flips CXX_VISIBILITY_PRESET and arms MOBILEGL_PIPE_POISON through a second, unrelated
# arm of its #if, so the debug switch would change what the lane is measuring.)
run: |
cmake -S . -B "${BUILD_DIR}" -G Ninja \
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \
-DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_BUILD_TYPE=Release \
-DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
-DMOBILEGL_BUILD_TEST=ON \
-DMOBILEGL_BUILD_BENCHMARK=OFF \
-DMOBILEGL_BUILD_INTEGRATION_TEST=ON \
-DMOBILEGL_ITEST_VK_ICD="${VK_ICD_FILENAMES}" \
-DMOBILEGL_BUILD_TRACE_REPLAY=OFF \
-DMOBILEGL_PIPE_VERIFY=ON \
-DCMAKE_POLICY_VERSION_MINIMUM=3.5
- name: Build
run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)"
# The lane is worthless if the option silently did not take, and that is a one-character
# mistake away at all times (a typo'd -D is not an error in CMake). Two checks, both cheap:
# the comparator's entry point must be in the library, and the fill entry point with it.
#
# `nm` and NOT `nm -D`. The library is built CXX_VISIBILITY_PRESET hidden in every non-Debug
# configuration (CMakeLists.txt:600-604) and the MGPipe entry points are plain namespace
# functions with no export attribute, so not one of them appears in the DYNAMIC table: on a
# perfectly healthy verify build `nm -D --defined-only ... | grep -c MGPipe` answers 0 out of
# ~11900 exported symbols, and a gate spelled that way is red forever for a reason that has
# nothing to do with what it claims to test. The static symbol table has them as local `t`
# entries, this artifact is never stripped, and `No MG_Remote in the pull build` below already
# uses this spelling. The symbol count guards the remaining hole: a stripped library would
# make both greps fail for a third, silent reason.
- name: Comparator symbols in verify library
run: |
test -f "${BUILD_DIR}/libMobileGL.so"
defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l)
if [ "${defined}" -lt 1000 ]; then
echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the two checks below could not have failed honestly"
exit 1
fi
if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeVerifyInputs"; then
echo "::error::libMobileGL.so defines no MGPipeVerifyInputs: -DMOBILEGL_PIPE_VERIFY=ON did not take, and every lane that consumes this artifact would run the comparator-free library and pass having compared nothing"
exit 1
fi
# The per-verb entry point, under EITHER of its two names. P2 renames
# MGPipeFillForVerb to MGPipeValidateForVerb (the body becomes the tracker's walk and
# the fill is one of its five steps), so this check has to accept both or it goes red on
# the rename for a reason that has nothing to do with what it tests. What it tests is
# unchanged: that the library HAS a per-verb entry point compiled in.
if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -qE "MGPipeValidateForVerb|MGPipeFillForVerb"; then
echo "::error::libMobileGL.so defines neither MGPipeValidateForVerb nor MGPipeFillForVerb: there is no per-verb entry point in this artifact, so nothing fills the block the comparator compares"
exit 1
fi
echo "libMobileGL.so defines MGPipeVerifyInputs and a per-verb entry point (${defined} defined symbols)"
- name: Show ccache stats
if: ${{ always() && needs.select-runner.result == 'success' }}
run: ccache --show-stats
- name: Release superseded ccache entry
if: github.ref_name == github.event.repository.default_branch
env:
GH_TOKEN: ${{ github.token }}
CACHE_KEY: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1
run: gh cache delete "${CACHE_KEY}" || true
- name: Save ccache
if: github.ref_name == github.event.repository.default_branch
continue-on-error: true
uses: actions/cache/save@v5
with:
path: .ccache
key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1
- name: Package Linux verify runtime
run: |
mkdir -p ci-artifacts
mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort)
tar \
--exclude='*/CMakeFiles' \
--exclude='*.o' \
--exclude='*.a' \
--exclude='*.ninja*' \
--exclude='build.ninja' \
--exclude='cmake_install.cmake' \
-czf ci-artifacts/mobilegl-linux-runtime-verify.tgz \
"${BUILD_DIR}/CTestTestfile.cmake" \
"${BUILD_DIR}/MobileGL/MG_Test" \
"${BUILD_DIR}/MobileGL/MG_IntegrationTest" \
"${SHARED_LIBS[@]}"
- name: Upload Linux verify runtime
uses: actions/upload-artifact@v7
with:
name: mobilegl-linux-runtime-verify
path: ci-artifacts/mobilegl-linux-runtime-verify.tgz
if-no-files-found: error
# The verify lane itself, plus the two negative controls that keep it falsifiable. The controls
# are ALWAYS-ON steps, not a manual exercise: a gate that can only be shown to work by someone
# remembering to break it on purpose is a gate that has already stopped working.
integration-verify:
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
timeout-minutes: 180
needs: [select-runner, build-linux-verify]
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
if: runner.environment != 'self-hosted'
uses: lukka/get-cmake@v4.3.3
- name: Install runtime dependencies
if: runner.environment != 'self-hosted'
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers
- name: Download Linux verify runtime
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-runtime-verify
path: .
- name: Unpack Linux verify runtime
run: |
tar -xzf mobilegl-linux-runtime-verify.tgz
test -f build-verify/libMobileGL.so
- name: Normalize CTest command paths
run: |
python - <<'PY'
from pathlib import Path
import re
for path in Path('build-verify').rglob('CTestTestfile.cmake'):
text = path.read_text()
text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text)
path.write_text(text)
PY
- name: Integration scenarios (verify)
working-directory: build-verify
# --no-tests=error is half the gate: the verify entries only exist when the library was
# configured with -DMOBILEGL_PIPE_VERIFY=ON, so a build that lost the option matches no
# tests and reds here instead of reporting a green run of nothing. The other half is
# PipeVerifyArmingScenario.Armed, which fails when the library never printed its arming
# line - the failure mode a bare `MOBILEGL_PIPE_VERIFY=1` cannot detect by itself.
#
# SCOPE, stated so nobody reads more into a green than is there: this is every integration
# ENTRY under the comparator, not every integration CONFIGURATION. The `integration` job
# runs a second, filtered pass with MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 for the
# upload ring's staged-copy tier; that pass is 186 entries here and, at the 5-10x the
# comparator costs, is not affordable inside this job's budget. The tier is covered by
# `integration`, unverified, and P2 can take it once the comparator's cost is known.
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH: "1"
MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS: "1"
MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER: "1"
run: |
ulimit -c unlimited
if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi
if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
ctest -V -L integration-verify --no-tests=error
else
ctest --output-on-failure -L integration-verify --no-tests=error
fi
# The push-only unit tests, on the verify runtime.
#
# WHY HERE AND NOT IN `test`. The entries themselves are registered in EVERY build - they
# have to be, or `ctest -N` would stop matching name-for-name between the pull and the push
# build (gate G2). What is push-only is what they assert about: MGPipeRenderStateSpans.cpp
# and PipeApply.cpp are appended to SOURCE_FILES inside the `if (MOBILEGL_PIPE_PUSH)` block,
# which is exactly how the pull build stays symbol-identical, so in a pull build each case
# opens with `#if !MOBILEGL_PIPE_PUSH GTEST_SKIP() << "push not compiled in"`. The `test`
# job therefore runs G6's chunk-table walk and G10's residual assertions as a column of
# skips: CI executes the NAMES and never one of the assertions. This job unpacks a build
# that compiled them, so it is the first place in CI where they actually run.
#
# This artifact already carries them - the packaging step above tars
# ${BUILD_DIR}/MobileGL/MG_Test whole - so the whole cost is the run, which is ~14 s for
# ~1490 entries. --no-tests=error, because a packaging change that stopped shipping the
# unit binaries would otherwise report a green run of nothing.
- name: Unit tests on the verify runtime (G6, G10)
working-directory: build-verify
run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)"
# The always-on P2 and P3a negative controls (G8, G10, G12), which are labelled
# integration-gpu and not integration-verify - they are about the handle key, the subsystem
# switch and the map-persistent counter, not about the comparator - so the lane above does
# not reach them. They are run HERE because this is the only CI job that unpacks a
# MOBILEGL_PIPE_PUSH build: every one of them reads a counter out of the library's summary
# line, and both the counters and their brackets are #if MOBILEGL_PIPE_PUSH, so in the pull
# `integration` job the entries exist (gate G2 requires the same names in both builds) but
# have nothing to assert.
#
# An arm whose subsystem has not landed on this tree SKIPS with the reason (never absent,
# never a green that asserted nothing), so this step is green through the P2 and P3a landing
# orders and starts asserting as each package arrives.
#
# The environment is the sibling step's, deliberately and in full: these entries run the
# same DirectVulkan binary through the same runner, so the three MOBILEGL_MAGMA_* fixes it
# needs apply here too, and a crash here has to leave a core for the same black-box flow.
# The step above is the only reason those lines exist in this job; a control that crashed
# without one would be the hardest failure in the job to diagnose.
#
# THE -R ALTERNATIVES ARE TEST-NAME PREFIXES, NOT LANE LABELS, and each one is deliberately
# the SHORTEST string that still selects only what it means to. `ResourceSubsystem` (not
# `ResourceSubsystemControl`) is what reaches the eight
# DirectGLES.ResourceSubsystemOn./Off.LargeArenaAdoptionScenario.* entries - the A/B lanes
# whose entire purpose is that the handle path and the legacy BufferBackendOps path must
# agree about an adopted store - as well as the two ResourceSubsystemControl. entries.
# `MapPersistentRoundtrip` is singular because LargeArenaAdoptionScenario's case is
# `AnAdoptionCostsExactlyOneMapPersistentRoundtrip`; the plural matched only the lane PREFIX
# of the other one. Both mistakes were silent: this is the only CI job that unpacks a push
# build, so an entry the filter misses is either never run under the P3a bits at all or runs
# only in the pull `integration` job, where a MOBILEGL_PIPE_PUSH value steers nothing
# (Config.h declares the field inside the push guard) and both arms are the same legacy path.
# A lane that cannot go red where it is installed is not a gate (ROADMAP.md:7).
# P4a ADDS THREE ALTERNATIVES, and each one is here because this is the only CI job that
# unpacks a push build:
# * `ObjectSubsystem` reaches the three DirectGLES.ObjectSubsystemControl. entries - the
# 0x1fff-vs-0x1ff A/B and the 0x9ff dependency refusal (G12). `ResourceSubsystem` does
# NOT match it: the two families are named apart on purpose, because they are different
# phases' switches and a filter that merged them would hide one behind the other.
# * `TextureParamsWithoutASamplerView` reaches G9's four cases, the scenario ROADMAP.md:20
# names by hand. It runs in the ambient lanes, which the label already selects - but this
# step is where those cases run against a PUSH library, and G9's whole claim is about the
# push path. All four cases are green on the contract commit - including the one D-E3
# expected to be red, for the reason the scenario's header records - so this row is green
# from the day it lands and goes red only if a reachability path stops syncing texture
# parameters at all, which is the coupling ARCHITECTURE.md:100 exists to remove.
# * `TextureUploadShape` is RECORDED, not gated (D-D4): it asserts that the two upload-shape
# counters could be read and that they agree, and prints the shape for MEASUREMENTS.md. It
# is in the filter so that the number is actually collected on every run - an unmeasured
# shape is not a recorded one - and because its own assertions can go red.
# As with the four before them, each alternative is the SHORTEST string that selects only what
# it means to.
# Handle-ABA, CSO, subsystem and texture-parameter controls (G8, G8b, G9, G10, G12).
- name: Push-build controls (G8, G8b, G9, G10, G12)
working-directory: build-verify
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH: "1"
MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS: "1"
MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER: "1"
run: |
ulimit -c unlimited
if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi
ctest --output-on-failure -L integration-gpu \
-R 'HandleRecycle|CsoContentAddressing|ResourceSubsystem|MapPersistentRoundtrip|ObjectSubsystem|TextureParamsWithoutASamplerView|TextureUploadShape' \
--no-tests=error -j 4
# The arming lanes' logs, and ONLY those. Each lane shares one MOBILEGL_LOG_FILE_PATH and the
# library opens it fopen(path, "w"), so after an ambient lane of 400-odd processes the file
# holds the LAST one - grepping it would say nothing about the other 405 and would red a
# healthy lane whenever the last entry happened not to issue a verb (which is what the
# PoisonOmissionScenario parent, the last ambient entry, does by construction: it forks,
# execve()s and reads files). The DirectGLES.VerifyArming. / DirectVulkan.VerifyArming.
# entries are one process each on a log path nothing else writes, so this grep means exactly
# what it says.
#
# What it proves: arming is a property of (this library, this environment), and these two
# processes ran the same library with the same MOBILEGL_PIPE_VERIFY=1 as their ~400 ambient
# siblings. It is not, and cannot be, a per-process census - the shared log cannot support one.
# It catches the case ctest cannot: an arming entry that SKIPPED still reports green.
- name: Verify lane arming check
working-directory: build-verify
run: |
shopt -s nullglob
logs=(MobileGL/MG_IntegrationTest/pipe-verify-arming-*.log)
if [ ${#logs[@]} -lt 2 ]; then
echo "::error::found ${#logs[@]} pipe-verify-arming-*.log (expected one per backend). The VerifyArming. entries did not run, so nothing in this job establishes that the comparator was ever armed."
exit 1
fi
for log in "${logs[@]}"; do
if ! grep -q "MGPipe: verify armed" "${log}"; then
echo "::error::${log} carries no arming line: that lane's process ran the whole scenario without the comparator, so every green entry beside it is green for no reason"
exit 1
fi
done
echo "arming line present in all ${#logs[@]} arming-lane log(s)"
# NEGATIVE CONTROL A (gate G4). The knob perturbs one field in the snapshot arm before the
# entry compare, so a working comparator must abort the run. This step passes when ctest
# FAILS - `if ctest ...; then error` - which is the only shape that can catch a comparator
# that silently compares nothing.
#
# The knob reaches the test process through the JOB environment: no ctest ENVIRONMENT
# property on the ambient Verify. entries names it (MG_IntegrationTest/CMakeLists.txt says
# so out loud), and a property entry would otherwise override this and the control would
# prove nothing. Same precedent as MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH in `integration`.
- name: Negative control A (corrupted snapshot)
working-directory: build-verify
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_PIPE_VERIFY_CORRUPT: GetRenderStateParameters
run: |
FILTER='DirectGLES\.Verify\..*ClearThenReadPixels'
# An empty selection would ALSO make ctest exit non-zero (--no-tests=error), and this
# step reads non-zero as "the control worked" - so the selection is counted first. A
# control that passes because it ran nothing is worse than no control.
matched=$(ctest -N -L integration-verify -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:')
if [ "${matched}" -lt 1 ]; then
echo "::error::negative control A selected ${matched} tests; its filter no longer matches anything"
exit 1
fi
if ctest --output-on-failure -L integration-verify -R "${FILTER}" --no-tests=error; then
echo "::error::MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left ${matched} verify entries GREEN. The comparator is not comparing, so every green entry above is green for no reason."
exit 1
fi
echo "the corrupted field turned ${matched} selected entries red, as it must"
# NEGATIVE CONTROL B (gate G5). The omission skips the STAMP of one field for one verb while
# still copying its value - indistinguishable from a fill row nobody wrote - so the poison
# must abort the glGenerateMipmap. Again: this step passes when ctest fails.
#
# The entry it targets is PoisonOmissionScenario.WithoutOmissionCompletes, which is green in
# the ambient lane above and is the ONLY integration entry in the tree that calls
# glGenerateMipmap at all. It deliberately does not skip itself when the knob is set, exactly
# so that this control has something to turn red.
- name: Negative control B (omitted fill point)
working-directory: build-verify
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_PIPE_POISON_OMIT: GenerateMipmap:GetActiveTextureUnit
run: |
FILTER='DirectGLES\.Verify\.PoisonOmissionScenario\.WithoutOmissionCompletes'
matched=$(ctest -N -L integration-verify -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:')
if [ "${matched}" -lt 1 ]; then
echo "::error::negative control B selected ${matched} tests; its filter no longer matches anything"
exit 1
fi
if ctest --output-on-failure -L integration-verify -R "${FILTER}" --no-tests=error; then
echo "::error::MOBILEGL_PIPE_POISON_OMIT=GenerateMipmap:GetActiveTextureUnit left the verify lane GREEN. The per-verb poison is not armed, so a forgotten fill row would ship silently."
exit 1
fi
echo "the omitted fill point turned the lane red, as it must"
- name: Upload verify lane logs
if: ${{ always() && needs.select-runner.result == 'success' }}
uses: actions/upload-artifact@v7
with:
name: integration-verify-logs
path: build-verify/MobileGL/MG_IntegrationTest/pipe-*.log*
if-no-files-found: warn
- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v7
with:
name: integration-verify-core-dumps
path: /tmp/core.*
if-no-files-found: ignore
# P7 WAVE 3 (V1), CONTRACT-P7 §6 - EXIT GATE 2's SECOND COLUMN: the comparator on the split
# (inproc) arm, both backends, and the two negative controls red on that arm.
#
# ITS OWN BUILD, because neither existing one can carry it: build-linux-verify configures
# without MOBILEGL_BUILD_DISAGGREGATED (so MG_Remote and the transport parser do not exist and
# `MOBILEGL_TRANSPORT=inproc` would be accepted and ignored), and build-linux-split configures
# without MOBILEGL_PIPE_VERIFY (so the comparator does not exist). The integration-verify-split
# entries are registered only when BOTH options are on, so `--no-tests=error` below reds a build
# that lost either one instead of reporting a green run of nothing. Built and run in one job:
# nothing downstream consumes this library.
#
# INPROC ONLY, BY THE INSTRUMENT'S NATURE: the comparator's two arms (the applier-written
# gPipeInputs and the frontend GLContext) are co-resident only when both roles share a process.
# MG_IntegrationTest/CMakeLists.txt's verify-split block records the spawn measurement.
integration-verify-split:
needs: select-runner
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
timeout-minutes: 180
permissions:
actions: write
contents: read
env:
BUILD_DIR: build-verify-split
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_COMPRESS: "true"
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 4G
CCACHE_NOHASHDIR: "true"
steps:
- name: Set swap space
if: runner.environment != 'self-hosted'
uses: pierotofy/set-swap-space@v1.0
with:
swap-size-gb: 32
- name: Checkout repo
uses: actions/checkout@v6
with:
# recursive for build-linux-split's reason: MOBILEGL_BUILD_DISAGGREGATED shadows itself
# back to OFF when 3rdparty/flatbuffers/include is missing.
submodules: recursive
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
if: runner.environment != 'self-hosted'
uses: lukka/get-cmake@v4.3.3
- name: Restore ccache
uses: actions/cache/restore@v5
with:
path: .ccache
key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1
restore-keys: |
${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-
- name: Prepare Vulkan SDK
if: runner.environment != 'self-hosted'
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Update glslang external sources
working-directory: 3rdparty/glslang
run: python update_glslang_sources.py
- name: Install build and runtime dependencies
if: runner.environment != 'self-hosted'
run: |
sudo apt-get update
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1
- name: Configure CMake
# Release/INFO for build-linux-verify's reason, and INFO is load-bearing here for
# build-linux-split's: ConfigLoader's resolved-transport line is logged at INFO and the
# arm-proof step below reads it back out of every entry's client log.
run: |
cmake -S . -B "${BUILD_DIR}" -G Ninja \
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \
-DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_BUILD_TYPE=Release \
-DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
-DMOBILEGL_BUILD_TEST=ON \
-DMOBILEGL_BUILD_BENCHMARK=OFF \
-DMOBILEGL_BUILD_INTEGRATION_TEST=ON \
-DMOBILEGL_ITEST_VK_ICD="${VK_ICD_FILENAMES}" \
-DMOBILEGL_BUILD_TRACE_REPLAY=OFF \
-DMOBILEGL_PIPE_PUSH=ON \
-DMOBILEGL_BUILD_DISAGGREGATED=ON \
-DMOBILEGL_BUILD_DISAGGREGATED_INPROC=ON \
-DMOBILEGL_PIPE_VERIFY=ON \
-DCMAKE_POLICY_VERSION_MINIMUM=3.5
- name: Build
run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)"
# Both options, both directions of "did it take" - the two jobs this one is the product of
# each check one of them, and this library has to pass both checks.
- name: Comparator and MG_Remote symbols
run: |
test -f "${BUILD_DIR}/libMobileGL.so"
defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l)
if [ "${defined}" -lt 1000 ]; then
echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the checks below could not have failed honestly"
exit 1
fi
if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeVerifyInputs"; then
echo "::error::libMobileGL.so defines no MGPipeVerifyInputs: -DMOBILEGL_PIPE_VERIFY=ON did not take, and the verify-split lane would compare nothing"
exit 1
fi
remote=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -c -i "MG_Remote" || true)
if [ "${remote}" -lt 1 ]; then
echo "::error::libMobileGL.so defines no MG_Remote symbol: -DMOBILEGL_BUILD_DISAGGREGATED=ON did not take, and every entry would run monolith under a name that says split"
exit 1
fi
echo "libMobileGL.so defines MGPipeVerifyInputs and ${remote} MG_Remote symbol(s) (${defined} defined symbols)"
- name: Show ccache stats
if: ${{ always() && needs.select-runner.result == 'success' }}
run: ccache --show-stats
- name: Release superseded ccache entry
if: github.ref_name == github.event.repository.default_branch
env:
GH_TOKEN: ${{ github.token }}
CACHE_KEY: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1
run: gh cache delete "${CACHE_KEY}" || true
- name: Save ccache
if: github.ref_name == github.event.repository.default_branch
continue-on-error: true
uses: actions/cache/save@v5
with:
path: .ccache
key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1
# The lane. `^...$` because ctest -L is a regex (ID-131) and `integration-verify` would
# otherwise also match this label and the monolith one would match both.
- name: Verify-split scenarios (inproc)
working-directory: build-verify-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
run: |
ulimit -c unlimited
if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi
# --output-junit, because the arm proof below has to tell a gtest SKIP from a pass: an
# entry that skipped reached no verb honestly, and the skip set moves with the driver.
ctest --output-on-failure -L '^integration-verify-split$' --no-tests=error -j 4 \
--output-junit "${RUNNER_TEMP}/verify-split.xml"
# THE ARM PROOF, PER ENTRY AND BY NAME. Every ambient entry has a private log path
# (SplitLogPaths.cmake.in), so this - unlike integration-verify's grep over a shared log -
# is a census:
# * ConfigLoader's `MOBILEGL_TRANSPORT=inproc` line in EVERY client log: an entry that ran
# monolith would have passed every assertion above and fails here by name;
# * `MGPipe: verify armed` in both VerifySplitArming. logs;
# * and then the census proper. This used to be a FLOOR - "at least 850 armed of at least
# 1000 logs" - which leaves the remaining ~155 unnamed, so the lane could lose fifty
# arms, gain fifty skips, and the floor would not move. It is now the two-sided rule
# dualblock-expected-fatals.txt already rides: each client log must ARM, or belong to an
# entry ctest reported SKIPPED, or be NAMED in Harness/verify-split-unarmed-expected.txt
# (56 names in five classes, each class saying why it reaches no verb). A listed entry
# that arms, and a listed name with no log at all, both fail - an exception set that
# keeps rows nothing needs is how a proof rots green.
- name: Verify-split per-entry arm proof
working-directory: build-verify-split/MobileGL/MG_IntegrationTest
run: |
shopt -s nullglob
logs=(verify-split-logs/*.client.log)
if [ ${#logs[@]} -lt 1000 ]; then
echo "::error::found ${#logs[@]} per-entry client logs under verify-split-logs (expected ~1064): the private log paths did not take, so nothing below is a per-entry proof"
exit 1
fi
missing=$(grep -L 'Config: MOBILEGL_TRANSPORT=inproc' "${logs[@]}" | wc -l)
if [ "${missing}" -ne 0 ]; then
grep -L 'Config: MOBILEGL_TRANSPORT=inproc' "${logs[@]}" | head -20
echo "::error::${missing} verify-split entries carry no resolved MOBILEGL_TRANSPORT=inproc line: they did not run on the split arm"
exit 1
fi
for backend in DirectGLES DirectVulkan; do
if ! grep -q "MGPipe: verify armed" "pipe-verify-split-arming-${backend}.client.log"; then
echo "::error::pipe-verify-split-arming-${backend}.client.log carries no arming line"
exit 1
fi
done
python3 "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
verify-split-arming verify-split-logs "${RUNNER_TEMP}/verify-split.xml" \
"${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/verify-split-unarmed-expected.txt"
# THE GREEN LANE'S LOGS, KEPT BEFORE THE CONTROLS OVERWRITE THEM. Each control step below
# re-runs ambient VerifySplit. entries with a knob exported, and an ambient entry writes to
# ITS OWN per-entry path under verify-split-logs/ - which the library opens "w". So by the
# time the upload step at the end of this job runs, twelve of those files are the CONTROL
# runs' logs sitting under the LANE's names: an artifact showing deliberate divergences,
# with the green run they were meant to be evidence of already gone. `if: always()` on the
# upload made that worse rather than better, because a failure investigation is exactly
# when the overwritten half is the half somebody needs.
- name: Keep lane logs before controls
if: ${{ always() && needs.select-runner.result == 'success' }}
working-directory: build-verify-split/MobileGL/MG_IntegrationTest
run: |
rm -rf verify-split-logs-lane
cp -a verify-split-logs verify-split-logs-lane
cp -p "${RUNNER_TEMP}/verify-split.xml" verify-split-logs-lane/ || true
echo "kept $(ls verify-split-logs-lane | wc -l) file(s) from the green lane run"
# NEGATIVE CONTROL A ON THE SPLIT ARM - integration-verify's step A, same knob, same shape
# (this step passes when ctest FAILS), aimed at the ambient VerifySplit. entries of BOTH
# backends. Measured at landing: 10 of 10 red, each with
# Fatal{PipeVerifyDiffer, "GetRenderStateParameters@Clear"|"@DrawArrays", ..., where=entry}.
- name: Negative control A (corrupted snapshot)
working-directory: build-verify-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_PIPE_VERIFY_CORRUPT: GetRenderStateParameters
run: |
FILTER='(DirectGLES|DirectVulkan)\.VerifySplit\..*ClearThenReadPixels'
matched=$(ctest -N -L '^integration-verify-split$' -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:')
if [ "${matched}" -lt 2 ]; then
echo "::error::split negative control A selected ${matched} tests; its filter no longer matches both backends"
exit 1
fi
if ctest --output-on-failure -L '^integration-verify-split$' -R "${FILTER}" --no-tests=error; then
echo "::error::MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left ${matched} verify-split entries GREEN. The comparator is not comparing on the split arm."
exit 1
fi
echo "the corrupted field turned ${matched} selected split entries red, as it must"
# NEGATIVE CONTROL A ON THE OTHER COMPARATOR (P7 wave 3, V1). The step above proves the
# ENTRY compare - client thread, verb boundary, `where=entry`. The compare-at-read hook runs
# on the SERVER's apply thread at every backend read and is the only comparator that arm ever
# runs, and until MG_Impl/Pipe/PipeFill.cpp's hook honoured MOBILEGL_PIPE_VERIFY_CORRUPT it
# had no control at all: a hook that had stopped comparing was indistinguishable from a hook
# with nothing to report.
#
# NOT THE "ctest MUST FAIL" SHAPE, and the reason is mechanical rather than a preference:
# GetPixelStoreParameters is in kReadback's fill mask, so with MOBILEGL_PIPE_VERIFY_FATAL at
# its default the client's entry compare aborts the process at the ReadPixels boundary and
# the server never reaches the read. The entries therefore pin FATAL=0, survive their own
# divergence, and each asserts from inside the process that the SERVER role's half of its
# private log carries the report. This step re-reads those logs from outside, so the claim
# does not rest only on the assertion that produced them.
#
# AND IT COUNTS, NEEDING TWO (V1 fix round 2). The server reads the pack field twice inside
# the ReadPixels window - PipeApplier::read_pixels saves the application's pack before it
# installs the neutral one, the backend's ReadPixels reads it after - and the hook perturbs
# its oracle once per read, because the ID-49 overwrite inside the window wipes the first
# perturbation before the second read's compare. A step satisfied by ONE line was satisfied
# by either perturbation alone and falsified neither. Measured at landing: 3 (DirectGLES,
# whose backend reads the field twice after the install) / 2 (DirectVulkan); without the
# re-applied perturbation both drop to 1, without the first one DirectVulkan drops to 1
# while DirectGLES stays at 2 - so BOTH backends are required, and the first perturbation's
# falsifier is the DirectVulkan entry.
#
# THE GREEN HALF is the lane run above: the same hook, the same window, with no corruption
# armed, across 1070 entries at zero divergences.
- name: Negative control A (server-side read)
working-directory: build-verify-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
run: |
FILTER='(DirectGLES|DirectVulkan)\.VerifySplitReadCorrupted\.'
matched=$(ctest -N -L '^integration-verify-split$' -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:')
if [ "${matched}" -lt 2 ]; then
echo "::error::split read-side control A selected ${matched} tests; its filter no longer matches both backends"
exit 1
fi
ctest --output-on-failure -L '^integration-verify-split$' -R "${FILTER}" --no-tests=error
for backend in DirectGLES DirectVulkan; do
log="MobileGL/MG_IntegrationTest/pipe-verify-split-read-corrupt-${backend}.server.log"
reads=$(grep -c 'Fatal{PipeVerifyDiffer, "GetPixelStoreParameters@ReadPixels".*where=read}' "${log}" 2>/dev/null || true)
reads=${reads:-0}
if [ "${reads}" -lt 2 ]; then
echo "::error::${log} carries ${reads} compare-at-read report(s) for the corrupted pack field, fewer than the two reads the server makes inside the ReadPixels window (the applier's saved-pack read and the backend's post-install read): the hook is not comparing on the server's apply thread, or one of its two perturbations no longer reaches its read because the ID-49 neutral-pack overwrite swallowed it"
exit 1
fi
done
echo "both backends reported GetPixelStoreParameters@ReadPixels where=read at least twice in the SERVER half, as they must"
# NEGATIVE CONTROL B ON THE SPLIT ARM, WITH THE SPLIT ARM'S PAIR. The monolith pair
# (GenerateMipmap:GetActiveTextureUnit) is inert here - the split backend's generate_mipmap
# resolves its texture by the verb's handle and never reads the active unit - so this step
# withholds ReadPixels:GetPixelStoreParameters, which the server reads under its own stamp.
# Measured at landing: 2 of 2 red, each child dying with
# Fatal{UnmigratedPipeInput, "GetPixelStoreParameters@ReadPixels"} on mgl-srv-apply.
- name: Negative control B (omitted stamp)
working-directory: build-verify-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_PIPE_POISON_OMIT: ReadPixels:GetPixelStoreParameters
run: |
FILTER='(DirectGLES|DirectVulkan)\.VerifySplit\.PoisonOmissionScenario\.WithoutOmissionCompletes'
matched=$(ctest -N -L '^integration-verify-split$' -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:')
if [ "${matched}" -lt 2 ]; then
echo "::error::split negative control B selected ${matched} tests; its filter no longer matches both backends"
exit 1
fi
if ctest --output-on-failure -L '^integration-verify-split$' -R "${FILTER}" --no-tests=error; then
echo "::error::MOBILEGL_PIPE_POISON_OMIT=ReadPixels:GetPixelStoreParameters left the verify-split lane GREEN. The server-side poison is not armed on the split arm."
exit 1
fi
echo "the omitted stamp turned the split lane red, as it must"
# BOTH SETS, AND THEY ARE DIFFERENT EVIDENCE. verify-split-logs-lane/ is the green run the
# arm proof read; verify-split-logs/ is whatever the control steps left behind, which for
# twelve entries is a deliberately red re-run under the same name. Uploading only the
# second - which is what this step did before the copy above existed - loses the first.
# The run log needs a signed-in reader; annotations do not. Name every failed ctest entry,
# with the lines of its output that say why, where anyone reading the run can see them.
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}"
- name: Upload verify-split lane logs
if: ${{ always() && needs.select-runner.result == 'success' }}
uses: actions/upload-artifact@v7
with:
name: integration-verify-split-logs
path: |
build-verify-split/MobileGL/MG_IntegrationTest/verify-split-logs-lane/**
build-verify-split/MobileGL/MG_IntegrationTest/verify-split-logs/**
build-verify-split/MobileGL/MG_IntegrationTest/pipe-*split*.log*
if-no-files-found: warn
- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v7
with:
name: integration-verify-split-core-dumps
path: /tmp/core.*
if-no-files-found: ignore
# P5's split build. The verify pair cloned, because the problem is the same one: a third
# configuration of the same sources whose whole value depends on the option having taken.
#
# This dedicated build keeps the protocol/strict controls independent of the branch's
# primary runtime mode. MOBILEGL_BUILD_DISAGGREGATED implies MOBILEGL_PIPE_PUSH (the split path decodes
# into the MGPipe applier and PIPE_PUSH is what compiles the applier), so the split arm needs a
# build of its own exactly as the verify arm does.
#
# THE `nm` STEP IS THE POINT OF THIS JOB. In a build without the option, MG_Remote is not
# compiled at all and MOBILEGL_TRANSPORT's parser does not exist - so the variable is accepted by
# the environment and silently ignored (CONTRACT-P5 5). Every downstream lane would then run
# monolith and go green under a name that says split. CMake will not complain about a typo'd -D,
# so the build-level assertion is the only guard, and its absence is precisely what would make
# the whole split arm meaningless.
build-linux-split:
needs: select-runner
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
timeout-minutes: 120
permissions:
actions: write
contents: read
env:
BUILD_DIR: build-split
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_COMPRESS: "true"
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 4G
CCACHE_NOHASHDIR: "true"
steps:
- name: Set swap space
if: runner.environment != 'self-hosted'
uses: pierotofy/set-swap-space@v1.0
with:
swap-size-gb: 32
- name: Checkout repo
uses: actions/checkout@v6
with:
# recursive, and load-bearing here beyond the usual: MOBILEGL_BUILD_DISAGGREGATED
# SHADOWS ITSELF BACK TO OFF when 3rdparty/flatbuffers/include is missing
# (CMakeLists.txt:471-486, a normal variable rather than a cache force, deliberately).
# A shallow checkout would therefore configure cleanly, build a monolith library, and
# be caught only by the nm step below.
submodules: recursive
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
if: runner.environment != 'self-hosted'
uses: lukka/get-cmake@v4.3.3
- name: Restore ccache
uses: actions/cache/restore@v5
with:
path: .ccache
key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1
restore-keys: |
${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-
- name: Prepare Vulkan SDK
if: runner.environment != 'self-hosted'
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Update glslang external sources
working-directory: 3rdparty/glslang
run: python update_glslang_sources.py
- name: Install build dependencies
if: runner.environment != 'self-hosted'
run: |
sudo apt-get update
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
- name: Show installed toolchain
run: |
ccache --version
"${MOBILEGL_CI_CC}" --version
"${MOBILEGL_CI_CXX}" --version
dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true
- name: Configure CMake
# Release/INFO like the shipped build, for the same reason build-linux-verify gives.
# INFO specifically matters here: ConfigLoader logs the resolved transport at INFO and
# that line is what run_trace_case.cmake and the integration-split lane read back as
# proof the transport resolved in THIS process.
#
# _INPROC implies _DISAGGREGATED implies _PIPE_PUSH; all three are passed anyway, because
# an implication that is asserted in two places is an implication nobody has to remember.
run: |
cmake -S . -B "${BUILD_DIR}" -G Ninja \
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \
-DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_BUILD_TYPE=Release \
-DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
-DMOBILEGL_BUILD_TEST=ON \
-DMOBILEGL_BUILD_BENCHMARK=OFF \
-DMOBILEGL_BUILD_INTEGRATION_TEST=ON \
-DMOBILEGL_ITEST_VK_ICD="${VK_ICD_FILENAMES}" \
-DMOBILEGL_BUILD_TRACE_REPLAY=OFF \
-DMOBILEGL_PIPE_PUSH=ON \
-DMOBILEGL_BUILD_DISAGGREGATED=ON \
-DMOBILEGL_BUILD_DISAGGREGATED_INPROC=ON \
-DCMAKE_POLICY_VERSION_MINIMUM=3.5
- name: Build
run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)"
# `nm` and NOT `nm -D`, for the reason build-linux-verify spells out: everything under
# MG_Remote is a plain namespace symbol in a CXX_VISIBILITY_PRESET=hidden Release build and
# none of it reaches the dynamic table. The symbol count guards the remaining hole - a
# stripped library would make the grep fail for a third, silent reason.
#
- name: MG_Remote symbols in split library
run: |
test -f "${BUILD_DIR}/libMobileGL.so"
defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l)
if [ "${defined}" -lt 1000 ]; then
echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the checks below could not have failed honestly"
exit 1
fi
remote=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -c -i "MG_Remote" || true)
if [ "${remote}" -lt 1 ]; then
echo "::error::libMobileGL.so defines no MG_Remote symbol: -DMOBILEGL_BUILD_DISAGGREGATED=ON did not take (a typo'd -D is not a CMake error, and the option shadows itself OFF when 3rdparty/flatbuffers/include is missing). Every lane that consumes this artifact would run MONOLITH while claiming to run split, because the MOBILEGL_TRANSPORT parser does not exist in such a build and the variable is accepted and ignored."
exit 1
fi
# The transport parser itself, which is the symbol the runtime evidence depends on.
if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeApply"; then
echo "::error::libMobileGL.so defines no MGPipeApply* entry point, so MOBILEGL_PIPE_PUSH did not take either and there is no applier for the split path to decode into"
exit 1
fi
echo "libMobileGL.so defines ${remote} MG_Remote symbol(s) and the MGPipe applier (${defined} defined symbols)"
# The link-closure ratchet (CONTRACT-P6.md 12.1, ID-P7-7): the set of frontend symbols a
# server-only image would still have to link, recomputed from the per-object partition in
# scripts/link_ratchet.py and compared against the stored LIST.
#
# THIS job and not `build-linux`: build-linux passes the same three options from
# ${MOBILEGL_CI_DISAGGREGATED}, which is ON only on feat/disaggregated and
# codex/fix-github-ci, so on every other branch it builds a monolith - MG_Remote would not
# be compiled at all, the SERVER half of the partition would be empty, and the gate would
# go green having measured nothing. This job turns the three options on unconditionally,
# and the nm step immediately above has just proved they took. The script re-reads them
# from CMakeCache.txt anyway and refuses the run if they are not all ON.
#
# Only NEW symbols fail. A symbol that has DISAPPEARED is the ratchet turning and is
# printed with an instruction to re-baseline in the same commit - the baseline stores the
# list rather than a count precisely so that clearing one symbol while adding another does
# not walk through at an unchanged total (ID-P7-7).
#
# It reads the per-object output of the MobileGL target, so it must run before anything
# that cleans the build tree; the packaging step below excludes *.o from the tarball but
# does not delete them.
- name: Server link-closure ratchet
run: |
mkdir -p ci-artifacts
python3 scripts/link_ratchet.py --self-test
python3 scripts/link_ratchet.py \
--build-dir "${BUILD_DIR}" \
--baseline scripts/data/link_ratchet_baseline.txt \
--assert-monotone \
--bucket \
--json ci-artifacts/link-ratchet.json
# Uploaded even when the gate fired: a red ratchet is exactly the moment someone needs the
# per-symbol definer/referrer attribution the JSON carries, and a job that failed before
# uploading its evidence is a job nobody can act on.
- name: Upload link ratchet
if: ${{ always() && needs.select-runner.result == 'success' }}
uses: actions/upload-artifact@v7
with:
name: link-ratchet
path: ci-artifacts/link-ratchet.json
if-no-files-found: warn
- name: Show ccache stats
if: ${{ always() && needs.select-runner.result == 'success' }}
run: ccache --show-stats
- name: Release superseded ccache entry
if: github.ref_name == github.event.repository.default_branch
env:
GH_TOKEN: ${{ github.token }}
CACHE_KEY: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1
run: gh cache delete "${CACHE_KEY}" || true
- name: Save ccache
if: github.ref_name == github.event.repository.default_branch
continue-on-error: true
uses: actions/cache/save@v5
with:
path: .ccache
key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1
- name: Package Linux split runtime
run: |
mkdir -p ci-artifacts
mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort)
tar \
--exclude='*/CMakeFiles' \
--exclude='*.o' \
--exclude='*.a' \
--exclude='*.ninja*' \
--exclude='build.ninja' \
--exclude='cmake_install.cmake' \
-czf ci-artifacts/mobilegl-linux-runtime-split.tgz \
"${BUILD_DIR}/CTestTestfile.cmake" \
"${BUILD_DIR}/MobileGL/MG_Test" \
"${BUILD_DIR}/MobileGL/MG_IntegrationTest" \
"${SHARED_LIBS[@]}"
- name: Upload Linux split runtime
uses: actions/upload-artifact@v7
with:
name: mobilegl-linux-runtime-split
path: ci-artifacts/mobilegl-linux-runtime-split.tgz
if-no-files-found: error
# The split lane itself: the three-arm shape ARCHITECTURE.md:521 asks for verbatim, plus the
# first real run of the five MG_Test/Wire suites, plus P5's two exit-gate negative controls.
integration-split:
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
timeout-minutes: 120
needs: [select-runner, build-linux-split]
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
if: runner.environment != 'self-hosted'
uses: lukka/get-cmake@v4.3.3
- name: Install runtime dependencies
if: runner.environment != 'self-hosted'
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers
- name: Download Linux split runtime
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-runtime-split
path: .
- name: Unpack Linux split runtime
run: |
tar -xzf mobilegl-linux-runtime-split.tgz
test -f build-split/libMobileGL.so
- name: Normalize CTest command paths
run: |
python - <<'PY'
from pathlib import Path
import re
for path in Path('build-split').rglob('CTestTestfile.cmake'):
text = path.read_text()
text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text)
path.write_text(text)
PY
# THE FIRST REAL RUN OF MG_Test/Wire. Those five suites - Framing, Ring, InProcessTransport,
# ProtocolSmoke and FdPassing - are registered only under MOBILEGL_BUILD_DISAGGREGATED
# (MG_Test/CMakeLists.txt), and before this job existed NO cmake invocation anywhere in this
# workflow passed that option. They had never been compiled by CI, let alone run.
# Exit gate 9.2. BEFORE the lanes run, because it costs a `ctest -N` and answers the one
# question a green lane cannot: did the spawn arm cover the same cases the split arm did?
# A spawn lane registered with fewer cases goes green while proving less, and nothing about
# a smaller green lane looks wrong.
- name: Spawn/split lane parity (9.2)
run: python3 scripts/ci/spawn_lane_parity.py build-split
# P8-A (ID-P8-2): every monolith case, tails normalised, is on a gated split arm or is named
# with a reason class in scripts/data/split_coverage_exemptions.txt. The same `ctest -N`
# cost as the parity step above, and the question it cannot answer: parity compares the
# split arms with EACH OTHER, so a case missing from all three passes it.
- name: Split coverage against monolith (P8-A)
run: |
python3 scripts/ci/split_coverage.py --self-test
python3 scripts/ci/split_coverage.py build-split
# P8-SE: PipeCatalogueTest's unmigrated-emulation list equals the MGPipeUnmigratedEmulation
# call sites in the tree, one site per name. A source lint: no build needed.
- name: Unmigrated-emulation list against its call sites (P8-SE)
run: |
python3 scripts/ci/unmigrated_emulation_sites.py --self-test
python3 scripts/ci/unmigrated_emulation_sites.py
- name: Unit tests on the split runtime
working-directory: build-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit-split.xml"
# P7 wave 2-F, PH-7 (1). THE TCP SUPERVISOR LANE NEEDS flatc, AND CI HAD NONE.
#
# `TcpLane.SupervisorProtocolControls` carries LABELS integration-tcp, so it has been in the
# selection below since wave 0 - as a SKIP, every run. The script builds the protocol's
# Python bindings with flatc, flatc is deliberately absent from the build graph
# (gen_protocol.py's header says why), and with none found it exits 77 and ctest reports
# the entry skipped. Skipped is what the entry looked like for the whole time it did not
# exist, which is the failure mode the plan's own row named ("referenced by NOTHING").
#
# So the job builds the SAME pinned flatc that flatc-check builds, from the same submodule,
# and the lane step below names the entry to junit_tally.py --require-entry-passed. If the
# build ever stops producing one, the lane reds by name rather than going green having run
# the supervisor's Busy / token / listen / version / build-policy / reap controls zero times.
# --check is passed so this step cannot quietly regenerate a different committed header;
# keeping that honest is flatc-check's job and this step must not overlap with it.
# P7 F2: `TcpLane.ControlFrameFuzz` (Ph fuzz arm 1, scripts/ci/ph_fuzz_control_frames.py)
# needs the same flatc and is required PASSED by name on the same junit_tally line.
- name: Check out FlatBuffers (TCP lane)
run: git submodule update --init 3rdparty/flatbuffers
- name: Build pinned flatc (TCP lane)
run: python3 scripts/gen_protocol.py --build-dir "${RUNNER_TEMP}/flatc-build" --check
# --no-tests=error is half the gate, exactly as in integration-verify: the integration-split
# entries exist only when the library was configured with -DMOBILEGL_BUILD_DISAGGREGATED=ON,
# so a build that lost the option matches nothing and reds here instead of reporting a green
# run of nothing. The other half is the transport-resolution check below.
#
# An entry whose owning package (c1 client, s1 session, v1 server) has not landed SKIPS with
# the reason and never goes green - MG_IntegrationTest/CMakeLists.txt probes MG_Remote for
# c0's signature stubs and disarms the lane while any remain.
- name: Split lanes (inproc, spawn, TCP)
working-directory: build-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
run: |
ulimit -c unlimited
if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi
export MOBILEGL_FLATC_EXECUTABLE="${RUNNER_TEMP}/flatc-build/flatc"
test -x "${MOBILEGL_FLATC_EXECUTABLE}"
ctest --output-on-failure -L integration-split --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/split-baseline.xml"
python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/split-baseline.xml" --require-split-ran
# Exit gate 9.3: the spawn arm green at the same count. It runs the same cases through a
# SECOND PROCESS, so a failure here that the line above did not see is a transport fact
# and not a phase one - which is the whole reason both lanes exist.
ctest --output-on-failure -L integration-spawn --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/spawn-baseline.xml"
# PH-6 / fuzz arm 3: the supervisor-level forfeit control rides this lane and must RUN.
python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/spawn-baseline.xml" --require-spawn-ran --require-entry-passed SpawnLane.EventForfeitPeer
ctest --show-only=json-v1 -L '^integration-tcp$' > "${RUNNER_TEMP}/tcp-discovery.json"
ctest --output-on-failure -L '^integration-tcp$' --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/tcp-baseline.xml"
python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/tcp-baseline.xml" --require-tcp-ran --require-run-ahead --discovery "${RUNNER_TEMP}/tcp-discovery.json"
python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/tcp-baseline.xml" --require-entry-passed TcpLane.SupervisorProtocolControls --require-entry-passed TcpLane.EventForfeitPeer --require-entry-passed TcpLane.ControlFrameFuzz
# Both full transport runs are hard gates. The per-case comparison below
# remains diagnostic because explicit mechanism controls retain their own mode.
#
# The entries that name MOBILEGL_TRANSPORT in their OWN property (the Split. lanes) keep
# their value in both passes, which is correct: they are the split family in both arms and
# the comparison is about the other 1100.
- name: Transport A/B (monolith vs inproc)
working-directory: build-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH: "1"
MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS: "1"
MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER: "1"
run: |
ulimit -c unlimited
if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi
count=$(ctest -N -L integration-gpu | grep -cE '^ *Test *#[0-9]+:')
if [ "${count}" -lt 1 ]; then
echo "::error::the split runtime registers ${count} integration-gpu entries"
exit 1
fi
echo "integration-gpu entries in the split build: ${count}"
MOBILEGL_TRANSPORT=monolith ctest --output-on-failure -L integration-gpu --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/arm-monolith.xml"
MOBILEGL_TRANSPORT=inproc MOBILEGL_IPC_ROLE_SPLIT_STATE=1 MOBILEGL_IPC_STRICT_ERRORS=1 ctest --output-on-failure -L integration-gpu --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/arm-inproc.xml"
# Both commands above must succeed; report differences among legitimate skips.
python3 - "${RUNNER_TEMP}/arm-monolith.xml" "${RUNNER_TEMP}/arm-inproc.xml" <<'PY'
import sys, xml.etree.ElementTree as ET
def rows(path):
out = {}
for case in ET.parse(path).getroot().iter('testcase'):
status = 'passed'
if case.find('failure') is not None or case.find('error') is not None:
status = 'failed'
elif case.find('skipped') is not None or case.get('status') in ('notrun', 'disabled'):
status = 'skipped'
out[case.get('name')] = status
return out
a, b = rows(sys.argv[1]), rows(sys.argv[2])
diff = sorted(set(a) ^ set(b)) + sorted(n for n in set(a) & set(b) if a[n] != b[n])
if diff:
for name in diff[:40]:
print(f"{name}: monolith={a.get(name, '<absent>')} inproc={b.get(name, '<absent>')}")
print(f"Transport comparison: {len(diff)} name/status differences after both full runs passed")
else:
print(f"the two arms agree on all {len(a)} entries, name and status")
PY
# THE RUNTIME HALF OF "THIS IS REALLY A SPLIT BUILD". The build-level nm check in
# build-linux-split proves the library CARRIES MG_Remote; this proves the transport
# RESOLVED in a process of this lane. ConfigLoader::InitTransport logs one INFO line when it
# selects InProcess, and the DirectGLES.Split.PersistentMapArm. entry retains its private
# counting log (all Split entries now have private paths; nothing else writes this file).
# The line is written during bring-up, before any scenario decides to skip, so this
# check is live from the day the lanes land rather than from the day they stop skipping.
- name: Transport resolution check
working-directory: build-split
run: |
# P6: the sink writes one file per ROLE from the base MOBILEGL_LOG_FILE_PATH; the
# transport-resolution line is ConfigLoader's, on the CLIENT thread, so it is in the
# .client half. The base name itself names no file now.
log=MobileGL/MG_IntegrationTest/persistent-map-arm-split-DirectGLES.client.log
if [ ! -f "${log}" ]; then
echo "::error::${log} does not exist: the DirectGLES.Split.PersistentMapArm. entry never ran (or the per-role split renamed it), so nothing in this job establishes that MOBILEGL_TRANSPORT ever resolved to inproc in a live process"
exit 1
fi
# THE DISTINCTIVE PART OF THE INFO LINE, not the bare KEY=VALUE (review finding M-5):
# ConfigLoader logs `Config: Accepted env variable: MOBILEGL_TRANSPORT=inproc` for the
# env dump too, unconditionally and in a PULL build, and at DEBUG that line is live.
if ! grep -q "MOBILEGL_TRANSPORT=inproc - the MGPipe record stream" "${log}"; then
echo "::error::${log} carries no transport-resolution line. ConfigLoader::InitTransport logs it at INFO when it selects InProcess, and that code exists only in a MOBILEGL_BUILD_DISAGGREGATED build - so this lane ran a monolith library while claiming to be the split lane. (A bare MOBILEGL_TRANSPORT=inproc substring is NOT accepted: the env dump prints one in every build.)"
exit 1
fi
echo "the split lane resolved MOBILEGL_TRANSPORT=inproc"
# EXIT GATE E1's NEGATIVE CONTROL and EXIT GATE E3(a)'s, in one step because they have the
# same two-state shape and the same reason for it.
#
# Both controls turn a knob that MUST make a split scenario red: MOBILEGL_IPC_VERB_BARRIER=0
# removes the lockstep fence R-1 rests on, and MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0 turns the
# persistent-map push off. A control of the verify lane's shape - "this step passes when
# ctest FAILS" - cannot be written yet, because while packages c1/s1/v1 are landing the
# Split entries SKIP and ctest reports green whatever the knob says, so an unconditional
# control would be red for the whole of P5 for a reason that is not a defect.
#
# So the expected state is DERIVED FROM BEHAVIOUR rather than assumed. The first version read
# MGITEST_REMOTE_CLIENT_PRESENT out of the generated *_tests.cmake, which was a restatement of
# the CMake source probe review finding M-1 falsified; the arming condition is a runtime fact
# inside each test process (MG_Config::Transport, ClientSession::Active() and
# ImplementedVerbCount(), read by Harness/SplitRuntimePeek), so the only honest way to ask it
# from a shell is to look at what the entries DID. When entries passed, the controls MUST
# fire; when every one of them skipped, the step says so loudly and does not pretend.
#
# THE BODY OF THIS STEP IS scripts/ci/split_negative_controls.sh, and the move is the point
# rather than tidiness. A `run:` block executes nowhere but on a runner, so these lines were
# unreviewable and untestable: when the wave-1 cross-family review said they were broken,
# CONFIRMING it needed a hand-made copy of them (wave1-codex-verify.md 8), and a copy is not
# the thing. scripts/ci/control_smoke_test.sh now drives the very file this step runs.
#
# What that smoke test pins, and what ID-46 finding 8 found missing: each control asserts its
# OWN failure reason. A non-zero ctest exit used to be enough, so a timeout, a setup abort or
# any unrelated assertion printed "turned N selected entries red, as it must" and this step
# went green. The arming run's `|| true` had the matching defect - it counted a case that ran
# and FAILED as evidence the lane was live, so the controls could be measured against a
# baseline that was already red.
- name: Negative controls (barrier, map push)
if: ${{ !cancelled() }}
working-directory: build-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
CONTROL_TMPDIR: ${{ runner.temp }}
run: bash "${GITHUB_WORKSPACE}/scripts/ci/split_negative_controls.sh"
# The run log needs a signed-in reader; annotations do not. Name every failed ctest entry,
# with the lines of its output that say why, where anyone reading the run can see them.
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}"
- name: Upload split lane logs
if: ${{ always() && needs.select-runner.result == 'success' }}
uses: actions/upload-artifact@v7
with:
name: integration-split-logs
path: |
build-split/MobileGL/MG_IntegrationTest/*.log*
build-split/MobileGL/MG_IntegrationTest/split-logs/*.log
${{ runner.temp }}/arm-inproc.xml
${{ runner.temp }}/arm-monolith.xml
if-no-files-found: warn
- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v7
with:
name: integration-split-core-dumps
path: /tmp/core.*
if-no-files-found: ignore
# Shares build-linux-split with the functional/transport job above. This job
# checks state ownership: strict residual-read rules, distinct client/server
# blocks, zero-residual assertions, and complete per-entry result accounting.
# Configured controls must pass without skips. Capability gaps and known
# limitations are reported separately and accepted only by exact name/reason.
integration-split-strict:
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
timeout-minutes: 60
needs: [select-runner, build-linux-split]
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
if: runner.environment != 'self-hosted'
uses: lukka/get-cmake@v4.3.3
- name: Install runtime dependencies
if: runner.environment != 'self-hosted'
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers vulkan-validationlayers
- name: Download Linux split runtime
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-runtime-split
path: .
- name: Unpack Linux split runtime
run: |
tar -xzf mobilegl-linux-runtime-split.tgz
test -f build-split/libMobileGL.so
- name: Normalize CTest command paths
run: |
python - <<'PY'
from pathlib import Path
import re
for path in Path('build-split').rglob('CTestTestfile.cmake'):
text = path.read_text()
text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text)
path.write_text(text)
PY
# The green half: no unit case may pull harder under strict. This includes the
# RemoteGuards suite (the layer-1/layer-2 Fatal drives) and FieldOwnershipTest's strict
# arms, so the guards are provably armed in the lane that gates on them.
- name: Unit tests on the split runtime (strict)
working-directory: build-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit-strict.xml"
# ---- P5e (ra), MG_Remote/CONTRACT-P5E.md §7: ONE STEP, TWO ARMS, AND THE TREE PICKS ----
#
# This lane was written as an EXPECTED-RED control: under strict every integration-split
# entry aborts on its first BARRIER-PULLED read, so the step gated on the run failing and
# on the strict marker being the reason. P5e turns it into a HARD GREEN GATE - once the
# client runs ahead, an unbarriered record's pull is a Fatal whatever this knob says
# (§3.3), so what strict adds is only the BARRIERED rows' pulls, and those are a short,
# named list. A revert of any handle arm in vi/sb/pg/tx2/fb then goes red HERE, by field
# and verb, instead of in a picture nobody diffs.
#
# THE ARM IS CHOSEN BY THE TREE, NOT BY A SECOND EDIT. kMGPipeP5eRunAheadReady in
# MG_Backend/Init.cpp is the one constant the integration commit flips, and it is exactly
# the fact this lane's expectation depends on: with it false no server publishes
# kCapRunAheadApply, every record is barriered, every scenario still aborts on its first
# pull and the expected-red control is the honest gate. Reading it here means the lane
# changes meaning in the same commit the runtime does - a lane whose flip is a separate
# human step is a lane that is wrong for as long as that step is outstanding.
- name: Strict split lane
working-directory: build-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
MOBILEGL_TRANSPORT: "inproc"
run: |
ulimit -c unlimited
ready=$(grep -oE 'kMGPipeP5eRunAheadReady = (true|false)' "${GITHUB_WORKSPACE}/MobileGL/MG_Backend/Init.cpp" | head -1 | awk '{print $3}')
echo "kMGPipeP5eRunAheadReady = ${ready}"
helper="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py"
expected="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/strict-expected-markers.txt"
# THE LOG SET COMES FROM ctest, NOT FROM A DIRECTORY (P5e gl, ID-119). This step used to
# grep MobileGL/MG_IntegrationTest/split-logs/, which holds 90 of the lane's entries.
# The 26 it could not see were the F1. readback block, both NamedBlit pairs, the four
# Ct. entries and PersistentMapArm - i.e. PRECISELY the readback population the
# allowlist is about, plus the only DirectVulkan entries in the lane. A directory is a
# guess about where the lane put its logs; `--show-only=json-v1` is the lane saying so.
ctest -L integration-split --show-only=json-v1 > "${RUNNER_TEMP}/lane.json"
# Stale logs from an earlier job would be read as this run's evidence: an entry that
# never starts leaves the previous file in place, and "no marker" is exactly what a
# lane that never ran looks like.
python3 - "${RUNNER_TEMP}/lane.json" "${helper}" <<'PY'
import importlib.util, json, os, sys
spec = importlib.util.spec_from_file_location("slp", sys.argv[2])
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
logs = slp.marker_log_paths(json.load(open(sys.argv[1])))
for path in set(logs.values()):
try:
os.unlink(path)
except OSError:
pass
print(f"reset {len(set(logs.values()))} private log(s) across {len(logs)} entries")
PY
rc=0
ctest -L integration-split --no-tests=error -j 4 > "${RUNNER_TEMP}/strict-split.txt" 2>&1 || rc=$?
tail -5 "${RUNNER_TEMP}/strict-split.txt"
echo "strict run: rc=${rc}"
python3 "${GITHUB_WORKSPACE}/scripts/gen_pipe_field_ownership.py" --print-admitted \
> "${RUNNER_TEMP}/allowed.txt"
echo "§7 allowlist (derived): $(wc -l < "${RUNNER_TEMP}/allowed.txt") admitted pair(s)"
if [ "${ready}" != "true" ]; then
# ---- the pre-flip arm ----
if [ "${rc}" -eq 0 ]; then
echo "::error::integration-split under MOBILEGL_IPC_STRICT_ERRORS=1 went GREEN while kMGPipeP5eRunAheadReady is false - the object-class residual pulls (P3b/P4b/P7) are still pinned, so strict must abort every drawing scenario. If P5e has just landed, flip that constant in the same commit."
exit 1
fi
# The census, and the ADMITTED half of the ratchet, run in this arm too: the Fatal
# half is expected here (it is what "red by design" MEANS), but a debt that appeared
# or vanished on the admitted side is news whichever arm the tree is in.
python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \
"${RUNNER_TEMP}/allowed.txt" "${expected}"
echo "integration-split under strict is red by design, and its marker census above is the evidence"
exit 0
fi
# ---- the post-flip arm: HARD GREEN, two-sided ----
#
# EVERY ENTRY MUST COMPLETE, and the census is what says why when one does not: an abort
# is either an unbarriered pull (§3.3's unconditional Fatal, a missed migration) or a
# barriered pull nothing admits, and both come out of the census by field and verb.
if [ "${rc}" -ne 0 ]; then
echo "::error::integration-split under MOBILEGL_IPC_STRICT_ERRORS=1 is RED on a run-ahead server. CONTRACT-P5E §7 makes this lane a hard green gate; the census below names the field and the verb of every entry that aborted."
python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \
"${RUNNER_TEMP}/allowed.txt" "${expected}" || true
exit 1
fi
# THE ADMITTED MARKERS COME FROM THE GENERATOR, NOT FROM THIS FILE (ID-116).
#
# A hand-kept copy of §7 lived here for one round and was wrong in BOTH directions at
# once: it omitted GetFramebufferBindingSlot@ReadPixels - 21 entries, the largest
# survivor in the lane - and it carried GetTextureObject@CopyImageSubData, which
# nothing made safe, because resource_copy_region is kWaitNone. Neither error is
# visible by reading the list; both are obvious the moment the list is DERIVED.
#
# The rule is ID-84 said in tables that already exist (ID-116, refined by ID-125): a
# <field>@<verb> pull is admitted iff the field's ownership row is BARRIER_PULLED, the
# verb has a stamp row, the field is inside the verb class's may-read mask, and EITHER
# the verb's wire op is statically barriered (kMGPipeWaitClasses != kWaitNone, i.e. the
# client really is parked behind the record) OR the field's retiring phase does not
# name P5e. The second disjunct is the honest reading of ID-84: this comparison only
# ever sees pulls on records the server stamped BARRIERED - an unbarriered one aborts
# before any of this - so the pull is legal by construction and the only question left
# is whether THIS phase still owes the migration. Without it the lane would go red on
# GetTransformFeedbackProgram@DrawArrays, a row CONTRACT-P5E §5.7 rules out of P5e
# entirely and no package here is allowed to touch.
# `--print-admitted` prints exactly the table MGPipeBarrierPullAdmitted() answers
# from, so this shell and the C++ cannot drift: retiring a verb's wait class narrows
# the lane with no edit here, and a new BARRIER_PULLED row widens it only on verbs
# that wait.
# THE RATCHET, BOTH SIDES (ID-119). `no-fatal` is what makes this arm the hard gate:
# - any Fatal marker fails;
# - an ADMITTED marker outside the generated allowlist fails, which can only mean the
# committed PipeFieldOwnership.inc and the generator disagree;
# - a pair that appears and is not in strict-expected-markers.txt fails: it is a debt
# somebody now owes and it gets written down with the phase that retires it;
# - AND a pair in that file that no longer appears fails, so the lane CANNOT ROT
# GREEN. Without that half, a retired debt leaves its row behind as a permanent
# licence and the next regression to re-introduce the read reads as "expected".
# ADMITTED-ESCALATED markers are checked against the expected set but NOT against the
# allowlist: they are a runtime fact about the record's payload (an open XFB span, a
# draw carrying client vertex arrays) that no table indexed by (field, verb) can carry,
# and widening the static list to hold them would forgive the ordinary draw path too.
python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \
"${RUNNER_TEMP}/allowed.txt" "${expected}" no-fatal
# `rsp`, AND THE PIN THAT REPLACED A VACUOUS ONE (ID-119). What stood here was a grep
# for `rsp=` over split-logs/ - a counter no entry in that directory emits, because
# PipeStats is off unless a lane asks for it - testing a sentence that was true of every
# possible implementation: CountBarrierPull's unbarriered arm is [[noreturn]] and runs
# BEFORE ++g_residualPulls, so an unbarriered pull can never reach the counter whatever
# the code does. It has been replaced by an assertion inside a lane entry that actually
# counts: DirectGLES.Split.StrictArming., which runs with MOBILEGL_PIPE_STATS=1,
# MOBILEGL_PIPE_STATS_PERIOD=1 and a private log path of its own (a whole-lane env flip
# races under -j, since the library opens that path "w"), and asserts
# vbs > 0 - the server stamped a verb boundary on a DRAWING inproc entry, which is
# ID-115's positive control: everything strict checks is downstream of
# that stamp, and before it existed "the lane is green" and "strict was
# never armed" were the same observation;
# rsp == 0 or rsp >= draws - the checkable shape of the debt. While the draw path's
# pull is unretired every draw pulls, so rsp scales with draws; the
# retirement is visible as rsp ceasing to scale. That is what the device
# measured (rsp ~= the draw count under inproc, 0 under monolith).
# So the assertion lives where the counter is, and this step only has to insist the
# entry RAN - a positive control that can be skipped away is not one.
if ! grep -q 'DirectGLES\.Split\.StrictArming\.' "${RUNNER_TEMP}/strict-split.txt"; then
echo "::error::the strict lane did not run DirectGLES.Split.StrictArming., so nothing in this run proves the server ever stamped a verb boundary. A green lane without it is indistinguishable from a lane where strict was never armed (ID-115)."
exit 1
fi
echo "integration-split under strict is GREEN, two-sided, with the arming control present"
# ---- P5f (f1), P5F-WIRE-COMPLETENESS.md §4: THE DUAL-BLOCK CENSUS LANE ---------------
#
# The same integration-split entries (plus Magma's own lane), re-run under
# These entries own assertions removed from generic transport registrations.
# Discovery, execution and zero-skip accounting must agree on the exact set.
- name: Specialized configuration controls
working-directory: build-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
run: |
helper="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py"
ctest --show-only=json-v1 > "${RUNNER_TEMP}/specialized-all.json"
python3 "${helper}" select-specialized "${RUNNER_TEMP}/specialized-all.json" \
"${RUNNER_TEMP}/specialized-selected.json" "${RUNNER_TEMP}/specialized-tests.txt"
ctest --tests-from-file "${RUNNER_TEMP}/specialized-tests.txt" --no-tests=error -j 4 \
--output-on-failure --output-junit "${RUNNER_TEMP}/specialized.xml"
python3 "${helper}" specialized-results "${RUNNER_TEMP}/specialized-all.json" \
"${RUNNER_TEMP}/specialized.xml"
# MOBILEGL_IPC_ROLE_SPLIT_STATE=1: the client's residual fill writes a client-role block
# and the backend reads a server-role one, so every BARRIER-PULLED read aborts with a
# named Fatal{UnmigratedPipeInput, "<field>@<verb>"}. THE RED IS THE CENSUS - it is the
# machine-checked list of what still depends on the two roles sharing one object, and it
# is ratcheted two-sided against Harness/dualblock-expected-fatals.txt: a new pair is a
# debt somebody now owes, a vanished pair must be removed in the commit that retired it.
#
# THE STEP ALSO FAILS WHEN THE LANE IS GREEN while the expected file still has rows -
# "green" would mean the knob stopped arming, which is exactly the decay ID-122 assigns
# to a control rather than to a lane's good mood. At P5f's exit the file is empty and the
# green arm is the correct one; delete the red expectation in the same commit.
# P5f wave 2: with the census empty, this lane is a hard green gate.
- name: Dual-block census lane
working-directory: build-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_TRANSPORT: "inproc"
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
run: |
ulimit -c unlimited
helper="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py"
expected="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/dualblock-expected-fatals.txt"
ctest -L 'integration-split|integration-magma-split' --show-only=json-v1 \
> "${RUNNER_TEMP}/dualblock-lane.json"
# Stale logs from an earlier step would be read as this run's evidence (ID-119's rule).
python3 - "${RUNNER_TEMP}/dualblock-lane.json" "${helper}" <<'PY'
import importlib.util, json, os, sys
spec = importlib.util.spec_from_file_location("slp", sys.argv[2])
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
logs = slp.marker_log_paths(json.load(open(sys.argv[1])))
for path in set(logs.values()):
try:
os.unlink(path)
except OSError:
pass
print(f"reset {len(set(logs.values()))} private log(s) across {len(logs)} entries")
PY
rc=0
ctest -L 'integration-split|integration-magma-split' --no-tests=error -j 4 \
--output-junit "${RUNNER_TEMP}/dualblock.xml" \
> "${RUNNER_TEMP}/dualblock-split.txt" 2>&1 || rc=$?
tail -5 "${RUNNER_TEMP}/dualblock-split.txt"
echo "dual-block census run: rc=${rc}"
rows=$(awk '!/^[[:space:]]*(#|$)/ { n++ } END { print n+0 }' "${expected}")
if [ "${rows}" -eq 0 ] && [ "${rc}" -ne 0 ]; then
echo "::error::empty dual-block census requires ctest success, including infrastructure/execution status"
exit "${rc}"
fi
if [ "${rc}" -eq 0 ] && [ "${rows}" -gt 0 ]; then
echo "::error::the dual-block census lane went GREEN with ${rows} expected pair(s) outstanding - the knob stopped arming, or the fields retired and dualblock-expected-fatals.txt was not emptied in the same commit"
exit 1
fi
python3 "${helper}" expect-fatal "${RUNNER_TEMP}/dualblock-lane.json" \
"${RUNNER_TEMP}/dualblock.xml" "${expected}"
# The per-frame zero-residual-read case must execute and pass on both backends.
- name: P5f per-frame zero-residual check
working-directory: build-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
run: |
ctest -L integration-p5f-rsp --show-only=json-v1 > "${RUNNER_TEMP}/rsp-lane.json"
ctest -L integration-p5f-rsp --no-tests=error --output-on-failure \
--output-junit "${RUNNER_TEMP}/rsp.xml"
python3 "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
require-green "${RUNNER_TEMP}/rsp-lane.json" "${RUNNER_TEMP}/rsp.xml" \
DirectGLES.Split.P5fRsp.F1WireScenario.EachWireFrameHasZeroResidualPulls \
DirectVulkan.Split.P5fRsp.F1WireScenario.EachWireFrameHasZeroResidualPulls
# The mechanism's own negative control (P5F §6): the same named case must be green with
# the knob on and red - on its own assertion text - with it off.
- name: Dual-block negative control (P5f f1)
working-directory: build-split
run: bash "${GITHUB_WORKSPACE}/scripts/ci/dualblock_negative_control.sh"
# ---- MAGMA'S OWN WIRE LANE: FRAMEBUFFER, TEXTURE AND APPLICATION BUFFERS ------------
#
# P5f fm: Magma's framebuffer, copy and mipmap verbs consume server records.
# The label remains separate so both backends' coverage stays explicit.
- name: Magma split lane (strict, dual-block)
working-directory: build-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
MOBILEGL_TRANSPORT: "inproc"
run: |
ctest -L integration-magma-split --no-tests=error -j 2 --output-on-failure
# ---- P7 PACKAGE L: THE SAME MAGMA CASES IN TWO PROCESSES -----------------------------
#
# EXIT GATE 1's FIRST HALF, and until this package there was nothing to run: every arm of
# the split registration macro hardcoded MOBILEGL_BACKEND_TYPE=DirectGLES, so the shared
# `integration-split` / `-spawn` / `-tcp` labels held ZERO DirectVulkan entries and
# `integration-magma-split` was inproc by construction. These two lanes are the same
# scenarios the step above runs, with the transport as their one difference - so a red
# here that the step above did not see is a TWO-PROCESS fact and not a Magma one.
#
# THEY ARE GATED BECAUSE THEY MEASURED GREEN, and only for that reason: 52 of 52 entries
# on each arm, with the two ClipDistance cases skipping for the reason the inproc arm's
# allowlist already names. The 21 entries the inproc arm has and these do not are
# MagmaRunAheadScenario and MagmaWireCacheScenario, which are inproc-only BY
# CONSTRUCTION (they hold and peek THIS process's server loop); the CMakeLists and
# spawn_lane_parity.py both name them rather than widening a tolerance.
#
# THE INFORMATIONAL TIER - `integration-magma-all-{split,spawn,tcp}`, every scenario the
# split arms run, on Magma - IS DELIBERATELY NOT HERE. Its job is to be able to go red
# while CI stays green; a case moves into a gating step with the commit that retires its
# blocker.
- name: Magma spawn and TCP lanes
working-directory: build-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
run: |
ulimit -c unlimited
if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi
ctest -L '^integration-magma-spawn$' --no-tests=error -j 2 --output-on-failure
ctest -L '^integration-magma-tcp$' --no-tests=error -j 2 --output-on-failure
# Every named Magma application-buffer wire case must execute and pass.
- name: Magma buffer wire cases (inproc)
working-directory: build-split
run: |
ctest -L integration-magma-buffers --show-only=json-v1 > "${RUNNER_TEMP}/magma-buffer-lane.json"
ctest -L integration-magma-buffers --no-tests=error -j 2 --output-on-failure \
--output-junit "${RUNNER_TEMP}/magma-buffers.xml"
python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
"${RUNNER_TEMP}/magma-buffer-lane.json" "${RUNNER_TEMP}/magma-buffers.xml" <<'PY'
import importlib.util, json, sys
spec = importlib.util.spec_from_file_location("slp", sys.argv[1])
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
cases = [
"F1WireScenario.EnabledVertexBufferDrawKeepsNamedP7Fatal",
"F1WireScenario.UniformBufferRangeAndRebindPixels",
"F1WireScenario.ShortUniformBufferRangePadsMissingBytes",
"F1WireScenario.SubWordUniformBufferRangePadsMissingBytes",
"F1WireScenario.UnalignedAtomicCounterRangeDeclinesAndTheSessionLives",
"F1WireScenario.ComputeWrittenVertexAndIndexBuffersDrawAndReadBack",
"F1WireScenario.ColorBlitToDefaultIgnoresViewportAndPreservesOrientation",
"F1WireScenario.ColorBlitToDefaultHonorsScissorAndReversedRect",
"IndexedDrawFamilyScenario.AnElementBufferDrawElementsOffsetSelectsTheRightQuad",
"IndexedDrawFamilyScenario.DrawElementsBaseVertexMovesTheSameIndicesToTheOtherQuad",
"IndexedDrawFamilyScenario.DrawRangeElementsDrawsTheRangedRun",
"IndexedDrawFamilyScenario.DrawElementsInstancedBaseVertexPaintsOneQuadPerInstance",
"IndexedDrawFamilyScenario.MultiDrawElementsBaseVertexPaintsEverySubDraw",
"IndexedDrawFamilyScenario.MultiDrawElementsIndirectDrawsFromTheIndirectBuffer",
"CrossFrameBufferScenario.VertexBufferSubData",
"CrossFrameBufferScenario.IndexBufferSubData",
"CrossFrameBufferScenario.VertexOrphanAndReupload",
"CrossFrameBufferScenario.IndexOrphanAndReupload",
"PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw",
"PersistentCoherentMapScenario.AWriteAfterAFrameBoundaryReachesTheNextFramesDraw",
"StorageBufferRegrowScenario.AGrownStoreIsVisibleThroughItsExistingIndexedBinding",
# P11 A2: the >= 16 MiB arena and its tier assertion (emulated).
"LargeArenaAdoptionScenario.SubDataAfterAnInFlightDrawReachesTheNextDraw",
"LargeArenaAdoptionScenario.RespecifiedVertexArenaKeepsVaoBindings",
"LargeArenaAdoptionScenario.RespecifiedIndexArenaKeepsVaoBinding",
"LargeArenaAdoptionScenario.ReadbackSeesTheLatestCpuWrite",
"LargeArenaAdoptionScenario.GpuWriteIntoTheArenaIsReadBack",
"LargeArenaAdoptionScenario.TheArenaLandsInTheTierItsLaneDeclares",
]
slp.require_green(json.load(open(sys.argv[2])), sys.argv[3],
["DirectVulkan.Split.Buffers." + name for name in cases])
PY
# P7 package L: THE SAME TWENTY-ONE, BY NAME, IN TWO PROCESSES (nineteen at package L; the
# sub-word uniform range and the unaligned atomic-counter decline joined later).
#
# `--no-tests=error` in the lane step above answers "did something run and pass"; it does
# not answer "did these twenty-one run". require_green does, and it is the check that would
# catch the failure mode this package could plausibly introduce - an arm that registers
# SOME of the lane and goes green on a smaller set. The label spelling is what keeps the
# three arms apart under `ctest -L`'s regex: the inproc arm keeps the historical
# `integration-magma-buffers` (which is what the step above require_greens), and the new
# arms put the arm word FIRST, so neither selection can reach the other.
#
# TcpServer.Start / TcpServer.Stop ARE IN THE tcp ARM'S REQUIRED LIST ON PURPOSE. Every
# tcp entry declares FIXTURES_REQUIRED mobilegl-tcp, so ctest lists the supervisor's two
# fixture entries under the label as well - and they must be green too: a lane whose
# supervisor failed to start has nothing to say about the twenty-one.
- name: Magma buffer wire cases (spawn, TCP)
working-directory: build-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
run: |
for arm in spawn tcp; do
case "${arm}" in
spawn) prefix=DirectVulkan.Spawn.Buffers. ;;
tcp) prefix=DirectVulkan.Tcp.Buffers. ;;
esac
label="integration-magma-${arm}-buffers"
ctest -L "^${label}$" --show-only=json-v1 > "${RUNNER_TEMP}/magma-buffer-${arm}.json"
ctest -L "^${label}$" --no-tests=error -j 2 --output-on-failure \
--output-junit "${RUNNER_TEMP}/magma-buffers-${arm}.xml"
python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
"${RUNNER_TEMP}/magma-buffer-${arm}.json" "${RUNNER_TEMP}/magma-buffers-${arm}.xml" \
"${prefix}" "${arm}" <<'PY'
import importlib.util, json, sys
spec = importlib.util.spec_from_file_location("slp", sys.argv[1])
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
cases = [
"F1WireScenario.EnabledVertexBufferDrawKeepsNamedP7Fatal",
"F1WireScenario.UniformBufferRangeAndRebindPixels",
"F1WireScenario.ShortUniformBufferRangePadsMissingBytes",
"F1WireScenario.SubWordUniformBufferRangePadsMissingBytes",
"F1WireScenario.UnalignedAtomicCounterRangeDeclinesAndTheSessionLives",
"F1WireScenario.ComputeWrittenVertexAndIndexBuffersDrawAndReadBack",
"F1WireScenario.ColorBlitToDefaultIgnoresViewportAndPreservesOrientation",
"F1WireScenario.ColorBlitToDefaultHonorsScissorAndReversedRect",
"IndexedDrawFamilyScenario.AnElementBufferDrawElementsOffsetSelectsTheRightQuad",
"IndexedDrawFamilyScenario.DrawElementsBaseVertexMovesTheSameIndicesToTheOtherQuad",
"IndexedDrawFamilyScenario.DrawRangeElementsDrawsTheRangedRun",
"IndexedDrawFamilyScenario.DrawElementsInstancedBaseVertexPaintsOneQuadPerInstance",
"IndexedDrawFamilyScenario.MultiDrawElementsBaseVertexPaintsEverySubDraw",
"IndexedDrawFamilyScenario.MultiDrawElementsIndirectDrawsFromTheIndirectBuffer",
"CrossFrameBufferScenario.VertexBufferSubData",
"CrossFrameBufferScenario.IndexBufferSubData",
"CrossFrameBufferScenario.VertexOrphanAndReupload",
"CrossFrameBufferScenario.IndexOrphanAndReupload",
"PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw",
"PersistentCoherentMapScenario.AWriteAfterAFrameBoundaryReachesTheNextFramesDraw",
"StorageBufferRegrowScenario.AGrownStoreIsVisibleThroughItsExistingIndexedBinding",
"LargeArenaAdoptionScenario.SubDataAfterAnInFlightDrawReachesTheNextDraw",
"LargeArenaAdoptionScenario.RespecifiedVertexArenaKeepsVaoBindings",
"LargeArenaAdoptionScenario.RespecifiedIndexArenaKeepsVaoBinding",
"LargeArenaAdoptionScenario.ReadbackSeesTheLatestCpuWrite",
"LargeArenaAdoptionScenario.GpuWriteIntoTheArenaIsReadBack",
"LargeArenaAdoptionScenario.TheArenaLandsInTheTierItsLaneDeclares",
]
required = [sys.argv[4] + name for name in cases]
if sys.argv[5] == "tcp":
required += ["TcpServer.Start", "TcpServer.Stop"]
slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], required)
PY
done
# Run-ahead must queue real GPU work and enforce credits 1 and 3.
- name: Magma run-ahead cases (credits 1, 3)
working-directory: build-split
run: |
ctest -L integration-magma-runahead --show-only=json-v1 > "${RUNNER_TEMP}/magma-runahead-lane.json"
ctest -L integration-magma-runahead --no-tests=error -j 2 --output-on-failure \
--output-junit "${RUNNER_TEMP}/magma-runahead.xml"
python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
"${RUNNER_TEMP}/magma-runahead-lane.json" "${RUNNER_TEMP}/magma-runahead.xml" <<'PY'
import importlib.util, json, sys
spec = importlib.util.spec_from_file_location("slp", sys.argv[1])
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
cases = [
"QueuedBufferVersionsAndDeletedNameReuseKeepTheirPixels",
"QueuedProgramRebindsKeepEachUniformSnapshot",
"QueuedGpuWritesReadBackTheLastDispatchBytes",
"MixedNativeAndConvertedGpuVertexStreamsKeepTheirReservation",
"TextureUploadCannotOvertakeThePreviouslyQueuedDraw",
"GpuClearSurvivesTheFirstStorageImageUpgrade",
"PresentCreditActuallyParksTheClientAtItsConfiguredLimit",
]
required = [f"DirectVulkan.Split.RunAhead.Credit{credit}.MagmaRunAheadScenario.{name}"
for credit in (1, 3) for name in cases]
slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], required)
PY
# Run-ahead off must go red on the lost queue lead; run-ahead back on must be green.
- name: Magma run-ahead negative control
run: |
python3 scripts/ci/magma_runahead_checks.py negative --build-dir build-split \
--out "${RUNNER_TEMP}/magma-runahead-negative" --execute
- name: Magma run-ahead sync validation
run: |
python3 scripts/ci/magma_runahead_checks.py validation --build-dir build-split \
--out "${RUNNER_TEMP}/magma-runahead-syncval" --execute
- name: Magma view/framebuffer cache sync validation
run: |
python3 scripts/ci/magma_cache_checks.py --build-dir build-split \
--out "${RUNNER_TEMP}/magma-cache-syncval" --execute
- name: Upload Magma run-ahead and cache evidence
if: ${{ always() && needs.select-runner.result == 'success' }}
uses: actions/upload-artifact@v7
with:
name: magma-runahead-checks
path: |
${{ runner.temp }}/magma-runahead-negative
${{ runner.temp }}/magma-runahead-syncval
${{ runner.temp }}/magma-cache-syncval
if-no-files-found: ignore
# The packed-float chain is generated by a real GPU shader blit, so this entry sits in the
# integration-magma-split lane with its siblings (CMakeLists.txt's fm loop) and carries
# their pixels assertion. Its named check survives the flip in the only direction that has
# evidence behind it: the case must EXECUTE, PASS, and never skip - a lane-level exit code
# alone would not separate "generated" from "quietly absent".
- name: Magma packed-float mipmap case (P7)
working-directory: build-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
MOBILEGL_TRANSPORT: "inproc"
run: |
p7_case='^DirectVulkan\.Split\.Fm\.F1WireScenario\.GenerateMipmapPackedFloatPixels$'
ctest -R "${p7_case}" --show-only=json-v1 > "${RUNNER_TEMP}/magma-p7-lane.json"
ctest -R "${p7_case}" --no-tests=error --output-on-failure \
--output-junit "${RUNNER_TEMP}/magma-p7.xml"
python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
"${RUNNER_TEMP}/magma-p7-lane.json" "${RUNNER_TEMP}/magma-p7.xml" <<'PY'
import importlib.util, json, sys
spec = importlib.util.spec_from_file_location("slp", sys.argv[1])
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
name = "DirectVulkan.Split.Fm.F1WireScenario.GenerateMipmapPackedFloatPixels"
slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], [name])
PY
# Client vertex/index inputs are copied into owned wire buffer resources.
# The original dedicated entries now have to execute successfully with run-ahead.
- name: Client-array split lane (run-ahead)
working-directory: build-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_TRANSPORT: "inproc"
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
MOBILEGL_IPC_RUN_AHEAD: "1"
run: |
ctest -L integration-clientarrays-split --show-only=json-v1 > "${RUNNER_TEMP}/clientarrays-lane.json"
ctest -L integration-clientarrays-split --no-tests=error --output-on-failure -j 1 \
--output-junit "${RUNNER_TEMP}/clientarrays.xml"
python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
"${RUNNER_TEMP}/clientarrays-lane.json" "${RUNNER_TEMP}/clientarrays.xml" <<'PY'
import importlib.util, json, sys
spec = importlib.util.spec_from_file_location("slp", sys.argv[1])
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
lane = json.load(open(sys.argv[2]))
names = [test["name"] for test in lane["tests"]]
if not names:
raise SystemExit("no client-array cases discovered")
slp.require_green(lane, sys.argv[3], names)
PY
# The run log needs a signed-in reader; annotations do not. Name every failed ctest entry,
# with the lines of its output that say why, where anyone reading the run can see them.
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}"
- name: Upload strict lane logs
if: ${{ always() && needs.select-runner.result == 'success' }}
uses: actions/upload-artifact@v7
with:
name: integration-split-strict-logs
path: |
build-split/MobileGL/MG_IntegrationTest/*.log*
build-split/MobileGL/MG_IntegrationTest/split-logs/*.log
${{ runner.temp }}/specialized*.json
${{ runner.temp }}/specialized-tests.txt
${{ runner.temp }}/specialized.xml
${{ runner.temp }}/dualblock-lane.json
${{ runner.temp }}/dualblock.xml
${{ runner.temp }}/dualblock-split.txt
if-no-files-found: warn
# MobileGL/MG_Remote/Protocol/generated/protocol_generated.h is COMMITTED, and
# flatc is deliberately absent from the default build graph (a codegen step in
# the graph is how the earlier branch ended up cross-compiling an arm64 flatc
# and trying to run it on the host). This job is what keeps the committed
# header honest: build the pinned flatc, regenerate, and fail on any diff.
# It needs no MobileGL build, so it does not depend on build-linux.
flatc-check:
needs: select-runner
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
if: runner.environment != 'self-hosted'
uses: lukka/get-cmake@v4.3.3
- name: Check out the FlatBuffers submodule only
# Just this one: the schema check has nothing to do with glslang,
# SPIRV-Cross or the trace fixtures.
run: git submodule update --init 3rdparty/flatbuffers
- name: Regenerate protocol_generated.h
run: python3 scripts/gen_protocol.py --build-dir "${{ runner.temp }}/flatc-build"
- name: Generated header staleness check
run: git diff --exit-code -- MobileGL/MG_Remote/Protocol/generated/protocol_generated.h
# P7 wave 2-F, F fix round (PH-7 (4) review). MOBILEGL_PROTOCOL_CONTROL_REVISION is mixed
# into wireFingerprint so a control-schema change refuses an old peer by name - but it is a
# hand-bumped integer, and nothing tied it to protocol.fbs's content. This step pins
# sha256(protocol.fbs) to the revision (scripts/ci/protocol_revision_pins.json): a schema
# edit that forgot the bump is red HERE, in the job that regenerates the header on every
# such edit, and not a silently absent field between an old server and a new client. Its
# self-test proves the check goes red on both perturbations before it says green.
- name: Protocol revision pin check
run: python3 scripts/ci/protocol_revision_pin.py --self-test
# P0.5 interface-purity gate A (ARCHITECTURE.md:501): the two extracted headers' include closure,
# asserted on `-H` output because `nm --undefined-only` is blind to "included but not called" -
# a header whose types are never named leaves no symbol behind, and "included at all" is exactly
# the coupling P1 and P7 have to sever. Needs a preprocessor and three header submodules, no
# CMake configure and no glslang sources, so like pipe-gates it does not depend on build-linux.
# The script's own --self-test is always on: a negative control that stopped tripping fails the
# job, because a gate that cannot go red is not a gate (ROADMAP.md:7).
include-graph-check:
name: Include-closure purity gate
needs: select-runner
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Check out header submodules
# ska/flat_hash_map.hpp, xxhash.h and vulkan/vulkan.h are the only submodule headers
# Includes.h reaches; glslang and spirv_cross are vendored under include/.
run: git submodule update --init include/ska 3rdparty/xxHash 3rdparty/Vulkan-Headers
- name: Install clang and X11 headers
if: runner.environment != 'self-hosted'
# Includes.h defines VK_USE_PLATFORM_XLIB_KHR before <vulkan/vulkan.h>, which then
# includes <X11/Xlib.h>; without libx11-dev every clang-mode probe dies in the
# preprocessor and the gate reports 5 problems that have nothing to do with purity.
run: sudo apt-get update && sudo apt-get install -y clang-20 libx11-dev
- name: Include-closure check and self-test
# --expect-probes 4 (contract-v2.md 7.6): an exit code cannot tell four probes from none,
# so a --probe typo or a manifest edit that selected nothing would run zero probes and
# exit 0 - the second half of the finding that added the flag. The count is the length of
# scripts/check_include_closure.py's PROBES list and changing one means changing the other.
#
# Use the compiler selected for this runner's installed toolchain.
run: python3 scripts/check_include_closure.py --mode both --compiler "${MOBILEGL_CI_CXX}" --self-test --require-all --expect-probes 4
benchmark:
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
needs: [select-runner, build-linux]
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
if: runner.environment != 'self-hosted'
uses: lukka/get-cmake@v4.3.3
- name: Install runtime dependencies
if: runner.environment != 'self-hosted'
# libegl-mesa0 is the EGL vendor library itself: DriverBench brings up a
# real GL context, and libegl1 is only glvnd's dispatch. It normally
# arrives as a Recommends of libegl1, which is too quiet a dependency for
# the one job that needs a working driver.
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers
- name: Download Linux runtime
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-runtime
path: .
- name: Unpack Linux runtime
run: tar -xzf mobilegl-linux-runtime.tgz
- name: Normalize CTest command paths
run: |
python - <<'PY'
from pathlib import Path
import re
for path in Path('build-linux').rglob('CTestTestfile.cmake'):
text = path.read_text()
text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text)
path.write_text(text)
PY
- name: Benchmark
working-directory: build-linux
run: |
ulimit -c unlimited
if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi
ctest -V -C Release -L benchmark --no-tests=error
- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v7
with:
name: benchmark-core-dumps
path: /tmp/core.*
if-no-files-found: ignore
build-retrace:
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
needs:
- select-runner
- build-linux
- test
- benchmark
- integration
permissions:
actions: write
contents: read
env:
BUILD_DIR: build-retrace
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_COMPRESS: "true"
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 4G
CCACHE_NOHASHDIR: "true"
MOBILEGL_LIBRARY: ${{ github.workspace }}/build-linux/libMobileGL.so
steps:
- name: Set swap space
if: runner.environment != 'self-hosted'
uses: pierotofy/set-swap-space@v1.0
with:
swap-size-gb: 32
- name: Checkout repo
uses: actions/checkout@v6
with:
submodules: recursive
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
if: runner.environment != 'self-hosted'
uses: lukka/get-cmake@v4.3.3
- name: Restore ccache
uses: actions/cache/restore@v5
with:
path: .ccache
key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1
restore-keys: |
${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-
- name: Prepare Vulkan SDK
if: runner.environment != 'self-hosted'
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Update glslang external sources
working-directory: 3rdparty/glslang
run: python update_glslang_sources.py
- name: Install dependencies
if: runner.environment != 'self-hosted'
run: |
sudo apt-get update
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
- name: Show installed toolchain
run: |
ccache --version
"${MOBILEGL_CI_CC}" --version
"${MOBILEGL_CI_CXX}" --version
"${MOBILEGL_CI_LLD}" --version
dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true
- name: Download Linux runtime
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-runtime
path: .
- name: Unpack Linux runtime
run: |
tar -xzf mobilegl-linux-runtime.tgz
test -f "${MOBILEGL_LIBRARY}"
- name: Configure CMake
run: |
if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
BUILD_TYPE=Debug
else
BUILD_TYPE=Release
fi
cmake -S . -B "${BUILD_DIR}" -G Ninja \
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \
-DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
-DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
-DMOBILEGL_BUILD_TEST=OFF \
-DMOBILEGL_BUILD_BENCHMARK=OFF \
-DMOBILEGL_BUILD_TRACE_REPLAY=ON \
-DMOBILEGL_TRACE_SOFTWARE_RENDERING="${MOBILEGL_CI_SOFTWARE_RENDERING}" \
-DMOBILEGL_TRACE_REPLAY_MOBILEGL_LIBRARY="${MOBILEGL_LIBRARY}" \
-DCMAKE_POLICY_VERSION_MINIMUM=3.5
- name: Build trace replay
run: cmake --build "${BUILD_DIR}" --target mobilegl_trace_replay --parallel "$(nproc)"
- name: Show ccache stats
if: ${{ always() && needs.select-runner.result == 'success' }}
run: ccache --show-stats
- name: Release superseded ccache entry
if: github.ref_name == github.event.repository.default_branch
env:
GH_TOKEN: ${{ github.token }}
CACHE_KEY: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1
run: gh cache delete "${CACHE_KEY}" || true
- name: Save ccache
if: github.ref_name == github.event.repository.default_branch
continue-on-error: true
uses: actions/cache/save@v5
with:
path: .ccache
key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1
- name: Normalize CTest command paths
run: |
python - <<'PY'
from pathlib import Path
import re
for path in Path('build-retrace').rglob('CTestTestfile.cmake'):
text = path.read_text()
text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text)
path.write_text(text)
PY
- name: Package trace replay
run: |
mkdir -p ci-artifacts
tar -czf ci-artifacts/mobilegl-trace-replay.tgz \
build-retrace/tools/trace_replay/mobilegl_trace_replay \
build-retrace/tools/trace_replay/CTestTestfile.cmake
- name: Upload trace replay
uses: actions/upload-artifact@v7
with:
name: mobilegl-trace-replay
path: ci-artifacts/mobilegl-trace-replay.tgz
if-no-files-found: error
trace-cases:
name: trace case matrix
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
env:
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci') && 'ON' || 'OFF' }}
needs:
- select-runner
- test
- benchmark
- integration
outputs:
matrix: ${{ steps.trace-cases.outputs.matrix }}
names: ${{ steps.trace-cases.outputs.names }}
verify-matrix: ${{ steps.trace-cases.outputs.verify-matrix }}
split-matrix: ${{ steps.trace-cases.outputs.split-matrix }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
# `dl` / exit gate S7. CHEAP AND EARLY, on a job that only reads the tree: it needs no
# build, so a death that names no family is reported in seconds rather than after the
# matrix. What it refuses is a std::abort() under MG_Remote/ with no Fatal{Family} marker,
# and a family word that appeared without being recorded - the vocabulary drifts on
# purpose or not at all (CONTRACT-P6 5.2's D2 ruling).
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Fatal census (S7)
run: python3 scripts/ci/fatal_census.py
# The decline half of rule I (P7 B3): every WireDeclines.def row has a site, every site a
# warning-or-error line. Same shape as the census - tree-only, no build. The self-test
# runs first: its fixtures (a site only in a comment or a string, a log in another
# branch, an MGLOG_D) must red, so a green audit is one that could have been red.
- name: Wire declines audit (B3)
run: |
python3 scripts/ci/wire_declines_audit.py --self-test
python3 scripts/ci/wire_declines_audit.py
- name: Load trace cases
id: trace-cases
run: |
python3 tools/trace_replay/trace_cases.py --self-test-transport-matrices
matrix_format=github-test-matrix
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then matrix_format=github-test-transport-matrix; fi
echo "matrix=$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}")" >> "$GITHUB_OUTPUT"
echo "names=$(python3 tools/trace_replay/trace_cases.py --ci --format names)" >> "$GITHUB_OUTPUT"
# The subset the verify build retraces ("verify": true in trace_cases.json). It is a
# SUBSET of the matrix above, so retrace-verify needs no fixtures of its own.
echo "verify-matrix=$(python3 tools/trace_replay/trace_cases.py --ci --format github-verify-matrix)" >> "$GITHUB_OUTPUT"
# The split subset: {backend, case, transport} over inproc AND spawn, both backends, for
# every CI case that has not said `"split": false`. Still a SUBSET of the matrix above,
# so retrace-split needs no fixtures of its own. P5 shipped it as one case (OpenRA, which
# the phase gate names); P6 widened it to the whole CI matrix, because every bug the
# spawn work found was a question about the SERVER's process rather than about one trace.
# trace_cases.py refuses a `split` case that is not in CI, so the subset stays a subset.
SPLIT_MATRIX=$(python3 tools/trace_replay/trace_cases.py --ci --format github-split-matrix)
# AND IT MUST NOT BE EMPTY. An empty `include` is not an error to GitHub - it skips the
# whole retrace-split job with no red anywhere - so the one way this subset can vanish
# silently is guarded here. trace_cases.py now also rejects an unknown manifest key, which
# was the hole: `"splitt": true` loaded clean and emptied the subset (review N-4).
if [ "$(printf '%s' "${SPLIT_MATRIX}" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["include"]))')" -lt 1 ]; then
echo "::error::the split retrace subset is EMPTY. 'split' is an OPT-OUT as of P6 - a CI case is in this subset unless it carries \"split\": false - so an empty subset means either every case opted out or the CI matrix itself is empty. Either way retrace-split would be skipped with no red, and exit gate E2 names OpenRA."
exit 1
fi
echo "split-matrix=${SPLIT_MATRIX}" >> "$GITHUB_OUTPUT"
trace-fixtures:
name: trace fixture (${{ matrix.case }})
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
needs: [select-runner, trace-cases]
strategy:
fail-fast: false
max-parallel: 4
matrix:
case: ${{ fromJSON(needs.trace-cases.outputs.names) }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Derive trace fixture cache key
id: fixture-key
run: bash .github/scripts/trace-fixture-cache.sh key '${{ matrix.case }}'
- name: Restore trace fixture cache
id: fixture-cache
if: steps.fixture-key.outputs.cacheable == 'true'
uses: actions/cache/restore@v5
with:
path: ${{ steps.fixture-key.outputs.paths }}
key: ${{ steps.fixture-key.outputs.key }}
- name: Verify restored trace fixture
id: fixture-verify
if: steps.fixture-cache.outputs.cache-hit == 'true'
run: |
if bash .github/scripts/trace-fixture-cache.sh verify '${{ matrix.case }}'; then
echo "ok=true" >> "$GITHUB_OUTPUT"
else
echo "ok=false" >> "$GITHUB_OUTPUT"
echo "::warning::Cached fixture for ${{ matrix.case }} failed verification; falling back to the download path"
bash .github/scripts/trace-fixture-cache.sh reset '${{ matrix.case }}'
fi
- name: Fetch trace fixture
if: steps.fixture-verify.outputs.ok != 'true'
run: bash .github/scripts/fetch-trace-fixture-lfs.sh '${{ matrix.case }}'
- name: Save trace fixture cache
if: steps.fixture-key.outputs.cacheable == 'true' && steps.fixture-cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v5
with:
path: ${{ steps.fixture-key.outputs.paths }}
key: ${{ steps.fixture-key.outputs.key }}
- name: Stage trace fixture
run: |
safe_case="$(printf '%s' '${{ matrix.case }}' | sed 's/[^A-Za-z0-9._-]/_/g')"
stage_dir="trace-fixtures/${safe_case}"
mkdir -p "${stage_dir}"
python3 tools/trace_replay/trace_cases.py --format fixture-files --case '${{ matrix.case }}' |
while IFS= read -r file; do
cp "${file}" "${stage_dir}/"
done
- name: Upload trace fixture
uses: actions/upload-artifact@v7
with:
name: trace-fixture-${{ matrix.case }}
path: trace-fixtures/**
if-no-files-found: error
retrace:
name: retrace (${{ matrix.backend }} / ${{ matrix.transport || 'monolith' }}, ${{ matrix.case }})
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
env:
MOBILEGL_TRANSPORT: ${{ matrix.transport || 'monolith' }}
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
MOBILEGL_IPC_RUN_AHEAD: "1"
needs:
- select-runner
- build-linux
- build-retrace
- trace-cases
- trace-fixtures
if: ${{ always() && needs.select-runner.result == 'success' && needs.build-linux.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }}
strategy:
fail-fast: false
max-parallel: 4
matrix: ${{ fromJSON(needs.trace-cases.outputs.matrix) }}
steps:
- name: Set swap space
if: runner.environment != 'self-hosted'
uses: pierotofy/set-swap-space@v1.0
with:
swap-size-gb: 16
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Download trace fixture
uses: actions/download-artifact@v8
with:
name: trace-fixture-${{ matrix.case }}
path: trace-fixture-download
- name: Install trace fixture
run: |
mkdir -p tools/trace_replay/fixtures
find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;
- name: Get CMake
if: runner.environment != 'self-hosted'
uses: lukka/get-cmake@v4.3.3
- name: Install runtime dependencies
if: runner.environment != 'self-hosted'
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers
test -e /usr/lib/x86_64-linux-gnu/libEGL.so
test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so
- name: Download Linux runtime
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-runtime
path: .
- name: Download trace replay
uses: actions/download-artifact@v8
with:
name: mobilegl-trace-replay
path: .
- name: Unpack retrace runtime
run: |
tar -xzf mobilegl-linux-runtime.tgz
tar -xzf mobilegl-trace-replay.tgz
test -f build-linux/libMobileGL.so
test -f build-retrace/tools/trace_replay/mobilegl_trace_replay
- name: Retrace and validate
working-directory: build-retrace/tools/trace_replay
run: |
ulimit -c unlimited
if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi
if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then
export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1
fi
if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
&& [ '${{ matrix.case }}' = 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' ]; then
export MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH=1
export MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS=1
export MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER=1
fi
# The blended depth-write quirk auto-enables only on Qualcomm, which no CI
# runner has, so force it on for the OIT case it exists to fix. ForceOn
# bypasses only the vendor gate, so this exercises the real strip on
# lavapipe. The Android AVD lane deliberately leaves it off, keeping the
# unstripped path covered for the same trace.
if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
&& [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then
export MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1
fi
ctest -V --no-tests=error -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$' --output-junit "${RUNNER_TEMP}/retrace.xml"
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py --excerpts 1 "${RUNNER_TEMP}"
- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v7
with:
name: retrace-core-dumps-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-${{ matrix.case }}
path: /tmp/core.*
if-no-files-found: ignore
- name: Upload actual image
if: ${{ always() && needs.select-runner.result == 'success' }}
uses: actions/upload-artifact@v7
with:
name: retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-${{ matrix.case }}
path: |
build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/**
build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/**
if-no-files-found: warn
retrace-summary:
name: retrace summary
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
needs: [select-runner, retrace]
if: ${{ always() && needs.select-runner.result == 'success' && needs.retrace.result != 'skipped' }}
env:
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci') && 'ON' || 'OFF' }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Set artifact metadata
run: |
echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
- name: Set up Node.js
uses: actions/setup-node@v7
with:
node-version: '22'
- name: Download monolith control results
uses: actions/download-artifact@v8
with:
pattern: retrace-result-monolith-control-monolith-*
path: retrace-artifacts/monolith-control
- name: Download inproc acceptance results
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
uses: actions/download-artifact@v8
with:
pattern: retrace-result-inproc-acceptance-inproc-*
path: retrace-artifacts/inproc-acceptance
- name: Render monolith control summary
run: |
node tools/trace_replay/render_retrace_summary.mjs \
--input retrace-artifacts/monolith-control \
--output-dir retrace-summary/monolith-control \
--title "MobileGL Linux monolith control" \
--group-label "Linux / monolith control" \
--html mobilegl-linux-monolith-control.html
- name: Render inproc acceptance summary
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
run: |
node tools/trace_replay/render_retrace_summary.mjs \
--input retrace-artifacts/inproc-acceptance \
--output-dir retrace-summary/inproc-acceptance \
--title "MobileGL Linux inproc acceptance" \
--group-label "Linux / inproc acceptance" \
--html mobilegl-linux-inproc-acceptance.html
- name: Upload monolith control summary
uses: actions/upload-artifact@v7
with:
name: mobilegl-linux-monolith-control-summary-${{ github.sha }}
path: retrace-summary/monolith-control/mobilegl-linux-monolith-control.html
archive: false
if-no-files-found: error
- name: Upload inproc acceptance summary
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
uses: actions/upload-artifact@v7
with:
name: mobilegl-linux-inproc-acceptance-summary-${{ github.sha }}
path: retrace-summary/inproc-acceptance/mobilegl-linux-inproc-acceptance.html
archive: false
if-no-files-found: error
# The trace half of the third CI mode. Same replay, same goldens, but the library underneath is
# the verify build and MOBILEGL_PIPE_VERIFY=1 is in the environment, so every backend read of
# frontend state is checked against a snapshot taken at the verb boundary. Eight cases rather
# than the full lane's 40 (tools/trace_replay/trace_cases.json, "verify": true): the comparator
# is budgeted at 5-10x, and the full sweep is a phase-exit / workflow_dispatch run.
retrace-verify:
name: retrace verify (${{ matrix.backend }}, ${{ matrix.case }})
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
timeout-minutes: 240
needs:
- select-runner
- build-linux-verify
- build-retrace
- trace-cases
- trace-fixtures
if: ${{ always() && needs.select-runner.result == 'success' && needs.build-linux-verify.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }}
strategy:
fail-fast: false
max-parallel: 4
matrix: ${{ fromJSON(needs.trace-cases.outputs.verify-matrix) }}
steps:
- name: Set swap space
if: runner.environment != 'self-hosted'
uses: pierotofy/set-swap-space@v1.0
with:
swap-size-gb: 16
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Download trace fixture
uses: actions/download-artifact@v8
with:
name: trace-fixture-${{ matrix.case }}
path: trace-fixture-download
- name: Install trace fixture
run: |
mkdir -p tools/trace_replay/fixtures
find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;
- name: Get CMake
if: runner.environment != 'self-hosted'
uses: lukka/get-cmake@v4.3.3
- name: Install runtime dependencies
if: runner.environment != 'self-hosted'
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers
test -e /usr/lib/x86_64-linux-gnu/libEGL.so
test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so
- name: Download Linux verify runtime
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-runtime-verify
path: .
- name: Download trace replay
uses: actions/download-artifact@v8
with:
name: mobilegl-trace-replay
path: .
- name: Unpack verify runtime (library under test)
# build-retrace's CTestTestfile.cmake has the absolute path
# <workspace>/build-linux/libMobileGL.so frozen into every case, so the swap happens here
# rather than through a variable: the verify .so is put where that path points. The nm
# check is what makes the swap falsifiable - a run against the ordinary library would
# carry no comparator, ignore MOBILEGL_PIPE_VERIFY entirely, and match its golden.
#
# `nm`, not `nm -D`, for the reason spelled out in build-linux-verify: everything MGPipe is
# hidden-visibility in a Release build and the dynamic table has none of it.
run: |
tar -xzf mobilegl-linux-runtime-verify.tgz
tar -xzf mobilegl-trace-replay.tgz
test -f build-verify/libMobileGL.so
test -f build-retrace/tools/trace_replay/mobilegl_trace_replay
mkdir -p build-linux
cp build-verify/libMobileGL.so build-linux/libMobileGL.so
if ! nm --defined-only build-linux/libMobileGL.so | grep -q MGPipeVerifyInputs; then
echo "::error::the library unpacked at build-linux/libMobileGL.so defines no MGPipeVerifyInputs, so this retrace would replay against a comparator-free build and pass on its golden having verified nothing"
exit 1
fi
echo "the library at build-linux/libMobileGL.so is the verify build"
- name: Retrace and validate (verify)
working-directory: build-retrace/tools/trace_replay
# run_trace_case.cmake turns MOBILEGL_PIPE_VERIFY into three assertions of its own (the
# arming line, no Fatal{PipeVerifyDiffer, no Fatal{UnmigratedPipeInput), so a case that
# somehow ran the wrong library reds here instead of passing on its golden.
# --timeout 10800: the 1800s cases run 5-10x slower with both comparator arms live, which
# is well past ctest's 1500s default.
run: |
ulimit -c unlimited
if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi
export MOBILEGL_PIPE_VERIFY=1
if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then
export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1
fi
if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
&& [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then
export MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1
fi
ctest -V --no-tests=error --timeout 10800 \
-R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$'
# The retrace lane's own always-on negative control, on one case so it costs one short trace:
# with a snapshot field corrupted, the SAME replay must fail. Without it, "40 traces, zero
# divergences" would be a statement about a comparator nobody watched.
#
# The rerun replays into the SAME case directory, so the verified run's images are put aside
# first and restored before the verdict: "Upload actual image" below runs `if: always()` and
# would otherwise ship the deliberately corrupted run's output under the name of the good one.
# The restore happens whichever way the control goes, which is why the ctest exit status is
# captured rather than tested inline.
- name: Negative control (corrupted snapshot)
if: ${{ matrix.case == 'OpenRA' && matrix.backend == 'DirectGLES' }}
working-directory: build-retrace/tools/trace_replay
run: |
export MOBILEGL_PIPE_VERIFY=1
export MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters
GOOD_OUTPUT="${RUNNER_TEMP}/openra-verified-output"
rm -rf "${GOOD_OUTPUT}"
if [ -d OpenRA ]; then
cp -a OpenRA "${GOOD_OUTPUT}"
fi
set +e
ctest -V --no-tests=error --timeout 10800 \
-R '^MobileGLTraceReplay\.OpenRA\.DirectGLES$'
control_rc=$?
set -e
if [ -d "${GOOD_OUTPUT}" ]; then
rm -rf OpenRA
mv "${GOOD_OUTPUT}" OpenRA
echo "restored the verified run's OpenRA output over the corrupted rerun's"
fi
if [ "${control_rc}" -eq 0 ]; then
echo "::error::MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left the OpenRA retrace GREEN, so the comparator is not comparing and the whole verify retrace lane proves nothing."
exit 1
fi
echo "the corrupted field turned the retrace red, as it must (ctest exit ${control_rc})"
- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v7
with:
name: retrace-verify-core-dumps-${{ matrix.backend }}-${{ matrix.case }}
path: /tmp/core.*
if-no-files-found: ignore
- name: Upload actual image
if: ${{ always() && needs.select-runner.result == 'success' }}
uses: actions/upload-artifact@v7
with:
name: retrace-verify-result-${{ matrix.backend }}-${{ matrix.case }}
path: |
build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/**
build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/**
if-no-files-found: warn
# The retrace arm of the split shape: retrace-verify with MOBILEGL_PIPE_VERIFY=1 swapped for
# MOBILEGL_TRANSPORT and the comparator's symbol swapped for an MG_Remote one. Exit gate E2 is
# one row - OpenRA at SSIM >= 0.99 - but the lane is no longer one row: as of P6 it is
# {backend, case, transport} over the whole CI matrix, both backends, inproc AND spawn.
#
# inproc IS THE CONTROL AND SPAWN IS THE CLAIM. Same library, same codec, same applier, same
# picture; the only difference is which PROCESS the server role runs in. Running only spawn
# would leave "did the transport break it?" and "did the phase break it?" answerable only
# together, which is the question every failure in this phase turned out to hinge on.
#
# IT RUNS THE UNCHANGED CTEST NAMES with the transport exported in the JOB environment, rather
# than the SPLIT-suffixed variant entries. build-retrace configures without
# -DMOBILEGL_BUILD_DISAGGREGATED, so the variant entries are deliberately not registered there
# (a name that says SPLIT in a build that cannot be one is worse than no name), and the
# unchanged names are what retrace-verify already proves this shape works with. The variant
# entries exist for a build that DOES configure the option - a local build-split with trace
# replay on - where `ctest -L retrace-split` is self-describing and needs no environment ritual.
#
# WHAT MAKES IT FALSIFIABLE is not the SSIM. A monolith run of OpenRA also scores 1.000: measured
# on this branch, a pull library under MOBILEGL_TRANSPORT=inproc produced ssim=1.0 and was caught
# only by run_trace_case.cmake's transport-resolution assertion. So there are two guards, and the
# SSIM is neither of them: the nm check below (this library carries MG_Remote) and the
# transport-resolution line in the library's own log (this PROCESS resolved inproc).
retrace-split:
name: retrace split (${{ matrix.backend }} / ${{ matrix.transport }}, ${{ matrix.case }})
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
timeout-minutes: 240
# The setup action selects Dozen on Arch WSL or lavapipe on GitHub.
needs:
- select-runner
- build-linux-monolith-control
- build-linux-split
- build-retrace
- trace-cases
- trace-fixtures
# The separate OFF/pull artifact is for the negative control ONLY: its library is what the control
# swaps in to prove the transport-resolution assertion is load-bearing.
if: ${{ always() && needs.select-runner.result == 'success' && needs.build-linux-monolith-control.result == 'success' && needs.build-linux-split.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }}
strategy:
fail-fast: false
max-parallel: 4
matrix: ${{ fromJSON(needs.trace-cases.outputs.split-matrix) }}
steps:
- name: Set swap space
if: runner.environment != 'self-hosted'
uses: pierotofy/set-swap-space@v1.0
with:
swap-size-gb: 16
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Download trace fixture
uses: actions/download-artifact@v8
with:
name: trace-fixture-${{ matrix.case }}
path: trace-fixture-download
- name: Install trace fixture
run: |
mkdir -p tools/trace_replay/fixtures
find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;
- name: Get CMake
if: runner.environment != 'self-hosted'
uses: lukka/get-cmake@v4.3.3
- name: Install runtime dependencies
if: runner.environment != 'self-hosted'
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers
test -e /usr/lib/x86_64-linux-gnu/libEGL.so
test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so
- name: Download Linux split runtime
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-runtime-split
path: .
- name: Download trace replay
uses: actions/download-artifact@v8
with:
name: mobilegl-trace-replay
path: .
# The PULL runtime, for the negative control at the end of this job and for nothing else.
- name: Download pull runtime (negative control)
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-monolith-control
path: pull-runtime
- name: Unpack the pull runtime (negative control)
run: |
tar -xzf pull-runtime/mobilegl-linux-monolith-control.tgz -C pull-runtime
test -f pull-runtime/build-monolith-control/libMobileGL.so
- name: Unpack split runtime (library under test)
# build-retrace's CTestTestfile.cmake has the absolute path
# <workspace>/build-linux/libMobileGL.so frozen into every case, so the swap happens here
# rather than through a variable, exactly as in retrace-verify. `nm`, not `nm -D`.
run: |
tar -xzf mobilegl-linux-runtime-split.tgz
tar -xzf mobilegl-trace-replay.tgz
test -f build-split/libMobileGL.so
test -f build-retrace/tools/trace_replay/mobilegl_trace_replay
mkdir -p build-linux
cp build-split/libMobileGL.so build-linux/libMobileGL.so
if ! nm --defined-only build-linux/libMobileGL.so | grep -q -i MG_Remote; then
echo "::error::the library unpacked at build-linux/libMobileGL.so defines no MG_Remote symbol, so this retrace would replay against a monolith build, ignore MOBILEGL_TRANSPORT entirely and match its golden having split nothing"
exit 1
fi
echo "the library at build-linux/libMobileGL.so is the split build"
# THE SPAWN ARM NEEDS A SECOND BINARY and the inproc arm needs none, which is the whole
# difference between the two at this level: inproc's server role is a thread in the replay
# process, spawn's is a PROCESS the client launches from this image. It is packaged by
# build-linux-split's `find -name '*.so'` sweep - libMobileGLServer.so is an executable that
# wears a .so name so an APK will install it somewhere exec-able - and a missing one must be
# a NAMED failure here rather than a LaunchServer refusal 78 jobs deep.
- name: Place the server image (spawn arm only)
if: ${{ matrix.transport == 'spawn' }}
run: |
if [ ! -f build-split/libMobileGLServer.so ]; then
echo "::error::the split runtime carries no libMobileGLServer.so, so transport=spawn has no server to launch. ConfigLoader would resolve Spawn, LaunchServer would refuse BY NAME, and this job would red on a message about an unresolvable image rather than about a packaging gap."
exit 1
fi
cp build-split/libMobileGLServer.so build-linux/libMobileGLServer.so
chmod +x build-linux/libMobileGLServer.so
echo "server image: $(readlink -f build-linux/libMobileGLServer.so)"
# THE MAGMA KNOBS GO TO $GITHUB_ENV, NOT TO ONE STEP'S SHELL: the replay below AND both
# negative controls after it must run the same repair set. Exported inside "Retrace and
# validate" only, they never reached the pull-library control, which replayed
# iterationrp x DirectVulkan unrepaired (ssim 0.028 < 0.98), reddened at
# run_trace_case.cmake:173 before the transport assertion, and failed as "the failure never
# says the library did not resolve the transport" - a red that is not the control's own.
- name: Magma repair knobs (all later steps)
run: |
# P7 package L: THE SAME MAGMA KNOBS THE MONOLITH RETRACE JOB EXPORTS, ALL OF THEM.
# This job exported none at all, so every DirectVulkan row of the split matrix was
# compared against a monolith row running a different repair set - and a split retrace
# whose control is not comparable cannot say whether a divergence is a transport fact.
# R11F_G11F_B10F fallback is whole-backend; the blended depth-write quirk auto-enables
# only on Qualcomm, which no runner has, so the OIT case forces it exactly as the
# monolith job does (ForceOn bypasses only the vendor gate).
if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then
echo "MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1" >> "${GITHUB_ENV}"
fi
if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
&& [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then
echo "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1" >> "${GITHUB_ENV}"
fi
# THE THREE ITERATIONRP KNOBS, FOR THE ONE CASE THE MONOLITH JOB NAMES.
# The iterationRP repairs are tri-state quirks that default to device auto-detection and
# lavapipe is on no auto list, so without them Program 203 runs unrepaired and misses its
# golden output. The monolith job has exported them since they landed; this job never
# did, so `iterationrp x DirectVulkan` was the one cell of the split matrix whose red
# could not be read as a transport fact - the monolith control it is compared against was
# running a different repair set. Exit gate 1 asks for the two halves to be COMPARABLE,
# and two runs that differ in a quirk knob are not.
if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
&& [ '${{ matrix.case }}' = 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' ]; then
echo "MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH=1" >> "${GITHUB_ENV}"
echo "MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS=1" >> "${GITHUB_ENV}"
echo "MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER=1" >> "${GITHUB_ENV}"
fi
- name: Retrace and validate (${{ matrix.transport }})
working-directory: build-retrace/tools/trace_replay
# run_trace_case.cmake turns MOBILEGL_TRANSPORT into assertions of its own - the library
# must have RESOLVED the transport in this process, and its log must carry no Fatal{ - so a
# case that somehow ran the wrong library reds here instead of passing on its golden. That
# log is also the only valid refusal census: the console sink is compiled out of this
# configuration, so a `ctest -V` transcript reports a false zero for Fatal{ lines.
run: |
ulimit -c unlimited
if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi
export MOBILEGL_TRANSPORT='${{ matrix.transport }}'
# Absolute, because the replay's working directory is not this one and LaunchServer
# resolves the image relative to nothing. Harmless on the inproc arm, which reads it
# only to decide it does not need it.
export MOBILEGL_IPC_SERVER_PATH="${GITHUB_WORKSPACE}/build-linux/libMobileGLServer.so"
ctest -V --no-tests=error --timeout 10800 \
-R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$'
# THE RETRACE LANE'S OWN ALWAYS-ON NEGATIVE CONTROL, and its absence was review finding M-6:
# this job was a clone of retrace-verify with the one step removed that makes the lane mean
# anything. retrace-verify's "Negative control (corrupted snapshot)" step requires a corrupted
# snapshot field to red the retrace, so that "79 traces, zero divergences" is not a statement
# about a comparator nobody watched; the same reasoning applies here word for word.
#
# The control swaps the PULL library into the frozen path and requires the SAME replay to
# fail. It is the sharpest one available, because OpenRA scores ssim 1.000000 either way -
# measured - so this step fails if and only if run_trace_case.cmake's transport-resolution
# assertion has stopped working, which is the single thing standing between this job and a
# green that ran monolith end to end.
#
# NOT the control BRIEF 7 E2 names ("patch the Clear emitter to drop one emission; SSIM must
# fall below threshold"). That one needs an emitter, i.e. package c1, and it is carried as an
# explicit debt in t1-v1.md rather than silently substituted - which is what the first
# version of this package did.
#
# The rerun replays into the same case directory, so the good run's images are put aside and
# restored whichever way the control goes; "Upload actual image" below runs `if: always()`
# and would otherwise ship the deliberately-wrong run's output under the good run's name.
# THE BODY OF THIS STEP IS scripts/ci/retrace_pull_library_control.sh, for the reason the
# split lane's control gives: a `run:` block cannot be executed off a runner, so these lines
# could not be tested until they ran in CI. scripts/ci/control_smoke_test.sh drives that file.
#
# Two holes ID-46 finding 8(b) found in this block, both CONFIRMED against the REAL ctest in a
# REAL build tree, both closed in the script: it had NO selection guard at all - unlike the
# split lane's run_control - so a case/backend regex matching nothing exited 8 through
# `--no-tests=error` and was read as "the pull library turned it red"; and only "non-zero
# ctest" was checked after the nm identity check, so a loader failure, a missing fixture or a
# timeout passed it. The red must now carry run_trace_case.cmake's own sentence.
- name: Negative control (pull library)
working-directory: build-retrace/tools/trace_replay
env:
CONTROL_TMPDIR: ${{ runner.temp }}
PULL_LIBRARY: ${{ github.workspace }}/pull-runtime/build-monolith-control/libMobileGL.so
FROZEN_LIBRARY: ${{ github.workspace }}/build-linux/libMobileGL.so
# The control has to run the arm it is a control FOR. Hard-coded inproc, it would go on
# proving something about the other arm while spawn ran unguarded.
SPLIT_TRANSPORT: ${{ matrix.transport }}
MOBILEGL_IPC_SERVER_PATH: ${{ github.workspace }}/build-linux/libMobileGLServer.so
run: >-
bash "${GITHUB_WORKSPACE}/scripts/ci/retrace_pull_library_control.sh"
'${{ matrix.case }}' '${{ matrix.backend }}'
# EXIT GATE E2's OTHER HALF, and the one BRIEF-P5 §7 E2 actually names: "drop an emission
# and the SSIM must fall below the threshold". The control above proves the lane runs a
# SPLIT library; it cannot prove the PICTURE came through the wire, because OpenRA scores
# ssim 1.000000 against a monolith library too and the transport assertion is what reds it.
#
# It drops the DRAWS and not the clears, and that is a measurement rather than a
# preference: with MOBILEGL_IPC_E2_DROP_CLEAR=1 armed, read, and all 29 of OpenRA's Clear
# records dropped, the retrace still scored ssim=1.000000 / mismatchPixels=0 - OpenRA
# overdraws every pixel it clears before the snapshot (joint-v1.md §3 found the green;
# scripts/ci/retrace_drop_draw_control.sh's header has the trace census that explains it).
# With the DrawVbo records dropped instead: 758 dropped, ssim=0.000036,
# mismatchPixels=295296.
#
# THE BODY IS A SCRIPT for the reason the two steps above give: a `run:` block cannot be
# executed off a runner, and scripts/ci/control_smoke_test.sh drives this file against a
# stubbed ctest in seven modes - including the two that matter most, a red whose SSIM never
# fell and a knob that armed and dropped nothing.
- name: Negative control (dropped draws)
working-directory: build-retrace/tools/trace_replay
env:
CONTROL_TMPDIR: ${{ runner.temp }}
LIBRARY_LOG: ${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.client.log
FROZEN_LIBRARY: ${{ github.workspace }}/build-linux/libMobileGL.so
# The control has to run the arm it is a control FOR. Hard-coded inproc, it would go on
# proving something about the other arm while spawn ran unguarded.
SPLIT_TRANSPORT: ${{ matrix.transport }}
MOBILEGL_IPC_SERVER_PATH: ${{ github.workspace }}/build-linux/libMobileGLServer.so
run: >-
bash "${GITHUB_WORKSPACE}/scripts/ci/retrace_drop_draw_control.sh"
'${{ matrix.case }}' '${{ matrix.backend }}'
# The refusal census, recorded rather than gated. run_trace_case.cmake already REDS the case
# on any Fatal{, so reaching here means the count is zero - but the number and the distinct
# slot names are what MEASUREMENTS wants from every split run, and reading them out of the
# library's own log is the only way to get them (ctest -V's transcript is a false zero).
- name: Refusal census from the library log
if: ${{ always() && needs.select-runner.result == 'success' }}
working-directory: build-retrace/tools/trace_replay
run: |
# BOTH ROLES, because under spawn every applier refusal is raised in the SERVER's
# process and a census reading only the client's file reports a confident zero for the
# half it cannot see.
log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.client.log"
server_log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.server.log"
if [ ! -f "${log}" ]; then
echo "no ${log} - the replay wrote no library log"
exit 0
fi
census_files="${log}"
if [ -f "${server_log}" ]; then census_files="${census_files} ${server_log}"; fi
echo "census over: ${census_files}"
echo "Fatal{ lines: $(cat ${census_files} | grep -c 'Fatal{' || true)"
cat ${census_files} | grep -o 'Fatal{[A-Za-z]*, "[^"]*"' | sort | uniq -c | sort -rn | head -20 || true
# The transport line is the CLIENT's: ConfigLoader resolves it there, and under spawn
# the server states its own separately.
grep -m1 "MOBILEGL_TRANSPORT=" "${log}" || echo "no transport line in ${log}"
- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v7
with:
name: retrace-split-core-dumps-${{ matrix.transport }}-${{ matrix.backend }}-${{ matrix.case }}
path: /tmp/core.*
if-no-files-found: ignore
- name: Upload actual image
if: ${{ always() && needs.select-runner.result == 'success' }}
uses: actions/upload-artifact@v7
with:
name: retrace-split-result-${{ matrix.transport }}-${{ matrix.backend }}-${{ matrix.case }}
path: |
build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/**
build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/**
if-no-files-found: warn
remove-artifact-clutter:
name: remove artifact clutter
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
# Every retrace arm downloads the shared fixtures, including split controls.
needs:
- select-runner
- retrace-summary
- retrace-verify
- retrace-split
if: ${{ always() && needs.select-runner.result == 'success' }}
permissions:
actions: write
steps:
- name: Delete intermediate Linux retrace artifacts
env:
GH_TOKEN: ${{ github.token }}
run: |
# Read the complete inventories before deleting anything. A failed API
# request must fail this step, not look like "no failed cases" through
# an unchecked process substitution.
gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100" \
--jq '.jobs[] | select((.name | startswith("retrace (")) or (.name | startswith("retrace verify (")) or (.name | startswith("retrace split ("))) | select(.conclusion == "failure" or .conclusion == "cancelled" or .conclusion == "timed_out" or .conclusion == "action_required") | .name' \
> "${RUNNER_TEMP}/cleanup-jobs.txt"
gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \
--jq '.artifacts[] | select(.name | startswith("trace-fixture-") or startswith("retrace-result-") or startswith("retrace-split-result-")) | [.id, .name] | @tsv' \
> "${RUNNER_TEMP}/cleanup-artifacts.txt"
declare -A failed_cases=()
while IFS= read -r job_name; do
case_name="${job_name#retrace split (*, }"
case_name="${case_name#retrace verify (*, }"
case_name="${case_name#retrace (*, }"
case_name="${case_name%)}"
failed_cases["${case_name}"]=1
done < "${RUNNER_TEMP}/cleanup-jobs.txt"
if ((${#failed_cases[@]})); then
echo "Retaining fixtures and results for failed retrace case(s):"
printf ' %s\n' "${!failed_cases[@]}"
else
echo "All retrace jobs succeeded; no fixtures need to be retained."
fi
deleted=0
retained=0
while IFS=$'\t' read -r artifact_id artifact_name; do
keep=0
if [[ "${artifact_name}" == trace-fixture-* ]]; then
case_name="${artifact_name#trace-fixture-}"
if [[ -v "failed_cases[${case_name}]" ]]; then
keep=1
fi
else
for case_name in "${!failed_cases[@]}"; do
if [[ "${artifact_name}" == *-"${case_name}" ]]; then keep=1; break; fi
done
fi
if ((keep)); then
echo "Retaining ${artifact_name} (${artifact_id}) for failed retrace."
((retained += 1))
continue
fi
echo "Deleting ${artifact_name} (${artifact_id})"
gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}"
((deleted += 1))
done < "${RUNNER_TEMP}/cleanup-artifacts.txt"
echo "Deleted ${deleted} intermediate Linux artifact(s); retained ${retained} failed-retrace artifact(s)."
pipe-gates:
name: MGPipe generators and hygiene gates
needs: select-runner
runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }}
# Deliberately independent of build-linux: these are source-level gates, they take
# seconds, and a broken build must not hide a drifted interface.
steps:
- name: Checkout repo
uses: actions/checkout@v6
# The seven generators all read MG_Pipe/*.def, so regenerating and diffing is what
# keeps the two interface tables, the wire records, the verify comparators, the
# PipeInputs field ids, the read-inventory coverage and the render-state member list
# from drifting apart from the catalogue. The generated files are committed
# deliberately: the build must not depend on python.
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Regenerate the MGPipe interface (G1-G7)
run: |
python3 scripts/gen_pipe.py
git diff --exit-code -- MobileGL/MG_Pipe/generated
# The generators' own negative controls: canned inputs that MUST trip each structural check
# (a field list that does not cover its struct's members, a verb set that is not the function
# table's). Regenerating and diffing above cannot see a check that silently stopped
# checking - a broken gate and a clean tree produce the same green.
- name: MGPipe generator self-test
run: python3 scripts/gen_pipe.py --self-test
# The link seam owns physical storage and keeps link dispatch out of callers.
- name: Link-seam purity check
run: python3 scripts/ci/link_seam_purity.py --self-test
# The same shape for Espryt's re-keyed memos (P3b/P4b, PLAN-PH-P34B-P7.md 1.2). P4a/P5f
# moved the resolved-binding, sampler-pass, image-sweep, twin-registry, unit-sampler and
# twin-lookup memos onto {slot, gen} handles and applier serials; the legacy pointer-keyed
# arms stayed because they are the PULL build's only arms and G1 pins that build's .text.
# Nothing checked that a NEW frontend-pointer key could not appear outside them.
#
# MOBILEGL_PIPE_LEGACY_MEMOS IS A POLICY EXEMPTION, NOT AN ABSENCE. It defaults ON
# (CMakeLists.txt:43) and no job here clears it, so the split builds in this workflow DO
# compile those arms - TwinLookupMemo among them. The gate's rule is "the legacy arm may
# keep pointer keys", which is a decision; only !MOBILEGL_BUILD_DISAGGREGATED is code a
# split build genuinely never sees.
#
# --self-test runs the negative controls and then the real scan in one invocation, as its
# sibling does; the scan also asserts that every memo family's anchor still matches and
# that every allow-list entry still suppresses something.
- name: Espryt memo-key purity check
run: python3 scripts/ci/espryt_memo_purity.py --self-test
# The same question for the symbol tool the P1 gate is written in terms of.
- name: Symbol report self-test
run: python3 scripts/symbol_report.py --self-test
# Per-draw fprintf/printf instrumentation has repeatedly been committed by accident,
# once inside a mutex critical section. Nothing under these two trees prints to a
# stdio stream today - MGLOG_D compiles out in INFO builds and is the only channel
# they are allowed to use - so this gate starts with no exceptions, and any addition
# to it needs a reason in the pull request rather than a quiet whitelist entry. The
# alternation names every stdio spelling, not just the two that were committed:
# fprintf to either stream, printf, puts, and the iostream pair.
- name: No stdio in MG_Backend or MG_State
run: |
if grep -rnE 'fprintf[[:space:]]*\((stderr|stdout)|(^|[^[:alnum:]_>.])printf[[:space:]]*\(|(^|[^[:alnum:]_>.:])puts[[:space:]]*\(|std::(cout|cerr)' \
MobileGL/MG_Backend MobileGL/MG_State; then
echo "::error::stdio instrumentation found; use MGLOG_D (compiled out in INFO builds)"
exit 1
fi
echo "no fprintf(stderr/stdout / printf( / puts( / std::cout|cerr under MobileGL/MG_Backend or MobileGL/MG_State"
# A GATE as of P2, which is when MG_Pipe/DirtySurface.def exists to diff the scan against
# (ROADMAP.md:18 puts the first mapping round in P2). --check fails BOTH directions: a
# mutator the scanner finds with no row in the def, and a row naming a mutator the scan no
# longer finds - so a deleted mutator cannot leave a stale row behind claiming coverage.
#
# --self-test is the half that keeps --check honest, and it is not optional. A completeness
# check that silently stopped checking produces exactly the same green as a complete
# mapping; the self-test feeds it two canned negative controls (a mutator withheld from the
# def, a row naming a function that does not exist) and fails if either fails to trip. Same
# shape as gen_pipe.py --self-test and check_include_closure.py above.
#
# What this gate does NOT cover is written into DirtySurface.def's header rather than left
# implicit: the scanner attributes a mutation inside a lambda to the enclosing function,
# reads a mutation published through a helper as deferred, and scans only MG_Impl/GLImpl -
# so the four MGP_NOTE_MUTATION sites in MG_State are outside it entirely. This is a
# completeness gate over what the scanner can see; the semantic proof is the verify lane.
- name: Dirty-surface mapping complete (G9)
run: |
python3 scripts/gen_pipe_dirty_surface.py --check
python3 scripts/gen_pipe_dirty_surface.py --self-test
# A GATE as of P5 (R-7.1, CONTRACT-P5.md table 2). The eighth generator, and the only one
# whose output the step above cannot cover: `gen_pipe.py` does not write
# PipeFieldOwnership.inc, so regenerating the seven and diffing MG_Pipe/generated leaves a
# drifted ownership table green. The build-level static_assert only catches an
# UNCLASSIFIED row; a hand edit that CHANGES a class compiles clean, and --check is the
# only thing in the tree that catches it.
#
# NO BRANCH GUARD, deliberately, unlike the two G5 steps below: this asks "does the
# committed table still follow from the .def files", which is a question every branch can
# answer and every branch wants answered. The table outlives feat/disaggregated.
#
# --self-test is not optional, for the reason the two gates above give and for one this
# generator learned the hard way: its first version counted eleven trips when one control
# was a silent duplicate of another, because the harness asked "did something exit" rather
# than "did THIS exit". Every control now asserts its own message, and a harness control
# asserts that the harness still rejects someone else's exit.
- name: Field-ownership table complete (R-7.1)
run: |
python3 scripts/gen_pipe_field_ownership.py --check
python3 scripts/gen_pipe_field_ownership.py --self-test
# R-16 APPLIED TO THE NEGATIVE CONTROLS THEMSELVES. The split lane's E1/E3(a) controls and the
# retrace lane's pull-library control are gates, and until ID-46 finding 8 neither could be
# made red by anyone: their bodies were `run:` blocks, which execute only on a runner. Both
# bodies now live in scripts/ci/, and this step runs them against a stubbed ctest that
# reproduces the finding - a NON-EMPTY selection failing with UNRELATED_CONTROL_FAILURE, and a
# case/backend regex matching no tests - and requires each control to report FAILED. The same
# stub, failing with the diagnostics the scenarios really emit, must make them report PASSED.
#
# NO BRANCH GUARD: this asks "do the negative controls still reject a red that is not theirs",
# which is a question every branch can answer and none of which depends on the TEMPORARY
# feat/disaggregated trigger at the top of this file. It costs a couple of seconds and needs
# no build.
# P7 adds the last two. THE DEVICE MATRIX DRIVER HAD NO CI STEP AT ALL: its 12 stub
# regressions (now 18, with the DirectVulkan half) were run by hand in WSL and recorded in
# a note, which is the same "a gate nobody can make red" shape the paragraph above is
# about - and it guards the tool that decides whether a device golden run PASSED.
# compare_actuals is the new half of that evidence (actual-vs-actual SSIM between
# repeats); both are stub/synthetic-only, need no build, no GPU and no device.
- name: Control and trace-tool smoke tests (R-16)
run: |
bash scripts/ci/control_smoke_test.sh
python3 scripts/ci/test_trace_infrastructure.py
python3 scripts/ci/tcp_lane_tools_test.py
python3 tools/trace_replay/test_run_tcp_matrix.py
python3 tools/trace_replay/test_compare_actuals.py
# Warning only for now: the disaggregation documents are still being written, and a
# lint that fails a rewrite in progress teaches people to ignore it. It becomes
# --strict when the documents settle.
- name: Documentation citation lint
run: |
shopt -s nullglob
documents=(docs/Disaggregated/*.md docs/Disaggregated/design/*.md docs/Disaggregated/guide/*.md)
if [ ${#documents[@]} -eq 0 ]; then
echo "no disaggregation documents to check"
exit 0
fi
python3 scripts/check_doc_citations.py "${documents[@]}" || true
Reference in New Issue View Git Blame Copy Permalink