Files
MobileGL/.github/workflows/test.yml
T

4052 lines
200 KiB
YAML

name: Test
run-name: ${{ (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main')) && format('Release validation ({0})', github.ref) || '' }}
on:
push:
branches:
- dev
- Feat/Backend-Direct-GLES
- Feat/Backend-Direct-Vulkan
# TEMPORARY, remove before merging the MGPipe work into dev: the disaggregation
# branch runs the full lane on every push so a phase's landing is not gated on
# someone remembering to dispatch the workflow by hand.
- feat/disaggregated
- codex/fix-github-ci
- cleanup
workflow_dispatch:
env:
# Software-rendered CI submissions may exceed the interactive GPU hang budget.
MOBILEGL_GPU_HANG_BUDGET_MS: "0"
# ONE LIVE RUN PER BRANCH. The disaggregation branch pushes (often docs-only) commits minutes apart
# and each push fans out to about 30 replay jobs here and the emulator shards in apk.yml; with no group, runs
# sat queued for 10+ hours behind their own successors. A manual dispatch gets a group of its own,
# so a push never cancels a manually started run; dev queues rather
# than cancels. No paths-ignore here, unlike apk.yml: this workflow hosts the doc citation lint.
# The "hosted" key leaves behind runs queued for the retired self-hosted runner, which cannot drain.
concurrency:
group: ${{ github.workflow }}-hosted-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.ref }}
cancel-in-progress: ${{ github.event_name == 'push' && github.ref != 'refs/heads/dev' }}
defaults:
run:
shell: bash
jobs:
# Temporarily keep all CI on GitHub-hosted runners while WSL D3D12 readback hangs are investigated.
build-linux:
runs-on: ubuntu-26.04
permissions:
actions: write
contents: read
outputs:
link-ratchet: ${{ steps.ratchet.outcome }}
image: ${{ steps.toolchain.outputs.image }}
compiler: ${{ steps.toolchain.outputs.compiler }}
build-type: ${{ steps.configure.outputs.build-type }}
env:
BUILD_DIR: build-linux
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }}
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_COMPRESS: "true"
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 4G
CCACHE_NOHASHDIR: "true"
steps:
- name: Checkout repo
uses: actions/checkout@v6
# Only the submodules the build reads, shallow and in parallel; the script fails when one
# of them did not arrive (scripts/ci/checkout_submodules.sh).
- name: Check out build submodules
timeout-minutes: 15
run: bash scripts/ci/checkout_submodules.sh library
- name: Set swap space
uses: ./.github/actions/set-swap-space
with:
swap-size-gb: 32
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Restore ccache
uses: actions/cache/restore@v5
with:
path: .ccache
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
restore-keys: |
${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-
- name: Prepare Vulkan SDK
id: netsetup77
continue-on-error: true
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Prepare Vulkan SDK (retry)
timeout-minutes: 15
if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup77.outcome == 'failure' }}
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Update glslang external sources
working-directory: 3rdparty/glslang
run: python3 update_glslang_sources.py
- name: Install build dependencies
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
run: |
sudo apt-get update
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
# The image and compiler are job outputs: integration-split prints them beside build-guard's.
- name: Show installed toolchain
id: toolchain
run: |
ccache --version
"${MOBILEGL_CI_CC}" --version
"${MOBILEGL_CI_CXX}" --version
"${MOBILEGL_CI_LLD}" --version
dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true
echo "image=${ImageOS:-} ${ImageVersion:-}" >> "${GITHUB_OUTPUT}"
echo "compiler=$("${MOBILEGL_CI_CXX}" --version | head -n 1)" >> "${GITHUB_OUTPUT}"
# The configure line lives in scripts/ci/configure_build.sh, shared with build-linux-split
# and build-guard. The build type is a job output: the split consumers reuse this build only
# when it is Release, the only type build-guard can find identical to the split line.
- name: Configure CMake
id: configure
env:
CI_BUILD_TYPE: ${{ secrets.ACTIONS_STEP_DEBUG == 'true' && 'Debug' || 'Release' }}
run: |
echo "build-type=${CI_BUILD_TYPE}" >> "${GITHUB_OUTPUT}"
bash scripts/ci/configure_build.sh primary "${BUILD_DIR}"
- name: Build
run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)"
- name: Primary runtime mode proof
run: |
python3 scripts/ci/runtime_mode_proof.py self-test
mode=monolith
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then mode=disaggregated; fi
python3 scripts/ci/runtime_mode_proof.py build --build-dir "${BUILD_DIR}" \
--mode "${mode}" --output "${BUILD_DIR}/runtime-mode.json"
# build-linux-split's server link-closure ratchet, on this build, which the split consumers
# take when build-guard proves the two configurations identical. This job does not wait for
# that verdict, so it runs on every disaggregation build. continue-on-error keeps a ratchet
# red from skipping this job's consumers; integration-split reds on it when it takes this
# build. The artifact has its own name, so it never replaces build-linux-split's.
- name: Server link-closure ratchet
id: ratchet
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
continue-on-error: true
run: |
mkdir -p ci-artifacts
python3 scripts/link_ratchet.py --self-test
python3 scripts/link_ratchet.py \
--build-dir "${BUILD_DIR}" \
--baseline scripts/data/link_ratchet_baseline.txt \
--assert-monotone \
--bucket \
--json ci-artifacts/link-ratchet-primary.json
- name: Upload link ratchet
if: ${{ always() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
uses: actions/upload-artifact@v7
with:
name: link-ratchet-primary
path: ci-artifacts/link-ratchet-primary.json
if-no-files-found: warn
- name: Show ccache stats
if: ${{ always() }}
run: ccache --show-stats
# Rewrite one rolling entry per job and branch. The upload stays cumulative - it
# carries every object restored at the top of this run plus the few TUs that
# changed - but Actions cache keys are immutable, so this branch's superseded
# blob is released first; --ref keeps the delete off other branches' entries.
# Running after the build means a failed build leaves the entry untouched.
- name: Release superseded ccache entry
env:
GH_TOKEN: ${{ github.token }}
CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true
- name: Save ccache
continue-on-error: true
uses: actions/cache/save@v5
with:
path: .ccache
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
- name: Prepare packaged test metadata
run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}"
# zstd on every core: the tarball is most of a gigabyte of test executables, and every
# consumer job waits for it. `tar --zstd -xf` reads the result unchanged.
- name: Package Linux runtime
run: |
mkdir -p ci-artifacts
mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort)
tar \
--exclude='*/CMakeFiles' \
--exclude='*.o' \
--exclude='*.a' \
--exclude='*.ninja*' \
--exclude='build.ninja' \
--exclude='cmake_install.cmake' \
-I 'zstd -T0' -cf ci-artifacts/mobilegl-linux-runtime.tar.zst \
"${BUILD_DIR}/runtime-mode.json" \
"${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" \
"${BUILD_DIR}/MobileGL/MG_Test" \
"${BUILD_DIR}/MobileGL/MG_Benchmark" \
"${BUILD_DIR}/MobileGL/MG_IntegrationTest" \
"${SHARED_LIBS[@]}"
# compression-level 0: the tarball is zstd already, and deflating it again in the upload
# only costs time. The artifact stays a zip holding this one file, as the consumers expect.
- name: Upload Linux runtime
uses: actions/upload-artifact@v7
with:
name: mobilegl-linux-runtime
path: ci-artifacts/mobilegl-linux-runtime.tar.zst
if-no-files-found: error
compression-level: 0
# The retrace legs load only the library, and the server image on the spawn arm, so they
# take this instead of the whole test runtime.
- name: Package retrace library
run: |
files=("${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/libMobileGL.so")
if [ -f "${BUILD_DIR}/libMobileGLServer.so" ]; then files+=("${BUILD_DIR}/libMobileGLServer.so"); fi
tar -czf ci-artifacts/mobilegl-linux-lib.tgz "${files[@]}"
- name: Upload retrace library
uses: actions/upload-artifact@v7
with:
name: mobilegl-linux-lib
path: ci-artifacts/mobilegl-linux-lib.tgz
if-no-files-found: error
# The shipped shape: a library with no MG_Remote, which FCL embeds. On the disaggregation
# branches the primary artifact carries the transport, so this is the only build of that shape
# there; retrace-shipped replays it.
build-linux-monolith-control:
name: build push monolith control (no MG_Remote)
runs-on: ubuntu-26.04
# actions: write is for this job's own rolling ccache entry, as in build-linux.
permissions:
actions: write
contents: read
env:
BUILD_DIR: build-push-control
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_COMPRESS: "true"
CCACHE_MAXSIZE: 4G
CCACHE_NOHASHDIR: "true"
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Check out build submodules
timeout-minutes: 15
run: bash scripts/ci/checkout_submodules.sh library
- name: Set swap space
uses: ./.github/actions/set-swap-space
with:
swap-size-gb: 32
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
# This job's own entry: its objects are compiled without MG_Remote, which no other build
# produces. The second restore key is build-linux's entry, which seeds the third-party
# objects on a branch's first run.
- name: Restore ccache
uses: actions/cache/restore@v5
with:
path: .ccache
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
restore-keys: |
${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-
${{ runner.os }}-ubuntu26-cpu-test-build-linux-ccache-
- name: Prepare Vulkan SDK
id: netsetup230
continue-on-error: true
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Prepare Vulkan SDK (retry)
timeout-minutes: 15
if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup230.outcome == 'failure' }}
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Update glslang external sources
working-directory: 3rdparty/glslang
run: python3 update_glslang_sources.py
- name: Install build dependencies
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
run: |
sudo apt-get update
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
# The push monolith without MG_Remote, the shape FCL embeds. Its runtime_mode_proof is the
# zero-MG_Remote-symbol check: a push library that links any MG_Remote symbol is red here.
# Its library is uploaded for retrace-shipped, a positive replay of that shape.
# ENABLE_PCH=OFF: ccache cannot cache a compile that uses glslang's precompiled header
# (see scripts/ci/configure_build.sh).
- name: Configure push control without MG_Remote
run: |
cmake -S . -B "${BUILD_DIR}" -G Ninja \
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \
-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DENABLE_PCH=OFF \
-DCMAKE_BUILD_TYPE=Release -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
-DMOBILEGL_BUILD_TEST=OFF -DMOBILEGL_BUILD_BENCHMARK=OFF \
-DMOBILEGL_BUILD_INTEGRATION_TEST=OFF -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \
-DMOBILEGL_BUILD_DISAGGREGATED=OFF -DMOBILEGL_BUILD_DISAGGREGATED_INPROC=OFF \
-DMOBILEGL_PIPE_VERIFY=OFF \
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5
- name: Build push control and prove it has zero MG_Remote symbols
run: |
cmake --build "${BUILD_DIR}" --target MobileGL --parallel "$(nproc)"
python3 scripts/ci/runtime_mode_proof.py build --build-dir "${BUILD_DIR}" \
--mode monolith --output "${BUILD_DIR}/runtime-mode.json"
- name: Show ccache stats
if: ${{ always() }}
run: ccache --show-stats
# One rolling entry per job and branch, as in build-linux: this branch's superseded blob is
# released before the save, and a failed build leaves the entry untouched.
- name: Release superseded ccache entry
env:
GH_TOKEN: ${{ github.token }}
CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true
- name: Save ccache
continue-on-error: true
uses: actions/cache/save@v5
with:
path: .ccache
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
- name: Prepare packaged test metadata
run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}"
- name: Package push control
run: |
tar -czf mobilegl-linux-monolith-control.tgz \
"${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/libMobileGL.so" "${BUILD_DIR}/runtime-mode.json"
- name: Upload push control
uses: actions/upload-artifact@v7
with:
name: mobilegl-linux-monolith-control
path: mobilegl-linux-monolith-control.tgz
if-no-files-found: error
test:
runs-on: ubuntu-26.04
needs: [build-linux]
# Every ctest entry is its own short-lived process, and none compiles enough to repay
# Initialize()'s glslang table prewarm. The retrace legs keep it: they replay real apps.
env:
MOBILEGL_DISABLE_SHADER_PREWARM: "1"
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Install runtime dependencies
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1 libgles2 libgl1-mesa-dri mesa-vulkan-drivers ${{ env.MOBILEGL_CI_DISAGGREGATED == 'ON' && 'libegl-mesa0' || '' }}
- name: Download Linux runtime
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-runtime
path: .
- name: Unpack Linux runtime
run: tar --zstd -xf mobilegl-linux-runtime.tar.zst
- name: Prepare downloaded test metadata
run: |
for root in build-linux build-verify build-split build-verify-split build-retrace; do
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
# On the three branches this is also the split runtime's unit run (integration-split skips
# its copy when build-linux-split reuses this build), so it keeps that run's hardening:
# REQUIRE_GPU makes the server loop's EGL cases fail rather than skip without a driver.
- name: Test
working-directory: build-linux
env:
MOBILEGL_ITEST_REQUIRE_GPU: ${{ env.MOBILEGL_CI_DISAGGREGATED == 'ON' && '1' || '' }}
run: |
ulimit -c unlimited
if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
ctest -V -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit.xml"
else
ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit.xml"
fi
- name: Disk and cores after the unit run
if: ${{ always() }}
run: |
df -h / /tmp
echo "cores: $(ls /tmp/core.* 2> /dev/null | wc -l), $(du -ch /tmp/core.* 2> /dev/null | tail -n 1)"
# P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py).
- name: Upload JUnit for the skip census
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: junit-test
path: ${{ runner.temp }}/*.xml
if-no-files-found: warn
- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v7
with:
name: unit-core-dumps
path: /tmp/core.*
if-no-files-found: ignore
integration:
name: integration (${{ matrix.transport }})
runs-on: ubuntu-26.04
needs: [build-linux]
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON((github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && '[{"lane":"monolith-control","transport":"monolith"},{"lane":"inproc-acceptance","transport":"inproc"}]' || '[{"lane":"monolith-control","transport":"monolith"}]') }}
env:
# Short-lived test processes skip the glslang prewarm, as in the test job.
MOBILEGL_DISABLE_SHADER_PREWARM: "1"
MOBILEGL_TRANSPORT: ${{ matrix.transport }}
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
MOBILEGL_IPC_RUN_AHEAD: "1"
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
with:
gpu: "true"
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Install runtime dependencies
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
# Same set as the benchmark job, for the same reason: the scenarios bring
# up real headless EGL (llvmpipe) and Vulkan (lavapipe) contexts, and
# libegl-mesa0 - the EGL vendor library behind glvnd's libegl1 dispatch -
# only arrives as a Recommends.
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers
- name: Download Linux runtime
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-runtime
path: .
- name: Unpack Linux runtime
run: tar --zstd -xf mobilegl-linux-runtime.tar.zst
- name: Prepare downloaded test metadata
run: |
for root in build-linux build-verify build-split build-verify-split build-retrace; do
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
- name: Integration scenarios
working-directory: build-linux
# REQUIRE_GPU makes a driverless runner FAIL instead of skipping every
# scenario - an all-skip run is otherwise indistinguishable from a pass,
# which is how a five-month-old draw-dropping bug survived unseen until
# this lane existed.
#
# Configure CMake with the selected runner's ICD; each test's ENVIRONMENT
# property takes precedence over ambient job variables.
#
# Cores are armed so that any crash - the harness pre-flight child's
# included - leaves /tmp/core.*.zst, which the failure-only step below ships
# as an artifact. Analyzing a downloaded core (zstd -d) against the runtime
# artifact's binary in an ubuntu-26.04 userspace reproduces the exact
# crash stack without burning a CI round on an in-workflow debugger.
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_ITRP_FIX: "ForceOn"
run: |
ulimit -c unlimited
# An entry whose own ENVIRONMENT sets MOBILEGL_TRANSPORT overrides the one variable the two
# legs differ in, so it runs identically in both; the inproc leg leaves those entries (and
# the fixtures only they need) to the monolith-control leg, which every branch runs, and
# that leg proves it ran every one of them. When transport_pinned.py cannot prove the
# trimmed selection lost exactly them, this leg runs every entry and writes its JUnits
# under -all stems, which the skip census holds to the full name lists.
EXCLUDE=()
MAIN_JUNIT=integration
STAGED_JUNIT=integration-staged-copy
if [ "${MOBILEGL_CI_DISAGGREGATED}" = ON ]; then
ctest -L integration-gpu --show-only=json-v1 > "${RUNNER_TEMP}/all.json"
fi
if [ "${MOBILEGL_TRANSPORT}" = inproc ]; then
if python3 ../scripts/ci/transport_pinned.py select "${RUNNER_TEMP}/all.json" --out "${RUNNER_TEMP}/transport-pinned.txt" \
&& ctest -L integration-gpu --exclude-from-file "${RUNNER_TEMP}/transport-pinned.txt" --show-only=json-v1 \
> "${RUNNER_TEMP}/trimmed.json" \
&& python3 ../scripts/ci/transport_pinned.py assert-exclusion "${RUNNER_TEMP}/all.json" \
"${RUNNER_TEMP}/trimmed.json" --out "${RUNNER_TEMP}/transport-pinned.json"; then
EXCLUDE=(--exclude-from-file "${RUNNER_TEMP}/transport-pinned.txt")
else
echo "::warning::the transport-pinned exclusion did not prove itself, so this leg runs every integration-gpu entry (integration-all.xml, integration-staged-copy-all.xml)"
MAIN_JUNIT=integration-all
STAGED_JUNIT=integration-staged-copy-all
fi
fi
# Second, filtered pass: with the range-invalidating map flush disabled,
# the buffer scenarios run on the upload ring's staged-copy tier - which
# the default pass never reaches (the map tier absorbs every flush on
# the CI drivers), so without this the Mali fallback tier would have zero CI
# coverage. The flag is NOT baked into the ctest ENVIRONMENT properties,
# so an inline env reaches the test processes (unlike the ICD pin above).
if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
ctest -V -L integration-gpu "${EXCLUDE[@]}" --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/${MAIN_JUNIT}.xml"
MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 ctest -V -L integration-gpu "${EXCLUDE[@]}" -j "$(nproc)" \
-R 'Buffer|Readback|Atomic|Ssbo|Arena' --no-tests=error --output-junit "${RUNNER_TEMP}/${STAGED_JUNIT}.xml"
else
ctest --output-on-failure -L integration-gpu "${EXCLUDE[@]}" --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/${MAIN_JUNIT}.xml"
MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 ctest --output-on-failure -L integration-gpu "${EXCLUDE[@]}" -j "$(nproc)" \
-R 'Buffer|Readback|Atomic|Ssbo|Arena' --no-tests=error --output-junit "${RUNNER_TEMP}/${STAGED_JUNIT}.xml"
fi
if [ "${MOBILEGL_TRANSPORT}" = monolith ] && [ "${MOBILEGL_CI_DISAGGREGATED}" = ON ]; then
python3 ../scripts/ci/transport_pinned.py require-ran "${RUNNER_TEMP}/all.json" "${RUNNER_TEMP}/${MAIN_JUNIT}.xml"
fi
# Both backend roles must execute inproc in the primary artifact.
- name: Inproc execution proof (both backends)
if: matrix.transport == 'inproc'
working-directory: build-linux
run: |
ctest -L integration-p5f-rsp --show-only=json-v1 > "${RUNNER_TEMP}/primary-inproc-lane.json"
ctest -L integration-p5f-rsp --no-tests=error --output-on-failure \
--output-junit "${RUNNER_TEMP}/primary-inproc.xml"
python3 ../scripts/ci/runtime_mode_proof.py logs \
--discovery "${RUNNER_TEMP}/primary-inproc-lane.json" \
--junit "${RUNNER_TEMP}/primary-inproc.xml" --output "${RUNNER_TEMP}/primary-inproc-proof.json"
# The run log needs a signed-in reader; annotations do not. Name every failed ctest entry,
# with the lines of its output that say why, where anyone reading the run can see them.
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}"
- name: Upload integration lane evidence
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: integration-evidence-${{ matrix.lane }}-${{ matrix.transport }}
path: |
${{ runner.temp }}/integration*.xml
${{ runner.temp }}/primary-inproc*
${{ runner.temp }}/transport-pinned*
${{ runner.temp }}/all.json
${{ runner.temp }}/trimmed.json
build-linux/runtime-mode.json
build-linux/MobileGL/MG_IntegrationTest/p5f-rsp-*.log
if-no-files-found: warn
# P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py).
- name: Upload JUnit for the skip census
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: junit-integration-${{ matrix.lane }}-${{ matrix.transport }}
path: ${{ runner.temp }}/*.xml
if-no-files-found: warn
- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v7
with:
name: integration-core-dumps-${{ matrix.lane }}-${{ matrix.transport }}
path: /tmp/core.*
if-no-files-found: ignore
# THE THIRD CI MODE (ARCHITECTURE.md 13.2-(2)): the same library, built with the PipeInputs
# comparator compiled in, running the integration suite and a trace subset with two state models
# in one address space. It is a second build rather than a flag on the first because
# MOBILEGL_PIPE_VERIFY is a compile-time option - the snapshot, the entry compare and the
# compare-at-read hook do not exist in the shipped library, and are never meant to.
build-linux-verify:
runs-on: ubuntu-26.04
timeout-minutes: 120
permissions:
actions: write
contents: read
env:
BUILD_DIR: build-verify
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_COMPRESS: "true"
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 4G
CCACHE_NOHASHDIR: "true"
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Check out build submodules
timeout-minutes: 15
run: bash scripts/ci/checkout_submodules.sh library
- name: Set swap space
uses: ./.github/actions/set-swap-space
with:
swap-size-gb: 32
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Restore ccache
uses: actions/cache/restore@v5
with:
path: .ccache
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
restore-keys: |
${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-
- name: Prepare Vulkan SDK
id: netsetup530
continue-on-error: true
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Prepare Vulkan SDK (retry)
timeout-minutes: 15
if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup530.outcome == 'failure' }}
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Update glslang external sources
working-directory: 3rdparty/glslang
run: python3 update_glslang_sources.py
- name: Install build dependencies
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
run: |
sudo apt-get update
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
- name: Show installed toolchain
run: |
ccache --version
"${MOBILEGL_CI_CC}" --version
"${MOBILEGL_CI_CXX}" --version
"${MOBILEGL_CI_LLD}" --version
dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true
- name: Configure CMake
# Release/INFO like the shipped build on purpose. The poison arms in this configuration
# through MOBILEGL_PIPE_VERIFY (PipeInputs.h derives MOBILEGL_PIPE_POISON from it), so this
# job needs neither a Debug log level nor MOBILEGL_BUILD_DISAGGREGATED - and a Debug build
# would compare a different library from the one the other lanes measure. (build-linux
# switches to Debug under ACTIONS_STEP_DEBUG; this job deliberately does not - a Debug
# build flips CXX_VISIBILITY_PRESET and arms MOBILEGL_PIPE_POISON through a second, unrelated
# arm of its #if, so the debug switch would change what the lane is measuring.)
# Both disaggregation options default ON, so they are pinned OFF here: a disaggregated
# library expects a server-role log that this monolith lane never writes.
# integration-verify-split is the split arm.
# ENABLE_PCH=OFF: ccache cannot cache a compile that uses glslang's precompiled header
# (see scripts/ci/configure_build.sh).
run: |
cmake -S . -B "${BUILD_DIR}" -G Ninja \
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \
-DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DENABLE_PCH=OFF \
-DCMAKE_BUILD_TYPE=Release \
-DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
-DMOBILEGL_BUILD_TEST=ON \
-DMOBILEGL_BUILD_BENCHMARK=OFF \
-DMOBILEGL_BUILD_INTEGRATION_TEST=ON \
-DMOBILEGL_ITEST_VK_ICD="${VK_ICD_FILENAMES}" \
-DMOBILEGL_BUILD_TRACE_REPLAY=OFF \
-DMOBILEGL_PIPE_VERIFY=ON \
-DMOBILEGL_BUILD_DISAGGREGATED=OFF \
-DMOBILEGL_BUILD_DISAGGREGATED_INPROC=OFF \
-DCMAKE_POLICY_VERSION_MINIMUM=3.5
- name: Build
run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)"
# The lane is worthless if the option silently did not take, and that is a one-character
# mistake away at all times (a typo'd -D is not an error in CMake). Two checks, both cheap:
# the comparator's entry point must be in the library, and the fill entry point with it.
#
# `nm` and NOT `nm -D`. The library is built CXX_VISIBILITY_PRESET hidden in every non-Debug
# configuration (CMakeLists.txt:600-604) and the MGPipe entry points are plain namespace
# functions with no export attribute, so not one of them appears in the DYNAMIC table: on a
# perfectly healthy verify build `nm -D --defined-only ... | grep -c MGPipe` answers 0 out of
# ~11900 exported symbols, and a gate spelled that way is red forever for a reason that has
# nothing to do with what it claims to test. The static symbol table has them as local `t`
# entries, this artifact is never stripped, and `No MG_Remote in the pull build` below already
# uses this spelling. The symbol count guards the remaining hole: a stripped library would
# make both greps fail for a third, silent reason.
- name: Comparator symbols in verify library
run: |
test -f "${BUILD_DIR}/libMobileGL.so"
defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l)
if [ "${defined}" -lt 1000 ]; then
echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the two checks below could not have failed honestly"
exit 1
fi
if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeVerifyInputs"; then
echo "::error::libMobileGL.so defines no MGPipeVerifyInputs: -DMOBILEGL_PIPE_VERIFY=ON did not take, and every lane that consumes this artifact would run the comparator-free library and pass having compared nothing"
exit 1
fi
# The per-verb entry point, under EITHER of its two names. P2 renames
# MGPipeFillForVerb to MGPipeValidateForVerb (the body becomes the tracker's walk and
# the fill is one of its five steps), so this check has to accept both or it goes red on
# the rename for a reason that has nothing to do with what it tests. What it tests is
# unchanged: that the library HAS a per-verb entry point compiled in.
if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -qE "MGPipeValidateForVerb|MGPipeFillForVerb"; then
echo "::error::libMobileGL.so defines neither MGPipeValidateForVerb nor MGPipeFillForVerb: there is no per-verb entry point in this artifact, so nothing fills the block the comparator compares"
exit 1
fi
echo "libMobileGL.so defines MGPipeVerifyInputs and a per-verb entry point (${defined} defined symbols)"
- name: Show ccache stats
if: ${{ always() }}
run: ccache --show-stats
- name: Release superseded ccache entry
env:
GH_TOKEN: ${{ github.token }}
CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true
- name: Save ccache
continue-on-error: true
uses: actions/cache/save@v5
with:
path: .ccache
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
- name: Prepare packaged test metadata
run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}"
- name: Package Linux verify runtime
run: |
mkdir -p ci-artifacts
mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort)
tar \
--exclude='*/CMakeFiles' \
--exclude='*.o' \
--exclude='*.a' \
--exclude='*.ninja*' \
--exclude='build.ninja' \
--exclude='cmake_install.cmake' \
-I 'zstd -T0' -cf ci-artifacts/mobilegl-linux-runtime-verify.tar.zst \
"${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" \
"${BUILD_DIR}/MobileGL/MG_Test" \
"${BUILD_DIR}/MobileGL/MG_IntegrationTest" \
"${SHARED_LIBS[@]}"
# compression-level 0: the tarball is zstd already (see 'Upload Linux runtime').
- name: Upload Linux verify runtime
uses: actions/upload-artifact@v7
with:
name: mobilegl-linux-runtime-verify
path: ci-artifacts/mobilegl-linux-runtime-verify.tar.zst
if-no-files-found: error
compression-level: 0
# The retrace legs load only the library, and the server image on the spawn arm, so they
# take this instead of the whole test runtime.
- name: Package retrace library
run: |
files=("${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/libMobileGL.so")
if [ -f "${BUILD_DIR}/libMobileGLServer.so" ]; then files+=("${BUILD_DIR}/libMobileGLServer.so"); fi
tar -czf ci-artifacts/mobilegl-linux-lib-verify.tgz "${files[@]}"
- name: Upload retrace library
uses: actions/upload-artifact@v7
with:
name: mobilegl-linux-lib-verify
path: ci-artifacts/mobilegl-linux-lib-verify.tgz
if-no-files-found: error
# The verify lane itself, plus the controls that keep it falsifiable: its own control entries,
# checked by name, and the comparator's unit cases on its library. They are always on, not a
# manual exercise: a gate that can only be shown to work by someone remembering to break it on
# purpose is a gate that has already stopped working.
integration-verify:
runs-on: ubuntu-26.04
timeout-minutes: 180
needs: [build-linux-verify]
# Short-lived test processes skip the glslang prewarm, as in the test job.
env:
MOBILEGL_DISABLE_SHADER_PREWARM: "1"
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
with:
gpu: "true"
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Install runtime dependencies
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers
- name: Download Linux verify runtime
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-runtime-verify
path: .
- name: Unpack Linux verify runtime
run: |
tar --zstd -xf mobilegl-linux-runtime-verify.tar.zst
test -f build-verify/libMobileGL.so
- name: Prepare downloaded test metadata
run: |
for root in build-linux build-verify build-split build-verify-split build-retrace; do
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
- name: Integration scenarios (verify)
working-directory: build-verify
# --no-tests=error is half the gate: the verify entries only exist when the library was
# configured with -DMOBILEGL_PIPE_VERIFY=ON, so a build that lost the option matches no
# tests and reds here instead of reporting a green run of nothing. The other half is
# PipeVerifyArmingScenario.Armed, which fails when the library never printed its arming
# line - the failure mode a bare `MOBILEGL_PIPE_VERIFY=1` cannot detect by itself.
#
# SCOPE, stated so nobody reads more into a green than is there: this is every integration
# ENTRY under the comparator, not every integration CONFIGURATION. The `integration` job
# runs a second, filtered pass with MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 for the
# upload ring's staged-copy tier; that pass is 186 entries here and, at the 5-10x the
# comparator costs, is not affordable inside this job's budget. The tier is covered by
# `integration`, unverified, and P2 can take it once the comparator's cost is known.
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_ITRP_FIX: "ForceOn"
run: |
ulimit -c unlimited
if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
ctest -V -L integration-verify --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/verify.xml"
else
ctest --output-on-failure -L integration-verify --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/verify.xml"
fi
# The unit suite again, on the verify library: cases compiled only under MOBILEGL_PIPE_VERIFY
# skip in the `test` job's build and assert only here, as do poison-gated cases on branches
# where that build is a monolith. The packaging step above ships MG_Test whole, so the cost
# is the run itself. --no-tests=error, so a packaging change that dropped the unit binaries
# fails here instead of passing with nothing run. Among them is the comparator's per-field
# control, PipeInputsTest.EveryCorruptedFieldIsNamedByTheComparator.
- name: Unit tests on the verify runtime (G6, G10)
working-directory: build-verify
run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit-verify.xml"
# Handle-ABA, CSO, subsystem-mask and texture-parameter controls, rerun on the verify library.
# They carry integration-gpu, not integration-verify, so the verify lane misses them.
# Each -R alternative is a test-name substring, the shortest that selects only its family:
# `ResourceSubsystem` also reaches the LargeArenaAdoption lanes and the mask refusals,
# `MapPersistentRoundtrip` is singular so it matches the case name as well as the lane prefix,
# and `ObjectSubsystem` and `ResourceSubsystem` stay distinct families. TextureUploadShape
# records the upload shape; it fails only when its two counters are unreadable or disagree.
- name: Push-build controls (G8, G8b, G9, G10, G12)
working-directory: build-verify
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_ITRP_FIX: "ForceOn"
run: |
ulimit -c unlimited
ctest --output-on-failure -L integration-gpu \
-R 'HandleRecycle|CsoContentAddressing|ResourceSubsystem|MapPersistentRoundtrip|ObjectSubsystem|TextureParamsWithoutASamplerView|TextureUploadShape' \
--no-tests=error -j 4
# The lane's own control entries, by name: each must have run and passed, and a skip is a
# failure here although ctest reports it green. VerifyArming asserts in-test that the
# comparator printed its armed line with fatal=1, under the ambient entries' own environment
# (MG_IntegrationTest/CMakeLists.txt builds both from one list), VerifyCorrupted that a
# corrupted field is reported by name, and PoisonOmitted that an omitted stamp kills a
# forked child with its own Fatal.
- name: Verify lane control entries
run: |
python3 scripts/ci/junit_tally.py "${RUNNER_TEMP}/verify.xml" \
--require-entry-passed DirectGLES.VerifyArming.PipeVerifyArmingScenario.Armed \
--require-entry-passed DirectVulkan.VerifyArming.PipeVerifyArmingScenario.Armed \
--require-entry-passed DirectGLES.VerifyCorrupted.PipeVerifyArmingScenario.CorruptedFieldIsReported \
--require-entry-passed DirectVulkan.VerifyCorrupted.PipeVerifyArmingScenario.CorruptedFieldIsReported \
--require-entry-passed DirectGLES.PoisonOmitted.PoisonOmissionScenario.OmittedFieldAbortsOnThatVerb \
--require-entry-passed DirectVulkan.PoisonOmitted.PoisonOmissionScenario.OmittedFieldAbortsOnThatVerb
- name: Upload verify lane logs
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: integration-verify-logs
path: build-verify/MobileGL/MG_IntegrationTest/pipe-*.log*
if-no-files-found: warn
# P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py).
- name: Upload JUnit for the skip census
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: junit-integration-verify
path: ${{ runner.temp }}/*.xml
if-no-files-found: warn
- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v7
with:
name: integration-verify-core-dumps
path: /tmp/core.*
if-no-files-found: ignore
# The comparator on the split (inproc) arm, both backends, with its control entries checked by
# name.
#
# ITS OWN BUILD, because neither existing one can carry it: build-linux-verify configures
# without MOBILEGL_BUILD_DISAGGREGATED (so MG_Remote and the transport parser do not exist and
# `MOBILEGL_TRANSPORT=inproc` would be accepted and ignored), and build-linux-split configures
# without MOBILEGL_PIPE_VERIFY (so the comparator does not exist). The integration-verify-split
# entries are registered only when BOTH options are on, so `--no-tests=error` below reds a build
# that lost either one instead of reporting a green run of nothing. Built and run in one job:
# nothing downstream consumes this library.
#
# INPROC ONLY, BY THE INSTRUMENT'S NATURE: the comparator's two arms (the applier-written
# gPipeInputs and the frontend GLContext) are co-resident only when both roles share a process.
# MG_IntegrationTest/CMakeLists.txt's verify-split block records the spawn measurement.
integration-verify-split:
runs-on: ubuntu-26.04
needs: [build-linux-verify-split]
timeout-minutes: 180
# Short-lived test processes skip the glslang prewarm, as in the test job.
env:
MOBILEGL_DISABLE_SHADER_PREWARM: "1"
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
with:
gpu: "true"
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Install runtime dependencies
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers
- name: Download verify-split runtime
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-runtime-verify-split
path: .
- name: Unpack verify-split runtime
run: tar --zstd -xf mobilegl-linux-runtime-verify-split.tar.zst
# The lane. `^...$` because ctest -L is a regex (ID-131) and `integration-verify` would
# otherwise also match this label and the monolith one would match both.
- name: Prepare downloaded test metadata
run: |
for root in build-linux build-verify build-split build-verify-split build-retrace; do
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
- name: Verify-split scenarios (inproc)
working-directory: build-verify-split
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
run: |
ulimit -c unlimited
# --output-junit, because the arm proof below has to tell a gtest SKIP from a pass: an
# entry that skipped reached no verb honestly, and the skip set moves with the driver.
ctest --output-on-failure -L '^integration-verify-split$' --no-tests=error -j 4 \
--output-junit "${RUNNER_TEMP}/verify-split.xml"
# THE ARM PROOF, PER ENTRY AND BY NAME. Every ambient entry has a private log path
# (SplitLogPaths.cmake.in), so this - unlike integration-verify's grep over a shared log -
# is a census:
# * ConfigLoader's `MOBILEGL_TRANSPORT=inproc` line in EVERY client log: an entry that ran
# monolith would have passed every assertion above and fails here by name;
# * `MGPipe: verify armed` in both VerifySplitArming. logs;
# * and then the census proper. This used to be a FLOOR - "at least 850 armed of at least
# 1000 logs" - which leaves the remaining ~155 unnamed, so the lane could lose fifty
# arms, gain fifty skips, and the floor would not move. It is now the two-sided rule
# dualblock-expected-fatals.txt already rides: each client log must ARM, or belong to an
# entry ctest reported SKIPPED, or be NAMED in Harness/verify-split-unarmed-expected.txt
# (56 names in five classes, each class saying why it reaches no verb). A listed entry
# that arms, and a listed name with no log at all, both fail - an exception set that
# keeps rows nothing needs is how a proof rots green.
- name: Verify-split per-entry arm proof
working-directory: build-verify-split/MobileGL/MG_IntegrationTest
run: |
shopt -s nullglob
logs=(verify-split-logs/*.client.log)
if [ ${#logs[@]} -lt 1000 ]; then
echo "::error::found ${#logs[@]} per-entry client logs under verify-split-logs (expected ~1064): the private log paths did not take, so nothing below is a per-entry proof"
exit 1
fi
missing=$(grep -L 'Config: MOBILEGL_TRANSPORT=inproc' "${logs[@]}" | wc -l)
if [ "${missing}" -ne 0 ]; then
grep -L 'Config: MOBILEGL_TRANSPORT=inproc' "${logs[@]}" | head -20
echo "::error::${missing} verify-split entries carry no resolved MOBILEGL_TRANSPORT=inproc line: they did not run on the split arm"
exit 1
fi
for backend in DirectGLES DirectVulkan; do
if ! grep -q "MGPipe: verify armed" "pipe-verify-split-arming-${backend}.client.log"; then
echo "::error::pipe-verify-split-arming-${backend}.client.log carries no arming line"
exit 1
fi
done
python3 "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
verify-split-arming verify-split-logs "${RUNNER_TEMP}/verify-split.xml" \
"${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/verify-split-unarmed-expected.txt"
# The lane's own control entries, by name, as in integration-verify: VerifySplitReadCorrupted
# asserts in-test the two where=read reports in the server half, and VerifySplitPoisonOmitted
# reads both role halves of the forked child's log for its own Fatal.
- name: Verify-split control entries
run: |
args=()
for backend in DirectGLES DirectVulkan; do
args+=(--require-entry-passed "${backend}.VerifySplitArming.PipeVerifyArmingScenario.Armed")
args+=(--require-entry-passed "${backend}.VerifySplitCorrupted.PipeVerifyArmingScenario.CorruptedFieldIsReported")
args+=(--require-entry-passed "${backend}.VerifySplitReadCorrupted.PipeVerifyArmingScenario.CorruptedFieldIsReportedOnTheServerRead")
args+=(--require-entry-passed "${backend}.VerifySplitPoisonOmitted.PoisonOmissionScenario.OmittedFieldAbortsOnThatVerb")
done
python3 scripts/ci/junit_tally.py "${RUNNER_TEMP}/verify-split.xml" "${args[@]}"
# The run log needs a signed-in reader; annotations do not. Name every failed ctest entry,
# with the lines of its output that say why, where anyone reading the run can see them.
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}"
- name: Upload verify-split lane logs
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: integration-verify-split-logs
path: |
build-verify-split/MobileGL/MG_IntegrationTest/verify-split-logs/**
build-verify-split/MobileGL/MG_IntegrationTest/pipe-*split*.log*
if-no-files-found: warn
# P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py).
- name: Upload JUnit for the skip census
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: junit-integration-verify-split
path: ${{ runner.temp }}/*.xml
if-no-files-found: warn
- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v7
with:
name: integration-verify-split-core-dumps
path: /tmp/core.*
if-no-files-found: ignore
# Whether build-linux-split may skip its compile. On the disaggregation branches its configure
# line and build-linux's differ only in the benchmarks; this job configures both on one runner
# (configure only) and compares the trees (scripts/ci/build_fingerprint.py). Its verdict is
# identical, different or unknown, and its failure never fails the run: only 'identical' makes
# build-linux-split skip its compile and the split consumers take build-linux's artifacts,
# while 'different' and 'unknown' keep both builds. The builds run on the same runner image as
# this job, which records the image and compiler it saw; integration-split prints both sides.
# MOBILEGL_CI_DISAGGREGATED is build-linux's own expression, so a branch list that drifts
# between the two jobs configures an OFF primary here and gives 'different'.
build-guard:
name: build guard (split reuse)
runs-on: ubuntu-26.04
timeout-minutes: 150
if: ${{ github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main')) }}
# Consumers read only its outputs, and any verdict but 'identical' (none at all included)
# keeps both builds.
continue-on-error: true
outputs:
verdict: ${{ steps.guard.outputs.verdict || 'unknown' }}
image: ${{ steps.guard.outputs.image }}
compiler: ${{ steps.guard.outputs.compiler }}
env:
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }}
# Every step may fail without failing the job: a setup step that failed leaves a configure
# that fails, which the guard reports as 'unknown'.
steps:
- name: Checkout repo
continue-on-error: true
timeout-minutes: 15
uses: actions/checkout@v6
# The same submodule set as the two builds this job compares. The compare step runs only
# when every one of them arrived: without 3rdparty/flatbuffers both lines would configure
# the same library without MG_Remote, and compare as identical.
- name: Check out build submodules
id: submodules
continue-on-error: true
timeout-minutes: 15
run: bash scripts/ci/checkout_submodules.sh library
- name: Configure runner environment
continue-on-error: true
timeout-minutes: 15
uses: ./.github/actions/setup-test-runner
- name: Get CMake
continue-on-error: true
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Prepare Vulkan SDK
id: vulkan
continue-on-error: true
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Prepare Vulkan SDK (retry)
continue-on-error: true
timeout-minutes: 15
if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.vulkan.outcome == 'failure' }}
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Update glslang external sources
continue-on-error: true
timeout-minutes: 15
working-directory: 3rdparty/glslang
run: python3 update_glslang_sources.py || python3 update_glslang_sources.py
- name: Install build dependencies
continue-on-error: true
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
run: |
sudo apt-get update
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
# The self-test first: a guard whose canned cases fail decides nothing. The two configures
# share one FetchContent directory, so only the first fetches; each attempt is killed after
# 10 minutes and retried once, which this step's timeout leaves room for.
- name: Configure both lines and compare
id: guard
continue-on-error: true
timeout-minutes: 45
env:
CI_BUILD_TYPE: ${{ secrets.ACTIONS_STEP_DEBUG == 'true' && 'Debug' || 'Release' }}
SUBMODULES_OUTCOME: ${{ steps.submodules.outcome }}
run: |
if [ "${SUBMODULES_OUTCOME}" != success ]; then
echo "::warning::the submodule checkout did not succeed (outcome '${SUBMODULES_OUTCOME}'), so the guard decides nothing"
echo "verdict=unknown" >> "${GITHUB_OUTPUT}"
elif python3 scripts/ci/build_fingerprint.py --self-test > "${RUNNER_TEMP}/fingerprint-self-test.txt"; then
python3 scripts/ci/build_fingerprint.py guard --primary-dir build-guard-primary \
--split-dir build-guard-split --deps-dir build-guard-deps \
--github-output verdict --report "${RUNNER_TEMP}/build-guard.json"
else
cat "${RUNNER_TEMP}/fingerprint-self-test.txt"
echo "::warning::build_fingerprint.py's self-test failed, so the guard decides nothing"
echo "verdict=unknown" >> "${GITHUB_OUTPUT}"
fi
- name: Upload guard report
if: ${{ always() }}
continue-on-error: true
uses: actions/upload-artifact@v7
with:
name: build-guard
path: ${{ runner.temp }}/build-guard.json
if-no-files-found: warn
# The split build: the same sources with the transport and its in-process arm always on, which the
# split integration and retrace lanes consume. When build-guard proves its configuration
# identical to build-linux's (the disaggregation branches, benchmarks apart), this job builds
# nothing and the consumers take build-linux's artifacts; on any other verdict, and where
# build-guard does not run, it builds.
#
# The `nm` step is the point of a build of its own. In a build without the option, MG_Remote is
# not compiled and MOBILEGL_TRANSPORT's parser does not exist, so the variable is accepted and
# ignored and every downstream lane would run monolith under a name that says split. CMake does
# not complain about a typo'd -D, so the build-level assertion is the guard; in reuse mode
# build-linux's runtime_mode_proof (--mode disaggregated) is the same assertion and more.
build-linux-split:
runs-on: ubuntu-26.04
timeout-minutes: 120
needs: [build-guard]
# A build-guard that was skipped, failed or undecided means this job builds.
if: ${{ !cancelled() }}
permissions:
actions: write
contents: read
env:
BUILD_DIR: build-split
SPLIT_REUSE: ${{ needs.build-guard.outputs.verdict == 'identical' && 'true' || 'false' }}
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_COMPRESS: "true"
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 4G
CCACHE_NOHASHDIR: "true"
steps:
- name: Reuse decision
env:
GUARD_VERDICT: ${{ needs.build-guard.outputs.verdict }}
run: |
if [ "${SPLIT_REUSE}" = true ]; then
echo "build-guard found the two configurations identical: this job builds nothing and the split consumers take build-linux's artifacts"
else
echo "build-guard verdict '${GUARD_VERDICT:-none}': this job builds the split library"
fi
- name: Checkout repo
if: env.SPLIT_REUSE != 'true'
uses: actions/checkout@v6
# The submodule list is load-bearing here beyond the usual: MOBILEGL_BUILD_DISAGGREGATED
# shadows itself back to OFF when 3rdparty/flatbuffers/include is missing (a normal
# variable rather than a cache force, deliberately), so a checkout without it would
# configure cleanly and build a monolith library. The script fails when the header is not
# there, and the nm step below is the second check.
- name: Check out build submodules
if: env.SPLIT_REUSE != 'true'
timeout-minutes: 15
run: bash scripts/ci/checkout_submodules.sh library
- name: Configure runner environment
if: env.SPLIT_REUSE != 'true'
uses: ./.github/actions/setup-test-runner
- name: Get CMake
if: env.SPLIT_REUSE != 'true'
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Prepare Vulkan SDK
id: netsetup1274
continue-on-error: true
timeout-minutes: 15
if: ${{ env.SPLIT_REUSE != 'true' && (env.MOBILEGL_CI_NATIVE_GPU != '1') }}
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Prepare Vulkan SDK (retry)
timeout-minutes: 15
if: ${{ env.SPLIT_REUSE != 'true' && ((env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup1274.outcome == 'failure') }}
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Update glslang external sources
if: env.SPLIT_REUSE != 'true'
working-directory: 3rdparty/glslang
run: python3 update_glslang_sources.py
- name: Install build dependencies
timeout-minutes: 15
if: ${{ env.SPLIT_REUSE != 'true' && (env.MOBILEGL_CI_NATIVE_GPU != '1') }}
run: |
sudo apt-get update
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
- name: Show installed toolchain
if: env.SPLIT_REUSE != 'true'
run: |
ccache --version
"${MOBILEGL_CI_CC}" --version
"${MOBILEGL_CI_CXX}" --version
dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true
# Release/INFO like the shipped build; the line and the reasons are in
# scripts/ci/configure_build.sh, shared with build-linux.
- name: Configure CMake
if: env.SPLIT_REUSE != 'true'
run: bash scripts/ci/configure_build.sh split "${BUILD_DIR}"
- name: Set swap space
if: env.SPLIT_REUSE != 'true'
uses: ./.github/actions/set-swap-space
with:
swap-size-gb: 32
- name: Restore ccache
if: env.SPLIT_REUSE != 'true'
uses: actions/cache/restore@v5
with:
path: .ccache
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
restore-keys: |
${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-
- name: Build
if: env.SPLIT_REUSE != 'true'
run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)"
# `nm` and NOT `nm -D`, for the reason build-linux-verify spells out: everything under
# MG_Remote is a plain namespace symbol in a CXX_VISIBILITY_PRESET=hidden Release build and
# none of it reaches the dynamic table. The symbol count guards the remaining hole - a
# stripped library would make the grep fail for a third, silent reason.
#
- name: MG_Remote symbols in split library
if: env.SPLIT_REUSE != 'true'
run: |
test -f "${BUILD_DIR}/libMobileGL.so"
defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l)
if [ "${defined}" -lt 1000 ]; then
echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the checks below could not have failed honestly"
exit 1
fi
remote=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -c -i "MG_Remote" || true)
if [ "${remote}" -lt 1 ]; then
echo "::error::libMobileGL.so defines no MG_Remote symbol: -DMOBILEGL_BUILD_DISAGGREGATED=ON did not take (a typo'd -D is not a CMake error, and the option shadows itself OFF when 3rdparty/flatbuffers/include is missing). Every lane that consumes this artifact would run MONOLITH while claiming to run split, because the MOBILEGL_TRANSPORT parser does not exist in such a build and the variable is accepted and ignored."
exit 1
fi
# The transport parser itself, which is the symbol the runtime evidence depends on.
if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeApply"; then
echo "::error::libMobileGL.so defines no MGPipeApply* entry point, so MOBILEGL_PIPE_PUSH did not take either and there is no applier for the split path to decode into"
exit 1
fi
echo "libMobileGL.so defines ${remote} MG_Remote symbol(s) and the MGPipe applier (${defined} defined symbols)"
# Server link-closure ratchet (ARCHITECTURE.md §13.2): scripts/link_ratchet.py recomputes
# from the per-object partition the frontend symbols a server-only image still has to link
# and compares the list with the stored baseline. Only new symbols fail; a vanished one is
# printed with a request to re-baseline in the same commit. The baseline is a list, not a
# count, so swapping one symbol for another fails. It runs where MG_Remote is built (the
# script re-checks CMakeCache.txt) and before anything deletes the *.o files it reads: here,
# or in build-linux when this job reuses that build.
- name: Server link-closure ratchet
if: env.SPLIT_REUSE != 'true'
run: |
mkdir -p ci-artifacts
python3 scripts/link_ratchet.py --self-test
python3 scripts/link_ratchet.py \
--build-dir "${BUILD_DIR}" \
--baseline scripts/data/link_ratchet_baseline.txt \
--assert-monotone \
--bucket \
--json ci-artifacts/link-ratchet.json
# Uploaded even when the gate fired: a red ratchet is exactly the moment someone needs the
# per-symbol definer/referrer attribution the JSON carries.
- name: Upload link ratchet
if: ${{ always() && env.SPLIT_REUSE != 'true' }}
uses: actions/upload-artifact@v7
with:
name: link-ratchet
path: ci-artifacts/link-ratchet.json
if-no-files-found: warn
- name: Show ccache stats
if: ${{ always() && env.SPLIT_REUSE != 'true' }}
run: ccache --show-stats
# Only after a build: with nothing built, deleting would drop this branch's entry for good.
- name: Release superseded ccache entry
if: env.SPLIT_REUSE != 'true'
env:
GH_TOKEN: ${{ github.token }}
CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true
- name: Save ccache
if: env.SPLIT_REUSE != 'true'
continue-on-error: true
uses: actions/cache/save@v5
with:
path: .ccache
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
- name: Prepare packaged test metadata
if: env.SPLIT_REUSE != 'true'
run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}"
- name: Package Linux split runtime
if: env.SPLIT_REUSE != 'true'
run: |
mkdir -p ci-artifacts
mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort)
tar \
--exclude='*/CMakeFiles' \
--exclude='*.o' \
--exclude='*.a' \
--exclude='*.ninja*' \
--exclude='build.ninja' \
--exclude='cmake_install.cmake' \
-I 'zstd -T0' -cf ci-artifacts/mobilegl-linux-runtime-split.tar.zst \
"${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" \
"${BUILD_DIR}/MobileGL/MG_Test" \
"${BUILD_DIR}/MobileGL/MG_IntegrationTest" \
"${SHARED_LIBS[@]}"
# compression-level 0: the tarball is zstd already (see 'Upload Linux runtime').
- name: Upload Linux split runtime
if: env.SPLIT_REUSE != 'true'
uses: actions/upload-artifact@v7
with:
name: mobilegl-linux-runtime-split
path: ci-artifacts/mobilegl-linux-runtime-split.tar.zst
if-no-files-found: error
compression-level: 0
# The retrace legs load only the library, and the server image on the spawn arm, so they
# take this instead of the whole test runtime.
- name: Package retrace library
if: env.SPLIT_REUSE != 'true'
run: |
files=("${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/libMobileGL.so")
if [ -f "${BUILD_DIR}/libMobileGLServer.so" ]; then files+=("${BUILD_DIR}/libMobileGLServer.so"); fi
tar -czf ci-artifacts/mobilegl-linux-lib-split.tgz "${files[@]}"
- name: Upload retrace library
if: env.SPLIT_REUSE != 'true'
uses: actions/upload-artifact@v7
with:
name: mobilegl-linux-lib-split
path: ci-artifacts/mobilegl-linux-lib-split.tgz
if-no-files-found: error
# The split lane itself: the three-arm shape ARCHITECTURE.md §13.2 asks for verbatim, plus the
# first real run of the five MG_Test/Wire suites. P5's two exit-gate negative controls are in-test
# now (RemoteWaitBoundaryControl.AnUnwaitedPublishIsSeenAsNoWait and
# PersistentCoherentMapScenario.ThePushIsTheOnlyRouteAWriteTakesToTheServer).
integration-split:
runs-on: ubuntu-26.04
timeout-minutes: 120
needs: [build-guard, build-linux-split, build-linux]
if: ${{ !cancelled() && needs.build-linux-split.result == 'success' && (needs.build-guard.outputs.verdict != 'identical' || needs.build-linux.result == 'success') }}
# The skip census reads whether 'Default-knob resolution check' proved the IPC defaults.
outputs:
defaults-on: ${{ steps.resolution.outputs.defaults-on }}
# Short-lived test processes skip the glslang prewarm, as in the test job.
env:
MOBILEGL_DISABLE_SHADER_PREWARM: "1"
SPLIT_REUSE: ${{ needs.build-guard.outputs.verdict == 'identical' && 'true' || 'false' }}
SPLIT_BUILD_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }}
SPLIT_RUNTIME: ${{ needs.build-guard.outputs.verdict == 'identical' && 'mobilegl-linux-runtime' || 'mobilegl-linux-runtime-split' }}
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
# Which build these lanes take: build-linux's when build-guard proved the two configurations
# identical, build-linux-split's otherwise. build-guard compared them on its own runner, so
# the image and compiler it saw are printed beside build-linux's. It reds only when it would
# reuse a build-linux that is not Release (see scripts/ci/split_reuse_build_type.sh).
- name: Split runtime
env:
GUARD_VERDICT: ${{ needs.build-guard.outputs.verdict }}
GUARD_IMAGE: ${{ needs.build-guard.outputs.image }}
GUARD_COMPILER: ${{ needs.build-guard.outputs.compiler }}
PRIMARY_IMAGE: ${{ needs.build-linux.outputs.image }}
PRIMARY_COMPILER: ${{ needs.build-linux.outputs.compiler }}
PRIMARY_BUILD_TYPE: ${{ needs.build-linux.outputs.build-type }}
run: |
echo "split runtime: ${SPLIT_BUILD_DIR} (build-guard verdict '${GUARD_VERDICT:-none}')"
bash scripts/ci/split_reuse_build_type.sh
echo "build-guard ran on '${GUARD_IMAGE}' with '${GUARD_COMPILER}'"
echo "build-linux ran on '${PRIMARY_IMAGE}' with '${PRIMARY_COMPILER}'"
if [ "${SPLIT_REUSE}" = true ] && { [ "${GUARD_IMAGE}" != "${PRIMARY_IMAGE}" ] || [ "${GUARD_COMPILER}" != "${PRIMARY_COMPILER}" ]; }; then
echo "::notice::build-guard and build-linux saw different runner images or compilers; the reused build is the one build-linux configured"
fi
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
with:
gpu: "true"
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Install runtime dependencies
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers
- name: Download Linux split runtime
uses: actions/download-artifact@v8
with:
name: ${{ env.SPLIT_RUNTIME }}
path: .
- name: Unpack Linux split runtime
run: |
tar --zstd -xf "${SPLIT_RUNTIME}.tar.zst"
test -f "${SPLIT_BUILD_DIR}/libMobileGL.so"
if [ "${SPLIT_REUSE}" = "true" ]; then
python3 -c 'import json, sys; mode = json.load(open(sys.argv[1]))["mode"]; sys.exit(0 if mode == "disaggregated" else f"build-linux runtime mode is {mode}, not disaggregated")' \
"${SPLIT_BUILD_DIR}/runtime-mode.json"
fi
- name: Prepare downloaded test metadata
run: |
for root in build-linux build-verify build-split build-verify-split build-retrace; do
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
- name: Spawn/split lane parity (9.2)
run: python3 scripts/ci/spawn_lane_parity.py "${SPLIT_BUILD_DIR}"
# P8-A (ID-P8-2): every monolith case, tails normalized, is on a gated split arm or is named
# with a reason class in scripts/data/split_coverage_exemptions.txt. The same `ctest -N`
# cost as the parity step above, and the question it cannot answer: parity compares the
# split arms with EACH OTHER, so a case missing from all three passes it.
- name: Split coverage against monolith (P8-A)
run: |
python3 scripts/ci/split_coverage.py --self-test
python3 scripts/ci/split_coverage.py "${SPLIT_BUILD_DIR}"
# P8-SE: PipeCatalogTest's unmigrated-emulation list equals the MGPipeUnmigratedEmulation
# call sites in the tree, one site per name. A source lint: no build needed.
- name: Unmigrated-emulation list against its call sites (P8-SE)
run: |
python3 scripts/ci/unmigrated_emulation_sites.py --self-test
python3 scripts/ci/unmigrated_emulation_sites.py
# The test job runs the same unit lane, with the same REQUIRE_GPU, on this same build when
# build-linux-split was reused; the strict unit step in integration-split-strict always runs.
- name: Unit tests on the split runtime
if: env.SPLIT_REUSE != 'true'
working-directory: ${{ env.SPLIT_BUILD_DIR }}
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit-split.xml"
# TcpLane.SupervisorProtocolControls and TcpLane.ControlFrameFuzz build the protocol's Python
# bindings with flatc, and without one they exit 77 and ctest reports a skip. flatc is not in
# the build graph (see gen_protocol.py), so this builds the same pinned flatc as flatc-check,
# and the lane step requires both entries PASSED by name through junit_tally.py. --check keeps
# this step from regenerating the committed header; checking that header is flatc-check's job.
# The binary comes from the cache entry flatc-check also uses, keyed on the submodule's
# commit; gen_protocol.py builds it when the entry is missing and checks its version against
# the submodule either way.
- name: Check out FlatBuffers (TCP lane)
run: git submodule update --init 3rdparty/flatbuffers
- name: Pinned flatc cache key
id: flatc
run: echo "commit=$(git rev-parse HEAD:3rdparty/flatbuffers)" >> "${GITHUB_OUTPUT}"
- name: Cache pinned flatc
uses: actions/cache@v5
with:
path: ${{ runner.temp }}/flatc-build/flatc
key: ${{ runner.os }}-ubuntu26-flatc-${{ steps.flatc.outputs.commit }}
- name: Build pinned flatc (TCP lane)
run: python3 scripts/gen_protocol.py --build-dir "${RUNNER_TEMP}/flatc-build" --check
# --no-tests=error is half the gate, exactly as in integration-verify: the integration-split
# entries exist only when the library was configured with -DMOBILEGL_BUILD_DISAGGREGATED=ON,
# so a build that lost the option matches nothing and reds here instead of reporting a green
# run of nothing. The other half is in-test: the split fixture fails an armed entry whose
# process did not resolve the transport its registration claims.
#
# An entry whose owning package (c1 client, s1 session, v1 server) has not landed SKIPS with
# the reason and never goes green - MG_IntegrationTest/CMakeLists.txt probes MG_Remote for
# c0's signature stubs and disarms the lane while any remain.
#
# The three arms run as one ctest invocation: the TCP arm holds one resource lock, so on its
# own it runs one entry at a time and leaves the other cores idle. Each arm keeps its own
# selection, JUnit and checks.
- name: Split lanes (inproc, spawn, TCP)
working-directory: ${{ env.SPLIT_BUILD_DIR }}
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
run: |
ulimit -c unlimited
export MOBILEGL_FLATC_EXECUTABLE="${RUNNER_TEMP}/flatc-build/flatc"
test -x "${MOBILEGL_FLATC_EXECUTABLE}"
# Each arm is selected by its own label regex. scripts/ci/lane_selection.py turns the
# three selections into one list of exact names and fails when an arm selects nothing,
# which is what --no-tests=error says for an arm run on its own. junit_tally.py then
# carves each arm's JUnit out of the run, and fails when a selected name is missing
# from the run or reported twice.
ctest --show-only=json-v1 -L integration-split > "${RUNNER_TEMP}/split-discovery.json"
ctest --show-only=json-v1 -L integration-spawn > "${RUNNER_TEMP}/spawn-discovery.json"
ctest --show-only=json-v1 -L '^integration-tcp$' > "${RUNNER_TEMP}/tcp-discovery.json"
# A directory of its own: the job uploads RUNNER_TEMP/*.xml for the skip census, which
# has rows for the three arms and none for the run that holds them all.
mkdir -p "${RUNNER_TEMP}/split-arms"
python3 ../scripts/ci/lane_selection.py union "${RUNNER_TEMP}/split-arms/tests.txt" \
split="${RUNNER_TEMP}/split-discovery.json" spawn="${RUNNER_TEMP}/spawn-discovery.json" \
tcp="${RUNNER_TEMP}/tcp-discovery.json"
rc=0
ctest --output-on-failure --tests-from-file "${RUNNER_TEMP}/split-arms/tests.txt" --no-tests=error -j 4 \
--output-junit "${RUNNER_TEMP}/split-arms/all.xml" || rc=$?
# Carved before the exit code is acted on, so a red run still leaves each arm's JUnit
# for the annotations and the skip census.
carved=0
for arm in split spawn tcp; do
python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/split-arms/all.xml" \
--select-from "${RUNNER_TEMP}/${arm}-discovery.json" \
--write-selected "${RUNNER_TEMP}/${arm}-baseline.xml" || carved=1
done
if [ "${rc}" -ne 0 ]; then
echo "::error::the split, spawn and TCP arms' run failed (ctest exit ${rc})"
exit "${rc}"
fi
if [ "${carved}" -ne 0 ]; then
echo "::error::the run does not hold every entry of the split, spawn and TCP arms exactly once, so it cannot stand for the three arms"
exit 1
fi
python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/split-baseline.xml" --require-split-ran
# Exit gate 9.3: the spawn arm green at the same count. It runs the same cases through a
# second process, so a failure here that the inproc arm did not see is a transport fact
# and not a phase one - which is the whole reason both lanes exist.
# PH-6 / fuzz arm 3: the supervisor-level forfeit control rides this lane and must RUN.
python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/spawn-baseline.xml" --require-spawn-ran --require-entry-passed SpawnLane.EventForfeitPeer
python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/tcp-baseline.xml" --require-tcp-ran --require-run-ahead --discovery "${RUNNER_TEMP}/tcp-discovery.json"
python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/tcp-baseline.xml" --require-entry-passed TcpLane.SupervisorProtocolControls --require-entry-passed TcpLane.EventForfeitPeer --require-entry-passed TcpLane.ControlFrameFuzz
# Whether each entry ran the transport it claims is asserted in-test: the split fixture
# (Harness/ScenarioFixture.h) fails an armed entry whose process resolved another transport
# or whose server is not where the claim says, and junit_tally's --require-*-ran above
# require each arm to have passed entries. This step proves the IPC defaults the lanes
# rely on, from every passed entry's own client log; it runs right after the lane, so it
# reads the lane's own run rather than a later step's rerun of the same entry.
- name: Default-knob resolution check
id: resolution
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
working-directory: ${{ env.SPLIT_BUILD_DIR }}
run: |
# The IPC knobs every passed entry resolved, from its own client log: the defaults the
# lane relies on (run-ahead and the role-split state on, strict off) unless the entry
# pins its own. Proven defaults make Transport A/B below a repeat of the integration
# job's two legs; exit 3 says a default changed, and A/B then runs.
ctest -L integration-split --show-only=json-v1 > "${RUNNER_TEMP}/split-lane.json"
rc=0
python3 "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" ipc-config \
"${RUNNER_TEMP}/split-lane.json" "${RUNNER_TEMP}/split-baseline.xml" "${RUNNER_TEMP}/split-ipc.json" \
strict=0 role-split-state=1 run-ahead=1 --assumed strict,role-split-state,run-ahead || rc=$?
if [ "${rc}" -eq 0 ]; then
echo "defaults-on=true" >> "${GITHUB_OUTPUT}"
elif [ "${rc}" -eq 3 ]; then
echo "::warning::an IPC default differs from what the lanes assume (split-ipc.json names the entries), so Transport A/B runs"
echo "defaults-on=false" >> "${GITHUB_OUTPUT}"
else
exit "${rc}"
fi
# Both full transport runs are hard gates. The per-case comparison below
# remains diagnostic because explicit mechanism controls retain their own mode.
# With build-linux reused and the defaults proven above, the integration job's two legs
# already ran these entries on this library in these configurations, so this is skipped.
#
# The entries that name MOBILEGL_TRANSPORT in their OWN property (the Split. lanes) keep
# their value in both passes, which is correct: they are the split family in both arms and
# the comparison is about the other 1100.
- name: Transport A/B (monolith vs inproc)
if: env.SPLIT_REUSE != 'true' || steps.resolution.outputs.defaults-on != 'true'
working-directory: ${{ env.SPLIT_BUILD_DIR }}
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_ITRP_FIX: "ForceOn"
run: |
ulimit -c unlimited
count=$(ctest -N -L integration-gpu | grep -cE '^ *Test *#[0-9]+:')
if [ "${count}" -lt 1 ]; then
echo "::error::the split runtime registers ${count} integration-gpu entries"
exit 1
fi
echo "integration-gpu entries in the split build: ${count}"
MOBILEGL_TRANSPORT=monolith ctest --output-on-failure -L integration-gpu --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/arm-monolith.xml"
MOBILEGL_TRANSPORT=inproc MOBILEGL_IPC_ROLE_SPLIT_STATE=1 MOBILEGL_IPC_STRICT_ERRORS=1 ctest --output-on-failure -L integration-gpu --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/arm-inproc.xml"
# Both commands above must succeed; report differences among legitimate skips.
python3 - "${RUNNER_TEMP}/arm-monolith.xml" "${RUNNER_TEMP}/arm-inproc.xml" <<'PY'
import sys, xml.etree.ElementTree as ET
def rows(path):
out = {}
for case in ET.parse(path).getroot().iter('testcase'):
status = 'passed'
if case.find('failure') is not None or case.find('error') is not None:
status = 'failed'
elif case.find('skipped') is not None or case.get('status') in ('notrun', 'disabled'):
status = 'skipped'
out[case.get('name')] = status
return out
a, b = rows(sys.argv[1]), rows(sys.argv[2])
diff = sorted(set(a) ^ set(b)) + sorted(n for n in set(a) & set(b) if a[n] != b[n])
if diff:
for name in diff[:40]:
print(f"{name}: monolith={a.get(name, '<absent>')} inproc={b.get(name, '<absent>')}")
print(f"Transport comparison: {len(diff)} name/status differences after both full runs passed")
else:
print(f"the two arms agree on all {len(a)} entries, name and status")
PY
# The DirectGLES.Split. entries and their log-ownership check, once, in a job whose 'Split
# lanes' step never ran because an earlier step failed: without this, a red unit lane or
# flatc build would hide every split integration result of the run. When the lane step ran
# it left split-baseline.xml, and this step does nothing.
- name: Split lane fallback (lane step skipped)
if: ${{ !cancelled() }}
working-directory: ${{ env.SPLIT_BUILD_DIR }}
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
run: |
if [ -f "${RUNNER_TEMP}/split-baseline.xml" ]; then
echo "the lane step ran and left split-baseline.xml; nothing to fall back to"
exit 0
fi
echo "::notice::the lane step left no split-baseline.xml, so the DirectGLES.Split. entries run here"
python3 "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" check ctest "$PWD"
# A directory of its own: the job uploads RUNNER_TEMP/*.xml for the skip census, which
# has no row for this run.
mkdir -p "${RUNNER_TEMP}/split-fallback"
ctest --output-on-failure -L integration-split -R '^DirectGLES\.Split\.' -j 4 --no-tests=error \
--output-junit "${RUNNER_TEMP}/split-fallback/isplit.xml"
python3 "${GITHUB_WORKSPACE}/scripts/ci/junit_tally.py" "${RUNNER_TEMP}/split-fallback/isplit.xml" --require-split-ran
# build-linux ran the server link-closure ratchet on the build these lanes took, with
# continue-on-error so that a red there does not skip build-linux's own consumers.
- name: Server link-closure ratchet of the reused build
if: ${{ !cancelled() && env.SPLIT_REUSE == 'true' }}
env:
PRIMARY_RATCHET: ${{ needs.build-linux.outputs.link-ratchet }}
run: |
if [ "${PRIMARY_RATCHET}" != success ]; then
echo "::error::the server link-closure ratchet of the build-linux build these lanes took did not pass (outcome '${PRIMARY_RATCHET}'); build-linux's link-ratchet-primary artifact names the symbols"
exit 1
fi
echo "the server link-closure ratchet passed on the build-linux build these lanes took"
# The run log needs a signed-in reader; annotations do not. Name every failed ctest entry,
# with the lines of its output that say why, where anyone reading the run can see them.
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}"
- name: Upload split lane logs
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: integration-split-logs
path: |
${{ env.SPLIT_BUILD_DIR }}/MobileGL/MG_IntegrationTest/*.log*
${{ env.SPLIT_BUILD_DIR }}/MobileGL/MG_IntegrationTest/split-logs/*.log
${{ runner.temp }}/arm-inproc.xml
${{ runner.temp }}/arm-monolith.xml
if-no-files-found: warn
# P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py).
- name: Upload JUnit for the skip census
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: junit-integration-split
path: ${{ runner.temp }}/*.xml
if-no-files-found: warn
- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v7
with:
name: integration-split-core-dumps
path: /tmp/core.*
if-no-files-found: ignore
# Shares build-linux-split with the functional/transport job above. This job
# checks state ownership: strict residual-read rules, distinct client/server
# blocks, zero-residual assertions, and complete per-entry result accounting.
# Configured controls must pass without skips. Capability gaps and known
# limitations are reported separately and accepted only by exact name/reason.
integration-split-strict:
runs-on: ubuntu-26.04
timeout-minutes: 60
needs: [build-guard, build-linux-split, build-linux]
if: ${{ !cancelled() && needs.build-linux-split.result == 'success' && (needs.build-guard.outputs.verdict != 'identical' || needs.build-linux.result == 'success') }}
# Short-lived test processes skip the glslang prewarm, as in the test job.
env:
MOBILEGL_DISABLE_SHADER_PREWARM: "1"
SPLIT_REUSE: ${{ needs.build-guard.outputs.verdict == 'identical' && 'true' || 'false' }}
SPLIT_BUILD_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }}
SPLIT_RUNTIME: ${{ needs.build-guard.outputs.verdict == 'identical' && 'mobilegl-linux-runtime' || 'mobilegl-linux-runtime-split' }}
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
# build-linux's build when build-guard proved the two configurations identical,
# build-linux-split's otherwise; integration-split prints the guard's evidence. A build-linux
# that is not Release is never reused (scripts/ci/split_reuse_build_type.sh).
- name: Split runtime
env:
GUARD_VERDICT: ${{ needs.build-guard.outputs.verdict }}
PRIMARY_BUILD_TYPE: ${{ needs.build-linux.outputs.build-type }}
run: |
echo "split runtime: ${SPLIT_BUILD_DIR} (build-guard verdict '${GUARD_VERDICT:-none}')"
bash scripts/ci/split_reuse_build_type.sh
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
with:
gpu: "true"
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Install runtime dependencies
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers vulkan-validationlayers
- name: Download Linux split runtime
uses: actions/download-artifact@v8
with:
name: ${{ env.SPLIT_RUNTIME }}
path: .
- name: Unpack Linux split runtime
run: |
tar --zstd -xf "${SPLIT_RUNTIME}.tar.zst"
test -f "${SPLIT_BUILD_DIR}/libMobileGL.so"
if [ "${SPLIT_REUSE}" = "true" ]; then
python3 -c 'import json, sys; mode = json.load(open(sys.argv[1]))["mode"]; sys.exit(0 if mode == "disaggregated" else f"build-linux runtime mode is {mode}, not disaggregated")' \
"${SPLIT_BUILD_DIR}/runtime-mode.json"
fi
- name: Prepare downloaded test metadata
run: |
for root in build-linux build-verify build-split build-verify-split build-retrace; do
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
- name: Unit tests on the split runtime (strict)
working-directory: ${{ env.SPLIT_BUILD_DIR }}
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit-strict.xml"
# ---- Strict split lane: residual pulls under MOBILEGL_IPC_STRICT_ERRORS=1 ----
#
# An unbarriered pull is always fatal, and strict also stops any barriered pull that is not
# admitted (ARCHITECTURE.md §9.2.1). The lane must be green, and its markers must match the
# admitted list and strict-expected-markers.txt in both directions. On the disaggregation
# branches the dual-block step below runs these entries in this same environment and makes
# the same checks, so this step runs on dev only.
- name: Strict split lane
if: env.MOBILEGL_CI_DISAGGREGATED != 'ON'
working-directory: ${{ env.SPLIT_BUILD_DIR }}
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
MOBILEGL_TRANSPORT: "inproc"
run: |
ulimit -c unlimited
helper="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py"
expected="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/strict-expected-markers.txt"
# THE LOG SET COMES FROM ctest, NOT FROM A DIRECTORY (P5e gl, ID-119). This step used to
# grep MobileGL/MG_IntegrationTest/split-logs/, which holds 90 of the lane's entries.
# The 26 it could not see were the F1. readback block, both NamedBlit pairs, the four
# Ct. entries and PersistentMapArm - i.e. PRECISELY the readback population the
# allowlist is about, plus the only DirectVulkan entries in the lane. A directory is a
# guess about where the lane put its logs; `--show-only=json-v1` is the lane saying so.
ctest -L integration-split --show-only=json-v1 > "${RUNNER_TEMP}/lane.json"
# Stale logs from an earlier job would be read as this run's evidence: an entry that
# never starts leaves the previous file in place, and "no marker" is exactly what a
# lane that never ran looks like.
python3 - "${RUNNER_TEMP}/lane.json" "${helper}" <<'PY'
import importlib.util, json, os, sys
spec = importlib.util.spec_from_file_location("slp", sys.argv[2])
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
logs = slp.marker_log_paths(json.load(open(sys.argv[1])))
for path in set(logs.values()):
try:
os.unlink(path)
except OSError:
pass
print(f"reset {len(set(logs.values()))} private log(s) across {len(logs)} entries")
PY
rc=0
ctest -L integration-split --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/strict-split.xml" \
> "${RUNNER_TEMP}/strict-split.txt" 2>&1 || rc=$?
tail -5 "${RUNNER_TEMP}/strict-split.txt"
echo "strict run: rc=${rc}"
python3 "${GITHUB_WORKSPACE}/scripts/gen_pipe_field_ownership.py" --print-admitted \
> "${RUNNER_TEMP}/allowed.txt"
echo "§7 allowlist (derived): $(wc -l < "${RUNNER_TEMP}/allowed.txt") admitted pair(s)"
# Every entry must complete, and the census is what says why when one does not: an abort
# is either an unbarriered pull (§3.3's unconditional Fatal, a missed migration) or a
# barriered pull nothing admits, and both come out of the census by field and verb.
if [ "${rc}" -ne 0 ]; then
echo "::error::integration-split under MOBILEGL_IPC_STRICT_ERRORS=1 is RED on a run-ahead server. CONTRACT-P5E §7 makes this lane a hard green gate; the census below names the field and the verb of every entry that aborted."
python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \
"${RUNNER_TEMP}/allowed.txt" "${expected}" || true
exit 1
fi
# The admitted pairs come from gen_pipe_field_ownership.py --print-admitted, the table
# MGPipeBarrierPullAdmitted() reads, so this list and the C++ cannot drift. A pair is
# admitted when its verb has a stamp row, the field is BARRIER_PULLED and in the verb's
# may-read mask, and either every wire op stamping the verb waits or the field's retiring
# phase is not the generator's THIS_PHASE. The census fails on an admitted marker outside
# that list and on any difference from strict-expected-markers.txt in either direction;
# `no-fatal` adds any Fatal marker. Escalated markers depend on the record's payload (an
# open XFB span, client vertex arrays), so they are checked only against the expected set.
python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \
"${RUNNER_TEMP}/allowed.txt" "${expected}" no-fatal
# Arming proof: DirectGLES.Split.StrictArming. runs with MOBILEGL_PIPE_STATS=1 and a
# private log and asserts, over one drawing frame, that the server stamped a verb
# boundary (vbs > 0) and that no residual pull happened (rsp == 0). Only that entry
# enables PipeStats, so the counters are checked there, not grepped from these logs; rsp
# counts only barriered pulls, as an unbarriered one aborts before CountBarrierPull.
# This step only requires that the entry ran: a skipped positive control proves nothing.
if ! grep -q 'DirectGLES\.Split\.StrictArming\.' "${RUNNER_TEMP}/strict-split.txt"; then
echo "::error::the strict lane did not run DirectGLES.Split.StrictArming., so nothing in this run proves the server ever stamped a verb boundary. A green lane without it is indistinguishable from a lane where strict was never armed (ID-115)."
exit 1
fi
echo "integration-split under strict is GREEN, two-sided, with the arming control present"
# ---- P5f (f1), P5F-WIRE-COMPLETENESS.md §4: THE DUAL-BLOCK CENSUS LANE ---------------
#
# The same integration-split entries (plus Magma's own lane), re-run under
# These entries own assertions removed from generic transport registrations.
# Discovery, execution and zero-skip accounting must agree on the exact set.
- name: Specialized configuration controls
working-directory: ${{ env.SPLIT_BUILD_DIR }}
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
run: |
helper="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py"
ctest --show-only=json-v1 > "${RUNNER_TEMP}/specialized-all.json"
python3 "${helper}" select-specialized "${RUNNER_TEMP}/specialized-all.json" \
"${RUNNER_TEMP}/specialized-selected.json" "${RUNNER_TEMP}/specialized-tests.txt"
ctest --tests-from-file "${RUNNER_TEMP}/specialized-tests.txt" --no-tests=error -j 4 \
--output-on-failure --output-junit "${RUNNER_TEMP}/specialized.xml"
python3 "${helper}" specialized-results "${RUNNER_TEMP}/specialized-all.json" \
"${RUNNER_TEMP}/specialized.xml"
# MOBILEGL_IPC_ROLE_SPLIT_STATE=1: the client's residual fill writes a client-role block
# and the backend reads a server-role one, so every BARRIER-PULLED read aborts with a
# named Fatal{UnmigratedPipeInput, "<field>@<verb>"}. THE RED IS THE CENSUS - it is the
# machine-checked list of what still depends on the two roles sharing one object, and it
# is ratcheted two-sided against Harness/dualblock-expected-fatals.txt: a new pair is a
# debt somebody now owes, a vanished pair must be removed in the commit that retired it.
# With the census empty, this lane is a hard green gate.
# On the disaggregation branches the first run is also the strict split lane: the knob is
# left at its default, which every passed entry's own log must show is the dual block, so
# the environment is the strict lane's exactly. Its checks are made on that run (the
# integration-split part of the JUnit becomes strict-split.xml); a default that changed
# reruns the census with the knob set, as on dev.
- name: Dual-block census lane
id: dualblock
working-directory: ${{ env.SPLIT_BUILD_DIR }}
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_TRANSPORT: "inproc"
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
run: |
ulimit -c unlimited
helper="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py"
expected="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/dualblock-expected-fatals.txt"
strict_expected="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/strict-expected-markers.txt"
ctest -L 'integration-split|integration-magma-split' --show-only=json-v1 \
> "${RUNNER_TEMP}/dualblock-lane.json"
# Stale logs from an earlier step would be read as this run's evidence (ID-119's rule).
reset_logs() {
python3 - "${RUNNER_TEMP}/dualblock-lane.json" "${helper}" <<'PY'
import importlib.util, json, os, sys
spec = importlib.util.spec_from_file_location("slp", sys.argv[2])
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
logs = slp.marker_log_paths(json.load(open(sys.argv[1])))
for path in set(logs.values()):
try:
os.unlink(path)
except OSError:
pass
print(f"reset {len(set(logs.values()))} private log(s) across {len(logs)} entries")
PY
}
census_run() {
rc=0
ctest -L 'integration-split|integration-magma-split' --no-tests=error -j 4 \
--output-junit "${RUNNER_TEMP}/dualblock.xml" \
> "${RUNNER_TEMP}/dualblock-split.txt" 2>&1 || rc=$?
tail -5 "${RUNNER_TEMP}/dualblock-split.txt"
echo "dual-block census run: rc=${rc}"
}
reset_logs
if [ "${MOBILEGL_CI_DISAGGREGATED}" != ON ]; then
census_run
else
# Run A: the strict split lane's environment, with the role-split state at its default.
saved_role="${MOBILEGL_IPC_ROLE_SPLIT_STATE}"
unset MOBILEGL_IPC_ROLE_SPLIT_STATE
census_run
export MOBILEGL_IPC_ROLE_SPLIT_STATE="${saved_role}"
# The strict lane's JUnit is the integration-split part of this run, every name once.
ctest -L integration-split --show-only=json-v1 > "${RUNNER_TEMP}/lane.json"
selected=0
python3 "${GITHUB_WORKSPACE}/scripts/ci/junit_tally.py" "${RUNNER_TEMP}/dualblock.xml" \
--select-from "${RUNNER_TEMP}/lane.json" --write-selected "${RUNNER_TEMP}/strict-split.xml" || selected=$?
python3 "${GITHUB_WORKSPACE}/scripts/gen_pipe_field_ownership.py" --print-admitted \
> "${RUNNER_TEMP}/allowed.txt"
echo "allowlist (derived): $(wc -l < "${RUNNER_TEMP}/allowed.txt") admitted pair(s)"
if [ "${rc}" -ne 0 ]; then
echo "::error::the strict run of integration-split and integration-magma-split under MOBILEGL_IPC_STRICT_ERRORS=1 is RED on a run-ahead server. It is a hard green gate; the census below names the field and the verb of every integration-split entry that aborted, and dualblock-split.txt holds both labels' output."
python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \
"${RUNNER_TEMP}/allowed.txt" "${strict_expected}" || true
exit 1
fi
if [ "${selected}" -ne 0 ]; then
echo "::error::the dual-block run's JUnit does not hold every integration-split entry exactly once, so it cannot stand for the strict split lane"
exit 1
fi
# The strict lane's two-sided marker census and its arming proof, as in its own step.
python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \
"${RUNNER_TEMP}/allowed.txt" "${strict_expected}" no-fatal
if ! grep -q 'DirectGLES\.Split\.StrictArming\.' "${RUNNER_TEMP}/dualblock-split.txt"; then
echo "::error::the strict run did not run DirectGLES.Split.StrictArming., so nothing in this run proves the server ever stamped a verb boundary. A green lane without it is indistinguishable from a lane where strict was never armed."
exit 1
fi
echo "integration-split under strict is GREEN, two-sided, with the arming control present"
# Run A is the census run when every passed entry resolved the dual block (and strict
# and run-ahead) on its own; a default that changed reruns with the knob set.
ipc=0
python3 "${helper}" ipc-config "${RUNNER_TEMP}/dualblock-lane.json" "${RUNNER_TEMP}/dualblock.xml" \
"${RUNNER_TEMP}/dualblock-ipc.json" strict=1 role-split-state=1 run-ahead=1 \
--assumed role-split-state,run-ahead || ipc=$?
if [ "${ipc}" -eq 0 ]; then
echo "defaults-on=true" >> "${GITHUB_OUTPUT}"
elif [ "${ipc}" -eq 3 ]; then
echo "::warning::the role-split state or run-ahead default is not on (dualblock-ipc.json names the entries), so the census reruns with MOBILEGL_IPC_ROLE_SPLIT_STATE=1"
echo "defaults-on=false" >> "${GITHUB_OUTPUT}"
rm -f "${RUNNER_TEMP}/dualblock.xml"
reset_logs
census_run
else
exit "${ipc}"
fi
fi
rows=$(awk '!/^[[:space:]]*(#|$)/ { n++ } END { print n+0 }' "${expected}")
if [ "${rows}" -eq 0 ] && [ "${rc}" -ne 0 ]; then
echo "::error::empty dual-block census requires ctest success, including infrastructure/execution status"
exit "${rc}"
fi
python3 "${helper}" expect-fatal "${RUNNER_TEMP}/dualblock-lane.json" \
"${RUNNER_TEMP}/dualblock.xml" "${expected}"
# The per-frame zero-residual-read case must execute and pass on both backends.
- name: P5f per-frame zero-residual check
working-directory: ${{ env.SPLIT_BUILD_DIR }}
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
run: |
ctest -L integration-p5f-rsp --show-only=json-v1 > "${RUNNER_TEMP}/rsp-lane.json"
ctest -L integration-p5f-rsp --no-tests=error --output-on-failure \
--output-junit "${RUNNER_TEMP}/rsp.xml"
python3 "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
require-green "${RUNNER_TEMP}/rsp-lane.json" "${RUNNER_TEMP}/rsp.xml" \
DirectGLES.Split.P5fRsp.F1WireScenario.EachWireFrameHasZeroResidualPulls \
DirectVulkan.Split.P5fRsp.F1WireScenario.EachWireFrameHasZeroResidualPulls
# The mechanism's own negative control (P5F §6): the same named case must be green with
# the knob on and red - on its own assertion text - with it off.
- name: Dual-block negative control (P5f f1)
working-directory: ${{ env.SPLIT_BUILD_DIR }}
run: bash "${GITHUB_WORKSPACE}/scripts/ci/dualblock_negative_control.sh"
# ---- MAGMA'S OWN WIRE LANE: FRAMEBUFFER, TEXTURE AND APPLICATION BUFFERS ------------
#
# P5f fm: Magma's framebuffer, copy and mipmap verbs consume server records.
# The label remains separate so both backends' coverage stays explicit. The dual-block
# census run above covers the label in this same environment; with its defaults proven on
# the disaggregation branches this step would repeat it, so it runs only otherwise.
- name: Magma split lane (strict, dual-block)
if: env.MOBILEGL_CI_DISAGGREGATED != 'ON' || steps.dualblock.outputs.defaults-on != 'true'
working-directory: ${{ env.SPLIT_BUILD_DIR }}
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
MOBILEGL_TRANSPORT: "inproc"
run: |
ctest -L integration-magma-split --no-tests=error -j 2 --output-on-failure
# The Magma scenarios of the step above over the spawn and TCP transports, so a failure here
# that the inproc step did not show is a two-process fact, not a Magma one. Entries that can
# run only inproc (such as those that peek this process's server loop) are listed in
# spawn_lane_parity.py's MAGMA_INPROC_ONLY. The informational tier,
# integration-magma-all-{split,spawn,tcp}, is not gated: it may go red while CI stays green,
# and a case moves into a gating step once its blocker is fixed.
- name: Magma spawn and TCP lanes
working-directory: ${{ env.SPLIT_BUILD_DIR }}
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
run: |
ulimit -c unlimited
ctest -L '^integration-magma-spawn$' --no-tests=error -j 2 --output-on-failure
ctest -L '^integration-magma-tcp$' --no-tests=error -j 2 --output-on-failure
# Every named Magma application-buffer wire case must execute and pass.
- name: Magma buffer wire cases (inproc)
working-directory: ${{ env.SPLIT_BUILD_DIR }}
run: |
ctest -L integration-magma-buffers --show-only=json-v1 > "${RUNNER_TEMP}/magma-buffer-lane.json"
ctest -L integration-magma-buffers --no-tests=error -j 2 --output-on-failure \
--output-junit "${RUNNER_TEMP}/magma-buffers.xml"
python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
"${RUNNER_TEMP}/magma-buffer-lane.json" "${RUNNER_TEMP}/magma-buffers.xml" <<'PY'
import importlib.util, json, sys
spec = importlib.util.spec_from_file_location("slp", sys.argv[1])
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
cases = [
"F1WireScenario.EnabledVertexBufferDrawKeepsNamedP7Fatal",
"F1WireScenario.UniformBufferRangeAndRebindPixels",
"F1WireScenario.ShortUniformBufferRangePadsMissingBytes",
"F1WireScenario.SubWordUniformBufferRangePadsMissingBytes",
"F1WireScenario.UnalignedAtomicCounterRangeDeclinesAndTheSessionLives",
"F1WireScenario.ComputeWrittenVertexAndIndexBuffersDrawAndReadBack",
"F1WireScenario.ColorBlitToDefaultIgnoresViewportAndPreservesOrientation",
"F1WireScenario.ColorBlitToDefaultHonorsScissorAndReversedRect",
"IndexedDrawFamilyScenario.AnElementBufferDrawElementsOffsetSelectsTheRightQuad",
"IndexedDrawFamilyScenario.DrawElementsBaseVertexMovesTheSameIndicesToTheOtherQuad",
"IndexedDrawFamilyScenario.DrawRangeElementsDrawsTheRangedRun",
"IndexedDrawFamilyScenario.DrawElementsInstancedBaseVertexPaintsOneQuadPerInstance",
"IndexedDrawFamilyScenario.MultiDrawElementsBaseVertexPaintsEverySubDraw",
"IndexedDrawFamilyScenario.MultiDrawElementsIndirectDrawsFromTheIndirectBuffer",
"CrossFrameBufferScenario.VertexBufferSubData",
"CrossFrameBufferScenario.IndexBufferSubData",
"CrossFrameBufferScenario.VertexOrphanAndReupload",
"CrossFrameBufferScenario.IndexOrphanAndReupload",
"PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw",
"PersistentCoherentMapScenario.AWriteAfterAFrameBoundaryReachesTheNextFramesDraw",
"StorageBufferRegrowScenario.AGrownStoreIsVisibleThroughItsExistingIndexedBinding",
# P11 A2: the >= 16 MiB arena and its tier assertion (emulated).
"LargeArenaAdoptionScenario.SubDataAfterAnInFlightDrawReachesTheNextDraw",
"LargeArenaAdoptionScenario.RespecifiedVertexArenaKeepsVaoBindings",
"LargeArenaAdoptionScenario.RespecifiedIndexArenaKeepsVaoBinding",
"LargeArenaAdoptionScenario.ReadbackSeesTheLatestCpuWrite",
"LargeArenaAdoptionScenario.GpuWriteIntoTheArenaIsReadBack",
"LargeArenaAdoptionScenario.TheArenaLandsInTheTierItsLaneDeclares",
]
slp.require_green(json.load(open(sys.argv[2])), sys.argv[3],
["DirectVulkan.Split.Buffers." + name for name in cases])
PY
# P7 package L: THE SAME TWENTY-ONE, BY NAME, IN TWO PROCESSES (nineteen at package L; the
# sub-word uniform range and the unaligned atomic-counter decline joined later).
#
# `--no-tests=error` in the lane step above answers "did something run and pass"; it does
# not answer "did these twenty-one run". require_green does, and it is the check that would
# catch the failure mode this package could plausibly introduce - an arm that registers
# SOME of the lane and goes green on a smaller set. The label spelling is what keeps the
# three arms apart under `ctest -L`'s regex: the inproc arm keeps the historical
# `integration-magma-buffers` (which is what the step above require_greens), and the new
# arms put the arm word FIRST, so neither selection can reach the other.
#
# TcpServer.Start / TcpServer.Stop ARE IN THE tcp ARM'S REQUIRED LIST ON PURPOSE. Every
# tcp entry declares FIXTURES_REQUIRED mobilegl-tcp, so ctest lists the supervisor's two
# fixture entries under the label as well - and they must be green too: a lane whose
# supervisor failed to start has nothing to say about the twenty-one.
- name: Magma buffer wire cases (spawn, TCP)
working-directory: ${{ env.SPLIT_BUILD_DIR }}
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
run: |
for arm in spawn tcp; do
case "${arm}" in
spawn) prefix=DirectVulkan.Spawn.Buffers. ;;
tcp) prefix=DirectVulkan.Tcp.Buffers. ;;
esac
label="integration-magma-${arm}-buffers"
ctest -L "^${label}$" --show-only=json-v1 > "${RUNNER_TEMP}/magma-buffer-${arm}.json"
ctest -L "^${label}$" --no-tests=error -j 2 --output-on-failure \
--output-junit "${RUNNER_TEMP}/magma-buffers-${arm}.xml"
python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
"${RUNNER_TEMP}/magma-buffer-${arm}.json" "${RUNNER_TEMP}/magma-buffers-${arm}.xml" \
"${prefix}" "${arm}" <<'PY'
import importlib.util, json, sys
spec = importlib.util.spec_from_file_location("slp", sys.argv[1])
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
cases = [
"F1WireScenario.EnabledVertexBufferDrawKeepsNamedP7Fatal",
"F1WireScenario.UniformBufferRangeAndRebindPixels",
"F1WireScenario.ShortUniformBufferRangePadsMissingBytes",
"F1WireScenario.SubWordUniformBufferRangePadsMissingBytes",
"F1WireScenario.UnalignedAtomicCounterRangeDeclinesAndTheSessionLives",
"F1WireScenario.ComputeWrittenVertexAndIndexBuffersDrawAndReadBack",
"F1WireScenario.ColorBlitToDefaultIgnoresViewportAndPreservesOrientation",
"F1WireScenario.ColorBlitToDefaultHonorsScissorAndReversedRect",
"IndexedDrawFamilyScenario.AnElementBufferDrawElementsOffsetSelectsTheRightQuad",
"IndexedDrawFamilyScenario.DrawElementsBaseVertexMovesTheSameIndicesToTheOtherQuad",
"IndexedDrawFamilyScenario.DrawRangeElementsDrawsTheRangedRun",
"IndexedDrawFamilyScenario.DrawElementsInstancedBaseVertexPaintsOneQuadPerInstance",
"IndexedDrawFamilyScenario.MultiDrawElementsBaseVertexPaintsEverySubDraw",
"IndexedDrawFamilyScenario.MultiDrawElementsIndirectDrawsFromTheIndirectBuffer",
"CrossFrameBufferScenario.VertexBufferSubData",
"CrossFrameBufferScenario.IndexBufferSubData",
"CrossFrameBufferScenario.VertexOrphanAndReupload",
"CrossFrameBufferScenario.IndexOrphanAndReupload",
"PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw",
"PersistentCoherentMapScenario.AWriteAfterAFrameBoundaryReachesTheNextFramesDraw",
"StorageBufferRegrowScenario.AGrownStoreIsVisibleThroughItsExistingIndexedBinding",
"LargeArenaAdoptionScenario.SubDataAfterAnInFlightDrawReachesTheNextDraw",
"LargeArenaAdoptionScenario.RespecifiedVertexArenaKeepsVaoBindings",
"LargeArenaAdoptionScenario.RespecifiedIndexArenaKeepsVaoBinding",
"LargeArenaAdoptionScenario.ReadbackSeesTheLatestCpuWrite",
"LargeArenaAdoptionScenario.GpuWriteIntoTheArenaIsReadBack",
"LargeArenaAdoptionScenario.TheArenaLandsInTheTierItsLaneDeclares",
]
required = [sys.argv[4] + name for name in cases]
if sys.argv[5] == "tcp":
required += ["TcpServer.Start", "TcpServer.Stop"]
slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], required)
PY
done
# Run-ahead must queue real GPU work and enforce credits 1 and 3.
- name: Magma run-ahead cases (credits 1, 3)
working-directory: ${{ env.SPLIT_BUILD_DIR }}
run: |
ctest -L integration-magma-runahead --show-only=json-v1 > "${RUNNER_TEMP}/magma-runahead-lane.json"
ctest -L integration-magma-runahead --no-tests=error -j 2 --output-on-failure \
--output-junit "${RUNNER_TEMP}/magma-runahead.xml"
python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
"${RUNNER_TEMP}/magma-runahead-lane.json" "${RUNNER_TEMP}/magma-runahead.xml" <<'PY'
import importlib.util, json, sys
spec = importlib.util.spec_from_file_location("slp", sys.argv[1])
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
cases = [
"QueuedBufferVersionsAndDeletedNameReuseKeepTheirPixels",
"QueuedProgramRebindsKeepEachUniformSnapshot",
"QueuedGpuWritesReadBackTheLastDispatchBytes",
"MixedNativeAndConvertedGpuVertexStreamsKeepTheirReservation",
"TextureUploadCannotOvertakeThePreviouslyQueuedDraw",
"GpuClearSurvivesTheFirstStorageImageUpgrade",
"PresentCreditActuallyParksTheClientAtItsConfiguredLimit",
]
required = [f"DirectVulkan.Split.RunAhead.Credit{credit}.MagmaRunAheadScenario.{name}"
for credit in (1, 3) for name in cases]
slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], required)
PY
# Run-ahead off must go red on the lost queue lead; run-ahead back on must be green.
- name: Magma run-ahead negative control
run: |
python3 scripts/ci/magma_runahead_checks.py negative --build-dir "${SPLIT_BUILD_DIR}" \
--out "${RUNNER_TEMP}/magma-runahead-negative" --execute
- name: Magma run-ahead sync validation
run: |
python3 scripts/ci/magma_runahead_checks.py validation --build-dir "${SPLIT_BUILD_DIR}" \
--out "${RUNNER_TEMP}/magma-runahead-syncval" --execute
- name: Magma view/framebuffer cache sync validation
run: |
python3 scripts/ci/magma_cache_checks.py --build-dir "${SPLIT_BUILD_DIR}" \
--out "${RUNNER_TEMP}/magma-cache-syncval" --execute
- name: Upload Magma run-ahead and cache evidence
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: magma-runahead-checks
path: |
${{ runner.temp }}/magma-runahead-negative
${{ runner.temp }}/magma-runahead-syncval
${{ runner.temp }}/magma-cache-syncval
if-no-files-found: ignore
# The packed-float chain is generated by a real GPU shader blit, so this entry sits in the
# integration-magma-split lane with its siblings (CMakeLists.txt's fm loop) and carries
# their pixels assertion. Its named check survives the flip in the only direction that has
# evidence behind it: the case must EXECUTE, PASS, and never skip - a lane-level exit code
# alone would not separate "generated" from "quietly absent".
- name: Magma packed-float mipmap case (P7)
working-directory: ${{ env.SPLIT_BUILD_DIR }}
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
MOBILEGL_TRANSPORT: "inproc"
run: |
p7_case='^DirectVulkan\.Split\.Fm\.F1WireScenario\.GenerateMipmapPackedFloatPixels$'
ctest -R "${p7_case}" --show-only=json-v1 > "${RUNNER_TEMP}/magma-p7-lane.json"
ctest -R "${p7_case}" --no-tests=error --output-on-failure \
--output-junit "${RUNNER_TEMP}/magma-p7.xml"
python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
"${RUNNER_TEMP}/magma-p7-lane.json" "${RUNNER_TEMP}/magma-p7.xml" <<'PY'
import importlib.util, json, sys
spec = importlib.util.spec_from_file_location("slp", sys.argv[1])
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
name = "DirectVulkan.Split.Fm.F1WireScenario.GenerateMipmapPackedFloatPixels"
slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], [name])
PY
# Client vertex/index inputs are copied into owned wire buffer resources.
# The original dedicated entries now have to execute successfully with run-ahead.
- name: Client-array split lane (run-ahead)
working-directory: ${{ env.SPLIT_BUILD_DIR }}
env:
MOBILEGL_ITEST_REQUIRE_GPU: "1"
MOBILEGL_TRANSPORT: "inproc"
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
MOBILEGL_IPC_RUN_AHEAD: "1"
run: |
ctest -L integration-clientarrays-split --show-only=json-v1 > "${RUNNER_TEMP}/clientarrays-lane.json"
ctest -L integration-clientarrays-split --no-tests=error --output-on-failure -j 1 \
--output-junit "${RUNNER_TEMP}/clientarrays.xml"
python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
"${RUNNER_TEMP}/clientarrays-lane.json" "${RUNNER_TEMP}/clientarrays.xml" <<'PY'
import importlib.util, json, sys
spec = importlib.util.spec_from_file_location("slp", sys.argv[1])
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
lane = json.load(open(sys.argv[2]))
names = [test["name"] for test in lane["tests"]]
if not names:
raise SystemExit("no client-array cases discovered")
slp.require_green(lane, sys.argv[3], names)
PY
# The run log needs a signed-in reader; annotations do not. Name every failed ctest entry,
# with the lines of its output that say why, where anyone reading the run can see them.
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}"
# P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py).
- name: Upload JUnit for the skip census
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: junit-integration-split-strict
path: ${{ runner.temp }}/*.xml
if-no-files-found: warn
- name: Upload strict lane logs
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: integration-split-strict-logs
path: |
${{ env.SPLIT_BUILD_DIR }}/MobileGL/MG_IntegrationTest/*.log*
${{ env.SPLIT_BUILD_DIR }}/MobileGL/MG_IntegrationTest/split-logs/*.log
${{ runner.temp }}/specialized*.json
${{ runner.temp }}/specialized-tests.txt
${{ runner.temp }}/specialized.xml
${{ runner.temp }}/dualblock-lane.json
${{ runner.temp }}/dualblock.xml
${{ runner.temp }}/dualblock-split.txt
if-no-files-found: warn
# MobileGL/MG_Remote/Protocol/generated/protocol_generated.h is COMMITTED, and
# flatc is deliberately absent from the default build graph (a codegen step in
# the graph is how the earlier branch ended up cross-compiling an arm64 flatc
# and trying to run it on the host). This job is what keeps the committed
# header honest: build the pinned flatc, regenerate, and fail on any diff.
# It needs no MobileGL build, so it does not depend on build-linux.
flatc-check:
runs-on: ubuntu-26.04
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Check out the FlatBuffers submodule only
# Just this one: the schema check has nothing to do with glslang,
# SPIRV-Cross or the trace fixtures.
run: git submodule update --init 3rdparty/flatbuffers
# The pinned flatc, cached under the submodule's commit: gen_protocol.py builds it only
# when the build directory holds none, and checks its version against the submodule
# either way. The gitlink gives the commit without the submodule's own checkout.
# integration-split restores the same entry, so the path is the same string in both jobs.
- name: Pinned flatc cache key
id: flatc
run: echo "commit=$(git rev-parse HEAD:3rdparty/flatbuffers)" >> "${GITHUB_OUTPUT}"
- name: Cache pinned flatc
uses: actions/cache@v5
with:
path: ${{ runner.temp }}/flatc-build/flatc
key: ${{ runner.os }}-ubuntu26-flatc-${{ steps.flatc.outputs.commit }}
- name: Regenerate protocol_generated.h
run: python3 scripts/gen_protocol.py --build-dir "${{ runner.temp }}/flatc-build"
- name: Generated header staleness check
run: git diff --exit-code -- MobileGL/MG_Remote/Protocol/generated/protocol_generated.h
# P7 wave 2-F, F fix round (PH-7 (4) review). MOBILEGL_PROTOCOL_CONTROL_REVISION is mixed
# into wireFingerprint so a control-schema change refuses an old peer by name - but it is a
# hand-bumped integer, and nothing tied it to protocol.fbs's content. This step pins
# sha256(protocol.fbs) to the revision (scripts/ci/protocol_revision_pins.json): a schema
# edit that forgot the bump is red HERE, in the job that regenerates the header on every
# such edit, and not a silently absent field between an old server and a new client. Its
# self-test proves the check goes red on both perturbations before it says green.
- name: Protocol revision pin check
run: python3 scripts/ci/protocol_revision_pin.py --self-test
# P0.5 interface-purity gate A (ARCHITECTURE.md §13.2): the two extracted headers' include closure,
# asserted on `-H` output because `nm --undefined-only` is blind to "included but not called" -
# a header whose types are never named leaves no symbol behind, and "included at all" is exactly
# the coupling P1 and P7 have to sever. Needs a preprocessor and three header submodules, no
# CMake configure and no glslang sources, so like pipe-gates it does not depend on build-linux.
# The script's own --self-test is always on: a negative control that stopped tripping fails the
# job, because a gate that cannot go red is not a gate (ARCHITECTURE.md §13.2).
include-graph-check:
name: Include-closure purity gate
runs-on: ubuntu-26.04
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Check out header submodules
# ska/flat_hash_map.hpp, xxhash.h and vulkan/vulkan.h are the only submodule headers
# Includes.h reaches; glslang and spirv_cross are vendored under include/.
run: git submodule update --init include/ska 3rdparty/xxHash 3rdparty/Vulkan-Headers
- name: Install clang and X11 headers
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
# Includes.h defines VK_USE_PLATFORM_XLIB_KHR before <vulkan/vulkan.h>, which then
# includes <X11/Xlib.h>; without libx11-dev every clang-mode probe dies in the
# preprocessor and the gate reports 5 problems that have nothing to do with purity.
run: sudo apt-get update && sudo apt-get install -y clang-20 libx11-dev
- name: Include-closure check and self-test
# --expect-probes 5 (contract-v2.md 7.6; P13 W5 adds record-core): an exit code cannot tell five probes from none,
# so a --probe typo or a manifest edit that selected nothing would run zero probes and
# exit 0 - the second half of the finding that added the flag. The count is the length of
# scripts/check_include_closure.py's PROBES list and changing one means changing the other.
#
# Use the compiler selected for this runner's installed toolchain.
run: python3 scripts/check_include_closure.py --mode both --compiler "${MOBILEGL_CI_CXX}" --self-test --require-all --expect-probes 5
benchmark:
runs-on: ubuntu-26.04
needs: [build-linux]
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
with:
gpu: "true"
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Install runtime dependencies
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
# libegl-mesa0 is the EGL vendor library itself: DriverBench brings up a
# real GL context, and libegl1 is only glvnd's dispatch. It normally
# arrives as a Recommends of libegl1, which is too quiet a dependency for
# the one job that needs a working driver.
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers
- name: Download Linux runtime
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-runtime
path: .
- name: Unpack Linux runtime
run: tar --zstd -xf mobilegl-linux-runtime.tar.zst
- name: Prepare downloaded test metadata
run: |
for root in build-linux build-verify build-split build-verify-split build-retrace; do
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
- name: Benchmark
working-directory: build-linux
run: |
ulimit -c unlimited
ctest -V -C Release -L benchmark --no-tests=error
- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v7
with:
name: benchmark-core-dumps
path: /tmp/core.*
if-no-files-found: ignore
build-retrace:
runs-on: ubuntu-26.04
permissions:
actions: write
contents: read
env:
BUILD_DIR: build-retrace
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_COMPRESS: "true"
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 4G
CCACHE_NOHASHDIR: "true"
MOBILEGL_LIBRARY: ${{ github.workspace }}/build-linux/libMobileGL.so
steps:
- name: Checkout repo
uses: actions/checkout@v6
# The library set plus apitrace and the directories its build requires
# (scripts/ci/checkout_submodules.sh).
- name: Check out build submodules
timeout-minutes: 15
run: bash scripts/ci/checkout_submodules.sh retrace
- name: Set swap space
uses: ./.github/actions/set-swap-space
with:
swap-size-gb: 32
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Restore ccache
uses: actions/cache/restore@v5
with:
path: .ccache
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
restore-keys: |
${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-
- name: Prepare Vulkan SDK
id: netsetup2500
continue-on-error: true
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Prepare Vulkan SDK (retry)
timeout-minutes: 15
if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup2500.outcome == 'failure' }}
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Update glslang external sources
working-directory: 3rdparty/glslang
run: python3 update_glslang_sources.py
- name: Install dependencies
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
run: |
sudo apt-get update
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
- name: Show installed toolchain
run: |
ccache --version
"${MOBILEGL_CI_CC}" --version
"${MOBILEGL_CI_CXX}" --version
"${MOBILEGL_CI_LLD}" --version
dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true
- name: Prepare downloaded test metadata
run: |
for root in build-linux build-verify build-split build-verify-split build-retrace; do
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
- name: Configure CMake
run: |
if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
BUILD_TYPE=Debug
else
BUILD_TYPE=Release
fi
cmake -S . -B "${BUILD_DIR}" -G Ninja \
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \
-DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
-DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
-DMOBILEGL_BUILD_TEST=OFF \
-DMOBILEGL_BUILD_BENCHMARK=OFF \
-DMOBILEGL_BUILD_TRACE_REPLAY=ON \
-DMOBILEGL_TRACE_SOFTWARE_RENDERING="${MOBILEGL_CI_SOFTWARE_RENDERING}" \
-DMOBILEGL_TRACE_REPLAY_MOBILEGL_LIBRARY="${MOBILEGL_LIBRARY}" \
-DCMAKE_POLICY_VERSION_MINIMUM=3.5
- name: Build trace replay
run: cmake --build "${BUILD_DIR}" --target mobilegl_trace_replay --parallel "$(nproc)"
- name: Show ccache stats
if: ${{ always() }}
run: ccache --show-stats
- name: Release superseded ccache entry
env:
GH_TOKEN: ${{ github.token }}
CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true
- name: Save ccache
continue-on-error: true
uses: actions/cache/save@v5
with:
path: .ccache
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
- name: Prepare packaged test metadata
run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}"
- name: Package trace replay
run: |
mkdir -p ci-artifacts
tar -czf ci-artifacts/mobilegl-trace-replay.tgz \
build-retrace/.ci-source-root \
build-retrace/tools/trace_replay/mobilegl_trace_replay \
build-retrace/tools/trace_replay/CTestTestfile.cmake
- name: Upload trace replay
uses: actions/upload-artifact@v7
with:
name: mobilegl-trace-replay
path: ci-artifacts/mobilegl-trace-replay.tgz
if-no-files-found: error
trace-cases:
name: trace case matrix
runs-on: ubuntu-26.04
env:
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }}
# How many jobs replay one (lane, transport, backend) cell of the Linux retrace lanes. Each
# job replays its share of the cell's cases one after another; a cell with little to
# replay gets fewer jobs (tools/trace_replay/trace_cases.py, shard_matrix).
RETRACE_SHARDS_PER_CELL: "4"
outputs:
shards: ${{ steps.trace-cases.outputs.shards }}
verify-shards: ${{ steps.trace-cases.outputs.verify-shards }}
split-shards: ${{ steps.trace-cases.outputs.split-shards }}
split-spawn-shards: ${{ steps.trace-cases.outputs.split-spawn-shards }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
# `dl` / exit gate S7. CHEAP AND EARLY, on a job that only reads the tree: it needs no
# build, so a death that names no family is reported in seconds rather than after the
# matrix. What it refuses is a std::abort() under MG_Remote/ with no Fatal{Family} marker,
# and a family word that appeared without being recorded - the vocabulary drifts on
# purpose or not at all (CONTRACT-P6 5.2's D2 ruling).
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Fatal census (S7)
run: python3 scripts/ci/fatal_census.py
# The decline half of rule I (P7 B3): every WireDeclines.def row has a site, every site a
# warning-or-error line. Same shape as the census - tree-only, no build. The self-test
# runs first: its fixtures (a site only in a comment or a string, a log in another
# branch, an MGLOG_D) must red, so a green audit is one that could have been red.
- name: Wire declines audit (B3)
run: |
python3 scripts/ci/wire_declines_audit.py --self-test
python3 scripts/ci/wire_declines_audit.py
- name: Load trace cases
id: trace-cases
run: |
python3 tools/trace_replay/trace_cases.py --self-test-transport-matrices
# The retrace jobs replay shards: several cases of one (lane, transport, backend) cell
# per job. The first self-test proves every shard matrix replays exactly the rows of
# the per-case matrix it is built from; the second proves the shard runner gives each
# case the settings of its lane.
python3 tools/trace_replay/trace_cases.py --self-test-shard-matrices
python3 scripts/ci/retrace_shard.py --self-test
shards() {
python3 tools/trace_replay/trace_cases.py --ci --shards "${RETRACE_SHARDS_PER_CELL}" "$@"
}
shard_count() {
python3 -c 'import json,sys; print(len(json.load(sys.stdin)["include"]))'
}
shard_format=github-test-shard-matrix
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then shard_format=github-test-transport-shard-matrix; fi
# An empty `include` is not an error to GitHub: it skips the job with no red anywhere,
# so each matrix is checked before it is handed over.
require_shards() {
if [ "$(printf '%s' "$2" | shard_count)" -lt 1 ]; then
echo "::error::the $1 shard matrix is empty, so its job would be skipped with no red"
exit 1
fi
}
SHARDS=$(shards --format "${shard_format}")
require_shards retrace "${SHARDS}"
echo "shards=${SHARDS}" >> "$GITHUB_OUTPUT"
# The subset the verify build retraces ("verify": true in trace_cases.json).
VERIFY_SHARDS=$(shards --format github-verify-shard-matrix)
require_shards retrace-verify "${VERIFY_SHARDS}"
echo "verify-shards=${VERIFY_SHARDS}" >> "$GITHUB_OUTPUT"
# The split subset: {backend, case, transport} over inproc and spawn, both backends, for
# every CI case that has not said `"split": false`. It is the whole CI matrix because
# the bugs the spawn arm finds are about the server's process rather than about one
# trace. trace_cases.py refuses a `split` case that is not in CI.
SPLIT_SHARDS=$(shards --format github-split-shard-matrix)
# It must not be empty. An empty `include` is not an error to GitHub: it skips the
# whole retrace-split job with no red anywhere, so that is guarded here.
# trace_cases.py also rejects an unknown manifest key, since a mistyped `split` key
# would otherwise load clean and change the subset.
if [ "$(printf '%s' "${SPLIT_SHARDS}" | shard_count)" -lt 1 ]; then
echo "::error::the split retrace subset is empty. 'split' is an opt-out: a CI case is in this subset unless it carries \"split\": false, so an empty subset means either every case opted out or the CI matrix itself is empty. Either way retrace-split would be skipped with no red."
exit 1
fi
echo "split-shards=${SPLIT_SHARDS}" >> "$GITHUB_OUTPUT"
# On the disaggregation branches, the same subset without the inproc rows the
# inproc-acceptance retrace rows replay on the same library, which retrace-split uses when
# it reuses build-linux's library. OpenRA keeps its inproc rows: they carry the
# wire-volume evidence. Its self-test proves every dropped row is an acceptance row. It is
# only a saving, so it never fails this job: on a failed self-test or an empty subset it
# is not written, and retrace-split takes the full subset, as it does elsewhere.
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then
if python3 tools/trace_replay/trace_cases.py --self-test-split-matrix \
&& SPLIT_SPAWN_SHARDS=$(shards --format github-split-shard-matrix \
--transports spawn --keep-inproc OpenRA) \
&& [ "$(printf '%s' "${SPLIT_SPAWN_SHARDS}" | shard_count || echo 0)" -ge 1 ]; then
echo "split-spawn-shards=${SPLIT_SPAWN_SHARDS}" >> "$GITHUB_OUTPUT"
else
echo "::warning::the spawn-only split retrace subset failed its self-test or is empty, so retrace-split replays the full split subset"
fi
fi
echo "retrace shards: $(printf '%s' "${SHARDS}" | shard_count) retrace, $(printf '%s' "${VERIFY_SHARDS}" | shard_count) verify, $(printf '%s' "${SPLIT_SHARDS}" | shard_count) split (full subset)"
# One cache entry holds every Git LFS fixture of the CI cases, and each replay job restores
# that entry itself. This job only makes sure the entry exists. On a hit it downloads
# nothing. On a miss it starts from the newest older bundle, fetches from the mirror the
# cases whose pointers changed, saves the entry under the new key and confirms it is there,
# so that no replay job has to download fixtures itself. The key, the path and the steps are
# .github/scripts/trace-fixture-bundle.sh's; apk.yml uses the same entry. The timeout bounds
# a fixture download that stalls, which would otherwise hold every replay job behind it.
trace-fixtures:
name: trace fixtures
runs-on: ubuntu-26.04
timeout-minutes: 30
permissions:
actions: read
contents: read
outputs:
key: ${{ steps.bundle.outputs.key }}
path: ${{ steps.bundle.outputs.path }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Derive the fixture bundle key
id: bundle
run: bash .github/scripts/trace-fixture-bundle.sh key
- name: Look up the fixture bundle
id: lookup
uses: actions/cache/restore@v5
with:
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
lookup-only: true
- name: Restore the newest older bundle
if: steps.lookup.outputs.cache-hit != 'true'
uses: actions/cache/restore@v5
with:
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
restore-keys: ${{ steps.bundle.outputs.restore-prefix }}
- name: Complete the fixture bundle
if: steps.lookup.outputs.cache-hit != 'true'
run: bash .github/scripts/trace-fixture-bundle.sh ensure
# The APK workflow of the same push saves the same entry, and only one save can win.
# The steps below find the entry whichever workflow wrote it.
- name: Save the fixture bundle
if: steps.lookup.outputs.cache-hit != 'true'
continue-on-error: true
uses: actions/cache/save@v5
with:
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
# The APK workflow of the same push may be the one whose save won. Its entry is as good,
# once it is listed.
- name: Wait for the fixture bundle entry
if: steps.lookup.outputs.cache-hit != 'true'
env:
GH_TOKEN: ${{ github.token }}
run: bash .github/scripts/trace-fixture-bundle.sh wait '${{ steps.bundle.outputs.key }}'
- name: Confirm the fixture bundle entry
if: steps.lookup.outputs.cache-hit != 'true'
uses: actions/cache/restore@v5
with:
path: ${{ steps.bundle.outputs.path }}
key: ${{ steps.bundle.outputs.key }}
lookup-only: true
fail-on-cache-miss: true
# One job replays a shard: some of the cases of one lane, transport and backend, one after
# another, each through its own ctest entry (scripts/ci/retrace_shard.py). The shard's step
# summary has a row per case, and a failed case has an error annotation of its own.
retrace:
name: retrace (${{ matrix.backend }} / ${{ matrix.transport || 'monolith' }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
runs-on: ubuntu-26.04
timeout-minutes: 240
env:
MOBILEGL_TRANSPORT: ${{ matrix.transport || 'monolith' }}
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
MOBILEGL_IPC_STRICT_ERRORS: "1"
MOBILEGL_IPC_RUN_AHEAD: "1"
SHARD_NAMES: ${{ toJSON(matrix.names) }}
needs:
- build-linux
- build-retrace
- trace-cases
- trace-fixtures
if: ${{ !cancelled() && needs.build-linux.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }}
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.trace-cases.outputs.shards) }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Set swap space
if: runner.environment != 'self-hosted'
uses: ./.github/actions/set-swap-space
with:
swap-size-gb: 16
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
with:
gpu: "true"
# A miss does not fail the job: the entry may have been evicted since trace-fixtures
# confirmed it (a re-run of failed jobs on an older run does not run that job again).
- name: Restore trace fixtures
uses: actions/cache/restore@v5
with:
path: ${{ needs.trace-fixtures.outputs.path }}
key: ${{ needs.trace-fixtures.outputs.key }}
# Moves this shard's fixtures into place and checks them against their Git LFS pointers.
# A case the restored bundle does not hold is downloaded here, with a warning annotation;
# the timeout bounds a download that stalls.
- name: Install trace fixtures
timeout-minutes: 15
run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Install runtime dependencies
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers
test -e /usr/lib/x86_64-linux-gnu/libEGL.so
test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so
- name: Download Linux library
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-lib
path: .
- name: Download trace replay
uses: actions/download-artifact@v8
with:
name: mobilegl-trace-replay
path: .
- name: Unpack retrace runtime
run: |
tar -xzf mobilegl-linux-lib.tgz
tar -xzf mobilegl-trace-replay.tgz
test -f build-linux/libMobileGL.so
test -f build-retrace/tools/trace_replay/mobilegl_trace_replay
- name: Prepare downloaded test metadata
run: |
for root in build-linux build-verify build-split build-verify-split build-retrace; do
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
# The per-case settings are retrace_shard.py's "retrace" profile: the Magma fallback and
# the two per-case repair knobs on DirectVulkan, and on the inproc lane the IPC
# configuration the client must log, the split evidence check and the refusal census from
# the library's own logs. A failing case does not stop the shard; the step fails at the
# end if a case failed or a case has no verdict.
- name: Retrace and validate
run: |
python3 scripts/ci/retrace_shard.py run --profile retrace \
--backend '${{ matrix.backend }}' --transport "${MOBILEGL_TRANSPORT}" \
--lane '${{ matrix.lane || 'monolith-control' }}' \
--shard '${{ matrix.shard }}/${{ matrix.shards }}' \
--build-dir build-retrace/tools/trace_replay \
--result-dir "${RUNNER_TEMP}/retrace-shard"
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit"
- name: Upload core dumps
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-core-dumps-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }}
path: /tmp/core.*
if-no-files-found: ignore
# What retrace-summary renders: per case, the images and the replay's output directory.
# The summary embeds the images, so these are short-lived; a failed shard keeps its own
# copy below.
- name: Upload shard results
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }}
path: ${{ runner.temp }}/retrace-shard/results
if-no-files-found: warn
retention-days: 1
overwrite: true
- name: Upload failed shard results
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-failed-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }}
path: |
${{ runner.temp }}/retrace-shard/results
${{ runner.temp }}/retrace-shard/junit
${{ runner.temp }}/retrace-shard/logs
if-no-files-found: warn
overwrite: true
retrace-summary:
name: retrace summary
runs-on: ubuntu-26.04
needs: [retrace]
if: ${{ !cancelled() && needs.retrace.result != 'skipped' }}
# actions: write is for the last step, which deletes the artifacts this job rendered.
permissions:
actions: write
contents: read
env:
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Set artifact metadata
run: |
echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
- name: Set up Node.js
uses: actions/setup-node@v7
with:
node-version: '22'
- name: Download monolith control results
uses: actions/download-artifact@v8
with:
pattern: retrace-result-monolith-control-monolith-*
path: retrace-artifacts/monolith-control
- name: Download inproc acceptance results
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
uses: actions/download-artifact@v8
with:
pattern: retrace-result-inproc-acceptance-inproc-*
path: retrace-artifacts/inproc-acceptance
- name: Render monolith control summary
run: |
node tools/trace_replay/render_retrace_summary.mjs \
--input retrace-artifacts/monolith-control \
--output-dir retrace-summary/monolith-control \
--title "MobileGL Linux monolith control" \
--group-label "Linux / monolith control" \
--html mobilegl-linux-monolith-control.html
- name: Render inproc acceptance summary
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
run: |
node tools/trace_replay/render_retrace_summary.mjs \
--input retrace-artifacts/inproc-acceptance \
--output-dir retrace-summary/inproc-acceptance \
--title "MobileGL Linux inproc acceptance" \
--group-label "Linux / inproc acceptance" \
--html mobilegl-linux-inproc-acceptance.html
- name: Upload monolith control summary
uses: actions/upload-artifact@v7
with:
name: mobilegl-linux-monolith-control-summary-${{ github.sha }}
path: retrace-summary/monolith-control/mobilegl-linux-monolith-control.html
archive: false
if-no-files-found: error
- name: Upload inproc acceptance summary
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
uses: actions/upload-artifact@v7
with:
name: mobilegl-linux-inproc-acceptance-summary-${{ github.sha }}
path: retrace-summary/inproc-acceptance/mobilegl-linux-inproc-acceptance.html
archive: false
if-no-files-found: error
# The summaries embed every image, so the shard result artifacts they were rendered from
# have no reader left and are deleted here. That happens only when every shard passed: a
# failed shard has its own retrace-failed-* artifact, and after a failure the results
# stay for their one day of retention, so that re-running the failed shards renders every
# case again. A failed delete only leaves them to that retention.
- name: Delete the rendered shard results
if: ${{ needs.retrace.result == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
run: |
gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \
--jq '.artifacts[] | select(.name | startswith("retrace-result-")) | [.id, .name] | @tsv' \
> "${RUNNER_TEMP}/rendered-artifacts.txt"
deleted=0
while IFS=$'\t' read -r artifact_id artifact_name; do
echo "Deleting ${artifact_name} (${artifact_id})"
gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}"
deleted=$((deleted + 1))
done < "${RUNNER_TEMP}/rendered-artifacts.txt"
echo "Deleted ${deleted} rendered shard result artifact(s)."
# The trace half of the third CI mode. Same replay, same goldens, but the library underneath is
# the verify build and MOBILEGL_PIPE_VERIFY=1 is in the environment, so every backend read of
# frontend state is checked against a snapshot taken at the verb boundary. Eight cases rather
# than the full lane's 40 (tools/trace_replay/trace_cases.json, "verify": true): the comparator
# is budgeted at 5-10x, and the full sweep is a phase-exit / workflow_dispatch run.
retrace-verify:
name: retrace verify (${{ matrix.backend }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
runs-on: ubuntu-26.04
timeout-minutes: 240
env:
SHARD_NAMES: ${{ toJSON(matrix.names) }}
needs:
- build-linux-verify
- build-retrace
- trace-cases
- trace-fixtures
if: ${{ !cancelled() && needs.build-linux-verify.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }}
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.trace-cases.outputs.verify-shards) }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Set swap space
if: runner.environment != 'self-hosted'
uses: ./.github/actions/set-swap-space
with:
swap-size-gb: 16
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
with:
gpu: "true"
# A miss does not fail the job; the install step downloads what the bundle does not hold.
- name: Restore trace fixtures
uses: actions/cache/restore@v5
with:
path: ${{ needs.trace-fixtures.outputs.path }}
key: ${{ needs.trace-fixtures.outputs.key }}
- name: Install trace fixtures
timeout-minutes: 15
run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Install runtime dependencies
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers
test -e /usr/lib/x86_64-linux-gnu/libEGL.so
test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so
- name: Download Linux verify library
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-lib-verify
path: .
- name: Download trace replay
uses: actions/download-artifact@v8
with:
name: mobilegl-trace-replay
path: .
- name: Unpack verify runtime (library under test)
# build-retrace's CTestTestfile.cmake has the absolute path
# <workspace>/build-linux/libMobileGL.so frozen into every case, so the swap happens here
# rather than through a variable: the verify .so is put where that path points. The nm
# check is what makes the swap falsifiable - a run against the ordinary library would
# carry no comparator, ignore MOBILEGL_PIPE_VERIFY entirely, and match its golden.
#
# `nm`, not `nm -D`, for the reason spelled out in build-linux-verify: everything MGPipe is
# hidden-visibility in a Release build and the dynamic table has none of it.
run: |
tar -xzf mobilegl-linux-lib-verify.tgz
tar -xzf mobilegl-trace-replay.tgz
test -f build-verify/libMobileGL.so
test -f build-retrace/tools/trace_replay/mobilegl_trace_replay
mkdir -p build-linux
cp build-verify/libMobileGL.so build-linux/libMobileGL.so
if ! nm --defined-only build-linux/libMobileGL.so | grep -q MGPipeVerifyInputs; then
echo "::error::the library unpacked at build-linux/libMobileGL.so defines no MGPipeVerifyInputs, so this retrace would replay against a comparator-free build and pass on its golden having verified nothing"
exit 1
fi
echo "the library at build-linux/libMobileGL.so is the verify build"
- name: Prepare downloaded test metadata
run: |
for root in build-linux build-verify build-split build-verify-split build-retrace; do
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
# Every case replays with MOBILEGL_PIPE_VERIFY=1 (retrace_shard.py's "verify" profile).
# run_trace_case.cmake turns that into three assertions of its own (the arming line, no
# Fatal{PipeVerifyDiffer, no Fatal{UnmigratedPipeInput), so a case that somehow ran the
# wrong library reds here instead of passing on its golden. ctest's timeout is raised to
# 10800 s: the long cases run 5-10x slower with both comparator arms live, well past the
# 1500 s default.
#
# The shard that holds OpenRA on DirectGLES also runs the lane's negative control right
# after that case: with a snapshot field corrupted, the same replay must fail, and for the
# comparator's reason (scripts/ci/retrace_verify_reason.py). Without it, zero divergences
# would be a statement about a comparator nobody watched. The control has a verdict row of
# its own, and its output is discarded so that only the verified run is uploaded.
- name: Retrace and validate (verify)
run: |
python3 scripts/ci/retrace_shard.py run --profile verify \
--backend '${{ matrix.backend }}' --lane verify \
--shard '${{ matrix.shard }}/${{ matrix.shards }}' \
--build-dir build-retrace/tools/trace_replay \
--result-dir "${RUNNER_TEMP}/retrace-shard"
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit"
- name: Upload core dumps
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-verify-core-dumps-${{ matrix.backend }}-shard${{ matrix.shard }}
path: /tmp/core.*
if-no-files-found: ignore
# Kept for every shard, green or not: per case, the verified images and the replay's
# output directory, whose library log has the comparator's arming line.
- name: Upload shard results
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: retrace-verify-result-${{ matrix.backend }}-shard${{ matrix.shard }}
path: ${{ runner.temp }}/retrace-shard/results
if-no-files-found: warn
overwrite: true
- name: Upload failed shard results
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-verify-failed-${{ matrix.backend }}-shard${{ matrix.shard }}
path: |
${{ runner.temp }}/retrace-shard/results
${{ runner.temp }}/retrace-shard/junit
${{ runner.temp }}/retrace-shard/logs
if-no-files-found: warn
overwrite: true
# Trace retrace on the split library: the retrace-verify shape with MOBILEGL_TRANSPORT set
# instead of MOBILEGL_PIPE_VERIFY, over {backend, case, transport} with inproc and spawn.
# inproc is the control and spawn the claim: same library and picture, only the server role's
# process differs, so a spawn-only failure is a transport fact. It runs the unchanged ctest
# names with MOBILEGL_TRANSPORT exported around ctest, since build-retrace does not register the
# split variant entries. SSIM cannot catch a monolith run (OpenRA scores 1.000 either way); the
# guards are the nm check for MG_Remote, run_trace_case.cmake's transport assertions (and, on
# OpenRA, its wire-volume assertion), and the per-leg check of the evidence file they leave.
retrace-split:
name: retrace split (${{ matrix.backend }} / ${{ matrix.transport }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
runs-on: ubuntu-26.04
timeout-minutes: 240
# The setup action selects lavapipe on GitHub-hosted runners.
needs:
- build-guard
- build-linux-split
- build-linux
- build-retrace
- trace-cases
- trace-fixtures
if: ${{ !cancelled() && needs.build-linux-split.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' && (needs.build-guard.outputs.verdict != 'identical' || needs.build-linux.result == 'success') }}
env:
SPLIT_REUSE: ${{ needs.build-guard.outputs.verdict == 'identical' && 'true' || 'false' }}
SPLIT_LIB: ${{ needs.build-guard.outputs.verdict == 'identical' && 'mobilegl-linux-lib' || 'mobilegl-linux-lib-split' }}
SPLIT_LIB_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }}
SPLIT_BUILD_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }}
SHARD_NAMES: ${{ toJSON(matrix.names) }}
strategy:
fail-fast: false
# With build-linux reused, the inproc rows (except OpenRA's) are replayed by the retrace job's
# inproc-acceptance rows on the same library, strict and with the dual block and run-ahead
# on; the spawn rows below prove per case that those are also the defaults they run with.
# A shard holds one transport, so OpenRA's inproc rows are shards of their own.
matrix: ${{ fromJSON(needs.build-guard.outputs.verdict == 'identical' && needs.trace-cases.outputs.split-spawn-shards || needs.trace-cases.outputs.split-shards) }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Set swap space
if: runner.environment != 'self-hosted'
uses: ./.github/actions/set-swap-space
with:
swap-size-gb: 16
# build-linux's build when build-guard proved the two configurations identical,
# build-linux-split's otherwise; integration-split prints the guard's evidence. A build-linux
# that is not Release is never reused (scripts/ci/split_reuse_build_type.sh).
- name: Split runtime
env:
GUARD_VERDICT: ${{ needs.build-guard.outputs.verdict }}
PRIMARY_BUILD_TYPE: ${{ needs.build-linux.outputs.build-type }}
run: |
echo "split runtime: ${SPLIT_BUILD_DIR} (build-guard verdict '${GUARD_VERDICT:-none}')"
bash scripts/ci/split_reuse_build_type.sh
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
with:
gpu: "true"
# A miss does not fail the job; the install step downloads what the bundle does not hold.
- name: Restore trace fixtures
uses: actions/cache/restore@v5
with:
path: ${{ needs.trace-fixtures.outputs.path }}
key: ${{ needs.trace-fixtures.outputs.key }}
- name: Install trace fixtures
timeout-minutes: 15
run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Install runtime dependencies
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers
test -e /usr/lib/x86_64-linux-gnu/libEGL.so
test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so
- name: Download Linux split library
uses: actions/download-artifact@v8
with:
name: ${{ env.SPLIT_LIB }}
path: .
- name: Download trace replay
uses: actions/download-artifact@v8
with:
name: mobilegl-trace-replay
path: .
- name: Unpack split runtime (library under test)
# build-retrace's CTestTestfile.cmake has the absolute path
# <workspace>/build-linux/libMobileGL.so frozen into every case, so the swap happens here
# rather than through a variable, exactly as in retrace-verify. `nm`, not `nm -D`.
run: |
tar -xzf "${SPLIT_LIB}.tgz"
tar -xzf mobilegl-trace-replay.tgz
test -f "${SPLIT_LIB_DIR}/libMobileGL.so"
test -f build-retrace/tools/trace_replay/mobilegl_trace_replay
if [ "${SPLIT_LIB_DIR}" != build-linux ]; then
mkdir -p build-linux
cp "${SPLIT_LIB_DIR}/libMobileGL.so" build-linux/libMobileGL.so
fi
if ! nm --defined-only build-linux/libMobileGL.so | grep -q -i MG_Remote; then
echo "::error::the library unpacked at build-linux/libMobileGL.so defines no MG_Remote symbol, so this retrace would replay against a monolith build, ignore MOBILEGL_TRANSPORT entirely and match its golden having split nothing"
exit 1
fi
echo "the library at build-linux/libMobileGL.so is the split build"
- name: Prepare downloaded test metadata
run: |
for root in build-linux build-verify build-split build-verify-split build-retrace; do
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
done
# The spawn arm needs a second binary and the inproc arm needs none: inproc's server role
# is a thread in the replay process, spawn's is a process the client launches from this
# image. It ships in the split retrace library (libMobileGLServer.so is an executable
# with a .so name, so that an APK installs it somewhere it can be executed), and a missing
# one must be a named failure here rather than a LaunchServer refusal in every case.
- name: Place the server image (spawn arm only)
if: ${{ matrix.transport == 'spawn' }}
run: |
if [ ! -f "${SPLIT_LIB_DIR}/libMobileGLServer.so" ]; then
echo "::error::the split runtime carries no libMobileGLServer.so, so transport=spawn has no server to launch. ConfigLoader would resolve Spawn, LaunchServer would refuse BY NAME, and this job would red on a message about an unresolvable image rather than about a packaging gap."
exit 1
fi
if [ "${SPLIT_LIB_DIR}" != build-linux ]; then
cp "${SPLIT_LIB_DIR}/libMobileGLServer.so" build-linux/libMobileGLServer.so
fi
chmod +x build-linux/libMobileGLServer.so
echo "server image: $(readlink -f build-linux/libMobileGLServer.so)"
# The per-case settings are retrace_shard.py's "split" profile:
# - the Magma knobs the retrace job's monolith rows run with (the R11F_G11F_B10F fallback,
# the blended depth-write quirk on its one case, the iterationRP repairs on theirs), so a
# split row and its monolith control differ in the transport alone;
# - MOBILEGL_TRANSPORT and the absolute server image path;
# - on OpenRA, the wire-volume counters and the check that draw records crossed the wire;
# - with build-linux reused, the IPC defaults the client must log (strict=0).
# run_trace_case.cmake turns MOBILEGL_TRANSPORT into assertions of its own: the library must
# have resolved the transport in this process and its log must carry no Fatal{, so a case
# that somehow ran the wrong library reds here instead of passing on its golden. After
# ctest the evidence file those assertions leave is checked for every case, and the
# refusal census is printed from the library's own logs of both roles, which is the only
# valid source: the console sink is compiled out of this configuration.
- name: Retrace and validate (${{ matrix.transport }})
run: |
reuse=()
if [ "${SPLIT_REUSE}" = "true" ]; then reuse=(--split-reuse); fi
python3 scripts/ci/retrace_shard.py run --profile split \
--backend '${{ matrix.backend }}' --transport '${{ matrix.transport }}' --lane split \
--shard '${{ matrix.shard }}/${{ matrix.shards }}' \
--server-path "${GITHUB_WORKSPACE}/build-linux/libMobileGLServer.so" \
--build-dir build-retrace/tools/trace_replay \
--result-dir "${RUNNER_TEMP}/retrace-shard" "${reuse[@]}"
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit"
- name: Upload core dumps
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-split-core-dumps-${{ matrix.transport }}-${{ matrix.backend }}-shard${{ matrix.shard }}
path: /tmp/core.*
if-no-files-found: ignore
# Nothing reads a green shard's output, so only a failed shard uploads it.
- name: Upload failed shard results
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-split-failed-${{ matrix.transport }}-${{ matrix.backend }}-shard${{ matrix.shard }}
path: |
${{ runner.temp }}/retrace-shard/results
${{ runner.temp }}/retrace-shard/junit
${{ runner.temp }}/retrace-shard/logs
if-no-files-found: warn
overwrite: true
# The shipped library, replayed. build-linux-monolith-control's library has no MG_Remote, the
# shape FCL embeds; on the disaggregation branches nothing else replays that shape, so it runs
# there only (elsewhere the retrace job's library is that shape). One short case on both
# backends, with no transport set, must match its golden like any monolith row.
retrace-shipped:
name: retrace shipped library (${{ matrix.backend }}, ${{ matrix.case }})
runs-on: ubuntu-26.04
timeout-minutes: 60
needs:
- build-linux-monolith-control
- build-retrace
- trace-cases
if: ${{ !cancelled() && (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && needs.build-linux-monolith-control.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }}
strategy:
fail-fast: false
matrix:
backend: [DirectGLES, DirectVulkan]
case: [OpenRA]
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Set swap space
if: runner.environment != 'self-hosted'
uses: ./.github/actions/set-swap-space
with:
swap-size-gb: 16
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
with:
gpu: "true"
# This job's one case keeps its fixtures in Git itself, not in Git LFS, so the checkout
# has them and the fixture bundle is not restored. The check is that they are real files;
# a case with Git LFS fixtures would be downloaded here, with a warning annotation.
- name: Check trace fixture
run: bash .github/scripts/trace-fixture-bundle.sh install '${{ matrix.case }}'
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Install runtime dependencies
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
run: |
sudo apt-get update
sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers
test -e /usr/lib/x86_64-linux-gnu/libEGL.so
test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so
- name: Download shipped library
uses: actions/download-artifact@v8
with:
name: mobilegl-linux-monolith-control
path: .
- name: Download trace replay
uses: actions/download-artifact@v8
with:
name: mobilegl-trace-replay
path: .
# build-retrace froze <workspace>/build-linux/libMobileGL.so into every case, so the library
# is put where that path points, as retrace-verify does. `nm`, not `nm -D`: the MG_Remote
# symbols are hidden in a Release build and only the static table lists them.
- name: Unpack shipped library (library under test)
run: |
tar -xzf mobilegl-linux-monolith-control.tgz
tar -xzf mobilegl-trace-replay.tgz
test -f build-push-control/libMobileGL.so
test -f build-retrace/tools/trace_replay/mobilegl_trace_replay
mkdir -p build-linux
cp build-push-control/libMobileGL.so build-linux/libMobileGL.so
defined=$(nm --defined-only build-linux/libMobileGL.so | wc -l)
remote=$(nm --defined-only build-linux/libMobileGL.so | grep -c -i MG_Remote || true)
if [ "${defined}" -lt 1000 ] || [ "${remote}" -ne 0 ]; then
echo "::error::build-linux/libMobileGL.so defines ${defined} symbols and ${remote} MG_Remote symbol(s): it is not the shipped shape (no MG_Remote, not stripped)"
exit 1
fi
echo "the library at build-linux/libMobileGL.so is the shipped shape (${defined} defined symbols, no MG_Remote)"
- name: Prepare downloaded test metadata
run: python3 scripts/ci/prepare_test_runtime.py build-retrace
- name: Retrace and validate
working-directory: build-retrace/tools/trace_replay
run: |
ulimit -c unlimited
if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then
export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1
fi
ctest -V --no-tests=error -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$' \
--output-junit "${RUNNER_TEMP}/retrace.xml"
- name: Annotate failed entries
if: failure()
run: python3 scripts/ci/junit_annotate.py --excerpts 1 "${RUNNER_TEMP}"
- name: Upload core dumps
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-shipped-core-dumps-${{ matrix.backend }}-${{ matrix.case }}
path: /tmp/core.*
if-no-files-found: ignore
# Nothing reads a green leg's output, so only a failed leg uploads it.
- name: Upload actual image
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@v7
with:
name: retrace-shipped-result-${{ matrix.backend }}-${{ matrix.case }}
path: |
build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/**
build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/**
if-no-files-found: warn
pipe-gates:
name: MGPipe generators and hygiene gates
runs-on: ubuntu-26.04
# Deliberately independent of build-linux: these are source-level gates, they take
# seconds, and a broken build must not hide a drifted interface.
steps:
- name: Checkout repo
uses: actions/checkout@v6
# The seven generators all read MG_Pipe/*.def, so regenerating and diffing is what
# keeps the two interface tables, the wire records, the verify comparators, the
# PipeInputs field ids, the read-inventory coverage and the render-state member list
# from drifting apart from the catalog. The generated files are committed
# deliberately: the build must not depend on python.
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Test CTest runtime metadata preparation
run: python3 scripts/ci/test_prepare_test_runtime.py
- name: Regenerate the MGPipe interface (G1-G7)
run: |
python3 scripts/gen_pipe.py
git diff --exit-code -- MobileGL/MG_Pipe/generated
# The generators' own negative controls: canned inputs that MUST trip each structural check
# (a field list that does not cover its struct's members, a verb set that is not the function
# table's). Regenerating and diffing above cannot see a check that silently stopped
# checking - a broken gate and a clean tree produce the same green.
- name: MGPipe generator self-test
run: python3 scripts/gen_pipe.py --self-test
# The link seam owns physical storage and keeps link dispatch out of callers.
- name: Link-seam purity check
run: python3 scripts/ci/link_seam_purity.py --self-test
# The same shape for Espryt's re-keyed memos (P3b/P4b, PLAN-PH-P34B-P7.md 1.2). P4a/P5f
# moved the resolved-binding, sampler-pass, image-sweep, twin-registry, unit-sampler and
# twin-lookup memos onto {slot, gen} handles and applier serials; the legacy pointer-keyed
# arms stayed because they are the PULL build's only arms and G1 pins that build's .text.
# Nothing checked that a NEW frontend-pointer key could not appear outside them.
#
# P13 W3b: NO EXEMPTED ARM ANY MORE. The legacy pointer-keyed arms (MOBILEGL_PIPE_LEGACY_MEMOS)
# retired with the pull build, and the !MOBILEGL_BUILD_DISAGGREGATED arms are compiled into
# the library FCL embeds, so they get the same rule: no frontend-pointer key in any memo
# family, in any build.
#
# --self-test runs the negative controls and then the real scan in one invocation, as its
# sibling does; the scan also asserts that every memo family's anchor still matches and
# that every allow-list entry still suppresses something.
- name: Espryt memo-key purity check
run: python3 scripts/ci/espryt_memo_purity.py --self-test
# P13 W3b: the pull build's switches stay retired - a preprocessor conditional on
# MOBILEGL_PIPE_PUSH / MOBILEGL_PIPE_LEGACY_MEMOS, or an MGB_CTX spelling, coming back is a
# red here. --self-test plants each shape first and requires the gate to catch it.
- name: Retired pull-build switches stay retired (P13)
run: python3 scripts/ci/retired_switches.py --self-test
# The same question for the symbol tool the P1 gate is written in terms of.
- name: Symbol report self-test
run: python3 scripts/symbol_report.py --self-test
# Per-draw fprintf/printf instrumentation has repeatedly been committed by accident,
# once inside a mutex critical section. Nothing under these two trees prints to a
# stdio stream today - MGLOG_D compiles out in INFO builds and is the only channel
# they are allowed to use - so this gate starts with no exceptions, and any addition
# to it needs a reason in the pull request rather than a quiet whitelist entry. The
# alternation names every stdio spelling, not just the two that were committed:
# fprintf to either stream, printf, puts, and the iostream pair.
- name: No stdio in MG_Backend or MG_State
run: |
if grep -rnE 'fprintf[[:space:]]*\((stderr|stdout)|(^|[^[:alnum:]_>.])printf[[:space:]]*\(|(^|[^[:alnum:]_>.:])puts[[:space:]]*\(|std::(cout|cerr)' \
MobileGL/MG_Backend MobileGL/MG_State; then
echo "::error::stdio instrumentation found; use MGLOG_D (compiled out in INFO builds)"
exit 1
fi
echo "no fprintf(stderr/stdout / printf( / puts( / std::cout|cerr under MobileGL/MG_Backend or MobileGL/MG_State"
# Dirty-surface completeness (ARCHITECTURE.md §5.2.1). --check fails in both directions: a
# mutator the scanner finds with no row in MG_Pipe/DirtySurface.def, and a row naming a
# mutator the scan no longer finds. --self-test feeds canned negative controls and fails if
# one does not trip, so a check that stopped checking cannot stay green. The scanner matches
# text; the semantic proof is the verify lane.
- name: Dirty-surface mapping complete (G9)
run: |
python3 scripts/gen_pipe_dirty_surface.py --check
python3 scripts/gen_pipe_dirty_surface.py --self-test
# PipeFieldOwnership.inc is written by gen_pipe_field_ownership.py, not gen_pipe.py, so the
# gen_pipe.py regenerate-and-diff step above does not cover it. The build's static_assert
# catches only an unclassified row; --check also catches a changed class. No branch guard:
# every branch needs the committed table to follow from the .def files. Each --self-test
# control asserts its own message, and a harness control checks that one control's exit is
# never credited to another.
- name: Field-ownership table complete (R-7.1)
run: |
python3 scripts/gen_pipe_field_ownership.py --check
python3 scripts/gen_pipe_field_ownership.py --self-test
# Negative controls must themselves be able to go red (ARCHITECTURE.md §13.2). The Python
# tests run the trace and TCP lane tooling on stubs and synthetic data; test_run_trace_case.py
# feeds run_trace_case.cmake canned logs. No branch guard; no build, GPU or device.
- name: Control and trace-tool smoke tests (R-16)
run: |
python3 scripts/ci/test_trace_infrastructure.py
python3 scripts/ci/tcp_lane_tools_test.py
python3 tools/trace_replay/test_run_tcp_matrix.py
python3 tools/trace_replay/test_compare_actuals.py
python3 tools/trace_replay/test_run_trace_case.py
python3 scripts/ci/build_fingerprint.py --self-test
bash scripts/ci/split_reuse_build_type.sh --self-test
bash scripts/ci/test_set_swap_space.sh
python3 MobileGL/MG_IntegrationTest/Harness/test_split_log_paths.py
python3 scripts/ci/transport_pinned.py --self-test
# The helpers the build and split-lane jobs run before anything else can fail for them: the
# union of the split arms' selections, and the submodule lists, which are printed here
# without checking anything out.
- name: Build and lane helper self-tests
run: |
python3 scripts/ci/lane_selection.py --self-test
bash scripts/ci/checkout_submodules.sh --list library
bash scripts/ci/checkout_submodules.sh --list retrace
# Warning only for now: the disaggregation documents are still being written, and a
# lint that fails a rewrite in progress teaches people to ignore it. It becomes
# --strict when the documents settle.
- name: Documentation citation lint
run: |
shopt -s nullglob
documents=(docs/Disaggregated/*.md docs/Disaggregated/design/*.md docs/Disaggregated/guide/*.md)
if [ ${#documents[@]} -eq 0 ]; then
echo "no disaggregation documents to check"
exit 0
fi
python3 scripts/check_doc_citations.py "${documents[@]}" || true
build-linux-verify-split:
runs-on: ubuntu-26.04
timeout-minutes: 180
permissions:
actions: write
contents: read
env:
BUILD_DIR: build-verify-split
CCACHE_BASEDIR: ${{ github.workspace }}
CCACHE_COMPRESS: "true"
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 4G
CCACHE_NOHASHDIR: "true"
steps:
- name: Checkout repo
uses: actions/checkout@v6
# For build-linux-split's reason: MOBILEGL_BUILD_DISAGGREGATED shadows itself back to OFF
# when 3rdparty/flatbuffers/include is missing, and the script fails when it is.
- name: Check out build submodules
timeout-minutes: 15
run: bash scripts/ci/checkout_submodules.sh library
- name: Set swap space
uses: ./.github/actions/set-swap-space
with:
swap-size-gb: 32
- name: Configure runner environment
uses: ./.github/actions/setup-test-runner
- name: Get CMake
timeout-minutes: 15
uses: lukka/get-cmake@v4.3.3
with:
cmakeVersion: 3.31.10
- name: Restore ccache
uses: actions/cache/restore@v5
with:
path: .ccache
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
restore-keys: |
${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-
- name: Prepare Vulkan SDK
id: netsetup3726
continue-on-error: true
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Prepare Vulkan SDK (retry)
timeout-minutes: 15
if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup3726.outcome == 'failure' }}
uses: humbletim/setup-vulkan-sdk@v1.2.1
with:
vulkan-query-version: 1.4.304.1
vulkan-components: Vulkan-Headers, Vulkan-Loader
vulkan-use-cache: true
- name: Update glslang external sources
working-directory: 3rdparty/glslang
run: python3 update_glslang_sources.py
- name: Install build and runtime dependencies
timeout-minutes: 15
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
run: |
sudo apt-get update
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1
- name: Configure CMake
# Release/INFO for build-linux-verify's reason, and INFO is load-bearing here for
# build-linux-split's: ConfigLoader's resolved-transport line is logged at INFO and the
# arm-proof step below reads it back out of every entry's client log.
# ENABLE_PCH=OFF: ccache cannot cache a compile that uses glslang's precompiled header
# (see scripts/ci/configure_build.sh).
run: |
cmake -S . -B "${BUILD_DIR}" -G Ninja \
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \
-DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DENABLE_PCH=OFF \
-DCMAKE_BUILD_TYPE=Release \
-DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
-DMOBILEGL_BUILD_TEST=ON \
-DMOBILEGL_BUILD_BENCHMARK=OFF \
-DMOBILEGL_BUILD_INTEGRATION_TEST=ON \
-DMOBILEGL_ITEST_VK_ICD="${VK_ICD_FILENAMES}" \
-DMOBILEGL_BUILD_TRACE_REPLAY=OFF \
-DMOBILEGL_BUILD_DISAGGREGATED=ON \
-DMOBILEGL_BUILD_DISAGGREGATED_INPROC=ON \
-DMOBILEGL_PIPE_VERIFY=ON \
-DCMAKE_POLICY_VERSION_MINIMUM=3.5
- name: Build
run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)"
# Both options, both directions of "did it take" - the two jobs this one is the product of
# each check one of them, and this library has to pass both checks.
- name: Comparator and MG_Remote symbols
run: |
test -f "${BUILD_DIR}/libMobileGL.so"
defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l)
if [ "${defined}" -lt 1000 ]; then
echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the checks below could not have failed honestly"
exit 1
fi
if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeVerifyInputs"; then
echo "::error::libMobileGL.so defines no MGPipeVerifyInputs: -DMOBILEGL_PIPE_VERIFY=ON did not take, and the verify-split lane would compare nothing"
exit 1
fi
remote=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -c -i "MG_Remote" || true)
if [ "${remote}" -lt 1 ]; then
echo "::error::libMobileGL.so defines no MG_Remote symbol: -DMOBILEGL_BUILD_DISAGGREGATED=ON did not take, and every entry would run monolith under a name that says split"
exit 1
fi
echo "libMobileGL.so defines MGPipeVerifyInputs and ${remote} MG_Remote symbol(s) (${defined} defined symbols)"
- name: Show ccache stats
if: ${{ always() }}
run: ccache --show-stats
- name: Release superseded ccache entry
env:
GH_TOKEN: ${{ github.token }}
CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true
- name: Save ccache
continue-on-error: true
uses: actions/cache/save@v5
with:
path: .ccache
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
- name: Prepare packaged test metadata
run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}"
# The unit tests are built here, so this flavor still compiles and links them, but they are
# not packaged: the one consumer, integration-verify-split, runs only the
# integration-verify-split label, whose entries all live under MG_IntegrationTest.
- name: Package verify-split runtime
run: |
mkdir -p ci-artifacts
mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort)
tar --exclude='*/CMakeFiles' --exclude='*.o' --exclude='*.a' --exclude='*.ninja*' \
--exclude='build.ninja' --exclude='cmake_install.cmake' \
-I 'zstd -T0' -cf ci-artifacts/mobilegl-linux-runtime-verify-split.tar.zst \
"${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" \
"${BUILD_DIR}/MobileGL/MG_IntegrationTest" "${SHARED_LIBS[@]}"
# compression-level 0: the tarball is zstd already (see 'Upload Linux runtime').
- name: Upload verify-split runtime
uses: actions/upload-artifact@v7
with:
name: mobilegl-linux-runtime-verify-split
path: ci-artifacts/mobilegl-linux-runtime-verify-split.tar.zst
if-no-files-found: error
compression-level: 0
# P13 W1 (PLAN-P13 §4.2): THE SKIP CENSUS. Deleting a compile-time macro before the GTEST_SKIP
# or pull stub that guards on it turns tests "green by skipping"; a per-lane pass/fail count
# cannot see that. Every gtest lane uploads its JUnit as junit-*; this job keys each case by
# lane and name and holds the run against scripts/data/skip_census_baseline.json: a new skip
# (pass -> skip included) or a vanished name is red, a skip that turned into a pass is a note.
# A deliberate retirement edits the baseline in the same commit and names the cases.
skip-census:
name: skip census
runs-on: ubuntu-26.04
needs:
- build-guard
- test
- integration
- integration-verify
- integration-verify-split
- integration-split
- integration-split-strict
if: ${{ !cancelled() }}
steps:
- name: Checkout repo
uses: actions/checkout@v6
- name: Skip census self-test
run: python3 scripts/ci/skip_census.py --self-test
- name: Download lane JUnit
uses: actions/download-artifact@v8
with:
pattern: junit-*
path: junit
# The optional rows run on some paths only, and on the path this run took they are required:
# without a reused build (any build-guard verdict but identical) integration-split runs its
# unit lane and Transport A/B, and with one A/B still runs when the IPC defaults were not
# proven.
- name: Skip census
env:
GUARD_VERDICT: ${{ needs.build-guard.outputs.verdict }}
SPLIT_DEFAULTS_ON: ${{ needs.integration-split.outputs.defaults-on }}
run: |
required=()
if [ "${GUARD_VERDICT}" != identical ]; then
required+=(--require junit-integration-split/unit-split)
fi
if [ "${GUARD_VERDICT}" != identical ] || [ "${SPLIT_DEFAULTS_ON}" != true ]; then
required+=(--require junit-integration-split/arm-inproc --require junit-integration-split/arm-monolith)
fi
echo "build-guard verdict '${GUARD_VERDICT:-none}', split IPC defaults proven '${SPLIT_DEFAULTS_ON:-no}': ${required[*]:-no optional row required}"
python3 scripts/ci/skip_census.py check junit \
--baseline scripts/data/skip_census_baseline.json --report skip-census.json "${required[@]}"
- name: Upload census
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: skip-census
path: skip-census.json
if-no-files-found: warn