name: Test on: push: branches: - dev - Feat/Backend-Direct-GLES - Feat/Backend-Direct-Vulkan # TEMPORARY, remove before merging the MGPipe work into dev: the disaggregation # branch runs the full lane on every push so a phase's landing is not gated on # someone remembering to dispatch the workflow by hand. - feat/disaggregated - codex/fix-github-ci workflow_dispatch: env: # Software-rendered CI submissions may exceed the interactive GPU hang budget. MOBILEGL_GPU_HANG_BUDGET_MS: "0" # ONE LIVE RUN PER BRANCH. The disaggregation branch pushes (often docs-only) commits minutes apart # and each push fans out to ~330 retrace legs here and ~275 jobs in apk.yml; with no group, runs # sat queued for 10+ hours behind their own successors. A manual dispatch gets a group of its own, # so a push never cancels a manually started run; dev queues rather # than cancels. No paths-ignore here, unlike apk.yml: this workflow hosts the doc citation lint. concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.ref }} cancel-in-progress: ${{ github.event_name == 'push' && github.ref != 'refs/heads/dev' }} jobs: select-runner: name: Select Test runner runs-on: ubuntu-latest permissions: contents: read outputs: runner: ${{ steps.select.outputs.runner }} pool: ${{ steps.select.outputs.pool }} steps: - uses: actions/checkout@v6 - name: Test runner selection run: python3 scripts/ci/test_select_test_runner.py - name: Check preferred runner availability id: select env: RUNNER_STATUS_TOKEN: ${{ secrets.RUNNER_STATUS_TOKEN }} run: python3 scripts/ci/select_test_runner.py build-linux: needs: select-runner runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} permissions: actions: write contents: read env: BUILD_DIR: build-linux MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci') && 'ON' || 'OFF' }} CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "true" CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 4G CCACHE_NOHASHDIR: "true" steps: - name: Set swap space if: runner.environment != 'self-hosted' uses: pierotofy/set-swap-space@v1.0 with: swap-size-gb: 32 - name: Checkout repo uses: actions/checkout@v6 with: submodules: recursive - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake if: runner.environment != 'self-hosted' uses: lukka/get-cmake@v4.3.3 - name: Restore ccache uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1 restore-keys: | ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache- - name: Prepare Vulkan SDK if: runner.environment != 'self-hosted' uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Update glslang external sources working-directory: 3rdparty/glslang run: python update_glslang_sources.py - name: Install build dependencies if: runner.environment != 'self-hosted' run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build - name: Show installed toolchain run: | ccache --version "${MOBILEGL_CI_CC}" --version "${MOBILEGL_CI_CXX}" --version "${MOBILEGL_CI_LLD}" --version dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true - name: Configure CMake run: | if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then BUILD_TYPE=Debug else BUILD_TYPE=Release fi cmake -S . -B "${BUILD_DIR}" -G Ninja \ -DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \ -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=ON \ -DMOBILEGL_BUILD_BENCHMARK=ON \ -DMOBILEGL_BUILD_INTEGRATION_TEST=ON \ -DMOBILEGL_ITEST_VK_ICD="${VK_ICD_FILENAMES}" \ -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \ -DMOBILEGL_BUILD_DISAGGREGATED="${MOBILEGL_CI_DISAGGREGATED}" \ -DMOBILEGL_BUILD_DISAGGREGATED_INPROC="${MOBILEGL_CI_DISAGGREGATED}" \ -DMOBILEGL_PIPE_PUSH="${MOBILEGL_CI_DISAGGREGATED}" \ -DMOBILEGL_PIPE_VERIFY=OFF \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON \ -DBENCHMARK_ENABLE_TESTING=OFF \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 - name: Build run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)" - name: Primary runtime mode proof run: | python3 scripts/ci/runtime_mode_proof.py self-test mode=monolith if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then mode=disaggregated; fi python3 scripts/ci/runtime_mode_proof.py build --build-dir "${BUILD_DIR}" \ --mode "${mode}" --output "${BUILD_DIR}/runtime-mode.json" - name: Show ccache stats if: ${{ always() && needs.select-runner.result == 'success' }} run: ccache --show-stats # Rewrite one rolling entry per job on the default branch. The upload stays # cumulative - it carries every object restored at the top of this run plus # the few TUs that actually changed - but Actions cache keys are immutable, # so the superseded blob has to be released before the same key can be # re-uploaded. Running after the build means a failed build leaves the # existing entry untouched. The other trigger branches restore this entry # rather than each writing one of their own. - name: Release superseded ccache entry if: github.ref_name == github.event.repository.default_branch env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1 run: gh cache delete "${CACHE_KEY}" || true - name: Save ccache if: github.ref_name == github.event.repository.default_branch continue-on-error: true uses: actions/cache/save@v5 with: path: .ccache key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1 - name: Package Linux runtime run: | mkdir -p ci-artifacts mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort) tar \ --exclude='*/CMakeFiles' \ --exclude='*.o' \ --exclude='*.a' \ --exclude='*.ninja*' \ --exclude='build.ninja' \ --exclude='cmake_install.cmake' \ -czf ci-artifacts/mobilegl-linux-runtime.tgz \ "${BUILD_DIR}/runtime-mode.json" \ "${BUILD_DIR}/CTestTestfile.cmake" \ "${BUILD_DIR}/MobileGL/MG_Test" \ "${BUILD_DIR}/MobileGL/MG_Benchmark" \ "${BUILD_DIR}/MobileGL/MG_IntegrationTest" \ "${SHARED_LIBS[@]}" - name: Upload Linux runtime uses: actions/upload-artifact@v7 with: name: mobilegl-linux-runtime path: ci-artifacts/mobilegl-linux-runtime.tgz if-no-files-found: error # The pull-library negative control must not consume the primary artifact: # on disaggregation branches that artifact now contains the real transport. build-linux-monolith-control: name: build explicit pull runtime control needs: select-runner runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} env: BUILD_DIR: build-monolith-control CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_COMPRESS: "true" CCACHE_NOHASHDIR: "true" steps: - name: Set swap space if: runner.environment != 'self-hosted' uses: pierotofy/set-swap-space@v1.0 with: swap-size-gb: 32 - name: Checkout repo uses: actions/checkout@v6 with: submodules: recursive - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake if: runner.environment != 'self-hosted' uses: lukka/get-cmake@v4.3.3 - name: Restore pull compiler cache uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-build-linux-ccache-v1 - name: Prepare Vulkan SDK if: runner.environment != 'self-hosted' uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Update glslang external sources working-directory: 3rdparty/glslang run: python update_glslang_sources.py - name: Install build dependencies if: runner.environment != 'self-hosted' run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build - name: Configure explicit pull control run: | cmake -S . -B "${BUILD_DIR}" -G Ninja \ -DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_BUILD_TYPE=Release -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=OFF -DMOBILEGL_BUILD_BENCHMARK=OFF \ -DMOBILEGL_BUILD_INTEGRATION_TEST=OFF -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \ -DMOBILEGL_BUILD_DISAGGREGATED=OFF -DMOBILEGL_BUILD_DISAGGREGATED_INPROC=OFF \ -DMOBILEGL_PIPE_PUSH=OFF -DMOBILEGL_PIPE_VERIFY=OFF \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5 - name: Build and identify pull control run: | cmake --build "${BUILD_DIR}" --target MobileGL --parallel "$(nproc)" python3 scripts/ci/runtime_mode_proof.py build --build-dir "${BUILD_DIR}" \ --mode monolith --output "${BUILD_DIR}/runtime-mode.json" - name: Package pull control run: | tar -czf mobilegl-linux-monolith-control.tgz \ "${BUILD_DIR}/libMobileGL.so" "${BUILD_DIR}/runtime-mode.json" - name: Upload pull control uses: actions/upload-artifact@v7 with: name: mobilegl-linux-monolith-control path: mobilegl-linux-monolith-control.tgz if-no-files-found: error test: runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} needs: [select-runner, build-linux] steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake if: runner.environment != 'self-hosted' uses: lukka/get-cmake@v4.3.3 - name: Install runtime dependencies if: runner.environment != 'self-hosted' run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libgles2 libgl1-mesa-dri mesa-vulkan-drivers - name: Download Linux runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime path: . - name: Unpack Linux runtime run: tar -xzf mobilegl-linux-runtime.tgz - name: Normalize CTest command paths run: | python - <<'PY' from pathlib import Path import re for path in Path('build-linux').rglob('CTestTestfile.cmake'): text = path.read_text() text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text) path.write_text(text) PY - name: Test working-directory: build-linux run: | ulimit -c unlimited if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then ctest -V -L unit --no-tests=error else ctest --output-on-failure -L unit --no-tests=error fi - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: unit-core-dumps path: /tmp/core.* if-no-files-found: ignore integration: name: integration (${{ matrix.transport }}) runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} needs: [select-runner, build-linux] strategy: fail-fast: false matrix: include: ${{ fromJSON((github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci') && '[{"lane":"monolith-control","transport":"monolith"},{"lane":"inproc-acceptance","transport":"inproc"}]' || '[{"lane":"monolith-control","transport":"monolith"}]') }} env: MOBILEGL_TRANSPORT: ${{ matrix.transport }} MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" MOBILEGL_IPC_RUN_AHEAD: "1" steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake if: runner.environment != 'self-hosted' uses: lukka/get-cmake@v4.3.3 - name: Install runtime dependencies if: runner.environment != 'self-hosted' # Same set as the benchmark job, for the same reason: the scenarios bring # up real headless EGL (llvmpipe) and Vulkan (lavapipe) contexts, and # libegl-mesa0 - the EGL vendor library behind glvnd's libegl1 dispatch - # only arrives as a Recommends. run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers - name: Download Linux runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime path: . - name: Unpack Linux runtime run: tar -xzf mobilegl-linux-runtime.tgz - name: Normalize CTest command paths run: | python - <<'PY' from pathlib import Path import re for path in Path('build-linux').rglob('CTestTestfile.cmake'): text = path.read_text() text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text) path.write_text(text) PY - name: Integration scenarios working-directory: build-linux # REQUIRE_GPU makes a driverless runner FAIL instead of skipping every # scenario - an all-skip run is otherwise indistinguishable from a pass, # which is how a five-month-old draw-dropping bug survived unseen until # this lane existed. # # Configure CMake with the selected runner's ICD; each test's ENVIRONMENT # property takes precedence over ambient job variables. # # Cores are armed so that any crash - the harness pre-flight child's # included - leaves /tmp/core.*, which the failure-only step below ships # as an artifact. Analyzing a downloaded core against the runtime # artifact's binary in an ubuntu-24.04 userspace reproduces the exact # crash stack without burning a CI round on an in-workflow debugger. env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH: "1" MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS: "1" MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER: "1" run: | ulimit -c unlimited if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi # Second, filtered pass: with the range-invalidating map flush disabled, # the buffer scenarios run on the upload ring's staged-copy tier - which # the default pass never reaches (the map tier absorbs every flush on # Mesa), so without this the Mali fallback tier would have zero CI # coverage. The flag is NOT baked into the ctest ENVIRONMENT properties, # so an inline env reaches the test processes (unlike the ICD pin above). if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then ctest -V -L integration-gpu --no-tests=error --output-junit "${RUNNER_TEMP}/integration.xml" MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 ctest -V -L integration-gpu \ -R 'Buffer|Readback|Atomic|Ssbo|Arena' --no-tests=error --output-junit "${RUNNER_TEMP}/integration-staged-copy.xml" else ctest --output-on-failure -L integration-gpu --no-tests=error --output-junit "${RUNNER_TEMP}/integration.xml" MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 ctest --output-on-failure -L integration-gpu \ -R 'Buffer|Readback|Atomic|Ssbo|Arena' --no-tests=error --output-junit "${RUNNER_TEMP}/integration-staged-copy.xml" fi # Both backend roles must execute inproc in the primary artifact. - name: Inproc execution proof (both backends) if: matrix.transport == 'inproc' working-directory: build-linux run: | ctest -L integration-p5f-rsp --show-only=json-v1 > "${RUNNER_TEMP}/primary-inproc-lane.json" ctest -L integration-p5f-rsp --no-tests=error --output-on-failure \ --output-junit "${RUNNER_TEMP}/primary-inproc.xml" python3 ../scripts/ci/runtime_mode_proof.py logs \ --discovery "${RUNNER_TEMP}/primary-inproc-lane.json" \ --junit "${RUNNER_TEMP}/primary-inproc.xml" --output "${RUNNER_TEMP}/primary-inproc-proof.json" # The run log needs a signed-in reader; annotations do not. Name every failed ctest entry, # with the lines of its output that say why, where anyone reading the run can see them. - name: Annotate failed entries if: failure() run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}" - name: Upload integration lane evidence if: ${{ always() && needs.select-runner.result == 'success' }} uses: actions/upload-artifact@v7 with: name: integration-evidence-${{ matrix.lane }}-${{ matrix.transport }} path: | ${{ runner.temp }}/integration*.xml ${{ runner.temp }}/primary-inproc* build-linux/runtime-mode.json build-linux/MobileGL/MG_IntegrationTest/p5f-rsp-*.log if-no-files-found: warn - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: integration-core-dumps-${{ matrix.lane }}-${{ matrix.transport }} path: /tmp/core.* if-no-files-found: ignore # THE THIRD CI MODE (ARCHITECTURE.md 13.2-(2)): the same library, built with the PipeInputs # comparator compiled in, running the integration suite and a trace subset with two state models # in one address space. It is a second build rather than a flag on the first because # MOBILEGL_PIPE_VERIFY is a compile-time option - the snapshot, the entry compare and the # compare-at-read hook do not exist in the shipped library, and are never meant to. build-linux-verify: needs: select-runner runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} timeout-minutes: 120 permissions: actions: write contents: read env: BUILD_DIR: build-verify CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "true" CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 4G CCACHE_NOHASHDIR: "true" steps: - name: Set swap space if: runner.environment != 'self-hosted' uses: pierotofy/set-swap-space@v1.0 with: swap-size-gb: 32 - name: Checkout repo uses: actions/checkout@v6 with: submodules: recursive - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake if: runner.environment != 'self-hosted' uses: lukka/get-cmake@v4.3.3 - name: Restore ccache uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1 restore-keys: | ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache- - name: Prepare Vulkan SDK if: runner.environment != 'self-hosted' uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Update glslang external sources working-directory: 3rdparty/glslang run: python update_glslang_sources.py - name: Install build dependencies if: runner.environment != 'self-hosted' run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build - name: Show installed toolchain run: | ccache --version "${MOBILEGL_CI_CC}" --version "${MOBILEGL_CI_CXX}" --version "${MOBILEGL_CI_LLD}" --version dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true - name: Configure CMake # Release/INFO like the shipped build on purpose. The poison arms in this configuration # through MOBILEGL_PIPE_VERIFY (PipeInputs.h derives MOBILEGL_PIPE_POISON from it), so this # job needs neither a Debug log level nor MOBILEGL_BUILD_DISAGGREGATED - and a Debug build # would compare a different library from the one the other lanes measure. (build-linux # switches to Debug under ACTIONS_STEP_DEBUG; this job deliberately does not - a Debug # build flips CXX_VISIBILITY_PRESET and arms MOBILEGL_PIPE_POISON through a second, unrelated # arm of its #if, so the debug switch would change what the lane is measuring.) run: | cmake -S . -B "${BUILD_DIR}" -G Ninja \ -DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \ -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_BUILD_TYPE=Release \ -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=ON \ -DMOBILEGL_BUILD_BENCHMARK=OFF \ -DMOBILEGL_BUILD_INTEGRATION_TEST=ON \ -DMOBILEGL_ITEST_VK_ICD="${VK_ICD_FILENAMES}" \ -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \ -DMOBILEGL_PIPE_VERIFY=ON \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 - name: Build run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)" # The lane is worthless if the option silently did not take, and that is a one-character # mistake away at all times (a typo'd -D is not an error in CMake). Two checks, both cheap: # the comparator's entry point must be in the library, and the fill entry point with it. # # `nm` and NOT `nm -D`. The library is built CXX_VISIBILITY_PRESET hidden in every non-Debug # configuration (CMakeLists.txt:600-604) and the MGPipe entry points are plain namespace # functions with no export attribute, so not one of them appears in the DYNAMIC table: on a # perfectly healthy verify build `nm -D --defined-only ... | grep -c MGPipe` answers 0 out of # ~11900 exported symbols, and a gate spelled that way is red forever for a reason that has # nothing to do with what it claims to test. The static symbol table has them as local `t` # entries, this artifact is never stripped, and `No MG_Remote in the pull build` below already # uses this spelling. The symbol count guards the remaining hole: a stripped library would # make both greps fail for a third, silent reason. - name: Comparator symbols in verify library run: | test -f "${BUILD_DIR}/libMobileGL.so" defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l) if [ "${defined}" -lt 1000 ]; then echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the two checks below could not have failed honestly" exit 1 fi if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeVerifyInputs"; then echo "::error::libMobileGL.so defines no MGPipeVerifyInputs: -DMOBILEGL_PIPE_VERIFY=ON did not take, and every lane that consumes this artifact would run the comparator-free library and pass having compared nothing" exit 1 fi # The per-verb entry point, under EITHER of its two names. P2 renames # MGPipeFillForVerb to MGPipeValidateForVerb (the body becomes the tracker's walk and # the fill is one of its five steps), so this check has to accept both or it goes red on # the rename for a reason that has nothing to do with what it tests. What it tests is # unchanged: that the library HAS a per-verb entry point compiled in. if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -qE "MGPipeValidateForVerb|MGPipeFillForVerb"; then echo "::error::libMobileGL.so defines neither MGPipeValidateForVerb nor MGPipeFillForVerb: there is no per-verb entry point in this artifact, so nothing fills the block the comparator compares" exit 1 fi echo "libMobileGL.so defines MGPipeVerifyInputs and a per-verb entry point (${defined} defined symbols)" - name: Show ccache stats if: ${{ always() && needs.select-runner.result == 'success' }} run: ccache --show-stats - name: Release superseded ccache entry if: github.ref_name == github.event.repository.default_branch env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1 run: gh cache delete "${CACHE_KEY}" || true - name: Save ccache if: github.ref_name == github.event.repository.default_branch continue-on-error: true uses: actions/cache/save@v5 with: path: .ccache key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1 - name: Package Linux verify runtime run: | mkdir -p ci-artifacts mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort) tar \ --exclude='*/CMakeFiles' \ --exclude='*.o' \ --exclude='*.a' \ --exclude='*.ninja*' \ --exclude='build.ninja' \ --exclude='cmake_install.cmake' \ -czf ci-artifacts/mobilegl-linux-runtime-verify.tgz \ "${BUILD_DIR}/CTestTestfile.cmake" \ "${BUILD_DIR}/MobileGL/MG_Test" \ "${BUILD_DIR}/MobileGL/MG_IntegrationTest" \ "${SHARED_LIBS[@]}" - name: Upload Linux verify runtime uses: actions/upload-artifact@v7 with: name: mobilegl-linux-runtime-verify path: ci-artifacts/mobilegl-linux-runtime-verify.tgz if-no-files-found: error # The verify lane itself, plus the two negative controls that keep it falsifiable. The controls # are ALWAYS-ON steps, not a manual exercise: a gate that can only be shown to work by someone # remembering to break it on purpose is a gate that has already stopped working. integration-verify: runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} timeout-minutes: 180 needs: [select-runner, build-linux-verify] steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake if: runner.environment != 'self-hosted' uses: lukka/get-cmake@v4.3.3 - name: Install runtime dependencies if: runner.environment != 'self-hosted' run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers - name: Download Linux verify runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime-verify path: . - name: Unpack Linux verify runtime run: | tar -xzf mobilegl-linux-runtime-verify.tgz test -f build-verify/libMobileGL.so - name: Normalize CTest command paths run: | python - <<'PY' from pathlib import Path import re for path in Path('build-verify').rglob('CTestTestfile.cmake'): text = path.read_text() text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text) path.write_text(text) PY - name: Integration scenarios (verify) working-directory: build-verify # --no-tests=error is half the gate: the verify entries only exist when the library was # configured with -DMOBILEGL_PIPE_VERIFY=ON, so a build that lost the option matches no # tests and reds here instead of reporting a green run of nothing. The other half is # PipeVerifyArmingScenario.Armed, which fails when the library never printed its arming # line - the failure mode a bare `MOBILEGL_PIPE_VERIFY=1` cannot detect by itself. # # SCOPE, stated so nobody reads more into a green than is there: this is every integration # ENTRY under the comparator, not every integration CONFIGURATION. The `integration` job # runs a second, filtered pass with MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 for the # upload ring's staged-copy tier; that pass is 186 entries here and, at the 5-10x the # comparator costs, is not affordable inside this job's budget. The tier is covered by # `integration`, unverified, and P2 can take it once the comparator's cost is known. env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH: "1" MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS: "1" MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER: "1" run: | ulimit -c unlimited if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then ctest -V -L integration-verify --no-tests=error else ctest --output-on-failure -L integration-verify --no-tests=error fi # The push-only unit tests, on the verify runtime. # # WHY HERE AND NOT IN `test`. The entries themselves are registered in EVERY build - they # have to be, or `ctest -N` would stop matching name-for-name between the pull and the push # build (gate G2). What is push-only is what they assert about: MGPipeRenderStateSpans.cpp # and PipeApply.cpp are appended to SOURCE_FILES inside the `if (MOBILEGL_PIPE_PUSH)` block, # which is exactly how the pull build stays symbol-identical, so in a pull build each case # opens with `#if !MOBILEGL_PIPE_PUSH GTEST_SKIP() << "push not compiled in"`. The `test` # job therefore runs G6's chunk-table walk and G10's residual assertions as a column of # skips: CI executes the NAMES and never one of the assertions. This job unpacks a build # that compiled them, so it is the first place in CI where they actually run. # # This artifact already carries them - the packaging step above tars # ${BUILD_DIR}/MobileGL/MG_Test whole - so the whole cost is the run, which is ~14 s for # ~1490 entries. --no-tests=error, because a packaging change that stopped shipping the # unit binaries would otherwise report a green run of nothing. - name: Unit tests on the verify runtime (G6, G10) working-directory: build-verify run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" # The always-on P2 and P3a negative controls (G8, G10, G12), which are labelled # integration-gpu and not integration-verify - they are about the handle key, the subsystem # switch and the map-persistent counter, not about the comparator - so the lane above does # not reach them. They are run HERE because this is the only CI job that unpacks a # MOBILEGL_PIPE_PUSH build: every one of them reads a counter out of the library's summary # line, and both the counters and their brackets are #if MOBILEGL_PIPE_PUSH, so in the pull # `integration` job the entries exist (gate G2 requires the same names in both builds) but # have nothing to assert. # # An arm whose subsystem has not landed on this tree SKIPS with the reason (never absent, # never a green that asserted nothing), so this step is green through the P2 and P3a landing # orders and starts asserting as each package arrives. # # The environment is the sibling step's, deliberately and in full: these entries run the # same DirectVulkan binary through the same runner, so the three MOBILEGL_MAGMA_* fixes it # needs apply here too, and a crash here has to leave a core for the same black-box flow. # The step above is the only reason those lines exist in this job; a control that crashed # without one would be the hardest failure in the job to diagnose. # # THE -R ALTERNATIVES ARE TEST-NAME PREFIXES, NOT LANE LABELS, and each one is deliberately # the SHORTEST string that still selects only what it means to. `ResourceSubsystem` (not # `ResourceSubsystemControl`) is what reaches the eight # DirectGLES.ResourceSubsystemOn./Off.LargeArenaAdoptionScenario.* entries - the A/B lanes # whose entire purpose is that the handle path and the legacy BufferBackendOps path must # agree about an adopted store - as well as the two ResourceSubsystemControl. entries. # `MapPersistentRoundtrip` is singular because LargeArenaAdoptionScenario's case is # `AnAdoptionCostsExactlyOneMapPersistentRoundtrip`; the plural matched only the lane PREFIX # of the other one. Both mistakes were silent: this is the only CI job that unpacks a push # build, so an entry the filter misses is either never run under the P3a bits at all or runs # only in the pull `integration` job, where a MOBILEGL_PIPE_PUSH value steers nothing # (Config.h declares the field inside the push guard) and both arms are the same legacy path. # A lane that cannot go red where it is installed is not a gate (ROADMAP.md:7). # P4a ADDS THREE ALTERNATIVES, and each one is here because this is the only CI job that # unpacks a push build: # * `ObjectSubsystem` reaches the three DirectGLES.ObjectSubsystemControl. entries - the # 0x1fff-vs-0x1ff A/B and the 0x9ff dependency refusal (G12). `ResourceSubsystem` does # NOT match it: the two families are named apart on purpose, because they are different # phases' switches and a filter that merged them would hide one behind the other. # * `TextureParamsWithoutASamplerView` reaches G9's four cases, the scenario ROADMAP.md:20 # names by hand. It runs in the ambient lanes, which the label already selects - but this # step is where those cases run against a PUSH library, and G9's whole claim is about the # push path. All four cases are green on the contract commit - including the one D-E3 # expected to be red, for the reason the scenario's header records - so this row is green # from the day it lands and goes red only if a reachability path stops syncing texture # parameters at all, which is the coupling ARCHITECTURE.md:100 exists to remove. # * `TextureUploadShape` is RECORDED, not gated (D-D4): it asserts that the two upload-shape # counters could be read and that they agree, and prints the shape for MEASUREMENTS.md. It # is in the filter so that the number is actually collected on every run - an unmeasured # shape is not a recorded one - and because its own assertions can go red. # As with the four before them, each alternative is the SHORTEST string that selects only what # it means to. # Handle-ABA, CSO, subsystem and texture-parameter controls (G8, G8b, G9, G10, G12). - name: Push-build controls (G8, G8b, G9, G10, G12) working-directory: build-verify env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH: "1" MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS: "1" MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER: "1" run: | ulimit -c unlimited if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi ctest --output-on-failure -L integration-gpu \ -R 'HandleRecycle|CsoContentAddressing|ResourceSubsystem|MapPersistentRoundtrip|ObjectSubsystem|TextureParamsWithoutASamplerView|TextureUploadShape' \ --no-tests=error -j 4 # The arming lanes' logs, and ONLY those. Each lane shares one MOBILEGL_LOG_FILE_PATH and the # library opens it fopen(path, "w"), so after an ambient lane of 400-odd processes the file # holds the LAST one - grepping it would say nothing about the other 405 and would red a # healthy lane whenever the last entry happened not to issue a verb (which is what the # PoisonOmissionScenario parent, the last ambient entry, does by construction: it forks, # execve()s and reads files). The DirectGLES.VerifyArming. / DirectVulkan.VerifyArming. # entries are one process each on a log path nothing else writes, so this grep means exactly # what it says. # # What it proves: arming is a property of (this library, this environment), and these two # processes ran the same library with the same MOBILEGL_PIPE_VERIFY=1 as their ~400 ambient # siblings. It is not, and cannot be, a per-process census - the shared log cannot support one. # It catches the case ctest cannot: an arming entry that SKIPPED still reports green. - name: Verify lane arming check working-directory: build-verify run: | shopt -s nullglob logs=(MobileGL/MG_IntegrationTest/pipe-verify-arming-*.log) if [ ${#logs[@]} -lt 2 ]; then echo "::error::found ${#logs[@]} pipe-verify-arming-*.log (expected one per backend). The VerifyArming. entries did not run, so nothing in this job establishes that the comparator was ever armed." exit 1 fi for log in "${logs[@]}"; do if ! grep -q "MGPipe: verify armed" "${log}"; then echo "::error::${log} carries no arming line: that lane's process ran the whole scenario without the comparator, so every green entry beside it is green for no reason" exit 1 fi done echo "arming line present in all ${#logs[@]} arming-lane log(s)" # NEGATIVE CONTROL A (gate G4). The knob perturbs one field in the snapshot arm before the # entry compare, so a working comparator must abort the run. This step passes when ctest # FAILS - `if ctest ...; then error` - which is the only shape that can catch a comparator # that silently compares nothing. # # The knob reaches the test process through the JOB environment: no ctest ENVIRONMENT # property on the ambient Verify. entries names it (MG_IntegrationTest/CMakeLists.txt says # so out loud), and a property entry would otherwise override this and the control would # prove nothing. Same precedent as MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH in `integration`. - name: Negative control A (corrupted snapshot) working-directory: build-verify env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_PIPE_VERIFY_CORRUPT: GetRenderStateParameters run: | FILTER='DirectGLES\.Verify\..*ClearThenReadPixels' # An empty selection would ALSO make ctest exit non-zero (--no-tests=error), and this # step reads non-zero as "the control worked" - so the selection is counted first. A # control that passes because it ran nothing is worse than no control. matched=$(ctest -N -L integration-verify -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:') if [ "${matched}" -lt 1 ]; then echo "::error::negative control A selected ${matched} tests; its filter no longer matches anything" exit 1 fi if ctest --output-on-failure -L integration-verify -R "${FILTER}" --no-tests=error; then echo "::error::MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left ${matched} verify entries GREEN. The comparator is not comparing, so every green entry above is green for no reason." exit 1 fi echo "the corrupted field turned ${matched} selected entries red, as it must" # NEGATIVE CONTROL B (gate G5). The omission skips the STAMP of one field for one verb while # still copying its value - indistinguishable from a fill row nobody wrote - so the poison # must abort the glGenerateMipmap. Again: this step passes when ctest fails. # # The entry it targets is PoisonOmissionScenario.WithoutOmissionCompletes, which is green in # the ambient lane above and is the ONLY integration entry in the tree that calls # glGenerateMipmap at all. It deliberately does not skip itself when the knob is set, exactly # so that this control has something to turn red. - name: Negative control B (omitted fill point) working-directory: build-verify env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_PIPE_POISON_OMIT: GenerateMipmap:GetActiveTextureUnit run: | FILTER='DirectGLES\.Verify\.PoisonOmissionScenario\.WithoutOmissionCompletes' matched=$(ctest -N -L integration-verify -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:') if [ "${matched}" -lt 1 ]; then echo "::error::negative control B selected ${matched} tests; its filter no longer matches anything" exit 1 fi if ctest --output-on-failure -L integration-verify -R "${FILTER}" --no-tests=error; then echo "::error::MOBILEGL_PIPE_POISON_OMIT=GenerateMipmap:GetActiveTextureUnit left the verify lane GREEN. The per-verb poison is not armed, so a forgotten fill row would ship silently." exit 1 fi echo "the omitted fill point turned the lane red, as it must" - name: Upload verify lane logs if: ${{ always() && needs.select-runner.result == 'success' }} uses: actions/upload-artifact@v7 with: name: integration-verify-logs path: build-verify/MobileGL/MG_IntegrationTest/pipe-*.log* if-no-files-found: warn - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: integration-verify-core-dumps path: /tmp/core.* if-no-files-found: ignore # P7 WAVE 3 (V1), CONTRACT-P7 §6 - EXIT GATE 2's SECOND COLUMN: the comparator on the split # (inproc) arm, both backends, and the two negative controls red on that arm. # # ITS OWN BUILD, because neither existing one can carry it: build-linux-verify configures # without MOBILEGL_BUILD_DISAGGREGATED (so MG_Remote and the transport parser do not exist and # `MOBILEGL_TRANSPORT=inproc` would be accepted and ignored), and build-linux-split configures # without MOBILEGL_PIPE_VERIFY (so the comparator does not exist). The integration-verify-split # entries are registered only when BOTH options are on, so `--no-tests=error` below reds a build # that lost either one instead of reporting a green run of nothing. Built and run in one job: # nothing downstream consumes this library. # # INPROC ONLY, BY THE INSTRUMENT'S NATURE: the comparator's two arms (the applier-written # gPipeInputs and the frontend GLContext) are co-resident only when both roles share a process. # MG_IntegrationTest/CMakeLists.txt's verify-split block records the spawn measurement. integration-verify-split: needs: select-runner runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} timeout-minutes: 180 permissions: actions: write contents: read env: BUILD_DIR: build-verify-split CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "true" CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 4G CCACHE_NOHASHDIR: "true" steps: - name: Set swap space if: runner.environment != 'self-hosted' uses: pierotofy/set-swap-space@v1.0 with: swap-size-gb: 32 - name: Checkout repo uses: actions/checkout@v6 with: # recursive for build-linux-split's reason: MOBILEGL_BUILD_DISAGGREGATED shadows itself # back to OFF when 3rdparty/flatbuffers/include is missing. submodules: recursive - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake if: runner.environment != 'self-hosted' uses: lukka/get-cmake@v4.3.3 - name: Restore ccache uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1 restore-keys: | ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache- - name: Prepare Vulkan SDK if: runner.environment != 'self-hosted' uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Update glslang external sources working-directory: 3rdparty/glslang run: python update_glslang_sources.py - name: Install build and runtime dependencies if: runner.environment != 'self-hosted' run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1 - name: Configure CMake # Release/INFO for build-linux-verify's reason, and INFO is load-bearing here for # build-linux-split's: ConfigLoader's resolved-transport line is logged at INFO and the # arm-proof step below reads it back out of every entry's client log. run: | cmake -S . -B "${BUILD_DIR}" -G Ninja \ -DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \ -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_BUILD_TYPE=Release \ -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=ON \ -DMOBILEGL_BUILD_BENCHMARK=OFF \ -DMOBILEGL_BUILD_INTEGRATION_TEST=ON \ -DMOBILEGL_ITEST_VK_ICD="${VK_ICD_FILENAMES}" \ -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \ -DMOBILEGL_PIPE_PUSH=ON \ -DMOBILEGL_BUILD_DISAGGREGATED=ON \ -DMOBILEGL_BUILD_DISAGGREGATED_INPROC=ON \ -DMOBILEGL_PIPE_VERIFY=ON \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 - name: Build run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)" # Both options, both directions of "did it take" - the two jobs this one is the product of # each check one of them, and this library has to pass both checks. - name: Comparator and MG_Remote symbols run: | test -f "${BUILD_DIR}/libMobileGL.so" defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l) if [ "${defined}" -lt 1000 ]; then echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the checks below could not have failed honestly" exit 1 fi if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeVerifyInputs"; then echo "::error::libMobileGL.so defines no MGPipeVerifyInputs: -DMOBILEGL_PIPE_VERIFY=ON did not take, and the verify-split lane would compare nothing" exit 1 fi remote=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -c -i "MG_Remote" || true) if [ "${remote}" -lt 1 ]; then echo "::error::libMobileGL.so defines no MG_Remote symbol: -DMOBILEGL_BUILD_DISAGGREGATED=ON did not take, and every entry would run monolith under a name that says split" exit 1 fi echo "libMobileGL.so defines MGPipeVerifyInputs and ${remote} MG_Remote symbol(s) (${defined} defined symbols)" - name: Show ccache stats if: ${{ always() && needs.select-runner.result == 'success' }} run: ccache --show-stats - name: Release superseded ccache entry if: github.ref_name == github.event.repository.default_branch env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1 run: gh cache delete "${CACHE_KEY}" || true - name: Save ccache if: github.ref_name == github.event.repository.default_branch continue-on-error: true uses: actions/cache/save@v5 with: path: .ccache key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1 # The lane. `^...$` because ctest -L is a regex (ID-131) and `integration-verify` would # otherwise also match this label and the monolith one would match both. - name: Verify-split scenarios (inproc) working-directory: build-verify-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" run: | ulimit -c unlimited if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi # --output-junit, because the arm proof below has to tell a gtest SKIP from a pass: an # entry that skipped reached no verb honestly, and the skip set moves with the driver. ctest --output-on-failure -L '^integration-verify-split$' --no-tests=error -j 4 \ --output-junit "${RUNNER_TEMP}/verify-split.xml" # THE ARM PROOF, PER ENTRY AND BY NAME. Every ambient entry has a private log path # (SplitLogPaths.cmake.in), so this - unlike integration-verify's grep over a shared log - # is a census: # * ConfigLoader's `MOBILEGL_TRANSPORT=inproc` line in EVERY client log: an entry that ran # monolith would have passed every assertion above and fails here by name; # * `MGPipe: verify armed` in both VerifySplitArming. logs; # * and then the census proper. This used to be a FLOOR - "at least 850 armed of at least # 1000 logs" - which leaves the remaining ~155 unnamed, so the lane could lose fifty # arms, gain fifty skips, and the floor would not move. It is now the two-sided rule # dualblock-expected-fatals.txt already rides: each client log must ARM, or belong to an # entry ctest reported SKIPPED, or be NAMED in Harness/verify-split-unarmed-expected.txt # (56 names in five classes, each class saying why it reaches no verb). A listed entry # that arms, and a listed name with no log at all, both fail - an exception set that # keeps rows nothing needs is how a proof rots green. - name: Verify-split per-entry arm proof working-directory: build-verify-split/MobileGL/MG_IntegrationTest run: | shopt -s nullglob logs=(verify-split-logs/*.client.log) if [ ${#logs[@]} -lt 1000 ]; then echo "::error::found ${#logs[@]} per-entry client logs under verify-split-logs (expected ~1064): the private log paths did not take, so nothing below is a per-entry proof" exit 1 fi missing=$(grep -L 'Config: MOBILEGL_TRANSPORT=inproc' "${logs[@]}" | wc -l) if [ "${missing}" -ne 0 ]; then grep -L 'Config: MOBILEGL_TRANSPORT=inproc' "${logs[@]}" | head -20 echo "::error::${missing} verify-split entries carry no resolved MOBILEGL_TRANSPORT=inproc line: they did not run on the split arm" exit 1 fi for backend in DirectGLES DirectVulkan; do if ! grep -q "MGPipe: verify armed" "pipe-verify-split-arming-${backend}.client.log"; then echo "::error::pipe-verify-split-arming-${backend}.client.log carries no arming line" exit 1 fi done python3 "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \ verify-split-arming verify-split-logs "${RUNNER_TEMP}/verify-split.xml" \ "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/verify-split-unarmed-expected.txt" # THE GREEN LANE'S LOGS, KEPT BEFORE THE CONTROLS OVERWRITE THEM. Each control step below # re-runs ambient VerifySplit. entries with a knob exported, and an ambient entry writes to # ITS OWN per-entry path under verify-split-logs/ - which the library opens "w". So by the # time the upload step at the end of this job runs, twelve of those files are the CONTROL # runs' logs sitting under the LANE's names: an artifact showing deliberate divergences, # with the green run they were meant to be evidence of already gone. `if: always()` on the # upload made that worse rather than better, because a failure investigation is exactly # when the overwritten half is the half somebody needs. - name: Keep lane logs before controls if: ${{ always() && needs.select-runner.result == 'success' }} working-directory: build-verify-split/MobileGL/MG_IntegrationTest run: | rm -rf verify-split-logs-lane cp -a verify-split-logs verify-split-logs-lane cp -p "${RUNNER_TEMP}/verify-split.xml" verify-split-logs-lane/ || true echo "kept $(ls verify-split-logs-lane | wc -l) file(s) from the green lane run" # NEGATIVE CONTROL A ON THE SPLIT ARM - integration-verify's step A, same knob, same shape # (this step passes when ctest FAILS), aimed at the ambient VerifySplit. entries of BOTH # backends. Measured at landing: 10 of 10 red, each with # Fatal{PipeVerifyDiffer, "GetRenderStateParameters@Clear"|"@DrawArrays", ..., where=entry}. - name: Negative control A (corrupted snapshot) working-directory: build-verify-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_PIPE_VERIFY_CORRUPT: GetRenderStateParameters run: | FILTER='(DirectGLES|DirectVulkan)\.VerifySplit\..*ClearThenReadPixels' matched=$(ctest -N -L '^integration-verify-split$' -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:') if [ "${matched}" -lt 2 ]; then echo "::error::split negative control A selected ${matched} tests; its filter no longer matches both backends" exit 1 fi if ctest --output-on-failure -L '^integration-verify-split$' -R "${FILTER}" --no-tests=error; then echo "::error::MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left ${matched} verify-split entries GREEN. The comparator is not comparing on the split arm." exit 1 fi echo "the corrupted field turned ${matched} selected split entries red, as it must" # NEGATIVE CONTROL A ON THE OTHER COMPARATOR (P7 wave 3, V1). The step above proves the # ENTRY compare - client thread, verb boundary, `where=entry`. The compare-at-read hook runs # on the SERVER's apply thread at every backend read and is the only comparator that arm ever # runs, and until MG_Impl/Pipe/PipeFill.cpp's hook honoured MOBILEGL_PIPE_VERIFY_CORRUPT it # had no control at all: a hook that had stopped comparing was indistinguishable from a hook # with nothing to report. # # NOT THE "ctest MUST FAIL" SHAPE, and the reason is mechanical rather than a preference: # GetPixelStoreParameters is in kReadback's fill mask, so with MOBILEGL_PIPE_VERIFY_FATAL at # its default the client's entry compare aborts the process at the ReadPixels boundary and # the server never reaches the read. The entries therefore pin FATAL=0, survive their own # divergence, and each asserts from inside the process that the SERVER role's half of its # private log carries the report. This step re-reads those logs from outside, so the claim # does not rest only on the assertion that produced them. # # AND IT COUNTS, NEEDING TWO (V1 fix round 2). The server reads the pack field twice inside # the ReadPixels window - PipeApplier::read_pixels saves the application's pack before it # installs the neutral one, the backend's ReadPixels reads it after - and the hook perturbs # its oracle once per read, because the ID-49 overwrite inside the window wipes the first # perturbation before the second read's compare. A step satisfied by ONE line was satisfied # by either perturbation alone and falsified neither. Measured at landing: 3 (DirectGLES, # whose backend reads the field twice after the install) / 2 (DirectVulkan); without the # re-applied perturbation both drop to 1, without the first one DirectVulkan drops to 1 # while DirectGLES stays at 2 - so BOTH backends are required, and the first perturbation's # falsifier is the DirectVulkan entry. # # THE GREEN HALF is the lane run above: the same hook, the same window, with no corruption # armed, across 1070 entries at zero divergences. - name: Negative control A (server-side read) working-directory: build-verify-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" run: | FILTER='(DirectGLES|DirectVulkan)\.VerifySplitReadCorrupted\.' matched=$(ctest -N -L '^integration-verify-split$' -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:') if [ "${matched}" -lt 2 ]; then echo "::error::split read-side control A selected ${matched} tests; its filter no longer matches both backends" exit 1 fi ctest --output-on-failure -L '^integration-verify-split$' -R "${FILTER}" --no-tests=error for backend in DirectGLES DirectVulkan; do log="MobileGL/MG_IntegrationTest/pipe-verify-split-read-corrupt-${backend}.server.log" reads=$(grep -c 'Fatal{PipeVerifyDiffer, "GetPixelStoreParameters@ReadPixels".*where=read}' "${log}" 2>/dev/null || true) reads=${reads:-0} if [ "${reads}" -lt 2 ]; then echo "::error::${log} carries ${reads} compare-at-read report(s) for the corrupted pack field, fewer than the two reads the server makes inside the ReadPixels window (the applier's saved-pack read and the backend's post-install read): the hook is not comparing on the server's apply thread, or one of its two perturbations no longer reaches its read because the ID-49 neutral-pack overwrite swallowed it" exit 1 fi done echo "both backends reported GetPixelStoreParameters@ReadPixels where=read at least twice in the SERVER half, as they must" # NEGATIVE CONTROL B ON THE SPLIT ARM, WITH THE SPLIT ARM'S PAIR. The monolith pair # (GenerateMipmap:GetActiveTextureUnit) is inert here - the split backend's generate_mipmap # resolves its texture by the verb's handle and never reads the active unit - so this step # withholds ReadPixels:GetPixelStoreParameters, which the server reads under its own stamp. # Measured at landing: 2 of 2 red, each child dying with # Fatal{UnmigratedPipeInput, "GetPixelStoreParameters@ReadPixels"} on mgl-srv-apply. - name: Negative control B (omitted stamp) working-directory: build-verify-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_PIPE_POISON_OMIT: ReadPixels:GetPixelStoreParameters run: | FILTER='(DirectGLES|DirectVulkan)\.VerifySplit\.PoisonOmissionScenario\.WithoutOmissionCompletes' matched=$(ctest -N -L '^integration-verify-split$' -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:') if [ "${matched}" -lt 2 ]; then echo "::error::split negative control B selected ${matched} tests; its filter no longer matches both backends" exit 1 fi if ctest --output-on-failure -L '^integration-verify-split$' -R "${FILTER}" --no-tests=error; then echo "::error::MOBILEGL_PIPE_POISON_OMIT=ReadPixels:GetPixelStoreParameters left the verify-split lane GREEN. The server-side poison is not armed on the split arm." exit 1 fi echo "the omitted stamp turned the split lane red, as it must" # BOTH SETS, AND THEY ARE DIFFERENT EVIDENCE. verify-split-logs-lane/ is the green run the # arm proof read; verify-split-logs/ is whatever the control steps left behind, which for # twelve entries is a deliberately red re-run under the same name. Uploading only the # second - which is what this step did before the copy above existed - loses the first. # The run log needs a signed-in reader; annotations do not. Name every failed ctest entry, # with the lines of its output that say why, where anyone reading the run can see them. - name: Annotate failed entries if: failure() run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}" - name: Upload verify-split lane logs if: ${{ always() && needs.select-runner.result == 'success' }} uses: actions/upload-artifact@v7 with: name: integration-verify-split-logs path: | build-verify-split/MobileGL/MG_IntegrationTest/verify-split-logs-lane/** build-verify-split/MobileGL/MG_IntegrationTest/verify-split-logs/** build-verify-split/MobileGL/MG_IntegrationTest/pipe-*split*.log* if-no-files-found: warn - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: integration-verify-split-core-dumps path: /tmp/core.* if-no-files-found: ignore # P5's split build. The verify pair cloned, because the problem is the same one: a third # configuration of the same sources whose whole value depends on the option having taken. # # This dedicated build keeps the protocol/strict controls independent of the branch's # primary runtime mode. MOBILEGL_BUILD_DISAGGREGATED implies MOBILEGL_PIPE_PUSH (the split path decodes # into the MGPipe applier and PIPE_PUSH is what compiles the applier), so the split arm needs a # build of its own exactly as the verify arm does. # # THE `nm` STEP IS THE POINT OF THIS JOB. In a build without the option, MG_Remote is not # compiled at all and MOBILEGL_TRANSPORT's parser does not exist - so the variable is accepted by # the environment and silently ignored (CONTRACT-P5 5). Every downstream lane would then run # monolith and go green under a name that says split. CMake will not complain about a typo'd -D, # so the build-level assertion is the only guard, and its absence is precisely what would make # the whole split arm meaningless. build-linux-split: needs: select-runner runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} timeout-minutes: 120 permissions: actions: write contents: read env: BUILD_DIR: build-split CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "true" CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 4G CCACHE_NOHASHDIR: "true" steps: - name: Set swap space if: runner.environment != 'self-hosted' uses: pierotofy/set-swap-space@v1.0 with: swap-size-gb: 32 - name: Checkout repo uses: actions/checkout@v6 with: # recursive, and load-bearing here beyond the usual: MOBILEGL_BUILD_DISAGGREGATED # SHADOWS ITSELF BACK TO OFF when 3rdparty/flatbuffers/include is missing # (CMakeLists.txt:471-486, a normal variable rather than a cache force, deliberately). # A shallow checkout would therefore configure cleanly, build a monolith library, and # be caught only by the nm step below. submodules: recursive - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake if: runner.environment != 'self-hosted' uses: lukka/get-cmake@v4.3.3 - name: Restore ccache uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1 restore-keys: | ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache- - name: Prepare Vulkan SDK if: runner.environment != 'self-hosted' uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Update glslang external sources working-directory: 3rdparty/glslang run: python update_glslang_sources.py - name: Install build dependencies if: runner.environment != 'self-hosted' run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build - name: Show installed toolchain run: | ccache --version "${MOBILEGL_CI_CC}" --version "${MOBILEGL_CI_CXX}" --version dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true - name: Configure CMake # Release/INFO like the shipped build, for the same reason build-linux-verify gives. # INFO specifically matters here: ConfigLoader logs the resolved transport at INFO and # that line is what run_trace_case.cmake and the integration-split lane read back as # proof the transport resolved in THIS process. # # _INPROC implies _DISAGGREGATED implies _PIPE_PUSH; all three are passed anyway, because # an implication that is asserted in two places is an implication nobody has to remember. run: | cmake -S . -B "${BUILD_DIR}" -G Ninja \ -DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \ -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_BUILD_TYPE=Release \ -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=ON \ -DMOBILEGL_BUILD_BENCHMARK=OFF \ -DMOBILEGL_BUILD_INTEGRATION_TEST=ON \ -DMOBILEGL_ITEST_VK_ICD="${VK_ICD_FILENAMES}" \ -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \ -DMOBILEGL_PIPE_PUSH=ON \ -DMOBILEGL_BUILD_DISAGGREGATED=ON \ -DMOBILEGL_BUILD_DISAGGREGATED_INPROC=ON \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 - name: Build run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)" # `nm` and NOT `nm -D`, for the reason build-linux-verify spells out: everything under # MG_Remote is a plain namespace symbol in a CXX_VISIBILITY_PRESET=hidden Release build and # none of it reaches the dynamic table. The symbol count guards the remaining hole - a # stripped library would make the grep fail for a third, silent reason. # - name: MG_Remote symbols in split library run: | test -f "${BUILD_DIR}/libMobileGL.so" defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l) if [ "${defined}" -lt 1000 ]; then echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the checks below could not have failed honestly" exit 1 fi remote=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -c -i "MG_Remote" || true) if [ "${remote}" -lt 1 ]; then echo "::error::libMobileGL.so defines no MG_Remote symbol: -DMOBILEGL_BUILD_DISAGGREGATED=ON did not take (a typo'd -D is not a CMake error, and the option shadows itself OFF when 3rdparty/flatbuffers/include is missing). Every lane that consumes this artifact would run MONOLITH while claiming to run split, because the MOBILEGL_TRANSPORT parser does not exist in such a build and the variable is accepted and ignored." exit 1 fi # The transport parser itself, which is the symbol the runtime evidence depends on. if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeApply"; then echo "::error::libMobileGL.so defines no MGPipeApply* entry point, so MOBILEGL_PIPE_PUSH did not take either and there is no applier for the split path to decode into" exit 1 fi echo "libMobileGL.so defines ${remote} MG_Remote symbol(s) and the MGPipe applier (${defined} defined symbols)" # The link-closure ratchet (CONTRACT-P6.md 12.1, ID-P7-7): the set of frontend symbols a # server-only image would still have to link, recomputed from the per-object partition in # scripts/link_ratchet.py and compared against the stored LIST. # # THIS job and not `build-linux`: build-linux passes the same three options from # ${MOBILEGL_CI_DISAGGREGATED}, which is ON only on feat/disaggregated and # codex/fix-github-ci, so on every other branch it builds a monolith - MG_Remote would not # be compiled at all, the SERVER half of the partition would be empty, and the gate would # go green having measured nothing. This job turns the three options on unconditionally, # and the nm step immediately above has just proved they took. The script re-reads them # from CMakeCache.txt anyway and refuses the run if they are not all ON. # # Only NEW symbols fail. A symbol that has DISAPPEARED is the ratchet turning and is # printed with an instruction to re-baseline in the same commit - the baseline stores the # list rather than a count precisely so that clearing one symbol while adding another does # not walk through at an unchanged total (ID-P7-7). # # It reads the per-object output of the MobileGL target, so it must run before anything # that cleans the build tree; the packaging step below excludes *.o from the tarball but # does not delete them. - name: Server link-closure ratchet run: | mkdir -p ci-artifacts python3 scripts/link_ratchet.py --self-test python3 scripts/link_ratchet.py \ --build-dir "${BUILD_DIR}" \ --baseline scripts/data/link_ratchet_baseline.txt \ --assert-monotone \ --bucket \ --json ci-artifacts/link-ratchet.json # Uploaded even when the gate fired: a red ratchet is exactly the moment someone needs the # per-symbol definer/referrer attribution the JSON carries, and a job that failed before # uploading its evidence is a job nobody can act on. - name: Upload link ratchet if: ${{ always() && needs.select-runner.result == 'success' }} uses: actions/upload-artifact@v7 with: name: link-ratchet path: ci-artifacts/link-ratchet.json if-no-files-found: warn - name: Show ccache stats if: ${{ always() && needs.select-runner.result == 'success' }} run: ccache --show-stats - name: Release superseded ccache entry if: github.ref_name == github.event.repository.default_branch env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1 run: gh cache delete "${CACHE_KEY}" || true - name: Save ccache if: github.ref_name == github.event.repository.default_branch continue-on-error: true uses: actions/cache/save@v5 with: path: .ccache key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1 - name: Package Linux split runtime run: | mkdir -p ci-artifacts mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort) tar \ --exclude='*/CMakeFiles' \ --exclude='*.o' \ --exclude='*.a' \ --exclude='*.ninja*' \ --exclude='build.ninja' \ --exclude='cmake_install.cmake' \ -czf ci-artifacts/mobilegl-linux-runtime-split.tgz \ "${BUILD_DIR}/CTestTestfile.cmake" \ "${BUILD_DIR}/MobileGL/MG_Test" \ "${BUILD_DIR}/MobileGL/MG_IntegrationTest" \ "${SHARED_LIBS[@]}" - name: Upload Linux split runtime uses: actions/upload-artifact@v7 with: name: mobilegl-linux-runtime-split path: ci-artifacts/mobilegl-linux-runtime-split.tgz if-no-files-found: error # The split lane itself: the three-arm shape ARCHITECTURE.md:521 asks for verbatim, plus the # first real run of the five MG_Test/Wire suites, plus P5's two exit-gate negative controls. integration-split: runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} timeout-minutes: 120 needs: [select-runner, build-linux-split] steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake if: runner.environment != 'self-hosted' uses: lukka/get-cmake@v4.3.3 - name: Install runtime dependencies if: runner.environment != 'self-hosted' run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers - name: Download Linux split runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime-split path: . - name: Unpack Linux split runtime run: | tar -xzf mobilegl-linux-runtime-split.tgz test -f build-split/libMobileGL.so - name: Normalize CTest command paths run: | python - <<'PY' from pathlib import Path import re for path in Path('build-split').rglob('CTestTestfile.cmake'): text = path.read_text() text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text) path.write_text(text) PY # THE FIRST REAL RUN OF MG_Test/Wire. Those five suites - Framing, Ring, InProcessTransport, # ProtocolSmoke and FdPassing - are registered only under MOBILEGL_BUILD_DISAGGREGATED # (MG_Test/CMakeLists.txt), and before this job existed NO cmake invocation anywhere in this # workflow passed that option. They had never been compiled by CI, let alone run. # Exit gate 9.2. BEFORE the lanes run, because it costs a `ctest -N` and answers the one # question a green lane cannot: did the spawn arm cover the same cases the split arm did? # A spawn lane registered with fewer cases goes green while proving less, and nothing about # a smaller green lane looks wrong. - name: Spawn/split lane parity (9.2) run: python3 scripts/ci/spawn_lane_parity.py build-split # P8-A (ID-P8-2): every monolith case, tails normalised, is on a gated split arm or is named # with a reason class in scripts/data/split_coverage_exemptions.txt. The same `ctest -N` # cost as the parity step above, and the question it cannot answer: parity compares the # split arms with EACH OTHER, so a case missing from all three passes it. - name: Split coverage against monolith (P8-A) run: | python3 scripts/ci/split_coverage.py --self-test python3 scripts/ci/split_coverage.py build-split # P8-SE: PipeCatalogueTest's unmigrated-emulation list equals the MGPipeUnmigratedEmulation # call sites in the tree, one site per name. A source lint: no build needed. - name: Unmigrated-emulation list against its call sites (P8-SE) run: | python3 scripts/ci/unmigrated_emulation_sites.py --self-test python3 scripts/ci/unmigrated_emulation_sites.py - name: Unit tests on the split runtime working-directory: build-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit-split.xml" # P7 wave 2-F, PH-7 (1). THE TCP SUPERVISOR LANE NEEDS flatc, AND CI HAD NONE. # # `TcpLane.SupervisorProtocolControls` carries LABELS integration-tcp, so it has been in the # selection below since wave 0 - as a SKIP, every run. The script builds the protocol's # Python bindings with flatc, flatc is deliberately absent from the build graph # (gen_protocol.py's header says why), and with none found it exits 77 and ctest reports # the entry skipped. Skipped is what the entry looked like for the whole time it did not # exist, which is the failure mode the plan's own row named ("referenced by NOTHING"). # # So the job builds the SAME pinned flatc that flatc-check builds, from the same submodule, # and the lane step below names the entry to junit_tally.py --require-entry-passed. If the # build ever stops producing one, the lane reds by name rather than going green having run # the supervisor's Busy / token / listen / version / build-policy / reap controls zero times. # --check is passed so this step cannot quietly regenerate a different committed header; # keeping that honest is flatc-check's job and this step must not overlap with it. # P7 F2: `TcpLane.ControlFrameFuzz` (Ph fuzz arm 1, scripts/ci/ph_fuzz_control_frames.py) # needs the same flatc and is required PASSED by name on the same junit_tally line. - name: Check out FlatBuffers (TCP lane) run: git submodule update --init 3rdparty/flatbuffers - name: Build pinned flatc (TCP lane) run: python3 scripts/gen_protocol.py --build-dir "${RUNNER_TEMP}/flatc-build" --check # --no-tests=error is half the gate, exactly as in integration-verify: the integration-split # entries exist only when the library was configured with -DMOBILEGL_BUILD_DISAGGREGATED=ON, # so a build that lost the option matches nothing and reds here instead of reporting a green # run of nothing. The other half is the transport-resolution check below. # # An entry whose owning package (c1 client, s1 session, v1 server) has not landed SKIPS with # the reason and never goes green - MG_IntegrationTest/CMakeLists.txt probes MG_Remote for # c0's signature stubs and disarms the lane while any remain. - name: Split lanes (inproc, spawn, TCP) working-directory: build-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" run: | ulimit -c unlimited if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi export MOBILEGL_FLATC_EXECUTABLE="${RUNNER_TEMP}/flatc-build/flatc" test -x "${MOBILEGL_FLATC_EXECUTABLE}" ctest --output-on-failure -L integration-split --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/split-baseline.xml" python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/split-baseline.xml" --require-split-ran # Exit gate 9.3: the spawn arm green at the same count. It runs the same cases through a # SECOND PROCESS, so a failure here that the line above did not see is a transport fact # and not a phase one - which is the whole reason both lanes exist. ctest --output-on-failure -L integration-spawn --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/spawn-baseline.xml" # PH-6 / fuzz arm 3: the supervisor-level forfeit control rides this lane and must RUN. python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/spawn-baseline.xml" --require-spawn-ran --require-entry-passed SpawnLane.EventForfeitPeer ctest --show-only=json-v1 -L '^integration-tcp$' > "${RUNNER_TEMP}/tcp-discovery.json" ctest --output-on-failure -L '^integration-tcp$' --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/tcp-baseline.xml" python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/tcp-baseline.xml" --require-tcp-ran --require-run-ahead --discovery "${RUNNER_TEMP}/tcp-discovery.json" python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/tcp-baseline.xml" --require-entry-passed TcpLane.SupervisorProtocolControls --require-entry-passed TcpLane.EventForfeitPeer --require-entry-passed TcpLane.ControlFrameFuzz # Both full transport runs are hard gates. The per-case comparison below # remains diagnostic because explicit mechanism controls retain their own mode. # # The entries that name MOBILEGL_TRANSPORT in their OWN property (the Split. lanes) keep # their value in both passes, which is correct: they are the split family in both arms and # the comparison is about the other 1100. - name: Transport A/B (monolith vs inproc) working-directory: build-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH: "1" MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS: "1" MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER: "1" run: | ulimit -c unlimited if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi count=$(ctest -N -L integration-gpu | grep -cE '^ *Test *#[0-9]+:') if [ "${count}" -lt 1 ]; then echo "::error::the split runtime registers ${count} integration-gpu entries" exit 1 fi echo "integration-gpu entries in the split build: ${count}" MOBILEGL_TRANSPORT=monolith ctest --output-on-failure -L integration-gpu --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/arm-monolith.xml" MOBILEGL_TRANSPORT=inproc MOBILEGL_IPC_ROLE_SPLIT_STATE=1 MOBILEGL_IPC_STRICT_ERRORS=1 ctest --output-on-failure -L integration-gpu --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/arm-inproc.xml" # Both commands above must succeed; report differences among legitimate skips. python3 - "${RUNNER_TEMP}/arm-monolith.xml" "${RUNNER_TEMP}/arm-inproc.xml" <<'PY' import sys, xml.etree.ElementTree as ET def rows(path): out = {} for case in ET.parse(path).getroot().iter('testcase'): status = 'passed' if case.find('failure') is not None or case.find('error') is not None: status = 'failed' elif case.find('skipped') is not None or case.get('status') in ('notrun', 'disabled'): status = 'skipped' out[case.get('name')] = status return out a, b = rows(sys.argv[1]), rows(sys.argv[2]) diff = sorted(set(a) ^ set(b)) + sorted(n for n in set(a) & set(b) if a[n] != b[n]) if diff: for name in diff[:40]: print(f"{name}: monolith={a.get(name, '')} inproc={b.get(name, '')}") print(f"Transport comparison: {len(diff)} name/status differences after both full runs passed") else: print(f"the two arms agree on all {len(a)} entries, name and status") PY # THE RUNTIME HALF OF "THIS IS REALLY A SPLIT BUILD". The build-level nm check in # build-linux-split proves the library CARRIES MG_Remote; this proves the transport # RESOLVED in a process of this lane. ConfigLoader::InitTransport logs one INFO line when it # selects InProcess, and the DirectGLES.Split.PersistentMapArm. entry retains its private # counting log (all Split entries now have private paths; nothing else writes this file). # The line is written during bring-up, before any scenario decides to skip, so this # check is live from the day the lanes land rather than from the day they stop skipping. - name: Transport resolution check working-directory: build-split run: | # P6: the sink writes one file per ROLE from the base MOBILEGL_LOG_FILE_PATH; the # transport-resolution line is ConfigLoader's, on the CLIENT thread, so it is in the # .client half. The base name itself names no file now. log=MobileGL/MG_IntegrationTest/persistent-map-arm-split-DirectGLES.client.log if [ ! -f "${log}" ]; then echo "::error::${log} does not exist: the DirectGLES.Split.PersistentMapArm. entry never ran (or the per-role split renamed it), so nothing in this job establishes that MOBILEGL_TRANSPORT ever resolved to inproc in a live process" exit 1 fi # THE DISTINCTIVE PART OF THE INFO LINE, not the bare KEY=VALUE (review finding M-5): # ConfigLoader logs `Config: Accepted env variable: MOBILEGL_TRANSPORT=inproc` for the # env dump too, unconditionally and in a PULL build, and at DEBUG that line is live. if ! grep -q "MOBILEGL_TRANSPORT=inproc - the MGPipe record stream" "${log}"; then echo "::error::${log} carries no transport-resolution line. ConfigLoader::InitTransport logs it at INFO when it selects InProcess, and that code exists only in a MOBILEGL_BUILD_DISAGGREGATED build - so this lane ran a monolith library while claiming to be the split lane. (A bare MOBILEGL_TRANSPORT=inproc substring is NOT accepted: the env dump prints one in every build.)" exit 1 fi echo "the split lane resolved MOBILEGL_TRANSPORT=inproc" # EXIT GATE E1's NEGATIVE CONTROL and EXIT GATE E3(a)'s, in one step because they have the # same two-state shape and the same reason for it. # # Both controls turn a knob that MUST make a split scenario red: MOBILEGL_IPC_VERB_BARRIER=0 # removes the lockstep fence R-1 rests on, and MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0 turns the # persistent-map push off. A control of the verify lane's shape - "this step passes when # ctest FAILS" - cannot be written yet, because while packages c1/s1/v1 are landing the # Split entries SKIP and ctest reports green whatever the knob says, so an unconditional # control would be red for the whole of P5 for a reason that is not a defect. # # So the expected state is DERIVED FROM BEHAVIOUR rather than assumed. The first version read # MGITEST_REMOTE_CLIENT_PRESENT out of the generated *_tests.cmake, which was a restatement of # the CMake source probe review finding M-1 falsified; the arming condition is a runtime fact # inside each test process (MG_Config::Transport, ClientSession::Active() and # ImplementedVerbCount(), read by Harness/SplitRuntimePeek), so the only honest way to ask it # from a shell is to look at what the entries DID. When entries passed, the controls MUST # fire; when every one of them skipped, the step says so loudly and does not pretend. # # THE BODY OF THIS STEP IS scripts/ci/split_negative_controls.sh, and the move is the point # rather than tidiness. A `run:` block executes nowhere but on a runner, so these lines were # unreviewable and untestable: when the wave-1 cross-family review said they were broken, # CONFIRMING it needed a hand-made copy of them (wave1-codex-verify.md 8), and a copy is not # the thing. scripts/ci/control_smoke_test.sh now drives the very file this step runs. # # What that smoke test pins, and what ID-46 finding 8 found missing: each control asserts its # OWN failure reason. A non-zero ctest exit used to be enough, so a timeout, a setup abort or # any unrelated assertion printed "turned N selected entries red, as it must" and this step # went green. The arming run's `|| true` had the matching defect - it counted a case that ran # and FAILED as evidence the lane was live, so the controls could be measured against a # baseline that was already red. - name: Negative controls (barrier, map push) if: ${{ !cancelled() }} working-directory: build-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" CONTROL_TMPDIR: ${{ runner.temp }} run: bash "${GITHUB_WORKSPACE}/scripts/ci/split_negative_controls.sh" # The run log needs a signed-in reader; annotations do not. Name every failed ctest entry, # with the lines of its output that say why, where anyone reading the run can see them. - name: Annotate failed entries if: failure() run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}" - name: Upload split lane logs if: ${{ always() && needs.select-runner.result == 'success' }} uses: actions/upload-artifact@v7 with: name: integration-split-logs path: | build-split/MobileGL/MG_IntegrationTest/*.log* build-split/MobileGL/MG_IntegrationTest/split-logs/*.log ${{ runner.temp }}/arm-inproc.xml ${{ runner.temp }}/arm-monolith.xml if-no-files-found: warn - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: integration-split-core-dumps path: /tmp/core.* if-no-files-found: ignore # Shares build-linux-split with the functional/transport job above. This job # checks state ownership: strict residual-read rules, distinct client/server # blocks, zero-residual assertions, and complete per-entry result accounting. # Configured controls must pass without skips. Capability gaps and known # limitations are reported separately and accepted only by exact name/reason. integration-split-strict: runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} timeout-minutes: 60 needs: [select-runner, build-linux-split] steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake if: runner.environment != 'self-hosted' uses: lukka/get-cmake@v4.3.3 - name: Install runtime dependencies if: runner.environment != 'self-hosted' run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers vulkan-validationlayers - name: Download Linux split runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime-split path: . - name: Unpack Linux split runtime run: | tar -xzf mobilegl-linux-runtime-split.tgz test -f build-split/libMobileGL.so - name: Normalize CTest command paths run: | python - <<'PY' from pathlib import Path import re for path in Path('build-split').rglob('CTestTestfile.cmake'): text = path.read_text() text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text) path.write_text(text) PY # The green half: no unit case may pull harder under strict. This includes the # RemoteGuards suite (the layer-1/layer-2 Fatal drives) and FieldOwnershipTest's strict # arms, so the guards are provably armed in the lane that gates on them. - name: Unit tests on the split runtime (strict) working-directory: build-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit-strict.xml" # ---- P5e (ra), MG_Remote/CONTRACT-P5E.md §7: ONE STEP, TWO ARMS, AND THE TREE PICKS ---- # # This lane was written as an EXPECTED-RED control: under strict every integration-split # entry aborts on its first BARRIER-PULLED read, so the step gated on the run failing and # on the strict marker being the reason. P5e turns it into a HARD GREEN GATE - once the # client runs ahead, an unbarriered record's pull is a Fatal whatever this knob says # (§3.3), so what strict adds is only the BARRIERED rows' pulls, and those are a short, # named list. A revert of any handle arm in vi/sb/pg/tx2/fb then goes red HERE, by field # and verb, instead of in a picture nobody diffs. # # THE ARM IS CHOSEN BY THE TREE, NOT BY A SECOND EDIT. kMGPipeP5eRunAheadReady in # MG_Backend/Init.cpp is the one constant the integration commit flips, and it is exactly # the fact this lane's expectation depends on: with it false no server publishes # kCapRunAheadApply, every record is barriered, every scenario still aborts on its first # pull and the expected-red control is the honest gate. Reading it here means the lane # changes meaning in the same commit the runtime does - a lane whose flip is a separate # human step is a lane that is wrong for as long as that step is outstanding. - name: Strict split lane working-directory: build-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" MOBILEGL_TRANSPORT: "inproc" run: | ulimit -c unlimited ready=$(grep -oE 'kMGPipeP5eRunAheadReady = (true|false)' "${GITHUB_WORKSPACE}/MobileGL/MG_Backend/Init.cpp" | head -1 | awk '{print $3}') echo "kMGPipeP5eRunAheadReady = ${ready}" helper="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" expected="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/strict-expected-markers.txt" # THE LOG SET COMES FROM ctest, NOT FROM A DIRECTORY (P5e gl, ID-119). This step used to # grep MobileGL/MG_IntegrationTest/split-logs/, which holds 90 of the lane's entries. # The 26 it could not see were the F1. readback block, both NamedBlit pairs, the four # Ct. entries and PersistentMapArm - i.e. PRECISELY the readback population the # allowlist is about, plus the only DirectVulkan entries in the lane. A directory is a # guess about where the lane put its logs; `--show-only=json-v1` is the lane saying so. ctest -L integration-split --show-only=json-v1 > "${RUNNER_TEMP}/lane.json" # Stale logs from an earlier job would be read as this run's evidence: an entry that # never starts leaves the previous file in place, and "no marker" is exactly what a # lane that never ran looks like. python3 - "${RUNNER_TEMP}/lane.json" "${helper}" <<'PY' import importlib.util, json, os, sys spec = importlib.util.spec_from_file_location("slp", sys.argv[2]) slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp) logs = slp.marker_log_paths(json.load(open(sys.argv[1]))) for path in set(logs.values()): try: os.unlink(path) except OSError: pass print(f"reset {len(set(logs.values()))} private log(s) across {len(logs)} entries") PY rc=0 ctest -L integration-split --no-tests=error -j 4 > "${RUNNER_TEMP}/strict-split.txt" 2>&1 || rc=$? tail -5 "${RUNNER_TEMP}/strict-split.txt" echo "strict run: rc=${rc}" python3 "${GITHUB_WORKSPACE}/scripts/gen_pipe_field_ownership.py" --print-admitted \ > "${RUNNER_TEMP}/allowed.txt" echo "§7 allowlist (derived): $(wc -l < "${RUNNER_TEMP}/allowed.txt") admitted pair(s)" if [ "${ready}" != "true" ]; then # ---- the pre-flip arm ---- if [ "${rc}" -eq 0 ]; then echo "::error::integration-split under MOBILEGL_IPC_STRICT_ERRORS=1 went GREEN while kMGPipeP5eRunAheadReady is false - the object-class residual pulls (P3b/P4b/P7) are still pinned, so strict must abort every drawing scenario. If P5e has just landed, flip that constant in the same commit." exit 1 fi # The census, and the ADMITTED half of the ratchet, run in this arm too: the Fatal # half is expected here (it is what "red by design" MEANS), but a debt that appeared # or vanished on the admitted side is news whichever arm the tree is in. python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \ "${RUNNER_TEMP}/allowed.txt" "${expected}" echo "integration-split under strict is red by design, and its marker census above is the evidence" exit 0 fi # ---- the post-flip arm: HARD GREEN, two-sided ---- # # EVERY ENTRY MUST COMPLETE, and the census is what says why when one does not: an abort # is either an unbarriered pull (§3.3's unconditional Fatal, a missed migration) or a # barriered pull nothing admits, and both come out of the census by field and verb. if [ "${rc}" -ne 0 ]; then echo "::error::integration-split under MOBILEGL_IPC_STRICT_ERRORS=1 is RED on a run-ahead server. CONTRACT-P5E §7 makes this lane a hard green gate; the census below names the field and the verb of every entry that aborted." python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \ "${RUNNER_TEMP}/allowed.txt" "${expected}" || true exit 1 fi # THE ADMITTED MARKERS COME FROM THE GENERATOR, NOT FROM THIS FILE (ID-116). # # A hand-kept copy of §7 lived here for one round and was wrong in BOTH directions at # once: it omitted GetFramebufferBindingSlot@ReadPixels - 21 entries, the largest # survivor in the lane - and it carried GetTextureObject@CopyImageSubData, which # nothing made safe, because resource_copy_region is kWaitNone. Neither error is # visible by reading the list; both are obvious the moment the list is DERIVED. # # The rule is ID-84 said in tables that already exist (ID-116, refined by ID-125): a # @ pull is admitted iff the field's ownership row is BARRIER_PULLED, the # verb has a stamp row, the field is inside the verb class's may-read mask, and EITHER # the verb's wire op is statically barriered (kMGPipeWaitClasses != kWaitNone, i.e. the # client really is parked behind the record) OR the field's retiring phase does not # name P5e. The second disjunct is the honest reading of ID-84: this comparison only # ever sees pulls on records the server stamped BARRIERED - an unbarriered one aborts # before any of this - so the pull is legal by construction and the only question left # is whether THIS phase still owes the migration. Without it the lane would go red on # GetTransformFeedbackProgram@DrawArrays, a row CONTRACT-P5E §5.7 rules out of P5e # entirely and no package here is allowed to touch. # `--print-admitted` prints exactly the table MGPipeBarrierPullAdmitted() answers # from, so this shell and the C++ cannot drift: retiring a verb's wait class narrows # the lane with no edit here, and a new BARRIER_PULLED row widens it only on verbs # that wait. # THE RATCHET, BOTH SIDES (ID-119). `no-fatal` is what makes this arm the hard gate: # - any Fatal marker fails; # - an ADMITTED marker outside the generated allowlist fails, which can only mean the # committed PipeFieldOwnership.inc and the generator disagree; # - a pair that appears and is not in strict-expected-markers.txt fails: it is a debt # somebody now owes and it gets written down with the phase that retires it; # - AND a pair in that file that no longer appears fails, so the lane CANNOT ROT # GREEN. Without that half, a retired debt leaves its row behind as a permanent # licence and the next regression to re-introduce the read reads as "expected". # ADMITTED-ESCALATED markers are checked against the expected set but NOT against the # allowlist: they are a runtime fact about the record's payload (an open XFB span, a # draw carrying client vertex arrays) that no table indexed by (field, verb) can carry, # and widening the static list to hold them would forgive the ordinary draw path too. python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \ "${RUNNER_TEMP}/allowed.txt" "${expected}" no-fatal # `rsp`, AND THE PIN THAT REPLACED A VACUOUS ONE (ID-119). What stood here was a grep # for `rsp=` over split-logs/ - a counter no entry in that directory emits, because # PipeStats is off unless a lane asks for it - testing a sentence that was true of every # possible implementation: CountBarrierPull's unbarriered arm is [[noreturn]] and runs # BEFORE ++g_residualPulls, so an unbarriered pull can never reach the counter whatever # the code does. It has been replaced by an assertion inside a lane entry that actually # counts: DirectGLES.Split.StrictArming., which runs with MOBILEGL_PIPE_STATS=1, # MOBILEGL_PIPE_STATS_PERIOD=1 and a private log path of its own (a whole-lane env flip # races under -j, since the library opens that path "w"), and asserts # vbs > 0 - the server stamped a verb boundary on a DRAWING inproc entry, which is # ID-115's positive control: everything strict checks is downstream of # that stamp, and before it existed "the lane is green" and "strict was # never armed" were the same observation; # rsp == 0 or rsp >= draws - the checkable shape of the debt. While the draw path's # pull is unretired every draw pulls, so rsp scales with draws; the # retirement is visible as rsp ceasing to scale. That is what the device # measured (rsp ~= the draw count under inproc, 0 under monolith). # So the assertion lives where the counter is, and this step only has to insist the # entry RAN - a positive control that can be skipped away is not one. if ! grep -q 'DirectGLES\.Split\.StrictArming\.' "${RUNNER_TEMP}/strict-split.txt"; then echo "::error::the strict lane did not run DirectGLES.Split.StrictArming., so nothing in this run proves the server ever stamped a verb boundary. A green lane without it is indistinguishable from a lane where strict was never armed (ID-115)." exit 1 fi echo "integration-split under strict is GREEN, two-sided, with the arming control present" # ---- P5f (f1), P5F-WIRE-COMPLETENESS.md §4: THE DUAL-BLOCK CENSUS LANE --------------- # # The same integration-split entries (plus Magma's own lane), re-run under # These entries own assertions removed from generic transport registrations. # Discovery, execution and zero-skip accounting must agree on the exact set. - name: Specialized configuration controls working-directory: build-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" run: | helper="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" ctest --show-only=json-v1 > "${RUNNER_TEMP}/specialized-all.json" python3 "${helper}" select-specialized "${RUNNER_TEMP}/specialized-all.json" \ "${RUNNER_TEMP}/specialized-selected.json" "${RUNNER_TEMP}/specialized-tests.txt" ctest --tests-from-file "${RUNNER_TEMP}/specialized-tests.txt" --no-tests=error -j 4 \ --output-on-failure --output-junit "${RUNNER_TEMP}/specialized.xml" python3 "${helper}" specialized-results "${RUNNER_TEMP}/specialized-all.json" \ "${RUNNER_TEMP}/specialized.xml" # MOBILEGL_IPC_ROLE_SPLIT_STATE=1: the client's residual fill writes a client-role block # and the backend reads a server-role one, so every BARRIER-PULLED read aborts with a # named Fatal{UnmigratedPipeInput, "@"}. THE RED IS THE CENSUS - it is the # machine-checked list of what still depends on the two roles sharing one object, and it # is ratcheted two-sided against Harness/dualblock-expected-fatals.txt: a new pair is a # debt somebody now owes, a vanished pair must be removed in the commit that retired it. # # THE STEP ALSO FAILS WHEN THE LANE IS GREEN while the expected file still has rows - # "green" would mean the knob stopped arming, which is exactly the decay ID-122 assigns # to a control rather than to a lane's good mood. At P5f's exit the file is empty and the # green arm is the correct one; delete the red expectation in the same commit. # P5f wave 2: with the census empty, this lane is a hard green gate. - name: Dual-block census lane working-directory: build-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_TRANSPORT: "inproc" MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" run: | ulimit -c unlimited helper="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" expected="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/dualblock-expected-fatals.txt" ctest -L 'integration-split|integration-magma-split' --show-only=json-v1 \ > "${RUNNER_TEMP}/dualblock-lane.json" # Stale logs from an earlier step would be read as this run's evidence (ID-119's rule). python3 - "${RUNNER_TEMP}/dualblock-lane.json" "${helper}" <<'PY' import importlib.util, json, os, sys spec = importlib.util.spec_from_file_location("slp", sys.argv[2]) slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp) logs = slp.marker_log_paths(json.load(open(sys.argv[1]))) for path in set(logs.values()): try: os.unlink(path) except OSError: pass print(f"reset {len(set(logs.values()))} private log(s) across {len(logs)} entries") PY rc=0 ctest -L 'integration-split|integration-magma-split' --no-tests=error -j 4 \ --output-junit "${RUNNER_TEMP}/dualblock.xml" \ > "${RUNNER_TEMP}/dualblock-split.txt" 2>&1 || rc=$? tail -5 "${RUNNER_TEMP}/dualblock-split.txt" echo "dual-block census run: rc=${rc}" rows=$(awk '!/^[[:space:]]*(#|$)/ { n++ } END { print n+0 }' "${expected}") if [ "${rows}" -eq 0 ] && [ "${rc}" -ne 0 ]; then echo "::error::empty dual-block census requires ctest success, including infrastructure/execution status" exit "${rc}" fi if [ "${rc}" -eq 0 ] && [ "${rows}" -gt 0 ]; then echo "::error::the dual-block census lane went GREEN with ${rows} expected pair(s) outstanding - the knob stopped arming, or the fields retired and dualblock-expected-fatals.txt was not emptied in the same commit" exit 1 fi python3 "${helper}" expect-fatal "${RUNNER_TEMP}/dualblock-lane.json" \ "${RUNNER_TEMP}/dualblock.xml" "${expected}" # The per-frame zero-residual-read case must execute and pass on both backends. - name: P5f per-frame zero-residual check working-directory: build-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" run: | ctest -L integration-p5f-rsp --show-only=json-v1 > "${RUNNER_TEMP}/rsp-lane.json" ctest -L integration-p5f-rsp --no-tests=error --output-on-failure \ --output-junit "${RUNNER_TEMP}/rsp.xml" python3 "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \ require-green "${RUNNER_TEMP}/rsp-lane.json" "${RUNNER_TEMP}/rsp.xml" \ DirectGLES.Split.P5fRsp.F1WireScenario.EachWireFrameHasZeroResidualPulls \ DirectVulkan.Split.P5fRsp.F1WireScenario.EachWireFrameHasZeroResidualPulls # The mechanism's own negative control (P5F §6): the same named case must be green with # the knob on and red - on its own assertion text - with it off. - name: Dual-block negative control (P5f f1) working-directory: build-split run: bash "${GITHUB_WORKSPACE}/scripts/ci/dualblock_negative_control.sh" # ---- MAGMA'S OWN WIRE LANE: FRAMEBUFFER, TEXTURE AND APPLICATION BUFFERS ------------ # # P5f fm: Magma's framebuffer, copy and mipmap verbs consume server records. # The label remains separate so both backends' coverage stays explicit. - name: Magma split lane (strict, dual-block) working-directory: build-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" MOBILEGL_TRANSPORT: "inproc" run: | ctest -L integration-magma-split --no-tests=error -j 2 --output-on-failure # ---- P7 PACKAGE L: THE SAME MAGMA CASES IN TWO PROCESSES ----------------------------- # # EXIT GATE 1's FIRST HALF, and until this package there was nothing to run: every arm of # the split registration macro hardcoded MOBILEGL_BACKEND_TYPE=DirectGLES, so the shared # `integration-split` / `-spawn` / `-tcp` labels held ZERO DirectVulkan entries and # `integration-magma-split` was inproc by construction. These two lanes are the same # scenarios the step above runs, with the transport as their one difference - so a red # here that the step above did not see is a TWO-PROCESS fact and not a Magma one. # # THEY ARE GATED BECAUSE THEY MEASURED GREEN, and only for that reason: 52 of 52 entries # on each arm, with the two ClipDistance cases skipping for the reason the inproc arm's # allowlist already names. The 21 entries the inproc arm has and these do not are # MagmaRunAheadScenario and MagmaWireCacheScenario, which are inproc-only BY # CONSTRUCTION (they hold and peek THIS process's server loop); the CMakeLists and # spawn_lane_parity.py both name them rather than widening a tolerance. # # THE INFORMATIONAL TIER - `integration-magma-all-{split,spawn,tcp}`, every scenario the # split arms run, on Magma - IS DELIBERATELY NOT HERE. Its job is to be able to go red # while CI stays green; a case moves into a gating step with the commit that retires its # blocker. - name: Magma spawn and TCP lanes working-directory: build-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" run: | ulimit -c unlimited if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi ctest -L '^integration-magma-spawn$' --no-tests=error -j 2 --output-on-failure ctest -L '^integration-magma-tcp$' --no-tests=error -j 2 --output-on-failure # Every named Magma application-buffer wire case must execute and pass. - name: Magma buffer wire cases (inproc) working-directory: build-split run: | ctest -L integration-magma-buffers --show-only=json-v1 > "${RUNNER_TEMP}/magma-buffer-lane.json" ctest -L integration-magma-buffers --no-tests=error -j 2 --output-on-failure \ --output-junit "${RUNNER_TEMP}/magma-buffers.xml" python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \ "${RUNNER_TEMP}/magma-buffer-lane.json" "${RUNNER_TEMP}/magma-buffers.xml" <<'PY' import importlib.util, json, sys spec = importlib.util.spec_from_file_location("slp", sys.argv[1]) slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp) cases = [ "F1WireScenario.EnabledVertexBufferDrawKeepsNamedP7Fatal", "F1WireScenario.UniformBufferRangeAndRebindPixels", "F1WireScenario.ShortUniformBufferRangePadsMissingBytes", "F1WireScenario.SubWordUniformBufferRangePadsMissingBytes", "F1WireScenario.UnalignedAtomicCounterRangeDeclinesAndTheSessionLives", "F1WireScenario.ComputeWrittenVertexAndIndexBuffersDrawAndReadBack", "F1WireScenario.ColorBlitToDefaultIgnoresViewportAndPreservesOrientation", "F1WireScenario.ColorBlitToDefaultHonorsScissorAndReversedRect", "IndexedDrawFamilyScenario.AnElementBufferDrawElementsOffsetSelectsTheRightQuad", "IndexedDrawFamilyScenario.DrawElementsBaseVertexMovesTheSameIndicesToTheOtherQuad", "IndexedDrawFamilyScenario.DrawRangeElementsDrawsTheRangedRun", "IndexedDrawFamilyScenario.DrawElementsInstancedBaseVertexPaintsOneQuadPerInstance", "IndexedDrawFamilyScenario.MultiDrawElementsBaseVertexPaintsEverySubDraw", "IndexedDrawFamilyScenario.MultiDrawElementsIndirectDrawsFromTheIndirectBuffer", "CrossFrameBufferScenario.VertexBufferSubData", "CrossFrameBufferScenario.IndexBufferSubData", "CrossFrameBufferScenario.VertexOrphanAndReupload", "CrossFrameBufferScenario.IndexOrphanAndReupload", "PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw", "PersistentCoherentMapScenario.AWriteAfterAFrameBoundaryReachesTheNextFramesDraw", "StorageBufferRegrowScenario.AGrownStoreIsVisibleThroughItsExistingIndexedBinding", # P11 A2: the >= 16 MiB arena and its tier assertion (emulated). "LargeArenaAdoptionScenario.SubDataAfterAnInFlightDrawReachesTheNextDraw", "LargeArenaAdoptionScenario.RespecifiedVertexArenaKeepsVaoBindings", "LargeArenaAdoptionScenario.RespecifiedIndexArenaKeepsVaoBinding", "LargeArenaAdoptionScenario.ReadbackSeesTheLatestCpuWrite", "LargeArenaAdoptionScenario.GpuWriteIntoTheArenaIsReadBack", "LargeArenaAdoptionScenario.TheArenaLandsInTheTierItsLaneDeclares", ] slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], ["DirectVulkan.Split.Buffers." + name for name in cases]) PY # P7 package L: THE SAME TWENTY-ONE, BY NAME, IN TWO PROCESSES (nineteen at package L; the # sub-word uniform range and the unaligned atomic-counter decline joined later). # # `--no-tests=error` in the lane step above answers "did something run and pass"; it does # not answer "did these twenty-one run". require_green does, and it is the check that would # catch the failure mode this package could plausibly introduce - an arm that registers # SOME of the lane and goes green on a smaller set. The label spelling is what keeps the # three arms apart under `ctest -L`'s regex: the inproc arm keeps the historical # `integration-magma-buffers` (which is what the step above require_greens), and the new # arms put the arm word FIRST, so neither selection can reach the other. # # TcpServer.Start / TcpServer.Stop ARE IN THE tcp ARM'S REQUIRED LIST ON PURPOSE. Every # tcp entry declares FIXTURES_REQUIRED mobilegl-tcp, so ctest lists the supervisor's two # fixture entries under the label as well - and they must be green too: a lane whose # supervisor failed to start has nothing to say about the twenty-one. - name: Magma buffer wire cases (spawn, TCP) working-directory: build-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" run: | for arm in spawn tcp; do case "${arm}" in spawn) prefix=DirectVulkan.Spawn.Buffers. ;; tcp) prefix=DirectVulkan.Tcp.Buffers. ;; esac label="integration-magma-${arm}-buffers" ctest -L "^${label}$" --show-only=json-v1 > "${RUNNER_TEMP}/magma-buffer-${arm}.json" ctest -L "^${label}$" --no-tests=error -j 2 --output-on-failure \ --output-junit "${RUNNER_TEMP}/magma-buffers-${arm}.xml" python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \ "${RUNNER_TEMP}/magma-buffer-${arm}.json" "${RUNNER_TEMP}/magma-buffers-${arm}.xml" \ "${prefix}" "${arm}" <<'PY' import importlib.util, json, sys spec = importlib.util.spec_from_file_location("slp", sys.argv[1]) slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp) cases = [ "F1WireScenario.EnabledVertexBufferDrawKeepsNamedP7Fatal", "F1WireScenario.UniformBufferRangeAndRebindPixels", "F1WireScenario.ShortUniformBufferRangePadsMissingBytes", "F1WireScenario.SubWordUniformBufferRangePadsMissingBytes", "F1WireScenario.UnalignedAtomicCounterRangeDeclinesAndTheSessionLives", "F1WireScenario.ComputeWrittenVertexAndIndexBuffersDrawAndReadBack", "F1WireScenario.ColorBlitToDefaultIgnoresViewportAndPreservesOrientation", "F1WireScenario.ColorBlitToDefaultHonorsScissorAndReversedRect", "IndexedDrawFamilyScenario.AnElementBufferDrawElementsOffsetSelectsTheRightQuad", "IndexedDrawFamilyScenario.DrawElementsBaseVertexMovesTheSameIndicesToTheOtherQuad", "IndexedDrawFamilyScenario.DrawRangeElementsDrawsTheRangedRun", "IndexedDrawFamilyScenario.DrawElementsInstancedBaseVertexPaintsOneQuadPerInstance", "IndexedDrawFamilyScenario.MultiDrawElementsBaseVertexPaintsEverySubDraw", "IndexedDrawFamilyScenario.MultiDrawElementsIndirectDrawsFromTheIndirectBuffer", "CrossFrameBufferScenario.VertexBufferSubData", "CrossFrameBufferScenario.IndexBufferSubData", "CrossFrameBufferScenario.VertexOrphanAndReupload", "CrossFrameBufferScenario.IndexOrphanAndReupload", "PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw", "PersistentCoherentMapScenario.AWriteAfterAFrameBoundaryReachesTheNextFramesDraw", "StorageBufferRegrowScenario.AGrownStoreIsVisibleThroughItsExistingIndexedBinding", "LargeArenaAdoptionScenario.SubDataAfterAnInFlightDrawReachesTheNextDraw", "LargeArenaAdoptionScenario.RespecifiedVertexArenaKeepsVaoBindings", "LargeArenaAdoptionScenario.RespecifiedIndexArenaKeepsVaoBinding", "LargeArenaAdoptionScenario.ReadbackSeesTheLatestCpuWrite", "LargeArenaAdoptionScenario.GpuWriteIntoTheArenaIsReadBack", "LargeArenaAdoptionScenario.TheArenaLandsInTheTierItsLaneDeclares", ] required = [sys.argv[4] + name for name in cases] if sys.argv[5] == "tcp": required += ["TcpServer.Start", "TcpServer.Stop"] slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], required) PY done # Run-ahead must queue real GPU work and enforce credits 1 and 3. - name: Magma run-ahead cases (credits 1, 3) working-directory: build-split run: | ctest -L integration-magma-runahead --show-only=json-v1 > "${RUNNER_TEMP}/magma-runahead-lane.json" ctest -L integration-magma-runahead --no-tests=error -j 2 --output-on-failure \ --output-junit "${RUNNER_TEMP}/magma-runahead.xml" python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \ "${RUNNER_TEMP}/magma-runahead-lane.json" "${RUNNER_TEMP}/magma-runahead.xml" <<'PY' import importlib.util, json, sys spec = importlib.util.spec_from_file_location("slp", sys.argv[1]) slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp) cases = [ "QueuedBufferVersionsAndDeletedNameReuseKeepTheirPixels", "QueuedProgramRebindsKeepEachUniformSnapshot", "QueuedGpuWritesReadBackTheLastDispatchBytes", "MixedNativeAndConvertedGpuVertexStreamsKeepTheirReservation", "TextureUploadCannotOvertakeThePreviouslyQueuedDraw", "GpuClearSurvivesTheFirstStorageImageUpgrade", "PresentCreditActuallyParksTheClientAtItsConfiguredLimit", ] required = [f"DirectVulkan.Split.RunAhead.Credit{credit}.MagmaRunAheadScenario.{name}" for credit in (1, 3) for name in cases] slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], required) PY # Run-ahead off must go red on the lost queue lead; run-ahead back on must be green. - name: Magma run-ahead negative control run: | python3 scripts/ci/magma_runahead_checks.py negative --build-dir build-split \ --out "${RUNNER_TEMP}/magma-runahead-negative" --execute - name: Magma run-ahead sync validation run: | python3 scripts/ci/magma_runahead_checks.py validation --build-dir build-split \ --out "${RUNNER_TEMP}/magma-runahead-syncval" --execute - name: Magma view/framebuffer cache sync validation run: | python3 scripts/ci/magma_cache_checks.py --build-dir build-split \ --out "${RUNNER_TEMP}/magma-cache-syncval" --execute - name: Upload Magma run-ahead and cache evidence if: ${{ always() && needs.select-runner.result == 'success' }} uses: actions/upload-artifact@v7 with: name: magma-runahead-checks path: | ${{ runner.temp }}/magma-runahead-negative ${{ runner.temp }}/magma-runahead-syncval ${{ runner.temp }}/magma-cache-syncval if-no-files-found: ignore # The packed-float chain is generated by a real GPU shader blit, so this entry sits in the # integration-magma-split lane with its siblings (CMakeLists.txt's fm loop) and carries # their pixels assertion. Its named check survives the flip in the only direction that has # evidence behind it: the case must EXECUTE, PASS, and never skip - a lane-level exit code # alone would not separate "generated" from "quietly absent". - name: Magma packed-float mipmap case (P7) working-directory: build-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" MOBILEGL_TRANSPORT: "inproc" run: | p7_case='^DirectVulkan\.Split\.Fm\.F1WireScenario\.GenerateMipmapPackedFloatPixels$' ctest -R "${p7_case}" --show-only=json-v1 > "${RUNNER_TEMP}/magma-p7-lane.json" ctest -R "${p7_case}" --no-tests=error --output-on-failure \ --output-junit "${RUNNER_TEMP}/magma-p7.xml" python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \ "${RUNNER_TEMP}/magma-p7-lane.json" "${RUNNER_TEMP}/magma-p7.xml" <<'PY' import importlib.util, json, sys spec = importlib.util.spec_from_file_location("slp", sys.argv[1]) slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp) name = "DirectVulkan.Split.Fm.F1WireScenario.GenerateMipmapPackedFloatPixels" slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], [name]) PY # Client vertex/index inputs are copied into owned wire buffer resources. # The original dedicated entries now have to execute successfully with run-ahead. - name: Client-array split lane (run-ahead) working-directory: build-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_TRANSPORT: "inproc" MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" MOBILEGL_IPC_RUN_AHEAD: "1" run: | ctest -L integration-clientarrays-split --show-only=json-v1 > "${RUNNER_TEMP}/clientarrays-lane.json" ctest -L integration-clientarrays-split --no-tests=error --output-on-failure -j 1 \ --output-junit "${RUNNER_TEMP}/clientarrays.xml" python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \ "${RUNNER_TEMP}/clientarrays-lane.json" "${RUNNER_TEMP}/clientarrays.xml" <<'PY' import importlib.util, json, sys spec = importlib.util.spec_from_file_location("slp", sys.argv[1]) slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp) lane = json.load(open(sys.argv[2])) names = [test["name"] for test in lane["tests"]] if not names: raise SystemExit("no client-array cases discovered") slp.require_green(lane, sys.argv[3], names) PY # The run log needs a signed-in reader; annotations do not. Name every failed ctest entry, # with the lines of its output that say why, where anyone reading the run can see them. - name: Annotate failed entries if: failure() run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}" - name: Upload strict lane logs if: ${{ always() && needs.select-runner.result == 'success' }} uses: actions/upload-artifact@v7 with: name: integration-split-strict-logs path: | build-split/MobileGL/MG_IntegrationTest/*.log* build-split/MobileGL/MG_IntegrationTest/split-logs/*.log ${{ runner.temp }}/specialized*.json ${{ runner.temp }}/specialized-tests.txt ${{ runner.temp }}/specialized.xml ${{ runner.temp }}/dualblock-lane.json ${{ runner.temp }}/dualblock.xml ${{ runner.temp }}/dualblock-split.txt if-no-files-found: warn # MobileGL/MG_Remote/Protocol/generated/protocol_generated.h is COMMITTED, and # flatc is deliberately absent from the default build graph (a codegen step in # the graph is how the earlier branch ended up cross-compiling an arm64 flatc # and trying to run it on the host). This job is what keeps the committed # header honest: build the pinned flatc, regenerate, and fail on any diff. # It needs no MobileGL build, so it does not depend on build-linux. flatc-check: needs: select-runner runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake if: runner.environment != 'self-hosted' uses: lukka/get-cmake@v4.3.3 - name: Check out the FlatBuffers submodule only # Just this one: the schema check has nothing to do with glslang, # SPIRV-Cross or the trace fixtures. run: git submodule update --init 3rdparty/flatbuffers - name: Regenerate protocol_generated.h run: python3 scripts/gen_protocol.py --build-dir "${{ runner.temp }}/flatc-build" - name: Generated header staleness check run: git diff --exit-code -- MobileGL/MG_Remote/Protocol/generated/protocol_generated.h # P7 wave 2-F, F fix round (PH-7 (4) review). MOBILEGL_PROTOCOL_CONTROL_REVISION is mixed # into wireFingerprint so a control-schema change refuses an old peer by name - but it is a # hand-bumped integer, and nothing tied it to protocol.fbs's content. This step pins # sha256(protocol.fbs) to the revision (scripts/ci/protocol_revision_pins.json): a schema # edit that forgot the bump is red HERE, in the job that regenerates the header on every # such edit, and not a silently absent field between an old server and a new client. Its # self-test proves the check goes red on both perturbations before it says green. - name: Protocol revision pin check run: python3 scripts/ci/protocol_revision_pin.py --self-test # P0.5 interface-purity gate A (ARCHITECTURE.md:501): the two extracted headers' include closure, # asserted on `-H` output because `nm --undefined-only` is blind to "included but not called" - # a header whose types are never named leaves no symbol behind, and "included at all" is exactly # the coupling P1 and P7 have to sever. Needs a preprocessor and three header submodules, no # CMake configure and no glslang sources, so like pipe-gates it does not depend on build-linux. # The script's own --self-test is always on: a negative control that stopped tripping fails the # job, because a gate that cannot go red is not a gate (ROADMAP.md:7). include-graph-check: name: Include-closure purity gate needs: select-runner runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Check out header submodules # ska/flat_hash_map.hpp, xxhash.h and vulkan/vulkan.h are the only submodule headers # Includes.h reaches; glslang and spirv_cross are vendored under include/. run: git submodule update --init include/ska 3rdparty/xxHash 3rdparty/Vulkan-Headers - name: Install clang and X11 headers if: runner.environment != 'self-hosted' # Includes.h defines VK_USE_PLATFORM_XLIB_KHR before , which then # includes ; without libx11-dev every clang-mode probe dies in the # preprocessor and the gate reports 5 problems that have nothing to do with purity. run: sudo apt-get update && sudo apt-get install -y clang-20 libx11-dev - name: Include-closure check and self-test # --expect-probes 4 (contract-v2.md 7.6): an exit code cannot tell four probes from none, # so a --probe typo or a manifest edit that selected nothing would run zero probes and # exit 0 - the second half of the finding that added the flag. The count is the length of # scripts/check_include_closure.py's PROBES list and changing one means changing the other. # # Use the compiler selected for this runner's installed toolchain. run: python3 scripts/check_include_closure.py --mode both --compiler "${MOBILEGL_CI_CXX}" --self-test --require-all --expect-probes 4 benchmark: runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} needs: [select-runner, build-linux] steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake if: runner.environment != 'self-hosted' uses: lukka/get-cmake@v4.3.3 - name: Install runtime dependencies if: runner.environment != 'self-hosted' # libegl-mesa0 is the EGL vendor library itself: DriverBench brings up a # real GL context, and libegl1 is only glvnd's dispatch. It normally # arrives as a Recommends of libegl1, which is too quiet a dependency for # the one job that needs a working driver. run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers - name: Download Linux runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime path: . - name: Unpack Linux runtime run: tar -xzf mobilegl-linux-runtime.tgz - name: Normalize CTest command paths run: | python - <<'PY' from pathlib import Path import re for path in Path('build-linux').rglob('CTestTestfile.cmake'): text = path.read_text() text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text) path.write_text(text) PY - name: Benchmark working-directory: build-linux run: | ulimit -c unlimited if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi ctest -V -C Release -L benchmark --no-tests=error - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: benchmark-core-dumps path: /tmp/core.* if-no-files-found: ignore build-retrace: runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} needs: - select-runner - build-linux - test - benchmark - integration permissions: actions: write contents: read env: BUILD_DIR: build-retrace CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "true" CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 4G CCACHE_NOHASHDIR: "true" MOBILEGL_LIBRARY: ${{ github.workspace }}/build-linux/libMobileGL.so steps: - name: Set swap space if: runner.environment != 'self-hosted' uses: pierotofy/set-swap-space@v1.0 with: swap-size-gb: 32 - name: Checkout repo uses: actions/checkout@v6 with: submodules: recursive - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake if: runner.environment != 'self-hosted' uses: lukka/get-cmake@v4.3.3 - name: Restore ccache uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1 restore-keys: | ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache- - name: Prepare Vulkan SDK if: runner.environment != 'self-hosted' uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Update glslang external sources working-directory: 3rdparty/glslang run: python update_glslang_sources.py - name: Install dependencies if: runner.environment != 'self-hosted' run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build - name: Show installed toolchain run: | ccache --version "${MOBILEGL_CI_CC}" --version "${MOBILEGL_CI_CXX}" --version "${MOBILEGL_CI_LLD}" --version dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true - name: Download Linux runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime path: . - name: Unpack Linux runtime run: | tar -xzf mobilegl-linux-runtime.tgz test -f "${MOBILEGL_LIBRARY}" - name: Configure CMake run: | if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then BUILD_TYPE=Debug else BUILD_TYPE=Release fi cmake -S . -B "${BUILD_DIR}" -G Ninja \ -DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \ -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=OFF \ -DMOBILEGL_BUILD_BENCHMARK=OFF \ -DMOBILEGL_BUILD_TRACE_REPLAY=ON \ -DMOBILEGL_TRACE_SOFTWARE_RENDERING="${MOBILEGL_CI_SOFTWARE_RENDERING}" \ -DMOBILEGL_TRACE_REPLAY_MOBILEGL_LIBRARY="${MOBILEGL_LIBRARY}" \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 - name: Build trace replay run: cmake --build "${BUILD_DIR}" --target mobilegl_trace_replay --parallel "$(nproc)" - name: Show ccache stats if: ${{ always() && needs.select-runner.result == 'success' }} run: ccache --show-stats - name: Release superseded ccache entry if: github.ref_name == github.event.repository.default_branch env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1 run: gh cache delete "${CACHE_KEY}" || true - name: Save ccache if: github.ref_name == github.event.repository.default_branch continue-on-error: true uses: actions/cache/save@v5 with: path: .ccache key: ${{ runner.os }}-${{ needs.select-runner.outputs.pool }}-test-${{ github.job }}-ccache-v1 - name: Normalize CTest command paths run: | python - <<'PY' from pathlib import Path import re for path in Path('build-retrace').rglob('CTestTestfile.cmake'): text = path.read_text() text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text) path.write_text(text) PY - name: Package trace replay run: | mkdir -p ci-artifacts tar -czf ci-artifacts/mobilegl-trace-replay.tgz \ build-retrace/tools/trace_replay/mobilegl_trace_replay \ build-retrace/tools/trace_replay/CTestTestfile.cmake - name: Upload trace replay uses: actions/upload-artifact@v7 with: name: mobilegl-trace-replay path: ci-artifacts/mobilegl-trace-replay.tgz if-no-files-found: error trace-cases: name: trace case matrix runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} env: MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci') && 'ON' || 'OFF' }} needs: - select-runner - test - benchmark - integration outputs: matrix: ${{ steps.trace-cases.outputs.matrix }} names: ${{ steps.trace-cases.outputs.names }} verify-matrix: ${{ steps.trace-cases.outputs.verify-matrix }} split-matrix: ${{ steps.trace-cases.outputs.split-matrix }} steps: - name: Checkout repo uses: actions/checkout@v6 # `dl` / exit gate S7. CHEAP AND EARLY, on a job that only reads the tree: it needs no # build, so a death that names no family is reported in seconds rather than after the # matrix. What it refuses is a std::abort() under MG_Remote/ with no Fatal{Family} marker, # and a family word that appeared without being recorded - the vocabulary drifts on # purpose or not at all (CONTRACT-P6 5.2's D2 ruling). - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Fatal census (S7) run: python3 scripts/ci/fatal_census.py # The decline half of rule I (P7 B3): every WireDeclines.def row has a site, every site a # warning-or-error line. Same shape as the census - tree-only, no build. The self-test # runs first: its fixtures (a site only in a comment or a string, a log in another # branch, an MGLOG_D) must red, so a green audit is one that could have been red. - name: Wire declines audit (B3) run: | python3 scripts/ci/wire_declines_audit.py --self-test python3 scripts/ci/wire_declines_audit.py - name: Load trace cases id: trace-cases run: | python3 tools/trace_replay/trace_cases.py --self-test-transport-matrices matrix_format=github-test-matrix if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then matrix_format=github-test-transport-matrix; fi echo "matrix=$(python3 tools/trace_replay/trace_cases.py --ci --format "${matrix_format}")" >> "$GITHUB_OUTPUT" echo "names=$(python3 tools/trace_replay/trace_cases.py --ci --format names)" >> "$GITHUB_OUTPUT" # The subset the verify build retraces ("verify": true in trace_cases.json). It is a # SUBSET of the matrix above, so retrace-verify needs no fixtures of its own. echo "verify-matrix=$(python3 tools/trace_replay/trace_cases.py --ci --format github-verify-matrix)" >> "$GITHUB_OUTPUT" # The split subset: {backend, case, transport} over inproc AND spawn, both backends, for # every CI case that has not said `"split": false`. Still a SUBSET of the matrix above, # so retrace-split needs no fixtures of its own. P5 shipped it as one case (OpenRA, which # the phase gate names); P6 widened it to the whole CI matrix, because every bug the # spawn work found was a question about the SERVER's process rather than about one trace. # trace_cases.py refuses a `split` case that is not in CI, so the subset stays a subset. SPLIT_MATRIX=$(python3 tools/trace_replay/trace_cases.py --ci --format github-split-matrix) # AND IT MUST NOT BE EMPTY. An empty `include` is not an error to GitHub - it skips the # whole retrace-split job with no red anywhere - so the one way this subset can vanish # silently is guarded here. trace_cases.py now also rejects an unknown manifest key, which # was the hole: `"splitt": true` loaded clean and emptied the subset (review N-4). if [ "$(printf '%s' "${SPLIT_MATRIX}" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["include"]))')" -lt 1 ]; then echo "::error::the split retrace subset is EMPTY. 'split' is an OPT-OUT as of P6 - a CI case is in this subset unless it carries \"split\": false - so an empty subset means either every case opted out or the CI matrix itself is empty. Either way retrace-split would be skipped with no red, and exit gate E2 names OpenRA." exit 1 fi echo "split-matrix=${SPLIT_MATRIX}" >> "$GITHUB_OUTPUT" trace-fixtures: name: trace fixture (${{ matrix.case }}) runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} needs: [select-runner, trace-cases] strategy: fail-fast: false max-parallel: 4 matrix: case: ${{ fromJSON(needs.trace-cases.outputs.names) }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Derive trace fixture cache key id: fixture-key run: bash .github/scripts/trace-fixture-cache.sh key '${{ matrix.case }}' - name: Restore trace fixture cache id: fixture-cache if: steps.fixture-key.outputs.cacheable == 'true' uses: actions/cache/restore@v5 with: path: ${{ steps.fixture-key.outputs.paths }} key: ${{ steps.fixture-key.outputs.key }} - name: Verify restored trace fixture id: fixture-verify if: steps.fixture-cache.outputs.cache-hit == 'true' run: | if bash .github/scripts/trace-fixture-cache.sh verify '${{ matrix.case }}'; then echo "ok=true" >> "$GITHUB_OUTPUT" else echo "ok=false" >> "$GITHUB_OUTPUT" echo "::warning::Cached fixture for ${{ matrix.case }} failed verification; falling back to the download path" bash .github/scripts/trace-fixture-cache.sh reset '${{ matrix.case }}' fi - name: Fetch trace fixture if: steps.fixture-verify.outputs.ok != 'true' run: bash .github/scripts/fetch-trace-fixture-lfs.sh '${{ matrix.case }}' - name: Save trace fixture cache if: steps.fixture-key.outputs.cacheable == 'true' && steps.fixture-cache.outputs.cache-hit != 'true' uses: actions/cache/save@v5 with: path: ${{ steps.fixture-key.outputs.paths }} key: ${{ steps.fixture-key.outputs.key }} - name: Stage trace fixture run: | safe_case="$(printf '%s' '${{ matrix.case }}' | sed 's/[^A-Za-z0-9._-]/_/g')" stage_dir="trace-fixtures/${safe_case}" mkdir -p "${stage_dir}" python3 tools/trace_replay/trace_cases.py --format fixture-files --case '${{ matrix.case }}' | while IFS= read -r file; do cp "${file}" "${stage_dir}/" done - name: Upload trace fixture uses: actions/upload-artifact@v7 with: name: trace-fixture-${{ matrix.case }} path: trace-fixtures/** if-no-files-found: error retrace: name: retrace (${{ matrix.backend }} / ${{ matrix.transport || 'monolith' }}, ${{ matrix.case }}) runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} env: MOBILEGL_TRANSPORT: ${{ matrix.transport || 'monolith' }} MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" MOBILEGL_IPC_RUN_AHEAD: "1" needs: - select-runner - build-linux - build-retrace - trace-cases - trace-fixtures if: ${{ always() && needs.select-runner.result == 'success' && needs.build-linux.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }} strategy: fail-fast: false max-parallel: 4 matrix: ${{ fromJSON(needs.trace-cases.outputs.matrix) }} steps: - name: Set swap space if: runner.environment != 'self-hosted' uses: pierotofy/set-swap-space@v1.0 with: swap-size-gb: 16 - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Download trace fixture uses: actions/download-artifact@v8 with: name: trace-fixture-${{ matrix.case }} path: trace-fixture-download - name: Install trace fixture run: | mkdir -p tools/trace_replay/fixtures find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \; - name: Get CMake if: runner.environment != 'self-hosted' uses: lukka/get-cmake@v4.3.3 - name: Install runtime dependencies if: runner.environment != 'self-hosted' run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers test -e /usr/lib/x86_64-linux-gnu/libEGL.so test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so - name: Download Linux runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime path: . - name: Download trace replay uses: actions/download-artifact@v8 with: name: mobilegl-trace-replay path: . - name: Unpack retrace runtime run: | tar -xzf mobilegl-linux-runtime.tgz tar -xzf mobilegl-trace-replay.tgz test -f build-linux/libMobileGL.so test -f build-retrace/tools/trace_replay/mobilegl_trace_replay - name: Retrace and validate working-directory: build-retrace/tools/trace_replay run: | ulimit -c unlimited if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1 fi if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \ && [ '${{ matrix.case }}' = 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' ]; then export MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH=1 export MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS=1 export MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER=1 fi # The blended depth-write quirk auto-enables only on Qualcomm, which no CI # runner has, so force it on for the OIT case it exists to fix. ForceOn # bypasses only the vendor gate, so this exercises the real strip on # lavapipe. The Android AVD lane deliberately leaves it off, keeping the # unstripped path covered for the same trace. if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \ && [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then export MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1 fi ctest -V --no-tests=error -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$' --output-junit "${RUNNER_TEMP}/retrace.xml" - name: Annotate failed entries if: failure() run: python3 scripts/ci/junit_annotate.py --excerpts 1 "${RUNNER_TEMP}" - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: retrace-core-dumps-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-${{ matrix.case }} path: /tmp/core.* if-no-files-found: ignore - name: Upload actual image if: ${{ always() && needs.select-runner.result == 'success' }} uses: actions/upload-artifact@v7 with: name: retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-${{ matrix.case }} path: | build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/** build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/** if-no-files-found: warn retrace-summary: name: retrace summary runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} needs: [select-runner, retrace] if: ${{ always() && needs.select-runner.result == 'success' && needs.retrace.result != 'skipped' }} env: MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci') && 'ON' || 'OFF' }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Set artifact metadata run: | echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV" - name: Set up Node.js uses: actions/setup-node@v7 with: node-version: '22' - name: Download monolith control results uses: actions/download-artifact@v8 with: pattern: retrace-result-monolith-control-monolith-* path: retrace-artifacts/monolith-control - name: Download inproc acceptance results if: env.MOBILEGL_CI_DISAGGREGATED == 'ON' uses: actions/download-artifact@v8 with: pattern: retrace-result-inproc-acceptance-inproc-* path: retrace-artifacts/inproc-acceptance - name: Render monolith control summary run: | node tools/trace_replay/render_retrace_summary.mjs \ --input retrace-artifacts/monolith-control \ --output-dir retrace-summary/monolith-control \ --title "MobileGL Linux monolith control" \ --group-label "Linux / monolith control" \ --html mobilegl-linux-monolith-control.html - name: Render inproc acceptance summary if: env.MOBILEGL_CI_DISAGGREGATED == 'ON' run: | node tools/trace_replay/render_retrace_summary.mjs \ --input retrace-artifacts/inproc-acceptance \ --output-dir retrace-summary/inproc-acceptance \ --title "MobileGL Linux inproc acceptance" \ --group-label "Linux / inproc acceptance" \ --html mobilegl-linux-inproc-acceptance.html - name: Upload monolith control summary uses: actions/upload-artifact@v7 with: name: mobilegl-linux-monolith-control-summary-${{ github.sha }} path: retrace-summary/monolith-control/mobilegl-linux-monolith-control.html archive: false if-no-files-found: error - name: Upload inproc acceptance summary if: env.MOBILEGL_CI_DISAGGREGATED == 'ON' uses: actions/upload-artifact@v7 with: name: mobilegl-linux-inproc-acceptance-summary-${{ github.sha }} path: retrace-summary/inproc-acceptance/mobilegl-linux-inproc-acceptance.html archive: false if-no-files-found: error # The trace half of the third CI mode. Same replay, same goldens, but the library underneath is # the verify build and MOBILEGL_PIPE_VERIFY=1 is in the environment, so every backend read of # frontend state is checked against a snapshot taken at the verb boundary. Eight cases rather # than the full lane's 40 (tools/trace_replay/trace_cases.json, "verify": true): the comparator # is budgeted at 5-10x, and the full sweep is a phase-exit / workflow_dispatch run. retrace-verify: name: retrace verify (${{ matrix.backend }}, ${{ matrix.case }}) runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} timeout-minutes: 240 needs: - select-runner - build-linux-verify - build-retrace - trace-cases - trace-fixtures if: ${{ always() && needs.select-runner.result == 'success' && needs.build-linux-verify.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }} strategy: fail-fast: false max-parallel: 4 matrix: ${{ fromJSON(needs.trace-cases.outputs.verify-matrix) }} steps: - name: Set swap space if: runner.environment != 'self-hosted' uses: pierotofy/set-swap-space@v1.0 with: swap-size-gb: 16 - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Download trace fixture uses: actions/download-artifact@v8 with: name: trace-fixture-${{ matrix.case }} path: trace-fixture-download - name: Install trace fixture run: | mkdir -p tools/trace_replay/fixtures find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \; - name: Get CMake if: runner.environment != 'self-hosted' uses: lukka/get-cmake@v4.3.3 - name: Install runtime dependencies if: runner.environment != 'self-hosted' run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers test -e /usr/lib/x86_64-linux-gnu/libEGL.so test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so - name: Download Linux verify runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime-verify path: . - name: Download trace replay uses: actions/download-artifact@v8 with: name: mobilegl-trace-replay path: . - name: Unpack verify runtime (library under test) # build-retrace's CTestTestfile.cmake has the absolute path # /build-linux/libMobileGL.so frozen into every case, so the swap happens here # rather than through a variable: the verify .so is put where that path points. The nm # check is what makes the swap falsifiable - a run against the ordinary library would # carry no comparator, ignore MOBILEGL_PIPE_VERIFY entirely, and match its golden. # # `nm`, not `nm -D`, for the reason spelled out in build-linux-verify: everything MGPipe is # hidden-visibility in a Release build and the dynamic table has none of it. run: | tar -xzf mobilegl-linux-runtime-verify.tgz tar -xzf mobilegl-trace-replay.tgz test -f build-verify/libMobileGL.so test -f build-retrace/tools/trace_replay/mobilegl_trace_replay mkdir -p build-linux cp build-verify/libMobileGL.so build-linux/libMobileGL.so if ! nm --defined-only build-linux/libMobileGL.so | grep -q MGPipeVerifyInputs; then echo "::error::the library unpacked at build-linux/libMobileGL.so defines no MGPipeVerifyInputs, so this retrace would replay against a comparator-free build and pass on its golden having verified nothing" exit 1 fi echo "the library at build-linux/libMobileGL.so is the verify build" - name: Retrace and validate (verify) working-directory: build-retrace/tools/trace_replay # run_trace_case.cmake turns MOBILEGL_PIPE_VERIFY into three assertions of its own (the # arming line, no Fatal{PipeVerifyDiffer, no Fatal{UnmigratedPipeInput), so a case that # somehow ran the wrong library reds here instead of passing on its golden. # --timeout 10800: the 1800s cases run 5-10x slower with both comparator arms live, which # is well past ctest's 1500s default. run: | ulimit -c unlimited if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi export MOBILEGL_PIPE_VERIFY=1 if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1 fi if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \ && [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then export MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1 fi ctest -V --no-tests=error --timeout 10800 \ -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$' # The retrace lane's own always-on negative control, on one case so it costs one short trace: # with a snapshot field corrupted, the SAME replay must fail. Without it, "40 traces, zero # divergences" would be a statement about a comparator nobody watched. # # The rerun replays into the SAME case directory, so the verified run's images are put aside # first and restored before the verdict: "Upload actual image" below runs `if: always()` and # would otherwise ship the deliberately corrupted run's output under the name of the good one. # The restore happens whichever way the control goes, which is why the ctest exit status is # captured rather than tested inline. - name: Negative control (corrupted snapshot) if: ${{ matrix.case == 'OpenRA' && matrix.backend == 'DirectGLES' }} working-directory: build-retrace/tools/trace_replay run: | export MOBILEGL_PIPE_VERIFY=1 export MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters GOOD_OUTPUT="${RUNNER_TEMP}/openra-verified-output" rm -rf "${GOOD_OUTPUT}" if [ -d OpenRA ]; then cp -a OpenRA "${GOOD_OUTPUT}" fi set +e ctest -V --no-tests=error --timeout 10800 \ -R '^MobileGLTraceReplay\.OpenRA\.DirectGLES$' control_rc=$? set -e if [ -d "${GOOD_OUTPUT}" ]; then rm -rf OpenRA mv "${GOOD_OUTPUT}" OpenRA echo "restored the verified run's OpenRA output over the corrupted rerun's" fi if [ "${control_rc}" -eq 0 ]; then echo "::error::MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left the OpenRA retrace GREEN, so the comparator is not comparing and the whole verify retrace lane proves nothing." exit 1 fi echo "the corrupted field turned the retrace red, as it must (ctest exit ${control_rc})" - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: retrace-verify-core-dumps-${{ matrix.backend }}-${{ matrix.case }} path: /tmp/core.* if-no-files-found: ignore - name: Upload actual image if: ${{ always() && needs.select-runner.result == 'success' }} uses: actions/upload-artifact@v7 with: name: retrace-verify-result-${{ matrix.backend }}-${{ matrix.case }} path: | build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/** build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/** if-no-files-found: warn # The retrace arm of the split shape: retrace-verify with MOBILEGL_PIPE_VERIFY=1 swapped for # MOBILEGL_TRANSPORT and the comparator's symbol swapped for an MG_Remote one. Exit gate E2 is # one row - OpenRA at SSIM >= 0.99 - but the lane is no longer one row: as of P6 it is # {backend, case, transport} over the whole CI matrix, both backends, inproc AND spawn. # # inproc IS THE CONTROL AND SPAWN IS THE CLAIM. Same library, same codec, same applier, same # picture; the only difference is which PROCESS the server role runs in. Running only spawn # would leave "did the transport break it?" and "did the phase break it?" answerable only # together, which is the question every failure in this phase turned out to hinge on. # # IT RUNS THE UNCHANGED CTEST NAMES with the transport exported in the JOB environment, rather # than the SPLIT-suffixed variant entries. build-retrace configures without # -DMOBILEGL_BUILD_DISAGGREGATED, so the variant entries are deliberately not registered there # (a name that says SPLIT in a build that cannot be one is worse than no name), and the # unchanged names are what retrace-verify already proves this shape works with. The variant # entries exist for a build that DOES configure the option - a local build-split with trace # replay on - where `ctest -L retrace-split` is self-describing and needs no environment ritual. # # WHAT MAKES IT FALSIFIABLE is not the SSIM. A monolith run of OpenRA also scores 1.000: measured # on this branch, a pull library under MOBILEGL_TRANSPORT=inproc produced ssim=1.0 and was caught # only by run_trace_case.cmake's transport-resolution assertion. So there are two guards, and the # SSIM is neither of them: the nm check below (this library carries MG_Remote) and the # transport-resolution line in the library's own log (this PROCESS resolved inproc). retrace-split: name: retrace split (${{ matrix.backend }} / ${{ matrix.transport }}, ${{ matrix.case }}) runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} timeout-minutes: 240 # The setup action selects Dozen on Arch WSL or lavapipe on GitHub. needs: - select-runner - build-linux-monolith-control - build-linux-split - build-retrace - trace-cases - trace-fixtures # The separate OFF/pull artifact is for the negative control ONLY: its library is what the control # swaps in to prove the transport-resolution assertion is load-bearing. if: ${{ always() && needs.select-runner.result == 'success' && needs.build-linux-monolith-control.result == 'success' && needs.build-linux-split.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }} strategy: fail-fast: false max-parallel: 4 matrix: ${{ fromJSON(needs.trace-cases.outputs.split-matrix) }} steps: - name: Set swap space if: runner.environment != 'self-hosted' uses: pierotofy/set-swap-space@v1.0 with: swap-size-gb: 16 - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Download trace fixture uses: actions/download-artifact@v8 with: name: trace-fixture-${{ matrix.case }} path: trace-fixture-download - name: Install trace fixture run: | mkdir -p tools/trace_replay/fixtures find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \; - name: Get CMake if: runner.environment != 'self-hosted' uses: lukka/get-cmake@v4.3.3 - name: Install runtime dependencies if: runner.environment != 'self-hosted' run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers test -e /usr/lib/x86_64-linux-gnu/libEGL.so test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so - name: Download Linux split runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime-split path: . - name: Download trace replay uses: actions/download-artifact@v8 with: name: mobilegl-trace-replay path: . # The PULL runtime, for the negative control at the end of this job and for nothing else. - name: Download pull runtime (negative control) uses: actions/download-artifact@v8 with: name: mobilegl-linux-monolith-control path: pull-runtime - name: Unpack the pull runtime (negative control) run: | tar -xzf pull-runtime/mobilegl-linux-monolith-control.tgz -C pull-runtime test -f pull-runtime/build-monolith-control/libMobileGL.so - name: Unpack split runtime (library under test) # build-retrace's CTestTestfile.cmake has the absolute path # /build-linux/libMobileGL.so frozen into every case, so the swap happens here # rather than through a variable, exactly as in retrace-verify. `nm`, not `nm -D`. run: | tar -xzf mobilegl-linux-runtime-split.tgz tar -xzf mobilegl-trace-replay.tgz test -f build-split/libMobileGL.so test -f build-retrace/tools/trace_replay/mobilegl_trace_replay mkdir -p build-linux cp build-split/libMobileGL.so build-linux/libMobileGL.so if ! nm --defined-only build-linux/libMobileGL.so | grep -q -i MG_Remote; then echo "::error::the library unpacked at build-linux/libMobileGL.so defines no MG_Remote symbol, so this retrace would replay against a monolith build, ignore MOBILEGL_TRANSPORT entirely and match its golden having split nothing" exit 1 fi echo "the library at build-linux/libMobileGL.so is the split build" # THE SPAWN ARM NEEDS A SECOND BINARY and the inproc arm needs none, which is the whole # difference between the two at this level: inproc's server role is a thread in the replay # process, spawn's is a PROCESS the client launches from this image. It is packaged by # build-linux-split's `find -name '*.so'` sweep - libMobileGLServer.so is an executable that # wears a .so name so an APK will install it somewhere exec-able - and a missing one must be # a NAMED failure here rather than a LaunchServer refusal 78 jobs deep. - name: Place the server image (spawn arm only) if: ${{ matrix.transport == 'spawn' }} run: | if [ ! -f build-split/libMobileGLServer.so ]; then echo "::error::the split runtime carries no libMobileGLServer.so, so transport=spawn has no server to launch. ConfigLoader would resolve Spawn, LaunchServer would refuse BY NAME, and this job would red on a message about an unresolvable image rather than about a packaging gap." exit 1 fi cp build-split/libMobileGLServer.so build-linux/libMobileGLServer.so chmod +x build-linux/libMobileGLServer.so echo "server image: $(readlink -f build-linux/libMobileGLServer.so)" # THE MAGMA KNOBS GO TO $GITHUB_ENV, NOT TO ONE STEP'S SHELL: the replay below AND both # negative controls after it must run the same repair set. Exported inside "Retrace and # validate" only, they never reached the pull-library control, which replayed # iterationrp x DirectVulkan unrepaired (ssim 0.028 < 0.98), reddened at # run_trace_case.cmake:173 before the transport assertion, and failed as "the failure never # says the library did not resolve the transport" - a red that is not the control's own. - name: Magma repair knobs (all later steps) run: | # P7 package L: THE SAME MAGMA KNOBS THE MONOLITH RETRACE JOB EXPORTS, ALL OF THEM. # This job exported none at all, so every DirectVulkan row of the split matrix was # compared against a monolith row running a different repair set - and a split retrace # whose control is not comparable cannot say whether a divergence is a transport fact. # R11F_G11F_B10F fallback is whole-backend; the blended depth-write quirk auto-enables # only on Qualcomm, which no runner has, so the OIT case forces it exactly as the # monolith job does (ForceOn bypasses only the vendor gate). if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then echo "MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1" >> "${GITHUB_ENV}" fi if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \ && [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then echo "MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1" >> "${GITHUB_ENV}" fi # THE THREE ITERATIONRP KNOBS, FOR THE ONE CASE THE MONOLITH JOB NAMES. # The iterationRP repairs are tri-state quirks that default to device auto-detection and # lavapipe is on no auto list, so without them Program 203 runs unrepaired and misses its # golden output. The monolith job has exported them since they landed; this job never # did, so `iterationrp x DirectVulkan` was the one cell of the split matrix whose red # could not be read as a transport fact - the monolith control it is compared against was # running a different repair set. Exit gate 1 asks for the two halves to be COMPARABLE, # and two runs that differ in a quirk knob are not. if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \ && [ '${{ matrix.case }}' = 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' ]; then echo "MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH=1" >> "${GITHUB_ENV}" echo "MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS=1" >> "${GITHUB_ENV}" echo "MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER=1" >> "${GITHUB_ENV}" fi - name: Retrace and validate (${{ matrix.transport }}) working-directory: build-retrace/tools/trace_replay # run_trace_case.cmake turns MOBILEGL_TRANSPORT into assertions of its own - the library # must have RESOLVED the transport in this process, and its log must carry no Fatal{ - so a # case that somehow ran the wrong library reds here instead of passing on its golden. That # log is also the only valid refusal census: the console sink is compiled out of this # configuration, so a `ctest -V` transcript reports a false zero for Fatal{ lines. run: | ulimit -c unlimited if [ "$RUNNER_ENVIRONMENT" != self-hosted ]; then sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'; fi export MOBILEGL_TRANSPORT='${{ matrix.transport }}' # Absolute, because the replay's working directory is not this one and LaunchServer # resolves the image relative to nothing. Harmless on the inproc arm, which reads it # only to decide it does not need it. export MOBILEGL_IPC_SERVER_PATH="${GITHUB_WORKSPACE}/build-linux/libMobileGLServer.so" ctest -V --no-tests=error --timeout 10800 \ -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$' # THE RETRACE LANE'S OWN ALWAYS-ON NEGATIVE CONTROL, and its absence was review finding M-6: # this job was a clone of retrace-verify with the one step removed that makes the lane mean # anything. retrace-verify's "Negative control (corrupted snapshot)" step requires a corrupted # snapshot field to red the retrace, so that "79 traces, zero divergences" is not a statement # about a comparator nobody watched; the same reasoning applies here word for word. # # The control swaps the PULL library into the frozen path and requires the SAME replay to # fail. It is the sharpest one available, because OpenRA scores ssim 1.000000 either way - # measured - so this step fails if and only if run_trace_case.cmake's transport-resolution # assertion has stopped working, which is the single thing standing between this job and a # green that ran monolith end to end. # # NOT the control BRIEF 7 E2 names ("patch the Clear emitter to drop one emission; SSIM must # fall below threshold"). That one needs an emitter, i.e. package c1, and it is carried as an # explicit debt in t1-v1.md rather than silently substituted - which is what the first # version of this package did. # # The rerun replays into the same case directory, so the good run's images are put aside and # restored whichever way the control goes; "Upload actual image" below runs `if: always()` # and would otherwise ship the deliberately-wrong run's output under the good run's name. # THE BODY OF THIS STEP IS scripts/ci/retrace_pull_library_control.sh, for the reason the # split lane's control gives: a `run:` block cannot be executed off a runner, so these lines # could not be tested until they ran in CI. scripts/ci/control_smoke_test.sh drives that file. # # Two holes ID-46 finding 8(b) found in this block, both CONFIRMED against the REAL ctest in a # REAL build tree, both closed in the script: it had NO selection guard at all - unlike the # split lane's run_control - so a case/backend regex matching nothing exited 8 through # `--no-tests=error` and was read as "the pull library turned it red"; and only "non-zero # ctest" was checked after the nm identity check, so a loader failure, a missing fixture or a # timeout passed it. The red must now carry run_trace_case.cmake's own sentence. - name: Negative control (pull library) working-directory: build-retrace/tools/trace_replay env: CONTROL_TMPDIR: ${{ runner.temp }} PULL_LIBRARY: ${{ github.workspace }}/pull-runtime/build-monolith-control/libMobileGL.so FROZEN_LIBRARY: ${{ github.workspace }}/build-linux/libMobileGL.so # The control has to run the arm it is a control FOR. Hard-coded inproc, it would go on # proving something about the other arm while spawn ran unguarded. SPLIT_TRANSPORT: ${{ matrix.transport }} MOBILEGL_IPC_SERVER_PATH: ${{ github.workspace }}/build-linux/libMobileGLServer.so run: >- bash "${GITHUB_WORKSPACE}/scripts/ci/retrace_pull_library_control.sh" '${{ matrix.case }}' '${{ matrix.backend }}' # EXIT GATE E2's OTHER HALF, and the one BRIEF-P5 §7 E2 actually names: "drop an emission # and the SSIM must fall below the threshold". The control above proves the lane runs a # SPLIT library; it cannot prove the PICTURE came through the wire, because OpenRA scores # ssim 1.000000 against a monolith library too and the transport assertion is what reds it. # # It drops the DRAWS and not the clears, and that is a measurement rather than a # preference: with MOBILEGL_IPC_E2_DROP_CLEAR=1 armed, read, and all 29 of OpenRA's Clear # records dropped, the retrace still scored ssim=1.000000 / mismatchPixels=0 - OpenRA # overdraws every pixel it clears before the snapshot (joint-v1.md §3 found the green; # scripts/ci/retrace_drop_draw_control.sh's header has the trace census that explains it). # With the DrawVbo records dropped instead: 758 dropped, ssim=0.000036, # mismatchPixels=295296. # # THE BODY IS A SCRIPT for the reason the two steps above give: a `run:` block cannot be # executed off a runner, and scripts/ci/control_smoke_test.sh drives this file against a # stubbed ctest in seven modes - including the two that matter most, a red whose SSIM never # fell and a knob that armed and dropped nothing. - name: Negative control (dropped draws) working-directory: build-retrace/tools/trace_replay env: CONTROL_TMPDIR: ${{ runner.temp }} LIBRARY_LOG: ${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.client.log FROZEN_LIBRARY: ${{ github.workspace }}/build-linux/libMobileGL.so # The control has to run the arm it is a control FOR. Hard-coded inproc, it would go on # proving something about the other arm while spawn ran unguarded. SPLIT_TRANSPORT: ${{ matrix.transport }} MOBILEGL_IPC_SERVER_PATH: ${{ github.workspace }}/build-linux/libMobileGLServer.so run: >- bash "${GITHUB_WORKSPACE}/scripts/ci/retrace_drop_draw_control.sh" '${{ matrix.case }}' '${{ matrix.backend }}' # The refusal census, recorded rather than gated. run_trace_case.cmake already REDS the case # on any Fatal{, so reaching here means the count is zero - but the number and the distinct # slot names are what MEASUREMENTS wants from every split run, and reading them out of the # library's own log is the only way to get them (ctest -V's transcript is a false zero). - name: Refusal census from the library log if: ${{ always() && needs.select-runner.result == 'success' }} working-directory: build-retrace/tools/trace_replay run: | # BOTH ROLES, because under spawn every applier refusal is raised in the SERVER's # process and a census reading only the client's file reports a confident zero for the # half it cannot see. log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.client.log" server_log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.server.log" if [ ! -f "${log}" ]; then echo "no ${log} - the replay wrote no library log" exit 0 fi census_files="${log}" if [ -f "${server_log}" ]; then census_files="${census_files} ${server_log}"; fi echo "census over: ${census_files}" echo "Fatal{ lines: $(cat ${census_files} | grep -c 'Fatal{' || true)" cat ${census_files} | grep -o 'Fatal{[A-Za-z]*, "[^"]*"' | sort | uniq -c | sort -rn | head -20 || true # The transport line is the CLIENT's: ConfigLoader resolves it there, and under spawn # the server states its own separately. grep -m1 "MOBILEGL_TRANSPORT=" "${log}" || echo "no transport line in ${log}" - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: retrace-split-core-dumps-${{ matrix.transport }}-${{ matrix.backend }}-${{ matrix.case }} path: /tmp/core.* if-no-files-found: ignore - name: Upload actual image if: ${{ always() && needs.select-runner.result == 'success' }} uses: actions/upload-artifact@v7 with: name: retrace-split-result-${{ matrix.transport }}-${{ matrix.backend }}-${{ matrix.case }} path: | build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/** build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/** if-no-files-found: warn remove-artifact-clutter: name: remove artifact clutter runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} # Every retrace arm downloads the shared fixtures, including split controls. needs: - select-runner - retrace-summary - retrace-verify - retrace-split if: ${{ always() && needs.select-runner.result == 'success' }} permissions: actions: write steps: - name: Delete intermediate Linux retrace artifacts env: GH_TOKEN: ${{ github.token }} run: | # Read the complete inventories before deleting anything. A failed API # request must fail this step, not look like "no failed cases" through # an unchecked process substitution. gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100" \ --jq '.jobs[] | select((.name | startswith("retrace (")) or (.name | startswith("retrace verify (")) or (.name | startswith("retrace split ("))) | select(.conclusion == "failure" or .conclusion == "cancelled" or .conclusion == "timed_out" or .conclusion == "action_required") | .name' \ > "${RUNNER_TEMP}/cleanup-jobs.txt" gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \ --jq '.artifacts[] | select(.name | startswith("trace-fixture-") or startswith("retrace-result-") or startswith("retrace-split-result-")) | [.id, .name] | @tsv' \ > "${RUNNER_TEMP}/cleanup-artifacts.txt" declare -A failed_cases=() while IFS= read -r job_name; do case_name="${job_name#retrace split (*, }" case_name="${case_name#retrace verify (*, }" case_name="${case_name#retrace (*, }" case_name="${case_name%)}" failed_cases["${case_name}"]=1 done < "${RUNNER_TEMP}/cleanup-jobs.txt" if ((${#failed_cases[@]})); then echo "Retaining fixtures and results for failed retrace case(s):" printf ' %s\n' "${!failed_cases[@]}" else echo "All retrace jobs succeeded; no fixtures need to be retained." fi deleted=0 retained=0 while IFS=$'\t' read -r artifact_id artifact_name; do keep=0 if [[ "${artifact_name}" == trace-fixture-* ]]; then case_name="${artifact_name#trace-fixture-}" if [[ -v "failed_cases[${case_name}]" ]]; then keep=1 fi else for case_name in "${!failed_cases[@]}"; do if [[ "${artifact_name}" == *-"${case_name}" ]]; then keep=1; break; fi done fi if ((keep)); then echo "Retaining ${artifact_name} (${artifact_id}) for failed retrace." ((retained += 1)) continue fi echo "Deleting ${artifact_name} (${artifact_id})" gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}" ((deleted += 1)) done < "${RUNNER_TEMP}/cleanup-artifacts.txt" echo "Deleted ${deleted} intermediate Linux artifact(s); retained ${retained} failed-retrace artifact(s)." pipe-gates: name: MGPipe generators and hygiene gates needs: select-runner runs-on: ${{ fromJSON(needs.select-runner.outputs.runner) }} # Deliberately independent of build-linux: these are source-level gates, they take # seconds, and a broken build must not hide a drifted interface. steps: - name: Checkout repo uses: actions/checkout@v6 # The seven generators all read MG_Pipe/*.def, so regenerating and diffing is what # keeps the two interface tables, the wire records, the verify comparators, the # PipeInputs field ids, the read-inventory coverage and the render-state member list # from drifting apart from the catalogue. The generated files are committed # deliberately: the build must not depend on python. - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Regenerate the MGPipe interface (G1-G7) run: | python3 scripts/gen_pipe.py git diff --exit-code -- MobileGL/MG_Pipe/generated # The generators' own negative controls: canned inputs that MUST trip each structural check # (a field list that does not cover its struct's members, a verb set that is not the function # table's). Regenerating and diffing above cannot see a check that silently stopped # checking - a broken gate and a clean tree produce the same green. - name: MGPipe generator self-test run: python3 scripts/gen_pipe.py --self-test # The link seam owns physical storage and keeps link dispatch out of callers. - name: Link-seam purity check run: python3 scripts/ci/link_seam_purity.py --self-test # The same shape for Espryt's re-keyed memos (P3b/P4b, PLAN-PH-P34B-P7.md 1.2). P4a/P5f # moved the resolved-binding, sampler-pass, image-sweep, twin-registry, unit-sampler and # twin-lookup memos onto {slot, gen} handles and applier serials; the legacy pointer-keyed # arms stayed because they are the PULL build's only arms and G1 pins that build's .text. # Nothing checked that a NEW frontend-pointer key could not appear outside them. # # MOBILEGL_PIPE_LEGACY_MEMOS IS A POLICY EXEMPTION, NOT AN ABSENCE. It defaults ON # (CMakeLists.txt:43) and no job here clears it, so the split builds in this workflow DO # compile those arms - TwinLookupMemo among them. The gate's rule is "the legacy arm may # keep pointer keys", which is a decision; only !MOBILEGL_BUILD_DISAGGREGATED is code a # split build genuinely never sees. # # --self-test runs the negative controls and then the real scan in one invocation, as its # sibling does; the scan also asserts that every memo family's anchor still matches and # that every allow-list entry still suppresses something. - name: Espryt memo-key purity check run: python3 scripts/ci/espryt_memo_purity.py --self-test # The same question for the symbol tool the P1 gate is written in terms of. - name: Symbol report self-test run: python3 scripts/symbol_report.py --self-test # Per-draw fprintf/printf instrumentation has repeatedly been committed by accident, # once inside a mutex critical section. Nothing under these two trees prints to a # stdio stream today - MGLOG_D compiles out in INFO builds and is the only channel # they are allowed to use - so this gate starts with no exceptions, and any addition # to it needs a reason in the pull request rather than a quiet whitelist entry. The # alternation names every stdio spelling, not just the two that were committed: # fprintf to either stream, printf, puts, and the iostream pair. - name: No stdio in MG_Backend or MG_State run: | if grep -rnE 'fprintf[[:space:]]*\((stderr|stdout)|(^|[^[:alnum:]_>.])printf[[:space:]]*\(|(^|[^[:alnum:]_>.:])puts[[:space:]]*\(|std::(cout|cerr)' \ MobileGL/MG_Backend MobileGL/MG_State; then echo "::error::stdio instrumentation found; use MGLOG_D (compiled out in INFO builds)" exit 1 fi echo "no fprintf(stderr/stdout / printf( / puts( / std::cout|cerr under MobileGL/MG_Backend or MobileGL/MG_State" # A GATE as of P2, which is when MG_Pipe/DirtySurface.def exists to diff the scan against # (ROADMAP.md:18 puts the first mapping round in P2). --check fails BOTH directions: a # mutator the scanner finds with no row in the def, and a row naming a mutator the scan no # longer finds - so a deleted mutator cannot leave a stale row behind claiming coverage. # # --self-test is the half that keeps --check honest, and it is not optional. A completeness # check that silently stopped checking produces exactly the same green as a complete # mapping; the self-test feeds it two canned negative controls (a mutator withheld from the # def, a row naming a function that does not exist) and fails if either fails to trip. Same # shape as gen_pipe.py --self-test and check_include_closure.py above. # # What this gate does NOT cover is written into DirtySurface.def's header rather than left # implicit: the scanner attributes a mutation inside a lambda to the enclosing function, # reads a mutation published through a helper as deferred, and scans only MG_Impl/GLImpl - # so the four MGP_NOTE_MUTATION sites in MG_State are outside it entirely. This is a # completeness gate over what the scanner can see; the semantic proof is the verify lane. - name: Dirty-surface mapping complete (G9) run: | python3 scripts/gen_pipe_dirty_surface.py --check python3 scripts/gen_pipe_dirty_surface.py --self-test # A GATE as of P5 (R-7.1, CONTRACT-P5.md table 2). The eighth generator, and the only one # whose output the step above cannot cover: `gen_pipe.py` does not write # PipeFieldOwnership.inc, so regenerating the seven and diffing MG_Pipe/generated leaves a # drifted ownership table green. The build-level static_assert only catches an # UNCLASSIFIED row; a hand edit that CHANGES a class compiles clean, and --check is the # only thing in the tree that catches it. # # NO BRANCH GUARD, deliberately, unlike the two G5 steps below: this asks "does the # committed table still follow from the .def files", which is a question every branch can # answer and every branch wants answered. The table outlives feat/disaggregated. # # --self-test is not optional, for the reason the two gates above give and for one this # generator learned the hard way: its first version counted eleven trips when one control # was a silent duplicate of another, because the harness asked "did something exit" rather # than "did THIS exit". Every control now asserts its own message, and a harness control # asserts that the harness still rejects someone else's exit. - name: Field-ownership table complete (R-7.1) run: | python3 scripts/gen_pipe_field_ownership.py --check python3 scripts/gen_pipe_field_ownership.py --self-test # R-16 APPLIED TO THE NEGATIVE CONTROLS THEMSELVES. The split lane's E1/E3(a) controls and the # retrace lane's pull-library control are gates, and until ID-46 finding 8 neither could be # made red by anyone: their bodies were `run:` blocks, which execute only on a runner. Both # bodies now live in scripts/ci/, and this step runs them against a stubbed ctest that # reproduces the finding - a NON-EMPTY selection failing with UNRELATED_CONTROL_FAILURE, and a # case/backend regex matching no tests - and requires each control to report FAILED. The same # stub, failing with the diagnostics the scenarios really emit, must make them report PASSED. # # NO BRANCH GUARD: this asks "do the negative controls still reject a red that is not theirs", # which is a question every branch can answer and none of which depends on the TEMPORARY # feat/disaggregated trigger at the top of this file. It costs a couple of seconds and needs # no build. # P7 adds the last two. THE DEVICE MATRIX DRIVER HAD NO CI STEP AT ALL: its 12 stub # regressions (now 18, with the DirectVulkan half) were run by hand in WSL and recorded in # a note, which is the same "a gate nobody can make red" shape the paragraph above is # about - and it guards the tool that decides whether a device golden run PASSED. # compare_actuals is the new half of that evidence (actual-vs-actual SSIM between # repeats); both are stub/synthetic-only, need no build, no GPU and no device. - name: Control and trace-tool smoke tests (R-16) run: | bash scripts/ci/control_smoke_test.sh python3 scripts/ci/test_trace_infrastructure.py python3 scripts/ci/tcp_lane_tools_test.py python3 tools/trace_replay/test_run_tcp_matrix.py python3 tools/trace_replay/test_compare_actuals.py # Warning only for now: the disaggregation documents are still being written, and a # lint that fails a rewrite in progress teaches people to ignore it. It becomes # --strict when the documents settle. - name: Documentation citation lint run: | shopt -s nullglob documents=(docs/Disaggregated/*.md docs/Disaggregated/design/*.md docs/Disaggregated/guide/*.md) if [ ${#documents[@]} -eq 0 ]; then echo "no disaggregation documents to check" exit 0 fi python3 scripts/check_doc_citations.py "${documents[@]}" || true