MobileGL/.github/workflows/test.yml

name: Test

on:
  push:
    branches:
      - dev
      - Feat/Backend-Direct-GLES
      - Feat/Backend-Direct-Vulkan
      # TEMPORARY, remove before merging the MGPipe work into dev: the disaggregation
      # branch runs the full lane on every push so a phase's landing is not gated on
      # someone remembering to dispatch the workflow by hand.
      - feat/disaggregated
  workflow_dispatch:
    inputs:
      baseline_sha:
        description: >-
          The commit monolith-symbol-report compares this tree against. P1's G1 says the pull
          build is byte-identical to feat/disaggregated@087685d1, and that is what the default
          names. The trigger set is unchanged: this job runs on workflow_dispatch only.
        required: false
        default: "087685d1"

jobs:
  build-linux:
    runs-on: ubuntu-latest
    permissions:
      actions: write
      contents: read
    env:
      BUILD_DIR: build-linux
      CCACHE_BASEDIR: ${{ github.workspace }}
      CCACHE_COMPRESS: "true"
      CCACHE_DIR: ${{ github.workspace }}/.ccache
      CCACHE_MAXSIZE: 4G
      CCACHE_NOHASHDIR: "true"

    steps:
      - name: Set Swap Space
        uses: pierotofy/set-swap-space@v1.0
        with:
          swap-size-gb: 32

      - name: Checkout repo
        uses: actions/checkout@v6
        with:
          submodules: recursive

      - name: Get CMake
        uses: lukka/get-cmake@v4.3.3

      - name: Restore ccache
        uses: actions/cache/restore@v5
        with:
          path: .ccache
          key: ${{ runner.os }}-test-${{ github.job }}-ccache-v1
          restore-keys: |
            ${{ runner.os }}-test-${{ github.job }}-ccache-

      - name: Prepare Vulkan SDK
        uses: humbletim/setup-vulkan-sdk@v1.2.1
        with:
          vulkan-query-version: 1.4.304.1
          vulkan-components: Vulkan-Headers, Vulkan-Loader
          vulkan-use-cache: true

      - name: Update glslang external sources
        working-directory: 3rdparty/glslang
        run: python update_glslang_sources.py

      - name: Install build dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build

      - name: Show installed toolchain
        run: |
          ccache --version
          clang-20 --version
          clang++-20 --version
          ld.lld-20 --version || ld.lld --version || true
          dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true

      - name: Configure CMake
        run: |
          if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
            BUILD_TYPE=Debug
          else
            BUILD_TYPE=Release
          fi

          cmake -S . -B "${BUILD_DIR}" -G Ninja \
            -DCMAKE_C_COMPILER=clang-20 \
            -DCMAKE_CXX_COMPILER=clang++-20 \
            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
            -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
            -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
            -DMOBILEGL_BUILD_TEST=ON \
            -DMOBILEGL_BUILD_BENCHMARK=ON \
            -DMOBILEGL_BUILD_INTEGRATION_TEST=ON \
            -DMOBILEGL_ITEST_VK_ICD=/usr/share/vulkan/icd.d/lvp_icd.json \
            -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \
            -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON \
            -DBENCHMARK_ENABLE_TESTING=OFF \
            -DCMAKE_POLICY_VERSION_MINIMUM=3.5

      - name: Build
        run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)"

      - name: Show ccache stats
        if: always()
        run: ccache --show-stats

      # Rewrite one rolling entry per job on the default branch. The upload stays
      # cumulative - it carries every object restored at the top of this run plus
      # the few TUs that actually changed - but Actions cache keys are immutable,
      # so the superseded blob has to be released before the same key can be
      # re-uploaded. Running after the build means a failed build leaves the
      # existing entry untouched. The other trigger branches restore this entry
      # rather than each writing one of their own.
      - name: Release superseded ccache entry
        if: github.ref_name == github.event.repository.default_branch
        env:
          GH_TOKEN: ${{ github.token }}
          CACHE_KEY: ${{ runner.os }}-test-${{ github.job }}-ccache-v1
        run: gh cache delete "${CACHE_KEY}" || true

      - name: Save ccache
        if: github.ref_name == github.event.repository.default_branch
        continue-on-error: true
        uses: actions/cache/save@v5
        with:
          path: .ccache
          key: ${{ runner.os }}-test-${{ github.job }}-ccache-v1

      - name: Package Linux runtime
        run: |
          mkdir -p ci-artifacts
          mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort)
          tar \
            --exclude='*/CMakeFiles' \
            --exclude='*.o' \
            --exclude='*.a' \
            --exclude='*.ninja*' \
            --exclude='build.ninja' \
            --exclude='cmake_install.cmake' \
            -czf ci-artifacts/mobilegl-linux-runtime.tgz \
            "${BUILD_DIR}/CTestTestfile.cmake" \
            "${BUILD_DIR}/MobileGL/MG_Test" \
            "${BUILD_DIR}/MobileGL/MG_Benchmark" \
            "${BUILD_DIR}/MobileGL/MG_IntegrationTest" \
            "${SHARED_LIBS[@]}"

      - name: Upload Linux runtime
        uses: actions/upload-artifact@v7
        with:
          name: mobilegl-linux-runtime
          path: ci-artifacts/mobilegl-linux-runtime.tgz
          if-no-files-found: error

  test:
    runs-on: ubuntu-latest
    needs: build-linux

    steps:
      - name: Checkout repo
        uses: actions/checkout@v6

      - name: Get CMake
        uses: lukka/get-cmake@v4.3.3

      - name: Install runtime dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libvulkan1 libegl1 libgles2 libgl1-mesa-dri mesa-vulkan-drivers

      - name: Download Linux runtime
        uses: actions/download-artifact@v8
        with:
          name: mobilegl-linux-runtime
          path: .

      - name: Unpack Linux runtime
        run: tar -xzf mobilegl-linux-runtime.tgz

      - name: Normalize CTest command paths
        run: |
          python - <<'PY'
          from pathlib import Path
          import re

          for path in Path('build-linux').rglob('CTestTestfile.cmake'):
              text = path.read_text()
              text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text)
              path.write_text(text)
          PY

      - name: Test
        working-directory: build-linux
        run: |
          ulimit -c unlimited
          sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'
          if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
            ctest -V -L unit --no-tests=error
          else
            ctest --output-on-failure -L unit --no-tests=error
          fi

      - name: Upload core dumps
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: unit-core-dumps
          path: /tmp/core.*
          if-no-files-found: ignore

  integration:
    runs-on: ubuntu-latest
    needs: build-linux

    steps:
      - name: Checkout repo
        uses: actions/checkout@v6

      - name: Get CMake
        uses: lukka/get-cmake@v4.3.3

      - name: Install runtime dependencies
        # Same set as the benchmark job, for the same reason: the scenarios bring
        # up real headless EGL (llvmpipe) and Vulkan (lavapipe) contexts, and
        # libegl-mesa0 - the EGL vendor library behind glvnd's libegl1 dispatch -
        # only arrives as a Recommends.
        run: |
          sudo apt-get update
          sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers

      - name: Download Linux runtime
        uses: actions/download-artifact@v8
        with:
          name: mobilegl-linux-runtime
          path: .

      - name: Unpack Linux runtime
        run: tar -xzf mobilegl-linux-runtime.tgz

      - name: Normalize CTest command paths
        run: |
          python - <<'PY'
          from pathlib import Path
          import re

          for path in Path('build-linux').rglob('CTestTestfile.cmake'):
              text = path.read_text()
              text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text)
              path.write_text(text)
          PY

      - name: Integration scenarios
        working-directory: build-linux
        # REQUIRE_GPU makes a driverless runner FAIL instead of skipping every
        # scenario - an all-skip run is otherwise indistinguishable from a pass,
        # which is how a five-month-old draw-dropping bug survived unseen until
        # this lane existed.
        #
        # The lavapipe ICD pin lives in the build-linux configure
        # (-DMOBILEGL_ITEST_VK_ICD), NOT here: the configure bakes it into each
        # test's ctest ENVIRONMENT property, and a property entry OVERRIDES the
        # job environment - a VK_ICD_FILENAMES exported here would be silently
        # ignored while looking like it works. This lane runs on lavapipe
        # deterministically, not on whichever of the eight Mesa ICDs a GPU-less
        # runner enumerates first.
        #
        # Cores are armed so that any crash - the harness pre-flight child's
        # included - leaves /tmp/core.*, which the failure-only step below ships
        # as an artifact. Analyzing a downloaded core against the runtime
        # artifact's binary in an ubuntu-24.04 userspace reproduces the exact
        # crash stack without burning a CI round on an in-workflow debugger.
        env:
          MOBILEGL_ITEST_REQUIRE_GPU: "1"
          MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH: "1"
          MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS: "1"
          MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER: "1"
        run: |
          ulimit -c unlimited
          sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'
          # Second, filtered pass: with the range-invalidating map flush disabled,
          # the buffer scenarios run on the upload ring's staged-copy tier - which
          # the default pass never reaches (the map tier absorbs every flush on
          # Mesa), so without this the Mali fallback tier would have zero CI
          # coverage. The flag is NOT baked into the ctest ENVIRONMENT properties,
          # so an inline env reaches the test processes (unlike the ICD pin above).
          if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
            ctest -V -L integration-gpu --no-tests=error
            MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 ctest -V -L integration-gpu \
              -R 'Buffer|Readback|Atomic|Ssbo|Arena' --no-tests=error
          else
            ctest --output-on-failure -L integration-gpu --no-tests=error
            MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 ctest --output-on-failure -L integration-gpu \
              -R 'Buffer|Readback|Atomic|Ssbo|Arena' --no-tests=error
          fi

      - name: Upload core dumps
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: integration-core-dumps
          path: /tmp/core.*
          if-no-files-found: ignore

  # THE THIRD CI MODE (ARCHITECTURE.md 13.2-(2)): the same library, built with the PipeInputs
  # comparator compiled in, running the integration suite and a trace subset with two state models
  # in one address space. It is a second build rather than a flag on the first because
  # MOBILEGL_PIPE_VERIFY is a compile-time option - the snapshot, the entry compare and the
  # compare-at-read hook do not exist in the shipped library, and are never meant to.
  build-linux-verify:
    runs-on: ubuntu-latest
    timeout-minutes: 120
    permissions:
      actions: write
      contents: read
    env:
      BUILD_DIR: build-verify
      CCACHE_BASEDIR: ${{ github.workspace }}
      CCACHE_COMPRESS: "true"
      CCACHE_DIR: ${{ github.workspace }}/.ccache
      CCACHE_MAXSIZE: 4G
      CCACHE_NOHASHDIR: "true"

    steps:
      - name: Set Swap Space
        uses: pierotofy/set-swap-space@v1.0
        with:
          swap-size-gb: 32

      - name: Checkout repo
        uses: actions/checkout@v6
        with:
          submodules: recursive

      - name: Get CMake
        uses: lukka/get-cmake@v4.3.3

      - name: Restore ccache
        uses: actions/cache/restore@v5
        with:
          path: .ccache
          key: ${{ runner.os }}-test-${{ github.job }}-ccache-v1
          restore-keys: |
            ${{ runner.os }}-test-${{ github.job }}-ccache-

      - name: Prepare Vulkan SDK
        uses: humbletim/setup-vulkan-sdk@v1.2.1
        with:
          vulkan-query-version: 1.4.304.1
          vulkan-components: Vulkan-Headers, Vulkan-Loader
          vulkan-use-cache: true

      - name: Update glslang external sources
        working-directory: 3rdparty/glslang
        run: python update_glslang_sources.py

      - name: Install build dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build

      - name: Show installed toolchain
        run: |
          ccache --version
          clang-20 --version
          clang++-20 --version
          ld.lld-20 --version || ld.lld --version || true
          dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true

      - name: Configure CMake
        # Release/INFO like the shipped build on purpose. The poison arms in this configuration
        # through MOBILEGL_PIPE_VERIFY (PipeInputs.h derives MOBILEGL_PIPE_POISON from it), so this
        # job needs neither a Debug log level nor MOBILEGL_BUILD_DISAGGREGATED - and a Debug build
        # would compare a different library from the one the other lanes measure. (build-linux
        # switches to Debug under ACTIONS_STEP_DEBUG; this job deliberately does not - a Debug
        # build flips CXX_VISIBILITY_PRESET and arms MOBILEGL_PIPE_POISON through a second, unrelated
        # arm of its #if, so the debug switch would change what the lane is measuring.)
        run: |
          cmake -S . -B "${BUILD_DIR}" -G Ninja \
            -DCMAKE_C_COMPILER=clang-20 \
            -DCMAKE_CXX_COMPILER=clang++-20 \
            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
            -DCMAKE_BUILD_TYPE=Release \
            -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
            -DMOBILEGL_BUILD_TEST=ON \
            -DMOBILEGL_BUILD_BENCHMARK=OFF \
            -DMOBILEGL_BUILD_INTEGRATION_TEST=ON \
            -DMOBILEGL_ITEST_VK_ICD=/usr/share/vulkan/icd.d/lvp_icd.json \
            -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \
            -DMOBILEGL_PIPE_VERIFY=ON \
            -DCMAKE_POLICY_VERSION_MINIMUM=3.5

      - name: Build
        run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)"

      # The lane is worthless if the option silently did not take, and that is a one-character
      # mistake away at all times (a typo'd -D is not an error in CMake). Two checks, both cheap:
      # the comparator's entry point must be in the library, and the fill entry point with it.
      #
      # `nm` and NOT `nm -D`. The library is built CXX_VISIBILITY_PRESET hidden in every non-Debug
      # configuration (CMakeLists.txt:600-604) and the MGPipe entry points are plain namespace
      # functions with no export attribute, so not one of them appears in the DYNAMIC table: on a
      # perfectly healthy verify build `nm -D --defined-only ... | grep -c MGPipe` answers 0 out of
      # ~11900 exported symbols, and a gate spelled that way is red forever for a reason that has
      # nothing to do with what it claims to test. The static symbol table has them as local `t`
      # entries, this artifact is never stripped, and `No MG_Remote in the pull build` below already
      # uses this spelling. The symbol count guards the remaining hole: a stripped library would
      # make both greps fail for a third, silent reason.
      - name: The verify library really carries the comparator
        run: |
          test -f "${BUILD_DIR}/libMobileGL.so"
          defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l)
          if [ "${defined}" -lt 1000 ]; then
            echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the two checks below could not have failed honestly"
            exit 1
          fi
          if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeVerifyInputs"; then
            echo "::error::libMobileGL.so defines no MGPipeVerifyInputs: -DMOBILEGL_PIPE_VERIFY=ON did not take, and every lane that consumes this artifact would run the comparator-free library and pass having compared nothing"
            exit 1
          fi
          # The per-verb entry point, under EITHER of its two names. P2 renames
          # MGPipeFillForVerb to MGPipeValidateForVerb (the body becomes the tracker's walk and
          # the fill is one of its five steps), so this check has to accept both or it goes red on
          # the rename for a reason that has nothing to do with what it tests. What it tests is
          # unchanged: that the library HAS a per-verb entry point compiled in.
          if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -qE "MGPipeValidateForVerb|MGPipeFillForVerb"; then
            echo "::error::libMobileGL.so defines neither MGPipeValidateForVerb nor MGPipeFillForVerb: there is no per-verb entry point in this artifact, so nothing fills the block the comparator compares"
            exit 1
          fi
          echo "libMobileGL.so defines MGPipeVerifyInputs and a per-verb entry point (${defined} defined symbols)"

      - name: Show ccache stats
        if: always()
        run: ccache --show-stats

      - name: Release superseded ccache entry
        if: github.ref_name == github.event.repository.default_branch
        env:
          GH_TOKEN: ${{ github.token }}
          CACHE_KEY: ${{ runner.os }}-test-${{ github.job }}-ccache-v1
        run: gh cache delete "${CACHE_KEY}" || true

      - name: Save ccache
        if: github.ref_name == github.event.repository.default_branch
        continue-on-error: true
        uses: actions/cache/save@v5
        with:
          path: .ccache
          key: ${{ runner.os }}-test-${{ github.job }}-ccache-v1

      - name: Package Linux verify runtime
        run: |
          mkdir -p ci-artifacts
          mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort)
          tar \
            --exclude='*/CMakeFiles' \
            --exclude='*.o' \
            --exclude='*.a' \
            --exclude='*.ninja*' \
            --exclude='build.ninja' \
            --exclude='cmake_install.cmake' \
            -czf ci-artifacts/mobilegl-linux-runtime-verify.tgz \
            "${BUILD_DIR}/CTestTestfile.cmake" \
            "${BUILD_DIR}/MobileGL/MG_Test" \
            "${BUILD_DIR}/MobileGL/MG_IntegrationTest" \
            "${SHARED_LIBS[@]}"

      - name: Upload Linux verify runtime
        uses: actions/upload-artifact@v7
        with:
          name: mobilegl-linux-runtime-verify
          path: ci-artifacts/mobilegl-linux-runtime-verify.tgz
          if-no-files-found: error

  # The verify lane itself, plus the two negative controls that keep it falsifiable. The controls
  # are ALWAYS-ON steps, not a manual exercise: a gate that can only be shown to work by someone
  # remembering to break it on purpose is a gate that has already stopped working.
  integration-verify:
    runs-on: ubuntu-latest
    timeout-minutes: 180
    needs: build-linux-verify

    steps:
      - name: Checkout repo
        uses: actions/checkout@v6

      - name: Get CMake
        uses: lukka/get-cmake@v4.3.3

      - name: Install runtime dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers

      - name: Download Linux verify runtime
        uses: actions/download-artifact@v8
        with:
          name: mobilegl-linux-runtime-verify
          path: .

      - name: Unpack Linux verify runtime
        run: |
          tar -xzf mobilegl-linux-runtime-verify.tgz
          test -f build-verify/libMobileGL.so

      - name: Normalize CTest command paths
        run: |
          python - <<'PY'
          from pathlib import Path
          import re

          for path in Path('build-verify').rglob('CTestTestfile.cmake'):
              text = path.read_text()
              text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text)
              path.write_text(text)
          PY

      - name: Integration scenarios under MOBILEGL_PIPE_VERIFY
        working-directory: build-verify
        # --no-tests=error is half the gate: the verify entries only exist when the library was
        # configured with -DMOBILEGL_PIPE_VERIFY=ON, so a build that lost the option matches no
        # tests and reds here instead of reporting a green run of nothing. The other half is
        # PipeVerifyArmingScenario.Armed, which fails when the library never printed its arming
        # line - the failure mode a bare `MOBILEGL_PIPE_VERIFY=1` cannot detect by itself.
        #
        # SCOPE, stated so nobody reads more into a green than is there: this is every integration
        # ENTRY under the comparator, not every integration CONFIGURATION. The `integration` job
        # runs a second, filtered pass with MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 for the
        # upload ring's staged-copy tier; that pass is 186 entries here and, at the 5-10x the
        # comparator costs, is not affordable inside this job's budget. The tier is covered by
        # `integration`, unverified, and P2 can take it once the comparator's cost is known.
        env:
          MOBILEGL_ITEST_REQUIRE_GPU: "1"
          MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH: "1"
          MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS: "1"
          MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER: "1"
        run: |
          ulimit -c unlimited
          sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'
          if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
            ctest -V -L integration-verify --no-tests=error
          else
            ctest --output-on-failure -L integration-verify --no-tests=error
          fi

      # The push-only unit tests, on the verify runtime.
      #
      # WHY HERE AND NOT IN `test`. The entries themselves are registered in EVERY build - they
      # have to be, or `ctest -N` would stop matching name-for-name between the pull and the push
      # build (gate G2). What is push-only is what they assert about: MGPipeRenderStateSpans.cpp
      # and PipeApply.cpp are appended to SOURCE_FILES inside the `if (MOBILEGL_PIPE_PUSH)` block,
      # which is exactly how the pull build stays symbol-identical, so in a pull build each case
      # opens with `#if !MOBILEGL_PIPE_PUSH GTEST_SKIP() << "push not compiled in"`. The `test`
      # job therefore runs G6's chunk-table walk and G10's residual assertions as a column of
      # skips: CI executes the NAMES and never one of the assertions. This job unpacks a build
      # that compiled them, so it is the first place in CI where they actually run.
      #
      # This artifact already carries them - the packaging step above tars
      # ${BUILD_DIR}/MobileGL/MG_Test whole - so the whole cost is the run, which is ~14 s for
      # ~1490 entries. --no-tests=error, because a packaging change that stopped shipping the
      # unit binaries would otherwise report a green run of nothing.
      - name: Unit tests on the verify runtime (G6, G10)
        working-directory: build-verify
        run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)"

      # The always-on P2 and P3a negative controls (G8, G10, G12), which are labelled
      # integration-gpu and not integration-verify - they are about the handle key, the subsystem
      # switch and the map-persistent counter, not about the comparator - so the lane above does
      # not reach them. They are run HERE because this is the only CI job that unpacks a
      # MOBILEGL_PIPE_PUSH build: every one of them reads a counter out of the library's summary
      # line, and both the counters and their brackets are #if MOBILEGL_PIPE_PUSH, so in the pull
      # `integration` job the entries exist (gate G2 requires the same names in both builds) but
      # have nothing to assert.
      #
      # An arm whose subsystem has not landed on this tree SKIPS with the reason (never absent,
      # never a green that asserted nothing), so this step is green through the P2 and P3a landing
      # orders and starts asserting as each package arrives.
      #
      # The environment is the sibling step's, deliberately and in full: these entries run the
      # same DirectVulkan binary through the same runner, so the three MOBILEGL_MAGMA_* fixes it
      # needs apply here too, and a crash here has to leave a core for the same black-box flow.
      # The step above is the only reason those lines exist in this job; a control that crashed
      # without one would be the hardest failure in the job to diagnose.
      #
      # THE -R ALTERNATIVES ARE TEST-NAME PREFIXES, NOT LANE LABELS, and each one is deliberately
      # the SHORTEST string that still selects only what it means to. `ResourceSubsystem` (not
      # `ResourceSubsystemControl`) is what reaches the eight
      # DirectGLES.ResourceSubsystemOn./Off.LargeArenaAdoptionScenario.* entries - the A/B lanes
      # whose entire purpose is that the handle path and the legacy BufferBackendOps path must
      # agree about an adopted store - as well as the two ResourceSubsystemControl. entries.
      # `MapPersistentRoundtrip` is singular because LargeArenaAdoptionScenario's case is
      # `AnAdoptionCostsExactlyOneMapPersistentRoundtrip`; the plural matched only the lane PREFIX
      # of the other one. Both mistakes were silent: this is the only CI job that unpacks a push
      # build, so an entry the filter misses is either never run under the P3a bits at all or runs
      # only in the pull `integration` job, where a MOBILEGL_PIPE_PUSH value steers nothing
      # (Config.h declares the field inside the push guard) and both arms are the same legacy path.
      # A lane that cannot go red where it is installed is not a gate (ROADMAP.md:7).
      # P4a ADDS THREE ALTERNATIVES, and each one is here because this is the only CI job that
      # unpacks a push build:
      #   * `ObjectSubsystem` reaches the three DirectGLES.ObjectSubsystemControl. entries - the
      #     0x1fff-vs-0x1ff A/B and the 0x9ff dependency refusal (G12). `ResourceSubsystem` does
      #     NOT match it: the two families are named apart on purpose, because they are different
      #     phases' switches and a filter that merged them would hide one behind the other.
      #   * `TextureParamsWithoutASamplerView` reaches G9's four cases, the scenario ROADMAP.md:20
      #     names by hand. It runs in the ambient lanes, which the label already selects - but this
      #     step is where those cases run against a PUSH library, and G9's whole claim is about the
      #     push path. All four cases are green on the contract commit - including the one D-E3
      #     expected to be red, for the reason the scenario's header records - so this row is green
      #     from the day it lands and goes red only if a reachability path stops syncing texture
      #     parameters at all, which is the coupling ARCHITECTURE.md:100 exists to remove.
      #   * `TextureUploadShape` is RECORDED, not gated (D-D4): it asserts that the two upload-shape
      #     counters could be read and that they agree, and prints the shape for MEASUREMENTS.md. It
      #     is in the filter so that the number is actually collected on every run - an unmeasured
      #     shape is not a recorded one - and because its own assertions can go red.
      # As with the four before them, each alternative is the SHORTEST string that selects only what
      # it means to.
      - name: The handle-ABA, CSO, subsystem and texture-parameter controls (G8, G8b, G9, G10, G12)
        working-directory: build-verify
        env:
          MOBILEGL_ITEST_REQUIRE_GPU: "1"
          MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH: "1"
          MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS: "1"
          MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER: "1"
        run: |
          ulimit -c unlimited
          sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'
          ctest --output-on-failure -L integration-gpu \
            -R 'HandleRecycle|CsoContentAddressing|ResourceSubsystem|MapPersistentRoundtrip|ObjectSubsystem|TextureParamsWithoutASamplerView|TextureUploadShape' \
            --no-tests=error -j 4

      # The arming lanes' logs, and ONLY those. Each lane shares one MOBILEGL_LOG_FILE_PATH and the
      # library opens it fopen(path, "w"), so after an ambient lane of 400-odd processes the file
      # holds the LAST one - grepping it would say nothing about the other 405 and would red a
      # healthy lane whenever the last entry happened not to issue a verb (which is what the
      # PoisonOmissionScenario parent, the last ambient entry, does by construction: it forks,
      # execve()s and reads files). The DirectGLES.VerifyArming. / DirectVulkan.VerifyArming.
      # entries are one process each on a log path nothing else writes, so this grep means exactly
      # what it says.
      #
      # What it proves: arming is a property of (this library, this environment), and these two
      # processes ran the same library with the same MOBILEGL_PIPE_VERIFY=1 as their ~400 ambient
      # siblings. It is not, and cannot be, a per-process census - the shared log cannot support one.
      # It catches the case ctest cannot: an arming entry that SKIPPED still reports green.
      - name: The verify lanes armed the comparator
        working-directory: build-verify
        run: |
          shopt -s nullglob
          logs=(MobileGL/MG_IntegrationTest/pipe-verify-arming-*.log)
          if [ ${#logs[@]} -lt 2 ]; then
            echo "::error::found ${#logs[@]} pipe-verify-arming-*.log (expected one per backend). The VerifyArming. entries did not run, so nothing in this job establishes that the comparator was ever armed."
            exit 1
          fi
          for log in "${logs[@]}"; do
            if ! grep -q "MGPipe: verify armed" "${log}"; then
              echo "::error::${log} carries no arming line: that lane's process ran the whole scenario without the comparator, so every green entry beside it is green for no reason"
              exit 1
            fi
          done
          echo "arming line present in all ${#logs[@]} arming-lane log(s)"

      # NEGATIVE CONTROL A (gate G4). The knob perturbs one field in the snapshot arm before the
      # entry compare, so a working comparator must abort the run. This step passes when ctest
      # FAILS - `if ctest ...; then error` - which is the only shape that can catch a comparator
      # that silently compares nothing.
      #
      # The knob reaches the test process through the JOB environment: no ctest ENVIRONMENT
      # property on the ambient Verify. entries names it (MG_IntegrationTest/CMakeLists.txt says
      # so out loud), and a property entry would otherwise override this and the control would
      # prove nothing. Same precedent as MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH in `integration`.
      - name: Negative control A - a corrupted snapshot field must turn the lane red
        working-directory: build-verify
        env:
          MOBILEGL_ITEST_REQUIRE_GPU: "1"
          MOBILEGL_PIPE_VERIFY_CORRUPT: GetRenderStateParameters
        run: |
          FILTER='DirectGLES\.Verify\..*ClearThenReadPixels'
          # An empty selection would ALSO make ctest exit non-zero (--no-tests=error), and this
          # step reads non-zero as "the control worked" - so the selection is counted first. A
          # control that passes because it ran nothing is worse than no control.
          matched=$(ctest -N -L integration-verify -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:')
          if [ "${matched}" -lt 1 ]; then
            echo "::error::negative control A selected ${matched} tests; its filter no longer matches anything"
            exit 1
          fi
          if ctest --output-on-failure -L integration-verify -R "${FILTER}" --no-tests=error; then
            echo "::error::MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left ${matched} verify entries GREEN. The comparator is not comparing, so every green entry above is green for no reason."
            exit 1
          fi
          echo "the corrupted field turned ${matched} selected entries red, as it must"

      # NEGATIVE CONTROL B (gate G5). The omission skips the STAMP of one field for one verb while
      # still copying its value - indistinguishable from a fill row nobody wrote - so the poison
      # must abort the glGenerateMipmap. Again: this step passes when ctest fails.
      #
      # The entry it targets is PoisonOmissionScenario.WithoutOmissionCompletes, which is green in
      # the ambient lane above and is the ONLY integration entry in the tree that calls
      # glGenerateMipmap at all. It deliberately does not skip itself when the knob is set, exactly
      # so that this control has something to turn red.
      - name: Negative control B - an omitted fill point must turn the lane red on that verb
        working-directory: build-verify
        env:
          MOBILEGL_ITEST_REQUIRE_GPU: "1"
          MOBILEGL_PIPE_POISON_OMIT: GenerateMipmap:GetActiveTextureUnit
        run: |
          FILTER='DirectGLES\.Verify\.PoisonOmissionScenario\.WithoutOmissionCompletes'
          matched=$(ctest -N -L integration-verify -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:')
          if [ "${matched}" -lt 1 ]; then
            echo "::error::negative control B selected ${matched} tests; its filter no longer matches anything"
            exit 1
          fi
          if ctest --output-on-failure -L integration-verify -R "${FILTER}" --no-tests=error; then
            echo "::error::MOBILEGL_PIPE_POISON_OMIT=GenerateMipmap:GetActiveTextureUnit left the verify lane GREEN. The per-verb poison is not armed, so a forgotten fill row would ship silently."
            exit 1
          fi
          echo "the omitted fill point turned the lane red, as it must"

      - name: Upload verify lane logs
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: integration-verify-logs
          path: build-verify/MobileGL/MG_IntegrationTest/pipe-*.log*
          if-no-files-found: warn

      - name: Upload core dumps
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: integration-verify-core-dumps
          path: /tmp/core.*
          if-no-files-found: ignore

  # P5's split build. The verify pair cloned, because the problem is the same one: a third
  # configuration of the same sources whose whole value depends on the option having taken.
  #
  # IT CANNOT RIDE ON build-linux'S ARTIFACT. build-linux passes no -DMOBILEGL_PIPE_PUSH, so it is
  # a PULL build; MOBILEGL_BUILD_DISAGGREGATED implies MOBILEGL_PIPE_PUSH (the split path decodes
  # into the MGPipe applier and PIPE_PUSH is what compiles the applier), so the split arm needs a
  # build of its own exactly as the verify arm does.
  #
  # THE `nm` STEP IS THE POINT OF THIS JOB. In a build without the option, MG_Remote is not
  # compiled at all and MOBILEGL_TRANSPORT's parser does not exist - so the variable is accepted by
  # the environment and silently ignored (CONTRACT-P5 5). Every downstream lane would then run
  # monolith and go green under a name that says split. CMake will not complain about a typo'd -D,
  # so the build-level assertion is the only guard, and its absence is precisely what would make
  # the whole split arm meaningless.
  build-linux-split:
    runs-on: ubuntu-latest
    timeout-minutes: 120
    permissions:
      actions: write
      contents: read
    env:
      BUILD_DIR: build-split
      CCACHE_BASEDIR: ${{ github.workspace }}
      CCACHE_COMPRESS: "true"
      CCACHE_DIR: ${{ github.workspace }}/.ccache
      CCACHE_MAXSIZE: 4G
      CCACHE_NOHASHDIR: "true"

    steps:
      - name: Set Swap Space
        uses: pierotofy/set-swap-space@v1.0
        with:
          swap-size-gb: 32

      - name: Checkout repo
        uses: actions/checkout@v6
        with:
          # recursive, and load-bearing here beyond the usual: MOBILEGL_BUILD_DISAGGREGATED
          # SHADOWS ITSELF BACK TO OFF when 3rdparty/flatbuffers/include is missing
          # (CMakeLists.txt:471-486, a normal variable rather than a cache force, deliberately).
          # A shallow checkout would therefore configure cleanly, build a monolith library, and
          # be caught only by the nm step below.
          submodules: recursive

      - name: Get CMake
        uses: lukka/get-cmake@v4.3.3

      - name: Restore ccache
        uses: actions/cache/restore@v5
        with:
          path: .ccache
          key: ${{ runner.os }}-test-${{ github.job }}-ccache-v1
          restore-keys: |
            ${{ runner.os }}-test-${{ github.job }}-ccache-

      - name: Prepare Vulkan SDK
        uses: humbletim/setup-vulkan-sdk@v1.2.1
        with:
          vulkan-query-version: 1.4.304.1
          vulkan-components: Vulkan-Headers, Vulkan-Loader
          vulkan-use-cache: true

      - name: Update glslang external sources
        working-directory: 3rdparty/glslang
        run: python update_glslang_sources.py

      - name: Install build dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build

      - name: Show installed toolchain
        run: |
          ccache --version
          clang-20 --version
          clang++-20 --version
          dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true

      - name: Configure CMake
        # Release/INFO like the shipped build, for the same reason build-linux-verify gives.
        # INFO specifically matters here: ConfigLoader logs the resolved transport at INFO and
        # that line is what run_trace_case.cmake and the integration-split lane read back as
        # proof the transport resolved in THIS process.
        #
        # _INPROC implies _DISAGGREGATED implies _PIPE_PUSH; all three are passed anyway, because
        # an implication that is asserted in two places is an implication nobody has to remember.
        run: |
          cmake -S . -B "${BUILD_DIR}" -G Ninja \
            -DCMAKE_C_COMPILER=clang-20 \
            -DCMAKE_CXX_COMPILER=clang++-20 \
            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
            -DCMAKE_BUILD_TYPE=Release \
            -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
            -DMOBILEGL_BUILD_TEST=ON \
            -DMOBILEGL_BUILD_BENCHMARK=OFF \
            -DMOBILEGL_BUILD_INTEGRATION_TEST=ON \
            -DMOBILEGL_ITEST_VK_ICD=/usr/share/vulkan/icd.d/lvp_icd.json \
            -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \
            -DMOBILEGL_PIPE_PUSH=ON \
            -DMOBILEGL_BUILD_DISAGGREGATED=ON \
            -DMOBILEGL_BUILD_DISAGGREGATED_INPROC=ON \
            -DCMAKE_POLICY_VERSION_MINIMUM=3.5

      - name: Build
        run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)"

      # `nm` and NOT `nm -D`, for the reason build-linux-verify spells out: everything under
      # MG_Remote is a plain namespace symbol in a CXX_VISIBILITY_PRESET=hidden Release build and
      # none of it reaches the dynamic table. The symbol count guards the remaining hole - a
      # stripped library would make the grep fail for a third, silent reason.
      #
      # The mirror of this assertion already exists and is the G1 control: monolith-symbol-report
      # asserts that a -DMOBILEGL_BUILD_DISAGGREGATED=OFF library defines NO MG_Remote symbol. The
      # two together are ARCHITECTURE.md:524's surviving byte-level equality, in both directions.
      - name: The split library really carries MG_Remote
        run: |
          test -f "${BUILD_DIR}/libMobileGL.so"
          defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l)
          if [ "${defined}" -lt 1000 ]; then
            echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the checks below could not have failed honestly"
            exit 1
          fi
          remote=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -c -i "MG_Remote" || true)
          if [ "${remote}" -lt 1 ]; then
            echo "::error::libMobileGL.so defines no MG_Remote symbol: -DMOBILEGL_BUILD_DISAGGREGATED=ON did not take (a typo'd -D is not a CMake error, and the option shadows itself OFF when 3rdparty/flatbuffers/include is missing). Every lane that consumes this artifact would run MONOLITH while claiming to run split, because the MOBILEGL_TRANSPORT parser does not exist in such a build and the variable is accepted and ignored."
            exit 1
          fi
          # The transport parser itself, which is the symbol the runtime evidence depends on.
          if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeApply"; then
            echo "::error::libMobileGL.so defines no MGPipeApply* entry point, so MOBILEGL_PIPE_PUSH did not take either and there is no applier for the split path to decode into"
            exit 1
          fi
          echo "libMobileGL.so defines ${remote} MG_Remote symbol(s) and the MGPipe applier (${defined} defined symbols)"

      - name: Show ccache stats
        if: always()
        run: ccache --show-stats

      - name: Release superseded ccache entry
        if: github.ref_name == github.event.repository.default_branch
        env:
          GH_TOKEN: ${{ github.token }}
          CACHE_KEY: ${{ runner.os }}-test-${{ github.job }}-ccache-v1
        run: gh cache delete "${CACHE_KEY}" || true

      - name: Save ccache
        if: github.ref_name == github.event.repository.default_branch
        continue-on-error: true
        uses: actions/cache/save@v5
        with:
          path: .ccache
          key: ${{ runner.os }}-test-${{ github.job }}-ccache-v1

      - name: Package Linux split runtime
        run: |
          mkdir -p ci-artifacts
          mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort)
          tar \
            --exclude='*/CMakeFiles' \
            --exclude='*.o' \
            --exclude='*.a' \
            --exclude='*.ninja*' \
            --exclude='build.ninja' \
            --exclude='cmake_install.cmake' \
            -czf ci-artifacts/mobilegl-linux-runtime-split.tgz \
            "${BUILD_DIR}/CTestTestfile.cmake" \
            "${BUILD_DIR}/MobileGL/MG_Test" \
            "${BUILD_DIR}/MobileGL/MG_IntegrationTest" \
            "${SHARED_LIBS[@]}"

      - name: Upload Linux split runtime
        uses: actions/upload-artifact@v7
        with:
          name: mobilegl-linux-runtime-split
          path: ci-artifacts/mobilegl-linux-runtime-split.tgz
          if-no-files-found: error

  # The split lane itself: the three-arm shape ARCHITECTURE.md:521 asks for verbatim, plus the
  # first real run of the five MG_Test/Wire suites, plus P5's two exit-gate negative controls.
  integration-split:
    runs-on: ubuntu-latest
    timeout-minutes: 120
    needs: build-linux-split

    steps:
      - name: Checkout repo
        uses: actions/checkout@v6

      - name: Get CMake
        uses: lukka/get-cmake@v4.3.3

      - name: Install runtime dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers

      - name: Download Linux split runtime
        uses: actions/download-artifact@v8
        with:
          name: mobilegl-linux-runtime-split
          path: .

      - name: Unpack Linux split runtime
        run: |
          tar -xzf mobilegl-linux-runtime-split.tgz
          test -f build-split/libMobileGL.so

      - name: Normalize CTest command paths
        run: |
          python - <<'PY'
          from pathlib import Path
          import re

          for path in Path('build-split').rglob('CTestTestfile.cmake'):
              text = path.read_text()
              text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text)
              path.write_text(text)
          PY

      # THE FIRST REAL RUN OF MG_Test/Wire. Those five suites - Framing, Ring, InProcessTransport,
      # ProtocolSmoke and FdPassing - are registered only under MOBILEGL_BUILD_DISAGGREGATED
      # (MG_Test/CMakeLists.txt), and before this job existed NO cmake invocation anywhere in this
      # workflow passed that option. They had never been compiled by CI, let alone run.
      - name: Unit tests on the split runtime
        working-directory: build-split
        run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)"

      # --no-tests=error is half the gate, exactly as in integration-verify: the integration-split
      # entries exist only when the library was configured with -DMOBILEGL_BUILD_DISAGGREGATED=ON,
      # so a build that lost the option matches nothing and reds here instead of reporting a green
      # run of nothing. The other half is the transport-resolution check below.
      #
      # An entry whose owning package (c1 client, s1 session, v1 server) has not landed SKIPS with
      # the reason and never goes green - MG_IntegrationTest/CMakeLists.txt probes MG_Remote for
      # c0's signature stubs and disarms the lane while any remain.
      - name: Split scenarios under MOBILEGL_TRANSPORT=inproc
        working-directory: build-split
        env:
          MOBILEGL_ITEST_REQUIRE_GPU: "1"
        run: |
          ulimit -c unlimited
          sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'
          ctest --output-on-failure -L integration-split --no-tests=error -j 4

      # ARCHITECTURE.md:521 asks for `ctest -L integration-gpu` to be name-for-name identical
      # between the monolith and the split arm of the SAME build - the G2 shape extended to a
      # third arm. The knob goes in the JOB environment rather than in a ctest property, for the
      # reason the MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH precedent in `integration` gives: a
      # property would override it and the arm would not be an arm.
      #
      # The entries that name MOBILEGL_TRANSPORT in their OWN property (the Split. lanes) keep
      # their value in both passes, which is correct: they are the split family in both arms and
      # the comparison is about the other 1100.
      - name: The same integration entries under monolith and under inproc
        working-directory: build-split
        env:
          MOBILEGL_ITEST_REQUIRE_GPU: "1"
          MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH: "1"
          MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS: "1"
          MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER: "1"
        run: |
          ulimit -c unlimited
          sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'
          count=$(ctest -N -L integration-gpu | grep -cE '^ *Test *#[0-9]+:')
          if [ "${count}" -lt 1 ]; then
            echo "::error::the split runtime registers ${count} integration-gpu entries"
            exit 1
          fi
          echo "integration-gpu entries in the split build: ${count}"
          MOBILEGL_TRANSPORT=monolith ctest --output-on-failure -L integration-gpu --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/arm-monolith.xml"
          MOBILEGL_TRANSPORT=inproc   ctest --output-on-failure -L integration-gpu --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/arm-inproc.xml"
          # NAME **AND STATUS**, and it is the comparison this step claimed to make and did not
          # (review finding N-3): the first version wrote a names file and never read it, and
          # `--output-on-failure` treats a SKIPPED test as not-a-failure - so the very failure
          # ARCHITECTURE.md:521 is about, "an inproc arm that skipped forty entries the monolith arm
          # ran", was invisible here and caught only by the local gate. `ctest -N` cannot see it
          # either: this is one build directory, so the two arms have identical name lists by
          # construction and the difference is entirely in what each entry DID.
          python3 - "${RUNNER_TEMP}/arm-monolith.xml" "${RUNNER_TEMP}/arm-inproc.xml" <<'PY'
          import sys, xml.etree.ElementTree as ET
          def rows(path):
              out = {}
              for case in ET.parse(path).getroot().iter('testcase'):
                  status = 'passed'
                  if case.find('failure') is not None or case.find('error') is not None:
                      status = 'failed'
                  elif case.find('skipped') is not None or case.get('status') in ('notrun', 'disabled'):
                      status = 'skipped'
                  out[case.get('name')] = status
              return out
          a, b = rows(sys.argv[1]), rows(sys.argv[2])
          diff = sorted(set(a) ^ set(b)) + sorted(n for n in set(a) & set(b) if a[n] != b[n])
          if diff:
              for name in diff[:40]:
                  print(f"::error::{name}: monolith={a.get(name, '<absent>')} inproc={b.get(name, '<absent>')}")
              print(f"::error::the monolith and inproc arms of ctest -L integration-gpu differ on "
                    f"{len(diff)} entries. ARCHITECTURE.md:521 requires them identical name for name "
                    f"AND status; an entry that SKIPPED on one arm and ran on the other is the "
                    f"failure this compares for, and it is not a failure to --output-on-failure.")
              raise SystemExit(1)
          print(f"the two arms agree on all {len(a)} entries, name and status")
          PY

      # THE RUNTIME HALF OF "THIS IS REALLY A SPLIT BUILD". The build-level nm check in
      # build-linux-split proves the library CARRIES MG_Remote; this proves the transport
      # RESOLVED in a process of this lane. ConfigLoader::InitTransport logs one INFO line when it
      # selects InProcess, and the DirectGLES.Split.PersistentMapArm. entry is the one Split entry
      # with a MOBILEGL_LOG_FILE_PATH of its own (nothing else writes it, so the grep means what it
      # says). The line is written during bring-up, before any scenario decides to skip, so this
      # check is live from the day the lanes land rather than from the day they stop skipping.
      - name: The split lane really resolved the transport
        working-directory: build-split
        run: |
          log=MobileGL/MG_IntegrationTest/persistent-map-arm-split-DirectGLES.log
          if [ ! -f "${log}" ]; then
            echo "::error::${log} does not exist: the DirectGLES.Split.PersistentMapArm. entry never ran, so nothing in this job establishes that MOBILEGL_TRANSPORT ever resolved to inproc in a live process"
            exit 1
          fi
          # THE DISTINCTIVE PART OF THE INFO LINE, not the bare KEY=VALUE (review finding M-5):
          # ConfigLoader logs `Config: Accepted env variable: MOBILEGL_TRANSPORT=inproc` for the
          # env dump too, unconditionally and in a PULL build, and at DEBUG that line is live.
          if ! grep -q "MOBILEGL_TRANSPORT=inproc - the MGPipe record stream" "${log}"; then
            echo "::error::${log} carries no transport-resolution line. ConfigLoader::InitTransport logs it at INFO when it selects InProcess, and that code exists only in a MOBILEGL_BUILD_DISAGGREGATED build - so this lane ran a monolith library while claiming to be the split lane. (A bare MOBILEGL_TRANSPORT=inproc substring is NOT accepted: the env dump prints one in every build.)"
            exit 1
          fi
          echo "the split lane resolved MOBILEGL_TRANSPORT=inproc"

      # EXIT GATE E1's NEGATIVE CONTROL and EXIT GATE E3(a)'s, in one step because they have the
      # same two-state shape and the same reason for it.
      #
      # Both controls turn a knob that MUST make a split scenario red: MOBILEGL_IPC_VERB_BARRIER=0
      # removes the lockstep fence R-1 rests on, and MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0 turns the
      # persistent-map push off. A control of the verify lane's shape - "this step passes when
      # ctest FAILS" - cannot be written yet, because while packages c1/s1/v1 are landing the
      # Split entries SKIP and ctest reports green whatever the knob says, so an unconditional
      # control would be red for the whole of P5 for a reason that is not a defect.
      #
      # So the expected state is DERIVED rather than assumed, from the same fact the lanes derive
      # it from: MG_IntegrationTest/CMakeLists.txt puts MGITEST_REMOTE_CLIENT_PRESENT=1 into the
      # Split entries' ENVIRONMENT exactly when MG_Remote carries no c0 signature stub, and that
      # string is in the generated ctest include files this artifact ships. When it is there the
      # controls MUST fire; when it is not, the step says so loudly and does not pretend.
      - name: Negative controls - the verb barrier and the persistent-map push must be load-bearing
        working-directory: build-split
        env:
          MOBILEGL_ITEST_REQUIRE_GPU: "1"
        run: |
          # THE ARMED STATE IS DERIVED FROM BEHAVIOUR, not from a marker string in the generated
          # ctest files. The first version read MGITEST_REMOTE_CLIENT_PRESENT out of
          # *_tests.cmake, which was a restatement of the CMake source probe review finding M-1
          # falsified; the arming condition is now a runtime fact inside each test process, so the
          # only honest way to ask it from a shell is to look at what the entries DID.
          ctest -L integration-split -j 4 --no-tests=error --output-junit "${RUNNER_TEMP}/isplit.xml" || true
          armed=$(python3 - "${RUNNER_TEMP}/isplit.xml" <<'PY'
          import sys, xml.etree.ElementTree as ET
          ran = 0
          for case in ET.parse(sys.argv[1]).getroot().iter('testcase'):
              if case.find('skipped') is None and case.get('status') not in ('notrun', 'disabled'):
                  ran += 1
          print(ran)
          PY
          )
          echo "split entries that actually ran: ${armed}"
          if [ "${armed}" -lt 1 ]; then
            echo "::warning::every DirectGLES.Split. entry SKIPPED, so neither negative control can fire. The arming condition is a runtime fact - MG_Config::Transport, ClientSession::Active() and ImplementedVerbCount(), read by Harness/SplitRuntimePeek - and it becomes true on the commit that lands the last of c1/s1/v1. This step becomes a gate then, with no edit; it is not a green that asserted anything today."
            exit 0
          fi
          run_control() {
            name="$1"; filter="$2"; shift 2
            matched=$(ctest -N -L integration-split -R "${filter}" | grep -cE '^ *Test *#[0-9]+:')
            if [ "${matched}" -lt 1 ]; then
              echo "::error::${name} selected ${matched} tests; its filter no longer matches anything"
              exit 1
            fi
            if env "$@" ctest --output-on-failure -L integration-split -R "${filter}" --no-tests=error; then
              echo "::error::${name} left ${matched} split entries GREEN, so the knob it turns is not load-bearing and the gate it controls proves nothing."
              exit 1
            fi
            echo "${name} turned ${matched} selected entries red, as it must"
          }
          # E1: R-1's lockstep verb barrier. Without it the client keeps pulling fields from a live
          # GLContext while the server runs ahead, so the server reads future values.
          run_control "negative control E1 (MOBILEGL_IPC_VERB_BARRIER=0)" \
            'DirectGLES\.Split\.(Triangle|ClearThenReadPixels)' MOBILEGL_IPC_VERB_BARRIER=0
          # E3(a): the persistent-map push. 0 is admitted by ConfigLoader on purpose and is
          # documented there as this control.
          run_control "negative control E3(a) (MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0)" \
            'DirectGLES\.Split\.PersistentCoherentMapScenario' MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0

      - name: Upload split lane logs
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: integration-split-logs
          path: build-split/MobileGL/MG_IntegrationTest/*.log*
          if-no-files-found: warn

      - name: Upload core dumps
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: integration-split-core-dumps
          path: /tmp/core.*
          if-no-files-found: ignore

  # MobileGL/MG_Remote/Protocol/generated/protocol_generated.h is COMMITTED, and
  # flatc is deliberately absent from the default build graph (a codegen step in
  # the graph is how the earlier branch ended up cross-compiling an arm64 flatc
  # and trying to run it on the host). This job is what keeps the committed
  # header honest: build the pinned flatc, regenerate, and fail on any diff.
  # It needs no MobileGL build, so it does not depend on build-linux.
  flatc-check:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repo
        uses: actions/checkout@v6

      - name: Get CMake
        uses: lukka/get-cmake@v4.3.3

      - name: Check out the FlatBuffers submodule only
        # Just this one: the schema check has nothing to do with glslang,
        # SPIRV-Cross or the trace fixtures.
        run: git submodule update --init 3rdparty/flatbuffers

      - name: Regenerate protocol_generated.h
        run: python3 scripts/gen_protocol.py --build-dir "${{ runner.temp }}/flatc-build"

      - name: Fail if the committed header is stale
        run: git diff --exit-code -- MobileGL/MG_Remote/Protocol/generated/protocol_generated.h

  # P0.5 interface-purity gate A (ARCHITECTURE.md:501): the two extracted headers' include closure,
  # asserted on `-H` output because `nm --undefined-only` is blind to "included but not called" -
  # a header whose types are never named leaves no symbol behind, and "included at all" is exactly
  # the coupling P1 and P7 have to sever. Needs a preprocessor and three header submodules, no
  # CMake configure and no glslang sources, so like pipe-gates it does not depend on build-linux.
  # The script's own --self-test is always on: a negative control that stopped tripping fails the
  # job, because a gate that cannot go red is not a gate (ROADMAP.md:7).
  include-graph-check:
    name: Include-closure purity gate
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repo
        uses: actions/checkout@v6

      - name: Check out the three header submodules the closure needs
        # ska/flat_hash_map.hpp, xxhash.h and vulkan/vulkan.h are the only submodule headers
        # Includes.h reaches; glslang and spirv_cross are vendored under include/.
        run: git submodule update --init include/ska 3rdparty/xxHash 3rdparty/Vulkan-Headers

      - name: Install clang and the X11 headers vulkan.h pulls on Linux
        # Includes.h defines VK_USE_PLATFORM_XLIB_KHR before <vulkan/vulkan.h>, which then
        # includes <X11/Xlib.h>; without libx11-dev every clang-mode probe dies in the
        # preprocessor and the gate reports 5 problems that have nothing to do with purity.
        run: sudo apt-get update && sudo apt-get install -y clang-20 libx11-dev

      - name: Include-closure assertions and negative control
        # --expect-probes 4 (contract-v2.md 7.6): an exit code cannot tell four probes from none,
        # so a --probe typo or a manifest edit that selected nothing would run zero probes and
        # exit 0 - the second half of the finding that added the flag. The count is the length of
        # scripts/check_include_closure.py's PROBES list and changing one means changing the other.
        #
        # --compiler stays clang++-20, which is what the step above installs (Debian's clang-20
        # package ships /usr/bin/clang++-20). It is deliberately NOT the bare `clang++` the local
        # campaign gate spells: that spelling exists because the WSL box has no clang++-20, and
        # copying it here would trade a version-pinned compiler for whatever the runner has.
        run: python3 scripts/check_include_closure.py --mode both --compiler clang++-20 --self-test --require-all --expect-probes 4

  benchmark:
    runs-on: ubuntu-latest
    needs: build-linux

    steps:
      - name: Checkout repo
        uses: actions/checkout@v6

      - name: Get CMake
        uses: lukka/get-cmake@v4.3.3

      - name: Install runtime dependencies
        # libegl-mesa0 is the EGL vendor library itself: DriverBench brings up a
        # real GL context, and libegl1 is only glvnd's dispatch. It normally
        # arrives as a Recommends of libegl1, which is too quiet a dependency for
        # the one job that needs a working driver.
        run: |
          sudo apt-get update
          sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers

      - name: Download Linux runtime
        uses: actions/download-artifact@v8
        with:
          name: mobilegl-linux-runtime
          path: .

      - name: Unpack Linux runtime
        run: tar -xzf mobilegl-linux-runtime.tgz

      - name: Normalize CTest command paths
        run: |
          python - <<'PY'
          from pathlib import Path
          import re

          for path in Path('build-linux').rglob('CTestTestfile.cmake'):
              text = path.read_text()
              text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text)
              path.write_text(text)
          PY

      - name: Benchmark
        working-directory: build-linux
        run: |
          ulimit -c unlimited
          sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'
          ctest -V -C Release -L benchmark --no-tests=error

      - name: Upload core dumps
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: benchmark-core-dumps
          path: /tmp/core.*
          if-no-files-found: ignore

  build-retrace:
    runs-on: ubuntu-latest
    needs:
      - build-linux
      - test
      - benchmark
      - integration
    permissions:
      actions: write
      contents: read
    env:
      BUILD_DIR: build-retrace
      CCACHE_BASEDIR: ${{ github.workspace }}
      CCACHE_COMPRESS: "true"
      CCACHE_DIR: ${{ github.workspace }}/.ccache
      CCACHE_MAXSIZE: 4G
      CCACHE_NOHASHDIR: "true"
      MOBILEGL_LIBRARY: ${{ github.workspace }}/build-linux/libMobileGL.so

    steps:
      - name: Set Swap Space
        uses: pierotofy/set-swap-space@v1.0
        with:
          swap-size-gb: 32

      - name: Checkout repo
        uses: actions/checkout@v6
        with:
          submodules: recursive

      - name: Get CMake
        uses: lukka/get-cmake@v4.3.3

      - name: Restore ccache
        uses: actions/cache/restore@v5
        with:
          path: .ccache
          key: ${{ runner.os }}-test-${{ github.job }}-ccache-v1
          restore-keys: |
            ${{ runner.os }}-test-${{ github.job }}-ccache-

      - name: Prepare Vulkan SDK
        uses: humbletim/setup-vulkan-sdk@v1.2.1
        with:
          vulkan-query-version: 1.4.304.1
          vulkan-components: Vulkan-Headers, Vulkan-Loader
          vulkan-use-cache: true

      - name: Update glslang external sources
        working-directory: 3rdparty/glslang
        run: python update_glslang_sources.py

      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build

      - name: Show installed toolchain
        run: |
          ccache --version
          clang-20 --version
          clang++-20 --version
          ld.lld-20 --version || ld.lld --version || true
          dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true

      - name: Download Linux runtime
        uses: actions/download-artifact@v8
        with:
          name: mobilegl-linux-runtime
          path: .

      - name: Unpack Linux runtime
        run: |
          tar -xzf mobilegl-linux-runtime.tgz
          test -f "${MOBILEGL_LIBRARY}"

      - name: Configure CMake
        run: |
          if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
            BUILD_TYPE=Debug
          else
            BUILD_TYPE=Release
          fi

          cmake -S . -B "${BUILD_DIR}" -G Ninja \
            -DCMAKE_C_COMPILER=clang-20 \
            -DCMAKE_CXX_COMPILER=clang++-20 \
            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
            -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
            -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
            -DMOBILEGL_BUILD_TEST=OFF \
            -DMOBILEGL_BUILD_BENCHMARK=OFF \
            -DMOBILEGL_BUILD_TRACE_REPLAY=ON \
            -DMOBILEGL_TRACE_REPLAY_MOBILEGL_LIBRARY="${MOBILEGL_LIBRARY}" \
            -DCMAKE_POLICY_VERSION_MINIMUM=3.5

      - name: Build trace replay
        run: cmake --build "${BUILD_DIR}" --target mobilegl_trace_replay --parallel "$(nproc)"

      - name: Show ccache stats
        if: always()
        run: ccache --show-stats

      - name: Release superseded ccache entry
        if: github.ref_name == github.event.repository.default_branch
        env:
          GH_TOKEN: ${{ github.token }}
          CACHE_KEY: ${{ runner.os }}-test-${{ github.job }}-ccache-v1
        run: gh cache delete "${CACHE_KEY}" || true

      - name: Save ccache
        if: github.ref_name == github.event.repository.default_branch
        continue-on-error: true
        uses: actions/cache/save@v5
        with:
          path: .ccache
          key: ${{ runner.os }}-test-${{ github.job }}-ccache-v1

      - name: Normalize CTest command paths
        run: |
          python - <<'PY'
          from pathlib import Path
          import re

          for path in Path('build-retrace').rglob('CTestTestfile.cmake'):
              text = path.read_text()
              text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text)
              path.write_text(text)
          PY

      - name: Package trace replay
        run: |
          mkdir -p ci-artifacts
          tar -czf ci-artifacts/mobilegl-trace-replay.tgz \
            build-retrace/tools/trace_replay/mobilegl_trace_replay \
            build-retrace/tools/trace_replay/CTestTestfile.cmake

      - name: Upload trace replay
        uses: actions/upload-artifact@v7
        with:
          name: mobilegl-trace-replay
          path: ci-artifacts/mobilegl-trace-replay.tgz
          if-no-files-found: error

  trace-cases:
    name: trace case matrix
    runs-on: ubuntu-latest
    needs:
      - test
      - benchmark
      - integration
    outputs:
      matrix: ${{ steps.trace-cases.outputs.matrix }}
      names: ${{ steps.trace-cases.outputs.names }}
      verify-matrix: ${{ steps.trace-cases.outputs.verify-matrix }}
      split-matrix: ${{ steps.trace-cases.outputs.split-matrix }}
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6

      - name: Load trace cases
        id: trace-cases
        run: |
          echo "matrix=$(python3 tools/trace_replay/trace_cases.py --ci --format github-test-matrix)" >> "$GITHUB_OUTPUT"
          echo "names=$(python3 tools/trace_replay/trace_cases.py --ci --format names)" >> "$GITHUB_OUTPUT"
          # The subset the verify build retraces ("verify": true in trace_cases.json). It is a
          # SUBSET of the matrix above, so retrace-verify needs no fixtures of its own.
          echo "verify-matrix=$(python3 tools/trace_replay/trace_cases.py --ci --format github-verify-matrix)" >> "$GITHUB_OUTPUT"
          # P5's split subset ("split": true, DirectGLES only). Also a SUBSET of the matrix above,
          # so retrace-split needs no fixtures of its own either. It is one case today - OpenRA,
          # which is what the phase gate names - and trace_cases.py refuses a `split` case that is
          # not in CI or does not run DirectGLES, so the subset cannot silently become empty.
          SPLIT_MATRIX=$(python3 tools/trace_replay/trace_cases.py --ci --format github-split-matrix)
          # AND IT MUST NOT BE EMPTY. An empty `include` is not an error to GitHub - it skips the
          # whole retrace-split job with no red anywhere - so the one way this subset can vanish
          # silently is guarded here. trace_cases.py now also rejects an unknown manifest key, which
          # was the hole: `"splitt": true` loaded clean and emptied the subset (review N-4).
          if [ "$(printf '%s' "${SPLIT_MATRIX}" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["include"]))')" -lt 1 ]; then
            echo "::error::the split retrace subset is EMPTY. No case in trace_cases.json carries \"split\": true, so retrace-split would be skipped with no red. Exit gate E2 names OpenRA."
            exit 1
          fi
          echo "split-matrix=${SPLIT_MATRIX}" >> "$GITHUB_OUTPUT"

  trace-fixtures:
    name: trace fixture (${{ matrix.case }})
    runs-on: ubuntu-latest
    needs: trace-cases
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix:
        case: ${{ fromJSON(needs.trace-cases.outputs.names) }}
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6

      - name: Derive trace fixture cache key
        id: fixture-key
        run: bash .github/scripts/trace-fixture-cache.sh key '${{ matrix.case }}'

      - name: Restore trace fixture cache
        id: fixture-cache
        if: steps.fixture-key.outputs.cacheable == 'true'
        uses: actions/cache/restore@v5
        with:
          path: ${{ steps.fixture-key.outputs.paths }}
          key: ${{ steps.fixture-key.outputs.key }}

      - name: Verify restored trace fixture
        id: fixture-verify
        if: steps.fixture-cache.outputs.cache-hit == 'true'
        run: |
          if bash .github/scripts/trace-fixture-cache.sh verify '${{ matrix.case }}'; then
            echo "ok=true" >> "$GITHUB_OUTPUT"
          else
            echo "ok=false" >> "$GITHUB_OUTPUT"
            echo "::warning::Cached fixture for ${{ matrix.case }} failed verification; falling back to the download path"
            bash .github/scripts/trace-fixture-cache.sh reset '${{ matrix.case }}'
          fi

      - name: Fetch trace fixture
        if: steps.fixture-verify.outputs.ok != 'true'
        run: bash .github/scripts/fetch-trace-fixture-lfs.sh '${{ matrix.case }}'

      - name: Save trace fixture cache
        if: steps.fixture-key.outputs.cacheable == 'true' && steps.fixture-cache.outputs.cache-hit != 'true'
        uses: actions/cache/save@v5
        with:
          path: ${{ steps.fixture-key.outputs.paths }}
          key: ${{ steps.fixture-key.outputs.key }}

      - name: Stage trace fixture
        run: |
          safe_case="$(printf '%s' '${{ matrix.case }}' | sed 's/[^A-Za-z0-9._-]/_/g')"
          stage_dir="trace-fixtures/${safe_case}"
          mkdir -p "${stage_dir}"
          python3 tools/trace_replay/trace_cases.py --format fixture-files --case '${{ matrix.case }}' |
            while IFS= read -r file; do
              cp "${file}" "${stage_dir}/"
            done

      - name: Upload trace fixture
        uses: actions/upload-artifact@v7
        with:
          name: trace-fixture-${{ matrix.case }}
          path: trace-fixtures/**
          if-no-files-found: error

  retrace:
    name: retrace (${{ matrix.backend }}, ${{ matrix.case }})
    runs-on: ubuntu-latest
    needs:
      - build-linux
      - build-retrace
      - trace-cases
      - trace-fixtures
    if: ${{ always() && needs.build-linux.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }}
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix: ${{ fromJSON(needs.trace-cases.outputs.matrix) }}

    steps:
      - name: Set Swap Space
        uses: pierotofy/set-swap-space@v1.0
        with:
          swap-size-gb: 16

      - name: Checkout repo
        uses: actions/checkout@v6

      - name: Download trace fixture
        uses: actions/download-artifact@v8
        with:
          name: trace-fixture-${{ matrix.case }}
          path: trace-fixture-download

      - name: Install trace fixture
        run: |
          mkdir -p tools/trace_replay/fixtures
          find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;

      - name: Get CMake
        uses: lukka/get-cmake@v4.3.3

      - name: Install runtime dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers
          test -e /usr/lib/x86_64-linux-gnu/libEGL.so
          test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so

      - name: Download Linux runtime
        uses: actions/download-artifact@v8
        with:
          name: mobilegl-linux-runtime
          path: .

      - name: Download trace replay
        uses: actions/download-artifact@v8
        with:
          name: mobilegl-trace-replay
          path: .

      - name: Unpack retrace runtime
        run: |
          tar -xzf mobilegl-linux-runtime.tgz
          tar -xzf mobilegl-trace-replay.tgz
          test -f build-linux/libMobileGL.so
          test -f build-retrace/tools/trace_replay/mobilegl_trace_replay

      - name: Retrace and validate
        working-directory: build-retrace/tools/trace_replay
        run: |
          ulimit -c unlimited
          sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'
          if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then
            export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1
          fi
          if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
             && [ '${{ matrix.case }}' = 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' ]; then
            export MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH=1
            export MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS=1
            export MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER=1
          fi
          # The blended depth-write quirk auto-enables only on Qualcomm, which no CI
          # runner has, so force it on for the OIT case it exists to fix. ForceOn
          # bypasses only the vendor gate, so this exercises the real strip on
          # lavapipe. The Android AVD lane deliberately leaves it off, keeping the
          # unstripped path covered for the same trace.
          if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
             && [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then
            export MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1
          fi
          ctest -V --no-tests=error -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$'

      - name: Upload core dumps
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: retrace-core-dumps-${{ matrix.backend }}-${{ matrix.case }}
          path: /tmp/core.*
          if-no-files-found: ignore

      - name: Upload actual image
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: retrace-result-${{ matrix.backend }}-${{ matrix.case }}
          path: |
            build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/**
            build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/**
          if-no-files-found: warn

  retrace-summary:
    name: retrace summary
    runs-on: ubuntu-latest
    needs: retrace
    if: ${{ always() && needs.retrace.result != 'skipped' }}

    steps:
      - name: Checkout repo
        uses: actions/checkout@v6

      - name: Set artifact metadata
        run: |
          echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"

      - name: Set up Node.js
        uses: actions/setup-node@v7
        with:
          node-version: '22'

      - name: Download retrace results
        uses: actions/download-artifact@v8
        with:
          pattern: retrace-result-*
          path: retrace-artifacts

      - name: Render retrace summary
        run: |
          node tools/trace_replay/render_retrace_summary.mjs \
            --input retrace-artifacts \
            --output-dir retrace-summary \
            --title "MobileGL Linux retrace overview" \
            --group-label "Linux" \
            --html mobilegl-linux-retrace-overview.html

      - name: Upload retrace summary
        uses: actions/upload-artifact@v7
        with:
          path: retrace-summary/mobilegl-linux-retrace-overview.html
          archive: false
          if-no-files-found: error

  # The trace half of the third CI mode. Same replay, same goldens, but the library underneath is
  # the verify build and MOBILEGL_PIPE_VERIFY=1 is in the environment, so every backend read of
  # frontend state is checked against a snapshot taken at the verb boundary. Eight cases rather
  # than the full lane's 40 (tools/trace_replay/trace_cases.json, "verify": true): the comparator
  # is budgeted at 5-10x, and the full sweep is a phase-exit / workflow_dispatch run.
  retrace-verify:
    name: retrace verify (${{ matrix.backend }}, ${{ matrix.case }})
    runs-on: ubuntu-latest
    timeout-minutes: 240
    needs:
      - build-linux-verify
      - build-retrace
      - trace-cases
      - trace-fixtures
    if: ${{ always() && needs.build-linux-verify.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }}
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix: ${{ fromJSON(needs.trace-cases.outputs.verify-matrix) }}

    steps:
      - name: Set Swap Space
        uses: pierotofy/set-swap-space@v1.0
        with:
          swap-size-gb: 16

      - name: Checkout repo
        uses: actions/checkout@v6

      - name: Download trace fixture
        uses: actions/download-artifact@v8
        with:
          name: trace-fixture-${{ matrix.case }}
          path: trace-fixture-download

      - name: Install trace fixture
        run: |
          mkdir -p tools/trace_replay/fixtures
          find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;

      - name: Get CMake
        uses: lukka/get-cmake@v4.3.3

      - name: Install runtime dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers
          test -e /usr/lib/x86_64-linux-gnu/libEGL.so
          test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so

      - name: Download Linux verify runtime
        uses: actions/download-artifact@v8
        with:
          name: mobilegl-linux-runtime-verify
          path: .

      - name: Download trace replay
        uses: actions/download-artifact@v8
        with:
          name: mobilegl-trace-replay
          path: .

      - name: Unpack the VERIFY runtime as the library under test
        # build-retrace's CTestTestfile.cmake has the absolute path
        # <workspace>/build-linux/libMobileGL.so frozen into every case, so the swap happens here
        # rather than through a variable: the verify .so is put where that path points. The nm
        # check is what makes the swap falsifiable - a run against the ordinary library would
        # carry no comparator, ignore MOBILEGL_PIPE_VERIFY entirely, and match its golden.
        #
        # `nm`, not `nm -D`, for the reason spelled out in build-linux-verify: everything MGPipe is
        # hidden-visibility in a Release build and the dynamic table has none of it.
        run: |
          tar -xzf mobilegl-linux-runtime-verify.tgz
          tar -xzf mobilegl-trace-replay.tgz
          test -f build-verify/libMobileGL.so
          test -f build-retrace/tools/trace_replay/mobilegl_trace_replay
          mkdir -p build-linux
          cp build-verify/libMobileGL.so build-linux/libMobileGL.so
          if ! nm --defined-only build-linux/libMobileGL.so | grep -q MGPipeVerifyInputs; then
            echo "::error::the library unpacked at build-linux/libMobileGL.so defines no MGPipeVerifyInputs, so this retrace would replay against a comparator-free build and pass on its golden having verified nothing"
            exit 1
          fi
          echo "the library at build-linux/libMobileGL.so is the verify build"

      - name: Retrace and validate under MOBILEGL_PIPE_VERIFY
        working-directory: build-retrace/tools/trace_replay
        # run_trace_case.cmake turns MOBILEGL_PIPE_VERIFY into three assertions of its own (the
        # arming line, no Fatal{PipeVerifyDiffer, no Fatal{UnmigratedPipeInput), so a case that
        # somehow ran the wrong library reds here instead of passing on its golden.
        # --timeout 10800: the 1800s cases run 5-10x slower with both comparator arms live, which
        # is well past ctest's 1500s default.
        run: |
          ulimit -c unlimited
          sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'
          export MOBILEGL_PIPE_VERIFY=1
          if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then
            export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1
          fi
          if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \
             && [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then
            export MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1
          fi
          ctest -V --no-tests=error --timeout 10800 \
            -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$'

      # The retrace lane's own always-on negative control, on one case so it costs one short trace:
      # with a snapshot field corrupted, the SAME replay must fail. Without it, "40 traces, zero
      # divergences" would be a statement about a comparator nobody watched.
      #
      # The rerun replays into the SAME case directory, so the verified run's images are put aside
      # first and restored before the verdict: "Upload actual image" below runs `if: always()` and
      # would otherwise ship the deliberately corrupted run's output under the name of the good one.
      # The restore happens whichever way the control goes, which is why the ctest exit status is
      # captured rather than tested inline.
      - name: Negative control - a corrupted snapshot field must red this retrace
        if: ${{ matrix.case == 'OpenRA' && matrix.backend == 'DirectGLES' }}
        working-directory: build-retrace/tools/trace_replay
        run: |
          export MOBILEGL_PIPE_VERIFY=1
          export MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters
          GOOD_OUTPUT="${RUNNER_TEMP}/openra-verified-output"
          rm -rf "${GOOD_OUTPUT}"
          if [ -d OpenRA ]; then
            cp -a OpenRA "${GOOD_OUTPUT}"
          fi
          set +e
          ctest -V --no-tests=error --timeout 10800 \
            -R '^MobileGLTraceReplay\.OpenRA\.DirectGLES$'
          control_rc=$?
          set -e
          if [ -d "${GOOD_OUTPUT}" ]; then
            rm -rf OpenRA
            mv "${GOOD_OUTPUT}" OpenRA
            echo "restored the verified run's OpenRA output over the corrupted rerun's"
          fi
          if [ "${control_rc}" -eq 0 ]; then
            echo "::error::MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left the OpenRA retrace GREEN, so the comparator is not comparing and the whole verify retrace lane proves nothing."
            exit 1
          fi
          echo "the corrupted field turned the retrace red, as it must (ctest exit ${control_rc})"

      - name: Upload core dumps
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: retrace-verify-core-dumps-${{ matrix.backend }}-${{ matrix.case }}
          path: /tmp/core.*
          if-no-files-found: ignore

      - name: Upload actual image
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: retrace-verify-result-${{ matrix.backend }}-${{ matrix.case }}
          path: |
            build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/**
            build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/**
          if-no-files-found: warn

  # P5's retrace arm: retrace-verify with MOBILEGL_PIPE_VERIFY=1 swapped for
  # MOBILEGL_TRANSPORT=inproc and the comparator's symbol swapped for an MG_Remote one. Exit gate
  # E2 is one row - OpenRA at SSIM >= 0.99 - and trace_cases.json's `split: true` is where that
  # subset lives.
  #
  # IT RUNS THE UNCHANGED CTEST NAMES with the transport exported in the JOB environment, rather
  # than the SPLIT-suffixed variant entries. build-retrace configures without
  # -DMOBILEGL_BUILD_DISAGGREGATED, so the variant entries are deliberately not registered there
  # (a name that says SPLIT in a build that cannot be one is worse than no name), and the
  # unchanged names are what retrace-verify already proves this shape works with. The variant
  # entries exist for a build that DOES configure the option - a local build-split with trace
  # replay on - where `ctest -L retrace-split` is self-describing and needs no environment ritual.
  #
  # WHAT MAKES IT FALSIFIABLE is not the SSIM. A monolith run of OpenRA also scores 1.000: measured
  # on this branch, a pull library under MOBILEGL_TRANSPORT=inproc produced ssim=1.0 and was caught
  # only by run_trace_case.cmake's transport-resolution assertion. So there are two guards, and the
  # SSIM is neither of them: the nm check below (this library carries MG_Remote) and the
  # transport-resolution line in the library's own log (this PROCESS resolved inproc).
  retrace-split:
    name: retrace split (${{ matrix.backend }}, ${{ matrix.case }})
    runs-on: ubuntu-latest
    timeout-minutes: 240
    needs:
      - build-linux
      - build-linux-split
      - build-retrace
      - trace-cases
      - trace-fixtures
    # build-linux is needed for the negative control ONLY: its pull library is what the control
    # swaps in to prove the transport-resolution assertion is load-bearing.
    if: ${{ always() && needs.build-linux.result == 'success' && needs.build-linux-split.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }}
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix: ${{ fromJSON(needs.trace-cases.outputs.split-matrix) }}

    steps:
      - name: Set Swap Space
        uses: pierotofy/set-swap-space@v1.0
        with:
          swap-size-gb: 16

      - name: Checkout repo
        uses: actions/checkout@v6

      - name: Download trace fixture
        uses: actions/download-artifact@v8
        with:
          name: trace-fixture-${{ matrix.case }}
          path: trace-fixture-download

      - name: Install trace fixture
        run: |
          mkdir -p tools/trace_replay/fixtures
          find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \;

      - name: Get CMake
        uses: lukka/get-cmake@v4.3.3

      - name: Install runtime dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers
          test -e /usr/lib/x86_64-linux-gnu/libEGL.so
          test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so

      - name: Download Linux split runtime
        uses: actions/download-artifact@v8
        with:
          name: mobilegl-linux-runtime-split
          path: .

      - name: Download trace replay
        uses: actions/download-artifact@v8
        with:
          name: mobilegl-trace-replay
          path: .

      # The PULL runtime, for the negative control at the end of this job and for nothing else.
      - name: Download Linux pull runtime (negative control)
        uses: actions/download-artifact@v8
        with:
          name: mobilegl-linux-runtime
          path: pull-runtime

      - name: Unpack the pull runtime (negative control)
        run: |
          tar -xzf pull-runtime/mobilegl-linux-runtime.tgz -C pull-runtime
          test -f pull-runtime/build-linux/libMobileGL.so

      - name: Unpack the SPLIT runtime as the library under test
        # build-retrace's CTestTestfile.cmake has the absolute path
        # <workspace>/build-linux/libMobileGL.so frozen into every case, so the swap happens here
        # rather than through a variable, exactly as in retrace-verify. `nm`, not `nm -D`.
        run: |
          tar -xzf mobilegl-linux-runtime-split.tgz
          tar -xzf mobilegl-trace-replay.tgz
          test -f build-split/libMobileGL.so
          test -f build-retrace/tools/trace_replay/mobilegl_trace_replay
          mkdir -p build-linux
          cp build-split/libMobileGL.so build-linux/libMobileGL.so
          if ! nm --defined-only build-linux/libMobileGL.so | grep -q -i MG_Remote; then
            echo "::error::the library unpacked at build-linux/libMobileGL.so defines no MG_Remote symbol, so this retrace would replay against a monolith build, ignore MOBILEGL_TRANSPORT entirely and match its golden having split nothing"
            exit 1
          fi
          echo "the library at build-linux/libMobileGL.so is the split build"

      - name: Retrace and validate under MOBILEGL_TRANSPORT=inproc
        working-directory: build-retrace/tools/trace_replay
        # run_trace_case.cmake turns MOBILEGL_TRANSPORT into assertions of its own - the library
        # must have RESOLVED the transport in this process, and its log must carry no Fatal{ - so a
        # case that somehow ran the wrong library reds here instead of passing on its golden. That
        # log is also the only valid refusal census: the console sink is compiled out of this
        # configuration, so a `ctest -V` transcript reports a false zero for Fatal{ lines.
        run: |
          ulimit -c unlimited
          sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p'
          export MOBILEGL_TRANSPORT=inproc
          ctest -V --no-tests=error --timeout 10800 \
            -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$'

      # THE RETRACE LANE'S OWN ALWAYS-ON NEGATIVE CONTROL, and its absence was review finding M-6:
      # this job was a clone of retrace-verify with the one step removed that makes the lane mean
      # anything. retrace-verify carries "a corrupted snapshot field must red this retrace" so that
      # "79 traces, zero divergences" is not a statement about a comparator nobody watched; the
      # same sentence applies here word for word.
      #
      # The control swaps the PULL library into the frozen path and requires the SAME replay to
      # fail. It is the sharpest one available, because OpenRA scores ssim 1.000000 either way -
      # measured - so this step fails if and only if run_trace_case.cmake's transport-resolution
      # assertion has stopped working, which is the single thing standing between this job and a
      # green that ran monolith end to end.
      #
      # NOT the control BRIEF 7 E2 names ("patch the Clear emitter to drop one emission; SSIM must
      # fall below threshold"). That one needs an emitter, i.e. package c1, and it is carried as an
      # explicit debt in t1-v1.md rather than silently substituted - which is what the first
      # version of this package did.
      #
      # The rerun replays into the same case directory, so the good run's images are put aside and
      # restored whichever way the control goes; "Upload actual image" below runs `if: always()`
      # and would otherwise ship the deliberately-wrong run's output under the good run's name.
      - name: Negative control - the PULL library must red this split retrace
        working-directory: build-retrace/tools/trace_replay
        run: |
          set +e
          GOOD_OUTPUT="${RUNNER_TEMP}/split-verified-output"
          rm -rf "${GOOD_OUTPUT}"
          if [ -d "${{ matrix.case }}" ]; then cp -a "${{ matrix.case }}" "${GOOD_OUTPUT}"; fi
          # The pull library, unpacked from build-linux's artifact, over the frozen path every
          # case has baked in. It defines no MG_Remote symbol, so ConfigLoader has no transport
          # parser and MOBILEGL_TRANSPORT=inproc is accepted and ignored - the exact shape of "the
          # split lane ran monolith".
          cp "${GITHUB_WORKSPACE}/pull-runtime/build-linux/libMobileGL.so" \
             "${GITHUB_WORKSPACE}/build-linux/libMobileGL.so"
          if nm --defined-only "${GITHUB_WORKSPACE}/build-linux/libMobileGL.so" | grep -q -i MG_Remote; then
            echo "::error::the control's own library defines MG_Remote symbols, so it is not a pull build and this control would prove nothing"
            exit 1
          fi
          export MOBILEGL_TRANSPORT=inproc
          ctest -V --no-tests=error --timeout 10800 \
            -R "^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$"
          control_rc=$?
          set -e
          if [ -d "${GOOD_OUTPUT}" ]; then
            rm -rf "${{ matrix.case }}"; mv "${GOOD_OUTPUT}" "${{ matrix.case }}"
            echo "restored the verified run's output over the control's"
          fi
          if [ "${control_rc}" -eq 0 ]; then
            echo "::error::a PULL library passed the split retrace. OpenRA scores ssim 1.000000 under a monolith library too (measured), so the picture is not and cannot be this lane's gate - run_trace_case.cmake's transport-resolution assertion is, and it has stopped working. Every green in this job is then a monolith run under a name that says split."
            exit 1
          fi
          echo "the pull library turned the split retrace red, as it must (ctest exit ${control_rc})"

      # The refusal census, recorded rather than gated. run_trace_case.cmake already REDS the case
      # on any Fatal{, so reaching here means the count is zero - but the number and the distinct
      # slot names are what MEASUREMENTS wants from every split run, and reading them out of the
      # library's own log is the only way to get them (ctest -V's transcript is a false zero).
      - name: Refusal census from the library log
        if: always()
        working-directory: build-retrace/tools/trace_replay
        run: |
          log="${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.log"
          if [ ! -f "${log}" ]; then
            echo "no ${log} - the replay wrote no library log"
            exit 0
          fi
          echo "Fatal{ lines: $(grep -c 'Fatal{' "${log}" || true)"
          grep -o 'Fatal{[A-Za-z]*, "[^"]*"' "${log}" | sort | uniq -c | sort -rn | head -20 || true
          grep -m1 "MOBILEGL_TRANSPORT=" "${log}" || echo "no transport line in ${log}"

      - name: Upload core dumps
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: retrace-split-core-dumps-${{ matrix.backend }}-${{ matrix.case }}
          path: /tmp/core.*
          if-no-files-found: ignore

      - name: Upload actual image
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: retrace-split-result-${{ matrix.backend }}-${{ matrix.case }}
          path: |
            build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/**
            build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/**
          if-no-files-found: warn

  # G1's own job: the pull build must be the tree before P1, symbol for symbol and byte for byte.
  # workflow_dispatch only - it builds the library twice from scratch, and its answer is about a
  # BASELINE rather than about this push, so a per-push run would be measuring the wrong pair.
  monolith-symbol-report:
    name: monolith symbol report
    runs-on: ubuntu-latest
    timeout-minutes: 180
    if: ${{ github.event_name == 'workflow_dispatch' }}
    env:
      CCACHE_BASEDIR: ${{ github.workspace }}
      CCACHE_COMPRESS: "true"
      CCACHE_DIR: ${{ github.workspace }}/.ccache
      CCACHE_MAXSIZE: 4G
      CCACHE_NOHASHDIR: "true"

    steps:
      - name: Set Swap Space
        uses: pierotofy/set-swap-space@v1.0
        with:
          swap-size-gb: 32

      - name: Checkout repo
        uses: actions/checkout@v6
        with:
          submodules: recursive
          fetch-depth: 0

      - name: Get CMake
        uses: lukka/get-cmake@v4.3.3

      - name: Restore ccache
        uses: actions/cache/restore@v5
        with:
          path: .ccache
          key: ${{ runner.os }}-test-${{ github.job }}-ccache-v1
          restore-keys: |
            ${{ runner.os }}-test-${{ github.job }}-ccache-

      - name: Prepare Vulkan SDK
        uses: humbletim/setup-vulkan-sdk@v1.2.1
        with:
          vulkan-query-version: 1.4.304.1
          vulkan-components: Vulkan-Headers, Vulkan-Loader
          vulkan-use-cache: true

      - name: Install build dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build binutils

      # Both sides with IDENTICAL flags, LTO off, the same compiler and the same standard library:
      # symbol_report.py's guard rails (scripts/symbol_report.py) say a mismatched pair "adds"
      # thousands of symbols and the comparison then means nothing. The library alone - no tests,
      # no benchmark, no integration test, no trace replay - because those targets do not ship.
      - name: Build the baseline library (${{ inputs.baseline_sha }})
        run: |
          git worktree add ../baseline "${{ inputs.baseline_sha }}"
          cd ../baseline
          git submodule update --init --recursive
          (cd 3rdparty/glslang && python update_glslang_sources.py)
          cmake -S . -B build-sym-base -G Ninja \
            -DCMAKE_C_COMPILER=clang-20 -DCMAKE_CXX_COMPILER=clang++-20 \
            -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
            -DCMAKE_BUILD_TYPE=Release \
            -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
            -DMOBILEGL_BUILD_TEST=OFF -DMOBILEGL_BUILD_BENCHMARK=OFF \
            -DMOBILEGL_BUILD_INTEGRATION_TEST=OFF -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \
            -DMOBILEGL_BUILD_DISAGGREGATED=OFF \
            -DMOBILEGL_PIPE_PUSH=OFF -DMOBILEGL_PIPE_VERIFY=OFF \
            -DMOBILEGL_ENABLE_LTO=OFF \
            -DCMAKE_POLICY_VERSION_MINIMUM=3.5
          cmake --build build-sym-base --parallel "$(nproc)"
          cp build-sym-base/libMobileGL.so "${GITHUB_WORKSPACE}/libMobileGL-baseline.so"

      - name: Build the head library
        run: |
          (cd 3rdparty/glslang && python update_glslang_sources.py)
          cmake -S . -B build-sym-head -G Ninja \
            -DCMAKE_C_COMPILER=clang-20 -DCMAKE_CXX_COMPILER=clang++-20 \
            -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
            -DCMAKE_BUILD_TYPE=Release \
            -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
            -DMOBILEGL_BUILD_TEST=OFF -DMOBILEGL_BUILD_BENCHMARK=OFF \
            -DMOBILEGL_BUILD_INTEGRATION_TEST=OFF -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \
            -DMOBILEGL_BUILD_DISAGGREGATED=OFF \
            -DMOBILEGL_PIPE_PUSH=OFF -DMOBILEGL_PIPE_VERIFY=OFF \
            -DMOBILEGL_ENABLE_LTO=OFF \
            -DCMAKE_POLICY_VERSION_MINIMUM=3.5
          cmake --build build-sym-head --parallel "$(nproc)"

      # The monolith must not have grown a remote half. ARCHITECTURE.md:506: MG_Remote lives behind
      # MOBILEGL_BUILD_DISAGGREGATED and nothing of it may reach a shipped pull build.
      - name: No MG_Remote in the pull build
        run: |
          if nm --defined-only build-sym-head/libMobileGL.so | grep -q MG_Remote; then
            echo "::error::the pull build defines MG_Remote symbols; the disaggregated half leaked into the monolith"
            nm --defined-only build-sym-head/libMobileGL.so | grep MG_Remote | head -20
            exit 1
          fi
          echo "no MG_Remote symbols in the pull build"

      - name: Symbol report (G1)
        run: |
          python3 scripts/symbol_report.py \
            --before libMobileGL-baseline.so \
            --after build-sym-head/libMobileGL.so \
            --threshold 0 \
            --fail-on-symbol-set-change \
            --fail-on-added-bytes 0 \
            --markdown symbol-report.md \
            --json symbol-report.json

      - name: Upload the symbol report
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: monolith-symbol-report
          path: |
            symbol-report.md
            symbol-report.json
          if-no-files-found: error

  remove-artifact-clutter:
    name: remove artifact clutter
    runs-on: ubuntu-latest
    # (d) retrace-verify too: this job deletes the trace-fixture-* artifacts, and the verify
    # retraces download the same ones.
    needs:
      - retrace-summary
      - retrace-verify
    if: always()
    permissions:
      actions: write
    steps:
      - name: Delete intermediate Linux retrace artifacts
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          # Both retrace lanes, not just the pull one: `retrace verify (backend, case)` downloads
          # the same trace-fixture-<case> artifact, and a failed verify retrace is exactly when
          # someone needs that fixture to reproduce locally. The two prefixes are stripped in
          # order, longest first, because "retrace (" is not a prefix of "retrace verify (".
          declare -A failed_cases=()
          while IFS= read -r job_name; do
            case_name="${job_name#retrace verify (*, }"
            case_name="${case_name#retrace (*, }"
            case_name="${case_name%)}"
            failed_cases["${case_name}"]=1
          done < <(
            gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100" \
              --jq '.jobs[] | select((.name | startswith("retrace (")) or (.name | startswith("retrace verify ("))) | select(.conclusion == "failure" or .conclusion == "cancelled" or .conclusion == "timed_out" or .conclusion == "action_required") | .name'
          )

          if ((${#failed_cases[@]})); then
            echo "Retaining fixtures for failed retrace case(s):"
            printf '  %s\n' "${!failed_cases[@]}"
          else
            echo "All retrace jobs succeeded; no fixtures need to be retained."
          fi

          deleted=0
          retained=0
          while IFS=$'\t' read -r artifact_id artifact_name; do
            if [[ "${artifact_name}" == trace-fixture-* ]]; then
              case_name="${artifact_name#trace-fixture-}"
              if [[ -v "failed_cases[${case_name}]" ]]; then
                echo "Retaining ${artifact_name} (${artifact_id}) for failed retrace."
                ((retained += 1))
                continue
              fi
            fi

            echo "Deleting ${artifact_name} (${artifact_id})"
            gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}"
            ((deleted += 1))
          done < <(
            gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \
              --jq '.artifacts[] | select(.name | startswith("trace-fixture-") or startswith("retrace-result-")) | [.id, .name] | @tsv'
          )

          echo "Deleted ${deleted} intermediate Linux artifact(s); retained ${retained} failed-retrace fixture(s)."

  pipe-gates:
    name: MGPipe generators and hygiene gates
    runs-on: ubuntu-latest
    # Deliberately independent of build-linux: these are source-level gates, they take
    # seconds, and a broken build must not hide a drifted interface.
    env:
      # THE CURRENT PHASE's base ref, for the two G5 region gates below. It is 37da3c3a - P4a's
      # base ref, INTEGRATOR-DECISIONS ID-1 - and NOT the workflow's baseline_sha input: that input
      # is the SYMBOL baseline (087685d1, P1's G1 reading) and it is empty on a push, whereas these
      # gates ask "did the do-not-touch list move since the phase started".
      #
      # IT MOVED FROM P3a's 44c2b5cf TO P4a's 37da3c3a WITH THE PHASE, and that is a deliberate
      # narrowing rather than a loss: P3a's eleven functions were compared against 44c2b5cf at P3a's
      # own exit and were byte-identical there, so 37da3c3a carries the same bodies (measured: the
      # eleven shas at 37da3c3a are the eleven shas at 44c2b5cf, and FlushPendingRangesFrom's is
      # still the sha pinned in the script at 3e298c9a). What the two gates now both answer is "did
      # anything on the list move during P4a", which is the question this phase can act on.
      BASELINE: "37da3c3a"

    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
        with:
          # The G5 gate reads Managers.cpp at BASELINE with `git show`, which a depth-1 checkout
          # does not have. Nothing else in this job needs history.
          fetch-depth: 0

      # The seven generators all read MG_Pipe/*.def, so regenerating and diffing is what
      # keeps the two interface tables, the wire records, the verify comparators, the
      # PipeInputs field ids, the read-inventory coverage and the render-state member list
      # from drifting apart from the catalogue. The generated files are committed
      # deliberately: the build must not depend on python.
      - name: Regenerate the MGPipe interface (G1-G7)
        run: |
          python3 scripts/gen_pipe.py
          git diff --exit-code -- MobileGL/MG_Pipe/generated

      # The generators' own negative controls: canned inputs that MUST trip each structural check
      # (a field list that does not cover its struct's members, a verb set that is not the function
      # table's). Regenerating and diffing above cannot see a check that silently stopped
      # checking - a broken gate and a clean tree produce the same green.
      - name: The MGPipe generators' checks can still fail
        run: python3 scripts/gen_pipe.py --self-test

      # The same question for the symbol tool the P1 gate is written in terms of.
      - name: The symbol report's buckets and gates can still fail
        run: python3 scripts/symbol_report.py --self-test

      # Per-draw fprintf/printf instrumentation has repeatedly been committed by accident,
      # once inside a mutex critical section. Nothing under these two trees prints to a
      # stdio stream today - MGLOG_D compiles out in INFO builds and is the only channel
      # they are allowed to use - so this gate starts with no exceptions, and any addition
      # to it needs a reason in the pull request rather than a quiet whitelist entry. The
      # alternation names every stdio spelling, not just the two that were committed:
      # fprintf to either stream, printf, puts, and the iostream pair.
      - name: No stdio instrumentation in MG_Backend or MG_State
        run: |
          if grep -rnE 'fprintf[[:space:]]*\((stderr|stdout)|(^|[^[:alnum:]_>.])printf[[:space:]]*\(|(^|[^[:alnum:]_>.:])puts[[:space:]]*\(|std::(cout|cerr)' \
              MobileGL/MG_Backend MobileGL/MG_State; then
            echo "::error::stdio instrumentation found; use MGLOG_D (compiled out in INFO builds)"
            exit 1
          fi
          echo "no fprintf(stderr/stdout / printf( / puts( / std::cout|cerr under MobileGL/MG_Backend or MobileGL/MG_State"

      # A GATE as of P2, which is when MG_Pipe/DirtySurface.def exists to diff the scan against
      # (ROADMAP.md:18 puts the first mapping round in P2). --check fails BOTH directions: a
      # mutator the scanner finds with no row in the def, and a row naming a mutator the scan no
      # longer finds - so a deleted mutator cannot leave a stale row behind claiming coverage.
      #
      # --self-test is the half that keeps --check honest, and it is not optional. A completeness
      # check that silently stopped checking produces exactly the same green as a complete
      # mapping; the self-test feeds it two canned negative controls (a mutator withheld from the
      # def, a row naming a function that does not exist) and fails if either fails to trip. Same
      # shape as gen_pipe.py --self-test and check_include_closure.py above.
      #
      # What this gate does NOT cover is written into DirtySurface.def's header rather than left
      # implicit: the scanner attributes a mutation inside a lambda to the enclosing function,
      # reads a mutation published through a helper as deferred, and scans only MG_Impl/GLImpl -
      # so the four MGP_NOTE_MUTATION sites in MG_State are outside it entirely. This is a
      # completeness gate over what the scanner can see; the semantic proof is the verify lane.
      - name: MGPipe dirty-surface mapping is complete (G9)
        run: |
          python3 scripts/gen_pipe_dirty_surface.py --check
          python3 scripts/gen_pipe_dirty_surface.py --self-test

      # A GATE as of P3a (G5). "pool 与延迟释放原样搬" (ROADMAP.md:19) is meant literally: the
      # buffer pool, the deferred-release drain and the three persistently mapped rings move
      # VERBATIM, and ARCHITECTURE.md:515 says why - their retire happens only inside Present, so
      # a batching or ordering change there starves them, and nothing else in this workflow can
      # see it. P3a rewrites the rest of Managers.cpp by design, so a file diff says nothing; the
      # script extracts the ELEVEN named bodies and compares their hashes on their own.
      #
      # Eleven and not ten (ID-15): Managers.cpp carries the three-tier flush drain TWICE, once
      # per preprocessor arm, and a push build compiles only FlushPendingRangesFrom while the
      # untouched FlushPendingRangesNow lives in the `#else`. Hashing the pull name alone would
      # protect text no shipping build compiles, so both are hashed - the pull ladder against
      # BASELINE, the push ladder against a sha pinned in the script at 3e298c9a, because that
      # one was born in P3a and has no body at the base ref to compare with.
      #
      # Scoped to the disaggregation branch and to a manual dispatch, deliberately: the question
      # is "did these eleven move since P3a started", and BASELINE is P3a's base ref. On dev,
      # where unrelated buffer fixes land on their own schedule, the same comparison would be
      # asking a question nobody posed - it belongs with the TEMPORARY trigger lines at the top
      # of this file and retires with them.
      #
      # --self-test is the half that keeps it honest, and it is not optional: a comparison that
      # silently stopped comparing produces exactly the same green as eleven untouched bodies. It
      # runs six canned controls - eleven bodies extracted, an untouched copy compared equal, an
      # edit OUTSIDE them ignored, and each of the three perturbation targets (ClearBufferPool
      # and BOTH flush ladders) reported BY NAME - and fails if any of them does not answer.
      # Same shape as gen_pipe.py --self-test above.
      - name: The buffer pool, the deferred-release drain and the rings did not move (G5)
        if: ${{ github.ref == 'refs/heads/feat/disaggregated' || github.event_name == 'workflow_dispatch' }}
        run: bash scripts/p3a_untouched_regions.sh "${BASELINE}" HEAD

      - name: The untouched-region gate can still fail (G5)
        if: ${{ github.ref == 'refs/heads/feat/disaggregated' || github.event_name == 'workflow_dispatch' }}
        run: bash scripts/p3a_untouched_regions.sh --self-test

      # A GATE AS OF P4a (G5), and a SECOND script rather than an edit to the one above. P4a extends
      # the same claim to the rest of ARCHITECTURE.md:318's do-not-touch list - the unpack PBO ring's
      # staging repack and its two helpers, the attachment permutation, the D24S8 sampling-emulation
      # core and the format-caveat handler - which is SEVENTEEN regions across THREE files
      # (BRIEF-P4A.md D-N: P3a's eleven, which P4a must not touch either, plus P4a's six). The
      # parent's SOURCE_PATH is a single file, so the extension needed a per-region source path and
      # a region KIND (DepthStencilSamplingReadImpl is a namespace, not a function); everything else
      # about the extraction is its parent's, verbatim.
      #
      # Both scripts run. The parent keeps answering its own question against its own eleven, so a
      # regression in either half names itself, and neither gate can be silenced by editing the
      # other's list.
      #
      # Same feat/disaggregated-or-dispatch guard as the P3a step, for the same reason: the question
      # is "did these move since the phase started", and on dev - where unrelated buffer and texture
      # fixes land on their own schedule - it would be a question nobody posed. It belongs with the
      # TEMPORARY trigger lines at the top of this file and retires with them.
      #
      # --self-test is the half that keeps it honest and is not optional: a comparison that silently
      # stopped comparing produces exactly the same green as seventeen untouched regions. It runs
      # three positive controls (seventeen regions extracted, an untouched copy compared equal, an
      # edit OUTSIDE them invisible in all three files) and FOUR negative ones - ClearBufferPool,
      # FlushPendingRangesNow, RecomputeBackendColorSlots and StageBlocksIntoUnpackRing, each
      # perturbed on its own and each required to be named BY NAME - and fails if any of them does
      # not answer.
      - name: The unpack ring, the attachment permutation, the D24S8 core and the format caveat did not move (G5)
        if: ${{ github.ref == 'refs/heads/feat/disaggregated' || github.event_name == 'workflow_dispatch' }}
        run: bash scripts/p4a_untouched_regions.sh "${BASELINE}" HEAD

      - name: The P4a untouched-region gate can still fail (G5)
        if: ${{ github.ref == 'refs/heads/feat/disaggregated' || github.event_name == 'workflow_dispatch' }}
        run: bash scripts/p4a_untouched_regions.sh --self-test

      # Warning only for now: the disaggregation documents are still being written, and a
      # lint that fails a rewrite in progress teaches people to ignore it. It becomes
      # --strict when the documents settle.
      - name: Documentation citation lint
        run: |
          shopt -s nullglob
          documents=(docs/Disaggregated/*.md)
          if [ ${#documents[@]} -eq 0 ]; then
            echo "no disaggregation documents to check"
            exit 0
          fi
          python3 scripts/check_doc_citations.py "${documents[@]}" || true