name: Test on: push: branches: - dev - Feat/Backend-Direct-GLES - Feat/Backend-Direct-Vulkan # TEMPORARY, remove before merging the MGPipe work into dev: the disaggregation # branch runs the full lane on every push so a phase's landing is not gated on # someone remembering to dispatch the workflow by hand. - feat/disaggregated workflow_dispatch: inputs: baseline_sha: description: >- The commit monolith-symbol-report compares this tree against. P1's G1 says the pull build is byte-identical to feat/disaggregated@087685d1, and that is what the default names. The trigger set is unchanged: this job runs on workflow_dispatch only. required: false default: "087685d1" jobs: build-linux: runs-on: ubuntu-latest permissions: actions: write contents: read env: BUILD_DIR: build-linux CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "true" CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 4G CCACHE_NOHASHDIR: "true" steps: - name: Set Swap Space uses: pierotofy/set-swap-space@v1.0 with: swap-size-gb: 32 - name: Checkout repo uses: actions/checkout@v6 with: submodules: recursive - name: Get CMake uses: lukka/get-cmake@v4.3.3 - name: Restore ccache uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-test-${{ github.job }}-ccache-v1 restore-keys: | ${{ runner.os }}-test-${{ github.job }}-ccache- - name: Prepare Vulkan SDK uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Update glslang external sources working-directory: 3rdparty/glslang run: python update_glslang_sources.py - name: Install build dependencies run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build - name: Show installed toolchain run: | ccache --version clang-20 --version clang++-20 --version ld.lld-20 --version || ld.lld --version || true dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true - name: Configure CMake run: | if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then BUILD_TYPE=Debug else BUILD_TYPE=Release fi cmake -S . -B "${BUILD_DIR}" -G Ninja \ -DCMAKE_C_COMPILER=clang-20 \ -DCMAKE_CXX_COMPILER=clang++-20 \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=ON \ -DMOBILEGL_BUILD_BENCHMARK=ON \ -DMOBILEGL_BUILD_INTEGRATION_TEST=ON \ -DMOBILEGL_ITEST_VK_ICD=/usr/share/vulkan/icd.d/lvp_icd.json \ -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \ -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON \ -DBENCHMARK_ENABLE_TESTING=OFF \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 - name: Build run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)" - name: Show ccache stats if: always() run: ccache --show-stats # Rewrite one rolling entry per job on the default branch. The upload stays # cumulative - it carries every object restored at the top of this run plus # the few TUs that actually changed - but Actions cache keys are immutable, # so the superseded blob has to be released before the same key can be # re-uploaded. Running after the build means a failed build leaves the # existing entry untouched. The other trigger branches restore this entry # rather than each writing one of their own. - name: Release superseded ccache entry if: github.ref_name == github.event.repository.default_branch env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ runner.os }}-test-${{ github.job }}-ccache-v1 run: gh cache delete "${CACHE_KEY}" || true - name: Save ccache if: github.ref_name == github.event.repository.default_branch continue-on-error: true uses: actions/cache/save@v5 with: path: .ccache key: ${{ runner.os }}-test-${{ github.job }}-ccache-v1 - name: Package Linux runtime run: | mkdir -p ci-artifacts mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort) tar \ --exclude='*/CMakeFiles' \ --exclude='*.o' \ --exclude='*.a' \ --exclude='*.ninja*' \ --exclude='build.ninja' \ --exclude='cmake_install.cmake' \ -czf ci-artifacts/mobilegl-linux-runtime.tgz \ "${BUILD_DIR}/CTestTestfile.cmake" \ "${BUILD_DIR}/MobileGL/MG_Test" \ "${BUILD_DIR}/MobileGL/MG_Benchmark" \ "${BUILD_DIR}/MobileGL/MG_IntegrationTest" \ "${SHARED_LIBS[@]}" - name: Upload Linux runtime uses: actions/upload-artifact@v7 with: name: mobilegl-linux-runtime path: ci-artifacts/mobilegl-linux-runtime.tgz if-no-files-found: error test: runs-on: ubuntu-latest needs: build-linux steps: - name: Checkout repo uses: actions/checkout@v6 - name: Get CMake uses: lukka/get-cmake@v4.3.3 - name: Install runtime dependencies run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libgles2 libgl1-mesa-dri mesa-vulkan-drivers - name: Download Linux runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime path: . - name: Unpack Linux runtime run: tar -xzf mobilegl-linux-runtime.tgz - name: Normalize CTest command paths run: | python - <<'PY' from pathlib import Path import re for path in Path('build-linux').rglob('CTestTestfile.cmake'): text = path.read_text() text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text) path.write_text(text) PY - name: Test working-directory: build-linux run: | ulimit -c unlimited sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p' if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then ctest -V -L unit --no-tests=error else ctest --output-on-failure -L unit --no-tests=error fi - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: unit-core-dumps path: /tmp/core.* if-no-files-found: ignore integration: runs-on: ubuntu-latest needs: build-linux steps: - name: Checkout repo uses: actions/checkout@v6 - name: Get CMake uses: lukka/get-cmake@v4.3.3 - name: Install runtime dependencies # Same set as the benchmark job, for the same reason: the scenarios bring # up real headless EGL (llvmpipe) and Vulkan (lavapipe) contexts, and # libegl-mesa0 - the EGL vendor library behind glvnd's libegl1 dispatch - # only arrives as a Recommends. run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers - name: Download Linux runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime path: . - name: Unpack Linux runtime run: tar -xzf mobilegl-linux-runtime.tgz - name: Normalize CTest command paths run: | python - <<'PY' from pathlib import Path import re for path in Path('build-linux').rglob('CTestTestfile.cmake'): text = path.read_text() text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text) path.write_text(text) PY - name: Integration scenarios working-directory: build-linux # REQUIRE_GPU makes a driverless runner FAIL instead of skipping every # scenario - an all-skip run is otherwise indistinguishable from a pass, # which is how a five-month-old draw-dropping bug survived unseen until # this lane existed. # # The lavapipe ICD pin lives in the build-linux configure # (-DMOBILEGL_ITEST_VK_ICD), NOT here: the configure bakes it into each # test's ctest ENVIRONMENT property, and a property entry OVERRIDES the # job environment - a VK_ICD_FILENAMES exported here would be silently # ignored while looking like it works. This lane runs on lavapipe # deterministically, not on whichever of the eight Mesa ICDs a GPU-less # runner enumerates first. # # Cores are armed so that any crash - the harness pre-flight child's # included - leaves /tmp/core.*, which the failure-only step below ships # as an artifact. Analyzing a downloaded core against the runtime # artifact's binary in an ubuntu-24.04 userspace reproduces the exact # crash stack without burning a CI round on an in-workflow debugger. env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH: "1" MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS: "1" MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER: "1" run: | ulimit -c unlimited sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p' # Second, filtered pass: with the range-invalidating map flush disabled, # the buffer scenarios run on the upload ring's staged-copy tier - which # the default pass never reaches (the map tier absorbs every flush on # Mesa), so without this the Mali fallback tier would have zero CI # coverage. The flag is NOT baked into the ctest ENVIRONMENT properties, # so an inline env reaches the test processes (unlike the ICD pin above). if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then ctest -V -L integration-gpu --no-tests=error MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 ctest -V -L integration-gpu \ -R 'Buffer|Readback|Atomic|Ssbo|Arena' --no-tests=error else ctest --output-on-failure -L integration-gpu --no-tests=error MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 ctest --output-on-failure -L integration-gpu \ -R 'Buffer|Readback|Atomic|Ssbo|Arena' --no-tests=error fi - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: integration-core-dumps path: /tmp/core.* if-no-files-found: ignore # THE THIRD CI MODE (ARCHITECTURE.md 13.2-(2)): the same library, built with the PipeInputs # comparator compiled in, running the integration suite and a trace subset with two state models # in one address space. It is a second build rather than a flag on the first because # MOBILEGL_PIPE_VERIFY is a compile-time option - the snapshot, the entry compare and the # compare-at-read hook do not exist in the shipped library, and are never meant to. build-linux-verify: runs-on: ubuntu-latest timeout-minutes: 120 permissions: actions: write contents: read env: BUILD_DIR: build-verify CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "true" CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 4G CCACHE_NOHASHDIR: "true" steps: - name: Set Swap Space uses: pierotofy/set-swap-space@v1.0 with: swap-size-gb: 32 - name: Checkout repo uses: actions/checkout@v6 with: submodules: recursive - name: Get CMake uses: lukka/get-cmake@v4.3.3 - name: Restore ccache uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-test-${{ github.job }}-ccache-v1 restore-keys: | ${{ runner.os }}-test-${{ github.job }}-ccache- - name: Prepare Vulkan SDK uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Update glslang external sources working-directory: 3rdparty/glslang run: python update_glslang_sources.py - name: Install build dependencies run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build - name: Show installed toolchain run: | ccache --version clang-20 --version clang++-20 --version ld.lld-20 --version || ld.lld --version || true dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true - name: Configure CMake # Release/INFO like the shipped build on purpose. The poison arms in this configuration # through MOBILEGL_PIPE_VERIFY (PipeInputs.h derives MOBILEGL_PIPE_POISON from it), so this # job needs neither a Debug log level nor MOBILEGL_BUILD_DISAGGREGATED - and a Debug build # would compare a different library from the one the other lanes measure. (build-linux # switches to Debug under ACTIONS_STEP_DEBUG; this job deliberately does not - a Debug # build flips CXX_VISIBILITY_PRESET and arms MOBILEGL_PIPE_POISON through a second, unrelated # arm of its #if, so the debug switch would change what the lane is measuring.) run: | cmake -S . -B "${BUILD_DIR}" -G Ninja \ -DCMAKE_C_COMPILER=clang-20 \ -DCMAKE_CXX_COMPILER=clang++-20 \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_BUILD_TYPE=Release \ -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=ON \ -DMOBILEGL_BUILD_BENCHMARK=OFF \ -DMOBILEGL_BUILD_INTEGRATION_TEST=ON \ -DMOBILEGL_ITEST_VK_ICD=/usr/share/vulkan/icd.d/lvp_icd.json \ -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \ -DMOBILEGL_PIPE_VERIFY=ON \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 - name: Build run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)" # The lane is worthless if the option silently did not take, and that is a one-character # mistake away at all times (a typo'd -D is not an error in CMake). Two checks, both cheap: # the comparator's entry point must be in the library, and the fill entry point with it. # # `nm` and NOT `nm -D`. The library is built CXX_VISIBILITY_PRESET hidden in every non-Debug # configuration (CMakeLists.txt:600-604) and the MGPipe entry points are plain namespace # functions with no export attribute, so not one of them appears in the DYNAMIC table: on a # perfectly healthy verify build `nm -D --defined-only ... | grep -c MGPipe` answers 0 out of # ~11900 exported symbols, and a gate spelled that way is red forever for a reason that has # nothing to do with what it claims to test. The static symbol table has them as local `t` # entries, this artifact is never stripped, and `No MG_Remote in the pull build` below already # uses this spelling. The symbol count guards the remaining hole: a stripped library would # make both greps fail for a third, silent reason. - name: The verify library really carries the comparator run: | test -f "${BUILD_DIR}/libMobileGL.so" defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l) if [ "${defined}" -lt 1000 ]; then echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the two checks below could not have failed honestly" exit 1 fi if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeVerifyInputs"; then echo "::error::libMobileGL.so defines no MGPipeVerifyInputs: -DMOBILEGL_PIPE_VERIFY=ON did not take, and every lane that consumes this artifact would run the comparator-free library and pass having compared nothing" exit 1 fi # The per-verb entry point, under EITHER of its two names. P2 renames # MGPipeFillForVerb to MGPipeValidateForVerb (the body becomes the tracker's walk and # the fill is one of its five steps), so this check has to accept both or it goes red on # the rename for a reason that has nothing to do with what it tests. What it tests is # unchanged: that the library HAS a per-verb entry point compiled in. if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -qE "MGPipeValidateForVerb|MGPipeFillForVerb"; then echo "::error::libMobileGL.so defines neither MGPipeValidateForVerb nor MGPipeFillForVerb: there is no per-verb entry point in this artifact, so nothing fills the block the comparator compares" exit 1 fi echo "libMobileGL.so defines MGPipeVerifyInputs and a per-verb entry point (${defined} defined symbols)" - name: Show ccache stats if: always() run: ccache --show-stats - name: Release superseded ccache entry if: github.ref_name == github.event.repository.default_branch env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ runner.os }}-test-${{ github.job }}-ccache-v1 run: gh cache delete "${CACHE_KEY}" || true - name: Save ccache if: github.ref_name == github.event.repository.default_branch continue-on-error: true uses: actions/cache/save@v5 with: path: .ccache key: ${{ runner.os }}-test-${{ github.job }}-ccache-v1 - name: Package Linux verify runtime run: | mkdir -p ci-artifacts mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort) tar \ --exclude='*/CMakeFiles' \ --exclude='*.o' \ --exclude='*.a' \ --exclude='*.ninja*' \ --exclude='build.ninja' \ --exclude='cmake_install.cmake' \ -czf ci-artifacts/mobilegl-linux-runtime-verify.tgz \ "${BUILD_DIR}/CTestTestfile.cmake" \ "${BUILD_DIR}/MobileGL/MG_Test" \ "${BUILD_DIR}/MobileGL/MG_IntegrationTest" \ "${SHARED_LIBS[@]}" - name: Upload Linux verify runtime uses: actions/upload-artifact@v7 with: name: mobilegl-linux-runtime-verify path: ci-artifacts/mobilegl-linux-runtime-verify.tgz if-no-files-found: error # The verify lane itself, plus the two negative controls that keep it falsifiable. The controls # are ALWAYS-ON steps, not a manual exercise: a gate that can only be shown to work by someone # remembering to break it on purpose is a gate that has already stopped working. integration-verify: runs-on: ubuntu-latest timeout-minutes: 180 needs: build-linux-verify steps: - name: Checkout repo uses: actions/checkout@v6 - name: Get CMake uses: lukka/get-cmake@v4.3.3 - name: Install runtime dependencies run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers - name: Download Linux verify runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime-verify path: . - name: Unpack Linux verify runtime run: | tar -xzf mobilegl-linux-runtime-verify.tgz test -f build-verify/libMobileGL.so - name: Normalize CTest command paths run: | python - <<'PY' from pathlib import Path import re for path in Path('build-verify').rglob('CTestTestfile.cmake'): text = path.read_text() text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text) path.write_text(text) PY - name: Integration scenarios under MOBILEGL_PIPE_VERIFY working-directory: build-verify # --no-tests=error is half the gate: the verify entries only exist when the library was # configured with -DMOBILEGL_PIPE_VERIFY=ON, so a build that lost the option matches no # tests and reds here instead of reporting a green run of nothing. The other half is # PipeVerifyArmingScenario.Armed, which fails when the library never printed its arming # line - the failure mode a bare `MOBILEGL_PIPE_VERIFY=1` cannot detect by itself. # # SCOPE, stated so nobody reads more into a green than is there: this is every integration # ENTRY under the comparator, not every integration CONFIGURATION. The `integration` job # runs a second, filtered pass with MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 for the # upload ring's staged-copy tier; that pass is 186 entries here and, at the 5-10x the # comparator costs, is not affordable inside this job's budget. The tier is covered by # `integration`, unverified, and P2 can take it once the comparator's cost is known. env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH: "1" MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS: "1" MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER: "1" run: | ulimit -c unlimited sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p' if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then ctest -V -L integration-verify --no-tests=error else ctest --output-on-failure -L integration-verify --no-tests=error fi # The push-only unit tests, on the verify runtime. # # WHY HERE AND NOT IN `test`. The `test` job builds the PULL library, and G6's chunk-table # walk and G10's residual assertions live in MG_Test/Pipe, compiled only under # MOBILEGL_PIPE_PUSH (MGPipeRenderStateSpans.cpp and PipeApply.cpp are appended to # SOURCE_FILES inside the `if (MOBILEGL_PIPE_PUSH)` block, which is exactly how the pull # build stays symbol-identical). So before P2 those tests ran in no CI job at all: they # existed, they were green locally, and CI never executed one of them. # # This artifact already carries them - the packaging step above tars # ${BUILD_DIR}/MobileGL/MG_Test whole - so the whole cost is the run, which is ~14 s for # ~1490 entries. --no-tests=error, because a packaging change that stopped shipping the # unit binaries would otherwise report a green run of nothing. - name: Unit tests on the verify runtime (G6, G10) working-directory: build-verify run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" # The two always-on P2 negative controls (G8, G12), which are labelled integration-gpu and # not integration-verify - they are about the handle key and the CSO switch, not about the # comparator - so the lane above does not reach them. They are run HERE because this is the # only CI job that unpacks a MOBILEGL_PIPE_PUSH build: CsoContentAddressingScenario reads # the two CSO counters out of the library's summary line and both the counters and the # cso[] bracket are #if MOBILEGL_PIPE_PUSH, so in the pull `integration` job the entries do # not exist at all. # # An arm whose subsystem has not landed on this tree SKIPS with the reason (never absent, # never a green that asserted nothing), so this step is green through the P2 landing order # and starts asserting as each package arrives. # # The environment is the sibling step's, deliberately and in full: these entries run the # same DirectVulkan binary through the same runner, so the three MOBILEGL_MAGMA_* fixes it # needs apply here too, and a crash here has to leave a core for the same black-box flow. # The step above is the only reason those lines exist in this job; a control that crashed # without one would be the hardest failure in the job to diagnose. - name: The handle-ABA and CSO-content-addressing controls (G8, G12) working-directory: build-verify env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH: "1" MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS: "1" MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER: "1" run: | ulimit -c unlimited sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p' ctest --output-on-failure -L integration-gpu \ -R 'HandleRecycle|CsoContentAddressing' --no-tests=error -j 4 # The arming lanes' logs, and ONLY those. Each lane shares one MOBILEGL_LOG_FILE_PATH and the # library opens it fopen(path, "w"), so after an ambient lane of 400-odd processes the file # holds the LAST one - grepping it would say nothing about the other 405 and would red a # healthy lane whenever the last entry happened not to issue a verb (which is what the # PoisonOmissionScenario parent, the last ambient entry, does by construction: it forks, # execve()s and reads files). The DirectGLES.VerifyArming. / DirectVulkan.VerifyArming. # entries are one process each on a log path nothing else writes, so this grep means exactly # what it says. # # What it proves: arming is a property of (this library, this environment), and these two # processes ran the same library with the same MOBILEGL_PIPE_VERIFY=1 as their ~400 ambient # siblings. It is not, and cannot be, a per-process census - the shared log cannot support one. # It catches the case ctest cannot: an arming entry that SKIPPED still reports green. - name: The verify lanes armed the comparator working-directory: build-verify run: | shopt -s nullglob logs=(MobileGL/MG_IntegrationTest/pipe-verify-arming-*.log) if [ ${#logs[@]} -lt 2 ]; then echo "::error::found ${#logs[@]} pipe-verify-arming-*.log (expected one per backend). The VerifyArming. entries did not run, so nothing in this job establishes that the comparator was ever armed." exit 1 fi for log in "${logs[@]}"; do if ! grep -q "MGPipe: verify armed" "${log}"; then echo "::error::${log} carries no arming line: that lane's process ran the whole scenario without the comparator, so every green entry beside it is green for no reason" exit 1 fi done echo "arming line present in all ${#logs[@]} arming-lane log(s)" # NEGATIVE CONTROL A (gate G4). The knob perturbs one field in the snapshot arm before the # entry compare, so a working comparator must abort the run. This step passes when ctest # FAILS - `if ctest ...; then error` - which is the only shape that can catch a comparator # that silently compares nothing. # # The knob reaches the test process through the JOB environment: no ctest ENVIRONMENT # property on the ambient Verify. entries names it (MG_IntegrationTest/CMakeLists.txt says # so out loud), and a property entry would otherwise override this and the control would # prove nothing. Same precedent as MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH in `integration`. - name: Negative control A - a corrupted snapshot field must turn the lane red working-directory: build-verify env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_PIPE_VERIFY_CORRUPT: GetRenderStateParameters run: | FILTER='DirectGLES\.Verify\..*ClearThenReadPixels' # An empty selection would ALSO make ctest exit non-zero (--no-tests=error), and this # step reads non-zero as "the control worked" - so the selection is counted first. A # control that passes because it ran nothing is worse than no control. matched=$(ctest -N -L integration-verify -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:') if [ "${matched}" -lt 1 ]; then echo "::error::negative control A selected ${matched} tests; its filter no longer matches anything" exit 1 fi if ctest --output-on-failure -L integration-verify -R "${FILTER}" --no-tests=error; then echo "::error::MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left ${matched} verify entries GREEN. The comparator is not comparing, so every green entry above is green for no reason." exit 1 fi echo "the corrupted field turned ${matched} selected entries red, as it must" # NEGATIVE CONTROL B (gate G5). The omission skips the STAMP of one field for one verb while # still copying its value - indistinguishable from a fill row nobody wrote - so the poison # must abort the glGenerateMipmap. Again: this step passes when ctest fails. # # The entry it targets is PoisonOmissionScenario.WithoutOmissionCompletes, which is green in # the ambient lane above and is the ONLY integration entry in the tree that calls # glGenerateMipmap at all. It deliberately does not skip itself when the knob is set, exactly # so that this control has something to turn red. - name: Negative control B - an omitted fill point must turn the lane red on that verb working-directory: build-verify env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_PIPE_POISON_OMIT: GenerateMipmap:GetActiveTextureUnit run: | FILTER='DirectGLES\.Verify\.PoisonOmissionScenario\.WithoutOmissionCompletes' matched=$(ctest -N -L integration-verify -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:') if [ "${matched}" -lt 1 ]; then echo "::error::negative control B selected ${matched} tests; its filter no longer matches anything" exit 1 fi if ctest --output-on-failure -L integration-verify -R "${FILTER}" --no-tests=error; then echo "::error::MOBILEGL_PIPE_POISON_OMIT=GenerateMipmap:GetActiveTextureUnit left the verify lane GREEN. The per-verb poison is not armed, so a forgotten fill row would ship silently." exit 1 fi echo "the omitted fill point turned the lane red, as it must" - name: Upload verify lane logs if: always() uses: actions/upload-artifact@v7 with: name: integration-verify-logs path: build-verify/MobileGL/MG_IntegrationTest/pipe-*.log* if-no-files-found: warn - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: integration-verify-core-dumps path: /tmp/core.* if-no-files-found: ignore # MobileGL/MG_Remote/Protocol/generated/protocol_generated.h is COMMITTED, and # flatc is deliberately absent from the default build graph (a codegen step in # the graph is how the earlier branch ended up cross-compiling an arm64 flatc # and trying to run it on the host). This job is what keeps the committed # header honest: build the pinned flatc, regenerate, and fail on any diff. # It needs no MobileGL build, so it does not depend on build-linux. flatc-check: runs-on: ubuntu-latest steps: - name: Checkout repo uses: actions/checkout@v6 - name: Get CMake uses: lukka/get-cmake@v4.3.3 - name: Check out the FlatBuffers submodule only # Just this one: the schema check has nothing to do with glslang, # SPIRV-Cross or the trace fixtures. run: git submodule update --init 3rdparty/flatbuffers - name: Regenerate protocol_generated.h run: python3 scripts/gen_protocol.py --build-dir "${{ runner.temp }}/flatc-build" - name: Fail if the committed header is stale run: git diff --exit-code -- MobileGL/MG_Remote/Protocol/generated/protocol_generated.h # P0.5 interface-purity gate A (ARCHITECTURE.md:501): the two extracted headers' include closure, # asserted on `-H` output because `nm --undefined-only` is blind to "included but not called" - # a header whose types are never named leaves no symbol behind, and "included at all" is exactly # the coupling P1 and P7 have to sever. Needs a preprocessor and three header submodules, no # CMake configure and no glslang sources, so like pipe-gates it does not depend on build-linux. # The script's own --self-test is always on: a negative control that stopped tripping fails the # job, because a gate that cannot go red is not a gate (ROADMAP.md:7). include-graph-check: name: Include-closure purity gate runs-on: ubuntu-latest steps: - name: Checkout repo uses: actions/checkout@v6 - name: Check out the three header submodules the closure needs # ska/flat_hash_map.hpp, xxhash.h and vulkan/vulkan.h are the only submodule headers # Includes.h reaches; glslang and spirv_cross are vendored under include/. run: git submodule update --init include/ska 3rdparty/xxHash 3rdparty/Vulkan-Headers - name: Install clang and the X11 headers vulkan.h pulls on Linux # Includes.h defines VK_USE_PLATFORM_XLIB_KHR before , which then # includes ; without libx11-dev every clang-mode probe dies in the # preprocessor and the gate reports 5 problems that have nothing to do with purity. run: sudo apt-get update && sudo apt-get install -y clang-20 libx11-dev - name: Include-closure assertions and negative control run: python3 scripts/check_include_closure.py --mode both --compiler clang++-20 --self-test --require-all benchmark: runs-on: ubuntu-latest needs: build-linux steps: - name: Checkout repo uses: actions/checkout@v6 - name: Get CMake uses: lukka/get-cmake@v4.3.3 - name: Install runtime dependencies # libegl-mesa0 is the EGL vendor library itself: DriverBench brings up a # real GL context, and libegl1 is only glvnd's dispatch. It normally # arrives as a Recommends of libegl1, which is too quiet a dependency for # the one job that needs a working driver. run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers - name: Download Linux runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime path: . - name: Unpack Linux runtime run: tar -xzf mobilegl-linux-runtime.tgz - name: Normalize CTest command paths run: | python - <<'PY' from pathlib import Path import re for path in Path('build-linux').rglob('CTestTestfile.cmake'): text = path.read_text() text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text) path.write_text(text) PY - name: Benchmark working-directory: build-linux run: | ulimit -c unlimited sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p' ctest -V -C Release -L benchmark --no-tests=error - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: benchmark-core-dumps path: /tmp/core.* if-no-files-found: ignore build-retrace: runs-on: ubuntu-latest needs: - build-linux - test - benchmark - integration permissions: actions: write contents: read env: BUILD_DIR: build-retrace CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "true" CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 4G CCACHE_NOHASHDIR: "true" MOBILEGL_LIBRARY: ${{ github.workspace }}/build-linux/libMobileGL.so steps: - name: Set Swap Space uses: pierotofy/set-swap-space@v1.0 with: swap-size-gb: 32 - name: Checkout repo uses: actions/checkout@v6 with: submodules: recursive - name: Get CMake uses: lukka/get-cmake@v4.3.3 - name: Restore ccache uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-test-${{ github.job }}-ccache-v1 restore-keys: | ${{ runner.os }}-test-${{ github.job }}-ccache- - name: Prepare Vulkan SDK uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Update glslang external sources working-directory: 3rdparty/glslang run: python update_glslang_sources.py - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build - name: Show installed toolchain run: | ccache --version clang-20 --version clang++-20 --version ld.lld-20 --version || ld.lld --version || true dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true - name: Download Linux runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime path: . - name: Unpack Linux runtime run: | tar -xzf mobilegl-linux-runtime.tgz test -f "${MOBILEGL_LIBRARY}" - name: Configure CMake run: | if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then BUILD_TYPE=Debug else BUILD_TYPE=Release fi cmake -S . -B "${BUILD_DIR}" -G Ninja \ -DCMAKE_C_COMPILER=clang-20 \ -DCMAKE_CXX_COMPILER=clang++-20 \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=OFF \ -DMOBILEGL_BUILD_BENCHMARK=OFF \ -DMOBILEGL_BUILD_TRACE_REPLAY=ON \ -DMOBILEGL_TRACE_REPLAY_MOBILEGL_LIBRARY="${MOBILEGL_LIBRARY}" \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 - name: Build trace replay run: cmake --build "${BUILD_DIR}" --target mobilegl_trace_replay --parallel "$(nproc)" - name: Show ccache stats if: always() run: ccache --show-stats - name: Release superseded ccache entry if: github.ref_name == github.event.repository.default_branch env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ runner.os }}-test-${{ github.job }}-ccache-v1 run: gh cache delete "${CACHE_KEY}" || true - name: Save ccache if: github.ref_name == github.event.repository.default_branch continue-on-error: true uses: actions/cache/save@v5 with: path: .ccache key: ${{ runner.os }}-test-${{ github.job }}-ccache-v1 - name: Normalize CTest command paths run: | python - <<'PY' from pathlib import Path import re for path in Path('build-retrace').rglob('CTestTestfile.cmake'): text = path.read_text() text = re.sub(r'"[^"]*/cmake-[^"]*/bin/cmake"', '"cmake"', text) path.write_text(text) PY - name: Package trace replay run: | mkdir -p ci-artifacts tar -czf ci-artifacts/mobilegl-trace-replay.tgz \ build-retrace/tools/trace_replay/mobilegl_trace_replay \ build-retrace/tools/trace_replay/CTestTestfile.cmake - name: Upload trace replay uses: actions/upload-artifact@v7 with: name: mobilegl-trace-replay path: ci-artifacts/mobilegl-trace-replay.tgz if-no-files-found: error trace-cases: name: trace case matrix runs-on: ubuntu-latest needs: - test - benchmark - integration outputs: matrix: ${{ steps.trace-cases.outputs.matrix }} names: ${{ steps.trace-cases.outputs.names }} verify-matrix: ${{ steps.trace-cases.outputs.verify-matrix }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Load trace cases id: trace-cases run: | echo "matrix=$(python3 tools/trace_replay/trace_cases.py --ci --format github-test-matrix)" >> "$GITHUB_OUTPUT" echo "names=$(python3 tools/trace_replay/trace_cases.py --ci --format names)" >> "$GITHUB_OUTPUT" # The subset the verify build retraces ("verify": true in trace_cases.json). It is a # SUBSET of the matrix above, so retrace-verify needs no fixtures of its own. echo "verify-matrix=$(python3 tools/trace_replay/trace_cases.py --ci --format github-verify-matrix)" >> "$GITHUB_OUTPUT" trace-fixtures: name: trace fixture (${{ matrix.case }}) runs-on: ubuntu-latest needs: trace-cases strategy: fail-fast: false max-parallel: 4 matrix: case: ${{ fromJSON(needs.trace-cases.outputs.names) }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Derive trace fixture cache key id: fixture-key run: bash .github/scripts/trace-fixture-cache.sh key '${{ matrix.case }}' - name: Restore trace fixture cache id: fixture-cache if: steps.fixture-key.outputs.cacheable == 'true' uses: actions/cache/restore@v5 with: path: ${{ steps.fixture-key.outputs.paths }} key: ${{ steps.fixture-key.outputs.key }} - name: Verify restored trace fixture id: fixture-verify if: steps.fixture-cache.outputs.cache-hit == 'true' run: | if bash .github/scripts/trace-fixture-cache.sh verify '${{ matrix.case }}'; then echo "ok=true" >> "$GITHUB_OUTPUT" else echo "ok=false" >> "$GITHUB_OUTPUT" echo "::warning::Cached fixture for ${{ matrix.case }} failed verification; falling back to the download path" bash .github/scripts/trace-fixture-cache.sh reset '${{ matrix.case }}' fi - name: Fetch trace fixture if: steps.fixture-verify.outputs.ok != 'true' run: bash .github/scripts/fetch-trace-fixture-lfs.sh '${{ matrix.case }}' - name: Save trace fixture cache if: steps.fixture-key.outputs.cacheable == 'true' && steps.fixture-cache.outputs.cache-hit != 'true' uses: actions/cache/save@v5 with: path: ${{ steps.fixture-key.outputs.paths }} key: ${{ steps.fixture-key.outputs.key }} - name: Stage trace fixture run: | safe_case="$(printf '%s' '${{ matrix.case }}' | sed 's/[^A-Za-z0-9._-]/_/g')" stage_dir="trace-fixtures/${safe_case}" mkdir -p "${stage_dir}" python3 tools/trace_replay/trace_cases.py --format fixture-files --case '${{ matrix.case }}' | while IFS= read -r file; do cp "${file}" "${stage_dir}/" done - name: Upload trace fixture uses: actions/upload-artifact@v7 with: name: trace-fixture-${{ matrix.case }} path: trace-fixtures/** if-no-files-found: error retrace: name: retrace (${{ matrix.backend }}, ${{ matrix.case }}) runs-on: ubuntu-latest needs: - build-linux - build-retrace - trace-cases - trace-fixtures if: ${{ always() && needs.build-linux.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }} strategy: fail-fast: false max-parallel: 4 matrix: ${{ fromJSON(needs.trace-cases.outputs.matrix) }} steps: - name: Set Swap Space uses: pierotofy/set-swap-space@v1.0 with: swap-size-gb: 16 - name: Checkout repo uses: actions/checkout@v6 - name: Download trace fixture uses: actions/download-artifact@v8 with: name: trace-fixture-${{ matrix.case }} path: trace-fixture-download - name: Install trace fixture run: | mkdir -p tools/trace_replay/fixtures find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \; - name: Get CMake uses: lukka/get-cmake@v4.3.3 - name: Install runtime dependencies run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers test -e /usr/lib/x86_64-linux-gnu/libEGL.so test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so - name: Download Linux runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime path: . - name: Download trace replay uses: actions/download-artifact@v8 with: name: mobilegl-trace-replay path: . - name: Unpack retrace runtime run: | tar -xzf mobilegl-linux-runtime.tgz tar -xzf mobilegl-trace-replay.tgz test -f build-linux/libMobileGL.so test -f build-retrace/tools/trace_replay/mobilegl_trace_replay - name: Retrace and validate working-directory: build-retrace/tools/trace_replay run: | ulimit -c unlimited sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p' if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1 fi if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \ && [ '${{ matrix.case }}' = 'minecraft-1.21.4-fabric-iris-iterationrp-in-world' ]; then export MOBILEGL_MAGMA_FIX_ITERATIONRP_SUBGROUP_SCRATCH=1 export MOBILEGL_MAGMA_DERIVE_NUM_SUBGROUPS=1 export MOBILEGL_MAGMA_ITERATIONRP_FIX_BARRIER=1 fi # The blended depth-write quirk auto-enables only on Qualcomm, which no CI # runner has, so force it on for the OIT case it exists to fix. ForceOn # bypasses only the vendor gate, so this exercises the real strip on # lavapipe. The Android AVD lane deliberately leaves it off, keeping the # unstripped path covered for the same trace. if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \ && [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then export MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1 fi ctest -V --no-tests=error -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$' - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: retrace-core-dumps-${{ matrix.backend }}-${{ matrix.case }} path: /tmp/core.* if-no-files-found: ignore - name: Upload actual image if: always() uses: actions/upload-artifact@v7 with: name: retrace-result-${{ matrix.backend }}-${{ matrix.case }} path: | build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/** build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/** if-no-files-found: warn retrace-summary: name: retrace summary runs-on: ubuntu-latest needs: retrace if: ${{ always() && needs.retrace.result != 'skipped' }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Set artifact metadata run: | echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV" - name: Set up Node.js uses: actions/setup-node@v7 with: node-version: '22' - name: Download retrace results uses: actions/download-artifact@v8 with: pattern: retrace-result-* path: retrace-artifacts - name: Render retrace summary run: | node tools/trace_replay/render_retrace_summary.mjs \ --input retrace-artifacts \ --output-dir retrace-summary \ --title "MobileGL Linux retrace overview" \ --group-label "Linux" \ --html mobilegl-linux-retrace-overview.html - name: Upload retrace summary uses: actions/upload-artifact@v7 with: path: retrace-summary/mobilegl-linux-retrace-overview.html archive: false if-no-files-found: error # The trace half of the third CI mode. Same replay, same goldens, but the library underneath is # the verify build and MOBILEGL_PIPE_VERIFY=1 is in the environment, so every backend read of # frontend state is checked against a snapshot taken at the verb boundary. Eight cases rather # than the full lane's 40 (tools/trace_replay/trace_cases.json, "verify": true): the comparator # is budgeted at 5-10x, and the full sweep is a phase-exit / workflow_dispatch run. retrace-verify: name: retrace verify (${{ matrix.backend }}, ${{ matrix.case }}) runs-on: ubuntu-latest timeout-minutes: 240 needs: - build-linux-verify - build-retrace - trace-cases - trace-fixtures if: ${{ always() && needs.build-linux-verify.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }} strategy: fail-fast: false max-parallel: 4 matrix: ${{ fromJSON(needs.trace-cases.outputs.verify-matrix) }} steps: - name: Set Swap Space uses: pierotofy/set-swap-space@v1.0 with: swap-size-gb: 16 - name: Checkout repo uses: actions/checkout@v6 - name: Download trace fixture uses: actions/download-artifact@v8 with: name: trace-fixture-${{ matrix.case }} path: trace-fixture-download - name: Install trace fixture run: | mkdir -p tools/trace_replay/fixtures find trace-fixture-download -type f -exec cp {} tools/trace_replay/fixtures/ \; - name: Get CMake uses: lukka/get-cmake@v4.3.3 - name: Install runtime dependencies run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers test -e /usr/lib/x86_64-linux-gnu/libEGL.so test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so - name: Download Linux verify runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime-verify path: . - name: Download trace replay uses: actions/download-artifact@v8 with: name: mobilegl-trace-replay path: . - name: Unpack the VERIFY runtime as the library under test # build-retrace's CTestTestfile.cmake has the absolute path # /build-linux/libMobileGL.so frozen into every case, so the swap happens here # rather than through a variable: the verify .so is put where that path points. The nm # check is what makes the swap falsifiable - a run against the ordinary library would # carry no comparator, ignore MOBILEGL_PIPE_VERIFY entirely, and match its golden. # # `nm`, not `nm -D`, for the reason spelled out in build-linux-verify: everything MGPipe is # hidden-visibility in a Release build and the dynamic table has none of it. run: | tar -xzf mobilegl-linux-runtime-verify.tgz tar -xzf mobilegl-trace-replay.tgz test -f build-verify/libMobileGL.so test -f build-retrace/tools/trace_replay/mobilegl_trace_replay mkdir -p build-linux cp build-verify/libMobileGL.so build-linux/libMobileGL.so if ! nm --defined-only build-linux/libMobileGL.so | grep -q MGPipeVerifyInputs; then echo "::error::the library unpacked at build-linux/libMobileGL.so defines no MGPipeVerifyInputs, so this retrace would replay against a comparator-free build and pass on its golden having verified nothing" exit 1 fi echo "the library at build-linux/libMobileGL.so is the verify build" - name: Retrace and validate under MOBILEGL_PIPE_VERIFY working-directory: build-retrace/tools/trace_replay # run_trace_case.cmake turns MOBILEGL_PIPE_VERIFY into three assertions of its own (the # arming line, no Fatal{PipeVerifyDiffer, no Fatal{UnmigratedPipeInput), so a case that # somehow ran the wrong library reds here instead of passing on its golden. # --timeout 10800: the 1800s cases run 5-10x slower with both comparator arms live, which # is well past ctest's 1500s default. run: | ulimit -c unlimited sudo sysctl -w kernel.core_pattern='/tmp/core.%e.%p' export MOBILEGL_PIPE_VERIFY=1 if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1 fi if [ '${{ matrix.backend }}' = 'DirectVulkan' ] \ && [ '${{ matrix.case }}' = 'improved-transparency-minecraft-26.3' ]; then export MOBILEGL_MAGMA_DISABLE_BLENDED_DEPTH_WRITE=1 fi ctest -V --no-tests=error --timeout 10800 \ -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$' # The retrace lane's own always-on negative control, on one case so it costs one short trace: # with a snapshot field corrupted, the SAME replay must fail. Without it, "40 traces, zero # divergences" would be a statement about a comparator nobody watched. # # The rerun replays into the SAME case directory, so the verified run's images are put aside # first and restored before the verdict: "Upload actual image" below runs `if: always()` and # would otherwise ship the deliberately corrupted run's output under the name of the good one. # The restore happens whichever way the control goes, which is why the ctest exit status is # captured rather than tested inline. - name: Negative control - a corrupted snapshot field must red this retrace if: ${{ matrix.case == 'OpenRA' && matrix.backend == 'DirectGLES' }} working-directory: build-retrace/tools/trace_replay run: | export MOBILEGL_PIPE_VERIFY=1 export MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters GOOD_OUTPUT="${RUNNER_TEMP}/openra-verified-output" rm -rf "${GOOD_OUTPUT}" if [ -d OpenRA ]; then cp -a OpenRA "${GOOD_OUTPUT}" fi set +e ctest -V --no-tests=error --timeout 10800 \ -R '^MobileGLTraceReplay\.OpenRA\.DirectGLES$' control_rc=$? set -e if [ -d "${GOOD_OUTPUT}" ]; then rm -rf OpenRA mv "${GOOD_OUTPUT}" OpenRA echo "restored the verified run's OpenRA output over the corrupted rerun's" fi if [ "${control_rc}" -eq 0 ]; then echo "::error::MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left the OpenRA retrace GREEN, so the comparator is not comparing and the whole verify retrace lane proves nothing." exit 1 fi echo "the corrupted field turned the retrace red, as it must (ctest exit ${control_rc})" - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: retrace-verify-core-dumps-${{ matrix.backend }}-${{ matrix.case }} path: /tmp/core.* if-no-files-found: ignore - name: Upload actual image if: always() uses: actions/upload-artifact@v7 with: name: retrace-verify-result-${{ matrix.backend }}-${{ matrix.case }} path: | build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/** build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/** if-no-files-found: warn # G1's own job: the pull build must be the tree before P1, symbol for symbol and byte for byte. # workflow_dispatch only - it builds the library twice from scratch, and its answer is about a # BASELINE rather than about this push, so a per-push run would be measuring the wrong pair. monolith-symbol-report: name: monolith symbol report runs-on: ubuntu-latest timeout-minutes: 180 if: ${{ github.event_name == 'workflow_dispatch' }} env: CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "true" CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 4G CCACHE_NOHASHDIR: "true" steps: - name: Set Swap Space uses: pierotofy/set-swap-space@v1.0 with: swap-size-gb: 32 - name: Checkout repo uses: actions/checkout@v6 with: submodules: recursive fetch-depth: 0 - name: Get CMake uses: lukka/get-cmake@v4.3.3 - name: Restore ccache uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-test-${{ github.job }}-ccache-v1 restore-keys: | ${{ runner.os }}-test-${{ github.job }}-ccache- - name: Prepare Vulkan SDK uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Install build dependencies run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build binutils # Both sides with IDENTICAL flags, LTO off, the same compiler and the same standard library: # symbol_report.py's guard rails (scripts/symbol_report.py) say a mismatched pair "adds" # thousands of symbols and the comparison then means nothing. The library alone - no tests, # no benchmark, no integration test, no trace replay - because those targets do not ship. - name: Build the baseline library (${{ inputs.baseline_sha }}) run: | git worktree add ../baseline "${{ inputs.baseline_sha }}" cd ../baseline git submodule update --init --recursive (cd 3rdparty/glslang && python update_glslang_sources.py) cmake -S . -B build-sym-base -G Ninja \ -DCMAKE_C_COMPILER=clang-20 -DCMAKE_CXX_COMPILER=clang++-20 \ -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_BUILD_TYPE=Release \ -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=OFF -DMOBILEGL_BUILD_BENCHMARK=OFF \ -DMOBILEGL_BUILD_INTEGRATION_TEST=OFF -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \ -DMOBILEGL_BUILD_DISAGGREGATED=OFF \ -DMOBILEGL_PIPE_PUSH=OFF -DMOBILEGL_PIPE_VERIFY=OFF \ -DMOBILEGL_ENABLE_LTO=OFF \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 cmake --build build-sym-base --parallel "$(nproc)" cp build-sym-base/libMobileGL.so "${GITHUB_WORKSPACE}/libMobileGL-baseline.so" - name: Build the head library run: | (cd 3rdparty/glslang && python update_glslang_sources.py) cmake -S . -B build-sym-head -G Ninja \ -DCMAKE_C_COMPILER=clang-20 -DCMAKE_CXX_COMPILER=clang++-20 \ -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_BUILD_TYPE=Release \ -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=OFF -DMOBILEGL_BUILD_BENCHMARK=OFF \ -DMOBILEGL_BUILD_INTEGRATION_TEST=OFF -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \ -DMOBILEGL_BUILD_DISAGGREGATED=OFF \ -DMOBILEGL_PIPE_PUSH=OFF -DMOBILEGL_PIPE_VERIFY=OFF \ -DMOBILEGL_ENABLE_LTO=OFF \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 cmake --build build-sym-head --parallel "$(nproc)" # The monolith must not have grown a remote half. ARCHITECTURE.md:506: MG_Remote lives behind # MOBILEGL_BUILD_DISAGGREGATED and nothing of it may reach a shipped pull build. - name: No MG_Remote in the pull build run: | if nm --defined-only build-sym-head/libMobileGL.so | grep -q MG_Remote; then echo "::error::the pull build defines MG_Remote symbols; the disaggregated half leaked into the monolith" nm --defined-only build-sym-head/libMobileGL.so | grep MG_Remote | head -20 exit 1 fi echo "no MG_Remote symbols in the pull build" - name: Symbol report (G1) run: | python3 scripts/symbol_report.py \ --before libMobileGL-baseline.so \ --after build-sym-head/libMobileGL.so \ --threshold 0 \ --fail-on-symbol-set-change \ --fail-on-added-bytes 0 \ --markdown symbol-report.md \ --json symbol-report.json - name: Upload the symbol report if: always() uses: actions/upload-artifact@v7 with: name: monolith-symbol-report path: | symbol-report.md symbol-report.json if-no-files-found: error remove-artifact-clutter: name: remove artifact clutter runs-on: ubuntu-latest # (d) retrace-verify too: this job deletes the trace-fixture-* artifacts, and the verify # retraces download the same ones. needs: - retrace-summary - retrace-verify if: always() permissions: actions: write steps: - name: Delete intermediate Linux retrace artifacts env: GH_TOKEN: ${{ github.token }} run: | # Both retrace lanes, not just the pull one: `retrace verify (backend, case)` downloads # the same trace-fixture- artifact, and a failed verify retrace is exactly when # someone needs that fixture to reproduce locally. The two prefixes are stripped in # order, longest first, because "retrace (" is not a prefix of "retrace verify (". declare -A failed_cases=() while IFS= read -r job_name; do case_name="${job_name#retrace verify (*, }" case_name="${case_name#retrace (*, }" case_name="${case_name%)}" failed_cases["${case_name}"]=1 done < <( gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100" \ --jq '.jobs[] | select((.name | startswith("retrace (")) or (.name | startswith("retrace verify ("))) | select(.conclusion == "failure" or .conclusion == "cancelled" or .conclusion == "timed_out" or .conclusion == "action_required") | .name' ) if ((${#failed_cases[@]})); then echo "Retaining fixtures for failed retrace case(s):" printf ' %s\n' "${!failed_cases[@]}" else echo "All retrace jobs succeeded; no fixtures need to be retained." fi deleted=0 retained=0 while IFS=$'\t' read -r artifact_id artifact_name; do if [[ "${artifact_name}" == trace-fixture-* ]]; then case_name="${artifact_name#trace-fixture-}" if [[ -v "failed_cases[${case_name}]" ]]; then echo "Retaining ${artifact_name} (${artifact_id}) for failed retrace." ((retained += 1)) continue fi fi echo "Deleting ${artifact_name} (${artifact_id})" gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}" ((deleted += 1)) done < <( gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \ --jq '.artifacts[] | select(.name | startswith("trace-fixture-") or startswith("retrace-result-")) | [.id, .name] | @tsv' ) echo "Deleted ${deleted} intermediate Linux artifact(s); retained ${retained} failed-retrace fixture(s)." pipe-gates: name: MGPipe generators and hygiene gates runs-on: ubuntu-latest # Deliberately independent of build-linux: these are source-level gates, they take # seconds, and a broken build must not hide a drifted interface. steps: - name: Checkout repo uses: actions/checkout@v6 # The seven generators all read MG_Pipe/*.def, so regenerating and diffing is what # keeps the two interface tables, the wire records, the verify comparators, the # PipeInputs field ids, the read-inventory coverage and the render-state member list # from drifting apart from the catalogue. The generated files are committed # deliberately: the build must not depend on python. - name: Regenerate the MGPipe interface (G1-G7) run: | python3 scripts/gen_pipe.py git diff --exit-code -- MobileGL/MG_Pipe/generated # The generators' own negative controls: canned inputs that MUST trip each structural check # (a field list that does not cover its struct's members, a verb set that is not the function # table's). Regenerating and diffing above cannot see a check that silently stopped # checking - a broken gate and a clean tree produce the same green. - name: The MGPipe generators' checks can still fail run: python3 scripts/gen_pipe.py --self-test # The same question for the symbol tool the P1 gate is written in terms of. - name: The symbol report's buckets and gates can still fail run: python3 scripts/symbol_report.py --self-test # Per-draw fprintf/printf instrumentation has repeatedly been committed by accident, # once inside a mutex critical section. Nothing under these two trees prints to a # stdio stream today - MGLOG_D compiles out in INFO builds and is the only channel # they are allowed to use - so this gate starts with no exceptions, and any addition # to it needs a reason in the pull request rather than a quiet whitelist entry. The # alternation names every stdio spelling, not just the two that were committed: # fprintf to either stream, printf, puts, and the iostream pair. - name: No stdio instrumentation in MG_Backend or MG_State run: | if grep -rnE 'fprintf[[:space:]]*\((stderr|stdout)|(^|[^[:alnum:]_>.])printf[[:space:]]*\(|(^|[^[:alnum:]_>.:])puts[[:space:]]*\(|std::(cout|cerr)' \ MobileGL/MG_Backend MobileGL/MG_State; then echo "::error::stdio instrumentation found; use MGLOG_D (compiled out in INFO builds)" exit 1 fi echo "no fprintf(stderr/stdout / printf( / puts( / std::cout|cerr under MobileGL/MG_Backend or MobileGL/MG_State" # A GATE as of P2, which is when MG_Pipe/DirtySurface.def exists to diff the scan against # (ROADMAP.md:18 puts the first mapping round in P2). --check fails BOTH directions: a # mutator the scanner finds with no row in the def, and a row naming a mutator the scan no # longer finds - so a deleted mutator cannot leave a stale row behind claiming coverage. # # --self-test is the half that keeps --check honest, and it is not optional. A completeness # check that silently stopped checking produces exactly the same green as a complete # mapping; the self-test feeds it two canned negative controls (a mutator withheld from the # def, a row naming a function that does not exist) and fails if either fails to trip. Same # shape as gen_pipe.py --self-test and check_include_closure.py above. # # What this gate does NOT cover is written into DirtySurface.def's header rather than left # implicit: the scanner attributes a mutation inside a lambda to the enclosing function, # reads a mutation published through a helper as deferred, and scans only MG_Impl/GLImpl - # so the four MGP_NOTE_MUTATION sites in MG_State are outside it entirely. This is a # completeness gate over what the scanner can see; the semantic proof is the verify lane. - name: MGPipe dirty-surface mapping is complete (G9) run: | python3 scripts/gen_pipe_dirty_surface.py --check python3 scripts/gen_pipe_dirty_surface.py --self-test # Warning only for now: the disaggregation documents are still being written, and a # lint that fails a rewrite in progress teaches people to ignore it. It becomes # --strict when the documents settle. - name: Documentation citation lint run: | shopt -s nullglob documents=(docs/Disaggregated/*.md) if [ ${#documents[@]} -eq 0 ]; then echo "no disaggregation documents to check" exit 0 fi python3 scripts/check_doc_citations.py "${documents[@]}" || true