name: Test run-name: ${{ (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main')) && format('Release validation ({0})', github.ref) || '' }} on: push: branches: - dev - Feat/Backend-Direct-GLES - Feat/Backend-Direct-Vulkan # TEMPORARY, remove before merging the MGPipe work into dev: the disaggregation # branch runs the full lane on every push so a phase's landing is not gated on # someone remembering to dispatch the workflow by hand. - feat/disaggregated - codex/fix-github-ci - cleanup workflow_dispatch: env: # Software-rendered CI submissions may exceed the interactive GPU hang budget. MOBILEGL_GPU_HANG_BUDGET_MS: "0" # ONE LIVE RUN PER BRANCH. The disaggregation branch pushes (often docs-only) commits minutes apart # and each push fans out to about 30 replay jobs here and the emulator shards in apk.yml; with no group, runs # sat queued for 10+ hours behind their own successors. A manual dispatch gets a group of its own, # so a push never cancels a manually started run; dev queues rather # than cancels. No paths-ignore here, unlike apk.yml: this workflow hosts the doc citation lint. # The "hosted" key leaves behind runs queued for the retired self-hosted runner, which cannot drain. concurrency: group: ${{ github.workflow }}-hosted-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.ref }} cancel-in-progress: ${{ github.event_name == 'push' && github.ref != 'refs/heads/dev' }} defaults: run: shell: bash jobs: # Temporarily keep all CI on GitHub-hosted runners while WSL D3D12 readback hangs are investigated. build-linux: runs-on: ubuntu-26.04 permissions: actions: write contents: read outputs: link-ratchet: ${{ steps.ratchet.outcome }} image: ${{ steps.toolchain.outputs.image }} compiler: ${{ steps.toolchain.outputs.compiler }} build-type: ${{ steps.configure.outputs.build-type }} env: BUILD_DIR: build-linux MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }} CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "true" CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 4G CCACHE_NOHASHDIR: "true" steps: - name: Checkout repo uses: actions/checkout@v6 # Only the submodules the build reads, shallow and in parallel; the script fails when one # of them did not arrive (scripts/ci/checkout_submodules.sh). - name: Check out build submodules timeout-minutes: 15 run: bash scripts/ci/checkout_submodules.sh library - name: Set swap space uses: ./.github/actions/set-swap-space with: swap-size-gb: 32 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Restore ccache uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 restore-keys: | ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache- - name: Prepare Vulkan SDK id: netsetup77 continue-on-error: true timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Prepare Vulkan SDK (retry) timeout-minutes: 15 if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup77.outcome == 'failure' }} uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Update glslang external sources working-directory: 3rdparty/glslang run: python3 update_glslang_sources.py - name: Install build dependencies timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build # The image and compiler are job outputs: integration-split prints them beside build-guard's. - name: Show installed toolchain id: toolchain run: | ccache --version "${MOBILEGL_CI_CC}" --version "${MOBILEGL_CI_CXX}" --version "${MOBILEGL_CI_LLD}" --version dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true echo "image=${ImageOS:-} ${ImageVersion:-}" >> "${GITHUB_OUTPUT}" echo "compiler=$("${MOBILEGL_CI_CXX}" --version | head -n 1)" >> "${GITHUB_OUTPUT}" # The configure line lives in scripts/ci/configure_build.sh, shared with build-linux-split # and build-guard. The build type is a job output: the split consumers reuse this build only # when it is Release, the only type build-guard can find identical to the split line. - name: Configure CMake id: configure env: CI_BUILD_TYPE: ${{ secrets.ACTIONS_STEP_DEBUG == 'true' && 'Debug' || 'Release' }} run: | echo "build-type=${CI_BUILD_TYPE}" >> "${GITHUB_OUTPUT}" bash scripts/ci/configure_build.sh primary "${BUILD_DIR}" - name: Build run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)" - name: Primary runtime mode proof run: | python3 scripts/ci/runtime_mode_proof.py self-test mode=monolith if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then mode=disaggregated; fi python3 scripts/ci/runtime_mode_proof.py build --build-dir "${BUILD_DIR}" \ --mode "${mode}" --output "${BUILD_DIR}/runtime-mode.json" # build-linux-split's server link-closure ratchet, on this build, which the split consumers # take when build-guard proves the two configurations identical. This job does not wait for # that verdict, so it runs on every disaggregation build. continue-on-error keeps a ratchet # red from skipping this job's consumers; integration-split reds on it when it takes this # build. The artifact has its own name, so it never replaces build-linux-split's. - name: Server link-closure ratchet id: ratchet if: env.MOBILEGL_CI_DISAGGREGATED == 'ON' continue-on-error: true run: | mkdir -p ci-artifacts python3 scripts/link_ratchet.py --self-test python3 scripts/link_ratchet.py \ --build-dir "${BUILD_DIR}" \ --baseline scripts/data/link_ratchet_baseline.txt \ --assert-monotone \ --bucket \ --json ci-artifacts/link-ratchet-primary.json - name: Upload link ratchet if: ${{ always() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }} uses: actions/upload-artifact@v7 with: name: link-ratchet-primary path: ci-artifacts/link-ratchet-primary.json if-no-files-found: warn - name: Show ccache stats if: ${{ always() }} run: ccache --show-stats # Rewrite one rolling entry per job and branch. The upload stays cumulative - it # carries every object restored at the top of this run plus the few TUs that # changed - but Actions cache keys are immutable, so this branch's superseded # blob is released first; --ref keeps the delete off other branches' entries. # Running after the build means a failed build leaves the entry untouched. - name: Release superseded ccache entry env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true - name: Save ccache continue-on-error: true uses: actions/cache/save@v5 with: path: .ccache key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 - name: Prepare packaged test metadata run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}" # zstd on every core: the tarball is most of a gigabyte of test executables, and every # consumer job waits for it. `tar --zstd -xf` reads the result unchanged. - name: Package Linux runtime run: | mkdir -p ci-artifacts mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort) tar \ --exclude='*/CMakeFiles' \ --exclude='*.o' \ --exclude='*.a' \ --exclude='*.ninja*' \ --exclude='build.ninja' \ --exclude='cmake_install.cmake' \ -I 'zstd -T0' -cf ci-artifacts/mobilegl-linux-runtime.tar.zst \ "${BUILD_DIR}/runtime-mode.json" \ "${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" \ "${BUILD_DIR}/MobileGL/MG_Test" \ "${BUILD_DIR}/MobileGL/MG_Benchmark" \ "${BUILD_DIR}/MobileGL/MG_IntegrationTest" \ "${SHARED_LIBS[@]}" # compression-level 0: the tarball is zstd already, and deflating it again in the upload # only costs time. The artifact stays a zip holding this one file, as the consumers expect. - name: Upload Linux runtime uses: actions/upload-artifact@v7 with: name: mobilegl-linux-runtime path: ci-artifacts/mobilegl-linux-runtime.tar.zst if-no-files-found: error compression-level: 0 # The retrace legs load only the library, and the server image on the spawn arm, so they # take this instead of the whole test runtime. - name: Package retrace library run: | files=("${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/libMobileGL.so") if [ -f "${BUILD_DIR}/libMobileGLServer.so" ]; then files+=("${BUILD_DIR}/libMobileGLServer.so"); fi tar -czf ci-artifacts/mobilegl-linux-lib.tgz "${files[@]}" - name: Upload retrace library uses: actions/upload-artifact@v7 with: name: mobilegl-linux-lib path: ci-artifacts/mobilegl-linux-lib.tgz if-no-files-found: error # The shipped shape: a library with no MG_Remote, which FCL embeds. On the disaggregation # branches the primary artifact carries the transport, so this is the only build of that shape # there; retrace-shipped replays it. build-linux-monolith-control: name: build push monolith control (no MG_Remote) runs-on: ubuntu-26.04 # actions: write is for this job's own rolling ccache entry, as in build-linux. permissions: actions: write contents: read env: BUILD_DIR: build-push-control CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_COMPRESS: "true" CCACHE_MAXSIZE: 4G CCACHE_NOHASHDIR: "true" steps: - name: Checkout repo uses: actions/checkout@v6 - name: Check out build submodules timeout-minutes: 15 run: bash scripts/ci/checkout_submodules.sh library - name: Set swap space uses: ./.github/actions/set-swap-space with: swap-size-gb: 32 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 # This job's own entry: its objects are compiled without MG_Remote, which no other build # produces. The second restore key is build-linux's entry, which seeds the third-party # objects on a branch's first run. - name: Restore ccache uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 restore-keys: | ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache- ${{ runner.os }}-ubuntu26-cpu-test-build-linux-ccache- - name: Prepare Vulkan SDK id: netsetup230 continue-on-error: true timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Prepare Vulkan SDK (retry) timeout-minutes: 15 if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup230.outcome == 'failure' }} uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Update glslang external sources working-directory: 3rdparty/glslang run: python3 update_glslang_sources.py - name: Install build dependencies timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build # The push monolith without MG_Remote, the shape FCL embeds. Its runtime_mode_proof is the # zero-MG_Remote-symbol check: a push library that links any MG_Remote symbol is red here. # Its library is uploaded for retrace-shipped, a positive replay of that shape. # ENABLE_PCH=OFF: ccache cannot cache a compile that uses glslang's precompiled header # (see scripts/ci/configure_build.sh). - name: Configure push control without MG_Remote run: | cmake -S . -B "${BUILD_DIR}" -G Ninja \ -DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DENABLE_PCH=OFF \ -DCMAKE_BUILD_TYPE=Release -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=OFF -DMOBILEGL_BUILD_BENCHMARK=OFF \ -DMOBILEGL_BUILD_INTEGRATION_TEST=OFF -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \ -DMOBILEGL_BUILD_DISAGGREGATED=OFF -DMOBILEGL_BUILD_DISAGGREGATED_INPROC=OFF \ -DMOBILEGL_PIPE_VERIFY=OFF \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5 - name: Build push control and prove it has zero MG_Remote symbols run: | cmake --build "${BUILD_DIR}" --target MobileGL --parallel "$(nproc)" python3 scripts/ci/runtime_mode_proof.py build --build-dir "${BUILD_DIR}" \ --mode monolith --output "${BUILD_DIR}/runtime-mode.json" - name: Show ccache stats if: ${{ always() }} run: ccache --show-stats # One rolling entry per job and branch, as in build-linux: this branch's superseded blob is # released before the save, and a failed build leaves the entry untouched. - name: Release superseded ccache entry env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true - name: Save ccache continue-on-error: true uses: actions/cache/save@v5 with: path: .ccache key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 - name: Prepare packaged test metadata run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}" - name: Package push control run: | tar -czf mobilegl-linux-monolith-control.tgz \ "${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/libMobileGL.so" "${BUILD_DIR}/runtime-mode.json" - name: Upload push control uses: actions/upload-artifact@v7 with: name: mobilegl-linux-monolith-control path: mobilegl-linux-monolith-control.tgz if-no-files-found: error test: runs-on: ubuntu-26.04 needs: [build-linux] # Every ctest entry is its own short-lived process, and none compiles enough to repay # Initialize()'s glslang table prewarm. The retrace legs keep it: they replay real apps. env: MOBILEGL_DISABLE_SHADER_PREWARM: "1" MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Install runtime dependencies timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libgles2 libgl1-mesa-dri mesa-vulkan-drivers ${{ env.MOBILEGL_CI_DISAGGREGATED == 'ON' && 'libegl-mesa0' || '' }} - name: Download Linux runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime path: . - name: Unpack Linux runtime run: tar --zstd -xf mobilegl-linux-runtime.tar.zst - name: Prepare downloaded test metadata run: | for root in build-linux build-verify build-split build-verify-split build-retrace; do if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi done # On the three branches this is also the split runtime's unit run (integration-split skips # its copy when build-linux-split reuses this build), so it keeps that run's hardening: # REQUIRE_GPU makes the server loop's EGL cases fail rather than skip without a driver. - name: Test working-directory: build-linux env: MOBILEGL_ITEST_REQUIRE_GPU: ${{ env.MOBILEGL_CI_DISAGGREGATED == 'ON' && '1' || '' }} run: | ulimit -c unlimited if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then ctest -V -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit.xml" else ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit.xml" fi - name: Disk and cores after the unit run if: ${{ always() }} run: | df -h / /tmp echo "cores: $(ls /tmp/core.* 2> /dev/null | wc -l), $(du -ch /tmp/core.* 2> /dev/null | tail -n 1)" # P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py). - name: Upload JUnit for the skip census if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: junit-test path: ${{ runner.temp }}/*.xml if-no-files-found: warn - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: unit-core-dumps path: /tmp/core.* if-no-files-found: ignore integration: name: integration (${{ matrix.transport }}) runs-on: ubuntu-26.04 needs: [build-linux] strategy: fail-fast: false matrix: include: ${{ fromJSON((github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && '[{"lane":"monolith-control","transport":"monolith"},{"lane":"inproc-acceptance","transport":"inproc"}]' || '[{"lane":"monolith-control","transport":"monolith"}]') }} env: # Short-lived test processes skip the glslang prewarm, as in the test job. MOBILEGL_DISABLE_SHADER_PREWARM: "1" MOBILEGL_TRANSPORT: ${{ matrix.transport }} MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" MOBILEGL_IPC_RUN_AHEAD: "1" MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner with: gpu: "true" - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Install runtime dependencies timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' # Same set as the benchmark job, for the same reason: the scenarios bring # up real headless EGL (llvmpipe) and Vulkan (lavapipe) contexts, and # libegl-mesa0 - the EGL vendor library behind glvnd's libegl1 dispatch - # only arrives as a Recommends. run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers - name: Download Linux runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime path: . - name: Unpack Linux runtime run: tar --zstd -xf mobilegl-linux-runtime.tar.zst - name: Prepare downloaded test metadata run: | for root in build-linux build-verify build-split build-verify-split build-retrace; do if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi done - name: Integration scenarios working-directory: build-linux # REQUIRE_GPU makes a driverless runner FAIL instead of skipping every # scenario - an all-skip run is otherwise indistinguishable from a pass, # which is how a five-month-old draw-dropping bug survived unseen until # this lane existed. # # Configure CMake with the selected runner's ICD; each test's ENVIRONMENT # property takes precedence over ambient job variables. # # Cores are armed so that any crash - the harness pre-flight child's # included - leaves /tmp/core.*.zst, which the failure-only step below ships # as an artifact. Analyzing a downloaded core (zstd -d) against the runtime # artifact's binary in an ubuntu-26.04 userspace reproduces the exact # crash stack without burning a CI round on an in-workflow debugger. env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_ITRP_FIX: "ForceOn" run: | ulimit -c unlimited # An entry whose own ENVIRONMENT sets MOBILEGL_TRANSPORT overrides the one variable the two # legs differ in, so it runs identically in both; the inproc leg leaves those entries (and # the fixtures only they need) to the monolith-control leg, which every branch runs, and # that leg proves it ran every one of them. When transport_pinned.py cannot prove the # trimmed selection lost exactly them, this leg runs every entry and writes its JUnits # under -all stems, which the skip census holds to the full name lists. EXCLUDE=() MAIN_JUNIT=integration STAGED_JUNIT=integration-staged-copy if [ "${MOBILEGL_CI_DISAGGREGATED}" = ON ]; then ctest -L integration-gpu --show-only=json-v1 > "${RUNNER_TEMP}/all.json" fi if [ "${MOBILEGL_TRANSPORT}" = inproc ]; then if python3 ../scripts/ci/transport_pinned.py select "${RUNNER_TEMP}/all.json" --out "${RUNNER_TEMP}/transport-pinned.txt" \ && ctest -L integration-gpu --exclude-from-file "${RUNNER_TEMP}/transport-pinned.txt" --show-only=json-v1 \ > "${RUNNER_TEMP}/trimmed.json" \ && python3 ../scripts/ci/transport_pinned.py assert-exclusion "${RUNNER_TEMP}/all.json" \ "${RUNNER_TEMP}/trimmed.json" --out "${RUNNER_TEMP}/transport-pinned.json"; then EXCLUDE=(--exclude-from-file "${RUNNER_TEMP}/transport-pinned.txt") else echo "::warning::the transport-pinned exclusion did not prove itself, so this leg runs every integration-gpu entry (integration-all.xml, integration-staged-copy-all.xml)" MAIN_JUNIT=integration-all STAGED_JUNIT=integration-staged-copy-all fi fi # Second, filtered pass: with the range-invalidating map flush disabled, # the buffer scenarios run on the upload ring's staged-copy tier - which # the default pass never reaches (the map tier absorbs every flush on # the CI drivers), so without this the Mali fallback tier would have zero CI # coverage. The flag is NOT baked into the ctest ENVIRONMENT properties, # so an inline env reaches the test processes (unlike the ICD pin above). if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then ctest -V -L integration-gpu "${EXCLUDE[@]}" --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/${MAIN_JUNIT}.xml" MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 ctest -V -L integration-gpu "${EXCLUDE[@]}" -j "$(nproc)" \ -R 'Buffer|Readback|Atomic|Ssbo|Arena' --no-tests=error --output-junit "${RUNNER_TEMP}/${STAGED_JUNIT}.xml" else ctest --output-on-failure -L integration-gpu "${EXCLUDE[@]}" --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/${MAIN_JUNIT}.xml" MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 ctest --output-on-failure -L integration-gpu "${EXCLUDE[@]}" -j "$(nproc)" \ -R 'Buffer|Readback|Atomic|Ssbo|Arena' --no-tests=error --output-junit "${RUNNER_TEMP}/${STAGED_JUNIT}.xml" fi if [ "${MOBILEGL_TRANSPORT}" = monolith ] && [ "${MOBILEGL_CI_DISAGGREGATED}" = ON ]; then python3 ../scripts/ci/transport_pinned.py require-ran "${RUNNER_TEMP}/all.json" "${RUNNER_TEMP}/${MAIN_JUNIT}.xml" fi # Both backend roles must execute inproc in the primary artifact. - name: Inproc execution proof (both backends) if: matrix.transport == 'inproc' working-directory: build-linux run: | ctest -L integration-p5f-rsp --show-only=json-v1 > "${RUNNER_TEMP}/primary-inproc-lane.json" ctest -L integration-p5f-rsp --no-tests=error --output-on-failure \ --output-junit "${RUNNER_TEMP}/primary-inproc.xml" python3 ../scripts/ci/runtime_mode_proof.py logs \ --discovery "${RUNNER_TEMP}/primary-inproc-lane.json" \ --junit "${RUNNER_TEMP}/primary-inproc.xml" --output "${RUNNER_TEMP}/primary-inproc-proof.json" # The run log needs a signed-in reader; annotations do not. Name every failed ctest entry, # with the lines of its output that say why, where anyone reading the run can see them. - name: Annotate failed entries if: failure() run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}" - name: Upload integration lane evidence if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: integration-evidence-${{ matrix.lane }}-${{ matrix.transport }} path: | ${{ runner.temp }}/integration*.xml ${{ runner.temp }}/primary-inproc* ${{ runner.temp }}/transport-pinned* ${{ runner.temp }}/all.json ${{ runner.temp }}/trimmed.json build-linux/runtime-mode.json build-linux/MobileGL/MG_IntegrationTest/p5f-rsp-*.log if-no-files-found: warn # P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py). - name: Upload JUnit for the skip census if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: junit-integration-${{ matrix.lane }}-${{ matrix.transport }} path: ${{ runner.temp }}/*.xml if-no-files-found: warn - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: integration-core-dumps-${{ matrix.lane }}-${{ matrix.transport }} path: /tmp/core.* if-no-files-found: ignore # THE THIRD CI MODE (ARCHITECTURE.md 13.2-(2)): the same library, built with the PipeInputs # comparator compiled in, running the integration suite and a trace subset with two state models # in one address space. It is a second build rather than a flag on the first because # MOBILEGL_PIPE_VERIFY is a compile-time option - the snapshot, the entry compare and the # compare-at-read hook do not exist in the shipped library, and are never meant to. build-linux-verify: runs-on: ubuntu-26.04 timeout-minutes: 120 permissions: actions: write contents: read env: BUILD_DIR: build-verify CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "true" CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 4G CCACHE_NOHASHDIR: "true" steps: - name: Checkout repo uses: actions/checkout@v6 - name: Check out build submodules timeout-minutes: 15 run: bash scripts/ci/checkout_submodules.sh library - name: Set swap space uses: ./.github/actions/set-swap-space with: swap-size-gb: 32 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Restore ccache uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 restore-keys: | ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache- - name: Prepare Vulkan SDK id: netsetup530 continue-on-error: true timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Prepare Vulkan SDK (retry) timeout-minutes: 15 if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup530.outcome == 'failure' }} uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Update glslang external sources working-directory: 3rdparty/glslang run: python3 update_glslang_sources.py - name: Install build dependencies timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build - name: Show installed toolchain run: | ccache --version "${MOBILEGL_CI_CC}" --version "${MOBILEGL_CI_CXX}" --version "${MOBILEGL_CI_LLD}" --version dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true - name: Configure CMake # Release/INFO like the shipped build on purpose. The poison arms in this configuration # through MOBILEGL_PIPE_VERIFY (PipeInputs.h derives MOBILEGL_PIPE_POISON from it), so this # job needs neither a Debug log level nor MOBILEGL_BUILD_DISAGGREGATED - and a Debug build # would compare a different library from the one the other lanes measure. (build-linux # switches to Debug under ACTIONS_STEP_DEBUG; this job deliberately does not - a Debug # build flips CXX_VISIBILITY_PRESET and arms MOBILEGL_PIPE_POISON through a second, unrelated # arm of its #if, so the debug switch would change what the lane is measuring.) # Both disaggregation options default ON, so they are pinned OFF here: a disaggregated # library expects a server-role log that this monolith lane never writes. # integration-verify-split is the split arm. # ENABLE_PCH=OFF: ccache cannot cache a compile that uses glslang's precompiled header # (see scripts/ci/configure_build.sh). run: | cmake -S . -B "${BUILD_DIR}" -G Ninja \ -DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \ -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DENABLE_PCH=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=ON \ -DMOBILEGL_BUILD_BENCHMARK=OFF \ -DMOBILEGL_BUILD_INTEGRATION_TEST=ON \ -DMOBILEGL_ITEST_VK_ICD="${VK_ICD_FILENAMES}" \ -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \ -DMOBILEGL_PIPE_VERIFY=ON \ -DMOBILEGL_BUILD_DISAGGREGATED=OFF \ -DMOBILEGL_BUILD_DISAGGREGATED_INPROC=OFF \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 - name: Build run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)" # The lane is worthless if the option silently did not take, and that is a one-character # mistake away at all times (a typo'd -D is not an error in CMake). Two checks, both cheap: # the comparator's entry point must be in the library, and the fill entry point with it. # # `nm` and NOT `nm -D`. The library is built CXX_VISIBILITY_PRESET hidden in every non-Debug # configuration (CMakeLists.txt:600-604) and the MGPipe entry points are plain namespace # functions with no export attribute, so not one of them appears in the DYNAMIC table: on a # perfectly healthy verify build `nm -D --defined-only ... | grep -c MGPipe` answers 0 out of # ~11900 exported symbols, and a gate spelled that way is red forever for a reason that has # nothing to do with what it claims to test. The static symbol table has them as local `t` # entries, this artifact is never stripped, and `No MG_Remote in the pull build` below already # uses this spelling. The symbol count guards the remaining hole: a stripped library would # make both greps fail for a third, silent reason. - name: Comparator symbols in verify library run: | test -f "${BUILD_DIR}/libMobileGL.so" defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l) if [ "${defined}" -lt 1000 ]; then echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the two checks below could not have failed honestly" exit 1 fi if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeVerifyInputs"; then echo "::error::libMobileGL.so defines no MGPipeVerifyInputs: -DMOBILEGL_PIPE_VERIFY=ON did not take, and every lane that consumes this artifact would run the comparator-free library and pass having compared nothing" exit 1 fi # The per-verb entry point, under EITHER of its two names. P2 renames # MGPipeFillForVerb to MGPipeValidateForVerb (the body becomes the tracker's walk and # the fill is one of its five steps), so this check has to accept both or it goes red on # the rename for a reason that has nothing to do with what it tests. What it tests is # unchanged: that the library HAS a per-verb entry point compiled in. if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -qE "MGPipeValidateForVerb|MGPipeFillForVerb"; then echo "::error::libMobileGL.so defines neither MGPipeValidateForVerb nor MGPipeFillForVerb: there is no per-verb entry point in this artifact, so nothing fills the block the comparator compares" exit 1 fi echo "libMobileGL.so defines MGPipeVerifyInputs and a per-verb entry point (${defined} defined symbols)" - name: Show ccache stats if: ${{ always() }} run: ccache --show-stats - name: Release superseded ccache entry env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true - name: Save ccache continue-on-error: true uses: actions/cache/save@v5 with: path: .ccache key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 - name: Prepare packaged test metadata run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}" - name: Package Linux verify runtime run: | mkdir -p ci-artifacts mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort) tar \ --exclude='*/CMakeFiles' \ --exclude='*.o' \ --exclude='*.a' \ --exclude='*.ninja*' \ --exclude='build.ninja' \ --exclude='cmake_install.cmake' \ -I 'zstd -T0' -cf ci-artifacts/mobilegl-linux-runtime-verify.tar.zst \ "${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" \ "${BUILD_DIR}/MobileGL/MG_Test" \ "${BUILD_DIR}/MobileGL/MG_IntegrationTest" \ "${SHARED_LIBS[@]}" # compression-level 0: the tarball is zstd already (see 'Upload Linux runtime'). - name: Upload Linux verify runtime uses: actions/upload-artifact@v7 with: name: mobilegl-linux-runtime-verify path: ci-artifacts/mobilegl-linux-runtime-verify.tar.zst if-no-files-found: error compression-level: 0 # The retrace legs load only the library, and the server image on the spawn arm, so they # take this instead of the whole test runtime. - name: Package retrace library run: | files=("${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/libMobileGL.so") if [ -f "${BUILD_DIR}/libMobileGLServer.so" ]; then files+=("${BUILD_DIR}/libMobileGLServer.so"); fi tar -czf ci-artifacts/mobilegl-linux-lib-verify.tgz "${files[@]}" - name: Upload retrace library uses: actions/upload-artifact@v7 with: name: mobilegl-linux-lib-verify path: ci-artifacts/mobilegl-linux-lib-verify.tgz if-no-files-found: error # The verify lane itself, plus the controls that keep it falsifiable: its own control entries, # checked by name, and the comparator's unit cases on its library. They are always on, not a # manual exercise: a gate that can only be shown to work by someone remembering to break it on # purpose is a gate that has already stopped working. integration-verify: runs-on: ubuntu-26.04 timeout-minutes: 180 needs: [build-linux-verify] # Short-lived test processes skip the glslang prewarm, as in the test job. env: MOBILEGL_DISABLE_SHADER_PREWARM: "1" steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner with: gpu: "true" - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Install runtime dependencies timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers - name: Download Linux verify runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime-verify path: . - name: Unpack Linux verify runtime run: | tar --zstd -xf mobilegl-linux-runtime-verify.tar.zst test -f build-verify/libMobileGL.so - name: Prepare downloaded test metadata run: | for root in build-linux build-verify build-split build-verify-split build-retrace; do if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi done - name: Integration scenarios (verify) working-directory: build-verify # --no-tests=error is half the gate: the verify entries only exist when the library was # configured with -DMOBILEGL_PIPE_VERIFY=ON, so a build that lost the option matches no # tests and reds here instead of reporting a green run of nothing. The other half is # PipeVerifyArmingScenario.Armed, which fails when the library never printed its arming # line - the failure mode a bare `MOBILEGL_PIPE_VERIFY=1` cannot detect by itself. # # SCOPE, stated so nobody reads more into a green than is there: this is every integration # ENTRY under the comparator, not every integration CONFIGURATION. The `integration` job # runs a second, filtered pass with MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 for the # upload ring's staged-copy tier; that pass is 186 entries here and, at the 5-10x the # comparator costs, is not affordable inside this job's budget. The tier is covered by # `integration`, unverified, and P2 can take it once the comparator's cost is known. env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_ITRP_FIX: "ForceOn" run: | ulimit -c unlimited if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then ctest -V -L integration-verify --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/verify.xml" else ctest --output-on-failure -L integration-verify --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/verify.xml" fi # The unit suite again, on the verify library: cases compiled only under MOBILEGL_PIPE_VERIFY # skip in the `test` job's build and assert only here, as do poison-gated cases on branches # where that build is a monolith. The packaging step above ships MG_Test whole, so the cost # is the run itself. --no-tests=error, so a packaging change that dropped the unit binaries # fails here instead of passing with nothing run. Among them is the comparator's per-field # control, PipeInputsTest.EveryCorruptedFieldIsNamedByTheComparator. - name: Unit tests on the verify runtime (G6, G10) working-directory: build-verify run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit-verify.xml" # Handle-ABA, CSO, subsystem-mask and texture-parameter controls, rerun on the verify library. # They carry integration-gpu, not integration-verify, so the verify lane misses them. # Each -R alternative is a test-name substring, the shortest that selects only its family: # `ResourceSubsystem` also reaches the LargeArenaAdoption lanes and the mask refusals, # `MapPersistentRoundtrip` is singular so it matches the case name as well as the lane prefix, # and `ObjectSubsystem` and `ResourceSubsystem` stay distinct families. TextureUploadShape # records the upload shape; it fails only when its two counters are unreadable or disagree. - name: Push-build controls (G8, G8b, G9, G10, G12) working-directory: build-verify env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_ITRP_FIX: "ForceOn" run: | ulimit -c unlimited ctest --output-on-failure -L integration-gpu \ -R 'HandleRecycle|CsoContentAddressing|ResourceSubsystem|MapPersistentRoundtrip|ObjectSubsystem|TextureParamsWithoutASamplerView|TextureUploadShape' \ --no-tests=error -j 4 # The lane's own control entries, by name: each must have run and passed, and a skip is a # failure here although ctest reports it green. VerifyArming asserts in-test that the # comparator printed its armed line with fatal=1, under the ambient entries' own environment # (MG_IntegrationTest/CMakeLists.txt builds both from one list), VerifyCorrupted that a # corrupted field is reported by name, and PoisonOmitted that an omitted stamp kills a # forked child with its own Fatal. - name: Verify lane control entries run: | python3 scripts/ci/junit_tally.py "${RUNNER_TEMP}/verify.xml" \ --require-entry-passed DirectGLES.VerifyArming.PipeVerifyArmingScenario.Armed \ --require-entry-passed DirectVulkan.VerifyArming.PipeVerifyArmingScenario.Armed \ --require-entry-passed DirectGLES.VerifyCorrupted.PipeVerifyArmingScenario.CorruptedFieldIsReported \ --require-entry-passed DirectVulkan.VerifyCorrupted.PipeVerifyArmingScenario.CorruptedFieldIsReported \ --require-entry-passed DirectGLES.PoisonOmitted.PoisonOmissionScenario.OmittedFieldAbortsOnThatVerb \ --require-entry-passed DirectVulkan.PoisonOmitted.PoisonOmissionScenario.OmittedFieldAbortsOnThatVerb - name: Upload verify lane logs if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: integration-verify-logs path: build-verify/MobileGL/MG_IntegrationTest/pipe-*.log* if-no-files-found: warn # P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py). - name: Upload JUnit for the skip census if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: junit-integration-verify path: ${{ runner.temp }}/*.xml if-no-files-found: warn - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: integration-verify-core-dumps path: /tmp/core.* if-no-files-found: ignore # The comparator on the split (inproc) arm, both backends, with its control entries checked by # name. # # ITS OWN BUILD, because neither existing one can carry it: build-linux-verify configures # without MOBILEGL_BUILD_DISAGGREGATED (so MG_Remote and the transport parser do not exist and # `MOBILEGL_TRANSPORT=inproc` would be accepted and ignored), and build-linux-split configures # without MOBILEGL_PIPE_VERIFY (so the comparator does not exist). The integration-verify-split # entries are registered only when BOTH options are on, so `--no-tests=error` below reds a build # that lost either one instead of reporting a green run of nothing. Built and run in one job: # nothing downstream consumes this library. # # INPROC ONLY, BY THE INSTRUMENT'S NATURE: the comparator's two arms (the applier-written # gPipeInputs and the frontend GLContext) are co-resident only when both roles share a process. # MG_IntegrationTest/CMakeLists.txt's verify-split block records the spawn measurement. integration-verify-split: runs-on: ubuntu-26.04 needs: [build-linux-verify-split] timeout-minutes: 180 # Short-lived test processes skip the glslang prewarm, as in the test job. env: MOBILEGL_DISABLE_SHADER_PREWARM: "1" steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner with: gpu: "true" - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Install runtime dependencies timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers - name: Download verify-split runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime-verify-split path: . - name: Unpack verify-split runtime run: tar --zstd -xf mobilegl-linux-runtime-verify-split.tar.zst # The lane. `^...$` because ctest -L is a regex (ID-131) and `integration-verify` would # otherwise also match this label and the monolith one would match both. - name: Prepare downloaded test metadata run: | for root in build-linux build-verify build-split build-verify-split build-retrace; do if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi done - name: Verify-split scenarios (inproc) working-directory: build-verify-split env: MOBILEGL_ITEST_REQUIRE_GPU: "1" run: | ulimit -c unlimited # --output-junit, because the arm proof below has to tell a gtest SKIP from a pass: an # entry that skipped reached no verb honestly, and the skip set moves with the driver. ctest --output-on-failure -L '^integration-verify-split$' --no-tests=error -j 4 \ --output-junit "${RUNNER_TEMP}/verify-split.xml" # THE ARM PROOF, PER ENTRY AND BY NAME. Every ambient entry has a private log path # (SplitLogPaths.cmake.in), so this - unlike integration-verify's grep over a shared log - # is a census: # * ConfigLoader's `MOBILEGL_TRANSPORT=inproc` line in EVERY client log: an entry that ran # monolith would have passed every assertion above and fails here by name; # * `MGPipe: verify armed` in both VerifySplitArming. logs; # * and then the census proper. This used to be a FLOOR - "at least 850 armed of at least # 1000 logs" - which leaves the remaining ~155 unnamed, so the lane could lose fifty # arms, gain fifty skips, and the floor would not move. It is now the two-sided rule # dualblock-expected-fatals.txt already rides: each client log must ARM, or belong to an # entry ctest reported SKIPPED, or be NAMED in Harness/verify-split-unarmed-expected.txt # (56 names in five classes, each class saying why it reaches no verb). A listed entry # that arms, and a listed name with no log at all, both fail - an exception set that # keeps rows nothing needs is how a proof rots green. - name: Verify-split per-entry arm proof working-directory: build-verify-split/MobileGL/MG_IntegrationTest run: | shopt -s nullglob logs=(verify-split-logs/*.client.log) if [ ${#logs[@]} -lt 1000 ]; then echo "::error::found ${#logs[@]} per-entry client logs under verify-split-logs (expected ~1064): the private log paths did not take, so nothing below is a per-entry proof" exit 1 fi missing=$(grep -L 'Config: MOBILEGL_TRANSPORT=inproc' "${logs[@]}" | wc -l) if [ "${missing}" -ne 0 ]; then grep -L 'Config: MOBILEGL_TRANSPORT=inproc' "${logs[@]}" | head -20 echo "::error::${missing} verify-split entries carry no resolved MOBILEGL_TRANSPORT=inproc line: they did not run on the split arm" exit 1 fi for backend in DirectGLES DirectVulkan; do if ! grep -q "MGPipe: verify armed" "pipe-verify-split-arming-${backend}.client.log"; then echo "::error::pipe-verify-split-arming-${backend}.client.log carries no arming line" exit 1 fi done python3 "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \ verify-split-arming verify-split-logs "${RUNNER_TEMP}/verify-split.xml" \ "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/verify-split-unarmed-expected.txt" # The lane's own control entries, by name, as in integration-verify: VerifySplitReadCorrupted # asserts in-test the two where=read reports in the server half, and VerifySplitPoisonOmitted # reads both role halves of the forked child's log for its own Fatal. - name: Verify-split control entries run: | args=() for backend in DirectGLES DirectVulkan; do args+=(--require-entry-passed "${backend}.VerifySplitArming.PipeVerifyArmingScenario.Armed") args+=(--require-entry-passed "${backend}.VerifySplitCorrupted.PipeVerifyArmingScenario.CorruptedFieldIsReported") args+=(--require-entry-passed "${backend}.VerifySplitReadCorrupted.PipeVerifyArmingScenario.CorruptedFieldIsReportedOnTheServerRead") args+=(--require-entry-passed "${backend}.VerifySplitPoisonOmitted.PoisonOmissionScenario.OmittedFieldAbortsOnThatVerb") done python3 scripts/ci/junit_tally.py "${RUNNER_TEMP}/verify-split.xml" "${args[@]}" # The run log needs a signed-in reader; annotations do not. Name every failed ctest entry, # with the lines of its output that say why, where anyone reading the run can see them. - name: Annotate failed entries if: failure() run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}" - name: Upload verify-split lane logs if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: integration-verify-split-logs path: | build-verify-split/MobileGL/MG_IntegrationTest/verify-split-logs/** build-verify-split/MobileGL/MG_IntegrationTest/pipe-*split*.log* if-no-files-found: warn # P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py). - name: Upload JUnit for the skip census if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: junit-integration-verify-split path: ${{ runner.temp }}/*.xml if-no-files-found: warn - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: integration-verify-split-core-dumps path: /tmp/core.* if-no-files-found: ignore # Whether build-linux-split may skip its compile. On the disaggregation branches its configure # line and build-linux's differ only in the benchmarks; this job configures both on one runner # (configure only) and compares the trees (scripts/ci/build_fingerprint.py). Its verdict is # identical, different or unknown, and its failure never fails the run: only 'identical' makes # build-linux-split skip its compile and the split consumers take build-linux's artifacts, # while 'different' and 'unknown' keep both builds. The builds run on the same runner image as # this job, which records the image and compiler it saw; integration-split prints both sides. # MOBILEGL_CI_DISAGGREGATED is build-linux's own expression, so a branch list that drifts # between the two jobs configures an OFF primary here and gives 'different'. build-guard: name: build guard (split reuse) runs-on: ubuntu-26.04 timeout-minutes: 150 if: ${{ github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main')) }} # Consumers read only its outputs, and any verdict but 'identical' (none at all included) # keeps both builds. continue-on-error: true outputs: verdict: ${{ steps.guard.outputs.verdict || 'unknown' }} image: ${{ steps.guard.outputs.image }} compiler: ${{ steps.guard.outputs.compiler }} env: MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }} # Every step may fail without failing the job: a setup step that failed leaves a configure # that fails, which the guard reports as 'unknown'. steps: - name: Checkout repo continue-on-error: true timeout-minutes: 15 uses: actions/checkout@v6 # The same submodule set as the two builds this job compares. The compare step runs only # when every one of them arrived: without 3rdparty/flatbuffers both lines would configure # the same library without MG_Remote, and compare as identical. - name: Check out build submodules id: submodules continue-on-error: true timeout-minutes: 15 run: bash scripts/ci/checkout_submodules.sh library - name: Configure runner environment continue-on-error: true timeout-minutes: 15 uses: ./.github/actions/setup-test-runner - name: Get CMake continue-on-error: true timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Prepare Vulkan SDK id: vulkan continue-on-error: true timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Prepare Vulkan SDK (retry) continue-on-error: true timeout-minutes: 15 if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.vulkan.outcome == 'failure' }} uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Update glslang external sources continue-on-error: true timeout-minutes: 15 working-directory: 3rdparty/glslang run: python3 update_glslang_sources.py || python3 update_glslang_sources.py - name: Install build dependencies continue-on-error: true timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build # The self-test first: a guard whose canned cases fail decides nothing. The two configures # share one FetchContent directory, so only the first fetches; each attempt is killed after # 10 minutes and retried once, which this step's timeout leaves room for. - name: Configure both lines and compare id: guard continue-on-error: true timeout-minutes: 45 env: CI_BUILD_TYPE: ${{ secrets.ACTIONS_STEP_DEBUG == 'true' && 'Debug' || 'Release' }} SUBMODULES_OUTCOME: ${{ steps.submodules.outcome }} run: | if [ "${SUBMODULES_OUTCOME}" != success ]; then echo "::warning::the submodule checkout did not succeed (outcome '${SUBMODULES_OUTCOME}'), so the guard decides nothing" echo "verdict=unknown" >> "${GITHUB_OUTPUT}" elif python3 scripts/ci/build_fingerprint.py --self-test > "${RUNNER_TEMP}/fingerprint-self-test.txt"; then python3 scripts/ci/build_fingerprint.py guard --primary-dir build-guard-primary \ --split-dir build-guard-split --deps-dir build-guard-deps \ --github-output verdict --report "${RUNNER_TEMP}/build-guard.json" else cat "${RUNNER_TEMP}/fingerprint-self-test.txt" echo "::warning::build_fingerprint.py's self-test failed, so the guard decides nothing" echo "verdict=unknown" >> "${GITHUB_OUTPUT}" fi - name: Upload guard report if: ${{ always() }} continue-on-error: true uses: actions/upload-artifact@v7 with: name: build-guard path: ${{ runner.temp }}/build-guard.json if-no-files-found: warn # The split build: the same sources with the transport and its in-process arm always on, which the # split integration and retrace lanes consume. When build-guard proves its configuration # identical to build-linux's (the disaggregation branches, benchmarks apart), this job builds # nothing and the consumers take build-linux's artifacts; on any other verdict, and where # build-guard does not run, it builds. # # The `nm` step is the point of a build of its own. In a build without the option, MG_Remote is # not compiled and MOBILEGL_TRANSPORT's parser does not exist, so the variable is accepted and # ignored and every downstream lane would run monolith under a name that says split. CMake does # not complain about a typo'd -D, so the build-level assertion is the guard; in reuse mode # build-linux's runtime_mode_proof (--mode disaggregated) is the same assertion and more. build-linux-split: runs-on: ubuntu-26.04 timeout-minutes: 120 needs: [build-guard] # A build-guard that was skipped, failed or undecided means this job builds. if: ${{ !cancelled() }} permissions: actions: write contents: read env: BUILD_DIR: build-split SPLIT_REUSE: ${{ needs.build-guard.outputs.verdict == 'identical' && 'true' || 'false' }} CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "true" CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 4G CCACHE_NOHASHDIR: "true" steps: - name: Reuse decision env: GUARD_VERDICT: ${{ needs.build-guard.outputs.verdict }} run: | if [ "${SPLIT_REUSE}" = true ]; then echo "build-guard found the two configurations identical: this job builds nothing and the split consumers take build-linux's artifacts" else echo "build-guard verdict '${GUARD_VERDICT:-none}': this job builds the split library" fi - name: Checkout repo if: env.SPLIT_REUSE != 'true' uses: actions/checkout@v6 # The submodule list is load-bearing here beyond the usual: MOBILEGL_BUILD_DISAGGREGATED # shadows itself back to OFF when 3rdparty/flatbuffers/include is missing (a normal # variable rather than a cache force, deliberately), so a checkout without it would # configure cleanly and build a monolith library. The script fails when the header is not # there, and the nm step below is the second check. - name: Check out build submodules if: env.SPLIT_REUSE != 'true' timeout-minutes: 15 run: bash scripts/ci/checkout_submodules.sh library - name: Configure runner environment if: env.SPLIT_REUSE != 'true' uses: ./.github/actions/setup-test-runner - name: Get CMake if: env.SPLIT_REUSE != 'true' timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Prepare Vulkan SDK id: netsetup1274 continue-on-error: true timeout-minutes: 15 if: ${{ env.SPLIT_REUSE != 'true' && (env.MOBILEGL_CI_NATIVE_GPU != '1') }} uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Prepare Vulkan SDK (retry) timeout-minutes: 15 if: ${{ env.SPLIT_REUSE != 'true' && ((env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup1274.outcome == 'failure') }} uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Update glslang external sources if: env.SPLIT_REUSE != 'true' working-directory: 3rdparty/glslang run: python3 update_glslang_sources.py - name: Install build dependencies timeout-minutes: 15 if: ${{ env.SPLIT_REUSE != 'true' && (env.MOBILEGL_CI_NATIVE_GPU != '1') }} run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build - name: Show installed toolchain if: env.SPLIT_REUSE != 'true' run: | ccache --version "${MOBILEGL_CI_CC}" --version "${MOBILEGL_CI_CXX}" --version dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true # Release/INFO like the shipped build; the line and the reasons are in # scripts/ci/configure_build.sh, shared with build-linux. - name: Configure CMake if: env.SPLIT_REUSE != 'true' run: bash scripts/ci/configure_build.sh split "${BUILD_DIR}" - name: Set swap space if: env.SPLIT_REUSE != 'true' uses: ./.github/actions/set-swap-space with: swap-size-gb: 32 - name: Restore ccache if: env.SPLIT_REUSE != 'true' uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 restore-keys: | ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache- - name: Build if: env.SPLIT_REUSE != 'true' run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)" # `nm` and NOT `nm -D`, for the reason build-linux-verify spells out: everything under # MG_Remote is a plain namespace symbol in a CXX_VISIBILITY_PRESET=hidden Release build and # none of it reaches the dynamic table. The symbol count guards the remaining hole - a # stripped library would make the grep fail for a third, silent reason. # - name: MG_Remote symbols in split library if: env.SPLIT_REUSE != 'true' run: | test -f "${BUILD_DIR}/libMobileGL.so" defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l) if [ "${defined}" -lt 1000 ]; then echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the checks below could not have failed honestly" exit 1 fi remote=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -c -i "MG_Remote" || true) if [ "${remote}" -lt 1 ]; then echo "::error::libMobileGL.so defines no MG_Remote symbol: -DMOBILEGL_BUILD_DISAGGREGATED=ON did not take (a typo'd -D is not a CMake error, and the option shadows itself OFF when 3rdparty/flatbuffers/include is missing). Every lane that consumes this artifact would run MONOLITH while claiming to run split, because the MOBILEGL_TRANSPORT parser does not exist in such a build and the variable is accepted and ignored." exit 1 fi # The transport parser itself, which is the symbol the runtime evidence depends on. if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeApply"; then echo "::error::libMobileGL.so defines no MGPipeApply* entry point, so MOBILEGL_PIPE_PUSH did not take either and there is no applier for the split path to decode into" exit 1 fi echo "libMobileGL.so defines ${remote} MG_Remote symbol(s) and the MGPipe applier (${defined} defined symbols)" # Server link-closure ratchet (ARCHITECTURE.md §13.2): scripts/link_ratchet.py recomputes # from the per-object partition the frontend symbols a server-only image still has to link # and compares the list with the stored baseline. Only new symbols fail; a vanished one is # printed with a request to re-baseline in the same commit. The baseline is a list, not a # count, so swapping one symbol for another fails. It runs where MG_Remote is built (the # script re-checks CMakeCache.txt) and before anything deletes the *.o files it reads: here, # or in build-linux when this job reuses that build. - name: Server link-closure ratchet if: env.SPLIT_REUSE != 'true' run: | mkdir -p ci-artifacts python3 scripts/link_ratchet.py --self-test python3 scripts/link_ratchet.py \ --build-dir "${BUILD_DIR}" \ --baseline scripts/data/link_ratchet_baseline.txt \ --assert-monotone \ --bucket \ --json ci-artifacts/link-ratchet.json # Uploaded even when the gate fired: a red ratchet is exactly the moment someone needs the # per-symbol definer/referrer attribution the JSON carries. - name: Upload link ratchet if: ${{ always() && env.SPLIT_REUSE != 'true' }} uses: actions/upload-artifact@v7 with: name: link-ratchet path: ci-artifacts/link-ratchet.json if-no-files-found: warn - name: Show ccache stats if: ${{ always() && env.SPLIT_REUSE != 'true' }} run: ccache --show-stats # Only after a build: with nothing built, deleting would drop this branch's entry for good. - name: Release superseded ccache entry if: env.SPLIT_REUSE != 'true' env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true - name: Save ccache if: env.SPLIT_REUSE != 'true' continue-on-error: true uses: actions/cache/save@v5 with: path: .ccache key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 - name: Prepare packaged test metadata if: env.SPLIT_REUSE != 'true' run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}" - name: Package Linux split runtime if: env.SPLIT_REUSE != 'true' run: | mkdir -p ci-artifacts mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort) tar \ --exclude='*/CMakeFiles' \ --exclude='*.o' \ --exclude='*.a' \ --exclude='*.ninja*' \ --exclude='build.ninja' \ --exclude='cmake_install.cmake' \ -I 'zstd -T0' -cf ci-artifacts/mobilegl-linux-runtime-split.tar.zst \ "${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" \ "${BUILD_DIR}/MobileGL/MG_Test" \ "${BUILD_DIR}/MobileGL/MG_IntegrationTest" \ "${SHARED_LIBS[@]}" # compression-level 0: the tarball is zstd already (see 'Upload Linux runtime'). - name: Upload Linux split runtime if: env.SPLIT_REUSE != 'true' uses: actions/upload-artifact@v7 with: name: mobilegl-linux-runtime-split path: ci-artifacts/mobilegl-linux-runtime-split.tar.zst if-no-files-found: error compression-level: 0 # The retrace legs load only the library, and the server image on the spawn arm, so they # take this instead of the whole test runtime. - name: Package retrace library if: env.SPLIT_REUSE != 'true' run: | files=("${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/libMobileGL.so") if [ -f "${BUILD_DIR}/libMobileGLServer.so" ]; then files+=("${BUILD_DIR}/libMobileGLServer.so"); fi tar -czf ci-artifacts/mobilegl-linux-lib-split.tgz "${files[@]}" - name: Upload retrace library if: env.SPLIT_REUSE != 'true' uses: actions/upload-artifact@v7 with: name: mobilegl-linux-lib-split path: ci-artifacts/mobilegl-linux-lib-split.tgz if-no-files-found: error # The split lane itself: the three-arm shape ARCHITECTURE.md §13.2 asks for verbatim, plus the # first real run of the five MG_Test/Wire suites. P5's two exit-gate negative controls are in-test # now (RemoteWaitBoundaryControl.AnUnwaitedPublishIsSeenAsNoWait and # PersistentCoherentMapScenario.ThePushIsTheOnlyRouteAWriteTakesToTheServer). integration-split: runs-on: ubuntu-26.04 timeout-minutes: 120 needs: [build-guard, build-linux-split, build-linux] if: ${{ !cancelled() && needs.build-linux-split.result == 'success' && (needs.build-guard.outputs.verdict != 'identical' || needs.build-linux.result == 'success') }} # The skip census reads whether 'Default-knob resolution check' proved the IPC defaults. outputs: defaults-on: ${{ steps.resolution.outputs.defaults-on }} # Short-lived test processes skip the glslang prewarm, as in the test job. env: MOBILEGL_DISABLE_SHADER_PREWARM: "1" SPLIT_REUSE: ${{ needs.build-guard.outputs.verdict == 'identical' && 'true' || 'false' }} SPLIT_BUILD_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }} SPLIT_RUNTIME: ${{ needs.build-guard.outputs.verdict == 'identical' && 'mobilegl-linux-runtime' || 'mobilegl-linux-runtime-split' }} MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }} steps: - name: Checkout repo uses: actions/checkout@v6 # Which build these lanes take: build-linux's when build-guard proved the two configurations # identical, build-linux-split's otherwise. build-guard compared them on its own runner, so # the image and compiler it saw are printed beside build-linux's. It reds only when it would # reuse a build-linux that is not Release (see scripts/ci/split_reuse_build_type.sh). - name: Split runtime env: GUARD_VERDICT: ${{ needs.build-guard.outputs.verdict }} GUARD_IMAGE: ${{ needs.build-guard.outputs.image }} GUARD_COMPILER: ${{ needs.build-guard.outputs.compiler }} PRIMARY_IMAGE: ${{ needs.build-linux.outputs.image }} PRIMARY_COMPILER: ${{ needs.build-linux.outputs.compiler }} PRIMARY_BUILD_TYPE: ${{ needs.build-linux.outputs.build-type }} run: | echo "split runtime: ${SPLIT_BUILD_DIR} (build-guard verdict '${GUARD_VERDICT:-none}')" bash scripts/ci/split_reuse_build_type.sh echo "build-guard ran on '${GUARD_IMAGE}' with '${GUARD_COMPILER}'" echo "build-linux ran on '${PRIMARY_IMAGE}' with '${PRIMARY_COMPILER}'" if [ "${SPLIT_REUSE}" = true ] && { [ "${GUARD_IMAGE}" != "${PRIMARY_IMAGE}" ] || [ "${GUARD_COMPILER}" != "${PRIMARY_COMPILER}" ]; }; then echo "::notice::build-guard and build-linux saw different runner images or compilers; the reused build is the one build-linux configured" fi - name: Configure runner environment uses: ./.github/actions/setup-test-runner with: gpu: "true" - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Install runtime dependencies timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers - name: Download Linux split runtime uses: actions/download-artifact@v8 with: name: ${{ env.SPLIT_RUNTIME }} path: . - name: Unpack Linux split runtime run: | tar --zstd -xf "${SPLIT_RUNTIME}.tar.zst" test -f "${SPLIT_BUILD_DIR}/libMobileGL.so" if [ "${SPLIT_REUSE}" = "true" ]; then python3 -c 'import json, sys; mode = json.load(open(sys.argv[1]))["mode"]; sys.exit(0 if mode == "disaggregated" else f"build-linux runtime mode is {mode}, not disaggregated")' \ "${SPLIT_BUILD_DIR}/runtime-mode.json" fi - name: Prepare downloaded test metadata run: | for root in build-linux build-verify build-split build-verify-split build-retrace; do if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi done - name: Spawn/split lane parity (9.2) run: python3 scripts/ci/spawn_lane_parity.py "${SPLIT_BUILD_DIR}" # P8-A (ID-P8-2): every monolith case, tails normalized, is on a gated split arm or is named # with a reason class in scripts/data/split_coverage_exemptions.txt. The same `ctest -N` # cost as the parity step above, and the question it cannot answer: parity compares the # split arms with EACH OTHER, so a case missing from all three passes it. - name: Split coverage against monolith (P8-A) run: | python3 scripts/ci/split_coverage.py --self-test python3 scripts/ci/split_coverage.py "${SPLIT_BUILD_DIR}" # P8-SE: PipeCatalogTest's unmigrated-emulation list equals the MGPipeUnmigratedEmulation # call sites in the tree, one site per name. A source lint: no build needed. - name: Unmigrated-emulation list against its call sites (P8-SE) run: | python3 scripts/ci/unmigrated_emulation_sites.py --self-test python3 scripts/ci/unmigrated_emulation_sites.py # The test job runs the same unit lane, with the same REQUIRE_GPU, on this same build when # build-linux-split was reused; the strict unit step in integration-split-strict always runs. - name: Unit tests on the split runtime if: env.SPLIT_REUSE != 'true' working-directory: ${{ env.SPLIT_BUILD_DIR }} env: MOBILEGL_ITEST_REQUIRE_GPU: "1" run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit-split.xml" # TcpLane.SupervisorProtocolControls and TcpLane.ControlFrameFuzz build the protocol's Python # bindings with flatc, and without one they exit 77 and ctest reports a skip. flatc is not in # the build graph (see gen_protocol.py), so this builds the same pinned flatc as flatc-check, # and the lane step requires both entries PASSED by name through junit_tally.py. --check keeps # this step from regenerating the committed header; checking that header is flatc-check's job. # The binary comes from the cache entry flatc-check also uses, keyed on the submodule's # commit; gen_protocol.py builds it when the entry is missing and checks its version against # the submodule either way. - name: Check out FlatBuffers (TCP lane) run: git submodule update --init 3rdparty/flatbuffers - name: Pinned flatc cache key id: flatc run: echo "commit=$(git rev-parse HEAD:3rdparty/flatbuffers)" >> "${GITHUB_OUTPUT}" - name: Cache pinned flatc uses: actions/cache@v5 with: path: ${{ runner.temp }}/flatc-build/flatc key: ${{ runner.os }}-ubuntu26-flatc-${{ steps.flatc.outputs.commit }} - name: Build pinned flatc (TCP lane) run: python3 scripts/gen_protocol.py --build-dir "${RUNNER_TEMP}/flatc-build" --check # --no-tests=error is half the gate, exactly as in integration-verify: the integration-split # entries exist only when the library was configured with -DMOBILEGL_BUILD_DISAGGREGATED=ON, # so a build that lost the option matches nothing and reds here instead of reporting a green # run of nothing. The other half is in-test: the split fixture fails an armed entry whose # process did not resolve the transport its registration claims. # # An entry whose owning package (c1 client, s1 session, v1 server) has not landed SKIPS with # the reason and never goes green - MG_IntegrationTest/CMakeLists.txt probes MG_Remote for # c0's signature stubs and disarms the lane while any remain. # # The three arms run as one ctest invocation: the TCP arm holds one resource lock, so on its # own it runs one entry at a time and leaves the other cores idle. Each arm keeps its own # selection, JUnit and checks. - name: Split lanes (inproc, spawn, TCP) working-directory: ${{ env.SPLIT_BUILD_DIR }} env: MOBILEGL_ITEST_REQUIRE_GPU: "1" run: | ulimit -c unlimited export MOBILEGL_FLATC_EXECUTABLE="${RUNNER_TEMP}/flatc-build/flatc" test -x "${MOBILEGL_FLATC_EXECUTABLE}" # Each arm is selected by its own label regex. scripts/ci/lane_selection.py turns the # three selections into one list of exact names and fails when an arm selects nothing, # which is what --no-tests=error says for an arm run on its own. junit_tally.py then # carves each arm's JUnit out of the run, and fails when a selected name is missing # from the run or reported twice. ctest --show-only=json-v1 -L integration-split > "${RUNNER_TEMP}/split-discovery.json" ctest --show-only=json-v1 -L integration-spawn > "${RUNNER_TEMP}/spawn-discovery.json" ctest --show-only=json-v1 -L '^integration-tcp$' > "${RUNNER_TEMP}/tcp-discovery.json" # A directory of its own: the job uploads RUNNER_TEMP/*.xml for the skip census, which # has rows for the three arms and none for the run that holds them all. mkdir -p "${RUNNER_TEMP}/split-arms" python3 ../scripts/ci/lane_selection.py union "${RUNNER_TEMP}/split-arms/tests.txt" \ split="${RUNNER_TEMP}/split-discovery.json" spawn="${RUNNER_TEMP}/spawn-discovery.json" \ tcp="${RUNNER_TEMP}/tcp-discovery.json" rc=0 ctest --output-on-failure --tests-from-file "${RUNNER_TEMP}/split-arms/tests.txt" --no-tests=error -j 4 \ --output-junit "${RUNNER_TEMP}/split-arms/all.xml" || rc=$? # Carved before the exit code is acted on, so a red run still leaves each arm's JUnit # for the annotations and the skip census. carved=0 for arm in split spawn tcp; do python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/split-arms/all.xml" \ --select-from "${RUNNER_TEMP}/${arm}-discovery.json" \ --write-selected "${RUNNER_TEMP}/${arm}-baseline.xml" || carved=1 done if [ "${rc}" -ne 0 ]; then echo "::error::the split, spawn and TCP arms' run failed (ctest exit ${rc})" exit "${rc}" fi if [ "${carved}" -ne 0 ]; then echo "::error::the run does not hold every entry of the split, spawn and TCP arms exactly once, so it cannot stand for the three arms" exit 1 fi python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/split-baseline.xml" --require-split-ran # Exit gate 9.3: the spawn arm green at the same count. It runs the same cases through a # second process, so a failure here that the inproc arm did not see is a transport fact # and not a phase one - which is the whole reason both lanes exist. # PH-6 / fuzz arm 3: the supervisor-level forfeit control rides this lane and must RUN. python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/spawn-baseline.xml" --require-spawn-ran --require-entry-passed SpawnLane.EventForfeitPeer python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/tcp-baseline.xml" --require-tcp-ran --require-run-ahead --discovery "${RUNNER_TEMP}/tcp-discovery.json" python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/tcp-baseline.xml" --require-entry-passed TcpLane.SupervisorProtocolControls --require-entry-passed TcpLane.EventForfeitPeer --require-entry-passed TcpLane.ControlFrameFuzz # Whether each entry ran the transport it claims is asserted in-test: the split fixture # (Harness/ScenarioFixture.h) fails an armed entry whose process resolved another transport # or whose server is not where the claim says, and junit_tally's --require-*-ran above # require each arm to have passed entries. This step proves the IPC defaults the lanes # rely on, from every passed entry's own client log; it runs right after the lane, so it # reads the lane's own run rather than a later step's rerun of the same entry. - name: Default-knob resolution check id: resolution if: env.MOBILEGL_CI_DISAGGREGATED == 'ON' working-directory: ${{ env.SPLIT_BUILD_DIR }} run: | # The IPC knobs every passed entry resolved, from its own client log: the defaults the # lane relies on (run-ahead and the role-split state on, strict off) unless the entry # pins its own. Proven defaults make Transport A/B below a repeat of the integration # job's two legs; exit 3 says a default changed, and A/B then runs. ctest -L integration-split --show-only=json-v1 > "${RUNNER_TEMP}/split-lane.json" rc=0 python3 "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" ipc-config \ "${RUNNER_TEMP}/split-lane.json" "${RUNNER_TEMP}/split-baseline.xml" "${RUNNER_TEMP}/split-ipc.json" \ strict=0 role-split-state=1 run-ahead=1 --assumed strict,role-split-state,run-ahead || rc=$? if [ "${rc}" -eq 0 ]; then echo "defaults-on=true" >> "${GITHUB_OUTPUT}" elif [ "${rc}" -eq 3 ]; then echo "::warning::an IPC default differs from what the lanes assume (split-ipc.json names the entries), so Transport A/B runs" echo "defaults-on=false" >> "${GITHUB_OUTPUT}" else exit "${rc}" fi # Both full transport runs are hard gates. The per-case comparison below # remains diagnostic because explicit mechanism controls retain their own mode. # With build-linux reused and the defaults proven above, the integration job's two legs # already ran these entries on this library in these configurations, so this is skipped. # # The entries that name MOBILEGL_TRANSPORT in their OWN property (the Split. lanes) keep # their value in both passes, which is correct: they are the split family in both arms and # the comparison is about the other 1100. - name: Transport A/B (monolith vs inproc) if: env.SPLIT_REUSE != 'true' || steps.resolution.outputs.defaults-on != 'true' working-directory: ${{ env.SPLIT_BUILD_DIR }} env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_ITRP_FIX: "ForceOn" run: | ulimit -c unlimited count=$(ctest -N -L integration-gpu | grep -cE '^ *Test *#[0-9]+:') if [ "${count}" -lt 1 ]; then echo "::error::the split runtime registers ${count} integration-gpu entries" exit 1 fi echo "integration-gpu entries in the split build: ${count}" MOBILEGL_TRANSPORT=monolith ctest --output-on-failure -L integration-gpu --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/arm-monolith.xml" MOBILEGL_TRANSPORT=inproc MOBILEGL_IPC_ROLE_SPLIT_STATE=1 MOBILEGL_IPC_STRICT_ERRORS=1 ctest --output-on-failure -L integration-gpu --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/arm-inproc.xml" # Both commands above must succeed; report differences among legitimate skips. python3 - "${RUNNER_TEMP}/arm-monolith.xml" "${RUNNER_TEMP}/arm-inproc.xml" <<'PY' import sys, xml.etree.ElementTree as ET def rows(path): out = {} for case in ET.parse(path).getroot().iter('testcase'): status = 'passed' if case.find('failure') is not None or case.find('error') is not None: status = 'failed' elif case.find('skipped') is not None or case.get('status') in ('notrun', 'disabled'): status = 'skipped' out[case.get('name')] = status return out a, b = rows(sys.argv[1]), rows(sys.argv[2]) diff = sorted(set(a) ^ set(b)) + sorted(n for n in set(a) & set(b) if a[n] != b[n]) if diff: for name in diff[:40]: print(f"{name}: monolith={a.get(name, '')} inproc={b.get(name, '')}") print(f"Transport comparison: {len(diff)} name/status differences after both full runs passed") else: print(f"the two arms agree on all {len(a)} entries, name and status") PY # The DirectGLES.Split. entries and their log-ownership check, once, in a job whose 'Split # lanes' step never ran because an earlier step failed: without this, a red unit lane or # flatc build would hide every split integration result of the run. When the lane step ran # it left split-baseline.xml, and this step does nothing. - name: Split lane fallback (lane step skipped) if: ${{ !cancelled() }} working-directory: ${{ env.SPLIT_BUILD_DIR }} env: MOBILEGL_ITEST_REQUIRE_GPU: "1" run: | if [ -f "${RUNNER_TEMP}/split-baseline.xml" ]; then echo "the lane step ran and left split-baseline.xml; nothing to fall back to" exit 0 fi echo "::notice::the lane step left no split-baseline.xml, so the DirectGLES.Split. entries run here" python3 "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" check ctest "$PWD" # A directory of its own: the job uploads RUNNER_TEMP/*.xml for the skip census, which # has no row for this run. mkdir -p "${RUNNER_TEMP}/split-fallback" ctest --output-on-failure -L integration-split -R '^DirectGLES\.Split\.' -j 4 --no-tests=error \ --output-junit "${RUNNER_TEMP}/split-fallback/isplit.xml" python3 "${GITHUB_WORKSPACE}/scripts/ci/junit_tally.py" "${RUNNER_TEMP}/split-fallback/isplit.xml" --require-split-ran # build-linux ran the server link-closure ratchet on the build these lanes took, with # continue-on-error so that a red there does not skip build-linux's own consumers. - name: Server link-closure ratchet of the reused build if: ${{ !cancelled() && env.SPLIT_REUSE == 'true' }} env: PRIMARY_RATCHET: ${{ needs.build-linux.outputs.link-ratchet }} run: | if [ "${PRIMARY_RATCHET}" != success ]; then echo "::error::the server link-closure ratchet of the build-linux build these lanes took did not pass (outcome '${PRIMARY_RATCHET}'); build-linux's link-ratchet-primary artifact names the symbols" exit 1 fi echo "the server link-closure ratchet passed on the build-linux build these lanes took" # The run log needs a signed-in reader; annotations do not. Name every failed ctest entry, # with the lines of its output that say why, where anyone reading the run can see them. - name: Annotate failed entries if: failure() run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}" - name: Upload split lane logs if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: integration-split-logs path: | ${{ env.SPLIT_BUILD_DIR }}/MobileGL/MG_IntegrationTest/*.log* ${{ env.SPLIT_BUILD_DIR }}/MobileGL/MG_IntegrationTest/split-logs/*.log ${{ runner.temp }}/arm-inproc.xml ${{ runner.temp }}/arm-monolith.xml if-no-files-found: warn # P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py). - name: Upload JUnit for the skip census if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: junit-integration-split path: ${{ runner.temp }}/*.xml if-no-files-found: warn - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: integration-split-core-dumps path: /tmp/core.* if-no-files-found: ignore # Shares build-linux-split with the functional/transport job above. This job # checks state ownership: strict residual-read rules, distinct client/server # blocks, zero-residual assertions, and complete per-entry result accounting. # Configured controls must pass without skips. Capability gaps and known # limitations are reported separately and accepted only by exact name/reason. integration-split-strict: runs-on: ubuntu-26.04 timeout-minutes: 60 needs: [build-guard, build-linux-split, build-linux] if: ${{ !cancelled() && needs.build-linux-split.result == 'success' && (needs.build-guard.outputs.verdict != 'identical' || needs.build-linux.result == 'success') }} # Short-lived test processes skip the glslang prewarm, as in the test job. env: MOBILEGL_DISABLE_SHADER_PREWARM: "1" SPLIT_REUSE: ${{ needs.build-guard.outputs.verdict == 'identical' && 'true' || 'false' }} SPLIT_BUILD_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }} SPLIT_RUNTIME: ${{ needs.build-guard.outputs.verdict == 'identical' && 'mobilegl-linux-runtime' || 'mobilegl-linux-runtime-split' }} MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }} steps: - name: Checkout repo uses: actions/checkout@v6 # build-linux's build when build-guard proved the two configurations identical, # build-linux-split's otherwise; integration-split prints the guard's evidence. A build-linux # that is not Release is never reused (scripts/ci/split_reuse_build_type.sh). - name: Split runtime env: GUARD_VERDICT: ${{ needs.build-guard.outputs.verdict }} PRIMARY_BUILD_TYPE: ${{ needs.build-linux.outputs.build-type }} run: | echo "split runtime: ${SPLIT_BUILD_DIR} (build-guard verdict '${GUARD_VERDICT:-none}')" bash scripts/ci/split_reuse_build_type.sh - name: Configure runner environment uses: ./.github/actions/setup-test-runner with: gpu: "true" - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Install runtime dependencies timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers vulkan-validationlayers - name: Download Linux split runtime uses: actions/download-artifact@v8 with: name: ${{ env.SPLIT_RUNTIME }} path: . - name: Unpack Linux split runtime run: | tar --zstd -xf "${SPLIT_RUNTIME}.tar.zst" test -f "${SPLIT_BUILD_DIR}/libMobileGL.so" if [ "${SPLIT_REUSE}" = "true" ]; then python3 -c 'import json, sys; mode = json.load(open(sys.argv[1]))["mode"]; sys.exit(0 if mode == "disaggregated" else f"build-linux runtime mode is {mode}, not disaggregated")' \ "${SPLIT_BUILD_DIR}/runtime-mode.json" fi - name: Prepare downloaded test metadata run: | for root in build-linux build-verify build-split build-verify-split build-retrace; do if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi done - name: Unit tests on the split runtime (strict) working-directory: ${{ env.SPLIT_BUILD_DIR }} env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit-strict.xml" # ---- Strict split lane: residual pulls under MOBILEGL_IPC_STRICT_ERRORS=1 ---- # # An unbarriered pull is always fatal, and strict also stops any barriered pull that is not # admitted (ARCHITECTURE.md §9.2.1). The lane must be green, and its markers must match the # admitted list and strict-expected-markers.txt in both directions. On the disaggregation # branches the dual-block step below runs these entries in this same environment and makes # the same checks, so this step runs on dev only. - name: Strict split lane if: env.MOBILEGL_CI_DISAGGREGATED != 'ON' working-directory: ${{ env.SPLIT_BUILD_DIR }} env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" MOBILEGL_TRANSPORT: "inproc" run: | ulimit -c unlimited helper="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" expected="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/strict-expected-markers.txt" # THE LOG SET COMES FROM ctest, NOT FROM A DIRECTORY (P5e gl, ID-119). This step used to # grep MobileGL/MG_IntegrationTest/split-logs/, which holds 90 of the lane's entries. # The 26 it could not see were the F1. readback block, both NamedBlit pairs, the four # Ct. entries and PersistentMapArm - i.e. PRECISELY the readback population the # allowlist is about, plus the only DirectVulkan entries in the lane. A directory is a # guess about where the lane put its logs; `--show-only=json-v1` is the lane saying so. ctest -L integration-split --show-only=json-v1 > "${RUNNER_TEMP}/lane.json" # Stale logs from an earlier job would be read as this run's evidence: an entry that # never starts leaves the previous file in place, and "no marker" is exactly what a # lane that never ran looks like. python3 - "${RUNNER_TEMP}/lane.json" "${helper}" <<'PY' import importlib.util, json, os, sys spec = importlib.util.spec_from_file_location("slp", sys.argv[2]) slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp) logs = slp.marker_log_paths(json.load(open(sys.argv[1]))) for path in set(logs.values()): try: os.unlink(path) except OSError: pass print(f"reset {len(set(logs.values()))} private log(s) across {len(logs)} entries") PY rc=0 ctest -L integration-split --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/strict-split.xml" \ > "${RUNNER_TEMP}/strict-split.txt" 2>&1 || rc=$? tail -5 "${RUNNER_TEMP}/strict-split.txt" echo "strict run: rc=${rc}" python3 "${GITHUB_WORKSPACE}/scripts/gen_pipe_field_ownership.py" --print-admitted \ > "${RUNNER_TEMP}/allowed.txt" echo "§7 allowlist (derived): $(wc -l < "${RUNNER_TEMP}/allowed.txt") admitted pair(s)" # Every entry must complete, and the census is what says why when one does not: an abort # is either an unbarriered pull (§3.3's unconditional Fatal, a missed migration) or a # barriered pull nothing admits, and both come out of the census by field and verb. if [ "${rc}" -ne 0 ]; then echo "::error::integration-split under MOBILEGL_IPC_STRICT_ERRORS=1 is RED on a run-ahead server. CONTRACT-P5E §7 makes this lane a hard green gate; the census below names the field and the verb of every entry that aborted." python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \ "${RUNNER_TEMP}/allowed.txt" "${expected}" || true exit 1 fi # The admitted pairs come from gen_pipe_field_ownership.py --print-admitted, the table # MGPipeBarrierPullAdmitted() reads, so this list and the C++ cannot drift. A pair is # admitted when its verb has a stamp row, the field is BARRIER_PULLED and in the verb's # may-read mask, and either every wire op stamping the verb waits or the field's retiring # phase is not the generator's THIS_PHASE. The census fails on an admitted marker outside # that list and on any difference from strict-expected-markers.txt in either direction; # `no-fatal` adds any Fatal marker. Escalated markers depend on the record's payload (an # open XFB span, client vertex arrays), so they are checked only against the expected set. python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \ "${RUNNER_TEMP}/allowed.txt" "${expected}" no-fatal # Arming proof: DirectGLES.Split.StrictArming. runs with MOBILEGL_PIPE_STATS=1 and a # private log and asserts, over one drawing frame, that the server stamped a verb # boundary (vbs > 0) and that no residual pull happened (rsp == 0). Only that entry # enables PipeStats, so the counters are checked there, not grepped from these logs; rsp # counts only barriered pulls, as an unbarriered one aborts before CountBarrierPull. # This step only requires that the entry ran: a skipped positive control proves nothing. if ! grep -q 'DirectGLES\.Split\.StrictArming\.' "${RUNNER_TEMP}/strict-split.txt"; then echo "::error::the strict lane did not run DirectGLES.Split.StrictArming., so nothing in this run proves the server ever stamped a verb boundary. A green lane without it is indistinguishable from a lane where strict was never armed (ID-115)." exit 1 fi echo "integration-split under strict is GREEN, two-sided, with the arming control present" # ---- P5f (f1), P5F-WIRE-COMPLETENESS.md §4: THE DUAL-BLOCK CENSUS LANE --------------- # # The same integration-split entries (plus Magma's own lane), re-run under # These entries own assertions removed from generic transport registrations. # Discovery, execution and zero-skip accounting must agree on the exact set. - name: Specialized configuration controls working-directory: ${{ env.SPLIT_BUILD_DIR }} env: MOBILEGL_ITEST_REQUIRE_GPU: "1" run: | helper="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" ctest --show-only=json-v1 > "${RUNNER_TEMP}/specialized-all.json" python3 "${helper}" select-specialized "${RUNNER_TEMP}/specialized-all.json" \ "${RUNNER_TEMP}/specialized-selected.json" "${RUNNER_TEMP}/specialized-tests.txt" ctest --tests-from-file "${RUNNER_TEMP}/specialized-tests.txt" --no-tests=error -j 4 \ --output-on-failure --output-junit "${RUNNER_TEMP}/specialized.xml" python3 "${helper}" specialized-results "${RUNNER_TEMP}/specialized-all.json" \ "${RUNNER_TEMP}/specialized.xml" # MOBILEGL_IPC_ROLE_SPLIT_STATE=1: the client's residual fill writes a client-role block # and the backend reads a server-role one, so every BARRIER-PULLED read aborts with a # named Fatal{UnmigratedPipeInput, "@"}. THE RED IS THE CENSUS - it is the # machine-checked list of what still depends on the two roles sharing one object, and it # is ratcheted two-sided against Harness/dualblock-expected-fatals.txt: a new pair is a # debt somebody now owes, a vanished pair must be removed in the commit that retired it. # With the census empty, this lane is a hard green gate. # On the disaggregation branches the first run is also the strict split lane: the knob is # left at its default, which every passed entry's own log must show is the dual block, so # the environment is the strict lane's exactly. Its checks are made on that run (the # integration-split part of the JUnit becomes strict-split.xml); a default that changed # reruns the census with the knob set, as on dev. - name: Dual-block census lane id: dualblock working-directory: ${{ env.SPLIT_BUILD_DIR }} env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_TRANSPORT: "inproc" MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" run: | ulimit -c unlimited helper="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" expected="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/dualblock-expected-fatals.txt" strict_expected="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/strict-expected-markers.txt" ctest -L 'integration-split|integration-magma-split' --show-only=json-v1 \ > "${RUNNER_TEMP}/dualblock-lane.json" # Stale logs from an earlier step would be read as this run's evidence (ID-119's rule). reset_logs() { python3 - "${RUNNER_TEMP}/dualblock-lane.json" "${helper}" <<'PY' import importlib.util, json, os, sys spec = importlib.util.spec_from_file_location("slp", sys.argv[2]) slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp) logs = slp.marker_log_paths(json.load(open(sys.argv[1]))) for path in set(logs.values()): try: os.unlink(path) except OSError: pass print(f"reset {len(set(logs.values()))} private log(s) across {len(logs)} entries") PY } census_run() { rc=0 ctest -L 'integration-split|integration-magma-split' --no-tests=error -j 4 \ --output-junit "${RUNNER_TEMP}/dualblock.xml" \ > "${RUNNER_TEMP}/dualblock-split.txt" 2>&1 || rc=$? tail -5 "${RUNNER_TEMP}/dualblock-split.txt" echo "dual-block census run: rc=${rc}" } reset_logs if [ "${MOBILEGL_CI_DISAGGREGATED}" != ON ]; then census_run else # Run A: the strict split lane's environment, with the role-split state at its default. saved_role="${MOBILEGL_IPC_ROLE_SPLIT_STATE}" unset MOBILEGL_IPC_ROLE_SPLIT_STATE census_run export MOBILEGL_IPC_ROLE_SPLIT_STATE="${saved_role}" # The strict lane's JUnit is the integration-split part of this run, every name once. ctest -L integration-split --show-only=json-v1 > "${RUNNER_TEMP}/lane.json" selected=0 python3 "${GITHUB_WORKSPACE}/scripts/ci/junit_tally.py" "${RUNNER_TEMP}/dualblock.xml" \ --select-from "${RUNNER_TEMP}/lane.json" --write-selected "${RUNNER_TEMP}/strict-split.xml" || selected=$? python3 "${GITHUB_WORKSPACE}/scripts/gen_pipe_field_ownership.py" --print-admitted \ > "${RUNNER_TEMP}/allowed.txt" echo "allowlist (derived): $(wc -l < "${RUNNER_TEMP}/allowed.txt") admitted pair(s)" if [ "${rc}" -ne 0 ]; then echo "::error::the strict run of integration-split and integration-magma-split under MOBILEGL_IPC_STRICT_ERRORS=1 is RED on a run-ahead server. It is a hard green gate; the census below names the field and the verb of every integration-split entry that aborted, and dualblock-split.txt holds both labels' output." python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \ "${RUNNER_TEMP}/allowed.txt" "${strict_expected}" || true exit 1 fi if [ "${selected}" -ne 0 ]; then echo "::error::the dual-block run's JUnit does not hold every integration-split entry exactly once, so it cannot stand for the strict split lane" exit 1 fi # The strict lane's two-sided marker census and its arming proof, as in its own step. python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \ "${RUNNER_TEMP}/allowed.txt" "${strict_expected}" no-fatal if ! grep -q 'DirectGLES\.Split\.StrictArming\.' "${RUNNER_TEMP}/dualblock-split.txt"; then echo "::error::the strict run did not run DirectGLES.Split.StrictArming., so nothing in this run proves the server ever stamped a verb boundary. A green lane without it is indistinguishable from a lane where strict was never armed." exit 1 fi echo "integration-split under strict is GREEN, two-sided, with the arming control present" # Run A is the census run when every passed entry resolved the dual block (and strict # and run-ahead) on its own; a default that changed reruns with the knob set. ipc=0 python3 "${helper}" ipc-config "${RUNNER_TEMP}/dualblock-lane.json" "${RUNNER_TEMP}/dualblock.xml" \ "${RUNNER_TEMP}/dualblock-ipc.json" strict=1 role-split-state=1 run-ahead=1 \ --assumed role-split-state,run-ahead || ipc=$? if [ "${ipc}" -eq 0 ]; then echo "defaults-on=true" >> "${GITHUB_OUTPUT}" elif [ "${ipc}" -eq 3 ]; then echo "::warning::the role-split state or run-ahead default is not on (dualblock-ipc.json names the entries), so the census reruns with MOBILEGL_IPC_ROLE_SPLIT_STATE=1" echo "defaults-on=false" >> "${GITHUB_OUTPUT}" rm -f "${RUNNER_TEMP}/dualblock.xml" reset_logs census_run else exit "${ipc}" fi fi rows=$(awk '!/^[[:space:]]*(#|$)/ { n++ } END { print n+0 }' "${expected}") if [ "${rows}" -eq 0 ] && [ "${rc}" -ne 0 ]; then echo "::error::empty dual-block census requires ctest success, including infrastructure/execution status" exit "${rc}" fi python3 "${helper}" expect-fatal "${RUNNER_TEMP}/dualblock-lane.json" \ "${RUNNER_TEMP}/dualblock.xml" "${expected}" # The per-frame zero-residual-read case must execute and pass on both backends. - name: P5f per-frame zero-residual check working-directory: ${{ env.SPLIT_BUILD_DIR }} env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" run: | ctest -L integration-p5f-rsp --show-only=json-v1 > "${RUNNER_TEMP}/rsp-lane.json" ctest -L integration-p5f-rsp --no-tests=error --output-on-failure \ --output-junit "${RUNNER_TEMP}/rsp.xml" python3 "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \ require-green "${RUNNER_TEMP}/rsp-lane.json" "${RUNNER_TEMP}/rsp.xml" \ DirectGLES.Split.P5fRsp.F1WireScenario.EachWireFrameHasZeroResidualPulls \ DirectVulkan.Split.P5fRsp.F1WireScenario.EachWireFrameHasZeroResidualPulls # The mechanism's own negative control (P5F §6): the same named case must be green with # the knob on and red - on its own assertion text - with it off. - name: Dual-block negative control (P5f f1) working-directory: ${{ env.SPLIT_BUILD_DIR }} run: bash "${GITHUB_WORKSPACE}/scripts/ci/dualblock_negative_control.sh" # ---- MAGMA'S OWN WIRE LANE: FRAMEBUFFER, TEXTURE AND APPLICATION BUFFERS ------------ # # P5f fm: Magma's framebuffer, copy and mipmap verbs consume server records. # The label remains separate so both backends' coverage stays explicit. The dual-block # census run above covers the label in this same environment; with its defaults proven on # the disaggregation branches this step would repeat it, so it runs only otherwise. - name: Magma split lane (strict, dual-block) if: env.MOBILEGL_CI_DISAGGREGATED != 'ON' || steps.dualblock.outputs.defaults-on != 'true' working-directory: ${{ env.SPLIT_BUILD_DIR }} env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" MOBILEGL_TRANSPORT: "inproc" run: | ctest -L integration-magma-split --no-tests=error -j 2 --output-on-failure # The Magma scenarios of the step above over the spawn and TCP transports, so a failure here # that the inproc step did not show is a two-process fact, not a Magma one. Entries that can # run only inproc (such as those that peek this process's server loop) are listed in # spawn_lane_parity.py's MAGMA_INPROC_ONLY. The informational tier, # integration-magma-all-{split,spawn,tcp}, is not gated: it may go red while CI stays green, # and a case moves into a gating step once its blocker is fixed. - name: Magma spawn and TCP lanes working-directory: ${{ env.SPLIT_BUILD_DIR }} env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" run: | ulimit -c unlimited ctest -L '^integration-magma-spawn$' --no-tests=error -j 2 --output-on-failure ctest -L '^integration-magma-tcp$' --no-tests=error -j 2 --output-on-failure # Every named Magma application-buffer wire case must execute and pass. - name: Magma buffer wire cases (inproc) working-directory: ${{ env.SPLIT_BUILD_DIR }} run: | ctest -L integration-magma-buffers --show-only=json-v1 > "${RUNNER_TEMP}/magma-buffer-lane.json" ctest -L integration-magma-buffers --no-tests=error -j 2 --output-on-failure \ --output-junit "${RUNNER_TEMP}/magma-buffers.xml" python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \ "${RUNNER_TEMP}/magma-buffer-lane.json" "${RUNNER_TEMP}/magma-buffers.xml" <<'PY' import importlib.util, json, sys spec = importlib.util.spec_from_file_location("slp", sys.argv[1]) slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp) cases = [ "F1WireScenario.EnabledVertexBufferDrawKeepsNamedP7Fatal", "F1WireScenario.UniformBufferRangeAndRebindPixels", "F1WireScenario.ShortUniformBufferRangePadsMissingBytes", "F1WireScenario.SubWordUniformBufferRangePadsMissingBytes", "F1WireScenario.UnalignedAtomicCounterRangeDeclinesAndTheSessionLives", "F1WireScenario.ComputeWrittenVertexAndIndexBuffersDrawAndReadBack", "F1WireScenario.ColorBlitToDefaultIgnoresViewportAndPreservesOrientation", "F1WireScenario.ColorBlitToDefaultHonorsScissorAndReversedRect", "IndexedDrawFamilyScenario.AnElementBufferDrawElementsOffsetSelectsTheRightQuad", "IndexedDrawFamilyScenario.DrawElementsBaseVertexMovesTheSameIndicesToTheOtherQuad", "IndexedDrawFamilyScenario.DrawRangeElementsDrawsTheRangedRun", "IndexedDrawFamilyScenario.DrawElementsInstancedBaseVertexPaintsOneQuadPerInstance", "IndexedDrawFamilyScenario.MultiDrawElementsBaseVertexPaintsEverySubDraw", "IndexedDrawFamilyScenario.MultiDrawElementsIndirectDrawsFromTheIndirectBuffer", "CrossFrameBufferScenario.VertexBufferSubData", "CrossFrameBufferScenario.IndexBufferSubData", "CrossFrameBufferScenario.VertexOrphanAndReupload", "CrossFrameBufferScenario.IndexOrphanAndReupload", "PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw", "PersistentCoherentMapScenario.AWriteAfterAFrameBoundaryReachesTheNextFramesDraw", "StorageBufferRegrowScenario.AGrownStoreIsVisibleThroughItsExistingIndexedBinding", # P11 A2: the >= 16 MiB arena and its tier assertion (emulated). "LargeArenaAdoptionScenario.SubDataAfterAnInFlightDrawReachesTheNextDraw", "LargeArenaAdoptionScenario.RespecifiedVertexArenaKeepsVaoBindings", "LargeArenaAdoptionScenario.RespecifiedIndexArenaKeepsVaoBinding", "LargeArenaAdoptionScenario.ReadbackSeesTheLatestCpuWrite", "LargeArenaAdoptionScenario.GpuWriteIntoTheArenaIsReadBack", "LargeArenaAdoptionScenario.TheArenaLandsInTheTierItsLaneDeclares", ] slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], ["DirectVulkan.Split.Buffers." + name for name in cases]) PY # P7 package L: THE SAME TWENTY-ONE, BY NAME, IN TWO PROCESSES (nineteen at package L; the # sub-word uniform range and the unaligned atomic-counter decline joined later). # # `--no-tests=error` in the lane step above answers "did something run and pass"; it does # not answer "did these twenty-one run". require_green does, and it is the check that would # catch the failure mode this package could plausibly introduce - an arm that registers # SOME of the lane and goes green on a smaller set. The label spelling is what keeps the # three arms apart under `ctest -L`'s regex: the inproc arm keeps the historical # `integration-magma-buffers` (which is what the step above require_greens), and the new # arms put the arm word FIRST, so neither selection can reach the other. # # TcpServer.Start / TcpServer.Stop ARE IN THE tcp ARM'S REQUIRED LIST ON PURPOSE. Every # tcp entry declares FIXTURES_REQUIRED mobilegl-tcp, so ctest lists the supervisor's two # fixture entries under the label as well - and they must be green too: a lane whose # supervisor failed to start has nothing to say about the twenty-one. - name: Magma buffer wire cases (spawn, TCP) working-directory: ${{ env.SPLIT_BUILD_DIR }} env: MOBILEGL_ITEST_REQUIRE_GPU: "1" run: | for arm in spawn tcp; do case "${arm}" in spawn) prefix=DirectVulkan.Spawn.Buffers. ;; tcp) prefix=DirectVulkan.Tcp.Buffers. ;; esac label="integration-magma-${arm}-buffers" ctest -L "^${label}$" --show-only=json-v1 > "${RUNNER_TEMP}/magma-buffer-${arm}.json" ctest -L "^${label}$" --no-tests=error -j 2 --output-on-failure \ --output-junit "${RUNNER_TEMP}/magma-buffers-${arm}.xml" python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \ "${RUNNER_TEMP}/magma-buffer-${arm}.json" "${RUNNER_TEMP}/magma-buffers-${arm}.xml" \ "${prefix}" "${arm}" <<'PY' import importlib.util, json, sys spec = importlib.util.spec_from_file_location("slp", sys.argv[1]) slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp) cases = [ "F1WireScenario.EnabledVertexBufferDrawKeepsNamedP7Fatal", "F1WireScenario.UniformBufferRangeAndRebindPixels", "F1WireScenario.ShortUniformBufferRangePadsMissingBytes", "F1WireScenario.SubWordUniformBufferRangePadsMissingBytes", "F1WireScenario.UnalignedAtomicCounterRangeDeclinesAndTheSessionLives", "F1WireScenario.ComputeWrittenVertexAndIndexBuffersDrawAndReadBack", "F1WireScenario.ColorBlitToDefaultIgnoresViewportAndPreservesOrientation", "F1WireScenario.ColorBlitToDefaultHonorsScissorAndReversedRect", "IndexedDrawFamilyScenario.AnElementBufferDrawElementsOffsetSelectsTheRightQuad", "IndexedDrawFamilyScenario.DrawElementsBaseVertexMovesTheSameIndicesToTheOtherQuad", "IndexedDrawFamilyScenario.DrawRangeElementsDrawsTheRangedRun", "IndexedDrawFamilyScenario.DrawElementsInstancedBaseVertexPaintsOneQuadPerInstance", "IndexedDrawFamilyScenario.MultiDrawElementsBaseVertexPaintsEverySubDraw", "IndexedDrawFamilyScenario.MultiDrawElementsIndirectDrawsFromTheIndirectBuffer", "CrossFrameBufferScenario.VertexBufferSubData", "CrossFrameBufferScenario.IndexBufferSubData", "CrossFrameBufferScenario.VertexOrphanAndReupload", "CrossFrameBufferScenario.IndexOrphanAndReupload", "PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw", "PersistentCoherentMapScenario.AWriteAfterAFrameBoundaryReachesTheNextFramesDraw", "StorageBufferRegrowScenario.AGrownStoreIsVisibleThroughItsExistingIndexedBinding", "LargeArenaAdoptionScenario.SubDataAfterAnInFlightDrawReachesTheNextDraw", "LargeArenaAdoptionScenario.RespecifiedVertexArenaKeepsVaoBindings", "LargeArenaAdoptionScenario.RespecifiedIndexArenaKeepsVaoBinding", "LargeArenaAdoptionScenario.ReadbackSeesTheLatestCpuWrite", "LargeArenaAdoptionScenario.GpuWriteIntoTheArenaIsReadBack", "LargeArenaAdoptionScenario.TheArenaLandsInTheTierItsLaneDeclares", ] required = [sys.argv[4] + name for name in cases] if sys.argv[5] == "tcp": required += ["TcpServer.Start", "TcpServer.Stop"] slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], required) PY done # Run-ahead must queue real GPU work and enforce credits 1 and 3. - name: Magma run-ahead cases (credits 1, 3) working-directory: ${{ env.SPLIT_BUILD_DIR }} run: | ctest -L integration-magma-runahead --show-only=json-v1 > "${RUNNER_TEMP}/magma-runahead-lane.json" ctest -L integration-magma-runahead --no-tests=error -j 2 --output-on-failure \ --output-junit "${RUNNER_TEMP}/magma-runahead.xml" python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \ "${RUNNER_TEMP}/magma-runahead-lane.json" "${RUNNER_TEMP}/magma-runahead.xml" <<'PY' import importlib.util, json, sys spec = importlib.util.spec_from_file_location("slp", sys.argv[1]) slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp) cases = [ "QueuedBufferVersionsAndDeletedNameReuseKeepTheirPixels", "QueuedProgramRebindsKeepEachUniformSnapshot", "QueuedGpuWritesReadBackTheLastDispatchBytes", "MixedNativeAndConvertedGpuVertexStreamsKeepTheirReservation", "TextureUploadCannotOvertakeThePreviouslyQueuedDraw", "GpuClearSurvivesTheFirstStorageImageUpgrade", "PresentCreditActuallyParksTheClientAtItsConfiguredLimit", ] required = [f"DirectVulkan.Split.RunAhead.Credit{credit}.MagmaRunAheadScenario.{name}" for credit in (1, 3) for name in cases] slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], required) PY # Run-ahead off must go red on the lost queue lead; run-ahead back on must be green. - name: Magma run-ahead negative control run: | python3 scripts/ci/magma_runahead_checks.py negative --build-dir "${SPLIT_BUILD_DIR}" \ --out "${RUNNER_TEMP}/magma-runahead-negative" --execute - name: Magma run-ahead sync validation run: | python3 scripts/ci/magma_runahead_checks.py validation --build-dir "${SPLIT_BUILD_DIR}" \ --out "${RUNNER_TEMP}/magma-runahead-syncval" --execute - name: Magma view/framebuffer cache sync validation run: | python3 scripts/ci/magma_cache_checks.py --build-dir "${SPLIT_BUILD_DIR}" \ --out "${RUNNER_TEMP}/magma-cache-syncval" --execute - name: Upload Magma run-ahead and cache evidence if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: magma-runahead-checks path: | ${{ runner.temp }}/magma-runahead-negative ${{ runner.temp }}/magma-runahead-syncval ${{ runner.temp }}/magma-cache-syncval if-no-files-found: ignore # The packed-float chain is generated by a real GPU shader blit, so this entry sits in the # integration-magma-split lane with its siblings (CMakeLists.txt's fm loop) and carries # their pixels assertion. Its named check survives the flip in the only direction that has # evidence behind it: the case must EXECUTE, PASS, and never skip - a lane-level exit code # alone would not separate "generated" from "quietly absent". - name: Magma packed-float mipmap case (P7) working-directory: ${{ env.SPLIT_BUILD_DIR }} env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" MOBILEGL_TRANSPORT: "inproc" run: | p7_case='^DirectVulkan\.Split\.Fm\.F1WireScenario\.GenerateMipmapPackedFloatPixels$' ctest -R "${p7_case}" --show-only=json-v1 > "${RUNNER_TEMP}/magma-p7-lane.json" ctest -R "${p7_case}" --no-tests=error --output-on-failure \ --output-junit "${RUNNER_TEMP}/magma-p7.xml" python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \ "${RUNNER_TEMP}/magma-p7-lane.json" "${RUNNER_TEMP}/magma-p7.xml" <<'PY' import importlib.util, json, sys spec = importlib.util.spec_from_file_location("slp", sys.argv[1]) slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp) name = "DirectVulkan.Split.Fm.F1WireScenario.GenerateMipmapPackedFloatPixels" slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], [name]) PY # Client vertex/index inputs are copied into owned wire buffer resources. # The original dedicated entries now have to execute successfully with run-ahead. - name: Client-array split lane (run-ahead) working-directory: ${{ env.SPLIT_BUILD_DIR }} env: MOBILEGL_ITEST_REQUIRE_GPU: "1" MOBILEGL_TRANSPORT: "inproc" MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" MOBILEGL_IPC_RUN_AHEAD: "1" run: | ctest -L integration-clientarrays-split --show-only=json-v1 > "${RUNNER_TEMP}/clientarrays-lane.json" ctest -L integration-clientarrays-split --no-tests=error --output-on-failure -j 1 \ --output-junit "${RUNNER_TEMP}/clientarrays.xml" python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \ "${RUNNER_TEMP}/clientarrays-lane.json" "${RUNNER_TEMP}/clientarrays.xml" <<'PY' import importlib.util, json, sys spec = importlib.util.spec_from_file_location("slp", sys.argv[1]) slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp) lane = json.load(open(sys.argv[2])) names = [test["name"] for test in lane["tests"]] if not names: raise SystemExit("no client-array cases discovered") slp.require_green(lane, sys.argv[3], names) PY # The run log needs a signed-in reader; annotations do not. Name every failed ctest entry, # with the lines of its output that say why, where anyone reading the run can see them. - name: Annotate failed entries if: failure() run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}" # P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py). - name: Upload JUnit for the skip census if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: junit-integration-split-strict path: ${{ runner.temp }}/*.xml if-no-files-found: warn - name: Upload strict lane logs if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: integration-split-strict-logs path: | ${{ env.SPLIT_BUILD_DIR }}/MobileGL/MG_IntegrationTest/*.log* ${{ env.SPLIT_BUILD_DIR }}/MobileGL/MG_IntegrationTest/split-logs/*.log ${{ runner.temp }}/specialized*.json ${{ runner.temp }}/specialized-tests.txt ${{ runner.temp }}/specialized.xml ${{ runner.temp }}/dualblock-lane.json ${{ runner.temp }}/dualblock.xml ${{ runner.temp }}/dualblock-split.txt if-no-files-found: warn # MobileGL/MG_Remote/Protocol/generated/protocol_generated.h is COMMITTED, and # flatc is deliberately absent from the default build graph (a codegen step in # the graph is how the earlier branch ended up cross-compiling an arm64 flatc # and trying to run it on the host). This job is what keeps the committed # header honest: build the pinned flatc, regenerate, and fail on any diff. # It needs no MobileGL build, so it does not depend on build-linux. flatc-check: runs-on: ubuntu-26.04 steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Check out the FlatBuffers submodule only # Just this one: the schema check has nothing to do with glslang, # SPIRV-Cross or the trace fixtures. run: git submodule update --init 3rdparty/flatbuffers # The pinned flatc, cached under the submodule's commit: gen_protocol.py builds it only # when the build directory holds none, and checks its version against the submodule # either way. The gitlink gives the commit without the submodule's own checkout. # integration-split restores the same entry, so the path is the same string in both jobs. - name: Pinned flatc cache key id: flatc run: echo "commit=$(git rev-parse HEAD:3rdparty/flatbuffers)" >> "${GITHUB_OUTPUT}" - name: Cache pinned flatc uses: actions/cache@v5 with: path: ${{ runner.temp }}/flatc-build/flatc key: ${{ runner.os }}-ubuntu26-flatc-${{ steps.flatc.outputs.commit }} - name: Regenerate protocol_generated.h run: python3 scripts/gen_protocol.py --build-dir "${{ runner.temp }}/flatc-build" - name: Generated header staleness check run: git diff --exit-code -- MobileGL/MG_Remote/Protocol/generated/protocol_generated.h # P7 wave 2-F, F fix round (PH-7 (4) review). MOBILEGL_PROTOCOL_CONTROL_REVISION is mixed # into wireFingerprint so a control-schema change refuses an old peer by name - but it is a # hand-bumped integer, and nothing tied it to protocol.fbs's content. This step pins # sha256(protocol.fbs) to the revision (scripts/ci/protocol_revision_pins.json): a schema # edit that forgot the bump is red HERE, in the job that regenerates the header on every # such edit, and not a silently absent field between an old server and a new client. Its # self-test proves the check goes red on both perturbations before it says green. - name: Protocol revision pin check run: python3 scripts/ci/protocol_revision_pin.py --self-test # P0.5 interface-purity gate A (ARCHITECTURE.md §13.2): the two extracted headers' include closure, # asserted on `-H` output because `nm --undefined-only` is blind to "included but not called" - # a header whose types are never named leaves no symbol behind, and "included at all" is exactly # the coupling P1 and P7 have to sever. Needs a preprocessor and three header submodules, no # CMake configure and no glslang sources, so like pipe-gates it does not depend on build-linux. # The script's own --self-test is always on: a negative control that stopped tripping fails the # job, because a gate that cannot go red is not a gate (ARCHITECTURE.md §13.2). include-graph-check: name: Include-closure purity gate runs-on: ubuntu-26.04 steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Check out header submodules # ska/flat_hash_map.hpp, xxhash.h and vulkan/vulkan.h are the only submodule headers # Includes.h reaches; glslang and spirv_cross are vendored under include/. run: git submodule update --init include/ska 3rdparty/xxHash 3rdparty/Vulkan-Headers - name: Install clang and X11 headers timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' # Includes.h defines VK_USE_PLATFORM_XLIB_KHR before , which then # includes ; without libx11-dev every clang-mode probe dies in the # preprocessor and the gate reports 5 problems that have nothing to do with purity. run: sudo apt-get update && sudo apt-get install -y clang-20 libx11-dev - name: Include-closure check and self-test # --expect-probes 5 (contract-v2.md 7.6; P13 W5 adds record-core): an exit code cannot tell five probes from none, # so a --probe typo or a manifest edit that selected nothing would run zero probes and # exit 0 - the second half of the finding that added the flag. The count is the length of # scripts/check_include_closure.py's PROBES list and changing one means changing the other. # # Use the compiler selected for this runner's installed toolchain. run: python3 scripts/check_include_closure.py --mode both --compiler "${MOBILEGL_CI_CXX}" --self-test --require-all --expect-probes 5 benchmark: runs-on: ubuntu-26.04 needs: [build-linux] steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner with: gpu: "true" - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Install runtime dependencies timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' # libegl-mesa0 is the EGL vendor library itself: DriverBench brings up a # real GL context, and libegl1 is only glvnd's dispatch. It normally # arrives as a Recommends of libegl1, which is too quiet a dependency for # the one job that needs a working driver. run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers - name: Download Linux runtime uses: actions/download-artifact@v8 with: name: mobilegl-linux-runtime path: . - name: Unpack Linux runtime run: tar --zstd -xf mobilegl-linux-runtime.tar.zst - name: Prepare downloaded test metadata run: | for root in build-linux build-verify build-split build-verify-split build-retrace; do if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi done - name: Benchmark working-directory: build-linux run: | ulimit -c unlimited ctest -V -C Release -L benchmark --no-tests=error - name: Upload core dumps if: failure() uses: actions/upload-artifact@v7 with: name: benchmark-core-dumps path: /tmp/core.* if-no-files-found: ignore build-retrace: runs-on: ubuntu-26.04 permissions: actions: write contents: read env: BUILD_DIR: build-retrace CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "true" CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 4G CCACHE_NOHASHDIR: "true" MOBILEGL_LIBRARY: ${{ github.workspace }}/build-linux/libMobileGL.so steps: - name: Checkout repo uses: actions/checkout@v6 # The library set plus apitrace and the directories its build requires # (scripts/ci/checkout_submodules.sh). - name: Check out build submodules timeout-minutes: 15 run: bash scripts/ci/checkout_submodules.sh retrace - name: Set swap space uses: ./.github/actions/set-swap-space with: swap-size-gb: 32 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Restore ccache uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 restore-keys: | ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache- - name: Prepare Vulkan SDK id: netsetup2500 continue-on-error: true timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Prepare Vulkan SDK (retry) timeout-minutes: 15 if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup2500.outcome == 'failure' }} uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Update glslang external sources working-directory: 3rdparty/glslang run: python3 update_glslang_sources.py - name: Install dependencies timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build - name: Show installed toolchain run: | ccache --version "${MOBILEGL_CI_CC}" --version "${MOBILEGL_CI_CXX}" --version "${MOBILEGL_CI_LLD}" --version dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true - name: Prepare downloaded test metadata run: | for root in build-linux build-verify build-split build-verify-split build-retrace; do if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi done - name: Configure CMake run: | if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then BUILD_TYPE=Debug else BUILD_TYPE=Release fi cmake -S . -B "${BUILD_DIR}" -G Ninja \ -DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \ -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=OFF \ -DMOBILEGL_BUILD_BENCHMARK=OFF \ -DMOBILEGL_BUILD_TRACE_REPLAY=ON \ -DMOBILEGL_TRACE_SOFTWARE_RENDERING="${MOBILEGL_CI_SOFTWARE_RENDERING}" \ -DMOBILEGL_TRACE_REPLAY_MOBILEGL_LIBRARY="${MOBILEGL_LIBRARY}" \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 - name: Build trace replay run: cmake --build "${BUILD_DIR}" --target mobilegl_trace_replay --parallel "$(nproc)" - name: Show ccache stats if: ${{ always() }} run: ccache --show-stats - name: Release superseded ccache entry env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true - name: Save ccache continue-on-error: true uses: actions/cache/save@v5 with: path: .ccache key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 - name: Prepare packaged test metadata run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}" - name: Package trace replay run: | mkdir -p ci-artifacts tar -czf ci-artifacts/mobilegl-trace-replay.tgz \ build-retrace/.ci-source-root \ build-retrace/tools/trace_replay/mobilegl_trace_replay \ build-retrace/tools/trace_replay/CTestTestfile.cmake - name: Upload trace replay uses: actions/upload-artifact@v7 with: name: mobilegl-trace-replay path: ci-artifacts/mobilegl-trace-replay.tgz if-no-files-found: error trace-cases: name: trace case matrix runs-on: ubuntu-26.04 env: MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }} # How many jobs replay one (lane, transport, backend) cell of the Linux retrace lanes. Each # job replays its share of the cell's cases one after another; a cell with little to # replay gets fewer jobs (tools/trace_replay/trace_cases.py, shard_matrix). RETRACE_SHARDS_PER_CELL: "4" outputs: shards: ${{ steps.trace-cases.outputs.shards }} verify-shards: ${{ steps.trace-cases.outputs.verify-shards }} split-shards: ${{ steps.trace-cases.outputs.split-shards }} split-spawn-shards: ${{ steps.trace-cases.outputs.split-spawn-shards }} steps: - name: Checkout repo uses: actions/checkout@v6 # `dl` / exit gate S7. CHEAP AND EARLY, on a job that only reads the tree: it needs no # build, so a death that names no family is reported in seconds rather than after the # matrix. What it refuses is a std::abort() under MG_Remote/ with no Fatal{Family} marker, # and a family word that appeared without being recorded - the vocabulary drifts on # purpose or not at all (CONTRACT-P6 5.2's D2 ruling). - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Fatal census (S7) run: python3 scripts/ci/fatal_census.py # The decline half of rule I (P7 B3): every WireDeclines.def row has a site, every site a # warning-or-error line. Same shape as the census - tree-only, no build. The self-test # runs first: its fixtures (a site only in a comment or a string, a log in another # branch, an MGLOG_D) must red, so a green audit is one that could have been red. - name: Wire declines audit (B3) run: | python3 scripts/ci/wire_declines_audit.py --self-test python3 scripts/ci/wire_declines_audit.py - name: Load trace cases id: trace-cases run: | python3 tools/trace_replay/trace_cases.py --self-test-transport-matrices # The retrace jobs replay shards: several cases of one (lane, transport, backend) cell # per job. The first self-test proves every shard matrix replays exactly the rows of # the per-case matrix it is built from; the second proves the shard runner gives each # case the settings of its lane. python3 tools/trace_replay/trace_cases.py --self-test-shard-matrices python3 scripts/ci/retrace_shard.py --self-test shards() { python3 tools/trace_replay/trace_cases.py --ci --shards "${RETRACE_SHARDS_PER_CELL}" "$@" } shard_count() { python3 -c 'import json,sys; print(len(json.load(sys.stdin)["include"]))' } shard_format=github-test-shard-matrix if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then shard_format=github-test-transport-shard-matrix; fi # An empty `include` is not an error to GitHub: it skips the job with no red anywhere, # so each matrix is checked before it is handed over. require_shards() { if [ "$(printf '%s' "$2" | shard_count)" -lt 1 ]; then echo "::error::the $1 shard matrix is empty, so its job would be skipped with no red" exit 1 fi } SHARDS=$(shards --format "${shard_format}") require_shards retrace "${SHARDS}" echo "shards=${SHARDS}" >> "$GITHUB_OUTPUT" # The subset the verify build retraces ("verify": true in trace_cases.json). VERIFY_SHARDS=$(shards --format github-verify-shard-matrix) require_shards retrace-verify "${VERIFY_SHARDS}" echo "verify-shards=${VERIFY_SHARDS}" >> "$GITHUB_OUTPUT" # The split subset: {backend, case, transport} over inproc and spawn, both backends, for # every CI case that has not said `"split": false`. It is the whole CI matrix because # the bugs the spawn arm finds are about the server's process rather than about one # trace. trace_cases.py refuses a `split` case that is not in CI. SPLIT_SHARDS=$(shards --format github-split-shard-matrix) # It must not be empty. An empty `include` is not an error to GitHub: it skips the # whole retrace-split job with no red anywhere, so that is guarded here. # trace_cases.py also rejects an unknown manifest key, since a mistyped `split` key # would otherwise load clean and change the subset. if [ "$(printf '%s' "${SPLIT_SHARDS}" | shard_count)" -lt 1 ]; then echo "::error::the split retrace subset is empty. 'split' is an opt-out: a CI case is in this subset unless it carries \"split\": false, so an empty subset means either every case opted out or the CI matrix itself is empty. Either way retrace-split would be skipped with no red." exit 1 fi echo "split-shards=${SPLIT_SHARDS}" >> "$GITHUB_OUTPUT" # On the disaggregation branches, the same subset without the inproc rows the # inproc-acceptance retrace rows replay on the same library, which retrace-split uses when # it reuses build-linux's library. OpenRA keeps its inproc rows: they carry the # wire-volume evidence. Its self-test proves every dropped row is an acceptance row. It is # only a saving, so it never fails this job: on a failed self-test or an empty subset it # is not written, and retrace-split takes the full subset, as it does elsewhere. if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then if python3 tools/trace_replay/trace_cases.py --self-test-split-matrix \ && SPLIT_SPAWN_SHARDS=$(shards --format github-split-shard-matrix \ --transports spawn --keep-inproc OpenRA) \ && [ "$(printf '%s' "${SPLIT_SPAWN_SHARDS}" | shard_count || echo 0)" -ge 1 ]; then echo "split-spawn-shards=${SPLIT_SPAWN_SHARDS}" >> "$GITHUB_OUTPUT" else echo "::warning::the spawn-only split retrace subset failed its self-test or is empty, so retrace-split replays the full split subset" fi fi echo "retrace shards: $(printf '%s' "${SHARDS}" | shard_count) retrace, $(printf '%s' "${VERIFY_SHARDS}" | shard_count) verify, $(printf '%s' "${SPLIT_SHARDS}" | shard_count) split (full subset)" # One cache entry holds every Git LFS fixture of the CI cases, and each replay job restores # that entry itself. This job only makes sure the entry exists. On a hit it downloads # nothing. On a miss it starts from the newest older bundle, fetches from the mirror the # cases whose pointers changed, saves the entry under the new key and confirms it is there, # so that no replay job has to download fixtures itself. The key, the path and the steps are # .github/scripts/trace-fixture-bundle.sh's; apk.yml uses the same entry. The timeout bounds # a fixture download that stalls, which would otherwise hold every replay job behind it. trace-fixtures: name: trace fixtures runs-on: ubuntu-26.04 timeout-minutes: 30 permissions: actions: read contents: read outputs: key: ${{ steps.bundle.outputs.key }} path: ${{ steps.bundle.outputs.path }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Derive the fixture bundle key id: bundle run: bash .github/scripts/trace-fixture-bundle.sh key - name: Look up the fixture bundle id: lookup uses: actions/cache/restore@v5 with: path: ${{ steps.bundle.outputs.path }} key: ${{ steps.bundle.outputs.key }} lookup-only: true - name: Restore the newest older bundle if: steps.lookup.outputs.cache-hit != 'true' uses: actions/cache/restore@v5 with: path: ${{ steps.bundle.outputs.path }} key: ${{ steps.bundle.outputs.key }} restore-keys: ${{ steps.bundle.outputs.restore-prefix }} - name: Complete the fixture bundle if: steps.lookup.outputs.cache-hit != 'true' run: bash .github/scripts/trace-fixture-bundle.sh ensure # The APK workflow of the same push saves the same entry, and only one save can win. # The steps below find the entry whichever workflow wrote it. - name: Save the fixture bundle if: steps.lookup.outputs.cache-hit != 'true' continue-on-error: true uses: actions/cache/save@v5 with: path: ${{ steps.bundle.outputs.path }} key: ${{ steps.bundle.outputs.key }} # The APK workflow of the same push may be the one whose save won. Its entry is as good, # once it is listed. - name: Wait for the fixture bundle entry if: steps.lookup.outputs.cache-hit != 'true' env: GH_TOKEN: ${{ github.token }} run: bash .github/scripts/trace-fixture-bundle.sh wait '${{ steps.bundle.outputs.key }}' - name: Confirm the fixture bundle entry if: steps.lookup.outputs.cache-hit != 'true' uses: actions/cache/restore@v5 with: path: ${{ steps.bundle.outputs.path }} key: ${{ steps.bundle.outputs.key }} lookup-only: true fail-on-cache-miss: true # One job replays a shard: some of the cases of one lane, transport and backend, one after # another, each through its own ctest entry (scripts/ci/retrace_shard.py). The shard's step # summary has a row per case, and a failed case has an error annotation of its own. retrace: name: retrace (${{ matrix.backend }} / ${{ matrix.transport || 'monolith' }}, shard ${{ matrix.shard }}/${{ matrix.shards }}) runs-on: ubuntu-26.04 timeout-minutes: 240 env: MOBILEGL_TRANSPORT: ${{ matrix.transport || 'monolith' }} MOBILEGL_IPC_ROLE_SPLIT_STATE: "1" MOBILEGL_IPC_STRICT_ERRORS: "1" MOBILEGL_IPC_RUN_AHEAD: "1" SHARD_NAMES: ${{ toJSON(matrix.names) }} needs: - build-linux - build-retrace - trace-cases - trace-fixtures if: ${{ !cancelled() && needs.build-linux.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }} strategy: fail-fast: false matrix: ${{ fromJSON(needs.trace-cases.outputs.shards) }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Set swap space if: runner.environment != 'self-hosted' uses: ./.github/actions/set-swap-space with: swap-size-gb: 16 - name: Configure runner environment uses: ./.github/actions/setup-test-runner with: gpu: "true" # A miss does not fail the job: the entry may have been evicted since trace-fixtures # confirmed it (a re-run of failed jobs on an older run does not run that job again). - name: Restore trace fixtures uses: actions/cache/restore@v5 with: path: ${{ needs.trace-fixtures.outputs.path }} key: ${{ needs.trace-fixtures.outputs.key }} # Moves this shard's fixtures into place and checks them against their Git LFS pointers. # A case the restored bundle does not hold is downloaded here, with a warning annotation; # the timeout bounds a download that stalls. - name: Install trace fixtures timeout-minutes: 15 run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }} - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Install runtime dependencies timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers test -e /usr/lib/x86_64-linux-gnu/libEGL.so test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so - name: Download Linux library uses: actions/download-artifact@v8 with: name: mobilegl-linux-lib path: . - name: Download trace replay uses: actions/download-artifact@v8 with: name: mobilegl-trace-replay path: . - name: Unpack retrace runtime run: | tar -xzf mobilegl-linux-lib.tgz tar -xzf mobilegl-trace-replay.tgz test -f build-linux/libMobileGL.so test -f build-retrace/tools/trace_replay/mobilegl_trace_replay - name: Prepare downloaded test metadata run: | for root in build-linux build-verify build-split build-verify-split build-retrace; do if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi done # The per-case settings are retrace_shard.py's "retrace" profile: the Magma fallback and # the two per-case repair knobs on DirectVulkan, and on the inproc lane the IPC # configuration the client must log, the split evidence check and the refusal census from # the library's own logs. A failing case does not stop the shard; the step fails at the # end if a case failed or a case has no verdict. - name: Retrace and validate run: | python3 scripts/ci/retrace_shard.py run --profile retrace \ --backend '${{ matrix.backend }}' --transport "${MOBILEGL_TRANSPORT}" \ --lane '${{ matrix.lane || 'monolith-control' }}' \ --shard '${{ matrix.shard }}/${{ matrix.shards }}' \ --build-dir build-retrace/tools/trace_replay \ --result-dir "${RUNNER_TEMP}/retrace-shard" - name: Annotate failed entries if: failure() run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit" - name: Upload core dumps if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v7 with: name: retrace-core-dumps-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }} path: /tmp/core.* if-no-files-found: ignore # What retrace-summary renders: per case, the images and the replay's output directory. # The summary embeds the images, so these are short-lived; a failed shard keeps its own # copy below. - name: Upload shard results if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }} path: ${{ runner.temp }}/retrace-shard/results if-no-files-found: warn retention-days: 1 overwrite: true - name: Upload failed shard results if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v7 with: name: retrace-failed-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }} path: | ${{ runner.temp }}/retrace-shard/results ${{ runner.temp }}/retrace-shard/junit ${{ runner.temp }}/retrace-shard/logs if-no-files-found: warn overwrite: true retrace-summary: name: retrace summary runs-on: ubuntu-26.04 needs: [retrace] if: ${{ !cancelled() && needs.retrace.result != 'skipped' }} # actions: write is for the last step, which deletes the artifacts this job rendered. permissions: actions: write contents: read env: MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Set artifact metadata run: | echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV" - name: Set up Node.js uses: actions/setup-node@v7 with: node-version: '22' - name: Download monolith control results uses: actions/download-artifact@v8 with: pattern: retrace-result-monolith-control-monolith-* path: retrace-artifacts/monolith-control - name: Download inproc acceptance results if: env.MOBILEGL_CI_DISAGGREGATED == 'ON' uses: actions/download-artifact@v8 with: pattern: retrace-result-inproc-acceptance-inproc-* path: retrace-artifacts/inproc-acceptance - name: Render monolith control summary run: | node tools/trace_replay/render_retrace_summary.mjs \ --input retrace-artifacts/monolith-control \ --output-dir retrace-summary/monolith-control \ --title "MobileGL Linux monolith control" \ --group-label "Linux / monolith control" \ --html mobilegl-linux-monolith-control.html - name: Render inproc acceptance summary if: env.MOBILEGL_CI_DISAGGREGATED == 'ON' run: | node tools/trace_replay/render_retrace_summary.mjs \ --input retrace-artifacts/inproc-acceptance \ --output-dir retrace-summary/inproc-acceptance \ --title "MobileGL Linux inproc acceptance" \ --group-label "Linux / inproc acceptance" \ --html mobilegl-linux-inproc-acceptance.html - name: Upload monolith control summary uses: actions/upload-artifact@v7 with: name: mobilegl-linux-monolith-control-summary-${{ github.sha }} path: retrace-summary/monolith-control/mobilegl-linux-monolith-control.html archive: false if-no-files-found: error - name: Upload inproc acceptance summary if: env.MOBILEGL_CI_DISAGGREGATED == 'ON' uses: actions/upload-artifact@v7 with: name: mobilegl-linux-inproc-acceptance-summary-${{ github.sha }} path: retrace-summary/inproc-acceptance/mobilegl-linux-inproc-acceptance.html archive: false if-no-files-found: error # The summaries embed every image, so the shard result artifacts they were rendered from # have no reader left and are deleted here. That happens only when every shard passed: a # failed shard has its own retrace-failed-* artifact, and after a failure the results # stay for their one day of retention, so that re-running the failed shards renders every # case again. A failed delete only leaves them to that retention. - name: Delete the rendered shard results if: ${{ needs.retrace.result == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ github.token }} run: | gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \ --jq '.artifacts[] | select(.name | startswith("retrace-result-")) | [.id, .name] | @tsv' \ > "${RUNNER_TEMP}/rendered-artifacts.txt" deleted=0 while IFS=$'\t' read -r artifact_id artifact_name; do echo "Deleting ${artifact_name} (${artifact_id})" gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}" deleted=$((deleted + 1)) done < "${RUNNER_TEMP}/rendered-artifacts.txt" echo "Deleted ${deleted} rendered shard result artifact(s)." # The trace half of the third CI mode. Same replay, same goldens, but the library underneath is # the verify build and MOBILEGL_PIPE_VERIFY=1 is in the environment, so every backend read of # frontend state is checked against a snapshot taken at the verb boundary. Eight cases rather # than the full lane's 40 (tools/trace_replay/trace_cases.json, "verify": true): the comparator # is budgeted at 5-10x, and the full sweep is a phase-exit / workflow_dispatch run. retrace-verify: name: retrace verify (${{ matrix.backend }}, shard ${{ matrix.shard }}/${{ matrix.shards }}) runs-on: ubuntu-26.04 timeout-minutes: 240 env: SHARD_NAMES: ${{ toJSON(matrix.names) }} needs: - build-linux-verify - build-retrace - trace-cases - trace-fixtures if: ${{ !cancelled() && needs.build-linux-verify.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }} strategy: fail-fast: false matrix: ${{ fromJSON(needs.trace-cases.outputs.verify-shards) }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Set swap space if: runner.environment != 'self-hosted' uses: ./.github/actions/set-swap-space with: swap-size-gb: 16 - name: Configure runner environment uses: ./.github/actions/setup-test-runner with: gpu: "true" # A miss does not fail the job; the install step downloads what the bundle does not hold. - name: Restore trace fixtures uses: actions/cache/restore@v5 with: path: ${{ needs.trace-fixtures.outputs.path }} key: ${{ needs.trace-fixtures.outputs.key }} - name: Install trace fixtures timeout-minutes: 15 run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }} - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Install runtime dependencies timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers test -e /usr/lib/x86_64-linux-gnu/libEGL.so test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so - name: Download Linux verify library uses: actions/download-artifact@v8 with: name: mobilegl-linux-lib-verify path: . - name: Download trace replay uses: actions/download-artifact@v8 with: name: mobilegl-trace-replay path: . - name: Unpack verify runtime (library under test) # build-retrace's CTestTestfile.cmake has the absolute path # /build-linux/libMobileGL.so frozen into every case, so the swap happens here # rather than through a variable: the verify .so is put where that path points. The nm # check is what makes the swap falsifiable - a run against the ordinary library would # carry no comparator, ignore MOBILEGL_PIPE_VERIFY entirely, and match its golden. # # `nm`, not `nm -D`, for the reason spelled out in build-linux-verify: everything MGPipe is # hidden-visibility in a Release build and the dynamic table has none of it. run: | tar -xzf mobilegl-linux-lib-verify.tgz tar -xzf mobilegl-trace-replay.tgz test -f build-verify/libMobileGL.so test -f build-retrace/tools/trace_replay/mobilegl_trace_replay mkdir -p build-linux cp build-verify/libMobileGL.so build-linux/libMobileGL.so if ! nm --defined-only build-linux/libMobileGL.so | grep -q MGPipeVerifyInputs; then echo "::error::the library unpacked at build-linux/libMobileGL.so defines no MGPipeVerifyInputs, so this retrace would replay against a comparator-free build and pass on its golden having verified nothing" exit 1 fi echo "the library at build-linux/libMobileGL.so is the verify build" - name: Prepare downloaded test metadata run: | for root in build-linux build-verify build-split build-verify-split build-retrace; do if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi done # Every case replays with MOBILEGL_PIPE_VERIFY=1 (retrace_shard.py's "verify" profile). # run_trace_case.cmake turns that into three assertions of its own (the arming line, no # Fatal{PipeVerifyDiffer, no Fatal{UnmigratedPipeInput), so a case that somehow ran the # wrong library reds here instead of passing on its golden. ctest's timeout is raised to # 10800 s: the long cases run 5-10x slower with both comparator arms live, well past the # 1500 s default. # # The shard that holds OpenRA on DirectGLES also runs the lane's negative control right # after that case: with a snapshot field corrupted, the same replay must fail, and for the # comparator's reason (scripts/ci/retrace_verify_reason.py). Without it, zero divergences # would be a statement about a comparator nobody watched. The control has a verdict row of # its own, and its output is discarded so that only the verified run is uploaded. - name: Retrace and validate (verify) run: | python3 scripts/ci/retrace_shard.py run --profile verify \ --backend '${{ matrix.backend }}' --lane verify \ --shard '${{ matrix.shard }}/${{ matrix.shards }}' \ --build-dir build-retrace/tools/trace_replay \ --result-dir "${RUNNER_TEMP}/retrace-shard" - name: Annotate failed entries if: failure() run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit" - name: Upload core dumps if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v7 with: name: retrace-verify-core-dumps-${{ matrix.backend }}-shard${{ matrix.shard }} path: /tmp/core.* if-no-files-found: ignore # Kept for every shard, green or not: per case, the verified images and the replay's # output directory, whose library log has the comparator's arming line. - name: Upload shard results if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: retrace-verify-result-${{ matrix.backend }}-shard${{ matrix.shard }} path: ${{ runner.temp }}/retrace-shard/results if-no-files-found: warn overwrite: true - name: Upload failed shard results if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v7 with: name: retrace-verify-failed-${{ matrix.backend }}-shard${{ matrix.shard }} path: | ${{ runner.temp }}/retrace-shard/results ${{ runner.temp }}/retrace-shard/junit ${{ runner.temp }}/retrace-shard/logs if-no-files-found: warn overwrite: true # Trace retrace on the split library: the retrace-verify shape with MOBILEGL_TRANSPORT set # instead of MOBILEGL_PIPE_VERIFY, over {backend, case, transport} with inproc and spawn. # inproc is the control and spawn the claim: same library and picture, only the server role's # process differs, so a spawn-only failure is a transport fact. It runs the unchanged ctest # names with MOBILEGL_TRANSPORT exported around ctest, since build-retrace does not register the # split variant entries. SSIM cannot catch a monolith run (OpenRA scores 1.000 either way); the # guards are the nm check for MG_Remote, run_trace_case.cmake's transport assertions (and, on # OpenRA, its wire-volume assertion), and the per-leg check of the evidence file they leave. retrace-split: name: retrace split (${{ matrix.backend }} / ${{ matrix.transport }}, shard ${{ matrix.shard }}/${{ matrix.shards }}) runs-on: ubuntu-26.04 timeout-minutes: 240 # The setup action selects lavapipe on GitHub-hosted runners. needs: - build-guard - build-linux-split - build-linux - build-retrace - trace-cases - trace-fixtures if: ${{ !cancelled() && needs.build-linux-split.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' && (needs.build-guard.outputs.verdict != 'identical' || needs.build-linux.result == 'success') }} env: SPLIT_REUSE: ${{ needs.build-guard.outputs.verdict == 'identical' && 'true' || 'false' }} SPLIT_LIB: ${{ needs.build-guard.outputs.verdict == 'identical' && 'mobilegl-linux-lib' || 'mobilegl-linux-lib-split' }} SPLIT_LIB_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }} SPLIT_BUILD_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }} SHARD_NAMES: ${{ toJSON(matrix.names) }} strategy: fail-fast: false # With build-linux reused, the inproc rows (except OpenRA's) are replayed by the retrace job's # inproc-acceptance rows on the same library, strict and with the dual block and run-ahead # on; the spawn rows below prove per case that those are also the defaults they run with. # A shard holds one transport, so OpenRA's inproc rows are shards of their own. matrix: ${{ fromJSON(needs.build-guard.outputs.verdict == 'identical' && needs.trace-cases.outputs.split-spawn-shards || needs.trace-cases.outputs.split-shards) }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Set swap space if: runner.environment != 'self-hosted' uses: ./.github/actions/set-swap-space with: swap-size-gb: 16 # build-linux's build when build-guard proved the two configurations identical, # build-linux-split's otherwise; integration-split prints the guard's evidence. A build-linux # that is not Release is never reused (scripts/ci/split_reuse_build_type.sh). - name: Split runtime env: GUARD_VERDICT: ${{ needs.build-guard.outputs.verdict }} PRIMARY_BUILD_TYPE: ${{ needs.build-linux.outputs.build-type }} run: | echo "split runtime: ${SPLIT_BUILD_DIR} (build-guard verdict '${GUARD_VERDICT:-none}')" bash scripts/ci/split_reuse_build_type.sh - name: Configure runner environment uses: ./.github/actions/setup-test-runner with: gpu: "true" # A miss does not fail the job; the install step downloads what the bundle does not hold. - name: Restore trace fixtures uses: actions/cache/restore@v5 with: path: ${{ needs.trace-fixtures.outputs.path }} key: ${{ needs.trace-fixtures.outputs.key }} - name: Install trace fixtures timeout-minutes: 15 run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }} - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Install runtime dependencies timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers test -e /usr/lib/x86_64-linux-gnu/libEGL.so test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so - name: Download Linux split library uses: actions/download-artifact@v8 with: name: ${{ env.SPLIT_LIB }} path: . - name: Download trace replay uses: actions/download-artifact@v8 with: name: mobilegl-trace-replay path: . - name: Unpack split runtime (library under test) # build-retrace's CTestTestfile.cmake has the absolute path # /build-linux/libMobileGL.so frozen into every case, so the swap happens here # rather than through a variable, exactly as in retrace-verify. `nm`, not `nm -D`. run: | tar -xzf "${SPLIT_LIB}.tgz" tar -xzf mobilegl-trace-replay.tgz test -f "${SPLIT_LIB_DIR}/libMobileGL.so" test -f build-retrace/tools/trace_replay/mobilegl_trace_replay if [ "${SPLIT_LIB_DIR}" != build-linux ]; then mkdir -p build-linux cp "${SPLIT_LIB_DIR}/libMobileGL.so" build-linux/libMobileGL.so fi if ! nm --defined-only build-linux/libMobileGL.so | grep -q -i MG_Remote; then echo "::error::the library unpacked at build-linux/libMobileGL.so defines no MG_Remote symbol, so this retrace would replay against a monolith build, ignore MOBILEGL_TRANSPORT entirely and match its golden having split nothing" exit 1 fi echo "the library at build-linux/libMobileGL.so is the split build" - name: Prepare downloaded test metadata run: | for root in build-linux build-verify build-split build-verify-split build-retrace; do if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi done # The spawn arm needs a second binary and the inproc arm needs none: inproc's server role # is a thread in the replay process, spawn's is a process the client launches from this # image. It ships in the split retrace library (libMobileGLServer.so is an executable # with a .so name, so that an APK installs it somewhere it can be executed), and a missing # one must be a named failure here rather than a LaunchServer refusal in every case. - name: Place the server image (spawn arm only) if: ${{ matrix.transport == 'spawn' }} run: | if [ ! -f "${SPLIT_LIB_DIR}/libMobileGLServer.so" ]; then echo "::error::the split runtime carries no libMobileGLServer.so, so transport=spawn has no server to launch. ConfigLoader would resolve Spawn, LaunchServer would refuse BY NAME, and this job would red on a message about an unresolvable image rather than about a packaging gap." exit 1 fi if [ "${SPLIT_LIB_DIR}" != build-linux ]; then cp "${SPLIT_LIB_DIR}/libMobileGLServer.so" build-linux/libMobileGLServer.so fi chmod +x build-linux/libMobileGLServer.so echo "server image: $(readlink -f build-linux/libMobileGLServer.so)" # The per-case settings are retrace_shard.py's "split" profile: # - the Magma knobs the retrace job's monolith rows run with (the R11F_G11F_B10F fallback, # the blended depth-write quirk on its one case, the iterationRP repairs on theirs), so a # split row and its monolith control differ in the transport alone; # - MOBILEGL_TRANSPORT and the absolute server image path; # - on OpenRA, the wire-volume counters and the check that draw records crossed the wire; # - with build-linux reused, the IPC defaults the client must log (strict=0). # run_trace_case.cmake turns MOBILEGL_TRANSPORT into assertions of its own: the library must # have resolved the transport in this process and its log must carry no Fatal{, so a case # that somehow ran the wrong library reds here instead of passing on its golden. After # ctest the evidence file those assertions leave is checked for every case, and the # refusal census is printed from the library's own logs of both roles, which is the only # valid source: the console sink is compiled out of this configuration. - name: Retrace and validate (${{ matrix.transport }}) run: | reuse=() if [ "${SPLIT_REUSE}" = "true" ]; then reuse=(--split-reuse); fi python3 scripts/ci/retrace_shard.py run --profile split \ --backend '${{ matrix.backend }}' --transport '${{ matrix.transport }}' --lane split \ --shard '${{ matrix.shard }}/${{ matrix.shards }}' \ --server-path "${GITHUB_WORKSPACE}/build-linux/libMobileGLServer.so" \ --build-dir build-retrace/tools/trace_replay \ --result-dir "${RUNNER_TEMP}/retrace-shard" "${reuse[@]}" - name: Annotate failed entries if: failure() run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit" - name: Upload core dumps if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v7 with: name: retrace-split-core-dumps-${{ matrix.transport }}-${{ matrix.backend }}-shard${{ matrix.shard }} path: /tmp/core.* if-no-files-found: ignore # Nothing reads a green shard's output, so only a failed shard uploads it. - name: Upload failed shard results if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v7 with: name: retrace-split-failed-${{ matrix.transport }}-${{ matrix.backend }}-shard${{ matrix.shard }} path: | ${{ runner.temp }}/retrace-shard/results ${{ runner.temp }}/retrace-shard/junit ${{ runner.temp }}/retrace-shard/logs if-no-files-found: warn overwrite: true # The shipped library, replayed. build-linux-monolith-control's library has no MG_Remote, the # shape FCL embeds; on the disaggregation branches nothing else replays that shape, so it runs # there only (elsewhere the retrace job's library is that shape). One short case on both # backends, with no transport set, must match its golden like any monolith row. retrace-shipped: name: retrace shipped library (${{ matrix.backend }}, ${{ matrix.case }}) runs-on: ubuntu-26.04 timeout-minutes: 60 needs: - build-linux-monolith-control - build-retrace - trace-cases if: ${{ !cancelled() && (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && needs.build-linux-monolith-control.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }} strategy: fail-fast: false matrix: backend: [DirectGLES, DirectVulkan] case: [OpenRA] steps: - name: Checkout repo uses: actions/checkout@v6 - name: Set swap space if: runner.environment != 'self-hosted' uses: ./.github/actions/set-swap-space with: swap-size-gb: 16 - name: Configure runner environment uses: ./.github/actions/setup-test-runner with: gpu: "true" # This job's one case keeps its fixtures in Git itself, not in Git LFS, so the checkout # has them and the fixture bundle is not restored. The check is that they are real files; # a case with Git LFS fixtures would be downloaded here, with a warning annotation. - name: Check trace fixture run: bash .github/scripts/trace-fixture-bundle.sh install '${{ matrix.case }}' - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Install runtime dependencies timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' run: | sudo apt-get update sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers test -e /usr/lib/x86_64-linux-gnu/libEGL.so test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so - name: Download shipped library uses: actions/download-artifact@v8 with: name: mobilegl-linux-monolith-control path: . - name: Download trace replay uses: actions/download-artifact@v8 with: name: mobilegl-trace-replay path: . # build-retrace froze /build-linux/libMobileGL.so into every case, so the library # is put where that path points, as retrace-verify does. `nm`, not `nm -D`: the MG_Remote # symbols are hidden in a Release build and only the static table lists them. - name: Unpack shipped library (library under test) run: | tar -xzf mobilegl-linux-monolith-control.tgz tar -xzf mobilegl-trace-replay.tgz test -f build-push-control/libMobileGL.so test -f build-retrace/tools/trace_replay/mobilegl_trace_replay mkdir -p build-linux cp build-push-control/libMobileGL.so build-linux/libMobileGL.so defined=$(nm --defined-only build-linux/libMobileGL.so | wc -l) remote=$(nm --defined-only build-linux/libMobileGL.so | grep -c -i MG_Remote || true) if [ "${defined}" -lt 1000 ] || [ "${remote}" -ne 0 ]; then echo "::error::build-linux/libMobileGL.so defines ${defined} symbols and ${remote} MG_Remote symbol(s): it is not the shipped shape (no MG_Remote, not stripped)" exit 1 fi echo "the library at build-linux/libMobileGL.so is the shipped shape (${defined} defined symbols, no MG_Remote)" - name: Prepare downloaded test metadata run: python3 scripts/ci/prepare_test_runtime.py build-retrace - name: Retrace and validate working-directory: build-retrace/tools/trace_replay run: | ulimit -c unlimited if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1 fi ctest -V --no-tests=error -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$' \ --output-junit "${RUNNER_TEMP}/retrace.xml" - name: Annotate failed entries if: failure() run: python3 scripts/ci/junit_annotate.py --excerpts 1 "${RUNNER_TEMP}" - name: Upload core dumps if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v7 with: name: retrace-shipped-core-dumps-${{ matrix.backend }}-${{ matrix.case }} path: /tmp/core.* if-no-files-found: ignore # Nothing reads a green leg's output, so only a failed leg uploads it. - name: Upload actual image if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v7 with: name: retrace-shipped-result-${{ matrix.backend }}-${{ matrix.case }} path: | build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/** build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/** if-no-files-found: warn pipe-gates: name: MGPipe generators and hygiene gates runs-on: ubuntu-26.04 # Deliberately independent of build-linux: these are source-level gates, they take # seconds, and a broken build must not hide a drifted interface. steps: - name: Checkout repo uses: actions/checkout@v6 # The seven generators all read MG_Pipe/*.def, so regenerating and diffing is what # keeps the two interface tables, the wire records, the verify comparators, the # PipeInputs field ids, the read-inventory coverage and the render-state member list # from drifting apart from the catalog. The generated files are committed # deliberately: the build must not depend on python. - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Test CTest runtime metadata preparation run: python3 scripts/ci/test_prepare_test_runtime.py - name: Regenerate the MGPipe interface (G1-G7) run: | python3 scripts/gen_pipe.py git diff --exit-code -- MobileGL/MG_Pipe/generated # The generators' own negative controls: canned inputs that MUST trip each structural check # (a field list that does not cover its struct's members, a verb set that is not the function # table's). Regenerating and diffing above cannot see a check that silently stopped # checking - a broken gate and a clean tree produce the same green. - name: MGPipe generator self-test run: python3 scripts/gen_pipe.py --self-test # The link seam owns physical storage and keeps link dispatch out of callers. - name: Link-seam purity check run: python3 scripts/ci/link_seam_purity.py --self-test # The same shape for Espryt's re-keyed memos (P3b/P4b, PLAN-PH-P34B-P7.md 1.2). P4a/P5f # moved the resolved-binding, sampler-pass, image-sweep, twin-registry, unit-sampler and # twin-lookup memos onto {slot, gen} handles and applier serials; the legacy pointer-keyed # arms stayed because they are the PULL build's only arms and G1 pins that build's .text. # Nothing checked that a NEW frontend-pointer key could not appear outside them. # # P13 W3b: NO EXEMPTED ARM ANY MORE. The legacy pointer-keyed arms (MOBILEGL_PIPE_LEGACY_MEMOS) # retired with the pull build, and the !MOBILEGL_BUILD_DISAGGREGATED arms are compiled into # the library FCL embeds, so they get the same rule: no frontend-pointer key in any memo # family, in any build. # # --self-test runs the negative controls and then the real scan in one invocation, as its # sibling does; the scan also asserts that every memo family's anchor still matches and # that every allow-list entry still suppresses something. - name: Espryt memo-key purity check run: python3 scripts/ci/espryt_memo_purity.py --self-test # P13 W3b: the pull build's switches stay retired - a preprocessor conditional on # MOBILEGL_PIPE_PUSH / MOBILEGL_PIPE_LEGACY_MEMOS, or an MGB_CTX spelling, coming back is a # red here. --self-test plants each shape first and requires the gate to catch it. - name: Retired pull-build switches stay retired (P13) run: python3 scripts/ci/retired_switches.py --self-test # The same question for the symbol tool the P1 gate is written in terms of. - name: Symbol report self-test run: python3 scripts/symbol_report.py --self-test # Per-draw fprintf/printf instrumentation has repeatedly been committed by accident, # once inside a mutex critical section. Nothing under these two trees prints to a # stdio stream today - MGLOG_D compiles out in INFO builds and is the only channel # they are allowed to use - so this gate starts with no exceptions, and any addition # to it needs a reason in the pull request rather than a quiet whitelist entry. The # alternation names every stdio spelling, not just the two that were committed: # fprintf to either stream, printf, puts, and the iostream pair. - name: No stdio in MG_Backend or MG_State run: | if grep -rnE 'fprintf[[:space:]]*\((stderr|stdout)|(^|[^[:alnum:]_>.])printf[[:space:]]*\(|(^|[^[:alnum:]_>.:])puts[[:space:]]*\(|std::(cout|cerr)' \ MobileGL/MG_Backend MobileGL/MG_State; then echo "::error::stdio instrumentation found; use MGLOG_D (compiled out in INFO builds)" exit 1 fi echo "no fprintf(stderr/stdout / printf( / puts( / std::cout|cerr under MobileGL/MG_Backend or MobileGL/MG_State" # Dirty-surface completeness (ARCHITECTURE.md §5.2.1). --check fails in both directions: a # mutator the scanner finds with no row in MG_Pipe/DirtySurface.def, and a row naming a # mutator the scan no longer finds. --self-test feeds canned negative controls and fails if # one does not trip, so a check that stopped checking cannot stay green. The scanner matches # text; the semantic proof is the verify lane. - name: Dirty-surface mapping complete (G9) run: | python3 scripts/gen_pipe_dirty_surface.py --check python3 scripts/gen_pipe_dirty_surface.py --self-test # PipeFieldOwnership.inc is written by gen_pipe_field_ownership.py, not gen_pipe.py, so the # gen_pipe.py regenerate-and-diff step above does not cover it. The build's static_assert # catches only an unclassified row; --check also catches a changed class. No branch guard: # every branch needs the committed table to follow from the .def files. Each --self-test # control asserts its own message, and a harness control checks that one control's exit is # never credited to another. - name: Field-ownership table complete (R-7.1) run: | python3 scripts/gen_pipe_field_ownership.py --check python3 scripts/gen_pipe_field_ownership.py --self-test # Negative controls must themselves be able to go red (ARCHITECTURE.md §13.2). The Python # tests run the trace and TCP lane tooling on stubs and synthetic data; test_run_trace_case.py # feeds run_trace_case.cmake canned logs. No branch guard; no build, GPU or device. - name: Control and trace-tool smoke tests (R-16) run: | python3 scripts/ci/test_trace_infrastructure.py python3 scripts/ci/tcp_lane_tools_test.py python3 tools/trace_replay/test_run_tcp_matrix.py python3 tools/trace_replay/test_compare_actuals.py python3 tools/trace_replay/test_run_trace_case.py python3 scripts/ci/build_fingerprint.py --self-test bash scripts/ci/split_reuse_build_type.sh --self-test bash scripts/ci/test_set_swap_space.sh python3 MobileGL/MG_IntegrationTest/Harness/test_split_log_paths.py python3 scripts/ci/transport_pinned.py --self-test # The helpers the build and split-lane jobs run before anything else can fail for them: the # union of the split arms' selections, and the submodule lists, which are printed here # without checking anything out. - name: Build and lane helper self-tests run: | python3 scripts/ci/lane_selection.py --self-test bash scripts/ci/checkout_submodules.sh --list library bash scripts/ci/checkout_submodules.sh --list retrace # Warning only for now: the disaggregation documents are still being written, and a # lint that fails a rewrite in progress teaches people to ignore it. It becomes # --strict when the documents settle. - name: Documentation citation lint run: | shopt -s nullglob documents=(docs/Disaggregated/*.md docs/Disaggregated/design/*.md docs/Disaggregated/guide/*.md) if [ ${#documents[@]} -eq 0 ]; then echo "no disaggregation documents to check" exit 0 fi python3 scripts/check_doc_citations.py "${documents[@]}" || true build-linux-verify-split: runs-on: ubuntu-26.04 timeout-minutes: 180 permissions: actions: write contents: read env: BUILD_DIR: build-verify-split CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_COMPRESS: "true" CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 4G CCACHE_NOHASHDIR: "true" steps: - name: Checkout repo uses: actions/checkout@v6 # For build-linux-split's reason: MOBILEGL_BUILD_DISAGGREGATED shadows itself back to OFF # when 3rdparty/flatbuffers/include is missing, and the script fails when it is. - name: Check out build submodules timeout-minutes: 15 run: bash scripts/ci/checkout_submodules.sh library - name: Set swap space uses: ./.github/actions/set-swap-space with: swap-size-gb: 32 - name: Configure runner environment uses: ./.github/actions/setup-test-runner - name: Get CMake timeout-minutes: 15 uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - name: Restore ccache uses: actions/cache/restore@v5 with: path: .ccache key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 restore-keys: | ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache- - name: Prepare Vulkan SDK id: netsetup3726 continue-on-error: true timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Prepare Vulkan SDK (retry) timeout-minutes: 15 if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup3726.outcome == 'failure' }} uses: humbletim/setup-vulkan-sdk@v1.2.1 with: vulkan-query-version: 1.4.304.1 vulkan-components: Vulkan-Headers, Vulkan-Loader vulkan-use-cache: true - name: Update glslang external sources working-directory: 3rdparty/glslang run: python3 update_glslang_sources.py - name: Install build and runtime dependencies timeout-minutes: 15 if: env.MOBILEGL_CI_NATIVE_GPU != '1' run: | sudo apt-get update sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1 - name: Configure CMake # Release/INFO for build-linux-verify's reason, and INFO is load-bearing here for # build-linux-split's: ConfigLoader's resolved-transport line is logged at INFO and the # arm-proof step below reads it back out of every entry's client log. # ENABLE_PCH=OFF: ccache cannot cache a compile that uses glslang's precompiled header # (see scripts/ci/configure_build.sh). run: | cmake -S . -B "${BUILD_DIR}" -G Ninja \ -DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \ -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DENABLE_PCH=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=ON \ -DMOBILEGL_BUILD_BENCHMARK=OFF \ -DMOBILEGL_BUILD_INTEGRATION_TEST=ON \ -DMOBILEGL_ITEST_VK_ICD="${VK_ICD_FILENAMES}" \ -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \ -DMOBILEGL_BUILD_DISAGGREGATED=ON \ -DMOBILEGL_BUILD_DISAGGREGATED_INPROC=ON \ -DMOBILEGL_PIPE_VERIFY=ON \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 - name: Build run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)" # Both options, both directions of "did it take" - the two jobs this one is the product of # each check one of them, and this library has to pass both checks. - name: Comparator and MG_Remote symbols run: | test -f "${BUILD_DIR}/libMobileGL.so" defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l) if [ "${defined}" -lt 1000 ]; then echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the checks below could not have failed honestly" exit 1 fi if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeVerifyInputs"; then echo "::error::libMobileGL.so defines no MGPipeVerifyInputs: -DMOBILEGL_PIPE_VERIFY=ON did not take, and the verify-split lane would compare nothing" exit 1 fi remote=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -c -i "MG_Remote" || true) if [ "${remote}" -lt 1 ]; then echo "::error::libMobileGL.so defines no MG_Remote symbol: -DMOBILEGL_BUILD_DISAGGREGATED=ON did not take, and every entry would run monolith under a name that says split" exit 1 fi echo "libMobileGL.so defines MGPipeVerifyInputs and ${remote} MG_Remote symbol(s) (${defined} defined symbols)" - name: Show ccache stats if: ${{ always() }} run: ccache --show-stats - name: Release superseded ccache entry env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true - name: Save ccache continue-on-error: true uses: actions/cache/save@v5 with: path: .ccache key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 - name: Prepare packaged test metadata run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}" # The unit tests are built here, so this flavor still compiles and links them, but they are # not packaged: the one consumer, integration-verify-split, runs only the # integration-verify-split label, whose entries all live under MG_IntegrationTest. - name: Package verify-split runtime run: | mkdir -p ci-artifacts mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort) tar --exclude='*/CMakeFiles' --exclude='*.o' --exclude='*.a' --exclude='*.ninja*' \ --exclude='build.ninja' --exclude='cmake_install.cmake' \ -I 'zstd -T0' -cf ci-artifacts/mobilegl-linux-runtime-verify-split.tar.zst \ "${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" \ "${BUILD_DIR}/MobileGL/MG_IntegrationTest" "${SHARED_LIBS[@]}" # compression-level 0: the tarball is zstd already (see 'Upload Linux runtime'). - name: Upload verify-split runtime uses: actions/upload-artifact@v7 with: name: mobilegl-linux-runtime-verify-split path: ci-artifacts/mobilegl-linux-runtime-verify-split.tar.zst if-no-files-found: error compression-level: 0 # P13 W1 (PLAN-P13 §4.2): THE SKIP CENSUS. Deleting a compile-time macro before the GTEST_SKIP # or pull stub that guards on it turns tests "green by skipping"; a per-lane pass/fail count # cannot see that. Every gtest lane uploads its JUnit as junit-*; this job keys each case by # lane and name and holds the run against scripts/data/skip_census_baseline.json: a new skip # (pass -> skip included) or a vanished name is red, a skip that turned into a pass is a note. # A deliberate retirement edits the baseline in the same commit and names the cases. skip-census: name: skip census runs-on: ubuntu-26.04 needs: - build-guard - test - integration - integration-verify - integration-verify-split - integration-split - integration-split-strict if: ${{ !cancelled() }} steps: - name: Checkout repo uses: actions/checkout@v6 - name: Skip census self-test run: python3 scripts/ci/skip_census.py --self-test - name: Download lane JUnit uses: actions/download-artifact@v8 with: pattern: junit-* path: junit # The optional rows run on some paths only, and on the path this run took they are required: # without a reused build (any build-guard verdict but identical) integration-split runs its # unit lane and Transport A/B, and with one A/B still runs when the IPC defaults were not # proven. - name: Skip census env: GUARD_VERDICT: ${{ needs.build-guard.outputs.verdict }} SPLIT_DEFAULTS_ON: ${{ needs.integration-split.outputs.defaults-on }} run: | required=() if [ "${GUARD_VERDICT}" != identical ]; then required+=(--require junit-integration-split/unit-split) fi if [ "${GUARD_VERDICT}" != identical ] || [ "${SPLIT_DEFAULTS_ON}" != true ]; then required+=(--require junit-integration-split/arm-inproc --require junit-integration-split/arm-monolith) fi echo "build-guard verdict '${GUARD_VERDICT:-none}', split IPC defaults proven '${SPLIT_DEFAULTS_ON:-no}': ${required[*]:-no optional row required}" python3 scripts/ci/skip_census.py check junit \ --baseline scripts/data/skip_census_baseline.json --report skip-census.json "${required[@]}" - name: Upload census if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: skip-census path: skip-census.json if-no-files-found: warn