mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-11-21 16:52:19 +09:00
4052 lines
200 KiB
YAML
4052 lines
200 KiB
YAML
name: Test
|
|
run-name: ${{ (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main')) && format('Release validation ({0})', github.ref) || '' }}
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- dev
|
|
- Feat/Backend-Direct-GLES
|
|
- Feat/Backend-Direct-Vulkan
|
|
# TEMPORARY, remove before merging the MGPipe work into dev: the disaggregation
|
|
# branch runs the full lane on every push so a phase's landing is not gated on
|
|
# someone remembering to dispatch the workflow by hand.
|
|
- feat/disaggregated
|
|
- codex/fix-github-ci
|
|
- cleanup
|
|
workflow_dispatch:
|
|
|
|
env:
|
|
# Software-rendered CI submissions may exceed the interactive GPU hang budget.
|
|
MOBILEGL_GPU_HANG_BUDGET_MS: "0"
|
|
|
|
# ONE LIVE RUN PER BRANCH. The disaggregation branch pushes (often docs-only) commits minutes apart
|
|
# and each push fans out to about 30 replay jobs here and the emulator shards in apk.yml; with no group, runs
|
|
# sat queued for 10+ hours behind their own successors. A manual dispatch gets a group of its own,
|
|
# so a push never cancels a manually started run; dev queues rather
|
|
# than cancels. No paths-ignore here, unlike apk.yml: this workflow hosts the doc citation lint.
|
|
# The "hosted" key leaves behind runs queued for the retired self-hosted runner, which cannot drain.
|
|
concurrency:
|
|
group: ${{ github.workflow }}-hosted-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.ref }}
|
|
cancel-in-progress: ${{ github.event_name == 'push' && github.ref != 'refs/heads/dev' }}
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
jobs:
|
|
# Temporarily keep all CI on GitHub-hosted runners while WSL D3D12 readback hangs are investigated.
|
|
build-linux:
|
|
runs-on: ubuntu-26.04
|
|
permissions:
|
|
actions: write
|
|
contents: read
|
|
outputs:
|
|
link-ratchet: ${{ steps.ratchet.outcome }}
|
|
image: ${{ steps.toolchain.outputs.image }}
|
|
compiler: ${{ steps.toolchain.outputs.compiler }}
|
|
build-type: ${{ steps.configure.outputs.build-type }}
|
|
env:
|
|
BUILD_DIR: build-linux
|
|
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }}
|
|
CCACHE_BASEDIR: ${{ github.workspace }}
|
|
CCACHE_COMPRESS: "true"
|
|
CCACHE_DIR: ${{ github.workspace }}/.ccache
|
|
CCACHE_MAXSIZE: 4G
|
|
CCACHE_NOHASHDIR: "true"
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
# Only the submodules the build reads, shallow and in parallel; the script fails when one
|
|
# of them did not arrive (scripts/ci/checkout_submodules.sh).
|
|
- name: Check out build submodules
|
|
timeout-minutes: 15
|
|
run: bash scripts/ci/checkout_submodules.sh library
|
|
|
|
- name: Set swap space
|
|
uses: ./.github/actions/set-swap-space
|
|
with:
|
|
swap-size-gb: 32
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Restore ccache
|
|
uses: actions/cache/restore@v5
|
|
with:
|
|
path: .ccache
|
|
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
restore-keys: |
|
|
${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-
|
|
|
|
- name: Prepare Vulkan SDK
|
|
id: netsetup77
|
|
continue-on-error: true
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
uses: humbletim/setup-vulkan-sdk@v1.2.1
|
|
with:
|
|
vulkan-query-version: 1.4.304.1
|
|
vulkan-components: Vulkan-Headers, Vulkan-Loader
|
|
vulkan-use-cache: true
|
|
|
|
- name: Prepare Vulkan SDK (retry)
|
|
timeout-minutes: 15
|
|
if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup77.outcome == 'failure' }}
|
|
uses: humbletim/setup-vulkan-sdk@v1.2.1
|
|
with:
|
|
vulkan-query-version: 1.4.304.1
|
|
vulkan-components: Vulkan-Headers, Vulkan-Loader
|
|
vulkan-use-cache: true
|
|
|
|
- name: Update glslang external sources
|
|
working-directory: 3rdparty/glslang
|
|
run: python3 update_glslang_sources.py
|
|
|
|
- name: Install build dependencies
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
|
|
|
|
# The image and compiler are job outputs: integration-split prints them beside build-guard's.
|
|
- name: Show installed toolchain
|
|
id: toolchain
|
|
run: |
|
|
ccache --version
|
|
"${MOBILEGL_CI_CC}" --version
|
|
"${MOBILEGL_CI_CXX}" --version
|
|
"${MOBILEGL_CI_LLD}" --version
|
|
dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true
|
|
echo "image=${ImageOS:-} ${ImageVersion:-}" >> "${GITHUB_OUTPUT}"
|
|
echo "compiler=$("${MOBILEGL_CI_CXX}" --version | head -n 1)" >> "${GITHUB_OUTPUT}"
|
|
|
|
# The configure line lives in scripts/ci/configure_build.sh, shared with build-linux-split
|
|
# and build-guard. The build type is a job output: the split consumers reuse this build only
|
|
# when it is Release, the only type build-guard can find identical to the split line.
|
|
- name: Configure CMake
|
|
id: configure
|
|
env:
|
|
CI_BUILD_TYPE: ${{ secrets.ACTIONS_STEP_DEBUG == 'true' && 'Debug' || 'Release' }}
|
|
run: |
|
|
echo "build-type=${CI_BUILD_TYPE}" >> "${GITHUB_OUTPUT}"
|
|
bash scripts/ci/configure_build.sh primary "${BUILD_DIR}"
|
|
|
|
- name: Build
|
|
run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)"
|
|
|
|
- name: Primary runtime mode proof
|
|
run: |
|
|
python3 scripts/ci/runtime_mode_proof.py self-test
|
|
mode=monolith
|
|
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then mode=disaggregated; fi
|
|
python3 scripts/ci/runtime_mode_proof.py build --build-dir "${BUILD_DIR}" \
|
|
--mode "${mode}" --output "${BUILD_DIR}/runtime-mode.json"
|
|
|
|
# build-linux-split's server link-closure ratchet, on this build, which the split consumers
|
|
# take when build-guard proves the two configurations identical. This job does not wait for
|
|
# that verdict, so it runs on every disaggregation build. continue-on-error keeps a ratchet
|
|
# red from skipping this job's consumers; integration-split reds on it when it takes this
|
|
# build. The artifact has its own name, so it never replaces build-linux-split's.
|
|
- name: Server link-closure ratchet
|
|
id: ratchet
|
|
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
|
|
continue-on-error: true
|
|
run: |
|
|
mkdir -p ci-artifacts
|
|
python3 scripts/link_ratchet.py --self-test
|
|
python3 scripts/link_ratchet.py \
|
|
--build-dir "${BUILD_DIR}" \
|
|
--baseline scripts/data/link_ratchet_baseline.txt \
|
|
--assert-monotone \
|
|
--bucket \
|
|
--json ci-artifacts/link-ratchet-primary.json
|
|
|
|
- name: Upload link ratchet
|
|
if: ${{ always() && env.MOBILEGL_CI_DISAGGREGATED == 'ON' }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: link-ratchet-primary
|
|
path: ci-artifacts/link-ratchet-primary.json
|
|
if-no-files-found: warn
|
|
|
|
- name: Show ccache stats
|
|
if: ${{ always() }}
|
|
run: ccache --show-stats
|
|
|
|
# Rewrite one rolling entry per job and branch. The upload stays cumulative - it
|
|
# carries every object restored at the top of this run plus the few TUs that
|
|
# changed - but Actions cache keys are immutable, so this branch's superseded
|
|
# blob is released first; --ref keeps the delete off other branches' entries.
|
|
# Running after the build means a failed build leaves the entry untouched.
|
|
- name: Release superseded ccache entry
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true
|
|
|
|
- name: Save ccache
|
|
continue-on-error: true
|
|
uses: actions/cache/save@v5
|
|
with:
|
|
path: .ccache
|
|
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
|
|
- name: Prepare packaged test metadata
|
|
run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}"
|
|
|
|
# zstd on every core: the tarball is most of a gigabyte of test executables, and every
|
|
# consumer job waits for it. `tar --zstd -xf` reads the result unchanged.
|
|
- name: Package Linux runtime
|
|
run: |
|
|
mkdir -p ci-artifacts
|
|
mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort)
|
|
tar \
|
|
--exclude='*/CMakeFiles' \
|
|
--exclude='*.o' \
|
|
--exclude='*.a' \
|
|
--exclude='*.ninja*' \
|
|
--exclude='build.ninja' \
|
|
--exclude='cmake_install.cmake' \
|
|
-I 'zstd -T0' -cf ci-artifacts/mobilegl-linux-runtime.tar.zst \
|
|
"${BUILD_DIR}/runtime-mode.json" \
|
|
"${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" \
|
|
"${BUILD_DIR}/MobileGL/MG_Test" \
|
|
"${BUILD_DIR}/MobileGL/MG_Benchmark" \
|
|
"${BUILD_DIR}/MobileGL/MG_IntegrationTest" \
|
|
"${SHARED_LIBS[@]}"
|
|
|
|
# compression-level 0: the tarball is zstd already, and deflating it again in the upload
|
|
# only costs time. The artifact stays a zip holding this one file, as the consumers expect.
|
|
- name: Upload Linux runtime
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: mobilegl-linux-runtime
|
|
path: ci-artifacts/mobilegl-linux-runtime.tar.zst
|
|
if-no-files-found: error
|
|
compression-level: 0
|
|
|
|
# The retrace legs load only the library, and the server image on the spawn arm, so they
|
|
# take this instead of the whole test runtime.
|
|
- name: Package retrace library
|
|
run: |
|
|
files=("${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/libMobileGL.so")
|
|
if [ -f "${BUILD_DIR}/libMobileGLServer.so" ]; then files+=("${BUILD_DIR}/libMobileGLServer.so"); fi
|
|
tar -czf ci-artifacts/mobilegl-linux-lib.tgz "${files[@]}"
|
|
|
|
- name: Upload retrace library
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: mobilegl-linux-lib
|
|
path: ci-artifacts/mobilegl-linux-lib.tgz
|
|
if-no-files-found: error
|
|
|
|
# The shipped shape: a library with no MG_Remote, which FCL embeds. On the disaggregation
|
|
# branches the primary artifact carries the transport, so this is the only build of that shape
|
|
# there; retrace-shipped replays it.
|
|
build-linux-monolith-control:
|
|
name: build push monolith control (no MG_Remote)
|
|
runs-on: ubuntu-26.04
|
|
# actions: write is for this job's own rolling ccache entry, as in build-linux.
|
|
permissions:
|
|
actions: write
|
|
contents: read
|
|
env:
|
|
BUILD_DIR: build-push-control
|
|
CCACHE_BASEDIR: ${{ github.workspace }}
|
|
CCACHE_DIR: ${{ github.workspace }}/.ccache
|
|
CCACHE_COMPRESS: "true"
|
|
CCACHE_MAXSIZE: 4G
|
|
CCACHE_NOHASHDIR: "true"
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Check out build submodules
|
|
timeout-minutes: 15
|
|
run: bash scripts/ci/checkout_submodules.sh library
|
|
|
|
- name: Set swap space
|
|
uses: ./.github/actions/set-swap-space
|
|
with:
|
|
swap-size-gb: 32
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
# This job's own entry: its objects are compiled without MG_Remote, which no other build
|
|
# produces. The second restore key is build-linux's entry, which seeds the third-party
|
|
# objects on a branch's first run.
|
|
- name: Restore ccache
|
|
uses: actions/cache/restore@v5
|
|
with:
|
|
path: .ccache
|
|
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
restore-keys: |
|
|
${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-
|
|
${{ runner.os }}-ubuntu26-cpu-test-build-linux-ccache-
|
|
- name: Prepare Vulkan SDK
|
|
id: netsetup230
|
|
continue-on-error: true
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
uses: humbletim/setup-vulkan-sdk@v1.2.1
|
|
with:
|
|
vulkan-query-version: 1.4.304.1
|
|
vulkan-components: Vulkan-Headers, Vulkan-Loader
|
|
vulkan-use-cache: true
|
|
|
|
- name: Prepare Vulkan SDK (retry)
|
|
timeout-minutes: 15
|
|
if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup230.outcome == 'failure' }}
|
|
uses: humbletim/setup-vulkan-sdk@v1.2.1
|
|
with:
|
|
vulkan-query-version: 1.4.304.1
|
|
vulkan-components: Vulkan-Headers, Vulkan-Loader
|
|
vulkan-use-cache: true
|
|
- name: Update glslang external sources
|
|
working-directory: 3rdparty/glslang
|
|
run: python3 update_glslang_sources.py
|
|
- name: Install build dependencies
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
|
|
# The push monolith without MG_Remote, the shape FCL embeds. Its runtime_mode_proof is the
|
|
# zero-MG_Remote-symbol check: a push library that links any MG_Remote symbol is red here.
|
|
# Its library is uploaded for retrace-shipped, a positive replay of that shape.
|
|
# ENABLE_PCH=OFF: ccache cannot cache a compile that uses glslang's precompiled header
|
|
# (see scripts/ci/configure_build.sh).
|
|
- name: Configure push control without MG_Remote
|
|
run: |
|
|
cmake -S . -B "${BUILD_DIR}" -G Ninja \
|
|
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \
|
|
-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
|
|
-DENABLE_PCH=OFF \
|
|
-DCMAKE_BUILD_TYPE=Release -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
|
|
-DMOBILEGL_BUILD_TEST=OFF -DMOBILEGL_BUILD_BENCHMARK=OFF \
|
|
-DMOBILEGL_BUILD_INTEGRATION_TEST=OFF -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \
|
|
-DMOBILEGL_BUILD_DISAGGREGATED=OFF -DMOBILEGL_BUILD_DISAGGREGATED_INPROC=OFF \
|
|
-DMOBILEGL_PIPE_VERIFY=OFF \
|
|
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5
|
|
- name: Build push control and prove it has zero MG_Remote symbols
|
|
run: |
|
|
cmake --build "${BUILD_DIR}" --target MobileGL --parallel "$(nproc)"
|
|
python3 scripts/ci/runtime_mode_proof.py build --build-dir "${BUILD_DIR}" \
|
|
--mode monolith --output "${BUILD_DIR}/runtime-mode.json"
|
|
|
|
- name: Show ccache stats
|
|
if: ${{ always() }}
|
|
run: ccache --show-stats
|
|
|
|
# One rolling entry per job and branch, as in build-linux: this branch's superseded blob is
|
|
# released before the save, and a failed build leaves the entry untouched.
|
|
- name: Release superseded ccache entry
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true
|
|
|
|
- name: Save ccache
|
|
continue-on-error: true
|
|
uses: actions/cache/save@v5
|
|
with:
|
|
path: .ccache
|
|
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
|
|
- name: Prepare packaged test metadata
|
|
run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}"
|
|
|
|
- name: Package push control
|
|
run: |
|
|
tar -czf mobilegl-linux-monolith-control.tgz \
|
|
"${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/libMobileGL.so" "${BUILD_DIR}/runtime-mode.json"
|
|
- name: Upload push control
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: mobilegl-linux-monolith-control
|
|
path: mobilegl-linux-monolith-control.tgz
|
|
if-no-files-found: error
|
|
|
|
test:
|
|
runs-on: ubuntu-26.04
|
|
needs: [build-linux]
|
|
|
|
# Every ctest entry is its own short-lived process, and none compiles enough to repay
|
|
# Initialize()'s glslang table prewarm. The retrace legs keep it: they replay real apps.
|
|
env:
|
|
MOBILEGL_DISABLE_SHADER_PREWARM: "1"
|
|
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }}
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Install runtime dependencies
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y libvulkan1 libegl1 libgles2 libgl1-mesa-dri mesa-vulkan-drivers ${{ env.MOBILEGL_CI_DISAGGREGATED == 'ON' && 'libegl-mesa0' || '' }}
|
|
|
|
- name: Download Linux runtime
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: mobilegl-linux-runtime
|
|
path: .
|
|
|
|
- name: Unpack Linux runtime
|
|
run: tar --zstd -xf mobilegl-linux-runtime.tar.zst
|
|
|
|
- name: Prepare downloaded test metadata
|
|
run: |
|
|
for root in build-linux build-verify build-split build-verify-split build-retrace; do
|
|
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
|
|
done
|
|
|
|
# On the three branches this is also the split runtime's unit run (integration-split skips
|
|
# its copy when build-linux-split reuses this build), so it keeps that run's hardening:
|
|
# REQUIRE_GPU makes the server loop's EGL cases fail rather than skip without a driver.
|
|
- name: Test
|
|
working-directory: build-linux
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: ${{ env.MOBILEGL_CI_DISAGGREGATED == 'ON' && '1' || '' }}
|
|
run: |
|
|
ulimit -c unlimited
|
|
if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
|
|
ctest -V -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit.xml"
|
|
else
|
|
ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit.xml"
|
|
fi
|
|
|
|
- name: Disk and cores after the unit run
|
|
if: ${{ always() }}
|
|
run: |
|
|
df -h / /tmp
|
|
echo "cores: $(ls /tmp/core.* 2> /dev/null | wc -l), $(du -ch /tmp/core.* 2> /dev/null | tail -n 1)"
|
|
|
|
# P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py).
|
|
- name: Upload JUnit for the skip census
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: junit-test
|
|
path: ${{ runner.temp }}/*.xml
|
|
if-no-files-found: warn
|
|
|
|
- name: Upload core dumps
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: unit-core-dumps
|
|
path: /tmp/core.*
|
|
if-no-files-found: ignore
|
|
|
|
integration:
|
|
name: integration (${{ matrix.transport }})
|
|
runs-on: ubuntu-26.04
|
|
needs: [build-linux]
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include: ${{ fromJSON((github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && '[{"lane":"monolith-control","transport":"monolith"},{"lane":"inproc-acceptance","transport":"inproc"}]' || '[{"lane":"monolith-control","transport":"monolith"}]') }}
|
|
env:
|
|
# Short-lived test processes skip the glslang prewarm, as in the test job.
|
|
MOBILEGL_DISABLE_SHADER_PREWARM: "1"
|
|
MOBILEGL_TRANSPORT: ${{ matrix.transport }}
|
|
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
|
|
MOBILEGL_IPC_STRICT_ERRORS: "1"
|
|
MOBILEGL_IPC_RUN_AHEAD: "1"
|
|
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }}
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
with:
|
|
gpu: "true"
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Install runtime dependencies
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
# Same set as the benchmark job, for the same reason: the scenarios bring
|
|
# up real headless EGL (llvmpipe) and Vulkan (lavapipe) contexts, and
|
|
# libegl-mesa0 - the EGL vendor library behind glvnd's libegl1 dispatch -
|
|
# only arrives as a Recommends.
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers
|
|
|
|
- name: Download Linux runtime
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: mobilegl-linux-runtime
|
|
path: .
|
|
|
|
- name: Unpack Linux runtime
|
|
run: tar --zstd -xf mobilegl-linux-runtime.tar.zst
|
|
|
|
- name: Prepare downloaded test metadata
|
|
run: |
|
|
for root in build-linux build-verify build-split build-verify-split build-retrace; do
|
|
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
|
|
done
|
|
|
|
- name: Integration scenarios
|
|
working-directory: build-linux
|
|
# REQUIRE_GPU makes a driverless runner FAIL instead of skipping every
|
|
# scenario - an all-skip run is otherwise indistinguishable from a pass,
|
|
# which is how a five-month-old draw-dropping bug survived unseen until
|
|
# this lane existed.
|
|
#
|
|
# Configure CMake with the selected runner's ICD; each test's ENVIRONMENT
|
|
# property takes precedence over ambient job variables.
|
|
#
|
|
# Cores are armed so that any crash - the harness pre-flight child's
|
|
# included - leaves /tmp/core.*.zst, which the failure-only step below ships
|
|
# as an artifact. Analyzing a downloaded core (zstd -d) against the runtime
|
|
# artifact's binary in an ubuntu-26.04 userspace reproduces the exact
|
|
# crash stack without burning a CI round on an in-workflow debugger.
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
MOBILEGL_ITRP_FIX: "ForceOn"
|
|
run: |
|
|
ulimit -c unlimited
|
|
# An entry whose own ENVIRONMENT sets MOBILEGL_TRANSPORT overrides the one variable the two
|
|
# legs differ in, so it runs identically in both; the inproc leg leaves those entries (and
|
|
# the fixtures only they need) to the monolith-control leg, which every branch runs, and
|
|
# that leg proves it ran every one of them. When transport_pinned.py cannot prove the
|
|
# trimmed selection lost exactly them, this leg runs every entry and writes its JUnits
|
|
# under -all stems, which the skip census holds to the full name lists.
|
|
EXCLUDE=()
|
|
MAIN_JUNIT=integration
|
|
STAGED_JUNIT=integration-staged-copy
|
|
if [ "${MOBILEGL_CI_DISAGGREGATED}" = ON ]; then
|
|
ctest -L integration-gpu --show-only=json-v1 > "${RUNNER_TEMP}/all.json"
|
|
fi
|
|
if [ "${MOBILEGL_TRANSPORT}" = inproc ]; then
|
|
if python3 ../scripts/ci/transport_pinned.py select "${RUNNER_TEMP}/all.json" --out "${RUNNER_TEMP}/transport-pinned.txt" \
|
|
&& ctest -L integration-gpu --exclude-from-file "${RUNNER_TEMP}/transport-pinned.txt" --show-only=json-v1 \
|
|
> "${RUNNER_TEMP}/trimmed.json" \
|
|
&& python3 ../scripts/ci/transport_pinned.py assert-exclusion "${RUNNER_TEMP}/all.json" \
|
|
"${RUNNER_TEMP}/trimmed.json" --out "${RUNNER_TEMP}/transport-pinned.json"; then
|
|
EXCLUDE=(--exclude-from-file "${RUNNER_TEMP}/transport-pinned.txt")
|
|
else
|
|
echo "::warning::the transport-pinned exclusion did not prove itself, so this leg runs every integration-gpu entry (integration-all.xml, integration-staged-copy-all.xml)"
|
|
MAIN_JUNIT=integration-all
|
|
STAGED_JUNIT=integration-staged-copy-all
|
|
fi
|
|
fi
|
|
# Second, filtered pass: with the range-invalidating map flush disabled,
|
|
# the buffer scenarios run on the upload ring's staged-copy tier - which
|
|
# the default pass never reaches (the map tier absorbs every flush on
|
|
# the CI drivers), so without this the Mali fallback tier would have zero CI
|
|
# coverage. The flag is NOT baked into the ctest ENVIRONMENT properties,
|
|
# so an inline env reaches the test processes (unlike the ICD pin above).
|
|
if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
|
|
ctest -V -L integration-gpu "${EXCLUDE[@]}" --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/${MAIN_JUNIT}.xml"
|
|
MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 ctest -V -L integration-gpu "${EXCLUDE[@]}" -j "$(nproc)" \
|
|
-R 'Buffer|Readback|Atomic|Ssbo|Arena' --no-tests=error --output-junit "${RUNNER_TEMP}/${STAGED_JUNIT}.xml"
|
|
else
|
|
ctest --output-on-failure -L integration-gpu "${EXCLUDE[@]}" --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/${MAIN_JUNIT}.xml"
|
|
MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 ctest --output-on-failure -L integration-gpu "${EXCLUDE[@]}" -j "$(nproc)" \
|
|
-R 'Buffer|Readback|Atomic|Ssbo|Arena' --no-tests=error --output-junit "${RUNNER_TEMP}/${STAGED_JUNIT}.xml"
|
|
fi
|
|
if [ "${MOBILEGL_TRANSPORT}" = monolith ] && [ "${MOBILEGL_CI_DISAGGREGATED}" = ON ]; then
|
|
python3 ../scripts/ci/transport_pinned.py require-ran "${RUNNER_TEMP}/all.json" "${RUNNER_TEMP}/${MAIN_JUNIT}.xml"
|
|
fi
|
|
|
|
# Both backend roles must execute inproc in the primary artifact.
|
|
- name: Inproc execution proof (both backends)
|
|
if: matrix.transport == 'inproc'
|
|
working-directory: build-linux
|
|
run: |
|
|
ctest -L integration-p5f-rsp --show-only=json-v1 > "${RUNNER_TEMP}/primary-inproc-lane.json"
|
|
ctest -L integration-p5f-rsp --no-tests=error --output-on-failure \
|
|
--output-junit "${RUNNER_TEMP}/primary-inproc.xml"
|
|
python3 ../scripts/ci/runtime_mode_proof.py logs \
|
|
--discovery "${RUNNER_TEMP}/primary-inproc-lane.json" \
|
|
--junit "${RUNNER_TEMP}/primary-inproc.xml" --output "${RUNNER_TEMP}/primary-inproc-proof.json"
|
|
|
|
# The run log needs a signed-in reader; annotations do not. Name every failed ctest entry,
|
|
# with the lines of its output that say why, where anyone reading the run can see them.
|
|
- name: Annotate failed entries
|
|
if: failure()
|
|
run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}"
|
|
|
|
- name: Upload integration lane evidence
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: integration-evidence-${{ matrix.lane }}-${{ matrix.transport }}
|
|
path: |
|
|
${{ runner.temp }}/integration*.xml
|
|
${{ runner.temp }}/primary-inproc*
|
|
${{ runner.temp }}/transport-pinned*
|
|
${{ runner.temp }}/all.json
|
|
${{ runner.temp }}/trimmed.json
|
|
build-linux/runtime-mode.json
|
|
build-linux/MobileGL/MG_IntegrationTest/p5f-rsp-*.log
|
|
if-no-files-found: warn
|
|
|
|
# P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py).
|
|
- name: Upload JUnit for the skip census
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: junit-integration-${{ matrix.lane }}-${{ matrix.transport }}
|
|
path: ${{ runner.temp }}/*.xml
|
|
if-no-files-found: warn
|
|
|
|
- name: Upload core dumps
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: integration-core-dumps-${{ matrix.lane }}-${{ matrix.transport }}
|
|
path: /tmp/core.*
|
|
if-no-files-found: ignore
|
|
|
|
# THE THIRD CI MODE (ARCHITECTURE.md 13.2-(2)): the same library, built with the PipeInputs
|
|
# comparator compiled in, running the integration suite and a trace subset with two state models
|
|
# in one address space. It is a second build rather than a flag on the first because
|
|
# MOBILEGL_PIPE_VERIFY is a compile-time option - the snapshot, the entry compare and the
|
|
# compare-at-read hook do not exist in the shipped library, and are never meant to.
|
|
build-linux-verify:
|
|
runs-on: ubuntu-26.04
|
|
timeout-minutes: 120
|
|
permissions:
|
|
actions: write
|
|
contents: read
|
|
env:
|
|
BUILD_DIR: build-verify
|
|
CCACHE_BASEDIR: ${{ github.workspace }}
|
|
CCACHE_COMPRESS: "true"
|
|
CCACHE_DIR: ${{ github.workspace }}/.ccache
|
|
CCACHE_MAXSIZE: 4G
|
|
CCACHE_NOHASHDIR: "true"
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Check out build submodules
|
|
timeout-minutes: 15
|
|
run: bash scripts/ci/checkout_submodules.sh library
|
|
|
|
- name: Set swap space
|
|
uses: ./.github/actions/set-swap-space
|
|
with:
|
|
swap-size-gb: 32
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Restore ccache
|
|
uses: actions/cache/restore@v5
|
|
with:
|
|
path: .ccache
|
|
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
restore-keys: |
|
|
${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-
|
|
|
|
- name: Prepare Vulkan SDK
|
|
id: netsetup530
|
|
continue-on-error: true
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
uses: humbletim/setup-vulkan-sdk@v1.2.1
|
|
with:
|
|
vulkan-query-version: 1.4.304.1
|
|
vulkan-components: Vulkan-Headers, Vulkan-Loader
|
|
vulkan-use-cache: true
|
|
|
|
- name: Prepare Vulkan SDK (retry)
|
|
timeout-minutes: 15
|
|
if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup530.outcome == 'failure' }}
|
|
uses: humbletim/setup-vulkan-sdk@v1.2.1
|
|
with:
|
|
vulkan-query-version: 1.4.304.1
|
|
vulkan-components: Vulkan-Headers, Vulkan-Loader
|
|
vulkan-use-cache: true
|
|
|
|
- name: Update glslang external sources
|
|
working-directory: 3rdparty/glslang
|
|
run: python3 update_glslang_sources.py
|
|
|
|
- name: Install build dependencies
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
|
|
|
|
- name: Show installed toolchain
|
|
run: |
|
|
ccache --version
|
|
"${MOBILEGL_CI_CC}" --version
|
|
"${MOBILEGL_CI_CXX}" --version
|
|
"${MOBILEGL_CI_LLD}" --version
|
|
dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true
|
|
|
|
- name: Configure CMake
|
|
# Release/INFO like the shipped build on purpose. The poison arms in this configuration
|
|
# through MOBILEGL_PIPE_VERIFY (PipeInputs.h derives MOBILEGL_PIPE_POISON from it), so this
|
|
# job needs neither a Debug log level nor MOBILEGL_BUILD_DISAGGREGATED - and a Debug build
|
|
# would compare a different library from the one the other lanes measure. (build-linux
|
|
# switches to Debug under ACTIONS_STEP_DEBUG; this job deliberately does not - a Debug
|
|
# build flips CXX_VISIBILITY_PRESET and arms MOBILEGL_PIPE_POISON through a second, unrelated
|
|
# arm of its #if, so the debug switch would change what the lane is measuring.)
|
|
# Both disaggregation options default ON, so they are pinned OFF here: a disaggregated
|
|
# library expects a server-role log that this monolith lane never writes.
|
|
# integration-verify-split is the split arm.
|
|
# ENABLE_PCH=OFF: ccache cannot cache a compile that uses glslang's precompiled header
|
|
# (see scripts/ci/configure_build.sh).
|
|
run: |
|
|
cmake -S . -B "${BUILD_DIR}" -G Ninja \
|
|
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \
|
|
-DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \
|
|
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
|
|
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
|
|
-DENABLE_PCH=OFF \
|
|
-DCMAKE_BUILD_TYPE=Release \
|
|
-DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
|
|
-DMOBILEGL_BUILD_TEST=ON \
|
|
-DMOBILEGL_BUILD_BENCHMARK=OFF \
|
|
-DMOBILEGL_BUILD_INTEGRATION_TEST=ON \
|
|
-DMOBILEGL_ITEST_VK_ICD="${VK_ICD_FILENAMES}" \
|
|
-DMOBILEGL_BUILD_TRACE_REPLAY=OFF \
|
|
-DMOBILEGL_PIPE_VERIFY=ON \
|
|
-DMOBILEGL_BUILD_DISAGGREGATED=OFF \
|
|
-DMOBILEGL_BUILD_DISAGGREGATED_INPROC=OFF \
|
|
-DCMAKE_POLICY_VERSION_MINIMUM=3.5
|
|
|
|
- name: Build
|
|
run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)"
|
|
|
|
# The lane is worthless if the option silently did not take, and that is a one-character
|
|
# mistake away at all times (a typo'd -D is not an error in CMake). Two checks, both cheap:
|
|
# the comparator's entry point must be in the library, and the fill entry point with it.
|
|
#
|
|
# `nm` and NOT `nm -D`. The library is built CXX_VISIBILITY_PRESET hidden in every non-Debug
|
|
# configuration (CMakeLists.txt:600-604) and the MGPipe entry points are plain namespace
|
|
# functions with no export attribute, so not one of them appears in the DYNAMIC table: on a
|
|
# perfectly healthy verify build `nm -D --defined-only ... | grep -c MGPipe` answers 0 out of
|
|
# ~11900 exported symbols, and a gate spelled that way is red forever for a reason that has
|
|
# nothing to do with what it claims to test. The static symbol table has them as local `t`
|
|
# entries, this artifact is never stripped, and `No MG_Remote in the pull build` below already
|
|
# uses this spelling. The symbol count guards the remaining hole: a stripped library would
|
|
# make both greps fail for a third, silent reason.
|
|
- name: Comparator symbols in verify library
|
|
run: |
|
|
test -f "${BUILD_DIR}/libMobileGL.so"
|
|
defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l)
|
|
if [ "${defined}" -lt 1000 ]; then
|
|
echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the two checks below could not have failed honestly"
|
|
exit 1
|
|
fi
|
|
if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeVerifyInputs"; then
|
|
echo "::error::libMobileGL.so defines no MGPipeVerifyInputs: -DMOBILEGL_PIPE_VERIFY=ON did not take, and every lane that consumes this artifact would run the comparator-free library and pass having compared nothing"
|
|
exit 1
|
|
fi
|
|
# The per-verb entry point, under EITHER of its two names. P2 renames
|
|
# MGPipeFillForVerb to MGPipeValidateForVerb (the body becomes the tracker's walk and
|
|
# the fill is one of its five steps), so this check has to accept both or it goes red on
|
|
# the rename for a reason that has nothing to do with what it tests. What it tests is
|
|
# unchanged: that the library HAS a per-verb entry point compiled in.
|
|
if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -qE "MGPipeValidateForVerb|MGPipeFillForVerb"; then
|
|
echo "::error::libMobileGL.so defines neither MGPipeValidateForVerb nor MGPipeFillForVerb: there is no per-verb entry point in this artifact, so nothing fills the block the comparator compares"
|
|
exit 1
|
|
fi
|
|
echo "libMobileGL.so defines MGPipeVerifyInputs and a per-verb entry point (${defined} defined symbols)"
|
|
|
|
- name: Show ccache stats
|
|
if: ${{ always() }}
|
|
run: ccache --show-stats
|
|
|
|
- name: Release superseded ccache entry
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true
|
|
|
|
- name: Save ccache
|
|
continue-on-error: true
|
|
uses: actions/cache/save@v5
|
|
with:
|
|
path: .ccache
|
|
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
|
|
- name: Prepare packaged test metadata
|
|
run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}"
|
|
|
|
- name: Package Linux verify runtime
|
|
run: |
|
|
mkdir -p ci-artifacts
|
|
mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort)
|
|
tar \
|
|
--exclude='*/CMakeFiles' \
|
|
--exclude='*.o' \
|
|
--exclude='*.a' \
|
|
--exclude='*.ninja*' \
|
|
--exclude='build.ninja' \
|
|
--exclude='cmake_install.cmake' \
|
|
-I 'zstd -T0' -cf ci-artifacts/mobilegl-linux-runtime-verify.tar.zst \
|
|
"${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" \
|
|
"${BUILD_DIR}/MobileGL/MG_Test" \
|
|
"${BUILD_DIR}/MobileGL/MG_IntegrationTest" \
|
|
"${SHARED_LIBS[@]}"
|
|
|
|
# compression-level 0: the tarball is zstd already (see 'Upload Linux runtime').
|
|
- name: Upload Linux verify runtime
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: mobilegl-linux-runtime-verify
|
|
path: ci-artifacts/mobilegl-linux-runtime-verify.tar.zst
|
|
if-no-files-found: error
|
|
compression-level: 0
|
|
|
|
# The retrace legs load only the library, and the server image on the spawn arm, so they
|
|
# take this instead of the whole test runtime.
|
|
- name: Package retrace library
|
|
run: |
|
|
files=("${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/libMobileGL.so")
|
|
if [ -f "${BUILD_DIR}/libMobileGLServer.so" ]; then files+=("${BUILD_DIR}/libMobileGLServer.so"); fi
|
|
tar -czf ci-artifacts/mobilegl-linux-lib-verify.tgz "${files[@]}"
|
|
|
|
- name: Upload retrace library
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: mobilegl-linux-lib-verify
|
|
path: ci-artifacts/mobilegl-linux-lib-verify.tgz
|
|
if-no-files-found: error
|
|
|
|
# The verify lane itself, plus the controls that keep it falsifiable: its own control entries,
|
|
# checked by name, and the comparator's unit cases on its library. They are always on, not a
|
|
# manual exercise: a gate that can only be shown to work by someone remembering to break it on
|
|
# purpose is a gate that has already stopped working.
|
|
integration-verify:
|
|
runs-on: ubuntu-26.04
|
|
timeout-minutes: 180
|
|
needs: [build-linux-verify]
|
|
|
|
# Short-lived test processes skip the glslang prewarm, as in the test job.
|
|
env:
|
|
MOBILEGL_DISABLE_SHADER_PREWARM: "1"
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
with:
|
|
gpu: "true"
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Install runtime dependencies
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers
|
|
|
|
- name: Download Linux verify runtime
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: mobilegl-linux-runtime-verify
|
|
path: .
|
|
|
|
- name: Unpack Linux verify runtime
|
|
run: |
|
|
tar --zstd -xf mobilegl-linux-runtime-verify.tar.zst
|
|
test -f build-verify/libMobileGL.so
|
|
|
|
- name: Prepare downloaded test metadata
|
|
run: |
|
|
for root in build-linux build-verify build-split build-verify-split build-retrace; do
|
|
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
|
|
done
|
|
|
|
- name: Integration scenarios (verify)
|
|
working-directory: build-verify
|
|
# --no-tests=error is half the gate: the verify entries only exist when the library was
|
|
# configured with -DMOBILEGL_PIPE_VERIFY=ON, so a build that lost the option matches no
|
|
# tests and reds here instead of reporting a green run of nothing. The other half is
|
|
# PipeVerifyArmingScenario.Armed, which fails when the library never printed its arming
|
|
# line - the failure mode a bare `MOBILEGL_PIPE_VERIFY=1` cannot detect by itself.
|
|
#
|
|
# SCOPE, stated so nobody reads more into a green than is there: this is every integration
|
|
# ENTRY under the comparator, not every integration CONFIGURATION. The `integration` job
|
|
# runs a second, filtered pass with MOBILEGL_ESPRYT_DISABLE_INVALIDATE_FLUSH=1 for the
|
|
# upload ring's staged-copy tier; that pass is 186 entries here and, at the 5-10x the
|
|
# comparator costs, is not affordable inside this job's budget. The tier is covered by
|
|
# `integration`, unverified, and P2 can take it once the comparator's cost is known.
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
MOBILEGL_ITRP_FIX: "ForceOn"
|
|
run: |
|
|
ulimit -c unlimited
|
|
if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
|
|
ctest -V -L integration-verify --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/verify.xml"
|
|
else
|
|
ctest --output-on-failure -L integration-verify --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/verify.xml"
|
|
fi
|
|
|
|
# The unit suite again, on the verify library: cases compiled only under MOBILEGL_PIPE_VERIFY
|
|
# skip in the `test` job's build and assert only here, as do poison-gated cases on branches
|
|
# where that build is a monolith. The packaging step above ships MG_Test whole, so the cost
|
|
# is the run itself. --no-tests=error, so a packaging change that dropped the unit binaries
|
|
# fails here instead of passing with nothing run. Among them is the comparator's per-field
|
|
# control, PipeInputsTest.EveryCorruptedFieldIsNamedByTheComparator.
|
|
- name: Unit tests on the verify runtime (G6, G10)
|
|
working-directory: build-verify
|
|
run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit-verify.xml"
|
|
|
|
# Handle-ABA, CSO, subsystem-mask and texture-parameter controls, rerun on the verify library.
|
|
# They carry integration-gpu, not integration-verify, so the verify lane misses them.
|
|
# Each -R alternative is a test-name substring, the shortest that selects only its family:
|
|
# `ResourceSubsystem` also reaches the LargeArenaAdoption lanes and the mask refusals,
|
|
# `MapPersistentRoundtrip` is singular so it matches the case name as well as the lane prefix,
|
|
# and `ObjectSubsystem` and `ResourceSubsystem` stay distinct families. TextureUploadShape
|
|
# records the upload shape; it fails only when its two counters are unreadable or disagree.
|
|
- name: Push-build controls (G8, G8b, G9, G10, G12)
|
|
working-directory: build-verify
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
MOBILEGL_ITRP_FIX: "ForceOn"
|
|
run: |
|
|
ulimit -c unlimited
|
|
ctest --output-on-failure -L integration-gpu \
|
|
-R 'HandleRecycle|CsoContentAddressing|ResourceSubsystem|MapPersistentRoundtrip|ObjectSubsystem|TextureParamsWithoutASamplerView|TextureUploadShape' \
|
|
--no-tests=error -j 4
|
|
|
|
# The lane's own control entries, by name: each must have run and passed, and a skip is a
|
|
# failure here although ctest reports it green. VerifyArming asserts in-test that the
|
|
# comparator printed its armed line with fatal=1, under the ambient entries' own environment
|
|
# (MG_IntegrationTest/CMakeLists.txt builds both from one list), VerifyCorrupted that a
|
|
# corrupted field is reported by name, and PoisonOmitted that an omitted stamp kills a
|
|
# forked child with its own Fatal.
|
|
- name: Verify lane control entries
|
|
run: |
|
|
python3 scripts/ci/junit_tally.py "${RUNNER_TEMP}/verify.xml" \
|
|
--require-entry-passed DirectGLES.VerifyArming.PipeVerifyArmingScenario.Armed \
|
|
--require-entry-passed DirectVulkan.VerifyArming.PipeVerifyArmingScenario.Armed \
|
|
--require-entry-passed DirectGLES.VerifyCorrupted.PipeVerifyArmingScenario.CorruptedFieldIsReported \
|
|
--require-entry-passed DirectVulkan.VerifyCorrupted.PipeVerifyArmingScenario.CorruptedFieldIsReported \
|
|
--require-entry-passed DirectGLES.PoisonOmitted.PoisonOmissionScenario.OmittedFieldAbortsOnThatVerb \
|
|
--require-entry-passed DirectVulkan.PoisonOmitted.PoisonOmissionScenario.OmittedFieldAbortsOnThatVerb
|
|
|
|
- name: Upload verify lane logs
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: integration-verify-logs
|
|
path: build-verify/MobileGL/MG_IntegrationTest/pipe-*.log*
|
|
if-no-files-found: warn
|
|
|
|
# P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py).
|
|
- name: Upload JUnit for the skip census
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: junit-integration-verify
|
|
path: ${{ runner.temp }}/*.xml
|
|
if-no-files-found: warn
|
|
|
|
- name: Upload core dumps
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: integration-verify-core-dumps
|
|
path: /tmp/core.*
|
|
if-no-files-found: ignore
|
|
|
|
# The comparator on the split (inproc) arm, both backends, with its control entries checked by
|
|
# name.
|
|
#
|
|
# ITS OWN BUILD, because neither existing one can carry it: build-linux-verify configures
|
|
# without MOBILEGL_BUILD_DISAGGREGATED (so MG_Remote and the transport parser do not exist and
|
|
# `MOBILEGL_TRANSPORT=inproc` would be accepted and ignored), and build-linux-split configures
|
|
# without MOBILEGL_PIPE_VERIFY (so the comparator does not exist). The integration-verify-split
|
|
# entries are registered only when BOTH options are on, so `--no-tests=error` below reds a build
|
|
# that lost either one instead of reporting a green run of nothing. Built and run in one job:
|
|
# nothing downstream consumes this library.
|
|
#
|
|
# INPROC ONLY, BY THE INSTRUMENT'S NATURE: the comparator's two arms (the applier-written
|
|
# gPipeInputs and the frontend GLContext) are co-resident only when both roles share a process.
|
|
# MG_IntegrationTest/CMakeLists.txt's verify-split block records the spawn measurement.
|
|
integration-verify-split:
|
|
runs-on: ubuntu-26.04
|
|
needs: [build-linux-verify-split]
|
|
timeout-minutes: 180
|
|
|
|
# Short-lived test processes skip the glslang prewarm, as in the test job.
|
|
env:
|
|
MOBILEGL_DISABLE_SHADER_PREWARM: "1"
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
with:
|
|
gpu: "true"
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
- name: Install runtime dependencies
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers
|
|
- name: Download verify-split runtime
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: mobilegl-linux-runtime-verify-split
|
|
path: .
|
|
- name: Unpack verify-split runtime
|
|
run: tar --zstd -xf mobilegl-linux-runtime-verify-split.tar.zst
|
|
|
|
# The lane. `^...$` because ctest -L is a regex (ID-131) and `integration-verify` would
|
|
# otherwise also match this label and the monolith one would match both.
|
|
- name: Prepare downloaded test metadata
|
|
run: |
|
|
for root in build-linux build-verify build-split build-verify-split build-retrace; do
|
|
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
|
|
done
|
|
|
|
- name: Verify-split scenarios (inproc)
|
|
working-directory: build-verify-split
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
run: |
|
|
ulimit -c unlimited
|
|
# --output-junit, because the arm proof below has to tell a gtest SKIP from a pass: an
|
|
# entry that skipped reached no verb honestly, and the skip set moves with the driver.
|
|
ctest --output-on-failure -L '^integration-verify-split$' --no-tests=error -j 4 \
|
|
--output-junit "${RUNNER_TEMP}/verify-split.xml"
|
|
|
|
# THE ARM PROOF, PER ENTRY AND BY NAME. Every ambient entry has a private log path
|
|
# (SplitLogPaths.cmake.in), so this - unlike integration-verify's grep over a shared log -
|
|
# is a census:
|
|
# * ConfigLoader's `MOBILEGL_TRANSPORT=inproc` line in EVERY client log: an entry that ran
|
|
# monolith would have passed every assertion above and fails here by name;
|
|
# * `MGPipe: verify armed` in both VerifySplitArming. logs;
|
|
# * and then the census proper. This used to be a FLOOR - "at least 850 armed of at least
|
|
# 1000 logs" - which leaves the remaining ~155 unnamed, so the lane could lose fifty
|
|
# arms, gain fifty skips, and the floor would not move. It is now the two-sided rule
|
|
# dualblock-expected-fatals.txt already rides: each client log must ARM, or belong to an
|
|
# entry ctest reported SKIPPED, or be NAMED in Harness/verify-split-unarmed-expected.txt
|
|
# (56 names in five classes, each class saying why it reaches no verb). A listed entry
|
|
# that arms, and a listed name with no log at all, both fail - an exception set that
|
|
# keeps rows nothing needs is how a proof rots green.
|
|
- name: Verify-split per-entry arm proof
|
|
working-directory: build-verify-split/MobileGL/MG_IntegrationTest
|
|
run: |
|
|
shopt -s nullglob
|
|
logs=(verify-split-logs/*.client.log)
|
|
if [ ${#logs[@]} -lt 1000 ]; then
|
|
echo "::error::found ${#logs[@]} per-entry client logs under verify-split-logs (expected ~1064): the private log paths did not take, so nothing below is a per-entry proof"
|
|
exit 1
|
|
fi
|
|
missing=$(grep -L 'Config: MOBILEGL_TRANSPORT=inproc' "${logs[@]}" | wc -l)
|
|
if [ "${missing}" -ne 0 ]; then
|
|
grep -L 'Config: MOBILEGL_TRANSPORT=inproc' "${logs[@]}" | head -20
|
|
echo "::error::${missing} verify-split entries carry no resolved MOBILEGL_TRANSPORT=inproc line: they did not run on the split arm"
|
|
exit 1
|
|
fi
|
|
for backend in DirectGLES DirectVulkan; do
|
|
if ! grep -q "MGPipe: verify armed" "pipe-verify-split-arming-${backend}.client.log"; then
|
|
echo "::error::pipe-verify-split-arming-${backend}.client.log carries no arming line"
|
|
exit 1
|
|
fi
|
|
done
|
|
python3 "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
|
|
verify-split-arming verify-split-logs "${RUNNER_TEMP}/verify-split.xml" \
|
|
"${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/verify-split-unarmed-expected.txt"
|
|
|
|
# The lane's own control entries, by name, as in integration-verify: VerifySplitReadCorrupted
|
|
# asserts in-test the two where=read reports in the server half, and VerifySplitPoisonOmitted
|
|
# reads both role halves of the forked child's log for its own Fatal.
|
|
- name: Verify-split control entries
|
|
run: |
|
|
args=()
|
|
for backend in DirectGLES DirectVulkan; do
|
|
args+=(--require-entry-passed "${backend}.VerifySplitArming.PipeVerifyArmingScenario.Armed")
|
|
args+=(--require-entry-passed "${backend}.VerifySplitCorrupted.PipeVerifyArmingScenario.CorruptedFieldIsReported")
|
|
args+=(--require-entry-passed "${backend}.VerifySplitReadCorrupted.PipeVerifyArmingScenario.CorruptedFieldIsReportedOnTheServerRead")
|
|
args+=(--require-entry-passed "${backend}.VerifySplitPoisonOmitted.PoisonOmissionScenario.OmittedFieldAbortsOnThatVerb")
|
|
done
|
|
python3 scripts/ci/junit_tally.py "${RUNNER_TEMP}/verify-split.xml" "${args[@]}"
|
|
|
|
# The run log needs a signed-in reader; annotations do not. Name every failed ctest entry,
|
|
# with the lines of its output that say why, where anyone reading the run can see them.
|
|
- name: Annotate failed entries
|
|
if: failure()
|
|
run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}"
|
|
|
|
- name: Upload verify-split lane logs
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: integration-verify-split-logs
|
|
path: |
|
|
build-verify-split/MobileGL/MG_IntegrationTest/verify-split-logs/**
|
|
build-verify-split/MobileGL/MG_IntegrationTest/pipe-*split*.log*
|
|
if-no-files-found: warn
|
|
|
|
# P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py).
|
|
- name: Upload JUnit for the skip census
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: junit-integration-verify-split
|
|
path: ${{ runner.temp }}/*.xml
|
|
if-no-files-found: warn
|
|
|
|
- name: Upload core dumps
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: integration-verify-split-core-dumps
|
|
path: /tmp/core.*
|
|
if-no-files-found: ignore
|
|
|
|
# Whether build-linux-split may skip its compile. On the disaggregation branches its configure
|
|
# line and build-linux's differ only in the benchmarks; this job configures both on one runner
|
|
# (configure only) and compares the trees (scripts/ci/build_fingerprint.py). Its verdict is
|
|
# identical, different or unknown, and its failure never fails the run: only 'identical' makes
|
|
# build-linux-split skip its compile and the split consumers take build-linux's artifacts,
|
|
# while 'different' and 'unknown' keep both builds. The builds run on the same runner image as
|
|
# this job, which records the image and compiler it saw; integration-split prints both sides.
|
|
# MOBILEGL_CI_DISAGGREGATED is build-linux's own expression, so a branch list that drifts
|
|
# between the two jobs configures an OFF primary here and gives 'different'.
|
|
build-guard:
|
|
name: build guard (split reuse)
|
|
runs-on: ubuntu-26.04
|
|
timeout-minutes: 150
|
|
if: ${{ github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main')) }}
|
|
# Consumers read only its outputs, and any verdict but 'identical' (none at all included)
|
|
# keeps both builds.
|
|
continue-on-error: true
|
|
outputs:
|
|
verdict: ${{ steps.guard.outputs.verdict || 'unknown' }}
|
|
image: ${{ steps.guard.outputs.image }}
|
|
compiler: ${{ steps.guard.outputs.compiler }}
|
|
env:
|
|
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }}
|
|
|
|
# Every step may fail without failing the job: a setup step that failed leaves a configure
|
|
# that fails, which the guard reports as 'unknown'.
|
|
steps:
|
|
- name: Checkout repo
|
|
continue-on-error: true
|
|
timeout-minutes: 15
|
|
uses: actions/checkout@v6
|
|
|
|
# The same submodule set as the two builds this job compares. The compare step runs only
|
|
# when every one of them arrived: without 3rdparty/flatbuffers both lines would configure
|
|
# the same library without MG_Remote, and compare as identical.
|
|
- name: Check out build submodules
|
|
id: submodules
|
|
continue-on-error: true
|
|
timeout-minutes: 15
|
|
run: bash scripts/ci/checkout_submodules.sh library
|
|
|
|
- name: Configure runner environment
|
|
continue-on-error: true
|
|
timeout-minutes: 15
|
|
uses: ./.github/actions/setup-test-runner
|
|
|
|
- name: Get CMake
|
|
continue-on-error: true
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Prepare Vulkan SDK
|
|
id: vulkan
|
|
continue-on-error: true
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
uses: humbletim/setup-vulkan-sdk@v1.2.1
|
|
with:
|
|
vulkan-query-version: 1.4.304.1
|
|
vulkan-components: Vulkan-Headers, Vulkan-Loader
|
|
vulkan-use-cache: true
|
|
|
|
- name: Prepare Vulkan SDK (retry)
|
|
continue-on-error: true
|
|
timeout-minutes: 15
|
|
if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.vulkan.outcome == 'failure' }}
|
|
uses: humbletim/setup-vulkan-sdk@v1.2.1
|
|
with:
|
|
vulkan-query-version: 1.4.304.1
|
|
vulkan-components: Vulkan-Headers, Vulkan-Loader
|
|
vulkan-use-cache: true
|
|
|
|
- name: Update glslang external sources
|
|
continue-on-error: true
|
|
timeout-minutes: 15
|
|
working-directory: 3rdparty/glslang
|
|
run: python3 update_glslang_sources.py || python3 update_glslang_sources.py
|
|
|
|
- name: Install build dependencies
|
|
continue-on-error: true
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
|
|
|
|
# The self-test first: a guard whose canned cases fail decides nothing. The two configures
|
|
# share one FetchContent directory, so only the first fetches; each attempt is killed after
|
|
# 10 minutes and retried once, which this step's timeout leaves room for.
|
|
- name: Configure both lines and compare
|
|
id: guard
|
|
continue-on-error: true
|
|
timeout-minutes: 45
|
|
env:
|
|
CI_BUILD_TYPE: ${{ secrets.ACTIONS_STEP_DEBUG == 'true' && 'Debug' || 'Release' }}
|
|
SUBMODULES_OUTCOME: ${{ steps.submodules.outcome }}
|
|
run: |
|
|
if [ "${SUBMODULES_OUTCOME}" != success ]; then
|
|
echo "::warning::the submodule checkout did not succeed (outcome '${SUBMODULES_OUTCOME}'), so the guard decides nothing"
|
|
echo "verdict=unknown" >> "${GITHUB_OUTPUT}"
|
|
elif python3 scripts/ci/build_fingerprint.py --self-test > "${RUNNER_TEMP}/fingerprint-self-test.txt"; then
|
|
python3 scripts/ci/build_fingerprint.py guard --primary-dir build-guard-primary \
|
|
--split-dir build-guard-split --deps-dir build-guard-deps \
|
|
--github-output verdict --report "${RUNNER_TEMP}/build-guard.json"
|
|
else
|
|
cat "${RUNNER_TEMP}/fingerprint-self-test.txt"
|
|
echo "::warning::build_fingerprint.py's self-test failed, so the guard decides nothing"
|
|
echo "verdict=unknown" >> "${GITHUB_OUTPUT}"
|
|
fi
|
|
|
|
- name: Upload guard report
|
|
if: ${{ always() }}
|
|
continue-on-error: true
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: build-guard
|
|
path: ${{ runner.temp }}/build-guard.json
|
|
if-no-files-found: warn
|
|
|
|
# The split build: the same sources with the transport and its in-process arm always on, which the
|
|
# split integration and retrace lanes consume. When build-guard proves its configuration
|
|
# identical to build-linux's (the disaggregation branches, benchmarks apart), this job builds
|
|
# nothing and the consumers take build-linux's artifacts; on any other verdict, and where
|
|
# build-guard does not run, it builds.
|
|
#
|
|
# The `nm` step is the point of a build of its own. In a build without the option, MG_Remote is
|
|
# not compiled and MOBILEGL_TRANSPORT's parser does not exist, so the variable is accepted and
|
|
# ignored and every downstream lane would run monolith under a name that says split. CMake does
|
|
# not complain about a typo'd -D, so the build-level assertion is the guard; in reuse mode
|
|
# build-linux's runtime_mode_proof (--mode disaggregated) is the same assertion and more.
|
|
build-linux-split:
|
|
runs-on: ubuntu-26.04
|
|
timeout-minutes: 120
|
|
needs: [build-guard]
|
|
# A build-guard that was skipped, failed or undecided means this job builds.
|
|
if: ${{ !cancelled() }}
|
|
permissions:
|
|
actions: write
|
|
contents: read
|
|
env:
|
|
BUILD_DIR: build-split
|
|
SPLIT_REUSE: ${{ needs.build-guard.outputs.verdict == 'identical' && 'true' || 'false' }}
|
|
CCACHE_BASEDIR: ${{ github.workspace }}
|
|
CCACHE_COMPRESS: "true"
|
|
CCACHE_DIR: ${{ github.workspace }}/.ccache
|
|
CCACHE_MAXSIZE: 4G
|
|
CCACHE_NOHASHDIR: "true"
|
|
|
|
steps:
|
|
- name: Reuse decision
|
|
env:
|
|
GUARD_VERDICT: ${{ needs.build-guard.outputs.verdict }}
|
|
run: |
|
|
if [ "${SPLIT_REUSE}" = true ]; then
|
|
echo "build-guard found the two configurations identical: this job builds nothing and the split consumers take build-linux's artifacts"
|
|
else
|
|
echo "build-guard verdict '${GUARD_VERDICT:-none}': this job builds the split library"
|
|
fi
|
|
|
|
- name: Checkout repo
|
|
if: env.SPLIT_REUSE != 'true'
|
|
uses: actions/checkout@v6
|
|
|
|
# The submodule list is load-bearing here beyond the usual: MOBILEGL_BUILD_DISAGGREGATED
|
|
# shadows itself back to OFF when 3rdparty/flatbuffers/include is missing (a normal
|
|
# variable rather than a cache force, deliberately), so a checkout without it would
|
|
# configure cleanly and build a monolith library. The script fails when the header is not
|
|
# there, and the nm step below is the second check.
|
|
- name: Check out build submodules
|
|
if: env.SPLIT_REUSE != 'true'
|
|
timeout-minutes: 15
|
|
run: bash scripts/ci/checkout_submodules.sh library
|
|
|
|
- name: Configure runner environment
|
|
if: env.SPLIT_REUSE != 'true'
|
|
uses: ./.github/actions/setup-test-runner
|
|
|
|
- name: Get CMake
|
|
if: env.SPLIT_REUSE != 'true'
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Prepare Vulkan SDK
|
|
id: netsetup1274
|
|
continue-on-error: true
|
|
timeout-minutes: 15
|
|
if: ${{ env.SPLIT_REUSE != 'true' && (env.MOBILEGL_CI_NATIVE_GPU != '1') }}
|
|
uses: humbletim/setup-vulkan-sdk@v1.2.1
|
|
with:
|
|
vulkan-query-version: 1.4.304.1
|
|
vulkan-components: Vulkan-Headers, Vulkan-Loader
|
|
vulkan-use-cache: true
|
|
|
|
- name: Prepare Vulkan SDK (retry)
|
|
timeout-minutes: 15
|
|
if: ${{ env.SPLIT_REUSE != 'true' && ((env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup1274.outcome == 'failure') }}
|
|
uses: humbletim/setup-vulkan-sdk@v1.2.1
|
|
with:
|
|
vulkan-query-version: 1.4.304.1
|
|
vulkan-components: Vulkan-Headers, Vulkan-Loader
|
|
vulkan-use-cache: true
|
|
|
|
- name: Update glslang external sources
|
|
if: env.SPLIT_REUSE != 'true'
|
|
working-directory: 3rdparty/glslang
|
|
run: python3 update_glslang_sources.py
|
|
|
|
- name: Install build dependencies
|
|
timeout-minutes: 15
|
|
if: ${{ env.SPLIT_REUSE != 'true' && (env.MOBILEGL_CI_NATIVE_GPU != '1') }}
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
|
|
|
|
- name: Show installed toolchain
|
|
if: env.SPLIT_REUSE != 'true'
|
|
run: |
|
|
ccache --version
|
|
"${MOBILEGL_CI_CC}" --version
|
|
"${MOBILEGL_CI_CXX}" --version
|
|
dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true
|
|
|
|
# Release/INFO like the shipped build; the line and the reasons are in
|
|
# scripts/ci/configure_build.sh, shared with build-linux.
|
|
- name: Configure CMake
|
|
if: env.SPLIT_REUSE != 'true'
|
|
run: bash scripts/ci/configure_build.sh split "${BUILD_DIR}"
|
|
|
|
- name: Set swap space
|
|
if: env.SPLIT_REUSE != 'true'
|
|
uses: ./.github/actions/set-swap-space
|
|
with:
|
|
swap-size-gb: 32
|
|
|
|
- name: Restore ccache
|
|
if: env.SPLIT_REUSE != 'true'
|
|
uses: actions/cache/restore@v5
|
|
with:
|
|
path: .ccache
|
|
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
restore-keys: |
|
|
${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-
|
|
|
|
- name: Build
|
|
if: env.SPLIT_REUSE != 'true'
|
|
run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)"
|
|
|
|
# `nm` and NOT `nm -D`, for the reason build-linux-verify spells out: everything under
|
|
# MG_Remote is a plain namespace symbol in a CXX_VISIBILITY_PRESET=hidden Release build and
|
|
# none of it reaches the dynamic table. The symbol count guards the remaining hole - a
|
|
# stripped library would make the grep fail for a third, silent reason.
|
|
#
|
|
- name: MG_Remote symbols in split library
|
|
if: env.SPLIT_REUSE != 'true'
|
|
run: |
|
|
test -f "${BUILD_DIR}/libMobileGL.so"
|
|
defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l)
|
|
if [ "${defined}" -lt 1000 ]; then
|
|
echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the checks below could not have failed honestly"
|
|
exit 1
|
|
fi
|
|
remote=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -c -i "MG_Remote" || true)
|
|
if [ "${remote}" -lt 1 ]; then
|
|
echo "::error::libMobileGL.so defines no MG_Remote symbol: -DMOBILEGL_BUILD_DISAGGREGATED=ON did not take (a typo'd -D is not a CMake error, and the option shadows itself OFF when 3rdparty/flatbuffers/include is missing). Every lane that consumes this artifact would run MONOLITH while claiming to run split, because the MOBILEGL_TRANSPORT parser does not exist in such a build and the variable is accepted and ignored."
|
|
exit 1
|
|
fi
|
|
# The transport parser itself, which is the symbol the runtime evidence depends on.
|
|
if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeApply"; then
|
|
echo "::error::libMobileGL.so defines no MGPipeApply* entry point, so MOBILEGL_PIPE_PUSH did not take either and there is no applier for the split path to decode into"
|
|
exit 1
|
|
fi
|
|
echo "libMobileGL.so defines ${remote} MG_Remote symbol(s) and the MGPipe applier (${defined} defined symbols)"
|
|
|
|
# Server link-closure ratchet (ARCHITECTURE.md §13.2): scripts/link_ratchet.py recomputes
|
|
# from the per-object partition the frontend symbols a server-only image still has to link
|
|
# and compares the list with the stored baseline. Only new symbols fail; a vanished one is
|
|
# printed with a request to re-baseline in the same commit. The baseline is a list, not a
|
|
# count, so swapping one symbol for another fails. It runs where MG_Remote is built (the
|
|
# script re-checks CMakeCache.txt) and before anything deletes the *.o files it reads: here,
|
|
# or in build-linux when this job reuses that build.
|
|
- name: Server link-closure ratchet
|
|
if: env.SPLIT_REUSE != 'true'
|
|
run: |
|
|
mkdir -p ci-artifacts
|
|
python3 scripts/link_ratchet.py --self-test
|
|
python3 scripts/link_ratchet.py \
|
|
--build-dir "${BUILD_DIR}" \
|
|
--baseline scripts/data/link_ratchet_baseline.txt \
|
|
--assert-monotone \
|
|
--bucket \
|
|
--json ci-artifacts/link-ratchet.json
|
|
|
|
# Uploaded even when the gate fired: a red ratchet is exactly the moment someone needs the
|
|
# per-symbol definer/referrer attribution the JSON carries.
|
|
- name: Upload link ratchet
|
|
if: ${{ always() && env.SPLIT_REUSE != 'true' }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: link-ratchet
|
|
path: ci-artifacts/link-ratchet.json
|
|
if-no-files-found: warn
|
|
|
|
- name: Show ccache stats
|
|
if: ${{ always() && env.SPLIT_REUSE != 'true' }}
|
|
run: ccache --show-stats
|
|
|
|
# Only after a build: with nothing built, deleting would drop this branch's entry for good.
|
|
- name: Release superseded ccache entry
|
|
if: env.SPLIT_REUSE != 'true'
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true
|
|
|
|
- name: Save ccache
|
|
if: env.SPLIT_REUSE != 'true'
|
|
continue-on-error: true
|
|
uses: actions/cache/save@v5
|
|
with:
|
|
path: .ccache
|
|
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
|
|
- name: Prepare packaged test metadata
|
|
if: env.SPLIT_REUSE != 'true'
|
|
run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}"
|
|
|
|
- name: Package Linux split runtime
|
|
if: env.SPLIT_REUSE != 'true'
|
|
run: |
|
|
mkdir -p ci-artifacts
|
|
mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort)
|
|
tar \
|
|
--exclude='*/CMakeFiles' \
|
|
--exclude='*.o' \
|
|
--exclude='*.a' \
|
|
--exclude='*.ninja*' \
|
|
--exclude='build.ninja' \
|
|
--exclude='cmake_install.cmake' \
|
|
-I 'zstd -T0' -cf ci-artifacts/mobilegl-linux-runtime-split.tar.zst \
|
|
"${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" \
|
|
"${BUILD_DIR}/MobileGL/MG_Test" \
|
|
"${BUILD_DIR}/MobileGL/MG_IntegrationTest" \
|
|
"${SHARED_LIBS[@]}"
|
|
|
|
# compression-level 0: the tarball is zstd already (see 'Upload Linux runtime').
|
|
- name: Upload Linux split runtime
|
|
if: env.SPLIT_REUSE != 'true'
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: mobilegl-linux-runtime-split
|
|
path: ci-artifacts/mobilegl-linux-runtime-split.tar.zst
|
|
if-no-files-found: error
|
|
compression-level: 0
|
|
|
|
# The retrace legs load only the library, and the server image on the spawn arm, so they
|
|
# take this instead of the whole test runtime.
|
|
- name: Package retrace library
|
|
if: env.SPLIT_REUSE != 'true'
|
|
run: |
|
|
files=("${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/libMobileGL.so")
|
|
if [ -f "${BUILD_DIR}/libMobileGLServer.so" ]; then files+=("${BUILD_DIR}/libMobileGLServer.so"); fi
|
|
tar -czf ci-artifacts/mobilegl-linux-lib-split.tgz "${files[@]}"
|
|
|
|
- name: Upload retrace library
|
|
if: env.SPLIT_REUSE != 'true'
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: mobilegl-linux-lib-split
|
|
path: ci-artifacts/mobilegl-linux-lib-split.tgz
|
|
if-no-files-found: error
|
|
|
|
# The split lane itself: the three-arm shape ARCHITECTURE.md §13.2 asks for verbatim, plus the
|
|
# first real run of the five MG_Test/Wire suites. P5's two exit-gate negative controls are in-test
|
|
# now (RemoteWaitBoundaryControl.AnUnwaitedPublishIsSeenAsNoWait and
|
|
# PersistentCoherentMapScenario.ThePushIsTheOnlyRouteAWriteTakesToTheServer).
|
|
integration-split:
|
|
runs-on: ubuntu-26.04
|
|
timeout-minutes: 120
|
|
needs: [build-guard, build-linux-split, build-linux]
|
|
if: ${{ !cancelled() && needs.build-linux-split.result == 'success' && (needs.build-guard.outputs.verdict != 'identical' || needs.build-linux.result == 'success') }}
|
|
# The skip census reads whether 'Default-knob resolution check' proved the IPC defaults.
|
|
outputs:
|
|
defaults-on: ${{ steps.resolution.outputs.defaults-on }}
|
|
|
|
# Short-lived test processes skip the glslang prewarm, as in the test job.
|
|
env:
|
|
MOBILEGL_DISABLE_SHADER_PREWARM: "1"
|
|
SPLIT_REUSE: ${{ needs.build-guard.outputs.verdict == 'identical' && 'true' || 'false' }}
|
|
SPLIT_BUILD_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }}
|
|
SPLIT_RUNTIME: ${{ needs.build-guard.outputs.verdict == 'identical' && 'mobilegl-linux-runtime' || 'mobilegl-linux-runtime-split' }}
|
|
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }}
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
# Which build these lanes take: build-linux's when build-guard proved the two configurations
|
|
# identical, build-linux-split's otherwise. build-guard compared them on its own runner, so
|
|
# the image and compiler it saw are printed beside build-linux's. It reds only when it would
|
|
# reuse a build-linux that is not Release (see scripts/ci/split_reuse_build_type.sh).
|
|
- name: Split runtime
|
|
env:
|
|
GUARD_VERDICT: ${{ needs.build-guard.outputs.verdict }}
|
|
GUARD_IMAGE: ${{ needs.build-guard.outputs.image }}
|
|
GUARD_COMPILER: ${{ needs.build-guard.outputs.compiler }}
|
|
PRIMARY_IMAGE: ${{ needs.build-linux.outputs.image }}
|
|
PRIMARY_COMPILER: ${{ needs.build-linux.outputs.compiler }}
|
|
PRIMARY_BUILD_TYPE: ${{ needs.build-linux.outputs.build-type }}
|
|
run: |
|
|
echo "split runtime: ${SPLIT_BUILD_DIR} (build-guard verdict '${GUARD_VERDICT:-none}')"
|
|
bash scripts/ci/split_reuse_build_type.sh
|
|
echo "build-guard ran on '${GUARD_IMAGE}' with '${GUARD_COMPILER}'"
|
|
echo "build-linux ran on '${PRIMARY_IMAGE}' with '${PRIMARY_COMPILER}'"
|
|
if [ "${SPLIT_REUSE}" = true ] && { [ "${GUARD_IMAGE}" != "${PRIMARY_IMAGE}" ] || [ "${GUARD_COMPILER}" != "${PRIMARY_COMPILER}" ]; }; then
|
|
echo "::notice::build-guard and build-linux saw different runner images or compilers; the reused build is the one build-linux configured"
|
|
fi
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
with:
|
|
gpu: "true"
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Install runtime dependencies
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers
|
|
|
|
- name: Download Linux split runtime
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: ${{ env.SPLIT_RUNTIME }}
|
|
path: .
|
|
|
|
- name: Unpack Linux split runtime
|
|
run: |
|
|
tar --zstd -xf "${SPLIT_RUNTIME}.tar.zst"
|
|
test -f "${SPLIT_BUILD_DIR}/libMobileGL.so"
|
|
if [ "${SPLIT_REUSE}" = "true" ]; then
|
|
python3 -c 'import json, sys; mode = json.load(open(sys.argv[1]))["mode"]; sys.exit(0 if mode == "disaggregated" else f"build-linux runtime mode is {mode}, not disaggregated")' \
|
|
"${SPLIT_BUILD_DIR}/runtime-mode.json"
|
|
fi
|
|
|
|
- name: Prepare downloaded test metadata
|
|
run: |
|
|
for root in build-linux build-verify build-split build-verify-split build-retrace; do
|
|
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
|
|
done
|
|
|
|
- name: Spawn/split lane parity (9.2)
|
|
run: python3 scripts/ci/spawn_lane_parity.py "${SPLIT_BUILD_DIR}"
|
|
|
|
# P8-A (ID-P8-2): every monolith case, tails normalized, is on a gated split arm or is named
|
|
# with a reason class in scripts/data/split_coverage_exemptions.txt. The same `ctest -N`
|
|
# cost as the parity step above, and the question it cannot answer: parity compares the
|
|
# split arms with EACH OTHER, so a case missing from all three passes it.
|
|
- name: Split coverage against monolith (P8-A)
|
|
run: |
|
|
python3 scripts/ci/split_coverage.py --self-test
|
|
python3 scripts/ci/split_coverage.py "${SPLIT_BUILD_DIR}"
|
|
|
|
# P8-SE: PipeCatalogTest's unmigrated-emulation list equals the MGPipeUnmigratedEmulation
|
|
# call sites in the tree, one site per name. A source lint: no build needed.
|
|
- name: Unmigrated-emulation list against its call sites (P8-SE)
|
|
run: |
|
|
python3 scripts/ci/unmigrated_emulation_sites.py --self-test
|
|
python3 scripts/ci/unmigrated_emulation_sites.py
|
|
|
|
# The test job runs the same unit lane, with the same REQUIRE_GPU, on this same build when
|
|
# build-linux-split was reused; the strict unit step in integration-split-strict always runs.
|
|
- name: Unit tests on the split runtime
|
|
if: env.SPLIT_REUSE != 'true'
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit-split.xml"
|
|
|
|
# TcpLane.SupervisorProtocolControls and TcpLane.ControlFrameFuzz build the protocol's Python
|
|
# bindings with flatc, and without one they exit 77 and ctest reports a skip. flatc is not in
|
|
# the build graph (see gen_protocol.py), so this builds the same pinned flatc as flatc-check,
|
|
# and the lane step requires both entries PASSED by name through junit_tally.py. --check keeps
|
|
# this step from regenerating the committed header; checking that header is flatc-check's job.
|
|
# The binary comes from the cache entry flatc-check also uses, keyed on the submodule's
|
|
# commit; gen_protocol.py builds it when the entry is missing and checks its version against
|
|
# the submodule either way.
|
|
- name: Check out FlatBuffers (TCP lane)
|
|
run: git submodule update --init 3rdparty/flatbuffers
|
|
|
|
- name: Pinned flatc cache key
|
|
id: flatc
|
|
run: echo "commit=$(git rev-parse HEAD:3rdparty/flatbuffers)" >> "${GITHUB_OUTPUT}"
|
|
|
|
- name: Cache pinned flatc
|
|
uses: actions/cache@v5
|
|
with:
|
|
path: ${{ runner.temp }}/flatc-build/flatc
|
|
key: ${{ runner.os }}-ubuntu26-flatc-${{ steps.flatc.outputs.commit }}
|
|
|
|
- name: Build pinned flatc (TCP lane)
|
|
run: python3 scripts/gen_protocol.py --build-dir "${RUNNER_TEMP}/flatc-build" --check
|
|
|
|
# --no-tests=error is half the gate, exactly as in integration-verify: the integration-split
|
|
# entries exist only when the library was configured with -DMOBILEGL_BUILD_DISAGGREGATED=ON,
|
|
# so a build that lost the option matches nothing and reds here instead of reporting a green
|
|
# run of nothing. The other half is in-test: the split fixture fails an armed entry whose
|
|
# process did not resolve the transport its registration claims.
|
|
#
|
|
# An entry whose owning package (c1 client, s1 session, v1 server) has not landed SKIPS with
|
|
# the reason and never goes green - MG_IntegrationTest/CMakeLists.txt probes MG_Remote for
|
|
# c0's signature stubs and disarms the lane while any remain.
|
|
#
|
|
# The three arms run as one ctest invocation: the TCP arm holds one resource lock, so on its
|
|
# own it runs one entry at a time and leaves the other cores idle. Each arm keeps its own
|
|
# selection, JUnit and checks.
|
|
- name: Split lanes (inproc, spawn, TCP)
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
run: |
|
|
ulimit -c unlimited
|
|
export MOBILEGL_FLATC_EXECUTABLE="${RUNNER_TEMP}/flatc-build/flatc"
|
|
test -x "${MOBILEGL_FLATC_EXECUTABLE}"
|
|
# Each arm is selected by its own label regex. scripts/ci/lane_selection.py turns the
|
|
# three selections into one list of exact names and fails when an arm selects nothing,
|
|
# which is what --no-tests=error says for an arm run on its own. junit_tally.py then
|
|
# carves each arm's JUnit out of the run, and fails when a selected name is missing
|
|
# from the run or reported twice.
|
|
ctest --show-only=json-v1 -L integration-split > "${RUNNER_TEMP}/split-discovery.json"
|
|
ctest --show-only=json-v1 -L integration-spawn > "${RUNNER_TEMP}/spawn-discovery.json"
|
|
ctest --show-only=json-v1 -L '^integration-tcp$' > "${RUNNER_TEMP}/tcp-discovery.json"
|
|
# A directory of its own: the job uploads RUNNER_TEMP/*.xml for the skip census, which
|
|
# has rows for the three arms and none for the run that holds them all.
|
|
mkdir -p "${RUNNER_TEMP}/split-arms"
|
|
python3 ../scripts/ci/lane_selection.py union "${RUNNER_TEMP}/split-arms/tests.txt" \
|
|
split="${RUNNER_TEMP}/split-discovery.json" spawn="${RUNNER_TEMP}/spawn-discovery.json" \
|
|
tcp="${RUNNER_TEMP}/tcp-discovery.json"
|
|
rc=0
|
|
ctest --output-on-failure --tests-from-file "${RUNNER_TEMP}/split-arms/tests.txt" --no-tests=error -j 4 \
|
|
--output-junit "${RUNNER_TEMP}/split-arms/all.xml" || rc=$?
|
|
# Carved before the exit code is acted on, so a red run still leaves each arm's JUnit
|
|
# for the annotations and the skip census.
|
|
carved=0
|
|
for arm in split spawn tcp; do
|
|
python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/split-arms/all.xml" \
|
|
--select-from "${RUNNER_TEMP}/${arm}-discovery.json" \
|
|
--write-selected "${RUNNER_TEMP}/${arm}-baseline.xml" || carved=1
|
|
done
|
|
if [ "${rc}" -ne 0 ]; then
|
|
echo "::error::the split, spawn and TCP arms' run failed (ctest exit ${rc})"
|
|
exit "${rc}"
|
|
fi
|
|
if [ "${carved}" -ne 0 ]; then
|
|
echo "::error::the run does not hold every entry of the split, spawn and TCP arms exactly once, so it cannot stand for the three arms"
|
|
exit 1
|
|
fi
|
|
python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/split-baseline.xml" --require-split-ran
|
|
# Exit gate 9.3: the spawn arm green at the same count. It runs the same cases through a
|
|
# second process, so a failure here that the inproc arm did not see is a transport fact
|
|
# and not a phase one - which is the whole reason both lanes exist.
|
|
# PH-6 / fuzz arm 3: the supervisor-level forfeit control rides this lane and must RUN.
|
|
python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/spawn-baseline.xml" --require-spawn-ran --require-entry-passed SpawnLane.EventForfeitPeer
|
|
python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/tcp-baseline.xml" --require-tcp-ran --require-run-ahead --discovery "${RUNNER_TEMP}/tcp-discovery.json"
|
|
python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/tcp-baseline.xml" --require-entry-passed TcpLane.SupervisorProtocolControls --require-entry-passed TcpLane.EventForfeitPeer --require-entry-passed TcpLane.ControlFrameFuzz
|
|
|
|
# Whether each entry ran the transport it claims is asserted in-test: the split fixture
|
|
# (Harness/ScenarioFixture.h) fails an armed entry whose process resolved another transport
|
|
# or whose server is not where the claim says, and junit_tally's --require-*-ran above
|
|
# require each arm to have passed entries. This step proves the IPC defaults the lanes
|
|
# rely on, from every passed entry's own client log; it runs right after the lane, so it
|
|
# reads the lane's own run rather than a later step's rerun of the same entry.
|
|
- name: Default-knob resolution check
|
|
id: resolution
|
|
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
run: |
|
|
# The IPC knobs every passed entry resolved, from its own client log: the defaults the
|
|
# lane relies on (run-ahead and the role-split state on, strict off) unless the entry
|
|
# pins its own. Proven defaults make Transport A/B below a repeat of the integration
|
|
# job's two legs; exit 3 says a default changed, and A/B then runs.
|
|
ctest -L integration-split --show-only=json-v1 > "${RUNNER_TEMP}/split-lane.json"
|
|
rc=0
|
|
python3 "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" ipc-config \
|
|
"${RUNNER_TEMP}/split-lane.json" "${RUNNER_TEMP}/split-baseline.xml" "${RUNNER_TEMP}/split-ipc.json" \
|
|
strict=0 role-split-state=1 run-ahead=1 --assumed strict,role-split-state,run-ahead || rc=$?
|
|
if [ "${rc}" -eq 0 ]; then
|
|
echo "defaults-on=true" >> "${GITHUB_OUTPUT}"
|
|
elif [ "${rc}" -eq 3 ]; then
|
|
echo "::warning::an IPC default differs from what the lanes assume (split-ipc.json names the entries), so Transport A/B runs"
|
|
echo "defaults-on=false" >> "${GITHUB_OUTPUT}"
|
|
else
|
|
exit "${rc}"
|
|
fi
|
|
|
|
# Both full transport runs are hard gates. The per-case comparison below
|
|
# remains diagnostic because explicit mechanism controls retain their own mode.
|
|
# With build-linux reused and the defaults proven above, the integration job's two legs
|
|
# already ran these entries on this library in these configurations, so this is skipped.
|
|
#
|
|
# The entries that name MOBILEGL_TRANSPORT in their OWN property (the Split. lanes) keep
|
|
# their value in both passes, which is correct: they are the split family in both arms and
|
|
# the comparison is about the other 1100.
|
|
- name: Transport A/B (monolith vs inproc)
|
|
if: env.SPLIT_REUSE != 'true' || steps.resolution.outputs.defaults-on != 'true'
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
MOBILEGL_ITRP_FIX: "ForceOn"
|
|
run: |
|
|
ulimit -c unlimited
|
|
count=$(ctest -N -L integration-gpu | grep -cE '^ *Test *#[0-9]+:')
|
|
if [ "${count}" -lt 1 ]; then
|
|
echo "::error::the split runtime registers ${count} integration-gpu entries"
|
|
exit 1
|
|
fi
|
|
echo "integration-gpu entries in the split build: ${count}"
|
|
MOBILEGL_TRANSPORT=monolith ctest --output-on-failure -L integration-gpu --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/arm-monolith.xml"
|
|
MOBILEGL_TRANSPORT=inproc MOBILEGL_IPC_ROLE_SPLIT_STATE=1 MOBILEGL_IPC_STRICT_ERRORS=1 ctest --output-on-failure -L integration-gpu --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/arm-inproc.xml"
|
|
# Both commands above must succeed; report differences among legitimate skips.
|
|
python3 - "${RUNNER_TEMP}/arm-monolith.xml" "${RUNNER_TEMP}/arm-inproc.xml" <<'PY'
|
|
import sys, xml.etree.ElementTree as ET
|
|
def rows(path):
|
|
out = {}
|
|
for case in ET.parse(path).getroot().iter('testcase'):
|
|
status = 'passed'
|
|
if case.find('failure') is not None or case.find('error') is not None:
|
|
status = 'failed'
|
|
elif case.find('skipped') is not None or case.get('status') in ('notrun', 'disabled'):
|
|
status = 'skipped'
|
|
out[case.get('name')] = status
|
|
return out
|
|
a, b = rows(sys.argv[1]), rows(sys.argv[2])
|
|
diff = sorted(set(a) ^ set(b)) + sorted(n for n in set(a) & set(b) if a[n] != b[n])
|
|
if diff:
|
|
for name in diff[:40]:
|
|
print(f"{name}: monolith={a.get(name, '<absent>')} inproc={b.get(name, '<absent>')}")
|
|
print(f"Transport comparison: {len(diff)} name/status differences after both full runs passed")
|
|
else:
|
|
print(f"the two arms agree on all {len(a)} entries, name and status")
|
|
PY
|
|
|
|
# The DirectGLES.Split. entries and their log-ownership check, once, in a job whose 'Split
|
|
# lanes' step never ran because an earlier step failed: without this, a red unit lane or
|
|
# flatc build would hide every split integration result of the run. When the lane step ran
|
|
# it left split-baseline.xml, and this step does nothing.
|
|
- name: Split lane fallback (lane step skipped)
|
|
if: ${{ !cancelled() }}
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
run: |
|
|
if [ -f "${RUNNER_TEMP}/split-baseline.xml" ]; then
|
|
echo "the lane step ran and left split-baseline.xml; nothing to fall back to"
|
|
exit 0
|
|
fi
|
|
echo "::notice::the lane step left no split-baseline.xml, so the DirectGLES.Split. entries run here"
|
|
python3 "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" check ctest "$PWD"
|
|
# A directory of its own: the job uploads RUNNER_TEMP/*.xml for the skip census, which
|
|
# has no row for this run.
|
|
mkdir -p "${RUNNER_TEMP}/split-fallback"
|
|
ctest --output-on-failure -L integration-split -R '^DirectGLES\.Split\.' -j 4 --no-tests=error \
|
|
--output-junit "${RUNNER_TEMP}/split-fallback/isplit.xml"
|
|
python3 "${GITHUB_WORKSPACE}/scripts/ci/junit_tally.py" "${RUNNER_TEMP}/split-fallback/isplit.xml" --require-split-ran
|
|
|
|
# build-linux ran the server link-closure ratchet on the build these lanes took, with
|
|
# continue-on-error so that a red there does not skip build-linux's own consumers.
|
|
- name: Server link-closure ratchet of the reused build
|
|
if: ${{ !cancelled() && env.SPLIT_REUSE == 'true' }}
|
|
env:
|
|
PRIMARY_RATCHET: ${{ needs.build-linux.outputs.link-ratchet }}
|
|
run: |
|
|
if [ "${PRIMARY_RATCHET}" != success ]; then
|
|
echo "::error::the server link-closure ratchet of the build-linux build these lanes took did not pass (outcome '${PRIMARY_RATCHET}'); build-linux's link-ratchet-primary artifact names the symbols"
|
|
exit 1
|
|
fi
|
|
echo "the server link-closure ratchet passed on the build-linux build these lanes took"
|
|
|
|
# The run log needs a signed-in reader; annotations do not. Name every failed ctest entry,
|
|
# with the lines of its output that say why, where anyone reading the run can see them.
|
|
- name: Annotate failed entries
|
|
if: failure()
|
|
run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}"
|
|
|
|
- name: Upload split lane logs
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: integration-split-logs
|
|
path: |
|
|
${{ env.SPLIT_BUILD_DIR }}/MobileGL/MG_IntegrationTest/*.log*
|
|
${{ env.SPLIT_BUILD_DIR }}/MobileGL/MG_IntegrationTest/split-logs/*.log
|
|
${{ runner.temp }}/arm-inproc.xml
|
|
${{ runner.temp }}/arm-monolith.xml
|
|
if-no-files-found: warn
|
|
|
|
# P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py).
|
|
- name: Upload JUnit for the skip census
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: junit-integration-split
|
|
path: ${{ runner.temp }}/*.xml
|
|
if-no-files-found: warn
|
|
|
|
- name: Upload core dumps
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: integration-split-core-dumps
|
|
path: /tmp/core.*
|
|
if-no-files-found: ignore
|
|
|
|
# Shares build-linux-split with the functional/transport job above. This job
|
|
# checks state ownership: strict residual-read rules, distinct client/server
|
|
# blocks, zero-residual assertions, and complete per-entry result accounting.
|
|
# Configured controls must pass without skips. Capability gaps and known
|
|
# limitations are reported separately and accepted only by exact name/reason.
|
|
integration-split-strict:
|
|
runs-on: ubuntu-26.04
|
|
timeout-minutes: 60
|
|
needs: [build-guard, build-linux-split, build-linux]
|
|
if: ${{ !cancelled() && needs.build-linux-split.result == 'success' && (needs.build-guard.outputs.verdict != 'identical' || needs.build-linux.result == 'success') }}
|
|
|
|
# Short-lived test processes skip the glslang prewarm, as in the test job.
|
|
env:
|
|
MOBILEGL_DISABLE_SHADER_PREWARM: "1"
|
|
SPLIT_REUSE: ${{ needs.build-guard.outputs.verdict == 'identical' && 'true' || 'false' }}
|
|
SPLIT_BUILD_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }}
|
|
SPLIT_RUNTIME: ${{ needs.build-guard.outputs.verdict == 'identical' && 'mobilegl-linux-runtime' || 'mobilegl-linux-runtime-split' }}
|
|
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }}
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
# build-linux's build when build-guard proved the two configurations identical,
|
|
# build-linux-split's otherwise; integration-split prints the guard's evidence. A build-linux
|
|
# that is not Release is never reused (scripts/ci/split_reuse_build_type.sh).
|
|
- name: Split runtime
|
|
env:
|
|
GUARD_VERDICT: ${{ needs.build-guard.outputs.verdict }}
|
|
PRIMARY_BUILD_TYPE: ${{ needs.build-linux.outputs.build-type }}
|
|
run: |
|
|
echo "split runtime: ${SPLIT_BUILD_DIR} (build-guard verdict '${GUARD_VERDICT:-none}')"
|
|
bash scripts/ci/split_reuse_build_type.sh
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
with:
|
|
gpu: "true"
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Install runtime dependencies
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers vulkan-validationlayers
|
|
|
|
- name: Download Linux split runtime
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: ${{ env.SPLIT_RUNTIME }}
|
|
path: .
|
|
|
|
- name: Unpack Linux split runtime
|
|
run: |
|
|
tar --zstd -xf "${SPLIT_RUNTIME}.tar.zst"
|
|
test -f "${SPLIT_BUILD_DIR}/libMobileGL.so"
|
|
if [ "${SPLIT_REUSE}" = "true" ]; then
|
|
python3 -c 'import json, sys; mode = json.load(open(sys.argv[1]))["mode"]; sys.exit(0 if mode == "disaggregated" else f"build-linux runtime mode is {mode}, not disaggregated")' \
|
|
"${SPLIT_BUILD_DIR}/runtime-mode.json"
|
|
fi
|
|
|
|
- name: Prepare downloaded test metadata
|
|
run: |
|
|
for root in build-linux build-verify build-split build-verify-split build-retrace; do
|
|
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
|
|
done
|
|
|
|
- name: Unit tests on the split runtime (strict)
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
MOBILEGL_IPC_STRICT_ERRORS: "1"
|
|
run: ctest --output-on-failure -L unit --no-tests=error -j "$(nproc)" --output-junit "${RUNNER_TEMP}/unit-strict.xml"
|
|
|
|
# ---- Strict split lane: residual pulls under MOBILEGL_IPC_STRICT_ERRORS=1 ----
|
|
#
|
|
# An unbarriered pull is always fatal, and strict also stops any barriered pull that is not
|
|
# admitted (ARCHITECTURE.md §9.2.1). The lane must be green, and its markers must match the
|
|
# admitted list and strict-expected-markers.txt in both directions. On the disaggregation
|
|
# branches the dual-block step below runs these entries in this same environment and makes
|
|
# the same checks, so this step runs on dev only.
|
|
- name: Strict split lane
|
|
if: env.MOBILEGL_CI_DISAGGREGATED != 'ON'
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
MOBILEGL_IPC_STRICT_ERRORS: "1"
|
|
MOBILEGL_TRANSPORT: "inproc"
|
|
run: |
|
|
ulimit -c unlimited
|
|
helper="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py"
|
|
expected="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/strict-expected-markers.txt"
|
|
|
|
# THE LOG SET COMES FROM ctest, NOT FROM A DIRECTORY (P5e gl, ID-119). This step used to
|
|
# grep MobileGL/MG_IntegrationTest/split-logs/, which holds 90 of the lane's entries.
|
|
# The 26 it could not see were the F1. readback block, both NamedBlit pairs, the four
|
|
# Ct. entries and PersistentMapArm - i.e. PRECISELY the readback population the
|
|
# allowlist is about, plus the only DirectVulkan entries in the lane. A directory is a
|
|
# guess about where the lane put its logs; `--show-only=json-v1` is the lane saying so.
|
|
ctest -L integration-split --show-only=json-v1 > "${RUNNER_TEMP}/lane.json"
|
|
|
|
# Stale logs from an earlier job would be read as this run's evidence: an entry that
|
|
# never starts leaves the previous file in place, and "no marker" is exactly what a
|
|
# lane that never ran looks like.
|
|
python3 - "${RUNNER_TEMP}/lane.json" "${helper}" <<'PY'
|
|
import importlib.util, json, os, sys
|
|
spec = importlib.util.spec_from_file_location("slp", sys.argv[2])
|
|
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
|
|
logs = slp.marker_log_paths(json.load(open(sys.argv[1])))
|
|
for path in set(logs.values()):
|
|
try:
|
|
os.unlink(path)
|
|
except OSError:
|
|
pass
|
|
print(f"reset {len(set(logs.values()))} private log(s) across {len(logs)} entries")
|
|
PY
|
|
|
|
rc=0
|
|
ctest -L integration-split --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/strict-split.xml" \
|
|
> "${RUNNER_TEMP}/strict-split.txt" 2>&1 || rc=$?
|
|
tail -5 "${RUNNER_TEMP}/strict-split.txt"
|
|
echo "strict run: rc=${rc}"
|
|
|
|
python3 "${GITHUB_WORKSPACE}/scripts/gen_pipe_field_ownership.py" --print-admitted \
|
|
> "${RUNNER_TEMP}/allowed.txt"
|
|
echo "§7 allowlist (derived): $(wc -l < "${RUNNER_TEMP}/allowed.txt") admitted pair(s)"
|
|
|
|
# Every entry must complete, and the census is what says why when one does not: an abort
|
|
# is either an unbarriered pull (§3.3's unconditional Fatal, a missed migration) or a
|
|
# barriered pull nothing admits, and both come out of the census by field and verb.
|
|
if [ "${rc}" -ne 0 ]; then
|
|
echo "::error::integration-split under MOBILEGL_IPC_STRICT_ERRORS=1 is RED on a run-ahead server. CONTRACT-P5E §7 makes this lane a hard green gate; the census below names the field and the verb of every entry that aborted."
|
|
python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \
|
|
"${RUNNER_TEMP}/allowed.txt" "${expected}" || true
|
|
exit 1
|
|
fi
|
|
|
|
# The admitted pairs come from gen_pipe_field_ownership.py --print-admitted, the table
|
|
# MGPipeBarrierPullAdmitted() reads, so this list and the C++ cannot drift. A pair is
|
|
# admitted when its verb has a stamp row, the field is BARRIER_PULLED and in the verb's
|
|
# may-read mask, and either every wire op stamping the verb waits or the field's retiring
|
|
# phase is not the generator's THIS_PHASE. The census fails on an admitted marker outside
|
|
# that list and on any difference from strict-expected-markers.txt in either direction;
|
|
# `no-fatal` adds any Fatal marker. Escalated markers depend on the record's payload (an
|
|
# open XFB span, client vertex arrays), so they are checked only against the expected set.
|
|
python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \
|
|
"${RUNNER_TEMP}/allowed.txt" "${expected}" no-fatal
|
|
|
|
# Arming proof: DirectGLES.Split.StrictArming. runs with MOBILEGL_PIPE_STATS=1 and a
|
|
# private log and asserts, over one drawing frame, that the server stamped a verb
|
|
# boundary (vbs > 0) and that no residual pull happened (rsp == 0). Only that entry
|
|
# enables PipeStats, so the counters are checked there, not grepped from these logs; rsp
|
|
# counts only barriered pulls, as an unbarriered one aborts before CountBarrierPull.
|
|
# This step only requires that the entry ran: a skipped positive control proves nothing.
|
|
if ! grep -q 'DirectGLES\.Split\.StrictArming\.' "${RUNNER_TEMP}/strict-split.txt"; then
|
|
echo "::error::the strict lane did not run DirectGLES.Split.StrictArming., so nothing in this run proves the server ever stamped a verb boundary. A green lane without it is indistinguishable from a lane where strict was never armed (ID-115)."
|
|
exit 1
|
|
fi
|
|
echo "integration-split under strict is GREEN, two-sided, with the arming control present"
|
|
|
|
# ---- P5f (f1), P5F-WIRE-COMPLETENESS.md §4: THE DUAL-BLOCK CENSUS LANE ---------------
|
|
#
|
|
# The same integration-split entries (plus Magma's own lane), re-run under
|
|
# These entries own assertions removed from generic transport registrations.
|
|
# Discovery, execution and zero-skip accounting must agree on the exact set.
|
|
- name: Specialized configuration controls
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
run: |
|
|
helper="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py"
|
|
ctest --show-only=json-v1 > "${RUNNER_TEMP}/specialized-all.json"
|
|
python3 "${helper}" select-specialized "${RUNNER_TEMP}/specialized-all.json" \
|
|
"${RUNNER_TEMP}/specialized-selected.json" "${RUNNER_TEMP}/specialized-tests.txt"
|
|
ctest --tests-from-file "${RUNNER_TEMP}/specialized-tests.txt" --no-tests=error -j 4 \
|
|
--output-on-failure --output-junit "${RUNNER_TEMP}/specialized.xml"
|
|
python3 "${helper}" specialized-results "${RUNNER_TEMP}/specialized-all.json" \
|
|
"${RUNNER_TEMP}/specialized.xml"
|
|
|
|
# MOBILEGL_IPC_ROLE_SPLIT_STATE=1: the client's residual fill writes a client-role block
|
|
# and the backend reads a server-role one, so every BARRIER-PULLED read aborts with a
|
|
# named Fatal{UnmigratedPipeInput, "<field>@<verb>"}. THE RED IS THE CENSUS - it is the
|
|
# machine-checked list of what still depends on the two roles sharing one object, and it
|
|
# is ratcheted two-sided against Harness/dualblock-expected-fatals.txt: a new pair is a
|
|
# debt somebody now owes, a vanished pair must be removed in the commit that retired it.
|
|
# With the census empty, this lane is a hard green gate.
|
|
# On the disaggregation branches the first run is also the strict split lane: the knob is
|
|
# left at its default, which every passed entry's own log must show is the dual block, so
|
|
# the environment is the strict lane's exactly. Its checks are made on that run (the
|
|
# integration-split part of the JUnit becomes strict-split.xml); a default that changed
|
|
# reruns the census with the knob set, as on dev.
|
|
- name: Dual-block census lane
|
|
id: dualblock
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
MOBILEGL_TRANSPORT: "inproc"
|
|
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
|
|
MOBILEGL_IPC_STRICT_ERRORS: "1"
|
|
run: |
|
|
ulimit -c unlimited
|
|
helper="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py"
|
|
expected="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/dualblock-expected-fatals.txt"
|
|
strict_expected="${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/strict-expected-markers.txt"
|
|
|
|
ctest -L 'integration-split|integration-magma-split' --show-only=json-v1 \
|
|
> "${RUNNER_TEMP}/dualblock-lane.json"
|
|
|
|
# Stale logs from an earlier step would be read as this run's evidence (ID-119's rule).
|
|
reset_logs() {
|
|
python3 - "${RUNNER_TEMP}/dualblock-lane.json" "${helper}" <<'PY'
|
|
import importlib.util, json, os, sys
|
|
spec = importlib.util.spec_from_file_location("slp", sys.argv[2])
|
|
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
|
|
logs = slp.marker_log_paths(json.load(open(sys.argv[1])))
|
|
for path in set(logs.values()):
|
|
try:
|
|
os.unlink(path)
|
|
except OSError:
|
|
pass
|
|
print(f"reset {len(set(logs.values()))} private log(s) across {len(logs)} entries")
|
|
PY
|
|
}
|
|
census_run() {
|
|
rc=0
|
|
ctest -L 'integration-split|integration-magma-split' --no-tests=error -j 4 \
|
|
--output-junit "${RUNNER_TEMP}/dualblock.xml" \
|
|
> "${RUNNER_TEMP}/dualblock-split.txt" 2>&1 || rc=$?
|
|
tail -5 "${RUNNER_TEMP}/dualblock-split.txt"
|
|
echo "dual-block census run: rc=${rc}"
|
|
}
|
|
reset_logs
|
|
|
|
if [ "${MOBILEGL_CI_DISAGGREGATED}" != ON ]; then
|
|
census_run
|
|
else
|
|
# Run A: the strict split lane's environment, with the role-split state at its default.
|
|
saved_role="${MOBILEGL_IPC_ROLE_SPLIT_STATE}"
|
|
unset MOBILEGL_IPC_ROLE_SPLIT_STATE
|
|
census_run
|
|
export MOBILEGL_IPC_ROLE_SPLIT_STATE="${saved_role}"
|
|
|
|
# The strict lane's JUnit is the integration-split part of this run, every name once.
|
|
ctest -L integration-split --show-only=json-v1 > "${RUNNER_TEMP}/lane.json"
|
|
selected=0
|
|
python3 "${GITHUB_WORKSPACE}/scripts/ci/junit_tally.py" "${RUNNER_TEMP}/dualblock.xml" \
|
|
--select-from "${RUNNER_TEMP}/lane.json" --write-selected "${RUNNER_TEMP}/strict-split.xml" || selected=$?
|
|
python3 "${GITHUB_WORKSPACE}/scripts/gen_pipe_field_ownership.py" --print-admitted \
|
|
> "${RUNNER_TEMP}/allowed.txt"
|
|
echo "allowlist (derived): $(wc -l < "${RUNNER_TEMP}/allowed.txt") admitted pair(s)"
|
|
if [ "${rc}" -ne 0 ]; then
|
|
echo "::error::the strict run of integration-split and integration-magma-split under MOBILEGL_IPC_STRICT_ERRORS=1 is RED on a run-ahead server. It is a hard green gate; the census below names the field and the verb of every integration-split entry that aborted, and dualblock-split.txt holds both labels' output."
|
|
python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \
|
|
"${RUNNER_TEMP}/allowed.txt" "${strict_expected}" || true
|
|
exit 1
|
|
fi
|
|
if [ "${selected}" -ne 0 ]; then
|
|
echo "::error::the dual-block run's JUnit does not hold every integration-split entry exactly once, so it cannot stand for the strict split lane"
|
|
exit 1
|
|
fi
|
|
# The strict lane's two-sided marker census and its arming proof, as in its own step.
|
|
python3 "${helper}" markers "${RUNNER_TEMP}/lane.json" '.' \
|
|
"${RUNNER_TEMP}/allowed.txt" "${strict_expected}" no-fatal
|
|
if ! grep -q 'DirectGLES\.Split\.StrictArming\.' "${RUNNER_TEMP}/dualblock-split.txt"; then
|
|
echo "::error::the strict run did not run DirectGLES.Split.StrictArming., so nothing in this run proves the server ever stamped a verb boundary. A green lane without it is indistinguishable from a lane where strict was never armed."
|
|
exit 1
|
|
fi
|
|
echo "integration-split under strict is GREEN, two-sided, with the arming control present"
|
|
|
|
# Run A is the census run when every passed entry resolved the dual block (and strict
|
|
# and run-ahead) on its own; a default that changed reruns with the knob set.
|
|
ipc=0
|
|
python3 "${helper}" ipc-config "${RUNNER_TEMP}/dualblock-lane.json" "${RUNNER_TEMP}/dualblock.xml" \
|
|
"${RUNNER_TEMP}/dualblock-ipc.json" strict=1 role-split-state=1 run-ahead=1 \
|
|
--assumed role-split-state,run-ahead || ipc=$?
|
|
if [ "${ipc}" -eq 0 ]; then
|
|
echo "defaults-on=true" >> "${GITHUB_OUTPUT}"
|
|
elif [ "${ipc}" -eq 3 ]; then
|
|
echo "::warning::the role-split state or run-ahead default is not on (dualblock-ipc.json names the entries), so the census reruns with MOBILEGL_IPC_ROLE_SPLIT_STATE=1"
|
|
echo "defaults-on=false" >> "${GITHUB_OUTPUT}"
|
|
rm -f "${RUNNER_TEMP}/dualblock.xml"
|
|
reset_logs
|
|
census_run
|
|
else
|
|
exit "${ipc}"
|
|
fi
|
|
fi
|
|
|
|
rows=$(awk '!/^[[:space:]]*(#|$)/ { n++ } END { print n+0 }' "${expected}")
|
|
if [ "${rows}" -eq 0 ] && [ "${rc}" -ne 0 ]; then
|
|
echo "::error::empty dual-block census requires ctest success, including infrastructure/execution status"
|
|
exit "${rc}"
|
|
fi
|
|
python3 "${helper}" expect-fatal "${RUNNER_TEMP}/dualblock-lane.json" \
|
|
"${RUNNER_TEMP}/dualblock.xml" "${expected}"
|
|
|
|
# The per-frame zero-residual-read case must execute and pass on both backends.
|
|
- name: P5f per-frame zero-residual check
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
|
|
MOBILEGL_IPC_STRICT_ERRORS: "1"
|
|
run: |
|
|
ctest -L integration-p5f-rsp --show-only=json-v1 > "${RUNNER_TEMP}/rsp-lane.json"
|
|
ctest -L integration-p5f-rsp --no-tests=error --output-on-failure \
|
|
--output-junit "${RUNNER_TEMP}/rsp.xml"
|
|
python3 "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
|
|
require-green "${RUNNER_TEMP}/rsp-lane.json" "${RUNNER_TEMP}/rsp.xml" \
|
|
DirectGLES.Split.P5fRsp.F1WireScenario.EachWireFrameHasZeroResidualPulls \
|
|
DirectVulkan.Split.P5fRsp.F1WireScenario.EachWireFrameHasZeroResidualPulls
|
|
|
|
# The mechanism's own negative control (P5F §6): the same named case must be green with
|
|
# the knob on and red - on its own assertion text - with it off.
|
|
- name: Dual-block negative control (P5f f1)
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
run: bash "${GITHUB_WORKSPACE}/scripts/ci/dualblock_negative_control.sh"
|
|
|
|
# ---- MAGMA'S OWN WIRE LANE: FRAMEBUFFER, TEXTURE AND APPLICATION BUFFERS ------------
|
|
#
|
|
# P5f fm: Magma's framebuffer, copy and mipmap verbs consume server records.
|
|
# The label remains separate so both backends' coverage stays explicit. The dual-block
|
|
# census run above covers the label in this same environment; with its defaults proven on
|
|
# the disaggregation branches this step would repeat it, so it runs only otherwise.
|
|
- name: Magma split lane (strict, dual-block)
|
|
if: env.MOBILEGL_CI_DISAGGREGATED != 'ON' || steps.dualblock.outputs.defaults-on != 'true'
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
MOBILEGL_IPC_STRICT_ERRORS: "1"
|
|
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
|
|
MOBILEGL_TRANSPORT: "inproc"
|
|
run: |
|
|
ctest -L integration-magma-split --no-tests=error -j 2 --output-on-failure
|
|
|
|
# The Magma scenarios of the step above over the spawn and TCP transports, so a failure here
|
|
# that the inproc step did not show is a two-process fact, not a Magma one. Entries that can
|
|
# run only inproc (such as those that peek this process's server loop) are listed in
|
|
# spawn_lane_parity.py's MAGMA_INPROC_ONLY. The informational tier,
|
|
# integration-magma-all-{split,spawn,tcp}, is not gated: it may go red while CI stays green,
|
|
# and a case moves into a gating step once its blocker is fixed.
|
|
- name: Magma spawn and TCP lanes
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
MOBILEGL_IPC_STRICT_ERRORS: "1"
|
|
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
|
|
run: |
|
|
ulimit -c unlimited
|
|
ctest -L '^integration-magma-spawn$' --no-tests=error -j 2 --output-on-failure
|
|
ctest -L '^integration-magma-tcp$' --no-tests=error -j 2 --output-on-failure
|
|
|
|
# Every named Magma application-buffer wire case must execute and pass.
|
|
- name: Magma buffer wire cases (inproc)
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
run: |
|
|
ctest -L integration-magma-buffers --show-only=json-v1 > "${RUNNER_TEMP}/magma-buffer-lane.json"
|
|
ctest -L integration-magma-buffers --no-tests=error -j 2 --output-on-failure \
|
|
--output-junit "${RUNNER_TEMP}/magma-buffers.xml"
|
|
python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
|
|
"${RUNNER_TEMP}/magma-buffer-lane.json" "${RUNNER_TEMP}/magma-buffers.xml" <<'PY'
|
|
import importlib.util, json, sys
|
|
spec = importlib.util.spec_from_file_location("slp", sys.argv[1])
|
|
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
|
|
cases = [
|
|
"F1WireScenario.EnabledVertexBufferDrawKeepsNamedP7Fatal",
|
|
"F1WireScenario.UniformBufferRangeAndRebindPixels",
|
|
"F1WireScenario.ShortUniformBufferRangePadsMissingBytes",
|
|
"F1WireScenario.SubWordUniformBufferRangePadsMissingBytes",
|
|
"F1WireScenario.UnalignedAtomicCounterRangeDeclinesAndTheSessionLives",
|
|
"F1WireScenario.ComputeWrittenVertexAndIndexBuffersDrawAndReadBack",
|
|
"F1WireScenario.ColorBlitToDefaultIgnoresViewportAndPreservesOrientation",
|
|
"F1WireScenario.ColorBlitToDefaultHonorsScissorAndReversedRect",
|
|
"IndexedDrawFamilyScenario.AnElementBufferDrawElementsOffsetSelectsTheRightQuad",
|
|
"IndexedDrawFamilyScenario.DrawElementsBaseVertexMovesTheSameIndicesToTheOtherQuad",
|
|
"IndexedDrawFamilyScenario.DrawRangeElementsDrawsTheRangedRun",
|
|
"IndexedDrawFamilyScenario.DrawElementsInstancedBaseVertexPaintsOneQuadPerInstance",
|
|
"IndexedDrawFamilyScenario.MultiDrawElementsBaseVertexPaintsEverySubDraw",
|
|
"IndexedDrawFamilyScenario.MultiDrawElementsIndirectDrawsFromTheIndirectBuffer",
|
|
"CrossFrameBufferScenario.VertexBufferSubData",
|
|
"CrossFrameBufferScenario.IndexBufferSubData",
|
|
"CrossFrameBufferScenario.VertexOrphanAndReupload",
|
|
"CrossFrameBufferScenario.IndexOrphanAndReupload",
|
|
"PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw",
|
|
"PersistentCoherentMapScenario.AWriteAfterAFrameBoundaryReachesTheNextFramesDraw",
|
|
"StorageBufferRegrowScenario.AGrownStoreIsVisibleThroughItsExistingIndexedBinding",
|
|
# P11 A2: the >= 16 MiB arena and its tier assertion (emulated).
|
|
"LargeArenaAdoptionScenario.SubDataAfterAnInFlightDrawReachesTheNextDraw",
|
|
"LargeArenaAdoptionScenario.RespecifiedVertexArenaKeepsVaoBindings",
|
|
"LargeArenaAdoptionScenario.RespecifiedIndexArenaKeepsVaoBinding",
|
|
"LargeArenaAdoptionScenario.ReadbackSeesTheLatestCpuWrite",
|
|
"LargeArenaAdoptionScenario.GpuWriteIntoTheArenaIsReadBack",
|
|
"LargeArenaAdoptionScenario.TheArenaLandsInTheTierItsLaneDeclares",
|
|
]
|
|
slp.require_green(json.load(open(sys.argv[2])), sys.argv[3],
|
|
["DirectVulkan.Split.Buffers." + name for name in cases])
|
|
PY
|
|
|
|
# P7 package L: THE SAME TWENTY-ONE, BY NAME, IN TWO PROCESSES (nineteen at package L; the
|
|
# sub-word uniform range and the unaligned atomic-counter decline joined later).
|
|
#
|
|
# `--no-tests=error` in the lane step above answers "did something run and pass"; it does
|
|
# not answer "did these twenty-one run". require_green does, and it is the check that would
|
|
# catch the failure mode this package could plausibly introduce - an arm that registers
|
|
# SOME of the lane and goes green on a smaller set. The label spelling is what keeps the
|
|
# three arms apart under `ctest -L`'s regex: the inproc arm keeps the historical
|
|
# `integration-magma-buffers` (which is what the step above require_greens), and the new
|
|
# arms put the arm word FIRST, so neither selection can reach the other.
|
|
#
|
|
# TcpServer.Start / TcpServer.Stop ARE IN THE tcp ARM'S REQUIRED LIST ON PURPOSE. Every
|
|
# tcp entry declares FIXTURES_REQUIRED mobilegl-tcp, so ctest lists the supervisor's two
|
|
# fixture entries under the label as well - and they must be green too: a lane whose
|
|
# supervisor failed to start has nothing to say about the twenty-one.
|
|
- name: Magma buffer wire cases (spawn, TCP)
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
run: |
|
|
for arm in spawn tcp; do
|
|
case "${arm}" in
|
|
spawn) prefix=DirectVulkan.Spawn.Buffers. ;;
|
|
tcp) prefix=DirectVulkan.Tcp.Buffers. ;;
|
|
esac
|
|
label="integration-magma-${arm}-buffers"
|
|
ctest -L "^${label}$" --show-only=json-v1 > "${RUNNER_TEMP}/magma-buffer-${arm}.json"
|
|
ctest -L "^${label}$" --no-tests=error -j 2 --output-on-failure \
|
|
--output-junit "${RUNNER_TEMP}/magma-buffers-${arm}.xml"
|
|
python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
|
|
"${RUNNER_TEMP}/magma-buffer-${arm}.json" "${RUNNER_TEMP}/magma-buffers-${arm}.xml" \
|
|
"${prefix}" "${arm}" <<'PY'
|
|
import importlib.util, json, sys
|
|
spec = importlib.util.spec_from_file_location("slp", sys.argv[1])
|
|
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
|
|
cases = [
|
|
"F1WireScenario.EnabledVertexBufferDrawKeepsNamedP7Fatal",
|
|
"F1WireScenario.UniformBufferRangeAndRebindPixels",
|
|
"F1WireScenario.ShortUniformBufferRangePadsMissingBytes",
|
|
"F1WireScenario.SubWordUniformBufferRangePadsMissingBytes",
|
|
"F1WireScenario.UnalignedAtomicCounterRangeDeclinesAndTheSessionLives",
|
|
"F1WireScenario.ComputeWrittenVertexAndIndexBuffersDrawAndReadBack",
|
|
"F1WireScenario.ColorBlitToDefaultIgnoresViewportAndPreservesOrientation",
|
|
"F1WireScenario.ColorBlitToDefaultHonorsScissorAndReversedRect",
|
|
"IndexedDrawFamilyScenario.AnElementBufferDrawElementsOffsetSelectsTheRightQuad",
|
|
"IndexedDrawFamilyScenario.DrawElementsBaseVertexMovesTheSameIndicesToTheOtherQuad",
|
|
"IndexedDrawFamilyScenario.DrawRangeElementsDrawsTheRangedRun",
|
|
"IndexedDrawFamilyScenario.DrawElementsInstancedBaseVertexPaintsOneQuadPerInstance",
|
|
"IndexedDrawFamilyScenario.MultiDrawElementsBaseVertexPaintsEverySubDraw",
|
|
"IndexedDrawFamilyScenario.MultiDrawElementsIndirectDrawsFromTheIndirectBuffer",
|
|
"CrossFrameBufferScenario.VertexBufferSubData",
|
|
"CrossFrameBufferScenario.IndexBufferSubData",
|
|
"CrossFrameBufferScenario.VertexOrphanAndReupload",
|
|
"CrossFrameBufferScenario.IndexOrphanAndReupload",
|
|
"PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw",
|
|
"PersistentCoherentMapScenario.AWriteAfterAFrameBoundaryReachesTheNextFramesDraw",
|
|
"StorageBufferRegrowScenario.AGrownStoreIsVisibleThroughItsExistingIndexedBinding",
|
|
"LargeArenaAdoptionScenario.SubDataAfterAnInFlightDrawReachesTheNextDraw",
|
|
"LargeArenaAdoptionScenario.RespecifiedVertexArenaKeepsVaoBindings",
|
|
"LargeArenaAdoptionScenario.RespecifiedIndexArenaKeepsVaoBinding",
|
|
"LargeArenaAdoptionScenario.ReadbackSeesTheLatestCpuWrite",
|
|
"LargeArenaAdoptionScenario.GpuWriteIntoTheArenaIsReadBack",
|
|
"LargeArenaAdoptionScenario.TheArenaLandsInTheTierItsLaneDeclares",
|
|
]
|
|
required = [sys.argv[4] + name for name in cases]
|
|
if sys.argv[5] == "tcp":
|
|
required += ["TcpServer.Start", "TcpServer.Stop"]
|
|
slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], required)
|
|
PY
|
|
done
|
|
|
|
# Run-ahead must queue real GPU work and enforce credits 1 and 3.
|
|
- name: Magma run-ahead cases (credits 1, 3)
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
run: |
|
|
ctest -L integration-magma-runahead --show-only=json-v1 > "${RUNNER_TEMP}/magma-runahead-lane.json"
|
|
ctest -L integration-magma-runahead --no-tests=error -j 2 --output-on-failure \
|
|
--output-junit "${RUNNER_TEMP}/magma-runahead.xml"
|
|
python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
|
|
"${RUNNER_TEMP}/magma-runahead-lane.json" "${RUNNER_TEMP}/magma-runahead.xml" <<'PY'
|
|
import importlib.util, json, sys
|
|
spec = importlib.util.spec_from_file_location("slp", sys.argv[1])
|
|
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
|
|
cases = [
|
|
"QueuedBufferVersionsAndDeletedNameReuseKeepTheirPixels",
|
|
"QueuedProgramRebindsKeepEachUniformSnapshot",
|
|
"QueuedGpuWritesReadBackTheLastDispatchBytes",
|
|
"MixedNativeAndConvertedGpuVertexStreamsKeepTheirReservation",
|
|
"TextureUploadCannotOvertakeThePreviouslyQueuedDraw",
|
|
"GpuClearSurvivesTheFirstStorageImageUpgrade",
|
|
"PresentCreditActuallyParksTheClientAtItsConfiguredLimit",
|
|
]
|
|
required = [f"DirectVulkan.Split.RunAhead.Credit{credit}.MagmaRunAheadScenario.{name}"
|
|
for credit in (1, 3) for name in cases]
|
|
slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], required)
|
|
PY
|
|
|
|
# Run-ahead off must go red on the lost queue lead; run-ahead back on must be green.
|
|
- name: Magma run-ahead negative control
|
|
run: |
|
|
python3 scripts/ci/magma_runahead_checks.py negative --build-dir "${SPLIT_BUILD_DIR}" \
|
|
--out "${RUNNER_TEMP}/magma-runahead-negative" --execute
|
|
|
|
- name: Magma run-ahead sync validation
|
|
run: |
|
|
python3 scripts/ci/magma_runahead_checks.py validation --build-dir "${SPLIT_BUILD_DIR}" \
|
|
--out "${RUNNER_TEMP}/magma-runahead-syncval" --execute
|
|
|
|
- name: Magma view/framebuffer cache sync validation
|
|
run: |
|
|
python3 scripts/ci/magma_cache_checks.py --build-dir "${SPLIT_BUILD_DIR}" \
|
|
--out "${RUNNER_TEMP}/magma-cache-syncval" --execute
|
|
|
|
- name: Upload Magma run-ahead and cache evidence
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: magma-runahead-checks
|
|
path: |
|
|
${{ runner.temp }}/magma-runahead-negative
|
|
${{ runner.temp }}/magma-runahead-syncval
|
|
${{ runner.temp }}/magma-cache-syncval
|
|
if-no-files-found: ignore
|
|
|
|
# The packed-float chain is generated by a real GPU shader blit, so this entry sits in the
|
|
# integration-magma-split lane with its siblings (CMakeLists.txt's fm loop) and carries
|
|
# their pixels assertion. Its named check survives the flip in the only direction that has
|
|
# evidence behind it: the case must EXECUTE, PASS, and never skip - a lane-level exit code
|
|
# alone would not separate "generated" from "quietly absent".
|
|
- name: Magma packed-float mipmap case (P7)
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
MOBILEGL_IPC_STRICT_ERRORS: "1"
|
|
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
|
|
MOBILEGL_TRANSPORT: "inproc"
|
|
run: |
|
|
p7_case='^DirectVulkan\.Split\.Fm\.F1WireScenario\.GenerateMipmapPackedFloatPixels$'
|
|
ctest -R "${p7_case}" --show-only=json-v1 > "${RUNNER_TEMP}/magma-p7-lane.json"
|
|
ctest -R "${p7_case}" --no-tests=error --output-on-failure \
|
|
--output-junit "${RUNNER_TEMP}/magma-p7.xml"
|
|
python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
|
|
"${RUNNER_TEMP}/magma-p7-lane.json" "${RUNNER_TEMP}/magma-p7.xml" <<'PY'
|
|
import importlib.util, json, sys
|
|
spec = importlib.util.spec_from_file_location("slp", sys.argv[1])
|
|
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
|
|
name = "DirectVulkan.Split.Fm.F1WireScenario.GenerateMipmapPackedFloatPixels"
|
|
slp.require_green(json.load(open(sys.argv[2])), sys.argv[3], [name])
|
|
PY
|
|
|
|
# Client vertex/index inputs are copied into owned wire buffer resources.
|
|
# The original dedicated entries now have to execute successfully with run-ahead.
|
|
- name: Client-array split lane (run-ahead)
|
|
working-directory: ${{ env.SPLIT_BUILD_DIR }}
|
|
env:
|
|
MOBILEGL_ITEST_REQUIRE_GPU: "1"
|
|
MOBILEGL_TRANSPORT: "inproc"
|
|
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
|
|
MOBILEGL_IPC_STRICT_ERRORS: "1"
|
|
MOBILEGL_IPC_RUN_AHEAD: "1"
|
|
run: |
|
|
ctest -L integration-clientarrays-split --show-only=json-v1 > "${RUNNER_TEMP}/clientarrays-lane.json"
|
|
ctest -L integration-clientarrays-split --no-tests=error --output-on-failure -j 1 \
|
|
--output-junit "${RUNNER_TEMP}/clientarrays.xml"
|
|
python3 - "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" \
|
|
"${RUNNER_TEMP}/clientarrays-lane.json" "${RUNNER_TEMP}/clientarrays.xml" <<'PY'
|
|
import importlib.util, json, sys
|
|
spec = importlib.util.spec_from_file_location("slp", sys.argv[1])
|
|
slp = importlib.util.module_from_spec(spec); spec.loader.exec_module(slp)
|
|
lane = json.load(open(sys.argv[2]))
|
|
names = [test["name"] for test in lane["tests"]]
|
|
if not names:
|
|
raise SystemExit("no client-array cases discovered")
|
|
slp.require_green(lane, sys.argv[3], names)
|
|
PY
|
|
|
|
# The run log needs a signed-in reader; annotations do not. Name every failed ctest entry,
|
|
# with the lines of its output that say why, where anyone reading the run can see them.
|
|
- name: Annotate failed entries
|
|
if: failure()
|
|
run: python3 scripts/ci/junit_annotate.py "${RUNNER_TEMP}"
|
|
|
|
# P13 W1: every JUnit this job wrote, for the skip census (scripts/ci/skip_census.py).
|
|
- name: Upload JUnit for the skip census
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: junit-integration-split-strict
|
|
path: ${{ runner.temp }}/*.xml
|
|
if-no-files-found: warn
|
|
|
|
- name: Upload strict lane logs
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: integration-split-strict-logs
|
|
path: |
|
|
${{ env.SPLIT_BUILD_DIR }}/MobileGL/MG_IntegrationTest/*.log*
|
|
${{ env.SPLIT_BUILD_DIR }}/MobileGL/MG_IntegrationTest/split-logs/*.log
|
|
${{ runner.temp }}/specialized*.json
|
|
${{ runner.temp }}/specialized-tests.txt
|
|
${{ runner.temp }}/specialized.xml
|
|
${{ runner.temp }}/dualblock-lane.json
|
|
${{ runner.temp }}/dualblock.xml
|
|
${{ runner.temp }}/dualblock-split.txt
|
|
if-no-files-found: warn
|
|
|
|
# MobileGL/MG_Remote/Protocol/generated/protocol_generated.h is COMMITTED, and
|
|
# flatc is deliberately absent from the default build graph (a codegen step in
|
|
# the graph is how the earlier branch ended up cross-compiling an arm64 flatc
|
|
# and trying to run it on the host). This job is what keeps the committed
|
|
# header honest: build the pinned flatc, regenerate, and fail on any diff.
|
|
# It needs no MobileGL build, so it does not depend on build-linux.
|
|
flatc-check:
|
|
runs-on: ubuntu-26.04
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Check out the FlatBuffers submodule only
|
|
# Just this one: the schema check has nothing to do with glslang,
|
|
# SPIRV-Cross or the trace fixtures.
|
|
run: git submodule update --init 3rdparty/flatbuffers
|
|
|
|
# The pinned flatc, cached under the submodule's commit: gen_protocol.py builds it only
|
|
# when the build directory holds none, and checks its version against the submodule
|
|
# either way. The gitlink gives the commit without the submodule's own checkout.
|
|
# integration-split restores the same entry, so the path is the same string in both jobs.
|
|
- name: Pinned flatc cache key
|
|
id: flatc
|
|
run: echo "commit=$(git rev-parse HEAD:3rdparty/flatbuffers)" >> "${GITHUB_OUTPUT}"
|
|
|
|
- name: Cache pinned flatc
|
|
uses: actions/cache@v5
|
|
with:
|
|
path: ${{ runner.temp }}/flatc-build/flatc
|
|
key: ${{ runner.os }}-ubuntu26-flatc-${{ steps.flatc.outputs.commit }}
|
|
|
|
- name: Regenerate protocol_generated.h
|
|
run: python3 scripts/gen_protocol.py --build-dir "${{ runner.temp }}/flatc-build"
|
|
|
|
- name: Generated header staleness check
|
|
run: git diff --exit-code -- MobileGL/MG_Remote/Protocol/generated/protocol_generated.h
|
|
|
|
# P7 wave 2-F, F fix round (PH-7 (4) review). MOBILEGL_PROTOCOL_CONTROL_REVISION is mixed
|
|
# into wireFingerprint so a control-schema change refuses an old peer by name - but it is a
|
|
# hand-bumped integer, and nothing tied it to protocol.fbs's content. This step pins
|
|
# sha256(protocol.fbs) to the revision (scripts/ci/protocol_revision_pins.json): a schema
|
|
# edit that forgot the bump is red HERE, in the job that regenerates the header on every
|
|
# such edit, and not a silently absent field between an old server and a new client. Its
|
|
# self-test proves the check goes red on both perturbations before it says green.
|
|
- name: Protocol revision pin check
|
|
run: python3 scripts/ci/protocol_revision_pin.py --self-test
|
|
|
|
# P0.5 interface-purity gate A (ARCHITECTURE.md §13.2): the two extracted headers' include closure,
|
|
# asserted on `-H` output because `nm --undefined-only` is blind to "included but not called" -
|
|
# a header whose types are never named leaves no symbol behind, and "included at all" is exactly
|
|
# the coupling P1 and P7 have to sever. Needs a preprocessor and three header submodules, no
|
|
# CMake configure and no glslang sources, so like pipe-gates it does not depend on build-linux.
|
|
# The script's own --self-test is always on: a negative control that stopped tripping fails the
|
|
# job, because a gate that cannot go red is not a gate (ARCHITECTURE.md §13.2).
|
|
include-graph-check:
|
|
name: Include-closure purity gate
|
|
runs-on: ubuntu-26.04
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
|
|
- name: Check out header submodules
|
|
# ska/flat_hash_map.hpp, xxhash.h and vulkan/vulkan.h are the only submodule headers
|
|
# Includes.h reaches; glslang and spirv_cross are vendored under include/.
|
|
run: git submodule update --init include/ska 3rdparty/xxHash 3rdparty/Vulkan-Headers
|
|
|
|
- name: Install clang and X11 headers
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
# Includes.h defines VK_USE_PLATFORM_XLIB_KHR before <vulkan/vulkan.h>, which then
|
|
# includes <X11/Xlib.h>; without libx11-dev every clang-mode probe dies in the
|
|
# preprocessor and the gate reports 5 problems that have nothing to do with purity.
|
|
run: sudo apt-get update && sudo apt-get install -y clang-20 libx11-dev
|
|
|
|
- name: Include-closure check and self-test
|
|
# --expect-probes 5 (contract-v2.md 7.6; P13 W5 adds record-core): an exit code cannot tell five probes from none,
|
|
# so a --probe typo or a manifest edit that selected nothing would run zero probes and
|
|
# exit 0 - the second half of the finding that added the flag. The count is the length of
|
|
# scripts/check_include_closure.py's PROBES list and changing one means changing the other.
|
|
#
|
|
# Use the compiler selected for this runner's installed toolchain.
|
|
run: python3 scripts/check_include_closure.py --mode both --compiler "${MOBILEGL_CI_CXX}" --self-test --require-all --expect-probes 5
|
|
|
|
benchmark:
|
|
runs-on: ubuntu-26.04
|
|
needs: [build-linux]
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
with:
|
|
gpu: "true"
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Install runtime dependencies
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
# libegl-mesa0 is the EGL vendor library itself: DriverBench brings up a
|
|
# real GL context, and libegl1 is only glvnd's dispatch. It normally
|
|
# arrives as a Recommends of libegl1, which is too quiet a dependency for
|
|
# the one job that needs a working driver.
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1-mesa-dri mesa-vulkan-drivers
|
|
|
|
- name: Download Linux runtime
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: mobilegl-linux-runtime
|
|
path: .
|
|
|
|
- name: Unpack Linux runtime
|
|
run: tar --zstd -xf mobilegl-linux-runtime.tar.zst
|
|
|
|
- name: Prepare downloaded test metadata
|
|
run: |
|
|
for root in build-linux build-verify build-split build-verify-split build-retrace; do
|
|
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
|
|
done
|
|
|
|
- name: Benchmark
|
|
working-directory: build-linux
|
|
run: |
|
|
ulimit -c unlimited
|
|
ctest -V -C Release -L benchmark --no-tests=error
|
|
|
|
- name: Upload core dumps
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: benchmark-core-dumps
|
|
path: /tmp/core.*
|
|
if-no-files-found: ignore
|
|
|
|
build-retrace:
|
|
runs-on: ubuntu-26.04
|
|
permissions:
|
|
actions: write
|
|
contents: read
|
|
env:
|
|
BUILD_DIR: build-retrace
|
|
CCACHE_BASEDIR: ${{ github.workspace }}
|
|
CCACHE_COMPRESS: "true"
|
|
CCACHE_DIR: ${{ github.workspace }}/.ccache
|
|
CCACHE_MAXSIZE: 4G
|
|
CCACHE_NOHASHDIR: "true"
|
|
MOBILEGL_LIBRARY: ${{ github.workspace }}/build-linux/libMobileGL.so
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
# The library set plus apitrace and the directories its build requires
|
|
# (scripts/ci/checkout_submodules.sh).
|
|
- name: Check out build submodules
|
|
timeout-minutes: 15
|
|
run: bash scripts/ci/checkout_submodules.sh retrace
|
|
|
|
- name: Set swap space
|
|
uses: ./.github/actions/set-swap-space
|
|
with:
|
|
swap-size-gb: 32
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Restore ccache
|
|
uses: actions/cache/restore@v5
|
|
with:
|
|
path: .ccache
|
|
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
restore-keys: |
|
|
${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-
|
|
|
|
- name: Prepare Vulkan SDK
|
|
id: netsetup2500
|
|
continue-on-error: true
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
uses: humbletim/setup-vulkan-sdk@v1.2.1
|
|
with:
|
|
vulkan-query-version: 1.4.304.1
|
|
vulkan-components: Vulkan-Headers, Vulkan-Loader
|
|
vulkan-use-cache: true
|
|
|
|
- name: Prepare Vulkan SDK (retry)
|
|
timeout-minutes: 15
|
|
if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup2500.outcome == 'failure' }}
|
|
uses: humbletim/setup-vulkan-sdk@v1.2.1
|
|
with:
|
|
vulkan-query-version: 1.4.304.1
|
|
vulkan-components: Vulkan-Headers, Vulkan-Loader
|
|
vulkan-use-cache: true
|
|
|
|
- name: Update glslang external sources
|
|
working-directory: 3rdparty/glslang
|
|
run: python3 update_glslang_sources.py
|
|
|
|
- name: Install dependencies
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build
|
|
|
|
- name: Show installed toolchain
|
|
run: |
|
|
ccache --version
|
|
"${MOBILEGL_CI_CC}" --version
|
|
"${MOBILEGL_CI_CXX}" --version
|
|
"${MOBILEGL_CI_LLD}" --version
|
|
dpkg -l 'libc++*' 'libegl*' 'libgles*' 'mesa*' 'vulkan*' || true
|
|
|
|
- name: Prepare downloaded test metadata
|
|
run: |
|
|
for root in build-linux build-verify build-split build-verify-split build-retrace; do
|
|
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
|
|
done
|
|
|
|
- name: Configure CMake
|
|
run: |
|
|
if [ "${{ secrets.ACTIONS_STEP_DEBUG }}" = "true" ]; then
|
|
BUILD_TYPE=Debug
|
|
else
|
|
BUILD_TYPE=Release
|
|
fi
|
|
|
|
cmake -S . -B "${BUILD_DIR}" -G Ninja \
|
|
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \
|
|
-DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \
|
|
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
|
|
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
|
|
-DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
|
|
-DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
|
|
-DMOBILEGL_BUILD_TEST=OFF \
|
|
-DMOBILEGL_BUILD_BENCHMARK=OFF \
|
|
-DMOBILEGL_BUILD_TRACE_REPLAY=ON \
|
|
-DMOBILEGL_TRACE_SOFTWARE_RENDERING="${MOBILEGL_CI_SOFTWARE_RENDERING}" \
|
|
-DMOBILEGL_TRACE_REPLAY_MOBILEGL_LIBRARY="${MOBILEGL_LIBRARY}" \
|
|
-DCMAKE_POLICY_VERSION_MINIMUM=3.5
|
|
|
|
- name: Build trace replay
|
|
run: cmake --build "${BUILD_DIR}" --target mobilegl_trace_replay --parallel "$(nproc)"
|
|
|
|
- name: Show ccache stats
|
|
if: ${{ always() }}
|
|
run: ccache --show-stats
|
|
|
|
- name: Release superseded ccache entry
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true
|
|
|
|
- name: Save ccache
|
|
continue-on-error: true
|
|
uses: actions/cache/save@v5
|
|
with:
|
|
path: .ccache
|
|
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
|
|
- name: Prepare packaged test metadata
|
|
run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}"
|
|
|
|
- name: Package trace replay
|
|
run: |
|
|
mkdir -p ci-artifacts
|
|
tar -czf ci-artifacts/mobilegl-trace-replay.tgz \
|
|
build-retrace/.ci-source-root \
|
|
build-retrace/tools/trace_replay/mobilegl_trace_replay \
|
|
build-retrace/tools/trace_replay/CTestTestfile.cmake
|
|
|
|
- name: Upload trace replay
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: mobilegl-trace-replay
|
|
path: ci-artifacts/mobilegl-trace-replay.tgz
|
|
if-no-files-found: error
|
|
|
|
trace-cases:
|
|
name: trace case matrix
|
|
runs-on: ubuntu-26.04
|
|
env:
|
|
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }}
|
|
# How many jobs replay one (lane, transport, backend) cell of the Linux retrace lanes. Each
|
|
# job replays its share of the cell's cases one after another; a cell with little to
|
|
# replay gets fewer jobs (tools/trace_replay/trace_cases.py, shard_matrix).
|
|
RETRACE_SHARDS_PER_CELL: "4"
|
|
outputs:
|
|
shards: ${{ steps.trace-cases.outputs.shards }}
|
|
verify-shards: ${{ steps.trace-cases.outputs.verify-shards }}
|
|
split-shards: ${{ steps.trace-cases.outputs.split-shards }}
|
|
split-spawn-shards: ${{ steps.trace-cases.outputs.split-spawn-shards }}
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
# `dl` / exit gate S7. CHEAP AND EARLY, on a job that only reads the tree: it needs no
|
|
# build, so a death that names no family is reported in seconds rather than after the
|
|
# matrix. What it refuses is a std::abort() under MG_Remote/ with no Fatal{Family} marker,
|
|
# and a family word that appeared without being recorded - the vocabulary drifts on
|
|
# purpose or not at all (CONTRACT-P6 5.2's D2 ruling).
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
|
|
- name: Fatal census (S7)
|
|
run: python3 scripts/ci/fatal_census.py
|
|
|
|
# The decline half of rule I (P7 B3): every WireDeclines.def row has a site, every site a
|
|
# warning-or-error line. Same shape as the census - tree-only, no build. The self-test
|
|
# runs first: its fixtures (a site only in a comment or a string, a log in another
|
|
# branch, an MGLOG_D) must red, so a green audit is one that could have been red.
|
|
- name: Wire declines audit (B3)
|
|
run: |
|
|
python3 scripts/ci/wire_declines_audit.py --self-test
|
|
python3 scripts/ci/wire_declines_audit.py
|
|
|
|
- name: Load trace cases
|
|
id: trace-cases
|
|
run: |
|
|
python3 tools/trace_replay/trace_cases.py --self-test-transport-matrices
|
|
# The retrace jobs replay shards: several cases of one (lane, transport, backend) cell
|
|
# per job. The first self-test proves every shard matrix replays exactly the rows of
|
|
# the per-case matrix it is built from; the second proves the shard runner gives each
|
|
# case the settings of its lane.
|
|
python3 tools/trace_replay/trace_cases.py --self-test-shard-matrices
|
|
python3 scripts/ci/retrace_shard.py --self-test
|
|
shards() {
|
|
python3 tools/trace_replay/trace_cases.py --ci --shards "${RETRACE_SHARDS_PER_CELL}" "$@"
|
|
}
|
|
shard_count() {
|
|
python3 -c 'import json,sys; print(len(json.load(sys.stdin)["include"]))'
|
|
}
|
|
shard_format=github-test-shard-matrix
|
|
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then shard_format=github-test-transport-shard-matrix; fi
|
|
# An empty `include` is not an error to GitHub: it skips the job with no red anywhere,
|
|
# so each matrix is checked before it is handed over.
|
|
require_shards() {
|
|
if [ "$(printf '%s' "$2" | shard_count)" -lt 1 ]; then
|
|
echo "::error::the $1 shard matrix is empty, so its job would be skipped with no red"
|
|
exit 1
|
|
fi
|
|
}
|
|
SHARDS=$(shards --format "${shard_format}")
|
|
require_shards retrace "${SHARDS}"
|
|
echo "shards=${SHARDS}" >> "$GITHUB_OUTPUT"
|
|
# The subset the verify build retraces ("verify": true in trace_cases.json).
|
|
VERIFY_SHARDS=$(shards --format github-verify-shard-matrix)
|
|
require_shards retrace-verify "${VERIFY_SHARDS}"
|
|
echo "verify-shards=${VERIFY_SHARDS}" >> "$GITHUB_OUTPUT"
|
|
# The split subset: {backend, case, transport} over inproc and spawn, both backends, for
|
|
# every CI case that has not said `"split": false`. It is the whole CI matrix because
|
|
# the bugs the spawn arm finds are about the server's process rather than about one
|
|
# trace. trace_cases.py refuses a `split` case that is not in CI.
|
|
SPLIT_SHARDS=$(shards --format github-split-shard-matrix)
|
|
# It must not be empty. An empty `include` is not an error to GitHub: it skips the
|
|
# whole retrace-split job with no red anywhere, so that is guarded here.
|
|
# trace_cases.py also rejects an unknown manifest key, since a mistyped `split` key
|
|
# would otherwise load clean and change the subset.
|
|
if [ "$(printf '%s' "${SPLIT_SHARDS}" | shard_count)" -lt 1 ]; then
|
|
echo "::error::the split retrace subset is empty. 'split' is an opt-out: a CI case is in this subset unless it carries \"split\": false, so an empty subset means either every case opted out or the CI matrix itself is empty. Either way retrace-split would be skipped with no red."
|
|
exit 1
|
|
fi
|
|
echo "split-shards=${SPLIT_SHARDS}" >> "$GITHUB_OUTPUT"
|
|
# On the disaggregation branches, the same subset without the inproc rows the
|
|
# inproc-acceptance retrace rows replay on the same library, which retrace-split uses when
|
|
# it reuses build-linux's library. OpenRA keeps its inproc rows: they carry the
|
|
# wire-volume evidence. Its self-test proves every dropped row is an acceptance row. It is
|
|
# only a saving, so it never fails this job: on a failed self-test or an empty subset it
|
|
# is not written, and retrace-split takes the full subset, as it does elsewhere.
|
|
if [ "${MOBILEGL_CI_DISAGGREGATED}" = "ON" ]; then
|
|
if python3 tools/trace_replay/trace_cases.py --self-test-split-matrix \
|
|
&& SPLIT_SPAWN_SHARDS=$(shards --format github-split-shard-matrix \
|
|
--transports spawn --keep-inproc OpenRA) \
|
|
&& [ "$(printf '%s' "${SPLIT_SPAWN_SHARDS}" | shard_count || echo 0)" -ge 1 ]; then
|
|
echo "split-spawn-shards=${SPLIT_SPAWN_SHARDS}" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "::warning::the spawn-only split retrace subset failed its self-test or is empty, so retrace-split replays the full split subset"
|
|
fi
|
|
fi
|
|
echo "retrace shards: $(printf '%s' "${SHARDS}" | shard_count) retrace, $(printf '%s' "${VERIFY_SHARDS}" | shard_count) verify, $(printf '%s' "${SPLIT_SHARDS}" | shard_count) split (full subset)"
|
|
|
|
# One cache entry holds every Git LFS fixture of the CI cases, and each replay job restores
|
|
# that entry itself. This job only makes sure the entry exists. On a hit it downloads
|
|
# nothing. On a miss it starts from the newest older bundle, fetches from the mirror the
|
|
# cases whose pointers changed, saves the entry under the new key and confirms it is there,
|
|
# so that no replay job has to download fixtures itself. The key, the path and the steps are
|
|
# .github/scripts/trace-fixture-bundle.sh's; apk.yml uses the same entry. The timeout bounds
|
|
# a fixture download that stalls, which would otherwise hold every replay job behind it.
|
|
trace-fixtures:
|
|
name: trace fixtures
|
|
runs-on: ubuntu-26.04
|
|
timeout-minutes: 30
|
|
permissions:
|
|
actions: read
|
|
contents: read
|
|
outputs:
|
|
key: ${{ steps.bundle.outputs.key }}
|
|
path: ${{ steps.bundle.outputs.path }}
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Derive the fixture bundle key
|
|
id: bundle
|
|
run: bash .github/scripts/trace-fixture-bundle.sh key
|
|
|
|
- name: Look up the fixture bundle
|
|
id: lookup
|
|
uses: actions/cache/restore@v5
|
|
with:
|
|
path: ${{ steps.bundle.outputs.path }}
|
|
key: ${{ steps.bundle.outputs.key }}
|
|
lookup-only: true
|
|
|
|
- name: Restore the newest older bundle
|
|
if: steps.lookup.outputs.cache-hit != 'true'
|
|
uses: actions/cache/restore@v5
|
|
with:
|
|
path: ${{ steps.bundle.outputs.path }}
|
|
key: ${{ steps.bundle.outputs.key }}
|
|
restore-keys: ${{ steps.bundle.outputs.restore-prefix }}
|
|
|
|
- name: Complete the fixture bundle
|
|
if: steps.lookup.outputs.cache-hit != 'true'
|
|
run: bash .github/scripts/trace-fixture-bundle.sh ensure
|
|
|
|
# The APK workflow of the same push saves the same entry, and only one save can win.
|
|
# The steps below find the entry whichever workflow wrote it.
|
|
- name: Save the fixture bundle
|
|
if: steps.lookup.outputs.cache-hit != 'true'
|
|
continue-on-error: true
|
|
uses: actions/cache/save@v5
|
|
with:
|
|
path: ${{ steps.bundle.outputs.path }}
|
|
key: ${{ steps.bundle.outputs.key }}
|
|
|
|
# The APK workflow of the same push may be the one whose save won. Its entry is as good,
|
|
# once it is listed.
|
|
- name: Wait for the fixture bundle entry
|
|
if: steps.lookup.outputs.cache-hit != 'true'
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: bash .github/scripts/trace-fixture-bundle.sh wait '${{ steps.bundle.outputs.key }}'
|
|
|
|
- name: Confirm the fixture bundle entry
|
|
if: steps.lookup.outputs.cache-hit != 'true'
|
|
uses: actions/cache/restore@v5
|
|
with:
|
|
path: ${{ steps.bundle.outputs.path }}
|
|
key: ${{ steps.bundle.outputs.key }}
|
|
lookup-only: true
|
|
fail-on-cache-miss: true
|
|
|
|
# One job replays a shard: some of the cases of one lane, transport and backend, one after
|
|
# another, each through its own ctest entry (scripts/ci/retrace_shard.py). The shard's step
|
|
# summary has a row per case, and a failed case has an error annotation of its own.
|
|
retrace:
|
|
name: retrace (${{ matrix.backend }} / ${{ matrix.transport || 'monolith' }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
|
|
runs-on: ubuntu-26.04
|
|
timeout-minutes: 240
|
|
env:
|
|
MOBILEGL_TRANSPORT: ${{ matrix.transport || 'monolith' }}
|
|
MOBILEGL_IPC_ROLE_SPLIT_STATE: "1"
|
|
MOBILEGL_IPC_STRICT_ERRORS: "1"
|
|
MOBILEGL_IPC_RUN_AHEAD: "1"
|
|
SHARD_NAMES: ${{ toJSON(matrix.names) }}
|
|
needs:
|
|
- build-linux
|
|
- build-retrace
|
|
- trace-cases
|
|
- trace-fixtures
|
|
if: ${{ !cancelled() && needs.build-linux.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix: ${{ fromJSON(needs.trace-cases.outputs.shards) }}
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set swap space
|
|
if: runner.environment != 'self-hosted'
|
|
uses: ./.github/actions/set-swap-space
|
|
with:
|
|
swap-size-gb: 16
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
with:
|
|
gpu: "true"
|
|
|
|
# A miss does not fail the job: the entry may have been evicted since trace-fixtures
|
|
# confirmed it (a re-run of failed jobs on an older run does not run that job again).
|
|
- name: Restore trace fixtures
|
|
uses: actions/cache/restore@v5
|
|
with:
|
|
path: ${{ needs.trace-fixtures.outputs.path }}
|
|
key: ${{ needs.trace-fixtures.outputs.key }}
|
|
|
|
# Moves this shard's fixtures into place and checks them against their Git LFS pointers.
|
|
# A case the restored bundle does not hold is downloaded here, with a warning annotation;
|
|
# the timeout bounds a download that stalls.
|
|
- name: Install trace fixtures
|
|
timeout-minutes: 15
|
|
run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Install runtime dependencies
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers
|
|
test -e /usr/lib/x86_64-linux-gnu/libEGL.so
|
|
test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so
|
|
|
|
- name: Download Linux library
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: mobilegl-linux-lib
|
|
path: .
|
|
|
|
- name: Download trace replay
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: mobilegl-trace-replay
|
|
path: .
|
|
|
|
- name: Unpack retrace runtime
|
|
run: |
|
|
tar -xzf mobilegl-linux-lib.tgz
|
|
tar -xzf mobilegl-trace-replay.tgz
|
|
test -f build-linux/libMobileGL.so
|
|
test -f build-retrace/tools/trace_replay/mobilegl_trace_replay
|
|
|
|
- name: Prepare downloaded test metadata
|
|
run: |
|
|
for root in build-linux build-verify build-split build-verify-split build-retrace; do
|
|
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
|
|
done
|
|
|
|
# The per-case settings are retrace_shard.py's "retrace" profile: the Magma fallback and
|
|
# the two per-case repair knobs on DirectVulkan, and on the inproc lane the IPC
|
|
# configuration the client must log, the split evidence check and the refusal census from
|
|
# the library's own logs. A failing case does not stop the shard; the step fails at the
|
|
# end if a case failed or a case has no verdict.
|
|
- name: Retrace and validate
|
|
run: |
|
|
python3 scripts/ci/retrace_shard.py run --profile retrace \
|
|
--backend '${{ matrix.backend }}' --transport "${MOBILEGL_TRANSPORT}" \
|
|
--lane '${{ matrix.lane || 'monolith-control' }}' \
|
|
--shard '${{ matrix.shard }}/${{ matrix.shards }}' \
|
|
--build-dir build-retrace/tools/trace_replay \
|
|
--result-dir "${RUNNER_TEMP}/retrace-shard"
|
|
|
|
- name: Annotate failed entries
|
|
if: failure()
|
|
run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit"
|
|
|
|
- name: Upload core dumps
|
|
if: ${{ failure() || cancelled() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: retrace-core-dumps-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }}
|
|
path: /tmp/core.*
|
|
if-no-files-found: ignore
|
|
|
|
# What retrace-summary renders: per case, the images and the replay's output directory.
|
|
# The summary embeds the images, so these are short-lived; a failed shard keeps its own
|
|
# copy below.
|
|
- name: Upload shard results
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: retrace-result-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }}
|
|
path: ${{ runner.temp }}/retrace-shard/results
|
|
if-no-files-found: warn
|
|
retention-days: 1
|
|
overwrite: true
|
|
|
|
- name: Upload failed shard results
|
|
if: ${{ failure() || cancelled() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: retrace-failed-${{ matrix.lane || 'monolith-control' }}-${{ matrix.transport || 'monolith' }}-${{ matrix.backend }}-shard${{ matrix.shard }}
|
|
path: |
|
|
${{ runner.temp }}/retrace-shard/results
|
|
${{ runner.temp }}/retrace-shard/junit
|
|
${{ runner.temp }}/retrace-shard/logs
|
|
if-no-files-found: warn
|
|
overwrite: true
|
|
|
|
retrace-summary:
|
|
name: retrace summary
|
|
runs-on: ubuntu-26.04
|
|
needs: [retrace]
|
|
if: ${{ !cancelled() && needs.retrace.result != 'skipped' }}
|
|
# actions: write is for the last step, which deletes the artifacts this job rendered.
|
|
permissions:
|
|
actions: write
|
|
contents: read
|
|
env:
|
|
MOBILEGL_CI_DISAGGREGATED: ${{ (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && 'ON' || 'OFF' }}
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
|
|
- name: Set artifact metadata
|
|
run: |
|
|
echo "date_today=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
|
|
|
|
- name: Set up Node.js
|
|
uses: actions/setup-node@v7
|
|
with:
|
|
node-version: '22'
|
|
|
|
- name: Download monolith control results
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
pattern: retrace-result-monolith-control-monolith-*
|
|
path: retrace-artifacts/monolith-control
|
|
|
|
- name: Download inproc acceptance results
|
|
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
pattern: retrace-result-inproc-acceptance-inproc-*
|
|
path: retrace-artifacts/inproc-acceptance
|
|
|
|
- name: Render monolith control summary
|
|
run: |
|
|
node tools/trace_replay/render_retrace_summary.mjs \
|
|
--input retrace-artifacts/monolith-control \
|
|
--output-dir retrace-summary/monolith-control \
|
|
--title "MobileGL Linux monolith control" \
|
|
--group-label "Linux / monolith control" \
|
|
--html mobilegl-linux-monolith-control.html
|
|
|
|
- name: Render inproc acceptance summary
|
|
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
|
|
run: |
|
|
node tools/trace_replay/render_retrace_summary.mjs \
|
|
--input retrace-artifacts/inproc-acceptance \
|
|
--output-dir retrace-summary/inproc-acceptance \
|
|
--title "MobileGL Linux inproc acceptance" \
|
|
--group-label "Linux / inproc acceptance" \
|
|
--html mobilegl-linux-inproc-acceptance.html
|
|
|
|
- name: Upload monolith control summary
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: mobilegl-linux-monolith-control-summary-${{ github.sha }}
|
|
path: retrace-summary/monolith-control/mobilegl-linux-monolith-control.html
|
|
archive: false
|
|
if-no-files-found: error
|
|
|
|
- name: Upload inproc acceptance summary
|
|
if: env.MOBILEGL_CI_DISAGGREGATED == 'ON'
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: mobilegl-linux-inproc-acceptance-summary-${{ github.sha }}
|
|
path: retrace-summary/inproc-acceptance/mobilegl-linux-inproc-acceptance.html
|
|
archive: false
|
|
if-no-files-found: error
|
|
|
|
# The summaries embed every image, so the shard result artifacts they were rendered from
|
|
# have no reader left and are deleted here. That happens only when every shard passed: a
|
|
# failed shard has its own retrace-failed-* artifact, and after a failure the results
|
|
# stay for their one day of retention, so that re-running the failed shards renders every
|
|
# case again. A failed delete only leaves them to that retention.
|
|
- name: Delete the rendered shard results
|
|
if: ${{ needs.retrace.result == 'success' }}
|
|
continue-on-error: true
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: |
|
|
gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \
|
|
--jq '.artifacts[] | select(.name | startswith("retrace-result-")) | [.id, .name] | @tsv' \
|
|
> "${RUNNER_TEMP}/rendered-artifacts.txt"
|
|
deleted=0
|
|
while IFS=$'\t' read -r artifact_id artifact_name; do
|
|
echo "Deleting ${artifact_name} (${artifact_id})"
|
|
gh api --method DELETE "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}"
|
|
deleted=$((deleted + 1))
|
|
done < "${RUNNER_TEMP}/rendered-artifacts.txt"
|
|
echo "Deleted ${deleted} rendered shard result artifact(s)."
|
|
|
|
# The trace half of the third CI mode. Same replay, same goldens, but the library underneath is
|
|
# the verify build and MOBILEGL_PIPE_VERIFY=1 is in the environment, so every backend read of
|
|
# frontend state is checked against a snapshot taken at the verb boundary. Eight cases rather
|
|
# than the full lane's 40 (tools/trace_replay/trace_cases.json, "verify": true): the comparator
|
|
# is budgeted at 5-10x, and the full sweep is a phase-exit / workflow_dispatch run.
|
|
retrace-verify:
|
|
name: retrace verify (${{ matrix.backend }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
|
|
runs-on: ubuntu-26.04
|
|
timeout-minutes: 240
|
|
env:
|
|
SHARD_NAMES: ${{ toJSON(matrix.names) }}
|
|
needs:
|
|
- build-linux-verify
|
|
- build-retrace
|
|
- trace-cases
|
|
- trace-fixtures
|
|
if: ${{ !cancelled() && needs.build-linux-verify.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix: ${{ fromJSON(needs.trace-cases.outputs.verify-shards) }}
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set swap space
|
|
if: runner.environment != 'self-hosted'
|
|
uses: ./.github/actions/set-swap-space
|
|
with:
|
|
swap-size-gb: 16
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
with:
|
|
gpu: "true"
|
|
|
|
# A miss does not fail the job; the install step downloads what the bundle does not hold.
|
|
- name: Restore trace fixtures
|
|
uses: actions/cache/restore@v5
|
|
with:
|
|
path: ${{ needs.trace-fixtures.outputs.path }}
|
|
key: ${{ needs.trace-fixtures.outputs.key }}
|
|
|
|
- name: Install trace fixtures
|
|
timeout-minutes: 15
|
|
run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Install runtime dependencies
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers
|
|
test -e /usr/lib/x86_64-linux-gnu/libEGL.so
|
|
test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so
|
|
|
|
- name: Download Linux verify library
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: mobilegl-linux-lib-verify
|
|
path: .
|
|
|
|
- name: Download trace replay
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: mobilegl-trace-replay
|
|
path: .
|
|
|
|
- name: Unpack verify runtime (library under test)
|
|
# build-retrace's CTestTestfile.cmake has the absolute path
|
|
# <workspace>/build-linux/libMobileGL.so frozen into every case, so the swap happens here
|
|
# rather than through a variable: the verify .so is put where that path points. The nm
|
|
# check is what makes the swap falsifiable - a run against the ordinary library would
|
|
# carry no comparator, ignore MOBILEGL_PIPE_VERIFY entirely, and match its golden.
|
|
#
|
|
# `nm`, not `nm -D`, for the reason spelled out in build-linux-verify: everything MGPipe is
|
|
# hidden-visibility in a Release build and the dynamic table has none of it.
|
|
run: |
|
|
tar -xzf mobilegl-linux-lib-verify.tgz
|
|
tar -xzf mobilegl-trace-replay.tgz
|
|
test -f build-verify/libMobileGL.so
|
|
test -f build-retrace/tools/trace_replay/mobilegl_trace_replay
|
|
mkdir -p build-linux
|
|
cp build-verify/libMobileGL.so build-linux/libMobileGL.so
|
|
if ! nm --defined-only build-linux/libMobileGL.so | grep -q MGPipeVerifyInputs; then
|
|
echo "::error::the library unpacked at build-linux/libMobileGL.so defines no MGPipeVerifyInputs, so this retrace would replay against a comparator-free build and pass on its golden having verified nothing"
|
|
exit 1
|
|
fi
|
|
echo "the library at build-linux/libMobileGL.so is the verify build"
|
|
|
|
- name: Prepare downloaded test metadata
|
|
run: |
|
|
for root in build-linux build-verify build-split build-verify-split build-retrace; do
|
|
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
|
|
done
|
|
|
|
# Every case replays with MOBILEGL_PIPE_VERIFY=1 (retrace_shard.py's "verify" profile).
|
|
# run_trace_case.cmake turns that into three assertions of its own (the arming line, no
|
|
# Fatal{PipeVerifyDiffer, no Fatal{UnmigratedPipeInput), so a case that somehow ran the
|
|
# wrong library reds here instead of passing on its golden. ctest's timeout is raised to
|
|
# 10800 s: the long cases run 5-10x slower with both comparator arms live, well past the
|
|
# 1500 s default.
|
|
#
|
|
# The shard that holds OpenRA on DirectGLES also runs the lane's negative control right
|
|
# after that case: with a snapshot field corrupted, the same replay must fail, and for the
|
|
# comparator's reason (scripts/ci/retrace_verify_reason.py). Without it, zero divergences
|
|
# would be a statement about a comparator nobody watched. The control has a verdict row of
|
|
# its own, and its output is discarded so that only the verified run is uploaded.
|
|
- name: Retrace and validate (verify)
|
|
run: |
|
|
python3 scripts/ci/retrace_shard.py run --profile verify \
|
|
--backend '${{ matrix.backend }}' --lane verify \
|
|
--shard '${{ matrix.shard }}/${{ matrix.shards }}' \
|
|
--build-dir build-retrace/tools/trace_replay \
|
|
--result-dir "${RUNNER_TEMP}/retrace-shard"
|
|
|
|
- name: Annotate failed entries
|
|
if: failure()
|
|
run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit"
|
|
|
|
- name: Upload core dumps
|
|
if: ${{ failure() || cancelled() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: retrace-verify-core-dumps-${{ matrix.backend }}-shard${{ matrix.shard }}
|
|
path: /tmp/core.*
|
|
if-no-files-found: ignore
|
|
|
|
# Kept for every shard, green or not: per case, the verified images and the replay's
|
|
# output directory, whose library log has the comparator's arming line.
|
|
- name: Upload shard results
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: retrace-verify-result-${{ matrix.backend }}-shard${{ matrix.shard }}
|
|
path: ${{ runner.temp }}/retrace-shard/results
|
|
if-no-files-found: warn
|
|
overwrite: true
|
|
|
|
- name: Upload failed shard results
|
|
if: ${{ failure() || cancelled() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: retrace-verify-failed-${{ matrix.backend }}-shard${{ matrix.shard }}
|
|
path: |
|
|
${{ runner.temp }}/retrace-shard/results
|
|
${{ runner.temp }}/retrace-shard/junit
|
|
${{ runner.temp }}/retrace-shard/logs
|
|
if-no-files-found: warn
|
|
overwrite: true
|
|
|
|
# Trace retrace on the split library: the retrace-verify shape with MOBILEGL_TRANSPORT set
|
|
# instead of MOBILEGL_PIPE_VERIFY, over {backend, case, transport} with inproc and spawn.
|
|
# inproc is the control and spawn the claim: same library and picture, only the server role's
|
|
# process differs, so a spawn-only failure is a transport fact. It runs the unchanged ctest
|
|
# names with MOBILEGL_TRANSPORT exported around ctest, since build-retrace does not register the
|
|
# split variant entries. SSIM cannot catch a monolith run (OpenRA scores 1.000 either way); the
|
|
# guards are the nm check for MG_Remote, run_trace_case.cmake's transport assertions (and, on
|
|
# OpenRA, its wire-volume assertion), and the per-leg check of the evidence file they leave.
|
|
retrace-split:
|
|
name: retrace split (${{ matrix.backend }} / ${{ matrix.transport }}, shard ${{ matrix.shard }}/${{ matrix.shards }})
|
|
runs-on: ubuntu-26.04
|
|
timeout-minutes: 240
|
|
# The setup action selects lavapipe on GitHub-hosted runners.
|
|
needs:
|
|
- build-guard
|
|
- build-linux-split
|
|
- build-linux
|
|
- build-retrace
|
|
- trace-cases
|
|
- trace-fixtures
|
|
if: ${{ !cancelled() && needs.build-linux-split.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' && needs.trace-fixtures.result == 'success' && (needs.build-guard.outputs.verdict != 'identical' || needs.build-linux.result == 'success') }}
|
|
env:
|
|
SPLIT_REUSE: ${{ needs.build-guard.outputs.verdict == 'identical' && 'true' || 'false' }}
|
|
SPLIT_LIB: ${{ needs.build-guard.outputs.verdict == 'identical' && 'mobilegl-linux-lib' || 'mobilegl-linux-lib-split' }}
|
|
SPLIT_LIB_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }}
|
|
SPLIT_BUILD_DIR: ${{ needs.build-guard.outputs.verdict == 'identical' && 'build-linux' || 'build-split' }}
|
|
SHARD_NAMES: ${{ toJSON(matrix.names) }}
|
|
strategy:
|
|
fail-fast: false
|
|
# With build-linux reused, the inproc rows (except OpenRA's) are replayed by the retrace job's
|
|
# inproc-acceptance rows on the same library, strict and with the dual block and run-ahead
|
|
# on; the spawn rows below prove per case that those are also the defaults they run with.
|
|
# A shard holds one transport, so OpenRA's inproc rows are shards of their own.
|
|
matrix: ${{ fromJSON(needs.build-guard.outputs.verdict == 'identical' && needs.trace-cases.outputs.split-spawn-shards || needs.trace-cases.outputs.split-shards) }}
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set swap space
|
|
if: runner.environment != 'self-hosted'
|
|
uses: ./.github/actions/set-swap-space
|
|
with:
|
|
swap-size-gb: 16
|
|
|
|
# build-linux's build when build-guard proved the two configurations identical,
|
|
# build-linux-split's otherwise; integration-split prints the guard's evidence. A build-linux
|
|
# that is not Release is never reused (scripts/ci/split_reuse_build_type.sh).
|
|
- name: Split runtime
|
|
env:
|
|
GUARD_VERDICT: ${{ needs.build-guard.outputs.verdict }}
|
|
PRIMARY_BUILD_TYPE: ${{ needs.build-linux.outputs.build-type }}
|
|
run: |
|
|
echo "split runtime: ${SPLIT_BUILD_DIR} (build-guard verdict '${GUARD_VERDICT:-none}')"
|
|
bash scripts/ci/split_reuse_build_type.sh
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
with:
|
|
gpu: "true"
|
|
|
|
# A miss does not fail the job; the install step downloads what the bundle does not hold.
|
|
- name: Restore trace fixtures
|
|
uses: actions/cache/restore@v5
|
|
with:
|
|
path: ${{ needs.trace-fixtures.outputs.path }}
|
|
key: ${{ needs.trace-fixtures.outputs.key }}
|
|
|
|
- name: Install trace fixtures
|
|
timeout-minutes: 15
|
|
run: bash .github/scripts/trace-fixture-bundle.sh install ${{ join(matrix.names, ' ') }}
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Install runtime dependencies
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers
|
|
test -e /usr/lib/x86_64-linux-gnu/libEGL.so
|
|
test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so
|
|
|
|
- name: Download Linux split library
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: ${{ env.SPLIT_LIB }}
|
|
path: .
|
|
|
|
- name: Download trace replay
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: mobilegl-trace-replay
|
|
path: .
|
|
|
|
- name: Unpack split runtime (library under test)
|
|
# build-retrace's CTestTestfile.cmake has the absolute path
|
|
# <workspace>/build-linux/libMobileGL.so frozen into every case, so the swap happens here
|
|
# rather than through a variable, exactly as in retrace-verify. `nm`, not `nm -D`.
|
|
run: |
|
|
tar -xzf "${SPLIT_LIB}.tgz"
|
|
tar -xzf mobilegl-trace-replay.tgz
|
|
test -f "${SPLIT_LIB_DIR}/libMobileGL.so"
|
|
test -f build-retrace/tools/trace_replay/mobilegl_trace_replay
|
|
if [ "${SPLIT_LIB_DIR}" != build-linux ]; then
|
|
mkdir -p build-linux
|
|
cp "${SPLIT_LIB_DIR}/libMobileGL.so" build-linux/libMobileGL.so
|
|
fi
|
|
if ! nm --defined-only build-linux/libMobileGL.so | grep -q -i MG_Remote; then
|
|
echo "::error::the library unpacked at build-linux/libMobileGL.so defines no MG_Remote symbol, so this retrace would replay against a monolith build, ignore MOBILEGL_TRANSPORT entirely and match its golden having split nothing"
|
|
exit 1
|
|
fi
|
|
echo "the library at build-linux/libMobileGL.so is the split build"
|
|
|
|
- name: Prepare downloaded test metadata
|
|
run: |
|
|
for root in build-linux build-verify build-split build-verify-split build-retrace; do
|
|
if [ -d "$root" ]; then python3 scripts/ci/prepare_test_runtime.py "$root"; fi
|
|
done
|
|
|
|
# The spawn arm needs a second binary and the inproc arm needs none: inproc's server role
|
|
# is a thread in the replay process, spawn's is a process the client launches from this
|
|
# image. It ships in the split retrace library (libMobileGLServer.so is an executable
|
|
# with a .so name, so that an APK installs it somewhere it can be executed), and a missing
|
|
# one must be a named failure here rather than a LaunchServer refusal in every case.
|
|
- name: Place the server image (spawn arm only)
|
|
if: ${{ matrix.transport == 'spawn' }}
|
|
run: |
|
|
if [ ! -f "${SPLIT_LIB_DIR}/libMobileGLServer.so" ]; then
|
|
echo "::error::the split runtime carries no libMobileGLServer.so, so transport=spawn has no server to launch. ConfigLoader would resolve Spawn, LaunchServer would refuse BY NAME, and this job would red on a message about an unresolvable image rather than about a packaging gap."
|
|
exit 1
|
|
fi
|
|
if [ "${SPLIT_LIB_DIR}" != build-linux ]; then
|
|
cp "${SPLIT_LIB_DIR}/libMobileGLServer.so" build-linux/libMobileGLServer.so
|
|
fi
|
|
chmod +x build-linux/libMobileGLServer.so
|
|
echo "server image: $(readlink -f build-linux/libMobileGLServer.so)"
|
|
|
|
# The per-case settings are retrace_shard.py's "split" profile:
|
|
# - the Magma knobs the retrace job's monolith rows run with (the R11F_G11F_B10F fallback,
|
|
# the blended depth-write quirk on its one case, the iterationRP repairs on theirs), so a
|
|
# split row and its monolith control differ in the transport alone;
|
|
# - MOBILEGL_TRANSPORT and the absolute server image path;
|
|
# - on OpenRA, the wire-volume counters and the check that draw records crossed the wire;
|
|
# - with build-linux reused, the IPC defaults the client must log (strict=0).
|
|
# run_trace_case.cmake turns MOBILEGL_TRANSPORT into assertions of its own: the library must
|
|
# have resolved the transport in this process and its log must carry no Fatal{, so a case
|
|
# that somehow ran the wrong library reds here instead of passing on its golden. After
|
|
# ctest the evidence file those assertions leave is checked for every case, and the
|
|
# refusal census is printed from the library's own logs of both roles, which is the only
|
|
# valid source: the console sink is compiled out of this configuration.
|
|
- name: Retrace and validate (${{ matrix.transport }})
|
|
run: |
|
|
reuse=()
|
|
if [ "${SPLIT_REUSE}" = "true" ]; then reuse=(--split-reuse); fi
|
|
python3 scripts/ci/retrace_shard.py run --profile split \
|
|
--backend '${{ matrix.backend }}' --transport '${{ matrix.transport }}' --lane split \
|
|
--shard '${{ matrix.shard }}/${{ matrix.shards }}' \
|
|
--server-path "${GITHUB_WORKSPACE}/build-linux/libMobileGLServer.so" \
|
|
--build-dir build-retrace/tools/trace_replay \
|
|
--result-dir "${RUNNER_TEMP}/retrace-shard" "${reuse[@]}"
|
|
|
|
- name: Annotate failed entries
|
|
if: failure()
|
|
run: python3 scripts/ci/junit_annotate.py --excerpts 3 "${RUNNER_TEMP}/retrace-shard/junit"
|
|
|
|
- name: Upload core dumps
|
|
if: ${{ failure() || cancelled() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: retrace-split-core-dumps-${{ matrix.transport }}-${{ matrix.backend }}-shard${{ matrix.shard }}
|
|
path: /tmp/core.*
|
|
if-no-files-found: ignore
|
|
|
|
# Nothing reads a green shard's output, so only a failed shard uploads it.
|
|
- name: Upload failed shard results
|
|
if: ${{ failure() || cancelled() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: retrace-split-failed-${{ matrix.transport }}-${{ matrix.backend }}-shard${{ matrix.shard }}
|
|
path: |
|
|
${{ runner.temp }}/retrace-shard/results
|
|
${{ runner.temp }}/retrace-shard/junit
|
|
${{ runner.temp }}/retrace-shard/logs
|
|
if-no-files-found: warn
|
|
overwrite: true
|
|
|
|
# The shipped library, replayed. build-linux-monolith-control's library has no MG_Remote, the
|
|
# shape FCL embeds; on the disaggregation branches nothing else replays that shape, so it runs
|
|
# there only (elsewhere the retrace job's library is that shape). One short case on both
|
|
# backends, with no transport set, must match its golden like any monolith row.
|
|
retrace-shipped:
|
|
name: retrace shipped library (${{ matrix.backend }}, ${{ matrix.case }})
|
|
runs-on: ubuntu-26.04
|
|
timeout-minutes: 60
|
|
needs:
|
|
- build-linux-monolith-control
|
|
- build-retrace
|
|
- trace-cases
|
|
if: ${{ !cancelled() && (github.ref_name == 'dev' || github.ref_name == 'feat/disaggregated' || github.ref_name == 'codex/fix-github-ci' || github.ref_name == 'cleanup' || (github.event_name == 'workflow_dispatch' && (github.ref_type == 'tag' || github.ref == 'refs/heads/main'))) && needs.build-linux-monolith-control.result == 'success' && needs.build-retrace.result == 'success' && needs.trace-cases.result == 'success' }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
backend: [DirectGLES, DirectVulkan]
|
|
case: [OpenRA]
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set swap space
|
|
if: runner.environment != 'self-hosted'
|
|
uses: ./.github/actions/set-swap-space
|
|
with:
|
|
swap-size-gb: 16
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
with:
|
|
gpu: "true"
|
|
|
|
# This job's one case keeps its fixtures in Git itself, not in Git LFS, so the checkout
|
|
# has them and the fixture bundle is not restored. The check is that they are real files;
|
|
# a case with Git LFS fixtures would be downloaded here, with a warning annotation.
|
|
- name: Check trace fixture
|
|
run: bash .github/scripts/trace-fixture-bundle.sh install '${{ matrix.case }}'
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Install runtime dependencies
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y libvulkan1 libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers
|
|
test -e /usr/lib/x86_64-linux-gnu/libEGL.so
|
|
test -e /usr/lib/x86_64-linux-gnu/libGLESv2.so
|
|
|
|
- name: Download shipped library
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: mobilegl-linux-monolith-control
|
|
path: .
|
|
|
|
- name: Download trace replay
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: mobilegl-trace-replay
|
|
path: .
|
|
|
|
# build-retrace froze <workspace>/build-linux/libMobileGL.so into every case, so the library
|
|
# is put where that path points, as retrace-verify does. `nm`, not `nm -D`: the MG_Remote
|
|
# symbols are hidden in a Release build and only the static table lists them.
|
|
- name: Unpack shipped library (library under test)
|
|
run: |
|
|
tar -xzf mobilegl-linux-monolith-control.tgz
|
|
tar -xzf mobilegl-trace-replay.tgz
|
|
test -f build-push-control/libMobileGL.so
|
|
test -f build-retrace/tools/trace_replay/mobilegl_trace_replay
|
|
mkdir -p build-linux
|
|
cp build-push-control/libMobileGL.so build-linux/libMobileGL.so
|
|
defined=$(nm --defined-only build-linux/libMobileGL.so | wc -l)
|
|
remote=$(nm --defined-only build-linux/libMobileGL.so | grep -c -i MG_Remote || true)
|
|
if [ "${defined}" -lt 1000 ] || [ "${remote}" -ne 0 ]; then
|
|
echo "::error::build-linux/libMobileGL.so defines ${defined} symbols and ${remote} MG_Remote symbol(s): it is not the shipped shape (no MG_Remote, not stripped)"
|
|
exit 1
|
|
fi
|
|
echo "the library at build-linux/libMobileGL.so is the shipped shape (${defined} defined symbols, no MG_Remote)"
|
|
|
|
- name: Prepare downloaded test metadata
|
|
run: python3 scripts/ci/prepare_test_runtime.py build-retrace
|
|
|
|
- name: Retrace and validate
|
|
working-directory: build-retrace/tools/trace_replay
|
|
run: |
|
|
ulimit -c unlimited
|
|
if [ '${{ matrix.backend }}' = 'DirectVulkan' ]; then
|
|
export MOBILEGL_MAGMA_R11G11B10F_FALLBACK=1
|
|
fi
|
|
ctest -V --no-tests=error -R '^MobileGLTraceReplay\.${{ matrix.case }}\.${{ matrix.backend }}$' \
|
|
--output-junit "${RUNNER_TEMP}/retrace.xml"
|
|
|
|
- name: Annotate failed entries
|
|
if: failure()
|
|
run: python3 scripts/ci/junit_annotate.py --excerpts 1 "${RUNNER_TEMP}"
|
|
|
|
- name: Upload core dumps
|
|
if: ${{ failure() || cancelled() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: retrace-shipped-core-dumps-${{ matrix.backend }}-${{ matrix.case }}
|
|
path: /tmp/core.*
|
|
if-no-files-found: ignore
|
|
|
|
# Nothing reads a green leg's output, so only a failed leg uploads it.
|
|
- name: Upload actual image
|
|
if: ${{ failure() || cancelled() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: retrace-shipped-result-${{ matrix.backend }}-${{ matrix.case }}
|
|
path: |
|
|
build-retrace/tools/trace_replay/${{ matrix.case }}/actual-images/**
|
|
build-retrace/tools/trace_replay/${{ matrix.case }}/${{ matrix.backend }}/output/**
|
|
if-no-files-found: warn
|
|
|
|
pipe-gates:
|
|
name: MGPipe generators and hygiene gates
|
|
runs-on: ubuntu-26.04
|
|
# Deliberately independent of build-linux: these are source-level gates, they take
|
|
# seconds, and a broken build must not hide a drifted interface.
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
# The seven generators all read MG_Pipe/*.def, so regenerating and diffing is what
|
|
# keeps the two interface tables, the wire records, the verify comparators, the
|
|
# PipeInputs field ids, the read-inventory coverage and the render-state member list
|
|
# from drifting apart from the catalog. The generated files are committed
|
|
# deliberately: the build must not depend on python.
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
- name: Test CTest runtime metadata preparation
|
|
run: python3 scripts/ci/test_prepare_test_runtime.py
|
|
|
|
- name: Regenerate the MGPipe interface (G1-G7)
|
|
run: |
|
|
python3 scripts/gen_pipe.py
|
|
git diff --exit-code -- MobileGL/MG_Pipe/generated
|
|
|
|
# The generators' own negative controls: canned inputs that MUST trip each structural check
|
|
# (a field list that does not cover its struct's members, a verb set that is not the function
|
|
# table's). Regenerating and diffing above cannot see a check that silently stopped
|
|
# checking - a broken gate and a clean tree produce the same green.
|
|
- name: MGPipe generator self-test
|
|
run: python3 scripts/gen_pipe.py --self-test
|
|
|
|
# The link seam owns physical storage and keeps link dispatch out of callers.
|
|
- name: Link-seam purity check
|
|
run: python3 scripts/ci/link_seam_purity.py --self-test
|
|
|
|
# The same shape for Espryt's re-keyed memos (P3b/P4b, PLAN-PH-P34B-P7.md 1.2). P4a/P5f
|
|
# moved the resolved-binding, sampler-pass, image-sweep, twin-registry, unit-sampler and
|
|
# twin-lookup memos onto {slot, gen} handles and applier serials; the legacy pointer-keyed
|
|
# arms stayed because they are the PULL build's only arms and G1 pins that build's .text.
|
|
# Nothing checked that a NEW frontend-pointer key could not appear outside them.
|
|
#
|
|
# P13 W3b: NO EXEMPTED ARM ANY MORE. The legacy pointer-keyed arms (MOBILEGL_PIPE_LEGACY_MEMOS)
|
|
# retired with the pull build, and the !MOBILEGL_BUILD_DISAGGREGATED arms are compiled into
|
|
# the library FCL embeds, so they get the same rule: no frontend-pointer key in any memo
|
|
# family, in any build.
|
|
#
|
|
# --self-test runs the negative controls and then the real scan in one invocation, as its
|
|
# sibling does; the scan also asserts that every memo family's anchor still matches and
|
|
# that every allow-list entry still suppresses something.
|
|
- name: Espryt memo-key purity check
|
|
run: python3 scripts/ci/espryt_memo_purity.py --self-test
|
|
|
|
# P13 W3b: the pull build's switches stay retired - a preprocessor conditional on
|
|
# MOBILEGL_PIPE_PUSH / MOBILEGL_PIPE_LEGACY_MEMOS, or an MGB_CTX spelling, coming back is a
|
|
# red here. --self-test plants each shape first and requires the gate to catch it.
|
|
- name: Retired pull-build switches stay retired (P13)
|
|
run: python3 scripts/ci/retired_switches.py --self-test
|
|
|
|
# The same question for the symbol tool the P1 gate is written in terms of.
|
|
- name: Symbol report self-test
|
|
run: python3 scripts/symbol_report.py --self-test
|
|
|
|
# Per-draw fprintf/printf instrumentation has repeatedly been committed by accident,
|
|
# once inside a mutex critical section. Nothing under these two trees prints to a
|
|
# stdio stream today - MGLOG_D compiles out in INFO builds and is the only channel
|
|
# they are allowed to use - so this gate starts with no exceptions, and any addition
|
|
# to it needs a reason in the pull request rather than a quiet whitelist entry. The
|
|
# alternation names every stdio spelling, not just the two that were committed:
|
|
# fprintf to either stream, printf, puts, and the iostream pair.
|
|
- name: No stdio in MG_Backend or MG_State
|
|
run: |
|
|
if grep -rnE 'fprintf[[:space:]]*\((stderr|stdout)|(^|[^[:alnum:]_>.])printf[[:space:]]*\(|(^|[^[:alnum:]_>.:])puts[[:space:]]*\(|std::(cout|cerr)' \
|
|
MobileGL/MG_Backend MobileGL/MG_State; then
|
|
echo "::error::stdio instrumentation found; use MGLOG_D (compiled out in INFO builds)"
|
|
exit 1
|
|
fi
|
|
echo "no fprintf(stderr/stdout / printf( / puts( / std::cout|cerr under MobileGL/MG_Backend or MobileGL/MG_State"
|
|
|
|
# Dirty-surface completeness (ARCHITECTURE.md §5.2.1). --check fails in both directions: a
|
|
# mutator the scanner finds with no row in MG_Pipe/DirtySurface.def, and a row naming a
|
|
# mutator the scan no longer finds. --self-test feeds canned negative controls and fails if
|
|
# one does not trip, so a check that stopped checking cannot stay green. The scanner matches
|
|
# text; the semantic proof is the verify lane.
|
|
- name: Dirty-surface mapping complete (G9)
|
|
run: |
|
|
python3 scripts/gen_pipe_dirty_surface.py --check
|
|
python3 scripts/gen_pipe_dirty_surface.py --self-test
|
|
|
|
# PipeFieldOwnership.inc is written by gen_pipe_field_ownership.py, not gen_pipe.py, so the
|
|
# gen_pipe.py regenerate-and-diff step above does not cover it. The build's static_assert
|
|
# catches only an unclassified row; --check also catches a changed class. No branch guard:
|
|
# every branch needs the committed table to follow from the .def files. Each --self-test
|
|
# control asserts its own message, and a harness control checks that one control's exit is
|
|
# never credited to another.
|
|
- name: Field-ownership table complete (R-7.1)
|
|
run: |
|
|
python3 scripts/gen_pipe_field_ownership.py --check
|
|
python3 scripts/gen_pipe_field_ownership.py --self-test
|
|
|
|
# Negative controls must themselves be able to go red (ARCHITECTURE.md §13.2). The Python
|
|
# tests run the trace and TCP lane tooling on stubs and synthetic data; test_run_trace_case.py
|
|
# feeds run_trace_case.cmake canned logs. No branch guard; no build, GPU or device.
|
|
- name: Control and trace-tool smoke tests (R-16)
|
|
run: |
|
|
python3 scripts/ci/test_trace_infrastructure.py
|
|
python3 scripts/ci/tcp_lane_tools_test.py
|
|
python3 tools/trace_replay/test_run_tcp_matrix.py
|
|
python3 tools/trace_replay/test_compare_actuals.py
|
|
python3 tools/trace_replay/test_run_trace_case.py
|
|
python3 scripts/ci/build_fingerprint.py --self-test
|
|
bash scripts/ci/split_reuse_build_type.sh --self-test
|
|
bash scripts/ci/test_set_swap_space.sh
|
|
python3 MobileGL/MG_IntegrationTest/Harness/test_split_log_paths.py
|
|
python3 scripts/ci/transport_pinned.py --self-test
|
|
|
|
# The helpers the build and split-lane jobs run before anything else can fail for them: the
|
|
# union of the split arms' selections, and the submodule lists, which are printed here
|
|
# without checking anything out.
|
|
- name: Build and lane helper self-tests
|
|
run: |
|
|
python3 scripts/ci/lane_selection.py --self-test
|
|
bash scripts/ci/checkout_submodules.sh --list library
|
|
bash scripts/ci/checkout_submodules.sh --list retrace
|
|
|
|
# Warning only for now: the disaggregation documents are still being written, and a
|
|
# lint that fails a rewrite in progress teaches people to ignore it. It becomes
|
|
# --strict when the documents settle.
|
|
- name: Documentation citation lint
|
|
run: |
|
|
shopt -s nullglob
|
|
documents=(docs/Disaggregated/*.md docs/Disaggregated/design/*.md docs/Disaggregated/guide/*.md)
|
|
if [ ${#documents[@]} -eq 0 ]; then
|
|
echo "no disaggregation documents to check"
|
|
exit 0
|
|
fi
|
|
python3 scripts/check_doc_citations.py "${documents[@]}" || true
|
|
build-linux-verify-split:
|
|
runs-on: ubuntu-26.04
|
|
timeout-minutes: 180
|
|
permissions:
|
|
actions: write
|
|
contents: read
|
|
env:
|
|
BUILD_DIR: build-verify-split
|
|
CCACHE_BASEDIR: ${{ github.workspace }}
|
|
CCACHE_COMPRESS: "true"
|
|
CCACHE_DIR: ${{ github.workspace }}/.ccache
|
|
CCACHE_MAXSIZE: 4G
|
|
CCACHE_NOHASHDIR: "true"
|
|
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
|
|
# For build-linux-split's reason: MOBILEGL_BUILD_DISAGGREGATED shadows itself back to OFF
|
|
# when 3rdparty/flatbuffers/include is missing, and the script fails when it is.
|
|
- name: Check out build submodules
|
|
timeout-minutes: 15
|
|
run: bash scripts/ci/checkout_submodules.sh library
|
|
|
|
- name: Set swap space
|
|
uses: ./.github/actions/set-swap-space
|
|
with:
|
|
swap-size-gb: 32
|
|
|
|
- name: Configure runner environment
|
|
uses: ./.github/actions/setup-test-runner
|
|
|
|
- name: Get CMake
|
|
timeout-minutes: 15
|
|
uses: lukka/get-cmake@v4.3.3
|
|
with:
|
|
cmakeVersion: 3.31.10
|
|
|
|
- name: Restore ccache
|
|
uses: actions/cache/restore@v5
|
|
with:
|
|
path: .ccache
|
|
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
restore-keys: |
|
|
${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-
|
|
|
|
- name: Prepare Vulkan SDK
|
|
id: netsetup3726
|
|
continue-on-error: true
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
uses: humbletim/setup-vulkan-sdk@v1.2.1
|
|
with:
|
|
vulkan-query-version: 1.4.304.1
|
|
vulkan-components: Vulkan-Headers, Vulkan-Loader
|
|
vulkan-use-cache: true
|
|
|
|
- name: Prepare Vulkan SDK (retry)
|
|
timeout-minutes: 15
|
|
if: ${{ (env.MOBILEGL_CI_NATIVE_GPU != '1') && steps.netsetup3726.outcome == 'failure' }}
|
|
uses: humbletim/setup-vulkan-sdk@v1.2.1
|
|
with:
|
|
vulkan-query-version: 1.4.304.1
|
|
vulkan-components: Vulkan-Headers, Vulkan-Loader
|
|
vulkan-use-cache: true
|
|
|
|
- name: Update glslang external sources
|
|
working-directory: 3rdparty/glslang
|
|
run: python3 update_glslang_sources.py
|
|
|
|
- name: Install build and runtime dependencies
|
|
timeout-minutes: 15
|
|
if: env.MOBILEGL_CI_NATIVE_GPU != '1'
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y ccache clang-20 clang++-20 lld-20 libc++-20-dev libc++abi-20-dev libvulkan-dev libegl1-mesa-dev libgles2-mesa-dev libgl1-mesa-dri mesa-vulkan-drivers ninja-build libvulkan1 libegl1 libegl-mesa0 libgles2 libgl1
|
|
|
|
- name: Configure CMake
|
|
# Release/INFO for build-linux-verify's reason, and INFO is load-bearing here for
|
|
# build-linux-split's: ConfigLoader's resolved-transport line is logged at INFO and the
|
|
# arm-proof step below reads it back out of every entry's client log.
|
|
# ENABLE_PCH=OFF: ccache cannot cache a compile that uses glslang's precompiled header
|
|
# (see scripts/ci/configure_build.sh).
|
|
run: |
|
|
cmake -S . -B "${BUILD_DIR}" -G Ninja \
|
|
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \
|
|
-DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \
|
|
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
|
|
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
|
|
-DENABLE_PCH=OFF \
|
|
-DCMAKE_BUILD_TYPE=Release \
|
|
-DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \
|
|
-DMOBILEGL_BUILD_TEST=ON \
|
|
-DMOBILEGL_BUILD_BENCHMARK=OFF \
|
|
-DMOBILEGL_BUILD_INTEGRATION_TEST=ON \
|
|
-DMOBILEGL_ITEST_VK_ICD="${VK_ICD_FILENAMES}" \
|
|
-DMOBILEGL_BUILD_TRACE_REPLAY=OFF \
|
|
-DMOBILEGL_BUILD_DISAGGREGATED=ON \
|
|
-DMOBILEGL_BUILD_DISAGGREGATED_INPROC=ON \
|
|
-DMOBILEGL_PIPE_VERIFY=ON \
|
|
-DCMAKE_POLICY_VERSION_MINIMUM=3.5
|
|
|
|
- name: Build
|
|
run: cmake --build "${BUILD_DIR}" --parallel "$(nproc)"
|
|
|
|
# Both options, both directions of "did it take" - the two jobs this one is the product of
|
|
# each check one of them, and this library has to pass both checks.
|
|
- name: Comparator and MG_Remote symbols
|
|
run: |
|
|
test -f "${BUILD_DIR}/libMobileGL.so"
|
|
defined=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | wc -l)
|
|
if [ "${defined}" -lt 1000 ]; then
|
|
echo "::error::nm --defined-only sees only ${defined} symbols in ${BUILD_DIR}/libMobileGL.so - it looks stripped, so the checks below could not have failed honestly"
|
|
exit 1
|
|
fi
|
|
if ! nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -q "MGPipeVerifyInputs"; then
|
|
echo "::error::libMobileGL.so defines no MGPipeVerifyInputs: -DMOBILEGL_PIPE_VERIFY=ON did not take, and the verify-split lane would compare nothing"
|
|
exit 1
|
|
fi
|
|
remote=$(nm --defined-only "${BUILD_DIR}/libMobileGL.so" | grep -c -i "MG_Remote" || true)
|
|
if [ "${remote}" -lt 1 ]; then
|
|
echo "::error::libMobileGL.so defines no MG_Remote symbol: -DMOBILEGL_BUILD_DISAGGREGATED=ON did not take, and every entry would run monolith under a name that says split"
|
|
exit 1
|
|
fi
|
|
echo "libMobileGL.so defines MGPipeVerifyInputs and ${remote} MG_Remote symbol(s) (${defined} defined symbols)"
|
|
|
|
- name: Show ccache stats
|
|
if: ${{ always() }}
|
|
run: ccache --show-stats
|
|
|
|
- name: Release superseded ccache entry
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true
|
|
|
|
- name: Save ccache
|
|
continue-on-error: true
|
|
uses: actions/cache/save@v5
|
|
with:
|
|
path: .ccache
|
|
key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1
|
|
|
|
- name: Prepare packaged test metadata
|
|
run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}"
|
|
|
|
# The unit tests are built here, so this flavor still compiles and links them, but they are
|
|
# not packaged: the one consumer, integration-verify-split, runs only the
|
|
# integration-verify-split label, whose entries all live under MG_IntegrationTest.
|
|
- name: Package verify-split runtime
|
|
run: |
|
|
mkdir -p ci-artifacts
|
|
mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort)
|
|
tar --exclude='*/CMakeFiles' --exclude='*.o' --exclude='*.a' --exclude='*.ninja*' \
|
|
--exclude='build.ninja' --exclude='cmake_install.cmake' \
|
|
-I 'zstd -T0' -cf ci-artifacts/mobilegl-linux-runtime-verify-split.tar.zst \
|
|
"${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" \
|
|
"${BUILD_DIR}/MobileGL/MG_IntegrationTest" "${SHARED_LIBS[@]}"
|
|
# compression-level 0: the tarball is zstd already (see 'Upload Linux runtime').
|
|
- name: Upload verify-split runtime
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: mobilegl-linux-runtime-verify-split
|
|
path: ci-artifacts/mobilegl-linux-runtime-verify-split.tar.zst
|
|
if-no-files-found: error
|
|
compression-level: 0
|
|
|
|
# P13 W1 (PLAN-P13 §4.2): THE SKIP CENSUS. Deleting a compile-time macro before the GTEST_SKIP
|
|
# or pull stub that guards on it turns tests "green by skipping"; a per-lane pass/fail count
|
|
# cannot see that. Every gtest lane uploads its JUnit as junit-*; this job keys each case by
|
|
# lane and name and holds the run against scripts/data/skip_census_baseline.json: a new skip
|
|
# (pass -> skip included) or a vanished name is red, a skip that turned into a pass is a note.
|
|
# A deliberate retirement edits the baseline in the same commit and names the cases.
|
|
skip-census:
|
|
name: skip census
|
|
runs-on: ubuntu-26.04
|
|
needs:
|
|
- build-guard
|
|
- test
|
|
- integration
|
|
- integration-verify
|
|
- integration-verify-split
|
|
- integration-split
|
|
- integration-split-strict
|
|
if: ${{ !cancelled() }}
|
|
steps:
|
|
- name: Checkout repo
|
|
uses: actions/checkout@v6
|
|
- name: Skip census self-test
|
|
run: python3 scripts/ci/skip_census.py --self-test
|
|
- name: Download lane JUnit
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
pattern: junit-*
|
|
path: junit
|
|
# The optional rows run on some paths only, and on the path this run took they are required:
|
|
# without a reused build (any build-guard verdict but identical) integration-split runs its
|
|
# unit lane and Transport A/B, and with one A/B still runs when the IPC defaults were not
|
|
# proven.
|
|
- name: Skip census
|
|
env:
|
|
GUARD_VERDICT: ${{ needs.build-guard.outputs.verdict }}
|
|
SPLIT_DEFAULTS_ON: ${{ needs.integration-split.outputs.defaults-on }}
|
|
run: |
|
|
required=()
|
|
if [ "${GUARD_VERDICT}" != identical ]; then
|
|
required+=(--require junit-integration-split/unit-split)
|
|
fi
|
|
if [ "${GUARD_VERDICT}" != identical ] || [ "${SPLIT_DEFAULTS_ON}" != true ]; then
|
|
required+=(--require junit-integration-split/arm-inproc --require junit-integration-split/arm-monolith)
|
|
fi
|
|
echo "build-guard verdict '${GUARD_VERDICT:-none}', split IPC defaults proven '${SPLIT_DEFAULTS_ON:-no}': ${required[*]:-no optional row required}"
|
|
python3 scripts/ci/skip_census.py check junit \
|
|
--baseline scripts/data/skip_census_baseline.json --report skip-census.json "${required[@]}"
|
|
- name: Upload census
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: skip-census
|
|
path: skip-census.json
|
|
if-no-files-found: warn
|