diff --git a/.github/workflows/apk.yml b/.github/workflows/apk.yml index d2e6ee6b..f39c1851 100644 --- a/.github/workflows/apk.yml +++ b/.github/workflows/apk.yml @@ -81,64 +81,12 @@ jobs: # Only the submodules the two APKs are built from, shallow and in parallel, in place of a # recursive checkout of every submodule in the tree: the ones the root CMakeLists.txt - # configures, and apitrace with the eight directories its thirdparty/CMakeLists.txt - # requires, for the trace APK. Each row names a submodule and one file the build reads - # from it, and a row whose file is missing afterwards fails the step: without - # 3rdparty/flatbuffers the root CMakeLists.txt would go on and build a library without - # MG_Remote instead of stopping. A submodule the build starts to use needs a row here. + # configures, and apitrace with the directories its thirdparty/CMakeLists.txt requires, + # for the trace APK. The step fails when a file the build reads from one of them did + # not arrive (scripts/ci/checkout_submodules.sh). - name: Check out build submodules timeout-minutes: 15 - run: | - top=( - "3rdparty/glslang CMakeLists.txt" - "3rdparty/SPIRV-Cross CMakeLists.txt" - "3rdparty/SPIRV-Reflect CMakeLists.txt" - "3rdparty/VulkanMemoryAllocator CMakeLists.txt" - "3rdparty/Vulkan-Headers CMakeLists.txt" - "3rdparty/Vulkan-Utility-Libraries CMakeLists.txt" - "3rdparty/xxHash build/cmake/CMakeLists.txt" - "3rdparty/asio include/asio.hpp" - "3rdparty/flatbuffers include/flatbuffers/flatbuffers.h" - "include/ska flat_hash_map.hpp" - "3rdparty/apitrace thirdparty/CMakeLists.txt" - ) - apitrace=( - "thirdparty/brotli LICENSE" - "thirdparty/gtest LICENSE" - "thirdparty/libbacktrace LICENSE" - "thirdparty/libpng LICENSE" - "thirdparty/snappy COPYING" - "thirdparty/zlib README" - "thirdparty/zstd LICENSE" - "thirdparty/directxmath LICENSE" - ) - # update ...: the update is idempotent, so a fetch that failed on - # the network is tried again, three times in all. Then every row's file must exist. - update() { - local dir="$1" paths=() row attempt updated=0 missing=0 - shift - for row in "$@"; do - paths+=("${row%% *}") - done - for attempt in 1 2 3; do - if git -C "${dir}" submodule update --init --depth=1 --jobs 8 -- "${paths[@]}"; then - updated=1 - break - fi - echo "the submodule update in ${dir} failed (attempt ${attempt} of 3)" >&2 - sleep $((attempt * 5)) - done - [ "${updated}" -eq 1 ] || return 1 - for row in "$@"; do - if [ ! -f "${dir}/${row%% *}/${row#* }" ]; then - echo "::error::${dir}/${row%% *}/${row#* } is missing after the submodule update, so the APK build cannot use ${row%% *}" - missing=1 - fi - done - return "${missing}" - } - update . "${top[@]}" - update 3rdparty/apitrace "${apitrace[@]}" + run: bash scripts/ci/checkout_submodules.sh retrace - name: Set artifact metadata run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87733d5d..8265b0fb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,8 +56,12 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v6 - with: - submodules: recursive + + # Only the submodules the build reads, shallow and in parallel; the script fails when one + # of them did not arrive (scripts/ci/checkout_submodules.sh). + - name: Check out build submodules + timeout-minutes: 15 + run: bash scripts/ci/checkout_submodules.sh library - name: Set swap space uses: ./.github/actions/set-swap-space @@ -198,6 +202,8 @@ jobs: - name: Prepare packaged test metadata run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}" + # zstd on every core: the tarball is most of a gigabyte of test executables, and every + # consumer job waits for it. `tar --zstd -xf` reads the result unchanged. - name: Package Linux runtime run: | mkdir -p ci-artifacts @@ -209,7 +215,7 @@ jobs: --exclude='*.ninja*' \ --exclude='build.ninja' \ --exclude='cmake_install.cmake' \ - --zstd -cf ci-artifacts/mobilegl-linux-runtime.tar.zst \ + -I 'zstd -T0' -cf ci-artifacts/mobilegl-linux-runtime.tar.zst \ "${BUILD_DIR}/runtime-mode.json" \ "${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" \ "${BUILD_DIR}/MobileGL/MG_Test" \ @@ -217,12 +223,15 @@ jobs: "${BUILD_DIR}/MobileGL/MG_IntegrationTest" \ "${SHARED_LIBS[@]}" + # compression-level 0: the tarball is zstd already, and deflating it again in the upload + # only costs time. The artifact stays a zip holding this one file, as the consumers expect. - name: Upload Linux runtime uses: actions/upload-artifact@v7 with: name: mobilegl-linux-runtime path: ci-artifacts/mobilegl-linux-runtime.tar.zst if-no-files-found: error + compression-level: 0 # The retrace legs load only the library, and the server image on the spawn arm, so they # take this instead of the whole test runtime. @@ -245,17 +254,24 @@ jobs: build-linux-monolith-control: name: build push monolith control (no MG_Remote) runs-on: ubuntu-26.04 + # actions: write is for this job's own rolling ccache entry, as in build-linux. + permissions: + actions: write + contents: read env: BUILD_DIR: build-push-control CCACHE_BASEDIR: ${{ github.workspace }} CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_COMPRESS: "true" + CCACHE_MAXSIZE: 4G CCACHE_NOHASHDIR: "true" steps: - name: Checkout repo uses: actions/checkout@v6 - with: - submodules: recursive + + - name: Check out build submodules + timeout-minutes: 15 + run: bash scripts/ci/checkout_submodules.sh library - name: Set swap space uses: ./.github/actions/set-swap-space @@ -270,11 +286,17 @@ jobs: uses: lukka/get-cmake@v4.3.3 with: cmakeVersion: 3.31.10 - - name: Restore compiler cache + # This job's own entry: its objects are compiled without MG_Remote, which no other build + # produces. The second restore key is build-linux's entry, which seeds the third-party + # objects on a branch's first run. + - name: Restore ccache uses: actions/cache/restore@v5 with: path: .ccache - key: ${{ runner.os }}-ubuntu26-cpu-test-build-linux-ccache-v1 + key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 + restore-keys: | + ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache- + ${{ runner.os }}-ubuntu26-cpu-test-build-linux-ccache- - name: Prepare Vulkan SDK id: netsetup230 continue-on-error: true @@ -306,11 +328,14 @@ jobs: # The push monolith without MG_Remote, the shape FCL embeds. Its runtime_mode_proof is the # zero-MG_Remote-symbol check: a push library that links any MG_Remote symbol is red here. # Its library is uploaded for retrace-shipped, a positive replay of that shape. + # ENABLE_PCH=OFF: ccache cannot cache a compile that uses glslang's precompiled header + # (see scripts/ci/configure_build.sh). - name: Configure push control without MG_Remote run: | cmake -S . -B "${BUILD_DIR}" -G Ninja \ -DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DENABLE_PCH=OFF \ -DCMAKE_BUILD_TYPE=Release -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=OFF -DMOBILEGL_BUILD_BENCHMARK=OFF \ -DMOBILEGL_BUILD_INTEGRATION_TEST=OFF -DMOBILEGL_BUILD_TRACE_REPLAY=OFF \ @@ -322,6 +347,26 @@ jobs: cmake --build "${BUILD_DIR}" --target MobileGL --parallel "$(nproc)" python3 scripts/ci/runtime_mode_proof.py build --build-dir "${BUILD_DIR}" \ --mode monolith --output "${BUILD_DIR}/runtime-mode.json" + + - name: Show ccache stats + if: ${{ always() }} + run: ccache --show-stats + + # One rolling entry per job and branch, as in build-linux: this branch's superseded blob is + # released before the save, and a failed build leaves the entry untouched. + - name: Release superseded ccache entry + env: + GH_TOKEN: ${{ github.token }} + CACHE_KEY: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 + run: gh cache delete "${CACHE_KEY}" --ref "${GITHUB_REF}" || true + + - name: Save ccache + continue-on-error: true + uses: actions/cache/save@v5 + with: + path: .ccache + key: ${{ runner.os }}-ubuntu26-cpu-test-${{ github.job }}-ccache-v1 + - name: Prepare packaged test metadata run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}" @@ -613,8 +658,10 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v6 - with: - submodules: recursive + + - name: Check out build submodules + timeout-minutes: 15 + run: bash scripts/ci/checkout_submodules.sh library - name: Set swap space uses: ./.github/actions/set-swap-space @@ -688,12 +735,15 @@ jobs: # Both disaggregation options default ON, so they are pinned OFF here: a disaggregated # library expects a server-role log that this monolith lane never writes. # integration-verify-split is the split arm. + # ENABLE_PCH=OFF: ccache cannot cache a compile that uses glslang's precompiled header + # (see scripts/ci/configure_build.sh). run: | cmake -S . -B "${BUILD_DIR}" -G Ninja \ -DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \ -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DENABLE_PCH=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=ON \ @@ -776,18 +826,20 @@ jobs: --exclude='*.ninja*' \ --exclude='build.ninja' \ --exclude='cmake_install.cmake' \ - --zstd -cf ci-artifacts/mobilegl-linux-runtime-verify.tar.zst \ + -I 'zstd -T0' -cf ci-artifacts/mobilegl-linux-runtime-verify.tar.zst \ "${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" \ "${BUILD_DIR}/MobileGL/MG_Test" \ "${BUILD_DIR}/MobileGL/MG_IntegrationTest" \ "${SHARED_LIBS[@]}" + # compression-level 0: the tarball is zstd already (see 'Upload Linux runtime'). - name: Upload Linux verify runtime uses: actions/upload-artifact@v7 with: name: mobilegl-linux-runtime-verify path: ci-artifacts/mobilegl-linux-runtime-verify.tar.zst if-no-files-found: error + compression-level: 0 # The retrace legs load only the library, and the server image on the spawn arm, so they # take this instead of the whole test runtime. @@ -1135,8 +1187,15 @@ jobs: continue-on-error: true timeout-minutes: 15 uses: actions/checkout@v6 - with: - submodules: recursive + + # The same submodule set as the two builds this job compares. The compare step runs only + # when every one of them arrived: without 3rdparty/flatbuffers both lines would configure + # the same library without MG_Remote, and compare as identical. + - name: Check out build submodules + id: submodules + continue-on-error: true + timeout-minutes: 15 + run: bash scripts/ci/checkout_submodules.sh library - name: Configure runner environment continue-on-error: true @@ -1194,8 +1253,12 @@ jobs: timeout-minutes: 45 env: CI_BUILD_TYPE: ${{ secrets.ACTIONS_STEP_DEBUG == 'true' && 'Debug' || 'Release' }} + SUBMODULES_OUTCOME: ${{ steps.submodules.outcome }} run: | - if python3 scripts/ci/build_fingerprint.py --self-test > "${RUNNER_TEMP}/fingerprint-self-test.txt"; then + if [ "${SUBMODULES_OUTCOME}" != success ]; then + echo "::warning::the submodule checkout did not succeed (outcome '${SUBMODULES_OUTCOME}'), so the guard decides nothing" + echo "verdict=unknown" >> "${GITHUB_OUTPUT}" + elif python3 scripts/ci/build_fingerprint.py --self-test > "${RUNNER_TEMP}/fingerprint-self-test.txt"; then python3 scripts/ci/build_fingerprint.py guard --primary-dir build-guard-primary \ --split-dir build-guard-split --deps-dir build-guard-deps \ --github-output verdict --report "${RUNNER_TEMP}/build-guard.json" @@ -1257,12 +1320,16 @@ jobs: - name: Checkout repo if: env.SPLIT_REUSE != 'true' uses: actions/checkout@v6 - with: - # recursive, and load-bearing here beyond the usual: MOBILEGL_BUILD_DISAGGREGATED - # shadows itself back to OFF when 3rdparty/flatbuffers/include is missing (a normal - # variable rather than a cache force, deliberately). A shallow checkout would therefore - # configure cleanly, build a monolith library, and be caught only by the nm step below. - submodules: recursive + + # The submodule list is load-bearing here beyond the usual: MOBILEGL_BUILD_DISAGGREGATED + # shadows itself back to OFF when 3rdparty/flatbuffers/include is missing (a normal + # variable rather than a cache force, deliberately), so a checkout without it would + # configure cleanly and build a monolith library. The script fails when the header is not + # there, and the nm step below is the second check. + - name: Check out build submodules + if: env.SPLIT_REUSE != 'true' + timeout-minutes: 15 + run: bash scripts/ci/checkout_submodules.sh library - name: Configure runner environment if: env.SPLIT_REUSE != 'true' @@ -1431,12 +1498,13 @@ jobs: --exclude='*.ninja*' \ --exclude='build.ninja' \ --exclude='cmake_install.cmake' \ - --zstd -cf ci-artifacts/mobilegl-linux-runtime-split.tar.zst \ + -I 'zstd -T0' -cf ci-artifacts/mobilegl-linux-runtime-split.tar.zst \ "${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" \ "${BUILD_DIR}/MobileGL/MG_Test" \ "${BUILD_DIR}/MobileGL/MG_IntegrationTest" \ "${SHARED_LIBS[@]}" + # compression-level 0: the tarball is zstd already (see 'Upload Linux runtime'). - name: Upload Linux split runtime if: env.SPLIT_REUSE != 'true' uses: actions/upload-artifact@v7 @@ -1444,6 +1512,7 @@ jobs: name: mobilegl-linux-runtime-split path: ci-artifacts/mobilegl-linux-runtime-split.tar.zst if-no-files-found: error + compression-level: 0 # The retrace legs load only the library, and the server image on the spawn arm, so they # take this instead of the whole test runtime. @@ -1580,9 +1649,22 @@ jobs: # the build graph (see gen_protocol.py), so this builds the same pinned flatc as flatc-check, # and the lane step requires both entries PASSED by name through junit_tally.py. --check keeps # this step from regenerating the committed header; checking that header is flatc-check's job. + # The binary comes from the cache entry flatc-check also uses, keyed on the submodule's + # commit; gen_protocol.py builds it when the entry is missing and checks its version against + # the submodule either way. - name: Check out FlatBuffers (TCP lane) run: git submodule update --init 3rdparty/flatbuffers + - name: Pinned flatc cache key + id: flatc + run: echo "commit=$(git rev-parse HEAD:3rdparty/flatbuffers)" >> "${GITHUB_OUTPUT}" + + - name: Cache pinned flatc + uses: actions/cache@v5 + with: + path: ${{ runner.temp }}/flatc-build/flatc + key: ${{ runner.os }}-ubuntu26-flatc-${{ steps.flatc.outputs.commit }} + - name: Build pinned flatc (TCP lane) run: python3 scripts/gen_protocol.py --build-dir "${RUNNER_TEMP}/flatc-build" --check @@ -1595,6 +1677,10 @@ jobs: # An entry whose owning package (c1 client, s1 session, v1 server) has not landed SKIPS with # the reason and never goes green - MG_IntegrationTest/CMakeLists.txt probes MG_Remote for # c0's signature stubs and disarms the lane while any remain. + # + # The three arms run as one ctest invocation: the TCP arm holds one resource lock, so on its + # own it runs one entry at a time and leaves the other cores idle. Each arm keeps its own + # selection, JUnit and checks. - name: Split lanes (inproc, spawn, TCP) working-directory: ${{ env.SPLIT_BUILD_DIR }} env: @@ -1603,16 +1689,45 @@ jobs: ulimit -c unlimited export MOBILEGL_FLATC_EXECUTABLE="${RUNNER_TEMP}/flatc-build/flatc" test -x "${MOBILEGL_FLATC_EXECUTABLE}" - ctest --output-on-failure -L integration-split --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/split-baseline.xml" + # Each arm is selected by its own label regex. scripts/ci/lane_selection.py turns the + # three selections into one list of exact names and fails when an arm selects nothing, + # which is what --no-tests=error says for an arm run on its own. junit_tally.py then + # carves each arm's JUnit out of the run, and fails when a selected name is missing + # from the run or reported twice. + ctest --show-only=json-v1 -L integration-split > "${RUNNER_TEMP}/split-discovery.json" + ctest --show-only=json-v1 -L integration-spawn > "${RUNNER_TEMP}/spawn-discovery.json" + ctest --show-only=json-v1 -L '^integration-tcp$' > "${RUNNER_TEMP}/tcp-discovery.json" + # A directory of its own: the job uploads RUNNER_TEMP/*.xml for the skip census, which + # has rows for the three arms and none for the run that holds them all. + mkdir -p "${RUNNER_TEMP}/split-arms" + python3 ../scripts/ci/lane_selection.py union "${RUNNER_TEMP}/split-arms/tests.txt" \ + split="${RUNNER_TEMP}/split-discovery.json" spawn="${RUNNER_TEMP}/spawn-discovery.json" \ + tcp="${RUNNER_TEMP}/tcp-discovery.json" + rc=0 + ctest --output-on-failure --tests-from-file "${RUNNER_TEMP}/split-arms/tests.txt" --no-tests=error -j 4 \ + --output-junit "${RUNNER_TEMP}/split-arms/all.xml" || rc=$? + # Carved before the exit code is acted on, so a red run still leaves each arm's JUnit + # for the annotations and the skip census. + carved=0 + for arm in split spawn tcp; do + python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/split-arms/all.xml" \ + --select-from "${RUNNER_TEMP}/${arm}-discovery.json" \ + --write-selected "${RUNNER_TEMP}/${arm}-baseline.xml" || carved=1 + done + if [ "${rc}" -ne 0 ]; then + echo "::error::the split, spawn and TCP arms' run failed (ctest exit ${rc})" + exit "${rc}" + fi + if [ "${carved}" -ne 0 ]; then + echo "::error::the run does not hold every entry of the split, spawn and TCP arms exactly once, so it cannot stand for the three arms" + exit 1 + fi python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/split-baseline.xml" --require-split-ran # Exit gate 9.3: the spawn arm green at the same count. It runs the same cases through a - # SECOND PROCESS, so a failure here that the line above did not see is a transport fact + # second process, so a failure here that the inproc arm did not see is a transport fact # and not a phase one - which is the whole reason both lanes exist. - ctest --output-on-failure -L integration-spawn --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/spawn-baseline.xml" # PH-6 / fuzz arm 3: the supervisor-level forfeit control rides this lane and must RUN. python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/spawn-baseline.xml" --require-spawn-ran --require-entry-passed SpawnLane.EventForfeitPeer - ctest --show-only=json-v1 -L '^integration-tcp$' > "${RUNNER_TEMP}/tcp-discovery.json" - ctest --output-on-failure -L '^integration-tcp$' --no-tests=error -j 4 --output-junit "${RUNNER_TEMP}/tcp-baseline.xml" python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/tcp-baseline.xml" --require-tcp-ran --require-run-ahead --discovery "${RUNNER_TEMP}/tcp-discovery.json" python3 ../scripts/ci/junit_tally.py "${RUNNER_TEMP}/tcp-baseline.xml" --require-entry-passed TcpLane.SupervisorProtocolControls --require-entry-passed TcpLane.EventForfeitPeer --require-entry-passed TcpLane.ControlFrameFuzz @@ -2401,6 +2516,20 @@ jobs: # SPIRV-Cross or the trace fixtures. run: git submodule update --init 3rdparty/flatbuffers + # The pinned flatc, cached under the submodule's commit: gen_protocol.py builds it only + # when the build directory holds none, and checks its version against the submodule + # either way. The gitlink gives the commit without the submodule's own checkout. + # integration-split restores the same entry, so the path is the same string in both jobs. + - name: Pinned flatc cache key + id: flatc + run: echo "commit=$(git rev-parse HEAD:3rdparty/flatbuffers)" >> "${GITHUB_OUTPUT}" + + - name: Cache pinned flatc + uses: actions/cache@v5 + with: + path: ${{ runner.temp }}/flatc-build/flatc + key: ${{ runner.os }}-ubuntu26-flatc-${{ steps.flatc.outputs.commit }} + - name: Regenerate protocol_generated.h run: python3 scripts/gen_protocol.py --build-dir "${{ runner.temp }}/flatc-build" @@ -2533,8 +2662,12 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v6 - with: - submodules: recursive + + # The library set plus apitrace and the directories its build requires + # (scripts/ci/checkout_submodules.sh). + - name: Check out build submodules + timeout-minutes: 15 + run: bash scripts/ci/checkout_submodules.sh retrace - name: Set swap space uses: ./.github/actions/set-swap-space @@ -3672,6 +3805,15 @@ jobs: python3 MobileGL/MG_IntegrationTest/Harness/test_split_log_paths.py python3 scripts/ci/transport_pinned.py --self-test + # The helpers the build and split-lane jobs run before anything else can fail for them: the + # union of the split arms' selections, and the submodule lists, which are printed here + # without checking anything out. + - name: Build and lane helper self-tests + run: | + python3 scripts/ci/lane_selection.py --self-test + bash scripts/ci/checkout_submodules.sh --list library + bash scripts/ci/checkout_submodules.sh --list retrace + # Warning only for now: the disaggregation documents are still being written, and a # lint that fails a rewrite in progress teaches people to ignore it. It becomes # --strict when the documents settle. @@ -3701,10 +3843,12 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v6 - with: - # recursive for build-linux-split's reason: MOBILEGL_BUILD_DISAGGREGATED shadows itself - # back to OFF when 3rdparty/flatbuffers/include is missing. - submodules: recursive + + # For build-linux-split's reason: MOBILEGL_BUILD_DISAGGREGATED shadows itself back to OFF + # when 3rdparty/flatbuffers/include is missing, and the script fails when it is. + - name: Check out build submodules + timeout-minutes: 15 + run: bash scripts/ci/checkout_submodules.sh library - name: Set swap space uses: ./.github/actions/set-swap-space @@ -3763,12 +3907,15 @@ jobs: # Release/INFO for build-linux-verify's reason, and INFO is load-bearing here for # build-linux-split's: ConfigLoader's resolved-transport line is logged at INFO and the # arm-proof step below reads it back out of every entry's client log. + # ENABLE_PCH=OFF: ccache cannot cache a compile that uses glslang's precompiled header + # (see scripts/ci/configure_build.sh). run: | cmake -S . -B "${BUILD_DIR}" -G Ninja \ -DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" \ -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DENABLE_PCH=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DMOBILEGL_LOG_ACTIVE_LEVEL=MOBILEGL_LOG_LEVEL_INFO \ -DMOBILEGL_BUILD_TEST=ON \ @@ -3825,21 +3972,26 @@ jobs: - name: Prepare packaged test metadata run: python3 scripts/ci/prepare_test_runtime.py "${BUILD_DIR}" + # The unit tests are built here, so this flavor still compiles and links them, but they are + # not packaged: the one consumer, integration-verify-split, runs only the + # integration-verify-split label, whose entries all live under MG_IntegrationTest. - name: Package verify-split runtime run: | mkdir -p ci-artifacts mapfile -t SHARED_LIBS < <(find "${BUILD_DIR}" -type f \( -name '*.so' -o -name '*.so.*' \) -print | sort) tar --exclude='*/CMakeFiles' --exclude='*.o' --exclude='*.a' --exclude='*.ninja*' \ --exclude='build.ninja' --exclude='cmake_install.cmake' \ - --zstd -cf ci-artifacts/mobilegl-linux-runtime-verify-split.tar.zst \ - "${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" "${BUILD_DIR}/MobileGL/MG_Test" \ + -I 'zstd -T0' -cf ci-artifacts/mobilegl-linux-runtime-verify-split.tar.zst \ + "${BUILD_DIR}/.ci-source-root" "${BUILD_DIR}/CTestTestfile.cmake" \ "${BUILD_DIR}/MobileGL/MG_IntegrationTest" "${SHARED_LIBS[@]}" + # compression-level 0: the tarball is zstd already (see 'Upload Linux runtime'). - name: Upload verify-split runtime uses: actions/upload-artifact@v7 with: name: mobilegl-linux-runtime-verify-split path: ci-artifacts/mobilegl-linux-runtime-verify-split.tar.zst if-no-files-found: error + compression-level: 0 # P13 W1 (PLAN-P13 ยง4.2): THE SKIP CENSUS. Deleting a compile-time macro before the GTEST_SKIP # or pull stub that guards on it turns tests "green by skipping"; a per-lane pass/fail count diff --git a/docs/ci/ci-cleanup-log.md b/docs/ci/ci-cleanup-log.md index a09a263b..447b6186 100644 --- a/docs/ci/ci-cleanup-log.md +++ b/docs/ci/ci-cleanup-log.md @@ -69,6 +69,7 @@ them wrong. | 2026-11-15 | per-case `trace fixture ()` jobs, their per-case cache entries and artifacts, `.github/scripts/trace-fixture-cache.sh` | Test and MobileGL APK / `trace-fixtures` | 40 jobs and 40 cache entries per workflow for files that change rarely | one content-addressed cache entry for all Git LFS fixtures (`.github/scripts/trace-fixture-bundle.sh`); a replay job restores it and downloads a case only when the entry does not hold it | | 2026-11-15 | `remove-artifact-clutter` job | Test and MobileGL APK | it deleted the fixture artifacts and the green result artifacts after the run; the fixture artifacts no longer exist | the delete step at the end of `retrace-summary`; on Linux `retention-days: 1` on the summary inputs as well, `retrace-split` and `retrace-shipped` upload results only for a failed shard (their green results were deleted unread before), and `retrace-verify` keeps its results for every shard as before | | 2026-11-15 | emulator replays of the cases outside `android_smoke` on the monolith control and spawn acceptance lanes (56 of 79 rows per lane; the 12 smoke cases and the whole inproc lane stay) | MobileGL APK / `retrace` | the 237 emulator replays of a push held the 20 runner slots for most of an hour, and the launcher ships inproc; what goes is the same-commit Android monolith control and the Android spawn evidence (server exec from the native library directory, the pbuffer surface) for those 56 rows | Test replays every case on monolith, inproc and spawn on Linux; the Android inproc lane replays every case; `workflow_dispatch` with `android_full_matrix` runs the full emulator matrix; `trace_cases.py --self-test-transport-matrices` checks that the reduced lanes are the full matrix less exactly those rows | +| 2026-11-15 | the three separate ctest invocations of `Split lanes (inproc, spawn, TCP)` (`-L integration-split`, `-L integration-spawn`, `-L '^integration-tcp$'`) | Test / `integration-split` | every TCP entry holds one resource lock, so the TCP invocation ran one entry at a time while the other cores were idle; the three selections share no entry | one invocation of the union of the three selections: each arm is still discovered with its own label regex, `lane_selection.py union` writes the exact names and fails when an arm selects nothing, and `junit_tally.py --select-from/--write-selected` carves each arm's JUnit out of the run under its old stem and fails when a selected name is missing or reported twice; the per-arm tallies, the entries required by name, the run-ahead proof and the skip-census rows are unchanged | ## Kept after investigation diff --git a/scripts/ci/checkout_submodules.sh b/scripts/ci/checkout_submodules.sh new file mode 100644 index 00000000..5731e495 --- /dev/null +++ b/scripts/ci/checkout_submodules.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# MobileGL - scripts/ci/checkout_submodules.sh +# Copyright (c) 2025-2026 MobileGL-Dev +# Licensed under the GNU Lesser General Public License v3.0: +# https://www.gnu.org/licenses/gpl-3.0.txt +# https://www.gnu.org/licenses/lgpl-3.0.txt +# SPDX-License-Identifier: LGPL-3.0-only +# End of Source File Header + +# The submodules a CI build reads, checked out shallow and in parallel, in place of a +# recursive checkout of every submodule in the tree. +# library what the root CMakeLists.txt configures for the library, its tests and benchmarks. +# retrace the library set plus the trace replay tool's: apitrace and the eight directories +# its thirdparty/CMakeLists.txt requires. +# Each row names a submodule and one file the build reads from it. A row whose file is missing +# after the update fails the script: the configure would otherwise go on without it, and for +# 3rdparty/flatbuffers the root CMakeLists.txt then builds a library without MG_Remote instead +# of stopping. A submodule the build starts to use needs a row here. +# +# usage: checkout_submodules.sh (from the repository root) +# checkout_submodules.sh --list prints the rows, checks out nothing +set -euo pipefail + +usage="usage: checkout_submodules.sh [--list] " + +list_only=false +if [ "${1:-}" = --list ]; then + list_only=true + shift +fi +set_name="${1:?${usage}}" + +# +library=( + "3rdparty/glslang CMakeLists.txt" + "3rdparty/SPIRV-Cross CMakeLists.txt" + "3rdparty/SPIRV-Reflect CMakeLists.txt" + "3rdparty/VulkanMemoryAllocator CMakeLists.txt" + "3rdparty/Vulkan-Headers CMakeLists.txt" + "3rdparty/Vulkan-Utility-Libraries CMakeLists.txt" + "3rdparty/xxHash build/cmake/CMakeLists.txt" + "3rdparty/asio include/asio.hpp" + "3rdparty/flatbuffers include/flatbuffers/flatbuffers.h" + "include/ska flat_hash_map.hpp" +) +retrace=( + "3rdparty/apitrace thirdparty/CMakeLists.txt" +) +# Inside 3rdparty/apitrace: the files its thirdparty/CMakeLists.txt tests before it configures. +apitrace=( + "thirdparty/brotli LICENSE" + "thirdparty/gtest LICENSE" + "thirdparty/libbacktrace LICENSE" + "thirdparty/libpng LICENSE" + "thirdparty/snappy COPYING" + "thirdparty/zlib README" + "thirdparty/zstd LICENSE" + "thirdparty/directxmath LICENSE" +) + +case "${set_name}" in + library) top=("${library[@]}"); nested=() ;; + retrace) top=("${library[@]}" "${retrace[@]}"); nested=("${apitrace[@]}") ;; + *) + echo "checkout_submodules.sh: unknown set '${set_name}' (library or retrace)" >&2 + echo "${usage}" >&2 + exit 2 + ;; +esac + +if [ "${list_only}" = true ]; then + printf '%s\n' "${top[@]}" + if [ "${#nested[@]}" -gt 0 ]; then + printf '3rdparty/apitrace/%s\n' "${nested[@]}" + fi + exit 0 +fi + +# update ...: one shallow, parallel update of the rows' submodules. The update +# is idempotent, so a fetch that failed on the network is tried again, three times in all. +update() { + local dir="$1" + shift + local paths=() row attempt + for row in "$@"; do + paths+=("${row%% *}") + done + for attempt in 1 2 3; do + if git -C "${dir}" submodule update --init --depth=1 --jobs 8 -- "${paths[@]}"; then + return 0 + fi + echo "checkout_submodules.sh: the submodule update in ${dir} failed (attempt ${attempt} of 3)" >&2 + if [ "${attempt}" -lt 3 ]; then + sleep $((attempt * 5)) + fi + done + return 1 +} + +# verify ...: every row's file exists; prints each missing one and returns 1. +verify() { + local dir="$1" missing=0 row path file + shift + for row in "$@"; do + path="${row%% *}" + file="${row#* }" + if [ ! -f "${dir}/${path}/${file}" ]; then + echo "::error::checkout_submodules.sh: ${dir}/${path}/${file} is missing after the submodule update, so the '${set_name}' build cannot use ${path}" + missing=1 + fi + done + return "${missing}" +} + +update . "${top[@]}" +verify . "${top[@]}" +if [ "${#nested[@]}" -gt 0 ]; then + update 3rdparty/apitrace "${nested[@]}" + verify 3rdparty/apitrace "${nested[@]}" +fi +echo "checkout_submodules.sh: the '${set_name}' set is checked out ($(( ${#top[@]} + ${#nested[@]} )) submodules)" diff --git a/scripts/ci/configure_build.sh b/scripts/ci/configure_build.sh index dc1b10f5..924f6943 100644 --- a/scripts/ci/configure_build.sh +++ b/scripts/ci/configure_build.sh @@ -25,12 +25,17 @@ variant="${1:?usage: configure_build.sh }" build_dir="${2:?usage: configure_build.sh }" shift 2 +# ENABLE_PCH is glslang's option, on by default. ccache cannot cache a compile that uses a +# precompiled header, so with it on every glslang translation unit is compiled again in every +# run. It is in the common part so that both lines stay the same configuration for +# build_fingerprint.py. common=( -S . -B "${build_dir}" -G Ninja -DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}" -DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}" -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + -DENABLE_PCH=OFF ) case "${variant}" in diff --git a/scripts/ci/lane_selection.py b/scripts/ci/lane_selection.py new file mode 100644 index 00000000..66833d64 --- /dev/null +++ b/scripts/ci/lane_selection.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# MobileGL - scripts/ci/lane_selection.py +# Copyright (c) 2025-2026 MobileGL-Dev +# Licensed under the GNU Lesser General Public License v3.0: +# https://www.gnu.org/licenses/gpl-3.0.txt +# https://www.gnu.org/licenses/lgpl-3.0.txt +# SPDX-License-Identifier: LGPL-3.0-only +# End of Source File Header + +"""One ctest run for several lanes: the exact names of every lane, as one --tests-from-file list. + +A lane is a `ctest -L