[CI] (.github/workflows, docs/ci, scripts/ci): trim setup time in the build and split lane jobs

This commit is contained in:
2026-11-15 11:47:46 -04:00
parent 8165793e0d
commit 25c984927d
6 changed files with 454 additions and 89 deletions
+121
View File
@@ -0,0 +1,121 @@
#!/bin/bash
# MobileGL - scripts/ci/checkout_submodules.sh
# Copyright (c) 2025-2026 MobileGL-Dev
# Licensed under the GNU Lesser General Public License v3.0:
# https://www.gnu.org/licenses/gpl-3.0.txt
# https://www.gnu.org/licenses/lgpl-3.0.txt
# SPDX-License-Identifier: LGPL-3.0-only
# End of Source File Header
# The submodules a CI build reads, checked out shallow and in parallel, in place of a
# recursive checkout of every submodule in the tree.
# library what the root CMakeLists.txt configures for the library, its tests and benchmarks.
# retrace the library set plus the trace replay tool's: apitrace and the eight directories
# its thirdparty/CMakeLists.txt requires.
# Each row names a submodule and one file the build reads from it. A row whose file is missing
# after the update fails the script: the configure would otherwise go on without it, and for
# 3rdparty/flatbuffers the root CMakeLists.txt then builds a library without MG_Remote instead
# of stopping. A submodule the build starts to use needs a row here.
#
# usage: checkout_submodules.sh <library|retrace> (from the repository root)
# checkout_submodules.sh --list <library|retrace> prints the rows, checks out nothing
set -euo pipefail
usage="usage: checkout_submodules.sh [--list] <library|retrace>"
list_only=false
if [ "${1:-}" = --list ]; then
list_only=true
shift
fi
set_name="${1:?${usage}}"
# <submodule path> <file the build reads from it>
library=(
"3rdparty/glslang CMakeLists.txt"
"3rdparty/SPIRV-Cross CMakeLists.txt"
"3rdparty/SPIRV-Reflect CMakeLists.txt"
"3rdparty/VulkanMemoryAllocator CMakeLists.txt"
"3rdparty/Vulkan-Headers CMakeLists.txt"
"3rdparty/Vulkan-Utility-Libraries CMakeLists.txt"
"3rdparty/xxHash build/cmake/CMakeLists.txt"
"3rdparty/asio include/asio.hpp"
"3rdparty/flatbuffers include/flatbuffers/flatbuffers.h"
"include/ska flat_hash_map.hpp"
)
retrace=(
"3rdparty/apitrace thirdparty/CMakeLists.txt"
)
# Inside 3rdparty/apitrace: the files its thirdparty/CMakeLists.txt tests before it configures.
apitrace=(
"thirdparty/brotli LICENSE"
"thirdparty/gtest LICENSE"
"thirdparty/libbacktrace LICENSE"
"thirdparty/libpng LICENSE"
"thirdparty/snappy COPYING"
"thirdparty/zlib README"
"thirdparty/zstd LICENSE"
"thirdparty/directxmath LICENSE"
)
case "${set_name}" in
library) top=("${library[@]}"); nested=() ;;
retrace) top=("${library[@]}" "${retrace[@]}"); nested=("${apitrace[@]}") ;;
*)
echo "checkout_submodules.sh: unknown set '${set_name}' (library or retrace)" >&2
echo "${usage}" >&2
exit 2
;;
esac
if [ "${list_only}" = true ]; then
printf '%s\n' "${top[@]}"
if [ "${#nested[@]}" -gt 0 ]; then
printf '3rdparty/apitrace/%s\n' "${nested[@]}"
fi
exit 0
fi
# update <directory> <row>...: one shallow, parallel update of the rows' submodules. The update
# is idempotent, so a fetch that failed on the network is tried again, three times in all.
update() {
local dir="$1"
shift
local paths=() row attempt
for row in "$@"; do
paths+=("${row%% *}")
done
for attempt in 1 2 3; do
if git -C "${dir}" submodule update --init --depth=1 --jobs 8 -- "${paths[@]}"; then
return 0
fi
echo "checkout_submodules.sh: the submodule update in ${dir} failed (attempt ${attempt} of 3)" >&2
if [ "${attempt}" -lt 3 ]; then
sleep $((attempt * 5))
fi
done
return 1
}
# verify <directory> <row>...: every row's file exists; prints each missing one and returns 1.
verify() {
local dir="$1" missing=0 row path file
shift
for row in "$@"; do
path="${row%% *}"
file="${row#* }"
if [ ! -f "${dir}/${path}/${file}" ]; then
echo "::error::checkout_submodules.sh: ${dir}/${path}/${file} is missing after the submodule update, so the '${set_name}' build cannot use ${path}"
missing=1
fi
done
return "${missing}"
}
update . "${top[@]}"
verify . "${top[@]}"
if [ "${#nested[@]}" -gt 0 ]; then
update 3rdparty/apitrace "${nested[@]}"
verify 3rdparty/apitrace "${nested[@]}"
fi
echo "checkout_submodules.sh: the '${set_name}' set is checked out ($(( ${#top[@]} + ${#nested[@]} )) submodules)"
+5
View File
@@ -25,12 +25,17 @@ variant="${1:?usage: configure_build.sh <primary|split> <build dir>}"
build_dir="${2:?usage: configure_build.sh <primary|split> <build dir>}"
shift 2
# ENABLE_PCH is glslang's option, on by default. ccache cannot cache a compile that uses a
# precompiled header, so with it on every glslang translation unit is compiled again in every
# run. It is in the common part so that both lines stay the same configuration for
# build_fingerprint.py.
common=(
-S . -B "${build_dir}" -G Ninja
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}"
-DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}"
-DCMAKE_C_COMPILER_LAUNCHER=ccache
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-DENABLE_PCH=OFF
)
case "${variant}" in
+138
View File
@@ -0,0 +1,138 @@
#!/usr/bin/env python3
# MobileGL - scripts/ci/lane_selection.py
# Copyright (c) 2025-2026 MobileGL-Dev
# Licensed under the GNU Lesser General Public License v3.0:
# https://www.gnu.org/licenses/gpl-3.0.txt
# https://www.gnu.org/licenses/lgpl-3.0.txt
# SPDX-License-Identifier: LGPL-3.0-only
# End of Source File Header
"""One ctest run for several lanes: the exact names of every lane, as one --tests-from-file list.
A lane is a `ctest -L <label regex>` selection. Running the lanes of one job as one ctest
invocation lets a lane that holds a resource lock (the TCP arm runs one entry at a time) share
the run with the others. The lanes keep their own selections: each is discovered with its own
label regex, this script writes the union of the discovered names, ctest runs that list, and
junit_tally.py --select-from/--write-selected carves each lane's JUnit back out, which fails
when a lane's name is missing from the run or reported twice.
lane_selection.py union OUT.txt LANE=DISCOVERY.json [LANE=DISCOVERY.json ...]
lane_selection.py --self-test
DISCOVERY.json is `ctest -L <regex> --show-only=json-v1`. union fails when a lane selects
nothing (what --no-tests=error says for a lane run on its own), names an entry twice, or holds a
name a tests-from-file line cannot carry. It prints each lane's count and the size of the union.
"""
import json
from pathlib import Path
import sys
import tempfile
def lane_names(label, path):
"""The names one discovery selects, in order; raises ValueError when the lane is unusable."""
names = [test['name'] for test in json.loads(Path(path).read_text()).get('tests', [])]
if not names:
raise ValueError(f'lane {label}: {path} selects no test')
if len(names) != len(set(names)):
twice = sorted({name for name in names if names.count(name) > 1})
raise ValueError(f'lane {label}: {path} names {len(twice)} entr(ies) twice, e.g. {twice[0]}')
# ctest reads the list one name per line and trims each line.
bad = [name for name in names if name != name.strip() or '\n' in name or '\r' in name]
if bad:
raise ValueError(f'lane {label}: {len(bad)} name(s) cannot be written one per line, e.g. {bad[0]!r}')
return names
def union(lanes):
"""(sorted union of names, {label: count}, number of names more than one lane selects)."""
counts, seen, shared = {}, set(), set()
for label, path in lanes:
if label in counts:
raise ValueError(f'lane {label} is given twice')
names = lane_names(label, path)
counts[label] = len(names)
shared |= seen & set(names)
seen |= set(names)
return sorted(seen), counts, len(shared)
def parse_lanes(arguments):
lanes = []
for argument in arguments:
label, separator, path = argument.partition('=')
if not separator or not label or not path:
raise ValueError(f'expected LANE=DISCOVERY.json, got {argument!r}')
lanes.append((label, path))
if not lanes:
raise ValueError('no lane given')
return lanes
def run(argv):
if len(argv) < 3 or argv[0] != 'union':
print(__doc__, file=sys.stderr)
return 2
try:
names, counts, shared = union(parse_lanes(argv[2:]))
except (ValueError, OSError, json.JSONDecodeError) as error:
print(f'::error::lane_selection.py: {error}')
return 1
Path(argv[1]).write_text(''.join(name + '\n' for name in names))
summary = ', '.join(f'{label} {count}' for label, count in counts.items())
print(f'lane_selection.py: {len(names)} entr(ies) in one run ({summary}; {shared} in more than one lane)')
return 0
def self_test():
import contextlib
import io
def quiet(argv):
# The negative controls print ::error:: lines on purpose; keep them off a green run.
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
return run(argv)
with tempfile.TemporaryDirectory() as tmp:
tmp = Path(tmp)
def discovery(name, names):
path = tmp / name
path.write_text(json.dumps({'tests': [{'name': n} for n in names]}))
return str(path)
out = tmp / 'tests.txt'
split = discovery('split.json', ['DirectGLES.Split.A', 'DirectGLES.Split.B'])
spawn = discovery('spawn.json', ['DirectGLES.Spawn.A', 'SpawnLane.Control'])
tcp = discovery('tcp.json', ['TcpServer.Start', 'DirectGLES.Tcp.A', 'TcpServer.Stop'])
assert quiet(['union', str(out), f'split={split}', f'spawn={spawn}', f'tcp={tcp}']) == 0
assert out.read_text().splitlines() == sorted(
['DirectGLES.Split.A', 'DirectGLES.Split.B', 'DirectGLES.Spawn.A', 'SpawnLane.Control',
'TcpServer.Start', 'DirectGLES.Tcp.A', 'TcpServer.Stop']), 'the union is every lane, each name once'
# A name two lanes select is written once; each lane still carves its own copy.
shared = discovery('shared.json', ['DirectGLES.Split.A', 'Shared.Fixture'])
assert quiet(['union', str(out), f'split={split}', f'other={shared}']) == 0
assert out.read_text().splitlines() == ['DirectGLES.Split.A', 'DirectGLES.Split.B', 'Shared.Fixture']
out.unlink()
controls = {
'a lane that selects nothing': [f'split={split}', f"spawn={discovery('empty.json', [])}"],
'a lane that names an entry twice': [f"split={discovery('twice.json', ['A', 'A'])}"],
'a name with a trailing blank': [f"split={discovery('blank.json', ['A '])}"],
'a name with a line break': [f"split={discovery('break.json', ['A' + chr(10) + 'B'])}"],
'a lane given twice': [f'split={split}', f'split={spawn}'],
'a missing discovery file': [f"split={tmp / 'absent.json'}"],
'an argument without a lane name': [split],
}
for label, arguments in controls.items():
assert quiet(['union', str(out)] + arguments) == 1, f'{label} must fail'
assert not out.exists(), f'{label} must not leave a list behind'
assert quiet(['union', str(out)]) == 2, 'no lane is a usage error'
assert quiet(['nonsense', str(out), f'split={split}']) == 2, 'an unknown command is a usage error'
print('lane_selection self-test: ok')
return 0
if __name__ == '__main__':
if sys.argv[1:] == ['--self-test']:
sys.exit(self_test())
sys.exit(run(sys.argv[1:]))