mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-11-17 15:03:25 +09:00
[CI] (.github/workflows, docs/ci, scripts/ci): trim setup time in the build and split lane jobs
This commit is contained in:
@@ -0,0 +1,121 @@
|
||||
#!/bin/bash
|
||||
# MobileGL - scripts/ci/checkout_submodules.sh
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
# The submodules a CI build reads, checked out shallow and in parallel, in place of a
|
||||
# recursive checkout of every submodule in the tree.
|
||||
# library what the root CMakeLists.txt configures for the library, its tests and benchmarks.
|
||||
# retrace the library set plus the trace replay tool's: apitrace and the eight directories
|
||||
# its thirdparty/CMakeLists.txt requires.
|
||||
# Each row names a submodule and one file the build reads from it. A row whose file is missing
|
||||
# after the update fails the script: the configure would otherwise go on without it, and for
|
||||
# 3rdparty/flatbuffers the root CMakeLists.txt then builds a library without MG_Remote instead
|
||||
# of stopping. A submodule the build starts to use needs a row here.
|
||||
#
|
||||
# usage: checkout_submodules.sh <library|retrace> (from the repository root)
|
||||
# checkout_submodules.sh --list <library|retrace> prints the rows, checks out nothing
|
||||
set -euo pipefail
|
||||
|
||||
usage="usage: checkout_submodules.sh [--list] <library|retrace>"
|
||||
|
||||
list_only=false
|
||||
if [ "${1:-}" = --list ]; then
|
||||
list_only=true
|
||||
shift
|
||||
fi
|
||||
set_name="${1:?${usage}}"
|
||||
|
||||
# <submodule path> <file the build reads from it>
|
||||
library=(
|
||||
"3rdparty/glslang CMakeLists.txt"
|
||||
"3rdparty/SPIRV-Cross CMakeLists.txt"
|
||||
"3rdparty/SPIRV-Reflect CMakeLists.txt"
|
||||
"3rdparty/VulkanMemoryAllocator CMakeLists.txt"
|
||||
"3rdparty/Vulkan-Headers CMakeLists.txt"
|
||||
"3rdparty/Vulkan-Utility-Libraries CMakeLists.txt"
|
||||
"3rdparty/xxHash build/cmake/CMakeLists.txt"
|
||||
"3rdparty/asio include/asio.hpp"
|
||||
"3rdparty/flatbuffers include/flatbuffers/flatbuffers.h"
|
||||
"include/ska flat_hash_map.hpp"
|
||||
)
|
||||
retrace=(
|
||||
"3rdparty/apitrace thirdparty/CMakeLists.txt"
|
||||
)
|
||||
# Inside 3rdparty/apitrace: the files its thirdparty/CMakeLists.txt tests before it configures.
|
||||
apitrace=(
|
||||
"thirdparty/brotli LICENSE"
|
||||
"thirdparty/gtest LICENSE"
|
||||
"thirdparty/libbacktrace LICENSE"
|
||||
"thirdparty/libpng LICENSE"
|
||||
"thirdparty/snappy COPYING"
|
||||
"thirdparty/zlib README"
|
||||
"thirdparty/zstd LICENSE"
|
||||
"thirdparty/directxmath LICENSE"
|
||||
)
|
||||
|
||||
case "${set_name}" in
|
||||
library) top=("${library[@]}"); nested=() ;;
|
||||
retrace) top=("${library[@]}" "${retrace[@]}"); nested=("${apitrace[@]}") ;;
|
||||
*)
|
||||
echo "checkout_submodules.sh: unknown set '${set_name}' (library or retrace)" >&2
|
||||
echo "${usage}" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ "${list_only}" = true ]; then
|
||||
printf '%s\n' "${top[@]}"
|
||||
if [ "${#nested[@]}" -gt 0 ]; then
|
||||
printf '3rdparty/apitrace/%s\n' "${nested[@]}"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# update <directory> <row>...: one shallow, parallel update of the rows' submodules. The update
|
||||
# is idempotent, so a fetch that failed on the network is tried again, three times in all.
|
||||
update() {
|
||||
local dir="$1"
|
||||
shift
|
||||
local paths=() row attempt
|
||||
for row in "$@"; do
|
||||
paths+=("${row%% *}")
|
||||
done
|
||||
for attempt in 1 2 3; do
|
||||
if git -C "${dir}" submodule update --init --depth=1 --jobs 8 -- "${paths[@]}"; then
|
||||
return 0
|
||||
fi
|
||||
echo "checkout_submodules.sh: the submodule update in ${dir} failed (attempt ${attempt} of 3)" >&2
|
||||
if [ "${attempt}" -lt 3 ]; then
|
||||
sleep $((attempt * 5))
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# verify <directory> <row>...: every row's file exists; prints each missing one and returns 1.
|
||||
verify() {
|
||||
local dir="$1" missing=0 row path file
|
||||
shift
|
||||
for row in "$@"; do
|
||||
path="${row%% *}"
|
||||
file="${row#* }"
|
||||
if [ ! -f "${dir}/${path}/${file}" ]; then
|
||||
echo "::error::checkout_submodules.sh: ${dir}/${path}/${file} is missing after the submodule update, so the '${set_name}' build cannot use ${path}"
|
||||
missing=1
|
||||
fi
|
||||
done
|
||||
return "${missing}"
|
||||
}
|
||||
|
||||
update . "${top[@]}"
|
||||
verify . "${top[@]}"
|
||||
if [ "${#nested[@]}" -gt 0 ]; then
|
||||
update 3rdparty/apitrace "${nested[@]}"
|
||||
verify 3rdparty/apitrace "${nested[@]}"
|
||||
fi
|
||||
echo "checkout_submodules.sh: the '${set_name}' set is checked out ($(( ${#top[@]} + ${#nested[@]} )) submodules)"
|
||||
@@ -25,12 +25,17 @@ variant="${1:?usage: configure_build.sh <primary|split> <build dir>}"
|
||||
build_dir="${2:?usage: configure_build.sh <primary|split> <build dir>}"
|
||||
shift 2
|
||||
|
||||
# ENABLE_PCH is glslang's option, on by default. ccache cannot cache a compile that uses a
|
||||
# precompiled header, so with it on every glslang translation unit is compiled again in every
|
||||
# run. It is in the common part so that both lines stay the same configuration for
|
||||
# build_fingerprint.py.
|
||||
common=(
|
||||
-S . -B "${build_dir}" -G Ninja
|
||||
-DCMAKE_C_COMPILER="${MOBILEGL_CI_CC}"
|
||||
-DCMAKE_CXX_COMPILER="${MOBILEGL_CI_CXX}"
|
||||
-DCMAKE_C_COMPILER_LAUNCHER=ccache
|
||||
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
|
||||
-DENABLE_PCH=OFF
|
||||
)
|
||||
|
||||
case "${variant}" in
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
# MobileGL - scripts/ci/lane_selection.py
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
"""One ctest run for several lanes: the exact names of every lane, as one --tests-from-file list.
|
||||
|
||||
A lane is a `ctest -L <label regex>` selection. Running the lanes of one job as one ctest
|
||||
invocation lets a lane that holds a resource lock (the TCP arm runs one entry at a time) share
|
||||
the run with the others. The lanes keep their own selections: each is discovered with its own
|
||||
label regex, this script writes the union of the discovered names, ctest runs that list, and
|
||||
junit_tally.py --select-from/--write-selected carves each lane's JUnit back out, which fails
|
||||
when a lane's name is missing from the run or reported twice.
|
||||
|
||||
lane_selection.py union OUT.txt LANE=DISCOVERY.json [LANE=DISCOVERY.json ...]
|
||||
lane_selection.py --self-test
|
||||
|
||||
DISCOVERY.json is `ctest -L <regex> --show-only=json-v1`. union fails when a lane selects
|
||||
nothing (what --no-tests=error says for a lane run on its own), names an entry twice, or holds a
|
||||
name a tests-from-file line cannot carry. It prints each lane's count and the size of the union.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
|
||||
def lane_names(label, path):
|
||||
"""The names one discovery selects, in order; raises ValueError when the lane is unusable."""
|
||||
names = [test['name'] for test in json.loads(Path(path).read_text()).get('tests', [])]
|
||||
if not names:
|
||||
raise ValueError(f'lane {label}: {path} selects no test')
|
||||
if len(names) != len(set(names)):
|
||||
twice = sorted({name for name in names if names.count(name) > 1})
|
||||
raise ValueError(f'lane {label}: {path} names {len(twice)} entr(ies) twice, e.g. {twice[0]}')
|
||||
# ctest reads the list one name per line and trims each line.
|
||||
bad = [name for name in names if name != name.strip() or '\n' in name or '\r' in name]
|
||||
if bad:
|
||||
raise ValueError(f'lane {label}: {len(bad)} name(s) cannot be written one per line, e.g. {bad[0]!r}')
|
||||
return names
|
||||
|
||||
|
||||
def union(lanes):
|
||||
"""(sorted union of names, {label: count}, number of names more than one lane selects)."""
|
||||
counts, seen, shared = {}, set(), set()
|
||||
for label, path in lanes:
|
||||
if label in counts:
|
||||
raise ValueError(f'lane {label} is given twice')
|
||||
names = lane_names(label, path)
|
||||
counts[label] = len(names)
|
||||
shared |= seen & set(names)
|
||||
seen |= set(names)
|
||||
return sorted(seen), counts, len(shared)
|
||||
|
||||
|
||||
def parse_lanes(arguments):
|
||||
lanes = []
|
||||
for argument in arguments:
|
||||
label, separator, path = argument.partition('=')
|
||||
if not separator or not label or not path:
|
||||
raise ValueError(f'expected LANE=DISCOVERY.json, got {argument!r}')
|
||||
lanes.append((label, path))
|
||||
if not lanes:
|
||||
raise ValueError('no lane given')
|
||||
return lanes
|
||||
|
||||
|
||||
def run(argv):
|
||||
if len(argv) < 3 or argv[0] != 'union':
|
||||
print(__doc__, file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
names, counts, shared = union(parse_lanes(argv[2:]))
|
||||
except (ValueError, OSError, json.JSONDecodeError) as error:
|
||||
print(f'::error::lane_selection.py: {error}')
|
||||
return 1
|
||||
Path(argv[1]).write_text(''.join(name + '\n' for name in names))
|
||||
summary = ', '.join(f'{label} {count}' for label, count in counts.items())
|
||||
print(f'lane_selection.py: {len(names)} entr(ies) in one run ({summary}; {shared} in more than one lane)')
|
||||
return 0
|
||||
|
||||
|
||||
def self_test():
|
||||
import contextlib
|
||||
import io
|
||||
|
||||
def quiet(argv):
|
||||
# The negative controls print ::error:: lines on purpose; keep them off a green run.
|
||||
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
||||
return run(argv)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp = Path(tmp)
|
||||
|
||||
def discovery(name, names):
|
||||
path = tmp / name
|
||||
path.write_text(json.dumps({'tests': [{'name': n} for n in names]}))
|
||||
return str(path)
|
||||
|
||||
out = tmp / 'tests.txt'
|
||||
split = discovery('split.json', ['DirectGLES.Split.A', 'DirectGLES.Split.B'])
|
||||
spawn = discovery('spawn.json', ['DirectGLES.Spawn.A', 'SpawnLane.Control'])
|
||||
tcp = discovery('tcp.json', ['TcpServer.Start', 'DirectGLES.Tcp.A', 'TcpServer.Stop'])
|
||||
assert quiet(['union', str(out), f'split={split}', f'spawn={spawn}', f'tcp={tcp}']) == 0
|
||||
assert out.read_text().splitlines() == sorted(
|
||||
['DirectGLES.Split.A', 'DirectGLES.Split.B', 'DirectGLES.Spawn.A', 'SpawnLane.Control',
|
||||
'TcpServer.Start', 'DirectGLES.Tcp.A', 'TcpServer.Stop']), 'the union is every lane, each name once'
|
||||
# A name two lanes select is written once; each lane still carves its own copy.
|
||||
shared = discovery('shared.json', ['DirectGLES.Split.A', 'Shared.Fixture'])
|
||||
assert quiet(['union', str(out), f'split={split}', f'other={shared}']) == 0
|
||||
assert out.read_text().splitlines() == ['DirectGLES.Split.A', 'DirectGLES.Split.B', 'Shared.Fixture']
|
||||
out.unlink()
|
||||
controls = {
|
||||
'a lane that selects nothing': [f'split={split}', f"spawn={discovery('empty.json', [])}"],
|
||||
'a lane that names an entry twice': [f"split={discovery('twice.json', ['A', 'A'])}"],
|
||||
'a name with a trailing blank': [f"split={discovery('blank.json', ['A '])}"],
|
||||
'a name with a line break': [f"split={discovery('break.json', ['A' + chr(10) + 'B'])}"],
|
||||
'a lane given twice': [f'split={split}', f'split={spawn}'],
|
||||
'a missing discovery file': [f"split={tmp / 'absent.json'}"],
|
||||
'an argument without a lane name': [split],
|
||||
}
|
||||
for label, arguments in controls.items():
|
||||
assert quiet(['union', str(out)] + arguments) == 1, f'{label} must fail'
|
||||
assert not out.exists(), f'{label} must not leave a list behind'
|
||||
assert quiet(['union', str(out)]) == 2, 'no lane is a usage error'
|
||||
assert quiet(['nonsense', str(out), f'split={split}']) == 2, 'an unknown command is a usage error'
|
||||
print('lane_selection self-test: ok')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if sys.argv[1:] == ['--self-test']:
|
||||
sys.exit(self_test())
|
||||
sys.exit(run(sys.argv[1:]))
|
||||
Reference in New Issue
Block a user