[CI] (.github/workflows, docs/ci, scripts/ci, scripts/data): tally the split baseline, do not rerun it

- 'Split lanes' keeps its first ctest's exit code beside split-baseline.xml; the controls step tallies that JUnit (DirectGLES.Split. entries) with the same refusals instead of re-running 802 entries, on every branch
- the rerun stays as the fallback when the lane step never ran, the case where the baseline was the only run of those entries
- smoke cases: reuse, fallback, a red exit code under a green JUnit, and a JUnit with no DirectGLES.Split. entry
- skip census: retire lane junit-integration-split/isplit (nothing borrows it; split-baseline covers the names)
This commit is contained in:
2026-10-17 15:59:53 -04:00
parent 1e9f8bb0ff
commit c5e4ddd330
5 changed files with 100 additions and 777 deletions
+59 -2
View File
@@ -49,12 +49,64 @@ expect() {
fi
}
run_split() { # $1 = STUB_MODE
# The lane step's own baseline, as the workflow leaves it: the stub's JUnit for the whole label and
# its exit code beside it, in <base>.xml and <base>.rc.
lane_baseline() { # $1 = STUB_MODE, $2 = base path
env -i PATH="${STUB_DIR}:/usr/bin:/bin" STUB_MODE="$1" CONTROL_TMPDIR="${WORK}/tmp-$1" \
ctest -L integration-split --no-tests=error --output-junit "$2.xml" > /dev/null 2>&1
echo "$?" > "$2.rc"
}
# run_split <STUB_MODE> [baseline path]: the controls, tallying the lane step's baseline. A path
# with no files behind it is the job whose lane step never ran.
run_split() {
local base="${2:-${WORK}/base-$1}"
[ -n "${2:-}" ] || lane_baseline "$1" "${base}"
env -i PATH="${STUB_DIR}:/usr/bin:/bin" STUB_MODE="$1" \
CTEST=ctest CONTROL_TMPDIR="${WORK}/tmp-$1" \
CTEST=ctest CONTROL_TMPDIR="${WORK}/tmp-$1" SPLIT_BASELINE_JUNIT="${base}.xml" \
bash "${HERE}/split_negative_controls.sh"
}
# run_split_saying <sentence> <args for run_split...>: run_split, and its output must carry the
# sentence that says which baseline it used.
run_split_saying() {
local sentence="$1"
shift
local out
out=$(run_split "$@" 2>&1)
local rc=$?
printf '%s\n' "${out}"
printf '%s' "${out}" | grep -qF "${sentence}" || { echo "missing: ${sentence}"; return 1; }
return "${rc}"
}
# The lane step's JUnit says green but its exit code says red: ctest's "Not Run" is a skip in
# JUnit, so the code is part of the baseline and must be honoured.
run_split_red_code() {
lane_baseline evidence "${WORK}/redcode"
echo 8 > "${WORK}/redcode.rc"
run_split evidence "${WORK}/redcode"
}
# The lane step never ran: the controls rerun the baseline, and the rerun's JUnit stays out of
# CONTROL_TMPDIR/*.xml, which the job uploads for the skip census.
run_split_rerun() {
rm -f "${WORK}/tmp-evidence/isplit.xml"
run_split_saying "reran the DirectGLES.Split. baseline" evidence "${WORK}/absent" || return 1
if [ -e "${WORK}/tmp-evidence/isplit.xml" ]; then
echo "the rerun's JUnit landed in CONTROL_TMPDIR/*.xml"
return 1
fi
}
# The lane step ran nothing of the DirectGLES.Split. arm (only other entries of the label).
run_split_no_arm() {
printf '%s\n' '<testsuite><testcase name="SplitLogPaths.PrivateAndDistinct" status="run"/></testsuite>' \
> "${WORK}/noarm.xml"
echo 0 > "${WORK}/noarm.rc"
run_split evidence "${WORK}/noarm"
}
echo "=== the split lane's E1 / E3(a) controls (scripts/ci/split_negative_controls.sh)"
# THE FINDING, REPRODUCED. Non-empty selection, green baseline, and a red that is not the knob's.
expect FAILED "unrelated failure with a non-empty selection" -- run_split unrelated
@@ -72,6 +124,11 @@ expect FAILED "E1 restored boundary remains red" -- run_split e1-res
expect FAILED "E1 record never reaches the peer" -- run_split e1-no-peer
expect FAILED "E1 observed no real emission" -- run_split e1-no-emit
expect FAILED "E1 incorrectly waits every record" -- run_split e1-waitall
# The baseline is the lane step's own run; the controls rerun it only when that step never ran.
expect PASSED "the lane step's baseline is tallied, not rerun" -- run_split_saying "no rerun" evidence
expect PASSED "no lane JUnit: the controls rerun the baseline" -- run_split_rerun
expect FAILED "the lane step's exit code is red, its JUnit green" -- run_split_red_code
expect FAILED "the lane JUnit has no DirectGLES.Split. entry" -- run_split_no_arm
echo
echo "smoke test: ${passes} passed, ${failures} failed"
+32 -16
View File
@@ -9,14 +9,9 @@
# EXIT GATE E1's NEGATIVE CONTROL and EXIT GATE E3(a)'s.
#
# This file is the body of .github/workflows/test.yml's "Negative controls (barrier, map push)"
# step, which requires the verb barrier and the persistent-map push to be load-bearing. It lives in
# the repository rather than inline in the workflow for one reason: a workflow `run:` block cannot
# be executed anywhere except on a runner, so the logic below was unreviewable and untestable until
# it ran in CI - and when the wave-1 cross-family review claimed it was broken, confirming the
# claim needed a hand-made copy of these lines with their inputs stubbed (wave1-codex-verify.md 8).
# A copy is not the thing. The smoke test at scripts/ci/control_smoke_test.sh now runs THIS file,
# so the lines CI executes and the lines the smoke test proves are the same lines.
# The body of test.yml's "Negative controls (barrier, map push)" step, which requires the verb
# barrier and the persistent-map push to be load-bearing. It is a file, not an inline `run:` block,
# so scripts/ci/control_smoke_test.sh runs these same lines off a runner against a stubbed ctest.
#
# WHAT THE REVIEW FOUND (ID-46 finding 8, CONFIRMED by execution; ID-48 assigns it here).
# The previous version accepted ANY non-zero ctest exit as "the knob is load-bearing". A timeout, a
@@ -37,7 +32,7 @@
# carries none of them.
#
# WHY THE EVIDENCE IS THE SCENARIO'S ASSERTION TEXT AND NOT THE KNOB'S OWN LOG LINE.
# ConfigLoader logs a named line for both knobs (ConfigLoader.cpp:385-393, "is the R-1 NEGATIVE
# ConfigLoader logs a named line for both knobs ("is the R-1 NEGATIVE
# CONTROL", "is the E3(a) NEGATIVE CONTROL"), and it is tempting to grep for that. It is not
# evidence: it is written at config load, by every process in the run, whatever happens next. A
# setup abort would carry it too. It proves the knob was READ, never that the knob caused the red.
@@ -47,8 +42,11 @@
# overlap Fatal cannot fire with the default BATCH_WAITS=1. Pixel failures are not E1 evidence.
#
# Usage: split_negative_controls.sh [--self-test]
# CTEST ctest binary (default: ctest)
# CONTROL_TMPDIR scratch dir for the junit + output (default: ${RUNNER_TEMP:-/tmp})
# CTEST ctest binary (default: ctest)
# CONTROL_TMPDIR scratch dir for the junit + output (default: ${RUNNER_TEMP:-/tmp})
# SPLIT_BASELINE_JUNIT the JUnit an earlier `ctest -L integration-split` wrote; its exit code is
# read from the file of the same name ending in .rc. When both exist the
# baseline is tallied from them instead of re-run.
set -u
if [ "${1:-}" = "--self-test" ]; then
@@ -60,8 +58,6 @@ CTEST="${CTEST:-ctest}"
CONTROL_TMPDIR="${CONTROL_TMPDIR:-${RUNNER_TEMP:-/tmp}}"
mkdir -p "${CONTROL_TMPDIR}"
junit="${CONTROL_TMPDIR}/isplit.xml"
rm -f "${junit}"
log_helper="$(dirname "$0")/../../MobileGL/MG_IntegrationTest/Harness/split_log_paths.py"
# Check ownership even while the runtime lane is disarmed and will skip.
@@ -76,15 +72,35 @@ python3 "${log_helper}" check "${CTEST}" "$PWD" || exit 1
# ImplementedVerbCount(), read by Harness/SplitRuntimePeek), so the only honest way to ask it from a
# shell is to look at what the entries DID.
# The metadata ownership test shares the label but cannot arm runtime controls.
"${CTEST}" -L integration-split -R '^DirectGLES\.Split\.' -j 4 --no-tests=error --output-junit "${junit}"
baseline_rc=$?
#
# The lane step before this one already ran every integration-split entry on this runner and
# library, and left its JUnit and exit code. Both are read when they exist; only the
# DirectGLES.Split. entries count, as in the rerun. The rerun stays for a job whose lane step
# never ran (an earlier gate failed), where this baseline is the only run of those entries.
baseline_rc_file="${SPLIT_BASELINE_JUNIT:-}"
baseline_rc_file="${baseline_rc_file%.xml}.rc"
if [ -n "${SPLIT_BASELINE_JUNIT:-}" ] && [ -f "${SPLIT_BASELINE_JUNIT}" ] && [ -f "${baseline_rc_file}" ]; then
junit="${SPLIT_BASELINE_JUNIT}"
baseline_rc=$(tr -dc '0-9' < "${baseline_rc_file}")
baseline_rc="${baseline_rc:-1}"
echo "baseline tallied from ${junit} (the lane step's own run, ctest exit ${baseline_rc}); no rerun"
else
# A directory of its own: the job uploads CONTROL_TMPDIR/*.xml for the skip census, which has
# no row for this rerun.
mkdir -p "${CONTROL_TMPDIR}/split-controls"
junit="${CONTROL_TMPDIR}/split-controls/isplit.xml"
rm -f "${junit}"
echo "::notice::no baseline JUnit and exit code from the lane step (SPLIT_BASELINE_JUNIT='${SPLIT_BASELINE_JUNIT:-}'), so the controls reran the DirectGLES.Split. baseline"
"${CTEST}" -L integration-split -R '^DirectGLES\.Split\.' -j 4 --no-tests=error --output-junit "${junit}"
baseline_rc=$?
fi
if [ ! -f "${junit}" ]; then
echo "::error::the baseline run wrote no ${junit} (ctest exit ${baseline_rc}), so nothing below can tell an armed lane from a broken one"
exit 1
fi
tally=$(python3 "$(dirname "$0")/junit_tally.py" "${junit}")
tally=$(python3 "$(dirname "$0")/junit_tally.py" "${junit}" --require-split-ran)
if [ -z "${tally}" ]; then
echo "::error::could not tally ${junit}; a run whose result cannot be read is not an arming signal"
exit 1