mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-11-16 22:43:24 +09:00
[CI] (.github/workflows, docs/ci, scripts/ci, scripts/data): tally the split baseline, do not rerun it
- 'Split lanes' keeps its first ctest's exit code beside split-baseline.xml; the controls step tallies that JUnit (DirectGLES.Split. entries) with the same refusals instead of re-running 802 entries, on every branch - the rerun stays as the fallback when the lane step never ran, the case where the baseline was the only run of those entries - smoke cases: reuse, fallback, a red exit code under a green JUnit, and a JUnit with no DirectGLES.Split. entry - skip census: retire lane junit-integration-split/isplit (nothing borrows it; split-baseline covers the names)
This commit is contained in:
@@ -49,12 +49,64 @@ expect() {
|
||||
fi
|
||||
}
|
||||
|
||||
run_split() { # $1 = STUB_MODE
|
||||
# The lane step's own baseline, as the workflow leaves it: the stub's JUnit for the whole label and
|
||||
# its exit code beside it, in <base>.xml and <base>.rc.
|
||||
lane_baseline() { # $1 = STUB_MODE, $2 = base path
|
||||
env -i PATH="${STUB_DIR}:/usr/bin:/bin" STUB_MODE="$1" CONTROL_TMPDIR="${WORK}/tmp-$1" \
|
||||
ctest -L integration-split --no-tests=error --output-junit "$2.xml" > /dev/null 2>&1
|
||||
echo "$?" > "$2.rc"
|
||||
}
|
||||
|
||||
# run_split <STUB_MODE> [baseline path]: the controls, tallying the lane step's baseline. A path
|
||||
# with no files behind it is the job whose lane step never ran.
|
||||
run_split() {
|
||||
local base="${2:-${WORK}/base-$1}"
|
||||
[ -n "${2:-}" ] || lane_baseline "$1" "${base}"
|
||||
env -i PATH="${STUB_DIR}:/usr/bin:/bin" STUB_MODE="$1" \
|
||||
CTEST=ctest CONTROL_TMPDIR="${WORK}/tmp-$1" \
|
||||
CTEST=ctest CONTROL_TMPDIR="${WORK}/tmp-$1" SPLIT_BASELINE_JUNIT="${base}.xml" \
|
||||
bash "${HERE}/split_negative_controls.sh"
|
||||
}
|
||||
|
||||
# run_split_saying <sentence> <args for run_split...>: run_split, and its output must carry the
|
||||
# sentence that says which baseline it used.
|
||||
run_split_saying() {
|
||||
local sentence="$1"
|
||||
shift
|
||||
local out
|
||||
out=$(run_split "$@" 2>&1)
|
||||
local rc=$?
|
||||
printf '%s\n' "${out}"
|
||||
printf '%s' "${out}" | grep -qF "${sentence}" || { echo "missing: ${sentence}"; return 1; }
|
||||
return "${rc}"
|
||||
}
|
||||
|
||||
# The lane step's JUnit says green but its exit code says red: ctest's "Not Run" is a skip in
|
||||
# JUnit, so the code is part of the baseline and must be honoured.
|
||||
run_split_red_code() {
|
||||
lane_baseline evidence "${WORK}/redcode"
|
||||
echo 8 > "${WORK}/redcode.rc"
|
||||
run_split evidence "${WORK}/redcode"
|
||||
}
|
||||
|
||||
# The lane step never ran: the controls rerun the baseline, and the rerun's JUnit stays out of
|
||||
# CONTROL_TMPDIR/*.xml, which the job uploads for the skip census.
|
||||
run_split_rerun() {
|
||||
rm -f "${WORK}/tmp-evidence/isplit.xml"
|
||||
run_split_saying "reran the DirectGLES.Split. baseline" evidence "${WORK}/absent" || return 1
|
||||
if [ -e "${WORK}/tmp-evidence/isplit.xml" ]; then
|
||||
echo "the rerun's JUnit landed in CONTROL_TMPDIR/*.xml"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# The lane step ran nothing of the DirectGLES.Split. arm (only other entries of the label).
|
||||
run_split_no_arm() {
|
||||
printf '%s\n' '<testsuite><testcase name="SplitLogPaths.PrivateAndDistinct" status="run"/></testsuite>' \
|
||||
> "${WORK}/noarm.xml"
|
||||
echo 0 > "${WORK}/noarm.rc"
|
||||
run_split evidence "${WORK}/noarm"
|
||||
}
|
||||
|
||||
echo "=== the split lane's E1 / E3(a) controls (scripts/ci/split_negative_controls.sh)"
|
||||
# THE FINDING, REPRODUCED. Non-empty selection, green baseline, and a red that is not the knob's.
|
||||
expect FAILED "unrelated failure with a non-empty selection" -- run_split unrelated
|
||||
@@ -72,6 +124,11 @@ expect FAILED "E1 restored boundary remains red" -- run_split e1-res
|
||||
expect FAILED "E1 record never reaches the peer" -- run_split e1-no-peer
|
||||
expect FAILED "E1 observed no real emission" -- run_split e1-no-emit
|
||||
expect FAILED "E1 incorrectly waits every record" -- run_split e1-waitall
|
||||
# The baseline is the lane step's own run; the controls rerun it only when that step never ran.
|
||||
expect PASSED "the lane step's baseline is tallied, not rerun" -- run_split_saying "no rerun" evidence
|
||||
expect PASSED "no lane JUnit: the controls rerun the baseline" -- run_split_rerun
|
||||
expect FAILED "the lane step's exit code is red, its JUnit green" -- run_split_red_code
|
||||
expect FAILED "the lane JUnit has no DirectGLES.Split. entry" -- run_split_no_arm
|
||||
|
||||
echo
|
||||
echo "smoke test: ${passes} passed, ${failures} failed"
|
||||
|
||||
@@ -9,14 +9,9 @@
|
||||
|
||||
# EXIT GATE E1's NEGATIVE CONTROL and EXIT GATE E3(a)'s.
|
||||
#
|
||||
# This file is the body of .github/workflows/test.yml's "Negative controls (barrier, map push)"
|
||||
# step, which requires the verb barrier and the persistent-map push to be load-bearing. It lives in
|
||||
# the repository rather than inline in the workflow for one reason: a workflow `run:` block cannot
|
||||
# be executed anywhere except on a runner, so the logic below was unreviewable and untestable until
|
||||
# it ran in CI - and when the wave-1 cross-family review claimed it was broken, confirming the
|
||||
# claim needed a hand-made copy of these lines with their inputs stubbed (wave1-codex-verify.md 8).
|
||||
# A copy is not the thing. The smoke test at scripts/ci/control_smoke_test.sh now runs THIS file,
|
||||
# so the lines CI executes and the lines the smoke test proves are the same lines.
|
||||
# The body of test.yml's "Negative controls (barrier, map push)" step, which requires the verb
|
||||
# barrier and the persistent-map push to be load-bearing. It is a file, not an inline `run:` block,
|
||||
# so scripts/ci/control_smoke_test.sh runs these same lines off a runner against a stubbed ctest.
|
||||
#
|
||||
# WHAT THE REVIEW FOUND (ID-46 finding 8, CONFIRMED by execution; ID-48 assigns it here).
|
||||
# The previous version accepted ANY non-zero ctest exit as "the knob is load-bearing". A timeout, a
|
||||
@@ -37,7 +32,7 @@
|
||||
# carries none of them.
|
||||
#
|
||||
# WHY THE EVIDENCE IS THE SCENARIO'S ASSERTION TEXT AND NOT THE KNOB'S OWN LOG LINE.
|
||||
# ConfigLoader logs a named line for both knobs (ConfigLoader.cpp:385-393, "is the R-1 NEGATIVE
|
||||
# ConfigLoader logs a named line for both knobs ("is the R-1 NEGATIVE
|
||||
# CONTROL", "is the E3(a) NEGATIVE CONTROL"), and it is tempting to grep for that. It is not
|
||||
# evidence: it is written at config load, by every process in the run, whatever happens next. A
|
||||
# setup abort would carry it too. It proves the knob was READ, never that the knob caused the red.
|
||||
@@ -47,8 +42,11 @@
|
||||
# overlap Fatal cannot fire with the default BATCH_WAITS=1. Pixel failures are not E1 evidence.
|
||||
#
|
||||
# Usage: split_negative_controls.sh [--self-test]
|
||||
# CTEST ctest binary (default: ctest)
|
||||
# CONTROL_TMPDIR scratch dir for the junit + output (default: ${RUNNER_TEMP:-/tmp})
|
||||
# CTEST ctest binary (default: ctest)
|
||||
# CONTROL_TMPDIR scratch dir for the junit + output (default: ${RUNNER_TEMP:-/tmp})
|
||||
# SPLIT_BASELINE_JUNIT the JUnit an earlier `ctest -L integration-split` wrote; its exit code is
|
||||
# read from the file of the same name ending in .rc. When both exist the
|
||||
# baseline is tallied from them instead of re-run.
|
||||
set -u
|
||||
|
||||
if [ "${1:-}" = "--self-test" ]; then
|
||||
@@ -60,8 +58,6 @@ CTEST="${CTEST:-ctest}"
|
||||
CONTROL_TMPDIR="${CONTROL_TMPDIR:-${RUNNER_TEMP:-/tmp}}"
|
||||
mkdir -p "${CONTROL_TMPDIR}"
|
||||
|
||||
junit="${CONTROL_TMPDIR}/isplit.xml"
|
||||
rm -f "${junit}"
|
||||
log_helper="$(dirname "$0")/../../MobileGL/MG_IntegrationTest/Harness/split_log_paths.py"
|
||||
|
||||
# Check ownership even while the runtime lane is disarmed and will skip.
|
||||
@@ -76,15 +72,35 @@ python3 "${log_helper}" check "${CTEST}" "$PWD" || exit 1
|
||||
# ImplementedVerbCount(), read by Harness/SplitRuntimePeek), so the only honest way to ask it from a
|
||||
# shell is to look at what the entries DID.
|
||||
# The metadata ownership test shares the label but cannot arm runtime controls.
|
||||
"${CTEST}" -L integration-split -R '^DirectGLES\.Split\.' -j 4 --no-tests=error --output-junit "${junit}"
|
||||
baseline_rc=$?
|
||||
#
|
||||
# The lane step before this one already ran every integration-split entry on this runner and
|
||||
# library, and left its JUnit and exit code. Both are read when they exist; only the
|
||||
# DirectGLES.Split. entries count, as in the rerun. The rerun stays for a job whose lane step
|
||||
# never ran (an earlier gate failed), where this baseline is the only run of those entries.
|
||||
baseline_rc_file="${SPLIT_BASELINE_JUNIT:-}"
|
||||
baseline_rc_file="${baseline_rc_file%.xml}.rc"
|
||||
if [ -n "${SPLIT_BASELINE_JUNIT:-}" ] && [ -f "${SPLIT_BASELINE_JUNIT}" ] && [ -f "${baseline_rc_file}" ]; then
|
||||
junit="${SPLIT_BASELINE_JUNIT}"
|
||||
baseline_rc=$(tr -dc '0-9' < "${baseline_rc_file}")
|
||||
baseline_rc="${baseline_rc:-1}"
|
||||
echo "baseline tallied from ${junit} (the lane step's own run, ctest exit ${baseline_rc}); no rerun"
|
||||
else
|
||||
# A directory of its own: the job uploads CONTROL_TMPDIR/*.xml for the skip census, which has
|
||||
# no row for this rerun.
|
||||
mkdir -p "${CONTROL_TMPDIR}/split-controls"
|
||||
junit="${CONTROL_TMPDIR}/split-controls/isplit.xml"
|
||||
rm -f "${junit}"
|
||||
echo "::notice::no baseline JUnit and exit code from the lane step (SPLIT_BASELINE_JUNIT='${SPLIT_BASELINE_JUNIT:-}'), so the controls reran the DirectGLES.Split. baseline"
|
||||
"${CTEST}" -L integration-split -R '^DirectGLES\.Split\.' -j 4 --no-tests=error --output-junit "${junit}"
|
||||
baseline_rc=$?
|
||||
fi
|
||||
|
||||
if [ ! -f "${junit}" ]; then
|
||||
echo "::error::the baseline run wrote no ${junit} (ctest exit ${baseline_rc}), so nothing below can tell an armed lane from a broken one"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tally=$(python3 "$(dirname "$0")/junit_tally.py" "${junit}")
|
||||
tally=$(python3 "$(dirname "$0")/junit_tally.py" "${junit}" --require-split-ran)
|
||||
if [ -z "${tally}" ]; then
|
||||
echo "::error::could not tally ${junit}; a run whose result cannot be read is not an arming signal"
|
||||
exit 1
|
||||
|
||||
Reference in New Issue
Block a user