[Fix] (CI): make split controls mandatory and isolate runner evidence

This commit is contained in:
2026-09-16 12:34:33 -04:00
parent 37fc4fdb0f
commit 6fa6a925bd
14 changed files with 148 additions and 51 deletions
+24
View File
@@ -0,0 +1,24 @@
#!/usr/bin/env python3
"""Summarize the indebted broad lane without turning debt into a hard gate."""
import collections
import sys
import xml.etree.ElementTree as ET
cases = ET.parse(sys.argv[1]).getroot().findall('.//testcase')
if not cases:
sys.exit('Census FAILED: no executed test records')
counts = collections.Counter()
for case in cases:
failure = case.find('failure')
if failure is not None:
reason = failure.get('message', '')
status = 'aborted' if 'aborted' in reason.lower() else 'failed'
elif case.find('skipped') is not None:
status = 'skipped'
else:
status = 'passed'
counts[status] += 1
print('### Broad inproc census (ID-65: recorded debt, not a gate)')
print(f'CTest exit: {sys.argv[2]}; total: {len(cases)}')
print('\n| passed | skipped | aborted | failed |\n|---:|---:|---:|---:|')
print('| ' + ' | '.join(str(counts[k]) for k in ('passed', 'skipped', 'aborted', 'failed')) + ' |')
+12 -5
View File
@@ -56,10 +56,15 @@ run_split() { # $1 = STUB_MODE
run_retrace() { # $1 = STUB_MODE
cd "${WORK}" || return 127
mkdir -p "${WORK}/OpenRA"
local rc=0
env -i PATH="${STUB_DIR}:/usr/bin:/bin" STUB_MODE="$1" \
CTEST=ctest CONTROL_TMPDIR="${WORK}/tmp-$1" \
PULL_LIBRARY="${WORK}/pull.so" FROZEN_LIBRARY="${WORK}/frozen.so" \
bash "${HERE}/retrace_pull_library_control.sh" OpenRA DirectGLES
bash "${HERE}/retrace_pull_library_control.sh" OpenRA DirectGLES || rc=$?
cmp -s "${WORK}/frozen.so" "${WORK}/split.so" || {
echo 'F6 FAILED: pull control did not restore the split library'; return 1;
}
return "${rc}"
}
run_drop_draw() { # $1 = STUB_MODE
@@ -67,7 +72,7 @@ run_drop_draw() { # $1 = STUB_MODE
mkdir -p "${WORK}/OpenRA"
env -i PATH="${STUB_DIR}:/usr/bin:/bin" STUB_MODE="$1" \
CTEST=ctest CONTROL_TMPDIR="${WORK}/tmp-$1" \
LIBRARY_LOG="${WORK}/tmp-$1/mobilegl.log" \
FROZEN_LIBRARY="${WORK}/frozen.so" LIBRARY_LOG="${WORK}/tmp-$1/mobilegl.log" \
bash "${HERE}/retrace_drop_draw_control.sh" OpenRA DirectGLES
}
@@ -80,8 +85,8 @@ expect PASSED "the scenarios' own diagnostic" -- run_split evidenc
expect FAILED "the knob leaves the selection green" -- run_split green
# The arming counter's half of the finding: a baseline that is already red cannot arm anything.
expect FAILED "the baseline is already red" -- run_split red-baseline
# The disarmed lane, which is a legitimate exit 0 while c1/s1/v1 are landing.
expect PASSED "every split entry skipped (lane not armed)" -- run_split all-skipped
# P5 is complete: losing the runtime implementation must no longer disarm the gate.
expect FAILED "every split entry skipped (implementation lost)" -- run_split all-skipped
echo
echo "=== the retrace lane's pull-library control (scripts/ci/retrace_pull_library_control.sh)"
@@ -93,7 +98,9 @@ else
echo "no cc available; the retrace half of this smoke test needs one" >&2
exit 1
fi
: > "${WORK}/frozen.so"
printf '%s\n' 'int MG_Remote_stub(void) { return 1; }' > "${WORK}/split.c"
cc -shared -fPIC -o "${WORK}/frozen.so" "${WORK}/split.c" || exit 1
cp "${WORK}/frozen.so" "${WORK}/split.so"
# THE FINDING, part (b): a regex matching no tests. --no-tests=error exits non-zero and the old
# control read that as "the pull library turned it red".
+9 -4
View File
@@ -20,9 +20,11 @@ import sys
import xml.etree.ElementTree as ET
def tally(path):
def tally(path, split_only=False):
passed = failed = skipped = 0
for case in ET.parse(path).getroot().iter('testcase'):
if split_only and not case.get('name', '').startswith('DirectGLES.Split.'):
continue
if case.find('failure') is not None or case.find('error') is not None:
failed += 1
elif case.find('skipped') is not None or case.get('status') in ('notrun', 'disabled'):
@@ -33,15 +35,18 @@ def tally(path):
def main():
if len(sys.argv) != 2:
print("usage: junit_tally.py <junit.xml>", file=sys.stderr)
if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] != '--require-split-ran'):
print("usage: junit_tally.py <junit.xml> [--require-split-ran]", file=sys.stderr)
return 2
try:
passed, failed, skipped = tally(sys.argv[1])
passed, failed, skipped = tally(sys.argv[1], split_only=len(sys.argv) == 3)
except Exception as exc: # a malformed file is not "zero of everything"
print(f"junit_tally: cannot parse {sys.argv[1]}: {exc}", file=sys.stderr)
return 1
print(f"{passed} {failed} {skipped}")
if len(sys.argv) == 3 and (passed == 0 or failed):
print('split baseline FAILED: no successful split runtime entries or an already-red selection', file=sys.stderr)
return 1
return 0
+1 -1
View File
@@ -55,7 +55,7 @@ split, retrace, dropdraw = sys.argv[1], sys.argv[2], sys.argv[3]
# other two catching the smoke test's "unrelated failure" case, and the case never flips - which
# is what this script measured the first time the perturbation actually applied.
rules = [
(split, [('LINE', 'grep -qE "${evidence}"', ('if ! ', 'elif ! ')),
(split, [('LINE', '"${log_helper}" assertion', ('if ! ', 'elif ! ')),
('SUBST', '"[A-Za-z_][A-Za-z_0-9]*"\\}\' || exit 1', '"[A-Za-z_][A-Za-z_0-9]*"\\}\' || true'),
('SUBST', '"${private_evidence}" "${name}" || exit 1',
'"${private_evidence}" "${name}" || true')]),
+10
View File
@@ -50,6 +50,14 @@ CTEST="${CTEST:-ctest}"
CONTROL_TMPDIR="${CONTROL_TMPDIR:-${RUNNER_TEMP:-/tmp}}"
LIBRARY_LOG="${LIBRARY_LOG:-${CASE}/${BACKEND}/output/mobilegl.log}"
mkdir -p "${CONTROL_TMPDIR}"
FROZEN_LIBRARY="${FROZEN_LIBRARY:?FROZEN_LIBRARY must name the split library the replay loads}"
symbols=$(nm --defined-only "${FROZEN_LIBRARY}") || exit 1
remote_count=$(printf '%s\n' "${symbols}" | grep -ic MG_Remote || true)
echo "draw-drop control library: ${FROZEN_LIBRARY}: MG_Remote=${remote_count}"
if [ "${remote_count}" -lt 1 ]; then
echo '::error::draw-drop control requires a split library: MG_Remote=0'
exit 1
fi
selector="^MobileGLTraceReplay\.${CASE}\.${BACKEND}$"
@@ -66,6 +74,8 @@ restore_good_output() {
echo "restored the verified run's output over the control's"
fi
}
trap restore_good_output EXIT
trap 'exit 130' INT TERM
matched=$("${CTEST}" -N -R "${selector}" | grep -cE '^ *Test *#[0-9]+:')
if [ "${matched}" -lt 1 ]; then
+17 -2
View File
@@ -69,6 +69,18 @@ restore_good_output() {
fi
}
# Restore the exact split library on success, failure, and interruption. The following
# draw-drop control uses this same frozen path.
saved_library=$(mktemp "${CONTROL_TMPDIR}/split-library.XXXXXX") || exit 1
cp -p "${FROZEN_LIBRARY}" "${saved_library}" || exit 1
restore_control() {
cp -p "${saved_library}" "${FROZEN_LIBRARY}"
rm -f "${saved_library}"
restore_good_output
}
trap restore_control EXIT
trap 'exit 130' INT TERM
# HOLE 1: COUNT THE SELECTION FIRST. `--no-tests=error` turns an empty selection into a non-zero
# exit, which is indistinguishable from a working control unless the selection is counted.
matched=$("${CTEST}" -N -R "${selector}" | grep -cE '^ *Test *#[0-9]+:')
@@ -82,8 +94,11 @@ fi
# in. It defines no MG_Remote symbol, so ConfigLoader has no transport parser and
# MOBILEGL_TRANSPORT=inproc is accepted and ignored - the exact shape of "the split lane ran
# monolith".
cp "${PULL_LIBRARY}" "${FROZEN_LIBRARY}"
if nm --defined-only "${FROZEN_LIBRARY}" | grep -q -i MG_Remote; then
cp "${PULL_LIBRARY}" "${FROZEN_LIBRARY}" || exit 1
symbols=$(nm --defined-only "${FROZEN_LIBRARY}") || exit 1
remote_count=$(printf '%s\n' "${symbols}" | grep -ic MG_Remote || true)
echo "pull control library: ${FROZEN_LIBRARY}: MG_Remote=${remote_count}"
if [ "${remote_count}" -ne 0 ]; then
restore_good_output
echo "::error::the control's own library defines MG_Remote symbols, so it is not a pull build and this control would prove nothing"
exit 1
+8 -5
View File
@@ -51,6 +51,7 @@ CONTROL_TMPDIR="${CONTROL_TMPDIR:-${RUNNER_TEMP:-/tmp}}"
mkdir -p "${CONTROL_TMPDIR}"
junit="${CONTROL_TMPDIR}/isplit.xml"
rm -f "${junit}"
log_helper="$(dirname "$0")/../../MobileGL/MG_IntegrationTest/Harness/split_log_paths.py"
# Check ownership even while the runtime lane is disarmed and will skip.
@@ -87,14 +88,14 @@ echo "split entries - passed: ${baseline_passed}, failed: ${baseline_failed}, sk
# A RED BASELINE DISARMS THE CONTROLS RATHER THAN ARMING THEM (review finding 8, second half).
# `|| true` plus a "not skipped" counter used to treat a case that ran and FAILED as evidence the
# lane was live. Turning an already-red entry red is not a measurement.
if [ "${baseline_failed}" -gt 0 ]; then
if [ "${baseline_failed}" -gt 0 ] || [ "${baseline_rc}" -ne 0 ]; then
echo "::error::${baseline_failed} DirectGLES.Split. entries are ALREADY RED with both knobs at their defaults, so neither negative control below can attribute its red to the knob it turns. Fix the lane first; a control measured against a red baseline is not a control. (This used to be swallowed by an unconditional '|| true' and counted as 'the lane is armed'.)"
exit 1
fi
if [ "${baseline_passed}" -lt 1 ]; then
echo "::warning::every DirectGLES.Split. entry SKIPPED, so neither negative control can fire. The arming condition is a runtime fact - MG_Config::Transport, ClientSession::Active() and ImplementedVerbCount(), read by Harness/SplitRuntimePeek - and it becomes true on the commit that lands the last of c1/s1/v1. This step becomes a gate then, with no edit; it is not a green that asserted anything today."
exit 0
echo "::error::split baseline FAILED: every DirectGLES.Split. entry SKIPPED; the split implementation did not execute"
exit 1
fi
# ---- the controls ---------------------------------------------------------------------------
@@ -145,7 +146,7 @@ run_control() {
if [ "${evidence}" = "private-barrier-fatal" ]; then
python3 "${log_helper}" evidence "${manifest}" "${filter}" \
'Fatal\{BarrierViolation, "[A-Za-z_][A-Za-z_0-9]*"\}' || exit 1
elif ! tr -s '[:space:]' ' ' < "${out}" | grep -qE "${evidence}"; then
elif ! python3 "${log_helper}" assertion "${manifest}" "${filter}" "${result}" "${evidence}"; then
echo "::error::${name} FAILED: red lacks its persistent-map push diagnostic. Required: ${evidence}"
exit 1
fi
@@ -179,8 +180,10 @@ run_control "negative control E1 (MOBILEGL_IPC_VERB_BARRIER=0)" \
# * the LIBRARY's own line in the entry's private file, saying the push was disabled by this
# knob. It did not exist until ID-65 assigned it (joint-v1.md 3), which is why this control
# used to rest on the pixels alone.
# TheMapLandsInTheArmItsLaneDeclares skips by design outside PersistentMapArm.
# Select only the pixel cases; a pre-flight skip in either remains a hard failure.
run_control "negative control E3(a) (MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0)" \
'DirectGLES\.Split\.(SmallRing\.)?PersistentCoherentMapScenario' \
'DirectGLES\.Split\.(SmallRing\.)?PersistentCoherentMapScenario\.(TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw|AWriteAfterAFrameBoundaryReachesTheNextFramesDraw)$' \
"the SECOND write through the same mapping, announced by nothing|frame 1's write through the SAME mapping, after a Present" \
'MGPipe: persistent-map push disabled - MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0' \
MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0
+2 -1
View File
@@ -7,7 +7,7 @@ trap 'rm -rf "${WORK}"' EXIT
cp "${HERE}/testdata/stub_ctest.sh" "${WORK}/ctest"
chmod +x "${WORK}/ctest"
passes=0
for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated e3-no-private skipped-selection notrun-selection missing-selection partial-fatal wrong-fatal; do
for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated e3-no-private skipped-selection e3-skipped-selection notrun-selection missing-selection partial-fatal wrong-fatal; do
mkdir -p "${WORK}/${mode}"
rc=0
STUB_MODE="${mode}" CTEST="${WORK}/ctest" CONTROL_TMPDIR="${WORK}/${mode}" \
@@ -25,6 +25,7 @@ for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated e3-no-p
case "${mode}" in
e3-no-private) message='no selected private log carries /MGPipe: persistent-map push disabled' ;;
skipped-selection) message='SplitLogPaths FAILED: E1 control: the knob killed the pre-flight, not the entry - 1 selected entries skipped' ;;
e3-skipped-selection) message='SplitLogPaths FAILED: E3(a) control: the knob killed the pre-flight, not the entry - 1 selected entries skipped' ;;
notrun-selection|missing-selection) message='SplitLogPaths FAILED: E1 control: 1 selected entries did not run' ;;
esac
if [[ "${mode}" = *-selection ]]; then
+19 -2
View File
@@ -24,7 +24,7 @@
# never says the push was disabled (the half ID-65 added)
# green baseline green; the control's own run PASSES (the knob is not load-bearing)
# red-baseline the baseline itself has a failed entry
# all-skipped the baseline is entirely skipped (the disarmed lane, a legitimate exit 0)
# all-skipped the baseline is entirely skipped (lost implementation, a hard failure)
# retrace-noselect `ctest -N` matches nothing; the run exits 8 the way --no-tests=error does
# retrace-unrelated one match; the run fails without naming the transport
# retrace-evidence one match; the run fails with run_trace_case.cmake's own sentence
@@ -76,10 +76,20 @@ write_junit() {
entry=DirectGLES.Split.ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels
[ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" != 0 ] || entry=DirectGLES.Split.PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw
body="<testcase name=\"${entry}\" status=\"fail\"><failure message=\"control red\"/></testcase>"
if [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" = 0 ]; then
case "${mode}" in
evidence|e3-no-private)
body="<testcase name=\"${entry}\" status=\"fail\"><failure/><system-out>the SECOND write through the same mapping, announced by nothing</system-out></testcase>" ;;
esac
fi
if [ "${mode}" = partial-fatal ] && [ "${MOBILEGL_IPC_VERB_BARRIER:-1}" = 0 ]; then
body="${body}<testcase name=\"DirectGLES.Split.TriangleScenario.SecondEntry\" status=\"fail\"><failure/></testcase>"
fi
case "${mode}" in
e3-skipped-selection)
if [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" = 0 ]; then
body="<testcase name=\"${entry}\" status=\"notrun\"><skipped/></testcase>"
fi ;;
skipped-selection) body="<testcase name=\"${entry}\" status=\"notrun\"><skipped/></testcase>" ;;
notrun-selection) body="<testcase name=\"${entry}\" status=\"notrun\"/>" ;;
missing-selection) body='' ;;
@@ -130,7 +140,7 @@ fi
# The control's own run.
if [ "${MOBILEGL_IPC_VERB_BARRIER:-1}" = 0 ]; then
case "${mode}" in
evidence|e3-unrelated|e3-no-private|skipped-selection|notrun-selection|missing-selection|partial-fatal) echo 'Fatal{BarrierViolation, "DrawVbo"}' > "${log}" ;;
evidence|e3-unrelated|e3-no-private|skipped-selection|e3-skipped-selection|notrun-selection|missing-selection|partial-fatal) echo 'Fatal{BarrierViolation, "DrawVbo"}' > "${log}" ;;
wrong-fatal) echo 'Fatal{ReplyMissing, "DrawVbo"}' > "${log}" ;;
missing-fatal) echo "library setup only; no fatal" > "${log}" ;;
stdout-fatal) echo 'Fatal{BarrierViolation, "DrawVbo"}' ;;
@@ -144,6 +154,13 @@ if [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" = 0 ] && [ "${mode}" = evidence ]
> "${CONTROL_TMPDIR}/pmap.log"
fi
case "${mode}" in
e3-skipped-selection)
if [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" = 0 ]; then
echo 'selected E3 entry ... ***Skipped'
exit 0
fi
exit 8
;;
skipped-selection|notrun-selection|missing-selection)
echo '1/1 Test #1: selected entry ... ***Skipped'
echo '100% tests passed, 0 tests failed out of 1'