mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-09-18 00:58:30 +09:00
[Fix] (CI): make split controls mandatory and isolate runner evidence
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Summarize the indebted broad lane without turning debt into a hard gate."""
|
||||
import collections
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
cases = ET.parse(sys.argv[1]).getroot().findall('.//testcase')
|
||||
if not cases:
|
||||
sys.exit('Census FAILED: no executed test records')
|
||||
counts = collections.Counter()
|
||||
for case in cases:
|
||||
failure = case.find('failure')
|
||||
if failure is not None:
|
||||
reason = failure.get('message', '')
|
||||
status = 'aborted' if 'aborted' in reason.lower() else 'failed'
|
||||
elif case.find('skipped') is not None:
|
||||
status = 'skipped'
|
||||
else:
|
||||
status = 'passed'
|
||||
counts[status] += 1
|
||||
print('### Broad inproc census (ID-65: recorded debt, not a gate)')
|
||||
print(f'CTest exit: {sys.argv[2]}; total: {len(cases)}')
|
||||
print('\n| passed | skipped | aborted | failed |\n|---:|---:|---:|---:|')
|
||||
print('| ' + ' | '.join(str(counts[k]) for k in ('passed', 'skipped', 'aborted', 'failed')) + ' |')
|
||||
@@ -56,10 +56,15 @@ run_split() { # $1 = STUB_MODE
|
||||
run_retrace() { # $1 = STUB_MODE
|
||||
cd "${WORK}" || return 127
|
||||
mkdir -p "${WORK}/OpenRA"
|
||||
local rc=0
|
||||
env -i PATH="${STUB_DIR}:/usr/bin:/bin" STUB_MODE="$1" \
|
||||
CTEST=ctest CONTROL_TMPDIR="${WORK}/tmp-$1" \
|
||||
PULL_LIBRARY="${WORK}/pull.so" FROZEN_LIBRARY="${WORK}/frozen.so" \
|
||||
bash "${HERE}/retrace_pull_library_control.sh" OpenRA DirectGLES
|
||||
bash "${HERE}/retrace_pull_library_control.sh" OpenRA DirectGLES || rc=$?
|
||||
cmp -s "${WORK}/frozen.so" "${WORK}/split.so" || {
|
||||
echo 'F6 FAILED: pull control did not restore the split library'; return 1;
|
||||
}
|
||||
return "${rc}"
|
||||
}
|
||||
|
||||
run_drop_draw() { # $1 = STUB_MODE
|
||||
@@ -67,7 +72,7 @@ run_drop_draw() { # $1 = STUB_MODE
|
||||
mkdir -p "${WORK}/OpenRA"
|
||||
env -i PATH="${STUB_DIR}:/usr/bin:/bin" STUB_MODE="$1" \
|
||||
CTEST=ctest CONTROL_TMPDIR="${WORK}/tmp-$1" \
|
||||
LIBRARY_LOG="${WORK}/tmp-$1/mobilegl.log" \
|
||||
FROZEN_LIBRARY="${WORK}/frozen.so" LIBRARY_LOG="${WORK}/tmp-$1/mobilegl.log" \
|
||||
bash "${HERE}/retrace_drop_draw_control.sh" OpenRA DirectGLES
|
||||
}
|
||||
|
||||
@@ -80,8 +85,8 @@ expect PASSED "the scenarios' own diagnostic" -- run_split evidenc
|
||||
expect FAILED "the knob leaves the selection green" -- run_split green
|
||||
# The arming counter's half of the finding: a baseline that is already red cannot arm anything.
|
||||
expect FAILED "the baseline is already red" -- run_split red-baseline
|
||||
# The disarmed lane, which is a legitimate exit 0 while c1/s1/v1 are landing.
|
||||
expect PASSED "every split entry skipped (lane not armed)" -- run_split all-skipped
|
||||
# P5 is complete: losing the runtime implementation must no longer disarm the gate.
|
||||
expect FAILED "every split entry skipped (implementation lost)" -- run_split all-skipped
|
||||
|
||||
echo
|
||||
echo "=== the retrace lane's pull-library control (scripts/ci/retrace_pull_library_control.sh)"
|
||||
@@ -93,7 +98,9 @@ else
|
||||
echo "no cc available; the retrace half of this smoke test needs one" >&2
|
||||
exit 1
|
||||
fi
|
||||
: > "${WORK}/frozen.so"
|
||||
printf '%s\n' 'int MG_Remote_stub(void) { return 1; }' > "${WORK}/split.c"
|
||||
cc -shared -fPIC -o "${WORK}/frozen.so" "${WORK}/split.c" || exit 1
|
||||
cp "${WORK}/frozen.so" "${WORK}/split.so"
|
||||
|
||||
# THE FINDING, part (b): a regex matching no tests. --no-tests=error exits non-zero and the old
|
||||
# control read that as "the pull library turned it red".
|
||||
|
||||
@@ -20,9 +20,11 @@ import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
|
||||
def tally(path):
|
||||
def tally(path, split_only=False):
|
||||
passed = failed = skipped = 0
|
||||
for case in ET.parse(path).getroot().iter('testcase'):
|
||||
if split_only and not case.get('name', '').startswith('DirectGLES.Split.'):
|
||||
continue
|
||||
if case.find('failure') is not None or case.find('error') is not None:
|
||||
failed += 1
|
||||
elif case.find('skipped') is not None or case.get('status') in ('notrun', 'disabled'):
|
||||
@@ -33,15 +35,18 @@ def tally(path):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: junit_tally.py <junit.xml>", file=sys.stderr)
|
||||
if len(sys.argv) not in (2, 3) or (len(sys.argv) == 3 and sys.argv[2] != '--require-split-ran'):
|
||||
print("usage: junit_tally.py <junit.xml> [--require-split-ran]", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
passed, failed, skipped = tally(sys.argv[1])
|
||||
passed, failed, skipped = tally(sys.argv[1], split_only=len(sys.argv) == 3)
|
||||
except Exception as exc: # a malformed file is not "zero of everything"
|
||||
print(f"junit_tally: cannot parse {sys.argv[1]}: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
print(f"{passed} {failed} {skipped}")
|
||||
if len(sys.argv) == 3 and (passed == 0 or failed):
|
||||
print('split baseline FAILED: no successful split runtime entries or an already-red selection', file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ split, retrace, dropdraw = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
# other two catching the smoke test's "unrelated failure" case, and the case never flips - which
|
||||
# is what this script measured the first time the perturbation actually applied.
|
||||
rules = [
|
||||
(split, [('LINE', 'grep -qE "${evidence}"', ('if ! ', 'elif ! ')),
|
||||
(split, [('LINE', '"${log_helper}" assertion', ('if ! ', 'elif ! ')),
|
||||
('SUBST', '"[A-Za-z_][A-Za-z_0-9]*"\\}\' || exit 1', '"[A-Za-z_][A-Za-z_0-9]*"\\}\' || true'),
|
||||
('SUBST', '"${private_evidence}" "${name}" || exit 1',
|
||||
'"${private_evidence}" "${name}" || true')]),
|
||||
|
||||
@@ -50,6 +50,14 @@ CTEST="${CTEST:-ctest}"
|
||||
CONTROL_TMPDIR="${CONTROL_TMPDIR:-${RUNNER_TEMP:-/tmp}}"
|
||||
LIBRARY_LOG="${LIBRARY_LOG:-${CASE}/${BACKEND}/output/mobilegl.log}"
|
||||
mkdir -p "${CONTROL_TMPDIR}"
|
||||
FROZEN_LIBRARY="${FROZEN_LIBRARY:?FROZEN_LIBRARY must name the split library the replay loads}"
|
||||
symbols=$(nm --defined-only "${FROZEN_LIBRARY}") || exit 1
|
||||
remote_count=$(printf '%s\n' "${symbols}" | grep -ic MG_Remote || true)
|
||||
echo "draw-drop control library: ${FROZEN_LIBRARY}: MG_Remote=${remote_count}"
|
||||
if [ "${remote_count}" -lt 1 ]; then
|
||||
echo '::error::draw-drop control requires a split library: MG_Remote=0'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
selector="^MobileGLTraceReplay\.${CASE}\.${BACKEND}$"
|
||||
|
||||
@@ -66,6 +74,8 @@ restore_good_output() {
|
||||
echo "restored the verified run's output over the control's"
|
||||
fi
|
||||
}
|
||||
trap restore_good_output EXIT
|
||||
trap 'exit 130' INT TERM
|
||||
|
||||
matched=$("${CTEST}" -N -R "${selector}" | grep -cE '^ *Test *#[0-9]+:')
|
||||
if [ "${matched}" -lt 1 ]; then
|
||||
|
||||
@@ -69,6 +69,18 @@ restore_good_output() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Restore the exact split library on success, failure, and interruption. The following
|
||||
# draw-drop control uses this same frozen path.
|
||||
saved_library=$(mktemp "${CONTROL_TMPDIR}/split-library.XXXXXX") || exit 1
|
||||
cp -p "${FROZEN_LIBRARY}" "${saved_library}" || exit 1
|
||||
restore_control() {
|
||||
cp -p "${saved_library}" "${FROZEN_LIBRARY}"
|
||||
rm -f "${saved_library}"
|
||||
restore_good_output
|
||||
}
|
||||
trap restore_control EXIT
|
||||
trap 'exit 130' INT TERM
|
||||
|
||||
# HOLE 1: COUNT THE SELECTION FIRST. `--no-tests=error` turns an empty selection into a non-zero
|
||||
# exit, which is indistinguishable from a working control unless the selection is counted.
|
||||
matched=$("${CTEST}" -N -R "${selector}" | grep -cE '^ *Test *#[0-9]+:')
|
||||
@@ -82,8 +94,11 @@ fi
|
||||
# in. It defines no MG_Remote symbol, so ConfigLoader has no transport parser and
|
||||
# MOBILEGL_TRANSPORT=inproc is accepted and ignored - the exact shape of "the split lane ran
|
||||
# monolith".
|
||||
cp "${PULL_LIBRARY}" "${FROZEN_LIBRARY}"
|
||||
if nm --defined-only "${FROZEN_LIBRARY}" | grep -q -i MG_Remote; then
|
||||
cp "${PULL_LIBRARY}" "${FROZEN_LIBRARY}" || exit 1
|
||||
symbols=$(nm --defined-only "${FROZEN_LIBRARY}") || exit 1
|
||||
remote_count=$(printf '%s\n' "${symbols}" | grep -ic MG_Remote || true)
|
||||
echo "pull control library: ${FROZEN_LIBRARY}: MG_Remote=${remote_count}"
|
||||
if [ "${remote_count}" -ne 0 ]; then
|
||||
restore_good_output
|
||||
echo "::error::the control's own library defines MG_Remote symbols, so it is not a pull build and this control would prove nothing"
|
||||
exit 1
|
||||
|
||||
@@ -51,6 +51,7 @@ CONTROL_TMPDIR="${CONTROL_TMPDIR:-${RUNNER_TEMP:-/tmp}}"
|
||||
mkdir -p "${CONTROL_TMPDIR}"
|
||||
|
||||
junit="${CONTROL_TMPDIR}/isplit.xml"
|
||||
rm -f "${junit}"
|
||||
log_helper="$(dirname "$0")/../../MobileGL/MG_IntegrationTest/Harness/split_log_paths.py"
|
||||
|
||||
# Check ownership even while the runtime lane is disarmed and will skip.
|
||||
@@ -87,14 +88,14 @@ echo "split entries - passed: ${baseline_passed}, failed: ${baseline_failed}, sk
|
||||
# A RED BASELINE DISARMS THE CONTROLS RATHER THAN ARMING THEM (review finding 8, second half).
|
||||
# `|| true` plus a "not skipped" counter used to treat a case that ran and FAILED as evidence the
|
||||
# lane was live. Turning an already-red entry red is not a measurement.
|
||||
if [ "${baseline_failed}" -gt 0 ]; then
|
||||
if [ "${baseline_failed}" -gt 0 ] || [ "${baseline_rc}" -ne 0 ]; then
|
||||
echo "::error::${baseline_failed} DirectGLES.Split. entries are ALREADY RED with both knobs at their defaults, so neither negative control below can attribute its red to the knob it turns. Fix the lane first; a control measured against a red baseline is not a control. (This used to be swallowed by an unconditional '|| true' and counted as 'the lane is armed'.)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${baseline_passed}" -lt 1 ]; then
|
||||
echo "::warning::every DirectGLES.Split. entry SKIPPED, so neither negative control can fire. The arming condition is a runtime fact - MG_Config::Transport, ClientSession::Active() and ImplementedVerbCount(), read by Harness/SplitRuntimePeek - and it becomes true on the commit that lands the last of c1/s1/v1. This step becomes a gate then, with no edit; it is not a green that asserted anything today."
|
||||
exit 0
|
||||
echo "::error::split baseline FAILED: every DirectGLES.Split. entry SKIPPED; the split implementation did not execute"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ---- the controls ---------------------------------------------------------------------------
|
||||
@@ -145,7 +146,7 @@ run_control() {
|
||||
if [ "${evidence}" = "private-barrier-fatal" ]; then
|
||||
python3 "${log_helper}" evidence "${manifest}" "${filter}" \
|
||||
'Fatal\{BarrierViolation, "[A-Za-z_][A-Za-z_0-9]*"\}' || exit 1
|
||||
elif ! tr -s '[:space:]' ' ' < "${out}" | grep -qE "${evidence}"; then
|
||||
elif ! python3 "${log_helper}" assertion "${manifest}" "${filter}" "${result}" "${evidence}"; then
|
||||
echo "::error::${name} FAILED: red lacks its persistent-map push diagnostic. Required: ${evidence}"
|
||||
exit 1
|
||||
fi
|
||||
@@ -179,8 +180,10 @@ run_control "negative control E1 (MOBILEGL_IPC_VERB_BARRIER=0)" \
|
||||
# * the LIBRARY's own line in the entry's private file, saying the push was disabled by this
|
||||
# knob. It did not exist until ID-65 assigned it (joint-v1.md 3), which is why this control
|
||||
# used to rest on the pixels alone.
|
||||
# TheMapLandsInTheArmItsLaneDeclares skips by design outside PersistentMapArm.
|
||||
# Select only the pixel cases; a pre-flight skip in either remains a hard failure.
|
||||
run_control "negative control E3(a) (MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0)" \
|
||||
'DirectGLES\.Split\.(SmallRing\.)?PersistentCoherentMapScenario' \
|
||||
'DirectGLES\.Split\.(SmallRing\.)?PersistentCoherentMapScenario\.(TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw|AWriteAfterAFrameBoundaryReachesTheNextFramesDraw)$' \
|
||||
"the SECOND write through the same mapping, announced by nothing|frame 1's write through the SAME mapping, after a Present" \
|
||||
'MGPipe: persistent-map push disabled - MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0' \
|
||||
MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0
|
||||
|
||||
+2
-1
@@ -7,7 +7,7 @@ trap 'rm -rf "${WORK}"' EXIT
|
||||
cp "${HERE}/testdata/stub_ctest.sh" "${WORK}/ctest"
|
||||
chmod +x "${WORK}/ctest"
|
||||
passes=0
|
||||
for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated e3-no-private skipped-selection notrun-selection missing-selection partial-fatal wrong-fatal; do
|
||||
for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated e3-no-private skipped-selection e3-skipped-selection notrun-selection missing-selection partial-fatal wrong-fatal; do
|
||||
mkdir -p "${WORK}/${mode}"
|
||||
rc=0
|
||||
STUB_MODE="${mode}" CTEST="${WORK}/ctest" CONTROL_TMPDIR="${WORK}/${mode}" \
|
||||
@@ -25,6 +25,7 @@ for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated e3-no-p
|
||||
case "${mode}" in
|
||||
e3-no-private) message='no selected private log carries /MGPipe: persistent-map push disabled' ;;
|
||||
skipped-selection) message='SplitLogPaths FAILED: E1 control: the knob killed the pre-flight, not the entry - 1 selected entries skipped' ;;
|
||||
e3-skipped-selection) message='SplitLogPaths FAILED: E3(a) control: the knob killed the pre-flight, not the entry - 1 selected entries skipped' ;;
|
||||
notrun-selection|missing-selection) message='SplitLogPaths FAILED: E1 control: 1 selected entries did not run' ;;
|
||||
esac
|
||||
if [[ "${mode}" = *-selection ]]; then
|
||||
|
||||
Vendored
+19
-2
@@ -24,7 +24,7 @@
|
||||
# never says the push was disabled (the half ID-65 added)
|
||||
# green baseline green; the control's own run PASSES (the knob is not load-bearing)
|
||||
# red-baseline the baseline itself has a failed entry
|
||||
# all-skipped the baseline is entirely skipped (the disarmed lane, a legitimate exit 0)
|
||||
# all-skipped the baseline is entirely skipped (lost implementation, a hard failure)
|
||||
# retrace-noselect `ctest -N` matches nothing; the run exits 8 the way --no-tests=error does
|
||||
# retrace-unrelated one match; the run fails without naming the transport
|
||||
# retrace-evidence one match; the run fails with run_trace_case.cmake's own sentence
|
||||
@@ -76,10 +76,20 @@ write_junit() {
|
||||
entry=DirectGLES.Split.ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels
|
||||
[ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" != 0 ] || entry=DirectGLES.Split.PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw
|
||||
body="<testcase name=\"${entry}\" status=\"fail\"><failure message=\"control red\"/></testcase>"
|
||||
if [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" = 0 ]; then
|
||||
case "${mode}" in
|
||||
evidence|e3-no-private)
|
||||
body="<testcase name=\"${entry}\" status=\"fail\"><failure/><system-out>the SECOND write through the same mapping, announced by nothing</system-out></testcase>" ;;
|
||||
esac
|
||||
fi
|
||||
if [ "${mode}" = partial-fatal ] && [ "${MOBILEGL_IPC_VERB_BARRIER:-1}" = 0 ]; then
|
||||
body="${body}<testcase name=\"DirectGLES.Split.TriangleScenario.SecondEntry\" status=\"fail\"><failure/></testcase>"
|
||||
fi
|
||||
case "${mode}" in
|
||||
e3-skipped-selection)
|
||||
if [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" = 0 ]; then
|
||||
body="<testcase name=\"${entry}\" status=\"notrun\"><skipped/></testcase>"
|
||||
fi ;;
|
||||
skipped-selection) body="<testcase name=\"${entry}\" status=\"notrun\"><skipped/></testcase>" ;;
|
||||
notrun-selection) body="<testcase name=\"${entry}\" status=\"notrun\"/>" ;;
|
||||
missing-selection) body='' ;;
|
||||
@@ -130,7 +140,7 @@ fi
|
||||
# The control's own run.
|
||||
if [ "${MOBILEGL_IPC_VERB_BARRIER:-1}" = 0 ]; then
|
||||
case "${mode}" in
|
||||
evidence|e3-unrelated|e3-no-private|skipped-selection|notrun-selection|missing-selection|partial-fatal) echo 'Fatal{BarrierViolation, "DrawVbo"}' > "${log}" ;;
|
||||
evidence|e3-unrelated|e3-no-private|skipped-selection|e3-skipped-selection|notrun-selection|missing-selection|partial-fatal) echo 'Fatal{BarrierViolation, "DrawVbo"}' > "${log}" ;;
|
||||
wrong-fatal) echo 'Fatal{ReplyMissing, "DrawVbo"}' > "${log}" ;;
|
||||
missing-fatal) echo "library setup only; no fatal" > "${log}" ;;
|
||||
stdout-fatal) echo 'Fatal{BarrierViolation, "DrawVbo"}' ;;
|
||||
@@ -144,6 +154,13 @@ if [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" = 0 ] && [ "${mode}" = evidence ]
|
||||
> "${CONTROL_TMPDIR}/pmap.log"
|
||||
fi
|
||||
case "${mode}" in
|
||||
e3-skipped-selection)
|
||||
if [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" = 0 ]; then
|
||||
echo 'selected E3 entry ... ***Skipped'
|
||||
exit 0
|
||||
fi
|
||||
exit 8
|
||||
;;
|
||||
skipped-selection|notrun-selection|missing-selection)
|
||||
echo '1/1 Test #1: selected entry ... ***Skipped'
|
||||
echo '100% tests passed, 0 tests failed out of 1'
|
||||
|
||||
Reference in New Issue
Block a user