mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-09-17 00:28:31 +09:00
[Fix] (IntegrationTest): reject skipped split control selections
This commit is contained in:
@@ -5,6 +5,7 @@ from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
|
||||
def paths(document):
|
||||
@@ -47,13 +48,36 @@ def main():
|
||||
if mode == "reset":
|
||||
for path in selected.values():
|
||||
Path(path).unlink(missing_ok=True)
|
||||
elif mode == "results":
|
||||
cases = ET.parse(sys.argv[4]).getroot().findall(".//testcase")
|
||||
label = sys.argv[5]
|
||||
by_name = {}
|
||||
for case in cases:
|
||||
by_name.setdefault(case.get("name"), []).append(case)
|
||||
skipped = sum(any(c.find("skipped") is not None for c in by_name.get(n, []))
|
||||
for n in selected)
|
||||
# ID-62: pre-flight Fatal is not evidence that a selected entry ran.
|
||||
if skipped:
|
||||
raise ValueError(f"{label} control: the knob killed the pre-flight, not the entry - "
|
||||
f"{skipped} selected entries skipped")
|
||||
missing = sum(len(by_name.get(n, [])) != 1 or
|
||||
by_name[n][0].get("status") in ("notrun", "disabled") for n in selected)
|
||||
if missing:
|
||||
raise ValueError(f"{label} control: {missing} selected entries did not run")
|
||||
not_failed = sum(c.find("failure") is None or c.get("status") != "fail"
|
||||
for n in selected for c in by_name[n])
|
||||
if not_failed:
|
||||
raise ValueError(f"{label} control: {not_failed} selected entries did not fail")
|
||||
elif mode == "evidence":
|
||||
missing = []
|
||||
for name, path in selected.items():
|
||||
if Path(path).is_file() and re.search(sys.argv[4], Path(path).read_text(errors="replace")):
|
||||
print(f"private-log evidence: {name}: {path}")
|
||||
return
|
||||
raise ValueError("E1 FAILED: selected private logs lack expected Fatal{BarrierViolation, \"<slot>\"} line: "
|
||||
+ ", ".join(f"{n} ({p})" for n, p in selected.items()))
|
||||
else:
|
||||
missing.append(f"{name} ({path})")
|
||||
if missing:
|
||||
raise ValueError("E1 FAILED: selected private logs lack expected Fatal{BarrierViolation, \"<slot>\"} line: "
|
||||
+ ", ".join(missing))
|
||||
else:
|
||||
raise ValueError(f"unknown mode: {mode}")
|
||||
|
||||
@@ -61,5 +85,5 @@ def main():
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (ValueError, OSError, subprocess.CalledProcessError) as error:
|
||||
except (ValueError, OSError, ET.ParseError, subprocess.CalledProcessError) as error:
|
||||
sys.exit(f"SplitLogPaths FAILED: {error}")
|
||||
|
||||
@@ -12,10 +12,12 @@
|
||||
set -u
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
WORK="$(mktemp -d)" || exit 1
|
||||
trap 'cp "${WORK}/split.orig" "${HERE}/split_negative_controls.sh"; cp "${WORK}/retrace.orig" "${HERE}/retrace_pull_library_control.sh"; rm -rf "${WORK}"' EXIT
|
||||
HELPER="${HERE}/../../MobileGL/MG_IntegrationTest/Harness/split_log_paths.py"
|
||||
trap 'cp "${WORK}/split.orig" "${HERE}/split_negative_controls.sh"; cp "${WORK}/retrace.orig" "${HERE}/retrace_pull_library_control.sh"; cp "${WORK}/helper.orig" "${HELPER}"; rm -rf "${WORK}"' EXIT
|
||||
|
||||
cp "${HERE}/split_negative_controls.sh" "${WORK}/split.orig"
|
||||
cp "${HERE}/retrace_pull_library_control.sh" "${WORK}/retrace.orig"
|
||||
cp "${HELPER}" "${WORK}/helper.orig"
|
||||
|
||||
echo "=== baseline: the smoke test must be GREEN before anything is perturbed"
|
||||
if ! bash "${HERE}/control_smoke_test.sh" > "${WORK}/before.log" 2>&1; then
|
||||
@@ -77,4 +79,25 @@ if [ "${missed}" -ne 0 ]; then
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "P5_T1_CONTROL_SMOKE_REDCHECK_OK - removing the evidence checks reds the unrelated split, unrelated retrace and missing private-Fatal cases"
|
||||
cp "${WORK}/split.orig" "${HERE}/split_negative_controls.sh"
|
||||
cp "${WORK}/retrace.orig" "${HERE}/retrace_pull_library_control.sh"
|
||||
echo "=== ID-62 perturbation: remove only the skip check"
|
||||
python3 - "${HELPER}" <<'PY' || exit 1
|
||||
from pathlib import Path
|
||||
import sys
|
||||
path = Path(sys.argv[1])
|
||||
text = path.read_text()
|
||||
needle = ' if skipped:\n'
|
||||
if text.count(needle) != 1:
|
||||
raise SystemExit('expected exactly one skip check')
|
||||
path.write_text(text.replace(needle, ' if False: # skip check removed by red-check\n'))
|
||||
PY
|
||||
bash "${HERE}/control_smoke_test.sh" > "${WORK}/skips.log" 2>&1
|
||||
rc=$?
|
||||
cat "${WORK}/skips.log"
|
||||
if [ "${rc}" -eq 0 ] || ! grep -qFx 'NOT OK skipped-selection: control must report FAILED for its own reason' "${WORK}/skips.log"; then
|
||||
echo 'RED-CHECK FAILED: removing the skip check must red the skipped-selection message assertion'
|
||||
exit 1
|
||||
fi
|
||||
echo 'ID-62 red-once: removing only the skip check made skipped-selection red (smoke rc=1)'
|
||||
echo "P5_T1_CONTROL_SMOKE_REDCHECK_OK - evidence and skipped-selection checks each made their smoke cases red"
|
||||
|
||||
@@ -115,10 +115,17 @@ run_control() {
|
||||
python3 "${log_helper}" reset "${manifest}" "${filter}" || exit 1
|
||||
|
||||
out="${CONTROL_TMPDIR}/control-output.txt"
|
||||
env "$@" "${CTEST}" --output-on-failure -L integration-split -R "${filter}" --no-tests=error > "${out}" 2>&1
|
||||
result="${CONTROL_TMPDIR}/control.xml"
|
||||
rm -f "${result}"
|
||||
env "$@" "${CTEST}" --output-on-failure -L integration-split -R "${filter}" --no-tests=error --output-junit "${result}" > "${out}" 2>&1
|
||||
control_rc=$?
|
||||
cat "${out}"
|
||||
|
||||
# Inspect JUnit before exit status or private Fatal: a skipped pre-flight can carry both.
|
||||
label='E3(a)'
|
||||
[ "${evidence}" != private-barrier-fatal ] || label=E1
|
||||
python3 "${log_helper}" results "${manifest}" "${filter}" "${result}" "${label}" || exit 1
|
||||
|
||||
if [ "${control_rc}" -eq 0 ]; then
|
||||
echo "::error::${name} left ${matched} split entries GREEN, so the knob it turns is not load-bearing and the gate it controls proves nothing."
|
||||
exit 1
|
||||
|
||||
+11
-2
@@ -7,7 +7,7 @@ trap 'rm -rf "${WORK}"' EXIT
|
||||
cp "${HERE}/testdata/stub_ctest.sh" "${WORK}/ctest"
|
||||
chmod +x "${WORK}/ctest"
|
||||
passes=0
|
||||
for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated; do
|
||||
for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated skipped-selection notrun-selection missing-selection partial-fatal wrong-fatal; do
|
||||
mkdir -p "${WORK}/${mode}"
|
||||
rc=0
|
||||
STUB_MODE="${mode}" CTEST="${WORK}/ctest" CONTROL_TMPDIR="${WORK}/${mode}" \
|
||||
@@ -22,7 +22,16 @@ for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated; do
|
||||
else
|
||||
message='E1 FAILED: selected private logs lack expected Fatal'
|
||||
[ "${mode}" != e3-unrelated ] || message='FAILED: red lacks its persistent-map push diagnostic'
|
||||
[ "${rc}" != 0 ] && grep -q "${message}" "${WORK}/${mode}.out" || {
|
||||
case "${mode}" in
|
||||
skipped-selection) message='SplitLogPaths FAILED: E1 control: the knob killed the pre-flight, not the entry - 1 selected entries skipped' ;;
|
||||
notrun-selection|missing-selection) message='SplitLogPaths FAILED: E1 control: 1 selected entries did not run' ;;
|
||||
esac
|
||||
if [[ "${mode}" = *-selection ]]; then
|
||||
match=(-qFx)
|
||||
else
|
||||
match=(-qF)
|
||||
fi
|
||||
[ "${rc}" != 0 ] && grep "${match[@]}" "${message}" "${WORK}/${mode}.out" || {
|
||||
cat "${WORK}/${mode}.out"; echo "NOT OK ${mode}: control must report FAILED for its own reason"; exit 1;
|
||||
}
|
||||
fi
|
||||
|
||||
Vendored
+38
-4
@@ -15,6 +15,11 @@
|
||||
# stdout-fatal Fatal exists only on stdout, never in the private file
|
||||
# stale-fatal Fatal exists before reset, never from this control run
|
||||
# e3-unrelated E1 has its private Fatal; E3(a) fails for an unrelated reason
|
||||
# skipped-selection knob kills pre-flight; private Fatal exists, entry skips, ctest exits 0
|
||||
# notrun-selection selected entry was not run, despite a private Fatal
|
||||
# missing-selection selected entry is absent from the result XML
|
||||
# partial-fatal two entries fail, but only the first has its expected private Fatal
|
||||
# wrong-fatal entry fails with a different Fatal in its private file
|
||||
# green baseline green; the control's own run PASSES (the knob is not load-bearing)
|
||||
# red-baseline the baseline itself has a failed entry
|
||||
# all-skipped the baseline is entirely skipped (the disarmed lane, a legitimate exit 0)
|
||||
@@ -44,10 +49,31 @@ emit_listing() {
|
||||
return
|
||||
fi
|
||||
echo " Test #1: DirectGLES.Split.ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels"
|
||||
echo "Total Tests: 1"
|
||||
if [ "${mode}" = partial-fatal ]; then
|
||||
echo " Test #2: DirectGLES.Split.TriangleScenario.SecondEntry"
|
||||
echo "Total Tests: 2"
|
||||
else
|
||||
echo "Total Tests: 1"
|
||||
fi
|
||||
}
|
||||
|
||||
write_junit() {
|
||||
if [ "${MOBILEGL_IPC_VERB_BARRIER:-1}" = 0 ] || [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" = 0 ]; then
|
||||
entry=DirectGLES.Split.ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels
|
||||
[ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" != 0 ] || entry=DirectGLES.Split.PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw
|
||||
body="<testcase name=\"${entry}\" status=\"fail\"><failure message=\"control red\"/></testcase>"
|
||||
if [ "${mode}" = partial-fatal ] && [ "${MOBILEGL_IPC_VERB_BARRIER:-1}" = 0 ]; then
|
||||
body="${body}<testcase name=\"DirectGLES.Split.TriangleScenario.SecondEntry\" status=\"fail\"><failure/></testcase>"
|
||||
fi
|
||||
case "${mode}" in
|
||||
skipped-selection) body="<testcase name=\"${entry}\" status=\"notrun\"><skipped/></testcase>" ;;
|
||||
notrun-selection) body="<testcase name=\"${entry}\" status=\"notrun\"/>" ;;
|
||||
missing-selection) body='' ;;
|
||||
green) body="<testcase name=\"${entry}\" status=\"run\"/>" ;;
|
||||
esac
|
||||
printf '%s\n' "<testsuite>${body}</testsuite>" > "$1"
|
||||
return
|
||||
fi
|
||||
case "${mode}" in
|
||||
red-baseline)
|
||||
body='<testcase name="DirectGLES.Split.TriangleScenario.AVboBackedTriangleReachesReadPixels" status="failed"><failure message="already red"/></testcase>'
|
||||
@@ -65,7 +91,7 @@ write_junit() {
|
||||
# Model the library file sink separately from ctest stdout (ID-53).
|
||||
log="${CONTROL_TMPDIR}/entry.log"
|
||||
if [ "${json_requested:-0}" = 1 ]; then
|
||||
python3 -c 'import json, os; p=os.environ["CONTROL_TMPDIR"]; print(json.dumps({"tests": [{"name": "DirectGLES.Split."+n, "properties": [{"name": "LABELS", "value": ["integration-split"]}, {"name": "ENVIRONMENT", "value": ["MOBILEGL_LOG_FILE_PATH="+p+"/"+f]}]} for n,f in [("ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels", "entry.log"), ("PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw", "pmap.log")]]}))'
|
||||
python3 -c 'import json, os; p=os.environ["CONTROL_TMPDIR"]; entries=[("ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels", "entry.log"), ("PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw", "pmap.log")]; entries += [("TriangleScenario.SecondEntry", "second.log")] if os.environ["STUB_MODE"] == "partial-fatal" else []; print(json.dumps({"tests": [{"name": "DirectGLES.Split."+n, "properties": [{"name": "LABELS", "value": ["integration-split"]}, {"name": "ENVIRONMENT", "value": ["MOBILEGL_LOG_FILE_PATH="+p+"/"+f]}]} for n,f in entries]}))'
|
||||
if [ "${mode}" = stale-fatal ]; then
|
||||
echo 'Fatal{BarrierViolation, "DrawVbo"}' > "${log}"
|
||||
fi
|
||||
@@ -79,22 +105,30 @@ fi
|
||||
|
||||
if [ -n "${junit}" ]; then
|
||||
write_junit "${junit}"
|
||||
if [ "${MOBILEGL_IPC_VERB_BARRIER:-1}" != 0 ] && [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" != 0 ]; then
|
||||
case "${mode}" in
|
||||
red-baseline) echo "1/1 Test #1: ... ***Failed"; exit 8 ;;
|
||||
*) echo "100% tests passed, 0 tests failed out of 1"; exit 0 ;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
|
||||
# The control's own run.
|
||||
if [ "${MOBILEGL_IPC_VERB_BARRIER:-1}" = 0 ]; then
|
||||
case "${mode}" in
|
||||
evidence|e3-unrelated) echo 'Fatal{BarrierViolation, "DrawVbo"}' > "${log}" ;;
|
||||
evidence|e3-unrelated|skipped-selection|notrun-selection|missing-selection|partial-fatal) echo 'Fatal{BarrierViolation, "DrawVbo"}' > "${log}" ;;
|
||||
wrong-fatal) echo 'Fatal{ReplyMissing, "DrawVbo"}' > "${log}" ;;
|
||||
missing-fatal) echo "library setup only; no fatal" > "${log}" ;;
|
||||
stdout-fatal) echo 'Fatal{BarrierViolation, "DrawVbo"}' ;;
|
||||
esac
|
||||
fi
|
||||
case "${mode}" in
|
||||
unrelated|missing-fatal|stdout-fatal|stale-fatal|e3-unrelated)
|
||||
skipped-selection|notrun-selection|missing-selection)
|
||||
echo '1/1 Test #1: selected entry ... ***Skipped'
|
||||
echo '100% tests passed, 0 tests failed out of 1'
|
||||
exit 0
|
||||
;;
|
||||
unrelated|missing-fatal|stdout-fatal|stale-fatal|e3-unrelated|partial-fatal|wrong-fatal)
|
||||
echo "1/1 Test #1: DirectGLES.Split.ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels ...***Failed"
|
||||
echo "UNRELATED_CONTROL_FAILURE: the harness aborted in setup before the knob was read"
|
||||
if [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" = 0 ]; then
|
||||
|
||||
Reference in New Issue
Block a user