diff --git a/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py b/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py index b605ecc4..100ea019 100644 --- a/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py +++ b/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py @@ -5,6 +5,7 @@ from pathlib import Path import re import subprocess import sys +import xml.etree.ElementTree as ET def paths(document): @@ -47,13 +48,36 @@ def main(): if mode == "reset": for path in selected.values(): Path(path).unlink(missing_ok=True) + elif mode == "results": + cases = ET.parse(sys.argv[4]).getroot().findall(".//testcase") + label = sys.argv[5] + by_name = {} + for case in cases: + by_name.setdefault(case.get("name"), []).append(case) + skipped = sum(any(c.find("skipped") is not None for c in by_name.get(n, [])) + for n in selected) + # ID-62: pre-flight Fatal is not evidence that a selected entry ran. + if skipped: + raise ValueError(f"{label} control: the knob killed the pre-flight, not the entry - " + f"{skipped} selected entries skipped") + missing = sum(len(by_name.get(n, [])) != 1 or + by_name[n][0].get("status") in ("notrun", "disabled") for n in selected) + if missing: + raise ValueError(f"{label} control: {missing} selected entries did not run") + not_failed = sum(c.find("failure") is None or c.get("status") != "fail" + for n in selected for c in by_name[n]) + if not_failed: + raise ValueError(f"{label} control: {not_failed} selected entries did not fail") elif mode == "evidence": + missing = [] for name, path in selected.items(): if Path(path).is_file() and re.search(sys.argv[4], Path(path).read_text(errors="replace")): print(f"private-log evidence: {name}: {path}") - return - raise ValueError("E1 FAILED: selected private logs lack expected Fatal{BarrierViolation, \"\"} line: " - + ", ".join(f"{n} ({p})" for n, p in selected.items())) + else: + missing.append(f"{name} ({path})") + if missing: + raise ValueError("E1 FAILED: selected private logs lack expected Fatal{BarrierViolation, \"\"} line: " + + ", ".join(missing)) else: raise ValueError(f"unknown mode: {mode}") @@ -61,5 +85,5 @@ def main(): if __name__ == "__main__": try: main() - except (ValueError, OSError, subprocess.CalledProcessError) as error: + except (ValueError, OSError, ET.ParseError, subprocess.CalledProcessError) as error: sys.exit(f"SplitLogPaths FAILED: {error}") diff --git a/scripts/ci/redcheck_control_smoke_test.sh b/scripts/ci/redcheck_control_smoke_test.sh index b8ee301f..6fff05ff 100755 --- a/scripts/ci/redcheck_control_smoke_test.sh +++ b/scripts/ci/redcheck_control_smoke_test.sh @@ -12,10 +12,12 @@ set -u HERE="$(cd "$(dirname "$0")" && pwd)" WORK="$(mktemp -d)" || exit 1 -trap 'cp "${WORK}/split.orig" "${HERE}/split_negative_controls.sh"; cp "${WORK}/retrace.orig" "${HERE}/retrace_pull_library_control.sh"; rm -rf "${WORK}"' EXIT +HELPER="${HERE}/../../MobileGL/MG_IntegrationTest/Harness/split_log_paths.py" +trap 'cp "${WORK}/split.orig" "${HERE}/split_negative_controls.sh"; cp "${WORK}/retrace.orig" "${HERE}/retrace_pull_library_control.sh"; cp "${WORK}/helper.orig" "${HELPER}"; rm -rf "${WORK}"' EXIT cp "${HERE}/split_negative_controls.sh" "${WORK}/split.orig" cp "${HERE}/retrace_pull_library_control.sh" "${WORK}/retrace.orig" +cp "${HELPER}" "${WORK}/helper.orig" echo "=== baseline: the smoke test must be GREEN before anything is perturbed" if ! bash "${HERE}/control_smoke_test.sh" > "${WORK}/before.log" 2>&1; then @@ -77,4 +79,25 @@ if [ "${missed}" -ne 0 ]; then fi echo -echo "P5_T1_CONTROL_SMOKE_REDCHECK_OK - removing the evidence checks reds the unrelated split, unrelated retrace and missing private-Fatal cases" +cp "${WORK}/split.orig" "${HERE}/split_negative_controls.sh" +cp "${WORK}/retrace.orig" "${HERE}/retrace_pull_library_control.sh" +echo "=== ID-62 perturbation: remove only the skip check" +python3 - "${HELPER}" <<'PY' || exit 1 +from pathlib import Path +import sys +path = Path(sys.argv[1]) +text = path.read_text() +needle = ' if skipped:\n' +if text.count(needle) != 1: + raise SystemExit('expected exactly one skip check') +path.write_text(text.replace(needle, ' if False: # skip check removed by red-check\n')) +PY +bash "${HERE}/control_smoke_test.sh" > "${WORK}/skips.log" 2>&1 +rc=$? +cat "${WORK}/skips.log" +if [ "${rc}" -eq 0 ] || ! grep -qFx 'NOT OK skipped-selection: control must report FAILED for its own reason' "${WORK}/skips.log"; then + echo 'RED-CHECK FAILED: removing the skip check must red the skipped-selection message assertion' + exit 1 +fi +echo 'ID-62 red-once: removing only the skip check made skipped-selection red (smoke rc=1)' +echo "P5_T1_CONTROL_SMOKE_REDCHECK_OK - evidence and skipped-selection checks each made their smoke cases red" diff --git a/scripts/ci/split_negative_controls.sh b/scripts/ci/split_negative_controls.sh index 9c4644dd..6d4f90c4 100755 --- a/scripts/ci/split_negative_controls.sh +++ b/scripts/ci/split_negative_controls.sh @@ -115,10 +115,17 @@ run_control() { python3 "${log_helper}" reset "${manifest}" "${filter}" || exit 1 out="${CONTROL_TMPDIR}/control-output.txt" - env "$@" "${CTEST}" --output-on-failure -L integration-split -R "${filter}" --no-tests=error > "${out}" 2>&1 + result="${CONTROL_TMPDIR}/control.xml" + rm -f "${result}" + env "$@" "${CTEST}" --output-on-failure -L integration-split -R "${filter}" --no-tests=error --output-junit "${result}" > "${out}" 2>&1 control_rc=$? cat "${out}" + # Inspect JUnit before exit status or private Fatal: a skipped pre-flight can carry both. + label='E3(a)' + [ "${evidence}" != private-barrier-fatal ] || label=E1 + python3 "${log_helper}" results "${manifest}" "${filter}" "${result}" "${label}" || exit 1 + if [ "${control_rc}" -eq 0 ]; then echo "::error::${name} left ${matched} split entries GREEN, so the knob it turns is not load-bearing and the gate it controls proves nothing." exit 1 diff --git a/scripts/ci/testdata/split_private_log_smoke.sh b/scripts/ci/testdata/split_private_log_smoke.sh index 725cb5de..fa01dcdd 100644 --- a/scripts/ci/testdata/split_private_log_smoke.sh +++ b/scripts/ci/testdata/split_private_log_smoke.sh @@ -7,7 +7,7 @@ trap 'rm -rf "${WORK}"' EXIT cp "${HERE}/testdata/stub_ctest.sh" "${WORK}/ctest" chmod +x "${WORK}/ctest" passes=0 -for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated; do +for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated skipped-selection notrun-selection missing-selection partial-fatal wrong-fatal; do mkdir -p "${WORK}/${mode}" rc=0 STUB_MODE="${mode}" CTEST="${WORK}/ctest" CONTROL_TMPDIR="${WORK}/${mode}" \ @@ -22,7 +22,16 @@ for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated; do else message='E1 FAILED: selected private logs lack expected Fatal' [ "${mode}" != e3-unrelated ] || message='FAILED: red lacks its persistent-map push diagnostic' - [ "${rc}" != 0 ] && grep -q "${message}" "${WORK}/${mode}.out" || { + case "${mode}" in + skipped-selection) message='SplitLogPaths FAILED: E1 control: the knob killed the pre-flight, not the entry - 1 selected entries skipped' ;; + notrun-selection|missing-selection) message='SplitLogPaths FAILED: E1 control: 1 selected entries did not run' ;; + esac + if [[ "${mode}" = *-selection ]]; then + match=(-qFx) + else + match=(-qF) + fi + [ "${rc}" != 0 ] && grep "${match[@]}" "${message}" "${WORK}/${mode}.out" || { cat "${WORK}/${mode}.out"; echo "NOT OK ${mode}: control must report FAILED for its own reason"; exit 1; } fi diff --git a/scripts/ci/testdata/stub_ctest.sh b/scripts/ci/testdata/stub_ctest.sh index de9dba3e..e6b4ea15 100755 --- a/scripts/ci/testdata/stub_ctest.sh +++ b/scripts/ci/testdata/stub_ctest.sh @@ -15,6 +15,11 @@ # stdout-fatal Fatal exists only on stdout, never in the private file # stale-fatal Fatal exists before reset, never from this control run # e3-unrelated E1 has its private Fatal; E3(a) fails for an unrelated reason +# skipped-selection knob kills pre-flight; private Fatal exists, entry skips, ctest exits 0 +# notrun-selection selected entry was not run, despite a private Fatal +# missing-selection selected entry is absent from the result XML +# partial-fatal two entries fail, but only the first has its expected private Fatal +# wrong-fatal entry fails with a different Fatal in its private file # green baseline green; the control's own run PASSES (the knob is not load-bearing) # red-baseline the baseline itself has a failed entry # all-skipped the baseline is entirely skipped (the disarmed lane, a legitimate exit 0) @@ -44,10 +49,31 @@ emit_listing() { return fi echo " Test #1: DirectGLES.Split.ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels" - echo "Total Tests: 1" + if [ "${mode}" = partial-fatal ]; then + echo " Test #2: DirectGLES.Split.TriangleScenario.SecondEntry" + echo "Total Tests: 2" + else + echo "Total Tests: 1" + fi } write_junit() { + if [ "${MOBILEGL_IPC_VERB_BARRIER:-1}" = 0 ] || [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" = 0 ]; then + entry=DirectGLES.Split.ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels + [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" != 0 ] || entry=DirectGLES.Split.PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw + body="" + if [ "${mode}" = partial-fatal ] && [ "${MOBILEGL_IPC_VERB_BARRIER:-1}" = 0 ]; then + body="${body}" + fi + case "${mode}" in + skipped-selection) body="" ;; + notrun-selection) body="" ;; + missing-selection) body='' ;; + green) body="" ;; + esac + printf '%s\n' "${body}" > "$1" + return + fi case "${mode}" in red-baseline) body='' @@ -65,7 +91,7 @@ write_junit() { # Model the library file sink separately from ctest stdout (ID-53). log="${CONTROL_TMPDIR}/entry.log" if [ "${json_requested:-0}" = 1 ]; then - python3 -c 'import json, os; p=os.environ["CONTROL_TMPDIR"]; print(json.dumps({"tests": [{"name": "DirectGLES.Split."+n, "properties": [{"name": "LABELS", "value": ["integration-split"]}, {"name": "ENVIRONMENT", "value": ["MOBILEGL_LOG_FILE_PATH="+p+"/"+f]}]} for n,f in [("ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels", "entry.log"), ("PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw", "pmap.log")]]}))' + python3 -c 'import json, os; p=os.environ["CONTROL_TMPDIR"]; entries=[("ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels", "entry.log"), ("PersistentCoherentMapScenario.TwoWritesThroughTheCoherentPointerEachReachTheirOwnDraw", "pmap.log")]; entries += [("TriangleScenario.SecondEntry", "second.log")] if os.environ["STUB_MODE"] == "partial-fatal" else []; print(json.dumps({"tests": [{"name": "DirectGLES.Split."+n, "properties": [{"name": "LABELS", "value": ["integration-split"]}, {"name": "ENVIRONMENT", "value": ["MOBILEGL_LOG_FILE_PATH="+p+"/"+f]}]} for n,f in entries]}))' if [ "${mode}" = stale-fatal ]; then echo 'Fatal{BarrierViolation, "DrawVbo"}' > "${log}" fi @@ -79,22 +105,30 @@ fi if [ -n "${junit}" ]; then write_junit "${junit}" + if [ "${MOBILEGL_IPC_VERB_BARRIER:-1}" != 0 ] && [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" != 0 ]; then case "${mode}" in red-baseline) echo "1/1 Test #1: ... ***Failed"; exit 8 ;; *) echo "100% tests passed, 0 tests failed out of 1"; exit 0 ;; esac + fi fi # The control's own run. if [ "${MOBILEGL_IPC_VERB_BARRIER:-1}" = 0 ]; then case "${mode}" in - evidence|e3-unrelated) echo 'Fatal{BarrierViolation, "DrawVbo"}' > "${log}" ;; + evidence|e3-unrelated|skipped-selection|notrun-selection|missing-selection|partial-fatal) echo 'Fatal{BarrierViolation, "DrawVbo"}' > "${log}" ;; + wrong-fatal) echo 'Fatal{ReplyMissing, "DrawVbo"}' > "${log}" ;; missing-fatal) echo "library setup only; no fatal" > "${log}" ;; stdout-fatal) echo 'Fatal{BarrierViolation, "DrawVbo"}' ;; esac fi case "${mode}" in - unrelated|missing-fatal|stdout-fatal|stale-fatal|e3-unrelated) + skipped-selection|notrun-selection|missing-selection) + echo '1/1 Test #1: selected entry ... ***Skipped' + echo '100% tests passed, 0 tests failed out of 1' + exit 0 + ;; + unrelated|missing-fatal|stdout-fatal|stale-fatal|e3-unrelated|partial-fatal|wrong-fatal) echo "1/1 Test #1: DirectGLES.Split.ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels ...***Failed" echo "UNRELATED_CONTROL_FAILURE: the harness aborted in setup before the knob was read" if [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" = 0 ]; then