From 579118a1b32656bb8fe4016f8db4f01623ed6e0f Mon Sep 17 00:00:00 2001 From: Swung0x48 Date: Wed, 16 Sep 2026 11:09:21 -0400 Subject: [PATCH] [Test, CI] (scripts/ci, workflows): E2's draw-drop control requires the SSIM red AND the library's dropped-record count, E3(a) requires its private-log line as well as the pixels, and the smoke red-check finally perturbs the split control (ID-65) --- .github/workflows/test.yml | 26 ++++ .../Harness/split_log_paths.py | 11 ++ scripts/ci/control_smoke_test.sh | 30 ++++ scripts/ci/redcheck_control_smoke_test.sh | 100 ++++++++++--- scripts/ci/retrace_drop_draw_control.sh | 131 ++++++++++++++++++ scripts/ci/split_negative_controls.sh | 35 ++++- .../ci/testdata/split_private_log_smoke.sh | 5 +- scripts/ci/testdata/stub_ctest.sh | 73 +++++++++- 8 files changed, 383 insertions(+), 28 deletions(-) mode change 100755 => 100644 scripts/ci/control_smoke_test.sh mode change 100755 => 100644 scripts/ci/redcheck_control_smoke_test.sh create mode 100644 scripts/ci/retrace_drop_draw_control.sh mode change 100755 => 100644 scripts/ci/split_negative_controls.sh mode change 100755 => 100644 scripts/ci/testdata/stub_ctest.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ca022ed7..99c9cc45 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1954,6 +1954,32 @@ jobs: bash "${GITHUB_WORKSPACE}/scripts/ci/retrace_pull_library_control.sh" '${{ matrix.case }}' '${{ matrix.backend }}' + # EXIT GATE E2's OTHER HALF, and the one BRIEF-P5 §7 E2 actually names: "drop an emission + # and the SSIM must fall below the threshold". The control above proves the lane runs a + # SPLIT library; it cannot prove the PICTURE came through the wire, because OpenRA scores + # ssim 1.000000 against a monolith library too and the transport assertion is what reds it. + # + # It drops the DRAWS and not the clears, and that is a measurement rather than a + # preference: with MOBILEGL_IPC_E2_DROP_CLEAR=1 armed, read, and all 29 of OpenRA's Clear + # records dropped, the retrace still scored ssim=1.000000 / mismatchPixels=0 - OpenRA + # overdraws every pixel it clears before the snapshot (joint-v1.md §3 found the green; + # scripts/ci/retrace_drop_draw_control.sh's header has the trace census that explains it). + # With the DrawVbo records dropped instead: 758 dropped, ssim=0.000036, + # mismatchPixels=295296. + # + # THE BODY IS A SCRIPT for the reason the two steps above give: a `run:` block cannot be + # executed off a runner, and scripts/ci/control_smoke_test.sh drives this file against a + # stubbed ctest in seven modes - including the two that matter most, a red whose SSIM never + # fell and a knob that armed and dropped nothing. + - name: Negative control - dropping the draws on the wire must red this split retrace + working-directory: build-retrace/tools/trace_replay + env: + CONTROL_TMPDIR: ${{ runner.temp }} + LIBRARY_LOG: ${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.log + run: >- + bash "${GITHUB_WORKSPACE}/scripts/ci/retrace_drop_draw_control.sh" + '${{ matrix.case }}' '${{ matrix.backend }}' + # The refusal census, recorded rather than gated. run_trace_case.cmake already REDS the case # on any Fatal{, so reaching here means the count is zero - but the number and the distinct # slot names are what MEASUREMENTS wants from every split run, and reading them out of the diff --git a/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py b/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py index 41a82ecc..9bf5508a 100644 --- a/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py +++ b/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py @@ -48,10 +48,21 @@ def main(): for path in selected.values(): Path(path).unlink(missing_ok=True) elif mode == "evidence": + # argv[5], optional: the control's name, for the failure message. Without it the message + # is E1's, word for word - scripts/ci/testdata/split_private_log_smoke.sh greps for that + # sentence, and E1 was the only caller until E3(a) gained a library diagnostic of its own + # (ID-65: "no library diagnostic exists for block size 0 - x2 adds one"). + label = sys.argv[5] if len(sys.argv) > 5 else "" for name, path in selected.items(): if Path(path).is_file() and re.search(sys.argv[4], Path(path).read_text(errors="replace")): print(f"private-log evidence: {name}: {path}") return + if label: + raise ValueError(f"{label} FAILED: no selected private log carries /{sys.argv[4]}/. " + "The library's own line is the only channel for this: ctest's " + "transcript is a FALSE ZERO for library output, because the console " + "sink is compiled out of the configurations these lanes run. " + + ", ".join(f"{n} ({p})" for n, p in selected.items())) raise ValueError("E1 FAILED: selected private logs lack expected Fatal{BarrierViolation, \"\"} line: " + ", ".join(f"{n} ({p})" for n, p in selected.items())) else: diff --git a/scripts/ci/control_smoke_test.sh b/scripts/ci/control_smoke_test.sh old mode 100755 new mode 100644 index d26fc2fc..39c66141 --- a/scripts/ci/control_smoke_test.sh +++ b/scripts/ci/control_smoke_test.sh @@ -62,6 +62,15 @@ run_retrace() { # $1 = STUB_MODE bash "${HERE}/retrace_pull_library_control.sh" OpenRA DirectGLES } +run_drop_draw() { # $1 = STUB_MODE + cd "${WORK}" || return 127 + mkdir -p "${WORK}/OpenRA" + env -i PATH="${STUB_DIR}:/usr/bin:/bin" STUB_MODE="$1" \ + CTEST=ctest CONTROL_TMPDIR="${WORK}/tmp-$1" \ + LIBRARY_LOG="${WORK}/tmp-$1/mobilegl.log" \ + bash "${HERE}/retrace_drop_draw_control.sh" OpenRA DirectGLES +} + echo "=== the split lane's E1 / E3(a) controls (scripts/ci/split_negative_controls.sh)" # THE FINDING, REPRODUCED. Non-empty selection, green baseline, and a red that is not the knob's. expect FAILED "unrelated failure with a non-empty selection" -- run_split unrelated @@ -97,6 +106,27 @@ expect PASSED "run_trace_case.cmake's own sentence, wrapped" -- run_retrace retr # The pull library replaying green is the failure this control exists to catch. expect FAILED "a pull library passed the split retrace" -- run_retrace retrace-green +echo +echo "=== the retrace lane's draw-drop control (scripts/ci/retrace_drop_draw_control.sh)" +# Exit gate E2's picture control. The joint gate's finding is the reason it exists at all: the +# CLEAR-drop knob was armed, was read, dropped all 29 of OpenRA's clears - and the retrace still +# scored ssim 1.000000, because OpenRA overdraws every pixel it clears. So this control's three +# guards are each a different way for "the picture went red" to be someone else's red. +expect FAILED "empty selection (--no-tests=error exit)" -- run_drop_draw dropdraw-noselect +# The failure the gate exists to catch: the golden survives the loss of every draw. +expect FAILED "the retrace passed with every draw dropped" -- run_drop_draw dropdraw-green +# A red with no comparator output at all - a loader failure, a missing fixture, a timeout. +expect FAILED "red with no ssim summary in the output" -- run_drop_draw dropdraw-nossim +# A red whose SSIM is FINE: something else (a Fatal{, a transport assertion) reddened the case. +expect FAILED "red but the ssim is above the threshold" -- run_drop_draw dropdraw-ssimhigh +# A red picture with no evidence the knob was ever read by the process that produced it. +expect FAILED "no 'E2 control armed' line in the library log" -- run_drop_draw dropdraw-nolog +# The R-16 case the dropped-record COUNT exists for: the knob armed and dropped nothing, so the +# wrong picture came from somewhere else. +expect FAILED "the knob armed but dropped zero records" -- run_drop_draw dropdraw-zero +# The real thing: a red picture, a fallen SSIM, and N > 0 records the library says it dropped. +expect PASSED "ssim below threshold and N records dropped" -- run_drop_draw dropdraw-evidence + echo echo "smoke test: ${passes} passed, ${failures} failed" if [ "${failures}" -gt 0 ]; then diff --git a/scripts/ci/redcheck_control_smoke_test.sh b/scripts/ci/redcheck_control_smoke_test.sh old mode 100755 new mode 100644 index 6f540e1c..5b653abd --- a/scripts/ci/redcheck_control_smoke_test.sh +++ b/scripts/ci/redcheck_control_smoke_test.sh @@ -12,10 +12,11 @@ set -u HERE="$(cd "$(dirname "$0")" && pwd)" WORK="$(mktemp -d)" -trap 'cp "${WORK}/split.orig" "${HERE}/split_negative_controls.sh"; cp "${WORK}/retrace.orig" "${HERE}/retrace_pull_library_control.sh"; rm -rf "${WORK}"' EXIT +trap 'cp "${WORK}/split.orig" "${HERE}/split_negative_controls.sh"; cp "${WORK}/retrace.orig" "${HERE}/retrace_pull_library_control.sh"; cp "${WORK}/dropdraw.orig" "${HERE}/retrace_drop_draw_control.sh"; rm -rf "${WORK}"' EXIT cp "${HERE}/split_negative_controls.sh" "${WORK}/split.orig" cp "${HERE}/retrace_pull_library_control.sh" "${WORK}/retrace.orig" +cp "${HERE}/retrace_drop_draw_control.sh" "${WORK}/dropdraw.orig" echo "=== baseline: the smoke test must be GREEN before anything is perturbed" if ! bash "${HERE}/control_smoke_test.sh" > "${WORK}/before.log" 2>&1; then @@ -27,24 +28,77 @@ tail -1 "${WORK}/before.log" echo echo "=== perturbation: remove the evidence check from both controls" -python3 - "${HERE}/split_negative_controls.sh" "${HERE}/retrace_pull_library_control.sh" <<'PY' +python3 - "${HERE}/split_negative_controls.sh" "${HERE}/retrace_pull_library_control.sh" \ + "${HERE}/retrace_drop_draw_control.sh" <<'PY' import sys -split, retrace = sys.argv[1], sys.argv[2] -for path, needle in ((split, 'grep -qE "${evidence}"'), (retrace, 'grep -qF "${EVIDENCE}"')): +split, retrace, dropdraw = sys.argv[1], sys.argv[2], sys.argv[3] +# (file, [(needle, the prefix the line must start with)]). The draw-drop control has THREE +# evidence checks rather than one, because "the picture went red" has three different ways of +# being somebody else's red: the SSIM never fell, the library never said the knob armed, and the +# knob armed and dropped nothing. All three are reverted together, and the smoke cases that pin +# each of them must go red below. +# +# THE PREFIXES ARE A TUPLE, AND THAT IS A FIX, not a generalisation for its own sake. The first +# version matched only lines starting with `if ! `, and the split control's evidence check has +# been `elif ! tr -s ... | grep -qE "${evidence}"` since it was written - so this red-check found +# ZERO checks in it, the SystemExit below was not checked by the caller, and the run went on to +# smoke-test the UNPERTURBED controls and report "RED-CHECK FAILED: the controls accept an +# unrelated failure again and the smoke test still passed". It failed safe rather than green, so +# nothing was ever silently proved - but the one control it was most about was never perturbed. +# Reproduced on p5/joint@e61d0012 before this line changed. +# +# AND THE SPLIT CONTROL HAS THREE EVIDENCE CHECKS, NOT ONE, so all three are reverted. E1's is a +# read of the entry's own private library log, E3(a)'s ctest-output regex is the `elif` above, +# and E3(a)'s private-log half is ID-65's addition. Perturbing only one of the three leaves the +# other two catching the smoke test's "unrelated failure" case, and the case never flips - which +# is what this script measured the first time the perturbation actually applied. +rules = [ + (split, [('LINE', 'grep -qE "${evidence}"', ('if ! ', 'elif ! ')), + ('SUBST', '"[A-Za-z_][A-Za-z_0-9]*"\\}\' || exit 1', '"[A-Za-z_][A-Za-z_0-9]*"\\}\' || true'), + ('SUBST', '"${private_evidence}" "${name}" || exit 1', + '"${private_evidence}" "${name}" || true')]), + (retrace, [('LINE', 'grep -qF "${EVIDENCE}"', ('if ! ',))]), + (dropdraw, [('LINE', 'awk -v a="${ssim}"', ('if ! ',)), + ('LINE', '[ -z "${armed_line}" ]', ('if ',)), + ('LINE', '[ "${dropped}" -lt 1 ]', ('if ',))]), +] +for path, checks in rules: text = open(path).read() - out, hit = [], 0 - for line in text.splitlines(keepends=True): - if needle in line and line.lstrip().startswith('if ! '): - indent = line[:len(line) - len(line.lstrip())] - out.append(f"{indent}if false; then\n") - hit += 1 - else: - out.append(line) - if hit != 1: - raise SystemExit(f"expected exactly one evidence check in {path}, found {hit}") - open(path, 'w').write(''.join(out)) -print("both evidence checks reverted to 'any non-zero ctest exit is accepted'") + for rule in checks: + if rule[0] == 'SUBST': + _, old, new = rule + if text.count(old) != 1: + raise SystemExit(f"expected exactly one {old!r} in {path}, found {text.count(old)}") + text = text.replace(old, new) + continue + _, needle, prefixes = rule + out, hit = [], 0 + for line in text.splitlines(keepends=True): + if needle in line and line.lstrip().startswith(prefixes): + indent = line[:len(line) - len(line.lstrip())] + # KEEP THE KEYWORD. Turning an `elif` into an `if` splits the chain in two and + # leaves the second half's `fi` dangling - a syntax error, which the smoke test + # then reports as rc=2 on every case and which looks nothing like "the control + # accepted an unrelated failure". + keyword = 'elif' if line.lstrip().startswith('elif') else 'if' + out.append(f"{indent}{keyword} false; then\n") + hit += 1 + else: + out.append(line) + if hit != 1: + raise SystemExit(f"expected exactly one {needle!r} check in {path}, found {hit}") + text = ''.join(out) + open(path, 'w').write(text) +print("all seven evidence checks reverted to 'any non-zero ctest exit is accepted'") PY +# AND THE PERTURBATION'S OWN FAILURE IS FATAL. Without this, a SystemExit above left the controls +# UNTOUCHED and the run continued to smoke-test them, printing "RED-CHECK FAILED: the controls +# accept an unrelated failure again and the smoke test still passed" - a true statement about a +# perturbation that never happened, and the wrong diagnosis to hand whoever reads it. +if [ $? -ne 0 ]; then + echo "RED-CHECK ABORTED: the perturbation did not apply, so nothing below would be measuring it." + exit 1 +fi echo echo "=== the smoke test on the reverted controls (it MUST be red, on those two cases)" @@ -59,13 +113,19 @@ if [ "${rc}" -eq 0 ]; then fi missed=0 -grep -q "NOT OK unrelated failure with a non-empty selection" "${WORK}/after.log" || missed=1 -grep -q "NOT OK red without the transport-resolution message" "${WORK}/after.log" || missed=1 +for case in \ + "unrelated failure with a non-empty selection" \ + "red without the transport-resolution message" \ + "red but the ssim is above the threshold" \ + "no 'E2 control armed' line in the library log" \ + "the knob armed but dropped zero records"; do + grep -q "NOT OK ${case}" "${WORK}/after.log" || { echo "still green: ${case}"; missed=1; } +done if [ "${missed}" -ne 0 ]; then echo - echo "RED-CHECK FAILED: the smoke test went red, but not on the two cases the evidence check exists for." + echo "RED-CHECK FAILED: the smoke test went red, but not on the five cases the evidence checks exist for." exit 1 fi echo -echo "P5_T1_CONTROL_SMOKE_REDCHECK_OK - removing the evidence check reds exactly the two cases that pin it" +echo "P5_T1_CONTROL_SMOKE_REDCHECK_OK - removing the evidence checks reds exactly the five cases that pin them" diff --git a/scripts/ci/retrace_drop_draw_control.sh b/scripts/ci/retrace_drop_draw_control.sh new file mode 100644 index 00000000..c7597c0c --- /dev/null +++ b/scripts/ci/retrace_drop_draw_control.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# EXIT GATE E2's PICTURE CONTROL: dropping the draws on the wire must redden this retrace, and +# it must be THIS control's red. +# +# WHY THERE IS A SECOND CONTROL ON THIS LANE, and why it is the draws and not the clears. +# +# The retrace-split job already carries scripts/ci/retrace_pull_library_control.sh, which swaps a +# PULL library in and requires run_trace_case.cmake's transport-resolution assertion to catch it. +# That control proves the lane is running a SPLIT library. It does not prove that the PICTURE came +# through the wire, and it cannot: OpenRA scores ssim 1.000000 against a monolith library too, so +# the transport assertion is what reds it and the comparator is never consulted. BRIEF-P5 7 E2 +# names a second, sharper statement - "patch an emitter to drop a record and the SSIM must fall +# below the threshold" - and until now nothing executed it. +# +# c1 shipped MOBILEGL_IPC_E2_DROP_CLEAR for exactly that, and the joint gate ran it and it DID NOT +# WORK: with the knob armed and its WARN in the library's own log, the OpenRA retrace under inproc +# still scored ssim=1.000000 with mismatchPixels=0 (joint-v1.md 3, "E2 clear-drop control - NOT +# VERIFIED"). That is not a broken knob. `apitrace dump` over the 31249 replayed calls counts 30 +# glClear, 30 glXSwapBuffers and 788 glDrawArrays, and the final frame issues its clear at call +# 30197 and then covers the surface four times over with a terrain layer before the snapshot at +# 31249. A frame that overdraws every pixel it clears has a picture that does not depend on the +# clear. Measured here with the dropped-record count published: 29 Clear records dropped, ssim +# still 1.000000. The knob worked; the observable was invisible. +# +# So this control drops every DrawVbo record instead (MOBILEGL_IPC_E2_DROP_DRAW=1): the surface +# can then only carry the clear colour, and the golden is made of the geometry. Measured on the +# same head: 758 DrawVbo records dropped, ssim=0.000036, mismatchPixels=295296. +# +# WHAT THE CONTROL ASSERTS, and none of the three is the process exit code alone (R-16): +# 1. a NON-EMPTY selection, counted before the run - `--no-tests=error` turns an empty selection +# into a non-zero exit, which is how ID-46 finding 8(b) got a control to congratulate itself; +# 2. the SSIM ACTUALLY FELL: the numbers are parsed out of the retrace's own summary and +# compared, rather than "ctest was non-zero". A loader failure, a missing fixture, a timeout +# and a Fatal{ all exit non-zero and none of them is this control's red; +# 3. the LIBRARY SAID IT DROPPED SOMETHING: its own "E2 control armed ... N records dropped on +# the wire" line, with N > 0, from the log file the replay wrote. Without this, a knob that +# was never read - a stale library, a variable that did not reach the process, an emit table +# that fell through to the driver - would redden the picture for some other reason and pass. +# +# Usage: retrace_drop_draw_control.sh +# CTEST ctest binary (default: ctest) +# CONTROL_TMPDIR scratch dir (default: ${RUNNER_TEMP:-/tmp}) +# LIBRARY_LOG the replay's library log (default: //output/mobilegl.log) +set -u + +CASE="${1:?usage: retrace_drop_draw_control.sh }" +BACKEND="${2:?usage: retrace_drop_draw_control.sh }" + +CTEST="${CTEST:-ctest}" +CONTROL_TMPDIR="${CONTROL_TMPDIR:-${RUNNER_TEMP:-/tmp}}" +LIBRARY_LOG="${LIBRARY_LOG:-${CASE}/${BACKEND}/output/mobilegl.log}" +mkdir -p "${CONTROL_TMPDIR}" + +selector="^MobileGLTraceReplay\.${CASE}\.${BACKEND}$" + +# The rerun replays into the same case directory, so the good run's images are put aside and +# restored whichever way the control goes; "Upload actual image" runs `if: always()` and would +# otherwise ship the deliberately-wrong run's output under the good run's name. +GOOD_OUTPUT="${CONTROL_TMPDIR}/drop-draw-verified-output" +rm -rf "${GOOD_OUTPUT}" +if [ -d "${CASE}" ]; then cp -a "${CASE}" "${GOOD_OUTPUT}"; fi + +restore_good_output() { + if [ -d "${GOOD_OUTPUT}" ]; then + rm -rf "${CASE}"; mv "${GOOD_OUTPUT}" "${CASE}" + echo "restored the verified run's output over the control's" + fi +} + +matched=$("${CTEST}" -N -R "${selector}" | grep -cE '^ *Test *#[0-9]+:') +if [ "${matched}" -lt 1 ]; then + restore_good_output + echo "::error::the control selected ${matched} tests with -R '${selector}', so there is nothing for the dropped draws to redden. --no-tests=error would have exited non-zero on the empty selection and a control without this guard reads that as success (ID-46 finding 8b)." + exit 1 +fi + +# A PREVIOUS RUN'S LINE MUST NEVER ARM THIS ONE. The library opens its log fopen(path, "w"), so +# the replay truncates it - but only if the replay gets that far, and a run that died in the +# loader would leave the baseline's log in place with a perfectly good "control armed" line in +# it. Removing it first is the same rule split_negative_controls.sh's `reset` step follows. +rm -f "${LIBRARY_LOG}" + +out="${CONTROL_TMPDIR}/retrace-drop-draw-output.txt" +export MOBILEGL_TRANSPORT=inproc +export MOBILEGL_IPC_E2_DROP_DRAW=1 +"${CTEST}" -V --no-tests=error --timeout 10800 -R "${selector}" > "${out}" 2>&1 +control_rc=$? +unset MOBILEGL_IPC_E2_DROP_DRAW +cat "${out}" + +# Read the library's evidence BEFORE the good output is restored over it. +armed_line="" +dropped=0 +if [ -f "${LIBRARY_LOG}" ]; then + armed_line=$(grep 'MGPipe: E2 control armed' "${LIBRARY_LOG}" | tail -1) + dropped=$(printf '%s' "${armed_line}" | sed -n 's/.*armed[^,]*, \([0-9][0-9]*\) records dropped.*/\1/p') + dropped="${dropped:-0}" +fi +cp -f "${LIBRARY_LOG}" "${CONTROL_TMPDIR}/drop-draw-library.log" 2>/dev/null + +restore_good_output + +if [ "${control_rc}" -eq 0 ]; then + echo "::error::the split retrace PASSED with every DrawVbo record dropped on the wire. The golden is made of that geometry, so a green here means the picture did not come from the wire: the emit table fell through to the driver, the library under test is not the one the lane thinks it is, or MOBILEGL_IPC_E2_DROP_DRAW never reached the process. Exit gate E2 is exactly this statement and nothing weaker - MOBILEGL_IPC_E2_DROP_CLEAR is NOT a substitute (measured: 29 clears dropped, ssim still 1.000000, because OpenRA overdraws every pixel it clears)." + exit 1 +fi + +# 2. THE SSIM ACTUALLY FELL. Parsed, not inferred from the exit code. +ssim_line=$(grep -o 'ssim=[0-9.]*, ssimThreshold=[0-9.]*' "${out}" | tail -1) +if [ -z "${ssim_line}" ]; then + echo "::error::the split retrace went red (ctest exit ${control_rc}) with the draws dropped, but its output carries no 'ssim=..., ssimThreshold=...' summary at all, so the comparator never ran. A loader failure, a missing fixture, a timeout or a Fatal{ all land here and none of them is this control's red." + exit 1 +fi +ssim=${ssim_line#ssim=}; ssim=${ssim%%,*} +threshold=${ssim_line##*ssimThreshold=} +if ! awk -v a="${ssim}" -v b="${threshold}" 'BEGIN { exit !(a + 0 < b + 0) }'; then + echo "::error::the split retrace went red (ctest exit ${control_rc}) but its ${ssim_line} is NOT below the threshold, so the picture is not what reddened it. This control's whole claim is that the golden is made of the dropped geometry." + exit 1 +fi + +# 3. THE LIBRARY SAID IT DROPPED SOMETHING. +if [ -z "${armed_line}" ]; then + echo "::error::the split retrace went red with ssim ${ssim} < ${threshold}, but ${LIBRARY_LOG} carries no 'MGPipe: E2 control armed' line, so there is no evidence the knob was ever read by the process that produced the picture. A library that is not the one under test, an emit table that fell through to the driver, or an environment that did not reach the replay all produce a wrong picture for a reason that has nothing to do with this control." + exit 1 +fi +if [ "${dropped}" -lt 1 ]; then + echo "::error::the split retrace went red with ssim ${ssim} < ${threshold} and the knob announced itself - '${armed_line}' - but it reports ZERO records dropped. The emitter was never reached, so whatever changed the picture was not this control. This is the R-16 case the dropped-record COUNT exists for: the arming message alone proves only that the knob was read." + exit 1 +fi + +echo "the dropped draws turned the split retrace red for their own reason (ctest exit ${control_rc}): ${matched} selected case(s), ssim ${ssim} < ${threshold}, and the library dropped ${dropped} record(s) on the wire" diff --git a/scripts/ci/split_negative_controls.sh b/scripts/ci/split_negative_controls.sh old mode 100755 new mode 100644 index 6baf5d5e..e85a0653 --- a/scripts/ci/split_negative_controls.sh +++ b/scripts/ci/split_negative_controls.sh @@ -99,9 +99,19 @@ fi # ---- the controls --------------------------------------------------------------------------- # -# run_control ... +# run_control ... +# +# THE FOURTH ARGUMENT IS NEW AND E3(a) IS WHY. The header above argues that the knob's own +# ConfigLoader line is not evidence, and that is still true: it is written at config load, by +# every process in the run, whatever happens next. What IS evidence is a line the knob's +# BEHAVIOUR emits at the site that changed - and until now E3(a) had none, because +# PersistentMapTracker::PushBlocksFor simply `return`ed at blockBytes == 0. The joint gate +# recorded exactly that ("There is no Fatal for block size zero ... No nonexistent private-file +# Fatal is quoted", joint-v1.md 3) and ID-65 assigned the missing line here. With it, E3(a) no +# longer rests on a pixel assertion alone: the red must carry the scenario's own diagnostic AND +# the library's own statement that the push was disabled, from the entry's private file. run_control() { - name="$1"; filter="$2"; evidence="$3"; shift 3 + name="$1"; filter="$2"; evidence="$3"; private_evidence="$4"; shift 4 matched=$("${CTEST}" -N -L integration-split -R "${filter}" | grep -cE '^ *Test *#[0-9]+:') if [ "${matched}" -lt 1 ]; then @@ -133,6 +143,14 @@ run_control() { exit 1 fi + # ... and, where the knob has one, the LIBRARY's own line as well, out of the entry's private + # file. Both halves are required: the scenario assertion says the pixels were wrong, and this + # says the code path the knob turns off is the one that stopped running. A red that has only + # the first half is consistent with any other defect in the same scenario. + if [ -n "${private_evidence}" ]; then + python3 "${log_helper}" evidence "${manifest}" "${filter}" "${private_evidence}" "${name}" || exit 1 + fi + echo "${name} turned ${matched} selected entries red, and the red carries the scenario's own diagnostic, as it must" } @@ -141,12 +159,19 @@ run_control() { run_control "negative control E1 (MOBILEGL_IPC_VERB_BARRIER=0)" \ 'DirectGLES\.Split\.(SmallRing\.)?(Triangle|ClearThenReadPixels)' \ 'private-barrier-fatal' \ + '' \ MOBILEGL_IPC_VERB_BARRIER=0 -# E3(a): PersistentMapTracker.cpp returns at blockBytes == 0 (no Fatal). -# PersistentCoherentMapScenario.cpp:414-417 / 442-443 name the missing second write. -# Do not accept a generic source-line Failure: an unrelated assertion is not this red. +# E3(a): PersistentMapTracker::PushBlocksFor stops at blockBytes == 0 - deliberately, because 0 +# is the negative control and not "unlimited". Two independent halves are now required: +# * the SCENARIO's own assertion in ctest's output. PersistentCoherentMapScenario.cpp:414-417 / +# 442-443 name the missing second write; a generic source-line Failure is not accepted, +# because an unrelated assertion in the same case is not this red; +# * the LIBRARY's own line in the entry's private file, saying the push was disabled by this +# knob. It did not exist until ID-65 assigned it (joint-v1.md 3), which is why this control +# used to rest on the pixels alone. run_control "negative control E3(a) (MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0)" \ 'DirectGLES\.Split\.(SmallRing\.)?PersistentCoherentMapScenario' \ "the SECOND write through the same mapping, announced by nothing|frame 1's write through the SAME mapping, after a Present" \ + 'MGPipe: persistent-map push disabled - MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0' \ MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0 diff --git a/scripts/ci/testdata/split_private_log_smoke.sh b/scripts/ci/testdata/split_private_log_smoke.sh index cf226eea..37831b92 100644 --- a/scripts/ci/testdata/split_private_log_smoke.sh +++ b/scripts/ci/testdata/split_private_log_smoke.sh @@ -7,7 +7,7 @@ trap 'rm -rf "${WORK}"' EXIT cp "${HERE}/testdata/stub_ctest.sh" "${WORK}/ctest" chmod +x "${WORK}/ctest" passes=0 -for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated; do +for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated e3-no-private; do mkdir -p "${WORK}/${mode}" rc=0 STUB_MODE="${mode}" CTEST="${WORK}/ctest" CONTROL_TMPDIR="${WORK}/${mode}" \ @@ -19,6 +19,9 @@ for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated; do else message='E1 FAILED: selected private logs lack expected Fatal' [ "${mode}" != e3-unrelated ] || message='FAILED: red lacks its persistent-map push diagnostic' + # ID-65's half: the pixel assertion arrived, the library said nothing, and the control must + # refuse the red by naming the line it wanted rather than accepting the pixels alone. + [ "${mode}" != e3-no-private ] || message='no selected private log carries /MGPipe: persistent-map push disabled' [ "${rc}" != 0 ] && grep -q "${message}" "${WORK}/${mode}.out" || { cat "${WORK}/${mode}.out"; echo "NOT OK ${mode}: control must report FAILED for its own reason"; exit 1; } diff --git a/scripts/ci/testdata/stub_ctest.sh b/scripts/ci/testdata/stub_ctest.sh old mode 100755 new mode 100644 index de9dba3e..09a686c6 --- a/scripts/ci/testdata/stub_ctest.sh +++ b/scripts/ci/testdata/stub_ctest.sh @@ -15,6 +15,8 @@ # stdout-fatal Fatal exists only on stdout, never in the private file # stale-fatal Fatal exists before reset, never from this control run # e3-unrelated E1 has its private Fatal; E3(a) fails for an unrelated reason +# e3-no-private E1 has its private Fatal; E3(a) has its PIXEL assertion but the library +# never says the push was disabled (the half ID-65 added) # green baseline green; the control's own run PASSES (the knob is not load-bearing) # red-baseline the baseline itself has a failed entry # all-skipped the baseline is entirely skipped (the disarmed lane, a legitimate exit 0) @@ -22,6 +24,18 @@ # retrace-unrelated one match; the run fails without naming the transport # retrace-evidence one match; the run fails with run_trace_case.cmake's own sentence # retrace-green one match; the run PASSES +# +# EXIT GATE E2's DRAW-DROP CONTROL (scripts/ci/retrace_drop_draw_control.sh). The library's own +# log is modelled as a separate sink from ctest stdout, exactly as it is for the split lane: the +# console sink is compiled out of the configurations these lanes run, so the dropped-record count +# can only ever arrive through ${LIBRARY_LOG}. +# dropdraw-noselect `ctest -N` matches nothing +# dropdraw-green the retrace PASSES with the draws dropped +# dropdraw-nossim red, but no ssim summary at all (loader failure / timeout shape) +# dropdraw-ssimhigh red, and the ssim is ABOVE the threshold: something else reddened it +# dropdraw-nolog red with a low ssim, but the library never said the knob armed +# dropdraw-zero red with a low ssim, the knob armed, and it dropped NOTHING +# dropdraw-evidence red with a low ssim and N > 0 records dropped: the real thing set -u mode="${STUB_MODE:?STUB_MODE must be set}" @@ -39,7 +53,7 @@ listing_requested="${listing_requested:-0}" emit_listing() { echo "Test project /stub" - if [ "${mode}" = "retrace-noselect" ]; then + if [ "${mode}" = "retrace-noselect" ] || [ "${mode}" = "dropdraw-noselect" ]; then echo "Total Tests: 0" return fi @@ -88,12 +102,29 @@ fi # The control's own run. if [ "${MOBILEGL_IPC_VERB_BARRIER:-1}" = 0 ]; then case "${mode}" in - evidence|e3-unrelated) echo 'Fatal{BarrierViolation, "DrawVbo"}' > "${log}" ;; + evidence|e3-unrelated|e3-no-private) echo 'Fatal{BarrierViolation, "DrawVbo"}' > "${log}" ;; missing-fatal) echo "library setup only; no fatal" > "${log}" ;; stdout-fatal) echo 'Fatal{BarrierViolation, "DrawVbo"}' ;; esac fi +# E3(a)'s library line, in the PERSISTENT-MAP entry's own private file - a different file from +# E1's, exactly as the manifest above declares. `e3-no-private` is the mode that leaves it out: +# the pixel assertion arrives, the library says nothing, and the control must refuse the red. +if [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" = 0 ] && [ "${mode}" = evidence ]; then + echo 'MGPipe: persistent-map push disabled - MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0 is exit gate E3(a)'"'"'s NEGATIVE CONTROL' \ + > "${CONTROL_TMPDIR}/pmap.log" +fi case "${mode}" in + e3-no-private) + # E1's half passes (its private Fatal is written above); E3(a)'s pixel assertion arrives on + # stdout and its private line does not, so the control must stop at the second half. + echo "1/1 Test #1: DirectGLES.Split.ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels ...***Failed" + if [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" = 0 ]; then + echo "../MobileGL/MG_IntegrationTest/Scenarios/PersistentCoherentMapScenario.cpp:414: Failure" + echo "the SECOND write through the same mapping, announced by nothing: this is exit gate E3(b)" + fi + exit 8 + ;; unrelated|missing-fatal|stdout-fatal|stale-fatal|e3-unrelated) echo "1/1 Test #1: DirectGLES.Split.ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels ...***Failed" echo "UNRELATED_CONTROL_FAILURE: the harness aborted in setup before the knob was read" @@ -138,6 +169,44 @@ case "${mode}" in echo "100% tests passed, 0 tests failed out of 1" exit 0 ;; + dropdraw-*) + # The library's own log, written by the replay the way the real one is. The control removes + # it before the run, so anything here is this run's. + armed="" + case "${mode}" in + dropdraw-zero) + armed='MGPipe: E2 control armed - drop-draw=1 drop-clear=0, 0 records dropped on the wire (draw=0 clear=0), frame 29' ;; + dropdraw-nolog) armed="" ;; + *) + armed='MGPipe: E2 control armed - drop-draw=1 drop-clear=0, 758 records dropped on the wire (draw=758 clear=0), frame 29' ;; + esac + if [ -n "${LIBRARY_LOG:-}" ] && [ -n "${armed}" ]; then + mkdir -p "$(dirname "${LIBRARY_LOG}")" + printf '%s\n' "[10:38:30] [Linux mobilegl_trace_/WARN]: ${armed}" > "${LIBRARY_LOG}" + fi + case "${mode}" in + dropdraw-green) + echo "100% tests passed, 0 tests failed out of 1" + exit 0 + ;; + dropdraw-nossim) + echo "1/1 Test #1: MobileGLTraceReplay.OpenRA.DirectGLES ...***Failed" + echo "CMake Error: the replay could not load the library" + exit 8 + ;; + dropdraw-ssimhigh) + echo "1/1 Test #1: MobileGLTraceReplay.OpenRA.DirectGLES ...***Failed" + echo "-- retrace completed; ssim=1.000000, ssimThreshold=0.990000, mismatchPixels=0" + echo "CMake Error at run_trace_case.cmake:301 (message): 3 MGPipe Fatal(s)" + exit 8 + ;; + *) + echo "1/1 Test #1: MobileGLTraceReplay.OpenRA.DirectGLES ...***Failed" + echo "-- retrace completed; ssim=0.000036, ssimThreshold=0.990000, mismatchPixels=295296" + exit 8 + ;; + esac + ;; *) echo "stub_ctest: unknown STUB_MODE '${mode}'" >&2 exit 127