diff --git a/docs/Disaggregated/notes/p7/device-window-2/tools/30-gate3-selftest.sh b/docs/Disaggregated/notes/p7/device-window-2/tools/30-gate3-selftest.sh index da20bcc6..7b80e7ac 100755 --- a/docs/Disaggregated/notes/p7/device-window-2/tools/30-gate3-selftest.sh +++ b/docs/Disaggregated/notes/p7/device-window-2/tools/30-gate3-selftest.sh @@ -2,14 +2,19 @@ # 30-gate3-selftest.sh [--script PATH] (host only: no phone, no adb server, no device lock) # # Runs 30-gate3.sh (or PATH: red-once of an older driver) against a stub adb and a stub runner in a -# temp dir and checks its repeat bookkeeping (ID-P7-62: the monolith arm runs x --repeat, recorded -# as arms.txt monolith_repeat=) and its resume semantics: -# fresh --cases a,b: arms.txt says repeat=3 monolith_repeat=3; the runner is asked for the -# monolith x3; every monolith pair archives 3 repeats and writes its .done; exit 0 -# resume one monolith .done deleted: only that pair re-runs, again x3 -# incomplete a monolith repeat without its PNG: no .done, progress.tsv INCOMPLETE, exit 3; the -# same command again completes it -# frozen an arms.txt with monolith_repeat=3, resumed with --repeat 1: every arm keeps 3 +# temp dir and checks its repeat bookkeeping (ID-P7-62 final ruling: the monolith arm runs x5 on +# every case whatever --repeat, the split arms x --repeat; recorded as arms.txt monolith_repeat=) +# and its resume semantics: +# fresh --cases a,b: arms.txt says repeat=3 monolith_repeat=5; the runner is asked for the +# monolith x5, inproc / spawn x3, ra0 x1; every monolith pair archives 5 repeats and +# writes its .done; exit 0 +# decoupled a fresh --repeat 1 run: the split arms x1, the monolith still x5 (monolith_repeat=5) +# resume one monolith .done deleted: only that pair re-runs, again x5 +# incomplete a monolith repeat-05 (beyond the split arms' 3) without its PNG: no .done, +# progress.tsv INCOMPLETE, exit 3; the same command again completes it +# frozen an arms.txt with monolith_repeat=5, resumed with --repeat 1: monolith x5, split x3 +# interim an arms.txt with monolith_repeat=3 (a run the interim x3 script started): a resume +# keeps monolith x3, and the log says why # old-format an arms.txt without monolith_repeat= (a run the pre-ID-P7-62 script started): a # resume keeps that run's monolith x1 # Exit 0 when every check holds. @@ -83,51 +88,77 @@ reps() { find "$W2_LOGROOT/$1/gate3/archive/$2/$3-DirectVulkan" -maxdepth 1 -nam RUNS=0 echo "30-gate3 self-test: driver $SCRIPT, stubs in $TMP" -echo "[fresh] a new run records monolith_repeat and runs the monolith x --repeat" +echo "[fresh] a new run records monolith_repeat=5 and runs the monolith x5, the split arms x3" gate fresh --cases case-a,case-b; rc=$? G="$W2_LOGROOT/fresh/gate3" expect "exit 0" $rc "rc=$rc; $(tail -3 "$TMP/fresh.$RUNS.log")" -grep -qx 'monolith_repeat=3' "$G/arms.txt" && grep -qx 'repeat=3' "$G/arms.txt" -expect "arms.txt: repeat=3 and monolith_repeat=3" $? "$(cat "$G/arms.txt")" -[ "$(calls 'transport=monolith repeat=3 ')" = 2 ] && [ "$(calls 'transport=monolith repeat=1 ')" = 0 ] -expect "the runner is asked for the monolith x3 on both cases" $? "$(grep monolith "$FAKE_RUNLOG")" -[ "$(reps fresh monolith case-a)" = 3 ] && [ "$(reps fresh monolith case-b)" = 3 ] -expect "each monolith pair archives 3 repeats" $? "$(reps fresh monolith case-a)/$(reps fresh monolith case-b)" +grep -qx 'monolith_repeat=5' "$G/arms.txt" && grep -qx 'repeat=3' "$G/arms.txt" +expect "arms.txt: repeat=3 and monolith_repeat=5" $? "$(cat "$G/arms.txt")" +[ "$(calls 'transport=monolith repeat=5 ')" = 2 ] && [ "$(calls 'transport=monolith ')" = 2 ] +expect "the runner is asked for the monolith x5 on both cases (and for nothing else on the monolith)" $? \ + "$(grep monolith "$FAKE_RUNLOG")" +[ "$(reps fresh monolith case-a)" = 5 ] && [ "$(reps fresh monolith case-b)" = 5 ] +expect "each monolith pair archives 5 repeats" $? "$(reps fresh monolith case-a)/$(reps fresh monolith case-b)" [ "$(ls "$G"/state/*/*.done | wc -l)" = 8 ] && [ -f "$G/state/monolith/case-a.done" ] expect "every (arm, case) pair has its .done (8)" $? "$(ls "$G"/state/*/)" [ "$(calls 'transport=inproc repeat=3 ')" = 2 ] && [ "$(calls 'transport=spawn repeat=3 ')" = 2 ] \ - && [ "$(calls 'transport=inproc repeat=1 case=case-.* env=MOBILEGL_IPC_RUN_AHEAD=0')" = 2 ] + && [ "$(calls 'transport=inproc repeat=1 case=case-.* env=MOBILEGL_IPC_RUN_AHEAD=0')" = 2 ] \ + && [ "$(reps fresh inproc case-a)" = 3 ] && [ "$(reps fresh spawn case-b)" = 3 ] expect "the split arms keep x3 and ra0 x1" $? "$(cat "$FAKE_RUNLOG")" -echo "[resume] a deleted monolith .done re-runs that one pair, x3" +echo "[decoupled] the monolith count does not follow --repeat" +: > "$FAKE_RUNLOG" +gate decoupled --cases case-a --repeat 1; rc=$? +G="$W2_LOGROOT/decoupled/gate3" +[ $rc -eq 0 ] && grep -qx 'repeat=1' "$G/arms.txt" && grep -qx 'monolith_repeat=5' "$G/arms.txt" \ + && [ "$(calls 'transport=monolith repeat=5 ')" = 1 ] && [ "$(calls 'transport=inproc repeat=1 case=case-a env=$')" = 1 ] \ + && [ "$(calls 'transport=spawn repeat=1 ')" = 1 ] && [ "$(reps decoupled monolith case-a)" = 5 ] +expect "a fresh --repeat 1 run: inproc / spawn x1, the monolith still x5 (arms.txt monolith_repeat=5)" $? \ + "rc=$rc; $(cat "$G/arms.txt" 2>/dev/null); $(cat "$FAKE_RUNLOG")" + +echo "[resume] a deleted monolith .done re-runs that one pair, x5" +G="$W2_LOGROOT/fresh/gate3" : > "$FAKE_RUNLOG"; rm "$G/state/monolith/case-a.done" gate fresh; rc=$? -[ $rc -eq 0 ] && [ "$(wc -l < "$FAKE_RUNLOG")" = 1 ] && [ "$(calls 'transport=monolith repeat=3 case=case-a ')" = 1 ] \ - && [ -f "$G/state/monolith/case-a.done" ] && [ "$(reps fresh monolith case-a)" = 3 ] -expect "only monolith case-a re-ran, x3, and wrote its .done" $? "rc=$rc; $(cat "$FAKE_RUNLOG")" +[ $rc -eq 0 ] && [ "$(wc -l < "$FAKE_RUNLOG")" = 1 ] && [ "$(calls 'transport=monolith repeat=5 case=case-a ')" = 1 ] \ + && [ -f "$G/state/monolith/case-a.done" ] && [ "$(reps fresh monolith case-a)" = 5 ] +expect "only monolith case-a re-ran, x5, and wrote its .done" $? "rc=$rc; $(cat "$FAKE_RUNLOG")" -echo "[incomplete] a monolith repeat without its PNG leaves the pair open" +echo "[incomplete] a monolith repeat without its PNG leaves the pair open - also beyond the split arms' 3" : > "$FAKE_RUNLOG" -FAKE_DROP=monolith:case-b:2 gate incomplete --cases case-a,case-b; rc=$? +FAKE_DROP=monolith:case-b:5 gate incomplete --cases case-a,case-b; rc=$? G="$W2_LOGROOT/incomplete/gate3" [ $rc -eq 3 ] && [ ! -f "$G/state/monolith/case-b.done" ] && [ -f "$G/state/monolith/case-a.done" ] \ - && grep -q "^monolith case-b .* INCOMPLETE$" "$G/progress.tsv" -expect "monolith case-b repeat-02 without a PNG -> no .done, progress INCOMPLETE, exit 3" $? "rc=$rc; $(cat "$G/progress.tsv" 2>/dev/null)" + && grep -q "^monolith case-b .* INCOMPLETE$" "$G/progress.tsv" && grep -q 'monolith case-b .*repeat-05 lack' "$TMP/incomplete.$RUNS.log" +expect "monolith case-b repeat-05 without a PNG -> no .done, progress INCOMPLETE naming repeat-05, exit 3" $? \ + "rc=$rc; $(cat "$G/progress.tsv" 2>/dev/null)" : > "$FAKE_RUNLOG" gate incomplete; rc=$? [ $rc -eq 0 ] && [ -f "$G/state/monolith/case-b.done" ] && [ "$(wc -l < "$FAKE_RUNLOG")" = 1 ] \ - && [ "$(calls 'transport=monolith repeat=3 case=case-b ')" = 1 ] -expect "...the same command again re-runs that pair only (x3) and completes, exit 0" $? "rc=$rc; $(cat "$FAKE_RUNLOG")" + && [ "$(calls 'transport=monolith repeat=5 case=case-b ')" = 1 ] && [ "$(reps incomplete monolith case-b)" = 5 ] +expect "...the same command again re-runs that pair only (x5) and completes, exit 0" $? "rc=$rc; $(cat "$FAKE_RUNLOG")" echo "[frozen] a resume keeps the first run's counts, whatever --repeat it is given" G="$W2_LOGROOT/frozen/gate3"; mkdir -p "$G" printf 'case-a\n' > "$G/cases.txt" -printf 'arms=monolith inproc spawn inproc-ra0\nrepeat=3\nmonolith_repeat=3\n' > "$G/arms.txt" +printf 'arms=monolith inproc spawn inproc-ra0\nrepeat=3\nmonolith_repeat=5\n' > "$G/arms.txt" : > "$FAKE_RUNLOG" gate frozen --repeat 1; rc=$? -[ $rc -eq 0 ] && [ "$(calls 'transport=monolith repeat=3 ')" = 1 ] && [ "$(calls 'transport=inproc repeat=3 ')" = 1 ] \ +[ $rc -eq 0 ] && [ "$(calls 'transport=monolith repeat=5 ')" = 1 ] && [ "$(calls 'transport=inproc repeat=3 ')" = 1 ] \ && [ "$(calls 'transport=spawn repeat=3 ')" = 1 ] -expect "arms.txt repeat=3 monolith_repeat=3 resumed with --repeat 1 -> monolith, inproc, spawn x3" $? "rc=$rc; $(cat "$FAKE_RUNLOG")" +expect "arms.txt repeat=3 monolith_repeat=5 resumed with --repeat 1 -> monolith x5, inproc / spawn x3" $? \ + "rc=$rc; $(cat "$FAKE_RUNLOG")" + +echo "[interim] an arms.txt the interim x3 script froze (monolith_repeat=3) keeps its monolith x3" +G="$W2_LOGROOT/interim/gate3"; mkdir -p "$G" +printf 'case-a\n' > "$G/cases.txt" +printf 'arms=monolith inproc spawn inproc-ra0\nrepeat=3\nmonolith_repeat=3\napk=x\ntools=y\n' > "$G/arms.txt" +: > "$FAKE_RUNLOG" +gate interim; rc=$? +[ $rc -eq 0 ] && [ "$(calls 'transport=monolith repeat=3 ')" = 1 ] && [ "$(calls 'transport=monolith ')" = 1 ] \ + && [ "$(reps interim monolith case-a)" = 3 ] && [ -f "$G/state/monolith/case-a.done" ] \ + && grep -qx 'monolith_repeat=3' "$G/arms.txt" && grep -q 'monolith repeat frozen at 3' "$TMP/interim.$RUNS.log" +expect "monolith x3 with its .done, arms.txt untouched, and the log says why" $? "rc=$rc; $(cat "$FAKE_RUNLOG")" echo "[old-format] an arms.txt the pre-ID-P7-62 script froze (no monolith_repeat=) keeps its monolith x1" G="$W2_LOGROOT/old/gate3"; mkdir -p "$G" diff --git a/docs/Disaggregated/notes/p7/device-window-2/tools/30-gate3.sh b/docs/Disaggregated/notes/p7/device-window-2/tools/30-gate3.sh index 60756ad4..e4aab14d 100755 --- a/docs/Disaggregated/notes/p7/device-window-2/tools/30-gate3.sh +++ b/docs/Disaggregated/notes/p7/device-window-2/tools/30-gate3.sh @@ -2,10 +2,12 @@ # 30-gate3.sh [--cases a,b,...] [--repeat N] [--arms monolith,inproc,spawn,inproc-ra0] # # CONTRACT-P7 7.1/7.2 exit gate 3, one reboot-clean session, DirectVulkan x --use-pbuffer: -# monolith xN the same-session control every split reading is scored against; xN (ID-P7-62) -# so the session itself shows whether the monolith is bit-reproducible - a case -# whose monolith is not (the Iris sundial-lite / bliss / derivative packs on Adreno -# 830) is judged by 60-reduce.py's distributional check instead of bit-identity +# monolith x5 the same-session control every split reading is scored against; x5 +# (MONOLITH_REPEAT, the ID-P7-62 final ruling; whatever --repeat) so the session +# itself shows whether the monolith is bit-reproducible - a case whose monolith is +# not (the Iris sundial-lite / bliss / derivative packs on Adreno 830) is judged by +# 60-reduce.py's distributional check instead of bit-identity, whose nearest- +# monolith clause needs >= 3 passes and is steadier with more # inproc xN (N = --repeat, default 3) must be bit-identical across passes # spawn xN same, server role in a second process # inproc-ra0 x1 MOBILEGL_IPC_RUN_AHEAD=0 lockstep control, recorded only @@ -32,6 +34,8 @@ set -uo pipefail [ $# -ge 1 ] || die "usage: $0 [--cases a,b] [--repeat N] [--arms list]" STAMP=$1; shift +# ID-P7-62 final ruling: the monolith arm runs x5 on every case (the split arms stay x --repeat). +MONOLITH_REPEAT=5 CASES_ARG=""; REPEAT=3; ARMS="monolith inproc spawn inproc-ra0" while [ $# -gt 0 ]; do case "$1" in @@ -73,7 +77,7 @@ fi mapfile -t CASES < "$OUT/cases.txt" [ ${#CASES[@]} -gt 0 ] || die "empty case list" [ -f "$OUT/arms.txt" ] || printf 'arms=%s\nrepeat=%s\nmonolith_repeat=%s\napk=%s\ntools=%s\n' \ - "$ARMS" "$REPEAT" "$REPEAT" "$W2_APK" "$W2_TOOLS" > "$OUT/arms.txt" + "$ARMS" "$REPEAT" "$MONOLITH_REPEAT" "$W2_APK" "$W2_TOOLS" > "$OUT/arms.txt" # The repeat counts are frozen with the case list: a resume started without the first run's --repeat # (RUNBOOK: "30-gate3.sh ") must not re-run a pair with another count than its siblings'. FROZEN_REPEAT=$(sed -n 's/^repeat=//p' "$OUT/arms.txt") @@ -81,11 +85,14 @@ if [ -n "$FROZEN_REPEAT" ] && [ "$FROZEN_REPEAT" != "$REPEAT" ]; then log "repeat frozen at $FROZEN_REPEAT by $OUT/arms.txt (asked: $REPEAT)" REPEAT=$FROZEN_REPEAT fi -# The monolith runs x the same repeat from ID-P7-62 on (arms.txt monolith_repeat=). An arms.txt -# without the key was frozen by the earlier script, whose monolith pairs ran x1: a resume of that -# run keeps x1, so its monolith pairs all have one count (60-reduce.py reads the same key). +# The monolith count is frozen the same way (arms.txt monolith_repeat=, MONOLITH_REPEAT on a fresh +# start). A run the interim ID-P7-62 script started says 3 and a resume keeps 3; an arms.txt without +# the key was frozen by the earlier script, whose monolith pairs ran x1: a resume of that run keeps +# x1. Either way every monolith pair of one run has one count (60-reduce.py reads the same key). MONO_REPEAT=$(sed -n 's/^monolith_repeat=//p' "$OUT/arms.txt") -if [ -z "$MONO_REPEAT" ]; then +if [ -n "$MONO_REPEAT" ] && [ "$MONO_REPEAT" != "$MONOLITH_REPEAT" ]; then + log "monolith repeat frozen at $MONO_REPEAT by $OUT/arms.txt (this script starts a run at $MONOLITH_REPEAT)" +elif [ -z "$MONO_REPEAT" ]; then MONO_REPEAT=1 log "monolith repeat frozen at 1 by $OUT/arms.txt (no monolith_repeat=: started by the pre-ID-P7-62 script)" fi diff --git a/docs/Disaggregated/notes/p7/device-window-2/tools/60-reduce-selftest.py b/docs/Disaggregated/notes/p7/device-window-2/tools/60-reduce-selftest.py index 5c15a6f4..dfc12617 100644 --- a/docs/Disaggregated/notes/p7/device-window-2/tools/60-reduce-selftest.py +++ b/docs/Disaggregated/notes/p7/device-window-2/tools/60-reduce-selftest.py @@ -6,8 +6,8 @@ reducer on each exactly as window.sh does (a subprocess, --logroot --too the printed verdict, verdict.json and the exit status. Gate 3 is canned at the gate's own denominator - the canonical 36, derived here as 30-gate3.sh derives gate3/cases.txt (the tree's trace_cases.json CI split DirectVulkan cases minus CONTRACT-P7 7.1's three exclusions) - with -monolith x3 (ID-P7-62) / inproc x3 / spawn x3 / ra0, one boot_id, reboot-clean, unless a scenario -says otherwise. The scenarios: +monolith x5 (ID-P7-62 final ruling) / inproc x3 / spawn x3 / ra0, one boot_id, reboot-clean, +unless a scenario says otherwise. The scenarios: control gate 3 and all five CTS blocks equal to $BASE -> GATE3 PASS, CTS PASS, exit 0 missing-block ssbo never finished, dsa's report is empty, texture's has no results; and a @@ -33,24 +33,28 @@ says otherwise. The scenarios: NEEDS-ADJUDICATION naming it; one case differing, adjudicated (+ a stale line) -> PASS; one inproc repeat differs from its siblings -> FAIL (within-arm identity) unfinished a pair without .done; a repeat without an actual PNG -> GATE3 INCOMPLETE - monolith-determinism ID-P7-62, clause (2) in its nearest-monolith form, on synthetic 8x8 - pictures (grey 128 with runs of pixels at other levels): a bit-identical monolith - x3 + a nondeterministic inproc -> FAIL (strict clause); a nondeterministic - monolith in a chain (4 / 4 / 8 px apart, delta 2: D_mm 4, Delta_mm 2, bound - 1.25 x 4 + 32 = 37 px / 2 + 1 = 3) + split passes at most 37 px / delta 3 from - their nearest monolith pass (45 px from the farthest: outside the first form's - 1.25 x max-pair) -> PASS, exit 0, no adjudication line, the numbers printed, each - pass's nearest named (a px tie goes to the smaller delta); the stdlib decoder - (W2_REDUCE_PNG_DECODER=pure) -> the same numbers; a split pass 38 px from its - nearest -> FAIL; one at delta 4 from its nearest-by-px pass (M1, 4 px) -> FAIL - though another pass is delta 2 away; monolith x4 with a bit-identical twin pair - -> leave-one-out over passes (D_mm 4, not 20 over pictures), a split pass 40 px - from its nearest -> FAIL; one split ssim 0.0005+ from ONE monolith reading -> - FAIL; the pre-ID-P7-62 monolith x1 + a nondeterministic split -> FAIL naming the - >= 3 passes it lacks; a finished monolith pair short of arms.txt's - monolith_repeat -> FAIL; --extra-monolith DIR@BOOT adding two passes to that x1 - archive -> the rule applies, GATE3 PASS-WITH-EXTRA-READINGS (exit 1), without - @BOOT or with another boot -> INVALID-SESSION + monolith-determinism ID-P7-62 final ruling, clause (2) in its nearest-monolith form, on + synthetic 8x8 pictures (grey 128 with runs of pixels at other levels): a + bit-identical monolith x5 + a nondeterministic inproc -> FAIL (strict clause); a + nondeterministic monolith x5 in a chain (each pass 4 px / delta 2 from its + neighbour: D_mm 4, Delta_mm 2, bound 1.25 x 4 + 32 = 37 px / 1.5 x 2 + 8 = 11) + + split passes at most 37 px / delta 3 from their nearest monolith pass (45 px + from the farthest: outside the first form's 1.25 x max-pair) -> PASS, exit 0, no + adjudication line, the numbers printed, each pass's nearest named (a px tie goes + to the smaller delta); the stdlib decoder (W2_REDUCE_PNG_DECODER=pure) -> the same + numbers; a split pass 38 px from its nearest -> FAIL; one at delta 11 from its + nearest (above the interim Delta_mm + 1 = 3 and Delta_mm + 8 = 10, at 1.5 x 2 + 8) + -> PASS, at delta 12 -> FAIL; one at delta 12 from its nearest-by-px pass (M1, + 4 px) -> FAIL though another pass is delta 10 away; monolith x5 with a + bit-identical twin pair -> leave-one-out over passes (D_mm 4, not 20 over + pictures), a split pass 40 px from its nearest -> FAIL; one split ssim 0.0005+ + from ONE monolith reading -> FAIL; a run the interim script started (monolith x3, + monolith_repeat=3) -> the rule applies (>= 3 passes), PASS, the 5 it lacks noted; + the pre-ID-P7-62 monolith x1 + a nondeterministic split -> FAIL naming the >= 3 + passes it lacks; a finished monolith pair short of arms.txt's monolith_repeat=5 + -> FAIL; --extra-monolith DIR@BOOT adding two passes to that x1 archive -> the + rule applies, GATE3 PASS-WITH-EXTRA-READINGS (exit 1), without @BOOT or with + another boot -> INVALID-SESSION rate-gate the gate rate alone decides (no crash anywhere): k $BASE-Pass dsa cases turn Fail. k=1: 367/370, -0.270 pp -> PASS, exit 0; k=2: -0.541 pp -> FAIL on the rate; k=4: 364/6/0/1/0, -1.081 pp -> FAIL, CTS FAIL @@ -119,8 +123,12 @@ EXCLUDED = ("minecraft-1.21.1-neoforge-create-indirect-in-world", "minecraft-1.2 CASES, A, B = (), None, None OPENRA = "OpenRA" REPEAT = 3 +# 30-gate3.sh under the ID-P7-62 final ruling: the monolith x5 on every case. +MONO_REPEAT = 5 GONE = "/nonexistent/p7w7-tree-deadbeef/tools/cts/caselists/p7-%s-gl46.txt" -ARMS = (("monolith", REPEAT), ("inproc", REPEAT), ("spawn", REPEAT), ("inproc-ra0", 1)) +ARMS = (("monolith", MONO_REPEAT), ("inproc", REPEAT), ("spawn", REPEAT), ("inproc-ra0", 1)) +# The interim ID-P7-62 30-gate3.sh: monolith x3 (monolith_repeat=3 in gate3/arms.txt). +ARMS_MONO3 = (("monolith", 3),) + ARMS[1:] # The pre-ID-P7-62 30-gate3.sh: monolith x1, and no monolith_repeat= in gate3/arms.txt. ARMS_MONO1 = (("monolith", 1),) + ARMS[1:] ARMS_TXT_MONO1 = "arms=monolith inproc spawn inproc-ra0\nrepeat=3\n" @@ -486,7 +494,7 @@ def scenario_monolith_arm(k, cn, tree): e.get("verdict") == "FAIL" and overall(v) == "FAIL" and rc != 0 and any("NOT proven monolith" in r for r in e.get("reasons", [])), e) k.expect("the other case still PASS", case(v, B).get("verdict") == "PASS") - k.expect("arm identity line counts 107/108 monolith passes (x3)", "monolith 107/108 repeat(s)" in text, text) + k.expect("arm identity line counts 179/180 monolith passes (x5)", "monolith 179/180 repeat(s)" in text, text) out = cn.window("mono-nolog") (out / "gate3/archive/monolith" / (B + "-DirectVulkan") / "repeat-01" / "mobilegl.log").unlink() @@ -560,7 +568,7 @@ def scenario_arms(k, cn, tree): and any("arm spawn has no archive" in p for p in contract(v, "arms")), overall(v)) cn.window("ra0-planned", gate3_args={"arms": three, "arms_txt": "arms=monolith inproc spawn inproc-ra0\nrepeat=3\n" - "monolith_repeat=3\n"}) + "monolith_repeat=%d\n" % MONO_REPEAT}) rc, text, v = run_reducer(cn.root, "ra0-planned", tree) k.expect("inproc-ra0 named in arms.txt but not reached yet -> GATE3 INCOMPLETE (not PASS), exit != 0", overall(v) == "INCOMPLETE" and rc != 0 and contract(v, "arms") == [] @@ -623,7 +631,7 @@ def scenario_cross_arm(k, cn, tree): and g3(v, "cross_arm_unadjudicated") == list(CASES)) x = case(v, B).get("cross_arm", {}) k.expect("the list carries the archived px!=golden and the direct px counts (8x8: 64 between 128 and 129)", - x.get("px_vs_golden") == {"monolith": [17, 17, 17], "inproc": [17, 17, 17], "spawn": [17, 17, 17]} + x.get("px_vs_golden") == {"monolith": [17] * MONO_REPEAT, "inproc": [17, 17, 17], "spawn": [17, 17, 17]} and x.get("px_direct") == {"inproc-monolith": 0, "spawn-monolith": 64, "inproc-spawn": 64} and "px!=golden (archived mismatchPixels) monolith 17..17, inproc 17..17, spawn 17..17; px between the " "arms' first repeats: inproc-monolith 0, spawn-monolith 64, inproc-spawn 64" in text, x) @@ -683,26 +691,30 @@ def scenario_unfinished(k, cn, tree): def scenario_determinism(k, cn, tree): print("[monolith-determinism] ID-P7-62: a monolith not bit-identical over >= 3 passes swaps the split arms' " - "bit-identity clause for the distributional check, clause (2) in its nearest-monolith form") - # A nondeterministic monolith in a chain: M1 = px 0..7 at 130, M2 = px 0..3 at 130, M3 = solid - # 128. M1-M2 4 px / delta 2, M2-M3 4 px / 2, M1-M3 8 px / 2 -> each pass's nearest OTHER pass is - # 4 px / delta 2: D_mm 4, Delta_mm 2, bound 1.25 x 4 + 32 = 37 px / 2 + 1 = 3. (The first form, - # 1.25 x the max over all pairs, bounded the split at 10 px / delta 2.5.) - m1, m2, m3 = speck((0, 8, 130)), speck((0, 4, 130)), 128 - mono = {("monolith", A, 1): m1, ("monolith", A, 2): m2, ("monolith", A, 3): m3} - # inproc rep3: px 8..44 at 131 -> nearest M3 at 37 px / delta 3 (41 px from M2, 45 from M1); - # spawn rep2: px 0..3 at 130 + 8..27 at 129 -> nearest M2, 20 px / delta 1; spawn rep3: px 0..7 - # at 131 -> 8 px from both M1 (delta 1) and M2 (delta 3): the tie goes to the smaller delta. + "bit-identity clause for the distributional check, clause (2) in its nearest-monolith form " + "(final ruling: 1.25 x D_mm + 32 px, 1.5 x Delta_mm + 8 per channel; monolith x5)") + # A nondeterministic monolith x5 in a chain: M1 = px 0..7 at 130, M2 = px 0..3 at 130, M3 = solid + # 128, M4 = px 56..59 at 126, M5 = px 56..63 at 126. Each neighbour pair (M1-M2, M2-M3, M3-M4, + # M4-M5) is 4 px / delta 2 apart, every other pair 8..16 px / delta 2 -> each pass's nearest + # OTHER pass is 4 px / delta 2: D_mm 4, Delta_mm 2, bound 1.25 x 4 + 32 = 37 px / 1.5 x 2 + 8 = + # 11. (The first form, 1.25 x the max over all pairs, bounded the split at 20 px / delta 2.5; the + # interim one at 37 px / Delta_mm + 1 = 3.) + m1, m2, m3, m4, m5 = speck((0, 8, 130)), speck((0, 4, 130)), 128, speck((56, 4, 126)), speck((56, 8, 126)) + mono = {("monolith", A, 1): m1, ("monolith", A, 2): m2, ("monolith", A, 3): m3, ("monolith", A, 4): m4, + ("monolith", A, 5): m5} + # inproc rep3: px 8..44 at 131 -> nearest M3 at 37 px / delta 3 (41 px from M2 and M4, 45 from M1 + # and M5); spawn rep2: px 0..3 at 130 + 8..27 at 129 -> nearest M2, 20 px / delta 1; spawn rep3: + # px 0..7 at 131 -> 8 px from M1 (delta 1), M2 and M3 (delta 3): the tie goes to the smaller delta. inside = {("inproc", A, 1): m1, ("inproc", A, 2): m2, ("inproc", A, 3): speck((8, 37, 131)), ("spawn", A, 1): m3, ("spawn", A, 2): speck((0, 4, 130), (8, 20, 129)), ("spawn", A, 3): speck((0, 8, 131))} - rule_line = "nondeterministic monolith (3 distinct / 3 passes) -> distributional check (ID-P7-62)" - numbers = {"passes": 3, "distinct": 3, "rule": "distributional", "mono_nearest_px_max": 4, + rule_line = "nondeterministic monolith (5 distinct / 5 passes) -> distributional check (ID-P7-62)\n" + numbers = {"passes": 5, "distinct": 5, "rule": "distributional", "mono_nearest_px_max": 4, "mono_nearest_delta_max": 2, "split_nearest_px_max": 37, "split_nearest_delta_max": 3, - "px_bound": 37.0, "delta_bound": 3, "within": True} - numbers_line = ("monolith leave-one-out nearest: D_mm 4 px / Delta_mm 2 (3 passes); split-to-nearest-monolith max " + "px_bound": 37.0, "delta_bound": 11.0, "within": True} + numbers_line = ("monolith leave-one-out nearest: D_mm 4 px / Delta_mm 2 (5 passes); split-to-nearest-monolith max " "37 px / delta 3 (6 passes; most px: inproc rep3 -> monolith rep3 37 px / delta 3); bound 1.25 x 4 " - "+ 32 = 37.00 px / 2 + 1 = 3: within") + "+ 32 = 37.00 px / 1.50 x 2 + 8 = 11.00: within") def det(v): return case(v, A).get("monolith_determinism", {}) @@ -713,31 +725,37 @@ def scenario_determinism(k, cn, tree): def has(v, needle): return any(needle in r for r in case(v, A).get("reasons", [])) - # (a) the strict clause stands when the monolith is bit-identical over its 3 passes. + # (a) the strict clause stands when the monolith is bit-identical over its 5 passes. cn.window("det-strict", gate3_args={"picture": {("inproc", A, 2): speck((0, 4, 130))}}) rc, text, v = run_reducer(cn.root, "det-strict", tree) - k.expect("bit-identical monolith x3 + a nondeterministic inproc -> %s FAIL (strict clause), GATE3 FAIL" % A, + k.expect("bit-identical monolith x5 + a nondeterministic inproc -> %s FAIL (strict clause), GATE3 FAIL" % A, case(v, A).get("verdict") == "FAIL" and overall(v) == "FAIL" and rc != 0 and det(v).get("rule") == "strict" and has(v, "inproc repeats NOT bit-identical (2 distinct pictures); the same-session monolith is " - "bit-identical over 3 passes") and case(v, B).get("verdict") == "PASS", + "bit-identical over 5 passes") and case(v, B).get("verdict") == "PASS", (case(v, A).get("reasons"), det(v))) # (b) a nondeterministic monolith, every split pass within its nearest monolith pass's bound -> PASS. cn.window("det-inside", gate3_args={"picture": {**mono, **inside}}) rc, text, v = run_reducer(cn.root, "det-inside", tree) - k.expect("nondeterministic monolith (3 distinct / 3 passes) + every split pass within 1.25 x D_mm + 32 px / " - "Delta_mm + 1 of its nearest monolith pass (37 px / delta 3; 45 px / delta 3 from the farthest) -> %s " - "PASS, GATE3 PASS, exit 0" % A, + k.expect("nondeterministic monolith (5 distinct / 5 passes) + every split pass within 1.25 x D_mm + 32 px / " + "1.5 x Delta_mm + 8 of its nearest monolith pass (37 px / delta 3; 45 px / delta 3 from the farthest) -> " + "%s PASS, GATE3 PASS, exit 0" % A, case(v, A).get("verdict") == "PASS" and overall(v) == "PASS" and rc == 0 and "GATE3 PASS: 36/36 cases PASS\n" in text, (case(v, A).get("reasons"), overall(v), text[-400:])) - k.expect("...the rule line and the numbers: D_mm 4 px / Delta_mm 2 over 3 monolith passes, split-to-nearest max " - "37 px / delta 3 over 6 split passes, bound 37.00 px / 3", + k.expect("...the rule line and the numbers: D_mm 4 px / Delta_mm 2 over 5 monolith passes, split-to-nearest max " + "37 px / delta 3 over 6 split passes, bound 37.00 px / 11.00; the four constants in verdict.json", {key: det(v).get(key) for key in numbers} == numbers and rule_line in text and numbers_line in text - and len(det(v).get("split_nearest", [])) == 6 and g3(v, "determinism", {}).get("distributional") == [A], - (det(v), text[:1500])) - k.expect("...leave-one-out over the monolith passes: each pass's nearest OTHER pass is 4 px / delta 2", - [(n.get("pass"), n.get("px"), n.get("delta")) for n in det(v).get("mono_nearest", [])] - == [("monolith rep1", 4, 2), ("monolith rep2", 4, 2), ("monolith rep3", 4, 2)], det(v).get("mono_nearest")) + and len(det(v).get("split_nearest", [])) == 6 and g3(v, "determinism", {}).get("distributional") == [A] + and g3(v, "rule_62") == {"min_passes": 3, "expected_passes": 5, "spread_factor": 1.25, "px_headroom": 32, + "delta_factor": 1.5, "delta_headroom": 8} + and "monolith repeat=5 (30-gate3.sh runs x5, the ID-P7-62 rule needs >= 3)" in text, + (det(v), g3(v, "rule_62"), text[:1500])) + k.expect("...leave-one-out over the monolith passes: each pass's nearest OTHER pass is 4 px / delta 2 (a tie: " + "the first listed)", + [(n.get("pass"), n.get("nearest"), n.get("px"), n.get("delta")) for n in det(v).get("mono_nearest", [])] + == [("monolith rep1", "monolith rep2", 4, 2), ("monolith rep2", "monolith rep1", 4, 2), + ("monolith rep3", "monolith rep2", 4, 2), ("monolith rep4", "monolith rep3", 4, 2), + ("monolith rep5", "monolith rep4", 4, 2)], det(v).get("mono_nearest")) k.expect("...each split pass is scored against its NEAREST monolith pass (fewest px; a px tie goes to the " "smaller delta)", near(v, "inproc rep3") == {"pass": "inproc rep3", "nearest": "monolith rep3", "px": 37, "delta": 3, @@ -759,7 +777,7 @@ def scenario_determinism(k, cn, tree): % decoder, {key: det(v).get(key) for key in numbers} == numbers and det(v).get("decoder", "").startswith("pure-python") and rc == 0, det(v)) - # (c) outside the bound: in pixels, then in the per-channel delta to the nearest-by-px pass. + # (c) outside the pixel bound. cn.window("det-px", gate3_args={"picture": {**mono, **inside, ("inproc", A, 3): speck((8, 38, 131))}}) rc, text, v = run_reducer(cn.root, "det-px", tree) k.expect("...one split pass 38 px from its nearest monolith pass > 1.25 x 4 + 32 = 37 (delta 3 inside) -> %s " @@ -770,46 +788,87 @@ def scenario_determinism(k, cn, tree): and case(v, A).get("reasons") == ["ID-P7-62: inproc rep3 is 38 differing px from its nearest monolith pass " "(monolith rep3) > 1.25 x D_mm 4 + 32 (= 37.00)"], (det(v).get("split_nearest"), case(v, A).get("reasons"))) - # inproc rep3: px 0..3 at 126, 4..7 at 130 -> M1 4 px / delta 4 (the nearest), M2 8 px / 4, M3 8 px / 2. - cn.window("det-delta", gate3_args={"picture": {**mono, **inside, ("inproc", A, 3): speck((0, 4, 126), (4, 4, 130))}}) - rc, text, v = run_reducer(cn.root, "det-delta", tree) - k.expect("...one split pass at delta 4 from its nearest-by-px monolith pass (M1, 4 px) > 2 + 1 -> %s FAIL, though " - "another pass (M3, 8 px) is only delta 2 away" % A, + + # (d) the per-channel delta bound is 1.5 x Delta_mm + 8 = 11: inproc rep3 = px 0..3 at 141 is + # 4 px from M2 (delta 11) and from M3 (delta 13) -> the tie goes to M2, delta 11: above the + # interim Delta_mm + 1 = 3 and above Delta_mm + 8 = 10, at the bound -> PASS. px 0..3 at 142 -> + # M2 delta 12 > 11 -> FAIL. + cn.window("det-delta-in", gate3_args={"picture": {**mono, **inside, ("inproc", A, 3): speck((0, 4, 141))}}) + rc, text, v = run_reducer(cn.root, "det-delta-in", tree) + k.expect("...one split pass at delta 11 from its nearest monolith pass (M2, 4 px): > the interim Delta_mm + 1 = 3, " + "<= 1.5 x 2 + 8 = 11 -> %s PASS, GATE3 PASS, exit 0" % A, + case(v, A).get("verdict") == "PASS" and overall(v) == "PASS" and rc == 0 + and near(v, "inproc rep3") == {"pass": "inproc rep3", "nearest": "monolith rep2", "px": 4, "delta": 11, + "within": True} + and det(v).get("split_nearest_delta_max") == 11 and det(v).get("delta_bound") == 11.0 + and det(v).get("within") is True, + (det(v).get("split_nearest"), case(v, A).get("reasons"))) + cn.window("det-delta-out", gate3_args={"picture": {**mono, **inside, ("inproc", A, 3): speck((0, 4, 142))}}) + rc, text, v = run_reducer(cn.root, "det-delta-out", tree) + k.expect("...one split pass at delta 12 from its nearest monolith pass (M2, 4 px) > 1.5 x 2 + 8 = 11 -> %s FAIL, " + "GATE3 FAIL" % A, case(v, A).get("verdict") == "FAIL" and overall(v) == "FAIL" and rc != 0 - and det(v).get("split_nearest_delta_max") == 4 and det(v).get("within") is False + and near(v, "inproc rep3").get("delta") == 12 and near(v, "inproc rep3").get("within") is False and case(v, A).get("reasons") == ["ID-P7-62: inproc rep3's per-channel delta to its nearest monolith pass " - "(monolith rep1, 4 px) is 4 > Delta_mm 2 + 1 (= 3)"], + "(monolith rep2, 4 px) is 12 > 1.50 x Delta_mm 2 + 8 (= 11.00)"], + (det(v).get("split_nearest"), case(v, A).get("reasons"))) + # The delta is measured to the nearest-by-px pass: inproc rep3 = px 0..3 at 118, 4..7 at 130 -> + # M1 4 px / delta 12 (the nearest), M2 8 px / 12, M3 8 px / 10, M4 12 px / 10. + cn.window("det-delta", gate3_args={"picture": {**mono, **inside, ("inproc", A, 3): speck((0, 4, 118), (4, 4, 130))}}) + rc, text, v = run_reducer(cn.root, "det-delta", tree) + k.expect("...one split pass at delta 12 from its nearest-by-px monolith pass (M1, 4 px) > 11 -> %s FAIL, though " + "another pass (M3, 8 px) is only delta 10 away" % A, + case(v, A).get("verdict") == "FAIL" and overall(v) == "FAIL" and rc != 0 + and det(v).get("split_nearest_delta_max") == 12 and det(v).get("within") is False + and case(v, A).get("reasons") == ["ID-P7-62: inproc rep3's per-channel delta to its nearest monolith pass " + "(monolith rep1, 4 px) is 12 > 1.50 x Delta_mm 2 + 8 (= 11.00)"], (det(v).get("split_nearest"), case(v, A).get("reasons"))) - # (d) leave-one-out is over PASSES: a bit-identical twin pair is 0 px from each other. Monolith x4: - # rep1 = rep2 = solid, rep3 = px 0..19 at 130, rep4 = px 0..23 at 130 -> D_mm = max(0, 0, 4, 4) = 4 - # (over distinct pictures it would be 20); inproc rep3 = px 24..63 at 129 is 40 px from its nearest - # (rep1) > 37 -> FAIL. + # (e) leave-one-out is over PASSES: a bit-identical twin pair is 0 px from each other. Monolith x5: + # rep1 = rep2 = solid, rep3 / rep4 / rep5 = px 0..19 / 0..23 / 0..27 at 130 -> D_mm = max(0, 0, 4, + # 4, 4) = 4 (over distinct pictures it would be 20); inproc rep3 = px 24..63 at 129 is 40 px from + # its nearest (rep1) > 37 -> FAIL. twins = {("monolith", A, 1): 128, ("monolith", A, 2): 128, ("monolith", A, 3): speck((0, 20, 130)), - ("monolith", A, 4): speck((0, 24, 130)), ("inproc", A, 3): speck((24, 40, 129))} - cn.window("det-twins", gate3_args={"arms": (("monolith", 4),) + ARMS[1:], "picture": twins}) + ("monolith", A, 4): speck((0, 24, 130)), ("monolith", A, 5): speck((0, 28, 130)), + ("inproc", A, 3): speck((24, 40, 129))} + cn.window("det-twins", gate3_args={"picture": twins}) rc, text, v = run_reducer(cn.root, "det-twins", tree) - k.expect("monolith x4 with a bit-identical twin pair (3 distinct / 4 passes): D_mm 4 over the passes, not 20 " + k.expect("monolith x5 with a bit-identical twin pair (4 distinct / 5 passes): D_mm 4 over the passes, not 20 " "over the pictures; a split pass 40 px from its nearest -> %s FAIL" % A, case(v, A).get("verdict") == "FAIL" and rc != 0 and det(v).get("rule") == "distributional" - and (det(v).get("passes"), det(v).get("distinct")) == (4, 3) + and (det(v).get("passes"), det(v).get("distinct")) == (5, 4) and [(n.get("pass"), n.get("nearest"), n.get("px")) for n in det(v).get("mono_nearest", [])] == [("monolith rep1", "monolith rep2", 0), ("monolith rep2", "monolith rep1", 0), - ("monolith rep3", "monolith rep4", 4), ("monolith rep4", "monolith rep3", 4)] + ("monolith rep3", "monolith rep4", 4), ("monolith rep4", "monolith rep3", 4), + ("monolith rep5", "monolith rep4", 4)] and det(v).get("px_bound") == 37.0 and has(v, "inproc rep3 is 40 differing px from its nearest monolith pass (monolith rep1)"), (det(v), case(v, A).get("reasons"))) - # (e) clause (1) holds against EVERY monolith reading, not one of them. + # (f) clause (1) holds against EVERY monolith reading, not one of them. ssim = {("monolith", A, 1): 0.9978, ("monolith", A, 2): 0.998, ("monolith", A, 3): 0.9982, ("spawn", A, 2): 0.99831} cn.window("det-ssim", gate3_args={"picture": {**mono, **inside}, "ssim": ssim}) rc, text, v = run_reducer(cn.root, "det-ssim", tree) k.expect("...a spawn ssim 0.00051 from ONE monolith reading (0.00011 from another) -> %s FAIL" % A, case(v, A).get("verdict") == "FAIL" and rc != 0 and det(v).get("within") is True - and case(v, A).get("reasons") == ["spawn rep2 |ssim-monolith|=0.000510 > 0.0005 (max over 3 monolith readings)"], + and case(v, A).get("reasons") == ["spawn rep2 |ssim-monolith|=0.000510 > 0.0005 (max over 5 monolith readings)"], case(v, A).get("reasons")) - # (f) the pre-ID-P7-62 archives (monolith x1, no monolith_repeat= in arms.txt): the rule cannot apply. + # (g) a run the interim ID-P7-62 script started (monolith x3, arms.txt monolith_repeat=3): the + # rule needs >= 3 passes, so it applies; the 5 the final script runs are noted, not required. + cn.window("det-mono3", gate3_args={"arms": ARMS_MONO3, "picture": {**{key: val for key, val in mono.items() if key[2] <= 3}, + **inside}}) + rc, text, v = run_reducer(cn.root, "det-mono3", tree) + k.expect("monolith x3 (the interim 30-gate3.sh, monolith_repeat=3) + a nondeterministic split within the bound -> " + "the rule applies (3 distinct / 3), %s PASS, GATE3 PASS, exit 0, the 5 it lacks noted" % A, + case(v, A).get("verdict") == "PASS" and overall(v) == "PASS" and rc == 0 and g3(v, "monolith_repeat") == 3 + and (det(v).get("rule"), det(v).get("passes"), det(v).get("distinct")) == ("distributional", 3, 3) + and "nondeterministic monolith (3 distinct / 3 passes) -> distributional check (ID-P7-62) [3 < the 5 " + "monolith passes 30-gate3.sh runs; the rule needs >= 3]" in text + and "monolith repeat=3 (30-gate3.sh runs x5" in text, + (overall(v), case(v, A).get("reasons"), det(v), text[:800])) + + # (h) the pre-ID-P7-62 archives (monolith x1, no monolith_repeat= in arms.txt): the rule cannot apply. cn.window("det-mono1", gate3_args={"arms": ARMS_MONO1, "arms_txt": ARMS_TXT_MONO1, "picture": inside}) rc, text, v = run_reducer(cn.root, "det-mono1", tree) k.expect("monolith x1 (the earlier 30-gate3.sh) + a nondeterministic split -> %s FAIL naming the >= 3 monolith " @@ -820,20 +879,20 @@ def scenario_determinism(k, cn, tree): "rule needs >= 3 same-session monolith passes, this session has 1") and case(v, B).get("verdict") == "PASS", (det(v), case(v, A).get("reasons"))) - # (g) the monolith pair's own bookkeeping: arms.txt monolith_repeat=3 holds a finished pair to 3. + # (i) the monolith pair's own bookkeeping: arms.txt monolith_repeat=5 holds a finished pair to 5. out = cn.window("det-short") - shutil.rmtree(out / "gate3/archive/monolith" / (A + "-DirectVulkan") / "repeat-03") + shutil.rmtree(out / "gate3/archive/monolith" / (A + "-DirectVulkan") / "repeat-05") rc, text, v = run_reducer(cn.root, "det-short", tree) - k.expect("a finished monolith pair with 2 of arms.txt's monolith_repeat=3 -> %s FAIL" % A, + k.expect("a finished monolith pair with 4 of arms.txt's monolith_repeat=5 -> %s FAIL" % A, case(v, A).get("verdict") == "FAIL" and rc != 0 - and has(v, "monolith: 2 repeat(s) in a finished pair < gate3/arms.txt monolith_repeat=3"), case(v, A)) + and has(v, "monolith: 4 repeat(s) in a finished pair < gate3/arms.txt monolith_repeat=5"), case(v, A)) (out / "gate3/state/monolith" / (A + ".done")).unlink() rc, text, v = run_reducer(cn.root, "det-short", tree) k.expect("...the same pair without its .done -> INCOMPLETE (a resume re-runs it)", case(v, A).get("verdict") == "INCOMPLETE" and overall(v) == "INCOMPLETE" - and "monolith: 2/3 repeats" in case(v, A).get("reasons", []), case(v, A).get("reasons")) + and "monolith: 4/5 repeats" in case(v, A).get("reasons", []), case(v, A).get("reasons")) - # (h) --extra-monolith: the g3det E1-mono re-reduction shape (x1 archive = M3 + passes outside gate3/). + # (j) --extra-monolith: the g3det E1-mono re-reduction shape (x1 archive = M3 + passes outside gate3/). cn.window("det-extra", gate3_args={"arms": ARMS_MONO1, "arms_txt": ARMS_TXT_MONO1, "picture": inside}) extra = cn.extra_monolith(cn.root / "extra-unstamped", A, [m1, m2]) rc, text, v = run_reducer(cn.root, "det-extra", tree, extra=["%s@%s" % (extra, BOOT)]) diff --git a/docs/Disaggregated/notes/p7/device-window-2/tools/60-reduce.py b/docs/Disaggregated/notes/p7/device-window-2/tools/60-reduce.py index d878ca25..28dd4d99 100644 --- a/docs/Disaggregated/notes/p7/device-window-2/tools/60-reduce.py +++ b/docs/Disaggregated/notes/p7/device-window-2/tools/60-reduce.py @@ -24,8 +24,9 @@ GATE 3 (CONTRACT-P7 7.1/7.2), per case of gate3/cases.txt: every arm, carries one boot_id, equal to session/boot-id.txt. A second boot_id, a record that carries none, or no session/boot-id.txt makes the whole gate GATE3 INVALID-SESSION, whatever the pictures say (the gate is a same-session reading); - * the monolith arm (x gate3/arms.txt's monolith_repeat: 3 from ID-P7-62 on; an arms.txt without - the key is the earlier script's x1) is the same-session control: every pass must itself pass, + * the monolith arm (x gate3/arms.txt's monolith_repeat: 5 from the ID-P7-62 final ruling on, + whatever --repeat; 3 in a run the interim script started; an arms.txt without the key is the + earlier script's x1) is the same-session control: every pass must itself pass, and its role logs must show no 'Config: IPC' / MOBILEGL_TRANSPORT line (the arm really was the monolith); a finished monolith pair with fewer passes than monolith_repeat is that case FAIL; * inproc and spawn: the pair finished (state .done), every expected repeat present and @@ -38,19 +39,25 @@ GATE 3 (CONTRACT-P7 7.1/7.2), per case of gate3/cases.txt: presupposes a bit-reproducible reference. A case whose same-session monolith pictures are NOT all bit-identical over >= 3 passes has none, so for it the split arms' bit-identity clause is replaced by the distributional check: (1) the |ssim - m| clause above, for every monolith - reading m; (2) NEAREST MONOLITH (integrator ruling on ID-P7-62, replacing the first form's - 1.25 x the max over all pairs): px(a, b) = the pixels where any RGBA channel differs, delta(a, b) - = the largest per-channel difference; the nearest of a pass among candidate monolith passes is - the one with the fewest differing px (a tie: the smaller delta), and its delta is measured to - that same pass. D_mm / Delta_mm = the max over the monolith passes of px / delta to the nearest - OTHER monolith pass (leave-one-out, over passes: a bit-identical twin is 0 / 0); every split - pass s (inproc and spawn repeats) must hold px(s, nearest) <= 1.25 x D_mm + 32 AND - delta(s, nearest) <= Delta_mm + 1. The pixels are read from the archived actual PNGs (numpy + - PIL; without them, or with W2_REDUCE_PNG_DECODER=pure, compare_actuals' stdlib decoder - the - same numbers, slower). Such a case prints 'nondeterministic monolith (N distinct / M passes) - -> distributional check' and its numbers, and its cross-arm difference is decided by (2), not - by gate3/adjudication.tsv. A case whose monolith is bit-identical, or that has fewer than 3 - monolith passes (the pre-ID-P7-62 x1 archives), keeps the strict clause; + reading m; (2) NEAREST MONOLITH (the integrator's final ruling on ID-P7-62, replacing the + first form's 1.25 x the max over all pairs and the interim Delta_mm + 1): px(a, b) = the pixels + where any RGBA channel differs, delta(a, b) = the largest per-channel difference; the nearest + of a pass among candidate monolith passes is the one with the fewest differing px (a tie: the + smaller delta), and its delta is measured to that same pass. D_mm / Delta_mm = the max over the + monolith passes of px / delta to the nearest OTHER monolith pass (leave-one-out, over passes: a + bit-identical twin is 0 / 0); every split pass s (inproc and spawn repeats) must hold + px(s, nearest) <= 1.25 x D_mm + 32 AND delta(s, nearest) <= 1.5 x Delta_mm + 8 (delta is one + pixel's largest channel difference, so it gets a factor and a headroom of its own: scoring the + window-2 gate-3 split pictures against every monolith x3 subset of that session's monolith + passes, derivative passed 0% and pure monolith noise 70% under Delta_mm + 1; under + 1.5 x Delta_mm + 8 sundial / bliss / derivative pass 100 / 97.5 / 95% at x3 and 100 / 99.9 / + 100% at x5, noise 95-100%). The rule needs >= 3 monolith passes; 30-gate3.sh runs 5 (a case + judged on fewer notes it). The pixels are read from the archived actual PNGs (numpy + PIL; + without them, or with W2_REDUCE_PNG_DECODER=pure, compare_actuals' stdlib decoder - the same + numbers, slower). Such a case prints 'nondeterministic monolith (N distinct / M passes) -> + distributional check' and its numbers, and its cross-arm difference is decided by (2), not by + gate3/adjudication.tsv. A case whose monolith is bit-identical over all its passes, or that has + fewer than 3 monolith passes (the pre-ID-P7-62 x1 archives), keeps the strict clause; * --extra-monolith DIR[@BOOT_ID] (repeatable; an adjudication re-reduction, never the gate): adds the passes under DIR/-DirectVulkan/repeat-NN (a run_android_retrace_local.py --archive-dir tree) to the case's monolith readings. Each must be the session's: a repeat's @@ -126,12 +133,16 @@ TOL = 0.0005 # ID-P7-62 (g3det VERDICT.md section 2): a case whose same-session monolith is not bit-identical # over >= MONOLITH_RULE_PASSES passes is judged by the distributional check, whose clause (2) holds # every split pass to its nearest monolith pass within SPREAD_FACTOR x D_mm + PX_HEADROOM px and -# Delta_mm + DELTA_HEADROOM per channel (D_mm / Delta_mm: the monolith's leave-one-out nearest -# spread; integrator ruling on ID-P7-62). +# DELTA_FACTOR x Delta_mm + DELTA_HEADROOM per channel (D_mm / Delta_mm: the monolith's +# leave-one-out nearest spread; the integrator's final ruling on ID-P7-62). 30-gate3.sh runs the +# monolith x MONOLITH_EXPECTED_PASSES (recorded as arms.txt monolith_repeat=); the rule itself +# needs MONOLITH_RULE_PASSES. MONOLITH_RULE_PASSES = 3 +MONOLITH_EXPECTED_PASSES = 5 SPREAD_FACTOR = 1.25 PX_HEADROOM = 32 -DELTA_HEADROOM = 1 +DELTA_FACTOR = 1.5 +DELTA_HEADROOM = 8 RULING_62 = "ID-P7-62" UUID = re.compile(r"^(.+)@([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$") # CONTRACT-P7 7.1/7.2's constants: the gate is held to these, never to what gate3/arms.txt, @@ -445,8 +456,9 @@ def gate3(out, ca, extras=()): repeat = int(meta.get("repeat", str(CONTRACT_REPEATS))) except ValueError: repeat = 0 - # 30-gate3.sh records monolith_repeat= from ID-P7-62 on (the monolith runs x --repeat); an - # arms.txt without the key was written by the earlier script, which ran the monolith x1. + # 30-gate3.sh records monolith_repeat= from ID-P7-62 on (5 under the final ruling, 3 under the + # interim script); an arms.txt without the key was written by the earlier script, which ran the + # monolith x1. A finished monolith pair is held to that recorded count. try: mono_repeat = int(meta.get("monolith_repeat", "1")) except ValueError: @@ -656,7 +668,7 @@ def gate3(out, ca, extras=()): else: d_mm, delta_mm = det["mono_nearest_px_max"], det["mono_nearest_delta_max"] det["px_bound"] = SPREAD_FACTOR * d_mm + PX_HEADROOM - det["delta_bound"] = delta_mm + DELTA_HEADROOM + det["delta_bound"] = DELTA_FACTOR * delta_mm + DELTA_HEADROOM for n in det["split_nearest"]: n["within"] = n["px"] <= det["px_bound"] and n["delta"] <= det["delta_bound"] if n["px"] > det["px_bound"]: @@ -665,9 +677,9 @@ def gate3(out, ca, extras=()): d_mm, PX_HEADROOM, det["px_bound"])) if n["delta"] > det["delta_bound"]: reasons.append("%s: %s's per-channel delta to its nearest monolith pass (%s, %d px) is %d > " - "Delta_mm %d + %d (= %d)" % (RULING_62, n["pass"], n["nearest"], n["px"], - n["delta"], delta_mm, DELTA_HEADROOM, - det["delta_bound"])) + "%.2f x Delta_mm %d + %d (= %.2f)" % ( + RULING_62, n["pass"], n["nearest"], n["px"], n["delta"], DELTA_FACTOR, + delta_mm, DELTA_HEADROOM, det["delta_bound"])) det["within"] = all(n["within"] for n in det["split_nearest"]) # The RUN_AHEAD=0 control arm: required on every case (CONTRACT-P7 7.2 "RUN_AHEAD=0 对照臂一遍 # 记录"), and it must be that arm; its picture is recorded, never gating. @@ -756,6 +768,9 @@ def gate3(out, ca, extras=()): overall = "INCOMPLETE" rule_of = [(e["case"], e["monolith_determinism"]["rule"]) for e in results] return {"cases": results, "excluded": excluded, "repeat": repeat, "monolith_repeat": mono_repeat, + "rule_62": {"min_passes": MONOLITH_RULE_PASSES, "expected_passes": MONOLITH_EXPECTED_PASSES, + "spread_factor": SPREAD_FACTOR, "px_headroom": PX_HEADROOM, "delta_factor": DELTA_FACTOR, + "delta_headroom": DELTA_HEADROOM}, "determinism": {"distributional": [c for c, r in rule_of if r == "distributional"], "strict": [c for c, r in rule_of if r == "strict"], "strict_under_3_passes": [c for c, r in rule_of if r.startswith("strict (")]}, @@ -771,8 +786,10 @@ def gate3(out, ca, extras=()): def print_gate3(g3): - print("== GATE 3 (DirectVulkan x pbuffer; split repeat=%d, contract >= %d; monolith repeat=%d; " - "tol |ssim-monolith| <= %.4f) ==" % (g3["repeat"], CONTRACT_REPEATS, g3["monolith_repeat"], TOL)) + print("== GATE 3 (DirectVulkan x pbuffer; split repeat=%d, contract >= %d; monolith repeat=%d (30-gate3.sh runs " + "x%d, the %s rule needs >= %d); tol |ssim-monolith| <= %.4f) ==" % ( + g3["repeat"], CONTRACT_REPEATS, g3["monolith_repeat"], MONOLITH_EXPECTED_PASSES, RULING_62, + MONOLITH_RULE_PASSES, TOL)) s = g3["session"] print("-- session: boot_id %s; %d record(s) stamped; %s" % (s["session_boot_id"] or "?", s["records"], s["reboot_clean"] or "no session/reboot-clean.txt")) @@ -813,19 +830,22 @@ def print_gate3(g3): "yes" if e["split_eq_monolith"] else ("i==s" if e["inproc_eq_spawn"] else "no"))) det = e["monolith_determinism"] if det["rule"] == "distributional": - print("%-*s nondeterministic monolith (%d distinct / %d passes) -> distributional check (%s)" % ( - w, "", det["distinct"], det["passes"], RULING_62)) + print("%-*s nondeterministic monolith (%d distinct / %d passes) -> distributional check (%s)%s" % ( + w, "", det["distinct"], det["passes"], RULING_62, + " [%d < the %d monolith passes 30-gate3.sh runs; the rule needs >= %d]" % ( + det["passes"], MONOLITH_EXPECTED_PASSES, MONOLITH_RULE_PASSES) + if det["passes"] < MONOLITH_EXPECTED_PASSES else "")) if "error" not in det: worst = max(det["split_nearest"], key=lambda n: (n["px"], n["delta"]), default=None) print("%-*s monolith leave-one-out nearest: D_mm %d px / Delta_mm %d (%d passes); split-to-nearest-" - "monolith max %d px / delta %d (%d passes%s); bound %.2f x %d + %d = %.2f px / %d + %d = %d: %s; " - "max |ssim - monolith| %s; decoder %s" % ( + "monolith max %d px / delta %d (%d passes%s); bound %.2f x %d + %d = %.2f px / %.2f x %d + %d = " + "%.2f: %s; max |ssim - monolith| %s; decoder %s" % ( w, "", det["mono_nearest_px_max"], det["mono_nearest_delta_max"], len(det["mono_nearest"]), det["split_nearest_px_max"], det["split_nearest_delta_max"], len(det["split_nearest"]), "; most px: %s -> %s %d px / delta %d" % (worst["pass"], worst["nearest"], worst["px"], worst["delta"]) if worst else "", SPREAD_FACTOR, det["mono_nearest_px_max"], PX_HEADROOM, det["px_bound"], - det["mono_nearest_delta_max"], DELTA_HEADROOM, det["delta_bound"], + DELTA_FACTOR, det["mono_nearest_delta_max"], DELTA_HEADROOM, det["delta_bound"], "within" if det["within"] else "OUTSIDE", "-" if det["max_dssim"] is None else "%.1e" % det["max_dssim"], det["decoder"])) for reason in e["reasons"]: diff --git a/docs/Disaggregated/notes/p7/device-window-2/tools/window.sh b/docs/Disaggregated/notes/p7/device-window-2/tools/window.sh index 2f8489c9..42336d67 100755 --- a/docs/Disaggregated/notes/p7/device-window-2/tools/window.sh +++ b/docs/Disaggregated/notes/p7/device-window-2/tools/window.sh @@ -5,7 +5,7 @@ # 10-build.sh APK + host artefacts, stamp (default p7w7-) host only # 21-preflight.sh record found state, reboot-clean (unless --no-reboot), stay-on, pin # 20-install.sh adb install -r, on-device base.apk sha256 == built APK, dex2oat idle -# 30-gate3.sh 36 cases x {monolith x3, inproc x3, spawn x3, inproc RUN_AHEAD=0 x1} +# 30-gate3.sh 36 cases x {monolith x5, inproc x3, spawn x3, inproc RUN_AHEAD=0 x1} # 40-bsl-stats.sh bsl-esc-menu x {inproc, spawn} with MOBILEGL_PIPE_STATS + maps sampler # 50-cts-after.sh five KHR-GL46 blocks, inproc x DirectVulkan, AFTER lib deployed to mgcts # 60-reduce.py gate-3 verdict per case, bsl peaks, CTS delta table + new-crash list