From 5a844b1e507f6de8df003e1284d898c70076df9c Mon Sep 17 00:00:00 2001 From: Swung0x48 Date: Wed, 23 Sep 2026 11:22:07 -0400 Subject: [PATCH] [tools] (P7): window-2 reducer never passes an empty, unrateable or subset reading, and its rate gate is tested --- .../pre-14e1c8b9-cts/verdict-contract.txt | 11 +- .../tools/60-reduce-selftest.py | 240 ++++++++++++++++-- .../p7/device-window-2/tools/60-reduce.py | 210 +++++++++++---- 3 files changed, 384 insertions(+), 77 deletions(-) diff --git a/docs/Disaggregated/notes/p7/device-window-2/dry-run/pre-14e1c8b9-cts/verdict-contract.txt b/docs/Disaggregated/notes/p7/device-window-2/dry-run/pre-14e1c8b9-cts/verdict-contract.txt index 62abe771..72d7b918 100644 --- a/docs/Disaggregated/notes/p7/device-window-2/dry-run/pre-14e1c8b9-cts/verdict-contract.txt +++ b/docs/Disaggregated/notes/p7/device-window-2/dry-run/pre-14e1c8b9-cts/verdict-contract.txt @@ -1,11 +1,15 @@ # 60-reduce.py (contract formula, p7/devprep w2-reducer) on ~/w7/logs/devprep/w2/pre-14e1c8b9 - CTS AFTER dry run of # pipe head 14e1c8b9's arm64 lib; that run had no gate 3 (NOT-RUN below is expected). Pre-ruling reading: verdict.txt. +== GATE 3 == +GATE3 NOT-RUN (no gate3/cases.txt) + == CTS AFTER (inproc x DirectVulkan) vs BASE (monolith x DirectVulkan, p7w1) == gate (CONTRACT-P7 7.3): rate = Pass/(Pass+Fail) [NS, warnings W and crashes X are not in the denominator]; per block delta >= -0.5 pp and 0 NEW crash; all five blocks required. 'info' = the pre-ruling - Pass/(results-NS) delta, printed for comparison only, never gating. + Pass/(results-NS) delta (its rates on each block's 'info' line), for comparison only, never gating. block cases BASE P/F/NS/W/X rate AFTER P/F/NS/W/X rate delta pp (info) crash regr fixed verdict shader-image 125 63/6/56/0/0 91.304% 65/3/56/0/1 95.588% +4.284 (+2.899) 1 1 3 FAIL + info pre-ruling Pass/(results-NS): BASE 91.304% AFTER 94.203% delta +2.899 pp (not gating) ! 1 NEW crash(es) ! note: AFTER is 4.284 pp ABOVE BASE (not a regression; review the fixed list) NEW CRASH KHR-GL46.shader_image_load_store.incomplete_textures @@ -14,12 +18,15 @@ shader-image 125 63/6/56/0/0 91.304% 65/3/56/0/1 95.5 not->Pass KHR-GL46.shader_image_load_store.advanced-sync-textureUpdate not->Pass KHR-GL46.shader_image_load_store.non-layered_binding ssbo 124 124/0/0/0/0 100.000% 124/0/0/0/0 100.000% +0.000 (+0.000) 0 0 0 PASS + info pre-ruling Pass/(results-NS): BASE 100.000% AFTER 100.000% delta +0.000 pp (not gating) dsa 371 368/2/0/1/0 99.459% 366/3/0/1/1 99.187% -0.272 (-0.539) 1 2 0 FAIL + info pre-ruling Pass/(results-NS): BASE 99.191% AFTER 98.652% delta -0.539 pp (not gating) ! 1 NEW crash(es) NEW CRASH KHR-GL46.direct_state_access.renderbuffers_storage Pass->not KHR-GL46.direct_state_access.renderbuffers_storage Pass->not KHR-GL46.direct_state_access.renderbuffers_storage_multisample texture 1054 815/18/221/0/0 97.839% 823/10/221/0/0 98.800% +0.960 (+0.960) 0 1 9 PASS + info pre-ruling Pass/(results-NS): BASE 97.839% AFTER 98.800% delta +0.960 pp (not gating) ! note: AFTER is 0.960 pp ABOVE BASE (not a regression; review the fixed list) Pass->not KHR-GL46.texture_border_clamp.Texture2DArrayCompressed not->Pass KHR-GL46.texture_barrier.disjoint-texels @@ -32,6 +39,7 @@ texture 1054 815/18/221/0/0 97.839% 823/10/221/0/0 98.8 not->Pass KHR-GL46.texture_swizzle.functional_format_idx_8_target_idx_7 not->Pass KHR-GL46.texture_swizzle.functional_format_idx_8_target_idx_8 packed-pixels 4732 4732/0/0/0/0 100.000% 4732/0/0/0/0 100.000% +0.000 (+0.000) 0 0 0 PASS + info pre-ruling Pass/(results-NS): BASE 100.000% AFTER 100.000% delta +0.000 pp (not gating) ubo unrun (not in this glcts): the device glcts has no GTF module (ID-P7-16); recorded, not gated (GTF-GL46.gtf31.GL3Tests.uniform_buffer_object*) CTS FAIL @@ -48,3 +56,4 @@ CTS FAIL restore 11s 2026-09-23T10:08:36 -> 2026-09-23T10:08:47 WINDOW2 GATE3 NOT-RUN | CTS FAIL -> exit 1 +reduce-rc=1 diff --git a/docs/Disaggregated/notes/p7/device-window-2/tools/60-reduce-selftest.py b/docs/Disaggregated/notes/p7/device-window-2/tools/60-reduce-selftest.py index fc9a6ca6..65639a2f 100755 --- a/docs/Disaggregated/notes/p7/device-window-2/tools/60-reduce-selftest.py +++ b/docs/Disaggregated/notes/p7/device-window-2/tools/60-reduce-selftest.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -"""60-reduce-selftest.py [--tools TREE] [--dry-run-dir DIR] [--keep] +"""60-reduce-selftest.py [--tools TREE] [--dry-run-dir DIR] [--reducer PY] [--keep] Self-test of 60-reduce.py: builds canned window output directories in a temp dir and runs the reducer on each exactly as window.sh does (a subprocess, --logroot --tools), then checks -the printed verdict, verdict.json and the exit status. The scenarios: +the printed verdict, verdict.json and the exit status. Gate 3 is canned at the gate's own +denominator (36 cases x monolith/inproc x3/spawn x3/ra0, one boot_id, reboot-clean) unless a +scenario says otherwise. The scenarios: - control gate 3 (2 cases x monolith/inproc x3/spawn x3/ra0, one boot_id) and all five CTS - blocks equal to $BASE -> GATE3 PASS, CTS PASS, exit 0 + control gate 3 and all five CTS blocks equal to $BASE -> GATE3 PASS, CTS PASS, exit 0 missing-block ssbo never finished, dsa's report is empty, texture's has no results; and a window with no cts/ at all -> CTS INCOMPLETE, exit != 0 boot-id-change a spawn pair's .done / one repeat's boot_id.txt names another boot; a .done @@ -18,15 +19,33 @@ the printed verdict, verdict.json and the exit status. The scenarios: -> WARN line, GATE3 still PASS one inproc repeat differs from its siblings -> FAIL (within-arm identity) unfinished a pair without .done; a repeat without an actual PNG -> GATE3 INCOMPLETE + rate-gate the contract rate alone decides (no crash anywhere): k $BASE-Pass dsa cases + turn Fail. k=1: 367/370, -0.270 pp -> PASS, exit 0; k=2: -0.541 pp -> FAIL on + the rate; k=4: 364/6/0/1/0, -1.081 pp -> FAIL, CTS FAIL + vacuous no verdict from nothing: the caselist a run names is gone (the dry run's AFTER + data, two new crashes) -> every block MISSING, CTS INCOMPLETE - and with the + caselist copy the window keeps, the real reading (CTS FAIL); no caselist at all / + an empty one -> MISSING; every AFTER result NotSupported -> every block FAIL + not-a-gate a pass that is not the gate's is never PASS / exit 0: 2 of 36 cases -> GATE3 + PASS-SUBSET; not reboot-clean -> GATE3 PASS-NOT-REBOOT-CLEAN; a --limit CTS + block -> CTS PASS-SUBSET; each exit != 0 + warnings information lines: a case $BASE lacks (AFTER Fail) stays out of both rates + (texture delta 0) with a WARN; a Pass that became NotSupported -> WARN on the NS + count, block still PASS + check-block 60-reduce.py --check-block (50-cts-after.sh's test before a block's .done): + complete -> 0; unrun.txt non-empty / a case without result / caselist gone / + empty report -> 3 dry-run the pre-14e1c8b9 CTS dry run rebuilt from $BASE + its recorded differences (60-reduce-selftest-pre14e1c8b9.json): contract rate Pass/(Pass+Fail) gives dsa -0.272 pp (the pre-ruling Pass/(results-NS) -0.539 pp is printed as info - only), shader-image +4.284, texture +0.960, ssbo/packed-pixels 0; dsa and - shader-image FAIL on one NEW crash each, not on the rate -> CTS FAIL + only, with its rates 99.191% -> 98.652%), shader-image +4.284, texture +0.960, + ssbo/packed-pixels 0; dsa and shader-image FAIL on one NEW crash each, not on + the rate -> CTS FAIL When --dry-run-dir (default ~/w7/logs/devprep/w2/pre-14e1c8b9) exists, the reducer also runs on that real output (verdict JSON to the temp dir) and must give the same CTS numbers. TREE (compare_actuals.py + device-window-1/CTS-base + tools/cts/caselists) defaults to the MobileGL tree this file sits in, else $W2_TOOLS_OVERRIDE, else $W2_PIPE / ~/w7/pipe (only read). +--reducer runs the checks against another 60-reduce.py (red-once of an older reducer). Exit 0 when every check holds. """ import argparse @@ -47,8 +66,10 @@ CTS_BASE_REL = "docs/Disaggregated/notes/p7/device-window-1/CTS-base" BLOCKS = ("shader-image", "ssbo", "dsa", "texture", "packed-pixels") BOOT = "11111111-2222-3333-4444-555555555555" OTHER_BOOT = "99999999-8888-7777-6666-555555555555" -CASES = ("caseA-in-world", "caseB-main-menu") +# The gate's own denominator (CONTRACT-P7 7.2): 36 cases; the first two are the ones scenarios poke. +CASES = ("caseA-in-world", "caseB-main-menu") + tuple("case%02d-filler" % i for i in range(3, 37)) REPEAT = 3 +GONE = "/nonexistent/p7w7-tree-deadbeef/tools/cts/caselists/p7-%s-gl46.txt" ARMS = (("monolith", 1), ("inproc", REPEAT), ("spawn", REPEAT), ("inproc-ra0", 1)) DSA_STORAGE = "KHR-GL46.direct_state_access.renderbuffers_storage" SI_INCOMPLETE = "KHR-GL46.shader_image_load_store.incomplete_textures" @@ -101,16 +122,18 @@ class Canned: def png(self, value): return self.ca._png_bytes(8, 8, self.ca._solid(8, 8, value)) - def gate3(self, out, picture=None, boot=BOOT): + def gate3(self, out, picture=None, boot=BOOT, cases=CASES, reboot_clean=True): picture = picture or {} g = out / "gate3" - write(g / "cases.txt", "\n".join(CASES) + "\n") + write(g / "cases.txt", "\n".join(cases) + "\n") write(g / "excluded.txt", "x-red-on-monolith-case\n") write(g / "arms.txt", "arms=monolith inproc spawn inproc-ra0\nrepeat=%d\n" % REPEAT) write(out / "session/boot-id.txt", boot + "\n") - write(out / "session/reboot-clean.txt", "reboot-clean: 00000000-0000-0000-0000-000000000000 -> %s\n" % boot) + write(out / "session/reboot-clean.txt", + ("reboot-clean: 00000000-0000-0000-0000-000000000000 -> %s\n" % boot) if reboot_clean + else ("NOT reboot-clean (no --reboot): boot_id %s, up 5000.00s\n" % boot)) for arm, n in ARMS: - for case in CASES: + for case in cases: name = case + "-DirectVulkan" for i in range(1, n + 1): rep = g / "archive" / arm / name / ("repeat-%02d" % i) @@ -129,9 +152,18 @@ class Canned: "finished=2026-09-23T12:00:00-04:00\n" % (n, boot, boot)) return g - def cts(self, out, differences=None, drop=(), empty=(), no_results=()): - differences = differences or {} - base_dir = self.tree / CTS_BASE_REL + def base(self, b): + return dict(json.loads((self.tree / CTS_BASE_REL / ("report-%s.json" % b)).read_text(encoding="utf-8"))["results"]) + + def caselist(self, b): + return self.tree / "tools/cts/caselists" / ("p7-%s-gl46.txt" % b) + + def cts(self, out, differences=None, drop=(), empty=(), no_results=(), copy=True, named=None, lists=None): + """50-cts-after.sh's output for the five blocks: AFTER = $BASE + `differences` per block. + copy: keep runs//caselist.txt as 50-cts-after.sh does; named: b -> what caselist.path + says (None = the tools tree's list, '' = no caselist.path); lists: b -> the cases the run + was asked to run (a --limit subset, or a list with cases $BASE lacks).""" + differences, lists = differences or {}, lists or {} skip = set(read_list(self.tree / "tools/cts/caselists/p7-skip.txt")) c = out / "cts" c.mkdir(parents=True, exist_ok=True) @@ -139,15 +171,23 @@ class Canned: for b in BLOCKS: if b in drop: continue - caselist = self.tree / "tools/cts/caselists" / ("p7-%s-gl46.txt" % b) + caselist = self.caselist(b) run = c / "runs" / b - write(run / "caselist.path", str(caselist) + "\n") + results = self.base(b) + if b in lists: + caselist = c / "caselists" / ("p7-%s-gl46.canned.txt" % b) + write(caselist, "\n".join(lists[b]) + "\n") + results = {k: v for k, v in results.items() if k in set(lists[b])} + path = named(b) if named else str(caselist) + if path: + write(run / "caselist.path", path + "\n") + if copy: + write(run / "caselist.txt", caselist.read_bytes()) skipped = [x for x in read_list(caselist) if x in skip] if skipped: write(run / "skipped.txt", "\n".join(skipped) + "\n") write(run / "unrun.txt", "") write(run / "hung.txt", "") - results = dict(json.loads((base_dir / ("report-%s.json" % b)).read_text(encoding="utf-8"))["results"]) results.update(differences.get(b, {})) rebuilt[b] = results write(run / "crashed.txt", "".join(k + "\n" for k, v in sorted(results.items()) if v == "Crash")) @@ -162,11 +202,11 @@ class Canned: "counts": dict(Counter(results.values())), "results": results}, indent=1)) return rebuilt - def window(self, stamp, gate3=True, cts=True, **cts_args): + def window(self, stamp, gate3=True, cts=True, gate3_args=None, **cts_args): out = self.root / stamp out.mkdir(parents=True) if gate3: - self.gate3(out) + self.gate3(out, **(gate3_args or {})) else: write(out / "session/boot-id.txt", BOOT + "\n") if cts: @@ -184,6 +224,12 @@ def run_reducer(logroot, stamp, tree, json_out=None): return p.returncode, p.stdout, verdict +def run_check_block(run, report): + p = subprocess.run([sys.executable, str(REDUCER), "--check-block", str(run), str(report)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) + return p.returncode, p.stdout + + class Checks: def __init__(self): self.total, self.failed = 0, [] @@ -221,7 +267,8 @@ def scenario_control(k, cn, tree): cn.window("control") rc, text, v = run_reducer(cn.root, "control", tree) k.expect("exit 0", rc == 0, "rc=%d\n%s" % (rc, text)) - k.expect("GATE3 PASS", v.get("gate3", {}).get("overall") == "PASS", text) + k.expect("GATE3 PASS 36/36", v.get("gate3", {}).get("overall") == "PASS" and "GATE3 PASS: 36/36 cases PASS\n" in text, + text) k.expect("CTS PASS, all five blocks", v.get("cts", {}).get("overall") == "PASS" and [b["block"] for b in v["cts"]["blocks"]] == list(BLOCKS), text) k.expect("every block delta 0.000 pp", all(r3(block(v, b).get("delta_pp")) == 0.0 for b in BLOCKS)) @@ -300,7 +347,7 @@ def scenario_monolith_arm(k, cn, tree): e.get("verdict") == "FAIL" and v["gate3"]["overall"] == "FAIL" and rc != 0 and any("NOT proven monolith" in r for r in e.get("reasons", [])), e) k.expect("the other case still PASS", case(v, CASES[1]).get("verdict") == "PASS") - k.expect("arm identity line counts 1/2 monolith", "monolith 1/2 repeat(s)" in text, text) + k.expect("arm identity line counts 35/36 monolith", "monolith 35/36 repeat(s)" in text, text) out = cn.window("mono-nolog") (out / "gate3/archive/monolith" / (CASES[1] + "-DirectVulkan") / "repeat-01" / "mobilegl.log").unlink() @@ -357,6 +404,143 @@ def scenario_unfinished(k, cn, tree): and any("no *-actual.png" in r for r in e.get("reasons", [])), e) +def pass_cases(cn, b): + """$BASE-Pass cases of block b's caselist (not skipped), in caselist order.""" + base = cn.base(b) + skip = set(read_list(cn.tree / "tools/cts/caselists/p7-skip.txt")) + return [x for x in read_list(cn.caselist(b)) if x not in skip and base.get(x) == "Pass"] + + +def dryrun_differences(): + fixture = json.loads(FIXTURE.read_text(encoding="utf-8")) + return fixture, {b: fixture["blocks"][b]["after_differences"] for b in BLOCKS} + + +def scenario_rate_gate(k, cn, tree): + print("[rate-gate] the contract rate alone decides a block (no crash anywhere)") + flips = pass_cases(cn, "dsa") + cn.window("rate-1", differences={"dsa": {x: "Fail" for x in flips[:1]}}) + rc, text, v = run_reducer(cn.root, "rate-1", tree) + d = block(v, "dsa") + k.expect("1 dsa Pass->Fail: 367/370 = -0.270 pp -> dsa PASS, CTS PASS, window exit 0", + r3(d.get("delta_pp")) == -0.27 and d.get("verdict") == "PASS" and v.get("cts", {}).get("overall") == "PASS" + and rc == 0, (d.get("delta_pp"), d.get("verdict"), rc, text[-300:])) + cn.window("rate-2", gate3=False, differences={"dsa": {x: "Fail" for x in flips[:2]}}) + rc, text, v = run_reducer(cn.root, "rate-2", tree) + d = block(v, "dsa") + k.expect("2 dsa Pass->Fail: -0.541 pp < -0.5 pp -> dsa FAIL on the rate alone (0 new crash)", + r3(d.get("delta_pp")) == -0.541 and d.get("verdict") == "FAIL" and d.get("new_crash") == [] + and d.get("reasons") == ["delta -0.541 pp < -0.5 pp (Pass/(Pass+Fail))"], + (d.get("delta_pp"), d.get("verdict"), d.get("reasons"))) + cn.window("rate-4", gate3=False, differences={"dsa": {x: "Fail" for x in flips[:4]}}) + rc, text, v = run_reducer(cn.root, "rate-4", tree) + d = block(v, "dsa") + k.expect("4 dsa Pass->Fail: AFTER 364/6/0/1/0, -1.081 pp -> dsa FAIL, CTS FAIL, exit != 0", + [d.get("after", {}).get(x) for x in ("pass", "fail", "ns", "warn", "crash")] == [364, 6, 0, 1, 0] + and r3(d.get("delta_pp")) == -1.081 and d.get("verdict") == "FAIL" and d.get("new_crash") == [] + and v.get("cts", {}).get("overall") == "FAIL" and "\nCTS FAIL" in text and rc != 0, + (d.get("after"), d.get("delta_pp"), d.get("verdict"), rc)) + + +def scenario_vacuous(k, cn, tree): + print("[vacuous] a block with nothing to compare is never PASS") + _, diffs = dryrun_differences() + cn.window("gone", differences=diffs, copy=False, named=lambda b: GONE % b) + rc, text, v = run_reducer(cn.root, "gone", tree) + k.expect("caselist.path names a caselist that is gone (no copy), dry-run AFTER data -> every block MISSING, " + "CTS INCOMPLETE, exit != 0", + rc != 0 and v.get("cts", {}).get("overall") == "INCOMPLETE" and "CTS PASS" not in text + and all(block(v, b).get("verdict") == "MISSING" and "is gone" in " ".join(block(v, b).get("reasons", [])) + for b in BLOCKS), text) + cn.window("gone-copy", gate3=False, differences=diffs, named=lambda b: GONE % b) + rc, text, v = run_reducer(cn.root, "gone-copy", tree) + d = block(v, "dsa") + k.expect("...with the window's caselist.txt copy: the real reading (dsa -0.272 pp, 2 new crashes, CTS FAIL)", + r3(d.get("delta_pp")) == -0.272 and d.get("caselist", "").endswith("runs/dsa/caselist.txt") + and v.get("cts", {}).get("overall") == "FAIL", (d.get("delta_pp"), d.get("caselist"), v.get("cts", {}).get("overall"))) + cn.window("no-caselist", gate3=False, copy=False, named=lambda b: "") + rc, text, v = run_reducer(cn.root, "no-caselist", tree) + k.expect("no caselist.txt and no caselist.path -> every block MISSING ('unknown'), CTS INCOMPLETE", + rc != 0 and v.get("cts", {}).get("overall") == "INCOMPLETE" + and all("unknown" in " ".join(block(v, b).get("reasons", [])) for b in BLOCKS), text) + out = cn.window("empty-caselist", gate3=False) + write(out / "cts/runs/ssbo/caselist.txt", "# nothing asked\n") + rc, text, v = run_reducer(cn.root, "empty-caselist", tree) + k.expect("an empty caselist -> ssbo MISSING ('holds no case'), CTS INCOMPLETE", + block(v, "ssbo").get("verdict") == "MISSING" and "holds no case" in " ".join(block(v, "ssbo").get("reasons", [])) + and v.get("cts", {}).get("overall") == "INCOMPLETE", block(v, "ssbo")) + ns = {b: {x: "NotSupported" for x in cn.base(b)} for b in BLOCKS} + cn.window("all-ns", differences=ns) + rc, text, v = run_reducer(cn.root, "all-ns", tree) + k.expect("every AFTER result NotSupported -> every block FAIL ('AFTER has no Pass/Fail'), CTS FAIL, exit != 0", + rc != 0 and v.get("cts", {}).get("overall") == "FAIL" and "CTS PASS" not in text + and all(block(v, b).get("verdict") == "FAIL" and block(v, b).get("delta_pp") is None + and any("AFTER has no Pass/Fail" in r for r in block(v, b).get("reasons", [])) for b in BLOCKS), text) + k.expect("...and a WARN on each block's NotSupported count", all(("WARN %s: NotSupported $BASE" % b) in text for b in BLOCKS), text) + + +def scenario_not_a_gate(k, cn, tree): + print("[not-a-gate] a pass that is not the gate's own reading never exits 0") + cn.window("subset-g3", gate3_args={"cases": CASES[:2]}) + rc, text, v = run_reducer(cn.root, "subset-g3", tree) + k.expect("2 of 36 gate-3 cases, all passing -> GATE3 PASS-SUBSET, exit != 0", + v.get("gate3", {}).get("overall") == "PASS-SUBSET" and "GATE3 PASS-SUBSET: 2/2 cases PASS" in text + and "WINDOW2 GATE3 PASS-SUBSET | CTS PASS -> exit 1" in text and rc != 0, text[-500:]) + cn.window("not-clean", gate3_args={"reboot_clean": False}) + rc, text, v = run_reducer(cn.root, "not-clean", tree) + k.expect("36/36 in a session that is not reboot-clean -> GATE3 PASS-NOT-REBOOT-CLEAN, exit != 0", + v.get("gate3", {}).get("overall") == "PASS-NOT-REBOOT-CLEAN" and rc != 0 + and "session not reboot-clean" in text, text[-500:]) + first20 = read_list(cn.caselist("dsa"))[:20] + cn.window("subset-cts", lists={"dsa": first20}) + rc, text, v = run_reducer(cn.root, "subset-cts", tree) + k.expect("a --limit 20 dsa block, all passing -> CTS PASS-SUBSET (dsa marked), GATE3 PASS, exit != 0", + v.get("cts", {}).get("overall") == "PASS-SUBSET" and block(v, "dsa").get("subset") is True + and block(v, "dsa").get("cases") == 20 and v.get("gate3", {}).get("overall") == "PASS" and rc != 0 + and "SUBSET run: dsa" in text, text[-500:]) + + +def scenario_warnings(k, cn, tree): + print("[warnings] like-for-like rates and the NotSupported count are reported, not gated") + extra = "KHR-GL46.texture_selftest.not_in_base" + tex = read_list(cn.caselist("texture")) + [extra] + ssbo_ns = pass_cases(cn, "ssbo")[0] + cn.window("warn", gate3=False, lists={"texture": tex}, + differences={"texture": {extra: "Fail"}, "ssbo": {ssbo_ns: "NotSupported"}}) + rc, text, v = run_reducer(cn.root, "warn", tree) + t, s = block(v, "texture"), block(v, "ssbo") + k.expect("a case $BASE lacks (AFTER Fail) stays out of both rates: texture delta 0.000, PASS, compared = cases - 1", + r3(t.get("delta_pp")) == 0.0 and t.get("verdict") == "PASS" and t.get("compared") == t.get("cases", 0) - 1 + and t.get("missing_in_base") == [extra], (t.get("delta_pp"), t.get("verdict"), t.get("compared"), t.get("cases"))) + k.expect("...with a WARN line naming it", "WARN texture: 1 case(s) have no $BASE result (e.g. %s)" % extra in text, text) + k.expect("an ssbo Pass that became NotSupported: rate unchanged (PASS), WARN on the NS count, in Pass->not", + s.get("verdict") == "PASS" and r3(s.get("delta_pp")) == 0.0 and ssbo_ns in s.get("regressed", []) + and "WARN ssbo: NotSupported $BASE 0 -> AFTER 1" in text, (s.get("verdict"), s.get("delta_pp"), text[-400:])) + + +def scenario_check_block(k, cn, tree): + print("[check-block] 50-cts-after.sh's completeness test before a block's .done") + out = cn.window("cb", gate3=False) + c = out / "cts" + rc, text = run_check_block(c / "runs/dsa", c / "report-dsa.json") + k.expect("complete dsa block -> 0", rc == 0 and "complete: 371/371" in text, (rc, text)) + write(c / "runs/ssbo/unrun.txt", read_list(cn.caselist("ssbo"))[-1] + "\n") + rc, text = run_check_block(c / "runs/ssbo", c / "report-ssbo.json") + k.expect("unrun.txt non-empty -> 3", rc == 3 and "unrun.txt 1" in text, (rc, text)) + report = json.loads((c / "report-texture.json").read_text(encoding="utf-8")) + report["results"].pop(pass_cases(cn, "texture")[0]) + write(c / "report-texture.json", json.dumps(report)) + rc, text = run_check_block(c / "runs/texture", c / "report-texture.json") + k.expect("a case without a result -> 3", rc == 3 and "1 case(s) without an AFTER result" in text, (rc, text)) + (c / "runs/shader-image/caselist.txt").unlink() + write(c / "runs/shader-image/caselist.path", GONE % "shader-image" + "\n") + rc, text = run_check_block(c / "runs/shader-image", c / "report-shader-image.json") + k.expect("the caselist gone -> 3", rc == 3 and "is gone" in text, (rc, text)) + write(c / "report-packed-pixels.json", "") + rc, text = run_check_block(c / "runs/packed-pixels", c / "report-packed-pixels.json") + k.expect("an empty report -> 3", rc == 3 and "is empty" in text, (rc, text)) + + def check_dryrun_numbers(k, v, text, label): dsa, si, tex = block(v, "dsa"), block(v, "shader-image"), block(v, "texture") k.expect("%s: dsa contract delta -0.272 pp" % label, r3(dsa.get("delta_pp")) == -0.272, dsa.get("delta_pp")) @@ -365,6 +549,8 @@ def check_dryrun_numbers(k, v, text, label): if dsa.get("base") else False, dsa) k.expect("%s: dsa pre-ruling delta -0.539 pp kept as info" % label, r3(dsa.get("delta_pp_results_info")) == -0.539 and "(-0.539)" in text, dsa.get("delta_pp_results_info")) + k.expect("%s: dsa pre-ruling rates printed (info line 99.191%% -> 98.652%%)" % label, + "info pre-ruling Pass/(results-NS): BASE 99.191% AFTER 98.652% delta -0.539 pp (not gating)" in text, text) k.expect("%s: dsa FAIL on the new crash only, not on the rate" % label, dsa.get("verdict") == "FAIL" and dsa.get("reasons") == ["1 NEW crash(es)"] and dsa.get("new_crash") == [DSA_STORAGE], dsa.get("reasons")) @@ -384,8 +570,7 @@ def check_dryrun_numbers(k, v, text, label): def scenario_dryrun(k, cn, tree, dry_dir): print("[dry-run] contract formula on the pre-14e1c8b9 dry-run data") - fixture = json.loads(FIXTURE.read_text(encoding="utf-8")) - diffs = {b: fixture["blocks"][b]["after_differences"] for b in BLOCKS} + fixture, diffs = dryrun_differences() out = cn.root / "dryrun" out.mkdir() rebuilt = cn.cts(out, differences=diffs) @@ -413,17 +598,21 @@ def scenario_dryrun(k, cn, tree, dry_dir): def main(): + global REDUCER ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--tools", help="MobileGL tree (default: the tree holding this file, else $W2_TOOLS_OVERRIDE)") ap.add_argument("--dry-run-dir", default=str(Path.home() / "w7/logs/devprep/w2/pre-14e1c8b9")) + ap.add_argument("--reducer", default=str(REDUCER), help="the 60-reduce.py under test (default: beside this file)") ap.add_argument("--keep", action="store_true", help="keep the canned directories") args = ap.parse_args() + REDUCER = Path(args.reducer).resolve() tree = find_tree(args.tools) if tree is None: print("no MobileGL tree with tools/trace_replay/compare_actuals.py and %s: pass --tools" % CTS_BASE_REL, file=sys.stderr) return 2 sys.path.insert(0, str(tree / "tools" / "trace_replay")) + sys.dont_write_bytecode = True # the tree is only read (the working copy falls back to ~/w7/pipe) import compare_actuals as ca root = Path(tempfile.mkdtemp(prefix="w2-reduce-selftest-")) print("60-reduce self-test: reducer %s, tree %s, canned windows in %s" % (REDUCER, tree, root)) @@ -436,6 +625,11 @@ def main(): scenario_monolith_arm(k, cn, tree) scenario_cross_arm(k, cn, tree) scenario_unfinished(k, cn, tree) + scenario_rate_gate(k, cn, tree) + scenario_vacuous(k, cn, tree) + scenario_not_a_gate(k, cn, tree) + scenario_warnings(k, cn, tree) + scenario_check_block(k, cn, tree) scenario_dryrun(k, cn, tree, args.dry_run_dir) finally: if not args.keep: diff --git a/docs/Disaggregated/notes/p7/device-window-2/tools/60-reduce.py b/docs/Disaggregated/notes/p7/device-window-2/tools/60-reduce.py index a28d20e7..4fba2353 100755 --- a/docs/Disaggregated/notes/p7/device-window-2/tools/60-reduce.py +++ b/docs/Disaggregated/notes/p7/device-window-2/tools/60-reduce.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """60-reduce.py [--logroot DIR] [--tools TREE] [--json OUT] + 60-reduce.py --check-block RUN_DIR REPORT_JSON Device window 2 verdicts, read from the window's own output directory (/). @@ -18,22 +19,36 @@ GATE 3 (CONTRACT-P7 7.2), per case of gate3/cases.txt: * OpenRA: 1.000000 and mismatch 0 on every repeat of both split arms AND on monolith; * cross-arm picture identity (inproc vs spawn vs monolith) is REPORTED per case and a mismatch prints a WARN line - information, not a gate (ruling: integrator 2026-09-23); - * inproc-ra0 (RUN_AHEAD=0): recorded (ssim, sha, run-ahead=0 proof), never gating. + * inproc-ra0 (RUN_AHEAD=0): recorded (ssim, sha, run-ahead=0 proof), never gating; + * a reading that passes but is not the gate's - a denominator other than 36, or a session that + is not reboot-clean - is GATE3 PASS-SUBSET / PASS-NOT-REBOOT-CLEAN, never PASS. BSL STATS: per arm, maps-line / VmRSS / VmHWM peaks and the wbuf[] gauges (information). CTS AFTER (CONTRACT-P7 7.3), per block vs device-window-1/CTS-base/report-.json, over the cases the AFTER run was asked to run (so a --limit subset is compared like for like): * all five blocks (shader-image ssbo dsa texture packed-pixels) are REQUIRED, each with a - finished run (cts/runs//.done) and a non-empty cts/report-.json; a missing one - makes CTS INCOMPLETE. The UBO block (GTF-GL46 uniform_buffer_object) is recorded as - 'unrun (not in this glcts)': the device glcts carries no GTF module (ID-P7-16); + finished run (cts/runs//.done), a non-empty cts/report-.json with results, and + a readable, non-empty caselist (runs//caselist.txt, the copy 50-cts-after.sh keeps; + else the file caselist.path names); a missing one makes the block MISSING and CTS INCOMPLETE - + a block of 0 cases is never a reading. The UBO block (GTF-GL46 uniform_buffer_object) is + recorded as 'unrun (not in this glcts)': the device glcts carries no GTF module (ID-P7-16); * rate = Pass / (Pass + Fail), the contract formula: NotSupported, the warnings and the crashes - are NOT in the denominator; gate: delta >= -0.5 pp on this rate; + are NOT in the denominator; gate: delta >= -0.5 pp on this rate. Both rates are taken over the + cases that have a BASE and an AFTER result (a case BASE lacks prints a WARN, stays out of both + rates and still counts for new crashes). AFTER without a single Pass/Fail where BASE has some + is FAIL (the rate cannot be formed: every BASE Pass is gone); BASE without one is INCOMPLETE; * crashes (Crash/Timeout/InternalError/ResourceError/DeviceHang/Incomplete, plus hung.txt) are counted in their own column and are non-Pass; one that BASE does not already have is a NEW crash, a hard red of its own (a Fail that became a Crash is a new crash); - * information only, never gating: the pre-ruling rate Pass / (results - NotSupported), delta. -Exit status: 0 only when GATE 3 and CTS are both PASS; 1 otherwise (a section that did not run, -is INCOMPLETE / INVALID-SESSION, or FAILs); 2 on bad input. + * a block whose cases are not the tools tree's full p7--gl46.txt (a --limit run) makes an + all-PASS CTS reading 'PASS-SUBSET', never PASS; + * information only, never gating: the pre-ruling rate Pass / (results - NotSupported) of BASE + and AFTER and its delta, and a WARN when the NotSupported count moved. +Exit status: 0 only when GATE 3 is PASS (36/36, reboot-clean, one session) and CTS is PASS (five +full blocks); 1 otherwise (a section that did not run, a PASS-SUBSET / PASS-NOT-REBOOT-CLEAN +reading, INCOMPLETE / INVALID-SESSION, or FAIL); 2 on bad input. +--check-block RUN_DIR REPORT_JSON: the completeness test 50-cts-after.sh applies before it writes +a block's .done (the same code the verdict uses): exit 0 when the report has a result for every +non-skipped case of the run's caselist and unrun.txt is empty, 3 when not (the reason is printed). """ import argparse import hashlib @@ -45,6 +60,7 @@ from collections import Counter from pathlib import Path TOL = 0.0005 +GATE3_DENOMINATOR = 36 CTS_TOL_PP = 0.5 HARD = ("Crash", "Timeout", "InternalError", "ResourceError", "DeviceHang", "Incomplete") SPLIT_ARMS = ("inproc", "spawn") @@ -276,16 +292,29 @@ def gate3(out, ca): entry["reasons"] = incomplete + reasons results.append(entry) verdicts = [e["verdict"] for e in results] + # A reading that is not the gate's own (CONTRACT-P7 7.2: the 36-case denominator, one + # reboot-clean session) may pass, but never as PASS: exit 0 is reserved for the gate verdict. + not_a_gate = [] + if len(cases) != GATE3_DENOMINATOR: + not_a_gate.append("denominator is %d, not the gate's %d: a subset run is NOT a gate verdict" + % (len(cases), GATE3_DENOMINATOR)) + if not session["reboot_clean"].startswith("reboot-clean:"): + not_a_gate.append("session not reboot-clean: NOT a gate verdict") if not session["valid"]: overall = "INVALID-SESSION" elif verdicts and all(v == "PASS" for v in verdicts): overall = "PASS" + if len(cases) != GATE3_DENOMINATOR: + overall = "PASS-SUBSET" + elif not_a_gate: + overall = "PASS-NOT-REBOOT-CLEAN" elif "FAIL" in verdicts: overall = "FAIL" else: overall = "INCOMPLETE" return {"cases": results, "excluded": excluded, "repeat": repeat, "denominator": len(cases), - "overall": overall, "arms_present": arms_present, "session": session, "identity": identity} + "overall": overall, "not_a_gate": not_a_gate, "arms_present": arms_present, + "session": session, "identity": identity} def print_gate3(g3): @@ -334,12 +363,7 @@ def print_gate3(g3): print("-- excluded (red on monolith, CONTRACT-P7 7.1): " + (", ".join(g3["excluded"]) or "none")) n = len(g3["cases"]) npass = sum(e["verdict"] == "PASS" for e in g3["cases"]) - notes = [] - if n != 36: - notes.append("denominator is %d, not the gate's 36: a subset run is NOT a gate verdict" % n) - if not s["reboot_clean"].startswith("reboot-clean:"): - notes.append("session not reboot-clean: NOT a gate verdict") - print("GATE3 %s: %d/%d cases PASS%s" % (g3["overall"], npass, n, "".join(" (%s)" % x for x in notes))) + print("GATE3 %s: %d/%d cases PASS%s" % (g3["overall"], npass, n, "".join(" (%s)" % x for x in g3["not_a_gate"]))) # ------------------------------------------------------------------------------------------------ @@ -411,7 +435,49 @@ def load_results(path): return results, None -def cts_block(c, b, base_dir): +def caselist_of(run): + """-> (cases, source, None) for the cases a block's AFTER run was asked to run, or (None, + source, why) when that cannot be read. 50-cts-after.sh keeps a copy (runs//caselist.txt) + beside the path it ran (caselist.path), so the window's output reduces on its own after the + build tree that path names is gone; a named list that is gone or empty is NOT replaced by + the report's own cases (that would make a block of 0 - or of only what ran - look complete).""" + copy = run / "caselist.txt" + if copy.is_file(): + src = copy + elif (run / "caselist.path").is_file(): + src = Path((run / "caselist.path").read_text(encoding="utf-8", errors="replace").strip() or "(empty caselist.path)") + else: + return None, None, ("no runs/%s/caselist.txt or caselist.path: which cases the run was asked " + "to run is unknown" % run.name) + if not src.is_file(): + return None, src, "the caselist the run used is gone (%s) and runs/%s/caselist.txt holds no copy" % (src, run.name) + cases = read_list(src) + if not cases: + return None, src, "the caselist the run used (%s) holds no case" % src + return cases, src, None + + +def block_scope(run, after): + """The one completeness test of a finished block (the verdict below, and 50-cts-after.sh through + --check-block before it writes .done). -> (universe, source, missing, incomplete): universe = + the caselist minus run_cts's skipped.txt; `missing` says why there is no reading at all, + `incomplete` why the reading lacks cases.""" + cases, src, why = caselist_of(run) + if why: + return [], src, [why], [] + skipped = set(read_list(run / "skipped.txt")) + universe = [x for x in cases if x not in skipped] + if not universe: + return [], src, ["every case of %s is skipped: a block of 0 cases is no reading" % src], [] + unrun = read_list(run / "unrun.txt") + without = [x for x in universe if x not in after] + incomplete = [] + if unrun or without: + incomplete.append("%d case(s) without an AFTER result (unrun.txt %d)" % (len(without), len(unrun))) + return universe, src, [], incomplete + + +def cts_block(c, b, base_dir, canon_dir): run = c / "runs" / b if not c.is_dir(): return {"block": b, "verdict": "MISSING", "reasons": ["no cts/ directory: 50-cts-after.sh did not run"]} @@ -420,18 +486,17 @@ def cts_block(c, b, base_dir): after, why = load_results(c / ("report-%s.json" % b)) if why: return {"block": b, "verdict": "MISSING", "reasons": [why]} + universe, src, missing, incomplete = block_scope(run, after) + if missing: + return {"block": b, "verdict": "MISSING", "reasons": missing} base, why = load_results(base_dir / ("report-%s.json" % b)) if why: return {"block": b, "verdict": "NO-BASE", "reasons": ["$BASE: " + why + " in " + str(base_dir)]} - caselist = Path((run / "caselist.path").read_text().strip()) if (run / "caselist.path").is_file() else None - universe = read_list(caselist) if caselist else sorted(after) - skipped = set(read_list(run / "skipped.txt")) - universe = [x for x in universe if x not in skipped] unrun = read_list(run / "unrun.txt") hung = read_list(run / "hung.txt") - a_st = [after[x] for x in universe if x in after] - b_st = [base[x] for x in universe if x in base] - ra, rb = rate(a_st), rate(b_st) + # Like for like: both rates over the cases that have a BASE and an AFTER result. + both = [x for x in universe if x in after and x in base] + ra, rb = rate([after[x] for x in both]), rate([base[x] for x in both]) delta = None if ra["rate"] is None or rb["rate"] is None else (ra["rate"] - rb["rate"]) * 100.0 delta_results = (None if ra["results_rate"] is None or rb["results_rate"] is None else (ra["results_rate"] - rb["results_rate"]) * 100.0) @@ -440,35 +505,47 @@ def cts_block(c, b, base_dir): regress = sorted(x for x in universe if base.get(x) == "Pass" and after.get(x) not in (None, "Pass")) fixed = sorted(x for x in universe if base.get(x) not in (None, "Pass") and after.get(x) == "Pass") missing_base = sorted(x for x in universe if x not in base) - reasons = [] - verdict = "PASS" - if unrun or len(a_st) < len(universe): - verdict = "INCOMPLETE" - reasons.append("%d case(s) without an AFTER result (unrun.txt %d)" % (len(universe) - len(a_st), len(unrun))) + fail, notes, warnings = [], [], [] if new_crash: - verdict = "FAIL" - reasons.append("%d NEW crash(es)" % len(new_crash)) + fail.append("%d NEW crash(es)" % len(new_crash)) + if rb["rate"] is None: + incomplete.append("$BASE has no Pass/Fail over this block's %d case(s): no contract rate to compare against" + % len(both)) + elif ra["rate"] is None: + fail.append("AFTER has no Pass/Fail result (NS %d, W %d, X %d of %d): the contract rate cannot be formed " + "where $BASE's is %.3f%% - every $BASE Pass is lost" % ( + ra["ns"], ra["warn"], ra["crash"], ra["results"], 100 * rb["rate"])) if delta is not None and delta < -CTS_TOL_PP: - verdict = "FAIL" - reasons.append("delta %.3f pp < -%.1f pp (Pass/(Pass+Fail))" % (delta, CTS_TOL_PP)) + fail.append("delta %.3f pp < -%.1f pp (Pass/(Pass+Fail))" % (delta, CTS_TOL_PP)) if delta is not None and delta > CTS_TOL_PP: - reasons.append("note: AFTER is %.3f pp ABOVE BASE (not a regression; review the fixed list)" % delta) - return {"block": b, "cases": len(universe), "after": ra, "base": rb, "delta_pp": delta, - "delta_pp_results_info": delta_results, "new_crash": new_crash, "regressed": regress, - "fixed": fixed, "missing_in_base": missing_base, "unrun": len(unrun), "hung": hung, - "verdict": verdict, "reasons": reasons, "subset": bool(caselist and "first" in caselist.name)} + notes.append("note: AFTER is %.3f pp ABOVE BASE (not a regression; review the fixed list)" % delta) + if missing_base: + warnings.append("%d case(s) have no $BASE result (e.g. %s): left out of both rates; a crash among them " + "still counts as NEW" % (len(missing_base), missing_base[0])) + if ra["ns"] != rb["ns"]: + warnings.append("NotSupported $BASE %d -> AFTER %d (not in the contract rate; a Pass that became NS is in " + "the Pass->not list)" % (rb["ns"], ra["ns"])) + canonical = canon_dir / ("p7-%s-gl46.txt" % b) + subset = set(read_list(src)) != set(read_list(canonical)) + verdict = "FAIL" if fail else ("INCOMPLETE" if incomplete else "PASS") + return {"block": b, "cases": len(universe), "compared": len(both), "caselist": str(src), + "after": ra, "base": rb, "delta_pp": delta, "delta_pp_results_info": delta_results, + "new_crash": new_crash, "regressed": regress, "fixed": fixed, "missing_in_base": missing_base, + "unrun": len(unrun), "hung": hung, "verdict": verdict, "reasons": incomplete + fail + notes, + "warnings": warnings, "subset": subset} -def cts(out, base_dir): +def cts(out, base_dir, canon_dir): c = out / "cts" - blocks = [cts_block(c, b, base_dir) for b in CTS_BLOCKS] + blocks = [cts_block(c, b, base_dir, canon_dir) for b in CTS_BLOCKS] extra = sorted(p.stem[len("report-"):] for p in c.glob("report-*.json") if p.stem[len("report-"):] not in CTS_BLOCKS) if c.is_dir() else [] verdicts = [b["verdict"] for b in blocks] if "FAIL" in verdicts: overall = "FAIL" elif all(v == "PASS" for v in verdicts): - overall = "PASS" + # Passing on a --limit subset (or on a list that is not the tools tree's) is not gate 5. + overall = "PASS-SUBSET" if any(b.get("subset") for b in blocks) else "PASS" else: overall = "INCOMPLETE" return {"blocks": blocks, "overall": overall, "ubo": dict(UBO_BLOCK), "ignored_reports": extra, @@ -479,30 +556,34 @@ def print_cts(c): print("== CTS AFTER (inproc x DirectVulkan) vs BASE (monolith x DirectVulkan, p7w1) ==") print("gate (CONTRACT-P7 7.3): rate = Pass/(Pass+Fail) [NS, warnings W and crashes X are not in the denominator];") print(" per block delta >= -%.1f pp and 0 NEW crash; all five blocks required. 'info' = the pre-ruling" % CTS_TOL_PP) - print(" Pass/(results-NS) delta, printed for comparison only, never gating.") + print(" Pass/(results-NS) delta (its rates on each block's 'info' line), for comparison only, never gating.") print("%-14s %6s %-20s %8s %-20s %8s %9s %9s %5s %4s %5s %s" % ( "block", "cases", "BASE P/F/NS/W/X", "rate", "AFTER P/F/NS/W/X", "rate", "delta pp", "(info)", "crash", "regr", "fixed", "verdict")) + + def counts(r): + return "%d/%d/%d/%d/%d" % (r["pass"], r["fail"], r["ns"], r["warn"], r["crash"]) + + def pct(v): + return "%.3f%%" % (100 * v) if v is not None else "n/a" + + def pp(v): + return "%+.3f" % v if v is not None else "n/a" for b in c["blocks"]: if "after" not in b: print("%-14s %s: %s" % (b["block"], b["verdict"], "; ".join(b["reasons"]))) continue - - def counts(r): - return "%d/%d/%d/%d/%d" % (r["pass"], r["fail"], r["ns"], r["warn"], r["crash"]) - - def pct(v): - return "%.3f%%" % (100 * v) if v is not None else "n/a" - - def pp(v): - return "%+.3f" % v if v is not None else "n/a" print("%-14s %6d %-20s %8s %-20s %8s %9s %9s %5d %4d %5d %s%s" % ( b["block"], b["cases"], counts(b["base"]), pct(b["base"]["rate"]), counts(b["after"]), pct(b["after"]["rate"]), pp(b["delta_pp"]), "(%s)" % pp(b["delta_pp_results_info"]), len(b["new_crash"]), len(b["regressed"]), len(b["fixed"]), b["verdict"], " (SUBSET)" if b["subset"] else "")) + print(" info pre-ruling Pass/(results-NS): BASE %s AFTER %s delta %s pp (not gating)" % ( + pct(b["base"]["results_rate"]), pct(b["after"]["results_rate"]), pp(b["delta_pp_results_info"]))) for reason in b["reasons"]: print(" ! " + reason) + for warning in b["warnings"]: + print("WARN %s: %s" % (b["block"], warning)) for x in b["new_crash"]: print(" NEW CRASH " + x) for x in b["regressed"][:40]: @@ -520,7 +601,8 @@ def print_cts(c): tail += " (INCOMPLETE: %s)" % ", ".join("%s %s" % (b["block"], b["verdict"]) for b in c["blocks"] if b["block"] in c["incomplete_blocks"]) if any(b.get("subset") for b in c["blocks"]): - tail += " (SUBSET run: not a gate verdict)" + tail += " (SUBSET run: %s not the tools tree's full caselist; not a gate verdict)" % ", ".join( + b["block"] for b in c["blocks"] if b.get("subset")) print("CTS %s%s" % (c["overall"], tail)) @@ -542,13 +624,34 @@ def print_timing(out): print(" gate3 %-11s %2d case invocations, mean %.0fs, total %ds" % (arm, len(v), sum(v) / len(v), sum(v))) +def check_block(run, report): + """--check-block: 0 when the block is a complete AFTER reading, 3 when not (50-cts-after.sh).""" + after, why = load_results(report) + if why: + print("INCOMPLETE: " + why) + return 3 + universe, src, missing, incomplete = block_scope(run, after) + if missing or incomplete: + print("INCOMPLETE: " + "; ".join(missing + incomplete)) + return 3 + print("complete: %d/%d case(s) of %s have a result" % (len(universe), len(universe), src)) + return 0 + + def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("stamp") + ap.add_argument("stamp", nargs="?") ap.add_argument("--logroot", default=os.environ.get("W2_LOGROOT", str(Path.home() / "w7/logs/p7w7"))) ap.add_argument("--tools", help="MobileGL tree holding compare_actuals.py and CTS-base (default: build.env)") ap.add_argument("--json", help="write every section as JSON here (default /verdict.json)") + ap.add_argument("--check-block", nargs=2, metavar=("RUN_DIR", "REPORT_JSON"), + help="only test one block's completeness (50-cts-after.sh, before it writes .done)") args = ap.parse_args() + if args.check_block: + return check_block(Path(args.check_block[0]), Path(args.check_block[1])) + if not args.stamp: + ap.print_usage(sys.stderr) + return 2 out = Path(args.logroot) / args.stamp if not out.is_dir(): print("no such window output: %s" % out, file=sys.stderr) @@ -558,6 +661,7 @@ def main(): print("no tools tree (--tools)", file=sys.stderr) return 2 sys.path.insert(0, str(Path(tools) / "tools" / "trace_replay")) + sys.dont_write_bytecode = True # the tools tree is only read (it may be the read-only integration tree) import compare_actuals as ca base_dir = Path(tools) / "docs/Disaggregated/notes/p7/device-window-1/CTS-base" report = {"stamp": args.stamp, "out": str(out)} @@ -573,7 +677,7 @@ def main(): if b: print_bsl(b); print() report["bsl"] = b - c = cts(out, base_dir) + c = cts(out, base_dir, Path(tools) / "tools/cts/caselists") print_cts(c); print() report["cts"] = c print_timing(out)