mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-11-21 00:41:57 +09:00
[tools] (P7): window-2 reducer never passes an empty, unrateable or subset reading, and its rate gate is tested
This commit is contained in:
@@ -1,12 +1,13 @@
|
||||
#!/usr/bin/env python3
|
||||
"""60-reduce-selftest.py [--tools TREE] [--dry-run-dir DIR] [--keep]
|
||||
"""60-reduce-selftest.py [--tools TREE] [--dry-run-dir DIR] [--reducer PY] [--keep]
|
||||
|
||||
Self-test of 60-reduce.py: builds canned window output directories in a temp dir and runs the
|
||||
reducer on each exactly as window.sh does (a subprocess, <stamp> --logroot --tools), then checks
|
||||
the printed verdict, verdict.json and the exit status. The scenarios:
|
||||
the printed verdict, verdict.json and the exit status. Gate 3 is canned at the gate's own
|
||||
denominator (36 cases x monolith/inproc x3/spawn x3/ra0, one boot_id, reboot-clean) unless a
|
||||
scenario says otherwise. The scenarios:
|
||||
|
||||
control gate 3 (2 cases x monolith/inproc x3/spawn x3/ra0, one boot_id) and all five CTS
|
||||
blocks equal to $BASE -> GATE3 PASS, CTS PASS, exit 0
|
||||
control gate 3 and all five CTS blocks equal to $BASE -> GATE3 PASS, CTS PASS, exit 0
|
||||
missing-block ssbo never finished, dsa's report is empty, texture's has no results; and a
|
||||
window with no cts/ at all -> CTS INCOMPLETE, exit != 0
|
||||
boot-id-change a spawn pair's .done / one repeat's boot_id.txt names another boot; a .done
|
||||
@@ -18,15 +19,33 @@ the printed verdict, verdict.json and the exit status. The scenarios:
|
||||
-> WARN line, GATE3 still PASS
|
||||
one inproc repeat differs from its siblings -> FAIL (within-arm identity)
|
||||
unfinished a pair without .done; a repeat without an actual PNG -> GATE3 INCOMPLETE
|
||||
rate-gate the contract rate alone decides (no crash anywhere): k $BASE-Pass dsa cases
|
||||
turn Fail. k=1: 367/370, -0.270 pp -> PASS, exit 0; k=2: -0.541 pp -> FAIL on
|
||||
the rate; k=4: 364/6/0/1/0, -1.081 pp -> FAIL, CTS FAIL
|
||||
vacuous no verdict from nothing: the caselist a run names is gone (the dry run's AFTER
|
||||
data, two new crashes) -> every block MISSING, CTS INCOMPLETE - and with the
|
||||
caselist copy the window keeps, the real reading (CTS FAIL); no caselist at all /
|
||||
an empty one -> MISSING; every AFTER result NotSupported -> every block FAIL
|
||||
not-a-gate a pass that is not the gate's is never PASS / exit 0: 2 of 36 cases -> GATE3
|
||||
PASS-SUBSET; not reboot-clean -> GATE3 PASS-NOT-REBOOT-CLEAN; a --limit CTS
|
||||
block -> CTS PASS-SUBSET; each exit != 0
|
||||
warnings information lines: a case $BASE lacks (AFTER Fail) stays out of both rates
|
||||
(texture delta 0) with a WARN; a Pass that became NotSupported -> WARN on the NS
|
||||
count, block still PASS
|
||||
check-block 60-reduce.py --check-block (50-cts-after.sh's test before a block's .done):
|
||||
complete -> 0; unrun.txt non-empty / a case without result / caselist gone /
|
||||
empty report -> 3
|
||||
dry-run the pre-14e1c8b9 CTS dry run rebuilt from $BASE + its recorded differences
|
||||
(60-reduce-selftest-pre14e1c8b9.json): contract rate Pass/(Pass+Fail) gives
|
||||
dsa -0.272 pp (the pre-ruling Pass/(results-NS) -0.539 pp is printed as info
|
||||
only), shader-image +4.284, texture +0.960, ssbo/packed-pixels 0; dsa and
|
||||
shader-image FAIL on one NEW crash each, not on the rate -> CTS FAIL
|
||||
only, with its rates 99.191% -> 98.652%), shader-image +4.284, texture +0.960,
|
||||
ssbo/packed-pixels 0; dsa and shader-image FAIL on one NEW crash each, not on
|
||||
the rate -> CTS FAIL
|
||||
When --dry-run-dir (default ~/w7/logs/devprep/w2/pre-14e1c8b9) exists, the reducer also runs on
|
||||
that real output (verdict JSON to the temp dir) and must give the same CTS numbers.
|
||||
TREE (compare_actuals.py + device-window-1/CTS-base + tools/cts/caselists) defaults to the
|
||||
MobileGL tree this file sits in, else $W2_TOOLS_OVERRIDE, else $W2_PIPE / ~/w7/pipe (only read).
|
||||
--reducer runs the checks against another 60-reduce.py (red-once of an older reducer).
|
||||
Exit 0 when every check holds.
|
||||
"""
|
||||
import argparse
|
||||
@@ -47,8 +66,10 @@ CTS_BASE_REL = "docs/Disaggregated/notes/p7/device-window-1/CTS-base"
|
||||
BLOCKS = ("shader-image", "ssbo", "dsa", "texture", "packed-pixels")
|
||||
BOOT = "11111111-2222-3333-4444-555555555555"
|
||||
OTHER_BOOT = "99999999-8888-7777-6666-555555555555"
|
||||
CASES = ("caseA-in-world", "caseB-main-menu")
|
||||
# The gate's own denominator (CONTRACT-P7 7.2): 36 cases; the first two are the ones scenarios poke.
|
||||
CASES = ("caseA-in-world", "caseB-main-menu") + tuple("case%02d-filler" % i for i in range(3, 37))
|
||||
REPEAT = 3
|
||||
GONE = "/nonexistent/p7w7-tree-deadbeef/tools/cts/caselists/p7-%s-gl46.txt"
|
||||
ARMS = (("monolith", 1), ("inproc", REPEAT), ("spawn", REPEAT), ("inproc-ra0", 1))
|
||||
DSA_STORAGE = "KHR-GL46.direct_state_access.renderbuffers_storage"
|
||||
SI_INCOMPLETE = "KHR-GL46.shader_image_load_store.incomplete_textures"
|
||||
@@ -101,16 +122,18 @@ class Canned:
|
||||
def png(self, value):
|
||||
return self.ca._png_bytes(8, 8, self.ca._solid(8, 8, value))
|
||||
|
||||
def gate3(self, out, picture=None, boot=BOOT):
|
||||
def gate3(self, out, picture=None, boot=BOOT, cases=CASES, reboot_clean=True):
|
||||
picture = picture or {}
|
||||
g = out / "gate3"
|
||||
write(g / "cases.txt", "\n".join(CASES) + "\n")
|
||||
write(g / "cases.txt", "\n".join(cases) + "\n")
|
||||
write(g / "excluded.txt", "x-red-on-monolith-case\n")
|
||||
write(g / "arms.txt", "arms=monolith inproc spawn inproc-ra0\nrepeat=%d\n" % REPEAT)
|
||||
write(out / "session/boot-id.txt", boot + "\n")
|
||||
write(out / "session/reboot-clean.txt", "reboot-clean: 00000000-0000-0000-0000-000000000000 -> %s\n" % boot)
|
||||
write(out / "session/reboot-clean.txt",
|
||||
("reboot-clean: 00000000-0000-0000-0000-000000000000 -> %s\n" % boot) if reboot_clean
|
||||
else ("NOT reboot-clean (no --reboot): boot_id %s, up 5000.00s\n" % boot))
|
||||
for arm, n in ARMS:
|
||||
for case in CASES:
|
||||
for case in cases:
|
||||
name = case + "-DirectVulkan"
|
||||
for i in range(1, n + 1):
|
||||
rep = g / "archive" / arm / name / ("repeat-%02d" % i)
|
||||
@@ -129,9 +152,18 @@ class Canned:
|
||||
"finished=2026-09-23T12:00:00-04:00\n" % (n, boot, boot))
|
||||
return g
|
||||
|
||||
def cts(self, out, differences=None, drop=(), empty=(), no_results=()):
|
||||
differences = differences or {}
|
||||
base_dir = self.tree / CTS_BASE_REL
|
||||
def base(self, b):
|
||||
return dict(json.loads((self.tree / CTS_BASE_REL / ("report-%s.json" % b)).read_text(encoding="utf-8"))["results"])
|
||||
|
||||
def caselist(self, b):
|
||||
return self.tree / "tools/cts/caselists" / ("p7-%s-gl46.txt" % b)
|
||||
|
||||
def cts(self, out, differences=None, drop=(), empty=(), no_results=(), copy=True, named=None, lists=None):
|
||||
"""50-cts-after.sh's output for the five blocks: AFTER = $BASE + `differences` per block.
|
||||
copy: keep runs/<b>/caselist.txt as 50-cts-after.sh does; named: b -> what caselist.path
|
||||
says (None = the tools tree's list, '' = no caselist.path); lists: b -> the cases the run
|
||||
was asked to run (a --limit subset, or a list with cases $BASE lacks)."""
|
||||
differences, lists = differences or {}, lists or {}
|
||||
skip = set(read_list(self.tree / "tools/cts/caselists/p7-skip.txt"))
|
||||
c = out / "cts"
|
||||
c.mkdir(parents=True, exist_ok=True)
|
||||
@@ -139,15 +171,23 @@ class Canned:
|
||||
for b in BLOCKS:
|
||||
if b in drop:
|
||||
continue
|
||||
caselist = self.tree / "tools/cts/caselists" / ("p7-%s-gl46.txt" % b)
|
||||
caselist = self.caselist(b)
|
||||
run = c / "runs" / b
|
||||
write(run / "caselist.path", str(caselist) + "\n")
|
||||
results = self.base(b)
|
||||
if b in lists:
|
||||
caselist = c / "caselists" / ("p7-%s-gl46.canned.txt" % b)
|
||||
write(caselist, "\n".join(lists[b]) + "\n")
|
||||
results = {k: v for k, v in results.items() if k in set(lists[b])}
|
||||
path = named(b) if named else str(caselist)
|
||||
if path:
|
||||
write(run / "caselist.path", path + "\n")
|
||||
if copy:
|
||||
write(run / "caselist.txt", caselist.read_bytes())
|
||||
skipped = [x for x in read_list(caselist) if x in skip]
|
||||
if skipped:
|
||||
write(run / "skipped.txt", "\n".join(skipped) + "\n")
|
||||
write(run / "unrun.txt", "")
|
||||
write(run / "hung.txt", "")
|
||||
results = dict(json.loads((base_dir / ("report-%s.json" % b)).read_text(encoding="utf-8"))["results"])
|
||||
results.update(differences.get(b, {}))
|
||||
rebuilt[b] = results
|
||||
write(run / "crashed.txt", "".join(k + "\n" for k, v in sorted(results.items()) if v == "Crash"))
|
||||
@@ -162,11 +202,11 @@ class Canned:
|
||||
"counts": dict(Counter(results.values())), "results": results}, indent=1))
|
||||
return rebuilt
|
||||
|
||||
def window(self, stamp, gate3=True, cts=True, **cts_args):
|
||||
def window(self, stamp, gate3=True, cts=True, gate3_args=None, **cts_args):
|
||||
out = self.root / stamp
|
||||
out.mkdir(parents=True)
|
||||
if gate3:
|
||||
self.gate3(out)
|
||||
self.gate3(out, **(gate3_args or {}))
|
||||
else:
|
||||
write(out / "session/boot-id.txt", BOOT + "\n")
|
||||
if cts:
|
||||
@@ -184,6 +224,12 @@ def run_reducer(logroot, stamp, tree, json_out=None):
|
||||
return p.returncode, p.stdout, verdict
|
||||
|
||||
|
||||
def run_check_block(run, report):
|
||||
p = subprocess.run([sys.executable, str(REDUCER), "--check-block", str(run), str(report)],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
|
||||
return p.returncode, p.stdout
|
||||
|
||||
|
||||
class Checks:
|
||||
def __init__(self):
|
||||
self.total, self.failed = 0, []
|
||||
@@ -221,7 +267,8 @@ def scenario_control(k, cn, tree):
|
||||
cn.window("control")
|
||||
rc, text, v = run_reducer(cn.root, "control", tree)
|
||||
k.expect("exit 0", rc == 0, "rc=%d\n%s" % (rc, text))
|
||||
k.expect("GATE3 PASS", v.get("gate3", {}).get("overall") == "PASS", text)
|
||||
k.expect("GATE3 PASS 36/36", v.get("gate3", {}).get("overall") == "PASS" and "GATE3 PASS: 36/36 cases PASS\n" in text,
|
||||
text)
|
||||
k.expect("CTS PASS, all five blocks", v.get("cts", {}).get("overall") == "PASS"
|
||||
and [b["block"] for b in v["cts"]["blocks"]] == list(BLOCKS), text)
|
||||
k.expect("every block delta 0.000 pp", all(r3(block(v, b).get("delta_pp")) == 0.0 for b in BLOCKS))
|
||||
@@ -300,7 +347,7 @@ def scenario_monolith_arm(k, cn, tree):
|
||||
e.get("verdict") == "FAIL" and v["gate3"]["overall"] == "FAIL" and rc != 0
|
||||
and any("NOT proven monolith" in r for r in e.get("reasons", [])), e)
|
||||
k.expect("the other case still PASS", case(v, CASES[1]).get("verdict") == "PASS")
|
||||
k.expect("arm identity line counts 1/2 monolith", "monolith 1/2 repeat(s)" in text, text)
|
||||
k.expect("arm identity line counts 35/36 monolith", "monolith 35/36 repeat(s)" in text, text)
|
||||
|
||||
out = cn.window("mono-nolog")
|
||||
(out / "gate3/archive/monolith" / (CASES[1] + "-DirectVulkan") / "repeat-01" / "mobilegl.log").unlink()
|
||||
@@ -357,6 +404,143 @@ def scenario_unfinished(k, cn, tree):
|
||||
and any("no *-actual.png" in r for r in e.get("reasons", [])), e)
|
||||
|
||||
|
||||
def pass_cases(cn, b):
|
||||
"""$BASE-Pass cases of block b's caselist (not skipped), in caselist order."""
|
||||
base = cn.base(b)
|
||||
skip = set(read_list(cn.tree / "tools/cts/caselists/p7-skip.txt"))
|
||||
return [x for x in read_list(cn.caselist(b)) if x not in skip and base.get(x) == "Pass"]
|
||||
|
||||
|
||||
def dryrun_differences():
|
||||
fixture = json.loads(FIXTURE.read_text(encoding="utf-8"))
|
||||
return fixture, {b: fixture["blocks"][b]["after_differences"] for b in BLOCKS}
|
||||
|
||||
|
||||
def scenario_rate_gate(k, cn, tree):
|
||||
print("[rate-gate] the contract rate alone decides a block (no crash anywhere)")
|
||||
flips = pass_cases(cn, "dsa")
|
||||
cn.window("rate-1", differences={"dsa": {x: "Fail" for x in flips[:1]}})
|
||||
rc, text, v = run_reducer(cn.root, "rate-1", tree)
|
||||
d = block(v, "dsa")
|
||||
k.expect("1 dsa Pass->Fail: 367/370 = -0.270 pp -> dsa PASS, CTS PASS, window exit 0",
|
||||
r3(d.get("delta_pp")) == -0.27 and d.get("verdict") == "PASS" and v.get("cts", {}).get("overall") == "PASS"
|
||||
and rc == 0, (d.get("delta_pp"), d.get("verdict"), rc, text[-300:]))
|
||||
cn.window("rate-2", gate3=False, differences={"dsa": {x: "Fail" for x in flips[:2]}})
|
||||
rc, text, v = run_reducer(cn.root, "rate-2", tree)
|
||||
d = block(v, "dsa")
|
||||
k.expect("2 dsa Pass->Fail: -0.541 pp < -0.5 pp -> dsa FAIL on the rate alone (0 new crash)",
|
||||
r3(d.get("delta_pp")) == -0.541 and d.get("verdict") == "FAIL" and d.get("new_crash") == []
|
||||
and d.get("reasons") == ["delta -0.541 pp < -0.5 pp (Pass/(Pass+Fail))"],
|
||||
(d.get("delta_pp"), d.get("verdict"), d.get("reasons")))
|
||||
cn.window("rate-4", gate3=False, differences={"dsa": {x: "Fail" for x in flips[:4]}})
|
||||
rc, text, v = run_reducer(cn.root, "rate-4", tree)
|
||||
d = block(v, "dsa")
|
||||
k.expect("4 dsa Pass->Fail: AFTER 364/6/0/1/0, -1.081 pp -> dsa FAIL, CTS FAIL, exit != 0",
|
||||
[d.get("after", {}).get(x) for x in ("pass", "fail", "ns", "warn", "crash")] == [364, 6, 0, 1, 0]
|
||||
and r3(d.get("delta_pp")) == -1.081 and d.get("verdict") == "FAIL" and d.get("new_crash") == []
|
||||
and v.get("cts", {}).get("overall") == "FAIL" and "\nCTS FAIL" in text and rc != 0,
|
||||
(d.get("after"), d.get("delta_pp"), d.get("verdict"), rc))
|
||||
|
||||
|
||||
def scenario_vacuous(k, cn, tree):
|
||||
print("[vacuous] a block with nothing to compare is never PASS")
|
||||
_, diffs = dryrun_differences()
|
||||
cn.window("gone", differences=diffs, copy=False, named=lambda b: GONE % b)
|
||||
rc, text, v = run_reducer(cn.root, "gone", tree)
|
||||
k.expect("caselist.path names a caselist that is gone (no copy), dry-run AFTER data -> every block MISSING, "
|
||||
"CTS INCOMPLETE, exit != 0",
|
||||
rc != 0 and v.get("cts", {}).get("overall") == "INCOMPLETE" and "CTS PASS" not in text
|
||||
and all(block(v, b).get("verdict") == "MISSING" and "is gone" in " ".join(block(v, b).get("reasons", []))
|
||||
for b in BLOCKS), text)
|
||||
cn.window("gone-copy", gate3=False, differences=diffs, named=lambda b: GONE % b)
|
||||
rc, text, v = run_reducer(cn.root, "gone-copy", tree)
|
||||
d = block(v, "dsa")
|
||||
k.expect("...with the window's caselist.txt copy: the real reading (dsa -0.272 pp, 2 new crashes, CTS FAIL)",
|
||||
r3(d.get("delta_pp")) == -0.272 and d.get("caselist", "").endswith("runs/dsa/caselist.txt")
|
||||
and v.get("cts", {}).get("overall") == "FAIL", (d.get("delta_pp"), d.get("caselist"), v.get("cts", {}).get("overall")))
|
||||
cn.window("no-caselist", gate3=False, copy=False, named=lambda b: "")
|
||||
rc, text, v = run_reducer(cn.root, "no-caselist", tree)
|
||||
k.expect("no caselist.txt and no caselist.path -> every block MISSING ('unknown'), CTS INCOMPLETE",
|
||||
rc != 0 and v.get("cts", {}).get("overall") == "INCOMPLETE"
|
||||
and all("unknown" in " ".join(block(v, b).get("reasons", [])) for b in BLOCKS), text)
|
||||
out = cn.window("empty-caselist", gate3=False)
|
||||
write(out / "cts/runs/ssbo/caselist.txt", "# nothing asked\n")
|
||||
rc, text, v = run_reducer(cn.root, "empty-caselist", tree)
|
||||
k.expect("an empty caselist -> ssbo MISSING ('holds no case'), CTS INCOMPLETE",
|
||||
block(v, "ssbo").get("verdict") == "MISSING" and "holds no case" in " ".join(block(v, "ssbo").get("reasons", []))
|
||||
and v.get("cts", {}).get("overall") == "INCOMPLETE", block(v, "ssbo"))
|
||||
ns = {b: {x: "NotSupported" for x in cn.base(b)} for b in BLOCKS}
|
||||
cn.window("all-ns", differences=ns)
|
||||
rc, text, v = run_reducer(cn.root, "all-ns", tree)
|
||||
k.expect("every AFTER result NotSupported -> every block FAIL ('AFTER has no Pass/Fail'), CTS FAIL, exit != 0",
|
||||
rc != 0 and v.get("cts", {}).get("overall") == "FAIL" and "CTS PASS" not in text
|
||||
and all(block(v, b).get("verdict") == "FAIL" and block(v, b).get("delta_pp") is None
|
||||
and any("AFTER has no Pass/Fail" in r for r in block(v, b).get("reasons", [])) for b in BLOCKS), text)
|
||||
k.expect("...and a WARN on each block's NotSupported count", all(("WARN %s: NotSupported $BASE" % b) in text for b in BLOCKS), text)
|
||||
|
||||
|
||||
def scenario_not_a_gate(k, cn, tree):
|
||||
print("[not-a-gate] a pass that is not the gate's own reading never exits 0")
|
||||
cn.window("subset-g3", gate3_args={"cases": CASES[:2]})
|
||||
rc, text, v = run_reducer(cn.root, "subset-g3", tree)
|
||||
k.expect("2 of 36 gate-3 cases, all passing -> GATE3 PASS-SUBSET, exit != 0",
|
||||
v.get("gate3", {}).get("overall") == "PASS-SUBSET" and "GATE3 PASS-SUBSET: 2/2 cases PASS" in text
|
||||
and "WINDOW2 GATE3 PASS-SUBSET | CTS PASS -> exit 1" in text and rc != 0, text[-500:])
|
||||
cn.window("not-clean", gate3_args={"reboot_clean": False})
|
||||
rc, text, v = run_reducer(cn.root, "not-clean", tree)
|
||||
k.expect("36/36 in a session that is not reboot-clean -> GATE3 PASS-NOT-REBOOT-CLEAN, exit != 0",
|
||||
v.get("gate3", {}).get("overall") == "PASS-NOT-REBOOT-CLEAN" and rc != 0
|
||||
and "session not reboot-clean" in text, text[-500:])
|
||||
first20 = read_list(cn.caselist("dsa"))[:20]
|
||||
cn.window("subset-cts", lists={"dsa": first20})
|
||||
rc, text, v = run_reducer(cn.root, "subset-cts", tree)
|
||||
k.expect("a --limit 20 dsa block, all passing -> CTS PASS-SUBSET (dsa marked), GATE3 PASS, exit != 0",
|
||||
v.get("cts", {}).get("overall") == "PASS-SUBSET" and block(v, "dsa").get("subset") is True
|
||||
and block(v, "dsa").get("cases") == 20 and v.get("gate3", {}).get("overall") == "PASS" and rc != 0
|
||||
and "SUBSET run: dsa" in text, text[-500:])
|
||||
|
||||
|
||||
def scenario_warnings(k, cn, tree):
|
||||
print("[warnings] like-for-like rates and the NotSupported count are reported, not gated")
|
||||
extra = "KHR-GL46.texture_selftest.not_in_base"
|
||||
tex = read_list(cn.caselist("texture")) + [extra]
|
||||
ssbo_ns = pass_cases(cn, "ssbo")[0]
|
||||
cn.window("warn", gate3=False, lists={"texture": tex},
|
||||
differences={"texture": {extra: "Fail"}, "ssbo": {ssbo_ns: "NotSupported"}})
|
||||
rc, text, v = run_reducer(cn.root, "warn", tree)
|
||||
t, s = block(v, "texture"), block(v, "ssbo")
|
||||
k.expect("a case $BASE lacks (AFTER Fail) stays out of both rates: texture delta 0.000, PASS, compared = cases - 1",
|
||||
r3(t.get("delta_pp")) == 0.0 and t.get("verdict") == "PASS" and t.get("compared") == t.get("cases", 0) - 1
|
||||
and t.get("missing_in_base") == [extra], (t.get("delta_pp"), t.get("verdict"), t.get("compared"), t.get("cases")))
|
||||
k.expect("...with a WARN line naming it", "WARN texture: 1 case(s) have no $BASE result (e.g. %s)" % extra in text, text)
|
||||
k.expect("an ssbo Pass that became NotSupported: rate unchanged (PASS), WARN on the NS count, in Pass->not",
|
||||
s.get("verdict") == "PASS" and r3(s.get("delta_pp")) == 0.0 and ssbo_ns in s.get("regressed", [])
|
||||
and "WARN ssbo: NotSupported $BASE 0 -> AFTER 1" in text, (s.get("verdict"), s.get("delta_pp"), text[-400:]))
|
||||
|
||||
|
||||
def scenario_check_block(k, cn, tree):
|
||||
print("[check-block] 50-cts-after.sh's completeness test before a block's .done")
|
||||
out = cn.window("cb", gate3=False)
|
||||
c = out / "cts"
|
||||
rc, text = run_check_block(c / "runs/dsa", c / "report-dsa.json")
|
||||
k.expect("complete dsa block -> 0", rc == 0 and "complete: 371/371" in text, (rc, text))
|
||||
write(c / "runs/ssbo/unrun.txt", read_list(cn.caselist("ssbo"))[-1] + "\n")
|
||||
rc, text = run_check_block(c / "runs/ssbo", c / "report-ssbo.json")
|
||||
k.expect("unrun.txt non-empty -> 3", rc == 3 and "unrun.txt 1" in text, (rc, text))
|
||||
report = json.loads((c / "report-texture.json").read_text(encoding="utf-8"))
|
||||
report["results"].pop(pass_cases(cn, "texture")[0])
|
||||
write(c / "report-texture.json", json.dumps(report))
|
||||
rc, text = run_check_block(c / "runs/texture", c / "report-texture.json")
|
||||
k.expect("a case without a result -> 3", rc == 3 and "1 case(s) without an AFTER result" in text, (rc, text))
|
||||
(c / "runs/shader-image/caselist.txt").unlink()
|
||||
write(c / "runs/shader-image/caselist.path", GONE % "shader-image" + "\n")
|
||||
rc, text = run_check_block(c / "runs/shader-image", c / "report-shader-image.json")
|
||||
k.expect("the caselist gone -> 3", rc == 3 and "is gone" in text, (rc, text))
|
||||
write(c / "report-packed-pixels.json", "")
|
||||
rc, text = run_check_block(c / "runs/packed-pixels", c / "report-packed-pixels.json")
|
||||
k.expect("an empty report -> 3", rc == 3 and "is empty" in text, (rc, text))
|
||||
|
||||
|
||||
def check_dryrun_numbers(k, v, text, label):
|
||||
dsa, si, tex = block(v, "dsa"), block(v, "shader-image"), block(v, "texture")
|
||||
k.expect("%s: dsa contract delta -0.272 pp" % label, r3(dsa.get("delta_pp")) == -0.272, dsa.get("delta_pp"))
|
||||
@@ -365,6 +549,8 @@ def check_dryrun_numbers(k, v, text, label):
|
||||
if dsa.get("base") else False, dsa)
|
||||
k.expect("%s: dsa pre-ruling delta -0.539 pp kept as info" % label,
|
||||
r3(dsa.get("delta_pp_results_info")) == -0.539 and "(-0.539)" in text, dsa.get("delta_pp_results_info"))
|
||||
k.expect("%s: dsa pre-ruling rates printed (info line 99.191%% -> 98.652%%)" % label,
|
||||
"info pre-ruling Pass/(results-NS): BASE 99.191% AFTER 98.652% delta -0.539 pp (not gating)" in text, text)
|
||||
k.expect("%s: dsa FAIL on the new crash only, not on the rate" % label,
|
||||
dsa.get("verdict") == "FAIL" and dsa.get("reasons") == ["1 NEW crash(es)"]
|
||||
and dsa.get("new_crash") == [DSA_STORAGE], dsa.get("reasons"))
|
||||
@@ -384,8 +570,7 @@ def check_dryrun_numbers(k, v, text, label):
|
||||
|
||||
def scenario_dryrun(k, cn, tree, dry_dir):
|
||||
print("[dry-run] contract formula on the pre-14e1c8b9 dry-run data")
|
||||
fixture = json.loads(FIXTURE.read_text(encoding="utf-8"))
|
||||
diffs = {b: fixture["blocks"][b]["after_differences"] for b in BLOCKS}
|
||||
fixture, diffs = dryrun_differences()
|
||||
out = cn.root / "dryrun"
|
||||
out.mkdir()
|
||||
rebuilt = cn.cts(out, differences=diffs)
|
||||
@@ -413,17 +598,21 @@ def scenario_dryrun(k, cn, tree, dry_dir):
|
||||
|
||||
|
||||
def main():
|
||||
global REDUCER
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--tools", help="MobileGL tree (default: the tree holding this file, else $W2_TOOLS_OVERRIDE)")
|
||||
ap.add_argument("--dry-run-dir", default=str(Path.home() / "w7/logs/devprep/w2/pre-14e1c8b9"))
|
||||
ap.add_argument("--reducer", default=str(REDUCER), help="the 60-reduce.py under test (default: beside this file)")
|
||||
ap.add_argument("--keep", action="store_true", help="keep the canned directories")
|
||||
args = ap.parse_args()
|
||||
REDUCER = Path(args.reducer).resolve()
|
||||
tree = find_tree(args.tools)
|
||||
if tree is None:
|
||||
print("no MobileGL tree with tools/trace_replay/compare_actuals.py and %s: pass --tools" % CTS_BASE_REL,
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
sys.path.insert(0, str(tree / "tools" / "trace_replay"))
|
||||
sys.dont_write_bytecode = True # the tree is only read (the working copy falls back to ~/w7/pipe)
|
||||
import compare_actuals as ca
|
||||
root = Path(tempfile.mkdtemp(prefix="w2-reduce-selftest-"))
|
||||
print("60-reduce self-test: reducer %s, tree %s, canned windows in %s" % (REDUCER, tree, root))
|
||||
@@ -436,6 +625,11 @@ def main():
|
||||
scenario_monolith_arm(k, cn, tree)
|
||||
scenario_cross_arm(k, cn, tree)
|
||||
scenario_unfinished(k, cn, tree)
|
||||
scenario_rate_gate(k, cn, tree)
|
||||
scenario_vacuous(k, cn, tree)
|
||||
scenario_not_a_gate(k, cn, tree)
|
||||
scenario_warnings(k, cn, tree)
|
||||
scenario_check_block(k, cn, tree)
|
||||
scenario_dryrun(k, cn, tree, args.dry_run_dir)
|
||||
finally:
|
||||
if not args.keep:
|
||||
|
||||
Reference in New Issue
Block a user