mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-11-07 01:53:24 +09:00
[tools] (P7): the window-2 reducer takes an adjudicated lost CTS case out of its block's rate on both sides and prints it ADJUDICATED
This commit is contained in:
@@ -39,11 +39,14 @@ otherwise. The scenarios:
|
||||
cts-lost a $BASE-Pass case AFTER turns NotSupported / another non-Pass, non-Fail status
|
||||
(L): dsa Pass->NS -> NON-PASS in the rate (367/370, -0.270 pp; the literal
|
||||
Pass/(Pass+Fail) -0.001 pp printed as info), listed, dsa FAIL, CTS FAIL; with a
|
||||
cts/adjudication.tsv line -> dsa PASS, exit 0 (still counted in the rate); a line
|
||||
without a reason adjudicates nothing; ssbo Pass->NS adjudicated -> still FAIL on
|
||||
the rate (-0.806 pp); codex's example, 123 of 124 ssbo Passes -> NS: literal
|
||||
100% but gate rate 0.806% -> FAIL; Pass->CompatibilityWarning -> L; Pass->Crash
|
||||
adjudicated -> still FAIL (a NEW crash is never adjudicated)
|
||||
cts/adjudication.tsv line -> dsa PASS, exit 0, printed ADJUDICATED and OUT of the
|
||||
block's rate on both sides (367/369 vs 367/369, +0.000 pp); one adjudicated + one
|
||||
not -> L 1, -0.271 pp, FAIL on the unadjudicated one; 2 adjudicated NS + 2
|
||||
Pass->Fail -> -0.543 pp, FAIL on the rate; a line without a reason adjudicates
|
||||
nothing; ssbo Pass->NS adjudicated -> ssbo PASS (+0.000 pp, not -0.806); codex's
|
||||
example, 123 of 124 ssbo Passes -> NS: literal 100% but gate rate 0.806% -> FAIL;
|
||||
Pass->CompatibilityWarning -> L; Pass->Crash adjudicated -> still FAIL (a NEW crash
|
||||
is never adjudicated)
|
||||
vacuous no verdict from nothing: the caselist a run names is gone (the dry run's AFTER
|
||||
data, two new crashes) -> every block MISSING, CTS INCOMPLETE - and with the
|
||||
caselist copy the window keeps, the real reading (CTS FAIL); no caselist at all /
|
||||
@@ -672,10 +675,42 @@ def scenario_cts_lost(k, cn, tree):
|
||||
cn.window("lost-ns-adj", differences={"dsa": {ns: "NotSupported"}}, adjudication="%s\t%s\n" % (ns, reason))
|
||||
rc, text, v = run_reducer(cn.root, "lost-ns-adj", tree)
|
||||
d = block(v, "dsa")
|
||||
k.expect("...with a cts/adjudication.tsv line: dsa PASS, CTS PASS, window exit 0, still NON-PASS in the rate",
|
||||
d.get("verdict") == "PASS" and v["cts"]["overall"] == "PASS" and rc == 0 and r3(d.get("delta_pp")) == -0.27
|
||||
and d.get("lost") == [{"case": ns, "after": "NotSupported", "adjudication": reason}]
|
||||
and ("adjudicated: " + reason) in text, (d.get("verdict"), d.get("reasons"), rc))
|
||||
k.expect("...with a cts/adjudication.tsv line: dsa PASS, CTS PASS, window exit 0",
|
||||
d.get("verdict") == "PASS" and v["cts"]["overall"] == "PASS" and rc == 0
|
||||
and d.get("lost") == [{"case": ns, "after": "NotSupported", "adjudication": reason}],
|
||||
(d.get("verdict"), d.get("reasons"), rc))
|
||||
k.expect("...and OUT of the block's rate on both sides: BASE 367/369, AFTER 367/(367+2+0), +0.000 pp, L 0, "
|
||||
"printed ADJUDICATED (the literal info still counts it: -0.001 pp; the NS WARN stays)",
|
||||
r3(d.get("delta_pp")) == 0.0 and d.get("after", {}).get("lost") == 0
|
||||
and r3(100 * d.get("base", {}).get("rate", 0)) == 99.458 and r3(100 * d.get("after", {}).get("rate", 0)) == 99.458
|
||||
and d.get("lost_adjudicated") == [ns] and d.get("lost_unadjudicated") == []
|
||||
and ("ADJUDICATED %s (Pass -> NotSupported) out of the rate; adjudicated: %s" % (ns, reason)) in text
|
||||
and r3(d.get("delta_pp_literal_info")) == -0.001 and "WARN dsa: NotSupported $BASE 0 -> AFTER 1" in text,
|
||||
(d.get("delta_pp"), d.get("base"), d.get("after"), d.get("lost_adjudicated")))
|
||||
|
||||
# One adjudicated, one not, in one block: only the adjudicated one leaves the rate.
|
||||
cn.window("lost-mixed", gate3=False, differences={"dsa": {dsa[0]: "NotSupported", dsa[1]: "NotSupported"}},
|
||||
adjudication="%s\t%s\n" % (dsa[0], reason))
|
||||
rc, text, v = run_reducer(cn.root, "lost-mixed", tree)
|
||||
d = block(v, "dsa")
|
||||
k.expect("dsa 2 Pass->NS, 1 adjudicated: L 1, BASE 367/369 vs AFTER 366/(366+2+1) = -0.271 pp; FAIL on the "
|
||||
"unadjudicated one, one ADJUDICATED and one LOST line",
|
||||
r3(d.get("delta_pp")) == -0.271 and d.get("after", {}).get("lost") == 1 and d.get("verdict") == "FAIL"
|
||||
and d.get("lost_unadjudicated") == [dsa[1]] and d.get("lost_adjudicated") == [dsa[0]]
|
||||
and not any(r.startswith("delta ") for r in d.get("reasons", []))
|
||||
and ("LOST %s (Pass -> NotSupported) UNADJUDICATED" % dsa[1]) in text
|
||||
and ("ADJUDICATED %s (Pass -> NotSupported)" % dsa[0]) in text,
|
||||
(d.get("delta_pp"), d.get("after"), d.get("reasons")))
|
||||
|
||||
# An adjudication takes its own case out and nothing else: real Pass->Fail losses still gate.
|
||||
cn.window("lost-adj-fails", gate3=False, adjudication="".join("%s\t%s\n" % (x, reason) for x in dsa[:2]),
|
||||
differences={"dsa": {dsa[0]: "NotSupported", dsa[1]: "NotSupported", dsa[2]: "Fail", dsa[3]: "Fail"}})
|
||||
rc, text, v = run_reducer(cn.root, "lost-adj-fails", tree)
|
||||
d = block(v, "dsa")
|
||||
k.expect("dsa 2 adjudicated Pass->NS + 2 Pass->Fail: BASE 366/368 vs AFTER 364/368 = -0.543 pp -> FAIL on the rate",
|
||||
r3(d.get("delta_pp")) == -0.543 and d.get("verdict") == "FAIL" and d.get("lost_unadjudicated") == []
|
||||
and "delta -0.543 pp < -0.5 pp (Pass/(Pass+Fail+L))" in d.get("reasons", []),
|
||||
(d.get("delta_pp"), d.get("reasons")))
|
||||
|
||||
cn.window("lost-ns-noreason", gate3=False, differences={"dsa": {ns: "NotSupported"}}, adjudication=ns + "\n")
|
||||
rc, text, v = run_reducer(cn.root, "lost-ns-noreason", tree)
|
||||
@@ -688,9 +723,11 @@ def scenario_cts_lost(k, cn, tree):
|
||||
adjudication="%s\t%s\n" % (ssbo[0], reason))
|
||||
rc, text, v = run_reducer(cn.root, "lost-ssbo-adj", tree)
|
||||
s = block(v, "ssbo")
|
||||
k.expect("ssbo Pass->NS adjudicated: 123/(123+0+1) = -0.806 pp -> still FAIL on the rate (adjudication != rate)",
|
||||
s.get("verdict") == "FAIL" and r3(s.get("delta_pp")) == -0.806 and s.get("lost_unadjudicated") == []
|
||||
and "delta -0.806 pp < -0.5 pp (Pass/(Pass+Fail+L))" in s.get("reasons", []), (s.get("delta_pp"), s.get("reasons")))
|
||||
k.expect("ssbo Pass->NS adjudicated: out of the rate, 123/123 vs 123/123 = +0.000 pp -> ssbo PASS (it was "
|
||||
"-0.806 pp FAIL for ever while an adjudicated case stayed NON-PASS)",
|
||||
s.get("verdict") == "PASS" and r3(s.get("delta_pp")) == 0.0 and s.get("lost_unadjudicated") == []
|
||||
and s.get("lost_adjudicated") == [ssbo[0]] and not any(r.startswith("delta ") for r in s.get("reasons", [])),
|
||||
(s.get("delta_pp"), s.get("reasons")))
|
||||
|
||||
cn.window("lost-codex", gate3=False, differences={"ssbo": {x: "NotSupported" for x in ssbo[1:]}})
|
||||
rc, text, v = run_reducer(cn.root, "lost-codex", tree)
|
||||
@@ -712,9 +749,11 @@ def scenario_cts_lost(k, cn, tree):
|
||||
cn.window("lost-crash-adj", gate3=False, differences={"dsa": {ns: "Crash"}}, adjudication="%s\t%s\n" % (ns, reason))
|
||||
rc, text, v = run_reducer(cn.root, "lost-crash-adj", tree)
|
||||
d = block(v, "dsa")
|
||||
k.expect("dsa Pass->Crash adjudicated -> the L case is adjudicated, but the NEW crash still makes dsa FAIL",
|
||||
k.expect("dsa Pass->Crash adjudicated -> the L case is adjudicated (out of the rate, +0.000 pp), but the NEW "
|
||||
"crash still makes dsa FAIL",
|
||||
d.get("verdict") == "FAIL" and d.get("new_crash") == [ns] and d.get("lost_unadjudicated") == []
|
||||
and d.get("reasons", [None])[0] == "1 NEW crash(es)", (d.get("verdict"), d.get("reasons")))
|
||||
and r3(d.get("delta_pp")) == 0.0 and d.get("reasons", [None])[0] == "1 NEW crash(es)",
|
||||
(d.get("verdict"), d.get("delta_pp"), d.get("reasons")))
|
||||
|
||||
|
||||
def scenario_vacuous(k, cn, tree):
|
||||
|
||||
@@ -63,8 +63,11 @@ CTS AFTER (CONTRACT-P7 7.3), per block vs device-window-1/CTS-base/report-<block
|
||||
rates and still counts for new crashes). AFTER without a single Pass/Fail/L where BASE has some
|
||||
is FAIL (the rate cannot be formed); BASE without one is INCOMPLETE;
|
||||
* every L case is listed (Pass -> its AFTER status), and a block with one that
|
||||
cts/adjudication.tsv ('<case><TAB><reason>') does not list is FAIL. An adjudication waives
|
||||
only that rule: the case still counts as NON-PASS in the rate;
|
||||
cts/adjudication.tsv ('<case><TAB><reason>') does not list is FAIL. An ADJUDICATED one is
|
||||
printed 'ADJUDICATED' and leaves that block's rate on both sides - no longer an L case in AFTER's
|
||||
denominator, one Pass fewer in BASE's (ruling: integrator 2026-09-23 after the int3 critic: else
|
||||
a single ruled-legitimate loss in the 124-case ssbo block was -0.806 pp that nothing could clear);
|
||||
the literal and pre-ruling info rates still count it;
|
||||
* crashes (Crash/Timeout/InternalError/ResourceError/DeviceHang/Incomplete, plus hung.txt) are
|
||||
counted in their own column and are non-Pass; one that BASE does not already have is a NEW
|
||||
crash, a hard red of its own that no adjudication waives (a Fail that became a Crash is a new
|
||||
@@ -628,19 +631,24 @@ def read_list(path):
|
||||
return [l.strip() for l in path.read_text(encoding="utf-8").splitlines() if l.strip() and not l.startswith("#")]
|
||||
|
||||
|
||||
def rate(statuses, lost=0):
|
||||
"""The gate's block rate = Pass / (Pass + Fail + lost), `lost` = the $BASE-Pass cases this side
|
||||
turned NotSupported or another non-Pass, non-Fail status (a warning, a crash): a lost supported
|
||||
case is NON-PASS, it does not leave the denominator (ruling: integrator 2026-09-23 after codex
|
||||
closeout review). Other NotSupported / warnings / crashes stay out of the denominator; crashes
|
||||
get their own count. Information only: CONTRACT-P7 7.3's literal Pass / (Pass + Fail) as
|
||||
`literal_rate` and the pre-ruling Pass / (results - NotSupported) as `results_rate`."""
|
||||
def rate(statuses, lost=0, adjudicated=0):
|
||||
"""The gate's block rate = Pass / (Pass + Fail + lost), `lost` = the UNADJUDICATED $BASE-Pass
|
||||
cases this side turned NotSupported or another non-Pass, non-Fail status (a warning, a crash): a
|
||||
lost supported case is NON-PASS, it does not leave the denominator (ruling: integrator 2026-09-23
|
||||
after codex closeout review). An ADJUDICATED one leaves the block's rate on both sides (ruling:
|
||||
integrator 2026-09-23 after the int3 critic): not an L case here, and on the $BASE side
|
||||
`adjudicated` of its Passes are taken out. Other NotSupported / warnings / crashes stay out of the
|
||||
denominator; crashes get their own count. Information only: CONTRACT-P7 7.3's literal Pass /
|
||||
(Pass + Fail) as `literal_rate` and the pre-ruling Pass / (results - NotSupported) as
|
||||
`results_rate`, both over every compared case."""
|
||||
c = Counter(statuses)
|
||||
p, f, ns = c["Pass"], c["Fail"], c["NotSupported"]
|
||||
x = sum(c[s] for s in HARD)
|
||||
n = len(statuses)
|
||||
g = p - adjudicated
|
||||
return {"pass": p, "fail": f, "ns": ns, "crash": x, "warn": n - p - f - ns - x, "results": n, "lost": lost,
|
||||
"rate": (p / (p + f + lost)) if (p + f + lost) else None,
|
||||
"adjudicated": adjudicated,
|
||||
"rate": (g / (g + f + lost)) if (g + f + lost) else None,
|
||||
"literal_rate": (p / (p + f)) if (p + f) else None,
|
||||
"results_rate": (p / (n - ns)) if (n - ns) else None}
|
||||
|
||||
@@ -724,8 +732,12 @@ def cts_block(c, b, base_dir, canon_dir, adjudicated=None):
|
||||
both = [x for x in universe if x in after and x in base]
|
||||
# A $BASE Pass that AFTER turned NotSupported / a warning / a crash: NON-PASS in the gate rate,
|
||||
# listed, and FAIL unless adjudicated (ruling: integrator 2026-09-23 after codex closeout review).
|
||||
# An adjudicated one is out of the block's rate on BOTH sides (ruling: integrator 2026-09-23 after
|
||||
# the int3 critic - else one ruled-legitimate loss in a 124-case block was -0.806 pp for ever).
|
||||
lost = sorted(x for x in both if base[x] == "Pass" and after[x] not in ("Pass", "Fail"))
|
||||
ra, rb = rate([after[x] for x in both], lost=len(lost)), rate([base[x] for x in both])
|
||||
unadjudicated = [x for x in lost if x not in adjudicated]
|
||||
waived = len(lost) - len(unadjudicated)
|
||||
ra, rb = rate([after[x] for x in both], lost=len(unadjudicated)), rate([base[x] for x in both], adjudicated=waived)
|
||||
|
||||
def pp(key):
|
||||
return None if ra[key] is None or rb[key] is None else (ra[key] - rb[key]) * 100.0
|
||||
@@ -735,7 +747,6 @@ def cts_block(c, b, base_dir, canon_dir, adjudicated=None):
|
||||
regress = sorted(x for x in universe if base.get(x) == "Pass" and after.get(x) not in (None, "Pass"))
|
||||
fixed = sorted(x for x in universe if base.get(x) not in (None, "Pass") and after.get(x) == "Pass")
|
||||
missing_base = sorted(x for x in universe if x not in base)
|
||||
unadjudicated = [x for x in lost if x not in adjudicated]
|
||||
fail, notes, warnings = [], [], []
|
||||
if new_crash:
|
||||
fail.append("%d NEW crash(es)" % len(new_crash))
|
||||
@@ -753,9 +764,9 @@ def cts_block(c, b, base_dir, canon_dir, adjudicated=None):
|
||||
if unadjudicated:
|
||||
fail.append("%d $BASE-Pass case(s) lost to NotSupported / another non-Pass, non-Fail status with no "
|
||||
"cts/adjudication.tsv line (%s)" % (len(unadjudicated), CODEX_RULING))
|
||||
elif lost:
|
||||
notes.append("note: %d $BASE-Pass case(s) lost to NotSupported / another non-Pass, non-Fail status, each "
|
||||
"adjudicated in cts/adjudication.tsv (still NON-PASS in the rate)" % len(lost))
|
||||
if waived:
|
||||
notes.append("note: %d $BASE-Pass case(s) lost to NotSupported / another non-Pass, non-Fail status and "
|
||||
"adjudicated in cts/adjudication.tsv: out of the block's rate, BASE and AFTER" % waived)
|
||||
if missing_base:
|
||||
warnings.append("%d case(s) have no $BASE result (e.g. %s): left out of both rates; a crash among them "
|
||||
"still counts as NEW" % (len(missing_base), missing_base[0]))
|
||||
@@ -771,6 +782,7 @@ def cts_block(c, b, base_dir, canon_dir, adjudicated=None):
|
||||
"new_crash": new_crash, "regressed": regress, "fixed": fixed, "missing_in_base": missing_base,
|
||||
"lost": [{"case": x, "after": after[x], "adjudication": adjudicated.get(x)} for x in lost],
|
||||
"lost_unadjudicated": unadjudicated,
|
||||
"lost_adjudicated": [x for x in lost if x in adjudicated],
|
||||
"unrun": len(unrun), "hung": hung, "verdict": verdict, "reasons": incomplete + fail + notes,
|
||||
"warnings": warnings, "subset": subset}
|
||||
|
||||
@@ -802,10 +814,11 @@ def print_cts(c):
|
||||
print(" rate = Pass/(Pass+Fail+L), L = the $BASE-Pass cases AFTER turned NotSupported or another non-Pass,")
|
||||
print(" non-Fail status (a lost supported case is NON-PASS); other NS, warnings W and crashes X are not in the")
|
||||
print(" denominator. Per block: delta >= -%.1f pp, 0 NEW crash, and every L case adjudicated in" % CTS_TOL_PP)
|
||||
print(" cts/adjudication.tsv (else FAIL); all five blocks required. '(literal)' = the contract's literal")
|
||||
print(" Pass/(Pass+Fail) delta and 'info' the pre-ruling Pass/(results-NS): printed for comparison, never gating.")
|
||||
print("%-14s %6s %-20s %8s %-20s %4s %8s %9s %9s %5s %4s %5s %s" % (
|
||||
"block", "cases", "BASE P/F/NS/W/X", "rate", "AFTER P/F/NS/W/X", "L", "rate", "delta pp", "(literal)",
|
||||
print(" cts/adjudication.tsv (else FAIL); an adjudicated one (A) is out of the block's rate, BASE and AFTER.")
|
||||
print(" All five blocks required. '(literal)' = the contract's literal Pass/(Pass+Fail) delta and 'info' the")
|
||||
print(" pre-ruling Pass/(results-NS), both over every compared case: printed for comparison, never gating.")
|
||||
print("%-14s %6s %-20s %8s %-20s %4s %3s %8s %9s %9s %5s %4s %5s %s" % (
|
||||
"block", "cases", "BASE P/F/NS/W/X", "rate", "AFTER P/F/NS/W/X", "L", "A", "rate", "delta pp", "(literal)",
|
||||
"crash", "regr", "fixed", "verdict"))
|
||||
|
||||
def counts(r):
|
||||
@@ -820,9 +833,10 @@ def print_cts(c):
|
||||
if "after" not in b:
|
||||
print("%-14s %s: %s" % (b["block"], b["verdict"], "; ".join(b["reasons"])))
|
||||
continue
|
||||
print("%-14s %6d %-20s %8s %-20s %4d %8s %9s %9s %5d %4d %5d %s%s" % (
|
||||
print("%-14s %6d %-20s %8s %-20s %4d %3d %8s %9s %9s %5d %4d %5d %s%s" % (
|
||||
b["block"], b["cases"], counts(b["base"]), pct(b["base"]["rate"]), counts(b["after"]),
|
||||
b["after"]["lost"], pct(b["after"]["rate"]), pp(b["delta_pp"]), "(%s)" % pp(b["delta_pp_literal_info"]),
|
||||
b["after"]["lost"], len(b.get("lost_adjudicated", [])), pct(b["after"]["rate"]), pp(b["delta_pp"]),
|
||||
"(%s)" % pp(b["delta_pp_literal_info"]),
|
||||
len(b["new_crash"]), len(b["regressed"]), len(b["fixed"]), b["verdict"],
|
||||
" (SUBSET)" if b["subset"] else ""))
|
||||
print(" info literal Pass/(Pass+Fail): BASE %s AFTER %s delta %s pp (not gating)" % (
|
||||
@@ -836,8 +850,11 @@ def print_cts(c):
|
||||
for x in b["new_crash"]:
|
||||
print(" NEW CRASH " + x)
|
||||
for x in b["lost"]:
|
||||
print(" LOST %s (Pass -> %s) %s" % (x["case"], x["after"], "adjudicated: " + x["adjudication"]
|
||||
if x["adjudication"] else "UNADJUDICATED"))
|
||||
if x["adjudication"]:
|
||||
print(" ADJUDICATED %s (Pass -> %s) out of the rate; adjudicated: %s" % (
|
||||
x["case"], x["after"], x["adjudication"]))
|
||||
else:
|
||||
print(" LOST %s (Pass -> %s) UNADJUDICATED" % (x["case"], x["after"]))
|
||||
lost = {x["case"] for x in b["lost"]}
|
||||
others = [x for x in b["regressed"] if x not in lost]
|
||||
for x in others[:40]:
|
||||
|
||||
@@ -120,7 +120,9 @@ boot_id，与它不同立即停；每对的 `.done` 与每遍归档的 `repeat-N
|
||||
Pass / (Pass + Fail)，ssbo 124 个 `$BASE` Pass 里 123 个变 NS、1 个仍 Pass，读数仍是 100%）；其余 NS、warning、crash 不在分母里。
|
||||
表里 `L` 列是其个数；§7.3 字面的 Pass / (Pass + Fail) 照印在 `(literal)` 列与每块的 `info literal` 行，只供对照、**不判**。
|
||||
L 例逐条列 `LOST <case> (Pass -> <AFTER 状态>)`，块里有任何一条没在 `cts/adjudication.tsv`（每行 `<case><TAB><理由>`；没有理由的行不算，打 WARN）
|
||||
裁定 → 该块 **FAIL**；裁定只免这一条，该例仍按非 Pass 进 rate（所以 ssbo 1 例 Pass→NS 即使裁定，−0.806 pp 仍红）。块判据：delta ≥ −0.5 pp、
|
||||
裁定 → 该块 **FAIL**；**已裁定的 L 例印作 `ADJUDICATED <case> (Pass -> <状态>) out of the rate; adjudicated: <理由>`，并从该块两边的 rate 里拿掉**
|
||||
（不再算 AFTER 分母里的 L，`$BASE` 少一个 Pass；表里 `A` 列是其个数；`(literal)` 与旧口径两行 info 仍把它算在内。**ruling: integrator 2026-09-23 after the int3 critic**——
|
||||
否则 ssbo 1 例合理的 Pass→NS 裁定后仍是 −0.806 pp，永远红）。块判据：delta ≥ −0.5 pp、
|
||||
未裁定的 L 例 = 0、**新增 crash = 0 单独硬红且不可裁定**（AFTER 为 X 而 BASE 不是——Fail→Crash 也算——另加 hung.txt；Pass→Crash 既是新 crash 也是 L 例）、`unrun.txt` 为空。
|
||||
两边的 rate 只在**两边都有结果的用例**上算（同口径）；`$BASE` 没有的用例打 `WARN <block>: N case(s) have no $BASE result`，不进两边的 rate，
|
||||
其中的 crash 仍算新增。AFTER 一个 Pass / Fail 都没有而 `$BASE` 有（例如全变 NotSupported：`$BASE` 的 Pass 全成 L，rate 为 0）→ 该块 **FAIL**；
|
||||
@@ -140,12 +142,13 @@ boot_id，与它不同立即停；每对的 `.done` 与每遍归档的 `repeat-N
|
||||
**跨臂**：spawn 每例不同 → WARN + 像素差、`PASS-NEEDS-ADJUDICATION`，全部写进 `gate3/adjudication.tsv` → PASS，缺一例（无理由行）→ 仍 NEEDS-ADJUDICATION，
|
||||
只一例不同且已裁定（另有陈旧行）→ PASS；臂内不同 → FAIL；**只由 rate 定**的块（dsa 1 / 2 / 4 例 Pass→Fail：−0.270 pp PASS、−0.541 pp FAIL、
|
||||
364/6/0/1/0 −1.081 pp FAIL，无 crash）；**L 例**：dsa Pass→NS 未裁定 → 367/370、−0.270 pp（字面 −0.001 pp 只作信息）、列出、dsa FAIL，裁定后 PASS
|
||||
（仍进 rate），无理由的裁定行不算；ssbo Pass→NS 裁定后仍 −0.806 pp FAIL；codex 的例子 ssbo 123/124 变 NS：字面 100% 而门 rate 0.806% → FAIL；
|
||||
且出 rate（367/369 对 367/369、+0.000 pp、印 `ADJUDICATED`）；一例裁定一例未裁定 → L 1、−0.271 pp、因未裁定例 FAIL；2 例裁定 NS + 2 例 Pass→Fail →
|
||||
−0.543 pp 因 rate FAIL；无理由的裁定行不算；ssbo Pass→NS 裁定后 PASS（+0.000 pp，不再 −0.806）；codex 的例子 ssbo 123/124 变 NS：字面 100% 而门 rate 0.806% → FAIL；
|
||||
Pass→CompatibilityWarning 也是 L；Pass→Crash 裁定后仍因新 crash FAIL；caselist 没了 / 为空 / 全 NotSupported → MISSING / FAIL 而非 PASS；
|
||||
2 例门 3、非 reboot-clean、`--limit` CTS → `PASS-SUBSET` / `PASS-NOT-REBOOT-CLEAN` 且退出非 0；`$BASE` 缺的用例 → WARN；`--check-block` 的 0 / 3；
|
||||
dry run 数据（`60-reduce-selftest-pre14e1c8b9.json` = pre-14e1c8b9 五块相对 `$BASE` 的差异）上 dsa 门 rate −0.541 pp（字面 −0.272 pp、
|
||||
旧口径 −0.539 pp 只作信息）；`~/w7/logs/devprep/w2/pre-14e1c8b9` 在时再对原始输出验一遍。
|
||||
`--reducer <py>` 可对别的 reducer 跑同一套检查（red-once：`6afa077f` 的 reducer 113 项里红 54 项）。
|
||||
`--reducer <py>` 可对别的 reducer 跑同一套检查（red-once：`6afa077f` 的 reducer 113 项里红 54 项；`8f3d179c` 的 reducer 116 项里红 5 项——裁定例仍进 rate）。
|
||||
|
||||
## 4. 入库清单 → `docs/Disaggregated/notes/p7/device-window-2/`
|
||||
|
||||
|
||||
Reference in New Issue
Block a user