mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-09-11 21:58:31 +09:00
[Feat] (tools/cts): automate Windows WGL conformance runs
This commit is contained in:
@@ -0,0 +1,251 @@
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from . import cts_matrix_report as report
|
||||
except ImportError: # Allows `python test_cts_matrix_report.py`.
|
||||
import cts_matrix_report as report
|
||||
|
||||
|
||||
def qpa_case(case, status):
|
||||
return (
|
||||
f"#beginTestCaseResult {case}\n"
|
||||
f'<Result StatusCode="{status}"/>\n'
|
||||
"#endTestCaseResult\n"
|
||||
)
|
||||
|
||||
|
||||
class MatrixReportTests(unittest.TestCase):
|
||||
def test_utf8_bom_caselist_matches_runner_semantics(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
caselist = root / "cases.txt"
|
||||
caselist.write_bytes(b"\xef\xbb\xbfcase.a\n")
|
||||
results = root / "results"
|
||||
results.mkdir()
|
||||
(results / "run.qpa").write_text(
|
||||
qpa_case("case.a", "Pass"), encoding="utf-8"
|
||||
)
|
||||
|
||||
item = report.build_version_report("gl30", str(caselist), [str(results)])
|
||||
|
||||
self.assertEqual(1, item["expected"])
|
||||
self.assertEqual({"case.a": "Pass"}, item["cases"]["results"])
|
||||
self.assertEqual("OK", item["validation"]["state"])
|
||||
|
||||
def test_incomplete_qpa_is_unrun_not_a_completed_result(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
caselist = root / "cases.txt"
|
||||
caselist.write_text("case.a\n", encoding="utf-8")
|
||||
results = root / "results"
|
||||
results.mkdir()
|
||||
(results / "run.qpa").write_text(
|
||||
"#beginTestCaseResult case.a\n#endTestCaseResult\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(results / "unrun.txt").write_text("case.a\n", encoding="utf-8")
|
||||
|
||||
item = report.build_version_report("gl46", str(caselist), [str(results)])
|
||||
|
||||
self.assertEqual(0, item["result"])
|
||||
self.assertEqual(1, item["unrun"])
|
||||
self.assertEqual(["case.a"], item["cases"]["incomplete_results"])
|
||||
self.assertEqual("INCOMPLETE", item["validation"]["state"])
|
||||
|
||||
def test_incomplete_qpa_is_upgraded_by_crash_sidecar(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
caselist = root / "cases.txt"
|
||||
caselist.write_text("case.a\n", encoding="utf-8")
|
||||
results = root / "results"
|
||||
results.mkdir()
|
||||
(results / "run.qpa").write_text(
|
||||
"#beginTestCaseResult case.a\n", encoding="utf-8"
|
||||
)
|
||||
(results / "crashed.txt").write_text("case.a\n", encoding="utf-8")
|
||||
|
||||
item = report.build_version_report("gl46", str(caselist), [str(results)])
|
||||
|
||||
self.assertEqual("Crash", item["cases"]["results"]["case.a"])
|
||||
self.assertEqual([], item["cases"]["incomplete_results"])
|
||||
self.assertEqual("OK", item["validation"]["state"])
|
||||
|
||||
def test_chunk_numbers_above_four_digits_use_numeric_order(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
caselist = root / "cases.txt"
|
||||
caselist.write_text("case.a\n", encoding="utf-8")
|
||||
results = root / "results"
|
||||
results.mkdir()
|
||||
(results / "alpha.qpa").write_text("# no results\n", encoding="utf-8")
|
||||
(results / "chunk9999.qpa").write_text(
|
||||
qpa_case("case.a", "Fail"), encoding="utf-8"
|
||||
)
|
||||
(results / "chunk10000.qpa").write_text(
|
||||
qpa_case("case.a", "Pass"), encoding="utf-8"
|
||||
)
|
||||
(results / "zeta.qpa").write_text("# no results\n", encoding="utf-8")
|
||||
|
||||
item = report.build_version_report(
|
||||
"gl46", str(caselist), [str(results)]
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
["alpha.qpa", "chunk9999.qpa", "chunk10000.qpa", "zeta.qpa"],
|
||||
[Path(path).name for path in item["inputs"]["qpa_files"]],
|
||||
)
|
||||
self.assertEqual("Pass", item["cases"]["results"]["case.a"])
|
||||
self.assertEqual(1, item["duplicate"])
|
||||
|
||||
def test_qpa_sidecars_duplicates_and_expected_denominator(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
caselist = root / "gl30.txt"
|
||||
caselist.write_text("\n".join("abcdefg") + "\n", encoding="utf-8")
|
||||
results = root / "results"
|
||||
results.mkdir()
|
||||
(results / "chunk0000.qpa").write_text(
|
||||
qpa_case("a", "Fail") + "#beginTestCaseResult e\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(results / "chunk0001.qpa").write_text(
|
||||
qpa_case("a", "Pass")
|
||||
+ qpa_case("b", "NotSupported")
|
||||
+ qpa_case("c", "QualityWarning")
|
||||
+ qpa_case("d", "Fail"),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(results / "crashed.txt").write_text("e\n", encoding="utf-8")
|
||||
(results / "hung.txt").write_text("f\n", encoding="utf-8")
|
||||
(results / "unrun.txt").write_text("g\n", encoding="utf-8")
|
||||
|
||||
item = report.build_version_report(
|
||||
"gl30", str(caselist), [str(results)]
|
||||
)
|
||||
|
||||
self.assertEqual(item["expected"], 7)
|
||||
self.assertEqual(item["result"], 6)
|
||||
self.assertEqual(item["pass"], 1)
|
||||
self.assertEqual(item["accepted"], 3)
|
||||
self.assertEqual(item["crash"], 1)
|
||||
self.assertEqual(item["hang"], 1)
|
||||
self.assertEqual(item["unrun"], 1)
|
||||
self.assertEqual(item["duplicate"], 1)
|
||||
self.assertEqual(item["cases"]["results"]["a"], "Pass")
|
||||
self.assertEqual(item["cases"]["results"]["e"], "Crash")
|
||||
self.assertEqual(item["cases"]["results"]["f"], "DeviceHang")
|
||||
self.assertAlmostEqual(item["strict_pass_rate"], 1 / 7)
|
||||
self.assertAlmostEqual(item["conformance_accepted_rate"], 3 / 7)
|
||||
self.assertAlmostEqual(
|
||||
item["rates"]["measured_only_conformance_accepted"], 3 / 6
|
||||
)
|
||||
self.assertEqual(item["validation"]["state"], "INCOMPLETE")
|
||||
self.assertEqual(item["validation"]["errors"], [])
|
||||
self.assertTrue(
|
||||
item["validation"]["invariant_expected_equals_result_plus_unrun"]
|
||||
)
|
||||
|
||||
def test_missing_result_is_inferred_and_rejected_when_not_declared(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
caselist = root / "cases.txt"
|
||||
caselist.write_text("a\nb\n", encoding="utf-8")
|
||||
results = root / "results"
|
||||
results.mkdir()
|
||||
(results / "run.qpa").write_text(qpa_case("a", "Pass"), encoding="utf-8")
|
||||
|
||||
item = report.build_version_report(
|
||||
"gl31", str(caselist), [str(results)]
|
||||
)
|
||||
|
||||
self.assertEqual(item["unrun"], 1)
|
||||
self.assertEqual(item["cases"]["unrun"], ["b"])
|
||||
self.assertEqual(item["validation"]["state"], "ERROR")
|
||||
self.assertEqual(item["validation"]["undeclared_unrun"], ["b"])
|
||||
self.assertIn("not declared", item["validation"]["errors"][0])
|
||||
self.assertEqual(item["strict_pass_rate"], 0.5)
|
||||
|
||||
def test_cli_emits_markdown_json_and_weighted_overall(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
statuses = {
|
||||
"gl30": "Pass",
|
||||
"gl31": "Fail",
|
||||
"gl32": "NotSupported",
|
||||
"gl33": None,
|
||||
}
|
||||
argv = []
|
||||
for version, status in statuses.items():
|
||||
caselist = root / f"{version}.txt"
|
||||
caselist.write_text(f"{version}.case\n", encoding="utf-8")
|
||||
result_dir = root / f"{version}-results"
|
||||
result_dir.mkdir()
|
||||
qpa = result_dir / "run.qpa"
|
||||
qpa.write_text(
|
||||
qpa_case(f"{version}.case", status) if status else "# empty run\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
if status is None:
|
||||
(result_dir / "unrun.txt").write_text(
|
||||
f"{version}.case\n", encoding="utf-8"
|
||||
)
|
||||
argv.extend(
|
||||
[
|
||||
f"--{version}-caselist",
|
||||
str(caselist),
|
||||
f"--{version}-results",
|
||||
str(result_dir),
|
||||
]
|
||||
)
|
||||
json_path = root / "matrix.json"
|
||||
argv.extend(["--json", str(json_path)])
|
||||
|
||||
stdout = io.StringIO()
|
||||
with contextlib.redirect_stdout(stdout):
|
||||
rc = report.main(argv)
|
||||
|
||||
self.assertEqual(rc, 1) # GL33 is explicitly incomplete.
|
||||
markdown = stdout.getvalue()
|
||||
self.assertIn("| Suite | Expected | Result", markdown)
|
||||
self.assertIn("| **Overall (weighted)**", markdown)
|
||||
payload = json.loads(json_path.read_text(encoding="utf-8"))
|
||||
overall = payload["overall"]
|
||||
self.assertEqual(overall["expected"], 4)
|
||||
self.assertEqual(overall["result"], 3)
|
||||
self.assertEqual(overall["pass"], 1)
|
||||
self.assertEqual(overall["accepted"], 2)
|
||||
self.assertEqual(overall["unrun"], 1)
|
||||
self.assertEqual(overall["strict_pass_rate"], 0.25)
|
||||
self.assertEqual(overall["conformance_accepted_rate"], 0.5)
|
||||
self.assertEqual(overall["aggregation"], "weighted_by_expected_cases")
|
||||
self.assertEqual(overall["validation"]["state"], "INCOMPLETE")
|
||||
|
||||
def test_duplicate_caselist_and_unexpected_result_are_validation_errors(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
caselist = root / "cases.txt"
|
||||
caselist.write_text("a\na\n", encoding="utf-8")
|
||||
results = root / "results"
|
||||
results.mkdir()
|
||||
(results / "run.qpa").write_text(
|
||||
qpa_case("a", "Pass") + qpa_case("outside", "Pass"),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
item = report.build_version_report(
|
||||
"gl32", str(caselist), [str(results)]
|
||||
)
|
||||
|
||||
self.assertEqual(item["cases"]["duplicate_caselist_entries"], {"a": 2})
|
||||
self.assertEqual(item["cases"]["unexpected_results"], {"outside": "Pass"})
|
||||
self.assertEqual(item["validation"]["state"], "ERROR")
|
||||
self.assertEqual(len(item["validation"]["errors"]), 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user