From c5569e71b380cebc94aa70f66de38974f8004d95 Mon Sep 17 00:00:00 2001 From: Swung0x48 Date: Thu, 30 Jul 2026 22:59:45 -0400 Subject: [PATCH] [Feat] (tools/cts): automate Windows WGL conformance runs --- .gitignore | 2 + tools/cts/README.md | 29 +- tools/cts/scripts/cts_matrix_report.py | 545 +++++++++++ tools/cts/scripts/cts_multi_report.py | 540 +++++++++++ tools/cts/scripts/run_cts_windows.py | 878 +++++++++++++++++ tools/cts/scripts/test_cts_matrix_report.py | 251 +++++ tools/cts/scripts/test_cts_multi_report.py | 342 +++++++ tools/cts/scripts/test_run_cts_windows.py | 401 ++++++++ tools/cts/scripts/test_wgl_glcts_pipeline.py | 294 ++++++ tools/cts/scripts/wgl_glcts_pipeline.py | 914 ++++++++++++++++++ tools/cts/skills/README.md | 1 + .../skills/wgl-gl-cts-on-mobilegl/SKILL.md | 143 +++ .../wgl-gl-cts-on-mobilegl/agents/openai.yaml | 4 + 13 files changed, 4343 insertions(+), 1 deletion(-) create mode 100644 tools/cts/scripts/cts_matrix_report.py create mode 100644 tools/cts/scripts/cts_multi_report.py create mode 100644 tools/cts/scripts/run_cts_windows.py create mode 100644 tools/cts/scripts/test_cts_matrix_report.py create mode 100644 tools/cts/scripts/test_cts_multi_report.py create mode 100644 tools/cts/scripts/test_run_cts_windows.py create mode 100644 tools/cts/scripts/test_wgl_glcts_pipeline.py create mode 100644 tools/cts/scripts/wgl_glcts_pipeline.py create mode 100644 tools/cts/skills/wgl-gl-cts-on-mobilegl/SKILL.md create mode 100644 tools/cts/skills/wgl-gl-cts-on-mobilegl/agents/openai.yaml diff --git a/.gitignore b/.gitignore index c961228b..f64d7d26 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ MobileGL/MG*/cmake-build* /android-plugin/app/src/trace/jniLibs /android-plugin/local.properties tools/trace_replay/work/ +__pycache__/ +*.py[cod] diff --git a/tools/cts/README.md b/tools/cts/README.md index b8d35b08..1471741e 100644 --- a/tools/cts/README.md +++ b/tools/cts/README.md @@ -1,4 +1,31 @@ -# Running the OpenGL CTS (VK-GL-CTS / KHR-GL33) against MobileGL on Android +# Running the OpenGL CTS against MobileGL + +This directory contains two supported paths: + +- Android arm64 / MobileGL EGL: the KHR-GL33 workflow documented below and in + `skills/gl-cts-on-mobilegl/SKILL.md`. +- Windows x64 / MobileGL WGL: the GL30-GL46 pipeline in + `scripts/wgl_glcts_pipeline.py`, documented by + `skills/wgl-gl-cts-on-mobilegl/SKILL.md`. + +Windows prerequisites are Git, Python 3.9+, CMake, Visual Studio 2022's Desktop +C++ workload, and a Vulkan SDK visible to CMake. DirectVulkan also needs a +working Vulkan loader plus a GPU-vendor ICD and driver; the SDK alone is not a +GPU driver. + +For Windows, start with: + +```powershell +python tools\cts\scripts\wgl_glcts_pipeline.py --help +``` + +The pipeline builds MobileGL as a drop-in `opengl32.dll`, builds or reuses +`glcts.exe`, checks that WGL loaded MobileGL rather than the system driver, +resumes individual suites after crashes/timeouts, and writes Markdown plus JSON +reports below the printed `runs/` directory. Its +manifest records provenance and the runner settings used to validate a resume. + +## Android KHR-GL33 workflow Goal: measure how much of the OpenGL 3.3 core-profile conformance suite MobileGL passes, separately for each backend (`DirectGLES`, `DirectVulkan`). diff --git a/tools/cts/scripts/cts_matrix_report.py b/tools/cts/scripts/cts_matrix_report.py new file mode 100644 index 00000000..a67158aa --- /dev/null +++ b/tools/cts/scripts/cts_matrix_report.py @@ -0,0 +1,545 @@ +#!/usr/bin/env python +"""Build a GL 3.0--3.3 CTS conformance matrix from dEQP QPA logs. + +The report deliberately scores against the unique cases in each supplied +caselist. A case that has not produced a result therefore cannot disappear +from the denominator and make a partial run look conformant. + +QPA parsing and crash/hang sidecar handling follow :mod:`qpa_report`: + +* a later QPA observation of a case wins; +* ``crashed.txt`` upgrades a missing/incomplete result to ``Crash``; +* ``hung.txt`` upgrades a missing/incomplete/crash result to ``DeviceHang``. + +Example:: + + python cts_matrix_report.py \ + --gl30-caselist gl30-main.txt --gl30-results runs/gl30 \ + --gl31-caselist gl31-main.txt --gl31-results runs/gl31 \ + --gl32-caselist gl32-main.txt --gl32-results runs/gl32 \ + --gl33-caselist gl33-main.txt --gl33-results runs/gl33 \ + --json runs/cts-matrix.json +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from collections import Counter, defaultdict +from datetime import datetime, timezone +from typing import Iterable, Optional, Sequence + +try: # Works both as a directly executed script and as a package import. + from . import qpa_report +except ImportError: # pragma: no cover - exercised by the command-line tests + import qpa_report + + +VERSIONS = ("gl30", "gl31", "gl32", "gl33") +ACCEPTED_STATUSES = ( + "Pass", + "NotSupported", + "QualityWarning", + "CompatibilityWarning", + "Waiver", +) +ACCEPTED = frozenset(ACCEPTED_STATUSES) +CHUNK_QPA = re.compile(r"^chunk(\d+)\.qpa$", re.IGNORECASE) + + +class ReportInputError(ValueError): + """An input path cannot be used to construct a meaningful report.""" + + +def _read_non_comment_lines(path: str) -> list[str]: + try: + # Match run_cts_windows.py: Khronos lists are UTF-8 and may carry a BOM. + with open(path, "r", encoding="utf-8-sig", errors="strict") as fh: + return [ + line.strip() + for line in fh + if line.strip() and not line.lstrip().startswith("#") + ] + except (OSError, UnicodeError) as exc: + raise ReportInputError(f"cannot read {path}: {exc}") from exc + + +def read_caselist(path: str) -> tuple[list[str], dict[str, int]]: + """Return unique cases in file order and repeated caselist entries. + + The mustpass files consumed by glcts and ``run_cts.py`` are one case per + non-empty, non-comment line, so this intentionally uses the same syntax. + """ + + entries = _read_non_comment_lines(path) + counts = Counter(entries) + unique = list(dict.fromkeys(entries)) + duplicates = {case: count for case, count in counts.items() if count > 1} + return unique, duplicates + + +def _collect_qpa_files(paths: Sequence[str]) -> list[str]: + missing = [path for path in paths if not os.path.exists(path)] + if missing: + raise ReportInputError( + "result path(s) do not exist: " + ", ".join(sorted(missing)) + ) + + # qpa_report.collect provides the established directory-recursion rules. + # De-duplicate aliases so specifying the same directory twice does not + # manufacture duplicate observations. + files = qpa_report.collect(paths) + by_identity: dict[str, str] = {} + for path in files: + if not os.path.isfile(path): + raise ReportInputError(f"QPA input is not a file: {path}") + absolute = os.path.abspath(path) + by_identity.setdefault(os.path.normcase(absolute), absolute) + def order_key(value: str) -> tuple[str, str, int, str]: + absolute = os.path.abspath(value) + directory = os.path.normcase(os.path.dirname(absolute)) + filename = os.path.normcase(os.path.basename(absolute)) + match = CHUNK_QPA.fullmatch(filename) + if match: + # run_cts_windows.py uses a minimum width of four digits, not a + # fixed width. Numeric ordering is therefore required once a run + # reaches chunk10000; lexical ordering would put it before + # chunk9999 and break the later-observation-wins rule. + return directory, "chunk", int(match.group(1)), filename + # Preserve a deterministic, name-based position for foreign/legacy + # QPA files while grouping numeric runner chunks at the lexical + # position occupied by the "chunk" basename. + return directory, filename, -1, filename + + return sorted(by_identity.values(), key=order_key) + + +def _ratio(numerator: int, denominator: int) -> float: + return numerator / denominator if denominator else 0.0 + + +def _sidecar(paths: Sequence[str], name: str) -> set[str]: + """Load a run_cts.py sidecar with qpa_report-compatible lookup rules.""" + + return qpa_report.load_sidecar(paths, name) + + +def build_version_report( + version: str, caselist: str, result_paths: Sequence[str] +) -> dict: + """Build the serialisable report for one GL mustpass version.""" + + expected_cases, expected_duplicates = read_caselist(caselist) + expected = set(expected_cases) + qpa_files = _collect_qpa_files(result_paths) + + results: dict[str, str] = {} + observation_history: dict[str, list[dict[str, str]]] = defaultdict(list) + for qpa_file in qpa_files: + for case, status in qpa_report.parse_qpa(qpa_file): + observation_history[case].append( + {"file": qpa_file, "status": status} + ) + results[case] = status + + crashed = _sidecar(result_paths, "crashed.txt") + hung = _sidecar(result_paths, "hung.txt") + explicit_unrun = _sidecar(result_paths, "unrun.txt") + skipped = _sidecar(result_paths, "skipped.txt") + + # Keep this order and these guards in lock-step with qpa_report.py. + for case in crashed: + if results.get(case, "Incomplete") == "Incomplete": + results[case] = "Crash" + for case in hung: + if results.get(case, "Incomplete") in ("Incomplete", "Crash"): + results[case] = "DeviceHang" + + # A begin/end pair without , or a QPA truncated mid-case, is not a + # completed observation. Sidecars above may upgrade it to Crash/Hang; + # anything still Incomplete must stay in the expected denominator as unrun. + incomplete_results = { + case for case, status in results.items() if status == "Incomplete" + } + for case in incomplete_results: + del results[case] + + expected_results = { + case: status for case, status in results.items() if case in expected + } + unexpected_results = { + case: status for case, status in results.items() if case not in expected + } + + # Missing cases are inferred from the caselist even if unrun.txt itself is + # missing or stale. This is the invariant that prevents partial-run rate + # inflation. + unrun_cases = expected - set(expected_results) + declared_not_measured = explicit_unrun | skipped + undeclared_unrun = unrun_cases - declared_not_measured + stale_unrun = (explicit_unrun | skipped) & set(expected_results) + + counts = Counter(expected_results.values()) + strict_pass = counts["Pass"] + accepted = sum(counts[status] for status in ACCEPTED) + result_count = len(expected_results) + expected_count = len(expected) + crash_count = counts["Crash"] + hang_count = counts["DeviceHang"] + + duplicate_cases = { + case: { + "observations": len(history), + "extra_observations": len(history) - 1, + "final_status": results.get(case, "Incomplete"), + "history": history, + } + for case, history in sorted(observation_history.items()) + if len(history) > 1 + } + duplicate_observations = sum( + item["extra_observations"] for item in duplicate_cases.values() + ) + + sidecar_unknown = { + name: sorted(cases - expected) + for name, cases in ( + ("crashed.txt", crashed), + ("hung.txt", hung), + ("unrun.txt", explicit_unrun), + ("skipped.txt", skipped), + ) + if cases - expected + } + + errors: list[str] = [] + warnings: list[str] = [] + if not expected_count: + errors.append("caselist has no cases") + if expected_duplicates: + errors.append( + f"caselist has {sum(n - 1 for n in expected_duplicates.values())} " + "duplicate entry/entries" + ) + if not qpa_files: + errors.append("no .qpa files found") + if unexpected_results: + errors.append( + f"{len(unexpected_results)} result case(s) are absent from the caselist" + ) + if sidecar_unknown: + errors.append("one or more sidecars name cases absent from the caselist") + if undeclared_unrun: + errors.append( + f"{len(undeclared_unrun)} missing result case(s) are not declared by " + "unrun.txt/skipped.txt" + ) + if stale_unrun: + warnings.append( + f"{len(stale_unrun)} case(s) declared unrun/skipped also have a result" + ) + if duplicate_observations: + warnings.append( + f"{duplicate_observations} duplicate QPA observation(s); last result wins" + ) + if incomplete_results: + warnings.append( + f"{len(incomplete_results)} QPA case(s) ended without a final result and were treated as unrun" + ) + + if errors: + state = "ERROR" + elif unrun_cases: + state = "INCOMPLETE" + else: + state = "OK" + + return { + "version": version, + "inputs": { + "caselist": os.path.abspath(caselist), + "result_paths": [os.path.abspath(path) for path in result_paths], + "qpa_files": qpa_files, + }, + "expected": expected_count, + "result": result_count, + "pass": strict_pass, + "accepted": accepted, + "crash": crash_count, + "hang": hang_count, + "unrun": len(unrun_cases), + "duplicate": duplicate_observations, + "counts": dict(sorted(counts.items())), + "coverage": { + "numerator": result_count, + "denominator": expected_count, + "rate": _ratio(result_count, expected_count), + }, + "rates": { + # These are the report's conformance rates. Expected, not merely + # measured results, is the denominator. + "denominator": "expected", + "strict_pass_only": _ratio(strict_pass, expected_count), + "conformance_accepted": _ratio(accepted, expected_count), + # Useful for comparison with qpa_report.py, whose denominator is + # cases with a result. Never presented as the conformance rate. + "measured_only_strict_pass": _ratio(strict_pass, result_count), + "measured_only_conformance_accepted": _ratio( + accepted, result_count + ), + }, + "strict_pass_rate": _ratio(strict_pass, expected_count), + "conformance_accepted_rate": _ratio(accepted, expected_count), + "validation": { + "state": state, + "ok": state == "OK", + "errors": errors, + "warnings": warnings, + "invariant_expected_equals_result_plus_unrun": ( + expected_count == result_count + len(unrun_cases) + ), + "undeclared_unrun": sorted(undeclared_unrun), + "stale_unrun_or_skipped": sorted(stale_unrun), + "sidecar_cases_absent_from_caselist": sidecar_unknown, + }, + "cases": { + "results": dict(sorted(expected_results.items())), + "unrun": sorted(unrun_cases), + "unexpected_results": dict(sorted(unexpected_results.items())), + "incomplete_results": sorted(incomplete_results), + "duplicate_results": duplicate_cases, + "duplicate_caselist_entries": dict(sorted(expected_duplicates.items())), + }, + } + + +def build_matrix(suites: dict[str, tuple[str, Sequence[str]]]) -> dict: + """Build all four version reports and their case-weighted aggregate.""" + + version_reports = { + version: build_version_report(version, *suites[version]) + for version in VERSIONS + } + + totals = { + key: sum(report[key] for report in version_reports.values()) + for key in ( + "expected", + "result", + "pass", + "accepted", + "crash", + "hang", + "unrun", + "duplicate", + ) + } + status_counts: Counter[str] = Counter() + for report in version_reports.values(): + status_counts.update(report["counts"]) + + states = {report["validation"]["state"] for report in version_reports.values()} + if "ERROR" in states: + overall_state = "ERROR" + elif "INCOMPLETE" in states: + overall_state = "INCOMPLETE" + else: + overall_state = "OK" + + overall = { + **totals, + "counts": dict(sorted(status_counts.items())), + "aggregation": "weighted_by_expected_cases", + "coverage": { + "numerator": totals["result"], + "denominator": totals["expected"], + "rate": _ratio(totals["result"], totals["expected"]), + }, + "rates": { + "denominator": "expected", + "strict_pass_only": _ratio(totals["pass"], totals["expected"]), + "conformance_accepted": _ratio( + totals["accepted"], totals["expected"] + ), + "measured_only_strict_pass": _ratio( + totals["pass"], totals["result"] + ), + "measured_only_conformance_accepted": _ratio( + totals["accepted"], totals["result"] + ), + }, + "strict_pass_rate": _ratio(totals["pass"], totals["expected"]), + "conformance_accepted_rate": _ratio( + totals["accepted"], totals["expected"] + ), + "validation": { + "state": overall_state, + "ok": overall_state == "OK", + "invariant_expected_equals_result_plus_unrun": ( + totals["expected"] == totals["result"] + totals["unrun"] + ), + }, + } + + return { + "schema_version": 1, + "generated_at": datetime.now(timezone.utc).isoformat(), + "accepted_statuses": list(ACCEPTED_STATUSES), + "rate_policy": { + "denominator": "unique expected cases from each caselist", + "unrun_cases": "included in the denominator and never accepted", + "duplicate_results": "last QPA result wins, matching qpa_report.py", + }, + "versions": version_reports, + "overall": overall, + } + + +def _percent(numerator: int, denominator: int) -> str: + if not denominator: + return "n/a" + return f"{100.0 * numerator / denominator:.2f}% ({numerator}/{denominator})" + + +def render_markdown(report: dict) -> str: + """Render the compact terminal-facing conformance table.""" + + header = ( + "| Suite | Expected | Result | Pass | Accepted | Crash | Hang | Unrun | " + "Duplicate | Coverage | Strict Pass-only | Conformance-accepted | Validation |" + ) + separator = ( + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|:---:|" + ) + rows = [header, separator] + for version in VERSIONS: + item = report["versions"][version] + rows.append( + "| {version} | {expected} | {result} | {pass_count} | {accepted} | " + "{crash} | {hang} | {unrun} | {duplicate} | {coverage} | {strict} | " + "{accepted_rate} | {state} |".format( + version=version.upper(), + expected=item["expected"], + result=item["result"], + pass_count=item["pass"], + accepted=item["accepted"], + crash=item["crash"], + hang=item["hang"], + unrun=item["unrun"], + duplicate=item["duplicate"], + coverage=_percent(item["result"], item["expected"]), + strict=_percent(item["pass"], item["expected"]), + accepted_rate=_percent(item["accepted"], item["expected"]), + state=item["validation"]["state"], + ) + ) + + overall = report["overall"] + rows.append( + "| **Overall (weighted)** | **{expected}** | **{result}** | **{pass_count}** | " + "**{accepted}** | **{crash}** | **{hang}** | **{unrun}** | **{duplicate}** | " + "**{coverage}** | **{strict}** | **{accepted_rate}** | **{state}** |".format( + expected=overall["expected"], + result=overall["result"], + pass_count=overall["pass"], + accepted=overall["accepted"], + crash=overall["crash"], + hang=overall["hang"], + unrun=overall["unrun"], + duplicate=overall["duplicate"], + coverage=_percent(overall["result"], overall["expected"]), + strict=_percent(overall["pass"], overall["expected"]), + accepted_rate=_percent(overall["accepted"], overall["expected"]), + state=overall["validation"]["state"], + ) + ) + rows.extend( + ( + "", + "Rates use unique **Expected** caselist cases as the denominator; unrun cases " + "remain in that denominator and are not accepted.", + "Accepted statuses: " + ", ".join(f"`{s}`" for s in ACCEPTED_STATUSES) + ".", + "Duplicate is the number of extra QPA observations; the last observation wins.", + ) + ) + + details: list[str] = [] + for version in VERSIONS: + validation = report["versions"][version]["validation"] + messages = validation["errors"] + validation["warnings"] + if messages: + details.append( + f"- **{version.upper()} {validation['state']}**: " + "; ".join(messages) + ) + if details: + rows.extend(("", "Validation details:", "", *details)) + return "\n".join(rows) + + +def _write_json(path: str, report: dict) -> None: + parent = os.path.dirname(os.path.abspath(path)) + os.makedirs(parent, exist_ok=True) + with open(path, "w", encoding="utf-8", newline="\n") as fh: + json.dump(report, fh, indent=2, sort_keys=True) + fh.write("\n") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + for version in VERSIONS: + parser.add_argument( + f"--{version}-caselist", + f"--{version}-case-list", + required=True, + help=f"{version.upper()} mustpass caselist", + ) + parser.add_argument( + f"--{version}-results", + f"--{version}-result-dir", + f"--{version}-results-dir", + action="append", + required=True, + help=f"{version.upper()} result directory or QPA file (repeatable)", + ) + parser.add_argument( + "--json", + dest="json_out", + default="cts_matrix_report.json", + help="JSON output path (default: ./cts_matrix_report.json)", + ) + parser.add_argument( + "--allow-incomplete", + action="store_true", + help="return success even when validation is ERROR/INCOMPLETE", + ) + return parser + + +def main(argv: Optional[Iterable[str]] = None) -> int: + args = _parser().parse_args(argv) + suites = { + version: ( + getattr(args, f"{version}_caselist"), + getattr(args, f"{version}_results"), + ) + for version in VERSIONS + } + try: + report = build_matrix(suites) + _write_json(args.json_out, report) + except (OSError, ReportInputError) as exc: + print(f"cts_matrix_report: {exc}", file=sys.stderr) + return 2 + + print(render_markdown(report)) + print(f"\nJSON: {os.path.abspath(args.json_out)}") + if not args.allow_incomplete and not report["overall"]["validation"]["ok"]: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/cts/scripts/cts_multi_report.py b/tools/cts/scripts/cts_multi_report.py new file mode 100644 index 00000000..967b8c2d --- /dev/null +++ b/tools/cts/scripts/cts_multi_report.py @@ -0,0 +1,540 @@ +#!/usr/bin/env python +"""Summarise any number of GL CTS suites and MobileGL backends. + +Each repeatable suite specification consists of four values: backend, label, +caselist, and result directory. For example:: + + python cts_multi_report.py \ + --suite DirectGLES gl30 gl30-main.txt runs/gles/gl30 \ + --suite DirectVulkan gl30 gl30-main.txt runs/vulkan/gl30 \ + --markdown cts-summary.md --json cts-summary.json + +A compact comma form is accepted as well:: + + --suite=DirectGLES,gl31,gl31-main.txt,runs/gles/gl31 + +Per-suite parsing and validation deliberately delegate to +``cts_matrix_report`` so QPA ordering, sidecar upgrades, accepted statuses, +unrun handling, and duplicate-result semantics cannot drift between reports. +All conformance rates use unique expected caselist cases as their denominator. +Backend subtotals and the overall total are therefore case-weighted, not an +unweighted average of suite percentages. +""" + +from __future__ import annotations + +import argparse +from collections import Counter +from dataclasses import dataclass +from datetime import datetime, timezone +import hashlib +import json +import os +import sys +from typing import Iterable, Optional, Sequence + +try: # Direct script execution and package imports are both supported. + from . import cts_matrix_report +except ImportError: # pragma: no cover - covered through CLI-style tests + import cts_matrix_report + + +SUPPORTED_BACKENDS = ("DirectGLES", "DirectVulkan") +SUM_FIELDS = ( + "expected", + "result", + "pass", + "accepted", + "crash", + "hang", + "unrun", + "duplicate", +) + + +class MultiReportInputError(ValueError): + """Suite specifications cannot produce an unambiguous report.""" + + +@dataclass(frozen=True) +class SuiteSpec: + backend: str + label: str + caselist: str + result_dir: str + + @property + def suite_id(self) -> str: + return f"{self.backend}/{self.label}" + + +def _ratio(numerator: int, denominator: int) -> float: + return numerator / denominator if denominator else 0.0 + + +def _validation_state(items: Sequence[dict]) -> str: + states = {item["validation"]["state"] for item in items} + if "ERROR" in states: + return "ERROR" + if "INCOMPLETE" in states: + return "INCOMPLETE" + return "OK" + + +def aggregate_reports(items: Sequence[dict], suite_state_keys: Sequence[str]) -> dict: + """Return an expected-case-weighted aggregate for suite reports.""" + + if len(items) != len(suite_state_keys): + raise MultiReportInputError("internal suite/state key count mismatch") + + totals = { + field: sum(int(item[field]) for item in items) + for field in SUM_FIELDS + } + status_counts: Counter[str] = Counter() + for item in items: + status_counts.update(item["counts"]) + + state = _validation_state(items) + expected = totals["expected"] + result = totals["result"] + suite_states = { + key: item["validation"]["state"] + for key, item in zip(suite_state_keys, items) + } + return { + **totals, + "suite_count": len(items), + "counts": dict(sorted(status_counts.items())), + "aggregation": "weighted_by_expected_cases", + "coverage": { + "numerator": result, + "denominator": expected, + "rate": _ratio(result, expected), + }, + "rates": { + "denominator": "expected", + "strict_pass_only": _ratio(totals["pass"], expected), + "conformance_accepted": _ratio(totals["accepted"], expected), + "measured_only_strict_pass": _ratio(totals["pass"], result), + "measured_only_conformance_accepted": _ratio( + totals["accepted"], result + ), + }, + # Keep the convenient aliases used by cts_matrix_report consumers. + "strict_pass_rate": _ratio(totals["pass"], expected), + "conformance_accepted_rate": _ratio(totals["accepted"], expected), + "validation": { + "state": state, + "ok": state == "OK", + "suite_states": suite_states, + "invariant_expected_equals_result_plus_unrun": ( + expected == result + totals["unrun"] + ), + }, + } + + +def _caselist_fingerprint(path: str) -> tuple[str, int]: + cases, _duplicates = cts_matrix_report.read_caselist(path) + payload = "\n".join(cases).encode("utf-8") + b"\n" + return hashlib.sha256(payload).hexdigest(), len(cases) + + +def _read_provenance( + spec: SuiteSpec, + require_run_state: bool, + expected_run_identity: Optional[str], +) -> dict: + path = os.path.join(spec.result_dir, "run_state.json") + if not os.path.isfile(path): + if require_run_state or expected_run_identity is not None: + raise MultiReportInputError( + "suite result directory has no run_state.json; invocation provenance " + f"cannot be verified: {spec.result_dir}" + ) + return {"state": "UNVERIFIED", "run_state": None} + try: + with open(path, "r", encoding="utf-8") as handle: + state = json.load(handle) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise MultiReportInputError(f"cannot read suite run identity {path}: {exc}") from exc + if not isinstance(state, dict): + raise MultiReportInputError(f"suite run identity must be a JSON object: {path}") + if state.get("backend") != spec.backend: + raise MultiReportInputError( + f"suite {spec.suite_id} is labelled {spec.backend}, but run_state.json " + f"records {state.get('backend')!r}" + ) + fingerprint, case_count = _caselist_fingerprint(spec.caselist) + if state.get("caselist_sha256") != fingerprint or state.get("case_count") != case_count: + raise MultiReportInputError( + f"suite {spec.suite_id} run_state.json belongs to a different caselist" + ) + invocation_identity = state.get("invocation_identity") + if ( + expected_run_identity is not None + and invocation_identity != expected_run_identity + ): + raise MultiReportInputError( + f"suite {spec.suite_id} run_state.json belongs to a different CTS invocation" + ) + return { + "state": "VERIFIED", + "run_state": os.path.abspath(path), + "invocation_identity": invocation_identity, + } + + +def _validate_specs( + specs: Sequence[SuiteSpec], + require_run_state: bool, + expected_run_identity: Optional[str], +) -> dict[str, dict]: + if not specs: + raise MultiReportInputError("at least one --suite specification is required") + + seen: set[tuple[str, str]] = set() + seen_result_dirs: list[tuple[str, str]] = [] + provenance: dict[str, dict] = {} + for spec in specs: + if spec.backend not in SUPPORTED_BACKENDS: + raise MultiReportInputError( + f"unsupported backend {spec.backend!r}; expected one of " + + ", ".join(SUPPORTED_BACKENDS) + ) + if not spec.label.strip(): + raise MultiReportInputError("suite label cannot be empty") + identity = (spec.backend, spec.label) + if identity in seen: + raise MultiReportInputError( + f"duplicate suite specification for {spec.backend}/{spec.label}" + ) + seen.add(identity) + if not os.path.isdir(spec.result_dir): + raise MultiReportInputError( + f"suite result directory does not exist: {spec.result_dir}" + ) + physical_result_dir = os.path.normcase( + os.path.realpath(os.path.abspath(spec.result_dir)) + ) + for previous_dir, previous_suite in seen_result_dirs: + try: + common_dir = os.path.commonpath( + [previous_dir, physical_result_dir] + ) + except ValueError: + continue + if common_dir in (previous_dir, physical_result_dir): + raise MultiReportInputError( + f"suite {spec.suite_id} uses a result directory which overlaps " + f"{previous_suite}: {spec.result_dir}" + ) + seen_result_dirs.append((physical_result_dir, spec.suite_id)) + provenance[spec.suite_id] = _read_provenance( + spec, require_run_state, expected_run_identity + ) + return provenance + + +def build_report( + specs: Sequence[SuiteSpec], + require_run_state: bool = True, + expected_run_identity: Optional[str] = None, +) -> dict: + """Build suite, per-backend, and overall serialisable reports.""" + + provenance = _validate_specs( + specs, require_run_state, expected_run_identity + ) + suite_reports: list[dict] = [] + backend_order: list[str] = [] + + for spec in specs: + if spec.backend not in backend_order: + backend_order.append(spec.backend) + item = cts_matrix_report.build_version_report( + spec.label, spec.caselist, [spec.result_dir] + ) + # ``version`` is the generic label argument in build_version_report; + # expose explicit multi-report terminology while retaining all of its + # validation and case-level evidence. + item.pop("version", None) + item["backend"] = spec.backend + item["label"] = spec.label + item["suite_id"] = spec.suite_id + item["provenance"] = provenance[spec.suite_id] + if item["provenance"]["state"] == "UNVERIFIED": + item["validation"]["warnings"].append( + "result directory has no run_state.json; backend provenance is unverified" + ) + suite_reports.append(item) + + backends: dict[str, dict] = {} + for backend in backend_order: + backend_items = [ + item for item in suite_reports if item["backend"] == backend + ] + labels = [item["label"] for item in backend_items] + aggregate = aggregate_reports(backend_items, labels) + aggregate["backend"] = backend + aggregate["suite_labels"] = labels + backends[backend] = aggregate + + overall = aggregate_reports( + suite_reports, [item["suite_id"] for item in suite_reports] + ) + overall["backend_count"] = len(backends) + overall["backends"] = backend_order + + return { + "schema_version": 1, + "generated_at": datetime.now(timezone.utc).isoformat(), + "accepted_statuses": list(cts_matrix_report.ACCEPTED_STATUSES), + "rate_policy": { + "denominator": "unique expected cases from each suite caselist", + "unrun_cases": "included in the denominator and never accepted", + "backend_aggregation": "weighted by expected cases", + "overall_aggregation": "weighted by expected cases across backend-suite pairs", + "duplicate_results": "last QPA result wins, matching qpa_report.py", + }, + "suites": suite_reports, + "backends": backends, + "overall": overall, + } + + +def _percent(numerator: int, denominator: int) -> str: + if not denominator: + return "n/a" + return f"{100.0 * numerator / denominator:.2f}% ({numerator}/{denominator})" + + +def _markdown_cell(value: object) -> str: + return str(value).replace("|", r"\|").replace("\r", " ").replace("\n", " ") + + +def _table_row(backend: str, label: str, item: dict, bold: bool = False) -> str: + values = [ + backend, + label, + str(item["expected"]), + str(item["result"]), + str(item["pass"]), + str(item["accepted"]), + str(item["crash"]), + str(item["hang"]), + str(item["unrun"]), + str(item["duplicate"]), + _percent(item["result"], item["expected"]), + _percent(item["pass"], item["expected"]), + _percent(item["accepted"], item["expected"]), + item["validation"]["state"], + ] + values = [_markdown_cell(value) for value in values] + if bold: + values = [f"**{value}**" for value in values] + return "| " + " | ".join(values) + " |" + + +def render_markdown(report: dict) -> str: + lines = [ + "# GL CTS multi-suite conformance report", + "", + ( + "| Backend | Suite | Expected | Result | Pass | Accepted | Crash | Hang | " + "Unrun | Duplicate | Coverage | Strict Pass-only | Conformance-accepted | Validation |" + ), + "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|:---:|", + ] + + for backend in report["backends"]: + for item in report["suites"]: + if item["backend"] == backend: + lines.append(_table_row(backend, item["label"], item)) + subtotal = report["backends"][backend] + lines.append( + _table_row(backend, f"{backend} weighted subtotal", subtotal, bold=True) + ) + + lines.append( + _table_row( + "All backends", + "Overall weighted", + report["overall"], + bold=True, + ) + ) + lines.extend( + [ + "", + ( + "Rates use unique **Expected** caselist cases as the denominator. " + "Unrun cases remain in the denominator and are not accepted." + ), + "Accepted statuses: " + + ", ".join( + f"`{status}`" for status in report["accepted_statuses"] + ) + + ".", + ( + "Duplicate is the number of extra QPA observations; the final " + "observation wins." + ), + ] + ) + + details: list[str] = [] + for item in report["suites"]: + validation = item["validation"] + messages = validation["errors"] + validation["warnings"] + if messages: + details.append( + f"- **{_markdown_cell(item['suite_id'])} {validation['state']}**: " + + "; ".join(_markdown_cell(message) for message in messages) + ) + if details: + lines.extend(["", "## Validation details", "", *details]) + return "\n".join(lines) + "\n" + + +def _write_text(path: str, contents: str) -> None: + absolute = os.path.abspath(path) + os.makedirs(os.path.dirname(absolute), exist_ok=True) + with open(absolute, "w", encoding="utf-8", newline="\n") as handle: + handle.write(contents) + + +def _write_json(path: str, report: dict) -> None: + _write_text(path, json.dumps(report, indent=2, sort_keys=True) + "\n") + + +def _normalise_compact_suite_args(argv: Sequence[str]) -> list[str]: + """Expand ``--suite=b,l,c,r`` into the four-value argparse form.""" + + result: list[str] = [] + index = 0 + while index < len(argv): + token = argv[index] + if token.startswith("--suite="): + compact = token.split("=", 1)[1] + parts = compact.split(",", 3) + if len(parts) != 4: + raise MultiReportInputError( + "compact --suite expects backend,label,caselist,result-dir" + ) + result.extend(["--suite", *parts]) + index += 1 + continue + if token == "--suite" and index + 1 < len(argv) and argv[index + 1].count(",") >= 3: + parts = argv[index + 1].split(",", 3) + result.extend(["--suite", *parts]) + index += 2 + continue + result.append(token) + index += 1 + return result + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--suite", + action="append", + nargs=4, + required=True, + metavar=("BACKEND", "LABEL", "CASELIST", "RESULT_DIR"), + help=( + "suite specification; repeat for every backend/suite pair " + f"(backends: {', '.join(SUPPORTED_BACKENDS)})" + ), + ) + parser.add_argument( + "--markdown", + default="cts_multi_report.md", + help="Markdown output path (default: ./cts_multi_report.md)", + ) + parser.add_argument( + "--json", + dest="json_out", + default="cts_multi_report.json", + help="JSON output path (default: ./cts_multi_report.json)", + ) + parser.add_argument( + "--allow-incomplete", + action="store_true", + help="return success even when one or more suites are ERROR/INCOMPLETE", + ) + parser.add_argument( + "--adopt-legacy", + dest="allow_unverified_provenance", + action="store_true", + help="accept legacy result directories without run_state.json (provenance remains unverified)", + ) + parser.add_argument( + "--allow-unverified-provenance", + dest="allow_unverified_provenance", + action="store_true", + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--expected-run-identity", + help="require every suite run_state.json to contain this controller fingerprint", + ) + return parser + + +def _specs_from_args(values: Sequence[Sequence[str]]) -> list[SuiteSpec]: + return [ + SuiteSpec( + backend=backend.strip(), + label=label.strip(), + caselist=caselist, + result_dir=result_dir, + ) + for backend, label, caselist, result_dir in values + ] + + +def main(argv: Optional[Iterable[str]] = None) -> int: + raw_argv = list(argv) if argv is not None else sys.argv[1:] + try: + normalised = _normalise_compact_suite_args(raw_argv) + except MultiReportInputError as exc: + print(f"cts_multi_report: {exc}", file=sys.stderr) + return 2 + args = _parser().parse_args(normalised) + + if os.path.normcase(os.path.abspath(args.markdown)) == os.path.normcase( + os.path.abspath(args.json_out) + ): + print("cts_multi_report: Markdown and JSON paths must differ", file=sys.stderr) + return 2 + + try: + report = build_report( + _specs_from_args(args.suite), + require_run_state=not args.allow_unverified_provenance, + expected_run_identity=args.expected_run_identity, + ) + markdown = render_markdown(report) + _write_text(args.markdown, markdown) + _write_json(args.json_out, report) + except ( + OSError, + MultiReportInputError, + cts_matrix_report.ReportInputError, + ) as exc: + print(f"cts_multi_report: {exc}", file=sys.stderr) + return 2 + + print(markdown, end="") + print(f"\nMarkdown: {os.path.abspath(args.markdown)}") + print(f"JSON: {os.path.abspath(args.json_out)}") + if not args.allow_incomplete and not report["overall"]["validation"]["ok"]: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/cts/scripts/run_cts_windows.py b/tools/cts/scripts/run_cts_windows.py new file mode 100644 index 00000000..80b90da5 --- /dev/null +++ b/tools/cts/scripts/run_cts_windows.py @@ -0,0 +1,878 @@ +#!/usr/bin/env python +"""Run a local Windows glcts executable and resume across process failures. + +The desktop CTS normally runs a complete caselist in one process. That is a +poor fit for testing a developing OpenGL implementation: one access violation +or GPU hang prevents every later case from running. This driver gives each +invocation the cases which have not produced a result yet, preserves one QPA +and stdout/stderr pair per invocation, and starts another process after a +crash. + +Existing ``chunkNNNN.qpa`` files and ``crashed.txt``/``hung.txt`` sidecars are +read on startup, so invoking the same command and output directory resumes an +interrupted run. A timeout is based on *idle QPA time*, not total process wall +time: a healthy invocation may legitimately run thousands of cases for hours. + +Example (values beginning with ``--`` use argparse's ``=`` spelling):: + + py run_cts_windows.py \ + --exe D:\\glcts\\glcts.exe --workdir D:\\glcts \ + --caselist D:\\glcts\\mustpass\\gl30.txt --outdir D:\\results\\gl30 \ + --backend DirectVulkan \ + --deqp-arg=--deqp-surface-type=window +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Iterable, Optional, Sequence + + +CASE_START = re.compile(r"^#beginTestCaseResult\s+(\S+)") +CASE_END = re.compile(r"^#endTestCaseResult(?:\s|$)") +CASE_TERM = re.compile(r"^#terminateTestCaseResult(?:\s|$)") +CASE_RESULT = re.compile(r' str: + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def read_caselist(path: Path) -> list[str]: + """Read a dEQP text caselist, preserving order and removing duplicates.""" + + try: + lines = path.read_text(encoding="utf-8-sig", errors="strict").splitlines() + except (OSError, UnicodeError) as exc: + raise RunnerError(f"cannot read caselist {path}: {exc}") from exc + + cases: list[str] = [] + seen: set[str] = set() + for raw in lines: + case = raw.strip() + if not case or case.startswith("#") or case in seen: + continue + cases.append(case) + seen.add(case) + if not cases: + raise RunnerError(f"caselist contains no test cases: {path}") + return cases + + +def read_name_set(path: Path) -> set[str]: + if not path.is_file(): + return set() + try: + return { + line.strip() + for line in path.read_text(encoding="utf-8-sig", errors="replace").splitlines() + if line.strip() and not line.lstrip().startswith("#") + } + except OSError as exc: + raise RunnerError(f"cannot read recovery file {path}: {exc}") from exc + + +def _replace_with_retry(source: Path, destination: Path) -> None: + """Replace a state file, tolerating brief Windows access-denied races. + + Antivirus/indexing tools can momentarily open ``remaining.txt`` without + delete sharing. Windows then reports either ``PermissionError`` or a + generic ``OSError`` carrying ``winerror == 5``. Retry only those cases; + disk, path, and programming errors remain immediately visible. + """ + + for attempt in range(ATOMIC_REPLACE_ATTEMPTS): + try: + os.replace(source, destination) + return + except OSError as exc: + retryable = isinstance(exc, PermissionError) or getattr(exc, "winerror", None) == 5 + if not retryable or attempt + 1 >= ATOMIC_REPLACE_ATTEMPTS: + raise + delay = min( + ATOMIC_REPLACE_INITIAL_BACKOFF_SECONDS * (2**attempt), + ATOMIC_REPLACE_MAX_BACKOFF_SECONDS, + ) + time.sleep(delay) + + +def atomic_write_text(path: Path, text: str) -> None: + """Replace a small state file without exposing a partially-written copy.""" + + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=str(path.parent)) + temporary_path = Path(temporary) + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle: + handle.write(text) + handle.flush() + os.fsync(handle.fileno()) + _replace_with_retry(temporary_path, path) + finally: + try: + temporary_path.unlink() + except FileNotFoundError: + pass + + +def atomic_write_json(path: Path, value: object) -> None: + atomic_write_text(path, json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def write_case_file(path: Path, cases: Iterable[str]) -> None: + values = list(cases) + atomic_write_text(path, "\n".join(values) + ("\n" if values else "")) + + +def scan_qpa(path: Path) -> QpaProgress: + """Return cases with a final result and the unfinished tail, if any. + + ``#terminateTestCaseResult`` is a completed result (usually Crash or + Timeout). ``#endTestCaseResult`` only completes a case when its XML carried + a ````. A truncated case that already wrote Result is + also recoverable; a case with no Result remains eligible for a retry. + """ + + if not path.is_file(): + return QpaProgress([], None, 0) + + recorded: list[str] = [] + current: Optional[str] = None + has_result = False + begin_count = 0 + try: + with path.open("r", encoding="utf-8", errors="replace") as handle: + for raw_line in handle: + line = raw_line.lstrip("\ufeff") + match = CASE_START.match(line) + if match: + if current is not None and has_result: + recorded.append(current) + current = match.group(1) + has_result = False + begin_count += 1 + continue + if current is not None and CASE_RESULT.search(line): + has_result = True + continue + if current is not None and CASE_TERM.match(line): + recorded.append(current) + current = None + has_result = False + continue + if current is not None and CASE_END.match(line): + if has_result: + recorded.append(current) + current = None + has_result = False + except OSError as exc: + raise RunnerError(f"cannot read QPA {path}: {exc}") from exc + if current is not None and has_result: + recorded.append(current) + current = None + return QpaProgress(recorded, current, begin_count) + + +def numbered_files(outdir: Path, pattern: re.Pattern[str]) -> list[tuple[int, Path]]: + found: list[tuple[int, Path]] = [] + try: + children = list(outdir.iterdir()) + except OSError as exc: + raise RunnerError(f"cannot list output directory {outdir}: {exc}") from exc + for path in children: + match = pattern.match(path.name) + if match: + found.append((int(match.group(1)), path)) + found.sort(key=lambda item: item[0]) + return found + + +def next_chunk_number(outdir: Path) -> int: + numbers = [number for number, _path in numbered_files(outdir, CHUNK_ARTIFACT)] + return max(numbers, default=-1) + 1 + + +def load_meta_classifications(outdir: Path, expected: set[str]) -> tuple[set[str], set[str]]: + """Recover an atomic classification written just before sidecar updates.""" + + crashed: set[str] = set() + hung: set[str] = set() + for _number, path in numbered_files(outdir, CHUNK_META): + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + # A damaged metadata file is diagnostic only. QPA and sidecars are + # authoritative and must still allow recovery. + continue + if not isinstance(value, dict): + continue + case = value.get("classified_case") + classification = value.get("classification") + if not isinstance(case, str) or case not in expected: + continue + if classification == "DeviceHang": + hung.add(case) + elif classification == "Crash": + crashed.add(case) + crashed.difference_update(hung) + return crashed, hung + + +def result_qpa_files(outdir: Path) -> list[Path]: + """Return every QPA a directory-based report would consume.""" + + found: list[Path] = [] + try: + for root, directories, names in os.walk(outdir): + directories.sort(key=str.casefold) + for name in sorted(names, key=str.casefold): + if name.casefold().endswith(".qpa"): + found.append(Path(root) / name) + except OSError as exc: + raise RunnerError(f"cannot scan output directory {outdir}: {exc}") from exc + return found + + +def recover_results(outdir: Path, expected: set[str]) -> tuple[set[str], set[str], set[str]]: + recorded: set[str] = set() + for path in result_qpa_files(outdir): + progress = scan_qpa(path) + recorded.update(case for case in progress.recorded if case in expected) + + crashed = read_name_set(outdir / "crashed.txt") & expected + hung = read_name_set(outdir / "hung.txt") & expected + meta_crashed, meta_hung = load_meta_classifications(outdir, expected) + crashed.update(meta_crashed) + hung.update(meta_hung) + crashed.difference_update(hung) + return recorded, crashed, hung + + +def caselist_fingerprint(cases: Sequence[str]) -> str: + payload = "\n".join(cases).encode("utf-8") + b"\n" + return hashlib.sha256(payload).hexdigest() + + +def recovery_artifacts(outdir: Path) -> list[Path]: + """Return prior-run evidence which must not be adopted implicitly.""" + + found = set(result_qpa_files(outdir)) + try: + children = list(outdir.iterdir()) + except OSError as exc: + raise RunnerError(f"cannot list output directory {outdir}: {exc}") from exc + found.update( + path + for path in children + if CHUNK_ARTIFACT.match(path.name) + or path.name.casefold() in RECOVERY_SIDECAR_NAMES + ) + return sorted( + found, + key=lambda path: str(path.relative_to(outdir)).casefold(), + ) + + +def check_run_identity( + outdir: Path, + backend: str, + cases: Sequence[str], + invocation_identity: Optional[str] = None, + adopt_legacy: bool = False, +) -> None: + """Refuse to silently mix different suites/backends in one result dir.""" + + path = outdir / "run_state.json" + fingerprint = caselist_fingerprint(cases) + if path.is_file(): + try: + state = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise RunnerError(f"cannot read run identity {path}: {exc}") from exc + if not isinstance(state, dict): + raise RunnerError(f"run identity must be a JSON object: {path}") + if state.get("backend") != backend: + raise RunnerError( + f"output directory belongs to backend {state.get('backend')!r}, not {backend!r}: {outdir}" + ) + if state.get("caselist_sha256") != fingerprint: + raise RunnerError(f"output directory belongs to a different caselist: {outdir}") + stored_invocation_identity = state.get("invocation_identity") + if ( + stored_invocation_identity is not None + or invocation_identity is not None + ) and stored_invocation_identity != invocation_identity: + raise RunnerError(f"output directory belongs to a different CTS invocation: {outdir}") + return + + legacy_artifacts = recovery_artifacts(outdir) + if legacy_artifacts and not adopt_legacy: + examples = ", ".join(path.name for path in legacy_artifacts[:3]) + raise RunnerError( + "output directory contains CTS recovery artifacts but no run_state.json; " + f"refusing to adopt unverified legacy results ({examples}). Re-run with " + "--adopt-legacy only after verifying the backend, caselist, and invocation." + ) + + atomic_write_json( + path, + { + "version": 1, + "backend": backend, + "case_count": len(cases), + "caselist_sha256": fingerprint, + "invocation_identity": invocation_identity, + "adopted_legacy": bool(legacy_artifacts), + "created_utc": utc_now(), + }, + ) + + +def persist_sidecars( + outdir: Path, + ordered_cases: Sequence[str], + crashed: set[str], + hung: set[str], + remaining: Sequence[str], +) -> None: + write_case_file(outdir / "crashed.txt", (case for case in ordered_cases if case in crashed)) + write_case_file(outdir / "hung.txt", (case for case in ordered_cases if case in hung)) + write_case_file(outdir / "unrun.txt", remaining) + write_case_file(outdir / "remaining.txt", remaining) + + +def parse_environment(values: Sequence[str]) -> dict[str, str]: + result: dict[str, str] = {} + for value in values: + if "=" not in value: + raise RunnerError(f"--env expects NAME=VALUE, got {value!r}") + name, contents = value.split("=", 1) + if not name or "\x00" in name or "=" in name: + raise RunnerError(f"invalid environment variable name in {value!r}") + result[name] = contents + return result + + +def validate_deqp_args(values: Sequence[str]) -> None: + for value in values: + option = value.split("=", 1)[0].lower() + if option in CONTROLLED_DEQP_OPTIONS: + raise RunnerError(f"{option} is controlled by this runner and cannot be supplied via --deqp-arg") + + +def resolve_paths( + exe_value: str, + workdir_value: Optional[str], + caselist_value: str, + outdir_value: str, +) -> tuple[Path, Path, Path, Path]: + launch_dir = Path.cwd() + requested_exe = Path(exe_value).expanduser() + + if workdir_value: + workdir = Path(workdir_value).expanduser().resolve() + elif requested_exe.is_absolute(): + workdir = requested_exe.resolve().parent + else: + workdir = launch_dir + + if requested_exe.is_absolute(): + exe = requested_exe.resolve() + else: + in_workdir = (workdir / requested_exe).resolve() + in_launch_dir = (launch_dir / requested_exe).resolve() + exe = in_workdir if in_workdir.is_file() else in_launch_dir + + caselist = Path(caselist_value).expanduser().resolve() + outdir = Path(outdir_value).expanduser().resolve() + if not exe.is_file(): + raise RunnerError(f"glcts executable does not exist: {exe}") + if not workdir.is_dir(): + raise RunnerError(f"working directory does not exist: {workdir}") + if not caselist.is_file(): + raise RunnerError(f"caselist does not exist: {caselist}") + return exe, workdir, caselist, outdir + + +def qpa_signature(path: Path) -> Optional[tuple[int, int]]: + try: + stat = path.stat() + except FileNotFoundError: + return None + except OSError: + # A transient sharing violation must not kill a healthy process. The + # next poll will retry and the idle clock retains its previous value. + return None + return stat.st_size, stat.st_mtime_ns + + +def kill_process_tree(process: subprocess.Popen[bytes]) -> None: + """Force-stop the process and descendants, with a parent-only fallback.""" + + if process.poll() is not None: + return + + if os.name == "nt": + # /T is essential: CTS/platform helpers can outlive the top-level + # process, retain the QPA/DLL, and poison the next continuation round. + taskkill = Path(os.environ.get("SystemRoot", r"C:\Windows")) / "System32" / "taskkill.exe" + command = [str(taskkill), "/PID", str(process.pid), "/T", "/F"] + try: + subprocess.run( + command, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=20, + check=False, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + except (OSError, subprocess.TimeoutExpired): + pass + else: + try: + os.killpg(process.pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError, OSError): + pass + + try: + process.wait(timeout=10) + return + except subprocess.TimeoutExpired: + pass + + try: + process.kill() + except OSError: + pass + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + pass + + +def run_process( + command: Sequence[str], + workdir: Path, + environment: dict[str, str], + qpa_path: Path, + stdout_path: Path, + stderr_path: Path, + idle_timeout: float, + max_round_seconds: float, + poll_seconds: float, +) -> ProcessOutcome: + """Run one CTS chunk, killing its tree only after QPA progress stalls.""" + + started = time.monotonic() + with stdout_path.open("wb") as stdout_handle, stderr_path.open("wb") as stderr_handle: + popen_options: dict[str, object] = { + "cwd": str(workdir), + "env": environment, + "stdin": subprocess.DEVNULL, + "stdout": stdout_handle, + "stderr": stderr_handle, + } + if os.name == "nt": + popen_options["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) + else: + popen_options["start_new_session"] = True + + try: + process = subprocess.Popen(list(command), **popen_options) # type: ignore[arg-type] + except OSError as exc: + message = f"failed to launch {command[0]}: {exc}\n" + stderr_handle.write(message.encode("utf-8", errors="replace")) + stderr_handle.flush() + return ProcessOutcome(None, time.monotonic() - started, launch_error=str(exc)) + + last_signature = qpa_signature(qpa_path) + last_progress = time.monotonic() + timed_out = False + timeout_reason: Optional[str] = None + interrupted = False + + try: + while True: + try: + returncode = process.wait(timeout=poll_seconds) + break + except subprocess.TimeoutExpired: + pass + + now = time.monotonic() + signature = qpa_signature(qpa_path) + if signature is not None and signature != last_signature: + last_signature = signature + last_progress = now + + if idle_timeout > 0 and now - last_progress >= idle_timeout: + timed_out = True + timeout_reason = "qpa-idle" + kill_process_tree(process) + returncode = process.poll() + break + if max_round_seconds > 0 and now - started >= max_round_seconds: + timed_out = True + timeout_reason = "max-round" + kill_process_tree(process) + returncode = process.poll() + break + except KeyboardInterrupt: + interrupted = True + kill_process_tree(process) + returncode = process.poll() + + return ProcessOutcome( + returncode=returncode, + duration_seconds=time.monotonic() - started, + timed_out=timed_out, + timeout_reason=timeout_reason, + interrupted=interrupted, + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run Windows glcts against MobileGL, resuming across crashes and GPU hangs." + ) + parser.add_argument("--exe", required=True, help="path to glcts.exe") + parser.add_argument( + "--workdir", + help="glcts working directory (default: executable directory, or current directory for a relative exe)", + ) + parser.add_argument("--caselist", required=True, help="mustpass/caselist text file") + parser.add_argument("--outdir", required=True, help="persistent result directory") + parser.add_argument("--backend", required=True, choices=("DirectGLES", "DirectVulkan")) + parser.add_argument( + "--run-identity", + help="controller fingerprint for executable, data, arguments, and environment", + ) + parser.add_argument( + "--adopt-legacy", + action="store_true", + help=( + "adopt existing chunk/sidecar results which predate run_state.json; " + "disabled by default because their provenance cannot be verified" + ), + ) + parser.add_argument( + "--env", + action="append", + default=[], + metavar="NAME=VALUE", + help="extra child environment variable (repeatable)", + ) + parser.add_argument( + "--deqp-arg", + action="append", + default=[], + metavar="ARG", + help="extra glcts argument; repeat and use --deqp-arg=--option=value for leading dashes", + ) + parser.add_argument( + "--idle-timeout", + type=float, + default=300.0, + metavar="SECONDS", + help="kill a chunk after this many seconds with no QPA size/mtime change (0 disables; default: 300)", + ) + parser.add_argument( + "--max-round-seconds", + type=float, + default=0.0, + metavar="SECONDS", + help="optional total wall limit for one invocation (0 disables; default: 0)", + ) + parser.add_argument("--poll-seconds", type=float, default=1.0, help=argparse.SUPPRESS) + parser.add_argument( + "--max-rounds", + type=int, + default=10000, + help="maximum glcts invocations in this runner process (default: 10000)", + ) + parser.add_argument( + "--max-empty-streak", + type=int, + default=3, + help="abort after this many invocations record no case at all; no case is blamed (default: 3)", + ) + return parser + + +def execute(args: argparse.Namespace) -> int: + if args.idle_timeout < 0 or args.max_round_seconds < 0: + raise RunnerError("timeout values must be non-negative") + if args.poll_seconds <= 0: + raise RunnerError("--poll-seconds must be greater than zero") + if args.max_rounds <= 0 or args.max_empty_streak <= 0: + raise RunnerError("--max-rounds and --max-empty-streak must be greater than zero") + validate_deqp_args(args.deqp_arg) + extra_environment = parse_environment(args.env) + exe, workdir, caselist_path, outdir = resolve_paths( + args.exe, args.workdir, args.caselist, args.outdir + ) + outdir.mkdir(parents=True, exist_ok=True) + + cases = read_caselist(caselist_path) + expected = set(cases) + check_run_identity( + outdir, + args.backend, + cases, + args.run_identity, + adopt_legacy=args.adopt_legacy, + ) + recorded, crashed, hung = recover_results(outdir, expected) + accounted = recorded | crashed | hung + remaining = [case for case in cases if case not in accounted] + persist_sidecars(outdir, cases, crashed, hung, remaining) + + existing_qpas = len(result_qpa_files(outdir)) + print( + f"[run_cts_windows] {args.backend}: expected {len(cases)}, recovered {len(accounted)} " + f"({existing_qpas} QPA chunk(s), {len(crashed)} crash, {len(hung)} hang)" + ) + if not remaining: + print(f"[run_cts_windows] complete: all {len(cases)} expected cases are accounted") + return 0 + + environment = os.environ.copy() + environment.update(extra_environment) + # --backend is authoritative even if the inherited or extra environment + # already contains a different value. + environment["MOBILEGL_BACKEND_TYPE"] = args.backend + + common_arguments = [ + "--deqp-terminate-on-device-lost=disable", + "--deqp-log-images=disable", + "--deqp-log-shader-sources=disable", + ] + + chunk_number = next_chunk_number(outdir) + rounds = 0 + empty_streak = 0 + interrupted = False + fatal_launch_error = False + started_all = time.monotonic() + + while remaining and rounds < args.max_rounds: + prefix = f"chunk{chunk_number:04d}" + remaining_path = outdir / "remaining.txt" + qpa_path = outdir / f"{prefix}.qpa" + stdout_path = outdir / f"{prefix}.stdout.log" + stderr_path = outdir / f"{prefix}.stderr.log" + meta_path = outdir / f"{prefix}.meta.json" + + # The number allocator considers every chunk artifact, so these should + # be new. Refuse to truncate evidence if a foreign file races us. + for artifact in (qpa_path, stdout_path, stderr_path, meta_path): + if artifact.exists(): + raise RunnerError(f"refusing to overwrite existing chunk artifact: {artifact}") + write_case_file(remaining_path, remaining) + + command = [ + str(exe), + f"--deqp-caselist-file={remaining_path}", + f"--deqp-log-filename={qpa_path}", + *common_arguments, + *args.deqp_arg, + ] + print( + f"[run_cts_windows] {prefix}: launching {len(remaining)} remaining case(s); " + f"idle timeout {args.idle_timeout:g}s" + ) + chunk_started_utc = utc_now() + outcome = run_process( + command, + workdir, + environment, + qpa_path, + stdout_path, + stderr_path, + args.idle_timeout, + args.max_round_seconds, + args.poll_seconds, + ) + progress = scan_qpa(qpa_path) + + before = set(accounted) + for case in progress.recorded: + if case in expected: + recorded.add(case) + accounted.add(case) + + classification: Optional[str] = None + classified_case: Optional[str] = None + in_flight = progress.in_flight if progress.in_flight in expected else None + if not outcome.interrupted and in_flight is not None and in_flight not in accounted: + classified_case = in_flight + if outcome.timed_out: + classification = "DeviceHang" + hung.add(in_flight) + crashed.discard(in_flight) + else: + classification = "Crash" + crashed.add(in_flight) + accounted.add(in_flight) + + new_accounted = len(accounted - before) + if new_accounted: + empty_streak = 0 + elif progress.begin_count == 0: + # No #begin marker means there is no evidence that the first + # remaining case was reached. Retry the identical caselist, then + # abort rather than manufacturing a string of false Crash results. + empty_streak += 1 + else: + # A log containing only already-accounted cases is also no forward + # progress, but it is a different failure mode. Bound it with the + # same guard while retaining the QPA evidence. + empty_streak += 1 + + remaining = [case for case in cases if case not in accounted] + metadata = { + "version": 1, + "chunk": chunk_number, + "started_utc": chunk_started_utc, + "finished_utc": utc_now(), + "duration_seconds": round(outcome.duration_seconds, 3), + "returncode": outcome.returncode, + "timed_out": outcome.timed_out, + "timeout_reason": outcome.timeout_reason, + "interrupted": outcome.interrupted, + "launch_error": outcome.launch_error, + "qpa_begin_count": progress.begin_count, + "qpa_recorded_count": len(progress.recorded), + "in_flight": progress.in_flight, + "classification": classification, + "classified_case": classified_case, + "new_accounted": new_accounted, + "remaining": len(remaining), + } + # Metadata is committed first. If the runner itself dies between this + # write and the sidecars, recovery can reconstruct the classification. + atomic_write_json(meta_path, metadata) + persist_sidecars(outdir, cases, crashed, hung, remaining) + + rounds += 1 + elapsed_minutes = (time.monotonic() - started_all) / 60.0 + detail = "" + if classification: + detail = f", {classification}={classified_case}" + if outcome.timed_out: + detail += f", timeout={outcome.timeout_reason}" + print( + f"[run_cts_windows] {prefix}: +{new_accounted}, accounted " + f"{len(accounted)}/{len(cases)}, remaining {len(remaining)}{detail} " + f"({elapsed_minutes:.1f} min)" + ) + + chunk_number += 1 + if outcome.interrupted: + interrupted = True + print("[run_cts_windows] interrupted; process tree stopped and state preserved", file=sys.stderr) + break + if outcome.launch_error: + fatal_launch_error = True + print( + f"[run_cts_windows] launch failed; see {stderr_path.name}: {outcome.launch_error}", + file=sys.stderr, + ) + break + if empty_streak >= args.max_empty_streak: + print( + f"[run_cts_windows] aborting after {empty_streak} consecutive chunks made no " + "case progress; no unobserved case was labelled Crash/Hang", + file=sys.stderr, + ) + break + + # Recompute from the persisted evidence so the final completeness claim is + # subject to the exact same recovery path as a later invocation. + final_recorded, final_crashed, final_hung = recover_results(outdir, expected) + final_accounted = final_recorded | final_crashed | final_hung + final_remaining = [case for case in cases if case not in final_accounted] + persist_sidecars(outdir, cases, final_crashed, final_hung, final_remaining) + + if not final_remaining and final_accounted == expected: + print( + f"[run_cts_windows] complete: all {len(cases)} expected cases are accounted " + f"({len(final_crashed)} crash, {len(final_hung)} hang, {rounds} new invocation(s))" + ) + return 0 + + print( + f"[run_cts_windows] INCOMPLETE: {len(final_accounted)}/{len(cases)} accounted; " + f"{len(final_remaining)} listed in {outdir / 'unrun.txt'}", + file=sys.stderr, + ) + if interrupted: + return 130 + if fatal_launch_error: + return 3 + return 4 + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + try: + return execute(args) + except RunnerError as exc: + print(f"[run_cts_windows] ERROR: {exc}", file=sys.stderr) + return 2 + except OSError as exc: + print(f"[run_cts_windows] ERROR: filesystem/process operation failed: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/cts/scripts/test_cts_matrix_report.py b/tools/cts/scripts/test_cts_matrix_report.py new file mode 100644 index 00000000..fc3bafbf --- /dev/null +++ b/tools/cts/scripts/test_cts_matrix_report.py @@ -0,0 +1,251 @@ +import contextlib +import io +import json +import tempfile +import unittest +from pathlib import Path + +try: + from . import cts_matrix_report as report +except ImportError: # Allows `python test_cts_matrix_report.py`. + import cts_matrix_report as report + + +def qpa_case(case, status): + return ( + f"#beginTestCaseResult {case}\n" + f'\n' + "#endTestCaseResult\n" + ) + + +class MatrixReportTests(unittest.TestCase): + def test_utf8_bom_caselist_matches_runner_semantics(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + caselist = root / "cases.txt" + caselist.write_bytes(b"\xef\xbb\xbfcase.a\n") + results = root / "results" + results.mkdir() + (results / "run.qpa").write_text( + qpa_case("case.a", "Pass"), encoding="utf-8" + ) + + item = report.build_version_report("gl30", str(caselist), [str(results)]) + + self.assertEqual(1, item["expected"]) + self.assertEqual({"case.a": "Pass"}, item["cases"]["results"]) + self.assertEqual("OK", item["validation"]["state"]) + + def test_incomplete_qpa_is_unrun_not_a_completed_result(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + caselist = root / "cases.txt" + caselist.write_text("case.a\n", encoding="utf-8") + results = root / "results" + results.mkdir() + (results / "run.qpa").write_text( + "#beginTestCaseResult case.a\n#endTestCaseResult\n", + encoding="utf-8", + ) + (results / "unrun.txt").write_text("case.a\n", encoding="utf-8") + + item = report.build_version_report("gl46", str(caselist), [str(results)]) + + self.assertEqual(0, item["result"]) + self.assertEqual(1, item["unrun"]) + self.assertEqual(["case.a"], item["cases"]["incomplete_results"]) + self.assertEqual("INCOMPLETE", item["validation"]["state"]) + + def test_incomplete_qpa_is_upgraded_by_crash_sidecar(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + caselist = root / "cases.txt" + caselist.write_text("case.a\n", encoding="utf-8") + results = root / "results" + results.mkdir() + (results / "run.qpa").write_text( + "#beginTestCaseResult case.a\n", encoding="utf-8" + ) + (results / "crashed.txt").write_text("case.a\n", encoding="utf-8") + + item = report.build_version_report("gl46", str(caselist), [str(results)]) + + self.assertEqual("Crash", item["cases"]["results"]["case.a"]) + self.assertEqual([], item["cases"]["incomplete_results"]) + self.assertEqual("OK", item["validation"]["state"]) + + def test_chunk_numbers_above_four_digits_use_numeric_order(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + caselist = root / "cases.txt" + caselist.write_text("case.a\n", encoding="utf-8") + results = root / "results" + results.mkdir() + (results / "alpha.qpa").write_text("# no results\n", encoding="utf-8") + (results / "chunk9999.qpa").write_text( + qpa_case("case.a", "Fail"), encoding="utf-8" + ) + (results / "chunk10000.qpa").write_text( + qpa_case("case.a", "Pass"), encoding="utf-8" + ) + (results / "zeta.qpa").write_text("# no results\n", encoding="utf-8") + + item = report.build_version_report( + "gl46", str(caselist), [str(results)] + ) + + self.assertEqual( + ["alpha.qpa", "chunk9999.qpa", "chunk10000.qpa", "zeta.qpa"], + [Path(path).name for path in item["inputs"]["qpa_files"]], + ) + self.assertEqual("Pass", item["cases"]["results"]["case.a"]) + self.assertEqual(1, item["duplicate"]) + + def test_qpa_sidecars_duplicates_and_expected_denominator(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + caselist = root / "gl30.txt" + caselist.write_text("\n".join("abcdefg") + "\n", encoding="utf-8") + results = root / "results" + results.mkdir() + (results / "chunk0000.qpa").write_text( + qpa_case("a", "Fail") + "#beginTestCaseResult e\n", + encoding="utf-8", + ) + (results / "chunk0001.qpa").write_text( + qpa_case("a", "Pass") + + qpa_case("b", "NotSupported") + + qpa_case("c", "QualityWarning") + + qpa_case("d", "Fail"), + encoding="utf-8", + ) + (results / "crashed.txt").write_text("e\n", encoding="utf-8") + (results / "hung.txt").write_text("f\n", encoding="utf-8") + (results / "unrun.txt").write_text("g\n", encoding="utf-8") + + item = report.build_version_report( + "gl30", str(caselist), [str(results)] + ) + + self.assertEqual(item["expected"], 7) + self.assertEqual(item["result"], 6) + self.assertEqual(item["pass"], 1) + self.assertEqual(item["accepted"], 3) + self.assertEqual(item["crash"], 1) + self.assertEqual(item["hang"], 1) + self.assertEqual(item["unrun"], 1) + self.assertEqual(item["duplicate"], 1) + self.assertEqual(item["cases"]["results"]["a"], "Pass") + self.assertEqual(item["cases"]["results"]["e"], "Crash") + self.assertEqual(item["cases"]["results"]["f"], "DeviceHang") + self.assertAlmostEqual(item["strict_pass_rate"], 1 / 7) + self.assertAlmostEqual(item["conformance_accepted_rate"], 3 / 7) + self.assertAlmostEqual( + item["rates"]["measured_only_conformance_accepted"], 3 / 6 + ) + self.assertEqual(item["validation"]["state"], "INCOMPLETE") + self.assertEqual(item["validation"]["errors"], []) + self.assertTrue( + item["validation"]["invariant_expected_equals_result_plus_unrun"] + ) + + def test_missing_result_is_inferred_and_rejected_when_not_declared(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + caselist = root / "cases.txt" + caselist.write_text("a\nb\n", encoding="utf-8") + results = root / "results" + results.mkdir() + (results / "run.qpa").write_text(qpa_case("a", "Pass"), encoding="utf-8") + + item = report.build_version_report( + "gl31", str(caselist), [str(results)] + ) + + self.assertEqual(item["unrun"], 1) + self.assertEqual(item["cases"]["unrun"], ["b"]) + self.assertEqual(item["validation"]["state"], "ERROR") + self.assertEqual(item["validation"]["undeclared_unrun"], ["b"]) + self.assertIn("not declared", item["validation"]["errors"][0]) + self.assertEqual(item["strict_pass_rate"], 0.5) + + def test_cli_emits_markdown_json_and_weighted_overall(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + statuses = { + "gl30": "Pass", + "gl31": "Fail", + "gl32": "NotSupported", + "gl33": None, + } + argv = [] + for version, status in statuses.items(): + caselist = root / f"{version}.txt" + caselist.write_text(f"{version}.case\n", encoding="utf-8") + result_dir = root / f"{version}-results" + result_dir.mkdir() + qpa = result_dir / "run.qpa" + qpa.write_text( + qpa_case(f"{version}.case", status) if status else "# empty run\n", + encoding="utf-8", + ) + if status is None: + (result_dir / "unrun.txt").write_text( + f"{version}.case\n", encoding="utf-8" + ) + argv.extend( + [ + f"--{version}-caselist", + str(caselist), + f"--{version}-results", + str(result_dir), + ] + ) + json_path = root / "matrix.json" + argv.extend(["--json", str(json_path)]) + + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + rc = report.main(argv) + + self.assertEqual(rc, 1) # GL33 is explicitly incomplete. + markdown = stdout.getvalue() + self.assertIn("| Suite | Expected | Result", markdown) + self.assertIn("| **Overall (weighted)**", markdown) + payload = json.loads(json_path.read_text(encoding="utf-8")) + overall = payload["overall"] + self.assertEqual(overall["expected"], 4) + self.assertEqual(overall["result"], 3) + self.assertEqual(overall["pass"], 1) + self.assertEqual(overall["accepted"], 2) + self.assertEqual(overall["unrun"], 1) + self.assertEqual(overall["strict_pass_rate"], 0.25) + self.assertEqual(overall["conformance_accepted_rate"], 0.5) + self.assertEqual(overall["aggregation"], "weighted_by_expected_cases") + self.assertEqual(overall["validation"]["state"], "INCOMPLETE") + + def test_duplicate_caselist_and_unexpected_result_are_validation_errors(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + caselist = root / "cases.txt" + caselist.write_text("a\na\n", encoding="utf-8") + results = root / "results" + results.mkdir() + (results / "run.qpa").write_text( + qpa_case("a", "Pass") + qpa_case("outside", "Pass"), + encoding="utf-8", + ) + + item = report.build_version_report( + "gl32", str(caselist), [str(results)] + ) + + self.assertEqual(item["cases"]["duplicate_caselist_entries"], {"a": 2}) + self.assertEqual(item["cases"]["unexpected_results"], {"outside": "Pass"}) + self.assertEqual(item["validation"]["state"], "ERROR") + self.assertEqual(len(item["validation"]["errors"]), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/cts/scripts/test_cts_multi_report.py b/tools/cts/scripts/test_cts_multi_report.py new file mode 100644 index 00000000..ae8bd50a --- /dev/null +++ b/tools/cts/scripts/test_cts_multi_report.py @@ -0,0 +1,342 @@ +import contextlib +import io +import json +import tempfile +import unittest +from pathlib import Path + +try: + from . import cts_multi_report as report +except ImportError: # Allows `python test_cts_multi_report.py`. + import cts_multi_report as report + + +def qpa_case(case: str, status: str) -> str: + return ( + f"#beginTestCaseResult {case}\n" + f'\n' + "#endTestCaseResult\n" + ) + + +def write_run_state( + caselist: Path, + result_dir: Path, + backend: str, + invocation_identity=None, +) -> None: + fingerprint, case_count = report._caselist_fingerprint(str(caselist)) + (result_dir / "run_state.json").write_text( + json.dumps( + { + "version": 1, + "backend": backend, + "case_count": case_count, + "caselist_sha256": fingerprint, + "invocation_identity": invocation_identity, + } + ), + encoding="utf-8", + ) + + +def make_inputs( + root: Path, + name: str, + cases: list[str], + qpa: str, + backend: str = "DirectGLES", +): + caselist = root / f"{name}.txt" + caselist.write_text("\n".join(cases) + "\n", encoding="utf-8") + result_dir = root / f"{name}-results" + result_dir.mkdir() + (result_dir / "chunk0000.qpa").write_text(qpa, encoding="utf-8") + write_run_state(caselist, result_dir, backend) + return caselist, result_dir + + +class MultiReportTests(unittest.TestCase): + def test_backend_aggregate_is_weighted_by_expected_cases(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + small_cases, small_results = make_inputs( + root, "small", ["small.pass"], qpa_case("small.pass", "Pass") + ) + large_cases, large_results = make_inputs( + root, + "large", + ["large.pass", "large.crash", "large.hang"], + qpa_case("large.pass", "Pass"), + ) + (large_results / "crashed.txt").write_text( + "large.crash\n", encoding="utf-8" + ) + (large_results / "hung.txt").write_text( + "large.hang\n", encoding="utf-8" + ) + + payload = report.build_report( + [ + report.SuiteSpec( + "DirectGLES", "small", str(small_cases), str(small_results) + ), + report.SuiteSpec( + "DirectGLES", "large", str(large_cases), str(large_results) + ), + ] + ) + + aggregate = payload["backends"]["DirectGLES"] + self.assertEqual(4, aggregate["expected"]) + self.assertEqual(4, aggregate["result"]) + self.assertEqual(2, aggregate["pass"]) + self.assertEqual(2, aggregate["accepted"]) + self.assertEqual(1, aggregate["crash"]) + self.assertEqual(1, aggregate["hang"]) + self.assertEqual(0, aggregate["unrun"]) + # (1 accepted + 1 accepted) / (1 expected + 3 expected), not + # the unweighted mean of 100% and 33.3%. + self.assertEqual(0.5, aggregate["conformance_accepted_rate"]) + self.assertEqual("weighted_by_expected_cases", aggregate["aggregation"]) + self.assertEqual("OK", aggregate["validation"]["state"]) + + def test_declared_unrun_is_incomplete_and_cli_returns_nonzero(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + caselist, result_dir = make_inputs( + root, "missing", ["case.a", "case.b"], qpa_case("case.a", "Pass") + ) + (result_dir / "unrun.txt").write_text("case.b\n", encoding="utf-8") + markdown_path = root / "report.md" + json_path = root / "report.json" + argv = [ + "--suite", + "DirectGLES", + "gl30", + str(caselist), + str(result_dir), + "--markdown", + str(markdown_path), + "--json", + str(json_path), + ] + + with contextlib.redirect_stdout(io.StringIO()): + returncode = report.main(argv) + + self.assertEqual(1, returncode) + self.assertTrue(markdown_path.is_file()) + payload = json.loads(json_path.read_text(encoding="utf-8")) + suite = payload["suites"][0] + self.assertEqual(2, suite["expected"]) + self.assertEqual(1, suite["result"]) + self.assertEqual(1, suite["unrun"]) + self.assertEqual("INCOMPLETE", suite["validation"]["state"]) + self.assertEqual("INCOMPLETE", payload["overall"]["validation"]["state"]) + self.assertEqual(0.5, payload["overall"]["conformance_accepted_rate"]) + + def test_dual_backend_cli_outputs_markdown_and_json(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + caselist = root / "gl30.txt" + caselist.write_text("gl30.case\n", encoding="utf-8") + gles = root / "gles" + vulkan = root / "vulkan" + gles.mkdir() + vulkan.mkdir() + (gles / "run.qpa").write_text( + qpa_case("gl30.case", "Pass"), encoding="utf-8" + ) + (vulkan / "run.qpa").write_text( + qpa_case("gl30.case", "Fail"), encoding="utf-8" + ) + write_run_state(caselist, gles, "DirectGLES") + write_run_state(caselist, vulkan, "DirectVulkan") + markdown_path = root / "dual.md" + json_path = root / "dual.json" + argv = [ + f"--suite=DirectGLES,gl30,{caselist},{gles}", + "--suite", + "DirectVulkan", + "gl30", + str(caselist), + str(vulkan), + "--markdown", + str(markdown_path), + "--json", + str(json_path), + ] + + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + returncode = report.main(argv) + + self.assertEqual(0, returncode) + payload = json.loads(json_path.read_text(encoding="utf-8")) + self.assertEqual({"DirectGLES", "DirectVulkan"}, set(payload["backends"])) + self.assertEqual(1, payload["backends"]["DirectGLES"]["accepted"]) + self.assertEqual(0, payload["backends"]["DirectVulkan"]["accepted"]) + self.assertEqual(2, payload["overall"]["expected"]) + self.assertEqual(1, payload["overall"]["accepted"]) + self.assertEqual(0.5, payload["overall"]["conformance_accepted_rate"]) + markdown = markdown_path.read_text(encoding="utf-8") + self.assertIn("DirectGLES weighted subtotal", markdown) + self.assertIn("DirectVulkan weighted subtotal", markdown) + self.assertIn("Overall weighted", markdown) + self.assertIn("Markdown:", stdout.getvalue()) + + def test_duplicate_qpa_result_uses_last_observation(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + caselist, result_dir = make_inputs( + root, + "duplicate", + ["case.a"], + qpa_case("case.a", "Fail"), + backend="DirectVulkan", + ) + (result_dir / "chunk0001.qpa").write_text( + qpa_case("case.a", "Pass"), encoding="utf-8" + ) + + payload = report.build_report( + [ + report.SuiteSpec( + "DirectVulkan", "gl33", str(caselist), str(result_dir) + ) + ] + ) + + suite = payload["suites"][0] + self.assertEqual("Pass", suite["cases"]["results"]["case.a"]) + self.assertEqual(1, suite["duplicate"]) + self.assertEqual(1, payload["overall"]["duplicate"]) + self.assertEqual(1.0, payload["overall"]["strict_pass_rate"]) + self.assertEqual("OK", suite["validation"]["state"]) + self.assertIn("last result wins", suite["validation"]["warnings"][0]) + + def test_backend_provenance_mismatch_is_rejected(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + caselist, result_dir = make_inputs( + root, + "provenance", + ["case.a"], + qpa_case("case.a", "Pass"), + backend="DirectVulkan", + ) + with self.assertRaises(report.MultiReportInputError): + report.build_report( + [report.SuiteSpec("DirectGLES", "gl30", str(caselist), str(result_dir))] + ) + + def test_missing_provenance_requires_explicit_legacy_opt_in(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + caselist, result_dir = make_inputs( + root, "legacy", ["case.a"], qpa_case("case.a", "Pass") + ) + (result_dir / "run_state.json").unlink() + spec = report.SuiteSpec("DirectGLES", "gl30", str(caselist), str(result_dir)) + with self.assertRaises(report.MultiReportInputError): + report.build_report([spec]) + payload = report.build_report([spec], require_run_state=False) + self.assertEqual("UNVERIFIED", payload["suites"][0]["provenance"]["state"]) + + def test_expected_run_identity_accepts_match_and_rejects_mismatch(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + caselist, result_dir = make_inputs( + root, "identity", ["case.a"], qpa_case("case.a", "Pass") + ) + write_run_state( + caselist, result_dir, "DirectGLES", invocation_identity="identity-a" + ) + spec = report.SuiteSpec( + "DirectGLES", "gl30", str(caselist), str(result_dir) + ) + + payload = report.build_report( + [spec], expected_run_identity="identity-a" + ) + self.assertEqual( + "identity-a", + payload["suites"][0]["provenance"]["invocation_identity"], + ) + with self.assertRaises(report.MultiReportInputError): + report.build_report([spec], expected_run_identity="identity-b") + + def test_expected_identity_rejects_legacy_state_and_missing_state(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + caselist, result_dir = make_inputs( + root, "legacy-identity", ["case.a"], qpa_case("case.a", "Pass") + ) + spec = report.SuiteSpec( + "DirectGLES", "gl30", str(caselist), str(result_dir) + ) + with self.assertRaises(report.MultiReportInputError): + report.build_report([spec], expected_run_identity="identity-a") + + (result_dir / "run_state.json").unlink() + with self.assertRaises(report.MultiReportInputError): + report.build_report( + [spec], + require_run_state=False, + expected_run_identity="identity-a", + ) + + def test_duplicate_physical_result_directory_is_rejected(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + caselist, result_dir = make_inputs( + root, "duplicate-dir", ["case.a"], qpa_case("case.a", "Pass") + ) + with self.assertRaises(report.MultiReportInputError): + report.build_report( + [ + report.SuiteSpec( + "DirectGLES", "gl30", str(caselist), str(result_dir) + ), + report.SuiteSpec( + "DirectGLES", + "gl31", + str(caselist), + str(result_dir / "."), + ), + ] + ) + + def test_ancestor_and_descendant_result_directories_are_rejected(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + caselist = root / "cases.txt" + caselist.write_text("case.a\n", encoding="utf-8") + parent = root / "results" + child = parent / "nested" + child.mkdir(parents=True) + (parent / "chunk0000.qpa").write_text( + qpa_case("case.a", "Pass"), encoding="utf-8" + ) + (child / "chunk0000.qpa").write_text( + qpa_case("case.a", "Fail"), encoding="utf-8" + ) + write_run_state(caselist, parent, "DirectGLES") + write_run_state(caselist, child, "DirectVulkan") + + with self.assertRaises(report.MultiReportInputError): + report.build_report( + [ + report.SuiteSpec( + "DirectGLES", "gl30", str(caselist), str(parent) + ), + report.SuiteSpec( + "DirectVulkan", "gl30", str(caselist), str(child) + ), + ] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/cts/scripts/test_run_cts_windows.py b/tools/cts/scripts/test_run_cts_windows.py new file mode 100644 index 00000000..71a4f045 --- /dev/null +++ b/tools/cts/scripts/test_run_cts_windows.py @@ -0,0 +1,401 @@ +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest import mock + +import run_cts_windows as runner + + +def qpa_closed(case: str, status: str = "Pass") -> str: + return ( + f"#beginTestCaseResult {case}\n" + f'ok\n' + "#endTestCaseResult\n" + ) + + +def command_path(command, option): + prefix = option + "=" + return Path(next(value[len(prefix) :] for value in command if value.startswith(prefix))) + + +class AtomicWriteTests(unittest.TestCase): + def test_access_denied_retries_then_replace_succeeds(self): + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / "remaining.txt" + real_replace = runner.os.replace + attempts = 0 + + def flaky_replace(source, destination): + nonlocal attempts + attempts += 1 + if attempts == 1: + raise PermissionError(13, "temporarily denied", str(destination)) + if attempts == 2: + error = OSError("temporary WinError 5") + error.winerror = 5 + raise error + real_replace(source, destination) + + with mock.patch.object(runner.os, "replace", side_effect=flaky_replace), mock.patch.object( + runner.time, "sleep" + ) as sleep: + runner.atomic_write_text(target, "case.a\n") + + self.assertEqual(3, attempts) + self.assertEqual("case.a\n", target.read_text(encoding="utf-8")) + self.assertEqual(2, sleep.call_count) + self.assertEqual( + [ + mock.call(runner.ATOMIC_REPLACE_INITIAL_BACKOFF_SECONDS), + mock.call(runner.ATOMIC_REPLACE_INITIAL_BACKOFF_SECONDS * 2), + ], + sleep.call_args_list, + ) + self.assertEqual([], list(target.parent.glob(".remaining.txt.*.tmp"))) + + def test_permanent_access_denied_stops_after_bounded_attempts(self): + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / "remaining.txt" + + def always_denied(_source, _destination): + error = OSError("persistent WinError 5") + error.winerror = 5 + raise error + + with mock.patch.object( + runner.os, "replace", side_effect=always_denied + ) as replace, mock.patch.object(runner.time, "sleep") as sleep: + with self.assertRaises(OSError) as raised: + runner.atomic_write_text(target, "case.a\n") + + self.assertEqual(5, raised.exception.winerror) + self.assertEqual(runner.ATOMIC_REPLACE_ATTEMPTS, replace.call_count) + self.assertEqual(runner.ATOMIC_REPLACE_ATTEMPTS - 1, sleep.call_count) + self.assertFalse(target.exists()) + self.assertEqual([], list(target.parent.glob(".remaining.txt.*.tmp"))) + + def test_non_access_error_is_not_retried(self): + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / "remaining.txt" + error = OSError(28, "disk full") + with mock.patch.object( + runner.os, "replace", side_effect=error + ) as replace, mock.patch.object(runner.time, "sleep") as sleep: + with self.assertRaises(OSError): + runner.atomic_write_text(target, "case.a\n") + + self.assertEqual(1, replace.call_count) + sleep.assert_not_called() + + +class QpaParsingTests(unittest.TestCase): + def test_terminate_is_a_completed_result(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "chunk0000.qpa" + path.write_text( + "#beginTestCaseResult KHR-GL30.a\n" + "#terminateTestCaseResult Crash\n" + "#beginTestCaseResult KHR-GL30.b\n", + encoding="utf-8", + ) + progress = runner.scan_qpa(path) + self.assertEqual(["KHR-GL30.a"], progress.recorded) + self.assertEqual("KHR-GL30.b", progress.in_flight) + self.assertEqual(2, progress.begin_count) + + def test_end_without_result_is_not_accounted(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "chunk0000.qpa" + path.write_text( + "#beginTestCaseResult KHR-GL46.incomplete\n" + "#endTestCaseResult\n" + + qpa_closed("KHR-GL46.complete"), + encoding="utf-8", + ) + progress = runner.scan_qpa(path) + self.assertEqual(["KHR-GL46.complete"], progress.recorded) + self.assertIsNone(progress.in_flight) + + def test_result_written_before_truncated_eof_is_recovered(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "chunk0000.qpa" + path.write_text( + "#beginTestCaseResult KHR-GL46.complete\n" + 'ok\n', + encoding="utf-8", + ) + progress = runner.scan_qpa(path) + self.assertEqual(["KHR-GL46.complete"], progress.recorded) + self.assertIsNone(progress.in_flight) + + +class RunIdentityTests(unittest.TestCase): + def test_non_object_run_state_is_a_controlled_error(self): + with tempfile.TemporaryDirectory() as temporary: + outdir = Path(temporary) + (outdir / "run_state.json").write_text("null\n", encoding="utf-8") + with self.assertRaises(runner.RunnerError): + runner.check_run_identity(outdir, "DirectVulkan", ["case.a"]) + + def test_controller_identity_prevents_mixed_invocations(self): + with tempfile.TemporaryDirectory() as temporary: + outdir = Path(temporary) + runner.check_run_identity( + outdir, "DirectVulkan", ["case.a"], invocation_identity="identity-a" + ) + runner.check_run_identity( + outdir, "DirectVulkan", ["case.a"], invocation_identity="identity-a" + ) + with self.assertRaises(runner.RunnerError): + runner.check_run_identity( + outdir, "DirectVulkan", ["case.a"], invocation_identity="identity-b" + ) + + def test_controller_identity_cannot_be_downgraded_by_omission(self): + with tempfile.TemporaryDirectory() as temporary: + outdir = Path(temporary) + runner.check_run_identity( + outdir, "DirectVulkan", ["case.a"], invocation_identity="identity-a" + ) + with self.assertRaises(runner.RunnerError): + runner.check_run_identity(outdir, "DirectVulkan", ["case.a"]) + + def test_legacy_artifacts_require_explicit_adoption(self): + with tempfile.TemporaryDirectory() as temporary: + outdir = Path(temporary) + (outdir / "chunk0000.qpa").write_text( + qpa_closed("case.a"), encoding="utf-8" + ) + with self.assertRaises(runner.RunnerError): + runner.check_run_identity( + outdir, "DirectVulkan", ["case.a"], invocation_identity="identity-a" + ) + self.assertFalse((outdir / "run_state.json").exists()) + + runner.check_run_identity( + outdir, + "DirectVulkan", + ["case.a"], + invocation_identity="identity-a", + adopt_legacy=True, + ) + state = runner.json.loads( + (outdir / "run_state.json").read_text(encoding="utf-8") + ) + self.assertTrue(state["adopted_legacy"]) + + def test_foreign_nested_qpa_and_skipped_sidecar_are_legacy_evidence(self): + with tempfile.TemporaryDirectory() as temporary: + outdir = Path(temporary) + nested = outdir / "old" + nested.mkdir() + qpa = nested / "legacy.qpa" + qpa.write_text(qpa_closed("case.a"), encoding="utf-8") + skipped = outdir / "skipped.txt" + skipped.write_text("case.b\n", encoding="utf-8") + + self.assertEqual( + {qpa, skipped}, set(runner.recovery_artifacts(outdir)) + ) + with self.assertRaises(runner.RunnerError): + runner.check_run_identity( + outdir, "DirectVulkan", ["case.a", "case.b"] + ) + recorded, crashed, hung = runner.recover_results( + outdir, {"case.a", "case.b"} + ) + self.assertEqual({"case.a"}, recorded) + self.assertEqual(set(), crashed) + self.assertEqual(set(), hung) + + def test_non_object_or_non_string_meta_classification_is_ignored(self): + with tempfile.TemporaryDirectory() as temporary: + outdir = Path(temporary) + (outdir / "chunk0000.meta.json").write_text("null\n", encoding="utf-8") + (outdir / "chunk0001.meta.json").write_text("[]\n", encoding="utf-8") + (outdir / "chunk0002.meta.json").write_text( + '{"classified_case": [], "classification": "Crash"}\n', encoding="utf-8" + ) + self.assertEqual( + (set(), set()), runner.load_meta_classifications(outdir, {"case.a"}) + ) + + +class RunnerRecoveryTests(unittest.TestCase): + def run_args(self, root: Path, caselist: Path, outdir: Path, *extra: str): + return [ + "--exe", + sys.executable, + "--workdir", + str(root), + "--caselist", + str(caselist), + "--outdir", + str(outdir), + "--backend", + "DirectVulkan", + *extra, + ] + + def test_crash_tail_is_quarantined_and_next_chunk_resumes(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + caselist = root / "cases.txt" + outdir = root / "results" + caselist.write_text("KHR-GL30.a\nKHR-GL30.b\nKHR-GL30.c\n", encoding="utf-8") + seen_remaining = [] + + def fake_run(command, workdir, environment, qpa_path, stdout_path, stderr_path, *timeouts): + del workdir, timeouts + seen_remaining.append( + command_path(command, "--deqp-caselist-file") + .read_text(encoding="utf-8") + .splitlines() + ) + stdout_path.write_text("fake stdout\n", encoding="utf-8") + stderr_path.write_text("fake stderr\n", encoding="utf-8") + self.assertEqual("DirectVulkan", environment["MOBILEGL_BACKEND_TYPE"]) + if len(seen_remaining) == 1: + qpa_path.write_text( + qpa_closed("KHR-GL30.a") + "#beginTestCaseResult KHR-GL30.b\n", + encoding="utf-8", + ) + return runner.ProcessOutcome(0xC0000005, 0.1) + qpa_path.write_text(qpa_closed("KHR-GL30.c"), encoding="utf-8") + return runner.ProcessOutcome(0, 0.1) + + with mock.patch.object(runner, "run_process", side_effect=fake_run): + result = runner.main(self.run_args(root, caselist, outdir)) + + self.assertEqual(0, result) + self.assertEqual( + [ + ["KHR-GL30.a", "KHR-GL30.b", "KHR-GL30.c"], + ["KHR-GL30.c"], + ], + seen_remaining, + ) + self.assertEqual("KHR-GL30.b\n", (outdir / "crashed.txt").read_text(encoding="utf-8")) + self.assertEqual("", (outdir / "hung.txt").read_text(encoding="utf-8")) + self.assertEqual("", (outdir / "unrun.txt").read_text(encoding="utf-8")) + + def test_existing_qpa_and_sidecar_are_recovered(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + caselist = root / "cases.txt" + outdir = root / "results" + outdir.mkdir() + caselist.write_text("KHR-GL31.a\nKHR-GL31.b\nKHR-GL31.c\n", encoding="utf-8") + (outdir / "chunk0000.qpa").write_text(qpa_closed("KHR-GL31.a"), encoding="utf-8") + (outdir / "crashed.txt").write_text("KHR-GL31.b\n", encoding="utf-8") + seen_remaining = [] + + def fake_run(command, workdir, environment, qpa_path, stdout_path, stderr_path, *timeouts): + del workdir, environment, stdout_path, stderr_path, timeouts + seen_remaining.extend( + command_path(command, "--deqp-caselist-file") + .read_text(encoding="utf-8") + .splitlines() + ) + qpa_path.write_text(qpa_closed("KHR-GL31.c"), encoding="utf-8") + return runner.ProcessOutcome(0, 0.1) + + with mock.patch.object(runner, "run_process", side_effect=fake_run): + result = runner.main( + self.run_args(root, caselist, outdir, "--adopt-legacy") + ) + + self.assertEqual(0, result) + self.assertEqual(["KHR-GL31.c"], seen_remaining) + self.assertTrue((outdir / "chunk0001.qpa").is_file()) + + def test_repeated_no_output_aborts_without_false_case_blame(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + caselist = root / "cases.txt" + outdir = root / "results" + caselist.write_text("KHR-GL32.a\nKHR-GL32.b\n", encoding="utf-8") + seen_remaining = [] + + def fake_run(command, workdir, environment, qpa_path, stdout_path, stderr_path, *timeouts): + del workdir, environment, stdout_path, stderr_path, timeouts + seen_remaining.append( + command_path(command, "--deqp-caselist-file") + .read_text(encoding="utf-8") + .splitlines() + ) + qpa_path.write_text("#sessionInfo releaseName fake\n", encoding="utf-8") + return runner.ProcessOutcome( + 1, 0.1, timed_out=True, timeout_reason="qpa-idle" + ) + + with mock.patch.object(runner, "run_process", side_effect=fake_run): + result = runner.main( + self.run_args(root, caselist, outdir, "--max-empty-streak", "2") + ) + + self.assertEqual(4, result) + self.assertEqual( + [["KHR-GL32.a", "KHR-GL32.b"], ["KHR-GL32.a", "KHR-GL32.b"]], + seen_remaining, + ) + self.assertEqual("", (outdir / "crashed.txt").read_text(encoding="utf-8")) + self.assertEqual("", (outdir / "hung.txt").read_text(encoding="utf-8")) + self.assertEqual( + "KHR-GL32.a\nKHR-GL32.b\n", + (outdir / "unrun.txt").read_text(encoding="utf-8"), + ) + + +class ProcessTimeoutTests(unittest.TestCase): + def test_qpa_activity_prevents_idle_timeout(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + qpa = root / "active.qpa" + helper = ( + "import pathlib,sys,time\n" + "path=pathlib.Path(sys.argv[1])\n" + "for size in range(1, 9):\n" + " path.write_text('x' * size, encoding='utf-8')\n" + " time.sleep(0.08)\n" + ) + outcome = runner.run_process( + [sys.executable, "-c", helper, str(qpa)], + root, + dict(runner.os.environ), + qpa, + root / "stdout.log", + root / "stderr.log", + idle_timeout=0.2, + max_round_seconds=0, + poll_seconds=0.03, + ) + self.assertFalse(outcome.timed_out) + self.assertEqual(0, outcome.returncode) + + def test_idle_timeout_really_stops_process(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + started = time.monotonic() + outcome = runner.run_process( + [sys.executable, "-c", "import time; time.sleep(30)"], + root, + dict(runner.os.environ), + root / "never-created.qpa", + root / "stdout.log", + root / "stderr.log", + idle_timeout=0.2, + max_round_seconds=0, + poll_seconds=0.05, + ) + elapsed = time.monotonic() - started + self.assertTrue(outcome.timed_out) + self.assertEqual("qpa-idle", outcome.timeout_reason) + self.assertLess(elapsed, 10) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/cts/scripts/test_wgl_glcts_pipeline.py b/tools/cts/scripts/test_wgl_glcts_pipeline.py new file mode 100644 index 00000000..9fb0de91 --- /dev/null +++ b/tools/cts/scripts/test_wgl_glcts_pipeline.py @@ -0,0 +1,294 @@ +import argparse +import json +from pathlib import Path +import struct +import tempfile +import unittest + +import wgl_glcts_pipeline as pipeline + + +def write_fake_pe(path: Path, machine: int = pipeline.PE_MACHINE_AMD64, payload: bytes = b"") -> None: + data = bytearray(0x88) + data[0:2] = b"MZ" + struct.pack_into(" Path: + return Path(__file__).resolve().parents[3] + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def normalize_version(value: str) -> str: + normalized = value.strip().lower().removeprefix("gl").replace(".", "") + if normalized not in SUPPORTED_VERSIONS: + supported = ", ".join(f"gl{version}" for version in SUPPORTED_VERSIONS) + raise argparse.ArgumentTypeError(f"unsupported GL suite {value!r}; choose one of: {supported}") + return normalized + + +def gl_version_tuple(version: str) -> tuple[int, int]: + return int(version[0]), int(version[1]) + + +def parse_assignment(value: str) -> tuple[str, str]: + name, separator, setting = value.partition("=") + if not separator or not name or "\x00" in value: + raise argparse.ArgumentTypeError(f"expected NAME=VALUE, got {value!r}") + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name): + raise argparse.ArgumentTypeError(f"invalid environment variable name {name!r}") + return name, setting + + +def canonicalize_windows_environment( + items: Iterable[tuple[str, str]], +) -> dict[str, str]: + """Canonicalize environment keys using Windows' case-insensitive rules.""" + + result: dict[str, str] = {} + for name, value in items: + result[name.upper()] = value + return result + + +def tracked_run_environment( + inherited: Mapping[str, str], overrides: Mapping[str, str] +) -> tuple[dict[str, str], dict[str, str]]: + """Return tracked ambient values and the effective values used for identity.""" + + canonical_inherited = canonicalize_windows_environment(inherited.items()) + ambient = { + name: value + for name, value in canonical_inherited.items() + if name in TRACKED_ENVIRONMENT_NAMES + or name.startswith(TRACKED_ENVIRONMENT_PREFIXES) + } + effective = dict(ambient) + effective.update(canonicalize_windows_environment(overrides.items())) + return dict(sorted(ambient.items())), dict(sorted(effective.items())) + + +def command_text(command: Sequence[object]) -> str: + return subprocess.list2cmdline([str(part) for part in command]) + + +def run_command( + command: Sequence[object], + *, + cwd: Optional[Path] = None, + env: Optional[Mapping[str, str]] = None, + check: bool = True, + capture: bool = False, +) -> subprocess.CompletedProcess[str]: + rendered = command_text(command) + location = f" (cwd={cwd})" if cwd else "" + print(f"[wgl_glcts_pipeline] $ {rendered}{location}", flush=True) + completed = subprocess.run( + [str(part) for part in command], + cwd=str(cwd) if cwd else None, + env=dict(env) if env else None, + text=True, + capture_output=capture, + check=False, + ) + if check and completed.returncode != 0: + detail = "" + if capture: + detail = f"\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + raise PipelineError(f"command failed with exit code {completed.returncode}: {rendered}{detail}") + return completed + + +def require_file(path: Path, label: str) -> Path: + resolved = path.expanduser().resolve() + if not resolved.is_file(): + raise PipelineError(f"{label} does not exist or is not a file: {resolved}") + return resolved + + +def require_directory(path: Path, label: str) -> Path: + resolved = path.expanduser().resolve() + if not resolved.is_dir(): + raise PipelineError(f"{label} does not exist or is not a directory: {resolved}") + return resolved + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def sha256_directory(path: Path) -> str: + digest = hashlib.sha256() + files = sorted( + (candidate for candidate in path.rglob("*") if candidate.is_file()), + key=lambda candidate: candidate.relative_to(path).as_posix(), + ) + if not files: + raise PipelineError(f"directory contains no files to fingerprint: {path}") + for candidate in files: + relative = candidate.relative_to(path).as_posix() + digest.update(relative.encode("utf-8")) + digest.update(b"\0") + digest.update(sha256_file(candidate).encode("ascii")) + digest.update(b"\n") + return digest.hexdigest() + + +def pe_machine(path: Path) -> int: + with path.open("rb") as stream: + if stream.read(2) != b"MZ": + raise PipelineError(f"not a PE executable: {path}") + stream.seek(0x3C) + offset_bytes = stream.read(4) + if len(offset_bytes) != 4: + raise PipelineError(f"truncated PE header: {path}") + pe_offset = struct.unpack(" None: + machine = pe_machine(path) + if machine != PE_MACHINE_AMD64: + raise PipelineError(f"{label} must be an x64 PE (machine 0x8664), got 0x{machine:04x}: {path}") + + +def git_snapshot(path: Path) -> dict[str, object]: + snapshot: dict[str, object] = {"path": str(path)} + try: + head = run_command( + ["git", "-C", path, "rev-parse", "HEAD"], check=True, capture=True + ).stdout.strip() + status = run_command( + ["git", "-C", path, "status", "--porcelain"], check=True, capture=True + ).stdout + snapshot.update({"head": head, "dirty": bool(status.strip())}) + except (OSError, PipelineError): + snapshot.update({"head": None, "dirty": None}) + return snapshot + + +def generator_arguments(generator: str, architecture: str) -> list[str]: + arguments = ["-G", generator] + if generator.lower().startswith("visual studio"): + arguments.extend(["-A", architecture]) + return arguments + + +def mobilegl_configure_command( + repo_root: Path, + build_dir: Path, + generator: str, + architecture: str, + extra: Iterable[str], +) -> list[str]: + return [ + "cmake", + "-S", + str(repo_root), + "-B", + str(build_dir), + *generator_arguments(generator, architecture), + "-DMOBILEGL_BUILD_TEST=OFF", + "-DMOBILEGL_BUILD_BENCHMARK=OFF", + "-DMOBILEGL_BUILD_TRACE_REPLAY=OFF", + "-DMOBILEGL_ENABLE_TRACY=OFF", + "-DMOBILEGL_FORCE_RELEASE_OPT=ON", + *extra, + ] + + +def cts_configure_command( + cts_source: Path, + build_dir: Path, + generator: str, + architecture: str, + extra: Iterable[str], +) -> list[str]: + return [ + "cmake", + "-S", + str(cts_source), + "-B", + str(build_dir), + *generator_arguments(generator, architecture), + "-DDEQP_TARGET=default", + "-DDEQP_SUPPORT_DRM=OFF", + *extra, + ] + + +def build_command(build_dir: Path, configuration: str, target: str, jobs: int) -> list[str]: + command = ["cmake", "--build", str(build_dir), "--config", configuration, "--target", target] + if jobs > 0: + command.extend(["--parallel", str(jobs)]) + return command + + +def verify_mobilegl_sources(repo_root: Path) -> None: + required = ( + repo_root / "CMakeLists.txt", + repo_root / "MobileGL" / "MG_Impl" / "WGLImpl" / "WGLImpl.cpp", + repo_root / "3rdparty" / "glslang" / "CMakeLists.txt", + repo_root / "3rdparty" / "SPIRV-Cross" / "CMakeLists.txt", + repo_root / "3rdparty" / "Vulkan-Headers" / "CMakeLists.txt", + ) + missing = [str(path) for path in required if not path.is_file()] + if missing: + raise PipelineError( + "MobileGL source/submodules are incomplete:\n " + + "\n ".join(missing) + + f"\nRun: git -C {repo_root} submodule update --init --recursive" + ) + + +def verify_cts_sources(cts_source: Path) -> None: + required = ( + cts_source / "CMakeLists.txt", + cts_source / "external" / "openglcts" / "CMakeLists.txt", + ) + missing = [str(path) for path in required if not path.is_file()] + if missing: + raise PipelineError( + "VK-GL-CTS source/external packages are incomplete:\n " + + "\n ".join(missing) + + f"\nRun: {sys.executable} {cts_source / 'external' / 'fetch_sources.py'}" + ) + + +def discover_mobilegl_dll(build_dir: Path, configuration: str) -> Path: + preferred = ( + build_dir / configuration / "opengl32.dll", + build_dir / "MobileGL" / configuration / "opengl32.dll", + build_dir / "opengl32.dll", + ) + for candidate in preferred: + if candidate.is_file(): + return candidate.resolve() + candidates = sorted({path.resolve() for path in build_dir.rglob("opengl32.dll") if path.is_file()}) + if len(candidates) == 1: + return candidates[0] + if not candidates: + raise PipelineError(f"MobileGL build produced no opengl32.dll under {build_dir}") + raise PipelineError("multiple opengl32.dll candidates; pass --mobilegl-dll explicitly:\n " + "\n ".join(map(str, candidates))) + + +def discover_glcts_exe(build_dir: Path, configuration: str) -> Path: + preferred = ( + build_dir / "external" / "openglcts" / "modules" / configuration / "glcts.exe", + build_dir / "external" / "openglcts" / "modules" / "glcts.exe", + ) + for candidate in preferred: + if candidate.is_file(): + return candidate.resolve() + candidates = sorted({path.resolve() for path in build_dir.rglob("glcts.exe") if path.is_file()}) + if len(candidates) == 1: + return candidates[0] + if not candidates: + raise PipelineError(f"CTS build produced no glcts.exe under {build_dir}") + raise PipelineError("multiple glcts.exe candidates; pass --glcts-exe explicitly:\n " + "\n ".join(map(str, candidates))) + + +def default_cts_modules_dir(cts_build_dir: Path) -> Path: + return cts_build_dir / "external" / "openglcts" / "modules" + + +def find_caselist_root(cts_modules_dir: Path, cts_source: Path) -> Path: + relative = Path("gl_cts/data/mustpass/gl/khronos_mustpass/main") + candidates = (cts_modules_dir / relative, cts_source / "external" / "openglcts" / "modules" / relative) + for candidate in candidates: + if candidate.is_dir(): + return candidate.resolve() + raise PipelineError("Khronos GL mustpass directory was not found; checked:\n " + "\n ".join(map(str, candidates))) + + +def caselist_for(caselist_root: Path, version: str) -> Path: + return require_file(caselist_root / f"gl{version}-main.txt", f"GL{version} mustpass caselist") + + +def runtime_source_files( + glcts_exe: Path, + mobilegl_dll: Path, + backends: Sequence[str], + angle_dir: Optional[Path], +) -> dict[str, Path]: + files = {"glcts.exe": glcts_exe, "opengl32.dll": mobilegl_dll} + if "DirectGLES" in backends: + if angle_dir is None: + raise PipelineError("--angle-dir is required when DirectGLES is selected") + angle_dir = require_directory(angle_dir, "ANGLE runtime directory") + for name in ANGLE_REQUIRED_DLLS: + files[name] = require_file(angle_dir / name, f"ANGLE {name}") + for name in ANGLE_OPTIONAL_DLLS: + candidate = angle_dir / name + if candidate.is_file(): + files[name] = candidate.resolve() + return files + + +def runtime_fingerprint(files: Mapping[str, Path]) -> tuple[str, dict[str, str]]: + hashes = {name: sha256_file(path) for name, path in sorted(files.items())} + digest = hashlib.sha256() + for name, file_hash in hashes.items(): + digest.update(f"{name}\0{file_hash}\n".encode("utf-8")) + return digest.hexdigest(), hashes + + +def run_fingerprint( + runtime_hash: str, + cts_data_hash: str, + tool_hashes: Mapping[str, str], + caselist_hashes: Mapping[str, str], + deqp_args: Sequence[str], + environment: Mapping[str, str], + result_semantics: Optional[Mapping[str, object]] = None, +) -> str: + identity = { + "version": 2, + "runtime_fingerprint": runtime_hash, + "cts_data_sha256": cts_data_hash, + "tool_hashes": dict(sorted(tool_hashes.items())), + "caselist_hashes": dict(sorted(caselist_hashes.items())), + "deqp_args": list(deqp_args), + "environment": dict(sorted(environment.items())), + "result_semantics": dict(sorted((result_semantics or {}).items())), + } + return hashlib.sha256( + json.dumps(identity, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + + +def assemble_runtime( + work_root: Path, + sources: Mapping[str, Path], + fingerprint: str, + hashes: Mapping[str, str], +) -> Path: + runtime_dir = work_root / "runtime" / fingerprint[:16] + runtime_dir.mkdir(parents=True, exist_ok=True) + for name, source in sources.items(): + require_x64_pe(source, name) + destination = runtime_dir / name + if source.resolve() != destination.resolve(): + shutil.copy2(source, destination) + if sha256_file(destination) != hashes[name]: + raise PipelineError(f"runtime copy hash mismatch: {destination}") + manifest = { + "version": 1, + "created_utc": utc_now(), + "fingerprint": fingerprint, + "files": { + name: {"source": str(source), "sha256": hashes[name]} + for name, source in sorted(sources.items()) + }, + } + write_json(runtime_dir / "manifest.json", manifest) + return runtime_dir + + +def write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + ".tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(temporary, path) + + +def read_cases(caselist: Path) -> list[str]: + cases: list[str] = [] + for raw_line in caselist.read_text(encoding="utf-8-sig").splitlines(): + line = raw_line.strip() + if line and not line.startswith("#"): + cases.append(line) + if not cases: + raise PipelineError(f"caselist contains no cases: {caselist}") + return cases + + +def choose_preflight_case(caselist: Path) -> str: + cases = read_cases(caselist) + preferred_suffixes = ( + ".buffer_objects.gen_buffers", + ".CommonBugs.CommonBug_GetProgramivActiveUniformBlockMaxNameLength", + ) + for suffix in preferred_suffixes: + for case in cases: + if case.endswith(suffix): + return case + return cases[0] + + +def runner_command( + runner: Path, + runtime_exe: Path, + workdir: Path, + caselist: Path, + outdir: Path, + backend: str, + idle_timeout: float, + max_round_seconds: float, + max_rounds: int, + environment: Mapping[str, str], + deqp_args: Sequence[str], + run_identity: Optional[str] = None, +) -> list[str]: + command = [ + sys.executable, + str(runner), + "--exe", + str(runtime_exe), + "--workdir", + str(workdir), + "--caselist", + str(caselist), + "--outdir", + str(outdir), + "--backend", + backend, + "--idle-timeout", + str(idle_timeout), + "--max-round-seconds", + str(max_round_seconds), + "--max-rounds", + str(max_rounds), + ] + if run_identity is not None: + command.extend(["--run-identity", run_identity]) + for name, value in sorted(environment.items()): + command.extend(["--env", f"{name}={value}"]) + command.extend(f"--deqp-arg={argument}" for argument in deqp_args) + return command + + +def parse_preflight_identity( + qpa_files: Sequence[Path], + mobilegl_log: Path, + backend: str, + minimum_version: tuple[int, int], +) -> dict[str, object]: + if not qpa_files: + raise PipelineError(f"{backend} preflight produced no QPA file") + qpa_text = "\n".join(path.read_text(encoding="utf-8", errors="replace") for path in qpa_files) + vendors = SESSION_VENDOR.findall(qpa_text) + renderers = SESSION_RENDERER.findall(qpa_text) + command_lines = SESSION_COMMAND_LINE.findall(qpa_text) + if not vendors or "MobileGL" not in vendors[-1]: + raise PipelineError(f"{backend} preflight did not load MobileGL (vendor={vendors[-1] if vendors else None!r})") + expected_renderer = "Espryt" if backend == "DirectGLES" else "Magma" + if not renderers or expected_renderer not in renderers[-1]: + raise PipelineError( + f"{backend} preflight renderer mismatch: expected {expected_renderer!r}, " + f"got {renderers[-1] if renderers else None!r}" + ) + if not command_lines or "--deqp-gl-context-type=wgl" not in command_lines[-1]: + raise PipelineError(f"{backend} preflight did not record a WGL context") + if not mobilegl_log.is_file(): + raise PipelineError(f"{backend} preflight did not create MobileGL log: {mobilegl_log}") + log_text = mobilegl_log.read_text(encoding="utf-8", errors="replace") + versions = [(int(major), int(minor)) for major, minor in TARGET_GL_VERSION.findall(log_text)] + if not versions: + raise PipelineError(f"{backend} preflight log contains no target OpenGL version") + actual_version = versions[-1] + if actual_version < minimum_version: + raise PipelineError( + f"{backend} reports GL {actual_version[0]}.{actual_version[1]}, " + f"but selected suites require at least {minimum_version[0]}.{minimum_version[1]}" + ) + return { + "backend": backend, + "vendor": vendors[-1], + "renderer": renderers[-1], + "target_gl_version": f"{actual_version[0]}.{actual_version[1]}", + "qpa_files": [str(path) for path in qpa_files], + "mobilegl_log": str(mobilegl_log), + } + + +def run_preflight( + *, + runner: Path, + runtime_exe: Path, + cts_modules_dir: Path, + caselist: Path, + preflight_root: Path, + backend: str, + idle_timeout: float, + environment: Mapping[str, str], + deqp_args: Sequence[str], + minimum_version: tuple[int, int], + run_identity: str, +) -> dict[str, object]: + outdir = preflight_root / backend.lower() + outdir.mkdir(parents=True, exist_ok=True) + case_file = outdir / "case.txt" + case_file.write_text(choose_preflight_case(caselist) + "\n", encoding="utf-8") + log_path = outdir / "mobilegl.log" + child_environment = dict(environment) + child_environment["MOBILEGL_LOG_FILE_PATH"] = str(log_path) + command = runner_command( + runner, + runtime_exe, + cts_modules_dir, + case_file, + outdir, + backend, + min(idle_timeout, 120.0) if idle_timeout > 0 else 120.0, + 180.0, + 1, + child_environment, + deqp_args, + run_identity, + ) + # A developing driver may fail the chosen case or terminate during deinit. + # Identity is the gate: the QPA and MobileGL log must prove which WGL driver ran. + run_command(command, cwd=repository_root(), check=False) + identity = parse_preflight_identity(sorted(outdir.glob("chunk*.qpa")), log_path, backend, minimum_version) + print( + f"[wgl_glcts_pipeline] preflight {backend}: {identity['renderer']} | " + f"GL {identity['target_gl_version']}" + ) + return identity + + +def report_command( + reporter: Path, + suites: Sequence[tuple[str, str, Path, Path]], + markdown: Path, + json_out: Path, + allow_incomplete: bool, + expected_run_identity: Optional[str] = None, +) -> list[str]: + command = [sys.executable, str(reporter)] + for backend, label, caselist, results in suites: + command.extend(["--suite", backend, label, str(caselist), str(results)]) + command.extend(["--markdown", str(markdown), "--json", str(json_out)]) + if expected_run_identity is not None: + command.extend(["--expected-run-identity", expected_run_identity]) + if allow_incomplete: + command.append("--allow-incomplete") + return command + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Build MobileGL WGL, assemble a local glcts runtime, run GL30-GL46, and report results." + ) + parser.add_argument("--repo-root", type=Path, default=repository_root(), help="MobileGL worktree root") + parser.add_argument("--cts-source", type=Path, required=True, help="VK-GL-CTS source checkout") + parser.add_argument("--work-root", type=Path, required=True, help="build/result root (kept outside source)") + parser.add_argument("--angle-dir", type=Path, help="x64 ANGLE directory for DirectGLES") + parser.add_argument("--backends", nargs="+", choices=SUPPORTED_BACKENDS, default=list(SUPPORTED_BACKENDS)) + parser.add_argument("--versions", nargs="+", type=normalize_version, default=list(SUPPORTED_VERSIONS)) + parser.add_argument("--configuration", default="Release") + parser.add_argument("--generator", default="Visual Studio 17 2022") + parser.add_argument("--architecture", default="x64") + parser.add_argument("--jobs", type=int, default=max(1, os.cpu_count() or 1)) + parser.add_argument("--mobilegl-build-dir", type=Path) + parser.add_argument("--cts-build-dir", type=Path) + parser.add_argument("--mobilegl-dll", type=Path, help="reuse an existing MobileGL/opengl32 DLL") + parser.add_argument("--glcts-exe", type=Path, help="reuse an existing glcts.exe") + parser.add_argument("--cts-modules-dir", type=Path, help="glcts working directory containing gl_cts data") + parser.add_argument("--skip-mobilegl-build", action="store_true") + parser.add_argument("--skip-cts-build", action="store_true") + parser.add_argument("--skip-preflight", action="store_true") + parser.add_argument("--skip-run", action="store_true") + parser.add_argument("--skip-report", action="store_true") + parser.add_argument("--allow-incomplete-report", action="store_true") + parser.add_argument("--continue-on-suite-error", action="store_true") + parser.add_argument("--idle-timeout", type=float, default=300.0) + parser.add_argument("--max-round-seconds", type=float, default=0.0) + parser.add_argument("--max-rounds", type=int, default=10000) + parser.add_argument("--env", action="append", type=parse_assignment, default=[], metavar="NAME=VALUE") + parser.add_argument( + "--deqp-arg", action="append", default=[], metavar="ARG", help="extra glcts option; use --deqp-arg=--x=y" + ) + parser.add_argument( + "--mobilegl-cmake-arg", action="append", default=[], metavar="ARG", help="extra MobileGL configure option" + ) + parser.add_argument("--cts-cmake-arg", action="append", default=[], metavar="ARG", help="extra CTS configure option") + return parser + + +def execute(args: argparse.Namespace) -> int: + if os.name != "nt": + raise PipelineError("this pipeline builds and exercises the Windows WGL target and must run on Windows") + if shutil.which("cmake") is None and (not args.skip_mobilegl_build or not args.skip_cts_build): + raise PipelineError("cmake was not found on PATH") + if args.jobs < 0 or args.max_rounds <= 0: + raise PipelineError("--jobs must be >= 0 and --max-rounds must be > 0") + + repo_root = require_directory(args.repo_root, "MobileGL worktree") + cts_source = require_directory(args.cts_source, "VK-GL-CTS checkout") + work_root = args.work_root.expanduser().resolve() + work_root.mkdir(parents=True, exist_ok=True) + versions = list(dict.fromkeys(args.versions)) + backends = list(dict.fromkeys(args.backends)) + minimum_version = max(gl_version_tuple(version) for version in versions) + extra_environment = canonicalize_windows_environment(args.env) + reserved_environment = CONTROLLED_ENVIRONMENT_NAMES & set(extra_environment) + if reserved_environment: + raise PipelineError("the pipeline controls these environment variables: " + ", ".join(sorted(reserved_environment))) + + configuration = args.configuration + mobilegl_build_dir = (args.mobilegl_build_dir or work_root / f"mobilegl-build-{configuration.lower()}").resolve() + cts_build_dir = (args.cts_build_dir or work_root / f"cts-build-wgl-{configuration.lower()}").resolve() + + if not args.skip_mobilegl_build: + verify_mobilegl_sources(repo_root) + mobilegl_build_dir.mkdir(parents=True, exist_ok=True) + run_command( + mobilegl_configure_command( + repo_root, mobilegl_build_dir, args.generator, args.architecture, args.mobilegl_cmake_arg + ), + cwd=repo_root, + ) + run_command(build_command(mobilegl_build_dir, configuration, "MobileGL", args.jobs), cwd=repo_root) + mobilegl_dll = ( + require_file(args.mobilegl_dll, "MobileGL DLL") + if args.mobilegl_dll + else discover_mobilegl_dll(mobilegl_build_dir, configuration) + ) + + if not args.skip_cts_build: + verify_cts_sources(cts_source) + cts_build_dir.mkdir(parents=True, exist_ok=True) + run_command( + cts_configure_command(cts_source, cts_build_dir, args.generator, args.architecture, args.cts_cmake_arg), + cwd=cts_source, + ) + run_command(build_command(cts_build_dir, configuration, "glcts", args.jobs), cwd=cts_source) + glcts_exe = ( + require_file(args.glcts_exe, "glcts executable") + if args.glcts_exe + else discover_glcts_exe(cts_build_dir, configuration) + ) + cts_modules_dir = require_directory( + args.cts_modules_dir or default_cts_modules_dir(cts_build_dir), "CTS modules/working directory" + ) + cts_data_dir = require_directory(cts_modules_dir / "gl_cts" / "data", "CTS gl_cts data directory") + cts_data_hash = sha256_directory(cts_data_dir) + caselist_root = find_caselist_root(cts_modules_dir, cts_source) + caselists = {version: caselist_for(caselist_root, version) for version in versions} + + runtime_sources = runtime_source_files(glcts_exe, mobilegl_dll, backends, args.angle_dir) + fingerprint, hashes = runtime_fingerprint(runtime_sources) + runtime_dir = assemble_runtime(work_root, runtime_sources, fingerprint, hashes) + runtime_exe = runtime_dir / "glcts.exe" + runner = require_file(repo_root / "tools" / "cts" / "scripts" / "run_cts_windows.py", "Windows CTS runner") + reporter = require_file(repo_root / "tools" / "cts" / "scripts" / "cts_multi_report.py", "CTS reporter") + matrix_reporter = require_file( + repo_root / "tools" / "cts" / "scripts" / "cts_matrix_report.py", "CTS matrix reporter" + ) + qpa_reporter = require_file(repo_root / "tools" / "cts" / "scripts" / "qpa_report.py", "QPA parser") + tool_hashes = { + "wgl_glcts_pipeline.py": sha256_file(Path(__file__).resolve()), + "run_cts_windows.py": sha256_file(runner), + "cts_multi_report.py": sha256_file(reporter), + "cts_matrix_report.py": sha256_file(matrix_reporter), + "qpa_report.py": sha256_file(qpa_reporter), + } + deqp_args = [*DEFAULT_DEQP_ARGS, *args.deqp_arg] + caselist_hashes = {version: sha256_file(path) for version, path in caselists.items()} + ambient_environment, identity_environment = tracked_run_environment( + os.environ, extra_environment + ) + result_semantics = { + "idle_timeout_seconds": args.idle_timeout, + "max_round_seconds": args.max_round_seconds, + } + execution_settings = { + **result_semantics, + "max_rounds": args.max_rounds, + "continue_on_suite_error": args.continue_on_suite_error, + } + execution_fingerprint = run_fingerprint( + fingerprint, + cts_data_hash, + tool_hashes, + caselist_hashes, + deqp_args, + identity_environment, + result_semantics, + ) + run_root = work_root / "runs" / execution_fingerprint[:16] + report_root = run_root / "reports" + + manifest = { + "version": 1, + "created_utc": utc_now(), + "run_fingerprint": execution_fingerprint, + "runtime_fingerprint": fingerprint, + "runtime_hashes": hashes, + "tool_hashes": tool_hashes, + "cts_data": {"path": str(cts_data_dir), "sha256": cts_data_hash}, + "runtime_dir": str(runtime_dir), + "mobilegl": git_snapshot(repo_root), + "vk_gl_cts": git_snapshot(cts_source), + "configuration": configuration, + "generator": args.generator, + "architecture": args.architecture, + "backends": backends, + "versions": versions, + "caselists": {version: {"path": str(path), "sha256": caselist_hashes[version]} for version, path in caselists.items()}, + "deqp_args": deqp_args, + "environment_overrides": extra_environment, + "ambient_environment": ambient_environment, + "identity_environment": identity_environment, + "result_semantics": result_semantics, + "execution_settings": execution_settings, + } + write_json(run_root / "manifest.json", manifest) + print(f"[wgl_glcts_pipeline] runtime fingerprint: {fingerprint}") + print(f"[wgl_glcts_pipeline] run fingerprint: {execution_fingerprint}") + print(f"[wgl_glcts_pipeline] run root: {run_root}") + + identities: list[dict[str, object]] = [] + if not args.skip_preflight: + first_caselist = caselists[versions[0]] + preflight_root = run_root / "preflight" + for backend in backends: + identities.append( + run_preflight( + runner=runner, + runtime_exe=runtime_exe, + cts_modules_dir=cts_modules_dir, + caselist=first_caselist, + preflight_root=preflight_root, + backend=backend, + idle_timeout=args.idle_timeout, + environment=extra_environment, + deqp_args=deqp_args, + minimum_version=minimum_version, + run_identity=execution_fingerprint, + ) + ) + manifest["preflight"] = identities + write_json(run_root / "manifest.json", manifest) + + suites: list[tuple[str, str, Path, Path]] = [] + suite_errors: list[dict[str, object]] = [] + for backend in backends: + for version in versions: + label = f"gl{version}" + result_dir = run_root / "results" / backend.lower() / label + suites.append((backend, label, caselists[version], result_dir)) + if args.skip_run: + continue + result_dir.mkdir(parents=True, exist_ok=True) + child_environment = dict(extra_environment) + child_environment["MOBILEGL_LOG_FILE_PATH"] = str(result_dir / "mobilegl.log") + command = runner_command( + runner, + runtime_exe, + cts_modules_dir, + caselists[version], + result_dir, + backend, + args.idle_timeout, + args.max_round_seconds, + args.max_rounds, + child_environment, + deqp_args, + execution_fingerprint, + ) + completed = run_command(command, cwd=repo_root, check=False) + if completed.returncode != 0: + suite_errors.append({"backend": backend, "suite": label, "returncode": completed.returncode}) + if not args.continue_on_suite_error: + break + if suite_errors and not args.continue_on_suite_error: + break + + report_returncode: Optional[int] = None + if not args.skip_report: + report_root.mkdir(parents=True, exist_ok=True) + completed = run_command( + report_command( + reporter, + suites, + report_root / "gl-cts-summary.md", + report_root / "gl-cts-summary.json", + args.allow_incomplete_report or bool(suite_errors), + execution_fingerprint, + ), + cwd=repo_root, + check=False, + ) + report_returncode = completed.returncode + + manifest["suite_errors"] = suite_errors + manifest["report_returncode"] = report_returncode + manifest["finished_utc"] = utc_now() + write_json(run_root / "manifest.json", manifest) + if suite_errors: + print(f"[wgl_glcts_pipeline] {len(suite_errors)} suite runner(s) incomplete; see manifest/report", file=sys.stderr) + returncodes = {int(item["returncode"]) for item in suite_errors} + if 130 in returncodes: + return 130 + if 2 in returncodes: + return 2 + if 3 in returncodes: + return 3 + return 4 + if report_returncode is not None and report_returncode != 0: + print(f"[wgl_glcts_pipeline] report validation failed with exit code {report_returncode}", file=sys.stderr) + return report_returncode + return 0 + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + try: + return execute(args) + except PipelineError as exc: + print(f"[wgl_glcts_pipeline] ERROR: {exc}", file=sys.stderr) + return 2 + except OSError as exc: + print(f"[wgl_glcts_pipeline] ERROR: filesystem/process operation failed: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/cts/skills/README.md b/tools/cts/skills/README.md index 6b5831af..82bc6930 100644 --- a/tools/cts/skills/README.md +++ b/tools/cts/skills/README.md @@ -16,3 +16,4 @@ Each skill is a self-contained package, matching the layout used by | Skill | What it does | | --- | --- | | [gl-cts-on-mobilegl](gl-cts-on-mobilegl/SKILL.md) | Build VK-GL-CTS `glcts` as a standalone Android arm64 binary against MobileGL's own EGL, run KHR-GL33, and report a per-backend OpenGL 3.3 core conformance rate. | +| [wgl-gl-cts-on-mobilegl](wgl-gl-cts-on-mobilegl/SKILL.md) | Build MobileGL's Windows x64 WGL drop-in, run GL30-GL46 core CTS against DirectGLES and DirectVulkan, resume safely, and emit validated reports. | diff --git a/tools/cts/skills/wgl-gl-cts-on-mobilegl/SKILL.md b/tools/cts/skills/wgl-gl-cts-on-mobilegl/SKILL.md new file mode 100644 index 00000000..735dc56c --- /dev/null +++ b/tools/cts/skills/wgl-gl-cts-on-mobilegl/SKILL.md @@ -0,0 +1,143 @@ +--- +name: wgl-gl-cts-on-mobilegl +description: Build MobileGL as a Windows x64 WGL drop-in opengl32.dll, build or reuse VK-GL-CTS glcts, run Khronos GL30 through GL46 core suites against DirectGLES and DirectVulkan, resume after crashes or idle timeouts, and produce validated Markdown/JSON conformance reports. Use when Codex needs to compile MobileGL's WGL target, connect it to desktop OpenGL CTS on Windows, rerun selected GL core mustpass lists, or verify that CTS loaded MobileGL instead of the system OpenGL driver. +--- + +# WGL OpenGL CTS on MobileGL + +Use `tools/cts/scripts/wgl_glcts_pipeline.py` as the single entry point. It +configures both Visual Studio builds, assembles a private runtime, verifies the +loaded WGL implementation, calls the crash-resuming runner, and generates the +multi-suite report. + +## Prerequisites + +- Run on Windows x64 with Git, Python 3.9 or newer, CMake, Visual Studio 2022's + Desktop C++ workload, and a Vulkan SDK visible to MobileGL's CMake configure. +- Use the already selected MobileGL worktree. Inspect `git status` first and do + not create another worktree unless the user explicitly asks. +- Initialize MobileGL submodules: + +```powershell +git submodule update --init --recursive +``` + +- Prepare a VK-GL-CTS checkout at a citable release tag, then fetch its pinned + externals: + +```powershell +git -C D:\VK-GL-CTS checkout opengl-cts-4.6.8.1 +python D:\VK-GL-CTS\external\fetch_sources.py +``` + +- For DirectGLES, provide one x64 ANGLE directory containing matching + `libEGL.dll`, `libGLESv2.dll`, and `d3dcompiler_47.dll`. DirectVulkan does not + need ANGLE. +- For DirectVulkan, provide a working Vulkan loader plus a GPU-vendor ICD and + driver. The Vulkan SDK alone does not provide a usable GPU device. + +## Run the full matrix + +From the MobileGL worktree root: + +```powershell +python tools\cts\scripts\wgl_glcts_pipeline.py ` + --cts-source D:\VK-GL-CTS ` + --work-root D:\MobileGL-WGL-CTS ` + --angle-dir C:\path\to\angle-x64 ` + --backends DirectGLES DirectVulkan ` + --versions gl30 gl31 gl32 gl33 gl40 gl41 gl42 gl43 gl44 gl45 gl46 +``` + +The defaults deliberately reproduce the proven Windows setup: + +- Visual Studio 17 2022, x64, Release; +- VK-GL-CTS `DEQP_TARGET=default`, which selects desktop WGL on Windows; +- MobileGL copied beside `glcts.exe` as `opengl32.dll`; +- WGL context, FBO surface, `rgba8888d24s8`, hidden window, watchdog and crash + handler enabled; +- `--deqp-terminate-on-device-lost=disable` supplied by the underlying runner. + +Do not replace the FBO surface with the default framebuffer when comparing +backends; it changes the readback path and invalidates comparison with the +established runs. + +## Mandatory preflight + +Leave preflight enabled for a new binary. It must prove all of the following +before the full suite starts: + +- QPA vendor contains `MobileGL`; +- DirectGLES renderer contains `Espryt`, or DirectVulkan contains `Magma`; +- QPA command line records `--deqp-gl-context-type=wgl`; +- MobileGL's log reports a GL version at least as high as the highest selected + suite. + +Treat any preflight failure as a hard stop. It commonly means `glcts.exe` loaded +the system `opengl32.dll`, ANGLE DLLs have the wrong architecture, or the driver +still reports too low a GL version. + +## Common variants + +Run DirectVulkan only, without ANGLE: + +```powershell +python tools\cts\scripts\wgl_glcts_pipeline.py ` + --cts-source D:\VK-GL-CTS --work-root D:\MobileGL-WGL-CTS ` + --backends DirectVulkan --versions gl43 gl44 gl45 gl46 +``` + +Build and preflight without starting a multi-hour CTS run: + +```powershell +python tools\cts\scripts\wgl_glcts_pipeline.py ` + --cts-source D:\VK-GL-CTS --work-root D:\MobileGL-WGL-CTS ` + --angle-dir C:\path\to\angle-x64 ` + --skip-run --skip-report +``` + +Reuse previously built binaries with `--skip-mobilegl-build`, +`--skip-cts-build`, `--mobilegl-dll`, `--glcts-exe`, and +`--cts-modules-dir`. Continue to use a modules directory containing the +`gl_cts` data tree; the executable directory alone is insufficient. + +Pass extra dEQP options with the equals form so argparse does not consume the +leading dashes: + +```powershell +--deqp-arg=--deqp-log-images=enable +``` + +## Resume and artifacts + +Repeat the exact command and `--work-root` to resume. The runner recovers +completed QPA cases plus `crashed.txt` and `hung.txt`, then schedules only +unaccounted cases. + +The pipeline prints full SHA-256 fingerprints and uses their first 16 +hexadecimal characters as directory names: `runtime/` and +`runs/`. The run fingerprint covers the CTS data tree, +runner/report tools, caselists, dEQP arguments, explicit `--env` values, tracked +ambient GL/Vulkan environment, and the timeout settings that determine +Crash/Hang classification. The manifest records those inputs plus +`max_rounds`, suite-error continuation policy, source commits and dirty state, +preflight identity, suite errors, and report status. + +The runner refuses to attach a new `run_state.json` to old QPA or sidecar files +by default. Invoke `run_cts_windows.py --adopt-legacy` directly only after +verifying that those artifacts match the backend, caselist, binaries, and dEQP +arguments. Pipeline reports require every suite state to match the current run +fingerprint. + +Read the final outputs at: + +```text +/runs//reports/gl-cts-summary.md +/runs//reports/gl-cts-summary.json +``` + +Use the exact run root printed by the pipeline. + +Do not claim completeness when the report validation is incomplete or when the +manifest records suite errors. Keep `NotSupported` separate from hard failures +when prioritizing implementation work. diff --git a/tools/cts/skills/wgl-gl-cts-on-mobilegl/agents/openai.yaml b/tools/cts/skills/wgl-gl-cts-on-mobilegl/agents/openai.yaml new file mode 100644 index 00000000..aa6a2452 --- /dev/null +++ b/tools/cts/skills/wgl-gl-cts-on-mobilegl/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "WGL GL CTS on MobileGL" + short_description: "Build MobileGL WGL and run GL30-GL46 CTS on Windows" + default_prompt: "Use $wgl-gl-cts-on-mobilegl to build MobileGL's WGL DLL and run the selected OpenGL CTS suites on Windows."