mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-09-12 06:08:30 +09:00
[Feat] (tools/cts): automate Windows WGL conformance runs
This commit is contained in:
@@ -0,0 +1,545 @@
|
||||
#!/usr/bin/env python
|
||||
"""Build a GL 3.0--3.3 CTS conformance matrix from dEQP QPA logs.
|
||||
|
||||
The report deliberately scores against the unique cases in each supplied
|
||||
caselist. A case that has not produced a result therefore cannot disappear
|
||||
from the denominator and make a partial run look conformant.
|
||||
|
||||
QPA parsing and crash/hang sidecar handling follow :mod:`qpa_report`:
|
||||
|
||||
* a later QPA observation of a case wins;
|
||||
* ``crashed.txt`` upgrades a missing/incomplete result to ``Crash``;
|
||||
* ``hung.txt`` upgrades a missing/incomplete/crash result to ``DeviceHang``.
|
||||
|
||||
Example::
|
||||
|
||||
python cts_matrix_report.py \
|
||||
--gl30-caselist gl30-main.txt --gl30-results runs/gl30 \
|
||||
--gl31-caselist gl31-main.txt --gl31-results runs/gl31 \
|
||||
--gl32-caselist gl32-main.txt --gl32-results runs/gl32 \
|
||||
--gl33-caselist gl33-main.txt --gl33-results runs/gl33 \
|
||||
--json runs/cts-matrix.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from typing import Iterable, Optional, Sequence
|
||||
|
||||
try: # Works both as a directly executed script and as a package import.
|
||||
from . import qpa_report
|
||||
except ImportError: # pragma: no cover - exercised by the command-line tests
|
||||
import qpa_report
|
||||
|
||||
|
||||
VERSIONS = ("gl30", "gl31", "gl32", "gl33")
|
||||
ACCEPTED_STATUSES = (
|
||||
"Pass",
|
||||
"NotSupported",
|
||||
"QualityWarning",
|
||||
"CompatibilityWarning",
|
||||
"Waiver",
|
||||
)
|
||||
ACCEPTED = frozenset(ACCEPTED_STATUSES)
|
||||
CHUNK_QPA = re.compile(r"^chunk(\d+)\.qpa$", re.IGNORECASE)
|
||||
|
||||
|
||||
class ReportInputError(ValueError):
|
||||
"""An input path cannot be used to construct a meaningful report."""
|
||||
|
||||
|
||||
def _read_non_comment_lines(path: str) -> list[str]:
|
||||
try:
|
||||
# Match run_cts_windows.py: Khronos lists are UTF-8 and may carry a BOM.
|
||||
with open(path, "r", encoding="utf-8-sig", errors="strict") as fh:
|
||||
return [
|
||||
line.strip()
|
||||
for line in fh
|
||||
if line.strip() and not line.lstrip().startswith("#")
|
||||
]
|
||||
except (OSError, UnicodeError) as exc:
|
||||
raise ReportInputError(f"cannot read {path}: {exc}") from exc
|
||||
|
||||
|
||||
def read_caselist(path: str) -> tuple[list[str], dict[str, int]]:
|
||||
"""Return unique cases in file order and repeated caselist entries.
|
||||
|
||||
The mustpass files consumed by glcts and ``run_cts.py`` are one case per
|
||||
non-empty, non-comment line, so this intentionally uses the same syntax.
|
||||
"""
|
||||
|
||||
entries = _read_non_comment_lines(path)
|
||||
counts = Counter(entries)
|
||||
unique = list(dict.fromkeys(entries))
|
||||
duplicates = {case: count for case, count in counts.items() if count > 1}
|
||||
return unique, duplicates
|
||||
|
||||
|
||||
def _collect_qpa_files(paths: Sequence[str]) -> list[str]:
|
||||
missing = [path for path in paths if not os.path.exists(path)]
|
||||
if missing:
|
||||
raise ReportInputError(
|
||||
"result path(s) do not exist: " + ", ".join(sorted(missing))
|
||||
)
|
||||
|
||||
# qpa_report.collect provides the established directory-recursion rules.
|
||||
# De-duplicate aliases so specifying the same directory twice does not
|
||||
# manufacture duplicate observations.
|
||||
files = qpa_report.collect(paths)
|
||||
by_identity: dict[str, str] = {}
|
||||
for path in files:
|
||||
if not os.path.isfile(path):
|
||||
raise ReportInputError(f"QPA input is not a file: {path}")
|
||||
absolute = os.path.abspath(path)
|
||||
by_identity.setdefault(os.path.normcase(absolute), absolute)
|
||||
def order_key(value: str) -> tuple[str, str, int, str]:
|
||||
absolute = os.path.abspath(value)
|
||||
directory = os.path.normcase(os.path.dirname(absolute))
|
||||
filename = os.path.normcase(os.path.basename(absolute))
|
||||
match = CHUNK_QPA.fullmatch(filename)
|
||||
if match:
|
||||
# run_cts_windows.py uses a minimum width of four digits, not a
|
||||
# fixed width. Numeric ordering is therefore required once a run
|
||||
# reaches chunk10000; lexical ordering would put it before
|
||||
# chunk9999 and break the later-observation-wins rule.
|
||||
return directory, "chunk", int(match.group(1)), filename
|
||||
# Preserve a deterministic, name-based position for foreign/legacy
|
||||
# QPA files while grouping numeric runner chunks at the lexical
|
||||
# position occupied by the "chunk" basename.
|
||||
return directory, filename, -1, filename
|
||||
|
||||
return sorted(by_identity.values(), key=order_key)
|
||||
|
||||
|
||||
def _ratio(numerator: int, denominator: int) -> float:
|
||||
return numerator / denominator if denominator else 0.0
|
||||
|
||||
|
||||
def _sidecar(paths: Sequence[str], name: str) -> set[str]:
|
||||
"""Load a run_cts.py sidecar with qpa_report-compatible lookup rules."""
|
||||
|
||||
return qpa_report.load_sidecar(paths, name)
|
||||
|
||||
|
||||
def build_version_report(
|
||||
version: str, caselist: str, result_paths: Sequence[str]
|
||||
) -> dict:
|
||||
"""Build the serialisable report for one GL mustpass version."""
|
||||
|
||||
expected_cases, expected_duplicates = read_caselist(caselist)
|
||||
expected = set(expected_cases)
|
||||
qpa_files = _collect_qpa_files(result_paths)
|
||||
|
||||
results: dict[str, str] = {}
|
||||
observation_history: dict[str, list[dict[str, str]]] = defaultdict(list)
|
||||
for qpa_file in qpa_files:
|
||||
for case, status in qpa_report.parse_qpa(qpa_file):
|
||||
observation_history[case].append(
|
||||
{"file": qpa_file, "status": status}
|
||||
)
|
||||
results[case] = status
|
||||
|
||||
crashed = _sidecar(result_paths, "crashed.txt")
|
||||
hung = _sidecar(result_paths, "hung.txt")
|
||||
explicit_unrun = _sidecar(result_paths, "unrun.txt")
|
||||
skipped = _sidecar(result_paths, "skipped.txt")
|
||||
|
||||
# Keep this order and these guards in lock-step with qpa_report.py.
|
||||
for case in crashed:
|
||||
if results.get(case, "Incomplete") == "Incomplete":
|
||||
results[case] = "Crash"
|
||||
for case in hung:
|
||||
if results.get(case, "Incomplete") in ("Incomplete", "Crash"):
|
||||
results[case] = "DeviceHang"
|
||||
|
||||
# A begin/end pair without <Result>, or a QPA truncated mid-case, is not a
|
||||
# completed observation. Sidecars above may upgrade it to Crash/Hang;
|
||||
# anything still Incomplete must stay in the expected denominator as unrun.
|
||||
incomplete_results = {
|
||||
case for case, status in results.items() if status == "Incomplete"
|
||||
}
|
||||
for case in incomplete_results:
|
||||
del results[case]
|
||||
|
||||
expected_results = {
|
||||
case: status for case, status in results.items() if case in expected
|
||||
}
|
||||
unexpected_results = {
|
||||
case: status for case, status in results.items() if case not in expected
|
||||
}
|
||||
|
||||
# Missing cases are inferred from the caselist even if unrun.txt itself is
|
||||
# missing or stale. This is the invariant that prevents partial-run rate
|
||||
# inflation.
|
||||
unrun_cases = expected - set(expected_results)
|
||||
declared_not_measured = explicit_unrun | skipped
|
||||
undeclared_unrun = unrun_cases - declared_not_measured
|
||||
stale_unrun = (explicit_unrun | skipped) & set(expected_results)
|
||||
|
||||
counts = Counter(expected_results.values())
|
||||
strict_pass = counts["Pass"]
|
||||
accepted = sum(counts[status] for status in ACCEPTED)
|
||||
result_count = len(expected_results)
|
||||
expected_count = len(expected)
|
||||
crash_count = counts["Crash"]
|
||||
hang_count = counts["DeviceHang"]
|
||||
|
||||
duplicate_cases = {
|
||||
case: {
|
||||
"observations": len(history),
|
||||
"extra_observations": len(history) - 1,
|
||||
"final_status": results.get(case, "Incomplete"),
|
||||
"history": history,
|
||||
}
|
||||
for case, history in sorted(observation_history.items())
|
||||
if len(history) > 1
|
||||
}
|
||||
duplicate_observations = sum(
|
||||
item["extra_observations"] for item in duplicate_cases.values()
|
||||
)
|
||||
|
||||
sidecar_unknown = {
|
||||
name: sorted(cases - expected)
|
||||
for name, cases in (
|
||||
("crashed.txt", crashed),
|
||||
("hung.txt", hung),
|
||||
("unrun.txt", explicit_unrun),
|
||||
("skipped.txt", skipped),
|
||||
)
|
||||
if cases - expected
|
||||
}
|
||||
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
if not expected_count:
|
||||
errors.append("caselist has no cases")
|
||||
if expected_duplicates:
|
||||
errors.append(
|
||||
f"caselist has {sum(n - 1 for n in expected_duplicates.values())} "
|
||||
"duplicate entry/entries"
|
||||
)
|
||||
if not qpa_files:
|
||||
errors.append("no .qpa files found")
|
||||
if unexpected_results:
|
||||
errors.append(
|
||||
f"{len(unexpected_results)} result case(s) are absent from the caselist"
|
||||
)
|
||||
if sidecar_unknown:
|
||||
errors.append("one or more sidecars name cases absent from the caselist")
|
||||
if undeclared_unrun:
|
||||
errors.append(
|
||||
f"{len(undeclared_unrun)} missing result case(s) are not declared by "
|
||||
"unrun.txt/skipped.txt"
|
||||
)
|
||||
if stale_unrun:
|
||||
warnings.append(
|
||||
f"{len(stale_unrun)} case(s) declared unrun/skipped also have a result"
|
||||
)
|
||||
if duplicate_observations:
|
||||
warnings.append(
|
||||
f"{duplicate_observations} duplicate QPA observation(s); last result wins"
|
||||
)
|
||||
if incomplete_results:
|
||||
warnings.append(
|
||||
f"{len(incomplete_results)} QPA case(s) ended without a final result and were treated as unrun"
|
||||
)
|
||||
|
||||
if errors:
|
||||
state = "ERROR"
|
||||
elif unrun_cases:
|
||||
state = "INCOMPLETE"
|
||||
else:
|
||||
state = "OK"
|
||||
|
||||
return {
|
||||
"version": version,
|
||||
"inputs": {
|
||||
"caselist": os.path.abspath(caselist),
|
||||
"result_paths": [os.path.abspath(path) for path in result_paths],
|
||||
"qpa_files": qpa_files,
|
||||
},
|
||||
"expected": expected_count,
|
||||
"result": result_count,
|
||||
"pass": strict_pass,
|
||||
"accepted": accepted,
|
||||
"crash": crash_count,
|
||||
"hang": hang_count,
|
||||
"unrun": len(unrun_cases),
|
||||
"duplicate": duplicate_observations,
|
||||
"counts": dict(sorted(counts.items())),
|
||||
"coverage": {
|
||||
"numerator": result_count,
|
||||
"denominator": expected_count,
|
||||
"rate": _ratio(result_count, expected_count),
|
||||
},
|
||||
"rates": {
|
||||
# These are the report's conformance rates. Expected, not merely
|
||||
# measured results, is the denominator.
|
||||
"denominator": "expected",
|
||||
"strict_pass_only": _ratio(strict_pass, expected_count),
|
||||
"conformance_accepted": _ratio(accepted, expected_count),
|
||||
# Useful for comparison with qpa_report.py, whose denominator is
|
||||
# cases with a result. Never presented as the conformance rate.
|
||||
"measured_only_strict_pass": _ratio(strict_pass, result_count),
|
||||
"measured_only_conformance_accepted": _ratio(
|
||||
accepted, result_count
|
||||
),
|
||||
},
|
||||
"strict_pass_rate": _ratio(strict_pass, expected_count),
|
||||
"conformance_accepted_rate": _ratio(accepted, expected_count),
|
||||
"validation": {
|
||||
"state": state,
|
||||
"ok": state == "OK",
|
||||
"errors": errors,
|
||||
"warnings": warnings,
|
||||
"invariant_expected_equals_result_plus_unrun": (
|
||||
expected_count == result_count + len(unrun_cases)
|
||||
),
|
||||
"undeclared_unrun": sorted(undeclared_unrun),
|
||||
"stale_unrun_or_skipped": sorted(stale_unrun),
|
||||
"sidecar_cases_absent_from_caselist": sidecar_unknown,
|
||||
},
|
||||
"cases": {
|
||||
"results": dict(sorted(expected_results.items())),
|
||||
"unrun": sorted(unrun_cases),
|
||||
"unexpected_results": dict(sorted(unexpected_results.items())),
|
||||
"incomplete_results": sorted(incomplete_results),
|
||||
"duplicate_results": duplicate_cases,
|
||||
"duplicate_caselist_entries": dict(sorted(expected_duplicates.items())),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_matrix(suites: dict[str, tuple[str, Sequence[str]]]) -> dict:
|
||||
"""Build all four version reports and their case-weighted aggregate."""
|
||||
|
||||
version_reports = {
|
||||
version: build_version_report(version, *suites[version])
|
||||
for version in VERSIONS
|
||||
}
|
||||
|
||||
totals = {
|
||||
key: sum(report[key] for report in version_reports.values())
|
||||
for key in (
|
||||
"expected",
|
||||
"result",
|
||||
"pass",
|
||||
"accepted",
|
||||
"crash",
|
||||
"hang",
|
||||
"unrun",
|
||||
"duplicate",
|
||||
)
|
||||
}
|
||||
status_counts: Counter[str] = Counter()
|
||||
for report in version_reports.values():
|
||||
status_counts.update(report["counts"])
|
||||
|
||||
states = {report["validation"]["state"] for report in version_reports.values()}
|
||||
if "ERROR" in states:
|
||||
overall_state = "ERROR"
|
||||
elif "INCOMPLETE" in states:
|
||||
overall_state = "INCOMPLETE"
|
||||
else:
|
||||
overall_state = "OK"
|
||||
|
||||
overall = {
|
||||
**totals,
|
||||
"counts": dict(sorted(status_counts.items())),
|
||||
"aggregation": "weighted_by_expected_cases",
|
||||
"coverage": {
|
||||
"numerator": totals["result"],
|
||||
"denominator": totals["expected"],
|
||||
"rate": _ratio(totals["result"], totals["expected"]),
|
||||
},
|
||||
"rates": {
|
||||
"denominator": "expected",
|
||||
"strict_pass_only": _ratio(totals["pass"], totals["expected"]),
|
||||
"conformance_accepted": _ratio(
|
||||
totals["accepted"], totals["expected"]
|
||||
),
|
||||
"measured_only_strict_pass": _ratio(
|
||||
totals["pass"], totals["result"]
|
||||
),
|
||||
"measured_only_conformance_accepted": _ratio(
|
||||
totals["accepted"], totals["result"]
|
||||
),
|
||||
},
|
||||
"strict_pass_rate": _ratio(totals["pass"], totals["expected"]),
|
||||
"conformance_accepted_rate": _ratio(
|
||||
totals["accepted"], totals["expected"]
|
||||
),
|
||||
"validation": {
|
||||
"state": overall_state,
|
||||
"ok": overall_state == "OK",
|
||||
"invariant_expected_equals_result_plus_unrun": (
|
||||
totals["expected"] == totals["result"] + totals["unrun"]
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"accepted_statuses": list(ACCEPTED_STATUSES),
|
||||
"rate_policy": {
|
||||
"denominator": "unique expected cases from each caselist",
|
||||
"unrun_cases": "included in the denominator and never accepted",
|
||||
"duplicate_results": "last QPA result wins, matching qpa_report.py",
|
||||
},
|
||||
"versions": version_reports,
|
||||
"overall": overall,
|
||||
}
|
||||
|
||||
|
||||
def _percent(numerator: int, denominator: int) -> str:
|
||||
if not denominator:
|
||||
return "n/a"
|
||||
return f"{100.0 * numerator / denominator:.2f}% ({numerator}/{denominator})"
|
||||
|
||||
|
||||
def render_markdown(report: dict) -> str:
|
||||
"""Render the compact terminal-facing conformance table."""
|
||||
|
||||
header = (
|
||||
"| Suite | Expected | Result | Pass | Accepted | Crash | Hang | Unrun | "
|
||||
"Duplicate | Coverage | Strict Pass-only | Conformance-accepted | Validation |"
|
||||
)
|
||||
separator = (
|
||||
"|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|:---:|"
|
||||
)
|
||||
rows = [header, separator]
|
||||
for version in VERSIONS:
|
||||
item = report["versions"][version]
|
||||
rows.append(
|
||||
"| {version} | {expected} | {result} | {pass_count} | {accepted} | "
|
||||
"{crash} | {hang} | {unrun} | {duplicate} | {coverage} | {strict} | "
|
||||
"{accepted_rate} | {state} |".format(
|
||||
version=version.upper(),
|
||||
expected=item["expected"],
|
||||
result=item["result"],
|
||||
pass_count=item["pass"],
|
||||
accepted=item["accepted"],
|
||||
crash=item["crash"],
|
||||
hang=item["hang"],
|
||||
unrun=item["unrun"],
|
||||
duplicate=item["duplicate"],
|
||||
coverage=_percent(item["result"], item["expected"]),
|
||||
strict=_percent(item["pass"], item["expected"]),
|
||||
accepted_rate=_percent(item["accepted"], item["expected"]),
|
||||
state=item["validation"]["state"],
|
||||
)
|
||||
)
|
||||
|
||||
overall = report["overall"]
|
||||
rows.append(
|
||||
"| **Overall (weighted)** | **{expected}** | **{result}** | **{pass_count}** | "
|
||||
"**{accepted}** | **{crash}** | **{hang}** | **{unrun}** | **{duplicate}** | "
|
||||
"**{coverage}** | **{strict}** | **{accepted_rate}** | **{state}** |".format(
|
||||
expected=overall["expected"],
|
||||
result=overall["result"],
|
||||
pass_count=overall["pass"],
|
||||
accepted=overall["accepted"],
|
||||
crash=overall["crash"],
|
||||
hang=overall["hang"],
|
||||
unrun=overall["unrun"],
|
||||
duplicate=overall["duplicate"],
|
||||
coverage=_percent(overall["result"], overall["expected"]),
|
||||
strict=_percent(overall["pass"], overall["expected"]),
|
||||
accepted_rate=_percent(overall["accepted"], overall["expected"]),
|
||||
state=overall["validation"]["state"],
|
||||
)
|
||||
)
|
||||
rows.extend(
|
||||
(
|
||||
"",
|
||||
"Rates use unique **Expected** caselist cases as the denominator; unrun cases "
|
||||
"remain in that denominator and are not accepted.",
|
||||
"Accepted statuses: " + ", ".join(f"`{s}`" for s in ACCEPTED_STATUSES) + ".",
|
||||
"Duplicate is the number of extra QPA observations; the last observation wins.",
|
||||
)
|
||||
)
|
||||
|
||||
details: list[str] = []
|
||||
for version in VERSIONS:
|
||||
validation = report["versions"][version]["validation"]
|
||||
messages = validation["errors"] + validation["warnings"]
|
||||
if messages:
|
||||
details.append(
|
||||
f"- **{version.upper()} {validation['state']}**: " + "; ".join(messages)
|
||||
)
|
||||
if details:
|
||||
rows.extend(("", "Validation details:", "", *details))
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
def _write_json(path: str, report: dict) -> None:
|
||||
parent = os.path.dirname(os.path.abspath(path))
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8", newline="\n") as fh:
|
||||
json.dump(report, fh, indent=2, sort_keys=True)
|
||||
fh.write("\n")
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
for version in VERSIONS:
|
||||
parser.add_argument(
|
||||
f"--{version}-caselist",
|
||||
f"--{version}-case-list",
|
||||
required=True,
|
||||
help=f"{version.upper()} mustpass caselist",
|
||||
)
|
||||
parser.add_argument(
|
||||
f"--{version}-results",
|
||||
f"--{version}-result-dir",
|
||||
f"--{version}-results-dir",
|
||||
action="append",
|
||||
required=True,
|
||||
help=f"{version.upper()} result directory or QPA file (repeatable)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json",
|
||||
dest="json_out",
|
||||
default="cts_matrix_report.json",
|
||||
help="JSON output path (default: ./cts_matrix_report.json)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--allow-incomplete",
|
||||
action="store_true",
|
||||
help="return success even when validation is ERROR/INCOMPLETE",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Optional[Iterable[str]] = None) -> int:
|
||||
args = _parser().parse_args(argv)
|
||||
suites = {
|
||||
version: (
|
||||
getattr(args, f"{version}_caselist"),
|
||||
getattr(args, f"{version}_results"),
|
||||
)
|
||||
for version in VERSIONS
|
||||
}
|
||||
try:
|
||||
report = build_matrix(suites)
|
||||
_write_json(args.json_out, report)
|
||||
except (OSError, ReportInputError) as exc:
|
||||
print(f"cts_matrix_report: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
print(render_markdown(report))
|
||||
print(f"\nJSON: {os.path.abspath(args.json_out)}")
|
||||
if not args.allow_incomplete and not report["overall"]["validation"]["ok"]:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,540 @@
|
||||
#!/usr/bin/env python
|
||||
"""Summarise any number of GL CTS suites and MobileGL backends.
|
||||
|
||||
Each repeatable suite specification consists of four values: backend, label,
|
||||
caselist, and result directory. For example::
|
||||
|
||||
python cts_multi_report.py \
|
||||
--suite DirectGLES gl30 gl30-main.txt runs/gles/gl30 \
|
||||
--suite DirectVulkan gl30 gl30-main.txt runs/vulkan/gl30 \
|
||||
--markdown cts-summary.md --json cts-summary.json
|
||||
|
||||
A compact comma form is accepted as well::
|
||||
|
||||
--suite=DirectGLES,gl31,gl31-main.txt,runs/gles/gl31
|
||||
|
||||
Per-suite parsing and validation deliberately delegate to
|
||||
``cts_matrix_report`` so QPA ordering, sidecar upgrades, accepted statuses,
|
||||
unrun handling, and duplicate-result semantics cannot drift between reports.
|
||||
All conformance rates use unique expected caselist cases as their denominator.
|
||||
Backend subtotals and the overall total are therefore case-weighted, not an
|
||||
unweighted average of suite percentages.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import Iterable, Optional, Sequence
|
||||
|
||||
try: # Direct script execution and package imports are both supported.
|
||||
from . import cts_matrix_report
|
||||
except ImportError: # pragma: no cover - covered through CLI-style tests
|
||||
import cts_matrix_report
|
||||
|
||||
|
||||
SUPPORTED_BACKENDS = ("DirectGLES", "DirectVulkan")
|
||||
SUM_FIELDS = (
|
||||
"expected",
|
||||
"result",
|
||||
"pass",
|
||||
"accepted",
|
||||
"crash",
|
||||
"hang",
|
||||
"unrun",
|
||||
"duplicate",
|
||||
)
|
||||
|
||||
|
||||
class MultiReportInputError(ValueError):
|
||||
"""Suite specifications cannot produce an unambiguous report."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SuiteSpec:
|
||||
backend: str
|
||||
label: str
|
||||
caselist: str
|
||||
result_dir: str
|
||||
|
||||
@property
|
||||
def suite_id(self) -> str:
|
||||
return f"{self.backend}/{self.label}"
|
||||
|
||||
|
||||
def _ratio(numerator: int, denominator: int) -> float:
|
||||
return numerator / denominator if denominator else 0.0
|
||||
|
||||
|
||||
def _validation_state(items: Sequence[dict]) -> str:
|
||||
states = {item["validation"]["state"] for item in items}
|
||||
if "ERROR" in states:
|
||||
return "ERROR"
|
||||
if "INCOMPLETE" in states:
|
||||
return "INCOMPLETE"
|
||||
return "OK"
|
||||
|
||||
|
||||
def aggregate_reports(items: Sequence[dict], suite_state_keys: Sequence[str]) -> dict:
|
||||
"""Return an expected-case-weighted aggregate for suite reports."""
|
||||
|
||||
if len(items) != len(suite_state_keys):
|
||||
raise MultiReportInputError("internal suite/state key count mismatch")
|
||||
|
||||
totals = {
|
||||
field: sum(int(item[field]) for item in items)
|
||||
for field in SUM_FIELDS
|
||||
}
|
||||
status_counts: Counter[str] = Counter()
|
||||
for item in items:
|
||||
status_counts.update(item["counts"])
|
||||
|
||||
state = _validation_state(items)
|
||||
expected = totals["expected"]
|
||||
result = totals["result"]
|
||||
suite_states = {
|
||||
key: item["validation"]["state"]
|
||||
for key, item in zip(suite_state_keys, items)
|
||||
}
|
||||
return {
|
||||
**totals,
|
||||
"suite_count": len(items),
|
||||
"counts": dict(sorted(status_counts.items())),
|
||||
"aggregation": "weighted_by_expected_cases",
|
||||
"coverage": {
|
||||
"numerator": result,
|
||||
"denominator": expected,
|
||||
"rate": _ratio(result, expected),
|
||||
},
|
||||
"rates": {
|
||||
"denominator": "expected",
|
||||
"strict_pass_only": _ratio(totals["pass"], expected),
|
||||
"conformance_accepted": _ratio(totals["accepted"], expected),
|
||||
"measured_only_strict_pass": _ratio(totals["pass"], result),
|
||||
"measured_only_conformance_accepted": _ratio(
|
||||
totals["accepted"], result
|
||||
),
|
||||
},
|
||||
# Keep the convenient aliases used by cts_matrix_report consumers.
|
||||
"strict_pass_rate": _ratio(totals["pass"], expected),
|
||||
"conformance_accepted_rate": _ratio(totals["accepted"], expected),
|
||||
"validation": {
|
||||
"state": state,
|
||||
"ok": state == "OK",
|
||||
"suite_states": suite_states,
|
||||
"invariant_expected_equals_result_plus_unrun": (
|
||||
expected == result + totals["unrun"]
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _caselist_fingerprint(path: str) -> tuple[str, int]:
|
||||
cases, _duplicates = cts_matrix_report.read_caselist(path)
|
||||
payload = "\n".join(cases).encode("utf-8") + b"\n"
|
||||
return hashlib.sha256(payload).hexdigest(), len(cases)
|
||||
|
||||
|
||||
def _read_provenance(
|
||||
spec: SuiteSpec,
|
||||
require_run_state: bool,
|
||||
expected_run_identity: Optional[str],
|
||||
) -> dict:
|
||||
path = os.path.join(spec.result_dir, "run_state.json")
|
||||
if not os.path.isfile(path):
|
||||
if require_run_state or expected_run_identity is not None:
|
||||
raise MultiReportInputError(
|
||||
"suite result directory has no run_state.json; invocation provenance "
|
||||
f"cannot be verified: {spec.result_dir}"
|
||||
)
|
||||
return {"state": "UNVERIFIED", "run_state": None}
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
state = json.load(handle)
|
||||
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
raise MultiReportInputError(f"cannot read suite run identity {path}: {exc}") from exc
|
||||
if not isinstance(state, dict):
|
||||
raise MultiReportInputError(f"suite run identity must be a JSON object: {path}")
|
||||
if state.get("backend") != spec.backend:
|
||||
raise MultiReportInputError(
|
||||
f"suite {spec.suite_id} is labelled {spec.backend}, but run_state.json "
|
||||
f"records {state.get('backend')!r}"
|
||||
)
|
||||
fingerprint, case_count = _caselist_fingerprint(spec.caselist)
|
||||
if state.get("caselist_sha256") != fingerprint or state.get("case_count") != case_count:
|
||||
raise MultiReportInputError(
|
||||
f"suite {spec.suite_id} run_state.json belongs to a different caselist"
|
||||
)
|
||||
invocation_identity = state.get("invocation_identity")
|
||||
if (
|
||||
expected_run_identity is not None
|
||||
and invocation_identity != expected_run_identity
|
||||
):
|
||||
raise MultiReportInputError(
|
||||
f"suite {spec.suite_id} run_state.json belongs to a different CTS invocation"
|
||||
)
|
||||
return {
|
||||
"state": "VERIFIED",
|
||||
"run_state": os.path.abspath(path),
|
||||
"invocation_identity": invocation_identity,
|
||||
}
|
||||
|
||||
|
||||
def _validate_specs(
|
||||
specs: Sequence[SuiteSpec],
|
||||
require_run_state: bool,
|
||||
expected_run_identity: Optional[str],
|
||||
) -> dict[str, dict]:
|
||||
if not specs:
|
||||
raise MultiReportInputError("at least one --suite specification is required")
|
||||
|
||||
seen: set[tuple[str, str]] = set()
|
||||
seen_result_dirs: list[tuple[str, str]] = []
|
||||
provenance: dict[str, dict] = {}
|
||||
for spec in specs:
|
||||
if spec.backend not in SUPPORTED_BACKENDS:
|
||||
raise MultiReportInputError(
|
||||
f"unsupported backend {spec.backend!r}; expected one of "
|
||||
+ ", ".join(SUPPORTED_BACKENDS)
|
||||
)
|
||||
if not spec.label.strip():
|
||||
raise MultiReportInputError("suite label cannot be empty")
|
||||
identity = (spec.backend, spec.label)
|
||||
if identity in seen:
|
||||
raise MultiReportInputError(
|
||||
f"duplicate suite specification for {spec.backend}/{spec.label}"
|
||||
)
|
||||
seen.add(identity)
|
||||
if not os.path.isdir(spec.result_dir):
|
||||
raise MultiReportInputError(
|
||||
f"suite result directory does not exist: {spec.result_dir}"
|
||||
)
|
||||
physical_result_dir = os.path.normcase(
|
||||
os.path.realpath(os.path.abspath(spec.result_dir))
|
||||
)
|
||||
for previous_dir, previous_suite in seen_result_dirs:
|
||||
try:
|
||||
common_dir = os.path.commonpath(
|
||||
[previous_dir, physical_result_dir]
|
||||
)
|
||||
except ValueError:
|
||||
continue
|
||||
if common_dir in (previous_dir, physical_result_dir):
|
||||
raise MultiReportInputError(
|
||||
f"suite {spec.suite_id} uses a result directory which overlaps "
|
||||
f"{previous_suite}: {spec.result_dir}"
|
||||
)
|
||||
seen_result_dirs.append((physical_result_dir, spec.suite_id))
|
||||
provenance[spec.suite_id] = _read_provenance(
|
||||
spec, require_run_state, expected_run_identity
|
||||
)
|
||||
return provenance
|
||||
|
||||
|
||||
def build_report(
|
||||
specs: Sequence[SuiteSpec],
|
||||
require_run_state: bool = True,
|
||||
expected_run_identity: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""Build suite, per-backend, and overall serialisable reports."""
|
||||
|
||||
provenance = _validate_specs(
|
||||
specs, require_run_state, expected_run_identity
|
||||
)
|
||||
suite_reports: list[dict] = []
|
||||
backend_order: list[str] = []
|
||||
|
||||
for spec in specs:
|
||||
if spec.backend not in backend_order:
|
||||
backend_order.append(spec.backend)
|
||||
item = cts_matrix_report.build_version_report(
|
||||
spec.label, spec.caselist, [spec.result_dir]
|
||||
)
|
||||
# ``version`` is the generic label argument in build_version_report;
|
||||
# expose explicit multi-report terminology while retaining all of its
|
||||
# validation and case-level evidence.
|
||||
item.pop("version", None)
|
||||
item["backend"] = spec.backend
|
||||
item["label"] = spec.label
|
||||
item["suite_id"] = spec.suite_id
|
||||
item["provenance"] = provenance[spec.suite_id]
|
||||
if item["provenance"]["state"] == "UNVERIFIED":
|
||||
item["validation"]["warnings"].append(
|
||||
"result directory has no run_state.json; backend provenance is unverified"
|
||||
)
|
||||
suite_reports.append(item)
|
||||
|
||||
backends: dict[str, dict] = {}
|
||||
for backend in backend_order:
|
||||
backend_items = [
|
||||
item for item in suite_reports if item["backend"] == backend
|
||||
]
|
||||
labels = [item["label"] for item in backend_items]
|
||||
aggregate = aggregate_reports(backend_items, labels)
|
||||
aggregate["backend"] = backend
|
||||
aggregate["suite_labels"] = labels
|
||||
backends[backend] = aggregate
|
||||
|
||||
overall = aggregate_reports(
|
||||
suite_reports, [item["suite_id"] for item in suite_reports]
|
||||
)
|
||||
overall["backend_count"] = len(backends)
|
||||
overall["backends"] = backend_order
|
||||
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"accepted_statuses": list(cts_matrix_report.ACCEPTED_STATUSES),
|
||||
"rate_policy": {
|
||||
"denominator": "unique expected cases from each suite caselist",
|
||||
"unrun_cases": "included in the denominator and never accepted",
|
||||
"backend_aggregation": "weighted by expected cases",
|
||||
"overall_aggregation": "weighted by expected cases across backend-suite pairs",
|
||||
"duplicate_results": "last QPA result wins, matching qpa_report.py",
|
||||
},
|
||||
"suites": suite_reports,
|
||||
"backends": backends,
|
||||
"overall": overall,
|
||||
}
|
||||
|
||||
|
||||
def _percent(numerator: int, denominator: int) -> str:
|
||||
if not denominator:
|
||||
return "n/a"
|
||||
return f"{100.0 * numerator / denominator:.2f}% ({numerator}/{denominator})"
|
||||
|
||||
|
||||
def _markdown_cell(value: object) -> str:
|
||||
return str(value).replace("|", r"\|").replace("\r", " ").replace("\n", " ")
|
||||
|
||||
|
||||
def _table_row(backend: str, label: str, item: dict, bold: bool = False) -> str:
|
||||
values = [
|
||||
backend,
|
||||
label,
|
||||
str(item["expected"]),
|
||||
str(item["result"]),
|
||||
str(item["pass"]),
|
||||
str(item["accepted"]),
|
||||
str(item["crash"]),
|
||||
str(item["hang"]),
|
||||
str(item["unrun"]),
|
||||
str(item["duplicate"]),
|
||||
_percent(item["result"], item["expected"]),
|
||||
_percent(item["pass"], item["expected"]),
|
||||
_percent(item["accepted"], item["expected"]),
|
||||
item["validation"]["state"],
|
||||
]
|
||||
values = [_markdown_cell(value) for value in values]
|
||||
if bold:
|
||||
values = [f"**{value}**" for value in values]
|
||||
return "| " + " | ".join(values) + " |"
|
||||
|
||||
|
||||
def render_markdown(report: dict) -> str:
|
||||
lines = [
|
||||
"# GL CTS multi-suite conformance report",
|
||||
"",
|
||||
(
|
||||
"| Backend | Suite | Expected | Result | Pass | Accepted | Crash | Hang | "
|
||||
"Unrun | Duplicate | Coverage | Strict Pass-only | Conformance-accepted | Validation |"
|
||||
),
|
||||
"|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|:---:|",
|
||||
]
|
||||
|
||||
for backend in report["backends"]:
|
||||
for item in report["suites"]:
|
||||
if item["backend"] == backend:
|
||||
lines.append(_table_row(backend, item["label"], item))
|
||||
subtotal = report["backends"][backend]
|
||||
lines.append(
|
||||
_table_row(backend, f"{backend} weighted subtotal", subtotal, bold=True)
|
||||
)
|
||||
|
||||
lines.append(
|
||||
_table_row(
|
||||
"All backends",
|
||||
"Overall weighted",
|
||||
report["overall"],
|
||||
bold=True,
|
||||
)
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
(
|
||||
"Rates use unique **Expected** caselist cases as the denominator. "
|
||||
"Unrun cases remain in the denominator and are not accepted."
|
||||
),
|
||||
"Accepted statuses: "
|
||||
+ ", ".join(
|
||||
f"`{status}`" for status in report["accepted_statuses"]
|
||||
)
|
||||
+ ".",
|
||||
(
|
||||
"Duplicate is the number of extra QPA observations; the final "
|
||||
"observation wins."
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
details: list[str] = []
|
||||
for item in report["suites"]:
|
||||
validation = item["validation"]
|
||||
messages = validation["errors"] + validation["warnings"]
|
||||
if messages:
|
||||
details.append(
|
||||
f"- **{_markdown_cell(item['suite_id'])} {validation['state']}**: "
|
||||
+ "; ".join(_markdown_cell(message) for message in messages)
|
||||
)
|
||||
if details:
|
||||
lines.extend(["", "## Validation details", "", *details])
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _write_text(path: str, contents: str) -> None:
|
||||
absolute = os.path.abspath(path)
|
||||
os.makedirs(os.path.dirname(absolute), exist_ok=True)
|
||||
with open(absolute, "w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(contents)
|
||||
|
||||
|
||||
def _write_json(path: str, report: dict) -> None:
|
||||
_write_text(path, json.dumps(report, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def _normalise_compact_suite_args(argv: Sequence[str]) -> list[str]:
|
||||
"""Expand ``--suite=b,l,c,r`` into the four-value argparse form."""
|
||||
|
||||
result: list[str] = []
|
||||
index = 0
|
||||
while index < len(argv):
|
||||
token = argv[index]
|
||||
if token.startswith("--suite="):
|
||||
compact = token.split("=", 1)[1]
|
||||
parts = compact.split(",", 3)
|
||||
if len(parts) != 4:
|
||||
raise MultiReportInputError(
|
||||
"compact --suite expects backend,label,caselist,result-dir"
|
||||
)
|
||||
result.extend(["--suite", *parts])
|
||||
index += 1
|
||||
continue
|
||||
if token == "--suite" and index + 1 < len(argv) and argv[index + 1].count(",") >= 3:
|
||||
parts = argv[index + 1].split(",", 3)
|
||||
result.extend(["--suite", *parts])
|
||||
index += 2
|
||||
continue
|
||||
result.append(token)
|
||||
index += 1
|
||||
return result
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--suite",
|
||||
action="append",
|
||||
nargs=4,
|
||||
required=True,
|
||||
metavar=("BACKEND", "LABEL", "CASELIST", "RESULT_DIR"),
|
||||
help=(
|
||||
"suite specification; repeat for every backend/suite pair "
|
||||
f"(backends: {', '.join(SUPPORTED_BACKENDS)})"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--markdown",
|
||||
default="cts_multi_report.md",
|
||||
help="Markdown output path (default: ./cts_multi_report.md)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json",
|
||||
dest="json_out",
|
||||
default="cts_multi_report.json",
|
||||
help="JSON output path (default: ./cts_multi_report.json)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--allow-incomplete",
|
||||
action="store_true",
|
||||
help="return success even when one or more suites are ERROR/INCOMPLETE",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--adopt-legacy",
|
||||
dest="allow_unverified_provenance",
|
||||
action="store_true",
|
||||
help="accept legacy result directories without run_state.json (provenance remains unverified)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--allow-unverified-provenance",
|
||||
dest="allow_unverified_provenance",
|
||||
action="store_true",
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--expected-run-identity",
|
||||
help="require every suite run_state.json to contain this controller fingerprint",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def _specs_from_args(values: Sequence[Sequence[str]]) -> list[SuiteSpec]:
|
||||
return [
|
||||
SuiteSpec(
|
||||
backend=backend.strip(),
|
||||
label=label.strip(),
|
||||
caselist=caselist,
|
||||
result_dir=result_dir,
|
||||
)
|
||||
for backend, label, caselist, result_dir in values
|
||||
]
|
||||
|
||||
|
||||
def main(argv: Optional[Iterable[str]] = None) -> int:
|
||||
raw_argv = list(argv) if argv is not None else sys.argv[1:]
|
||||
try:
|
||||
normalised = _normalise_compact_suite_args(raw_argv)
|
||||
except MultiReportInputError as exc:
|
||||
print(f"cts_multi_report: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
args = _parser().parse_args(normalised)
|
||||
|
||||
if os.path.normcase(os.path.abspath(args.markdown)) == os.path.normcase(
|
||||
os.path.abspath(args.json_out)
|
||||
):
|
||||
print("cts_multi_report: Markdown and JSON paths must differ", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
try:
|
||||
report = build_report(
|
||||
_specs_from_args(args.suite),
|
||||
require_run_state=not args.allow_unverified_provenance,
|
||||
expected_run_identity=args.expected_run_identity,
|
||||
)
|
||||
markdown = render_markdown(report)
|
||||
_write_text(args.markdown, markdown)
|
||||
_write_json(args.json_out, report)
|
||||
except (
|
||||
OSError,
|
||||
MultiReportInputError,
|
||||
cts_matrix_report.ReportInputError,
|
||||
) as exc:
|
||||
print(f"cts_multi_report: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
print(markdown, end="")
|
||||
print(f"\nMarkdown: {os.path.abspath(args.markdown)}")
|
||||
print(f"JSON: {os.path.abspath(args.json_out)}")
|
||||
if not args.allow_incomplete and not report["overall"]["validation"]["ok"]:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,878 @@
|
||||
#!/usr/bin/env python
|
||||
"""Run a local Windows glcts executable and resume across process failures.
|
||||
|
||||
The desktop CTS normally runs a complete caselist in one process. That is a
|
||||
poor fit for testing a developing OpenGL implementation: one access violation
|
||||
or GPU hang prevents every later case from running. This driver gives each
|
||||
invocation the cases which have not produced a result yet, preserves one QPA
|
||||
and stdout/stderr pair per invocation, and starts another process after a
|
||||
crash.
|
||||
|
||||
Existing ``chunkNNNN.qpa`` files and ``crashed.txt``/``hung.txt`` sidecars are
|
||||
read on startup, so invoking the same command and output directory resumes an
|
||||
interrupted run. A timeout is based on *idle QPA time*, not total process wall
|
||||
time: a healthy invocation may legitimately run thousands of cases for hours.
|
||||
|
||||
Example (values beginning with ``--`` use argparse's ``=`` spelling)::
|
||||
|
||||
py run_cts_windows.py \
|
||||
--exe D:\\glcts\\glcts.exe --workdir D:\\glcts \
|
||||
--caselist D:\\glcts\\mustpass\\gl30.txt --outdir D:\\results\\gl30 \
|
||||
--backend DirectVulkan \
|
||||
--deqp-arg=--deqp-surface-type=window
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Iterable, Optional, Sequence
|
||||
|
||||
|
||||
CASE_START = re.compile(r"^#beginTestCaseResult\s+(\S+)")
|
||||
CASE_END = re.compile(r"^#endTestCaseResult(?:\s|$)")
|
||||
CASE_TERM = re.compile(r"^#terminateTestCaseResult(?:\s|$)")
|
||||
CASE_RESULT = re.compile(r'<Result\s+StatusCode="[^"]+"')
|
||||
CHUNK_ARTIFACT = re.compile(r"^chunk(\d+)(?:\.|$)", re.IGNORECASE)
|
||||
CHUNK_META = re.compile(r"^chunk(\d+)\.meta\.json$", re.IGNORECASE)
|
||||
RECOVERY_SIDECAR_NAMES = frozenset(
|
||||
{"crashed.txt", "hung.txt", "unrun.txt", "skipped.txt", "remaining.txt"}
|
||||
)
|
||||
CONTROLLED_DEQP_OPTIONS = {
|
||||
"--deqp-caselist-file",
|
||||
"--deqp-log-filename",
|
||||
}
|
||||
ATOMIC_REPLACE_ATTEMPTS = 8
|
||||
ATOMIC_REPLACE_INITIAL_BACKOFF_SECONDS = 0.025
|
||||
ATOMIC_REPLACE_MAX_BACKOFF_SECONDS = 0.2
|
||||
|
||||
|
||||
class RunnerError(Exception):
|
||||
"""A user/configuration error which should not be attributed to a case."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class QpaProgress:
|
||||
"""Cases recorded by a QPA and its unterminated tail, if any."""
|
||||
|
||||
recorded: list[str]
|
||||
in_flight: Optional[str]
|
||||
begin_count: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessOutcome:
|
||||
returncode: Optional[int]
|
||||
duration_seconds: float
|
||||
timed_out: bool = False
|
||||
timeout_reason: Optional[str] = None
|
||||
interrupted: bool = False
|
||||
launch_error: Optional[str] = None
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def read_caselist(path: Path) -> list[str]:
|
||||
"""Read a dEQP text caselist, preserving order and removing duplicates."""
|
||||
|
||||
try:
|
||||
lines = path.read_text(encoding="utf-8-sig", errors="strict").splitlines()
|
||||
except (OSError, UnicodeError) as exc:
|
||||
raise RunnerError(f"cannot read caselist {path}: {exc}") from exc
|
||||
|
||||
cases: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for raw in lines:
|
||||
case = raw.strip()
|
||||
if not case or case.startswith("#") or case in seen:
|
||||
continue
|
||||
cases.append(case)
|
||||
seen.add(case)
|
||||
if not cases:
|
||||
raise RunnerError(f"caselist contains no test cases: {path}")
|
||||
return cases
|
||||
|
||||
|
||||
def read_name_set(path: Path) -> set[str]:
|
||||
if not path.is_file():
|
||||
return set()
|
||||
try:
|
||||
return {
|
||||
line.strip()
|
||||
for line in path.read_text(encoding="utf-8-sig", errors="replace").splitlines()
|
||||
if line.strip() and not line.lstrip().startswith("#")
|
||||
}
|
||||
except OSError as exc:
|
||||
raise RunnerError(f"cannot read recovery file {path}: {exc}") from exc
|
||||
|
||||
|
||||
def _replace_with_retry(source: Path, destination: Path) -> None:
|
||||
"""Replace a state file, tolerating brief Windows access-denied races.
|
||||
|
||||
Antivirus/indexing tools can momentarily open ``remaining.txt`` without
|
||||
delete sharing. Windows then reports either ``PermissionError`` or a
|
||||
generic ``OSError`` carrying ``winerror == 5``. Retry only those cases;
|
||||
disk, path, and programming errors remain immediately visible.
|
||||
"""
|
||||
|
||||
for attempt in range(ATOMIC_REPLACE_ATTEMPTS):
|
||||
try:
|
||||
os.replace(source, destination)
|
||||
return
|
||||
except OSError as exc:
|
||||
retryable = isinstance(exc, PermissionError) or getattr(exc, "winerror", None) == 5
|
||||
if not retryable or attempt + 1 >= ATOMIC_REPLACE_ATTEMPTS:
|
||||
raise
|
||||
delay = min(
|
||||
ATOMIC_REPLACE_INITIAL_BACKOFF_SECONDS * (2**attempt),
|
||||
ATOMIC_REPLACE_MAX_BACKOFF_SECONDS,
|
||||
)
|
||||
time.sleep(delay)
|
||||
|
||||
|
||||
def atomic_write_text(path: Path, text: str) -> None:
|
||||
"""Replace a small state file without exposing a partially-written copy."""
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=str(path.parent))
|
||||
temporary_path = Path(temporary)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(text)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
_replace_with_retry(temporary_path, path)
|
||||
finally:
|
||||
try:
|
||||
temporary_path.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
def atomic_write_json(path: Path, value: object) -> None:
|
||||
atomic_write_text(path, json.dumps(value, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def write_case_file(path: Path, cases: Iterable[str]) -> None:
|
||||
values = list(cases)
|
||||
atomic_write_text(path, "\n".join(values) + ("\n" if values else ""))
|
||||
|
||||
|
||||
def scan_qpa(path: Path) -> QpaProgress:
|
||||
"""Return cases with a final result and the unfinished tail, if any.
|
||||
|
||||
``#terminateTestCaseResult`` is a completed result (usually Crash or
|
||||
Timeout). ``#endTestCaseResult`` only completes a case when its XML carried
|
||||
a ``<Result StatusCode=...>``. A truncated case that already wrote Result is
|
||||
also recoverable; a case with no Result remains eligible for a retry.
|
||||
"""
|
||||
|
||||
if not path.is_file():
|
||||
return QpaProgress([], None, 0)
|
||||
|
||||
recorded: list[str] = []
|
||||
current: Optional[str] = None
|
||||
has_result = False
|
||||
begin_count = 0
|
||||
try:
|
||||
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
||||
for raw_line in handle:
|
||||
line = raw_line.lstrip("\ufeff")
|
||||
match = CASE_START.match(line)
|
||||
if match:
|
||||
if current is not None and has_result:
|
||||
recorded.append(current)
|
||||
current = match.group(1)
|
||||
has_result = False
|
||||
begin_count += 1
|
||||
continue
|
||||
if current is not None and CASE_RESULT.search(line):
|
||||
has_result = True
|
||||
continue
|
||||
if current is not None and CASE_TERM.match(line):
|
||||
recorded.append(current)
|
||||
current = None
|
||||
has_result = False
|
||||
continue
|
||||
if current is not None and CASE_END.match(line):
|
||||
if has_result:
|
||||
recorded.append(current)
|
||||
current = None
|
||||
has_result = False
|
||||
except OSError as exc:
|
||||
raise RunnerError(f"cannot read QPA {path}: {exc}") from exc
|
||||
if current is not None and has_result:
|
||||
recorded.append(current)
|
||||
current = None
|
||||
return QpaProgress(recorded, current, begin_count)
|
||||
|
||||
|
||||
def numbered_files(outdir: Path, pattern: re.Pattern[str]) -> list[tuple[int, Path]]:
|
||||
found: list[tuple[int, Path]] = []
|
||||
try:
|
||||
children = list(outdir.iterdir())
|
||||
except OSError as exc:
|
||||
raise RunnerError(f"cannot list output directory {outdir}: {exc}") from exc
|
||||
for path in children:
|
||||
match = pattern.match(path.name)
|
||||
if match:
|
||||
found.append((int(match.group(1)), path))
|
||||
found.sort(key=lambda item: item[0])
|
||||
return found
|
||||
|
||||
|
||||
def next_chunk_number(outdir: Path) -> int:
|
||||
numbers = [number for number, _path in numbered_files(outdir, CHUNK_ARTIFACT)]
|
||||
return max(numbers, default=-1) + 1
|
||||
|
||||
|
||||
def load_meta_classifications(outdir: Path, expected: set[str]) -> tuple[set[str], set[str]]:
|
||||
"""Recover an atomic classification written just before sidecar updates."""
|
||||
|
||||
crashed: set[str] = set()
|
||||
hung: set[str] = set()
|
||||
for _number, path in numbered_files(outdir, CHUNK_META):
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError):
|
||||
# A damaged metadata file is diagnostic only. QPA and sidecars are
|
||||
# authoritative and must still allow recovery.
|
||||
continue
|
||||
if not isinstance(value, dict):
|
||||
continue
|
||||
case = value.get("classified_case")
|
||||
classification = value.get("classification")
|
||||
if not isinstance(case, str) or case not in expected:
|
||||
continue
|
||||
if classification == "DeviceHang":
|
||||
hung.add(case)
|
||||
elif classification == "Crash":
|
||||
crashed.add(case)
|
||||
crashed.difference_update(hung)
|
||||
return crashed, hung
|
||||
|
||||
|
||||
def result_qpa_files(outdir: Path) -> list[Path]:
|
||||
"""Return every QPA a directory-based report would consume."""
|
||||
|
||||
found: list[Path] = []
|
||||
try:
|
||||
for root, directories, names in os.walk(outdir):
|
||||
directories.sort(key=str.casefold)
|
||||
for name in sorted(names, key=str.casefold):
|
||||
if name.casefold().endswith(".qpa"):
|
||||
found.append(Path(root) / name)
|
||||
except OSError as exc:
|
||||
raise RunnerError(f"cannot scan output directory {outdir}: {exc}") from exc
|
||||
return found
|
||||
|
||||
|
||||
def recover_results(outdir: Path, expected: set[str]) -> tuple[set[str], set[str], set[str]]:
|
||||
recorded: set[str] = set()
|
||||
for path in result_qpa_files(outdir):
|
||||
progress = scan_qpa(path)
|
||||
recorded.update(case for case in progress.recorded if case in expected)
|
||||
|
||||
crashed = read_name_set(outdir / "crashed.txt") & expected
|
||||
hung = read_name_set(outdir / "hung.txt") & expected
|
||||
meta_crashed, meta_hung = load_meta_classifications(outdir, expected)
|
||||
crashed.update(meta_crashed)
|
||||
hung.update(meta_hung)
|
||||
crashed.difference_update(hung)
|
||||
return recorded, crashed, hung
|
||||
|
||||
|
||||
def caselist_fingerprint(cases: Sequence[str]) -> str:
|
||||
payload = "\n".join(cases).encode("utf-8") + b"\n"
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def recovery_artifacts(outdir: Path) -> list[Path]:
|
||||
"""Return prior-run evidence which must not be adopted implicitly."""
|
||||
|
||||
found = set(result_qpa_files(outdir))
|
||||
try:
|
||||
children = list(outdir.iterdir())
|
||||
except OSError as exc:
|
||||
raise RunnerError(f"cannot list output directory {outdir}: {exc}") from exc
|
||||
found.update(
|
||||
path
|
||||
for path in children
|
||||
if CHUNK_ARTIFACT.match(path.name)
|
||||
or path.name.casefold() in RECOVERY_SIDECAR_NAMES
|
||||
)
|
||||
return sorted(
|
||||
found,
|
||||
key=lambda path: str(path.relative_to(outdir)).casefold(),
|
||||
)
|
||||
|
||||
|
||||
def check_run_identity(
|
||||
outdir: Path,
|
||||
backend: str,
|
||||
cases: Sequence[str],
|
||||
invocation_identity: Optional[str] = None,
|
||||
adopt_legacy: bool = False,
|
||||
) -> None:
|
||||
"""Refuse to silently mix different suites/backends in one result dir."""
|
||||
|
||||
path = outdir / "run_state.json"
|
||||
fingerprint = caselist_fingerprint(cases)
|
||||
if path.is_file():
|
||||
try:
|
||||
state = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
raise RunnerError(f"cannot read run identity {path}: {exc}") from exc
|
||||
if not isinstance(state, dict):
|
||||
raise RunnerError(f"run identity must be a JSON object: {path}")
|
||||
if state.get("backend") != backend:
|
||||
raise RunnerError(
|
||||
f"output directory belongs to backend {state.get('backend')!r}, not {backend!r}: {outdir}"
|
||||
)
|
||||
if state.get("caselist_sha256") != fingerprint:
|
||||
raise RunnerError(f"output directory belongs to a different caselist: {outdir}")
|
||||
stored_invocation_identity = state.get("invocation_identity")
|
||||
if (
|
||||
stored_invocation_identity is not None
|
||||
or invocation_identity is not None
|
||||
) and stored_invocation_identity != invocation_identity:
|
||||
raise RunnerError(f"output directory belongs to a different CTS invocation: {outdir}")
|
||||
return
|
||||
|
||||
legacy_artifacts = recovery_artifacts(outdir)
|
||||
if legacy_artifacts and not adopt_legacy:
|
||||
examples = ", ".join(path.name for path in legacy_artifacts[:3])
|
||||
raise RunnerError(
|
||||
"output directory contains CTS recovery artifacts but no run_state.json; "
|
||||
f"refusing to adopt unverified legacy results ({examples}). Re-run with "
|
||||
"--adopt-legacy only after verifying the backend, caselist, and invocation."
|
||||
)
|
||||
|
||||
atomic_write_json(
|
||||
path,
|
||||
{
|
||||
"version": 1,
|
||||
"backend": backend,
|
||||
"case_count": len(cases),
|
||||
"caselist_sha256": fingerprint,
|
||||
"invocation_identity": invocation_identity,
|
||||
"adopted_legacy": bool(legacy_artifacts),
|
||||
"created_utc": utc_now(),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def persist_sidecars(
|
||||
outdir: Path,
|
||||
ordered_cases: Sequence[str],
|
||||
crashed: set[str],
|
||||
hung: set[str],
|
||||
remaining: Sequence[str],
|
||||
) -> None:
|
||||
write_case_file(outdir / "crashed.txt", (case for case in ordered_cases if case in crashed))
|
||||
write_case_file(outdir / "hung.txt", (case for case in ordered_cases if case in hung))
|
||||
write_case_file(outdir / "unrun.txt", remaining)
|
||||
write_case_file(outdir / "remaining.txt", remaining)
|
||||
|
||||
|
||||
def parse_environment(values: Sequence[str]) -> dict[str, str]:
|
||||
result: dict[str, str] = {}
|
||||
for value in values:
|
||||
if "=" not in value:
|
||||
raise RunnerError(f"--env expects NAME=VALUE, got {value!r}")
|
||||
name, contents = value.split("=", 1)
|
||||
if not name or "\x00" in name or "=" in name:
|
||||
raise RunnerError(f"invalid environment variable name in {value!r}")
|
||||
result[name] = contents
|
||||
return result
|
||||
|
||||
|
||||
def validate_deqp_args(values: Sequence[str]) -> None:
|
||||
for value in values:
|
||||
option = value.split("=", 1)[0].lower()
|
||||
if option in CONTROLLED_DEQP_OPTIONS:
|
||||
raise RunnerError(f"{option} is controlled by this runner and cannot be supplied via --deqp-arg")
|
||||
|
||||
|
||||
def resolve_paths(
|
||||
exe_value: str,
|
||||
workdir_value: Optional[str],
|
||||
caselist_value: str,
|
||||
outdir_value: str,
|
||||
) -> tuple[Path, Path, Path, Path]:
|
||||
launch_dir = Path.cwd()
|
||||
requested_exe = Path(exe_value).expanduser()
|
||||
|
||||
if workdir_value:
|
||||
workdir = Path(workdir_value).expanduser().resolve()
|
||||
elif requested_exe.is_absolute():
|
||||
workdir = requested_exe.resolve().parent
|
||||
else:
|
||||
workdir = launch_dir
|
||||
|
||||
if requested_exe.is_absolute():
|
||||
exe = requested_exe.resolve()
|
||||
else:
|
||||
in_workdir = (workdir / requested_exe).resolve()
|
||||
in_launch_dir = (launch_dir / requested_exe).resolve()
|
||||
exe = in_workdir if in_workdir.is_file() else in_launch_dir
|
||||
|
||||
caselist = Path(caselist_value).expanduser().resolve()
|
||||
outdir = Path(outdir_value).expanduser().resolve()
|
||||
if not exe.is_file():
|
||||
raise RunnerError(f"glcts executable does not exist: {exe}")
|
||||
if not workdir.is_dir():
|
||||
raise RunnerError(f"working directory does not exist: {workdir}")
|
||||
if not caselist.is_file():
|
||||
raise RunnerError(f"caselist does not exist: {caselist}")
|
||||
return exe, workdir, caselist, outdir
|
||||
|
||||
|
||||
def qpa_signature(path: Path) -> Optional[tuple[int, int]]:
|
||||
try:
|
||||
stat = path.stat()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except OSError:
|
||||
# A transient sharing violation must not kill a healthy process. The
|
||||
# next poll will retry and the idle clock retains its previous value.
|
||||
return None
|
||||
return stat.st_size, stat.st_mtime_ns
|
||||
|
||||
|
||||
def kill_process_tree(process: subprocess.Popen[bytes]) -> None:
|
||||
"""Force-stop the process and descendants, with a parent-only fallback."""
|
||||
|
||||
if process.poll() is not None:
|
||||
return
|
||||
|
||||
if os.name == "nt":
|
||||
# /T is essential: CTS/platform helpers can outlive the top-level
|
||||
# process, retain the QPA/DLL, and poison the next continuation round.
|
||||
taskkill = Path(os.environ.get("SystemRoot", r"C:\Windows")) / "System32" / "taskkill.exe"
|
||||
command = [str(taskkill), "/PID", str(process.pid), "/T", "/F"]
|
||||
try:
|
||||
subprocess.run(
|
||||
command,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=20,
|
||||
check=False,
|
||||
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
pass
|
||||
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
return
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
|
||||
try:
|
||||
process.kill()
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
|
||||
|
||||
def run_process(
|
||||
command: Sequence[str],
|
||||
workdir: Path,
|
||||
environment: dict[str, str],
|
||||
qpa_path: Path,
|
||||
stdout_path: Path,
|
||||
stderr_path: Path,
|
||||
idle_timeout: float,
|
||||
max_round_seconds: float,
|
||||
poll_seconds: float,
|
||||
) -> ProcessOutcome:
|
||||
"""Run one CTS chunk, killing its tree only after QPA progress stalls."""
|
||||
|
||||
started = time.monotonic()
|
||||
with stdout_path.open("wb") as stdout_handle, stderr_path.open("wb") as stderr_handle:
|
||||
popen_options: dict[str, object] = {
|
||||
"cwd": str(workdir),
|
||||
"env": environment,
|
||||
"stdin": subprocess.DEVNULL,
|
||||
"stdout": stdout_handle,
|
||||
"stderr": stderr_handle,
|
||||
}
|
||||
if os.name == "nt":
|
||||
popen_options["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
|
||||
else:
|
||||
popen_options["start_new_session"] = True
|
||||
|
||||
try:
|
||||
process = subprocess.Popen(list(command), **popen_options) # type: ignore[arg-type]
|
||||
except OSError as exc:
|
||||
message = f"failed to launch {command[0]}: {exc}\n"
|
||||
stderr_handle.write(message.encode("utf-8", errors="replace"))
|
||||
stderr_handle.flush()
|
||||
return ProcessOutcome(None, time.monotonic() - started, launch_error=str(exc))
|
||||
|
||||
last_signature = qpa_signature(qpa_path)
|
||||
last_progress = time.monotonic()
|
||||
timed_out = False
|
||||
timeout_reason: Optional[str] = None
|
||||
interrupted = False
|
||||
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
returncode = process.wait(timeout=poll_seconds)
|
||||
break
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
|
||||
now = time.monotonic()
|
||||
signature = qpa_signature(qpa_path)
|
||||
if signature is not None and signature != last_signature:
|
||||
last_signature = signature
|
||||
last_progress = now
|
||||
|
||||
if idle_timeout > 0 and now - last_progress >= idle_timeout:
|
||||
timed_out = True
|
||||
timeout_reason = "qpa-idle"
|
||||
kill_process_tree(process)
|
||||
returncode = process.poll()
|
||||
break
|
||||
if max_round_seconds > 0 and now - started >= max_round_seconds:
|
||||
timed_out = True
|
||||
timeout_reason = "max-round"
|
||||
kill_process_tree(process)
|
||||
returncode = process.poll()
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
interrupted = True
|
||||
kill_process_tree(process)
|
||||
returncode = process.poll()
|
||||
|
||||
return ProcessOutcome(
|
||||
returncode=returncode,
|
||||
duration_seconds=time.monotonic() - started,
|
||||
timed_out=timed_out,
|
||||
timeout_reason=timeout_reason,
|
||||
interrupted=interrupted,
|
||||
)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run Windows glcts against MobileGL, resuming across crashes and GPU hangs."
|
||||
)
|
||||
parser.add_argument("--exe", required=True, help="path to glcts.exe")
|
||||
parser.add_argument(
|
||||
"--workdir",
|
||||
help="glcts working directory (default: executable directory, or current directory for a relative exe)",
|
||||
)
|
||||
parser.add_argument("--caselist", required=True, help="mustpass/caselist text file")
|
||||
parser.add_argument("--outdir", required=True, help="persistent result directory")
|
||||
parser.add_argument("--backend", required=True, choices=("DirectGLES", "DirectVulkan"))
|
||||
parser.add_argument(
|
||||
"--run-identity",
|
||||
help="controller fingerprint for executable, data, arguments, and environment",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--adopt-legacy",
|
||||
action="store_true",
|
||||
help=(
|
||||
"adopt existing chunk/sidecar results which predate run_state.json; "
|
||||
"disabled by default because their provenance cannot be verified"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--env",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="NAME=VALUE",
|
||||
help="extra child environment variable (repeatable)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deqp-arg",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="ARG",
|
||||
help="extra glcts argument; repeat and use --deqp-arg=--option=value for leading dashes",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--idle-timeout",
|
||||
type=float,
|
||||
default=300.0,
|
||||
metavar="SECONDS",
|
||||
help="kill a chunk after this many seconds with no QPA size/mtime change (0 disables; default: 300)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-round-seconds",
|
||||
type=float,
|
||||
default=0.0,
|
||||
metavar="SECONDS",
|
||||
help="optional total wall limit for one invocation (0 disables; default: 0)",
|
||||
)
|
||||
parser.add_argument("--poll-seconds", type=float, default=1.0, help=argparse.SUPPRESS)
|
||||
parser.add_argument(
|
||||
"--max-rounds",
|
||||
type=int,
|
||||
default=10000,
|
||||
help="maximum glcts invocations in this runner process (default: 10000)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-empty-streak",
|
||||
type=int,
|
||||
default=3,
|
||||
help="abort after this many invocations record no case at all; no case is blamed (default: 3)",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def execute(args: argparse.Namespace) -> int:
|
||||
if args.idle_timeout < 0 or args.max_round_seconds < 0:
|
||||
raise RunnerError("timeout values must be non-negative")
|
||||
if args.poll_seconds <= 0:
|
||||
raise RunnerError("--poll-seconds must be greater than zero")
|
||||
if args.max_rounds <= 0 or args.max_empty_streak <= 0:
|
||||
raise RunnerError("--max-rounds and --max-empty-streak must be greater than zero")
|
||||
validate_deqp_args(args.deqp_arg)
|
||||
extra_environment = parse_environment(args.env)
|
||||
exe, workdir, caselist_path, outdir = resolve_paths(
|
||||
args.exe, args.workdir, args.caselist, args.outdir
|
||||
)
|
||||
outdir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
cases = read_caselist(caselist_path)
|
||||
expected = set(cases)
|
||||
check_run_identity(
|
||||
outdir,
|
||||
args.backend,
|
||||
cases,
|
||||
args.run_identity,
|
||||
adopt_legacy=args.adopt_legacy,
|
||||
)
|
||||
recorded, crashed, hung = recover_results(outdir, expected)
|
||||
accounted = recorded | crashed | hung
|
||||
remaining = [case for case in cases if case not in accounted]
|
||||
persist_sidecars(outdir, cases, crashed, hung, remaining)
|
||||
|
||||
existing_qpas = len(result_qpa_files(outdir))
|
||||
print(
|
||||
f"[run_cts_windows] {args.backend}: expected {len(cases)}, recovered {len(accounted)} "
|
||||
f"({existing_qpas} QPA chunk(s), {len(crashed)} crash, {len(hung)} hang)"
|
||||
)
|
||||
if not remaining:
|
||||
print(f"[run_cts_windows] complete: all {len(cases)} expected cases are accounted")
|
||||
return 0
|
||||
|
||||
environment = os.environ.copy()
|
||||
environment.update(extra_environment)
|
||||
# --backend is authoritative even if the inherited or extra environment
|
||||
# already contains a different value.
|
||||
environment["MOBILEGL_BACKEND_TYPE"] = args.backend
|
||||
|
||||
common_arguments = [
|
||||
"--deqp-terminate-on-device-lost=disable",
|
||||
"--deqp-log-images=disable",
|
||||
"--deqp-log-shader-sources=disable",
|
||||
]
|
||||
|
||||
chunk_number = next_chunk_number(outdir)
|
||||
rounds = 0
|
||||
empty_streak = 0
|
||||
interrupted = False
|
||||
fatal_launch_error = False
|
||||
started_all = time.monotonic()
|
||||
|
||||
while remaining and rounds < args.max_rounds:
|
||||
prefix = f"chunk{chunk_number:04d}"
|
||||
remaining_path = outdir / "remaining.txt"
|
||||
qpa_path = outdir / f"{prefix}.qpa"
|
||||
stdout_path = outdir / f"{prefix}.stdout.log"
|
||||
stderr_path = outdir / f"{prefix}.stderr.log"
|
||||
meta_path = outdir / f"{prefix}.meta.json"
|
||||
|
||||
# The number allocator considers every chunk artifact, so these should
|
||||
# be new. Refuse to truncate evidence if a foreign file races us.
|
||||
for artifact in (qpa_path, stdout_path, stderr_path, meta_path):
|
||||
if artifact.exists():
|
||||
raise RunnerError(f"refusing to overwrite existing chunk artifact: {artifact}")
|
||||
write_case_file(remaining_path, remaining)
|
||||
|
||||
command = [
|
||||
str(exe),
|
||||
f"--deqp-caselist-file={remaining_path}",
|
||||
f"--deqp-log-filename={qpa_path}",
|
||||
*common_arguments,
|
||||
*args.deqp_arg,
|
||||
]
|
||||
print(
|
||||
f"[run_cts_windows] {prefix}: launching {len(remaining)} remaining case(s); "
|
||||
f"idle timeout {args.idle_timeout:g}s"
|
||||
)
|
||||
chunk_started_utc = utc_now()
|
||||
outcome = run_process(
|
||||
command,
|
||||
workdir,
|
||||
environment,
|
||||
qpa_path,
|
||||
stdout_path,
|
||||
stderr_path,
|
||||
args.idle_timeout,
|
||||
args.max_round_seconds,
|
||||
args.poll_seconds,
|
||||
)
|
||||
progress = scan_qpa(qpa_path)
|
||||
|
||||
before = set(accounted)
|
||||
for case in progress.recorded:
|
||||
if case in expected:
|
||||
recorded.add(case)
|
||||
accounted.add(case)
|
||||
|
||||
classification: Optional[str] = None
|
||||
classified_case: Optional[str] = None
|
||||
in_flight = progress.in_flight if progress.in_flight in expected else None
|
||||
if not outcome.interrupted and in_flight is not None and in_flight not in accounted:
|
||||
classified_case = in_flight
|
||||
if outcome.timed_out:
|
||||
classification = "DeviceHang"
|
||||
hung.add(in_flight)
|
||||
crashed.discard(in_flight)
|
||||
else:
|
||||
classification = "Crash"
|
||||
crashed.add(in_flight)
|
||||
accounted.add(in_flight)
|
||||
|
||||
new_accounted = len(accounted - before)
|
||||
if new_accounted:
|
||||
empty_streak = 0
|
||||
elif progress.begin_count == 0:
|
||||
# No #begin marker means there is no evidence that the first
|
||||
# remaining case was reached. Retry the identical caselist, then
|
||||
# abort rather than manufacturing a string of false Crash results.
|
||||
empty_streak += 1
|
||||
else:
|
||||
# A log containing only already-accounted cases is also no forward
|
||||
# progress, but it is a different failure mode. Bound it with the
|
||||
# same guard while retaining the QPA evidence.
|
||||
empty_streak += 1
|
||||
|
||||
remaining = [case for case in cases if case not in accounted]
|
||||
metadata = {
|
||||
"version": 1,
|
||||
"chunk": chunk_number,
|
||||
"started_utc": chunk_started_utc,
|
||||
"finished_utc": utc_now(),
|
||||
"duration_seconds": round(outcome.duration_seconds, 3),
|
||||
"returncode": outcome.returncode,
|
||||
"timed_out": outcome.timed_out,
|
||||
"timeout_reason": outcome.timeout_reason,
|
||||
"interrupted": outcome.interrupted,
|
||||
"launch_error": outcome.launch_error,
|
||||
"qpa_begin_count": progress.begin_count,
|
||||
"qpa_recorded_count": len(progress.recorded),
|
||||
"in_flight": progress.in_flight,
|
||||
"classification": classification,
|
||||
"classified_case": classified_case,
|
||||
"new_accounted": new_accounted,
|
||||
"remaining": len(remaining),
|
||||
}
|
||||
# Metadata is committed first. If the runner itself dies between this
|
||||
# write and the sidecars, recovery can reconstruct the classification.
|
||||
atomic_write_json(meta_path, metadata)
|
||||
persist_sidecars(outdir, cases, crashed, hung, remaining)
|
||||
|
||||
rounds += 1
|
||||
elapsed_minutes = (time.monotonic() - started_all) / 60.0
|
||||
detail = ""
|
||||
if classification:
|
||||
detail = f", {classification}={classified_case}"
|
||||
if outcome.timed_out:
|
||||
detail += f", timeout={outcome.timeout_reason}"
|
||||
print(
|
||||
f"[run_cts_windows] {prefix}: +{new_accounted}, accounted "
|
||||
f"{len(accounted)}/{len(cases)}, remaining {len(remaining)}{detail} "
|
||||
f"({elapsed_minutes:.1f} min)"
|
||||
)
|
||||
|
||||
chunk_number += 1
|
||||
if outcome.interrupted:
|
||||
interrupted = True
|
||||
print("[run_cts_windows] interrupted; process tree stopped and state preserved", file=sys.stderr)
|
||||
break
|
||||
if outcome.launch_error:
|
||||
fatal_launch_error = True
|
||||
print(
|
||||
f"[run_cts_windows] launch failed; see {stderr_path.name}: {outcome.launch_error}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
break
|
||||
if empty_streak >= args.max_empty_streak:
|
||||
print(
|
||||
f"[run_cts_windows] aborting after {empty_streak} consecutive chunks made no "
|
||||
"case progress; no unobserved case was labelled Crash/Hang",
|
||||
file=sys.stderr,
|
||||
)
|
||||
break
|
||||
|
||||
# Recompute from the persisted evidence so the final completeness claim is
|
||||
# subject to the exact same recovery path as a later invocation.
|
||||
final_recorded, final_crashed, final_hung = recover_results(outdir, expected)
|
||||
final_accounted = final_recorded | final_crashed | final_hung
|
||||
final_remaining = [case for case in cases if case not in final_accounted]
|
||||
persist_sidecars(outdir, cases, final_crashed, final_hung, final_remaining)
|
||||
|
||||
if not final_remaining and final_accounted == expected:
|
||||
print(
|
||||
f"[run_cts_windows] complete: all {len(cases)} expected cases are accounted "
|
||||
f"({len(final_crashed)} crash, {len(final_hung)} hang, {rounds} new invocation(s))"
|
||||
)
|
||||
return 0
|
||||
|
||||
print(
|
||||
f"[run_cts_windows] INCOMPLETE: {len(final_accounted)}/{len(cases)} accounted; "
|
||||
f"{len(final_remaining)} listed in {outdir / 'unrun.txt'}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
if interrupted:
|
||||
return 130
|
||||
if fatal_launch_error:
|
||||
return 3
|
||||
return 4
|
||||
|
||||
|
||||
def main(argv: Optional[Sequence[str]] = None) -> int:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
return execute(args)
|
||||
except RunnerError as exc:
|
||||
print(f"[run_cts_windows] ERROR: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
except OSError as exc:
|
||||
print(f"[run_cts_windows] ERROR: filesystem/process operation failed: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,251 @@
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from . import cts_matrix_report as report
|
||||
except ImportError: # Allows `python test_cts_matrix_report.py`.
|
||||
import cts_matrix_report as report
|
||||
|
||||
|
||||
def qpa_case(case, status):
|
||||
return (
|
||||
f"#beginTestCaseResult {case}\n"
|
||||
f'<Result StatusCode="{status}"/>\n'
|
||||
"#endTestCaseResult\n"
|
||||
)
|
||||
|
||||
|
||||
class MatrixReportTests(unittest.TestCase):
|
||||
def test_utf8_bom_caselist_matches_runner_semantics(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
caselist = root / "cases.txt"
|
||||
caselist.write_bytes(b"\xef\xbb\xbfcase.a\n")
|
||||
results = root / "results"
|
||||
results.mkdir()
|
||||
(results / "run.qpa").write_text(
|
||||
qpa_case("case.a", "Pass"), encoding="utf-8"
|
||||
)
|
||||
|
||||
item = report.build_version_report("gl30", str(caselist), [str(results)])
|
||||
|
||||
self.assertEqual(1, item["expected"])
|
||||
self.assertEqual({"case.a": "Pass"}, item["cases"]["results"])
|
||||
self.assertEqual("OK", item["validation"]["state"])
|
||||
|
||||
def test_incomplete_qpa_is_unrun_not_a_completed_result(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
caselist = root / "cases.txt"
|
||||
caselist.write_text("case.a\n", encoding="utf-8")
|
||||
results = root / "results"
|
||||
results.mkdir()
|
||||
(results / "run.qpa").write_text(
|
||||
"#beginTestCaseResult case.a\n#endTestCaseResult\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(results / "unrun.txt").write_text("case.a\n", encoding="utf-8")
|
||||
|
||||
item = report.build_version_report("gl46", str(caselist), [str(results)])
|
||||
|
||||
self.assertEqual(0, item["result"])
|
||||
self.assertEqual(1, item["unrun"])
|
||||
self.assertEqual(["case.a"], item["cases"]["incomplete_results"])
|
||||
self.assertEqual("INCOMPLETE", item["validation"]["state"])
|
||||
|
||||
def test_incomplete_qpa_is_upgraded_by_crash_sidecar(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
caselist = root / "cases.txt"
|
||||
caselist.write_text("case.a\n", encoding="utf-8")
|
||||
results = root / "results"
|
||||
results.mkdir()
|
||||
(results / "run.qpa").write_text(
|
||||
"#beginTestCaseResult case.a\n", encoding="utf-8"
|
||||
)
|
||||
(results / "crashed.txt").write_text("case.a\n", encoding="utf-8")
|
||||
|
||||
item = report.build_version_report("gl46", str(caselist), [str(results)])
|
||||
|
||||
self.assertEqual("Crash", item["cases"]["results"]["case.a"])
|
||||
self.assertEqual([], item["cases"]["incomplete_results"])
|
||||
self.assertEqual("OK", item["validation"]["state"])
|
||||
|
||||
def test_chunk_numbers_above_four_digits_use_numeric_order(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
caselist = root / "cases.txt"
|
||||
caselist.write_text("case.a\n", encoding="utf-8")
|
||||
results = root / "results"
|
||||
results.mkdir()
|
||||
(results / "alpha.qpa").write_text("# no results\n", encoding="utf-8")
|
||||
(results / "chunk9999.qpa").write_text(
|
||||
qpa_case("case.a", "Fail"), encoding="utf-8"
|
||||
)
|
||||
(results / "chunk10000.qpa").write_text(
|
||||
qpa_case("case.a", "Pass"), encoding="utf-8"
|
||||
)
|
||||
(results / "zeta.qpa").write_text("# no results\n", encoding="utf-8")
|
||||
|
||||
item = report.build_version_report(
|
||||
"gl46", str(caselist), [str(results)]
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
["alpha.qpa", "chunk9999.qpa", "chunk10000.qpa", "zeta.qpa"],
|
||||
[Path(path).name for path in item["inputs"]["qpa_files"]],
|
||||
)
|
||||
self.assertEqual("Pass", item["cases"]["results"]["case.a"])
|
||||
self.assertEqual(1, item["duplicate"])
|
||||
|
||||
def test_qpa_sidecars_duplicates_and_expected_denominator(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
caselist = root / "gl30.txt"
|
||||
caselist.write_text("\n".join("abcdefg") + "\n", encoding="utf-8")
|
||||
results = root / "results"
|
||||
results.mkdir()
|
||||
(results / "chunk0000.qpa").write_text(
|
||||
qpa_case("a", "Fail") + "#beginTestCaseResult e\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(results / "chunk0001.qpa").write_text(
|
||||
qpa_case("a", "Pass")
|
||||
+ qpa_case("b", "NotSupported")
|
||||
+ qpa_case("c", "QualityWarning")
|
||||
+ qpa_case("d", "Fail"),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(results / "crashed.txt").write_text("e\n", encoding="utf-8")
|
||||
(results / "hung.txt").write_text("f\n", encoding="utf-8")
|
||||
(results / "unrun.txt").write_text("g\n", encoding="utf-8")
|
||||
|
||||
item = report.build_version_report(
|
||||
"gl30", str(caselist), [str(results)]
|
||||
)
|
||||
|
||||
self.assertEqual(item["expected"], 7)
|
||||
self.assertEqual(item["result"], 6)
|
||||
self.assertEqual(item["pass"], 1)
|
||||
self.assertEqual(item["accepted"], 3)
|
||||
self.assertEqual(item["crash"], 1)
|
||||
self.assertEqual(item["hang"], 1)
|
||||
self.assertEqual(item["unrun"], 1)
|
||||
self.assertEqual(item["duplicate"], 1)
|
||||
self.assertEqual(item["cases"]["results"]["a"], "Pass")
|
||||
self.assertEqual(item["cases"]["results"]["e"], "Crash")
|
||||
self.assertEqual(item["cases"]["results"]["f"], "DeviceHang")
|
||||
self.assertAlmostEqual(item["strict_pass_rate"], 1 / 7)
|
||||
self.assertAlmostEqual(item["conformance_accepted_rate"], 3 / 7)
|
||||
self.assertAlmostEqual(
|
||||
item["rates"]["measured_only_conformance_accepted"], 3 / 6
|
||||
)
|
||||
self.assertEqual(item["validation"]["state"], "INCOMPLETE")
|
||||
self.assertEqual(item["validation"]["errors"], [])
|
||||
self.assertTrue(
|
||||
item["validation"]["invariant_expected_equals_result_plus_unrun"]
|
||||
)
|
||||
|
||||
def test_missing_result_is_inferred_and_rejected_when_not_declared(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
caselist = root / "cases.txt"
|
||||
caselist.write_text("a\nb\n", encoding="utf-8")
|
||||
results = root / "results"
|
||||
results.mkdir()
|
||||
(results / "run.qpa").write_text(qpa_case("a", "Pass"), encoding="utf-8")
|
||||
|
||||
item = report.build_version_report(
|
||||
"gl31", str(caselist), [str(results)]
|
||||
)
|
||||
|
||||
self.assertEqual(item["unrun"], 1)
|
||||
self.assertEqual(item["cases"]["unrun"], ["b"])
|
||||
self.assertEqual(item["validation"]["state"], "ERROR")
|
||||
self.assertEqual(item["validation"]["undeclared_unrun"], ["b"])
|
||||
self.assertIn("not declared", item["validation"]["errors"][0])
|
||||
self.assertEqual(item["strict_pass_rate"], 0.5)
|
||||
|
||||
def test_cli_emits_markdown_json_and_weighted_overall(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
statuses = {
|
||||
"gl30": "Pass",
|
||||
"gl31": "Fail",
|
||||
"gl32": "NotSupported",
|
||||
"gl33": None,
|
||||
}
|
||||
argv = []
|
||||
for version, status in statuses.items():
|
||||
caselist = root / f"{version}.txt"
|
||||
caselist.write_text(f"{version}.case\n", encoding="utf-8")
|
||||
result_dir = root / f"{version}-results"
|
||||
result_dir.mkdir()
|
||||
qpa = result_dir / "run.qpa"
|
||||
qpa.write_text(
|
||||
qpa_case(f"{version}.case", status) if status else "# empty run\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
if status is None:
|
||||
(result_dir / "unrun.txt").write_text(
|
||||
f"{version}.case\n", encoding="utf-8"
|
||||
)
|
||||
argv.extend(
|
||||
[
|
||||
f"--{version}-caselist",
|
||||
str(caselist),
|
||||
f"--{version}-results",
|
||||
str(result_dir),
|
||||
]
|
||||
)
|
||||
json_path = root / "matrix.json"
|
||||
argv.extend(["--json", str(json_path)])
|
||||
|
||||
stdout = io.StringIO()
|
||||
with contextlib.redirect_stdout(stdout):
|
||||
rc = report.main(argv)
|
||||
|
||||
self.assertEqual(rc, 1) # GL33 is explicitly incomplete.
|
||||
markdown = stdout.getvalue()
|
||||
self.assertIn("| Suite | Expected | Result", markdown)
|
||||
self.assertIn("| **Overall (weighted)**", markdown)
|
||||
payload = json.loads(json_path.read_text(encoding="utf-8"))
|
||||
overall = payload["overall"]
|
||||
self.assertEqual(overall["expected"], 4)
|
||||
self.assertEqual(overall["result"], 3)
|
||||
self.assertEqual(overall["pass"], 1)
|
||||
self.assertEqual(overall["accepted"], 2)
|
||||
self.assertEqual(overall["unrun"], 1)
|
||||
self.assertEqual(overall["strict_pass_rate"], 0.25)
|
||||
self.assertEqual(overall["conformance_accepted_rate"], 0.5)
|
||||
self.assertEqual(overall["aggregation"], "weighted_by_expected_cases")
|
||||
self.assertEqual(overall["validation"]["state"], "INCOMPLETE")
|
||||
|
||||
def test_duplicate_caselist_and_unexpected_result_are_validation_errors(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
caselist = root / "cases.txt"
|
||||
caselist.write_text("a\na\n", encoding="utf-8")
|
||||
results = root / "results"
|
||||
results.mkdir()
|
||||
(results / "run.qpa").write_text(
|
||||
qpa_case("a", "Pass") + qpa_case("outside", "Pass"),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
item = report.build_version_report(
|
||||
"gl32", str(caselist), [str(results)]
|
||||
)
|
||||
|
||||
self.assertEqual(item["cases"]["duplicate_caselist_entries"], {"a": 2})
|
||||
self.assertEqual(item["cases"]["unexpected_results"], {"outside": "Pass"})
|
||||
self.assertEqual(item["validation"]["state"], "ERROR")
|
||||
self.assertEqual(len(item["validation"]["errors"]), 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,342 @@
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from . import cts_multi_report as report
|
||||
except ImportError: # Allows `python test_cts_multi_report.py`.
|
||||
import cts_multi_report as report
|
||||
|
||||
|
||||
def qpa_case(case: str, status: str) -> str:
|
||||
return (
|
||||
f"#beginTestCaseResult {case}\n"
|
||||
f'<Result StatusCode="{status}"/>\n'
|
||||
"#endTestCaseResult\n"
|
||||
)
|
||||
|
||||
|
||||
def write_run_state(
|
||||
caselist: Path,
|
||||
result_dir: Path,
|
||||
backend: str,
|
||||
invocation_identity=None,
|
||||
) -> None:
|
||||
fingerprint, case_count = report._caselist_fingerprint(str(caselist))
|
||||
(result_dir / "run_state.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"version": 1,
|
||||
"backend": backend,
|
||||
"case_count": case_count,
|
||||
"caselist_sha256": fingerprint,
|
||||
"invocation_identity": invocation_identity,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def make_inputs(
|
||||
root: Path,
|
||||
name: str,
|
||||
cases: list[str],
|
||||
qpa: str,
|
||||
backend: str = "DirectGLES",
|
||||
):
|
||||
caselist = root / f"{name}.txt"
|
||||
caselist.write_text("\n".join(cases) + "\n", encoding="utf-8")
|
||||
result_dir = root / f"{name}-results"
|
||||
result_dir.mkdir()
|
||||
(result_dir / "chunk0000.qpa").write_text(qpa, encoding="utf-8")
|
||||
write_run_state(caselist, result_dir, backend)
|
||||
return caselist, result_dir
|
||||
|
||||
|
||||
class MultiReportTests(unittest.TestCase):
|
||||
def test_backend_aggregate_is_weighted_by_expected_cases(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
small_cases, small_results = make_inputs(
|
||||
root, "small", ["small.pass"], qpa_case("small.pass", "Pass")
|
||||
)
|
||||
large_cases, large_results = make_inputs(
|
||||
root,
|
||||
"large",
|
||||
["large.pass", "large.crash", "large.hang"],
|
||||
qpa_case("large.pass", "Pass"),
|
||||
)
|
||||
(large_results / "crashed.txt").write_text(
|
||||
"large.crash\n", encoding="utf-8"
|
||||
)
|
||||
(large_results / "hung.txt").write_text(
|
||||
"large.hang\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
payload = report.build_report(
|
||||
[
|
||||
report.SuiteSpec(
|
||||
"DirectGLES", "small", str(small_cases), str(small_results)
|
||||
),
|
||||
report.SuiteSpec(
|
||||
"DirectGLES", "large", str(large_cases), str(large_results)
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
aggregate = payload["backends"]["DirectGLES"]
|
||||
self.assertEqual(4, aggregate["expected"])
|
||||
self.assertEqual(4, aggregate["result"])
|
||||
self.assertEqual(2, aggregate["pass"])
|
||||
self.assertEqual(2, aggregate["accepted"])
|
||||
self.assertEqual(1, aggregate["crash"])
|
||||
self.assertEqual(1, aggregate["hang"])
|
||||
self.assertEqual(0, aggregate["unrun"])
|
||||
# (1 accepted + 1 accepted) / (1 expected + 3 expected), not
|
||||
# the unweighted mean of 100% and 33.3%.
|
||||
self.assertEqual(0.5, aggregate["conformance_accepted_rate"])
|
||||
self.assertEqual("weighted_by_expected_cases", aggregate["aggregation"])
|
||||
self.assertEqual("OK", aggregate["validation"]["state"])
|
||||
|
||||
def test_declared_unrun_is_incomplete_and_cli_returns_nonzero(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
caselist, result_dir = make_inputs(
|
||||
root, "missing", ["case.a", "case.b"], qpa_case("case.a", "Pass")
|
||||
)
|
||||
(result_dir / "unrun.txt").write_text("case.b\n", encoding="utf-8")
|
||||
markdown_path = root / "report.md"
|
||||
json_path = root / "report.json"
|
||||
argv = [
|
||||
"--suite",
|
||||
"DirectGLES",
|
||||
"gl30",
|
||||
str(caselist),
|
||||
str(result_dir),
|
||||
"--markdown",
|
||||
str(markdown_path),
|
||||
"--json",
|
||||
str(json_path),
|
||||
]
|
||||
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
returncode = report.main(argv)
|
||||
|
||||
self.assertEqual(1, returncode)
|
||||
self.assertTrue(markdown_path.is_file())
|
||||
payload = json.loads(json_path.read_text(encoding="utf-8"))
|
||||
suite = payload["suites"][0]
|
||||
self.assertEqual(2, suite["expected"])
|
||||
self.assertEqual(1, suite["result"])
|
||||
self.assertEqual(1, suite["unrun"])
|
||||
self.assertEqual("INCOMPLETE", suite["validation"]["state"])
|
||||
self.assertEqual("INCOMPLETE", payload["overall"]["validation"]["state"])
|
||||
self.assertEqual(0.5, payload["overall"]["conformance_accepted_rate"])
|
||||
|
||||
def test_dual_backend_cli_outputs_markdown_and_json(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
caselist = root / "gl30.txt"
|
||||
caselist.write_text("gl30.case\n", encoding="utf-8")
|
||||
gles = root / "gles"
|
||||
vulkan = root / "vulkan"
|
||||
gles.mkdir()
|
||||
vulkan.mkdir()
|
||||
(gles / "run.qpa").write_text(
|
||||
qpa_case("gl30.case", "Pass"), encoding="utf-8"
|
||||
)
|
||||
(vulkan / "run.qpa").write_text(
|
||||
qpa_case("gl30.case", "Fail"), encoding="utf-8"
|
||||
)
|
||||
write_run_state(caselist, gles, "DirectGLES")
|
||||
write_run_state(caselist, vulkan, "DirectVulkan")
|
||||
markdown_path = root / "dual.md"
|
||||
json_path = root / "dual.json"
|
||||
argv = [
|
||||
f"--suite=DirectGLES,gl30,{caselist},{gles}",
|
||||
"--suite",
|
||||
"DirectVulkan",
|
||||
"gl30",
|
||||
str(caselist),
|
||||
str(vulkan),
|
||||
"--markdown",
|
||||
str(markdown_path),
|
||||
"--json",
|
||||
str(json_path),
|
||||
]
|
||||
|
||||
stdout = io.StringIO()
|
||||
with contextlib.redirect_stdout(stdout):
|
||||
returncode = report.main(argv)
|
||||
|
||||
self.assertEqual(0, returncode)
|
||||
payload = json.loads(json_path.read_text(encoding="utf-8"))
|
||||
self.assertEqual({"DirectGLES", "DirectVulkan"}, set(payload["backends"]))
|
||||
self.assertEqual(1, payload["backends"]["DirectGLES"]["accepted"])
|
||||
self.assertEqual(0, payload["backends"]["DirectVulkan"]["accepted"])
|
||||
self.assertEqual(2, payload["overall"]["expected"])
|
||||
self.assertEqual(1, payload["overall"]["accepted"])
|
||||
self.assertEqual(0.5, payload["overall"]["conformance_accepted_rate"])
|
||||
markdown = markdown_path.read_text(encoding="utf-8")
|
||||
self.assertIn("DirectGLES weighted subtotal", markdown)
|
||||
self.assertIn("DirectVulkan weighted subtotal", markdown)
|
||||
self.assertIn("Overall weighted", markdown)
|
||||
self.assertIn("Markdown:", stdout.getvalue())
|
||||
|
||||
def test_duplicate_qpa_result_uses_last_observation(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
caselist, result_dir = make_inputs(
|
||||
root,
|
||||
"duplicate",
|
||||
["case.a"],
|
||||
qpa_case("case.a", "Fail"),
|
||||
backend="DirectVulkan",
|
||||
)
|
||||
(result_dir / "chunk0001.qpa").write_text(
|
||||
qpa_case("case.a", "Pass"), encoding="utf-8"
|
||||
)
|
||||
|
||||
payload = report.build_report(
|
||||
[
|
||||
report.SuiteSpec(
|
||||
"DirectVulkan", "gl33", str(caselist), str(result_dir)
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
suite = payload["suites"][0]
|
||||
self.assertEqual("Pass", suite["cases"]["results"]["case.a"])
|
||||
self.assertEqual(1, suite["duplicate"])
|
||||
self.assertEqual(1, payload["overall"]["duplicate"])
|
||||
self.assertEqual(1.0, payload["overall"]["strict_pass_rate"])
|
||||
self.assertEqual("OK", suite["validation"]["state"])
|
||||
self.assertIn("last result wins", suite["validation"]["warnings"][0])
|
||||
|
||||
def test_backend_provenance_mismatch_is_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
caselist, result_dir = make_inputs(
|
||||
root,
|
||||
"provenance",
|
||||
["case.a"],
|
||||
qpa_case("case.a", "Pass"),
|
||||
backend="DirectVulkan",
|
||||
)
|
||||
with self.assertRaises(report.MultiReportInputError):
|
||||
report.build_report(
|
||||
[report.SuiteSpec("DirectGLES", "gl30", str(caselist), str(result_dir))]
|
||||
)
|
||||
|
||||
def test_missing_provenance_requires_explicit_legacy_opt_in(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
caselist, result_dir = make_inputs(
|
||||
root, "legacy", ["case.a"], qpa_case("case.a", "Pass")
|
||||
)
|
||||
(result_dir / "run_state.json").unlink()
|
||||
spec = report.SuiteSpec("DirectGLES", "gl30", str(caselist), str(result_dir))
|
||||
with self.assertRaises(report.MultiReportInputError):
|
||||
report.build_report([spec])
|
||||
payload = report.build_report([spec], require_run_state=False)
|
||||
self.assertEqual("UNVERIFIED", payload["suites"][0]["provenance"]["state"])
|
||||
|
||||
def test_expected_run_identity_accepts_match_and_rejects_mismatch(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
caselist, result_dir = make_inputs(
|
||||
root, "identity", ["case.a"], qpa_case("case.a", "Pass")
|
||||
)
|
||||
write_run_state(
|
||||
caselist, result_dir, "DirectGLES", invocation_identity="identity-a"
|
||||
)
|
||||
spec = report.SuiteSpec(
|
||||
"DirectGLES", "gl30", str(caselist), str(result_dir)
|
||||
)
|
||||
|
||||
payload = report.build_report(
|
||||
[spec], expected_run_identity="identity-a"
|
||||
)
|
||||
self.assertEqual(
|
||||
"identity-a",
|
||||
payload["suites"][0]["provenance"]["invocation_identity"],
|
||||
)
|
||||
with self.assertRaises(report.MultiReportInputError):
|
||||
report.build_report([spec], expected_run_identity="identity-b")
|
||||
|
||||
def test_expected_identity_rejects_legacy_state_and_missing_state(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
caselist, result_dir = make_inputs(
|
||||
root, "legacy-identity", ["case.a"], qpa_case("case.a", "Pass")
|
||||
)
|
||||
spec = report.SuiteSpec(
|
||||
"DirectGLES", "gl30", str(caselist), str(result_dir)
|
||||
)
|
||||
with self.assertRaises(report.MultiReportInputError):
|
||||
report.build_report([spec], expected_run_identity="identity-a")
|
||||
|
||||
(result_dir / "run_state.json").unlink()
|
||||
with self.assertRaises(report.MultiReportInputError):
|
||||
report.build_report(
|
||||
[spec],
|
||||
require_run_state=False,
|
||||
expected_run_identity="identity-a",
|
||||
)
|
||||
|
||||
def test_duplicate_physical_result_directory_is_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
caselist, result_dir = make_inputs(
|
||||
root, "duplicate-dir", ["case.a"], qpa_case("case.a", "Pass")
|
||||
)
|
||||
with self.assertRaises(report.MultiReportInputError):
|
||||
report.build_report(
|
||||
[
|
||||
report.SuiteSpec(
|
||||
"DirectGLES", "gl30", str(caselist), str(result_dir)
|
||||
),
|
||||
report.SuiteSpec(
|
||||
"DirectGLES",
|
||||
"gl31",
|
||||
str(caselist),
|
||||
str(result_dir / "."),
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
def test_ancestor_and_descendant_result_directories_are_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
caselist = root / "cases.txt"
|
||||
caselist.write_text("case.a\n", encoding="utf-8")
|
||||
parent = root / "results"
|
||||
child = parent / "nested"
|
||||
child.mkdir(parents=True)
|
||||
(parent / "chunk0000.qpa").write_text(
|
||||
qpa_case("case.a", "Pass"), encoding="utf-8"
|
||||
)
|
||||
(child / "chunk0000.qpa").write_text(
|
||||
qpa_case("case.a", "Fail"), encoding="utf-8"
|
||||
)
|
||||
write_run_state(caselist, parent, "DirectGLES")
|
||||
write_run_state(caselist, child, "DirectVulkan")
|
||||
|
||||
with self.assertRaises(report.MultiReportInputError):
|
||||
report.build_report(
|
||||
[
|
||||
report.SuiteSpec(
|
||||
"DirectGLES", "gl30", str(caselist), str(parent)
|
||||
),
|
||||
report.SuiteSpec(
|
||||
"DirectVulkan", "gl30", str(caselist), str(child)
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,401 @@
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import run_cts_windows as runner
|
||||
|
||||
|
||||
def qpa_closed(case: str, status: str = "Pass") -> str:
|
||||
return (
|
||||
f"#beginTestCaseResult {case}\n"
|
||||
f'<Result StatusCode="{status}">ok</Result>\n'
|
||||
"#endTestCaseResult\n"
|
||||
)
|
||||
|
||||
|
||||
def command_path(command, option):
|
||||
prefix = option + "="
|
||||
return Path(next(value[len(prefix) :] for value in command if value.startswith(prefix)))
|
||||
|
||||
|
||||
class AtomicWriteTests(unittest.TestCase):
|
||||
def test_access_denied_retries_then_replace_succeeds(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
target = Path(temporary) / "remaining.txt"
|
||||
real_replace = runner.os.replace
|
||||
attempts = 0
|
||||
|
||||
def flaky_replace(source, destination):
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
if attempts == 1:
|
||||
raise PermissionError(13, "temporarily denied", str(destination))
|
||||
if attempts == 2:
|
||||
error = OSError("temporary WinError 5")
|
||||
error.winerror = 5
|
||||
raise error
|
||||
real_replace(source, destination)
|
||||
|
||||
with mock.patch.object(runner.os, "replace", side_effect=flaky_replace), mock.patch.object(
|
||||
runner.time, "sleep"
|
||||
) as sleep:
|
||||
runner.atomic_write_text(target, "case.a\n")
|
||||
|
||||
self.assertEqual(3, attempts)
|
||||
self.assertEqual("case.a\n", target.read_text(encoding="utf-8"))
|
||||
self.assertEqual(2, sleep.call_count)
|
||||
self.assertEqual(
|
||||
[
|
||||
mock.call(runner.ATOMIC_REPLACE_INITIAL_BACKOFF_SECONDS),
|
||||
mock.call(runner.ATOMIC_REPLACE_INITIAL_BACKOFF_SECONDS * 2),
|
||||
],
|
||||
sleep.call_args_list,
|
||||
)
|
||||
self.assertEqual([], list(target.parent.glob(".remaining.txt.*.tmp")))
|
||||
|
||||
def test_permanent_access_denied_stops_after_bounded_attempts(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
target = Path(temporary) / "remaining.txt"
|
||||
|
||||
def always_denied(_source, _destination):
|
||||
error = OSError("persistent WinError 5")
|
||||
error.winerror = 5
|
||||
raise error
|
||||
|
||||
with mock.patch.object(
|
||||
runner.os, "replace", side_effect=always_denied
|
||||
) as replace, mock.patch.object(runner.time, "sleep") as sleep:
|
||||
with self.assertRaises(OSError) as raised:
|
||||
runner.atomic_write_text(target, "case.a\n")
|
||||
|
||||
self.assertEqual(5, raised.exception.winerror)
|
||||
self.assertEqual(runner.ATOMIC_REPLACE_ATTEMPTS, replace.call_count)
|
||||
self.assertEqual(runner.ATOMIC_REPLACE_ATTEMPTS - 1, sleep.call_count)
|
||||
self.assertFalse(target.exists())
|
||||
self.assertEqual([], list(target.parent.glob(".remaining.txt.*.tmp")))
|
||||
|
||||
def test_non_access_error_is_not_retried(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
target = Path(temporary) / "remaining.txt"
|
||||
error = OSError(28, "disk full")
|
||||
with mock.patch.object(
|
||||
runner.os, "replace", side_effect=error
|
||||
) as replace, mock.patch.object(runner.time, "sleep") as sleep:
|
||||
with self.assertRaises(OSError):
|
||||
runner.atomic_write_text(target, "case.a\n")
|
||||
|
||||
self.assertEqual(1, replace.call_count)
|
||||
sleep.assert_not_called()
|
||||
|
||||
|
||||
class QpaParsingTests(unittest.TestCase):
|
||||
def test_terminate_is_a_completed_result(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
path = Path(temporary) / "chunk0000.qpa"
|
||||
path.write_text(
|
||||
"#beginTestCaseResult KHR-GL30.a\n"
|
||||
"#terminateTestCaseResult Crash\n"
|
||||
"#beginTestCaseResult KHR-GL30.b\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
progress = runner.scan_qpa(path)
|
||||
self.assertEqual(["KHR-GL30.a"], progress.recorded)
|
||||
self.assertEqual("KHR-GL30.b", progress.in_flight)
|
||||
self.assertEqual(2, progress.begin_count)
|
||||
|
||||
def test_end_without_result_is_not_accounted(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
path = Path(temporary) / "chunk0000.qpa"
|
||||
path.write_text(
|
||||
"#beginTestCaseResult KHR-GL46.incomplete\n"
|
||||
"#endTestCaseResult\n"
|
||||
+ qpa_closed("KHR-GL46.complete"),
|
||||
encoding="utf-8",
|
||||
)
|
||||
progress = runner.scan_qpa(path)
|
||||
self.assertEqual(["KHR-GL46.complete"], progress.recorded)
|
||||
self.assertIsNone(progress.in_flight)
|
||||
|
||||
def test_result_written_before_truncated_eof_is_recovered(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
path = Path(temporary) / "chunk0000.qpa"
|
||||
path.write_text(
|
||||
"#beginTestCaseResult KHR-GL46.complete\n"
|
||||
'<Result StatusCode="Pass">ok</Result>\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
progress = runner.scan_qpa(path)
|
||||
self.assertEqual(["KHR-GL46.complete"], progress.recorded)
|
||||
self.assertIsNone(progress.in_flight)
|
||||
|
||||
|
||||
class RunIdentityTests(unittest.TestCase):
|
||||
def test_non_object_run_state_is_a_controlled_error(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
outdir = Path(temporary)
|
||||
(outdir / "run_state.json").write_text("null\n", encoding="utf-8")
|
||||
with self.assertRaises(runner.RunnerError):
|
||||
runner.check_run_identity(outdir, "DirectVulkan", ["case.a"])
|
||||
|
||||
def test_controller_identity_prevents_mixed_invocations(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
outdir = Path(temporary)
|
||||
runner.check_run_identity(
|
||||
outdir, "DirectVulkan", ["case.a"], invocation_identity="identity-a"
|
||||
)
|
||||
runner.check_run_identity(
|
||||
outdir, "DirectVulkan", ["case.a"], invocation_identity="identity-a"
|
||||
)
|
||||
with self.assertRaises(runner.RunnerError):
|
||||
runner.check_run_identity(
|
||||
outdir, "DirectVulkan", ["case.a"], invocation_identity="identity-b"
|
||||
)
|
||||
|
||||
def test_controller_identity_cannot_be_downgraded_by_omission(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
outdir = Path(temporary)
|
||||
runner.check_run_identity(
|
||||
outdir, "DirectVulkan", ["case.a"], invocation_identity="identity-a"
|
||||
)
|
||||
with self.assertRaises(runner.RunnerError):
|
||||
runner.check_run_identity(outdir, "DirectVulkan", ["case.a"])
|
||||
|
||||
def test_legacy_artifacts_require_explicit_adoption(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
outdir = Path(temporary)
|
||||
(outdir / "chunk0000.qpa").write_text(
|
||||
qpa_closed("case.a"), encoding="utf-8"
|
||||
)
|
||||
with self.assertRaises(runner.RunnerError):
|
||||
runner.check_run_identity(
|
||||
outdir, "DirectVulkan", ["case.a"], invocation_identity="identity-a"
|
||||
)
|
||||
self.assertFalse((outdir / "run_state.json").exists())
|
||||
|
||||
runner.check_run_identity(
|
||||
outdir,
|
||||
"DirectVulkan",
|
||||
["case.a"],
|
||||
invocation_identity="identity-a",
|
||||
adopt_legacy=True,
|
||||
)
|
||||
state = runner.json.loads(
|
||||
(outdir / "run_state.json").read_text(encoding="utf-8")
|
||||
)
|
||||
self.assertTrue(state["adopted_legacy"])
|
||||
|
||||
def test_foreign_nested_qpa_and_skipped_sidecar_are_legacy_evidence(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
outdir = Path(temporary)
|
||||
nested = outdir / "old"
|
||||
nested.mkdir()
|
||||
qpa = nested / "legacy.qpa"
|
||||
qpa.write_text(qpa_closed("case.a"), encoding="utf-8")
|
||||
skipped = outdir / "skipped.txt"
|
||||
skipped.write_text("case.b\n", encoding="utf-8")
|
||||
|
||||
self.assertEqual(
|
||||
{qpa, skipped}, set(runner.recovery_artifacts(outdir))
|
||||
)
|
||||
with self.assertRaises(runner.RunnerError):
|
||||
runner.check_run_identity(
|
||||
outdir, "DirectVulkan", ["case.a", "case.b"]
|
||||
)
|
||||
recorded, crashed, hung = runner.recover_results(
|
||||
outdir, {"case.a", "case.b"}
|
||||
)
|
||||
self.assertEqual({"case.a"}, recorded)
|
||||
self.assertEqual(set(), crashed)
|
||||
self.assertEqual(set(), hung)
|
||||
|
||||
def test_non_object_or_non_string_meta_classification_is_ignored(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
outdir = Path(temporary)
|
||||
(outdir / "chunk0000.meta.json").write_text("null\n", encoding="utf-8")
|
||||
(outdir / "chunk0001.meta.json").write_text("[]\n", encoding="utf-8")
|
||||
(outdir / "chunk0002.meta.json").write_text(
|
||||
'{"classified_case": [], "classification": "Crash"}\n', encoding="utf-8"
|
||||
)
|
||||
self.assertEqual(
|
||||
(set(), set()), runner.load_meta_classifications(outdir, {"case.a"})
|
||||
)
|
||||
|
||||
|
||||
class RunnerRecoveryTests(unittest.TestCase):
|
||||
def run_args(self, root: Path, caselist: Path, outdir: Path, *extra: str):
|
||||
return [
|
||||
"--exe",
|
||||
sys.executable,
|
||||
"--workdir",
|
||||
str(root),
|
||||
"--caselist",
|
||||
str(caselist),
|
||||
"--outdir",
|
||||
str(outdir),
|
||||
"--backend",
|
||||
"DirectVulkan",
|
||||
*extra,
|
||||
]
|
||||
|
||||
def test_crash_tail_is_quarantined_and_next_chunk_resumes(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
caselist = root / "cases.txt"
|
||||
outdir = root / "results"
|
||||
caselist.write_text("KHR-GL30.a\nKHR-GL30.b\nKHR-GL30.c\n", encoding="utf-8")
|
||||
seen_remaining = []
|
||||
|
||||
def fake_run(command, workdir, environment, qpa_path, stdout_path, stderr_path, *timeouts):
|
||||
del workdir, timeouts
|
||||
seen_remaining.append(
|
||||
command_path(command, "--deqp-caselist-file")
|
||||
.read_text(encoding="utf-8")
|
||||
.splitlines()
|
||||
)
|
||||
stdout_path.write_text("fake stdout\n", encoding="utf-8")
|
||||
stderr_path.write_text("fake stderr\n", encoding="utf-8")
|
||||
self.assertEqual("DirectVulkan", environment["MOBILEGL_BACKEND_TYPE"])
|
||||
if len(seen_remaining) == 1:
|
||||
qpa_path.write_text(
|
||||
qpa_closed("KHR-GL30.a") + "#beginTestCaseResult KHR-GL30.b\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return runner.ProcessOutcome(0xC0000005, 0.1)
|
||||
qpa_path.write_text(qpa_closed("KHR-GL30.c"), encoding="utf-8")
|
||||
return runner.ProcessOutcome(0, 0.1)
|
||||
|
||||
with mock.patch.object(runner, "run_process", side_effect=fake_run):
|
||||
result = runner.main(self.run_args(root, caselist, outdir))
|
||||
|
||||
self.assertEqual(0, result)
|
||||
self.assertEqual(
|
||||
[
|
||||
["KHR-GL30.a", "KHR-GL30.b", "KHR-GL30.c"],
|
||||
["KHR-GL30.c"],
|
||||
],
|
||||
seen_remaining,
|
||||
)
|
||||
self.assertEqual("KHR-GL30.b\n", (outdir / "crashed.txt").read_text(encoding="utf-8"))
|
||||
self.assertEqual("", (outdir / "hung.txt").read_text(encoding="utf-8"))
|
||||
self.assertEqual("", (outdir / "unrun.txt").read_text(encoding="utf-8"))
|
||||
|
||||
def test_existing_qpa_and_sidecar_are_recovered(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
caselist = root / "cases.txt"
|
||||
outdir = root / "results"
|
||||
outdir.mkdir()
|
||||
caselist.write_text("KHR-GL31.a\nKHR-GL31.b\nKHR-GL31.c\n", encoding="utf-8")
|
||||
(outdir / "chunk0000.qpa").write_text(qpa_closed("KHR-GL31.a"), encoding="utf-8")
|
||||
(outdir / "crashed.txt").write_text("KHR-GL31.b\n", encoding="utf-8")
|
||||
seen_remaining = []
|
||||
|
||||
def fake_run(command, workdir, environment, qpa_path, stdout_path, stderr_path, *timeouts):
|
||||
del workdir, environment, stdout_path, stderr_path, timeouts
|
||||
seen_remaining.extend(
|
||||
command_path(command, "--deqp-caselist-file")
|
||||
.read_text(encoding="utf-8")
|
||||
.splitlines()
|
||||
)
|
||||
qpa_path.write_text(qpa_closed("KHR-GL31.c"), encoding="utf-8")
|
||||
return runner.ProcessOutcome(0, 0.1)
|
||||
|
||||
with mock.patch.object(runner, "run_process", side_effect=fake_run):
|
||||
result = runner.main(
|
||||
self.run_args(root, caselist, outdir, "--adopt-legacy")
|
||||
)
|
||||
|
||||
self.assertEqual(0, result)
|
||||
self.assertEqual(["KHR-GL31.c"], seen_remaining)
|
||||
self.assertTrue((outdir / "chunk0001.qpa").is_file())
|
||||
|
||||
def test_repeated_no_output_aborts_without_false_case_blame(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
caselist = root / "cases.txt"
|
||||
outdir = root / "results"
|
||||
caselist.write_text("KHR-GL32.a\nKHR-GL32.b\n", encoding="utf-8")
|
||||
seen_remaining = []
|
||||
|
||||
def fake_run(command, workdir, environment, qpa_path, stdout_path, stderr_path, *timeouts):
|
||||
del workdir, environment, stdout_path, stderr_path, timeouts
|
||||
seen_remaining.append(
|
||||
command_path(command, "--deqp-caselist-file")
|
||||
.read_text(encoding="utf-8")
|
||||
.splitlines()
|
||||
)
|
||||
qpa_path.write_text("#sessionInfo releaseName fake\n", encoding="utf-8")
|
||||
return runner.ProcessOutcome(
|
||||
1, 0.1, timed_out=True, timeout_reason="qpa-idle"
|
||||
)
|
||||
|
||||
with mock.patch.object(runner, "run_process", side_effect=fake_run):
|
||||
result = runner.main(
|
||||
self.run_args(root, caselist, outdir, "--max-empty-streak", "2")
|
||||
)
|
||||
|
||||
self.assertEqual(4, result)
|
||||
self.assertEqual(
|
||||
[["KHR-GL32.a", "KHR-GL32.b"], ["KHR-GL32.a", "KHR-GL32.b"]],
|
||||
seen_remaining,
|
||||
)
|
||||
self.assertEqual("", (outdir / "crashed.txt").read_text(encoding="utf-8"))
|
||||
self.assertEqual("", (outdir / "hung.txt").read_text(encoding="utf-8"))
|
||||
self.assertEqual(
|
||||
"KHR-GL32.a\nKHR-GL32.b\n",
|
||||
(outdir / "unrun.txt").read_text(encoding="utf-8"),
|
||||
)
|
||||
|
||||
|
||||
class ProcessTimeoutTests(unittest.TestCase):
|
||||
def test_qpa_activity_prevents_idle_timeout(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
qpa = root / "active.qpa"
|
||||
helper = (
|
||||
"import pathlib,sys,time\n"
|
||||
"path=pathlib.Path(sys.argv[1])\n"
|
||||
"for size in range(1, 9):\n"
|
||||
" path.write_text('x' * size, encoding='utf-8')\n"
|
||||
" time.sleep(0.08)\n"
|
||||
)
|
||||
outcome = runner.run_process(
|
||||
[sys.executable, "-c", helper, str(qpa)],
|
||||
root,
|
||||
dict(runner.os.environ),
|
||||
qpa,
|
||||
root / "stdout.log",
|
||||
root / "stderr.log",
|
||||
idle_timeout=0.2,
|
||||
max_round_seconds=0,
|
||||
poll_seconds=0.03,
|
||||
)
|
||||
self.assertFalse(outcome.timed_out)
|
||||
self.assertEqual(0, outcome.returncode)
|
||||
|
||||
def test_idle_timeout_really_stops_process(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
started = time.monotonic()
|
||||
outcome = runner.run_process(
|
||||
[sys.executable, "-c", "import time; time.sleep(30)"],
|
||||
root,
|
||||
dict(runner.os.environ),
|
||||
root / "never-created.qpa",
|
||||
root / "stdout.log",
|
||||
root / "stderr.log",
|
||||
idle_timeout=0.2,
|
||||
max_round_seconds=0,
|
||||
poll_seconds=0.05,
|
||||
)
|
||||
elapsed = time.monotonic() - started
|
||||
self.assertTrue(outcome.timed_out)
|
||||
self.assertEqual("qpa-idle", outcome.timeout_reason)
|
||||
self.assertLess(elapsed, 10)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,294 @@
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
import wgl_glcts_pipeline as pipeline
|
||||
|
||||
|
||||
def write_fake_pe(path: Path, machine: int = pipeline.PE_MACHINE_AMD64, payload: bytes = b"") -> None:
|
||||
data = bytearray(0x88)
|
||||
data[0:2] = b"MZ"
|
||||
struct.pack_into("<I", data, 0x3C, 0x80)
|
||||
data[0x80:0x84] = b"PE\0\0"
|
||||
struct.pack_into("<H", data, 0x84, machine)
|
||||
path.write_bytes(bytes(data) + payload)
|
||||
|
||||
|
||||
class ArgumentTests(unittest.TestCase):
|
||||
def test_versions_accept_gl_and_dotted_spellings(self):
|
||||
self.assertEqual("30", pipeline.normalize_version("GL30"))
|
||||
self.assertEqual("46", pipeline.normalize_version("4.6"))
|
||||
with self.assertRaises(argparse.ArgumentTypeError):
|
||||
pipeline.normalize_version("4.7")
|
||||
|
||||
def test_environment_assignment_validation(self):
|
||||
self.assertEqual(("MOBILEGL_TEST", "a=b"), pipeline.parse_assignment("MOBILEGL_TEST=a=b"))
|
||||
with self.assertRaises(argparse.ArgumentTypeError):
|
||||
pipeline.parse_assignment("9BAD=value")
|
||||
|
||||
def test_windows_environment_keys_are_canonical_and_last_wins(self):
|
||||
self.assertEqual(
|
||||
{"FOO": "x", "PATH": "second"},
|
||||
pipeline.canonicalize_windows_environment(
|
||||
[("Path", "first"), ("FOO", "x"), ("pAtH", "second")]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class CommandTests(unittest.TestCase):
|
||||
def test_visual_studio_configure_commands_are_x64_and_wgl_default(self):
|
||||
mobilegl = pipeline.mobilegl_configure_command(
|
||||
Path("C:/src/MobileGL"), Path("D:/work/mg"), "Visual Studio 17 2022", "x64", []
|
||||
)
|
||||
self.assertIn("-A", mobilegl)
|
||||
self.assertIn("x64", mobilegl)
|
||||
self.assertIn("-DMOBILEGL_BUILD_TEST=OFF", mobilegl)
|
||||
|
||||
cts = pipeline.cts_configure_command(
|
||||
Path("D:/src/VK-GL-CTS"), Path("D:/work/cts"), "Visual Studio 17 2022", "x64", []
|
||||
)
|
||||
self.assertIn("-DDEQP_TARGET=default", cts)
|
||||
self.assertNotIn("-DDEQP_TARGET=mobilegl", cts)
|
||||
|
||||
def test_runner_command_contains_identity_preserving_wgl_flags(self):
|
||||
command = pipeline.runner_command(
|
||||
Path("runner.py"),
|
||||
Path("runtime/glcts.exe"),
|
||||
Path("cts/modules"),
|
||||
Path("gl46-main.txt"),
|
||||
Path("results/gl46"),
|
||||
"DirectVulkan",
|
||||
300,
|
||||
0,
|
||||
10000,
|
||||
{"MOBILEGL_LOG_FILE_PATH": "result/mobilegl.log"},
|
||||
pipeline.DEFAULT_DEQP_ARGS,
|
||||
"run-fingerprint",
|
||||
)
|
||||
self.assertIn("--backend", command)
|
||||
self.assertIn("DirectVulkan", command)
|
||||
self.assertIn("--deqp-arg=--deqp-gl-context-type=wgl", command)
|
||||
self.assertIn("--deqp-arg=--deqp-surface-type=fbo", command)
|
||||
self.assertIn("--env", command)
|
||||
self.assertIn("MOBILEGL_LOG_FILE_PATH=result/mobilegl.log", command)
|
||||
self.assertIn("--run-identity", command)
|
||||
self.assertIn("run-fingerprint", command)
|
||||
|
||||
def test_report_command_requires_the_pipeline_run_identity(self):
|
||||
command = pipeline.report_command(
|
||||
Path("report.py"),
|
||||
[("DirectVulkan", "gl46", Path("gl46.txt"), Path("results/gl46"))],
|
||||
Path("summary.md"),
|
||||
Path("summary.json"),
|
||||
False,
|
||||
"run-fingerprint",
|
||||
)
|
||||
self.assertIn("--expected-run-identity", command)
|
||||
self.assertIn("run-fingerprint", command)
|
||||
|
||||
|
||||
class RuntimeTests(unittest.TestCase):
|
||||
def test_pe_machine_rejects_non_x64(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
path = Path(temporary) / "x86.dll"
|
||||
write_fake_pe(path, machine=0x14C)
|
||||
self.assertEqual(0x14C, pipeline.pe_machine(path))
|
||||
with self.assertRaises(pipeline.PipelineError):
|
||||
pipeline.require_x64_pe(path, "test DLL")
|
||||
|
||||
def test_runtime_is_hash_keyed_and_copies_only_declared_files(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
glcts = root / "source-glcts.exe"
|
||||
mobilegl = root / "source-MobileGL.dll"
|
||||
write_fake_pe(glcts, payload=b"glcts")
|
||||
write_fake_pe(mobilegl, payload=b"mobilegl")
|
||||
sources = {"glcts.exe": glcts, "opengl32.dll": mobilegl}
|
||||
fingerprint, hashes = pipeline.runtime_fingerprint(sources)
|
||||
|
||||
runtime = pipeline.assemble_runtime(root / "work", sources, fingerprint, hashes)
|
||||
|
||||
self.assertEqual(fingerprint[:16], runtime.name)
|
||||
self.assertEqual(hashes["glcts.exe"], pipeline.sha256_file(runtime / "glcts.exe"))
|
||||
self.assertEqual(hashes["opengl32.dll"], pipeline.sha256_file(runtime / "opengl32.dll"))
|
||||
manifest = json.loads((runtime / "manifest.json").read_text(encoding="utf-8"))
|
||||
self.assertEqual(fingerprint, manifest["fingerprint"])
|
||||
self.assertFalse((runtime / "libEGL.dll").exists())
|
||||
|
||||
def test_run_fingerprint_changes_with_execution_semantics(self):
|
||||
base = pipeline.run_fingerprint(
|
||||
"runtime", "data", {"runner": "tool"}, {"30": "caselist"}, ["--deqp-surface-type=fbo"], {"FLAG": "1"}
|
||||
)
|
||||
self.assertEqual(
|
||||
base,
|
||||
pipeline.run_fingerprint(
|
||||
"runtime", "data", {"runner": "tool"}, {"30": "caselist"}, ["--deqp-surface-type=fbo"], {"FLAG": "1"}
|
||||
),
|
||||
)
|
||||
self.assertNotEqual(
|
||||
base,
|
||||
pipeline.run_fingerprint(
|
||||
"runtime", "data", {"runner": "tool"}, {"30": "caselist"}, ["--deqp-surface-type=window"], {"FLAG": "1"}
|
||||
),
|
||||
)
|
||||
self.assertNotEqual(
|
||||
base,
|
||||
pipeline.run_fingerprint(
|
||||
"runtime", "data", {"runner": "tool"}, {"30": "different"}, ["--deqp-surface-type=fbo"], {"FLAG": "1"}
|
||||
),
|
||||
)
|
||||
self.assertNotEqual(
|
||||
base,
|
||||
pipeline.run_fingerprint(
|
||||
"runtime", "different-data", {"runner": "tool"}, {"30": "caselist"}, ["--deqp-surface-type=fbo"], {"FLAG": "1"}
|
||||
),
|
||||
)
|
||||
self.assertNotEqual(
|
||||
base,
|
||||
pipeline.run_fingerprint(
|
||||
"runtime", "data", {"runner": "different-tool"}, {"30": "caselist"}, ["--deqp-surface-type=fbo"], {"FLAG": "1"}
|
||||
),
|
||||
)
|
||||
|
||||
timeout_baseline = pipeline.run_fingerprint(
|
||||
"runtime",
|
||||
"data",
|
||||
{"runner": "tool"},
|
||||
{"30": "caselist"},
|
||||
["--deqp-surface-type=fbo"],
|
||||
{"FLAG": "1"},
|
||||
{"idle_timeout_seconds": 300.0, "max_round_seconds": 0.0},
|
||||
)
|
||||
idle_changed = pipeline.run_fingerprint(
|
||||
"runtime",
|
||||
"data",
|
||||
{"runner": "tool"},
|
||||
{"30": "caselist"},
|
||||
["--deqp-surface-type=fbo"],
|
||||
{"FLAG": "1"},
|
||||
{"idle_timeout_seconds": 1.0, "max_round_seconds": 0.0},
|
||||
)
|
||||
max_round_changed = pipeline.run_fingerprint(
|
||||
"runtime",
|
||||
"data",
|
||||
{"runner": "tool"},
|
||||
{"30": "caselist"},
|
||||
["--deqp-surface-type=fbo"],
|
||||
{"FLAG": "1"},
|
||||
{"idle_timeout_seconds": 300.0, "max_round_seconds": 60.0},
|
||||
)
|
||||
self.assertNotEqual(timeout_baseline, idle_changed)
|
||||
self.assertNotEqual(timeout_baseline, max_round_changed)
|
||||
|
||||
def test_tracked_environment_is_case_insensitive_and_narrow(self):
|
||||
ambient, effective = pipeline.tracked_run_environment(
|
||||
{
|
||||
"Path": "ambient-path",
|
||||
"mobilegl_debug": "0",
|
||||
"LibGL_Driver": "ambient-libgl",
|
||||
"vK_iCd_fIlEnAmEs": "ambient-icd",
|
||||
"ANGLE_DEFAULT_PLATFORM": "vulkan",
|
||||
"Egl_Test": "1",
|
||||
"D3D_Feature": "1",
|
||||
"DxVk_Config": "ambient-dxvk",
|
||||
"HOME": "ignored",
|
||||
"PATH_EXTRA": "ignored",
|
||||
"MOBILEGL": "ignored",
|
||||
},
|
||||
{"pAtH": "explicit-path", "vk_icd_filenames": "explicit-icd", "CUSTOM": "kept"},
|
||||
)
|
||||
self.assertEqual("ambient-path", ambient["PATH"])
|
||||
self.assertNotIn("HOME", ambient)
|
||||
self.assertNotIn("PATH_EXTRA", ambient)
|
||||
self.assertNotIn("MOBILEGL", ambient)
|
||||
self.assertEqual("explicit-path", effective["PATH"])
|
||||
self.assertEqual("explicit-icd", effective["VK_ICD_FILENAMES"])
|
||||
self.assertEqual("kept", effective["CUSTOM"])
|
||||
|
||||
def test_reserved_environment_names_cannot_hide_behind_case(self):
|
||||
overrides = pipeline.canonicalize_windows_environment(
|
||||
[("mobilegl_backend_type", "DirectGLES")]
|
||||
)
|
||||
self.assertEqual(
|
||||
{"MOBILEGL_BACKEND_TYPE"},
|
||||
pipeline.CONTROLLED_ENVIRONMENT_NAMES & set(overrides),
|
||||
)
|
||||
|
||||
def test_directgles_requires_complete_angle_runtime(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
glcts = root / "glcts.exe"
|
||||
mobilegl = root / "MobileGL.dll"
|
||||
write_fake_pe(glcts)
|
||||
write_fake_pe(mobilegl)
|
||||
with self.assertRaises(pipeline.PipelineError):
|
||||
pipeline.runtime_source_files(glcts, mobilegl, ["DirectGLES"], root / "angle")
|
||||
|
||||
angle = root / "angle"
|
||||
angle.mkdir()
|
||||
for name in pipeline.ANGLE_REQUIRED_DLLS:
|
||||
write_fake_pe(angle / name, payload=name.encode("ascii"))
|
||||
files = pipeline.runtime_source_files(glcts, mobilegl, ["DirectGLES"], angle)
|
||||
self.assertEqual(
|
||||
{"glcts.exe", "opengl32.dll", *pipeline.ANGLE_REQUIRED_DLLS}, set(files)
|
||||
)
|
||||
|
||||
|
||||
class CaselistTests(unittest.TestCase):
|
||||
def test_preflight_prefers_small_buffer_case(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
caselist = Path(temporary) / "gl30-main.txt"
|
||||
caselist.write_text(
|
||||
"KHR-GL30.api.coverage\nKHR-GL30.buffer_objects.gen_buffers\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
self.assertEqual("KHR-GL30.buffer_objects.gen_buffers", pipeline.choose_preflight_case(caselist))
|
||||
|
||||
|
||||
class PreflightTests(unittest.TestCase):
|
||||
def write_identity(self, root: Path, renderer: str, version: str = "4.6"):
|
||||
qpa = root / "chunk0000.qpa"
|
||||
qpa.write_text(
|
||||
'#sessionInfo vendor "MobileGL-Dev"\n'
|
||||
f'#sessionInfo renderer "{renderer}"\n'
|
||||
'#sessionInfo commandLineParameters "--deqp-gl-context-type=wgl --deqp-surface-type=fbo"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
log = root / "mobilegl.log"
|
||||
log.write_text(f"Target OpenGL Version: {version}\n", encoding="utf-8")
|
||||
return qpa, log
|
||||
|
||||
def test_identity_accepts_both_mobilegl_renderers(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
qpa, log = self.write_identity(root, "Magma (MobileGL Core)")
|
||||
identity = pipeline.parse_preflight_identity([qpa], log, "DirectVulkan", (4, 6))
|
||||
self.assertEqual("4.6", identity["target_gl_version"])
|
||||
|
||||
qpa, log = self.write_identity(root, "Espryt (MobileGL Core)")
|
||||
identity = pipeline.parse_preflight_identity([qpa], log, "DirectGLES", (3, 3))
|
||||
self.assertIn("Espryt", identity["renderer"])
|
||||
|
||||
def test_identity_rejects_system_driver_or_low_version(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
qpa, log = self.write_identity(root, "NVIDIA GeForce RTX", version="4.6")
|
||||
qpa.write_text(
|
||||
'#sessionInfo vendor "NVIDIA Corporation"\n'
|
||||
'#sessionInfo renderer "NVIDIA GeForce RTX"\n'
|
||||
'#sessionInfo commandLineParameters "--deqp-gl-context-type=wgl"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
with self.assertRaises(pipeline.PipelineError):
|
||||
pipeline.parse_preflight_identity([qpa], log, "DirectVulkan", (4, 6))
|
||||
|
||||
qpa, log = self.write_identity(root, "Magma (MobileGL Core)", version="4.5")
|
||||
with self.assertRaises(pipeline.PipelineError):
|
||||
pipeline.parse_preflight_identity([qpa], log, "DirectVulkan", (4, 6))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,914 @@
|
||||
#!/usr/bin/env python
|
||||
"""Build MobileGL's Windows WGL shim and run Khronos OpenGL CTS suites.
|
||||
|
||||
The pipeline intentionally keeps the build, runtime, results, and reports in an
|
||||
explicit work root. Each run is keyed by the hashes of glcts.exe, opengl32.dll,
|
||||
and (for DirectGLES) the ANGLE runtime, so resuming can never silently combine
|
||||
results from different binaries.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from typing import Iterable, Mapping, Optional, Sequence
|
||||
|
||||
|
||||
SUPPORTED_VERSIONS = ("30", "31", "32", "33", "40", "41", "42", "43", "44", "45", "46")
|
||||
SUPPORTED_BACKENDS = ("DirectGLES", "DirectVulkan")
|
||||
ANGLE_REQUIRED_DLLS = ("libEGL.dll", "libGLESv2.dll", "d3dcompiler_47.dll")
|
||||
ANGLE_OPTIONAL_DLLS = ("dxcompiler.dll", "dxil.dll")
|
||||
PE_MACHINE_AMD64 = 0x8664
|
||||
TRACKED_ENVIRONMENT_NAMES = frozenset({"PATH"})
|
||||
TRACKED_ENVIRONMENT_PREFIXES = (
|
||||
"MOBILEGL_",
|
||||
"LIBGL_",
|
||||
"VK_",
|
||||
"ANGLE_",
|
||||
"EGL_",
|
||||
"D3D_",
|
||||
"DXVK_",
|
||||
)
|
||||
CONTROLLED_ENVIRONMENT_NAMES = frozenset(
|
||||
{"MOBILEGL_BACKEND_TYPE", "MOBILEGL_LOG_FILE_PATH"}
|
||||
)
|
||||
|
||||
DEFAULT_DEQP_ARGS = (
|
||||
"--deqp-gl-context-type=wgl",
|
||||
"--deqp-surface-type=fbo",
|
||||
"--deqp-gl-config-name=rgba8888d24s8",
|
||||
"--deqp-surface-width=64",
|
||||
"--deqp-surface-height=-1",
|
||||
"--deqp-base-seed=3",
|
||||
"--deqp-visibility=hidden",
|
||||
"--deqp-watchdog=enable",
|
||||
"--deqp-crashhandler=enable",
|
||||
)
|
||||
|
||||
SESSION_VENDOR = re.compile(r'^#sessionInfo vendor "([^"]*)"', re.MULTILINE)
|
||||
SESSION_RENDERER = re.compile(r'^#sessionInfo renderer "([^"]*)"', re.MULTILINE)
|
||||
SESSION_COMMAND_LINE = re.compile(r'^#sessionInfo commandLineParameters "([^"]*)"', re.MULTILINE)
|
||||
TARGET_GL_VERSION = re.compile(r"Target OpenGL Version:\s*(\d+)\.(\d+)")
|
||||
|
||||
|
||||
class PipelineError(RuntimeError):
|
||||
"""A configuration, build, or identity error."""
|
||||
|
||||
|
||||
def repository_root() -> Path:
|
||||
return Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def normalize_version(value: str) -> str:
|
||||
normalized = value.strip().lower().removeprefix("gl").replace(".", "")
|
||||
if normalized not in SUPPORTED_VERSIONS:
|
||||
supported = ", ".join(f"gl{version}" for version in SUPPORTED_VERSIONS)
|
||||
raise argparse.ArgumentTypeError(f"unsupported GL suite {value!r}; choose one of: {supported}")
|
||||
return normalized
|
||||
|
||||
|
||||
def gl_version_tuple(version: str) -> tuple[int, int]:
|
||||
return int(version[0]), int(version[1])
|
||||
|
||||
|
||||
def parse_assignment(value: str) -> tuple[str, str]:
|
||||
name, separator, setting = value.partition("=")
|
||||
if not separator or not name or "\x00" in value:
|
||||
raise argparse.ArgumentTypeError(f"expected NAME=VALUE, got {value!r}")
|
||||
if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name):
|
||||
raise argparse.ArgumentTypeError(f"invalid environment variable name {name!r}")
|
||||
return name, setting
|
||||
|
||||
|
||||
def canonicalize_windows_environment(
|
||||
items: Iterable[tuple[str, str]],
|
||||
) -> dict[str, str]:
|
||||
"""Canonicalize environment keys using Windows' case-insensitive rules."""
|
||||
|
||||
result: dict[str, str] = {}
|
||||
for name, value in items:
|
||||
result[name.upper()] = value
|
||||
return result
|
||||
|
||||
|
||||
def tracked_run_environment(
|
||||
inherited: Mapping[str, str], overrides: Mapping[str, str]
|
||||
) -> tuple[dict[str, str], dict[str, str]]:
|
||||
"""Return tracked ambient values and the effective values used for identity."""
|
||||
|
||||
canonical_inherited = canonicalize_windows_environment(inherited.items())
|
||||
ambient = {
|
||||
name: value
|
||||
for name, value in canonical_inherited.items()
|
||||
if name in TRACKED_ENVIRONMENT_NAMES
|
||||
or name.startswith(TRACKED_ENVIRONMENT_PREFIXES)
|
||||
}
|
||||
effective = dict(ambient)
|
||||
effective.update(canonicalize_windows_environment(overrides.items()))
|
||||
return dict(sorted(ambient.items())), dict(sorted(effective.items()))
|
||||
|
||||
|
||||
def command_text(command: Sequence[object]) -> str:
|
||||
return subprocess.list2cmdline([str(part) for part in command])
|
||||
|
||||
|
||||
def run_command(
|
||||
command: Sequence[object],
|
||||
*,
|
||||
cwd: Optional[Path] = None,
|
||||
env: Optional[Mapping[str, str]] = None,
|
||||
check: bool = True,
|
||||
capture: bool = False,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
rendered = command_text(command)
|
||||
location = f" (cwd={cwd})" if cwd else ""
|
||||
print(f"[wgl_glcts_pipeline] $ {rendered}{location}", flush=True)
|
||||
completed = subprocess.run(
|
||||
[str(part) for part in command],
|
||||
cwd=str(cwd) if cwd else None,
|
||||
env=dict(env) if env else None,
|
||||
text=True,
|
||||
capture_output=capture,
|
||||
check=False,
|
||||
)
|
||||
if check and completed.returncode != 0:
|
||||
detail = ""
|
||||
if capture:
|
||||
detail = f"\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}"
|
||||
raise PipelineError(f"command failed with exit code {completed.returncode}: {rendered}{detail}")
|
||||
return completed
|
||||
|
||||
|
||||
def require_file(path: Path, label: str) -> Path:
|
||||
resolved = path.expanduser().resolve()
|
||||
if not resolved.is_file():
|
||||
raise PipelineError(f"{label} does not exist or is not a file: {resolved}")
|
||||
return resolved
|
||||
|
||||
|
||||
def require_directory(path: Path, label: str) -> Path:
|
||||
resolved = path.expanduser().resolve()
|
||||
if not resolved.is_dir():
|
||||
raise PipelineError(f"{label} does not exist or is not a directory: {resolved}")
|
||||
return resolved
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def sha256_directory(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
files = sorted(
|
||||
(candidate for candidate in path.rglob("*") if candidate.is_file()),
|
||||
key=lambda candidate: candidate.relative_to(path).as_posix(),
|
||||
)
|
||||
if not files:
|
||||
raise PipelineError(f"directory contains no files to fingerprint: {path}")
|
||||
for candidate in files:
|
||||
relative = candidate.relative_to(path).as_posix()
|
||||
digest.update(relative.encode("utf-8"))
|
||||
digest.update(b"\0")
|
||||
digest.update(sha256_file(candidate).encode("ascii"))
|
||||
digest.update(b"\n")
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def pe_machine(path: Path) -> int:
|
||||
with path.open("rb") as stream:
|
||||
if stream.read(2) != b"MZ":
|
||||
raise PipelineError(f"not a PE executable: {path}")
|
||||
stream.seek(0x3C)
|
||||
offset_bytes = stream.read(4)
|
||||
if len(offset_bytes) != 4:
|
||||
raise PipelineError(f"truncated PE header: {path}")
|
||||
pe_offset = struct.unpack("<I", offset_bytes)[0]
|
||||
stream.seek(pe_offset)
|
||||
if stream.read(4) != b"PE\0\0":
|
||||
raise PipelineError(f"invalid PE signature: {path}")
|
||||
machine_bytes = stream.read(2)
|
||||
if len(machine_bytes) != 2:
|
||||
raise PipelineError(f"truncated PE COFF header: {path}")
|
||||
return struct.unpack("<H", machine_bytes)[0]
|
||||
|
||||
|
||||
def require_x64_pe(path: Path, label: str) -> None:
|
||||
machine = pe_machine(path)
|
||||
if machine != PE_MACHINE_AMD64:
|
||||
raise PipelineError(f"{label} must be an x64 PE (machine 0x8664), got 0x{machine:04x}: {path}")
|
||||
|
||||
|
||||
def git_snapshot(path: Path) -> dict[str, object]:
|
||||
snapshot: dict[str, object] = {"path": str(path)}
|
||||
try:
|
||||
head = run_command(
|
||||
["git", "-C", path, "rev-parse", "HEAD"], check=True, capture=True
|
||||
).stdout.strip()
|
||||
status = run_command(
|
||||
["git", "-C", path, "status", "--porcelain"], check=True, capture=True
|
||||
).stdout
|
||||
snapshot.update({"head": head, "dirty": bool(status.strip())})
|
||||
except (OSError, PipelineError):
|
||||
snapshot.update({"head": None, "dirty": None})
|
||||
return snapshot
|
||||
|
||||
|
||||
def generator_arguments(generator: str, architecture: str) -> list[str]:
|
||||
arguments = ["-G", generator]
|
||||
if generator.lower().startswith("visual studio"):
|
||||
arguments.extend(["-A", architecture])
|
||||
return arguments
|
||||
|
||||
|
||||
def mobilegl_configure_command(
|
||||
repo_root: Path,
|
||||
build_dir: Path,
|
||||
generator: str,
|
||||
architecture: str,
|
||||
extra: Iterable[str],
|
||||
) -> list[str]:
|
||||
return [
|
||||
"cmake",
|
||||
"-S",
|
||||
str(repo_root),
|
||||
"-B",
|
||||
str(build_dir),
|
||||
*generator_arguments(generator, architecture),
|
||||
"-DMOBILEGL_BUILD_TEST=OFF",
|
||||
"-DMOBILEGL_BUILD_BENCHMARK=OFF",
|
||||
"-DMOBILEGL_BUILD_TRACE_REPLAY=OFF",
|
||||
"-DMOBILEGL_ENABLE_TRACY=OFF",
|
||||
"-DMOBILEGL_FORCE_RELEASE_OPT=ON",
|
||||
*extra,
|
||||
]
|
||||
|
||||
|
||||
def cts_configure_command(
|
||||
cts_source: Path,
|
||||
build_dir: Path,
|
||||
generator: str,
|
||||
architecture: str,
|
||||
extra: Iterable[str],
|
||||
) -> list[str]:
|
||||
return [
|
||||
"cmake",
|
||||
"-S",
|
||||
str(cts_source),
|
||||
"-B",
|
||||
str(build_dir),
|
||||
*generator_arguments(generator, architecture),
|
||||
"-DDEQP_TARGET=default",
|
||||
"-DDEQP_SUPPORT_DRM=OFF",
|
||||
*extra,
|
||||
]
|
||||
|
||||
|
||||
def build_command(build_dir: Path, configuration: str, target: str, jobs: int) -> list[str]:
|
||||
command = ["cmake", "--build", str(build_dir), "--config", configuration, "--target", target]
|
||||
if jobs > 0:
|
||||
command.extend(["--parallel", str(jobs)])
|
||||
return command
|
||||
|
||||
|
||||
def verify_mobilegl_sources(repo_root: Path) -> None:
|
||||
required = (
|
||||
repo_root / "CMakeLists.txt",
|
||||
repo_root / "MobileGL" / "MG_Impl" / "WGLImpl" / "WGLImpl.cpp",
|
||||
repo_root / "3rdparty" / "glslang" / "CMakeLists.txt",
|
||||
repo_root / "3rdparty" / "SPIRV-Cross" / "CMakeLists.txt",
|
||||
repo_root / "3rdparty" / "Vulkan-Headers" / "CMakeLists.txt",
|
||||
)
|
||||
missing = [str(path) for path in required if not path.is_file()]
|
||||
if missing:
|
||||
raise PipelineError(
|
||||
"MobileGL source/submodules are incomplete:\n "
|
||||
+ "\n ".join(missing)
|
||||
+ f"\nRun: git -C {repo_root} submodule update --init --recursive"
|
||||
)
|
||||
|
||||
|
||||
def verify_cts_sources(cts_source: Path) -> None:
|
||||
required = (
|
||||
cts_source / "CMakeLists.txt",
|
||||
cts_source / "external" / "openglcts" / "CMakeLists.txt",
|
||||
)
|
||||
missing = [str(path) for path in required if not path.is_file()]
|
||||
if missing:
|
||||
raise PipelineError(
|
||||
"VK-GL-CTS source/external packages are incomplete:\n "
|
||||
+ "\n ".join(missing)
|
||||
+ f"\nRun: {sys.executable} {cts_source / 'external' / 'fetch_sources.py'}"
|
||||
)
|
||||
|
||||
|
||||
def discover_mobilegl_dll(build_dir: Path, configuration: str) -> Path:
|
||||
preferred = (
|
||||
build_dir / configuration / "opengl32.dll",
|
||||
build_dir / "MobileGL" / configuration / "opengl32.dll",
|
||||
build_dir / "opengl32.dll",
|
||||
)
|
||||
for candidate in preferred:
|
||||
if candidate.is_file():
|
||||
return candidate.resolve()
|
||||
candidates = sorted({path.resolve() for path in build_dir.rglob("opengl32.dll") if path.is_file()})
|
||||
if len(candidates) == 1:
|
||||
return candidates[0]
|
||||
if not candidates:
|
||||
raise PipelineError(f"MobileGL build produced no opengl32.dll under {build_dir}")
|
||||
raise PipelineError("multiple opengl32.dll candidates; pass --mobilegl-dll explicitly:\n " + "\n ".join(map(str, candidates)))
|
||||
|
||||
|
||||
def discover_glcts_exe(build_dir: Path, configuration: str) -> Path:
|
||||
preferred = (
|
||||
build_dir / "external" / "openglcts" / "modules" / configuration / "glcts.exe",
|
||||
build_dir / "external" / "openglcts" / "modules" / "glcts.exe",
|
||||
)
|
||||
for candidate in preferred:
|
||||
if candidate.is_file():
|
||||
return candidate.resolve()
|
||||
candidates = sorted({path.resolve() for path in build_dir.rglob("glcts.exe") if path.is_file()})
|
||||
if len(candidates) == 1:
|
||||
return candidates[0]
|
||||
if not candidates:
|
||||
raise PipelineError(f"CTS build produced no glcts.exe under {build_dir}")
|
||||
raise PipelineError("multiple glcts.exe candidates; pass --glcts-exe explicitly:\n " + "\n ".join(map(str, candidates)))
|
||||
|
||||
|
||||
def default_cts_modules_dir(cts_build_dir: Path) -> Path:
|
||||
return cts_build_dir / "external" / "openglcts" / "modules"
|
||||
|
||||
|
||||
def find_caselist_root(cts_modules_dir: Path, cts_source: Path) -> Path:
|
||||
relative = Path("gl_cts/data/mustpass/gl/khronos_mustpass/main")
|
||||
candidates = (cts_modules_dir / relative, cts_source / "external" / "openglcts" / "modules" / relative)
|
||||
for candidate in candidates:
|
||||
if candidate.is_dir():
|
||||
return candidate.resolve()
|
||||
raise PipelineError("Khronos GL mustpass directory was not found; checked:\n " + "\n ".join(map(str, candidates)))
|
||||
|
||||
|
||||
def caselist_for(caselist_root: Path, version: str) -> Path:
|
||||
return require_file(caselist_root / f"gl{version}-main.txt", f"GL{version} mustpass caselist")
|
||||
|
||||
|
||||
def runtime_source_files(
|
||||
glcts_exe: Path,
|
||||
mobilegl_dll: Path,
|
||||
backends: Sequence[str],
|
||||
angle_dir: Optional[Path],
|
||||
) -> dict[str, Path]:
|
||||
files = {"glcts.exe": glcts_exe, "opengl32.dll": mobilegl_dll}
|
||||
if "DirectGLES" in backends:
|
||||
if angle_dir is None:
|
||||
raise PipelineError("--angle-dir is required when DirectGLES is selected")
|
||||
angle_dir = require_directory(angle_dir, "ANGLE runtime directory")
|
||||
for name in ANGLE_REQUIRED_DLLS:
|
||||
files[name] = require_file(angle_dir / name, f"ANGLE {name}")
|
||||
for name in ANGLE_OPTIONAL_DLLS:
|
||||
candidate = angle_dir / name
|
||||
if candidate.is_file():
|
||||
files[name] = candidate.resolve()
|
||||
return files
|
||||
|
||||
|
||||
def runtime_fingerprint(files: Mapping[str, Path]) -> tuple[str, dict[str, str]]:
|
||||
hashes = {name: sha256_file(path) for name, path in sorted(files.items())}
|
||||
digest = hashlib.sha256()
|
||||
for name, file_hash in hashes.items():
|
||||
digest.update(f"{name}\0{file_hash}\n".encode("utf-8"))
|
||||
return digest.hexdigest(), hashes
|
||||
|
||||
|
||||
def run_fingerprint(
|
||||
runtime_hash: str,
|
||||
cts_data_hash: str,
|
||||
tool_hashes: Mapping[str, str],
|
||||
caselist_hashes: Mapping[str, str],
|
||||
deqp_args: Sequence[str],
|
||||
environment: Mapping[str, str],
|
||||
result_semantics: Optional[Mapping[str, object]] = None,
|
||||
) -> str:
|
||||
identity = {
|
||||
"version": 2,
|
||||
"runtime_fingerprint": runtime_hash,
|
||||
"cts_data_sha256": cts_data_hash,
|
||||
"tool_hashes": dict(sorted(tool_hashes.items())),
|
||||
"caselist_hashes": dict(sorted(caselist_hashes.items())),
|
||||
"deqp_args": list(deqp_args),
|
||||
"environment": dict(sorted(environment.items())),
|
||||
"result_semantics": dict(sorted((result_semantics or {}).items())),
|
||||
}
|
||||
return hashlib.sha256(
|
||||
json.dumps(identity, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def assemble_runtime(
|
||||
work_root: Path,
|
||||
sources: Mapping[str, Path],
|
||||
fingerprint: str,
|
||||
hashes: Mapping[str, str],
|
||||
) -> Path:
|
||||
runtime_dir = work_root / "runtime" / fingerprint[:16]
|
||||
runtime_dir.mkdir(parents=True, exist_ok=True)
|
||||
for name, source in sources.items():
|
||||
require_x64_pe(source, name)
|
||||
destination = runtime_dir / name
|
||||
if source.resolve() != destination.resolve():
|
||||
shutil.copy2(source, destination)
|
||||
if sha256_file(destination) != hashes[name]:
|
||||
raise PipelineError(f"runtime copy hash mismatch: {destination}")
|
||||
manifest = {
|
||||
"version": 1,
|
||||
"created_utc": utc_now(),
|
||||
"fingerprint": fingerprint,
|
||||
"files": {
|
||||
name: {"source": str(source), "sha256": hashes[name]}
|
||||
for name, source in sorted(sources.items())
|
||||
},
|
||||
}
|
||||
write_json(runtime_dir / "manifest.json", manifest)
|
||||
return runtime_dir
|
||||
|
||||
|
||||
def write_json(path: Path, value: object) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_name(path.name + ".tmp")
|
||||
temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
os.replace(temporary, path)
|
||||
|
||||
|
||||
def read_cases(caselist: Path) -> list[str]:
|
||||
cases: list[str] = []
|
||||
for raw_line in caselist.read_text(encoding="utf-8-sig").splitlines():
|
||||
line = raw_line.strip()
|
||||
if line and not line.startswith("#"):
|
||||
cases.append(line)
|
||||
if not cases:
|
||||
raise PipelineError(f"caselist contains no cases: {caselist}")
|
||||
return cases
|
||||
|
||||
|
||||
def choose_preflight_case(caselist: Path) -> str:
|
||||
cases = read_cases(caselist)
|
||||
preferred_suffixes = (
|
||||
".buffer_objects.gen_buffers",
|
||||
".CommonBugs.CommonBug_GetProgramivActiveUniformBlockMaxNameLength",
|
||||
)
|
||||
for suffix in preferred_suffixes:
|
||||
for case in cases:
|
||||
if case.endswith(suffix):
|
||||
return case
|
||||
return cases[0]
|
||||
|
||||
|
||||
def runner_command(
|
||||
runner: Path,
|
||||
runtime_exe: Path,
|
||||
workdir: Path,
|
||||
caselist: Path,
|
||||
outdir: Path,
|
||||
backend: str,
|
||||
idle_timeout: float,
|
||||
max_round_seconds: float,
|
||||
max_rounds: int,
|
||||
environment: Mapping[str, str],
|
||||
deqp_args: Sequence[str],
|
||||
run_identity: Optional[str] = None,
|
||||
) -> list[str]:
|
||||
command = [
|
||||
sys.executable,
|
||||
str(runner),
|
||||
"--exe",
|
||||
str(runtime_exe),
|
||||
"--workdir",
|
||||
str(workdir),
|
||||
"--caselist",
|
||||
str(caselist),
|
||||
"--outdir",
|
||||
str(outdir),
|
||||
"--backend",
|
||||
backend,
|
||||
"--idle-timeout",
|
||||
str(idle_timeout),
|
||||
"--max-round-seconds",
|
||||
str(max_round_seconds),
|
||||
"--max-rounds",
|
||||
str(max_rounds),
|
||||
]
|
||||
if run_identity is not None:
|
||||
command.extend(["--run-identity", run_identity])
|
||||
for name, value in sorted(environment.items()):
|
||||
command.extend(["--env", f"{name}={value}"])
|
||||
command.extend(f"--deqp-arg={argument}" for argument in deqp_args)
|
||||
return command
|
||||
|
||||
|
||||
def parse_preflight_identity(
|
||||
qpa_files: Sequence[Path],
|
||||
mobilegl_log: Path,
|
||||
backend: str,
|
||||
minimum_version: tuple[int, int],
|
||||
) -> dict[str, object]:
|
||||
if not qpa_files:
|
||||
raise PipelineError(f"{backend} preflight produced no QPA file")
|
||||
qpa_text = "\n".join(path.read_text(encoding="utf-8", errors="replace") for path in qpa_files)
|
||||
vendors = SESSION_VENDOR.findall(qpa_text)
|
||||
renderers = SESSION_RENDERER.findall(qpa_text)
|
||||
command_lines = SESSION_COMMAND_LINE.findall(qpa_text)
|
||||
if not vendors or "MobileGL" not in vendors[-1]:
|
||||
raise PipelineError(f"{backend} preflight did not load MobileGL (vendor={vendors[-1] if vendors else None!r})")
|
||||
expected_renderer = "Espryt" if backend == "DirectGLES" else "Magma"
|
||||
if not renderers or expected_renderer not in renderers[-1]:
|
||||
raise PipelineError(
|
||||
f"{backend} preflight renderer mismatch: expected {expected_renderer!r}, "
|
||||
f"got {renderers[-1] if renderers else None!r}"
|
||||
)
|
||||
if not command_lines or "--deqp-gl-context-type=wgl" not in command_lines[-1]:
|
||||
raise PipelineError(f"{backend} preflight did not record a WGL context")
|
||||
if not mobilegl_log.is_file():
|
||||
raise PipelineError(f"{backend} preflight did not create MobileGL log: {mobilegl_log}")
|
||||
log_text = mobilegl_log.read_text(encoding="utf-8", errors="replace")
|
||||
versions = [(int(major), int(minor)) for major, minor in TARGET_GL_VERSION.findall(log_text)]
|
||||
if not versions:
|
||||
raise PipelineError(f"{backend} preflight log contains no target OpenGL version")
|
||||
actual_version = versions[-1]
|
||||
if actual_version < minimum_version:
|
||||
raise PipelineError(
|
||||
f"{backend} reports GL {actual_version[0]}.{actual_version[1]}, "
|
||||
f"but selected suites require at least {minimum_version[0]}.{minimum_version[1]}"
|
||||
)
|
||||
return {
|
||||
"backend": backend,
|
||||
"vendor": vendors[-1],
|
||||
"renderer": renderers[-1],
|
||||
"target_gl_version": f"{actual_version[0]}.{actual_version[1]}",
|
||||
"qpa_files": [str(path) for path in qpa_files],
|
||||
"mobilegl_log": str(mobilegl_log),
|
||||
}
|
||||
|
||||
|
||||
def run_preflight(
|
||||
*,
|
||||
runner: Path,
|
||||
runtime_exe: Path,
|
||||
cts_modules_dir: Path,
|
||||
caselist: Path,
|
||||
preflight_root: Path,
|
||||
backend: str,
|
||||
idle_timeout: float,
|
||||
environment: Mapping[str, str],
|
||||
deqp_args: Sequence[str],
|
||||
minimum_version: tuple[int, int],
|
||||
run_identity: str,
|
||||
) -> dict[str, object]:
|
||||
outdir = preflight_root / backend.lower()
|
||||
outdir.mkdir(parents=True, exist_ok=True)
|
||||
case_file = outdir / "case.txt"
|
||||
case_file.write_text(choose_preflight_case(caselist) + "\n", encoding="utf-8")
|
||||
log_path = outdir / "mobilegl.log"
|
||||
child_environment = dict(environment)
|
||||
child_environment["MOBILEGL_LOG_FILE_PATH"] = str(log_path)
|
||||
command = runner_command(
|
||||
runner,
|
||||
runtime_exe,
|
||||
cts_modules_dir,
|
||||
case_file,
|
||||
outdir,
|
||||
backend,
|
||||
min(idle_timeout, 120.0) if idle_timeout > 0 else 120.0,
|
||||
180.0,
|
||||
1,
|
||||
child_environment,
|
||||
deqp_args,
|
||||
run_identity,
|
||||
)
|
||||
# A developing driver may fail the chosen case or terminate during deinit.
|
||||
# Identity is the gate: the QPA and MobileGL log must prove which WGL driver ran.
|
||||
run_command(command, cwd=repository_root(), check=False)
|
||||
identity = parse_preflight_identity(sorted(outdir.glob("chunk*.qpa")), log_path, backend, minimum_version)
|
||||
print(
|
||||
f"[wgl_glcts_pipeline] preflight {backend}: {identity['renderer']} | "
|
||||
f"GL {identity['target_gl_version']}"
|
||||
)
|
||||
return identity
|
||||
|
||||
|
||||
def report_command(
|
||||
reporter: Path,
|
||||
suites: Sequence[tuple[str, str, Path, Path]],
|
||||
markdown: Path,
|
||||
json_out: Path,
|
||||
allow_incomplete: bool,
|
||||
expected_run_identity: Optional[str] = None,
|
||||
) -> list[str]:
|
||||
command = [sys.executable, str(reporter)]
|
||||
for backend, label, caselist, results in suites:
|
||||
command.extend(["--suite", backend, label, str(caselist), str(results)])
|
||||
command.extend(["--markdown", str(markdown), "--json", str(json_out)])
|
||||
if expected_run_identity is not None:
|
||||
command.extend(["--expected-run-identity", expected_run_identity])
|
||||
if allow_incomplete:
|
||||
command.append("--allow-incomplete")
|
||||
return command
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Build MobileGL WGL, assemble a local glcts runtime, run GL30-GL46, and report results."
|
||||
)
|
||||
parser.add_argument("--repo-root", type=Path, default=repository_root(), help="MobileGL worktree root")
|
||||
parser.add_argument("--cts-source", type=Path, required=True, help="VK-GL-CTS source checkout")
|
||||
parser.add_argument("--work-root", type=Path, required=True, help="build/result root (kept outside source)")
|
||||
parser.add_argument("--angle-dir", type=Path, help="x64 ANGLE directory for DirectGLES")
|
||||
parser.add_argument("--backends", nargs="+", choices=SUPPORTED_BACKENDS, default=list(SUPPORTED_BACKENDS))
|
||||
parser.add_argument("--versions", nargs="+", type=normalize_version, default=list(SUPPORTED_VERSIONS))
|
||||
parser.add_argument("--configuration", default="Release")
|
||||
parser.add_argument("--generator", default="Visual Studio 17 2022")
|
||||
parser.add_argument("--architecture", default="x64")
|
||||
parser.add_argument("--jobs", type=int, default=max(1, os.cpu_count() or 1))
|
||||
parser.add_argument("--mobilegl-build-dir", type=Path)
|
||||
parser.add_argument("--cts-build-dir", type=Path)
|
||||
parser.add_argument("--mobilegl-dll", type=Path, help="reuse an existing MobileGL/opengl32 DLL")
|
||||
parser.add_argument("--glcts-exe", type=Path, help="reuse an existing glcts.exe")
|
||||
parser.add_argument("--cts-modules-dir", type=Path, help="glcts working directory containing gl_cts data")
|
||||
parser.add_argument("--skip-mobilegl-build", action="store_true")
|
||||
parser.add_argument("--skip-cts-build", action="store_true")
|
||||
parser.add_argument("--skip-preflight", action="store_true")
|
||||
parser.add_argument("--skip-run", action="store_true")
|
||||
parser.add_argument("--skip-report", action="store_true")
|
||||
parser.add_argument("--allow-incomplete-report", action="store_true")
|
||||
parser.add_argument("--continue-on-suite-error", action="store_true")
|
||||
parser.add_argument("--idle-timeout", type=float, default=300.0)
|
||||
parser.add_argument("--max-round-seconds", type=float, default=0.0)
|
||||
parser.add_argument("--max-rounds", type=int, default=10000)
|
||||
parser.add_argument("--env", action="append", type=parse_assignment, default=[], metavar="NAME=VALUE")
|
||||
parser.add_argument(
|
||||
"--deqp-arg", action="append", default=[], metavar="ARG", help="extra glcts option; use --deqp-arg=--x=y"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mobilegl-cmake-arg", action="append", default=[], metavar="ARG", help="extra MobileGL configure option"
|
||||
)
|
||||
parser.add_argument("--cts-cmake-arg", action="append", default=[], metavar="ARG", help="extra CTS configure option")
|
||||
return parser
|
||||
|
||||
|
||||
def execute(args: argparse.Namespace) -> int:
|
||||
if os.name != "nt":
|
||||
raise PipelineError("this pipeline builds and exercises the Windows WGL target and must run on Windows")
|
||||
if shutil.which("cmake") is None and (not args.skip_mobilegl_build or not args.skip_cts_build):
|
||||
raise PipelineError("cmake was not found on PATH")
|
||||
if args.jobs < 0 or args.max_rounds <= 0:
|
||||
raise PipelineError("--jobs must be >= 0 and --max-rounds must be > 0")
|
||||
|
||||
repo_root = require_directory(args.repo_root, "MobileGL worktree")
|
||||
cts_source = require_directory(args.cts_source, "VK-GL-CTS checkout")
|
||||
work_root = args.work_root.expanduser().resolve()
|
||||
work_root.mkdir(parents=True, exist_ok=True)
|
||||
versions = list(dict.fromkeys(args.versions))
|
||||
backends = list(dict.fromkeys(args.backends))
|
||||
minimum_version = max(gl_version_tuple(version) for version in versions)
|
||||
extra_environment = canonicalize_windows_environment(args.env)
|
||||
reserved_environment = CONTROLLED_ENVIRONMENT_NAMES & set(extra_environment)
|
||||
if reserved_environment:
|
||||
raise PipelineError("the pipeline controls these environment variables: " + ", ".join(sorted(reserved_environment)))
|
||||
|
||||
configuration = args.configuration
|
||||
mobilegl_build_dir = (args.mobilegl_build_dir or work_root / f"mobilegl-build-{configuration.lower()}").resolve()
|
||||
cts_build_dir = (args.cts_build_dir or work_root / f"cts-build-wgl-{configuration.lower()}").resolve()
|
||||
|
||||
if not args.skip_mobilegl_build:
|
||||
verify_mobilegl_sources(repo_root)
|
||||
mobilegl_build_dir.mkdir(parents=True, exist_ok=True)
|
||||
run_command(
|
||||
mobilegl_configure_command(
|
||||
repo_root, mobilegl_build_dir, args.generator, args.architecture, args.mobilegl_cmake_arg
|
||||
),
|
||||
cwd=repo_root,
|
||||
)
|
||||
run_command(build_command(mobilegl_build_dir, configuration, "MobileGL", args.jobs), cwd=repo_root)
|
||||
mobilegl_dll = (
|
||||
require_file(args.mobilegl_dll, "MobileGL DLL")
|
||||
if args.mobilegl_dll
|
||||
else discover_mobilegl_dll(mobilegl_build_dir, configuration)
|
||||
)
|
||||
|
||||
if not args.skip_cts_build:
|
||||
verify_cts_sources(cts_source)
|
||||
cts_build_dir.mkdir(parents=True, exist_ok=True)
|
||||
run_command(
|
||||
cts_configure_command(cts_source, cts_build_dir, args.generator, args.architecture, args.cts_cmake_arg),
|
||||
cwd=cts_source,
|
||||
)
|
||||
run_command(build_command(cts_build_dir, configuration, "glcts", args.jobs), cwd=cts_source)
|
||||
glcts_exe = (
|
||||
require_file(args.glcts_exe, "glcts executable")
|
||||
if args.glcts_exe
|
||||
else discover_glcts_exe(cts_build_dir, configuration)
|
||||
)
|
||||
cts_modules_dir = require_directory(
|
||||
args.cts_modules_dir or default_cts_modules_dir(cts_build_dir), "CTS modules/working directory"
|
||||
)
|
||||
cts_data_dir = require_directory(cts_modules_dir / "gl_cts" / "data", "CTS gl_cts data directory")
|
||||
cts_data_hash = sha256_directory(cts_data_dir)
|
||||
caselist_root = find_caselist_root(cts_modules_dir, cts_source)
|
||||
caselists = {version: caselist_for(caselist_root, version) for version in versions}
|
||||
|
||||
runtime_sources = runtime_source_files(glcts_exe, mobilegl_dll, backends, args.angle_dir)
|
||||
fingerprint, hashes = runtime_fingerprint(runtime_sources)
|
||||
runtime_dir = assemble_runtime(work_root, runtime_sources, fingerprint, hashes)
|
||||
runtime_exe = runtime_dir / "glcts.exe"
|
||||
runner = require_file(repo_root / "tools" / "cts" / "scripts" / "run_cts_windows.py", "Windows CTS runner")
|
||||
reporter = require_file(repo_root / "tools" / "cts" / "scripts" / "cts_multi_report.py", "CTS reporter")
|
||||
matrix_reporter = require_file(
|
||||
repo_root / "tools" / "cts" / "scripts" / "cts_matrix_report.py", "CTS matrix reporter"
|
||||
)
|
||||
qpa_reporter = require_file(repo_root / "tools" / "cts" / "scripts" / "qpa_report.py", "QPA parser")
|
||||
tool_hashes = {
|
||||
"wgl_glcts_pipeline.py": sha256_file(Path(__file__).resolve()),
|
||||
"run_cts_windows.py": sha256_file(runner),
|
||||
"cts_multi_report.py": sha256_file(reporter),
|
||||
"cts_matrix_report.py": sha256_file(matrix_reporter),
|
||||
"qpa_report.py": sha256_file(qpa_reporter),
|
||||
}
|
||||
deqp_args = [*DEFAULT_DEQP_ARGS, *args.deqp_arg]
|
||||
caselist_hashes = {version: sha256_file(path) for version, path in caselists.items()}
|
||||
ambient_environment, identity_environment = tracked_run_environment(
|
||||
os.environ, extra_environment
|
||||
)
|
||||
result_semantics = {
|
||||
"idle_timeout_seconds": args.idle_timeout,
|
||||
"max_round_seconds": args.max_round_seconds,
|
||||
}
|
||||
execution_settings = {
|
||||
**result_semantics,
|
||||
"max_rounds": args.max_rounds,
|
||||
"continue_on_suite_error": args.continue_on_suite_error,
|
||||
}
|
||||
execution_fingerprint = run_fingerprint(
|
||||
fingerprint,
|
||||
cts_data_hash,
|
||||
tool_hashes,
|
||||
caselist_hashes,
|
||||
deqp_args,
|
||||
identity_environment,
|
||||
result_semantics,
|
||||
)
|
||||
run_root = work_root / "runs" / execution_fingerprint[:16]
|
||||
report_root = run_root / "reports"
|
||||
|
||||
manifest = {
|
||||
"version": 1,
|
||||
"created_utc": utc_now(),
|
||||
"run_fingerprint": execution_fingerprint,
|
||||
"runtime_fingerprint": fingerprint,
|
||||
"runtime_hashes": hashes,
|
||||
"tool_hashes": tool_hashes,
|
||||
"cts_data": {"path": str(cts_data_dir), "sha256": cts_data_hash},
|
||||
"runtime_dir": str(runtime_dir),
|
||||
"mobilegl": git_snapshot(repo_root),
|
||||
"vk_gl_cts": git_snapshot(cts_source),
|
||||
"configuration": configuration,
|
||||
"generator": args.generator,
|
||||
"architecture": args.architecture,
|
||||
"backends": backends,
|
||||
"versions": versions,
|
||||
"caselists": {version: {"path": str(path), "sha256": caselist_hashes[version]} for version, path in caselists.items()},
|
||||
"deqp_args": deqp_args,
|
||||
"environment_overrides": extra_environment,
|
||||
"ambient_environment": ambient_environment,
|
||||
"identity_environment": identity_environment,
|
||||
"result_semantics": result_semantics,
|
||||
"execution_settings": execution_settings,
|
||||
}
|
||||
write_json(run_root / "manifest.json", manifest)
|
||||
print(f"[wgl_glcts_pipeline] runtime fingerprint: {fingerprint}")
|
||||
print(f"[wgl_glcts_pipeline] run fingerprint: {execution_fingerprint}")
|
||||
print(f"[wgl_glcts_pipeline] run root: {run_root}")
|
||||
|
||||
identities: list[dict[str, object]] = []
|
||||
if not args.skip_preflight:
|
||||
first_caselist = caselists[versions[0]]
|
||||
preflight_root = run_root / "preflight"
|
||||
for backend in backends:
|
||||
identities.append(
|
||||
run_preflight(
|
||||
runner=runner,
|
||||
runtime_exe=runtime_exe,
|
||||
cts_modules_dir=cts_modules_dir,
|
||||
caselist=first_caselist,
|
||||
preflight_root=preflight_root,
|
||||
backend=backend,
|
||||
idle_timeout=args.idle_timeout,
|
||||
environment=extra_environment,
|
||||
deqp_args=deqp_args,
|
||||
minimum_version=minimum_version,
|
||||
run_identity=execution_fingerprint,
|
||||
)
|
||||
)
|
||||
manifest["preflight"] = identities
|
||||
write_json(run_root / "manifest.json", manifest)
|
||||
|
||||
suites: list[tuple[str, str, Path, Path]] = []
|
||||
suite_errors: list[dict[str, object]] = []
|
||||
for backend in backends:
|
||||
for version in versions:
|
||||
label = f"gl{version}"
|
||||
result_dir = run_root / "results" / backend.lower() / label
|
||||
suites.append((backend, label, caselists[version], result_dir))
|
||||
if args.skip_run:
|
||||
continue
|
||||
result_dir.mkdir(parents=True, exist_ok=True)
|
||||
child_environment = dict(extra_environment)
|
||||
child_environment["MOBILEGL_LOG_FILE_PATH"] = str(result_dir / "mobilegl.log")
|
||||
command = runner_command(
|
||||
runner,
|
||||
runtime_exe,
|
||||
cts_modules_dir,
|
||||
caselists[version],
|
||||
result_dir,
|
||||
backend,
|
||||
args.idle_timeout,
|
||||
args.max_round_seconds,
|
||||
args.max_rounds,
|
||||
child_environment,
|
||||
deqp_args,
|
||||
execution_fingerprint,
|
||||
)
|
||||
completed = run_command(command, cwd=repo_root, check=False)
|
||||
if completed.returncode != 0:
|
||||
suite_errors.append({"backend": backend, "suite": label, "returncode": completed.returncode})
|
||||
if not args.continue_on_suite_error:
|
||||
break
|
||||
if suite_errors and not args.continue_on_suite_error:
|
||||
break
|
||||
|
||||
report_returncode: Optional[int] = None
|
||||
if not args.skip_report:
|
||||
report_root.mkdir(parents=True, exist_ok=True)
|
||||
completed = run_command(
|
||||
report_command(
|
||||
reporter,
|
||||
suites,
|
||||
report_root / "gl-cts-summary.md",
|
||||
report_root / "gl-cts-summary.json",
|
||||
args.allow_incomplete_report or bool(suite_errors),
|
||||
execution_fingerprint,
|
||||
),
|
||||
cwd=repo_root,
|
||||
check=False,
|
||||
)
|
||||
report_returncode = completed.returncode
|
||||
|
||||
manifest["suite_errors"] = suite_errors
|
||||
manifest["report_returncode"] = report_returncode
|
||||
manifest["finished_utc"] = utc_now()
|
||||
write_json(run_root / "manifest.json", manifest)
|
||||
if suite_errors:
|
||||
print(f"[wgl_glcts_pipeline] {len(suite_errors)} suite runner(s) incomplete; see manifest/report", file=sys.stderr)
|
||||
returncodes = {int(item["returncode"]) for item in suite_errors}
|
||||
if 130 in returncodes:
|
||||
return 130
|
||||
if 2 in returncodes:
|
||||
return 2
|
||||
if 3 in returncodes:
|
||||
return 3
|
||||
return 4
|
||||
if report_returncode is not None and report_returncode != 0:
|
||||
print(f"[wgl_glcts_pipeline] report validation failed with exit code {report_returncode}", file=sys.stderr)
|
||||
return report_returncode
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: Optional[Sequence[str]] = None) -> int:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
return execute(args)
|
||||
except PipelineError as exc:
|
||||
print(f"[wgl_glcts_pipeline] ERROR: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
except OSError as exc:
|
||||
print(f"[wgl_glcts_pipeline] ERROR: filesystem/process operation failed: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user