mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-09-07 19:58:32 +09:00
[Feat] (tools/cts): add crash-resuming local-host glcts runner
Local counterpart of run_cts.py for desktop Linux runs: re-invokes glcts with the not-yet-measured cases after a crash, quarantines timed-out cases with the dEQP watchdog enabled, and records crashed/hung/unrun lists so a partial run cannot read as a complete one.
This commit is contained in:
BZLZHH
@@ -0,0 +1,185 @@
|
||||
#!/usr/bin/env python
|
||||
"""Drive a glcts run on the local host, resuming across crashes.
|
||||
|
||||
Local-host counterpart of run_cts.py: MobileGL crashes on some cases and glcts
|
||||
takes the whole process down with it, so a single invocation stops at the first
|
||||
crash. This runner re-invokes glcts with only the cases that have no result
|
||||
yet, records the case that was open when the process died as "Crash" (or
|
||||
"Hang" on a timeout), and repeats until the list is exhausted.
|
||||
|
||||
Usage:
|
||||
python run_cts_local.py --backend DirectVulkan \\
|
||||
--glcts <path-to-glcts-binary> --lib <path-to-libMobileGL.so> \\
|
||||
--caselist <mustpass.txt> --outdir <dir> [--env K=V ...]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
CASE_START = re.compile(r"^#beginTestCaseResult\s+(\S+)")
|
||||
CASE_END = re.compile(r"^#endTestCaseResult")
|
||||
CASE_TERM = re.compile(r"^#terminateTestCaseResult")
|
||||
|
||||
|
||||
def completed_cases(qpa_path):
|
||||
"""Return (finished_case_names, last_started_case_or_None)."""
|
||||
finished = []
|
||||
current = None
|
||||
if not os.path.exists(qpa_path):
|
||||
return finished, None
|
||||
with open(qpa_path, "r", encoding="utf-8", errors="replace") as fh:
|
||||
for line in fh:
|
||||
m = CASE_START.match(line)
|
||||
if m:
|
||||
current = m.group(1)
|
||||
continue
|
||||
if current is not None and (CASE_END.match(line) or CASE_TERM.match(line)):
|
||||
finished.append(current)
|
||||
current = None
|
||||
return finished, current
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--backend", required=True, choices=["DirectGLES", "DirectVulkan"])
|
||||
ap.add_argument("--glcts", required=True, help="path to the glcts binary")
|
||||
ap.add_argument("--lib", required=True, help="path to libMobileGL.so")
|
||||
ap.add_argument("--caselist", required=True)
|
||||
ap.add_argument("--outdir", required=True)
|
||||
ap.add_argument("--surface", default="fbo", help="--deqp-surface-type value")
|
||||
ap.add_argument("--max-rounds", type=int, default=4000)
|
||||
ap.add_argument("--max-empty-streak", type=int, default=64,
|
||||
help="abort after this many consecutive chunks that produce no log at all")
|
||||
ap.add_argument("--chunk-timeout", type=int, default=1800,
|
||||
help="seconds before killing one glcts invocation (a wedged case never returns)")
|
||||
ap.add_argument("--skip-file", default=None,
|
||||
help="file of case names to exclude, e.g. cases known to wedge the host")
|
||||
ap.add_argument("--env", action="append", default=[], metavar="K=V",
|
||||
help="extra environment variable for glcts (repeatable)")
|
||||
args = ap.parse_args()
|
||||
|
||||
os.makedirs(args.outdir, exist_ok=True)
|
||||
glcts = os.path.abspath(args.glcts)
|
||||
lib = os.path.abspath(args.lib)
|
||||
# glcts resolves its gl_cts data tree relative to the binary's directory.
|
||||
workdir = os.path.dirname(glcts)
|
||||
|
||||
with open(args.caselist, "r", encoding="utf-8") as fh:
|
||||
remaining = [l.strip() for l in fh if l.strip() and not l.strip().startswith("#")]
|
||||
|
||||
skipped = []
|
||||
if args.skip_file and os.path.isfile(args.skip_file):
|
||||
with open(args.skip_file, "r", encoding="utf-8") as fh:
|
||||
skip = {l.strip() for l in fh if l.strip() and not l.strip().startswith("#")}
|
||||
skipped = [c for c in remaining if c in skip]
|
||||
remaining = [c for c in remaining if c not in skip]
|
||||
print(f"[run_cts_local] skipping {len(skipped)} case(s) from {args.skip_file}")
|
||||
|
||||
total = len(remaining)
|
||||
print(f"[run_cts_local] {args.backend}: {total} cases")
|
||||
|
||||
env = dict(os.environ)
|
||||
env["MOBILEGL_BACKEND_TYPE"] = args.backend
|
||||
env["MOBILEGL_CTS_LIB"] = lib
|
||||
for kv in args.env:
|
||||
k, _, v = kv.partition("=")
|
||||
env[k] = v
|
||||
|
||||
crashed = []
|
||||
hung = []
|
||||
done = set()
|
||||
chunk = 0
|
||||
started = time.time()
|
||||
empty_streak = 0
|
||||
|
||||
while remaining and chunk < args.max_rounds:
|
||||
listfile = os.path.abspath(os.path.join(args.outdir, "remaining.txt"))
|
||||
with open(listfile, "w", encoding="utf-8", newline="\n") as fh:
|
||||
fh.write("\n".join(remaining) + "\n")
|
||||
|
||||
qpa = os.path.abspath(os.path.join(args.outdir, f"chunk{chunk:04d}.qpa"))
|
||||
cmd = [
|
||||
glcts,
|
||||
f"--deqp-caselist-file={listfile}",
|
||||
f"--deqp-surface-type={args.surface}",
|
||||
"--deqp-terminate-on-device-lost=disable",
|
||||
# A wedged case aborts the process instead of stalling the chunk;
|
||||
# the runner then records it as Crash and resumes past it.
|
||||
"--deqp-watchdog=enable",
|
||||
"--deqp-log-images=disable",
|
||||
"--deqp-log-shader-sources=disable",
|
||||
f"--deqp-log-filename={qpa}",
|
||||
]
|
||||
timed_out = False
|
||||
try:
|
||||
subprocess.run(cmd, cwd=workdir, env=env, timeout=args.chunk_timeout,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
start_new_session=True)
|
||||
except subprocess.TimeoutExpired:
|
||||
timed_out = True
|
||||
print(f"[run_cts_local] chunk {chunk:04d} timed out after {args.chunk_timeout}s",
|
||||
file=sys.stderr)
|
||||
|
||||
finished, in_flight = completed_cases(qpa)
|
||||
for c in finished:
|
||||
done.add(c)
|
||||
|
||||
progressed = len(finished)
|
||||
if progressed > 0:
|
||||
empty_streak = 0
|
||||
if in_flight is not None:
|
||||
if timed_out:
|
||||
print(f"[run_cts_local] HANG in {in_flight} - quarantining it")
|
||||
hung.append(in_flight)
|
||||
else:
|
||||
crashed.append(in_flight)
|
||||
done.add(in_flight)
|
||||
progressed += 1
|
||||
elif progressed == 0:
|
||||
empty_streak += 1
|
||||
if empty_streak >= args.max_empty_streak:
|
||||
print(f"[run_cts_local] ABORTING: {empty_streak} consecutive chunks produced no "
|
||||
f"output. Something systemic is wrong; refusing to label the rest of the "
|
||||
f"suite as crashes.", file=sys.stderr)
|
||||
break
|
||||
victim = remaining[0]
|
||||
label = "Hang" if timed_out else "Crash"
|
||||
print(f"[run_cts_local] no output at all; recording {victim} as {label}")
|
||||
(hung if timed_out else crashed).append(victim)
|
||||
done.add(victim)
|
||||
progressed = 1
|
||||
|
||||
remaining = [c for c in remaining if c not in done]
|
||||
elapsed = time.time() - started
|
||||
print(
|
||||
f"[run_cts_local] chunk {chunk:04d}: +{progressed} (done {len(done)}/{total}, "
|
||||
f"crashes {len(crashed)}, hangs {len(hung)}, {elapsed / 60:.1f} min)"
|
||||
)
|
||||
chunk += 1
|
||||
|
||||
with open(os.path.join(args.outdir, "crashed.txt"), "w", encoding="utf-8", newline="\n") as fh:
|
||||
fh.write("\n".join(crashed) + ("\n" if crashed else ""))
|
||||
with open(os.path.join(args.outdir, "hung.txt"), "w", encoding="utf-8", newline="\n") as fh:
|
||||
fh.write("\n".join(hung) + ("\n" if hung else ""))
|
||||
with open(os.path.join(args.outdir, "unrun.txt"), "w", encoding="utf-8", newline="\n") as fh:
|
||||
fh.write("\n".join(remaining) + ("\n" if remaining else ""))
|
||||
if skipped:
|
||||
with open(os.path.join(args.outdir, "skipped.txt"), "w", encoding="utf-8", newline="\n") as fh:
|
||||
fh.write("\n".join(skipped) + "\n")
|
||||
|
||||
if remaining:
|
||||
print(f"[run_cts_local] WARNING: {len(remaining)} cases were never run (see unrun.txt)",
|
||||
file=sys.stderr)
|
||||
print(f"[run_cts_local] finished: {len(done)}/{total} cases, {len(crashed)} crashes, "
|
||||
f"{len(hung)} hangs, {chunk} invocations")
|
||||
print(f"[run_cts_local] qpa chunks in {args.outdir}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user