Files
MobileGL/tools/trace_replay/run_macos_window_retrace_local.py
T

546 lines
22 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import concurrent.futures
import json
import os
import signal
import shutil
import subprocess
import sys
import time
from pathlib import Path
from trace_cases import case_with_defaults, ci_trace_cases, load_trace_cases
ROOT = Path(__file__).resolve().parents[2]
FIXTURES = ROOT / "tools" / "trace_replay" / "fixtures"
RESULT_ROOT = ROOT / ".trace-work" / "macos-window-retrace-result"
WORK_ROOT = ROOT / ".trace-work" / "macos-window-retrace-work"
SUMMARY_DIR = ROOT / ".trace-work" / "macos-window-retrace-summary"
SUMMARY_HTML = "mobilegl-macos-window-vulkan-retrace-overview.html"
# Fixture mirrors, tried in order before Git LFS.
DEFAULT_MIRROR_BASES = [
"https://git.hit.moe/swung0x48/MobileGL/media/branch/dev/tools/trace_replay/fixtures",
"https://repo.miawa.cn/mgl/tools/trace_replay/fixtures",
]
DEFAULT_MIRROR_BASE = DEFAULT_MIRROR_BASES[0]
BACKENDS = ("DirectVulkan",)
CASES = load_trace_cases()
def host_machine():
try:
result = subprocess.run(["/usr/sbin/sysctl", "-n", "hw.optional.arm64"], text=True, capture_output=True, check=False)
if result.stdout.strip() == "1":
return "arm64"
except OSError:
pass
try:
result = subprocess.run(["/usr/bin/uname", "-m"], text=True, capture_output=True, check=False)
if result.stdout.strip():
return result.stdout.strip()
except OSError:
pass
return os.uname().machine
DEFAULT_BUILD_DIR = ROOT / (
"cmake-build-macos-trace-arm64" if host_machine() == "arm64" else "cmake-build-macos-trace"
)
def safe_case(name):
return "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in name)
def is_lfs_pointer(path):
return path.exists() and path.read_bytes()[:80].startswith(b"version https://git-lfs.github.com/spec/v1")
def is_bad_fixture(path):
return not path.exists() or path.stat().st_size <= 1024 or is_lfs_pointer(path)
def fixture_paths(case):
paths = [FIXTURES / case["trace_archive"], FIXTURES / case["golden"]]
if case.get("alternate_golden"):
paths.append(FIXTURES / case["alternate_golden"])
return paths
def file_arch(path):
if not path.exists():
return ""
try:
result = subprocess.run(["file", str(path)], text=True, capture_output=True, check=False)
except OSError:
return ""
text = result.stdout
if "x86_64" in text and "arm64" not in text:
return "x86_64"
if "arm64" in text and "x86_64" not in text:
return "arm64"
if "universal" in text or ("x86_64" in text and "arm64" in text):
return "universal"
return ""
def default_vulkan_icd(mobilegl_library):
env_value = os.environ.get("VK_ICD_FILENAMES")
if env_value:
return env_value
candidates = [
Path("/usr/local/share/vulkan/icd.d/MoltenVK_icd.json"),
Path("/opt/homebrew/etc/vulkan/icd.d/MoltenVK_icd.json"),
Path("/opt/homebrew/Cellar/molten-vk/1.4.1/etc/vulkan/icd.d/MoltenVK_icd.json"),
]
arch = file_arch(mobilegl_library)
if arch == "arm64":
candidates = candidates[1:] + candidates[:1]
for candidate in candidates:
if candidate.exists():
return str(candidate)
return ""
def download_fixture(path, mirror_bases):
"""Try each mirror in order; return on the first that serves a good file."""
if isinstance(mirror_bases, str):
mirror_bases = [mirror_bases]
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(path.suffix + ".tmp")
token = os.environ.get("MOBILEGL_TRACE_FIXTURE_MIRROR_TOKEN", "")
errors = []
for mirror_base in mirror_bases:
url = f"{mirror_base.rstrip('/')}/{path.name}"
command = ["curl", "-L", "--fail", "--retry", "3", "--retry-delay", "2",
"--continue-at", "-"]
if token:
command += ["--header", f"Authorization: token {token}"]
command += ["-o", str(tmp), url]
result = subprocess.run(command, text=True, capture_output=True)
if result.returncode != 0:
errors.append(f"{mirror_base}: {result.stderr.strip() or result.stdout.strip()}")
tmp.unlink(missing_ok=True)
continue
tmp.replace(path)
if is_bad_fixture(path):
errors.append(f"{mirror_base}: downloaded fixture is empty or still an LFS pointer")
continue
return path, True, ""
return path, False, "; ".join(errors)
def hydrate_fixtures(cases, fetch, mirror_bases, download_jobs):
required = []
seen = set()
for case in cases:
for path in fixture_paths(case):
if path in seen:
continue
seen.add(path)
if is_bad_fixture(path):
required.append(path)
if not required:
print("fixtures: all required files are present", flush=True)
return True
if not fetch:
for path in required:
print(f"missing fixture: {path}", file=sys.stderr)
return False
print(f"fixtures: downloading {len(required)} file(s) with {download_jobs} worker(s)", flush=True)
failures = []
with concurrent.futures.ThreadPoolExecutor(max_workers=download_jobs) as executor:
futures = [executor.submit(download_fixture, path, mirror_bases) for path in required]
for future in concurrent.futures.as_completed(futures):
path, ok, message = future.result()
if ok:
print(f" fetched {path.name}", flush=True)
else:
print(f" mirror fetch failed for {path.name}: {message}", file=sys.stderr, flush=True)
failures.append(path)
remaining = [path for path in required if is_bad_fixture(path)]
if remaining:
include = ",".join(str(path.relative_to(ROOT)) for path in remaining)
print(f"fixtures: falling back to git lfs for {include}", flush=True)
subprocess.run(["git", "lfs", "install", "--local"], cwd=ROOT, check=False)
subprocess.run(["git", "lfs", "pull", f"--include={include}", "--exclude="], cwd=ROOT, check=False)
remaining = [path for path in required if is_bad_fixture(path)]
if remaining:
for path in remaining:
print(f"failed to hydrate fixture: {path}", file=sys.stderr)
return False
return True
def copy_goldens(case, backend, result_dir):
for key, suffix in (("golden", "golden"), ("alternate_golden", "alternate-golden")):
value = case.get(key)
if not value:
continue
source = FIXTURES / value
if source.exists() and not is_lfs_pointer(source):
shutil.copyfile(source, result_dir / f"{safe_case(case['name'])}-{backend}-{suffix}.png")
def synthesize_result(case, backend, result_dir, status_code, message, elapsed_seconds):
result = {
"passed": False,
"statusCode": status_code,
"message": message,
"tracePath": str(WORK_ROOT / safe_case(case["name"]) / "input" / case["trace_file"]),
"goldenPath": str(FIXTURES / case["golden"]),
"alternateGoldenPaths": [
str(FIXTURES / case["alternate_golden"])
] if case.get("alternate_golden") else [],
"matchedGoldenPath": "",
"actualPath": str(result_dir / "actual.png"),
"diffPath": str(result_dir / f"{safe_case(case['name'])}-{backend}-diff.png"),
"backend": backend,
"targetCall": case["target_call"],
"width": case["width"],
"height": case["height"],
"cropX": case["crop_x"],
"cropY": case["crop_y"],
"cropWidth": case["crop_width"],
"cropHeight": case["crop_height"],
"ssim": -1,
"ssimThreshold": float(case["ssim_threshold"]),
"mismatchPixels": -1,
"elapsedSeconds": elapsed_seconds,
}
(result_dir / "result.json").write_text(json.dumps(result, indent=2), encoding="utf-8")
def recent_log_text(paths, max_bytes=16000):
chunks = []
for path in paths:
if not path.exists():
continue
try:
with path.open("rb") as file:
file.seek(0, os.SEEK_END)
size = file.tell()
file.seek(max(0, size - max_bytes), os.SEEK_SET)
text = file.read().decode("utf-8", errors="replace")
chunks.append(f"==> {path.name} <==\n{text}")
except OSError:
pass
return "\n".join(chunks)
def fatal_replay_message(result_dir):
text = recent_log_text([result_dir / "mobilegl.log", result_dir / "retrace.log"])
fatal_markers = ("[FATAL]", "Assertion failed:", "caught signal", "caught an unhandled exception")
if any(marker in text for marker in fatal_markers):
lines = [line for line in text.splitlines() if any(marker in line for marker in fatal_markers)]
return "\n".join(lines[-8:]) or "trace replay hit a fatal error"
return ""
def terminate_process_group(process):
if process.poll() is not None:
return
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
return
except OSError:
pass
try:
process.wait(timeout=1)
return
except subprocess.TimeoutExpired:
pass
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
return
except OSError:
pass
try:
process.wait(timeout=1)
except subprocess.TimeoutExpired:
pass
def render_summary(result_root, summary_dir, html_path):
summary_dir.mkdir(parents=True, exist_ok=True)
html_path = Path(html_path)
if html_path.is_absolute() or html_path.parent != Path("."):
html_output = html_path if html_path.is_absolute() else (ROOT / html_path).resolve()
html_output.parent.mkdir(parents=True, exist_ok=True)
html_name = html_output.name
else:
html_output = summary_dir / html_path
html_name = html_path.name
command = [
"node",
str(ROOT / "tools" / "trace_replay" / "render_retrace_summary.mjs"),
"--input",
str(result_root),
"--output-dir",
str(summary_dir),
"--title",
"MobileGL macOS window retrace overview",
"--group-label",
"macOS Window",
"--html",
html_name,
]
subprocess.run(command, cwd=ROOT, check=True)
rendered_html = summary_dir / html_name
if html_output != rendered_html:
shutil.copyfile(rendered_html, html_output)
if rendered_html != summary_dir / "index.html" and html_output != summary_dir / "index.html":
shutil.copyfile(rendered_html, summary_dir / "index.html")
return html_output
def run_case(case, backend, replay_exe, mobilegl_library, vulkan_icd):
result_dir = RESULT_ROOT / f"{safe_case(case['name'])}-{backend}"
case_work = WORK_ROOT / safe_case(case["name"])
input_dir = case_work / "input"
if result_dir.exists():
shutil.rmtree(result_dir)
if case_work.exists():
shutil.rmtree(case_work)
result_dir.mkdir(parents=True, exist_ok=True)
input_dir.mkdir(parents=True, exist_ok=True)
copy_goldens(case, backend, result_dir)
start = time.monotonic()
extract = subprocess.run(
["cmake", "-E", "tar", "xzf", str(FIXTURES / case["trace_archive"])],
cwd=input_dir,
text=True,
capture_output=True,
)
if extract.returncode != 0:
elapsed = time.monotonic() - start
(result_dir / "runner-stdout.log").write_text(extract.stdout, encoding="utf-8", errors="replace")
(result_dir / "runner-stderr.log").write_text(extract.stderr, encoding="utf-8", errors="replace")
synthesize_result(case, backend, result_dir, extract.returncode, "failed to extract trace archive", elapsed)
return extract.returncode, elapsed, "", False
alternate_args = []
if case.get("alternate_golden"):
alternate_args = ["--alternate-golden", str(FIXTURES / case["alternate_golden"])]
diff = result_dir / f"{safe_case(case['name'])}-{backend}-diff.png"
command = [
str(replay_exe),
"--trace",
str(input_dir / case["trace_file"]),
"--golden",
str(FIXTURES / case["golden"]),
*alternate_args,
"--diff",
str(diff),
"--output",
str(result_dir),
"--backend",
backend,
"--mobilegl-library",
str(mobilegl_library),
"--target-call",
str(case["target_call"]),
"--width",
str(case["width"]),
"--height",
str(case["height"]),
"--ssim-threshold",
str(case["ssim_threshold"]),
"--crop-x",
str(case["crop_x"]),
"--crop-y",
str(case["crop_y"]),
"--crop-width",
str(case["crop_width"]),
"--crop-height",
str(case["crop_height"]),
"--window-surface",
]
if case.get("coherent_as_flush"):
command.append("--coherent-as-flush")
env = os.environ.copy()
env["MOBILEGL_BACKEND_TYPE"] = backend
env["MOBILEGL_MAGMA_R11G11B10F_FALLBACK"] = "1"
if vulkan_icd:
env["VK_ICD_FILENAMES"] = vulkan_icd
stdout_path = result_dir / "runner-stdout.log"
stderr_path = result_dir / "runner-stderr.log"
with stdout_path.open("w", encoding="utf-8", errors="replace") as stdout_file, \
stderr_path.open("w", encoding="utf-8", errors="replace") as stderr_file:
process = subprocess.Popen(
command,
cwd=ROOT,
env=env,
stdout=stdout_file,
stderr=stderr_file,
start_new_session=True,
)
deadline = start + int(case["timeout_seconds"])
while process.poll() is None:
elapsed = time.monotonic() - start
fatal_message = fatal_replay_message(result_dir)
if fatal_message:
terminate_process_group(process)
synthesize_result(case, backend, result_dir, 133, fatal_message, elapsed)
return 133, elapsed, "", True
if time.monotonic() >= deadline:
terminate_process_group(process)
elapsed = time.monotonic() - start
synthesize_result(case, backend, result_dir, 124, f"timed out after {case['timeout_seconds']}s", elapsed)
return 124, elapsed, "", False
time.sleep(0.5)
elapsed = time.monotonic() - start
if not (result_dir / "result.json").exists():
stdout_text = stdout_path.read_text(encoding="utf-8", errors="replace") if stdout_path.exists() else ""
stderr_text = stderr_path.read_text(encoding="utf-8", errors="replace") if stderr_path.exists() else ""
message = (stdout_text + "\n" + stderr_text).strip()[-4000:] or f"process exited {process.returncode}"
synthesize_result(case, backend, result_dir, process.returncode, message, elapsed)
actual = result_dir / "actual.png"
if actual.exists():
shutil.copyfile(actual, result_dir / f"{safe_case(case['name'])}-{backend}-actual.png")
ssim = ""
try:
result = json.loads((result_dir / "result.json").read_text(encoding="utf-8"))
ssim = result.get("ssim", "")
except (OSError, json.JSONDecodeError):
pass
return process.returncode, elapsed, ssim, False
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--case", action="append", dest="cases", help="Case name to run; may be repeated.")
parser.add_argument("--all", action="store_true", help="Run every selected trace case.")
parser.add_argument("--ci", action="store_true", help="Select only cases enabled for CI.")
parser.add_argument("--skip-case", action="append", default=[], help="Mark a selected case skipped without launching it; may be repeated.")
parser.add_argument("--backend", action="append", choices=BACKENDS, help="Backend to run; default: DirectVulkan.")
parser.add_argument("--build-dir", type=Path, default=DEFAULT_BUILD_DIR, help="CMake build directory with mobilegl_trace_replay.")
parser.add_argument("--trace-replay-exe", type=Path, help="Path to mobilegl_trace_replay.")
parser.add_argument("--mobilegl-library", type=Path, help="Path to libMobileGL.dylib.")
parser.add_argument("--vulkan-icd", help="MoltenVK ICD JSON path; defaults to VK_ICD_FILENAMES or a detected local install.")
parser.add_argument("--result-root", type=Path, default=RESULT_ROOT)
parser.add_argument("--work-root", type=Path, default=WORK_ROOT)
parser.add_argument("--summary-dir", type=Path, default=SUMMARY_DIR)
parser.add_argument("--html", default=SUMMARY_HTML)
parser.add_argument("--keep-results", action="store_true", help="Do not clear previous result/work roots.")
parser.add_argument("--no-render", action="store_true", help="Do not render the HTML summary.")
parser.add_argument("--fetch-fixtures", action=argparse.BooleanOptionalAction, default=True, help="Hydrate missing fixtures before running.")
parser.add_argument("--fixture-mirror-base", action="append", dest="fixture_mirror_bases",
help="Fixture mirror base URL; repeatable, tried in order. "
"Defaults to the hit.moe mirror then the miawa mirror.")
parser.add_argument("--download-jobs", type=int, default=4, help="Parallel fixture download count.")
parser.add_argument("--continue-after-fatal", action="store_true", help="Continue launching later cases after a fatal native-window replay.")
return parser.parse_args()
def main():
global RESULT_ROOT, WORK_ROOT
args = parse_args()
RESULT_ROOT = args.result_root.resolve()
WORK_ROOT = args.work_root.resolve()
replay_exe = (args.trace_replay_exe or (args.build_dir / "tools" / "trace_replay" / "mobilegl_trace_replay")).resolve()
mobilegl_library = (args.mobilegl_library or (args.build_dir / "libMobileGL.dylib")).resolve()
vulkan_icd = args.vulkan_icd or default_vulkan_icd(mobilegl_library)
selected_backends = args.backend or ["DirectVulkan"]
cases = ci_trace_cases(CASES) if args.ci else CASES
selected_names = set(args.cases or [])
selected_cases = [case_with_defaults(case) for case in cases if args.all or case["name"] in selected_names]
if not selected_cases:
print("No cases selected. Use --all or --case NAME.", file=sys.stderr)
return 2
if not replay_exe.exists():
print(f"missing trace replay executable: {replay_exe}", file=sys.stderr)
return 2
if not mobilegl_library.exists():
print(f"missing MobileGL library: {mobilegl_library}", file=sys.stderr)
return 2
mirror_bases = args.fixture_mirror_bases or [
b for b in os.environ.get("MOBILEGL_TRACE_FIXTURE_MIRROR_BASES", "").split() or []
] or ([os.environ["MOBILEGL_TRACE_FIXTURE_MIRROR_BASE"]] if os.environ.get("MOBILEGL_TRACE_FIXTURE_MIRROR_BASE") else DEFAULT_MIRROR_BASES)
if not hydrate_fixtures(selected_cases, args.fetch_fixtures, mirror_bases, max(1, args.download_jobs)):
return 2
if not args.keep_results:
shutil.rmtree(RESULT_ROOT, ignore_errors=True)
shutil.rmtree(WORK_ROOT, ignore_errors=True)
RESULT_ROOT.mkdir(parents=True, exist_ok=True)
WORK_ROOT.mkdir(parents=True, exist_ok=True)
print(f"trace replay: {replay_exe} ({file_arch(replay_exe) or 'unknown arch'})", flush=True)
print(f"MobileGL: {mobilegl_library} ({file_arch(mobilegl_library) or 'unknown arch'})", flush=True)
if vulkan_icd:
print(f"Vulkan ICD: {vulkan_icd}", flush=True)
else:
print("Vulkan ICD: using loader defaults", flush=True)
results = []
stop_matrix = False
skipped_names = set(args.skip_case)
for case in selected_cases:
for backend in selected_backends:
if case["name"] in skipped_names:
result_dir = RESULT_ROOT / f"{safe_case(case['name'])}-{backend}"
result_dir.mkdir(parents=True, exist_ok=True)
copy_goldens(case, backend, result_dir)
synthesize_result(case, backend, result_dir, 125, "skipped by --skip-case", 0)
print(f"=== macOS window retrace: {case['name']} / {backend} ===", flush=True)
print(" SKIP rc=125 elapsed=0.0s ssim=", flush=True)
results.append({
"case": case["name"],
"backend": backend,
"status": "SKIP",
"returncode": 125,
"elapsedSeconds": 0,
"ssim": "",
})
continue
print(f"=== macOS window retrace: {case['name']} / {backend} ===", flush=True)
rc, elapsed, ssim, fatal = run_case(case, backend, replay_exe, mobilegl_library, vulkan_icd)
status = "PASS" if rc == 0 else "FAIL"
print(f" {status} rc={rc} elapsed={elapsed:.1f}s ssim={ssim}", flush=True)
results.append({
"case": case["name"],
"backend": backend,
"status": status,
"returncode": rc,
"elapsedSeconds": round(elapsed, 3),
"ssim": ssim,
})
if fatal and not args.continue_after_fatal:
print("stopping after fatal native-window replay; use --continue-after-fatal to keep launching later cases", file=sys.stderr)
stop_matrix = True
break
if stop_matrix:
break
summary_json = RESULT_ROOT / "macos-window-retrace-summary.json"
summary_json.write_text(json.dumps(results, indent=2), encoding="utf-8")
if not args.no_render:
html_output = render_summary(RESULT_ROOT, args.summary_dir.resolve(), args.html)
print(f"summary: {html_output}", flush=True)
failures = sum(1 for result in results if result["returncode"] != 0)
print(f"completed cases={len(results)} pass={len(results) - failures} fail={failures}", flush=True)
return 1 if failures else 0
if __name__ == "__main__":
raise SystemExit(main())