mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-11-16 06:23:23 +09:00
[CI, Tools] (...): replay trace cases in shards from one fixture cache
This commit is contained in:
@@ -74,3 +74,57 @@ adb shell am start -a top.mobilegl.plugin.TRACE_REPLAY \
|
||||
--es ssim_threshold 0.99
|
||||
adb shell run-as top.mobilegl.plugin.espryt.trace cat files/trace-replay/output/result.json
|
||||
```
|
||||
|
||||
## Emulator shards in CI
|
||||
|
||||
The APK workflow replays its cases through `trace-replay-shard.sh`: one emulator boot, one APK
|
||||
install, then an ordered list of cases for one lane, transport and backend. Each case still runs
|
||||
through `trace-replay-ci.sh`, with the same arguments, environment and transport proof as a
|
||||
one-case job.
|
||||
|
||||
```sh
|
||||
sh android-plugin/trace-replay-shard.sh \
|
||||
--apk-file MobileGL-plugin-trace-release.apk \
|
||||
--backend DirectVulkan --gpu lavapipe \
|
||||
--transport inproc --lane inproc-acceptance \
|
||||
--cases-file cases.json --deadline-seconds 7200
|
||||
```
|
||||
|
||||
The case list is a JSON array of case objects: the `cases` member of a row of
|
||||
`trace_cases.py --ci --format github-apk-transport-shard-matrix`, which is what the workflow
|
||||
passes, or the `case` members of `--format github-apk-transport-matrix` rows. `--dry-run` prints
|
||||
what each case would be run with and touches no device.
|
||||
|
||||
The workflow's matrix is that shard matrix: one row per emulator job, holding some of the cases
|
||||
of one lane, transport and backend. `--shard-seconds` (`RETRACE_SHARD_SECONDS` in `apk.yml`) sets
|
||||
how much hinted replay time one shard aims for, and `trace_cases.py` refuses to print a shard
|
||||
matrix whose shards, taken apart, are not exactly the per-case matrix. The fixtures of a shard's
|
||||
cases must be in `tools/trace_replay/fixtures` before the runner starts; the workflow restores
|
||||
them from one cache entry (`.github/scripts/trace-fixture-bundle.sh`).
|
||||
|
||||
Between cases the runner force-stops the app, checks that no process of the app's uid is left,
|
||||
clears the app data, removes the case's files from `/data/local/tmp` and from the host, and
|
||||
checks that the device still answers. It restarts the emulator when it does not, before the
|
||||
case that follows one that did not pass, and before the single retry of a case that
|
||||
`trace-replay-ci.sh` charged to the infrastructure (exit 75).
|
||||
`MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE=1` cold boots before every case.
|
||||
|
||||
The cases of a shard share the job's time. `--deadline-seconds` stops the runner that long after
|
||||
it starts, so that the job can still upload what it has: a case that is running then, or has not
|
||||
started, is recorded as `not-run`. `retrace_shard_budget.py` reads a shard matrix and adds two
|
||||
fields to every shard: `deadline_seconds`, the time in which every case of the shard could use
|
||||
its whole time budget (`timeout_seconds` plus 300 s) when the cases before it used all of
|
||||
theirs, with one infrastructure retry of the longest budget and a cold boot per case on top,
|
||||
and `timeout_minutes`, the limit of the shard's job. `--cap-seconds`
|
||||
(`RETRACE_SHARD_DEADLINE_CAP_SECONDS` in `apk.yml`) bounds the deadline, so a shard whose
|
||||
replays all hang cannot hold a runner for hours; a capped shard loses verdicts only when several
|
||||
of its replays hang to the end of their budgets.
|
||||
|
||||
A failing case does not stop the shard. Every case ends with `<case>-<backend>/case-status.json`
|
||||
(`passed`, `failed` or `not-run`), the result root gets `shard-status.json`, the job log gets one
|
||||
`::error` per case that did not pass, and the exit status is non-zero when a case did not pass or
|
||||
a verdict is missing. `--finalize` writes the summary inputs and a `not-run` verdict for every
|
||||
case without one, without a device; the workflow runs it after the replay step whatever happened.
|
||||
|
||||
`python3 android-plugin/test_trace_replay_shard.py` drives the runner with a fake adb, replay
|
||||
script and AVD script.
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
#!/usr/bin/env python3
|
||||
# MobileGL - android-plugin/retrace_artifact_cleanup.py
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
"""Delete the intermediate artifacts of an APK workflow run, keeping what a failure needs.
|
||||
|
||||
The retrace summary embeds every image it shows, so once it is uploaded the per-shard result
|
||||
artifacts are only worth keeping for cases that did not pass:
|
||||
|
||||
- a result artifact stays when its shard-status.json is missing, when its job failed, or when
|
||||
it holds a case that did not pass in any shard (so both backends and all transports of a
|
||||
failed case stay comparable);
|
||||
- every result artifact stays when a job failed without leaving a shard-status.json to say
|
||||
which cases it was running.
|
||||
|
||||
A canceled job retains nothing by itself; a failed verdict inside it still does. The trace
|
||||
fixtures are not artifacts: the retrace jobs restore them from a cache entry.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
RESULT_PREFIXES = ("MobileGL-android-retrace-result-", "retrace-result-")
|
||||
FAILED_CONCLUSIONS = ("failure", "timed_out", "action_required")
|
||||
# GitHub's secondary rate limit allows about 180 mutating REST calls a minute. Requests start
|
||||
# at most this often, whatever the number of workers.
|
||||
DELETE_INTERVAL_SECONDS = 60.0 / 170.0
|
||||
DELETE_WORKERS = 3
|
||||
MAX_REFUSED_DELETES = 5
|
||||
|
||||
|
||||
def pause(seconds):
|
||||
time.sleep(seconds)
|
||||
|
||||
|
||||
def read_rows(text, columns):
|
||||
rows = []
|
||||
for line in text.splitlines():
|
||||
if line.strip():
|
||||
fields = line.split("\t")
|
||||
rows.append(fields + [""] * (columns - len(fields)))
|
||||
return rows
|
||||
|
||||
|
||||
def shard_statuses(download_roots):
|
||||
"""Artifact name -> its shard-status.json, for every downloaded result artifact that has one."""
|
||||
statuses = {}
|
||||
for root in download_roots:
|
||||
root = Path(root)
|
||||
if not root.is_dir():
|
||||
continue
|
||||
for directory in root.iterdir():
|
||||
try:
|
||||
status = json.loads((directory / "shard-status.json").read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if isinstance(status, dict) and isinstance(status.get("cases"), list):
|
||||
statuses[directory.name] = status
|
||||
return statuses
|
||||
|
||||
|
||||
def plan(artifacts, jobs, statuses):
|
||||
"""Returns (ids to delete, lines explaining what stays)."""
|
||||
failed_job_ids = {job_id for job_id, conclusion, _ in jobs if conclusion in FAILED_CONCLUSIONS}
|
||||
status_job_ids = {str(status.get("job_id") or "") for status in statuses.values()} - {""}
|
||||
# Failed jobs that left no status file: nothing says which cases they were running.
|
||||
unexplained = failed_job_ids - status_job_ids
|
||||
if not status_job_ids and failed_job_ids:
|
||||
# The statuses carry no job ids to match against. Count instead: each failed job must
|
||||
# be explained by an artifact that shows a case without a pass.
|
||||
explaining = sum(1 for status in statuses.values()
|
||||
if any(record.get("status") != "passed" for record in status["cases"]))
|
||||
unexplained = set() if explaining >= len(failed_job_ids) else failed_job_ids
|
||||
|
||||
failed_cases = set() # did not pass somewhere: every result that holds one stays
|
||||
for status in statuses.values():
|
||||
not_passed = [record for record in status["cases"] if record.get("status") != "passed"]
|
||||
if status_job_ids:
|
||||
job_failed = str(status.get("job_id") or "") in failed_job_ids
|
||||
else:
|
||||
# Without job ids, a status that shows a case without a pass stands for a failed job.
|
||||
job_failed = bool(failed_job_ids) and bool(not_passed)
|
||||
for record in not_passed:
|
||||
if record.get("status") == "failed" or job_failed:
|
||||
failed_cases.add(record.get("case"))
|
||||
|
||||
delete, kept = [], []
|
||||
for artifact_id, name in artifacts:
|
||||
if not name.startswith(RESULT_PREFIXES):
|
||||
continue
|
||||
reason = ""
|
||||
status = statuses.get(name)
|
||||
if unexplained:
|
||||
reason = f"{len(unexplained)} failed job(s) left no shard status"
|
||||
elif status is None:
|
||||
reason = "no shard status was downloaded for it"
|
||||
elif str(status.get("job_id") or "") in failed_job_ids:
|
||||
reason = "its job failed"
|
||||
else:
|
||||
cases = sorted(failed_cases.intersection(record.get("case") for record in status["cases"]))
|
||||
if cases:
|
||||
reason = "holds failed case(s): " + ", ".join(cases)
|
||||
if reason:
|
||||
kept.append(f"{name} ({artifact_id}): {reason}")
|
||||
else:
|
||||
delete.append((artifact_id, name))
|
||||
return delete, kept
|
||||
|
||||
|
||||
class Deleter:
|
||||
"""Runs `command` once per artifact, with the artifact id appended to its last word."""
|
||||
|
||||
def __init__(self, command):
|
||||
self.command = command
|
||||
self.lock = threading.Lock()
|
||||
self.next_start = 0.0
|
||||
self.refused = []
|
||||
self.deleted = 0
|
||||
|
||||
def wait_for_slot(self):
|
||||
with self.lock:
|
||||
now = time.monotonic()
|
||||
start = max(now, self.next_start)
|
||||
self.next_start = start + DELETE_INTERVAL_SECONDS
|
||||
if start > now:
|
||||
pause(start - now)
|
||||
|
||||
def delete(self, artifact_id, name):
|
||||
for attempt in (1, 2):
|
||||
with self.lock:
|
||||
if len(self.refused) >= MAX_REFUSED_DELETES:
|
||||
return
|
||||
self.wait_for_slot()
|
||||
done = subprocess.run(self.command[:-1] + [self.command[-1] + artifact_id],
|
||||
capture_output=True, text=True)
|
||||
if done.returncode == 0:
|
||||
with self.lock:
|
||||
self.deleted += 1
|
||||
return
|
||||
if attempt == 1:
|
||||
# Give a rate limit time to clear before the one retry.
|
||||
with self.lock:
|
||||
self.next_start = max(self.next_start, time.monotonic() + 30)
|
||||
with self.lock:
|
||||
self.refused.append(f"{name} ({artifact_id}): {' '.join(done.stderr.split())[:200]}")
|
||||
|
||||
def run(self, entries):
|
||||
pending = list(entries)
|
||||
|
||||
def work():
|
||||
while True:
|
||||
with self.lock:
|
||||
if not pending:
|
||||
return
|
||||
artifact_id, name = pending.pop(0)
|
||||
self.delete(artifact_id, name)
|
||||
|
||||
threads = [threading.Thread(target=work) for _ in range(min(DELETE_WORKERS, len(pending)))]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
|
||||
def gh_rows(path, query, columns):
|
||||
done = subprocess.run(["gh", "api", "--paginate", path, "--jq", query], capture_output=True, text=True)
|
||||
if done.returncode != 0:
|
||||
raise RuntimeError(f"gh api {path} failed: {done.stderr.strip()}")
|
||||
return read_rows(done.stdout, columns)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--downloads", action="append", default=[],
|
||||
help="Directory holding one sub-directory per downloaded result artifact.")
|
||||
parser.add_argument("--artifacts-file", help="Tab-separated id and name per line, instead of asking gh.")
|
||||
parser.add_argument("--jobs-file", help="Tab-separated id, conclusion and name per line, instead of asking gh.")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Print the plan, delete nothing.")
|
||||
options = parser.parse_args(argv)
|
||||
|
||||
repository = os.environ.get("GITHUB_REPOSITORY", "")
|
||||
run_id = os.environ.get("GITHUB_RUN_ID", "")
|
||||
try:
|
||||
if options.artifacts_file:
|
||||
artifacts = read_rows(Path(options.artifacts_file).read_text(encoding="utf-8"), 2)
|
||||
else:
|
||||
artifacts = gh_rows(f"repos/{repository}/actions/runs/{run_id}/artifacts?per_page=100",
|
||||
".artifacts[] | [.id, .name] | @tsv", 2)
|
||||
if options.jobs_file:
|
||||
jobs = read_rows(Path(options.jobs_file).read_text(encoding="utf-8"), 3)
|
||||
else:
|
||||
# 50 rows per page: the jobs endpoint answers HTTP 502 for larger pages of a re-run
|
||||
# attempt of a run this size.
|
||||
jobs = gh_rows(f"repos/{repository}/actions/runs/{run_id}/jobs?per_page=50",
|
||||
'.jobs[] | [.id, .conclusion // "", .name] | @tsv', 3)
|
||||
except (OSError, RuntimeError) as error:
|
||||
print(f"::error::Could not read the run's artifacts and jobs; nothing was deleted: {error}")
|
||||
return 1
|
||||
|
||||
delete, kept = plan([(row[0], row[1]) for row in artifacts], [tuple(row[:3]) for row in jobs],
|
||||
shard_statuses(options.downloads))
|
||||
for line in kept:
|
||||
print(f"Retaining {line}")
|
||||
if options.dry_run:
|
||||
for artifact_id, name in delete:
|
||||
print(f"Would delete {name} ({artifact_id})")
|
||||
return 0
|
||||
|
||||
deleter = Deleter(shlex.split(os.environ.get("MOBILEGL_CLEANUP_DELETE_COMMAND") or
|
||||
f"gh api --method DELETE repos/{repository}/actions/artifacts/"))
|
||||
started = time.monotonic()
|
||||
deleter.run(delete)
|
||||
for line in deleter.refused:
|
||||
print(f"::warning::Could not delete intermediate artifact {line}")
|
||||
if len(deleter.refused) >= MAX_REFUSED_DELETES:
|
||||
print("::warning::Stopped deleting intermediate artifacts after repeated refusals; the rest stay until they expire.")
|
||||
print(f"Deleted {deleter.deleted} intermediate Android artifact(s) in {time.monotonic() - started:.0f}s; "
|
||||
f"retained {len(kept)}.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
# MobileGL - android-plugin/retrace_shard_budget.py
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
"""Give every emulator shard the deadline and the job timeout its own cases need.
|
||||
|
||||
The cases of a shard share one job, so one case that hangs spends time the cases after it
|
||||
need. This reads a shard matrix of trace_cases.py (--format github-apk-shard-matrix or
|
||||
github-apk-transport-shard-matrix) on standard input, or from --matrix-file, and prints it
|
||||
with two fields added to every shard: deadline_seconds, which the retrace job passes to
|
||||
trace-replay-shard.sh, and timeout_minutes, the limit of that job. The deadline is what the
|
||||
shard needs for every case to use its whole time budget (trace_replay_shard.needed_seconds),
|
||||
up to --cap-seconds. A capped shard still replays every case; only when several of its
|
||||
replays hang to the end of their budgets are the cases after them recorded as not-run.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import trace_replay_shard # noqa: E402 (a sibling file, found through the line above)
|
||||
|
||||
|
||||
def shard_label(row):
|
||||
backend = row.get("backend")
|
||||
backend = backend.get("name", "") if isinstance(backend, dict) else str(backend or "")
|
||||
return f"{backend} / {row.get('transport') or 'monolith'}, shard {row.get('shard')}/{row.get('shards')}"
|
||||
|
||||
|
||||
def shard_needs(matrix):
|
||||
"""(label, case count, seconds needed) for every shard of the matrix."""
|
||||
rows = matrix.get("include") if isinstance(matrix, dict) else None
|
||||
if not isinstance(rows, list) or not rows:
|
||||
raise ValueError("the matrix has no shard rows under 'include'")
|
||||
needs = []
|
||||
for row in rows:
|
||||
cases = row.get("cases") if isinstance(row, dict) else None
|
||||
if not isinstance(cases, list) or not cases:
|
||||
raise ValueError(f"a shard row has no 'cases' list: {str(row)[:200]}")
|
||||
needs.append((shard_label(row), len(cases), trace_replay_shard.needed_seconds(cases)))
|
||||
return needs
|
||||
|
||||
|
||||
# Minutes a retrace job has around its shard runner: the setup before the replays and the
|
||||
# uploads after them.
|
||||
JOB_OVERHEAD_MINUTES = 15
|
||||
|
||||
# A hosted job may run six hours.
|
||||
HOSTED_JOB_LIMIT_MINUTES = 360
|
||||
|
||||
|
||||
def with_limits(matrix, cap_seconds):
|
||||
"""The matrix with deadline_seconds and timeout_minutes on every shard, and the report."""
|
||||
needs = shard_needs(matrix)
|
||||
rows = []
|
||||
report = []
|
||||
for row, (label, cases, seconds) in zip(matrix["include"], needs):
|
||||
deadline = min(seconds, cap_seconds)
|
||||
rows.append({**row, "deadline_seconds": deadline,
|
||||
"timeout_minutes": -(-deadline // 60) + JOB_OVERHEAD_MINUTES})
|
||||
if seconds > cap_seconds:
|
||||
report.append(f"retrace shard budget: {label} would need {seconds} s for all of its "
|
||||
f"{cases} case(s) to use their whole budgets; capped at {cap_seconds} s")
|
||||
largest = max(needs, key=lambda entry: entry[2])
|
||||
report.append(f"retrace shard budget: {len(needs)} shard(s), {len(report)} capped at "
|
||||
f"{cap_seconds} s; the most a shard needs is {largest[2]} s ({largest[0]}, "
|
||||
f"{largest[1]} case(s))")
|
||||
return {**matrix, "include": rows}, report
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--cap-seconds", type=int, required=True,
|
||||
help="The longest deadline a shard is given.")
|
||||
parser.add_argument("--matrix-file", help="The shard matrix; standard input when not given.")
|
||||
options = parser.parse_args(argv)
|
||||
limit = (HOSTED_JOB_LIMIT_MINUTES - JOB_OVERHEAD_MINUTES) * 60
|
||||
if not 0 < options.cap_seconds <= limit:
|
||||
print(f"retrace shard budget: --cap-seconds must be between 1 and {limit}", file=sys.stderr)
|
||||
return 2
|
||||
if options.matrix_file:
|
||||
text = Path(options.matrix_file).read_text(encoding="utf-8")
|
||||
else:
|
||||
text = sys.stdin.read()
|
||||
try:
|
||||
matrix, report = with_limits(json.loads(text), options.cap_seconds)
|
||||
except (ValueError, KeyError, TypeError) as error:
|
||||
print(f"retrace shard budget: the shard matrix cannot be read: {error}", file=sys.stderr)
|
||||
return 2
|
||||
for line in report:
|
||||
print(line, file=sys.stderr)
|
||||
print(json.dumps(matrix, separators=(",", ":")))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env python3
|
||||
# MobileGL - android-plugin/test_retrace_artifact_cleanup.py
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
"""Pin which intermediate APK artifacts the cleanup keeps, and how fast it deletes the rest."""
|
||||
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
HERE = pathlib.Path(__file__).resolve().parent
|
||||
_spec = importlib.util.spec_from_file_location("retrace_artifact_cleanup", HERE / "retrace_artifact_cleanup.py")
|
||||
cleanup = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(cleanup)
|
||||
|
||||
RESULT = "MobileGL-android-retrace-result-"
|
||||
|
||||
|
||||
def status(job_id, **cases):
|
||||
return {"job_id": job_id, "cases": [{"case": name, "status": verdict} for name, verdict in cases.items()]}
|
||||
|
||||
|
||||
def names(entries):
|
||||
return sorted(name for _, name in entries)
|
||||
|
||||
|
||||
class PlanTest(unittest.TestCase):
|
||||
ARTIFACTS = [("1", RESULT + "mono-gles"), ("2", RESULT + "inproc-gles"), ("3", RESULT + "spawn-vk"),
|
||||
("4", "mobilegl-android-monolith-control.html"), ("5", "MobileGL-plugin-release-abc.apk"),
|
||||
("6", "MobileGL-plugin-trace-release-abc.apk")]
|
||||
|
||||
def test_a_green_run_deletes_every_intermediate_and_nothing_else(self):
|
||||
statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed"),
|
||||
RESULT + "inproc-gles": status("12", a="passed", b="passed"),
|
||||
RESULT + "spawn-vk": status("13", c="passed")}
|
||||
jobs = [("11", "success", "x"), ("12", "success", "y"), ("13", "success", "z"), ("14", "", "retrace summary")]
|
||||
delete, kept = cleanup.plan(self.ARTIFACTS, jobs, statuses)
|
||||
self.assertEqual(kept, [])
|
||||
self.assertEqual(names(delete), sorted(name for _, name in self.ARTIFACTS[:3]))
|
||||
|
||||
def test_a_failed_case_keeps_every_shard_that_ran_it(self):
|
||||
statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed"),
|
||||
RESULT + "inproc-gles": status("12", a="passed", b="failed"),
|
||||
RESULT + "spawn-vk": status("13", c="passed")}
|
||||
jobs = [("11", "success", "x"), ("12", "failure", "y"), ("13", "success", "z")]
|
||||
delete, kept = cleanup.plan(self.ARTIFACTS, jobs, statuses)
|
||||
# Only b's results are worth comparing, so the monolith shard stays for b and the
|
||||
# spawn shard goes.
|
||||
self.assertEqual(names(delete), [RESULT + "spawn-vk"])
|
||||
self.assertEqual(len(kept), 2)
|
||||
self.assertTrue(any("inproc-gles" in line and "its job failed" in line for line in kept))
|
||||
self.assertTrue(any("mono-gles" in line and "holds failed case(s): b" in line for line in kept))
|
||||
|
||||
def test_cases_a_failed_job_never_reached_are_kept(self):
|
||||
statuses = {RESULT + "mono-gles": status("11", a="passed", b="not-run"),
|
||||
RESULT + "spawn-vk": status("13", c="passed")}
|
||||
jobs = [("11", "timed_out", "x"), ("13", "success", "z")]
|
||||
delete, kept = cleanup.plan(self.ARTIFACTS[:1] + self.ARTIFACTS[2:], jobs, statuses)
|
||||
self.assertEqual(names(delete), [RESULT + "spawn-vk"])
|
||||
self.assertEqual(len(kept), 1)
|
||||
|
||||
def test_a_job_that_failed_after_its_cases_passed_keeps_its_results(self):
|
||||
statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed"),
|
||||
RESULT + "spawn-vk": status("13", c="passed")}
|
||||
jobs = [("11", "failure", "upload failed"), ("13", "success", "z")]
|
||||
delete, kept = cleanup.plan(self.ARTIFACTS[:1] + self.ARTIFACTS[2:], jobs, statuses)
|
||||
self.assertEqual(names(delete), [RESULT + "spawn-vk"])
|
||||
self.assertTrue("mono-gles" in kept[0] and "its job failed" in kept[0])
|
||||
|
||||
def test_a_canceled_job_keeps_nothing_by_itself(self):
|
||||
statuses = {RESULT + "mono-gles": status("11", a="passed", b="not-run"),
|
||||
RESULT + "inproc-gles": status("12", a="failed", b="not-run")}
|
||||
jobs = [("11", "cancelled", "x"), ("12", "cancelled", "y")]
|
||||
delete, kept = cleanup.plan(self.ARTIFACTS[:2], jobs, statuses)
|
||||
# Case a failed in the inproc shard, so both shards that ran it stay.
|
||||
self.assertEqual(delete, [])
|
||||
self.assertEqual(len(kept), 2)
|
||||
only_not_run = {RESULT + "mono-gles": status("11", a="passed", b="not-run")}
|
||||
delete, kept = cleanup.plan(self.ARTIFACTS[:1], [("11", "cancelled", "x")], only_not_run)
|
||||
self.assertEqual((names(delete), kept), ([RESULT + "mono-gles"], []))
|
||||
|
||||
def test_a_result_without_a_status_is_kept(self):
|
||||
statuses = {RESULT + "mono-gles": status("11", a="passed")}
|
||||
delete, kept = cleanup.plan(self.ARTIFACTS[:2], [("11", "success", "x")], statuses)
|
||||
self.assertEqual(names(delete), [RESULT + "mono-gles"])
|
||||
self.assertIn("no shard status", kept[0])
|
||||
|
||||
def test_a_failed_job_without_a_status_keeps_everything(self):
|
||||
statuses = {RESULT + "mono-gles": status("11", a="passed", b="passed")}
|
||||
jobs = [("11", "success", "x"), ("12", "failure", "died before its upload")]
|
||||
delete, kept = cleanup.plan(self.ARTIFACTS, jobs, statuses)
|
||||
self.assertEqual(delete, [])
|
||||
self.assertEqual(len(kept), 3)
|
||||
|
||||
def test_statuses_without_job_ids_fall_back_to_counting(self):
|
||||
artifacts = self.ARTIFACTS[:1] + self.ARTIFACTS[2:]
|
||||
explained = {RESULT + "mono-gles": status("", a="passed", b="not-run"),
|
||||
RESULT + "spawn-vk": status("", c="passed")}
|
||||
delete, _ = cleanup.plan(artifacts, [("12", "failure", "y")], explained)
|
||||
# The one status with a case that did not pass stands for the one failed job.
|
||||
self.assertEqual(names(delete), [RESULT + "spawn-vk"])
|
||||
unexplained = {RESULT + "mono-gles": status("", a="passed", b="passed"),
|
||||
RESULT + "spawn-vk": status("", c="passed")}
|
||||
delete, _ = cleanup.plan(artifacts, [("12", "failure", "y")], unexplained)
|
||||
self.assertEqual(delete, [])
|
||||
|
||||
|
||||
class RunTest(unittest.TestCase):
|
||||
@staticmethod
|
||||
def passed(artifacts):
|
||||
"""A shard status with one passed case for every artifact, each from a job of its own."""
|
||||
return {name: status(f"9{artifact_id}", a="passed") for artifact_id, name in artifacts}
|
||||
|
||||
def run_cleanup(self, artifacts, jobs, statuses, exit_code=0, dry_run=False):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = pathlib.Path(directory)
|
||||
for name, value in statuses.items():
|
||||
(root / "downloads" / "lane" / name).mkdir(parents=True)
|
||||
(root / "downloads" / "lane" / name / "shard-status.json").write_text(json.dumps(value))
|
||||
(root / "artifacts.tsv").write_text("".join(f"{i}\t{n}\n" for i, n in artifacts))
|
||||
(root / "jobs.tsv").write_text("".join("\t".join(job) + "\n" for job in jobs))
|
||||
# One file per call: the deletes run in parallel processes.
|
||||
(root / "deleted").mkdir()
|
||||
script = root / "delete.py"
|
||||
script.write_text("import pathlib, sys\n"
|
||||
"(pathlib.Path(sys.argv[1]) / sys.argv[2].replace('/', '_')).write_text('')\n"
|
||||
f"sys.exit({exit_code})\n")
|
||||
python = pathlib.Path(sys.executable).as_posix()
|
||||
command = f'"{python}" "{script.as_posix()}" "{(root / "deleted").as_posix()}" artifacts/'
|
||||
pauses = []
|
||||
output = io.StringIO()
|
||||
with mock.patch.dict(os.environ, {"MOBILEGL_CLEANUP_DELETE_COMMAND": command}), \
|
||||
mock.patch.object(cleanup, "pause", pauses.append), contextlib.redirect_stdout(output):
|
||||
code = cleanup.main(["--downloads", str(root / "downloads" / "lane"),
|
||||
"--artifacts-file", str(root / "artifacts.tsv"),
|
||||
"--jobs-file", str(root / "jobs.tsv")] + (["--dry-run"] if dry_run else []))
|
||||
deleted = sorted(entry.name.replace("_", "/") for entry in (root / "deleted").iterdir())
|
||||
return code, output.getvalue(), deleted, pauses
|
||||
|
||||
def test_deletes_are_spaced_under_the_secondary_rate_limit(self):
|
||||
artifacts = [(str(index), f"{RESULT}shard{index}") for index in range(12)]
|
||||
code, output, deleted, pauses = self.run_cleanup(artifacts, [], self.passed(artifacts))
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(deleted, sorted(f"artifacts/{index}" for index in range(12)))
|
||||
self.assertIn("Deleted 12 intermediate Android artifact(s)", output)
|
||||
# Twelve starts are booked at least 11 intervals apart, so the rate stays under 180 a minute.
|
||||
self.assertLess(60 / cleanup.DELETE_INTERVAL_SECONDS, 180)
|
||||
self.assertGreater(len(pauses), 0)
|
||||
|
||||
def test_refused_deletes_warn_and_leave_the_run_green(self):
|
||||
artifacts = [(str(index), f"{RESULT}shard{index}") for index in range(8)]
|
||||
code, output, _, _ = self.run_cleanup(artifacts, [], self.passed(artifacts), exit_code=1)
|
||||
self.assertEqual(code, 0)
|
||||
self.assertIn("::warning::Could not delete intermediate artifact", output)
|
||||
self.assertIn("Stopped deleting", output)
|
||||
self.assertIn("Deleted 0 intermediate", output)
|
||||
|
||||
def test_a_dry_run_deletes_nothing(self):
|
||||
artifacts = [("1", RESULT + "shard1")]
|
||||
code, output, deleted, _ = self.run_cleanup(artifacts, [], self.passed(artifacts), dry_run=True)
|
||||
self.assertEqual((code, deleted), (0, []))
|
||||
self.assertIn(f"Would delete {RESULT}shard1 (1)", output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,587 @@
|
||||
#!/usr/bin/env python3
|
||||
# MobileGL - android-plugin/test_trace_replay_shard.py
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
"""Drive the emulator shard runner with a fake adb, replay script and AVD script.
|
||||
|
||||
No device is needed. The fakes record every call, so the tests pin what each case is run
|
||||
with, that every case ends with a verdict, and what a failure, an infrastructure retry, a
|
||||
timeout or a dead emulator does to the rest of the shard.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
HERE = pathlib.Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def load(name):
|
||||
spec = importlib.util.spec_from_file_location(name, HERE / f"{name}.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
shard = load("trace_replay_shard")
|
||||
budget = load("retrace_shard_budget")
|
||||
|
||||
BLISS = "minecraft-1.21.4-fabric-iris-bliss-in-world"
|
||||
ITERATIONRP = "minecraft-1.21.4-fabric-iris-iterationrp-in-world"
|
||||
SPLIT = "MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1"
|
||||
|
||||
FAKE_COMMON = r'''
|
||||
import json, os, pathlib, sys
|
||||
state = pathlib.Path(os.environ["FAKE_STATE"])
|
||||
def record(name, entry):
|
||||
with open(state / name, "a", encoding="utf-8") as log:
|
||||
log.write(json.dumps(entry) + "\n")
|
||||
def scenario():
|
||||
path = state / "scenario.json"
|
||||
return json.loads(path.read_text()) if path.exists() else {}
|
||||
def count(name):
|
||||
path = state / f"count-{name}"
|
||||
value = int(path.read_text()) if path.exists() else 0
|
||||
path.write_text(str(value + 1))
|
||||
return value
|
||||
'''
|
||||
|
||||
FAKE_ADB = FAKE_COMMON + r'''
|
||||
args = sys.argv[1:]
|
||||
record("adb.log", args)
|
||||
plan = scenario()
|
||||
if args[:1] == ["get-state"]:
|
||||
print("offline" if count("get-state") < plan.get("offline_get_state_calls", 0) else "device")
|
||||
elif args[:3] == ["shell", "getprop", "sys.boot_completed"]:
|
||||
print("1")
|
||||
elif args[:3] == ["shell", "pm", "path"]:
|
||||
print("package:/data/app/trace/base.apk")
|
||||
elif args[:3] == ["shell", "pm", "clear"]:
|
||||
print("Failed" if plan.get("pm_clear_fails") else "Success")
|
||||
elif args[:1] == ["install"]:
|
||||
if count("install") < plan.get("failing_installs", 0):
|
||||
print("Failure [INSTALL_FAILED]")
|
||||
sys.exit(1)
|
||||
print("Success")
|
||||
elif args[:2] == ["shell", "run-as"] and args[3:] == ["id", "-un"]:
|
||||
print("u0_a77")
|
||||
elif args[:3] == ["shell", "ps", "-A"]:
|
||||
print("USER PID NAME")
|
||||
print("root 1 init")
|
||||
if count("ps") < plan.get("ps_calls_with_leftover", 0):
|
||||
print("u0_a77 4242 libMobileGLServer.so")
|
||||
'''
|
||||
|
||||
FAKE_REPLAY = FAKE_COMMON + r'''
|
||||
import time
|
||||
args = sys.argv[1:]
|
||||
def value(flag):
|
||||
return args[args.index(flag) + 1]
|
||||
case, backend, root = value("--case"), value("--backend"), pathlib.Path(value("--result-root"))
|
||||
record("replay.log", {"case": case, "arguments": args, "environment": {
|
||||
key: os.environ.get(key) for key in (
|
||||
"MOBILEGL_ESPRYT_USE_ANGLE", "MOBILEGL_TRACE_ANGLE_VARIANT", "MOBILEGL_MAGMA_R11G11B10F_FALLBACK",
|
||||
"MOBILEGL_ITRP_FIX", "MOBILEGL_TRACE_SKIP_INSTALL", "MOBILEGL_TRACE_POLL_INTERVAL")}})
|
||||
(state / "replay.pid").write_text(str(os.getpid()))
|
||||
steps = scenario().get("replay", {}).get(case, [])
|
||||
attempt = count("replay-" + case)
|
||||
step = steps[attempt] if attempt < len(steps) else {}
|
||||
time.sleep(step.get("sleep", 0))
|
||||
directory = root / f"{case}-{backend}"
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
code = step.get("exit", 0)
|
||||
if code == 75:
|
||||
with open(root / "infrastructure-failure-reason.txt", "a") as reasons:
|
||||
reasons.write(step.get("reason", "device-unavailable") + "\n")
|
||||
if step.get("result", code in (0, 1)):
|
||||
(directory / "result.json").write_text(json.dumps(
|
||||
{"passed": step.get("passed", code == 0), "ssim": 0.999, "message": step.get("message", "retrace completed")}))
|
||||
transport = "inproc" if "--require-inproc" in args else "spawn" if "--require-spawn" in args else ""
|
||||
if transport and code == 0 and step.get("proof", True):
|
||||
(directory / "transport-proof.json").write_text(json.dumps({"passed": True, "required_transport": transport}))
|
||||
sys.exit(code)
|
||||
'''
|
||||
|
||||
FAKE_AVD = FAKE_COMMON + r'''
|
||||
args = sys.argv[1:]
|
||||
record("avd.log", args)
|
||||
if args[:1] == ["start"]:
|
||||
exits = scenario().get("avd_start_exits", [])
|
||||
index = count("avd-start")
|
||||
sys.exit(exits[index] if index < len(exits) else 0)
|
||||
'''
|
||||
|
||||
|
||||
def case(name, **extra):
|
||||
base = {"name": name, "trace_archive": f"fixtures/{name}.tgz", "trace_file": "trace.trace",
|
||||
"golden": f"fixtures/{name}.png", "target_call": 1234, "width": 854, "height": 480,
|
||||
"ssim_threshold": 0.99, "crop_x": 0, "crop_y": 0, "crop_width": 0, "crop_height": 0,
|
||||
"timeout_seconds": 900}
|
||||
base.update(extra)
|
||||
return base
|
||||
|
||||
|
||||
class ShardTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temporary.cleanup)
|
||||
self.root = pathlib.Path(self.temporary.name)
|
||||
self.state = self.root / "state"
|
||||
self.state.mkdir()
|
||||
self.results = self.root / "results"
|
||||
(self.root / "fixtures").mkdir()
|
||||
self.apk = self.root / "trace.apk"
|
||||
self.apk.write_bytes(b"apk")
|
||||
self.emulator_log = self.root / "emulator.log"
|
||||
self.emulator_log.write_text("boot\n")
|
||||
python = pathlib.Path(sys.executable).as_posix()
|
||||
commands = {}
|
||||
for name, source in (("adb", FAKE_ADB), ("replay", FAKE_REPLAY), ("avd", FAKE_AVD)):
|
||||
script = self.root / f"fake_{name}.py"
|
||||
script.write_text(source)
|
||||
commands[name] = f'"{python}" "{script.as_posix()}"'
|
||||
self.environment = {
|
||||
"FAKE_STATE": str(self.state), "ADB": commands["adb"],
|
||||
"MOBILEGL_SHARD_REPLAY_COMMAND": commands["replay"], "MOBILEGL_SHARD_AVD_COMMAND": commands["avd"],
|
||||
"MOBILEGL_GPU_HANG_BUDGET_MS": "0", "AVD_NAME": "mobilegl-ci",
|
||||
"EMULATOR_LOG": str(self.emulator_log), "EMULATOR_PID_FILE": str(self.root / "emulator.pid"),
|
||||
"GITHUB_STEP_SUMMARY": str(self.root / "summary.md"),
|
||||
}
|
||||
|
||||
def fixture(self, name):
|
||||
path = self.root / "fixtures" / name
|
||||
path.write_bytes(b"png")
|
||||
return str(path)
|
||||
|
||||
def run_shard(self, cases, *extra, backend="DirectVulkan", transport="monolith", scenario=None, env=None):
|
||||
if scenario is not None:
|
||||
(self.state / "scenario.json").write_text(json.dumps(scenario))
|
||||
arguments = ["--apk-file", str(self.apk), "--backend", backend, "--gpu", "lavapipe",
|
||||
"--transport", transport, "--lane", f"{transport}-lane", "--cases-json", json.dumps(cases),
|
||||
"--result-root", str(self.results), "--fixture-root", str(self.root / "work"), *extra]
|
||||
output = io.StringIO()
|
||||
with mock.patch.dict(os.environ, {**self.environment, **(env or {})}), \
|
||||
mock.patch.object(shard, "pause", lambda seconds: None), \
|
||||
contextlib.redirect_stdout(output):
|
||||
status = shard.main(arguments)
|
||||
return status, output.getvalue()
|
||||
|
||||
def log(self, name):
|
||||
path = self.state / name
|
||||
return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else []
|
||||
|
||||
def status(self, name, backend="DirectVulkan"):
|
||||
return json.loads((self.results / f"{name}-{backend}" / "case-status.json").read_text())
|
||||
|
||||
def shard_status(self):
|
||||
return json.loads((self.results / "shard-status.json").read_text())
|
||||
|
||||
# ---- what each case is run with ----------------------------------------------------
|
||||
|
||||
def planned(self, cases, backend, transport):
|
||||
status, output = self.run_shard(cases, "--dry-run", backend=backend, transport=transport)
|
||||
self.assertEqual(status, 0)
|
||||
return [json.loads(line) for line in output.splitlines()]
|
||||
|
||||
def test_plain_case_arguments_per_transport(self):
|
||||
cases = [case("plain", alternate_golden="fixtures/plain-mali.png")]
|
||||
head = ["--apk-file", str(self.apk), "--package", "top.mobilegl.plugin.trace", "--backend", "DirectVulkan",
|
||||
"--result-root", str(self.results), "--fixture-root", str(self.root / "work"), "--case", "plain",
|
||||
"--trace-archive", "fixtures/plain.tgz", "--trace-file", "trace.trace",
|
||||
"--golden", "fixtures/plain.png", "--alternate-golden", "fixtures/plain-mali.png",
|
||||
"--target-call", "1234", "--width", "854", "--height", "480", "--ssim-threshold", "0.99",
|
||||
"--crop-x", "0", "--crop-y", "0", "--crop-width", "0", "--crop-height", "0",
|
||||
"--timeout-seconds", "900"]
|
||||
tails = {
|
||||
"monolith": ["--env", "MOBILEGL_TRANSPORT=monolith;MOBILEGL_GPU_HANG_BUDGET_MS=0"],
|
||||
"inproc": ["--env", f"MOBILEGL_TRANSPORT=inproc;{SPLIT};MOBILEGL_GPU_HANG_BUDGET_MS=0", "--require-inproc"],
|
||||
"spawn": ["--env", f"MOBILEGL_TRANSPORT=spawn;{SPLIT};MOBILEGL_GPU_HANG_BUDGET_MS=0",
|
||||
"--use-pbuffer", "--require-spawn"],
|
||||
}
|
||||
for transport, tail in tails.items():
|
||||
plan, = self.planned(cases, "DirectVulkan", transport)
|
||||
self.assertEqual(plan["arguments"], head + tail, transport)
|
||||
self.assertEqual(plan["timeout"], 1200)
|
||||
self.assertEqual(plan["environment"], {
|
||||
"MOBILEGL_ESPRYT_USE_ANGLE": "0", "MOBILEGL_TRACE_ANGLE_VARIANT": "ec889e6ea831",
|
||||
"MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "1", "MOBILEGL_ITRP_FIX": "Auto"})
|
||||
|
||||
def test_case_specific_settings(self):
|
||||
cases = [case(BLISS), case(ITERATIONRP), case("flush", coherent_as_flush=True),
|
||||
case("lod", avoid_angle_llvmpipe_explicit_lod_bias=True), case("plain", ssim_threshold=0.985)]
|
||||
gles = {plan["case"]: plan for plan in self.planned(cases, "DirectGLES", "monolith")}
|
||||
vulkan = {plan["case"]: plan for plan in self.planned(cases, "DirectVulkan", "monolith")}
|
||||
|
||||
def flags(plan):
|
||||
return [argument for argument in plan["arguments"][plan["arguments"].index("--env") + 2:]]
|
||||
|
||||
self.assertEqual(gles[BLISS]["environment"]["MOBILEGL_TRACE_ANGLE_VARIANT"], "90a62123d794")
|
||||
self.assertEqual(vulkan[BLISS]["environment"]["MOBILEGL_TRACE_ANGLE_VARIANT"], "90a62123d794")
|
||||
self.assertEqual(gles["plain"]["environment"], {
|
||||
"MOBILEGL_ESPRYT_USE_ANGLE": "1", "MOBILEGL_TRACE_ANGLE_VARIANT": "ec889e6ea831",
|
||||
"MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "0", "MOBILEGL_ITRP_FIX": "Auto"})
|
||||
self.assertEqual(gles[ITERATIONRP]["environment"]["MOBILEGL_ITRP_FIX"], "ForceOn")
|
||||
self.assertEqual(flags(gles[BLISS]), ["--avoid-angle-llvmpipe-sampler-mipmap-min-filter"])
|
||||
self.assertEqual(flags(vulkan[BLISS]), [])
|
||||
self.assertEqual(flags(gles["lod"]), ["--avoid-angle-llvmpipe-explicit-lod-bias"])
|
||||
self.assertEqual(flags(vulkan["lod"]), [])
|
||||
self.assertEqual(flags(gles["flush"]), ["--coherent-as-flush"])
|
||||
self.assertEqual(flags(vulkan["flush"]), ["--coherent-as-flush"])
|
||||
self.assertEqual(flags(gles["plain"]), [])
|
||||
arguments = gles["plain"]["arguments"]
|
||||
self.assertEqual(arguments[arguments.index("--ssim-threshold") + 1], "0.985")
|
||||
self.assertEqual(arguments[arguments.index("--alternate-golden") + 1], "")
|
||||
|
||||
def test_misconfigured_case_lists_run_nothing(self):
|
||||
for cases in ([case("a"), case("a")], [case("a", ci_backends=["DirectGLES"])], [{"name": "a"}], []):
|
||||
status, output = self.run_shard(cases)
|
||||
self.assertEqual(status, 2, cases)
|
||||
self.assertIn("::error title=Android retrace shard misconfigured::", output)
|
||||
self.assertEqual(self.log("adb.log"), [])
|
||||
|
||||
# ---- the loop ----------------------------------------------------------------------
|
||||
|
||||
def test_every_case_runs_in_order_on_one_install(self):
|
||||
cases = [case("a", golden=self.fixture("a.png"), alternate_golden=self.fixture("a-mali.png")),
|
||||
case("b", golden=self.fixture("b.png")), case("c")]
|
||||
status, output = self.run_shard(cases, transport="inproc")
|
||||
self.assertEqual(status, 0, output)
|
||||
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b", "c"])
|
||||
for entry in self.log("replay.log"):
|
||||
self.assertEqual(entry["environment"]["MOBILEGL_TRACE_SKIP_INSTALL"], "1")
|
||||
self.assertEqual(entry["environment"]["MOBILEGL_TRACE_POLL_INTERVAL"], "0.5")
|
||||
adb = self.log("adb.log")
|
||||
self.assertEqual(sum(1 for call in adb if call[:1] == ["install"]), 1)
|
||||
self.assertEqual(sum(1 for call in adb if call[:3] == ["shell", "pm", "clear"]), 3)
|
||||
self.assertEqual(sum(1 for call in adb if call[:3] == ["shell", "rm", "-f"]), 3)
|
||||
self.assertIn(["shell", "rm", "-f", "/data/local/tmp/mobilegl-b.trace", "/data/local/tmp/mobilegl-b.golden.png",
|
||||
"/data/local/tmp/mobilegl-b.alternate-golden.png"], adb)
|
||||
self.assertEqual(self.log("avd.log"), [])
|
||||
directory = self.results / "a-DirectVulkan"
|
||||
self.assertEqual(json.loads((directory / "lane.json").read_text()),
|
||||
{"lane": "inproc-lane", "transport": "inproc"})
|
||||
self.assertTrue((directory / "a-DirectVulkan-golden.png").is_file())
|
||||
self.assertTrue((directory / "a-DirectVulkan-alternate-golden.png").is_file())
|
||||
self.assertFalse((self.results / "c-DirectVulkan" / "c-DirectVulkan-golden.png").exists())
|
||||
whole = self.shard_status()
|
||||
self.assertTrue(whole["complete"])
|
||||
self.assertEqual([(entry["case"], entry["status"]) for entry in whole["cases"]],
|
||||
[("a", "passed"), ("b", "passed"), ("c", "passed")])
|
||||
self.assertIn("| b | passed | 1 |", (self.root / "summary.md").read_text())
|
||||
|
||||
def test_a_failed_case_does_not_stop_the_shard(self):
|
||||
status, output = self.run_shard([case("a"), case("b"), case("c")], scenario={
|
||||
"replay": {"b": [{"exit": 1, "message": "ssim=0.5"}]}})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b", "c"])
|
||||
self.assertEqual([self.status(name)["status"] for name in "abc"], ["passed", "failed", "passed"])
|
||||
self.assertIn("::error title=Android retrace failed (DirectVulkan / monolith)::b: exit 1: ssim=0.5", output)
|
||||
self.assertEqual(output.count("::error"), 1)
|
||||
self.assertTrue(self.shard_status()["complete"])
|
||||
self.assertTrue((self.results / "b-DirectVulkan" / "emulator.log").is_file())
|
||||
|
||||
def test_the_case_after_one_that_did_not_pass_starts_on_a_cold_boot(self):
|
||||
status, _ = self.run_shard([case("a"), case("b"), case("c"), case("d")], scenario={
|
||||
"replay": {"a": [{"exit": 75}], "b": [{"exit": 1}]}})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual([self.status(name)["status"] for name in "abcd"], ["passed", "failed", "passed", "passed"])
|
||||
# One restart for a's infrastructure retry and one before c. A case that passed on
|
||||
# its retry costs the next case nothing.
|
||||
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start", "stop", "start"])
|
||||
self.assertEqual(self.status("b")["notes"], [])
|
||||
self.assertEqual(self.status("c")["notes"], ["emulator restarted before the case: "
|
||||
"the case before it, b, did not pass"])
|
||||
self.assertEqual(self.status("d")["notes"], [])
|
||||
self.assertEqual(sum(1 for call in self.log("adb.log") if call[:1] == ["install"]), 3)
|
||||
|
||||
def test_a_failed_case_without_an_avd_to_restart_does_not_stop_the_shard(self):
|
||||
status, _ = self.run_shard([case("a"), case("b")], scenario={"replay": {"a": [{"exit": 1}]}},
|
||||
env={"AVD_NAME": ""})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual([self.status(name)["status"] for name in "ab"], ["failed", "passed"])
|
||||
self.assertEqual(self.log("avd.log"), [])
|
||||
|
||||
def test_an_exit_0_without_a_passed_result_or_proof_fails(self):
|
||||
status, _ = self.run_shard([case("a"), case("b")], transport="spawn", scenario={
|
||||
"replay": {"a": [{"exit": 0, "result": False}], "b": [{"exit": 0, "proof": False}]}})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertIn("without a passed result.json", self.status("a")["reason"])
|
||||
self.assertIn("without a passed spawn transport proof", self.status("b")["reason"])
|
||||
|
||||
def test_an_infrastructure_failure_restarts_and_retries_once(self):
|
||||
status, output = self.run_shard([case("a"), case("b")], scenario={
|
||||
"replay": {"a": [{"exit": 75, "reason": "angle-surface-lost", "result": True, "passed": False}]}})
|
||||
self.assertEqual(status, 0, output)
|
||||
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "a", "b"])
|
||||
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start"])
|
||||
self.assertEqual(self.log("avd.log")[1], [
|
||||
"start", "--avd-name", "mobilegl-ci", "--emulator-log", str(self.emulator_log),
|
||||
"--pid-file", str(self.root / "emulator.pid"), "--gpu", "lavapipe", "--boot-timeout", "300"])
|
||||
self.assertIn(["kill-server"], self.log("adb.log"))
|
||||
self.assertEqual(sum(1 for call in self.log("adb.log") if call[:1] == ["install"]), 2)
|
||||
record = self.status("a")
|
||||
self.assertEqual((record["status"], record["attempts"], record["exit_codes"]), ("passed", 2, [75, 0]))
|
||||
self.assertEqual(record["surface_lost_retries"], 1)
|
||||
kept = self.results / "a-DirectVulkan" / "first-attempt"
|
||||
self.assertTrue((kept / "result.first-attempt.json").is_file())
|
||||
self.assertFalse((kept / "result.json").exists())
|
||||
self.assertIn("::warning::Android emulator infrastructure failed", output)
|
||||
self.assertIn("surface-lost retries: 1 (DirectVulkan, a)", (self.root / "summary.md").read_text())
|
||||
self.assertTrue((self.results / "diagnostics" / "emulator-boot-1-ended.log").is_file())
|
||||
self.assertEqual(self.shard_status()["emulator_boots"], 2)
|
||||
|
||||
def test_a_second_infrastructure_failure_is_the_verdict(self):
|
||||
status, _ = self.run_shard([case("a"), case("b")], scenario={
|
||||
"replay": {"a": [{"exit": 75}, {"exit": 75}]}})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual(self.status("a")["exit_codes"], [75, 75])
|
||||
self.assertEqual(self.status("a")["status"], "failed")
|
||||
self.assertEqual(self.status("b")["status"], "passed")
|
||||
# The surface-lost count is this case's own: the reasons of earlier cases do not leak in.
|
||||
self.assertEqual(self.status("b")["surface_lost_retries"], 0)
|
||||
|
||||
def test_the_deadline_cuts_a_running_case_and_names_the_rest(self):
|
||||
status, output = self.run_shard([case("a"), case("b"), case("c")], "--deadline-seconds", "5", scenario={
|
||||
"replay": {"b": [{"sleep": 30}]}})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b"])
|
||||
# b never had its own 1200 s, so the cut is not charged to it as a failure.
|
||||
self.assertEqual([self.status(name)["status"] for name in "abc"], ["passed", "not-run", "not-run"])
|
||||
self.assertRegex(self.status("b")["reason"], r"^cut by the shard deadline after \d+ s of a 1200 s budget$")
|
||||
self.assertEqual(self.status("b")["exit_codes"], [124])
|
||||
self.assertIn("deadline passed before this case started", self.status("c")["reason"])
|
||||
self.assertIn("::error title=Android retrace not-run (DirectVulkan / monolith)::b: cut by the shard deadline",
|
||||
output)
|
||||
self.assertIn("::error title=Android retrace shard incomplete (DirectVulkan / monolith)::"
|
||||
"1 verdict(s) for 3 case(s)", output)
|
||||
self.assertFalse(self.shard_status()["complete"])
|
||||
# Nothing is booted for a case the deadline keeps from starting.
|
||||
self.assertEqual(self.log("avd.log"), [])
|
||||
|
||||
def test_a_case_that_runs_out_of_its_own_budget_fails(self):
|
||||
# The deadline is far away, so the one second a has is its own budget.
|
||||
with mock.patch.object(shard, "REPLAY_GRACE_SECONDS", 0):
|
||||
status, _ = self.run_shard([case("a", timeout_seconds=1), case("b")], "--deadline-seconds", "600",
|
||||
scenario={"replay": {"a": [{"sleep": 30}]}})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual([self.status(name)["status"] for name in "ab"], ["failed", "passed"])
|
||||
self.assertEqual(self.status("a")["reason"], "the replay script exceeded its time budget")
|
||||
self.assertTrue(self.shard_status()["complete"])
|
||||
|
||||
def test_an_infrastructure_retry_is_not_started_after_the_deadline(self):
|
||||
def restart_past_the_deadline(runner, reason):
|
||||
runner.started -= 1000
|
||||
return True
|
||||
|
||||
with mock.patch.object(shard.Shard, "restart_emulator", restart_past_the_deadline):
|
||||
status, _ = self.run_shard([case("a"), case("b")], "--deadline-seconds", "600",
|
||||
scenario={"replay": {"a": [{"exit": 75}]}})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a"])
|
||||
self.assertEqual([self.status(name)["status"] for name in "ab"], ["not-run", "not-run"])
|
||||
self.assertEqual(self.status("a")["reason"],
|
||||
"the shard deadline passed before the infrastructure retry could start")
|
||||
self.assertEqual(self.status("a")["exit_codes"], [75])
|
||||
|
||||
# ---- the time a shard needs --------------------------------------------------------
|
||||
|
||||
def test_needed_seconds_lets_every_case_use_its_budget(self):
|
||||
cases = [case("a", timeout_seconds=1800), case("b", timeout_seconds=900), case("c", timeout_seconds="180")]
|
||||
# Every case to the end of its budget, the longest once more, and the overhead per case.
|
||||
self.assertEqual(shard.needed_seconds(cases), (2100 + 1200 + 480) + 2100 + 3 * 180)
|
||||
self.assertEqual(shard.needed_seconds(cases[2:]), 480 + 480 + 180)
|
||||
|
||||
def test_a_deadline_of_needed_seconds_gives_every_hanging_case_its_verdict(self):
|
||||
cases = [case(name, timeout_seconds=1) for name in "abc"]
|
||||
# Scaled down: one second per case, and ten for what the fakes take around it.
|
||||
with mock.patch.object(shard, "REPLAY_GRACE_SECONDS", 0), \
|
||||
mock.patch.object(shard, "CASE_OVERHEAD_SECONDS", 10):
|
||||
deadline = shard.needed_seconds(cases)
|
||||
self.assertEqual(deadline, 3 + 1 + 30)
|
||||
status, _ = self.run_shard(cases, "--deadline-seconds", str(deadline), scenario={
|
||||
"replay": {name: [{"sleep": 60}] for name in "abc"}})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b", "c"])
|
||||
for name in "abc":
|
||||
self.assertEqual((self.status(name)["status"], self.status(name)["reason"]),
|
||||
("failed", "the replay script exceeded its time budget"))
|
||||
self.assertTrue(self.shard_status()["complete"])
|
||||
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start", "stop", "start"])
|
||||
|
||||
def test_every_shard_gets_its_own_deadline_and_job_timeout(self):
|
||||
matrix = {"include": [
|
||||
{"backend": {"name": "DirectGLES", "gpu": "software"}, "lane": "spawn-acceptance", "transport": "spawn",
|
||||
"shard": 1, "shards": 2, "names": ["a", "b"],
|
||||
"cases": [case("a", timeout_seconds=1800), case("b", timeout_seconds=1800)]},
|
||||
{"backend": {"name": "DirectVulkan", "gpu": "lavapipe"}, "shard": 2, "shards": 2, "names": ["c"],
|
||||
"cases": [case("c", timeout_seconds=180)]}]}
|
||||
matrix_file = self.root / "matrix.json"
|
||||
matrix_file.write_text(json.dumps(matrix))
|
||||
|
||||
def limits(cap, path=matrix_file):
|
||||
output, report = io.StringIO(), io.StringIO()
|
||||
with contextlib.redirect_stdout(output), contextlib.redirect_stderr(report):
|
||||
status = budget.main(["--cap-seconds", str(cap), "--matrix-file", str(path)])
|
||||
return status, output.getvalue(), report.getvalue()
|
||||
|
||||
self.assertEqual(budget.shard_needs(matrix), [("DirectGLES / spawn, shard 1/2", 2, 6660),
|
||||
("DirectVulkan / monolith, shard 2/2", 1, 1140)])
|
||||
# Under the cap every shard gets exactly what its cases need.
|
||||
status, output, report = limits(6660)
|
||||
self.assertEqual(status, 0, report)
|
||||
rows = json.loads(output)["include"]
|
||||
self.assertEqual([(row["deadline_seconds"], row["timeout_minutes"]) for row in rows],
|
||||
[(6660, 111 + budget.JOB_OVERHEAD_MINUTES), (1140, 19 + budget.JOB_OVERHEAD_MINUTES)])
|
||||
# Nothing but the two fields is added, and no case is dropped or changed.
|
||||
self.assertEqual([{key: value for key, value in row.items()
|
||||
if key not in ("deadline_seconds", "timeout_minutes")} for row in rows],
|
||||
matrix["include"])
|
||||
self.assertIn("2 shard(s), 0 capped at 6660 s; the most a shard needs is 6660 s", report)
|
||||
# Over the cap a shard is cut to it and named; the other shard is untouched.
|
||||
status, output, report = limits(6000)
|
||||
self.assertEqual(status, 0, report)
|
||||
rows = json.loads(output)["include"]
|
||||
self.assertEqual([(row["deadline_seconds"], row["timeout_minutes"]) for row in rows],
|
||||
[(6000, 100 + budget.JOB_OVERHEAD_MINUTES), (1140, 19 + budget.JOB_OVERHEAD_MINUTES)])
|
||||
self.assertIn("DirectGLES / spawn, shard 1/2 would need 6660 s for all of its 2 case(s)", report)
|
||||
self.assertNotIn("DirectVulkan / monolith, shard 2/2 would need", report)
|
||||
# A cap that could not fit in a hosted job, and a matrix that cannot be read.
|
||||
self.assertEqual(limits(0)[0], 2)
|
||||
self.assertEqual(limits(360 * 60)[0], 2)
|
||||
for broken in ({"include": []}, {"include": [{"names": ["a"]}]}, []):
|
||||
matrix_file.write_text(json.dumps(broken))
|
||||
status, output, _ = limits(7200)
|
||||
self.assertEqual((status, output), (2, ""), broken)
|
||||
|
||||
def test_the_workflow_takes_its_limits_from_the_matrix(self):
|
||||
text = (HERE.parent / ".github" / "workflows" / "apk.yml").read_text(encoding="utf-8")
|
||||
retrace = text[text.index("\n retrace:"):text.index("\n retrace-summary:")]
|
||||
cap, = re.findall(r'^\s+RETRACE_SHARD_DEADLINE_CAP_SECONDS: "(\d+)"\s*$', text, flags=re.MULTILINE)
|
||||
# The runner's deadline and the job's limit are the two fields the budget script adds.
|
||||
self.assertIn('--deadline-seconds "${{ matrix.deadline_seconds }}"', retrace)
|
||||
self.assertEqual(re.findall(r"^ timeout-minutes: (.+?)\s*$", retrace, flags=re.MULTILINE),
|
||||
["${{ matrix.timeout_minutes }}"])
|
||||
self.assertIn('--cap-seconds "${RETRACE_SHARD_DEADLINE_CAP_SECONDS}"', text)
|
||||
# The capped deadline and the job's overhead stay inside the six hours a hosted job may run.
|
||||
self.assertLessEqual(-(-int(cap) // 60) + budget.JOB_OVERHEAD_MINUTES, budget.HOSTED_JOB_LIMIT_MINUTES)
|
||||
|
||||
def test_an_unhealthy_emulator_is_restarted_before_the_next_case(self):
|
||||
status, output = self.run_shard([case("a"), case("b")], scenario={"offline_get_state_calls": 1})
|
||||
self.assertEqual(status, 0, output)
|
||||
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start"])
|
||||
self.assertIn("emulator restarted before the case: adb state is offline", self.status("a")["notes"])
|
||||
self.assertEqual(self.status("b")["notes"], [])
|
||||
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a", "b"])
|
||||
|
||||
def test_the_restart_switch_cold_boots_before_every_case_but_the_first(self):
|
||||
status, _ = self.run_shard([case("a"), case("b"), case("c")],
|
||||
env={"MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE": "1"})
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start", "stop", "start"])
|
||||
self.assertEqual(sum(1 for call in self.log("adb.log") if call[:1] == ["install"]), 3)
|
||||
|
||||
def test_an_emulator_that_does_not_come_back_leaves_named_rows(self):
|
||||
status, output = self.run_shard([case("a"), case("b"), case("c")], scenario={
|
||||
"replay": {"a": [{"exit": 75}]}, "avd_start_exits": [2, 2]})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual([entry["case"] for entry in self.log("replay.log")], ["a"])
|
||||
self.assertEqual([self.status(name)["status"] for name in "abc"], ["failed", "not-run", "not-run"])
|
||||
self.assertEqual(output.count("::error title=Android retrace"), 4)
|
||||
for name in "abc":
|
||||
self.assertTrue((self.results / f"{name}-DirectVulkan" / "lane.json").is_file())
|
||||
|
||||
def test_an_apk_that_does_not_install_runs_nothing(self):
|
||||
status, _ = self.run_shard([case("a"), case("b")], scenario={"failing_installs": 9})
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual(self.log("replay.log"), [])
|
||||
self.assertEqual([self.status(name)["status"] for name in "ab"], ["not-run", "not-run"])
|
||||
|
||||
def test_processes_left_after_force_stop_are_killed_and_recorded(self):
|
||||
# Listed on all four looks after the force-stop, gone after the kill.
|
||||
status, _ = self.run_shard([case("a"), case("b")], scenario={"ps_calls_with_leftover": 4})
|
||||
self.assertEqual(status, 0)
|
||||
self.assertIn(["shell", "run-as", "top.mobilegl.plugin.trace", "kill", "-9", "4242"], self.log("adb.log"))
|
||||
self.assertIn("processes left after force-stop: 4242 libMobileGLServer.so", self.status("a")["notes"])
|
||||
self.assertEqual(self.status("b")["notes"], [])
|
||||
self.assertEqual(self.log("avd.log"), [])
|
||||
|
||||
def test_a_process_that_is_still_exiting_is_not_reported(self):
|
||||
status, _ = self.run_shard([case("a")], scenario={"ps_calls_with_leftover": 2})
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(self.status("a")["notes"], [])
|
||||
self.assertNotIn("kill", [call[3] for call in self.log("adb.log") if call[:2] == ["shell", "run-as"]])
|
||||
|
||||
def test_processes_that_survive_the_kill_cost_an_emulator_restart(self):
|
||||
status, _ = self.run_shard([case("a")], scenario={"ps_calls_with_leftover": 10})
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual([call[0] for call in self.log("avd.log")], ["stop", "start"])
|
||||
self.assertIn("the emulator was restarted to remove them", self.status("a")["notes"])
|
||||
|
||||
def test_a_failed_pm_clear_falls_back_to_a_reinstall(self):
|
||||
status, _ = self.run_shard([case("a")], scenario={"pm_clear_fails": True})
|
||||
self.assertEqual(status, 0)
|
||||
adb = self.log("adb.log")
|
||||
self.assertIn(["uninstall", "top.mobilegl.plugin.trace"], adb)
|
||||
self.assertEqual(sum(1 for call in adb if call[:1] == ["install"]), 2)
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "signals and process groups")
|
||||
def test_a_terminated_runner_takes_the_running_replay_with_it(self):
|
||||
cases = [case("a"), case("b")]
|
||||
(self.state / "scenario.json").write_text(json.dumps({"replay": {"a": [{"sleep": 60}]}}))
|
||||
arguments = ["--apk-file", str(self.apk), "--backend", "DirectVulkan", "--gpu", "lavapipe",
|
||||
"--transport", "monolith", "--lane", "monolith-lane", "--cases-json", json.dumps(cases),
|
||||
"--result-root", str(self.results), "--fixture-root", str(self.root / "work")]
|
||||
runner = subprocess.Popen([sys.executable, str(HERE / "trace_replay_shard.py")] + arguments,
|
||||
env={**os.environ, **self.environment}, stdout=subprocess.DEVNULL)
|
||||
deadline = time.monotonic() + 30
|
||||
while not (self.state / "replay.pid").exists() and time.monotonic() < deadline:
|
||||
time.sleep(0.05)
|
||||
replay_pid = int((self.state / "replay.pid").read_text())
|
||||
runner.send_signal(signal.SIGTERM)
|
||||
self.assertEqual(runner.wait(timeout=30), 128 + signal.SIGTERM)
|
||||
for _ in range(100):
|
||||
try:
|
||||
os.kill(replay_pid, 0)
|
||||
except OSError:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
else:
|
||||
self.fail("the replay outlived the runner")
|
||||
# The step that runs afterwards names both cases.
|
||||
status, _ = self.run_shard(cases, "--finalize")
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual([self.status(name)["status"] for name in "ab"], ["not-run", "not-run"])
|
||||
|
||||
def test_finalize_names_every_case_without_a_device(self):
|
||||
cases = [case("a", golden=self.fixture("a.png")), case("b")]
|
||||
status, _ = self.run_shard(cases)
|
||||
self.assertEqual(status, 0)
|
||||
(self.results / "b-DirectVulkan" / "case-status.json").unlink()
|
||||
calls_before = len(self.log("adb.log"))
|
||||
status, _ = self.run_shard(cases + [case("c")], "--finalize")
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(len(self.log("adb.log")), calls_before)
|
||||
self.assertEqual([self.status(name)["status"] for name in "abc"], ["passed", "not-run", "not-run"])
|
||||
self.assertTrue((self.results / "c-DirectVulkan" / "lane.json").is_file())
|
||||
self.assertFalse(self.shard_status()["complete"])
|
||||
self.assertEqual(self.shard_status()["expected_cases"], ["a", "b", "c"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -62,6 +62,9 @@ copies benchmark.json (per-frame times plus mean/median/p95) out of the app, and
|
||||
"passed" only means the replay reached the end of the trace without an error.
|
||||
Pass --reuse-fixture to skip re-extracting and re-pushing the trace, for repeat
|
||||
runs of a case whose fixture is already in /data/local/tmp.
|
||||
Set MOBILEGL_TRACE_POLL_INTERVAL to the seconds between result polls (for example
|
||||
0.5) to wait on a wall clock: the wait then ends --timeout-seconds plus a fifth after
|
||||
the app starts, which is about how long that many one-second polls take.
|
||||
Pass --env "K=V;K=V" (or set MOBILEGL_TRACE_ENV) to hand arbitrary environment
|
||||
variables to the replay process. They are applied last, immediately before
|
||||
libMobileGL.so is loaded, so they override every flag above; an entry with no "="
|
||||
@@ -93,6 +96,21 @@ adb_device_path() {
|
||||
MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL='*' "${ADB}" "$@"
|
||||
}
|
||||
|
||||
# Milliseconds since the epoch; whole seconds where date has no %N.
|
||||
now_ms() {
|
||||
stamp="$(date +%s%3N 2>/dev/null || true)"
|
||||
case "${stamp}" in
|
||||
''|*[!0-9]*) stamp="$(($(date +%s) * 1000))" ;;
|
||||
esac
|
||||
printf '%s' "${stamp}"
|
||||
}
|
||||
|
||||
# Appends "<phase> <ms>" to phase-timings.txt in the result directory, so a run records where
|
||||
# its wall time went: push, install, replay, log pulls.
|
||||
mark_phase() {
|
||||
printf '%s %s\n' "$1" "$(now_ms)" >> "${result_dir}/phase-timings.txt" 2>/dev/null || true
|
||||
}
|
||||
|
||||
next_arg() {
|
||||
if [ "$#" -lt 2 ]; then
|
||||
die "$1 requires a value"
|
||||
@@ -411,6 +429,81 @@ copy_fixture_to_app() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Waits for result.json with one poll per second; --timeout-seconds counts the polls.
|
||||
wait_for_result_by_count() {
|
||||
# One failed pidof is not an exit: under memory pressure the adb shell call
|
||||
# itself can fail while the app is alive (seen with the process still in
|
||||
# pidof and the activity still initializing right after the "exit").
|
||||
missed_app_polls=0
|
||||
for _ in $(seq 1 "${timeout_seconds}"); do
|
||||
if adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
if adb_device_path shell pidof "${package_name}" >/dev/null 2>&1; then
|
||||
saw_app_process=1
|
||||
missed_app_polls=0
|
||||
else
|
||||
poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
|
||||
if [ "${poll_adb_state}" != "device" ]; then
|
||||
printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt"
|
||||
break
|
||||
fi
|
||||
if [ "${saw_app_process}" -eq 1 ]; then
|
||||
missed_app_polls=$((missed_app_polls + 1))
|
||||
if [ "${missed_app_polls}" -ge 3 ]; then
|
||||
app_exited=1
|
||||
break
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
# The same wait on a wall clock, selected by MOBILEGL_TRACE_POLL_INTERVAL (seconds between
|
||||
# polls, fractions allowed). One adb call per poll asks for result.json and for the app
|
||||
# process. The budget is --timeout-seconds of wall time plus a fifth: a poll of the
|
||||
# one-second loop takes about 1.2 s with its two adb calls, so a replay gets the same time
|
||||
# to finish under either loop. An exit needs three missed polls spread over at least three
|
||||
# seconds, so a short interval keeps the one-second loop's tolerance for adb calls that fail
|
||||
# under memory pressure.
|
||||
wait_for_result_by_clock() {
|
||||
poll_deadline=$((poll_started + timeout_seconds + timeout_seconds / 5))
|
||||
missed_app_polls=0
|
||||
first_missed_at=0
|
||||
while [ "$(date +%s)" -lt "${poll_deadline}" ]; do
|
||||
poll_output="$(adb_device_path shell "run-as ${package_name} ls ${app_dir}/output/result.json >/dev/null 2>&1 && echo mobilegl-poll-result; pidof ${package_name} >/dev/null 2>&1 && echo mobilegl-poll-alive; echo mobilegl-poll-end" 2>/dev/null | tr -d '\r' || true)"
|
||||
case "${poll_output}" in
|
||||
*mobilegl-poll-result*)
|
||||
break
|
||||
;;
|
||||
*mobilegl-poll-alive*)
|
||||
saw_app_process=1
|
||||
missed_app_polls=0
|
||||
;;
|
||||
*)
|
||||
poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
|
||||
if [ "${poll_adb_state}" != "device" ]; then
|
||||
printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt"
|
||||
break
|
||||
fi
|
||||
if [ "${saw_app_process}" -eq 1 ]; then
|
||||
poll_now="$(date +%s)"
|
||||
if [ "${missed_app_polls}" -eq 0 ]; then
|
||||
first_missed_at="${poll_now}"
|
||||
fi
|
||||
missed_app_polls=$((missed_app_polls + 1))
|
||||
if [ "${missed_app_polls}" -ge 3 ] && [ "$((poll_now - first_missed_at))" -ge 3 ]; then
|
||||
app_exited=1
|
||||
break
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
sleep "${MOBILEGL_TRACE_POLL_INTERVAL}"
|
||||
done
|
||||
}
|
||||
|
||||
run_retrace() {
|
||||
result_dir="${result_root}/${safe_case}-${backend}"
|
||||
alternate_golden_app_path=""
|
||||
@@ -427,13 +520,17 @@ run_retrace() {
|
||||
fi
|
||||
|
||||
mkdir -p "${result_dir}"
|
||||
printf 'script-started %s\n' "${script_started_ms}" > "${result_dir}/phase-timings.txt" 2>/dev/null || true
|
||||
mark_phase fixture-pushed
|
||||
# A repeat run of a case the previous invocation already installed and pushed only needs
|
||||
# the on-device copy back into a fresh app output directory.
|
||||
# MOBILEGL_TRACE_SKIP_INSTALL=1: the caller installed the APK itself (ColorOS shows an install-confirmation
|
||||
# dialog whose tap drops adb for a few seconds, so the install has to be done and settled outside a run).
|
||||
# dialog whose tap drops adb for a few seconds, so the install has to be done and settled outside a run;
|
||||
# trace-replay-shard.sh installs once per emulator boot).
|
||||
if [ "${reuse_fixture}" -eq 0 ] && [ "${MOBILEGL_TRACE_SKIP_INSTALL:-0}" != "1" ]; then
|
||||
"${ADB}" install -r "$(host_path_for_adb "${apk_file}")"
|
||||
fi
|
||||
mark_phase apk-installed
|
||||
copy_fixture_to_app
|
||||
adb_device_path shell am force-stop "${package_name}"
|
||||
"${ADB}" logcat -c
|
||||
@@ -502,39 +599,20 @@ run_retrace() {
|
||||
--ei crop_width "${crop_width}" \
|
||||
--ei crop_height "${crop_height}"
|
||||
adb_device_path shell "$@"
|
||||
mark_phase app-started
|
||||
|
||||
app_exited=0
|
||||
saw_app_process=0
|
||||
# One failed pidof is not an exit: under memory pressure the adb shell call
|
||||
# itself can fail while the app is alive (seen with the process still in
|
||||
# pidof and the activity still initializing right after the "exit").
|
||||
missed_app_polls=0
|
||||
poll_started="$(date +%s)"
|
||||
for _ in $(seq 1 "${timeout_seconds}"); do
|
||||
if adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
if adb_device_path shell pidof "${package_name}" >/dev/null 2>&1; then
|
||||
saw_app_process=1
|
||||
missed_app_polls=0
|
||||
else
|
||||
poll_adb_state="$(adb_device_path get-state 2>/dev/null | tr -d '\r' || true)"
|
||||
if [ "${poll_adb_state}" != "device" ]; then
|
||||
printf '%s\n' "${poll_adb_state:-unknown}" > "${result_dir}/adb-disconnected.txt"
|
||||
break
|
||||
fi
|
||||
if [ "${saw_app_process}" -eq 1 ]; then
|
||||
missed_app_polls=$((missed_app_polls + 1))
|
||||
if [ "${missed_app_polls}" -ge 3 ]; then
|
||||
app_exited=1
|
||||
break
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ -n "${MOBILEGL_TRACE_POLL_INTERVAL:-}" ]; then
|
||||
wait_for_result_by_clock
|
||||
else
|
||||
wait_for_result_by_count
|
||||
fi
|
||||
mark_phase wait-ended
|
||||
|
||||
collect_run_diagnostics "${result_dir}"
|
||||
mark_phase diagnostics-collected
|
||||
if ! adb_device_path shell run-as "${package_name}" ls "${app_dir}/output/result.json" >/dev/null 2>&1; then
|
||||
if [ -f "${result_dir}/adb-disconnected.txt" ]; then
|
||||
echo "trace-replay-ci.sh: device disconnected while waiting for result.json" >&2
|
||||
@@ -574,6 +652,7 @@ run_retrace() {
|
||||
copy_app_artifact "${app_dir}/output/benchmark.json" "${result_dir}/benchmark.json"
|
||||
fi
|
||||
copy_texture_2d_dumps
|
||||
mark_phase artifacts-copied
|
||||
|
||||
# A replay that wrote result.json but did not pass used to print nothing but
|
||||
# the JSON, which for a non-zero statusCode says only "retrace failed with
|
||||
@@ -696,6 +775,7 @@ PY
|
||||
fi
|
||||
}
|
||||
|
||||
script_started_ms="$(now_ms)"
|
||||
mkdir -p "${fixture_root}" "${result_root}"
|
||||
if [ "${reuse_fixture}" -eq 0 ]; then
|
||||
prepare_fixture
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
# MobileGL - android-plugin/trace-replay-shard.sh
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
# Replays an ordered list of trace cases for one (lane, transport, backend) on one emulator
|
||||
# boot, each through trace-replay-ci.sh. The loop, the per-case settings and the verdict files
|
||||
# are in trace_replay_shard.py; `--help` lists the arguments.
|
||||
#
|
||||
# sh android-plugin/trace-replay-shard.sh \
|
||||
# --apk-file FILE --backend DirectGLES|DirectVulkan --gpu GPU \
|
||||
# --transport monolith|inproc|spawn --lane LANE \
|
||||
# --cases-env VARIABLE | --cases-file FILE | --cases-json JSON \
|
||||
# [--deadline-seconds N] [--dry-run | --finalize]
|
||||
#
|
||||
# Environment:
|
||||
# AVD_NAME, EMULATOR_LOG, EMULATOR_PID_FILE the emulator to restart when it stops answering
|
||||
# MOBILEGL_GPU_HANG_BUDGET_MS forwarded to every replay (required)
|
||||
# MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE 1 cold boots the emulator before every case
|
||||
# MOBILEGL_SHARD_POLL_INTERVAL seconds between result polls (default 0.5);
|
||||
# empty selects trace-replay-ci.sh's 1 s loop
|
||||
set -eu
|
||||
|
||||
exec "${PYTHON:-python3}" "$(dirname "$0")/trace_replay_shard.py" "$@"
|
||||
@@ -0,0 +1,792 @@
|
||||
#!/usr/bin/env python3
|
||||
# MobileGL - android-plugin/trace_replay_shard.py
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
"""Replay an ordered list of trace cases on one emulator boot.
|
||||
|
||||
One shard is one (lane, transport, backend). Each case runs through trace-replay-ci.sh with
|
||||
the per-case and per-transport settings of the APK workflow, in a fresh app process with
|
||||
cleared app data. A failing case does not stop the shard: the case after it starts on a cold
|
||||
boot, every case gets a verdict file, and the exit status is non-zero when a case failed or a
|
||||
verdict is missing.
|
||||
|
||||
Layout under --result-root (the directory the workflow uploads):
|
||||
|
||||
<case>-<backend>/ what trace-replay-ci.sh writes, plus lane.json, the golden
|
||||
copies the summary renders, and case-status.json
|
||||
<case>-<backend>/first-attempt/ files of an attempt that was retried
|
||||
shard-status.json every case of the shard with its verdict
|
||||
diagnostics/ emulator logs kept across restarts
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
INFRASTRUCTURE_FAILURE_EXIT_CODE = 75
|
||||
TIMEOUT_EXIT_CODE = 124
|
||||
BACKENDS = ("DirectGLES", "DirectVulkan")
|
||||
TRANSPORTS = ("monolith", "inproc", "spawn")
|
||||
PASSED, FAILED, NOT_RUN = "passed", "failed", "not-run"
|
||||
|
||||
# Cases with settings of their own. Bliss needs the newer packaged ANGLE build and, on
|
||||
# DirectGLES, sampler mipmap min-filter downgrading on ANGLE llvmpipe; iterationRP needs its
|
||||
# SPIR-V repairs forced on.
|
||||
BLISS_CASE = "minecraft-1.21.4-fabric-iris-bliss-in-world"
|
||||
ITERATIONRP_CASE = "minecraft-1.21.4-fabric-iris-iterationrp-in-world"
|
||||
DEFAULT_ANGLE_VARIANT = "ec889e6ea831"
|
||||
BLISS_ANGLE_VARIANT = "90a62123d794"
|
||||
|
||||
# The inproc and spawn lanes pass the same client settings, so they differ only in which
|
||||
# process runs the server role.
|
||||
SPLIT_CLIENT_SETTINGS = "MOBILEGL_IPC_ROLE_SPLIT_STATE=1;MOBILEGL_IPC_STRICT_ERRORS=1;MOBILEGL_IPC_RUN_AHEAD=1"
|
||||
|
||||
REQUIRED_CASE_KEYS = ("name", "trace_archive", "trace_file", "golden", "target_call", "width", "height",
|
||||
"crop_x", "crop_y", "crop_width", "crop_height", "timeout_seconds")
|
||||
# Seconds trace-replay-ci.sh gets beyond the case's own budget for install, push and log pulls.
|
||||
REPLAY_GRACE_SECONDS = 300
|
||||
BOOT_TIMEOUT_SECONDS = 300
|
||||
# Seconds a case may take around its replays: the app reset, the failure diagnostics and the
|
||||
# cold boot that follows a case that did not pass.
|
||||
CASE_OVERHEAD_SECONDS = 180
|
||||
|
||||
|
||||
def say(message):
|
||||
print(message, flush=True)
|
||||
|
||||
|
||||
def pause(seconds):
|
||||
time.sleep(seconds)
|
||||
|
||||
|
||||
def safe_case(name):
|
||||
return re.sub(r"[^A-Za-z0-9._-]", "_", name)
|
||||
|
||||
|
||||
def as_text(value):
|
||||
if value is True:
|
||||
return "true"
|
||||
if value is False:
|
||||
return "false"
|
||||
return "" if value is None else str(value)
|
||||
|
||||
|
||||
def is_true(value):
|
||||
return value is True or value == "true"
|
||||
|
||||
|
||||
def load_cases(text, backend):
|
||||
"""The case objects of the APK matrix, as text ready for the command line."""
|
||||
try:
|
||||
# Numbers stay the literal text of the matrix, so 0.985 is passed on as 0.985.
|
||||
cases = json.loads(text, parse_float=str, parse_int=str)
|
||||
except json.JSONDecodeError as error:
|
||||
raise ValueError(f"the case list is not JSON: {error}") from error
|
||||
if not isinstance(cases, list) or not cases:
|
||||
raise ValueError("the case list must be a non-empty JSON array of case objects")
|
||||
names = set()
|
||||
for index, case in enumerate(cases):
|
||||
if not isinstance(case, dict):
|
||||
raise ValueError(f"case {index} is not an object; pass the case objects of the APK matrix")
|
||||
missing = [key for key in REQUIRED_CASE_KEYS if case.get(key) in (None, "")]
|
||||
if missing:
|
||||
raise ValueError(f"case {case.get('name', index)} lacks {', '.join(missing)}")
|
||||
name = case["name"]
|
||||
if name in names:
|
||||
raise ValueError(f"case {name} is listed twice")
|
||||
names.add(name)
|
||||
backends = case.get("ci_backends")
|
||||
if backends is not None and backend not in backends:
|
||||
raise ValueError(f"case {name} does not run on {backend} (ci_backends: {backends})")
|
||||
return cases
|
||||
|
||||
|
||||
def needed_seconds(cases):
|
||||
"""Wall time in which every case of a shard can use its whole time budget.
|
||||
|
||||
It covers each case running until its replay script is stopped, one infrastructure retry
|
||||
that does so again for the longest budget, and CASE_OVERHEAD_SECONDS per case. A shard
|
||||
deadline at or above this never stops a case that is within its own budget, unless more
|
||||
than one case needs a retry as long as its first attempt.
|
||||
"""
|
||||
budgets = [int(as_text(case["timeout_seconds"])) + REPLAY_GRACE_SECONDS for case in cases]
|
||||
return sum(budgets) + max(budgets) + CASE_OVERHEAD_SECONDS * len(budgets)
|
||||
|
||||
|
||||
def replay_environment(backend, case_name):
|
||||
return {
|
||||
"MOBILEGL_ESPRYT_USE_ANGLE": "1" if backend == "DirectGLES" else "0",
|
||||
"MOBILEGL_TRACE_ANGLE_VARIANT": BLISS_ANGLE_VARIANT if case_name == BLISS_CASE else DEFAULT_ANGLE_VARIANT,
|
||||
"MOBILEGL_MAGMA_R11G11B10F_FALLBACK": "1" if backend == "DirectVulkan" else "0",
|
||||
"MOBILEGL_ITRP_FIX": "ForceOn" if case_name == ITERATIONRP_CASE else "Auto",
|
||||
}
|
||||
|
||||
|
||||
def transport_arguments(transport, hang_budget_ms):
|
||||
hang_budget = f"MOBILEGL_GPU_HANG_BUDGET_MS={hang_budget_ms}"
|
||||
if transport == "inproc":
|
||||
return ["--env", f"MOBILEGL_TRANSPORT=inproc;{SPLIT_CLIENT_SETTINGS};{hang_budget}", "--require-inproc"]
|
||||
if transport == "spawn":
|
||||
# No MOBILEGL_IPC_SERVER_PATH: only the app can spell nativeLibraryDir, and
|
||||
# TraceReplayActivity resolves it when the environment asks for spawn. The pbuffer is
|
||||
# required: a spawned server cannot be handed the client's window.
|
||||
return ["--env", f"MOBILEGL_TRANSPORT=spawn;{SPLIT_CLIENT_SETTINGS};{hang_budget}",
|
||||
"--use-pbuffer", "--require-spawn"]
|
||||
return ["--env", f"MOBILEGL_TRANSPORT=monolith;{hang_budget}"]
|
||||
|
||||
|
||||
def replay_arguments(options, case, hang_budget_ms):
|
||||
name = case["name"]
|
||||
arguments = [
|
||||
"--apk-file", options.apk_file,
|
||||
"--package", options.package,
|
||||
"--backend", options.backend,
|
||||
"--result-root", options.result_root,
|
||||
"--fixture-root", options.fixture_root,
|
||||
"--case", name,
|
||||
"--trace-archive", as_text(case["trace_archive"]),
|
||||
"--trace-file", as_text(case["trace_file"]),
|
||||
"--golden", as_text(case["golden"]),
|
||||
"--alternate-golden", as_text(case.get("alternate_golden") or ""),
|
||||
"--target-call", as_text(case["target_call"]),
|
||||
"--width", as_text(case["width"]),
|
||||
"--height", as_text(case["height"]),
|
||||
"--ssim-threshold", as_text(case.get("ssim_threshold") or "0.99"),
|
||||
"--crop-x", as_text(case["crop_x"]),
|
||||
"--crop-y", as_text(case["crop_y"]),
|
||||
"--crop-width", as_text(case["crop_width"]),
|
||||
"--crop-height", as_text(case["crop_height"]),
|
||||
"--timeout-seconds", as_text(case["timeout_seconds"]),
|
||||
]
|
||||
arguments += transport_arguments(options.transport, hang_budget_ms)
|
||||
if options.backend == "DirectGLES" and name == BLISS_CASE:
|
||||
arguments.append("--avoid-angle-llvmpipe-sampler-mipmap-min-filter")
|
||||
if options.backend == "DirectGLES" and is_true(case.get("avoid_angle_llvmpipe_explicit_lod_bias")):
|
||||
arguments.append("--avoid-angle-llvmpipe-explicit-lod-bias")
|
||||
if is_true(case.get("coherent_as_flush")):
|
||||
arguments.append("--coherent-as-flush")
|
||||
return arguments
|
||||
|
||||
|
||||
def read_json(path):
|
||||
try:
|
||||
return json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def write_json(path, value):
|
||||
Path(path).write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def one_line(text):
|
||||
return " ".join(str(text).split())
|
||||
|
||||
|
||||
class Shard:
|
||||
def __init__(self, options, cases, environ):
|
||||
self.options = options
|
||||
self.cases = cases
|
||||
self.environ = dict(environ)
|
||||
self.adb_command = shlex.split(environ.get("ADB", "adb"))
|
||||
directory = Path(__file__).resolve().parent
|
||||
self.replay_command = shlex.split(environ.get("MOBILEGL_SHARD_REPLAY_COMMAND") or
|
||||
f"sh {shlex.quote(str(directory / 'trace-replay-ci.sh'))}")
|
||||
self.avd_command = shlex.split(environ.get("MOBILEGL_SHARD_AVD_COMMAND") or
|
||||
f"sh {shlex.quote(str(directory / 'run-avd-ci.sh'))}")
|
||||
self.hang_budget_ms = environ.get("MOBILEGL_GPU_HANG_BUDGET_MS")
|
||||
self.restart_each_case = environ.get("MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE", "0") == "1"
|
||||
self.poll_interval = environ.get("MOBILEGL_SHARD_POLL_INTERVAL", "0.5")
|
||||
self.avd_name = environ.get("AVD_NAME", "")
|
||||
self.emulator_log = environ.get("EMULATOR_LOG", "")
|
||||
self.emulator_pid_file = environ.get("EMULATOR_PID_FILE", "")
|
||||
self.result_root = Path(options.result_root)
|
||||
self.diagnostics = self.result_root / "diagnostics"
|
||||
self.started = time.monotonic()
|
||||
self.boots = 1
|
||||
self.install_seconds = []
|
||||
self.records = {}
|
||||
self.aborted = ""
|
||||
# The case before the current one when it did not pass; the next case gets a cold boot.
|
||||
self.unclean_case = ""
|
||||
|
||||
# ---- bookkeeping -------------------------------------------------------------------
|
||||
|
||||
def result_dir(self, case):
|
||||
return self.result_root / f"{safe_case(case['name'])}-{self.options.backend}"
|
||||
|
||||
def record_for(self, case, status, reason, **extra):
|
||||
record = {"case": case["name"], "backend": self.options.backend, "transport": self.options.transport,
|
||||
"lane": self.options.lane, "status": status, "reason": reason}
|
||||
record.update(extra)
|
||||
return record
|
||||
|
||||
def write_summary_inputs(self, case):
|
||||
"""lane.json and the golden copies the retrace summary reads next to each result."""
|
||||
directory = self.result_dir(case)
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
write_json(directory / "lane.json", {"lane": self.options.lane, "transport": self.options.transport})
|
||||
stem = f"{safe_case(case['name'])}-{self.options.backend}"
|
||||
for key, suffix in (("golden", "golden"), ("alternate_golden", "alternate-golden")):
|
||||
source = as_text(case.get(key) or "")
|
||||
if source and os.path.isfile(source) and os.path.getsize(source) > 0:
|
||||
shutil.copyfile(source, directory / f"{stem}-{suffix}.png")
|
||||
|
||||
def save_record(self, case, record):
|
||||
self.records[case["name"]] = record
|
||||
self.write_summary_inputs(case)
|
||||
write_json(self.result_dir(case) / "case-status.json", record)
|
||||
self.write_shard_status()
|
||||
|
||||
def write_shard_status(self, complete=False):
|
||||
self.result_root.mkdir(parents=True, exist_ok=True)
|
||||
records = [self.records.get(case["name"]) or
|
||||
self.record_for(case, NOT_RUN, "the shard has not reached this case") for case in self.cases]
|
||||
write_json(self.result_root / "shard-status.json", {
|
||||
"schema": 1, "lane": self.options.lane, "transport": self.options.transport,
|
||||
"backend": self.options.backend, "complete": complete,
|
||||
# The Actions job that ran the shard, so the artifact cleanup can tell whose job failed.
|
||||
"job_id": self.environ.get("MOBILEGL_SHARD_JOB_ID", ""),
|
||||
"expected_cases": [case["name"] for case in self.cases], "cases": records,
|
||||
"emulator_boots": self.boots, "install_seconds": self.install_seconds})
|
||||
|
||||
def remaining_seconds(self):
|
||||
if self.options.deadline_seconds <= 0:
|
||||
return None
|
||||
return self.options.deadline_seconds - (time.monotonic() - self.started)
|
||||
|
||||
def deadline_passed(self):
|
||||
remaining = self.remaining_seconds()
|
||||
return remaining is not None and remaining <= 0
|
||||
|
||||
# ---- processes ---------------------------------------------------------------------
|
||||
|
||||
def run(self, command, timeout=None, capture=False, extra_env=None):
|
||||
"""Returns (exit status, output). A timeout kills the whole process group and reads as 124."""
|
||||
environment = dict(self.environ)
|
||||
environment.update(extra_env or {})
|
||||
own_group = os.name == "posix"
|
||||
try:
|
||||
process = subprocess.Popen(command, env=environment, start_new_session=own_group,
|
||||
stdout=subprocess.PIPE if capture else None,
|
||||
stderr=subprocess.STDOUT if capture else None)
|
||||
except OSError as error:
|
||||
return 127, str(error)
|
||||
def kill():
|
||||
if own_group:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except OSError:
|
||||
pass
|
||||
process.kill()
|
||||
|
||||
try:
|
||||
output, _ = process.communicate(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
kill()
|
||||
output, _ = process.communicate()
|
||||
return TIMEOUT_EXIT_CODE, (output or b"").decode("utf-8", "replace")
|
||||
except BaseException:
|
||||
# The job was canceled or timed out. The child is in a session of its own, so it
|
||||
# would otherwise keep driving the device behind the steps that still run.
|
||||
kill()
|
||||
raise
|
||||
return process.returncode, (output or b"").decode("utf-8", "replace")
|
||||
|
||||
def adb(self, *arguments, timeout=60):
|
||||
status, output = self.run(self.adb_command + list(arguments), timeout=timeout, capture=True)
|
||||
return status, output.replace("\r", "")
|
||||
|
||||
# ---- emulator ----------------------------------------------------------------------
|
||||
|
||||
def can_restart(self):
|
||||
return bool(self.avd_name and self.options.gpu and self.emulator_log and self.emulator_pid_file)
|
||||
|
||||
def emulator_pid(self):
|
||||
try:
|
||||
return int(Path(self.emulator_pid_file).read_text().strip())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
def emulator_process_alive(self):
|
||||
pid = self.emulator_pid()
|
||||
if pid is None:
|
||||
return True # nothing to check against, for example a real device
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def health_problem(self):
|
||||
"""Empty when the device answers; otherwise what did not."""
|
||||
if not self.emulator_process_alive():
|
||||
return "the emulator process is gone"
|
||||
status, output = self.adb("get-state", timeout=15)
|
||||
if status != 0 or output.strip() != "device":
|
||||
return f"adb state is {output.strip() or 'unknown'}"
|
||||
status, output = self.adb("shell", "getprop", "sys.boot_completed", timeout=15)
|
||||
if status != 0 or output.strip() != "1":
|
||||
return "sys.boot_completed is not 1"
|
||||
# A framework call: boot_completed stays 1 across a system_server restart.
|
||||
status, output = self.adb("shell", "pm", "path", self.options.package, timeout=30)
|
||||
if status != 0 or "package:" not in output:
|
||||
return "the package manager does not list the trace APK"
|
||||
return ""
|
||||
|
||||
def keep_emulator_log(self, label):
|
||||
if self.emulator_log and os.path.isfile(self.emulator_log):
|
||||
self.diagnostics.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(self.emulator_log, self.diagnostics / f"emulator-boot-{self.boots}-{label}.log")
|
||||
|
||||
def prepare_booted_device(self):
|
||||
# A shard outlives the screen timeout a single replay never reached.
|
||||
self.adb("shell", "svc", "power", "stayon", "true", timeout=30)
|
||||
return self.install()
|
||||
|
||||
def install(self):
|
||||
started = time.monotonic()
|
||||
status, output = self.adb("install", "-r", self.options.apk_file, timeout=300)
|
||||
self.install_seconds.append(round(time.monotonic() - started, 1))
|
||||
say(output.strip())
|
||||
return status == 0
|
||||
|
||||
def restart_emulator(self, reason):
|
||||
"""Stop, cold boot and reinstall. False when the emulator did not come back."""
|
||||
if not self.can_restart():
|
||||
say(f"trace-replay-shard: cannot restart the emulator ({reason}): no AVD is configured")
|
||||
return False
|
||||
say(f"trace-replay-shard: restarting the emulator: {reason}")
|
||||
common = ["--avd-name", self.avd_name, "--emulator-log", self.emulator_log,
|
||||
"--pid-file", self.emulator_pid_file]
|
||||
for attempt in (1, 2):
|
||||
# The start truncates the log, and the boot that failed is the one worth reading.
|
||||
self.keep_emulator_log(f"attempt-{attempt}" if attempt > 1 else "ended")
|
||||
old_pid = self.emulator_pid()
|
||||
self.run(self.avd_command + ["stop"] + common, timeout=120)
|
||||
self.wait_for_exit(old_pid)
|
||||
self.adb("kill-server", timeout=30)
|
||||
pause(2)
|
||||
self.boots += 1
|
||||
status, _ = self.run(self.avd_command + ["start"] + common +
|
||||
["--gpu", self.options.gpu, "--boot-timeout", str(BOOT_TIMEOUT_SECONDS)],
|
||||
timeout=2 * BOOT_TIMEOUT_SECONDS + 60)
|
||||
if status == 0 and self.prepare_booted_device():
|
||||
return True
|
||||
say(f"trace-replay-shard: emulator restart attempt {attempt} failed")
|
||||
return False
|
||||
|
||||
def wait_for_exit(self, pid):
|
||||
if pid is None or os.name != "posix":
|
||||
return
|
||||
for _ in range(60):
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except OSError:
|
||||
return
|
||||
pause(0.5)
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# ---- app state ---------------------------------------------------------------------
|
||||
|
||||
def leftover_processes(self):
|
||||
"""Processes still running under the app's uid, as 'pid name' strings."""
|
||||
status, user = self.adb("shell", "run-as", self.options.package, "id", "-un", timeout=30)
|
||||
user = user.strip()
|
||||
if status != 0 or not re.fullmatch(r"[A-Za-z0-9_]+", user):
|
||||
return None
|
||||
status, listing = self.adb("shell", "ps", "-A", "-o", "USER,PID,NAME", timeout=30)
|
||||
if status != 0:
|
||||
return None
|
||||
found = []
|
||||
for line in listing.splitlines():
|
||||
fields = line.split()
|
||||
if len(fields) >= 3 and fields[0] == user and fields[1].isdigit():
|
||||
found.append(f"{fields[1]} {fields[2]}")
|
||||
return found
|
||||
|
||||
def settled_leftover_processes(self, polls):
|
||||
"""leftover_processes once the app's uid is empty, or as it still is after `polls` looks.
|
||||
|
||||
A killed process stays listed for a moment, so one look right after the kill would
|
||||
report every force-stop as incomplete.
|
||||
"""
|
||||
found = self.leftover_processes()
|
||||
for _ in range(polls - 1):
|
||||
if not found:
|
||||
break
|
||||
pause(0.5)
|
||||
found = self.leftover_processes()
|
||||
return found
|
||||
|
||||
def reset_app_state(self):
|
||||
"""Fresh process and fresh app data, as a new install gives. Returns notes for the record."""
|
||||
notes = []
|
||||
package = self.options.package
|
||||
self.adb("shell", "am", "force-stop", package, timeout=60)
|
||||
leftovers = self.settled_leftover_processes(polls=4)
|
||||
if leftovers is None:
|
||||
notes.append("the app's processes could not be listed after force-stop")
|
||||
if leftovers:
|
||||
# A force-stop is expected to take a spawned server with it; say so when it did not.
|
||||
notes.append("processes left after force-stop: " + ", ".join(leftovers))
|
||||
self.adb("shell", "run-as", package, "kill", "-9", *[entry.split()[0] for entry in leftovers],
|
||||
timeout=30)
|
||||
if self.settled_leftover_processes(polls=6):
|
||||
notes.append("the leftover processes could not be killed")
|
||||
if self.restart_emulator("app processes survived force-stop and kill"):
|
||||
notes.append("the emulator was restarted to remove them")
|
||||
status, output = self.adb("shell", "pm", "clear", package, timeout=60)
|
||||
probe_status, _ = self.adb("shell", "run-as", package, "ls", f"/data/user/0/{package}", timeout=30)
|
||||
if status != 0 or "Success" not in output or probe_status != 0:
|
||||
# The state a fresh install leaves, reached the other way.
|
||||
notes.append("pm clear did not leave a usable app data directory; reinstalled the APK")
|
||||
self.adb("uninstall", package, timeout=120)
|
||||
self.install()
|
||||
return notes
|
||||
|
||||
def remove_case_leftovers(self, case):
|
||||
name = safe_case(case["name"])
|
||||
base = f"/data/local/tmp/mobilegl-{name}"
|
||||
self.adb("shell", "rm", "-f", f"{base}.trace", f"{base}.golden.png", f"{base}.alternate-golden.png",
|
||||
timeout=60)
|
||||
self.adb("shell", "am", "force-stop", self.options.package, timeout=60)
|
||||
shutil.rmtree(Path(self.options.fixture_root) / name, ignore_errors=True)
|
||||
|
||||
# ---- one case ----------------------------------------------------------------------
|
||||
|
||||
def reason_lines(self):
|
||||
try:
|
||||
return (self.result_root / "infrastructure-failure-reason.txt").read_text().splitlines()
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
def run_attempt(self, case):
|
||||
"""Returns (exit status, seconds, how the shard deadline cut the attempt short or "")."""
|
||||
case_budget = int(as_text(case["timeout_seconds"])) + REPLAY_GRACE_SECONDS
|
||||
budget = case_budget
|
||||
remaining = self.remaining_seconds()
|
||||
if remaining is not None and remaining < case_budget:
|
||||
budget = max(remaining, 1.0)
|
||||
environment = replay_environment(self.options.backend, case["name"])
|
||||
# The shard installs the APK once per boot.
|
||||
environment["MOBILEGL_TRACE_SKIP_INSTALL"] = "1"
|
||||
environment["MOBILEGL_TRACE_POLL_INTERVAL"] = self.poll_interval
|
||||
started = time.monotonic()
|
||||
status, _ = self.run(self.replay_command + replay_arguments(self.options, case, self.hang_budget_ms),
|
||||
timeout=budget, extra_env=environment)
|
||||
elapsed = round(time.monotonic() - started, 1)
|
||||
cut = ""
|
||||
if status == TIMEOUT_EXIT_CODE and budget < case_budget:
|
||||
# The case never had its own budget, so the timeout says nothing about the case.
|
||||
cut = f"cut by the shard deadline after {elapsed:.0f} s of a {case_budget} s budget"
|
||||
return status, elapsed, cut
|
||||
|
||||
def keep_first_attempt(self, case):
|
||||
"""Move the retried attempt's files aside so the retry starts from an empty directory.
|
||||
|
||||
result.json is renamed: the summary keys rows on any result.json below a directory
|
||||
that names the case, and the retry's is the one that counts.
|
||||
"""
|
||||
directory = self.result_dir(case)
|
||||
if not directory.is_dir():
|
||||
return
|
||||
kept = directory / "first-attempt"
|
||||
kept.mkdir(exist_ok=True)
|
||||
for entry in directory.iterdir():
|
||||
if entry.is_file():
|
||||
name = "result.first-attempt.json" if entry.name == "result.json" else entry.name
|
||||
entry.replace(kept / name)
|
||||
|
||||
def verdict(self, case, status):
|
||||
directory = self.result_dir(case)
|
||||
if status == 0:
|
||||
result = read_json(directory / "result.json")
|
||||
if not isinstance(result, dict) or result.get("passed") is not True:
|
||||
return FAILED, "the replay exited 0 without a passed result.json"
|
||||
if self.options.transport != "monolith":
|
||||
proof = read_json(directory / "transport-proof.json")
|
||||
if not (isinstance(proof, dict) and proof.get("passed") is True and
|
||||
proof.get("required_transport") == self.options.transport):
|
||||
return FAILED, f"the replay exited 0 without a passed {self.options.transport} transport proof"
|
||||
return PASSED, ""
|
||||
if status == INFRASTRUCTURE_FAILURE_EXIT_CODE:
|
||||
return FAILED, "the emulator infrastructure failed again on the retry"
|
||||
if status == TIMEOUT_EXIT_CODE:
|
||||
return FAILED, "the replay script exceeded its time budget"
|
||||
result = read_json(directory / "result.json")
|
||||
if not isinstance(result, dict):
|
||||
return FAILED, f"exit {status} without a result.json"
|
||||
if result.get("passed") is True:
|
||||
# The picture matched, so what failed is a check that runs after it.
|
||||
return FAILED, f"exit {status} after a passed result.json: the transport proof or a log check failed"
|
||||
return FAILED, f"exit {status}: {one_line(result.get('message', ''))}"
|
||||
|
||||
def collect_failure_diagnostics(self, case):
|
||||
directory = self.result_dir(case)
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
logcat = directory / "logcat.txt"
|
||||
if not logcat.is_file() or logcat.stat().st_size == 0:
|
||||
# trace-replay-ci.sh pulls this itself unless it was killed first.
|
||||
_, output = self.adb("logcat", "-d", "-t", "20000", timeout=30)
|
||||
logcat.write_text(output, encoding="utf-8", errors="replace")
|
||||
app_output = f"/data/user/0/{self.options.package}/files/trace-replay/output"
|
||||
for name in ("retrace.log", "mobilegl.client.log", "mobilegl.server.log"):
|
||||
status, output = self.adb("exec-out", "run-as", self.options.package, "cat",
|
||||
f"{app_output}/{name}", timeout=30)
|
||||
if status == 0 and output and not output.startswith("cat: "):
|
||||
(directory / name).write_text(output, encoding="utf-8", errors="replace")
|
||||
if self.emulator_log and os.path.isfile(self.emulator_log):
|
||||
shutil.copyfile(self.emulator_log, directory / "emulator.log")
|
||||
# A vanished emulator looks the same whether the host OOM killer took it or the
|
||||
# renderer faulted. These two say which.
|
||||
_, memory = self.run(["free", "-h"], timeout=30, capture=True)
|
||||
(directory / "host-memory.txt").write_text(memory, encoding="utf-8")
|
||||
_, dmesg = self.run(["sudo", "-n", "dmesg", "-T"], timeout=30, capture=True)
|
||||
(directory / "host-dmesg.txt").write_text("\n".join(dmesg.splitlines()[-300:]) + "\n", encoding="utf-8")
|
||||
|
||||
def run_case(self, index, case):
|
||||
name = case["name"]
|
||||
notes = []
|
||||
seconds = {}
|
||||
case_started = time.monotonic()
|
||||
|
||||
if self.deadline_passed():
|
||||
return self.record_for(case, NOT_RUN, "the shard deadline passed before this case started")
|
||||
|
||||
restarted_for = ""
|
||||
if self.restart_each_case and index > 0:
|
||||
restarted_for = "MOBILEGL_SHARD_RESTART_EMULATOR_EACH_CASE=1"
|
||||
elif self.unclean_case and self.can_restart():
|
||||
# A crashed, hung or killed replay can leave the guest or the host renderer in a
|
||||
# state no adb query shows, and this case's verdict must not depend on it.
|
||||
restarted_for = f"the case before it, {self.unclean_case}, did not pass"
|
||||
else:
|
||||
restarted_for = self.health_problem()
|
||||
if restarted_for:
|
||||
notes.append(f"emulator restarted before the case: {restarted_for}")
|
||||
if not self.restart_emulator(restarted_for):
|
||||
self.aborted = "the emulator could not be restarted"
|
||||
return self.record_for(case, NOT_RUN, self.aborted, notes=notes)
|
||||
|
||||
notes += self.reset_app_state()
|
||||
seconds["reset"] = round(time.monotonic() - case_started, 1)
|
||||
|
||||
reasons_before = len(self.reason_lines())
|
||||
exit_codes = []
|
||||
replay_seconds = []
|
||||
status, elapsed, cut = self.run_attempt(case)
|
||||
exit_codes.append(status)
|
||||
replay_seconds.append(elapsed)
|
||||
surface_lost = 0
|
||||
if status == INFRASTRUCTURE_FAILURE_EXIT_CODE:
|
||||
say(f"::warning::Android emulator infrastructure failed; restarting it and retrying {name} once.")
|
||||
new_reasons = self.reason_lines()[reasons_before:]
|
||||
# A lost surface is retried, so it is counted where a rate spike shows.
|
||||
surface_lost = sum(1 for line in new_reasons if "angle-surface-lost" in line)
|
||||
if surface_lost:
|
||||
self.append_step_summary(f"surface-lost retries: {surface_lost} ({self.options.backend}, {name})\n")
|
||||
notes.append("infrastructure retry: " + (", ".join(new_reasons) or "no reason recorded"))
|
||||
self.keep_first_attempt(case)
|
||||
if not self.restart_emulator(f"infrastructure failure in {name}"):
|
||||
self.aborted = "the emulator could not be restarted"
|
||||
elif self.deadline_passed():
|
||||
cut = "the shard deadline passed before the infrastructure retry could start"
|
||||
else:
|
||||
notes += self.reset_app_state()
|
||||
status, elapsed, cut = self.run_attempt(case)
|
||||
exit_codes.append(status)
|
||||
replay_seconds.append(elapsed)
|
||||
seconds["replay"] = replay_seconds
|
||||
|
||||
finish_started = time.monotonic()
|
||||
if self.aborted:
|
||||
verdict, reason = FAILED, "the emulator did not come back for the infrastructure retry"
|
||||
elif cut:
|
||||
verdict, reason = NOT_RUN, cut
|
||||
else:
|
||||
verdict, reason = self.verdict(case, status)
|
||||
if verdict != PASSED:
|
||||
self.collect_failure_diagnostics(case)
|
||||
if not self.aborted:
|
||||
self.remove_case_leftovers(case)
|
||||
seconds["finish"] = round(time.monotonic() - finish_started, 1)
|
||||
seconds["total"] = round(time.monotonic() - case_started, 1)
|
||||
return self.record_for(case, verdict, reason, attempts=len(exit_codes), exit_codes=exit_codes,
|
||||
surface_lost_retries=surface_lost, seconds=seconds, notes=notes)
|
||||
|
||||
# ---- the shard ---------------------------------------------------------------------
|
||||
|
||||
def append_step_summary(self, text):
|
||||
path = self.environ.get("GITHUB_STEP_SUMMARY")
|
||||
if path:
|
||||
with open(path, "a", encoding="utf-8") as summary:
|
||||
summary.write(text)
|
||||
|
||||
def run_all(self):
|
||||
self.write_shard_status()
|
||||
say(f"trace-replay-shard: {len(self.cases)} case(s) on {self.options.backend} / "
|
||||
f"{self.options.transport} ({self.options.lane})")
|
||||
if not self.prepare_booted_device():
|
||||
if not self.restart_emulator("the trace APK did not install"):
|
||||
self.aborted = "the trace APK could not be installed"
|
||||
for index, case in enumerate(self.cases):
|
||||
say(f"::group::{case['name']} ({index + 1}/{len(self.cases)})")
|
||||
if self.aborted:
|
||||
record = self.record_for(case, NOT_RUN, self.aborted)
|
||||
else:
|
||||
try:
|
||||
record = self.run_case(index, case)
|
||||
except Exception as error: # one case's bookkeeping must not cost the others their verdicts
|
||||
record = self.record_for(case, FAILED, f"the shard runner failed: {one_line(error)}")
|
||||
self.unclean_case = "" if record["status"] == PASSED else case["name"]
|
||||
self.save_record(case, record)
|
||||
say(f"trace-replay-shard: {case['name']}: {record['status']}"
|
||||
+ (f" ({record['reason']})" if record["reason"] else ""))
|
||||
say("::endgroup::")
|
||||
return self.finish()
|
||||
|
||||
def finish(self):
|
||||
"""Tally from the files on disk, so a verdict the loop never wrote counts as missing."""
|
||||
label = f"{self.options.backend} / {self.options.transport}"
|
||||
rows = []
|
||||
verdicts = 0
|
||||
failed = []
|
||||
for case in self.cases:
|
||||
record = read_json(self.result_dir(case) / "case-status.json")
|
||||
if not isinstance(record, dict) or record.get("case") != case["name"]:
|
||||
record = self.record_for(case, NOT_RUN, "no verdict file was written")
|
||||
if record["status"] in (PASSED, FAILED):
|
||||
verdicts += 1
|
||||
if record["status"] == PASSED:
|
||||
result = read_json(self.result_dir(case) / "result.json")
|
||||
if not isinstance(result, dict) or result.get("passed") is not True:
|
||||
record = dict(record, status=FAILED, reason="the verdict is passed but result.json is not")
|
||||
if record["status"] != PASSED:
|
||||
failed.append(record)
|
||||
result = read_json(self.result_dir(case) / "result.json")
|
||||
ssim = result.get("ssim", "") if isinstance(result, dict) else ""
|
||||
seconds = record.get("seconds", {}).get("total", "")
|
||||
note = one_line("; ".join([record.get("reason", "")] + record.get("notes", [])).strip("; "))
|
||||
rows.append(f"| {case['name']} | {record['status']} | {record.get('attempts', 0)} | {seconds} | "
|
||||
f"{ssim} | {note.replace('|', '/')} |")
|
||||
self.write_shard_status(complete=verdicts == len(self.cases))
|
||||
|
||||
summary = [f"### Android retrace: {label} ({self.options.lane})", "",
|
||||
f"{len(self.cases) - len(failed)} of {len(self.cases)} passed, {self.boots} emulator boot(s).", "",
|
||||
"| case | status | attempts | seconds | SSIM | note |", "| --- | --- | --- | --- | --- | --- |"]
|
||||
self.append_step_summary("\n".join(summary + rows) + "\n\n")
|
||||
for record in failed:
|
||||
say(f"::error title=Android retrace {record['status']} ({label})::"
|
||||
f"{record['case']}: {record.get('reason') or record['status']}")
|
||||
if verdicts != len(self.cases):
|
||||
say(f"::error title=Android retrace shard incomplete ({label})::"
|
||||
f"{verdicts} verdict(s) for {len(self.cases)} case(s)")
|
||||
say(f"trace-replay-shard: {len(self.cases) - len(failed)} passed, {len(failed)} not passed, "
|
||||
f"{verdicts} verdict(s) for {len(self.cases)} case(s)")
|
||||
return 0 if not failed and verdicts == len(self.cases) else 1
|
||||
|
||||
def finalize(self):
|
||||
"""Give every case its summary inputs and a verdict file without touching the device.
|
||||
|
||||
Runs after the replay step whatever its outcome, so a shard that was killed or never
|
||||
started still uploads one named row per case.
|
||||
"""
|
||||
for case in self.cases:
|
||||
record = read_json(self.result_dir(case) / "case-status.json")
|
||||
if isinstance(record, dict) and record.get("case") == case["name"]:
|
||||
self.records[case["name"]] = record
|
||||
self.write_summary_inputs(case)
|
||||
else:
|
||||
self.save_record(case, self.record_for(case, NOT_RUN, "the shard did not reach this case"))
|
||||
previous = read_json(self.result_root / "shard-status.json")
|
||||
if isinstance(previous, dict):
|
||||
self.boots = previous.get("emulator_boots", self.boots)
|
||||
self.install_seconds = previous.get("install_seconds", self.install_seconds)
|
||||
self.write_shard_status(
|
||||
complete=all(self.records[case["name"]]["status"] in (PASSED, FAILED) for case in self.cases))
|
||||
return 0
|
||||
|
||||
def dry_run(self):
|
||||
for case in self.cases:
|
||||
say(json.dumps({
|
||||
"case": case["name"],
|
||||
"environment": replay_environment(self.options.backend, case["name"]),
|
||||
"arguments": replay_arguments(self.options, case, self.hang_budget_ms),
|
||||
"timeout": int(as_text(case["timeout_seconds"])) + REPLAY_GRACE_SECONDS,
|
||||
}, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def parse_arguments(argv):
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--apk-file", required=True, help="Trace APK to install once per emulator boot.")
|
||||
parser.add_argument("--package", default="top.mobilegl.plugin.trace")
|
||||
parser.add_argument("--backend", required=True, choices=BACKENDS)
|
||||
parser.add_argument("--transport", required=True, choices=TRANSPORTS)
|
||||
parser.add_argument("--lane", required=True, help="Lane name written to lane.json, e.g. inproc-acceptance.")
|
||||
parser.add_argument("--gpu", default="", help="Emulator -gpu mode, needed to restart the emulator.")
|
||||
source = parser.add_mutually_exclusive_group(required=True)
|
||||
source.add_argument("--cases-json", help="JSON array of the APK matrix's case objects.")
|
||||
source.add_argument("--cases-file", help="File holding that JSON array.")
|
||||
source.add_argument("--cases-env", help="Name of the environment variable holding that JSON array.")
|
||||
parser.add_argument("--result-root", default="android-retrace-result")
|
||||
parser.add_argument("--fixture-root", default="android-retrace-fixture")
|
||||
parser.add_argument("--deadline-seconds", type=int, default=0,
|
||||
help="Stop starting cases, and cut a running one, this long after the start; both are "
|
||||
"recorded as not-run. 0: no limit. retrace_shard_budget.py says what a shard needs.")
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
mode.add_argument("--dry-run", action="store_true",
|
||||
help="Print each case's environment and trace-replay-ci.sh arguments, run nothing.")
|
||||
mode.add_argument("--finalize", action="store_true",
|
||||
help="Only write the summary inputs and a not-run verdict for cases without one.")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
options = parse_arguments(argv)
|
||||
if options.cases_json is not None:
|
||||
text = options.cases_json
|
||||
elif options.cases_file is not None:
|
||||
text = Path(options.cases_file).read_text(encoding="utf-8")
|
||||
else:
|
||||
text = os.environ.get(options.cases_env, "")
|
||||
try:
|
||||
cases = load_cases(text, options.backend)
|
||||
except ValueError as error:
|
||||
say(f"::error title=Android retrace shard misconfigured::{error}")
|
||||
return 2
|
||||
shard = Shard(options, cases, os.environ)
|
||||
if shard.hang_budget_ms is None:
|
||||
say("::error title=Android retrace shard misconfigured::MOBILEGL_GPU_HANG_BUDGET_MS is not set")
|
||||
return 2
|
||||
if options.dry_run:
|
||||
return shard.dry_run()
|
||||
if options.finalize:
|
||||
return shard.finalize()
|
||||
if not os.path.isfile(options.apk_file):
|
||||
say(f"::error title=Android retrace shard misconfigured::APK does not exist: {options.apk_file}")
|
||||
return 2
|
||||
# A termination request unwinds through Shard.run, which kills the running child first.
|
||||
signal.signal(signal.SIGTERM, lambda number, frame: sys.exit(128 + number))
|
||||
return shard.run_all()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user