mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-11-21 00:41:57 +09:00
[CI, Tools] (...): replay trace cases in shards from one fixture cache
This commit is contained in:
@@ -0,0 +1,235 @@
|
||||
#!/usr/bin/env python3
|
||||
# MobileGL - android-plugin/retrace_artifact_cleanup.py
|
||||
# Copyright (c) 2025-2026 MobileGL-Dev
|
||||
# Licensed under the GNU Lesser General Public License v3.0:
|
||||
# https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
# SPDX-License-Identifier: LGPL-3.0-only
|
||||
# End of Source File Header
|
||||
|
||||
"""Delete the intermediate artifacts of an APK workflow run, keeping what a failure needs.
|
||||
|
||||
The retrace summary embeds every image it shows, so once it is uploaded the per-shard result
|
||||
artifacts are only worth keeping for cases that did not pass:
|
||||
|
||||
- a result artifact stays when its shard-status.json is missing, when its job failed, or when
|
||||
it holds a case that did not pass in any shard (so both backends and all transports of a
|
||||
failed case stay comparable);
|
||||
- every result artifact stays when a job failed without leaving a shard-status.json to say
|
||||
which cases it was running.
|
||||
|
||||
A canceled job retains nothing by itself; a failed verdict inside it still does. The trace
|
||||
fixtures are not artifacts: the retrace jobs restore them from a cache entry.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
RESULT_PREFIXES = ("MobileGL-android-retrace-result-", "retrace-result-")
|
||||
FAILED_CONCLUSIONS = ("failure", "timed_out", "action_required")
|
||||
# GitHub's secondary rate limit allows about 180 mutating REST calls a minute. Requests start
|
||||
# at most this often, whatever the number of workers.
|
||||
DELETE_INTERVAL_SECONDS = 60.0 / 170.0
|
||||
DELETE_WORKERS = 3
|
||||
MAX_REFUSED_DELETES = 5
|
||||
|
||||
|
||||
def pause(seconds):
|
||||
time.sleep(seconds)
|
||||
|
||||
|
||||
def read_rows(text, columns):
|
||||
rows = []
|
||||
for line in text.splitlines():
|
||||
if line.strip():
|
||||
fields = line.split("\t")
|
||||
rows.append(fields + [""] * (columns - len(fields)))
|
||||
return rows
|
||||
|
||||
|
||||
def shard_statuses(download_roots):
|
||||
"""Artifact name -> its shard-status.json, for every downloaded result artifact that has one."""
|
||||
statuses = {}
|
||||
for root in download_roots:
|
||||
root = Path(root)
|
||||
if not root.is_dir():
|
||||
continue
|
||||
for directory in root.iterdir():
|
||||
try:
|
||||
status = json.loads((directory / "shard-status.json").read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if isinstance(status, dict) and isinstance(status.get("cases"), list):
|
||||
statuses[directory.name] = status
|
||||
return statuses
|
||||
|
||||
|
||||
def plan(artifacts, jobs, statuses):
|
||||
"""Returns (ids to delete, lines explaining what stays)."""
|
||||
failed_job_ids = {job_id for job_id, conclusion, _ in jobs if conclusion in FAILED_CONCLUSIONS}
|
||||
status_job_ids = {str(status.get("job_id") or "") for status in statuses.values()} - {""}
|
||||
# Failed jobs that left no status file: nothing says which cases they were running.
|
||||
unexplained = failed_job_ids - status_job_ids
|
||||
if not status_job_ids and failed_job_ids:
|
||||
# The statuses carry no job ids to match against. Count instead: each failed job must
|
||||
# be explained by an artifact that shows a case without a pass.
|
||||
explaining = sum(1 for status in statuses.values()
|
||||
if any(record.get("status") != "passed" for record in status["cases"]))
|
||||
unexplained = set() if explaining >= len(failed_job_ids) else failed_job_ids
|
||||
|
||||
failed_cases = set() # did not pass somewhere: every result that holds one stays
|
||||
for status in statuses.values():
|
||||
not_passed = [record for record in status["cases"] if record.get("status") != "passed"]
|
||||
if status_job_ids:
|
||||
job_failed = str(status.get("job_id") or "") in failed_job_ids
|
||||
else:
|
||||
# Without job ids, a status that shows a case without a pass stands for a failed job.
|
||||
job_failed = bool(failed_job_ids) and bool(not_passed)
|
||||
for record in not_passed:
|
||||
if record.get("status") == "failed" or job_failed:
|
||||
failed_cases.add(record.get("case"))
|
||||
|
||||
delete, kept = [], []
|
||||
for artifact_id, name in artifacts:
|
||||
if not name.startswith(RESULT_PREFIXES):
|
||||
continue
|
||||
reason = ""
|
||||
status = statuses.get(name)
|
||||
if unexplained:
|
||||
reason = f"{len(unexplained)} failed job(s) left no shard status"
|
||||
elif status is None:
|
||||
reason = "no shard status was downloaded for it"
|
||||
elif str(status.get("job_id") or "") in failed_job_ids:
|
||||
reason = "its job failed"
|
||||
else:
|
||||
cases = sorted(failed_cases.intersection(record.get("case") for record in status["cases"]))
|
||||
if cases:
|
||||
reason = "holds failed case(s): " + ", ".join(cases)
|
||||
if reason:
|
||||
kept.append(f"{name} ({artifact_id}): {reason}")
|
||||
else:
|
||||
delete.append((artifact_id, name))
|
||||
return delete, kept
|
||||
|
||||
|
||||
class Deleter:
|
||||
"""Runs `command` once per artifact, with the artifact id appended to its last word."""
|
||||
|
||||
def __init__(self, command):
|
||||
self.command = command
|
||||
self.lock = threading.Lock()
|
||||
self.next_start = 0.0
|
||||
self.refused = []
|
||||
self.deleted = 0
|
||||
|
||||
def wait_for_slot(self):
|
||||
with self.lock:
|
||||
now = time.monotonic()
|
||||
start = max(now, self.next_start)
|
||||
self.next_start = start + DELETE_INTERVAL_SECONDS
|
||||
if start > now:
|
||||
pause(start - now)
|
||||
|
||||
def delete(self, artifact_id, name):
|
||||
for attempt in (1, 2):
|
||||
with self.lock:
|
||||
if len(self.refused) >= MAX_REFUSED_DELETES:
|
||||
return
|
||||
self.wait_for_slot()
|
||||
done = subprocess.run(self.command[:-1] + [self.command[-1] + artifact_id],
|
||||
capture_output=True, text=True)
|
||||
if done.returncode == 0:
|
||||
with self.lock:
|
||||
self.deleted += 1
|
||||
return
|
||||
if attempt == 1:
|
||||
# Give a rate limit time to clear before the one retry.
|
||||
with self.lock:
|
||||
self.next_start = max(self.next_start, time.monotonic() + 30)
|
||||
with self.lock:
|
||||
self.refused.append(f"{name} ({artifact_id}): {' '.join(done.stderr.split())[:200]}")
|
||||
|
||||
def run(self, entries):
|
||||
pending = list(entries)
|
||||
|
||||
def work():
|
||||
while True:
|
||||
with self.lock:
|
||||
if not pending:
|
||||
return
|
||||
artifact_id, name = pending.pop(0)
|
||||
self.delete(artifact_id, name)
|
||||
|
||||
threads = [threading.Thread(target=work) for _ in range(min(DELETE_WORKERS, len(pending)))]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
|
||||
def gh_rows(path, query, columns):
|
||||
done = subprocess.run(["gh", "api", "--paginate", path, "--jq", query], capture_output=True, text=True)
|
||||
if done.returncode != 0:
|
||||
raise RuntimeError(f"gh api {path} failed: {done.stderr.strip()}")
|
||||
return read_rows(done.stdout, columns)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--downloads", action="append", default=[],
|
||||
help="Directory holding one sub-directory per downloaded result artifact.")
|
||||
parser.add_argument("--artifacts-file", help="Tab-separated id and name per line, instead of asking gh.")
|
||||
parser.add_argument("--jobs-file", help="Tab-separated id, conclusion and name per line, instead of asking gh.")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Print the plan, delete nothing.")
|
||||
options = parser.parse_args(argv)
|
||||
|
||||
repository = os.environ.get("GITHUB_REPOSITORY", "")
|
||||
run_id = os.environ.get("GITHUB_RUN_ID", "")
|
||||
try:
|
||||
if options.artifacts_file:
|
||||
artifacts = read_rows(Path(options.artifacts_file).read_text(encoding="utf-8"), 2)
|
||||
else:
|
||||
artifacts = gh_rows(f"repos/{repository}/actions/runs/{run_id}/artifacts?per_page=100",
|
||||
".artifacts[] | [.id, .name] | @tsv", 2)
|
||||
if options.jobs_file:
|
||||
jobs = read_rows(Path(options.jobs_file).read_text(encoding="utf-8"), 3)
|
||||
else:
|
||||
# 50 rows per page: the jobs endpoint answers HTTP 502 for larger pages of a re-run
|
||||
# attempt of a run this size.
|
||||
jobs = gh_rows(f"repos/{repository}/actions/runs/{run_id}/jobs?per_page=50",
|
||||
'.jobs[] | [.id, .conclusion // "", .name] | @tsv', 3)
|
||||
except (OSError, RuntimeError) as error:
|
||||
print(f"::error::Could not read the run's artifacts and jobs; nothing was deleted: {error}")
|
||||
return 1
|
||||
|
||||
delete, kept = plan([(row[0], row[1]) for row in artifacts], [tuple(row[:3]) for row in jobs],
|
||||
shard_statuses(options.downloads))
|
||||
for line in kept:
|
||||
print(f"Retaining {line}")
|
||||
if options.dry_run:
|
||||
for artifact_id, name in delete:
|
||||
print(f"Would delete {name} ({artifact_id})")
|
||||
return 0
|
||||
|
||||
deleter = Deleter(shlex.split(os.environ.get("MOBILEGL_CLEANUP_DELETE_COMMAND") or
|
||||
f"gh api --method DELETE repos/{repository}/actions/artifacts/"))
|
||||
started = time.monotonic()
|
||||
deleter.run(delete)
|
||||
for line in deleter.refused:
|
||||
print(f"::warning::Could not delete intermediate artifact {line}")
|
||||
if len(deleter.refused) >= MAX_REFUSED_DELETES:
|
||||
print("::warning::Stopped deleting intermediate artifacts after repeated refusals; the rest stay until they expire.")
|
||||
print(f"Deleted {deleter.deleted} intermediate Android artifact(s) in {time.monotonic() - started:.0f}s; "
|
||||
f"retained {len(kept)}.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user