mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-11-16 14:33:24 +09:00
236 lines
9.6 KiB
Python
236 lines
9.6 KiB
Python
#!/usr/bin/env python3
|
|
# MobileGL - android-plugin/retrace_artifact_cleanup.py
|
|
# Copyright (c) 2025-2026 MobileGL-Dev
|
|
# Licensed under the GNU Lesser General Public License v3.0:
|
|
# https://www.gnu.org/licenses/gpl-3.0.txt
|
|
# https://www.gnu.org/licenses/lgpl-3.0.txt
|
|
# SPDX-License-Identifier: LGPL-3.0-only
|
|
# End of Source File Header
|
|
|
|
"""Delete the intermediate artifacts of an APK workflow run, keeping what a failure needs.
|
|
|
|
The retrace summary embeds every image it shows, so once it is uploaded the per-shard result
|
|
artifacts are only worth keeping for cases that did not pass:
|
|
|
|
- a result artifact stays when its shard-status.json is missing, when its job failed, or when
|
|
it holds a case that did not pass in any shard (so both backends and all transports of a
|
|
failed case stay comparable);
|
|
- every result artifact stays when a job failed without leaving a shard-status.json to say
|
|
which cases it was running.
|
|
|
|
A canceled job retains nothing by itself; a failed verdict inside it still does. The trace
|
|
fixtures are not artifacts: the retrace jobs restore them from a cache entry.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
RESULT_PREFIXES = ("MobileGL-android-retrace-result-", "retrace-result-")
|
|
FAILED_CONCLUSIONS = ("failure", "timed_out", "action_required")
|
|
# GitHub's secondary rate limit allows about 180 mutating REST calls a minute. Requests start
|
|
# at most this often, whatever the number of workers.
|
|
DELETE_INTERVAL_SECONDS = 60.0 / 170.0
|
|
DELETE_WORKERS = 3
|
|
MAX_REFUSED_DELETES = 5
|
|
|
|
|
|
def pause(seconds):
|
|
time.sleep(seconds)
|
|
|
|
|
|
def read_rows(text, columns):
|
|
rows = []
|
|
for line in text.splitlines():
|
|
if line.strip():
|
|
fields = line.split("\t")
|
|
rows.append(fields + [""] * (columns - len(fields)))
|
|
return rows
|
|
|
|
|
|
def shard_statuses(download_roots):
|
|
"""Artifact name -> its shard-status.json, for every downloaded result artifact that has one."""
|
|
statuses = {}
|
|
for root in download_roots:
|
|
root = Path(root)
|
|
if not root.is_dir():
|
|
continue
|
|
for directory in root.iterdir():
|
|
try:
|
|
status = json.loads((directory / "shard-status.json").read_text(encoding="utf-8"))
|
|
except (OSError, ValueError):
|
|
continue
|
|
if isinstance(status, dict) and isinstance(status.get("cases"), list):
|
|
statuses[directory.name] = status
|
|
return statuses
|
|
|
|
|
|
def plan(artifacts, jobs, statuses):
|
|
"""Returns (ids to delete, lines explaining what stays)."""
|
|
failed_job_ids = {job_id for job_id, conclusion, _ in jobs if conclusion in FAILED_CONCLUSIONS}
|
|
status_job_ids = {str(status.get("job_id") or "") for status in statuses.values()} - {""}
|
|
# Failed jobs that left no status file: nothing says which cases they were running.
|
|
unexplained = failed_job_ids - status_job_ids
|
|
if not status_job_ids and failed_job_ids:
|
|
# The statuses carry no job ids to match against. Count instead: each failed job must
|
|
# be explained by an artifact that shows a case without a pass.
|
|
explaining = sum(1 for status in statuses.values()
|
|
if any(record.get("status") != "passed" for record in status["cases"]))
|
|
unexplained = set() if explaining >= len(failed_job_ids) else failed_job_ids
|
|
|
|
failed_cases = set() # did not pass somewhere: every result that holds one stays
|
|
for status in statuses.values():
|
|
not_passed = [record for record in status["cases"] if record.get("status") != "passed"]
|
|
if status_job_ids:
|
|
job_failed = str(status.get("job_id") or "") in failed_job_ids
|
|
else:
|
|
# Without job ids, a status that shows a case without a pass stands for a failed job.
|
|
job_failed = bool(failed_job_ids) and bool(not_passed)
|
|
for record in not_passed:
|
|
if record.get("status") == "failed" or job_failed:
|
|
failed_cases.add(record.get("case"))
|
|
|
|
delete, kept = [], []
|
|
for artifact_id, name in artifacts:
|
|
if not name.startswith(RESULT_PREFIXES):
|
|
continue
|
|
reason = ""
|
|
status = statuses.get(name)
|
|
if unexplained:
|
|
reason = f"{len(unexplained)} failed job(s) left no shard status"
|
|
elif status is None:
|
|
reason = "no shard status was downloaded for it"
|
|
elif str(status.get("job_id") or "") in failed_job_ids:
|
|
reason = "its job failed"
|
|
else:
|
|
cases = sorted(failed_cases.intersection(record.get("case") for record in status["cases"]))
|
|
if cases:
|
|
reason = "holds failed case(s): " + ", ".join(cases)
|
|
if reason:
|
|
kept.append(f"{name} ({artifact_id}): {reason}")
|
|
else:
|
|
delete.append((artifact_id, name))
|
|
return delete, kept
|
|
|
|
|
|
class Deleter:
|
|
"""Runs `command` once per artifact, with the artifact id appended to its last word."""
|
|
|
|
def __init__(self, command):
|
|
self.command = command
|
|
self.lock = threading.Lock()
|
|
self.next_start = 0.0
|
|
self.refused = []
|
|
self.deleted = 0
|
|
|
|
def wait_for_slot(self):
|
|
with self.lock:
|
|
now = time.monotonic()
|
|
start = max(now, self.next_start)
|
|
self.next_start = start + DELETE_INTERVAL_SECONDS
|
|
if start > now:
|
|
pause(start - now)
|
|
|
|
def delete(self, artifact_id, name):
|
|
for attempt in (1, 2):
|
|
with self.lock:
|
|
if len(self.refused) >= MAX_REFUSED_DELETES:
|
|
return
|
|
self.wait_for_slot()
|
|
done = subprocess.run(self.command[:-1] + [self.command[-1] + artifact_id],
|
|
capture_output=True, text=True)
|
|
if done.returncode == 0:
|
|
with self.lock:
|
|
self.deleted += 1
|
|
return
|
|
if attempt == 1:
|
|
# Give a rate limit time to clear before the one retry.
|
|
with self.lock:
|
|
self.next_start = max(self.next_start, time.monotonic() + 30)
|
|
with self.lock:
|
|
self.refused.append(f"{name} ({artifact_id}): {' '.join(done.stderr.split())[:200]}")
|
|
|
|
def run(self, entries):
|
|
pending = list(entries)
|
|
|
|
def work():
|
|
while True:
|
|
with self.lock:
|
|
if not pending:
|
|
return
|
|
artifact_id, name = pending.pop(0)
|
|
self.delete(artifact_id, name)
|
|
|
|
threads = [threading.Thread(target=work) for _ in range(min(DELETE_WORKERS, len(pending)))]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join()
|
|
|
|
|
|
def gh_rows(path, query, columns):
|
|
done = subprocess.run(["gh", "api", "--paginate", path, "--jq", query], capture_output=True, text=True)
|
|
if done.returncode != 0:
|
|
raise RuntimeError(f"gh api {path} failed: {done.stderr.strip()}")
|
|
return read_rows(done.stdout, columns)
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
parser.add_argument("--downloads", action="append", default=[],
|
|
help="Directory holding one sub-directory per downloaded result artifact.")
|
|
parser.add_argument("--artifacts-file", help="Tab-separated id and name per line, instead of asking gh.")
|
|
parser.add_argument("--jobs-file", help="Tab-separated id, conclusion and name per line, instead of asking gh.")
|
|
parser.add_argument("--dry-run", action="store_true", help="Print the plan, delete nothing.")
|
|
options = parser.parse_args(argv)
|
|
|
|
repository = os.environ.get("GITHUB_REPOSITORY", "")
|
|
run_id = os.environ.get("GITHUB_RUN_ID", "")
|
|
try:
|
|
if options.artifacts_file:
|
|
artifacts = read_rows(Path(options.artifacts_file).read_text(encoding="utf-8"), 2)
|
|
else:
|
|
artifacts = gh_rows(f"repos/{repository}/actions/runs/{run_id}/artifacts?per_page=100",
|
|
".artifacts[] | [.id, .name] | @tsv", 2)
|
|
if options.jobs_file:
|
|
jobs = read_rows(Path(options.jobs_file).read_text(encoding="utf-8"), 3)
|
|
else:
|
|
# 50 rows per page: the jobs endpoint answers HTTP 502 for larger pages of a re-run
|
|
# attempt of a run this size.
|
|
jobs = gh_rows(f"repos/{repository}/actions/runs/{run_id}/jobs?per_page=50",
|
|
'.jobs[] | [.id, .conclusion // "", .name] | @tsv', 3)
|
|
except (OSError, RuntimeError) as error:
|
|
print(f"::error::Could not read the run's artifacts and jobs; nothing was deleted: {error}")
|
|
return 1
|
|
|
|
delete, kept = plan([(row[0], row[1]) for row in artifacts], [tuple(row[:3]) for row in jobs],
|
|
shard_statuses(options.downloads))
|
|
for line in kept:
|
|
print(f"Retaining {line}")
|
|
if options.dry_run:
|
|
for artifact_id, name in delete:
|
|
print(f"Would delete {name} ({artifact_id})")
|
|
return 0
|
|
|
|
deleter = Deleter(shlex.split(os.environ.get("MOBILEGL_CLEANUP_DELETE_COMMAND") or
|
|
f"gh api --method DELETE repos/{repository}/actions/artifacts/"))
|
|
started = time.monotonic()
|
|
deleter.run(delete)
|
|
for line in deleter.refused:
|
|
print(f"::warning::Could not delete intermediate artifact {line}")
|
|
if len(deleter.refused) >= MAX_REFUSED_DELETES:
|
|
print("::warning::Stopped deleting intermediate artifacts after repeated refusals; the rest stay until they expire.")
|
|
print(f"Deleted {deleter.deleted} intermediate Android artifact(s) in {time.monotonic() - started:.0f}s; "
|
|
f"retained {len(kept)}.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|