[Fix] (Bench): make the blend-toggle gate go red when the case it names stops running

- DriverBenchStateToggle was an entry that could not fail for the reason it was added. A case
  name matching nothing in kBenchCases selected nothing, run_case is void, and main returned 0
  unconditionally, so renaming or dropping mc_state_toggle left the entry green while measuring
  nothing - the exact state it was landed to end (ROADMAP.md:7).
- DriverBench now refuses an unknown case name before any GL work (exit 2, listing the cases it
  does have), so a caller that names a case - run_driver_bench.sh included - learns the case is
  gone instead of getting an empty CSV.
- Both ctest entries additionally require the case's own output row via PASS_REGULAR_EXPRESSION,
  so the gate stands on the evidence rather than on that check staying in the binary. The
  toggle entry pins the ops-per-frame column to 46, because the mc_* cases are deliberately
  excluded from the DRIVERBENCH_DRAWS scaling and 46 toggles per frame is part of what "this
  case still runs" means. A PASS_REGULAR_EXPRESSION makes ctest ignore the exit code, which is
  why the row is what is checked; the comment says so.
- Verified: renaming mc_state_toggle in kBenchCases -> DriverBenchStateToggle FAILS; setting its
  ops-per-frame to 45 -> FAILS; restored -> both entries pass again.
This commit is contained in:
2026-09-07 23:18:10 -04:00
parent af20dba6db
commit 08d14d85ef
2 changed files with 51 additions and 2 deletions
+29 -2
View File
@@ -11,8 +11,29 @@ endif()
add_executable(DriverBench DriverBench.c)
target_link_libraries(DriverBench PRIVATE dl)
# WHY EVERY ENTRY HERE CARRIES A PASS_REGULAR_EXPRESSION.
#
# DriverBench prints one CSV row per case it ran and exits 0 whatever it ran. Before this, a ctest
# entry naming a case therefore could not answer the only question it exists to ask: an argument
# matching nothing in kBenchCases selected no case, printed only the header row, and still exited
# 0. DriverBench.c now refuses an unknown case name (exit 2), which closes it at the source - but
# the entry must be able to go red for the reason it exists WITHOUT depending on that check
# staying in the binary, so each entry also requires the case's own output row to appear.
#
# The regex is what a healthy run of that case prints and nothing else does: the case name at the
# start of a line, then the frames / ops-per-frame / median-ms / ns-per-op / fps columns
# (run_case()). A rename, a drop from kBenchCases, a boot_egl() failure or
# a crash part-way through the case all remove that row and turn the entry red.
#
# Note that a PASS_REGULAR_EXPRESSION makes ctest ignore the process exit code (cmCTestRunTest:
# success is `retVal == 0 || !RequiredRegularExpressions.empty()`), which is why the row itself
# has to be the evidence rather than a companion to the rc.
add_test(NAME DriverBench COMMAND DriverBench draw_tiny)
set_tests_properties(DriverBench PROPERTIES LABELS benchmark)
# draw_tiny's a/ops scale with $DRIVERBENCH_DRAWS (main()), so only the shape of
# the row is pinned here, not the column values.
set_tests_properties(DriverBench PROPERTIES
LABELS benchmark
PASS_REGULAR_EXPRESSION "(^|\n)draw_tiny,[0-9]+,[0-9]+,[0-9.]+,[0-9.]+,[0-9.]+")
# The Blaze3D blend toggle, as its own entry.
#
@@ -31,4 +52,10 @@ set_tests_properties(DriverBench PROPERTIES LABELS benchmark)
# when unset) - the ctest entry is a "does this case still run" gate, not the measurement. The
# measurement is run_driver_bench.sh against each of {native, espryt, magma}.
add_test(NAME DriverBenchStateToggle COMMAND DriverBench mc_state_toggle)
set_tests_properties(DriverBenchStateToggle PROPERTIES LABELS benchmark)
# The ops-per-frame column is pinned to 46 here, unlike the entry above: the mc_* cases are
# excluded from the $DRIVERBENCH_DRAWS scaling on purpose ("the mc_* rates are measured and must
# not move, or the numbers stop being comparable", main()), so 46 toggles per frame
# is part of what "this case still runs" means. Change the workload and this entry says so.
set_tests_properties(DriverBenchStateToggle PROPERTIES
LABELS benchmark
PASS_REGULAR_EXPRESSION "(^|\n)mc_state_toggle,[0-9]+,46,[0-9.]+,[0-9.]+,[0-9.]+")
@@ -476,6 +476,28 @@ int main(int argc, char** argv) {
if (getenv("DRIVERBENCH_FRAMES")) g_frames = atoi(getenv("DRIVERBENCH_FRAMES"));
if (getenv("DRIVERBENCH_SPRITES")) g_mixSprites = atol(getenv("DRIVERBENCH_SPRITES"));
/* A requested case name that matches nothing used to select nothing, print the header row and
* exit 0 - so a caller that names a case (run_driver_bench.sh, and the two ctest entries in
* CMakeLists.txt) could not tell "the case ran" from "the case has been renamed or deleted".
* Refuse it here, before any GL work, so the refusal reaches a caller that has no display
* either, and name what does exist so the fix is obvious. */
int unknownCases = 0;
for (int j = 1; j < argc; ++j) {
int known = 0;
for (int i = 0; i < kBenchCaseCount; ++i)
if (strcmp(argv[j], kBenchCases[i].name) == 0) known = 1;
if (!known) {
fprintf(stderr, "DriverBench: no case named '%s'\n", argv[j]);
unknownCases = 1;
}
}
if (unknownCases) {
fprintf(stderr, "DriverBench: the %d cases in kBenchCases are:\n", kBenchCaseCount);
for (int i = 0; i < kBenchCaseCount; ++i)
fprintf(stderr, " %s\n", kBenchCases[i].name);
return 2;
}
if (boot_egl()) return 1;
build_resources();