diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0bea4fad..89074422 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -741,9 +741,10 @@ jobs: path: ci-artifacts/mobilegl-linux-lib-verify.tgz if-no-files-found: error - # The verify lane itself, plus the two negative controls that keep it falsifiable. The controls - # are ALWAYS-ON steps, not a manual exercise: a gate that can only be shown to work by someone - # remembering to break it on purpose is a gate that has already stopped working. + # The verify lane itself, plus the controls that keep it falsifiable: its own control entries, + # checked by name, and the two corrupted-snapshot steps. They are always on, not a manual + # exercise: a gate that can only be shown to work by someone remembering to break it on purpose + # is a gate that has already stopped working. integration-verify: runs-on: ubuntu-26.04 timeout-minutes: 180 @@ -844,35 +845,19 @@ jobs: -R 'HandleRecycle|CsoContentAddressing|ResourceSubsystem|MapPersistentRoundtrip|ObjectSubsystem|TextureParamsWithoutASamplerView|TextureUploadShape' \ --no-tests=error -j 4 - # The arming lanes' logs, and ONLY those. Each lane shares one MOBILEGL_LOG_FILE_PATH and the - # library opens it fopen(path, "w"), so after an ambient lane of 400-odd processes the file - # holds the LAST one - grepping it would say nothing about the other 405 and would red a - # healthy lane whenever the last entry happened not to issue a verb (which is what the - # PoisonOmissionScenario parent, the last ambient entry, does by construction: it forks, - # execve()s and reads files). The DirectGLES.VerifyArming. / DirectVulkan.VerifyArming. - # entries are one process each on a log path nothing else writes, so this grep means exactly - # what it says. - # - # What it proves: arming is a property of (this library, this environment), and these two - # processes ran the same library with the same MOBILEGL_PIPE_VERIFY=1 as their ~400 ambient - # siblings. It is not, and cannot be, a per-process census - the shared log cannot support one. - # It catches the case ctest cannot: an arming entry that SKIPPED still reports green. - - name: Verify lane arming check - working-directory: build-verify + # The lane's own control entries, by name: each must have run and passed, and a skip is a + # failure here although ctest reports it green. VerifyArming asserts in-test that the + # comparator printed its armed line, VerifyCorrupted that a corrupted field is reported by + # name, and PoisonOmitted that an omitted stamp kills a forked child with its own Fatal. + - name: Verify lane control entries run: | - shopt -s nullglob - logs=(MobileGL/MG_IntegrationTest/pipe-verify-arming-*.log) - if [ ${#logs[@]} -lt 2 ]; then - echo "::error::found ${#logs[@]} pipe-verify-arming-*.log (expected one per backend). The VerifyArming. entries did not run, so nothing in this job establishes that the comparator was ever armed." - exit 1 - fi - for log in "${logs[@]}"; do - if ! grep -q "MGPipe: verify armed" "${log}"; then - echo "::error::${log} carries no arming line: that lane's process ran the whole scenario without the comparator, so every green entry beside it is green for no reason" - exit 1 - fi - done - echo "arming line present in all ${#logs[@]} arming-lane log(s)" + python3 scripts/ci/junit_tally.py "${RUNNER_TEMP}/verify.xml" \ + --require-entry-passed DirectGLES.VerifyArming.PipeVerifyArmingScenario.Armed \ + --require-entry-passed DirectVulkan.VerifyArming.PipeVerifyArmingScenario.Armed \ + --require-entry-passed DirectGLES.VerifyCorrupted.PipeVerifyArmingScenario.CorruptedFieldIsReported \ + --require-entry-passed DirectVulkan.VerifyCorrupted.PipeVerifyArmingScenario.CorruptedFieldIsReported \ + --require-entry-passed DirectGLES.PoisonOmitted.PoisonOmissionScenario.OmittedFieldAbortsOnThatVerb \ + --require-entry-passed DirectVulkan.PoisonOmitted.PoisonOmissionScenario.OmittedFieldAbortsOnThatVerb # NEGATIVE CONTROL A (gate G4). The knob perturbs one field in the snapshot arm before the # entry compare, so a working comparator must abort the run. This step passes when ctest @@ -927,34 +912,6 @@ jobs: fi echo "the corrupted capability field turned ${matched} selected entries red, as it must" - # NEGATIVE CONTROL B (gate G5). The omission skips the STAMP of one field for one verb while - # still copying its value - indistinguishable from a fill row nobody wrote - so the poison - # must abort that verb. Again: this step passes when ctest fails. - # - # The entry it targets is PoisonOmissionScenario.WithoutOmissionCompletes, which is green in - # the ambient lane above and deliberately does not skip itself when the knob is set, exactly - # so that this control has something to turn red. P13 W5: the pair is the pack state at - # ReadPixels (the split step's pair below). Monolith now runs the record arm, whose - # generate_mipmap resolves its texture by the verb's handle and never reads the active unit, - # so GenerateMipmap:GetActiveTextureUnit left this lane green (Test 37515697754). - - name: Negative control B (omitted fill point) - working-directory: build-verify - env: - MOBILEGL_ITEST_REQUIRE_GPU: "1" - MOBILEGL_PIPE_POISON_OMIT: ReadPixels:GetPixelStoreParameters - run: | - FILTER='DirectGLES\.Verify\.PoisonOmissionScenario\.WithoutOmissionCompletes' - matched=$(ctest -N -L integration-verify -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:') - if [ "${matched}" -lt 1 ]; then - echo "::error::negative control B selected ${matched} tests; its filter no longer matches anything" - exit 1 - fi - if ctest --output-on-failure -L integration-verify -R "${FILTER}" --no-tests=error; then - echo "::error::MOBILEGL_PIPE_POISON_OMIT=ReadPixels:GetPixelStoreParameters left the verify lane GREEN. The per-verb poison is not armed, so a forgotten fill row would ship silently." - exit 1 - fi - echo "the omitted fill point turned the lane red, as it must" - - name: Upload verify lane logs if: ${{ always() }} uses: actions/upload-artifact@v7 @@ -980,8 +937,8 @@ jobs: path: /tmp/core.* if-no-files-found: ignore - # P7 WAVE 3 (V1), CONTRACT-P7 §6 - EXIT GATE 2's SECOND COLUMN: the comparator on the split - # (inproc) arm, both backends, and the two negative controls red on that arm. + # The comparator on the split (inproc) arm, both backends, with its control entries checked by + # name and the corrupted-snapshot control red on that arm. # # ITS OWN BUILD, because neither existing one can carry it: build-linux-verify configures # without MOBILEGL_BUILD_DISAGGREGATED (so MG_Remote and the transport parser do not exist and @@ -1087,10 +1044,24 @@ jobs: verify-split-arming verify-split-logs "${RUNNER_TEMP}/verify-split.xml" \ "${GITHUB_WORKSPACE}/MobileGL/MG_IntegrationTest/Harness/verify-split-unarmed-expected.txt" - # THE GREEN LANE'S LOGS, KEPT BEFORE THE CONTROLS OVERWRITE THEM. Each control step below + # The lane's own control entries, by name, as in integration-verify: VerifySplitReadCorrupted + # asserts in-test the two where=read reports in the server half, and VerifySplitPoisonOmitted + # reads both role halves of the forked child's log for its own Fatal. + - name: Verify-split control entries + run: | + args=() + for backend in DirectGLES DirectVulkan; do + args+=(--require-entry-passed "${backend}.VerifySplitArming.PipeVerifyArmingScenario.Armed") + args+=(--require-entry-passed "${backend}.VerifySplitCorrupted.PipeVerifyArmingScenario.CorruptedFieldIsReported") + args+=(--require-entry-passed "${backend}.VerifySplitReadCorrupted.PipeVerifyArmingScenario.CorruptedFieldIsReportedOnTheServerRead") + args+=(--require-entry-passed "${backend}.VerifySplitPoisonOmitted.PoisonOmissionScenario.OmittedFieldAbortsOnThatVerb") + done + python3 scripts/ci/junit_tally.py "${RUNNER_TEMP}/verify-split.xml" "${args[@]}" + + # THE GREEN LANE'S LOGS, KEPT BEFORE THE CONTROLS OVERWRITE THEM. The control step below # re-runs ambient VerifySplit. entries with a knob exported, and an ambient entry writes to # ITS OWN per-entry path under verify-split-logs/ - which the library opens "w". So by the - # time the upload step at the end of this job runs, twelve of those files are the CONTROL + # time the upload step at the end of this job runs, the selected files are the CONTROL # runs' logs sitting under the LANE's names: an artifact showing deliberate divergences, # with the green run they were meant to be evidence of already gone. `if: always()` on the # upload made that worse rather than better, because a failure investigation is exactly @@ -1127,64 +1098,9 @@ jobs: fi echo "the corrupted field turned ${matched} selected split entries red, as it must" - # Negative control for the compare-at-read hook, which the server's apply thread runs at each - # backend read; the step above covers only the client's entry compare. The entries pin - # MOBILEGL_PIPE_VERIFY_FATAL=0, or the client's entry compare would abort at the ReadPixels - # boundary before the server reads. This step re-reads each server log and needs two - # `where=read` reports per backend: the server reads the pack field twice in the ReadPixels - # window (the saved pack, then the backend's read under the neutral pack), and each read must - # catch its own perturbation. PipeVerifyArmingScenario.CorruptedFieldIsReportedOnTheServerRead - # has the details, including why both backends are needed. - - name: Negative control A (server-side read) - working-directory: build-verify-split - env: - MOBILEGL_ITEST_REQUIRE_GPU: "1" - run: | - FILTER='(DirectGLES|DirectVulkan)\.VerifySplitReadCorrupted\.' - matched=$(ctest -N -L '^integration-verify-split$' -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:') - if [ "${matched}" -lt 2 ]; then - echo "::error::split read-side control A selected ${matched} tests; its filter no longer matches both backends" - exit 1 - fi - ctest --output-on-failure -L '^integration-verify-split$' -R "${FILTER}" --no-tests=error - for backend in DirectGLES DirectVulkan; do - log="MobileGL/MG_IntegrationTest/pipe-verify-split-read-corrupt-${backend}.server.log" - reads=$(grep -c 'Fatal{PipeVerifyDiffer, "GetPixelStoreParameters@ReadPixels".*where=read}' "${log}" 2>/dev/null || true) - reads=${reads:-0} - if [ "${reads}" -lt 2 ]; then - echo "::error::${log} carries ${reads} compare-at-read report(s) for the corrupted pack field, fewer than the two reads the server makes inside the ReadPixels window (the applier's saved-pack read and the backend's post-install read): the hook is not comparing on the server's apply thread, or one of its two perturbations no longer reaches its read because the ID-49 neutral-pack overwrite swallowed it" - exit 1 - fi - done - echo "both backends reported GetPixelStoreParameters@ReadPixels where=read at least twice in the SERVER half, as they must" - - # NEGATIVE CONTROL B ON THE SPLIT ARM, WITH THE SAME PAIR. GenerateMipmap:GetActiveTextureUnit - # (the pre-P13 monolith pair) is inert here - the split backend's generate_mipmap - # resolves its texture by the verb's handle and never reads the active unit - so this step - # withholds ReadPixels:GetPixelStoreParameters, which the server reads under its own stamp. - # Measured at landing: 2 of 2 red, each child dying with - # Fatal{UnmigratedPipeInput, "GetPixelStoreParameters@ReadPixels"} on mgl-srv-apply. - - name: Negative control B (omitted stamp) - working-directory: build-verify-split - env: - MOBILEGL_ITEST_REQUIRE_GPU: "1" - MOBILEGL_PIPE_POISON_OMIT: ReadPixels:GetPixelStoreParameters - run: | - FILTER='(DirectGLES|DirectVulkan)\.VerifySplit\.PoisonOmissionScenario\.WithoutOmissionCompletes' - matched=$(ctest -N -L '^integration-verify-split$' -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:') - if [ "${matched}" -lt 2 ]; then - echo "::error::split negative control B selected ${matched} tests; its filter no longer matches both backends" - exit 1 - fi - if ctest --output-on-failure -L '^integration-verify-split$' -R "${FILTER}" --no-tests=error; then - echo "::error::MOBILEGL_PIPE_POISON_OMIT=ReadPixels:GetPixelStoreParameters left the verify-split lane GREEN. The server-side poison is not armed on the split arm." - exit 1 - fi - echo "the omitted stamp turned the split lane red, as it must" - # BOTH SETS, AND THEY ARE DIFFERENT EVIDENCE. verify-split-logs-lane/ is the green run the - # arm proof read; verify-split-logs/ is whatever the control steps left behind, which for - # twelve entries is a deliberately red re-run under the same name. Uploading only the + # arm proof read; verify-split-logs/ is whatever the control step left behind, which for + # its entries is a deliberately red re-run under the same name. Uploading only the # second - which is what this step did before the copy above existed - loses the first. # The run log needs a signed-in reader; annotations do not. Name every failed ctest entry, # with the lines of its output that say why, where anyone reading the run can see them. @@ -3020,8 +2936,8 @@ jobs: fi set +e ctest -V --no-tests=error --timeout 10800 \ - -R '^MobileGLTraceReplay\.OpenRA\.DirectGLES$' - control_rc=$? + -R '^MobileGLTraceReplay\.OpenRA\.DirectGLES$' 2>&1 | tee "${RUNNER_TEMP}/verify-control.txt" + control_rc=${PIPESTATUS[0]} set -e if [ -d "${GOOD_OUTPUT}" ]; then rm -rf OpenRA @@ -3032,6 +2948,12 @@ jobs: echo "::error::MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left the OpenRA retrace GREEN, so the comparator is not comparing and the whole verify retrace lane proves nothing." exit 1 fi + # Its own red: ctest -V prefixes every line with the test number, and CMake re-wraps text. + if ! sed -E 's/^[0-9]+: //' "${RUNNER_TEMP}/verify-control.txt" | tr -s '[:space:]' ' ' \ + | grep -qF 'Fatal{PipeVerifyDiffer, "GetRenderStateParameters@'; then + echo "::error::the corrupted-snapshot retrace went red without Fatal{PipeVerifyDiffer, \"GetRenderStateParameters@...\"}: something other than the comparator failed it" + exit 1 + fi echo "the corrupted field turned the retrace red, as it must (ctest exit ${control_rc})" - name: Upload core dumps diff --git a/MobileGL/MG_IntegrationTest/CMakeLists.txt b/MobileGL/MG_IntegrationTest/CMakeLists.txt index 161fec8b..e903c3cd 100644 --- a/MobileGL/MG_IntegrationTest/CMakeLists.txt +++ b/MobileGL/MG_IntegrationTest/CMakeLists.txt @@ -3632,9 +3632,9 @@ if (MOBILEGL_BUILD_DISAGGREGATED AND MOBILEGL_PIPE_VERIFY) # THE TWO NEGATIVE CONTROLS ON THIS ARM (CONTRACT-P7 §6), the monolith verify block's # four entries with the split environment underneath and nothing else changed: each is # green when its control REPORTS, so each goes red the day the comparator or the poison - # stops reaching the split arm. Their CI halves - the same knob exported against the - # ambient VerifySplit. entries, where ctest has to FAIL - live in test.yml's - # integration-verify-split job. + # stops reaching the split arm. test.yml's integration-verify-split job requires each of + # them to pass by name, and control A's CI half - the same knob exported against the + # ambient VerifySplit. entries, where ctest has to FAIL - runs there too. # # A (G4): the corruption perturbs the snapshot arm at the CLIENT's entry compare, so the # report lands in the client role's log, which is the half LibraryLog() reads. @@ -3667,11 +3667,10 @@ if (MOBILEGL_BUILD_DISAGGREGATED AND MOBILEGL_PIPE_VERIFY) # ServerPoisonOmission). PoisonOmissionScenario.cpp's kOmittedPairs carries both. mgl_itest_lane("${mglItestVerifyBackend}.VerifySplitPoisonOmitted." "PoisonOmissionScenario.OmittedFieldAbortsOnThatVerb" LABELS "integration-verify-split" TIMEOUT ${MGL_ITEST_VERIFY_TIMEOUT} LOCK "pipe-poison-omit-split-${mglItestVerifyBackend}.log" ENV "${mglItestVerifyArmEnv}\;MOBILEGL_PIPE_POISON_OMIT=ReadPixels:GetPixelStoreParameters\;MOBILEGL_LOG_FILE_PATH=${CMAKE_CURRENT_BINARY_DIR}/pipe-poison-omit-split-${mglItestVerifyBackend}.log") - # B's AMBIENT TARGET. The monolith verify lane's control-B step aims its knob at - # `Verify.PoisonOmissionScenario.WithoutOmissionCompletes`, the one integration entry that - # calls glGenerateMipmap at all - and the mechanism filter took that entry out of the - # ambient set above along with the rest of the scenario. Registered back by hand, with no - # knob, so the CI step has an entry on THIS arm to turn red. + # B's AMBIENT HALF. `Verify.PoisonOmissionScenario.WithoutOmissionCompletes` runs the + # scenario with nothing withheld, and the mechanism filter took it out of the ambient set + # above along with the rest of the scenario. Registered back by hand, with no knob, so + # this arm's ambient lane still runs it. mgl_itest_lane("${mglItestVerifyBackend}.VerifySplit." "PoisonOmissionScenario.WithoutOmissionCompletes" LABELS "integration-verify-split" TIMEOUT ${MGL_ITEST_VERIFY_TIMEOUT} TEST_LIST "MGL_VERIFY_SPLIT_TESTS_POISON_${mglItestVerifyBackend}" ENV "${mglItestVerifyArmEnv}") list(APPEND MGL_VERIFY_SPLIT_TEST_LISTS "MGL_VERIFY_SPLIT_TESTS_POISON_${mglItestVerifyBackend}") diff --git a/MobileGL/MG_IntegrationTest/Scenarios/PoisonOmissionScenario.cpp b/MobileGL/MG_IntegrationTest/Scenarios/PoisonOmissionScenario.cpp index 506fd79f..14a5d2b8 100644 --- a/MobileGL/MG_IntegrationTest/Scenarios/PoisonOmissionScenario.cpp +++ b/MobileGL/MG_IntegrationTest/Scenarios/PoisonOmissionScenario.cpp @@ -26,7 +26,7 @@ // about WHY: a poison that fired on every verb would look just as red. // // The knob is process-wide, so the two cases cannot share a lane: the first runs in the PoisonOmitted. -// entries, the second in the ambient Verify. entries (it skips when the knob IS set). +// entries, the second in the ambient Verify. entries (the first skips when the knob is unset). // // WHY THE SEQUENCE RUNS IN A SEPARATE PROCESS, AND WHY THAT PROCESS IS fork()+execve() AND NOT fork() // ALONE. The poison reports with MGLOG_F and then std::abort(), in the middle of a GL command - so the @@ -81,7 +81,7 @@ extern char** environ; namespace MGITest { namespace { - // What the PoisonOmitted. ctest entries and the CI negative-control steps name. Each pair is + // What the PoisonOmitted. ctest entries name, which CI requires to pass by name. Each pair is // spelled here so the assertion below is about the exact string the poison contracts to // print (ARCHITECTURE.md 9.2: Fatal{UnmigratedPipeInput, "@"}). // @@ -480,14 +480,12 @@ void main() { o_color = vec4(0.25, 0.5, 0.75, 1.0); } } } - // The sibling control, in the ambient Verify. lanes: the same sequence with the knob UNSET - // must run to completion and log no Fatal at all. + // The ambient half of the pair, in the ambient Verify. lanes: the same sequence with the + // knob UNSET must run to completion and log no Fatal at all. The PoisonOmitted. entries are + // the armed half (knob set, the verb must abort), and CI requires those entries by name. // - // It deliberately does NOT skip when MOBILEGL_PIPE_POISON_OMIT is set. This is the entry - // CI's always-on negative control B exports the knob at: a green entry that the omission - // turns red is the whole proof that the poison is armed, and an entry that politely skipped - // itself would report that green either way. Nothing else in the integration suite calls - // glGenerateMipmap, so this case is also the only possible target for that control. + // It does not skip when MOBILEGL_PIPE_POISON_OMIT is set: run by hand with the knob + // exported, it goes red, and the note in its failure says that is the poison firing. TEST_F(PoisonOmissionScenario, WithoutOmissionCompletes) { if (!Ready()) return; @@ -505,8 +503,7 @@ void main() { o_color = vec4(0.25, 0.5, 0.75, 1.0); } const std::string note = omissionArmed ? std::string(" NOTE: MOBILEGL_PIPE_POISON_OMIT is set in this process, so this failure is " - "what CI's negative control B is asking for - the poison IS armed, and this " - "entry going red is the proof.") + "expected: the poison IS armed, and this entry going red is the proof.") : std::string(); ASSERT_TRUE(WIFEXITED(status)) << "with no omission armed, a draw followed by glGenerateMipmap must complete; the child " diff --git a/docs/ci/ci-cleanup-log.md b/docs/ci/ci-cleanup-log.md index 8fb9480f..9bb315bd 100644 --- a/docs/ci/ci-cleanup-log.md +++ b/docs/ci/ci-cleanup-log.md @@ -41,6 +41,9 @@ them wrong. | 2026-10-17 | `scripts/ci/retrace_transport_control.sh`, `scripts/ci/retrace_drop_draw_control.sh`, the retrace and drop-draw halves of `control_smoke_test.sh` and `stub_ctest.sh`, and `scripts/ci/redcheck_control_smoke_test.sh` | Test / pipe-gates (R-16) | the bodies of the two controls above and their smoke cases; the redcheck script had no caller and mutated the deleted scripts | `test_run_trace_case.py` | | 2026-10-17 | unit `RemoteEmitTable.TheE2DropSwitchStartsDisarmed`, and the `MOBILEGL_IPC_E2_DROP_DRAW` / `MOBILEGL_IPC_E2_DROP_CLEAR` knobs with their per-draw and per-clear branches in the shipped emitter (`VerbPort.cpp`) | Test / unit (split); product code | the knobs existed only for the retired dropped-draws control (the clear knob was set by nothing); the unit case only toggled the clear setter | the wire-volume assertion in `run_trace_case.cmake` on the OpenRA split legs (DrawVbo records on the wire, draws on the server role) | | 2026-10-17 | the 802-entry `DirectGLES.Split.` baseline rerun inside `split_negative_controls.sh` (skip-census lane `junit-integration-split/isplit`) | Test / `integration-split`, every branch | a rerun of entries 'Split lanes' had run minutes earlier on the same runner and library (~2.5 min); its logs also overwrote the lane's in the uploaded artifact | the script tallies the lane step's `split-baseline.xml` and its exit code (`split-baseline.rc`) with the same three refusals; it still reruns when that step never ran (the 10-12/10-13 catches); the `split-baseline` census lane covers the names; smoke cases for the reuse, the fallback, a red exit code under a green JUnit and an empty arm | +| 2026-10-17 | `Negative control B (omitted fill point)` | Test / `integration-verify` | it re-ran `PoisonOmissionScenario.WithoutOmissionCompletes` with `MOBILEGL_PIPE_POISON_OMIT=ReadPixels:GetPixelStoreParameters` and accepted any red | the in-lane `DirectGLES/DirectVulkan.PoisonOmitted.PoisonOmissionScenario.OmittedFieldAbortsOnThatVerb` entries: the same pair, and they also require SIGABRT with the named `Fatal{UnmigratedPipeInput}`; now required to pass by name | +| 2026-10-17 | `Verify lane arming check` (grep of `pipe-verify-arming-*.log`) | Test / `integration-verify` | the VerifyArming entries assert the armed line in-test; the grep only added "the entry did not skip" | `Verify lane control entries`: `junit_tally.py --require-entry-passed` on the VerifyArming, VerifyCorrupted and PoisonOmitted entries of both backends (a skip fails) | +| 2026-10-17 | `Negative control A (server-side read)` and `Negative control B (omitted stamp)` | Test / `integration-verify-split` | the first re-ran the two VerifySplitReadCorrupted entries the lane had just run and grepped what they already assert in-test (at least two where=read reports in the server half); the second re-ran with the pair the in-lane VerifySplitPoisonOmitted entries already pin, and accepted any red | `Verify-split control entries`: `--require-entry-passed` on VerifySplitArming, VerifySplitCorrupted, VerifySplitReadCorrupted and VerifySplitPoisonOmitted of both backends | ## Kept after investigation diff --git a/tools/trace_replay/test_run_trace_case.py b/tools/trace_replay/test_run_trace_case.py index f2acc431..2ce33d7f 100644 --- a/tools/trace_replay/test_run_trace_case.py +++ b/tools/trace_replay/test_run_trace_case.py @@ -54,10 +54,16 @@ if canned: shutil.copy(os.path.join(canned, name), os.path.join(out, name)) with open(os.path.join(out, 'retrace.log'), 'w') as log: log.write('stub replay\n') +if os.environ.get('STUB_ABORT'): + sys.exit(134) with open(os.path.join(out, 'result.json'), 'w') as result: json.dump({'passed': True, 'ssim': 1.0}, result) ''' +# The retrace-verify control's reason check, as the workflow runs it on the tee'd ctest -V output. +VERIFY_CONTROL_REASON = ("sed -E 's/^[0-9]+: //' \"$1\" | tr -s '[:space:]' ' ' " + "| grep -qF 'Fatal{PipeVerifyDiffer, \"GetRenderStateParameters@'") + PREFIX = '[12:00:00] [Linux mobilegl_trace_/INFO]: ' MARKER = { 'inproc': 'Config: MOBILEGL_TRANSPORT=inproc - the MGPipe record stream crosses a real ring to an apply thread', @@ -117,7 +123,7 @@ class RunTraceCaseTests(unittest.TestCase): def tearDown(self): self.temp.cleanup() - def run_case(self, env, logs, tcp=False): + def run_case(self, env, logs, tcp=False, squashed=True): canned = self.root / 'canned' shutil.rmtree(canned, ignore_errors=True) canned.mkdir() @@ -136,7 +142,8 @@ class RunTraceCaseTests(unittest.TestCase): command += ['-DTRACE_IPC_CONTROL=tcp://127.0.0.1:1', '-DTRACE_IPC_TOKEN=x'] command += ['-P', str(SCRIPT)] result = subprocess.run(command, env=environment, capture_output=True, text=True) - return result.returncode, squash(result.stdout + result.stderr) + output = result.stdout + result.stderr + return result.returncode, squash(output) if squashed else output def split_env(self, transport, wire=False): env = {'MOBILEGL_TRANSPORT': 'spawn' if transport == 'tcp' else transport, @@ -264,6 +271,20 @@ class RunTraceCaseTests(unittest.TestCase): 'MGPipe: Fatal{PipeVerifyDiffer, "GetRenderStateParameters@Clear"}')} self.expect_red({'MOBILEGL_PIPE_VERIFY': '1'}, logs, 'MGPipe Fatal(s) under MOBILEGL_PIPE_VERIFY') + def test_v4_an_aborted_verify_run_names_its_divergence(self): + # The comparator aborts the replay before result.json, so the script dumps the library log; + # the retrace-verify control requires that dump to carry the divergence it caused. + fatal = 'MGPipe: Fatal{PipeVerifyDiffer, "GetRenderStateParameters@Clear"} field=3 where=entry' + env = {'MOBILEGL_PIPE_VERIFY': '1', 'STUB_ABORT': '1'} + rc, output = self.run_case(env, {'mobilegl.log': lines('MGPipe: verify armed', fatal)}) + self.assertNotEqual(rc, 0, output) + transcript = self.root / 'control.txt' + for log, expected in ((lines('MGPipe: verify armed', fatal), 0), (lines('MGPipe: verify armed'), 1)): + _, raw = self.run_case(env, {'mobilegl.log': log}, squashed=False) + transcript.write_text(''.join('1: ' + row + '\n' for row in raw.splitlines())) + check = subprocess.run(['bash', '-c', VERIFY_CONTROL_REASON, 'reason', str(transcript)]) + self.assertEqual(check.returncode, expected, raw) + def test_v3_verify_green(self): rc, output = self.run_case({'MOBILEGL_PIPE_VERIFY': '1'}, {'mobilegl.log': lines('MGPipe: verify armed')}) self.assertEqual(rc, 0, output)