diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 05a57124..475d0a31 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1034,6 +1034,52 @@ jobs: fi echo "${#logs[@]} entries resolved inproc; ${armed} of them armed the comparator" + # NEGATIVE CONTROL A ON THE SPLIT ARM - integration-verify's step A, same knob, same shape + # (this step passes when ctest FAILS), aimed at the ambient VerifySplit. entries of BOTH + # backends. Measured at landing: 10 of 10 red, each with + # Fatal{PipeVerifyDiffer, "GetRenderStateParameters@Clear"|"@DrawArrays", ..., where=entry}. + - name: Negative control A (split) - a corrupted snapshot field must turn the lane red + working-directory: build-verify-split + env: + MOBILEGL_ITEST_REQUIRE_GPU: "1" + MOBILEGL_PIPE_VERIFY_CORRUPT: GetRenderStateParameters + run: | + FILTER='(DirectGLES|DirectVulkan)\.VerifySplit\..*ClearThenReadPixels' + matched=$(ctest -N -L '^integration-verify-split$' -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:') + if [ "${matched}" -lt 2 ]; then + echo "::error::split negative control A selected ${matched} tests; its filter no longer matches both backends" + exit 1 + fi + if ctest --output-on-failure -L '^integration-verify-split$' -R "${FILTER}" --no-tests=error; then + echo "::error::MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters left ${matched} verify-split entries GREEN. The comparator is not comparing on the split arm." + exit 1 + fi + echo "the corrupted field turned ${matched} selected split entries red, as it must" + + # NEGATIVE CONTROL B ON THE SPLIT ARM, WITH THE SPLIT ARM'S PAIR. The monolith pair + # (GenerateMipmap:GetActiveTextureUnit) is inert here - the split backend's generate_mipmap + # resolves its texture by the verb's handle and never reads the active unit - so this step + # withholds ReadPixels:GetPixelStoreParameters, which the server reads under its own stamp. + # Measured at landing: 2 of 2 red, each child dying with + # Fatal{UnmigratedPipeInput, "GetPixelStoreParameters@ReadPixels"} on mgl-srv-apply. + - name: Negative control B (split) - an omitted stamp must turn the lane red on that verb + working-directory: build-verify-split + env: + MOBILEGL_ITEST_REQUIRE_GPU: "1" + MOBILEGL_PIPE_POISON_OMIT: ReadPixels:GetPixelStoreParameters + run: | + FILTER='(DirectGLES|DirectVulkan)\.VerifySplit\.PoisonOmissionScenario\.WithoutOmissionCompletes' + matched=$(ctest -N -L '^integration-verify-split$' -R "${FILTER}" | grep -cE '^ *Test *#[0-9]+:') + if [ "${matched}" -lt 2 ]; then + echo "::error::split negative control B selected ${matched} tests; its filter no longer matches both backends" + exit 1 + fi + if ctest --output-on-failure -L '^integration-verify-split$' -R "${FILTER}" --no-tests=error; then + echo "::error::MOBILEGL_PIPE_POISON_OMIT=ReadPixels:GetPixelStoreParameters left the verify-split lane GREEN. The server-side poison is not armed on the split arm." + exit 1 + fi + echo "the omitted stamp turned the split lane red, as it must" + - name: Upload verify-split lane logs if: always() uses: actions/upload-artifact@v7 diff --git a/MobileGL/MG_Backend/MGPipe/PipeInputs.cpp b/MobileGL/MG_Backend/MGPipe/PipeInputs.cpp index 03377b50..37e553b7 100644 --- a/MobileGL/MG_Backend/MGPipe/PipeInputs.cpp +++ b/MobileGL/MG_Backend/MGPipe/PipeInputs.cpp @@ -54,6 +54,60 @@ namespace MobileGL::MG_Pipe { namespace { Uint64 g_residualPulls = 0; +#if MOBILEGL_PIPE_VERIFY + // ---- P7 wave 3 (V1): negative control B's SERVER half ---------------------------- + // + // MOBILEGL_PIPE_POISON_OMIT withholds the STAMP of one (verb, field) pair, and until a + // verify build first ran on the split arm it was only ever read by the client's filler + // (MG_Impl/Pipe/PipeFill.cpp's ParsePoisonOmissionKnob). On the split arm that is not + // where the stamp the backend reads comes from: the stamp below re-stamps every field + // the verb's class can answer out of applied records, so it restamped the very field + // the client had withheld and the control went GREEN - measured, both backends, the + // child's glGenerateMipmap returned and the process exited 0 with + // GenerateMipmap:GetActiveTextureUnit omitted. The knob means "this verb's stamp of this + // field was never written", and under a transport the stamp that answers the backend is + // this one, so this one honours it too. + // + // PARSED HERE, NOT SHARED WITH THE CLIENT'S PARSER, because this file is in the server + // library and MG_Impl is not. A malformed value is the client's Fatal{PipeVerifyBadKnob} + // (it parses first, at the first fill of the process); this side simply does not arm. + // VERIFY BUILDS ONLY: the knob belongs to the verify harness, and a push build's stamp + // stays the branch-free fill the P5d profile asked for. + struct ServerPoisonOmission { + Bool Parsed = false; + String Value; + Bool Armed = false; + MGPipeVerb Verb = MGPipeVerb::kVerbCount; + MGPipeInputField Field = MGPipeInputField::kFieldCount; + }; + ServerPoisonOmission g_serverOmission; + + const ServerPoisonOmission& ServerOmission() { + const String& knob = MG_Config::Features.PipePoisonOmit; + if (g_serverOmission.Parsed && knob.size() == g_serverOmission.Value.size() && + (knob.empty() || knob == g_serverOmission.Value)) { + return g_serverOmission; + } + g_serverOmission = ServerPoisonOmission{}; + g_serverOmission.Parsed = true; + g_serverOmission.Value = knob; + const auto colon = knob.find(':'); + if (colon == String::npos || colon == 0 || colon + 1 >= knob.size()) return g_serverOmission; + const auto verb = MGPipeFindVerb(knob.substr(0, colon).c_str()); + const auto field = MGPipeFindInputField(knob.substr(colon + 1).c_str()); + if (!verb || !field) return g_serverOmission; + g_serverOmission.Armed = true; + g_serverOmission.Verb = *verb; + g_serverOmission.Field = *field; + return g_serverOmission; + } + + Bool ServerOmitsStamp(MGPipeVerb verb, MGPipeInputField field) { + const ServerPoisonOmission& omission = ServerOmission(); + return omission.Armed && omission.Verb == verb && omission.Field == field; + } +#endif // MOBILEGL_PIPE_VERIFY + // The strict arm of R-7.3. Same first line as the ordinary poison Fatal, so every // existing filter on Fatal{UnmigratedPipeInput still matches, plus the class and the // phase that retires it - a strict abort that did not say which phase owes the answer @@ -415,6 +469,15 @@ namespace MobileGL::MG_Pipe { const Uint64 bit = (answerable.Words[i / 64] >> (i % 64)) & Uint64{1}; filled.FilledGen[i] = serial & (Uint64{0} - bit); } +#if MOBILEGL_PIPE_VERIFY + // Negative control B's server half (ServerPoisonOmission above): the one withheld pair. + { + const ServerPoisonOmission& omission = ServerOmission(); + if (omission.Armed && omission.Verb == verb) { + filled.FilledGen[static_cast(omission.Field)] = 0; + } + } +#endif MGPipeStampAccess::SetServerStamped(inputs, true); // P5f (f1): the server block's identity rides the stamp (CONTRACT-P5E §3.2's // "SetIdentity moves to ApplyOne" - the stamp is the per-verb half of ApplyOne, and @@ -438,6 +501,12 @@ namespace MobileGL::MG_Pipe { // AND A READ OUTSIDE THE VERB'S OWN CLASS IS STILL FATAL even for a BARRIER-PULLED // field: the value it would be answered with was never copied for this verb, so // counting it would trade a loud staleness for a quiet one. +#if MOBILEGL_PIPE_VERIFY + // An injected omission is reported as what it models - a stamp nobody wrote, the + // client filler's own Fatal{UnmigratedPipeInput, "@"} - and never as the + // RoleViolation below, which would blame the wire for a withdrawal the control made. + if (serverStamped && ServerOmitsStamp(verb, field)) MGPipeInputPoisonFatalForVerb(field, verb); +#endif if (!serverStamped || kMGPipeFieldOwnership[static_cast(field)] != MGPipeFieldOwnership::kBarrierPulled || !FieldIsInVerbClass(field, verb)) { diff --git a/MobileGL/MG_IntegrationTest/CMakeLists.txt b/MobileGL/MG_IntegrationTest/CMakeLists.txt index af4a35ab..6263bd2b 100644 --- a/MobileGL/MG_IntegrationTest/CMakeLists.txt +++ b/MobileGL/MG_IntegrationTest/CMakeLists.txt @@ -3933,8 +3933,75 @@ if (MOBILEGL_BUILD_DISAGGREGATED AND MOBILEGL_PIPE_VERIFY) ENVIRONMENT "${mglItestVerifyArmEnv}\;MGITEST_PIPE_ARMING_LANE=1\;MOBILEGL_LOG_FILE_PATH=${CMAKE_CURRENT_BINARY_DIR}/pipe-verify-split-arming-${mglItestVerifyBackend}.log" RESOURCE_LOCK "pipe-verify-split-arming-${mglItestVerifyBackend}.log" ) + + # THE TWO NEGATIVE CONTROLS ON THIS ARM (CONTRACT-P7 §6), the monolith verify block's + # four entries with the split environment underneath and nothing else changed: each is + # green when its control REPORTS, so each goes red the day the comparator or the poison + # stops reaching the split arm. Their CI halves - the same knob exported against the + # ambient VerifySplit. entries, where ctest has to FAIL - live in test.yml's + # integration-verify-split job. + # + # A (G4): the corruption perturbs the snapshot arm at the CLIENT's entry compare, so the + # report lands in the client role's log, which is the half LibraryLog() reads. + gtest_discover_tests(MobileGLIntegrationTest + TEST_PREFIX "${mglItestVerifyBackend}.VerifySplitCorrupted." + TEST_FILTER "PipeVerifyArmingScenario.CorruptedFieldIsReported" + DISCOVERY_TIMEOUT 30 + PROPERTIES + LABELS "integration-verify-split" + TIMEOUT ${MGL_ITEST_VERIFY_TIMEOUT} + ENVIRONMENT "${mglItestVerifyArmEnv}\;MOBILEGL_PIPE_VERIFY_CORRUPT=GetRenderStateParameters\;MOBILEGL_PIPE_VERIFY_FATAL=0\;MOBILEGL_LOG_FILE_PATH=${CMAKE_CURRENT_BINARY_DIR}/pipe-verify-split-corrupt-${mglItestVerifyBackend}.log" + RESOURCE_LOCK "pipe-verify-split-corrupt-${mglItestVerifyBackend}.log" + ) + # B (G5): the child brings up its own inproc session from the inherited environment, and + # PoisonOmissionScenario's ReadChildLog reads BOTH role halves of the child's log, because + # under a transport the poisoned read can fire on either side of the seam. + # + # A DIFFERENT PAIR FROM THE MONOLITH ENTRIES', and the monolith pair is not "also fine + # here": GenerateMipmap:GetActiveTextureUnit is inert on this arm - the split backend's + # generate_mipmap resolves its texture from the verb's handle and never reads the active + # unit, so the child exited 0 with that stamp withheld (measured, both backends). + # ReadPixels:GetPixelStoreParameters is read on the server under the server's stamp, + # which is the stamp the knob has to withhold on this arm (PipeInputs.cpp's + # ServerPoisonOmission). PoisonOmissionScenario.cpp's kOmittedPairs carries both. + gtest_discover_tests(MobileGLIntegrationTest + TEST_PREFIX "${mglItestVerifyBackend}.VerifySplitPoisonOmitted." + TEST_FILTER "PoisonOmissionScenario.OmittedFieldAbortsOnThatVerb" + DISCOVERY_TIMEOUT 30 + PROPERTIES + LABELS "integration-verify-split" + TIMEOUT ${MGL_ITEST_VERIFY_TIMEOUT} + ENVIRONMENT "${mglItestVerifyArmEnv}\;MOBILEGL_PIPE_POISON_OMIT=ReadPixels:GetPixelStoreParameters\;MOBILEGL_LOG_FILE_PATH=${CMAKE_CURRENT_BINARY_DIR}/pipe-poison-omit-split-${mglItestVerifyBackend}.log" + RESOURCE_LOCK "pipe-poison-omit-split-${mglItestVerifyBackend}.log" + ) + # B's AMBIENT TARGET. The monolith verify lane's control-B step aims its knob at + # `Verify.PoisonOmissionScenario.WithoutOmissionCompletes`, the one integration entry that + # calls glGenerateMipmap at all - and the mechanism filter took that entry out of the + # ambient set above along with the rest of the scenario. Registered back by hand, with no + # knob, so the CI step has an entry on THIS arm to turn red. + gtest_discover_tests(MobileGLIntegrationTest + TEST_PREFIX "${mglItestVerifyBackend}.VerifySplit." + TEST_LIST "MGL_VERIFY_SPLIT_TESTS_POISON_${mglItestVerifyBackend}" + TEST_FILTER "PoisonOmissionScenario.WithoutOmissionCompletes" + DISCOVERY_TIMEOUT 30 + PROPERTIES + LABELS "integration-verify-split" + TIMEOUT ${MGL_ITEST_VERIFY_TIMEOUT} + ENVIRONMENT "${mglItestVerifyArmEnv}" + ) + list(APPEND MGL_VERIFY_SPLIT_TEST_LISTS "MGL_VERIFY_SPLIT_TESTS_POISON_${mglItestVerifyBackend}") endforeach() + # NOT ON SPAWN, AND THIS IS A PROPERTY OF THE INSTRUMENT, NOT A GAP IN THE REGISTRATION. The + # comparator is a same-address-space oracle: its two arms are the applier-written gPipeInputs + # and the frontend GLContext, and only inproc puts them in one process. Measured on this build + # (VerifySplitArming's DirectGLES entry with MOBILEGL_TRANSPORT=spawn): the client's entry + # compare aborts at the first verb with Fatal{PipeVerifyDiffer, "GetClearColor@Clear", verb=1, + # where=entry}, because the fill leaves every record-supplied field to the applier and the + # applier now writes ANOTHER process's block; and the server process runs no compare at all, + # because MG_Impl is not in it and the read hook finds no live context. A spawn verify lane + # would be red at bring-up on the first half and blind on the second. + message(STATUS "Integration tests: P7 wave 3 - the verify comparator on the split (inproc) arm, " "both backends, under integration-verify-split") endif() diff --git a/MobileGL/MG_IntegrationTest/Scenarios/PoisonOmissionScenario.cpp b/MobileGL/MG_IntegrationTest/Scenarios/PoisonOmissionScenario.cpp index acebdcb3..67404273 100644 --- a/MobileGL/MG_IntegrationTest/Scenarios/PoisonOmissionScenario.cpp +++ b/MobileGL/MG_IntegrationTest/Scenarios/PoisonOmissionScenario.cpp @@ -81,11 +81,29 @@ extern char** environ; namespace MGITest { namespace { - // What the PoisonOmitted. ctest entry and the CI negative-control step name. The pair is + // What the PoisonOmitted. ctest entries and the CI negative-control steps name. Each pair is // spelled here so the assertion below is about the exact string the poison contracts to // print (ARCHITECTURE.md 9.2: Fatal{UnmigratedPipeInput, "@"}). - constexpr const char* kOmittedVerb = "GenerateMipmap"; - constexpr const char* kOmittedField = "GetActiveTextureUnit"; + // + // TWO PAIRS SINCE P7 WAVE 3, ONE PER ARM, because the first one is structurally inert on + // the split arm. GenerateMipmap:GetActiveTextureUnit is the monolith control: the monolith + // backend resolves the texture to mip through the active unit. Under a transport it does + // not - since P5c hd the server's generate_mipmap resolves the texture from the verb's own + // handle (MGPipeApplier().VerbMipRes) and never reads the active unit - so omitting that + // stamp is omitting a stamp nobody reads, and the child exits 0 (measured on + // integration-verify-split, both backends). ReadPixels:GetPixelStoreParameters is the + // split control: the server's read_pixels backend call DOES read the pack half through the + // accessor, under the server's own stamp, which honours the knob in a verify build + // (MG_Backend/MGPipe/PipeInputs.cpp's ServerPoisonOmission). + struct OmittedPair { + const char* Verb; + const char* Field; + const char* EarlierVerbs[2]; // verbs the sequence runs BEFORE this one, which must not trip + }; + constexpr OmittedPair kOmittedPairs[] = { + {"GenerateMipmap", "GetActiveTextureUnit", {"@DrawArrays", "@DrawArrays"}}, + {"ReadPixels", "GetPixelStoreParameters", {"@DrawArrays", "@GenerateMipmap"}}, + }; constexpr const char* kFatalPrefix = "Fatal{UnmigratedPipeInput"; // Set only in the re-executed child, so the worker case below runs in that process and skips @@ -117,10 +135,41 @@ void main() { o_color = vec4(0.25, 0.5, 0.75, 1.0); } // Where the child is told to write ITS log. Empty when the lane configured no log path at // all, in which case the signal is the only evidence and the text assertions are skipped. - std::string ChildLogPath() { - const std::filesystem::path parent = LibraryLogPath(); - if (parent.empty()) return {}; - return (parent.string() + ".poison-child"); + // + // P7 WAVE 3 (V1): THIS IS A BASE NAME AND NOT A RESOLVED ONE, and the difference only + // became visible when verify and split first shared a build. It used to be + // LibraryLogPath() + ".poison-child" - and LibraryLogPath() is this lane's CLIENT half, + // which in a monolith verify build IS the raw MOBILEGL_LOG_FILE_PATH (so nothing moves + // there) but in a DISAGGREGATED one is already `.client.log`. Handing that to the + // child as its own MOBILEGL_LOG_FILE_PATH made the child's library apply the role rule a + // SECOND time, so the child wrote `.client.client.log.poison-child` while this + // parent read `.client.log.poison-child` and found nothing. The result was + // `the child aborted, but not with Fatal{UnmigratedPipeInput, "..."}. Child log:` followed + // by silence - in a run where the abort had happened exactly as designed. Measured on + // build-split with -DMOBILEGL_PIPE_VERIFY=ON, both backends, 2/1136 red in the monolith + // verify lane; there is no CI job today that configures both options at once. + std::string ChildLogBasePath() { + const char* base = std::getenv("MOBILEGL_LOG_FILE_PATH"); + if (base == nullptr || *base == '\0') return {}; + return std::string(base) + ".poison-child"; + } + + // The library's own role rule, applied to an arbitrary base. It is PipeStatsWindow.h's + // copy of Log.cpp's RoleLogPath, for the reason stated there (the helpers are hidden in + // the shipping .so and this module links that), differing only in taking the base as an + // argument rather than reading the environment. Keep all three in step. + std::string ChildRoleLogPath(const std::string& base, const char* roleSuffix) { +#if MOBILEGL_BUILD_DISAGGREGATED + const std::string::size_type slash = base.find_last_of("/\\"); + const std::string::size_type dot = base.find_last_of('.'); + if (dot == std::string::npos || (slash != std::string::npos && dot < slash)) { + return base + "." + roleSuffix; + } + return base.substr(0, dot) + "." + roleSuffix + base.substr(dot); +#else + (void)roleSuffix; + return base; +#endif } std::string ReadWholeFile(const std::string& path) { @@ -130,6 +179,20 @@ void main() { o_color = vec4(0.25, 0.5, 0.75, 1.0); } return std::string((std::istreambuf_iterator(file)), std::istreambuf_iterator()); } + // BOTH HALVES, because under a transport the poison can fire on either side of the seam: + // the fill is the client's and the read is the backend's, and in a split build the + // backend runs on the server role's thread. A reader that took the client half alone + // would report "the child aborted with no Fatal" for every server-side firing. + std::string ReadChildLog() { + const std::string base = ChildLogBasePath(); + if (base.empty()) return {}; + std::string all = ReadWholeFile(ChildRoleLogPath(base, "client")); +#if MOBILEGL_BUILD_DISAGGREGATED + all += ReadWholeFile(ChildRoleLogPath(base, "server")); +#endif + return all; + } + class PoisonOmissionScenario : public ScenarioTest { protected: // The sequence under test. Deliberately in this order: the DRAW comes first and must @@ -183,6 +246,12 @@ void main() { o_color = vec4(0.25, 0.5, 0.75, 1.0); } glFinish(); std::fprintf(stderr, "[itest] poison worker: glGenerateMipmap returned\n"); + // The split arm's verb (kOmittedPairs[1]): a readback, which is a round trip, so + // the server has applied it - and read the pack state - before this returns. + unsigned char centre[4] = {}; + glReadPixels(gl.Width() / 2, gl.Height() / 2, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, centre); + std::fprintf(stderr, "[itest] poison worker: glReadPixels returned\n"); + // The sequence is the WHOLE datum this child reports, so a GL error in it must be // part of the answer rather than something only a human reading stderr would see. // WithoutOmissionCompletes reads the child's exit status, and the status is built @@ -211,10 +280,14 @@ void main() { o_color = vec4(0.25, 0.5, 0.75, 1.0); } env.push_back(text); } env.push_back(std::string(kChildMarker) + "=1"); - const std::string childLog = ChildLogPath(); + const std::string childLog = ChildLogBasePath(); if (!childLog.empty()) { + // BOTH halves, for TruncateRoleLogs' reason one file over: a previous run's + // server log left standing would be concatenated into this run's read and a + // stale Fatal would answer for a child that never logged one. std::error_code ec; - std::filesystem::remove(childLog, ec); + std::filesystem::remove(ChildRoleLogPath(childLog, "client"), ec); + std::filesystem::remove(ChildRoleLogPath(childLog, "server"), ec); env.push_back("MOBILEGL_LOG_FILE_PATH=" + childLog); } @@ -309,19 +382,24 @@ void main() { o_color = vec4(0.25, 0.5, 0.75, 1.0); } "is what the PoisonOmitted. ctest entries set"; } const std::string knob = std::getenv("MOBILEGL_PIPE_POISON_OMIT"); - const std::string expectedPair = std::string(kOmittedField) + "@" + kOmittedVerb; - if (knob != std::string(kOmittedVerb) + ":" + kOmittedField) { - GTEST_SKIP() << "MOBILEGL_PIPE_POISON_OMIT is " << knob << ", but this case only knows " - << "how to provoke " << kOmittedVerb << ":" << kOmittedField; + const OmittedPair* pair = nullptr; + for (const OmittedPair& candidate : kOmittedPairs) { + if (knob == std::string(candidate.Verb) + ":" + candidate.Field) pair = &candidate; } + if (pair == nullptr) { + GTEST_SKIP() << "MOBILEGL_PIPE_POISON_OMIT is " << knob << ", but this case only knows " + << "how to provoke " << kOmittedPairs[0].Verb << ":" << kOmittedPairs[0].Field + << " and " << kOmittedPairs[1].Verb << ":" << kOmittedPairs[1].Field; + } + const std::string expectedPair = std::string(pair->Field) + "@" + pair->Verb; int status = 0; std::string reason; ASSERT_TRUE(RunSequenceInAChildProcess(status, reason)) << reason; - const std::string childLog = ReadWholeFile(ChildLogPath()); + const std::string childLog = ReadChildLog(); ASSERT_TRUE(WIFSIGNALED(status)) - << "with the stamp of " << expectedPair << " omitted, the glGenerateMipmap in the child " + << "with the stamp of " << expectedPair << " omitted, the " << pair->Verb << " in the child " << "had to read a field its verb never filled and abort. It " << DescribeStatus(status) << " instead - the poison is not armed (a build without MOBILEGL_PIPE_POISON, a filler " "that stamps what it was told to skip, or a backend that no longer reads the field " @@ -333,7 +411,7 @@ void main() { o_color = vec4(0.25, 0.5, 0.75, 1.0); } "Child log:\n" << childLog; - if (ChildLogPath().empty()) { + if (ChildLogBasePath().empty()) { GTEST_SKIP() << "the abort happened, but the lane set no MOBILEGL_LOG_FILE_PATH, so the " "Fatal's text cannot be read back; the PoisonOmitted. ctest entries set it"; } @@ -342,11 +420,13 @@ void main() { o_color = vec4(0.25, 0.5, 0.75, 1.0); } << "the child aborted, but not with Fatal{UnmigratedPipeInput, \"" << expectedPair << "\"} - that message is the whole diagnostic value of the poison. Child log:\n" << childLog; - EXPECT_EQ(childLog.find("@DrawArrays"), std::string::npos) - << "the draw that ran BEFORE the omitted verb also tripped the poison, so the omission " - "is not scoped to its verb: the fill classes are wrong, or the stamps are global. " - "Child log:\n" - << childLog; + for (const char* earlier : pair->EarlierVerbs) { + EXPECT_EQ(childLog.find(earlier), std::string::npos) + << "a verb that ran BEFORE the omitted one (" << earlier << ") also tripped the " + "poison, so the omission is not scoped to its verb: the fill classes are wrong, " + "or the stamps are global. Child log:\n" + << childLog; + } } // The sibling control, in the ambient Verify. lanes: the same sequence with the knob UNSET @@ -370,7 +450,7 @@ void main() { o_color = vec4(0.25, 0.5, 0.75, 1.0); } std::string reason; ASSERT_TRUE(RunSequenceInAChildProcess(status, reason)) << reason; - const std::string childLog = ReadWholeFile(ChildLogPath()); + const std::string childLog = ReadChildLog(); const std::string note = omissionArmed ? std::string(