From 9f528783134775421823699666c5884b05ac1ee9 Mon Sep 17 00:00:00 2001 From: Swung0x48 Date: Wed, 16 Sep 2026 11:09:09 -0400 Subject: [PATCH 1/3] [Feat] (MG_Remote, MG_Util): the wire ledger - maxrec/maxcap/ringwraps/ringpads/ringwaits on the MGPipe stats line and at session teardown, MOBILEGL_IPC_E2_DROP_DRAW beside the clear drop, and E3(a)'s named persistent-map-push-disabled line (ID-65) --- MobileGL/MG_Remote/Client/ClientSession.cpp | 40 +++++ MobileGL/MG_Remote/Client/ClientSession.h | 3 + MobileGL/MG_Remote/Client/EmitTables.cpp | 157 ++++++++++++++++-- MobileGL/MG_Remote/Client/EmitTables.h | 22 ++- .../MG_Remote/Client/PersistentMapTracker.cpp | 22 ++- .../MG_Remote/Client/PersistentMapTracker.h | 6 + MobileGL/MG_Remote/Wire/PipeWireCodec.cpp | 79 +++++++++ MobileGL/MG_Remote/Wire/PipeWireCodec.h | 52 ++++++ MobileGL/MG_Test/Wire/PipeWireCodecTest.cpp | 97 +++++++++++ MobileGL/MG_Util/Metrics/PipeStats.cpp | 55 ++++++ MobileGL/MG_Util/Metrics/PipeStats.h | 52 ++++++ 11 files changed, 559 insertions(+), 26 deletions(-) diff --git a/MobileGL/MG_Remote/Client/ClientSession.cpp b/MobileGL/MG_Remote/Client/ClientSession.cpp index 24b918aa..03a7b752 100644 --- a/MobileGL/MG_Remote/Client/ClientSession.cpp +++ b/MobileGL/MG_Remote/Client/ClientSession.cpp @@ -649,6 +649,7 @@ namespace MobileGL::MG_Remote::Client { // 4. and ONLY NOW may anything an emitter owns be released: a var-tail still // referenced by an unapplied record is a use-after-free the join is what prevents. LogMemory("teardown"); + LogWireLedger(); m_producer.Detach(); m_encoder = Wire::PipeWireEncoder(); m_events = Transport::EventRingConsumer(); @@ -900,6 +901,45 @@ namespace MobileGL::MG_Remote::Client { Transport::LogRoleMemory(phase, SampleMemory()); } + // R-10's AND R-9's numbers IN EVERY SPLIT PRIVATE LOG, not only in the lanes that set + // MOBILEGL_PIPE_STATS=1. + // + // WHY IT IS HERE AND NOT ONLY ON THE STATS LINE. `MGPipe stats:` is an opt-in channel: two + // ctest entries out of 21 set MOBILEGL_PIPE_STATS, and neither the retrace lanes nor the + // 19 ordinary split entries do. R-10's proof obligation is about THE PHASE, not about the + // two counting lanes - "no record on the reduced path comes near half the ring" has to be + // readable from any split run that happened, which is what ID-53's per-entry private log + // is for. One line per session teardown costs nothing and cannot be missed. + // + // IT IS ALSO WHERE THE PROOF FAILS SOFTLY. A record ABOVE the cap already aborts on the + // spot with Fatal{RingOverrun} (PipeWireCodec.cpp), so this line's job is the other half: + // a maximum that is merely CLOSE to the cap is not a crash and would otherwise be + // invisible until the day a workload crossed it. The percentage is printed for exactly + // that reason, and R-10 names the integrator as the person who decides between early + // chunking and a bigger default ring when it climbs. + void ClientSession::LogWireLedger() const { + const Uint64 maxRecord = m_encoder.MaxRecordBytesSeen(); + const Uint64 cap = m_encoder.MaxRecordBytesCap(); + // Integer permille rather than a float: this file has no and a "%.1f" of a + // ratio nobody can reproduce by hand is worse than two integers. + const Uint64 permille = cap != 0 ? (maxRecord * 1000ull) / cap : 0ull; + MGLOG_I("MG_Remote client: wire ledger: maxrec=%llu maxrecop=%s cap=%llu (%llu.%llu%% of " + "RingProducer::MaxRecordBytes, half of a %llu byte SEG_CMD) cmdbytes=%llu " + "ringwraps=%llu ringpads=%llu " + "ringwaits=%llu emitseq=%llu - R-10's proof obligation and R-9's producer " + "readings, published from the session that produced them", + static_cast(maxRecord), m_encoder.MaxRecordOpName(), + static_cast(cap), + static_cast(permille / 10), + static_cast(permille % 10), + static_cast(cap * 2), + static_cast(m_encoder.CmdBytesWritten()), + static_cast(m_encoder.CmdWraps()), + static_cast(m_encoder.CmdWrapPads()), + static_cast(m_encoder.StageReclaimWaits()), + static_cast(m_encoder.EmitSeq())); + } + #undef MGP5_C0_STUB } // namespace MobileGL::MG_Remote::Client diff --git a/MobileGL/MG_Remote/Client/ClientSession.h b/MobileGL/MG_Remote/Client/ClientSession.h index 2f971f9c..be165629 100644 --- a/MobileGL/MG_Remote/Client/ClientSession.h +++ b/MobileGL/MG_Remote/Client/ClientSession.h @@ -200,6 +200,9 @@ namespace MobileGL::MG_Remote::Client { // Peak-RSS accounting for t1 (RoleMemory.h). Transport::RoleMemorySample SampleMemory() const; void LogMemory(const char* phase) const; + // R-10's maximum record bytes and R-9's wrap/wait counts, at teardown, in whatever log + // this process writes. See the definition for why it is not only on the stats line. + void LogWireLedger() const; private: Wire::PipeWireEncoder m_encoder; diff --git a/MobileGL/MG_Remote/Client/EmitTables.cpp b/MobileGL/MG_Remote/Client/EmitTables.cpp index 39fba630..431475ba 100644 --- a/MobileGL/MG_Remote/Client/EmitTables.cpp +++ b/MobileGL/MG_Remote/Client/EmitTables.cpp @@ -30,6 +30,7 @@ #include #include +#include #include #include @@ -64,27 +65,85 @@ namespace MobileGL::MG_Remote::Client { Bool g_dropClearEmission = false; Uint64 g_droppedClearEmissions = 0; + Bool g_dropDrawEmission = false; + Uint64 g_droppedDrawEmissions = 0; + Uint64 g_presentOrdinal = 0; + Uint64 g_publishedMaxRecordBytes = 0; // E2's control has to be armable from OUTSIDE the process that runs the replay, because - // the statement it makes is about a trace lane and not about a unit case: "drop one - // Clear emission and OpenRA's SSIM falls below 0.99". A recompile would make the control + // the statement it makes is about a trace lane and not about a unit case: "drop an + // emission and OpenRA's SSIM falls below 0.99". A recompile would make the control // arm against source text, which is ID-22(a)'s defect. // // READ WITH getenv RATHER THAN THROUGH MG_Config, DELIBERATELY AND TEMPORARILY. Config.h - // is c0's and a new MOBILEGL_IPC_* knob goes through the integrator; this is a - // NEGATIVE-CONTROL switch no operator may ever set, and it announces itself at warning - // level every time it arms so it cannot be on by accident. Flagged for adoption into - // IpcTable if the integrator wants it there. - Bool ReadDropClearFromEnvironment() { - const char* value = std::getenv("MOBILEGL_IPC_E2_DROP_CLEAR"); - const Bool armed = value != nullptr && value[0] == '1' && value[1] == '\0'; - if (armed) { - MGLOG_W("MG_Remote client: MOBILEGL_IPC_E2_DROP_CLEAR=1 - E2's NEGATIVE CONTROL is " - "armed and every glClear will be DROPPED on the wire. This arm is expected " - "to fail its SSIM threshold; a lane that stays green with it set is not " - "going through the wire at all"); + // is c0's and a new MOBILEGL_IPC_* knob goes through the integrator; these are + // NEGATIVE-CONTROL switches no operator may ever set, and they announce themselves at + // warning level every time they arm so they cannot be on by accident. Flagged for + // adoption into IpcTable if the integrator wants them there. + Bool ReadControlKnob(const char* name) { + const char* value = std::getenv(name); + return value != nullptr && value[0] == '1' && value[1] == '\0'; + } + + // ---- WHY THERE ARE TWO OF THESE KNOBS, measured rather than assumed ----------------- + // + // MOBILEGL_IPC_E2_DROP_CLEAR came first and it does exactly what it says: every glClear + // stops at the client and no Clear record reaches the ring. It STILL COULD NOT TURN THE + // E2 RETRACE RED (joint-v1.md 3: SSIM 1.000000, mismatchPixels=0 with the knob armed and + // the arming WARN in the library's own log). That is not a broken knob, it is OpenRA: + // `apitrace dump` of openra.trace over the 31249 replayed calls counts 30 glClear, 30 + // glXSwapBuffers and 788 glDrawArrays, and the final frame issues its clear at call + // 30197 and then covers the surface four times over with a terrain layer + // (`glDrawArrays(GL_TRIANGLES, first=56064, count=16128)` x4, under a scissor of + // -24,-24,688x528 over a 640x480 surface) before the snapshot at 31249. A frame that + // overdraws every pixel it clears has a picture that does not depend on the clear, so + // "drop the clear" is a control whose observable is invisible to THIS trace - R-16's + // exact defect, a gate that cannot go red for its own reason. + // + // MOBILEGL_IPC_E2_DROP_DRAW is the honest form of the same statement for a trace lane: + // drop every DrawVbo record and the picture can only be the clear colour. It is the + // control that makes "the wire carried the frame" falsifiable, because the thing it + // removes is the thing the golden is made of. + // + // DROP_CLEAR IS KEPT rather than retired: it drops a real record, it now publishes the + // count it dropped (below), and a scenario whose picture DOES depend on its clear - + // ClearThenReadPixelsScenario is the reduced path's target A - is where it is + // observable. What it is no longer allowed to be is E2's retrace control. + void ArmControlKnobs() { + g_dropClearEmission = ReadControlKnob("MOBILEGL_IPC_E2_DROP_CLEAR"); + g_dropDrawEmission = ReadControlKnob("MOBILEGL_IPC_E2_DROP_DRAW"); + if (g_dropClearEmission) { + MGLOG_W("MG_Remote client: MOBILEGL_IPC_E2_DROP_CLEAR=1 - a NEGATIVE CONTROL is " + "armed and every glClear will be DROPPED on the wire. It is observable " + "only where the picture depends on the clear: OpenRA overdraws its whole " + "surface every frame, so this knob does NOT redden the E2 retrace " + "(measured, joint-v1.md 3) - MOBILEGL_IPC_E2_DROP_DRAW is the one that " + "does. The dropped count is published on the 'E2 control armed' line"); } - return armed; + if (g_dropDrawEmission) { + MGLOG_W("MG_Remote client: MOBILEGL_IPC_E2_DROP_DRAW=1 - E2's NEGATIVE CONTROL is " + "armed and every DrawVbo record will be DROPPED on the wire. The surface " + "can then only carry the clear colour, so this arm is expected to fail its " + "SSIM threshold; a lane that stays green with it set is not going through " + "the wire at all"); + } + } + + // THE CONTROL'S OWN EVIDENCE LINE, and it is emitted per frame rather than at teardown + // on purpose: a retrace that is killed by its own timeout, or whose library never runs + // MobileGL::Destroy, would leave a teardown-only line absent and the control would then + // have to accept a bare threshold failure - which is the thing R-16 forbids. One line + // per Present, only while a knob is armed, is bounded by the frame count and present + // whatever happens afterwards. + void LogE2ControlLine(Uint64 frameOrdinal) { + if (!g_dropClearEmission && !g_dropDrawEmission) return; + MGLOG_W("MGPipe: E2 control armed - drop-draw=%d drop-clear=%d, %llu records dropped " + "on the wire (draw=%llu clear=%llu), frame %llu", + g_dropDrawEmission ? 1 : 0, g_dropClearEmission ? 1 : 0, + static_cast(g_droppedDrawEmissions + g_droppedClearEmissions), + static_cast(g_droppedDrawEmissions), + static_cast(g_droppedClearEmissions), + static_cast(frameOrdinal)); } // ID-49's two halves, in one place so the emitter and its control read the same @@ -150,9 +209,11 @@ namespace MobileGL::MG_Remote::Client { BeforeReadOnlyVerb(); if (g_dropClearEmission) { - // E2's negative control. Everything above still ran, so the only difference + // A negative control. Everything above still ran, so the only difference // between this arm and the live one is the record - which is exactly the - // statement "the picture comes from the wire" that E2 exists to prove. + // statement "the picture comes from the wire" that E2 exists to prove. Its + // OBSERVABILITY is a property of the workload, not of this branch: see + // ArmControlKnobs for the measurement that took E2's retrace off this knob. ++g_droppedClearEmissions; return; } @@ -175,6 +236,16 @@ namespace MobileGL::MG_Remote::Client { ClientSession& session = RequireSession("DrawArrays"); BeforeDrawVerb(); + if (g_dropDrawEmission) { + // E2's LOAD-BEARING negative control. Same shape as the clear drop and the same + // rule: everything above still ran - the persistent-map push and the GPU-write + // mark walk both happened - so the ONLY difference from the live arm is that + // this frame's geometry never crossed the ring. A lane that still matches its + // golden with this armed did not get its picture from the wire. + ++g_droppedDrawEmissions; + return; + } + MG_Pipe::MGPDrawInfo info{}; info.Mode = static_cast(mode); info.IndexSize = 0; // arrays @@ -409,6 +480,50 @@ namespace MobileGL::MG_Remote::Client { ClientSession& session = RequireSession("Present"); BeforeReadOnlyVerb(); + // ---- R-10's and R-9's readings, published BEFORE the present record ------------ + // + // THE FRAME BOUNDARY IS THE RIGHT PLACE and the per-record path is the wrong one: + // the encoder keeps all five as run totals, so this is five relaxed stores per + // frame rather than five per record. Guarded by Enabled() like every other counting + // site in the tree, so the cost with MOBILEGL_PIPE_STATS unset is a global load and + // a predicted branch. + // + // AND IT IS BEFORE THE EmitAndWait BELOW, WHICH IS NOT A DETAIL. PipeStats::OnPresent + // is called by the SERVER's Present - i.e. from inside the apply of the very record + // this function is about to emit - so a publish placed after it lands one frame + // late, and the FIRST summary line of every run then reads `maxrec=0 maxcap=0`. + // Measured that way once: a zero that means "not published yet" is printed in the + // same shape as a zero that means "nothing crossed", and the second one is a real + // defect (an emit table that fell through to the driver). The Present record is 24 + // bytes and cannot be the maximum, so nothing is lost by reading one record early. + if (MG_Util::PipeStats::Enabled()) { + const Wire::PipeWireEncoder& encoder = session.Encoder(); + using MG_Util::PipeStats::Gauge; + MG_Util::PipeStats::PublishGauge(Gauge::MaxRecordBytes, encoder.MaxRecordBytesSeen()); + MG_Util::PipeStats::PublishGauge(Gauge::MaxRecordBytesCap, encoder.MaxRecordBytesCap()); + MG_Util::PipeStats::PublishGauge(Gauge::RingWraps, encoder.CmdWraps()); + MG_Util::PipeStats::PublishGauge(Gauge::RingWrapPads, encoder.CmdWrapPads()); + MG_Util::PipeStats::PublishGauge(Gauge::RingWaits, encoder.StageReclaimWaits()); + + // AND THE ROW, WHENEVER THE MAXIMUM MOVES. The summary line can carry the + // number but not the name - MG_Util is below MG_Remote and has no WireOpName - + // and the name is the actionable half: R-10 makes the integrator choose between + // early chunking and a bigger ring, and that is a decision about a record + // FAMILY. ClientSession::Stop prints the same pair at teardown, but a trace + // replay never reaches it (measured: the OpenRA lane's library log ends mid-run + // with no teardown line at all), so a stats-enabled run would otherwise publish + // a size with no row. Emitted only when the maximum actually grows, so it is + // bounded by the number of distinct maxima - five or six in a whole replay. + if (encoder.MaxRecordBytesSeen() > g_publishedMaxRecordBytes) { + g_publishedMaxRecordBytes = encoder.MaxRecordBytesSeen(); + MGLOG_I("MGPipe: wire ledger: new maximum record - maxrec=%llu " + "maxrecop=%s cap=%llu (R-10's proof obligation; P5 does not chunk)", + static_cast(g_publishedMaxRecordBytes), + encoder.MaxRecordOpName(), + static_cast(encoder.MaxRecordBytesCap())); + } + } + MG_Pipe::MGPPresent record{}; // FrameSerial 0 = "the server stamps its own". P5 has no client-side present credit // (MOBILEGL_IPC_PRESENT_CREDIT is P6's), so a client-minted serial would be a second @@ -423,6 +538,9 @@ namespace MobileGL::MG_Remote::Client { // keyed on pActiveBackendObject.get() (Core.cpp:34) and that pointer never changes // under split. session.PumpControlPlane(); + + ++g_presentOrdinal; + LogE2ControlLine(g_presentOrdinal); } // ============================================================================= @@ -599,7 +717,7 @@ namespace MobileGL::MG_Remote::Client { "the three classes no longer partition the 71 slots"); MG_Backend::GlobalBackendFunctionsTable BuildRemoteEmitTable() { - g_dropClearEmission = ReadDropClearFromEnvironment(); + ArmControlKnobs(); MG_Backend::GlobalBackendFunctionsTable table{}; // ---- class C first, so that a slot forgotten below stays Fatal rather than null. @@ -653,6 +771,9 @@ namespace MobileGL::MG_Remote::Client { void SetDropClearEmissionForNegativeControl(Bool drop) { g_dropClearEmission = drop; } Uint64 DroppedClearEmissions() { return g_droppedClearEmissions; } + void SetDropDrawEmissionForNegativeControl(Bool drop) { g_dropDrawEmission = drop; } + Uint64 DroppedDrawEmissions() { return g_droppedDrawEmissions; } + Bool ReadbackPackStateIsTightForTest(GLsizei width, Uint64 bytesPerPixel, const PixelStoreParameters& pack) { return ReadbackPackStateIsTight(width, bytesPerPixel, pack); diff --git a/MobileGL/MG_Remote/Client/EmitTables.h b/MobileGL/MG_Remote/Client/EmitTables.h index cb791e46..6a75dc88 100644 --- a/MobileGL/MG_Remote/Client/EmitTables.h +++ b/MobileGL/MG_Remote/Client/EmitTables.h @@ -102,19 +102,27 @@ namespace MobileGL::MG_Remote::Client { Uint32 LocallyAnsweredSlotCount(); // class A - answered from the caps mirror, R-15 Uint32 UnmigratedSlotCount(); // class C - Fatal{UnmigratedVerb} - // THE E2 NEGATIVE CONTROL (t1's debt against c1, BRIEF §7). When set, the Clear emitter + // THE E2 NEGATIVE CONTROLS (t1's debt against c1, BRIEF §7). When set, the named emitter // SKIPS its record - it still runs the pre-verb hooks and still returns - so a replay that - // is really going through the wire loses one clear per frame and its SSIM falls below the - // 0.99 threshold, while a replay that fell through to the driver is unaffected. It is a - // function rather than a knob in Config.h for two reasons: the control has to be settable - // from a test process that has already started, and a knob would be a - // MOBILEGL_IPC_-shaped name for something no operator may ever set. + // is really going through the wire loses that verb while a replay that fell through to the + // driver is unaffected. They are functions rather than knobs in Config.h for two reasons: + // a control has to be settable from a test process that has already started, and a knob + // would be a MOBILEGL_IPC_-shaped name for something no operator may ever set. // - // Emissions actually skipped, so the control can assert that it DID something rather than + // Emissions actually skipped, so a control can assert that it DID something rather than // that a picture changed - a control that silently never fired is the third shape of R-16's // "a gate that cannot go red for its own reason". + // + // WHICH ONE E2'S RETRACE USES, and it is not the clear. Measured on the joint head: with + // MOBILEGL_IPC_E2_DROP_CLEAR=1 armed and its WARN in the library's own log, the OpenRA + // retrace under inproc still scored ssim=1.000000 / mismatchPixels=0 (joint-v1.md §3), + // because OpenRA covers every pixel it clears before the snapshot. Dropping the DRAWS is + // the control whose observable the golden is actually made of. See EmitTables.cpp's + // ArmControlKnobs for the trace census that settled it. void SetDropClearEmissionForNegativeControl(Bool drop); Uint64 DroppedClearEmissions(); + void SetDropDrawEmissionForNegativeControl(Bool drop); + Uint64 DroppedDrawEmissions(); // ID-47. The CLIENT refuses a readback whose answer would not fit a reply slot, BEFORE it // emits the record, and names the read. THE REFUSAL ITSELF IS s1's - diff --git a/MobileGL/MG_Remote/Client/PersistentMapTracker.cpp b/MobileGL/MG_Remote/Client/PersistentMapTracker.cpp index d3cda550..252d9a9b 100644 --- a/MobileGL/MG_Remote/Client/PersistentMapTracker.cpp +++ b/MobileGL/MG_Remote/Client/PersistentMapTracker.cpp @@ -85,7 +85,27 @@ namespace MobileGL::MG_Remote::Client { // here would make the control green for the wrong reason - it has to disable the // push, so that PersistentCoherentMapScenario draws the last uploaded bytes and goes // red exactly the way an unpushed map does. - if (blockBytes == 0) return; + // + // AND IT SAYS SO, ONCE. Until now this was a silent `return`, so E3(a)'s red could + // only ever be the scenario's pixel assertion and the control had no way to tell "the + // push was disabled" apart from "the push was never armed, or never reached, or the + // knob never got here" (joint-v1.md §3: "There is no Fatal for block size zero"; + // ID-65 assigns the line to x2). The control now requires BOTH: the pixel red AND + // this line in the entry's own private log. It is MGLOG_W and not a Fatal because 0 + // is a legal configured value whose whole purpose is to keep running with the push + // off; aborting here would turn every E3(a) entry into a subprocess abort and take + // the pixel evidence with it. + if (blockBytes == 0) { + if (!m_blockZeroAnnounced) { + m_blockZeroAnnounced = true; + MGLOG_W("MGPipe: persistent-map push disabled - MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0 " + "is exit gate E3(a)'s NEGATIVE CONTROL, not 'unlimited': a live " + "persistent WRITE mapping's dirty blocks are NOT being pushed, so the " + "server draws whatever bytes last crossed by some other route. A lane " + "that stays green with this set is not getting its pixels from the push"); + } + return; + } const auto range = buffer.GetMappedRange(); const Uint64 begin = static_cast(range.start); diff --git a/MobileGL/MG_Remote/Client/PersistentMapTracker.h b/MobileGL/MG_Remote/Client/PersistentMapTracker.h index c8326a73..cffca073 100644 --- a/MobileGL/MG_Remote/Client/PersistentMapTracker.h +++ b/MobileGL/MG_Remote/Client/PersistentMapTracker.h @@ -96,6 +96,7 @@ namespace MobileGL::MG_Remote::Client { void ResetCountersForTest() { m_blocksPushed = 0; m_bytesPushed = 0; + m_blockZeroAnnounced = false; } void ClearForTest() { m_livePersistentMaps.clear(); @@ -111,6 +112,11 @@ namespace MobileGL::MG_Remote::Client { UnorderedMap m_livePersistentMaps; Uint64 m_blocksPushed = 0; Uint64 m_bytesPushed = 0; + // E3(a)'s diagnostic is emitted ONCE per process. PushBlocksFor runs at every validate + // point of every member, so an unlatched MGLOG_W would be one line per draw per mapping + // - and a control that has to grep a log cannot tell a message that fired from a + // message that flooded. + Bool m_blockZeroAnnounced = false; }; // What the client's emit table calls immediately BEFORE emitting any verb that can read a diff --git a/MobileGL/MG_Remote/Wire/PipeWireCodec.cpp b/MobileGL/MG_Remote/Wire/PipeWireCodec.cpp index 1f901c27..6085d7f6 100644 --- a/MobileGL/MG_Remote/Wire/PipeWireCodec.cpp +++ b/MobileGL/MG_Remote/Wire/PipeWireCodec.cpp @@ -770,6 +770,15 @@ namespace MobileGL::MG_Remote::Wire { // One try at reclaiming what the server has already retired. A second failure // means the bytes genuinely do not fit, which R-10 says P5 does not chunk and // must instead prove it never needs to. + // + // AND THIS IS P5'S ONE REAL BACK-PRESSURE EVENT, so it is counted here and + // published as `ringwaits=`. Reaching this line means the producer could not + // place a blob until the CONSUMER had retired earlier ones - the producer's + // progress depended on retiredSeq, which is exactly what R-9's "batching may + // only delay a watermark" is about. Exit gate E3(e)'s small-ring lane exists + // to make it happen at least once; a lane that never reaches it has a ring + // that is small only in its environment block. + ++m_stageReclaimWaits; ReclaimStagedBytes(); } } @@ -889,8 +898,48 @@ namespace MobileGL::MG_Remote::Wire { if ((callFlags & static_cast(kHasBlob)) != 0) ringFlags |= Transport::kRecHasBlob; if ((callFlags & static_cast(kVarTail)) != 0) ringFlags |= Transport::kRecVarTail; + // BOTH WRAP READINGS ARE TAKEN FROM THE PRODUCER'S OWN CURSOR, not from a flag Reserve + // does not return. The cursor is a monotonic byte count and the ring is indexed + // `cursor & mask`, so `cursor / capacity` is the number of times the byte area has been + // reused - and Reserve advances the cursor by `total` for a contiguous record and by + // `spaceToEnd + total` when it had to lay a kRecPad filler down to the boundary first + // (Ring.cpp). Everything below is exact arithmetic over those two facts, which keeps + // both counters in the encoder - where R-10's maximum already lives - rather than + // adding members to a Transport class the wire package does not own. + // + // AND THEY ARE TWO COUNTERS BECAUSE THEY ARE TWO EVENTS, which one measurement made + // unmissable: a workload whose records repeat at a uniform stride that DIVIDES the + // capacity lands on the boundary exactly, every time, for ever. Driving 1310824 bytes + // of clears and draws through a 1 MiB SEG_CMD produced ZERO pads - the ring went round + // once and a half and never straddled - so "did a pad happen" is NOT the question "did + // this ring wrap", and a lane that asked the first one while meaning the second would + // have gone red for a property of its own arithmetic. + // + // m_cmdWraps the head crossed a multiple of the capacity: the ring went round. + // Guaranteed once more bytes are written than the ring holds, which + // is what makes it something exit gate E3(e) can ASSERT. + // m_cmdWrapPads a kRecPad filler was laid because a record would have straddled + // the boundary. R-9's "a pad does not advance seq, both sides skip + // it and count again" is about THIS one, and it is RECORDED rather + // than asserted, because whether it ever happens is a property of + // the record sizes and not of the ring. + const Uint64 headBeforeReserve = m_cmd->LocalHead(); void* slot = m_cmd->Reserve(static_cast(op), ringFlags, total - sizeof(MGPWireRecHeader)); + if (slot != nullptr) { + const Uint64 headAfterReserve = m_cmd->LocalHead(); + if ((headAfterReserve - headBeforeReserve) > total) { + ++m_cmdWrapPads; + } + const Uint64 capacity = m_cmd->Capacity(); + if (capacity != 0) { + // A record is capped at capacity/2 and its pad at capacity/2 too, so one + // Reserve can cross at most one boundary; the subtraction is still written as + // a difference of quotients rather than as a Bool, because that stays correct + // if the cap ever changes. + m_cmdWraps += (headAfterReserve / capacity) - (headBeforeReserve / capacity); + } + } if (slot == nullptr) { // The ring is full, not the record too big - Reserve refuses an oversized record // above, and we already proved this one is not. The caller publishes, waits for @@ -967,6 +1016,13 @@ namespace MobileGL::MG_Remote::Wire { if (total > m_maxRecordBytes) { m_maxRecordBytes = total; + // WHICH ROW IT WAS, not just how big. R-10 makes the integrator choose between early + // chunking and a bigger default ring when the maximum climbs, and that choice is + // about a specific record family - a var-tail whose length the GL limits bound, or + // one the emitter has to split itself. A number with no row attached leaves the + // reader to guess which, and the guess in c1-v2.md §10 was SetGlobalConstants while + // the measured answer on the reduced path is a different row entirely. + m_maxRecordOp = op; } ++m_emitSeq; // The stage mark: where SEG_STAGE stood once everything this record names had been @@ -1077,6 +1133,29 @@ namespace MobileGL::MG_Remote::Wire { Uint64 PipeWireEncoder::MaxRecordBytesSeen() const { return m_maxRecordBytes; } + const char* PipeWireEncoder::MaxRecordOpName() const { + return m_maxRecordOp == MG_Pipe::MGPWireOp::kOpCount ? "none" : WireOpName(m_maxRecordOp); + } + + Uint64 PipeWireEncoder::MaxRecordBytesCap() const { + // Read from the ring, not recomputed from MOBILEGL_IPC_RING_MB: the number the proof + // has to hold against is the capacity this process's producer actually got, and the + // two differ the moment a session clamps or rounds the configured size. + return (m_cmd != nullptr && m_cmd->Valid()) ? m_cmd->MaxRecordBytes() : 0; + } + + Uint64 PipeWireEncoder::CmdWraps() const { return m_cmdWraps; } + + Uint64 PipeWireEncoder::CmdWrapPads() const { return m_cmdWrapPads; } + + Uint64 PipeWireEncoder::StageReclaimWaits() const { return m_stageReclaimWaits; } + + Uint64 PipeWireEncoder::CmdBytesWritten() const { + // LocalHead(), not RingControl::head: the producer's own cursor includes records + // reserved but not yet published, and this number is about what the PRODUCER wrote. + return (m_cmd != nullptr && m_cmd->Valid()) ? m_cmd->LocalHead() : 0; + } + // --------------------------------------------------------------------------------- // Decoder // --------------------------------------------------------------------------------- diff --git a/MobileGL/MG_Remote/Wire/PipeWireCodec.h b/MobileGL/MG_Remote/Wire/PipeWireCodec.h index ea2360bb..acbf108f 100644 --- a/MobileGL/MG_Remote/Wire/PipeWireCodec.h +++ b/MobileGL/MG_Remote/Wire/PipeWireCodec.h @@ -276,6 +276,54 @@ namespace MobileGL::MG_Remote::Wire { // R-10's proof obligation: the largest single record this encoder has written. Uint64 MaxRecordBytesSeen() const; + // The op whose record set that maximum, by name, or "none" before any record. Published + // beside the number so R-10's integrator decision names a row rather than a size. + const char* MaxRecordOpName() const; + + // THE CAP THAT NUMBER IS PROVED AGAINST, read from the ring rather than recomputed. + // RingProducer::MaxRecordBytes() == Capacity()/2, and Capacity() is + // MOBILEGL_IPC_RING_MB. Published beside MaxRecordBytesSeen() so a reader never has to + // multiply an environment variable to know whether the proof holds - which is the one + // arithmetic step between "4 MiB" and "half of the ring this process actually got". + // 0 when this encoder has no command ring (a default-constructed one). + Uint64 MaxRecordBytesCap() const; + + // ---- R-9's producer readings, and why they are three rather than one --------------- + // + // `CmdWraps()` counts SEG_CMD going ROUND: the number of times the producer's monotonic + // head crossed a multiple of the ring capacity and the byte area was reused from the + // start. It is what exit gate E3(e)'s small-ring lane asserts, because it is the one + // that is GUARANTEED once a workload writes more bytes than the ring holds, and + // therefore the one a lane can be red for not reaching. + // + // `CmdWrapPads()` counts the kRecPad fillers Reserve lays when a record would have + // STRADDLED that boundary. R-9's last clause - "a pad record does not advance seq, both + // sides must skip it and count again" - is about this one, and it is RECORDED rather + // than asserted: measured, a stream of clears and draws repeats at a stride that + // divides a power-of-two capacity exactly, so 1310824 bytes through a 1 MiB SEG_CMD + // produced one and a half trips round the ring and ZERO pads. A gate written against + // this number would have been red for the arithmetic of the record catalogue rather + // than for anything about the ring. + // + // `StageReclaimWaits()` counts every SEG_STAGE allocation that did not fit until the + // encoder reclaimed the runs the server had already retired - i.e. every time the + // producer's progress depended on the consumer's retiredSeq. That is the honest + // back-pressure reading in P5, and the reason the command ring has none: the verb + // barrier makes EmitAndWait wait for appliedSeq after EVERY record (R-1), so at most + // one record is ever in flight on SEG_CMD and a full command ring is not a wait but a + // Fatal{RingOverrun} (ClientSession.cpp). Publishing a "command ring waits" counter + // that can only ever be zero-or-dead is the decoration this file's counters are not. + Uint64 CmdWraps() const; + Uint64 CmdWrapPads() const; + Uint64 StageReclaimWaits() const; + + // Bytes this encoder has ever written into SEG_CMD, pad fillers included: the + // producer's monotonic head cursor. It is the DENOMINATOR the wrap count only means + // anything against - "0 wraps" is a defect when the run pushed more bytes than the ring + // holds and a tautology when it pushed fewer, and only this number tells those apart. + // It is also how E3(e)'s lane knows when it has driven enough work, without guessing a + // record size. 0 when there is no command ring. + Uint64 CmdBytesWritten() const; private: // {the record's seq, the SEG_STAGE cursor just past everything that record named}. @@ -307,6 +355,10 @@ namespace MobileGL::MG_Remote::Wire { SegmentTable* m_segments = nullptr; Uint64 m_emitSeq = kInvalidSeq; Uint64 m_maxRecordBytes = 0; + MG_Pipe::MGPWireOp m_maxRecordOp = MG_Pipe::MGPWireOp::kOpCount; + Uint64 m_cmdWraps = 0; + Uint64 m_cmdWrapPads = 0; + Uint64 m_stageReclaimWaits = 0; Vector m_stageMarks; SizeT m_stageMarkFront = 0; Uint8* m_stageBase = nullptr; diff --git a/MobileGL/MG_Test/Wire/PipeWireCodecTest.cpp b/MobileGL/MG_Test/Wire/PipeWireCodecTest.cpp index 4e1b8508..2dd13843 100644 --- a/MobileGL/MG_Test/Wire/PipeWireCodecTest.cpp +++ b/MobileGL/MG_Test/Wire/PipeWireCodecTest.cpp @@ -1161,6 +1161,74 @@ TEST_F(PipeWireCodecTest, MaxRecordBytesSeenStaysFarBelowHalfTheRing) { // MGPFramebufferState at 304, so a record only ever grows through its TAIL - which is why // the counter is on the encoder and not a constant. EXPECT_LT(8u + sizeof(MGPFramebufferState), wire.Cmd().MaxRecordBytes()); + + // AND THE CAP THE PROOF IS AGAINST IS PUBLISHED BY THE ENCODER ITSELF. Everything outside + // MG_Remote - the summary line's `maxrec=`/`maxcap=`, the session's teardown ledger, the + // integration lanes' Harness/WireLedgerChecks - compares against MaxRecordBytesCap() rather + // than against MOBILEGL_IPC_RING_MB / 2, because the cap moved twice in this phase without + // the environment variable changing (see the paragraph above). If those two ever disagree, + // every published `maxrec` percentage is measured against the wrong denominator, and this + // is the case that says so. + EXPECT_EQ(wire.Encoder().MaxRecordBytesCap(), wire.Cmd().MaxRecordBytes()); + EXPECT_EQ(wire.Encoder().MaxRecordBytesCap(), Wire2::kCmdBytes / 2); +} + +TEST_F(PipeWireCodecTest, TheCommandRingWrapCountIsWhatTheHeadActuallyDid) { + // R-9's wrap reading, and the distinction exit gate E3(e) turned out to depend on. + // + // `CmdWraps()` is the head crossing a multiple of the capacity - the ring going ROUND - + // and it is guaranteed once more bytes are written than the ring holds. `CmdWrapPads()` is + // the narrower event: a record that would have STRADDLED the boundary and needed a kRecPad + // filler, which is the case R-9's "a pad does not advance seq, both sides skip it and + // count again" is about. + // + // THEY ARE NOT THE SAME NUMBER, and assuming they were is what this case exists to + // prevent. A stream of identically sized records whose stride divides a power-of-two + // capacity lands on the boundary EXACTLY every time and never straddles it: measured in + // the split lane, 1310824 bytes of clears and draws through a 1 MiB SEG_CMD produced one + // wrap and ZERO pads. The first cut of E3(e)'s assertion read the pad count and went red + // for that arithmetic rather than for anything about the ring. + Wire2 wire; + EXPECT_EQ(wire.Encoder().CmdWraps(), 0u); + EXPECT_EQ(wire.Encoder().CmdWrapPads(), 0u); + EXPECT_EQ(wire.Encoder().CmdBytesWritten(), 0u); + + // The stride is MEASURED rather than computed from sizeof: Reserve rounds the header plus + // payload up to 8, and a case that restated that arithmetic would be asserting its own + // copy of Ring.cpp rather than what the producer did. + MGPBindRenderState bind{}; + ASSERT_NE(wire.Encoder().EncodeRecord(MGPWireOp::BindRenderState, &bind, sizeof(bind)), + kInvalidSeq); + bool applied = false; + ASSERT_TRUE(wire.PumpOne(&applied)); + const Uint64 stride = wire.Encoder().CmdBytesWritten(); + ASSERT_GT(stride, 0u); + EXPECT_EQ(wire.Encoder().CmdWraps(), 0u) << "one record cannot have taken the ring round"; + + // One trip round and a little more, draining after every record so the producer never meets + // its own tail. This is the same shape the split lane has under the verb barrier: one + // record in flight at a time, the ring recycled behind it - so a wrap here is a wrap + // there, and not an artefact of a backed-up queue. + const Uint64 records = (Wire2::kCmdBytes / stride) + 3; + for (Uint64 i = 1; i < records; ++i) { + ASSERT_NE(wire.Encoder().EncodeRecord(MGPWireOp::BindRenderState, &bind, sizeof(bind)), + kInvalidSeq) + << "the ring refused record " << i << " of " << records; + ASSERT_TRUE(wire.PumpOne(&applied)); + } + + EXPECT_GT(wire.Encoder().CmdBytesWritten(), Wire2::kCmdBytes); + EXPECT_EQ(wire.Encoder().CmdWraps(), 1u); + + // AND THE PAD COUNT IS A DIFFERENT NUMBER. With this record the stride is 24 bytes and the + // ring is 65536, which leaves a 16-byte remainder: exactly one record per trip finds fewer + // than 24 bytes to the boundary and gets a kRecPad filler. Change the stride to one that + // DIVIDES the capacity and the same trip produces no pad at all - measured in the split + // lane, where 1310824 bytes of clears and draws through a 1 MiB SEG_CMD reported + // ringwraps=1 ringpads=0. That is why exit gate E3(e) asserts the WRAP and only records + // the pad: a gate on the pad count would be a gate on the sizes in the record catalogue. + EXPECT_EQ(Wire2::kCmdBytes % stride, 16u) << "stride " << stride; + EXPECT_EQ(wire.Encoder().CmdWrapPads(), 1u); } TEST_F(PipeWireCodecTest, ABigProgramArchiveDoesNotGrowItsRecordAtAll) { @@ -1609,6 +1677,35 @@ TEST_F(PipeWireCodecTest, ANonZeroSizeWithNoSegmentIsFatal) { EXPECT_NE(r.Log.find("with no segment"), std::string::npos) << r.Log; } +TEST_F(PipeWireCodecTest, ARecordLargerThanHalfTheRingIsFatalRingOverrun) { + // R-10's PROOF OBLIGATION, FAILING ON PURPOSE - the red-once for everything the phase + // publishes as `maxrec=`. P5 does no chunking: a record above + // RingProducer::MaxRecordBytes() == Capacity()/2 must abort by name at the ENCODER, on the + // producing side, rather than becoming a nullptr from Reserve that some caller reads as + // "the ring is full, wait" - which on an EMPTY ring would be a wait that never ends. + // + // The oversized record is a REAL one from the catalogue with a long var-tail, not a forged + // header: MGPDrawInfo declares NumDraws and the encoder cross-checks the tail against the + // layout that number implies, so this is the shape a genuine emitter bug would take. + const ChildResult r = RunInChild([] { + Wire2 wire; + MGPDrawInfo info{}; + // Just over half the ring. Wire2's SEG_CMD is 64 KiB, so the cap is 32 KiB. + const Uint32 draws = + static_cast(((Wire2::kCmdBytes / 2) / sizeof(MGPDrawRange)) + 8); + info.NumDraws = draws; + std::vector ranges(draws); + (void)wire.Encoder().EncodeRecord(MGPWireOp::DrawVbo, &info, sizeof(info), ranges.data(), + ranges.size() * sizeof(MGPDrawRange)); + }); + ASSERT_TRUE(DiedOfAbort(r)) << DescribeStatus(r) << "\n" << r.Log; + EXPECT_NE(r.Log.find("Fatal{RingOverrun,"), std::string::npos) << r.Log; + EXPECT_NE(r.Log.find("exceeds RingProducer::MaxRecordBytes()"), std::string::npos) << r.Log; + // The diagnostic has to name R-10 and the decision it forces, because the person reading it + // has to choose between early chunking and a bigger ring and neither is a local fix. + EXPECT_NE(r.Log.find("does not chunk (R-10)"), std::string::npos) << r.Log; +} + TEST_F(PipeWireCodecTest, ARunThatLeavesItsSegmentIsFatal) { // R-2 arm 4. const ChildResult r = RunInChild([] { diff --git a/MobileGL/MG_Util/Metrics/PipeStats.cpp b/MobileGL/MG_Util/Metrics/PipeStats.cpp index 181c0c19..a988db8a 100644 --- a/MobileGL/MG_Util/Metrics/PipeStats.cpp +++ b/MobileGL/MG_Util/Metrics/PipeStats.cpp @@ -153,6 +153,13 @@ namespace MobileGL::MG_Util::PipeStats { Uint64 g_windowBaseGateMiss[kGateCount] = {}; Uint64 g_windowBaseFrames = 0; Bool g_shutdownDone = false; +#if MOBILEGL_PIPE_PUSH + // The gauges' storage. Relaxed atomics like every other counter here: the publisher is + // the GL thread at a frame boundary and the reader is whoever formats the line, which + // under split can be the apply thread. + Counter g_gauges[static_cast(Gauge::Count)] = {}; + constexpr Uint32 kGaugeCount = static_cast(Gauge::Count); +#endif // Frames per summary line, latched by Init() from MOBILEGL_PIPE_STATS_PERIOD. Uint64 g_summaryPeriod = kDefaultSummaryFramePeriod; @@ -258,6 +265,11 @@ namespace MobileGL::MG_Util::PipeStats { } g_frameCount.store(0, std::memory_order_relaxed); g_windowBaseFrames = 0; +#if MOBILEGL_PIPE_PUSH + for (Uint32 i = 0; i < kGaugeCount; ++i) { + g_gauges[i].store(0, std::memory_order_relaxed); + } +#endif } void EmitSummaryLine() { @@ -329,6 +341,19 @@ namespace MobileGL::MG_Util::PipeStats { Bump(g_totalCalls[index], count); } +#if MOBILEGL_PIPE_PUSH + // A STORE, NOT A BUMP, and the difference is the whole reason these are a separate kind. + // The publisher hands over its OWN run total (a maximum, or a count it has been keeping + // since the session opened), so accumulating deltas here would double every reading; and a + // maximum is not additive at all. Publishing the same value twice is a no-op, which is + // what makes it safe to call at every frame boundary. + void PublishGauge(Gauge gauge, Uint64 value) { + g_gauges[static_cast(gauge)].store(value, std::memory_order_relaxed); + } + + Uint64 GaugeValue(Gauge gauge) { return Read(g_gauges[static_cast(gauge)]); } +#endif + void CountGate(Gate gate, Bool hit) { const Uint32 index = static_cast(gate); if (hit) { @@ -489,6 +514,22 @@ namespace MobileGL::MG_Util::PipeStats { // tracks the draw count is a pull inside a loop, and one that tracks the frame count is // a pull per verb. Zero in every monolith lane by construction. line += " rsp=" + std::to_string(calls[static_cast(CallClass::ResidualPulls)]); + // P5's three wire gauges, and THEY ARE RUN TOTALS on a line whose every other field is + // a window - see the Gauge enum for the argument. `maxrec` is R-10's proof obligation + // (BRIEF 8 item 3): the largest single record this run wrote, in BYTES, beside the cap + // it has to stay under so nobody has to multiply MOBILEGL_IPC_RING_MB by hand. `maxcap` + // reads 0 when this process has no wire producer, which is what a monolith lane prints + // and is NOT the same statement as "the cap is zero". + // + // ringwraps / ringwaits are R-9's: SEG_CMD wrap pads and SEG_STAGE waits on retiredSeq. + // A small-ring lane whose ringwraps stays 0 ran the default lane's workload under a + // different environment block, which is exactly what exit gate E3(e) was recorded as + // NOT having proved (joint-v1.md 6). + line += " maxrec=" + std::to_string(Read(g_gauges[static_cast(Gauge::MaxRecordBytes)])); + line += " maxcap=" + std::to_string(Read(g_gauges[static_cast(Gauge::MaxRecordBytesCap)])); + line += " ringwraps=" + std::to_string(Read(g_gauges[static_cast(Gauge::RingWraps)])); + line += " ringpads=" + std::to_string(Read(g_gauges[static_cast(Gauge::RingWrapPads)])); + line += " ringwaits=" + std::to_string(Read(g_gauges[static_cast(Gauge::RingWaits)])); #endif line += "] gates["; for (Uint32 i = 0; i < kGateCount; ++i) { @@ -544,6 +585,20 @@ namespace MobileGL::MG_Util::PipeStats { ", \"miss\": " + std::to_string(Read(g_totalGateMiss[i])) + "}"; json += (i + 1 == kGateCount) ? "\n" : ",\n"; } +#if MOBILEGL_PIPE_PUSH + // The gauges, under their long names. Run totals here as on the summary line. + json += " },\n \"wire\": {\n"; + json += " \"max-record-bytes\": " + + std::to_string(Read(g_gauges[static_cast(Gauge::MaxRecordBytes)])) + ",\n"; + json += " \"max-record-bytes-cap\": " + + std::to_string(Read(g_gauges[static_cast(Gauge::MaxRecordBytesCap)])) + ",\n"; + json += " \"ring-wraps\": " + + std::to_string(Read(g_gauges[static_cast(Gauge::RingWraps)])) + ",\n"; + json += " \"ring-wrap-pads\": " + + std::to_string(Read(g_gauges[static_cast(Gauge::RingWrapPads)])) + ",\n"; + json += " \"ring-waits\": " + + std::to_string(Read(g_gauges[static_cast(Gauge::RingWaits)])) + "\n"; +#endif json += " },\n \"cmd-bytes-per-draw-histogram\": ["; for (Uint32 i = 0; i < kPayloadHistogramBuckets; ++i) { if (i != 0) { diff --git a/MobileGL/MG_Util/Metrics/PipeStats.h b/MobileGL/MG_Util/Metrics/PipeStats.h index 79932baa..bf7bce97 100644 --- a/MobileGL/MG_Util/Metrics/PipeStats.h +++ b/MobileGL/MG_Util/Metrics/PipeStats.h @@ -183,6 +183,58 @@ namespace MobileGL::MG_Util::PipeStats { Count }; +#if MOBILEGL_PIPE_PUSH + // P5's GAUGES, and they are a THIRD KIND of counter rather than three more CallClass rows. + // + // A ByteClass and a CallClass are SUMS this module owns and a call site increments. These + // three are neither: they are the wire producer's own running readings - a MAXIMUM and two + // RUN TOTALS that live on MG_Remote's encoder, which this module cannot see and must not + // link against (MG_Util is below MG_Remote, and the pull build has no MG_Remote at all). + // The owner publishes its current value at the frame boundary and this module prints the + // last one it was given. Summing them here would be wrong twice: a maximum is not additive, + // and the encoder already holds the run total, so adding deltas would double-count. + // + // THEY ARE RUN TOTALS ON A WINDOWED LINE, deliberately and against the file's own habit. + // Everything else on the summary line covers "since the previous line" because a run total + // over a workload whose shape changes hides the number P2 wants. These three are the + // opposite: "the largest record this run ever wrote" and "did the ring ever wrap" are + // questions about the RUN, and a windowed maximum would read 0 in every window that did not + // happen to contain the biggest record - which is the shape of a proof obligation that + // cannot fail. The label says so in the line itself (`maxrec=` is bytes, not bytes/frame). + // + // PUSH-ONLY for the reason every counter added since P2 is: the pull build must stay + // symbol-identical (gate G1), and a gauge whose only publisher is MG_Remote could never + // leave zero there. + enum class Gauge : Uint32 { + // R-10's PROOF OBLIGATION. The largest single record the wire encoder has written, in + // bytes, and the cap it must stay under - RingProducer::MaxRecordBytes() == + // MOBILEGL_IPC_RING_MB / 2. P5 does no chunking and has to prove it needs none; before + // this pair existed the only consumers of PipeWireEncoder::MaxRecordBytesSeen() were + // codec unit tests, so BRIEF 8 item 3 had no measurement from any real workload + // (joint-v1.md 5, "Maximum record bytes: NO MEASUREMENT"). + MaxRecordBytes = 0, + MaxRecordBytesCap, + // R-9's three producer readings. `RingWraps` is SEG_CMD going ROUND - the head crossing + // a multiple of the capacity - which is the event exit gate E3(e)'s small-ring lane + // asserts, because it is guaranteed once the workload writes more bytes than the ring + // holds. `RingWrapPads` is the kRecPad fillers laid when a record would have STRADDLED + // that boundary, which is R-9's "a pad does not advance seq" path and is RECORDED, not + // asserted: a uniform record stride over a power-of-two ring lands on the boundary + // exactly and never straddles it (measured). `RingWaits` is SEG_STAGE allocations that + // had to wait on the consumer's retiredSeq. See PipeWireCodec.h for why the command + // ring contributes no wait count while the verb barrier is armed. + RingWraps, + RingWrapPads, + RingWaits, + Count + }; + + // Publishes the owner's current reading. Cheap and unconditional on the caller's side: + // the call sites are per-frame, not per-record. + void PublishGauge(Gauge gauge, Uint64 value); + Uint64 GaugeValue(Gauge gauge); +#endif + // Memo gates. Each is a place where a backend decides "nothing moved, skip the work". // Hit == the gate short-circuited; Miss == it fell through and did the work. The six // are exactly the ones section 2.3.1 tabulates. From a7b43cbbcd1a93a18ac162405b507afd8d5d5cef Mon Sep 17 00:00:00 2001 From: Swung0x48 Date: Wed, 16 Sep 2026 11:09:15 -0400 Subject: [PATCH 2/3] [Test] (MG_IntegrationTest): exit gate E3(e) drives the small ring past its own capacity and asserts the wrap, and both split arms of TriangleScenario publish R-10's maximum record bytes against the ring's cap (ID-65) --- .../Harness/SplitRuntimePeek.cpp | 12 +- .../Harness/SplitRuntimePeek.h | 30 ++++ .../Harness/WireLedgerChecks.h | 146 ++++++++++++++++++ .../Scenarios/TriangleScenario.cpp | 63 ++++++++ 4 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 MobileGL/MG_IntegrationTest/Harness/WireLedgerChecks.h diff --git a/MobileGL/MG_IntegrationTest/Harness/SplitRuntimePeek.cpp b/MobileGL/MG_IntegrationTest/Harness/SplitRuntimePeek.cpp index 87c936c5..7cb8c851 100644 --- a/MobileGL/MG_IntegrationTest/Harness/SplitRuntimePeek.cpp +++ b/MobileGL/MG_IntegrationTest/Harness/SplitRuntimePeek.cpp @@ -47,7 +47,17 @@ namespace MGITest { if (session != nullptr) { // Encoder() returns the member; EmitSeq() returns m_emitSeq. Neither is a stub, and // neither emits anything - this is a read. - state.emitSeq = session->Encoder().EmitSeq(); + const MobileGL::MG_Remote::Wire::PipeWireEncoder& encoder = session->Encoder(); + state.emitSeq = encoder.EmitSeq(); + // The producer's ledger. Every one of these is a plain member read on the encoder + // or on the RingProducer it holds; none of them emits, publishes or waits, so a + // case may read them between two GL calls without changing what the next record is. + state.maxRecordBytes = encoder.MaxRecordBytesSeen(); + state.maxRecordBytesCap = encoder.MaxRecordBytesCap(); + state.cmdWraps = encoder.CmdWraps(); + state.cmdWrapPads = encoder.CmdWrapPads(); + state.cmdBytesWritten = encoder.CmdBytesWritten(); + state.stageReclaimWaits = encoder.StageReclaimWaits(); } #endif return state; diff --git a/MobileGL/MG_IntegrationTest/Harness/SplitRuntimePeek.h b/MobileGL/MG_IntegrationTest/Harness/SplitRuntimePeek.h index 3ee52772..9451f78e 100644 --- a/MobileGL/MG_IntegrationTest/Harness/SplitRuntimePeek.h +++ b/MobileGL/MG_IntegrationTest/Harness/SplitRuntimePeek.h @@ -70,6 +70,36 @@ namespace MGITest { unsigned int totalVerbSlots = 0; // The encoder's highest produced record ordinal, or 0 when there is no session. unsigned long long emitSeq = 0; + + // ---- the wire producer's ledger, for exit gates E3(e) and R-10's proof obligation --- + // + // All four are 0 when there is no session, which is why every case that reads them has + // to have passed SplitRuntimeSkipReason() first: 0 wraps in a process that never had a + // ring and 0 wraps in a process whose ring never filled are the same number and + // completely different facts. + // + // maxRecordBytes / maxRecordBytesCap: R-10 says P5 does no chunking and must prove it + // needs none. The cap is RingProducer::MaxRecordBytes() == MOBILEGL_IPC_RING_MB / 2, + // read from the ring this process actually got rather than recomputed from the + // environment. + // + // cmdWraps / cmdWrapPads / cmdBytesWritten: SEG_CMD cannot go round until more bytes + // have been written than the ring holds, so the byte count is the denominator without + // which the wrap count means nothing - "0 wraps" is a defect after 1.25 MiB through a + // 1 MiB ring and a tautology after 40 KiB. cmdWrapPads is the narrower R-9 event (a + // record STRADDLED the boundary and needed a kRecPad filler) and is recorded rather + // than asserted: a uniform record stride over a power-of-two ring lands on the + // boundary exactly and never straddles it. + // + // stageReclaimWaits: SEG_STAGE allocations that only fitted after the encoder reclaimed + // what the server had retired - P5's one real producer wait (see PipeWireCodec.h for + // why the command ring has none while the verb barrier is armed). + unsigned long long maxRecordBytes = 0; + unsigned long long maxRecordBytesCap = 0; + unsigned long long cmdWraps = 0; + unsigned long long cmdWrapPads = 0; + unsigned long long cmdBytesWritten = 0; + unsigned long long stageReclaimWaits = 0; }; SplitRuntimeState PeekSplitRuntime(); diff --git a/MobileGL/MG_IntegrationTest/Harness/WireLedgerChecks.h b/MobileGL/MG_IntegrationTest/Harness/WireLedgerChecks.h new file mode 100644 index 00000000..1c5eb512 --- /dev/null +++ b/MobileGL/MG_IntegrationTest/Harness/WireLedgerChecks.h @@ -0,0 +1,146 @@ +// MobileGL - MobileGL/MG_IntegrationTest/Harness/WireLedgerChecks.h +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header +// +// THE TWO ASSERTIONS THE WIRE PRODUCER'S LEDGER MAKES POSSIBLE, in one place so the scenarios +// that carry them cannot drift apart on what the numbers mean. +// +// R-10 (ExpectMaxRecordBytesUnderCap). P5 does no chunking and must instead PROVE it never +// needs any: no record may exceed RingProducer::MaxRecordBytes() == MOBILEGL_IPC_RING_MB / 2. +// Half of that proof is already a Fatal - PipeWireCodec.cpp aborts Fatal{RingOverrun} on a +// record ABOVE the cap - and it is the loud half. The quiet half is the one this assertion +// covers: the phase has to publish the MAXIMUM ACTUALLY SEEN on a real workload, so that a +// record creeping towards the cap is visible before the day it crosses. Until this landed the +// only readers of PipeWireEncoder::MaxRecordBytesSeen() were codec unit cases over synthetic +// records, and the joint gate recorded BRIEF 8 item 3 as "Maximum record bytes: NO MEASUREMENT" +// (joint-v1.md 5). +// +// R-9 / exit gate E3(e) (ExpectSmallRingWrappedAtLeastOnce). The SmallRing lane exists to run +// the reduced path over a ring small enough to force the wrap path - the kRecPad filler that +// both sides must SKIP WITHOUT ADVANCING seq, which is R-9's last clause and the one piece of +// ring behaviour no other lane reaches. It ran green from the day it was registered and proved +// nothing, because nobody counted: measured on the joint head, a whole split scenario writes +// about 40 KiB into SEG_CMD (emitseq 22-71, maxrec 784 bytes), so a 1 MiB "small" ring is 25 +// times larger than the traffic and the head never comes near its wrap boundary. The lane and +// the default lane were the same run under two names - which is exactly the shape ID-53's own +// comment warned about and t1-v1.md carried as a debt ("WHAT THIS LANE DOES NOT YET ASSERT"). +// +// So the lane now DRIVES ENOUGH WORK to overrun its own ring before it asserts, and the +// assertion prints the denominator with the count: "0 wraps" is a defect after 1.25 MiB through +// a 1 MiB ring and a tautology after 40 KiB, and a bare `EXPECT_GE(wraps, 1)` cannot tell those +// apart. The driving is ordinary GL - clears and draws through the scenario's own objects - and +// not a hand-built record: R-16 forbids an assertion that constructs the state it observes, and +// a test that called Reserve directly would be asserting that the ring wraps, not that the +// WORKLOAD makes it wrap. +// +// WHAT IS ASSERTED IS THE HEAD GOING ROUND, NOT THE kRecPad FILLER, and that distinction was +// forced by a measurement rather than chosen: the first cut of this assertion read the pad +// count, and 1310824 bytes of clears and draws through a 1 MiB SEG_CMD produced one and a half +// trips round the ring and ZERO pads. The reason is arithmetic, not a defect - a workload whose +// records repeat at a uniform stride that divides a power-of-two capacity lands on the boundary +// exactly, every time - so a gate written against the pad count would have been red for the +// sizes in the record catalogue and green the day one of them changed. The pad count is +// RECORDED beside the wrap count (ringpads= on the stats line, in the JUnit properties, and in +// the session's teardown ledger) so the number is available without being load-bearing. + +#pragma once + +#include + +#include + +#include "SplitLane.h" +#include "SplitRuntimePeek.h" + +namespace MGITest::WireLedger { + + // The SmallRing lane declares MOBILEGL_IPC_RING_MB=1 (MG_IntegrationTest/CMakeLists.txt; + // 1 MiB is ConfigLoader's floor for it). The byte target below is that size plus a + // quarter: enough that the head MUST have crossed the wrap boundary, and far enough under + // the DEFAULT 8 MiB ring that the same workload cannot wrap there - which is what makes + // "raise the ring back to the default and this assertion goes red" a real control rather + // than a description. + inline constexpr unsigned long long kSmallRingLaneCmdByteTarget = (5ull << 20) / 4; // 1.25 MiB + + // A bound on the drive loop, so a lane whose records shrank cannot spin forever. It is + // generous on purpose: the loop's exit condition is the BYTE COUNT, and this only turns an + // infinite loop into a named failure. + inline constexpr unsigned int kSmallRingLaneMaxIterations = 200000u; + + // R-10's reading, for any split lane. `where` names the case, because the number is a + // MEASUREMENT this phase has to publish and a reader needs to know which workload produced + // it. + inline void ExpectMaxRecordBytesUnderCap(const char* where) { + const SplitRuntimeState state = PeekSplitRuntime(); + ASSERT_TRUE(state.sessionActive) + << "the wire ledger was read in a process with no client session; the caller must " + "pass SplitLane::SkipReasonForSplitOnlyAssertions() first, because every field of " + "this ledger is 0 there and 0 is also a legal measurement"; + // Not merely "under the cap": a maximum of ZERO means the case emitted no record at all, + // which satisfies `< cap` perfectly and is the exact shape of an emit table that + // resolved the transport and then fell through to the driver. + EXPECT_GT(state.maxRecordBytes, 0u) + << where << ": the largest record this session wrote is 0 bytes, so nothing crossed " + "SEG_CMD. R-10's proof obligation has no subject and the lane did not go " + "through the wire"; + EXPECT_LT(state.maxRecordBytes, state.maxRecordBytesCap) + << where << ": R-10 - the largest record this session wrote is " << state.maxRecordBytes + << " bytes and RingProducer::MaxRecordBytes() is " << state.maxRecordBytesCap + << " (half of a " << (state.maxRecordBytesCap * 2) + << " byte SEG_CMD, i.e. MOBILEGL_IPC_RING_MB). P5 does NOT chunk: a record at or " + "above the cap is Fatal{RingOverrun} at the encoder, and a maximum that has " + "climbed to it is the proof obligation failing. Report it to the integrator, who " + "decides between early chunking (P8) and a bigger default ring"; + ::testing::Test::RecordProperty("max_record_bytes", + static_cast(state.maxRecordBytes)); + ::testing::Test::RecordProperty("max_record_bytes_cap", + static_cast(state.maxRecordBytesCap)); + } + + // E3(e)'s reading. The CALLER drives the workload; this only reads the result, so that the + // thing being asserted about is the workload and not this header. + inline void ExpectSmallRingWrappedAtLeastOnce(const char* where, + unsigned long long bytesDriven) { + const SplitRuntimeState state = PeekSplitRuntime(); + ASSERT_TRUE(state.sessionActive) + << "the wire ledger was read in a process with no client session"; + const unsigned long long capacity = state.maxRecordBytesCap * 2; // MaxRecordBytes == cap/2 + // THE WRAP FIRST, AND THE DENOMINATOR RIGHT BEHIND IT, both as EXPECT so that a red + // carries both sentences. Order matters for what the failure SAYS: the thing this gate + // is about is the missing wrap, and "the loop pushed fewer bytes than the ring holds" + // is the EXPLANATION for it, not a different failure. An ASSERT on the denominator + // would print only the explanation and the reader would have to infer the gate - which + // is how the red-once line for this control was measured, and why it is written this + // way round. + EXPECT_GE(state.cmdWraps, 1u) + << where << ": exit gate E3(e) - " << bytesDriven << " bytes were written into a " + << capacity + << " byte SEG_CMD and the producer's head NEVER WENT ROUND: no wrap, so this entry " + "exercised exactly what the default lane exercises and the word SmallRing in its " + "name asserts nothing. That is what the joint gate recorded as 'SmallRing entries " + "ran, but no back-pressure wait count was measured' (joint-v1.md 6). The usual " + "cause is the ring: MOBILEGL_IPC_RING_MB did not reach this process, or the lane " + "was given the DEFAULT 8 MiB ring - which is exactly how this assertion was " + "proved to be load-bearing (R-16), by re-running this entry's own command with " + "MOBILEGL_IPC_RING_MB=8 and nothing else changed"; + EXPECT_GT(bytesDriven, capacity) + << where << ": and the reason is the denominator - the drive loop pushed only " + << bytesDriven << " bytes through a " << capacity + << " byte SEG_CMD, which cannot reach a wrap boundary at all. The loop stops at " + "kSmallRingLaneCmdByteTarget, which is sized for the 1 MiB ring this lane " + "declares (MGL_ITEST_GLES_SPLIT_SMALL_RING_ENVIRONMENT); a larger ring needs a " + "larger workload and is not what this lane is for"; + ::testing::Test::RecordProperty("ring_wraps", static_cast(state.cmdWraps)); + ::testing::Test::RecordProperty("ring_wrap_pads", static_cast(state.cmdWrapPads)); + ::testing::Test::RecordProperty("ring_waits", static_cast(state.stageReclaimWaits)); + ::testing::Test::RecordProperty("cmd_bytes_written", static_cast(bytesDriven)); + } + + // Bytes the producer has written so far, or 0 outside a split process. + inline unsigned long long CmdBytesWritten() { return PeekSplitRuntime().cmdBytesWritten; } + +} // namespace MGITest::WireLedger diff --git a/MobileGL/MG_IntegrationTest/Scenarios/TriangleScenario.cpp b/MobileGL/MG_IntegrationTest/Scenarios/TriangleScenario.cpp index 7f0e67ce..520615c0 100644 --- a/MobileGL/MG_IntegrationTest/Scenarios/TriangleScenario.cpp +++ b/MobileGL/MG_IntegrationTest/Scenarios/TriangleScenario.cpp @@ -48,6 +48,8 @@ #include "../Harness/HeadlessGL.h" #include "../Harness/ScenarioFixture.h" +#include "../Harness/SplitLane.h" +#include "../Harness/WireLedgerChecks.h" #ifdef GLAPI #undef GLAPI @@ -163,6 +165,32 @@ void main() { oColor = vec4(vColor, 1.0); } EXPECT_TRUE(RegionIsMostly(image, 0, (w * 5) / 100, 0, (h * 5) / 100, color, 0.0, when)); } + // EXIT GATE E3(e)'s DRIVE LOOP. Ordinary GL through this scenario's own objects - + // a clear and a VBO-backed draw per iteration, no readback (a readback is a + // SEG_REPLY round trip per iteration and would make this cost seconds rather than + // milliseconds) - repeated until the producer has written more bytes into SEG_CMD + // than the lane's ring holds. Returns the bytes this loop drove. + // + // NOTHING HERE TOUCHES THE RING DIRECTLY. The loop's only input is the producer's + // own head cursor, read through Harness/SplitRuntimePeek, and its only output is + // GL calls the scenario already makes. R-16: an assertion may not construct the + // state it observes, and "the workload makes the ring wrap" is a different claim + // from "a test can make the ring wrap". + unsigned long long DriveUntilSmallRingOverruns() { + const unsigned long long before = WireLedger::CmdBytesWritten(); + unsigned long long driven = 0; + for (unsigned int i = 0; i < WireLedger::kSmallRingLaneMaxIterations; ++i) { + ClearTo(0.0f, 0.0f, (i & 1u) ? 1.0f : 0.0f, 1.0f); + glUseProgram(m_program); + glBindVertexArray(m_vao); + glDrawArrays(GL_TRIANGLES, 0, 3); + // Cheap: this is a member read on the encoder, not a wire round trip. + driven = WireLedger::CmdBytesWritten() - before; + if (driven > WireLedger::kSmallRingLaneCmdByteTarget) break; + } + return driven; + } + unsigned int m_program = 0; unsigned int m_vao = 0; unsigned int m_vbo = 0; @@ -205,6 +233,41 @@ void main() { oColor = vec4(vColor, 1.0); } "re-specification of either"); ExpectClearedCorner(second, "black", "frame 1's clear, which is the only thing that changed"); Gl().EndFrame(); + + // ---- the split lanes' two readings of the wire producer's ledger -------------------- + // + // They are HERE, at the end of the steady-state case, and not in a case of their own, + // for a reason that is about the gate and not about tidiness: `integration-split` is a + // NAMED census (19 ran / 2 skipped by design) and a new entry moves it, so the phase + // would have to re-baseline a number the joint gate just pinned. The measurement wants + // this workload anyway - BRIEF 8 item 3 names this case - and a reading taken after the + // case's own pixel assertions is a reading over a run that is known to have been + // correct. + // + // Both are skipped, loudly and by the same predicate every other split-only assertion + // uses, in the monolith lanes: there is no encoder there, and every field of the + // ledger reads 0. + const std::string skip = SplitLane::SkipReasonForSplitOnlyAssertions(); + if (!skip.empty()) { + RecordProperty("wire_ledger_skip_reason", skip); + return; + } + + // R-10's proof obligation over target B. Published in every split lane, small ring + // included - the cap moves with MOBILEGL_IPC_RING_MB, so the SmallRing lane is also the + // arm where a record closest to its cap would show up first. + WireLedger::ExpectMaxRecordBytesUnderCap( + "TriangleScenario.TheSameVboAndVaoRedrawAcrossAFrameBoundary"); + + // Exit gate E3(e). Only the small-ring lane drives the overrun: at the default 8 MiB + // the same loop would take eight times as long to say the same thing, and the point of + // the lane is that IT is the arm with a ring the workload can fill. + if (SplitLane::IsSmallRingLane()) { + const unsigned long long driven = DriveUntilSmallRingOverruns(); + Gl().EndFrame(); + WireLedger::ExpectSmallRingWrappedAtLeastOnce( + "TriangleScenario.TheSameVboAndVaoRedrawAcrossAFrameBoundary", driven); + } } } // namespace MGITest From 579118a1b32656bb8fe4016f8db4f01623ed6e0f Mon Sep 17 00:00:00 2001 From: Swung0x48 Date: Wed, 16 Sep 2026 11:09:21 -0400 Subject: [PATCH 3/3] [Test, CI] (scripts/ci, workflows): E2's draw-drop control requires the SSIM red AND the library's dropped-record count, E3(a) requires its private-log line as well as the pixels, and the smoke red-check finally perturbs the split control (ID-65) --- .github/workflows/test.yml | 26 ++++ .../Harness/split_log_paths.py | 11 ++ scripts/ci/control_smoke_test.sh | 30 ++++ scripts/ci/redcheck_control_smoke_test.sh | 100 ++++++++++--- scripts/ci/retrace_drop_draw_control.sh | 131 ++++++++++++++++++ scripts/ci/split_negative_controls.sh | 35 ++++- .../ci/testdata/split_private_log_smoke.sh | 5 +- scripts/ci/testdata/stub_ctest.sh | 73 +++++++++- 8 files changed, 383 insertions(+), 28 deletions(-) mode change 100755 => 100644 scripts/ci/control_smoke_test.sh mode change 100755 => 100644 scripts/ci/redcheck_control_smoke_test.sh create mode 100644 scripts/ci/retrace_drop_draw_control.sh mode change 100755 => 100644 scripts/ci/split_negative_controls.sh mode change 100755 => 100644 scripts/ci/testdata/stub_ctest.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ca022ed7..99c9cc45 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1954,6 +1954,32 @@ jobs: bash "${GITHUB_WORKSPACE}/scripts/ci/retrace_pull_library_control.sh" '${{ matrix.case }}' '${{ matrix.backend }}' + # EXIT GATE E2's OTHER HALF, and the one BRIEF-P5 §7 E2 actually names: "drop an emission + # and the SSIM must fall below the threshold". The control above proves the lane runs a + # SPLIT library; it cannot prove the PICTURE came through the wire, because OpenRA scores + # ssim 1.000000 against a monolith library too and the transport assertion is what reds it. + # + # It drops the DRAWS and not the clears, and that is a measurement rather than a + # preference: with MOBILEGL_IPC_E2_DROP_CLEAR=1 armed, read, and all 29 of OpenRA's Clear + # records dropped, the retrace still scored ssim=1.000000 / mismatchPixels=0 - OpenRA + # overdraws every pixel it clears before the snapshot (joint-v1.md §3 found the green; + # scripts/ci/retrace_drop_draw_control.sh's header has the trace census that explains it). + # With the DrawVbo records dropped instead: 758 dropped, ssim=0.000036, + # mismatchPixels=295296. + # + # THE BODY IS A SCRIPT for the reason the two steps above give: a `run:` block cannot be + # executed off a runner, and scripts/ci/control_smoke_test.sh drives this file against a + # stubbed ctest in seven modes - including the two that matter most, a red whose SSIM never + # fell and a knob that armed and dropped nothing. + - name: Negative control - dropping the draws on the wire must red this split retrace + working-directory: build-retrace/tools/trace_replay + env: + CONTROL_TMPDIR: ${{ runner.temp }} + LIBRARY_LOG: ${{ matrix.case }}/${{ matrix.backend }}/output/mobilegl.log + run: >- + bash "${GITHUB_WORKSPACE}/scripts/ci/retrace_drop_draw_control.sh" + '${{ matrix.case }}' '${{ matrix.backend }}' + # The refusal census, recorded rather than gated. run_trace_case.cmake already REDS the case # on any Fatal{, so reaching here means the count is zero - but the number and the distinct # slot names are what MEASUREMENTS wants from every split run, and reading them out of the diff --git a/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py b/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py index 41a82ecc..9bf5508a 100644 --- a/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py +++ b/MobileGL/MG_IntegrationTest/Harness/split_log_paths.py @@ -48,10 +48,21 @@ def main(): for path in selected.values(): Path(path).unlink(missing_ok=True) elif mode == "evidence": + # argv[5], optional: the control's name, for the failure message. Without it the message + # is E1's, word for word - scripts/ci/testdata/split_private_log_smoke.sh greps for that + # sentence, and E1 was the only caller until E3(a) gained a library diagnostic of its own + # (ID-65: "no library diagnostic exists for block size 0 - x2 adds one"). + label = sys.argv[5] if len(sys.argv) > 5 else "" for name, path in selected.items(): if Path(path).is_file() and re.search(sys.argv[4], Path(path).read_text(errors="replace")): print(f"private-log evidence: {name}: {path}") return + if label: + raise ValueError(f"{label} FAILED: no selected private log carries /{sys.argv[4]}/. " + "The library's own line is the only channel for this: ctest's " + "transcript is a FALSE ZERO for library output, because the console " + "sink is compiled out of the configurations these lanes run. " + + ", ".join(f"{n} ({p})" for n, p in selected.items())) raise ValueError("E1 FAILED: selected private logs lack expected Fatal{BarrierViolation, \"\"} line: " + ", ".join(f"{n} ({p})" for n, p in selected.items())) else: diff --git a/scripts/ci/control_smoke_test.sh b/scripts/ci/control_smoke_test.sh old mode 100755 new mode 100644 index d26fc2fc..39c66141 --- a/scripts/ci/control_smoke_test.sh +++ b/scripts/ci/control_smoke_test.sh @@ -62,6 +62,15 @@ run_retrace() { # $1 = STUB_MODE bash "${HERE}/retrace_pull_library_control.sh" OpenRA DirectGLES } +run_drop_draw() { # $1 = STUB_MODE + cd "${WORK}" || return 127 + mkdir -p "${WORK}/OpenRA" + env -i PATH="${STUB_DIR}:/usr/bin:/bin" STUB_MODE="$1" \ + CTEST=ctest CONTROL_TMPDIR="${WORK}/tmp-$1" \ + LIBRARY_LOG="${WORK}/tmp-$1/mobilegl.log" \ + bash "${HERE}/retrace_drop_draw_control.sh" OpenRA DirectGLES +} + echo "=== the split lane's E1 / E3(a) controls (scripts/ci/split_negative_controls.sh)" # THE FINDING, REPRODUCED. Non-empty selection, green baseline, and a red that is not the knob's. expect FAILED "unrelated failure with a non-empty selection" -- run_split unrelated @@ -97,6 +106,27 @@ expect PASSED "run_trace_case.cmake's own sentence, wrapped" -- run_retrace retr # The pull library replaying green is the failure this control exists to catch. expect FAILED "a pull library passed the split retrace" -- run_retrace retrace-green +echo +echo "=== the retrace lane's draw-drop control (scripts/ci/retrace_drop_draw_control.sh)" +# Exit gate E2's picture control. The joint gate's finding is the reason it exists at all: the +# CLEAR-drop knob was armed, was read, dropped all 29 of OpenRA's clears - and the retrace still +# scored ssim 1.000000, because OpenRA overdraws every pixel it clears. So this control's three +# guards are each a different way for "the picture went red" to be someone else's red. +expect FAILED "empty selection (--no-tests=error exit)" -- run_drop_draw dropdraw-noselect +# The failure the gate exists to catch: the golden survives the loss of every draw. +expect FAILED "the retrace passed with every draw dropped" -- run_drop_draw dropdraw-green +# A red with no comparator output at all - a loader failure, a missing fixture, a timeout. +expect FAILED "red with no ssim summary in the output" -- run_drop_draw dropdraw-nossim +# A red whose SSIM is FINE: something else (a Fatal{, a transport assertion) reddened the case. +expect FAILED "red but the ssim is above the threshold" -- run_drop_draw dropdraw-ssimhigh +# A red picture with no evidence the knob was ever read by the process that produced it. +expect FAILED "no 'E2 control armed' line in the library log" -- run_drop_draw dropdraw-nolog +# The R-16 case the dropped-record COUNT exists for: the knob armed and dropped nothing, so the +# wrong picture came from somewhere else. +expect FAILED "the knob armed but dropped zero records" -- run_drop_draw dropdraw-zero +# The real thing: a red picture, a fallen SSIM, and N > 0 records the library says it dropped. +expect PASSED "ssim below threshold and N records dropped" -- run_drop_draw dropdraw-evidence + echo echo "smoke test: ${passes} passed, ${failures} failed" if [ "${failures}" -gt 0 ]; then diff --git a/scripts/ci/redcheck_control_smoke_test.sh b/scripts/ci/redcheck_control_smoke_test.sh old mode 100755 new mode 100644 index 6f540e1c..5b653abd --- a/scripts/ci/redcheck_control_smoke_test.sh +++ b/scripts/ci/redcheck_control_smoke_test.sh @@ -12,10 +12,11 @@ set -u HERE="$(cd "$(dirname "$0")" && pwd)" WORK="$(mktemp -d)" -trap 'cp "${WORK}/split.orig" "${HERE}/split_negative_controls.sh"; cp "${WORK}/retrace.orig" "${HERE}/retrace_pull_library_control.sh"; rm -rf "${WORK}"' EXIT +trap 'cp "${WORK}/split.orig" "${HERE}/split_negative_controls.sh"; cp "${WORK}/retrace.orig" "${HERE}/retrace_pull_library_control.sh"; cp "${WORK}/dropdraw.orig" "${HERE}/retrace_drop_draw_control.sh"; rm -rf "${WORK}"' EXIT cp "${HERE}/split_negative_controls.sh" "${WORK}/split.orig" cp "${HERE}/retrace_pull_library_control.sh" "${WORK}/retrace.orig" +cp "${HERE}/retrace_drop_draw_control.sh" "${WORK}/dropdraw.orig" echo "=== baseline: the smoke test must be GREEN before anything is perturbed" if ! bash "${HERE}/control_smoke_test.sh" > "${WORK}/before.log" 2>&1; then @@ -27,24 +28,77 @@ tail -1 "${WORK}/before.log" echo echo "=== perturbation: remove the evidence check from both controls" -python3 - "${HERE}/split_negative_controls.sh" "${HERE}/retrace_pull_library_control.sh" <<'PY' +python3 - "${HERE}/split_negative_controls.sh" "${HERE}/retrace_pull_library_control.sh" \ + "${HERE}/retrace_drop_draw_control.sh" <<'PY' import sys -split, retrace = sys.argv[1], sys.argv[2] -for path, needle in ((split, 'grep -qE "${evidence}"'), (retrace, 'grep -qF "${EVIDENCE}"')): +split, retrace, dropdraw = sys.argv[1], sys.argv[2], sys.argv[3] +# (file, [(needle, the prefix the line must start with)]). The draw-drop control has THREE +# evidence checks rather than one, because "the picture went red" has three different ways of +# being somebody else's red: the SSIM never fell, the library never said the knob armed, and the +# knob armed and dropped nothing. All three are reverted together, and the smoke cases that pin +# each of them must go red below. +# +# THE PREFIXES ARE A TUPLE, AND THAT IS A FIX, not a generalisation for its own sake. The first +# version matched only lines starting with `if ! `, and the split control's evidence check has +# been `elif ! tr -s ... | grep -qE "${evidence}"` since it was written - so this red-check found +# ZERO checks in it, the SystemExit below was not checked by the caller, and the run went on to +# smoke-test the UNPERTURBED controls and report "RED-CHECK FAILED: the controls accept an +# unrelated failure again and the smoke test still passed". It failed safe rather than green, so +# nothing was ever silently proved - but the one control it was most about was never perturbed. +# Reproduced on p5/joint@e61d0012 before this line changed. +# +# AND THE SPLIT CONTROL HAS THREE EVIDENCE CHECKS, NOT ONE, so all three are reverted. E1's is a +# read of the entry's own private library log, E3(a)'s ctest-output regex is the `elif` above, +# and E3(a)'s private-log half is ID-65's addition. Perturbing only one of the three leaves the +# other two catching the smoke test's "unrelated failure" case, and the case never flips - which +# is what this script measured the first time the perturbation actually applied. +rules = [ + (split, [('LINE', 'grep -qE "${evidence}"', ('if ! ', 'elif ! ')), + ('SUBST', '"[A-Za-z_][A-Za-z_0-9]*"\\}\' || exit 1', '"[A-Za-z_][A-Za-z_0-9]*"\\}\' || true'), + ('SUBST', '"${private_evidence}" "${name}" || exit 1', + '"${private_evidence}" "${name}" || true')]), + (retrace, [('LINE', 'grep -qF "${EVIDENCE}"', ('if ! ',))]), + (dropdraw, [('LINE', 'awk -v a="${ssim}"', ('if ! ',)), + ('LINE', '[ -z "${armed_line}" ]', ('if ',)), + ('LINE', '[ "${dropped}" -lt 1 ]', ('if ',))]), +] +for path, checks in rules: text = open(path).read() - out, hit = [], 0 - for line in text.splitlines(keepends=True): - if needle in line and line.lstrip().startswith('if ! '): - indent = line[:len(line) - len(line.lstrip())] - out.append(f"{indent}if false; then\n") - hit += 1 - else: - out.append(line) - if hit != 1: - raise SystemExit(f"expected exactly one evidence check in {path}, found {hit}") - open(path, 'w').write(''.join(out)) -print("both evidence checks reverted to 'any non-zero ctest exit is accepted'") + for rule in checks: + if rule[0] == 'SUBST': + _, old, new = rule + if text.count(old) != 1: + raise SystemExit(f"expected exactly one {old!r} in {path}, found {text.count(old)}") + text = text.replace(old, new) + continue + _, needle, prefixes = rule + out, hit = [], 0 + for line in text.splitlines(keepends=True): + if needle in line and line.lstrip().startswith(prefixes): + indent = line[:len(line) - len(line.lstrip())] + # KEEP THE KEYWORD. Turning an `elif` into an `if` splits the chain in two and + # leaves the second half's `fi` dangling - a syntax error, which the smoke test + # then reports as rc=2 on every case and which looks nothing like "the control + # accepted an unrelated failure". + keyword = 'elif' if line.lstrip().startswith('elif') else 'if' + out.append(f"{indent}{keyword} false; then\n") + hit += 1 + else: + out.append(line) + if hit != 1: + raise SystemExit(f"expected exactly one {needle!r} check in {path}, found {hit}") + text = ''.join(out) + open(path, 'w').write(text) +print("all seven evidence checks reverted to 'any non-zero ctest exit is accepted'") PY +# AND THE PERTURBATION'S OWN FAILURE IS FATAL. Without this, a SystemExit above left the controls +# UNTOUCHED and the run continued to smoke-test them, printing "RED-CHECK FAILED: the controls +# accept an unrelated failure again and the smoke test still passed" - a true statement about a +# perturbation that never happened, and the wrong diagnosis to hand whoever reads it. +if [ $? -ne 0 ]; then + echo "RED-CHECK ABORTED: the perturbation did not apply, so nothing below would be measuring it." + exit 1 +fi echo echo "=== the smoke test on the reverted controls (it MUST be red, on those two cases)" @@ -59,13 +113,19 @@ if [ "${rc}" -eq 0 ]; then fi missed=0 -grep -q "NOT OK unrelated failure with a non-empty selection" "${WORK}/after.log" || missed=1 -grep -q "NOT OK red without the transport-resolution message" "${WORK}/after.log" || missed=1 +for case in \ + "unrelated failure with a non-empty selection" \ + "red without the transport-resolution message" \ + "red but the ssim is above the threshold" \ + "no 'E2 control armed' line in the library log" \ + "the knob armed but dropped zero records"; do + grep -q "NOT OK ${case}" "${WORK}/after.log" || { echo "still green: ${case}"; missed=1; } +done if [ "${missed}" -ne 0 ]; then echo - echo "RED-CHECK FAILED: the smoke test went red, but not on the two cases the evidence check exists for." + echo "RED-CHECK FAILED: the smoke test went red, but not on the five cases the evidence checks exist for." exit 1 fi echo -echo "P5_T1_CONTROL_SMOKE_REDCHECK_OK - removing the evidence check reds exactly the two cases that pin it" +echo "P5_T1_CONTROL_SMOKE_REDCHECK_OK - removing the evidence checks reds exactly the five cases that pin them" diff --git a/scripts/ci/retrace_drop_draw_control.sh b/scripts/ci/retrace_drop_draw_control.sh new file mode 100644 index 00000000..c7597c0c --- /dev/null +++ b/scripts/ci/retrace_drop_draw_control.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# EXIT GATE E2's PICTURE CONTROL: dropping the draws on the wire must redden this retrace, and +# it must be THIS control's red. +# +# WHY THERE IS A SECOND CONTROL ON THIS LANE, and why it is the draws and not the clears. +# +# The retrace-split job already carries scripts/ci/retrace_pull_library_control.sh, which swaps a +# PULL library in and requires run_trace_case.cmake's transport-resolution assertion to catch it. +# That control proves the lane is running a SPLIT library. It does not prove that the PICTURE came +# through the wire, and it cannot: OpenRA scores ssim 1.000000 against a monolith library too, so +# the transport assertion is what reds it and the comparator is never consulted. BRIEF-P5 7 E2 +# names a second, sharper statement - "patch an emitter to drop a record and the SSIM must fall +# below the threshold" - and until now nothing executed it. +# +# c1 shipped MOBILEGL_IPC_E2_DROP_CLEAR for exactly that, and the joint gate ran it and it DID NOT +# WORK: with the knob armed and its WARN in the library's own log, the OpenRA retrace under inproc +# still scored ssim=1.000000 with mismatchPixels=0 (joint-v1.md 3, "E2 clear-drop control - NOT +# VERIFIED"). That is not a broken knob. `apitrace dump` over the 31249 replayed calls counts 30 +# glClear, 30 glXSwapBuffers and 788 glDrawArrays, and the final frame issues its clear at call +# 30197 and then covers the surface four times over with a terrain layer before the snapshot at +# 31249. A frame that overdraws every pixel it clears has a picture that does not depend on the +# clear. Measured here with the dropped-record count published: 29 Clear records dropped, ssim +# still 1.000000. The knob worked; the observable was invisible. +# +# So this control drops every DrawVbo record instead (MOBILEGL_IPC_E2_DROP_DRAW=1): the surface +# can then only carry the clear colour, and the golden is made of the geometry. Measured on the +# same head: 758 DrawVbo records dropped, ssim=0.000036, mismatchPixels=295296. +# +# WHAT THE CONTROL ASSERTS, and none of the three is the process exit code alone (R-16): +# 1. a NON-EMPTY selection, counted before the run - `--no-tests=error` turns an empty selection +# into a non-zero exit, which is how ID-46 finding 8(b) got a control to congratulate itself; +# 2. the SSIM ACTUALLY FELL: the numbers are parsed out of the retrace's own summary and +# compared, rather than "ctest was non-zero". A loader failure, a missing fixture, a timeout +# and a Fatal{ all exit non-zero and none of them is this control's red; +# 3. the LIBRARY SAID IT DROPPED SOMETHING: its own "E2 control armed ... N records dropped on +# the wire" line, with N > 0, from the log file the replay wrote. Without this, a knob that +# was never read - a stale library, a variable that did not reach the process, an emit table +# that fell through to the driver - would redden the picture for some other reason and pass. +# +# Usage: retrace_drop_draw_control.sh +# CTEST ctest binary (default: ctest) +# CONTROL_TMPDIR scratch dir (default: ${RUNNER_TEMP:-/tmp}) +# LIBRARY_LOG the replay's library log (default: //output/mobilegl.log) +set -u + +CASE="${1:?usage: retrace_drop_draw_control.sh }" +BACKEND="${2:?usage: retrace_drop_draw_control.sh }" + +CTEST="${CTEST:-ctest}" +CONTROL_TMPDIR="${CONTROL_TMPDIR:-${RUNNER_TEMP:-/tmp}}" +LIBRARY_LOG="${LIBRARY_LOG:-${CASE}/${BACKEND}/output/mobilegl.log}" +mkdir -p "${CONTROL_TMPDIR}" + +selector="^MobileGLTraceReplay\.${CASE}\.${BACKEND}$" + +# The rerun replays into the same case directory, so the good run's images are put aside and +# restored whichever way the control goes; "Upload actual image" runs `if: always()` and would +# otherwise ship the deliberately-wrong run's output under the good run's name. +GOOD_OUTPUT="${CONTROL_TMPDIR}/drop-draw-verified-output" +rm -rf "${GOOD_OUTPUT}" +if [ -d "${CASE}" ]; then cp -a "${CASE}" "${GOOD_OUTPUT}"; fi + +restore_good_output() { + if [ -d "${GOOD_OUTPUT}" ]; then + rm -rf "${CASE}"; mv "${GOOD_OUTPUT}" "${CASE}" + echo "restored the verified run's output over the control's" + fi +} + +matched=$("${CTEST}" -N -R "${selector}" | grep -cE '^ *Test *#[0-9]+:') +if [ "${matched}" -lt 1 ]; then + restore_good_output + echo "::error::the control selected ${matched} tests with -R '${selector}', so there is nothing for the dropped draws to redden. --no-tests=error would have exited non-zero on the empty selection and a control without this guard reads that as success (ID-46 finding 8b)." + exit 1 +fi + +# A PREVIOUS RUN'S LINE MUST NEVER ARM THIS ONE. The library opens its log fopen(path, "w"), so +# the replay truncates it - but only if the replay gets that far, and a run that died in the +# loader would leave the baseline's log in place with a perfectly good "control armed" line in +# it. Removing it first is the same rule split_negative_controls.sh's `reset` step follows. +rm -f "${LIBRARY_LOG}" + +out="${CONTROL_TMPDIR}/retrace-drop-draw-output.txt" +export MOBILEGL_TRANSPORT=inproc +export MOBILEGL_IPC_E2_DROP_DRAW=1 +"${CTEST}" -V --no-tests=error --timeout 10800 -R "${selector}" > "${out}" 2>&1 +control_rc=$? +unset MOBILEGL_IPC_E2_DROP_DRAW +cat "${out}" + +# Read the library's evidence BEFORE the good output is restored over it. +armed_line="" +dropped=0 +if [ -f "${LIBRARY_LOG}" ]; then + armed_line=$(grep 'MGPipe: E2 control armed' "${LIBRARY_LOG}" | tail -1) + dropped=$(printf '%s' "${armed_line}" | sed -n 's/.*armed[^,]*, \([0-9][0-9]*\) records dropped.*/\1/p') + dropped="${dropped:-0}" +fi +cp -f "${LIBRARY_LOG}" "${CONTROL_TMPDIR}/drop-draw-library.log" 2>/dev/null + +restore_good_output + +if [ "${control_rc}" -eq 0 ]; then + echo "::error::the split retrace PASSED with every DrawVbo record dropped on the wire. The golden is made of that geometry, so a green here means the picture did not come from the wire: the emit table fell through to the driver, the library under test is not the one the lane thinks it is, or MOBILEGL_IPC_E2_DROP_DRAW never reached the process. Exit gate E2 is exactly this statement and nothing weaker - MOBILEGL_IPC_E2_DROP_CLEAR is NOT a substitute (measured: 29 clears dropped, ssim still 1.000000, because OpenRA overdraws every pixel it clears)." + exit 1 +fi + +# 2. THE SSIM ACTUALLY FELL. Parsed, not inferred from the exit code. +ssim_line=$(grep -o 'ssim=[0-9.]*, ssimThreshold=[0-9.]*' "${out}" | tail -1) +if [ -z "${ssim_line}" ]; then + echo "::error::the split retrace went red (ctest exit ${control_rc}) with the draws dropped, but its output carries no 'ssim=..., ssimThreshold=...' summary at all, so the comparator never ran. A loader failure, a missing fixture, a timeout or a Fatal{ all land here and none of them is this control's red." + exit 1 +fi +ssim=${ssim_line#ssim=}; ssim=${ssim%%,*} +threshold=${ssim_line##*ssimThreshold=} +if ! awk -v a="${ssim}" -v b="${threshold}" 'BEGIN { exit !(a + 0 < b + 0) }'; then + echo "::error::the split retrace went red (ctest exit ${control_rc}) but its ${ssim_line} is NOT below the threshold, so the picture is not what reddened it. This control's whole claim is that the golden is made of the dropped geometry." + exit 1 +fi + +# 3. THE LIBRARY SAID IT DROPPED SOMETHING. +if [ -z "${armed_line}" ]; then + echo "::error::the split retrace went red with ssim ${ssim} < ${threshold}, but ${LIBRARY_LOG} carries no 'MGPipe: E2 control armed' line, so there is no evidence the knob was ever read by the process that produced the picture. A library that is not the one under test, an emit table that fell through to the driver, or an environment that did not reach the replay all produce a wrong picture for a reason that has nothing to do with this control." + exit 1 +fi +if [ "${dropped}" -lt 1 ]; then + echo "::error::the split retrace went red with ssim ${ssim} < ${threshold} and the knob announced itself - '${armed_line}' - but it reports ZERO records dropped. The emitter was never reached, so whatever changed the picture was not this control. This is the R-16 case the dropped-record COUNT exists for: the arming message alone proves only that the knob was read." + exit 1 +fi + +echo "the dropped draws turned the split retrace red for their own reason (ctest exit ${control_rc}): ${matched} selected case(s), ssim ${ssim} < ${threshold}, and the library dropped ${dropped} record(s) on the wire" diff --git a/scripts/ci/split_negative_controls.sh b/scripts/ci/split_negative_controls.sh old mode 100755 new mode 100644 index 6baf5d5e..e85a0653 --- a/scripts/ci/split_negative_controls.sh +++ b/scripts/ci/split_negative_controls.sh @@ -99,9 +99,19 @@ fi # ---- the controls --------------------------------------------------------------------------- # -# run_control ... +# run_control ... +# +# THE FOURTH ARGUMENT IS NEW AND E3(a) IS WHY. The header above argues that the knob's own +# ConfigLoader line is not evidence, and that is still true: it is written at config load, by +# every process in the run, whatever happens next. What IS evidence is a line the knob's +# BEHAVIOUR emits at the site that changed - and until now E3(a) had none, because +# PersistentMapTracker::PushBlocksFor simply `return`ed at blockBytes == 0. The joint gate +# recorded exactly that ("There is no Fatal for block size zero ... No nonexistent private-file +# Fatal is quoted", joint-v1.md 3) and ID-65 assigned the missing line here. With it, E3(a) no +# longer rests on a pixel assertion alone: the red must carry the scenario's own diagnostic AND +# the library's own statement that the push was disabled, from the entry's private file. run_control() { - name="$1"; filter="$2"; evidence="$3"; shift 3 + name="$1"; filter="$2"; evidence="$3"; private_evidence="$4"; shift 4 matched=$("${CTEST}" -N -L integration-split -R "${filter}" | grep -cE '^ *Test *#[0-9]+:') if [ "${matched}" -lt 1 ]; then @@ -133,6 +143,14 @@ run_control() { exit 1 fi + # ... and, where the knob has one, the LIBRARY's own line as well, out of the entry's private + # file. Both halves are required: the scenario assertion says the pixels were wrong, and this + # says the code path the knob turns off is the one that stopped running. A red that has only + # the first half is consistent with any other defect in the same scenario. + if [ -n "${private_evidence}" ]; then + python3 "${log_helper}" evidence "${manifest}" "${filter}" "${private_evidence}" "${name}" || exit 1 + fi + echo "${name} turned ${matched} selected entries red, and the red carries the scenario's own diagnostic, as it must" } @@ -141,12 +159,19 @@ run_control() { run_control "negative control E1 (MOBILEGL_IPC_VERB_BARRIER=0)" \ 'DirectGLES\.Split\.(SmallRing\.)?(Triangle|ClearThenReadPixels)' \ 'private-barrier-fatal' \ + '' \ MOBILEGL_IPC_VERB_BARRIER=0 -# E3(a): PersistentMapTracker.cpp returns at blockBytes == 0 (no Fatal). -# PersistentCoherentMapScenario.cpp:414-417 / 442-443 name the missing second write. -# Do not accept a generic source-line Failure: an unrelated assertion is not this red. +# E3(a): PersistentMapTracker::PushBlocksFor stops at blockBytes == 0 - deliberately, because 0 +# is the negative control and not "unlimited". Two independent halves are now required: +# * the SCENARIO's own assertion in ctest's output. PersistentCoherentMapScenario.cpp:414-417 / +# 442-443 name the missing second write; a generic source-line Failure is not accepted, +# because an unrelated assertion in the same case is not this red; +# * the LIBRARY's own line in the entry's private file, saying the push was disabled by this +# knob. It did not exist until ID-65 assigned it (joint-v1.md 3), which is why this control +# used to rest on the pixels alone. run_control "negative control E3(a) (MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0)" \ 'DirectGLES\.Split\.(SmallRing\.)?PersistentCoherentMapScenario' \ "the SECOND write through the same mapping, announced by nothing|frame 1's write through the SAME mapping, after a Present" \ + 'MGPipe: persistent-map push disabled - MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0' \ MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0 diff --git a/scripts/ci/testdata/split_private_log_smoke.sh b/scripts/ci/testdata/split_private_log_smoke.sh index cf226eea..37831b92 100644 --- a/scripts/ci/testdata/split_private_log_smoke.sh +++ b/scripts/ci/testdata/split_private_log_smoke.sh @@ -7,7 +7,7 @@ trap 'rm -rf "${WORK}"' EXIT cp "${HERE}/testdata/stub_ctest.sh" "${WORK}/ctest" chmod +x "${WORK}/ctest" passes=0 -for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated; do +for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated e3-no-private; do mkdir -p "${WORK}/${mode}" rc=0 STUB_MODE="${mode}" CTEST="${WORK}/ctest" CONTROL_TMPDIR="${WORK}/${mode}" \ @@ -19,6 +19,9 @@ for mode in missing-fatal stdout-fatal stale-fatal evidence e3-unrelated; do else message='E1 FAILED: selected private logs lack expected Fatal' [ "${mode}" != e3-unrelated ] || message='FAILED: red lacks its persistent-map push diagnostic' + # ID-65's half: the pixel assertion arrived, the library said nothing, and the control must + # refuse the red by naming the line it wanted rather than accepting the pixels alone. + [ "${mode}" != e3-no-private ] || message='no selected private log carries /MGPipe: persistent-map push disabled' [ "${rc}" != 0 ] && grep -q "${message}" "${WORK}/${mode}.out" || { cat "${WORK}/${mode}.out"; echo "NOT OK ${mode}: control must report FAILED for its own reason"; exit 1; } diff --git a/scripts/ci/testdata/stub_ctest.sh b/scripts/ci/testdata/stub_ctest.sh old mode 100755 new mode 100644 index de9dba3e..09a686c6 --- a/scripts/ci/testdata/stub_ctest.sh +++ b/scripts/ci/testdata/stub_ctest.sh @@ -15,6 +15,8 @@ # stdout-fatal Fatal exists only on stdout, never in the private file # stale-fatal Fatal exists before reset, never from this control run # e3-unrelated E1 has its private Fatal; E3(a) fails for an unrelated reason +# e3-no-private E1 has its private Fatal; E3(a) has its PIXEL assertion but the library +# never says the push was disabled (the half ID-65 added) # green baseline green; the control's own run PASSES (the knob is not load-bearing) # red-baseline the baseline itself has a failed entry # all-skipped the baseline is entirely skipped (the disarmed lane, a legitimate exit 0) @@ -22,6 +24,18 @@ # retrace-unrelated one match; the run fails without naming the transport # retrace-evidence one match; the run fails with run_trace_case.cmake's own sentence # retrace-green one match; the run PASSES +# +# EXIT GATE E2's DRAW-DROP CONTROL (scripts/ci/retrace_drop_draw_control.sh). The library's own +# log is modelled as a separate sink from ctest stdout, exactly as it is for the split lane: the +# console sink is compiled out of the configurations these lanes run, so the dropped-record count +# can only ever arrive through ${LIBRARY_LOG}. +# dropdraw-noselect `ctest -N` matches nothing +# dropdraw-green the retrace PASSES with the draws dropped +# dropdraw-nossim red, but no ssim summary at all (loader failure / timeout shape) +# dropdraw-ssimhigh red, and the ssim is ABOVE the threshold: something else reddened it +# dropdraw-nolog red with a low ssim, but the library never said the knob armed +# dropdraw-zero red with a low ssim, the knob armed, and it dropped NOTHING +# dropdraw-evidence red with a low ssim and N > 0 records dropped: the real thing set -u mode="${STUB_MODE:?STUB_MODE must be set}" @@ -39,7 +53,7 @@ listing_requested="${listing_requested:-0}" emit_listing() { echo "Test project /stub" - if [ "${mode}" = "retrace-noselect" ]; then + if [ "${mode}" = "retrace-noselect" ] || [ "${mode}" = "dropdraw-noselect" ]; then echo "Total Tests: 0" return fi @@ -88,12 +102,29 @@ fi # The control's own run. if [ "${MOBILEGL_IPC_VERB_BARRIER:-1}" = 0 ]; then case "${mode}" in - evidence|e3-unrelated) echo 'Fatal{BarrierViolation, "DrawVbo"}' > "${log}" ;; + evidence|e3-unrelated|e3-no-private) echo 'Fatal{BarrierViolation, "DrawVbo"}' > "${log}" ;; missing-fatal) echo "library setup only; no fatal" > "${log}" ;; stdout-fatal) echo 'Fatal{BarrierViolation, "DrawVbo"}' ;; esac fi +# E3(a)'s library line, in the PERSISTENT-MAP entry's own private file - a different file from +# E1's, exactly as the manifest above declares. `e3-no-private` is the mode that leaves it out: +# the pixel assertion arrives, the library says nothing, and the control must refuse the red. +if [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" = 0 ] && [ "${mode}" = evidence ]; then + echo 'MGPipe: persistent-map push disabled - MOBILEGL_IPC_PERSISTENT_BLOCK_KB=0 is exit gate E3(a)'"'"'s NEGATIVE CONTROL' \ + > "${CONTROL_TMPDIR}/pmap.log" +fi case "${mode}" in + e3-no-private) + # E1's half passes (its private Fatal is written above); E3(a)'s pixel assertion arrives on + # stdout and its private line does not, so the control must stop at the second half. + echo "1/1 Test #1: DirectGLES.Split.ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels ...***Failed" + if [ "${MOBILEGL_IPC_PERSISTENT_BLOCK_KB:-64}" = 0 ]; then + echo "../MobileGL/MG_IntegrationTest/Scenarios/PersistentCoherentMapScenario.cpp:414: Failure" + echo "the SECOND write through the same mapping, announced by nothing: this is exit gate E3(b)" + fi + exit 8 + ;; unrelated|missing-fatal|stdout-fatal|stale-fatal|e3-unrelated) echo "1/1 Test #1: DirectGLES.Split.ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels ...***Failed" echo "UNRELATED_CONTROL_FAILURE: the harness aborted in setup before the knob was read" @@ -138,6 +169,44 @@ case "${mode}" in echo "100% tests passed, 0 tests failed out of 1" exit 0 ;; + dropdraw-*) + # The library's own log, written by the replay the way the real one is. The control removes + # it before the run, so anything here is this run's. + armed="" + case "${mode}" in + dropdraw-zero) + armed='MGPipe: E2 control armed - drop-draw=1 drop-clear=0, 0 records dropped on the wire (draw=0 clear=0), frame 29' ;; + dropdraw-nolog) armed="" ;; + *) + armed='MGPipe: E2 control armed - drop-draw=1 drop-clear=0, 758 records dropped on the wire (draw=758 clear=0), frame 29' ;; + esac + if [ -n "${LIBRARY_LOG:-}" ] && [ -n "${armed}" ]; then + mkdir -p "$(dirname "${LIBRARY_LOG}")" + printf '%s\n' "[10:38:30] [Linux mobilegl_trace_/WARN]: ${armed}" > "${LIBRARY_LOG}" + fi + case "${mode}" in + dropdraw-green) + echo "100% tests passed, 0 tests failed out of 1" + exit 0 + ;; + dropdraw-nossim) + echo "1/1 Test #1: MobileGLTraceReplay.OpenRA.DirectGLES ...***Failed" + echo "CMake Error: the replay could not load the library" + exit 8 + ;; + dropdraw-ssimhigh) + echo "1/1 Test #1: MobileGLTraceReplay.OpenRA.DirectGLES ...***Failed" + echo "-- retrace completed; ssim=1.000000, ssimThreshold=0.990000, mismatchPixels=0" + echo "CMake Error at run_trace_case.cmake:301 (message): 3 MGPipe Fatal(s)" + exit 8 + ;; + *) + echo "1/1 Test #1: MobileGLTraceReplay.OpenRA.DirectGLES ...***Failed" + echo "-- retrace completed; ssim=0.000036, ssimThreshold=0.990000, mismatchPixels=295296" + exit 8 + ;; + esac + ;; *) echo "stub_ctest: unknown STUB_MODE '${mode}'" >&2 exit 127