diff --git a/MobileGL/Config.h b/MobileGL/Config.h index db8f63d4..93d3b941 100644 --- a/MobileGL/Config.h +++ b/MobileGL/Config.h @@ -513,6 +513,17 @@ namespace MobileGL::MG_Config { Uint32 WireDeferredMb = 64; // MOBILEGL_IPC_SPIN_US: spin before parking on a doorbell, either direction. Uint32 SpinUs = 50; + // MOBILEGL_IPC_EVENT_WAIT_MS (PH-6, ID-P7-2): the SERVER's patience, in ms, for ONE + // reverse-channel event that finds SEG_EVENT full under run-ahead. The apply thread + // publishes, rings the client and parks until the client drains enough room; a client + // that has not made room when this runs out - it stopped draining, it drains a slot at + // a time, or it went away - FORFEITS the reverse channel: that event and every later + // one is dropped and counted (eventDropped), and the session stops by the ordinary + // stop path rather than by Fatal{EventRingOverflow}. It bounds the whole reservation, + // not one park, so a trickling peer cannot stretch it. Was a 30000 ms constant spent + // twice. Read by the server only (ServerSpawn.cpp passes it to a spawned child); the + // lockstep arm (no kCapRunAheadApply, i.e. Magma) never waits and keeps P5C's Fatal. + Uint32 EventWaitMs = 2000; // MOBILEGL_IPC_PERSISTENT_BLOCK_KB: block granularity of the persistent-map push. // 0 IS A NEGATIVE CONTROL, NOT "unlimited": it disables the push, and // PersistentCoherentMapScenario must go RED under it (exit gate E3(a)). diff --git a/MobileGL/ConfigLoader.cpp b/MobileGL/ConfigLoader.cpp index 5e317734..1c2ceee2 100644 --- a/MobileGL/ConfigLoader.cpp +++ b/MobileGL/ConfigLoader.cpp @@ -396,6 +396,10 @@ namespace MobileGL::MG_ConfigLoader { // control: no forced sync, and MagmaWireReclaimScenario's watermark case must go red. ipc.WireDeferredMb = QueryEnvUint32("MOBILEGL_IPC_WIRE_DEFERRED_MB", 64, 0, 65536); ipc.SpinUs = QueryEnvUint32("MOBILEGL_IPC_SPIN_US", 50, 0, 1000000); + // PH-6 (ID-P7-2; Config.h has the semantics). 0 is NOT admitted: a server with no + // patience at all would forfeit a healthy run-ahead client the first time its ring + // filled between two of the client's drains, which is an ordinary backlog. + ipc.EventWaitMs = QueryEnvUint32("MOBILEGL_IPC_EVENT_WAIT_MS", 2000, 1, 600000); // 0 is admitted ON PURPOSE and is the negative control of exit gate E3(a): it turns // the persistent-map push OFF, and PersistentCoherentMapScenario must go red. ipc.PersistentBlockKb = QueryEnvUint32("MOBILEGL_IPC_PERSISTENT_BLOCK_KB", 64, 0, 65536); @@ -469,10 +473,12 @@ namespace MobileGL::MG_ConfigLoader { if (MG_Config::Transport == MG_Config::TransportMode::Monolith) return; // One line, on the arm where these numbers decide behaviour, because every one of // them is a number a bug report has to quote. - MGLOG_I("Config: IPC ring=%uMiB stage=%uMiB wire-deferred=%uMiB spin=%uus persistent-block=%uKiB " + MGLOG_I("Config: IPC ring=%uMiB stage=%uMiB wire-deferred=%uMiB spin=%uus event-wait=%ums " + "persistent-block=%uKiB " "adopt-tier=%u verb-barrier=%u run-ahead=%u present-credit=%u control-timeout=%ums " "cold-start=%ums strict=%d audit=%d role-split-state=%d affinity='%s'", - ipc.RingMb, ipc.StageMb, ipc.WireDeferredMb, ipc.SpinUs, ipc.PersistentBlockKb, ipc.AdoptTier, + ipc.RingMb, ipc.StageMb, ipc.WireDeferredMb, ipc.SpinUs, ipc.EventWaitMs, + ipc.PersistentBlockKb, ipc.AdoptTier, ipc.VerbBarrier, ipc.RunAhead, ipc.PresentCredit, ipc.ControlTimeoutMs, ipc.ColdStartMs, static_cast(ipc.StrictErrors), static_cast(ipc.Audit), static_cast(ipc.RoleSplitState), ipc.ServerAffinity.c_str()); diff --git a/MobileGL/MG_Remote/Server/ServerLoop.cpp b/MobileGL/MG_Remote/Server/ServerLoop.cpp index 871b1cd8..b4daeb46 100644 --- a/MobileGL/MG_Remote/Server/ServerLoop.cpp +++ b/MobileGL/MG_Remote/Server/ServerLoop.cpp @@ -402,17 +402,27 @@ namespace MobileGL::MG_Remote::Server { // this bell (that pairing is the whole of the flow-control protocol; a cleared flag // with no bell is the lost wakeup the forward direction's publish-then-ring order // exists to prevent). - const auto ready = [this, signals, &ring] { - if (m_stopRequested.load(std::memory_order_acquire) || ControlIsPending()) return true; + // + // PH-6 (ID-P7-2) ADDS THE FORFEIT LATCH BESIDE STOP, AND FOR STOP'S REASON. A session whose + // reverse channel was forfeited (ServerSession::ForfeitReverseChannel) has a client that + // is not reading SEG_EVENT, so its ring stays full - and the ring test below would park + // this thread on `eventRingFull` for ever, waiting for a drain that the forfeit has + // already decided will not come. The latch wins over the ring exactly as Stop does, and + // the loop takes the same way out. + const auto stopOrForfeit = [this, &session] { + return m_stopRequested.load(std::memory_order_acquire) || session.ReverseChannelForfeited(); + }; + const auto ready = [this, signals, &ring, &stopOrForfeit] { + if (stopOrForfeit() || ControlIsPending()) return true; if (signals.EventRingFull->load(std::memory_order_acquire) != 0) return false; return signals.CmdHead->load(std::memory_order_acquire) != ring.LocalTail(); }; for (;;) { PumpControlRequest(); - if (m_stopRequested.load(std::memory_order_acquire)) break; + if (stopOrForfeit()) break; if (signals.EventRingFull->load(std::memory_order_acquire) == 0) DrainRing(); - if (m_stopRequested.load(std::memory_order_acquire)) break; + if (stopOrForfeit()) break; if (ready()) continue; const Uint64 waits = m_parks.fetch_add(1, std::memory_order_acq_rel) + 1; @@ -466,8 +476,23 @@ namespace MobileGL::MG_Remote::Server { // drain (ClientSession::Stop step 1) normally empties the ring first and is BOUNDED, so // a timeout there leaves records here; applying them costs nothing and dropping them // would leave the emitter's var-tails referenced by records nobody ever read. + // + // EXCEPT AFTER A FORFEIT (PH-6). That stop was not asked for by a client finishing its + // work; it was decided about a peer that stopped reading this session's answers, and + // every event the rest of its queue produced would only be one more counted drop - while + // a peer that keeps publishing could keep this final drain going as long as it liked, + // which is exactly the unbounded hold the forfeit exists to end. The queue is left + // unapplied; the rings die with the session. + const Bool forfeited = session.ReverseChannelForfeited(); PumpControlRequest(); - DrainRing(); + if (forfeited) { + MGLOG_E("MG_Remote server: mgl-srv-apply stops on ReverseChannelForfeit - %llu reverse-channel " + "event(s) dropped, applied seq %llu; the records still queued are not applied", + static_cast(session.ForfeitDrops()), + static_cast(session.Consumer().AppliedSeq())); + } else { + DrainRing(); + } // THE BACKEND IS DESTROYED ON THIS THREAD, WHILE IT IS STILL THE CONTEXT OWNER. // ~BackendObject_DirectGLES calls DestroyEGLContext (BackendObject_DirectGLES.cpp: @@ -633,6 +658,10 @@ namespace MobileGL::MG_Remote::Server { } if (!popped) break; ++applied; + // PH-6: the record whose event forfeited the reverse channel is the last one this + // session applies. Checked only AFTER a popped record, so the empty-ring answer above + // still costs its two loads and nothing else. + if (session.ReverseChannelForfeited()) break; } if (applied != 0) { // THE TWO TALLIES MUST AGREE, AND THAT IS WHAT MAKES R-9's BATCHING BAN CHECKABLE. diff --git a/MobileGL/MG_Remote/Server/ServerMain.cpp b/MobileGL/MG_Remote/Server/ServerMain.cpp index 0041ffce..d0c83d21 100644 --- a/MobileGL/MG_Remote/Server/ServerMain.cpp +++ b/MobileGL/MG_Remote/Server/ServerMain.cpp @@ -350,9 +350,26 @@ void SendLogAck(void* pointer) { WireLogError("MG_Remote server: pid=%d transport=spawn role=server ready control=%s data=%s", selfPid, tcp ? "tcp" : "unix", tcp ? "stream" : "shm"); std::vector buffer(64 * 1024); + // PH-6 (ID-P7-2): THE CONTROL WAIT IS BOUNDED, SO A SESSION CAN END FROM THE SERVER'S SIDE. + // It used to be kWaitForever, which left exactly one way out: the peer closing its control + // connection. A peer that forfeited the reverse channel is by definition one that stopped + // reading, and it may well keep that connection open - so the apply thread would stop and + // this child would sit here, holding the supervisor's one session slot, answering every + // later client Busy. Now each quiet interval asks whether the apply thread is still there; + // when it is not (forfeit, or a dead data bell), the session takes the ordinary exit below: + // Stop, Close, _exit(0). The interval is the supervisor's own accept poll. + constexpr std::uint32_t kControlPollMs = 250; for (;;) { std::uint64_t size = 0; - const auto result = control->ReceiveFrame({buffer.data(), buffer.size()}, &size, kWaitForever); + const auto result = control->ReceiveFrame({buffer.data(), buffer.size()}, &size, kControlPollMs); + if (result == MOBILEGL_ERR_TIMEOUT) { + if (loop.Running()) continue; + WireLogError("MG_Remote server: pid=%d the apply thread has stopped (%s, %llu event(s) dropped); " + "ending the session", + selfPid, session.ReverseChannelForfeited() ? "ReverseChannelForfeit" : "its bell died", + static_cast(session.ForfeitDrops())); + break; + } if (result == MOBILEGL_ERR_BUFFER_TOO_SMALL) { buffer.resize(static_cast(size)); continue; } if (result != MOBILEGL_OK) break; // THE FILE IDENTIFIER, ASKED FIRST AND SAID OUT LOUD (P7 wave 0). diff --git a/MobileGL/MG_Remote/Server/ServerSession.cpp b/MobileGL/MG_Remote/Server/ServerSession.cpp index 715aca79..1c1e841b 100644 --- a/MobileGL/MG_Remote/Server/ServerSession.cpp +++ b/MobileGL/MG_Remote/Server/ServerSession.cpp @@ -239,19 +239,39 @@ namespace MobileGL::MG_Remote::Server { // `eventRingFull` as well (SessionProducer::WaitFor*OrEventBacklog) - so a parked // client is always wakeable by the server that is waiting for it. // - // TWO NAMED REFUSALS SURVIVE, and they are the impossible cases rather than the busy - // ones: a single event larger than the ring can EVER hold (no amount of draining - // helps), and a wait that ran out of patience (the peer is gone or wedged). A server - // that does not publish kCapRunAheadApply keeps P5C's Fatal unchanged. + // ONE NAMED REFUSAL SURVIVES, the impossible case: a single event larger than the ring + // can EVER hold (no amount of draining helps). The two BUSY refusals that used to sit + // beside it - a wait that ran out of patience, and two rounds against a ring the record + // fits - are the PEER's behaviour and became the forfeit latch below (PH-6, ID-P7-2). + // A server that does not publish kCapRunAheadApply keeps P5C's Fatal unchanged. Bool ServerPublishesRunAhead(const ServerSession& session) { if (!session.CallMaskIsSet()) return false; return (session.CallMask() & static_cast(MG_Pipe::kCapRunAheadApply)) != 0; } - constexpr Uint32 kEventBacklogWaitMs = 30000; + // PH-6 (ID-P7-2): MOBILEGL_IPC_EVENT_WAIT_MS (Config.h). It was a 30000 ms constant, and + // it was spent TWICE per reservation (two rounds), so the plan's "60 seconds" was really + // 30 s against a peer that never drains and a few milliseconds against one that trickles. + Uint32 EventBacklogWaitMs() { +#if MOBILEGL_BUILD_DISAGGREGATED + return MG_Config::Ipc.EventWaitMs; +#else + return 2000; +#endif + } + + Uint32 MillisecondsSince(std::chrono::steady_clock::time_point start) { + const auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start).count(); + return elapsed <= 0 ? 0u : static_cast(elapsed); + } void* ReserveEventOrBlock(ServerSession& session, Transport::EventKind kind, const char* eventName, Uint64 payloadBytes) { + // FORFEITED: every event after the first drop is a drop too, at once - no Reserve and + // no second wait. The session is on its way down, and the peer has already shown it + // does not read this ring. + if (session.ReverseChannelForfeited()) return session.DropEventAfterForfeit(payloadBytes); void* slot = session.Events().Reserve(kind, payloadBytes); if (slot != nullptr) return slot; if (!ServerPublishesRunAhead(session)) FatalEventRingOverflow(eventName, payloadBytes); @@ -275,29 +295,66 @@ namespace MobileGL::MG_Remote::Server { const auto drained = [signals] { return signals.EventRingFull->load(std::memory_order_acquire) == 0; }; - for (Uint32 round = 0; round < 2; ++round) { + // ONE DEADLINE FOR THE WHOLE RESERVATION, NOT ONE PER PARK. The loop goes round as + // often as the client clears the latch - that is how a client that drains in pieces + // eventually makes room - but every round spends the same budget, so a peer that + // clears the latch without freeing enough (it drains one slot per interval, or it + // writes the flag itself) cannot hold the apply thread longer than the knob says. + // The old shape gave up after TWO rounds, and called that "a corrupt cursor set": + // a trickling peer reached it in milliseconds and took the process down with it. + const Uint32 budgetMs = EventBacklogWaitMs(); + const auto start = std::chrono::steady_clock::now(); + const auto deadline = start + std::chrono::milliseconds(budgetMs); + // Rounds in which the client DID clear the latch and the record still did not fit. + // Zero at the deadline means the client never drained at all; more means it drained + // too little, too slowly - two different peers, and the log says which one this was. + Uint32 shortDrains = 0; + const auto outOfPatience = [&]() { + return shortDrains == 0 + ? session.ForfeitReverseChannel("NotDraining", "the client did not drain SEG_EVENT", + eventName, payloadBytes, MillisecondsSince(start), + budgetMs, shortDrains) + : session.ForfeitReverseChannel("TooSlow", + "the client drained SEG_EVENT but never made room for " + "this record", + eventName, payloadBytes, MillisecondsSince(start), + budgetMs, shortDrains); + }; + for (;;) { // PUBLISH AND RING FIRST. The client cannot drain a ring whose head it has not // been shown, and the latch Reserve just set is only useful to a client that // gets a bell with it. session.PublishEvents(); - if (!drained() && !bell.Wait(*signals.ConsumerParked, drained, /*spinUs=*/0, - kEventBacklogWaitMs)) { - SessionFail(MGFatalFamily::EventRingOverflow, "MGPipe: Fatal{EventRingOverflow} - %s waited %u ms for the client " - "to drain SEG_EVENT and it never did. Under run-ahead the apply " - "thread parks on this ring (CONTRACT-P5E §2.6); a client that does " - "not drain is one that is not running", - eventName, kEventBacklogWaitMs); + // DEAD FIRST (the plan's own clause): a bell whose peer is gone answers every + // Wait with an immediate false, and reporting that as "waited N ms" would name + // a timeout for what is a hangup. + if (bell.Dead()) { + return session.ForfeitReverseChannel("PeerGone", "the client's doorbell is dead - the peer is gone", + eventName, payloadBytes, MillisecondsSince(start), budgetMs, + shortDrains); + } + if (!drained()) { + const auto left = std::chrono::duration_cast( + deadline - std::chrono::steady_clock::now()).count(); + if (left <= 0) return outOfPatience(); + if (!bell.Wait(*signals.ConsumerParked, drained, /*spinUs=*/0, static_cast(left))) { + if (bell.Dead()) { + return session.ForfeitReverseChannel( + "PeerGone", "the client's doorbell died while the server waited - the peer is gone", + eventName, payloadBytes, MillisecondsSince(start), budgetMs, shortDrains); + } + return outOfPatience(); + } } slot = session.Events().Reserve(kind, payloadBytes); if (slot != nullptr) return slot; + // The client cleared the latch and the record STILL does not fit: it drained less + // than this record needs. That is a slow drain, not a corrupt cursor set, and it + // gets the rest of the same budget - Reserve has just raised the latch again, so + // the next round parks until the client clears it once more. + ++shortDrains; + if (std::chrono::steady_clock::now() >= deadline) return outOfPatience(); } - // Two full rounds against a ring the record provably fits: something else is - // producing into it, which on this side is impossible (one producer, the apply - // thread, by construction). - SessionFail(MGFatalFamily::EventRingOverflow, "MGPipe: Fatal{EventRingOverflow} - %s could not reserve %llu bytes on an " - "emptied SEG_EVENT. The apply thread is SEG_EVENT's only producer, so a " - "drained ring that still refuses a record it fits is a corrupt cursor set", - eventName, static_cast(payloadBytes)); } // Accept owns the single process callback table. Its owner need not be the @@ -493,6 +550,10 @@ namespace MobileGL::MG_Remote::Server { } } releaseOnFailure{this}; m_transport = &transport; + // PH-6: a forfeit is THIS session's fact. A process that accepts again (inproc context + // loss, a test's second session) starts with a reverse channel it has not given up. + m_reverseChannelForfeit.store(false, std::memory_order_release); + m_forfeitDrops.store(0, std::memory_order_release); // MOBILEGL_IPC_RING_MB / _STAGE_MB unless SetSegmentSizes overrode them. Unconditional // on purpose - see SizesFromConfig. if (!m_sizesSet) { @@ -997,6 +1058,39 @@ namespace MobileGL::MG_Remote::Server { PublishEvents(); } + // PH-6 (ID-P7-2). THE FIRST DROP, AND THE LATCH IT RAISES. + // + // Logged ONCE, by name, with everything a triage needs: which event, how big, what the + // client did (never drained / drained too little / went away), how long the server waited + // against which budget. MGLOG_E and not a Fatal marker on purpose: nothing here is this + // process's defect, so neither the census nor a retrace's `Fatal{` scan may count it as one - + // and the session that follows is an ordinary stop, exit 0, not a SessionFault. + void* ServerSession::ForfeitReverseChannel(const char* cause, const char* why, const char* eventName, + Uint64 payloadBytes, Uint32 waitedMs, Uint32 budgetMs, + Uint32 shortDrains) { + if (!m_reverseChannelForfeit.exchange(true, std::memory_order_acq_rel)) { + MGLOG_E("MGPipe: ReverseChannelForfeit{%s} - %s needs %llu bytes on a full SEG_EVENT and %s " + "(waited %u ms of MOBILEGL_IPC_EVENT_WAIT_MS=%u, %u short drain(s)). The reverse " + "channel is lossless (CONTRACT-P5C §4.4), so a session that drops one event cannot " + "go on: this event and every later one is dropped and counted, and the session " + "stops by the ordinary stop path (PH-6, ID-P7-2)", + cause, eventName, static_cast(payloadBytes), why, waitedMs, budgetMs, + shortDrains); + } + return DropEventAfterForfeit(payloadBytes); + } + + // The producer still fills and publishes what it was given, because its code is the same + // whether the slot is SEG_EVENT or not; the scratch is where that write goes to die. Apply + // thread only (SEG_EVENT has one producer), so the one buffer needs no lock. + void* ServerSession::DropEventAfterForfeit(Uint64 payloadBytes) { + if (m_events != nullptr) m_events->CountDrop(); + m_forfeitDrops.fetch_add(1, std::memory_order_acq_rel); + const SizeT bytes = static_cast(payloadBytes == 0 ? 1 : payloadBytes); + if (m_forfeitScratch.size() < bytes) m_forfeitScratch.resize(bytes); + return m_forfeitScratch.data(); + } + // Both of these advance AND ring, through SessionConsumer. The free functions in namespace // Watermark do not ring: a client parked in WaitForPresentAck(kWaitForever) needs the pair. void ServerSession::AdvanceCompletedFrame(Uint64 serial) { diff --git a/MobileGL/MG_Remote/Server/ServerSession.h b/MobileGL/MG_Remote/Server/ServerSession.h index e5c4e612..3985d1f2 100644 --- a/MobileGL/MG_Remote/Server/ServerSession.h +++ b/MobileGL/MG_Remote/Server/ServerSession.h @@ -42,6 +42,7 @@ #pragma once #include "../Transport/ILink.h" #include +#include #include #include #include @@ -211,6 +212,38 @@ namespace MobileGL::MG_Remote::Server { void PostGlError(Uint32 code, const char* message); Transport::ITransport* Control_Plane(); + // ---- PH-6 (ID-P7-2): THE REVERSE CHANNEL IS FORFEITED, NOT FATAL ------------------- + // + // A run-ahead producer that finds SEG_EVENT full parks until the client drains room + // (CONTRACT-P5E §2.6). What a client that does not drain used to cost was two Fatals + // inside ReserveEventOrBlock - "waited 30000 ms", twice, and "a drained ring that still + // refuses a record it fits is a corrupt cursor set", which a peer that drains one slot at + // a time reaches in milliseconds. Both are the PEER's behaviour, not a defect of this + // process, so neither may abort it. + // + // What replaces them is a LATCH on this session. The first event that cannot be placed + // within MOBILEGL_IPC_EVENT_WAIT_MS raises it (ForfeitReverseChannel, which logs + // `ReverseChannelForfeit{}` once); that event and every later one is a counted + // drop (CountDrop on the shared page, ForfeitDrops() here) into scratch the producer may + // fill and forget; the apply loop's park predicate sees the latch and takes the ordinary + // stop path; ServerMain's control loop sees the apply thread gone and ends the session. + // The process exits 0 and the supervisor welcomes the next connection. The latch is + // set, read and reset on the apply thread's side of Accept/Close only, and read with + // acquire elsewhere (the control loop, tests). + Bool ReverseChannelForfeited() const { + return m_reverseChannelForfeit.load(std::memory_order_acquire); + } + // Drops since the latch went up, counted HERE because the shared page's eventDropped is + // on a page the peer can write. + Uint64 ForfeitDrops() const { return m_forfeitDrops.load(std::memory_order_acquire); } + // `cause` is one word (the braces' content); `why` the sentence; `shortDrains` how many + // times the client cleared the latch without making room. Returns the scratch the caller + // writes the dropped event into. + void* ForfeitReverseChannel(const char* cause, const char* why, const char* eventName, + Uint64 payloadBytes, Uint32 waitedMs, Uint32 budgetMs, + Uint32 shortDrains); + void* DropEventAfterForfeit(Uint64 payloadBytes); + // completedFrameSerial / presentAckSerial: the two watermarks only the server can // advance, kept together with the other three rather than poked into RingControl from // whatever code happens to notice a present finished. @@ -246,6 +279,10 @@ namespace MobileGL::MG_Remote::Server { Bool m_consumedSet = false; Bool m_sizesSet = false; Bool m_accepted = false; + // PH-6. Written by the apply thread (the only SEG_EVENT producer), reset by Accept. + std::atomic m_reverseChannelForfeit{false}; + std::atomic m_forfeitDrops{0}; + std::vector m_forfeitScratch; }; // Leak-at-exit like every other MG_Remote singleton (ID-8): no frontend destructor may diff --git a/MobileGL/MG_Remote/Server/ServerSpawn.cpp b/MobileGL/MG_Remote/Server/ServerSpawn.cpp index d751b0b7..d96dbc0a 100644 --- a/MobileGL/MG_Remote/Server/ServerSpawn.cpp +++ b/MobileGL/MG_Remote/Server/ServerSpawn.cpp @@ -97,10 +97,14 @@ namespace MobileGL::MG_Remote::Server { // audit value, none names an endpoint, a role, a path or a segment size, and ServerMain's // catch (b) keeps verifying the four it verifies (TRANSPORT, SERVER_PATH, RING_MB, // STAGE_MB). Everything else under the prefix still goes. + // + // PH-6 (ID-P7-2) adds MOBILEGL_IPC_EVENT_WAIT_MS on the same argument: it is the server's + // patience with a client that has stopped draining SEG_EVENT, nothing on the client reads + // it, and scrubbing it would make a spawned server wait the default whatever the lane set. bool ShouldScrub(const char* entry) { static constexpr const char* kServerOwned[] = { "MOBILEGL_IPC_WIRE_DEFERRED_MB=", "MOBILEGL_IPC_SPIN_US=", "MOBILEGL_IPC_SERVER_AFFINITY=", - "MOBILEGL_IPC_AUDIT=", + "MOBILEGL_IPC_AUDIT=", "MOBILEGL_IPC_EVENT_WAIT_MS=", }; for (const char* kept : kServerOwned) { if (std::strncmp(entry, kept, std::strlen(kept)) == 0) { diff --git a/MobileGL/MG_Test/Wire/RemoteClientTest.cpp b/MobileGL/MG_Test/Wire/RemoteClientTest.cpp index 3f6e24d4..57922244 100644 --- a/MobileGL/MG_Test/Wire/RemoteClientTest.cpp +++ b/MobileGL/MG_Test/Wire/RemoteClientTest.cpp @@ -2823,6 +2823,146 @@ TEST(RemoteRunAhead, AnEventBurstBehindAnUnwaitedSequenceFlowControlsInsteadOfAb ExpectChildSuccess(child); } +// =========================================================================================== +// PH-6 (ID-P7-2): a client that does not drain SEG_EVENT FORFEITS the reverse channel +// =========================================================================================== +// +// The flow-control case above is the healthy half: a client that drains late gets every event. +// These two are the other half, and they are why ReserveEventOrBlock's two BUSY refusals became +// a latch. Each child reads its OWN outcome - the latch, the server's drop tally, the shared +// page's eventDropped the client sees, the time spent against MOBILEGL_IPC_EVENT_WAIT_MS, and the +// apply thread leaving by itself - and exits non-zero on the first that is wrong; the parent then +// asserts the named line and the ABSENCE of Fatal{EventRingOverflow}. +namespace { + // Waits for mgl-srv-apply to leave on its own: the forfeit's stop is the ordinary one, so + // nobody calls Stop() to get it out. + bool ApplyThreadLeavesWithin(std::chrono::milliseconds budget) { + const auto until = std::chrono::steady_clock::now() + budget; + while (std::chrono::steady_clock::now() < until) { + if (!Srv::ServerLoopInstance().Running()) return true; + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return false; + } + + Uint64 MillisecondsSince(std::chrono::steady_clock::time_point start) { + return static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count()); + } +} // namespace + +// STONEWALL: the client never drains. 3000 GPU-write marks of ~1 KiB against a 256 KiB ring: +// ~250 fit, the next one waits MOBILEGL_IPC_EVENT_WAIT_MS (300 ms here) and forfeits, and the +// remaining ~2750 are drops that cost no wait at all - so the whole burst returns in a little over +// the knob, not in 30 s and not with an abort. +// +// THE RED (recorded in the F2 package note): put ReserveEventOrBlock back to its two 30000 ms +// rounds and this child is killed by StartRunAheadSession's alarm(15) inside the first wait - +// before that change it would have aborted at 30 s with Fatal{EventRingOverflow} "waited 30000 ms". +TEST(RemoteRunAhead, AStonewallingClientForfeitsTheReverseChannelWithinTheWaitKnob) { + const auto child = RunInChild([] { + MG_Config::Ipc.EventWaitMs = 300; + StartRunAheadSession(); + const auto start = std::chrono::steady_clock::now(); + Srv::ServerLoopInstance().RunProbeOnApplyThreadForTesting( + +[](void*) -> MobileGLResult { + MG_Pipe::MGPRange ranges[64]; + for (Uint32 r = 0; r < 64; ++r) ranges[r] = MG_Pipe::MGPRange{r * 64ull, 64ull}; + for (Uint32 i = 0; i < 3000; ++i) { + MG_Pipe::gMGPipeCallbacks.OnGpuWritten(MG_Pipe::MGPipeHandle{7, 1}, 64, ranges); + } + return MOBILEGL_OK; + }, + nullptr); + const Uint64 elapsedMs = MillisecondsSince(start); + auto& server = Srv::ServerSessionInstance(); + if (!server.ReverseChannelForfeited()) ::_exit(101); + const Uint64 drops = server.ForfeitDrops(); + // Some fitted (the ring was empty) and some did not (it holds ~250 of 3000). + if (drops == 0 || drops >= 3000u) ::_exit(102); + // The shared page's counter is the one a client can read; the server's own tally is the + // one a peer cannot write. They agree when nothing else counts. + if (ClientSessionInstance().Events().DroppedEvents() != drops) ::_exit(103); + // The knob, spent once: at least most of it, and nowhere near the old 30 s. + if (elapsedMs < 250u || elapsedMs > 5000u) ::_exit(104); + if (!ApplyThreadLeavesWithin(std::chrono::milliseconds(3000))) ::_exit(105); + ClientSessionInstance().Stop(); + }); + ExpectChildSuccess(child); + EXPECT_NE(child.Log.find("ReverseChannelForfeit{NotDraining} - kEventGpuWritten"), std::string::npos) + << child.Log; + EXPECT_NE(child.Log.find("mgl-srv-apply stops on ReverseChannelForfeit"), std::string::npos) << child.Log; + EXPECT_EQ(child.Log.find("Fatal{EventRingOverflow"), std::string::npos) << child.Log; +} + +// TRICKLE: the client drains, one record per 50 ms, and the record the server is waiting to place +// is twenty times bigger than what each drain frees. Every drain clears the latch and rings the +// server; every retry still does not fit. The old shape gave up after TWO such rounds and called +// it "a drained ring that still refuses a record it fits is a corrupt cursor set" - ~100 ms into a +// session whose only fault was a slow reader, and a Fatal. Now the rounds share one deadline: +// the server keeps retrying for the whole knob (600 ms here) and then forfeits by name, TooSlow, +// with the short drains counted. +// +// THE RED (recorded in the F2 package note): the old two-round ReserveEventOrBlock aborts this +// child with Fatal{EventRingOverflow} "could not reserve ... on an emptied SEG_EVENT". +TEST(RemoteRunAhead, ATricklingClientGetsTheWholeWaitThenForfeitsByNameInsteadOfAborting) { + const auto child = RunInChild([] { + MG_Config::Ipc.EventWaitMs = 600; + StartRunAheadSession(); + std::atomic stop{false}; + std::atomic popped{0}; + std::thread trickler([&stop, &popped] { + Transport::EventRingConsumer& events = ClientSessionInstance().Events(); + while (!stop.load(std::memory_order_acquire)) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + Transport::RingRecordView view; + if (events.Pop(view)) popped.fetch_add(1); + // Releases what was popped, clears the latch and rings the server: a real drain, + // just a stingy one. + events.Drained(); + } + }); + const auto start = std::chrono::steady_clock::now(); + Srv::ServerLoopInstance().RunProbeOnApplyThreadForTesting( + +[](void*) -> MobileGLResult { + // Fill with ONE-range marks (48 bytes a record) until under 4 KiB is left, without + // ever blocking - so the fill is not paced by the trickle - then ask for 64-range + // marks (~1 KiB each). The first of those that does not fit is the one the server + // waits for, and one popped 48-byte record never makes room for it. + auto& ring = Srv::ServerSessionInstance().Events().Ring(); + const MG_Pipe::MGPRange one{0, 64}; + while (ring.FreeBytes() > 4096u) { + MG_Pipe::gMGPipeCallbacks.OnGpuWritten(MG_Pipe::MGPipeHandle{7, 1}, 1, &one); + } + MG_Pipe::MGPRange ranges[64]; + for (Uint32 r = 0; r < 64; ++r) ranges[r] = MG_Pipe::MGPRange{r * 64ull, 64ull}; + for (Uint32 i = 0; i < 16; ++i) { + MG_Pipe::gMGPipeCallbacks.OnGpuWritten(MG_Pipe::MGPipeHandle{7, 1}, 64, ranges); + } + return MOBILEGL_OK; + }, + nullptr); + const Uint64 elapsedMs = MillisecondsSince(start); + stop.store(true, std::memory_order_release); + trickler.join(); + auto& server = Srv::ServerSessionInstance(); + if (!server.ReverseChannelForfeited()) ::_exit(101); + if (server.ForfeitDrops() == 0) ::_exit(102); + // The trickle really happened while the server waited - otherwise this is the stonewall + // case under another name. + if (popped.load() < 2u) ::_exit(103); + // The WHOLE budget was spent retrying, not two rounds of it. + if (elapsedMs < 540u || elapsedMs > 5000u) ::_exit(104); + if (!ApplyThreadLeavesWithin(std::chrono::milliseconds(3000))) ::_exit(105); + ClientSessionInstance().Stop(); + }); + ExpectChildSuccess(child); + EXPECT_NE(child.Log.find("ReverseChannelForfeit{TooSlow} - kEventGpuWritten"), std::string::npos) + << child.Log; + EXPECT_EQ(child.Log.find("Fatal{EventRingOverflow"), std::string::npos) << child.Log; +} + // §2.7 / ruling 13: the deferred-destroy queue stays, and an enqueue from an unbarriered // apply is the finding. Under strict it is the abort; this is the arm the lane owns. TEST(RemoteGuards, ADeferredDestroyFromAnUnbarrieredApplyIsFatalUnderStrict) { diff --git a/docs/Disaggregated/ARCHITECTURE.md b/docs/Disaggregated/ARCHITECTURE.md index 455e140c..95968296 100644 --- a/docs/Disaggregated/ARCHITECTURE.md +++ b/docs/Disaggregated/ARCHITECTURE.md @@ -653,6 +653,7 @@ CMake： | `MOBILEGL_IPC_RING_MB` | 8 | `SEG_CMD`；单条记录至多一半 | | `MOBILEGL_IPC_STAGE_MB` | 32 | `SEG_STAGE`；目标负载 profile 显式 256 | | `MOBILEGL_IPC_SPIN_US` | 50 | park 前自旋；P5d 三轮起自旋按一次性校准的迭代预算走、稳态不读时钟，`0` = 不自旋直接 park | +| `MOBILEGL_IPC_EVENT_WAIT_MS` | 2000（1–600000） | PH-6（ID-P7-2），只由 server 读（`ServerSpawn.cpp` 放行给 spawn 子进程）：run-ahead 下一条反向事件遇到满的 `SEG_EVENT` 时，server 等 client 腾出空间的**整笔预算**（不是每次 park 的）。到期、client 只涓流不腾够、或 client 的 bell 已死（先判死，不误报为超时）→ `ReverseChannelForfeit{NotDraining\|TooSlow\|PeerGone}`：该事件及其后全部计数丢弃（`eventDropped`），apply 循环按正常 Stop 退出，会话 exit 0，supervisor 照常欢迎下一连接。取代原先 30000 ms × 两轮后的 `Fatal{EventRingOverflow}`。lockstep 臂（无 `kCapRunAheadApply`，即 Magma）不等待，仍是 P5C 的 Fatal | | `MOBILEGL_IPC_PERSISTENT_BLOCK_KB` | 64 | persistent-map 推送块粒度；`0` 是 E3(a) 阴性对照 | | `MOBILEGL_IPC_PERSISTENT_HASH_SUPPRESS` | 1 | 推送只发内容变了的块（追踪 buffer 走 mprotect 位图，未追踪走内容哈希）；`0` 恢复全范围推送（A/B 对照） | | `MOBILEGL_IPC_BATCH_WAITS` | 1 | 值类记录（无 reply slot 的 kCtxState/kCtxCso/kCtxObject）发布即返回，barrier 推迟到下一个拉取类 verb；**`generate_mipmap` 例外**（它的 apply 读 `MGB_CTX->GetActiveTextureUnit()`，规则：只有 apply 不读残余填充字段的记录才可免等）；`0` 恢复 R-1 逐条 barrier（`MOBILEGL_PIPE_VERIFY` 强制 0） |