mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-11-07 01:53:24 +09:00
[MG_Remote, MG_Test, docs] (P7): forfeit the reverse channel on a non-draining peer (PH-6)
This commit is contained in:
@@ -513,6 +513,17 @@ namespace MobileGL::MG_Config {
|
||||
Uint32 WireDeferredMb = 64;
|
||||
// MOBILEGL_IPC_SPIN_US: spin before parking on a doorbell, either direction.
|
||||
Uint32 SpinUs = 50;
|
||||
// MOBILEGL_IPC_EVENT_WAIT_MS (PH-6, ID-P7-2): the SERVER's patience, in ms, for ONE
|
||||
// reverse-channel event that finds SEG_EVENT full under run-ahead. The apply thread
|
||||
// publishes, rings the client and parks until the client drains enough room; a client
|
||||
// that has not made room when this runs out - it stopped draining, it drains a slot at
|
||||
// a time, or it went away - FORFEITS the reverse channel: that event and every later
|
||||
// one is dropped and counted (eventDropped), and the session stops by the ordinary
|
||||
// stop path rather than by Fatal{EventRingOverflow}. It bounds the whole reservation,
|
||||
// not one park, so a trickling peer cannot stretch it. Was a 30000 ms constant spent
|
||||
// twice. Read by the server only (ServerSpawn.cpp passes it to a spawned child); the
|
||||
// lockstep arm (no kCapRunAheadApply, i.e. Magma) never waits and keeps P5C's Fatal.
|
||||
Uint32 EventWaitMs = 2000;
|
||||
// MOBILEGL_IPC_PERSISTENT_BLOCK_KB: block granularity of the persistent-map push.
|
||||
// 0 IS A NEGATIVE CONTROL, NOT "unlimited": it disables the push, and
|
||||
// PersistentCoherentMapScenario must go RED under it (exit gate E3(a)).
|
||||
|
||||
@@ -396,6 +396,10 @@ namespace MobileGL::MG_ConfigLoader {
|
||||
// control: no forced sync, and MagmaWireReclaimScenario's watermark case must go red.
|
||||
ipc.WireDeferredMb = QueryEnvUint32("MOBILEGL_IPC_WIRE_DEFERRED_MB", 64, 0, 65536);
|
||||
ipc.SpinUs = QueryEnvUint32("MOBILEGL_IPC_SPIN_US", 50, 0, 1000000);
|
||||
// PH-6 (ID-P7-2; Config.h has the semantics). 0 is NOT admitted: a server with no
|
||||
// patience at all would forfeit a healthy run-ahead client the first time its ring
|
||||
// filled between two of the client's drains, which is an ordinary backlog.
|
||||
ipc.EventWaitMs = QueryEnvUint32("MOBILEGL_IPC_EVENT_WAIT_MS", 2000, 1, 600000);
|
||||
// 0 is admitted ON PURPOSE and is the negative control of exit gate E3(a): it turns
|
||||
// the persistent-map push OFF, and PersistentCoherentMapScenario must go red.
|
||||
ipc.PersistentBlockKb = QueryEnvUint32("MOBILEGL_IPC_PERSISTENT_BLOCK_KB", 64, 0, 65536);
|
||||
@@ -469,10 +473,12 @@ namespace MobileGL::MG_ConfigLoader {
|
||||
if (MG_Config::Transport == MG_Config::TransportMode::Monolith) return;
|
||||
// One line, on the arm where these numbers decide behaviour, because every one of
|
||||
// them is a number a bug report has to quote.
|
||||
MGLOG_I("Config: IPC ring=%uMiB stage=%uMiB wire-deferred=%uMiB spin=%uus persistent-block=%uKiB "
|
||||
MGLOG_I("Config: IPC ring=%uMiB stage=%uMiB wire-deferred=%uMiB spin=%uus event-wait=%ums "
|
||||
"persistent-block=%uKiB "
|
||||
"adopt-tier=%u verb-barrier=%u run-ahead=%u present-credit=%u control-timeout=%ums "
|
||||
"cold-start=%ums strict=%d audit=%d role-split-state=%d affinity='%s'",
|
||||
ipc.RingMb, ipc.StageMb, ipc.WireDeferredMb, ipc.SpinUs, ipc.PersistentBlockKb, ipc.AdoptTier,
|
||||
ipc.RingMb, ipc.StageMb, ipc.WireDeferredMb, ipc.SpinUs, ipc.EventWaitMs,
|
||||
ipc.PersistentBlockKb, ipc.AdoptTier,
|
||||
ipc.VerbBarrier, ipc.RunAhead, ipc.PresentCredit, ipc.ControlTimeoutMs, ipc.ColdStartMs,
|
||||
static_cast<int>(ipc.StrictErrors), static_cast<int>(ipc.Audit),
|
||||
static_cast<int>(ipc.RoleSplitState), ipc.ServerAffinity.c_str());
|
||||
|
||||
@@ -402,17 +402,27 @@ namespace MobileGL::MG_Remote::Server {
|
||||
// this bell (that pairing is the whole of the flow-control protocol; a cleared flag
|
||||
// with no bell is the lost wakeup the forward direction's publish-then-ring order
|
||||
// exists to prevent).
|
||||
const auto ready = [this, signals, &ring] {
|
||||
if (m_stopRequested.load(std::memory_order_acquire) || ControlIsPending()) return true;
|
||||
//
|
||||
// PH-6 (ID-P7-2) ADDS THE FORFEIT LATCH BESIDE STOP, AND FOR STOP'S REASON. A session whose
|
||||
// reverse channel was forfeited (ServerSession::ForfeitReverseChannel) has a client that
|
||||
// is not reading SEG_EVENT, so its ring stays full - and the ring test below would park
|
||||
// this thread on `eventRingFull` for ever, waiting for a drain that the forfeit has
|
||||
// already decided will not come. The latch wins over the ring exactly as Stop does, and
|
||||
// the loop takes the same way out.
|
||||
const auto stopOrForfeit = [this, &session] {
|
||||
return m_stopRequested.load(std::memory_order_acquire) || session.ReverseChannelForfeited();
|
||||
};
|
||||
const auto ready = [this, signals, &ring, &stopOrForfeit] {
|
||||
if (stopOrForfeit() || ControlIsPending()) return true;
|
||||
if (signals.EventRingFull->load(std::memory_order_acquire) != 0) return false;
|
||||
return signals.CmdHead->load(std::memory_order_acquire) != ring.LocalTail();
|
||||
};
|
||||
|
||||
for (;;) {
|
||||
PumpControlRequest();
|
||||
if (m_stopRequested.load(std::memory_order_acquire)) break;
|
||||
if (stopOrForfeit()) break;
|
||||
if (signals.EventRingFull->load(std::memory_order_acquire) == 0) DrainRing();
|
||||
if (m_stopRequested.load(std::memory_order_acquire)) break;
|
||||
if (stopOrForfeit()) break;
|
||||
if (ready()) continue;
|
||||
|
||||
const Uint64 waits = m_parks.fetch_add(1, std::memory_order_acq_rel) + 1;
|
||||
@@ -466,8 +476,23 @@ namespace MobileGL::MG_Remote::Server {
|
||||
// drain (ClientSession::Stop step 1) normally empties the ring first and is BOUNDED, so
|
||||
// a timeout there leaves records here; applying them costs nothing and dropping them
|
||||
// would leave the emitter's var-tails referenced by records nobody ever read.
|
||||
//
|
||||
// EXCEPT AFTER A FORFEIT (PH-6). That stop was not asked for by a client finishing its
|
||||
// work; it was decided about a peer that stopped reading this session's answers, and
|
||||
// every event the rest of its queue produced would only be one more counted drop - while
|
||||
// a peer that keeps publishing could keep this final drain going as long as it liked,
|
||||
// which is exactly the unbounded hold the forfeit exists to end. The queue is left
|
||||
// unapplied; the rings die with the session.
|
||||
const Bool forfeited = session.ReverseChannelForfeited();
|
||||
PumpControlRequest();
|
||||
DrainRing();
|
||||
if (forfeited) {
|
||||
MGLOG_E("MG_Remote server: mgl-srv-apply stops on ReverseChannelForfeit - %llu reverse-channel "
|
||||
"event(s) dropped, applied seq %llu; the records still queued are not applied",
|
||||
static_cast<unsigned long long>(session.ForfeitDrops()),
|
||||
static_cast<unsigned long long>(session.Consumer().AppliedSeq()));
|
||||
} else {
|
||||
DrainRing();
|
||||
}
|
||||
|
||||
// THE BACKEND IS DESTROYED ON THIS THREAD, WHILE IT IS STILL THE CONTEXT OWNER.
|
||||
// ~BackendObject_DirectGLES calls DestroyEGLContext (BackendObject_DirectGLES.cpp:
|
||||
@@ -633,6 +658,10 @@ namespace MobileGL::MG_Remote::Server {
|
||||
}
|
||||
if (!popped) break;
|
||||
++applied;
|
||||
// PH-6: the record whose event forfeited the reverse channel is the last one this
|
||||
// session applies. Checked only AFTER a popped record, so the empty-ring answer above
|
||||
// still costs its two loads and nothing else.
|
||||
if (session.ReverseChannelForfeited()) break;
|
||||
}
|
||||
if (applied != 0) {
|
||||
// THE TWO TALLIES MUST AGREE, AND THAT IS WHAT MAKES R-9's BATCHING BAN CHECKABLE.
|
||||
|
||||
@@ -350,9 +350,26 @@ void SendLogAck(void* pointer) {
|
||||
WireLogError("MG_Remote server: pid=%d transport=spawn role=server ready control=%s data=%s",
|
||||
selfPid, tcp ? "tcp" : "unix", tcp ? "stream" : "shm");
|
||||
std::vector<std::uint8_t> buffer(64 * 1024);
|
||||
// PH-6 (ID-P7-2): THE CONTROL WAIT IS BOUNDED, SO A SESSION CAN END FROM THE SERVER'S SIDE.
|
||||
// It used to be kWaitForever, which left exactly one way out: the peer closing its control
|
||||
// connection. A peer that forfeited the reverse channel is by definition one that stopped
|
||||
// reading, and it may well keep that connection open - so the apply thread would stop and
|
||||
// this child would sit here, holding the supervisor's one session slot, answering every
|
||||
// later client Busy. Now each quiet interval asks whether the apply thread is still there;
|
||||
// when it is not (forfeit, or a dead data bell), the session takes the ordinary exit below:
|
||||
// Stop, Close, _exit(0). The interval is the supervisor's own accept poll.
|
||||
constexpr std::uint32_t kControlPollMs = 250;
|
||||
for (;;) {
|
||||
std::uint64_t size = 0;
|
||||
const auto result = control->ReceiveFrame({buffer.data(), buffer.size()}, &size, kWaitForever);
|
||||
const auto result = control->ReceiveFrame({buffer.data(), buffer.size()}, &size, kControlPollMs);
|
||||
if (result == MOBILEGL_ERR_TIMEOUT) {
|
||||
if (loop.Running()) continue;
|
||||
WireLogError("MG_Remote server: pid=%d the apply thread has stopped (%s, %llu event(s) dropped); "
|
||||
"ending the session",
|
||||
selfPid, session.ReverseChannelForfeited() ? "ReverseChannelForfeit" : "its bell died",
|
||||
static_cast<unsigned long long>(session.ForfeitDrops()));
|
||||
break;
|
||||
}
|
||||
if (result == MOBILEGL_ERR_BUFFER_TOO_SMALL) { buffer.resize(static_cast<std::size_t>(size)); continue; }
|
||||
if (result != MOBILEGL_OK) break;
|
||||
// THE FILE IDENTIFIER, ASKED FIRST AND SAID OUT LOUD (P7 wave 0).
|
||||
|
||||
@@ -239,19 +239,39 @@ namespace MobileGL::MG_Remote::Server {
|
||||
// `eventRingFull` as well (SessionProducer::WaitFor*OrEventBacklog) - so a parked
|
||||
// client is always wakeable by the server that is waiting for it.
|
||||
//
|
||||
// TWO NAMED REFUSALS SURVIVE, and they are the impossible cases rather than the busy
|
||||
// ones: a single event larger than the ring can EVER hold (no amount of draining
|
||||
// helps), and a wait that ran out of patience (the peer is gone or wedged). A server
|
||||
// that does not publish kCapRunAheadApply keeps P5C's Fatal unchanged.
|
||||
// ONE NAMED REFUSAL SURVIVES, the impossible case: a single event larger than the ring
|
||||
// can EVER hold (no amount of draining helps). The two BUSY refusals that used to sit
|
||||
// beside it - a wait that ran out of patience, and two rounds against a ring the record
|
||||
// fits - are the PEER's behaviour and became the forfeit latch below (PH-6, ID-P7-2).
|
||||
// A server that does not publish kCapRunAheadApply keeps P5C's Fatal unchanged.
|
||||
Bool ServerPublishesRunAhead(const ServerSession& session) {
|
||||
if (!session.CallMaskIsSet()) return false;
|
||||
return (session.CallMask() & static_cast<Uint64>(MG_Pipe::kCapRunAheadApply)) != 0;
|
||||
}
|
||||
|
||||
constexpr Uint32 kEventBacklogWaitMs = 30000;
|
||||
// PH-6 (ID-P7-2): MOBILEGL_IPC_EVENT_WAIT_MS (Config.h). It was a 30000 ms constant, and
|
||||
// it was spent TWICE per reservation (two rounds), so the plan's "60 seconds" was really
|
||||
// 30 s against a peer that never drains and a few milliseconds against one that trickles.
|
||||
Uint32 EventBacklogWaitMs() {
|
||||
#if MOBILEGL_BUILD_DISAGGREGATED
|
||||
return MG_Config::Ipc.EventWaitMs;
|
||||
#else
|
||||
return 2000;
|
||||
#endif
|
||||
}
|
||||
|
||||
Uint32 MillisecondsSince(std::chrono::steady_clock::time_point start) {
|
||||
const auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - start).count();
|
||||
return elapsed <= 0 ? 0u : static_cast<Uint32>(elapsed);
|
||||
}
|
||||
|
||||
void* ReserveEventOrBlock(ServerSession& session, Transport::EventKind kind,
|
||||
const char* eventName, Uint64 payloadBytes) {
|
||||
// FORFEITED: every event after the first drop is a drop too, at once - no Reserve and
|
||||
// no second wait. The session is on its way down, and the peer has already shown it
|
||||
// does not read this ring.
|
||||
if (session.ReverseChannelForfeited()) return session.DropEventAfterForfeit(payloadBytes);
|
||||
void* slot = session.Events().Reserve(kind, payloadBytes);
|
||||
if (slot != nullptr) return slot;
|
||||
if (!ServerPublishesRunAhead(session)) FatalEventRingOverflow(eventName, payloadBytes);
|
||||
@@ -275,29 +295,66 @@ namespace MobileGL::MG_Remote::Server {
|
||||
const auto drained = [signals] {
|
||||
return signals.EventRingFull->load(std::memory_order_acquire) == 0;
|
||||
};
|
||||
for (Uint32 round = 0; round < 2; ++round) {
|
||||
// ONE DEADLINE FOR THE WHOLE RESERVATION, NOT ONE PER PARK. The loop goes round as
|
||||
// often as the client clears the latch - that is how a client that drains in pieces
|
||||
// eventually makes room - but every round spends the same budget, so a peer that
|
||||
// clears the latch without freeing enough (it drains one slot per interval, or it
|
||||
// writes the flag itself) cannot hold the apply thread longer than the knob says.
|
||||
// The old shape gave up after TWO rounds, and called that "a corrupt cursor set":
|
||||
// a trickling peer reached it in milliseconds and took the process down with it.
|
||||
const Uint32 budgetMs = EventBacklogWaitMs();
|
||||
const auto start = std::chrono::steady_clock::now();
|
||||
const auto deadline = start + std::chrono::milliseconds(budgetMs);
|
||||
// Rounds in which the client DID clear the latch and the record still did not fit.
|
||||
// Zero at the deadline means the client never drained at all; more means it drained
|
||||
// too little, too slowly - two different peers, and the log says which one this was.
|
||||
Uint32 shortDrains = 0;
|
||||
const auto outOfPatience = [&]() {
|
||||
return shortDrains == 0
|
||||
? session.ForfeitReverseChannel("NotDraining", "the client did not drain SEG_EVENT",
|
||||
eventName, payloadBytes, MillisecondsSince(start),
|
||||
budgetMs, shortDrains)
|
||||
: session.ForfeitReverseChannel("TooSlow",
|
||||
"the client drained SEG_EVENT but never made room for "
|
||||
"this record",
|
||||
eventName, payloadBytes, MillisecondsSince(start),
|
||||
budgetMs, shortDrains);
|
||||
};
|
||||
for (;;) {
|
||||
// PUBLISH AND RING FIRST. The client cannot drain a ring whose head it has not
|
||||
// been shown, and the latch Reserve just set is only useful to a client that
|
||||
// gets a bell with it.
|
||||
session.PublishEvents();
|
||||
if (!drained() && !bell.Wait(*signals.ConsumerParked, drained, /*spinUs=*/0,
|
||||
kEventBacklogWaitMs)) {
|
||||
SessionFail(MGFatalFamily::EventRingOverflow, "MGPipe: Fatal{EventRingOverflow} - %s waited %u ms for the client "
|
||||
"to drain SEG_EVENT and it never did. Under run-ahead the apply "
|
||||
"thread parks on this ring (CONTRACT-P5E §2.6); a client that does "
|
||||
"not drain is one that is not running",
|
||||
eventName, kEventBacklogWaitMs);
|
||||
// DEAD FIRST (the plan's own clause): a bell whose peer is gone answers every
|
||||
// Wait with an immediate false, and reporting that as "waited N ms" would name
|
||||
// a timeout for what is a hangup.
|
||||
if (bell.Dead()) {
|
||||
return session.ForfeitReverseChannel("PeerGone", "the client's doorbell is dead - the peer is gone",
|
||||
eventName, payloadBytes, MillisecondsSince(start), budgetMs,
|
||||
shortDrains);
|
||||
}
|
||||
if (!drained()) {
|
||||
const auto left = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
deadline - std::chrono::steady_clock::now()).count();
|
||||
if (left <= 0) return outOfPatience();
|
||||
if (!bell.Wait(*signals.ConsumerParked, drained, /*spinUs=*/0, static_cast<Uint32>(left))) {
|
||||
if (bell.Dead()) {
|
||||
return session.ForfeitReverseChannel(
|
||||
"PeerGone", "the client's doorbell died while the server waited - the peer is gone",
|
||||
eventName, payloadBytes, MillisecondsSince(start), budgetMs, shortDrains);
|
||||
}
|
||||
return outOfPatience();
|
||||
}
|
||||
}
|
||||
slot = session.Events().Reserve(kind, payloadBytes);
|
||||
if (slot != nullptr) return slot;
|
||||
// The client cleared the latch and the record STILL does not fit: it drained less
|
||||
// than this record needs. That is a slow drain, not a corrupt cursor set, and it
|
||||
// gets the rest of the same budget - Reserve has just raised the latch again, so
|
||||
// the next round parks until the client clears it once more.
|
||||
++shortDrains;
|
||||
if (std::chrono::steady_clock::now() >= deadline) return outOfPatience();
|
||||
}
|
||||
// Two full rounds against a ring the record provably fits: something else is
|
||||
// producing into it, which on this side is impossible (one producer, the apply
|
||||
// thread, by construction).
|
||||
SessionFail(MGFatalFamily::EventRingOverflow, "MGPipe: Fatal{EventRingOverflow} - %s could not reserve %llu bytes on an "
|
||||
"emptied SEG_EVENT. The apply thread is SEG_EVENT's only producer, so a "
|
||||
"drained ring that still refuses a record it fits is a corrupt cursor set",
|
||||
eventName, static_cast<unsigned long long>(payloadBytes));
|
||||
}
|
||||
|
||||
// Accept owns the single process callback table. Its owner need not be the
|
||||
@@ -493,6 +550,10 @@ namespace MobileGL::MG_Remote::Server {
|
||||
}
|
||||
} releaseOnFailure{this};
|
||||
m_transport = &transport;
|
||||
// PH-6: a forfeit is THIS session's fact. A process that accepts again (inproc context
|
||||
// loss, a test's second session) starts with a reverse channel it has not given up.
|
||||
m_reverseChannelForfeit.store(false, std::memory_order_release);
|
||||
m_forfeitDrops.store(0, std::memory_order_release);
|
||||
// MOBILEGL_IPC_RING_MB / _STAGE_MB unless SetSegmentSizes overrode them. Unconditional
|
||||
// on purpose - see SizesFromConfig.
|
||||
if (!m_sizesSet) {
|
||||
@@ -997,6 +1058,39 @@ namespace MobileGL::MG_Remote::Server {
|
||||
PublishEvents();
|
||||
}
|
||||
|
||||
// PH-6 (ID-P7-2). THE FIRST DROP, AND THE LATCH IT RAISES.
|
||||
//
|
||||
// Logged ONCE, by name, with everything a triage needs: which event, how big, what the
|
||||
// client did (never drained / drained too little / went away), how long the server waited
|
||||
// against which budget. MGLOG_E and not a Fatal marker on purpose: nothing here is this
|
||||
// process's defect, so neither the census nor a retrace's `Fatal{` scan may count it as one -
|
||||
// and the session that follows is an ordinary stop, exit 0, not a SessionFault.
|
||||
void* ServerSession::ForfeitReverseChannel(const char* cause, const char* why, const char* eventName,
|
||||
Uint64 payloadBytes, Uint32 waitedMs, Uint32 budgetMs,
|
||||
Uint32 shortDrains) {
|
||||
if (!m_reverseChannelForfeit.exchange(true, std::memory_order_acq_rel)) {
|
||||
MGLOG_E("MGPipe: ReverseChannelForfeit{%s} - %s needs %llu bytes on a full SEG_EVENT and %s "
|
||||
"(waited %u ms of MOBILEGL_IPC_EVENT_WAIT_MS=%u, %u short drain(s)). The reverse "
|
||||
"channel is lossless (CONTRACT-P5C §4.4), so a session that drops one event cannot "
|
||||
"go on: this event and every later one is dropped and counted, and the session "
|
||||
"stops by the ordinary stop path (PH-6, ID-P7-2)",
|
||||
cause, eventName, static_cast<unsigned long long>(payloadBytes), why, waitedMs, budgetMs,
|
||||
shortDrains);
|
||||
}
|
||||
return DropEventAfterForfeit(payloadBytes);
|
||||
}
|
||||
|
||||
// The producer still fills and publishes what it was given, because its code is the same
|
||||
// whether the slot is SEG_EVENT or not; the scratch is where that write goes to die. Apply
|
||||
// thread only (SEG_EVENT has one producer), so the one buffer needs no lock.
|
||||
void* ServerSession::DropEventAfterForfeit(Uint64 payloadBytes) {
|
||||
if (m_events != nullptr) m_events->CountDrop();
|
||||
m_forfeitDrops.fetch_add(1, std::memory_order_acq_rel);
|
||||
const SizeT bytes = static_cast<SizeT>(payloadBytes == 0 ? 1 : payloadBytes);
|
||||
if (m_forfeitScratch.size() < bytes) m_forfeitScratch.resize(bytes);
|
||||
return m_forfeitScratch.data();
|
||||
}
|
||||
|
||||
// Both of these advance AND ring, through SessionConsumer. The free functions in namespace
|
||||
// Watermark do not ring: a client parked in WaitForPresentAck(kWaitForever) needs the pair.
|
||||
void ServerSession::AdvanceCompletedFrame(Uint64 serial) {
|
||||
|
||||
@@ -42,6 +42,7 @@
|
||||
#pragma once
|
||||
#include "../Transport/ILink.h"
|
||||
#include <Includes.h>
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <vector>
|
||||
@@ -211,6 +212,38 @@ namespace MobileGL::MG_Remote::Server {
|
||||
void PostGlError(Uint32 code, const char* message);
|
||||
Transport::ITransport* Control_Plane();
|
||||
|
||||
// ---- PH-6 (ID-P7-2): THE REVERSE CHANNEL IS FORFEITED, NOT FATAL -------------------
|
||||
//
|
||||
// A run-ahead producer that finds SEG_EVENT full parks until the client drains room
|
||||
// (CONTRACT-P5E §2.6). What a client that does not drain used to cost was two Fatals
|
||||
// inside ReserveEventOrBlock - "waited 30000 ms", twice, and "a drained ring that still
|
||||
// refuses a record it fits is a corrupt cursor set", which a peer that drains one slot at
|
||||
// a time reaches in milliseconds. Both are the PEER's behaviour, not a defect of this
|
||||
// process, so neither may abort it.
|
||||
//
|
||||
// What replaces them is a LATCH on this session. The first event that cannot be placed
|
||||
// within MOBILEGL_IPC_EVENT_WAIT_MS raises it (ForfeitReverseChannel, which logs
|
||||
// `ReverseChannelForfeit{<cause>}` once); that event and every later one is a counted
|
||||
// drop (CountDrop on the shared page, ForfeitDrops() here) into scratch the producer may
|
||||
// fill and forget; the apply loop's park predicate sees the latch and takes the ordinary
|
||||
// stop path; ServerMain's control loop sees the apply thread gone and ends the session.
|
||||
// The process exits 0 and the supervisor welcomes the next connection. The latch is
|
||||
// set, read and reset on the apply thread's side of Accept/Close only, and read with
|
||||
// acquire elsewhere (the control loop, tests).
|
||||
Bool ReverseChannelForfeited() const {
|
||||
return m_reverseChannelForfeit.load(std::memory_order_acquire);
|
||||
}
|
||||
// Drops since the latch went up, counted HERE because the shared page's eventDropped is
|
||||
// on a page the peer can write.
|
||||
Uint64 ForfeitDrops() const { return m_forfeitDrops.load(std::memory_order_acquire); }
|
||||
// `cause` is one word (the braces' content); `why` the sentence; `shortDrains` how many
|
||||
// times the client cleared the latch without making room. Returns the scratch the caller
|
||||
// writes the dropped event into.
|
||||
void* ForfeitReverseChannel(const char* cause, const char* why, const char* eventName,
|
||||
Uint64 payloadBytes, Uint32 waitedMs, Uint32 budgetMs,
|
||||
Uint32 shortDrains);
|
||||
void* DropEventAfterForfeit(Uint64 payloadBytes);
|
||||
|
||||
// completedFrameSerial / presentAckSerial: the two watermarks only the server can
|
||||
// advance, kept together with the other three rather than poked into RingControl from
|
||||
// whatever code happens to notice a present finished.
|
||||
@@ -246,6 +279,10 @@ namespace MobileGL::MG_Remote::Server {
|
||||
Bool m_consumedSet = false;
|
||||
Bool m_sizesSet = false;
|
||||
Bool m_accepted = false;
|
||||
// PH-6. Written by the apply thread (the only SEG_EVENT producer), reset by Accept.
|
||||
std::atomic<Bool> m_reverseChannelForfeit{false};
|
||||
std::atomic<Uint64> m_forfeitDrops{0};
|
||||
std::vector<Uint8> m_forfeitScratch;
|
||||
};
|
||||
|
||||
// Leak-at-exit like every other MG_Remote singleton (ID-8): no frontend destructor may
|
||||
|
||||
@@ -97,10 +97,14 @@ namespace MobileGL::MG_Remote::Server {
|
||||
// audit value, none names an endpoint, a role, a path or a segment size, and ServerMain's
|
||||
// catch (b) keeps verifying the four it verifies (TRANSPORT, SERVER_PATH, RING_MB,
|
||||
// STAGE_MB). Everything else under the prefix still goes.
|
||||
//
|
||||
// PH-6 (ID-P7-2) adds MOBILEGL_IPC_EVENT_WAIT_MS on the same argument: it is the server's
|
||||
// patience with a client that has stopped draining SEG_EVENT, nothing on the client reads
|
||||
// it, and scrubbing it would make a spawned server wait the default whatever the lane set.
|
||||
bool ShouldScrub(const char* entry) {
|
||||
static constexpr const char* kServerOwned[] = {
|
||||
"MOBILEGL_IPC_WIRE_DEFERRED_MB=", "MOBILEGL_IPC_SPIN_US=", "MOBILEGL_IPC_SERVER_AFFINITY=",
|
||||
"MOBILEGL_IPC_AUDIT=",
|
||||
"MOBILEGL_IPC_AUDIT=", "MOBILEGL_IPC_EVENT_WAIT_MS=",
|
||||
};
|
||||
for (const char* kept : kServerOwned) {
|
||||
if (std::strncmp(entry, kept, std::strlen(kept)) == 0) {
|
||||
|
||||
@@ -2823,6 +2823,146 @@ TEST(RemoteRunAhead, AnEventBurstBehindAnUnwaitedSequenceFlowControlsInsteadOfAb
|
||||
ExpectChildSuccess(child);
|
||||
}
|
||||
|
||||
// ===========================================================================================
|
||||
// PH-6 (ID-P7-2): a client that does not drain SEG_EVENT FORFEITS the reverse channel
|
||||
// ===========================================================================================
|
||||
//
|
||||
// The flow-control case above is the healthy half: a client that drains late gets every event.
|
||||
// These two are the other half, and they are why ReserveEventOrBlock's two BUSY refusals became
|
||||
// a latch. Each child reads its OWN outcome - the latch, the server's drop tally, the shared
|
||||
// page's eventDropped the client sees, the time spent against MOBILEGL_IPC_EVENT_WAIT_MS, and the
|
||||
// apply thread leaving by itself - and exits non-zero on the first that is wrong; the parent then
|
||||
// asserts the named line and the ABSENCE of Fatal{EventRingOverflow}.
|
||||
namespace {
|
||||
// Waits for mgl-srv-apply to leave on its own: the forfeit's stop is the ordinary one, so
|
||||
// nobody calls Stop() to get it out.
|
||||
bool ApplyThreadLeavesWithin(std::chrono::milliseconds budget) {
|
||||
const auto until = std::chrono::steady_clock::now() + budget;
|
||||
while (std::chrono::steady_clock::now() < until) {
|
||||
if (!Srv::ServerLoopInstance().Running()) return true;
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(5));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
Uint64 MillisecondsSince(std::chrono::steady_clock::time_point start) {
|
||||
return static_cast<Uint64>(std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - start)
|
||||
.count());
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// STONEWALL: the client never drains. 3000 GPU-write marks of ~1 KiB against a 256 KiB ring:
|
||||
// ~250 fit, the next one waits MOBILEGL_IPC_EVENT_WAIT_MS (300 ms here) and forfeits, and the
|
||||
// remaining ~2750 are drops that cost no wait at all - so the whole burst returns in a little over
|
||||
// the knob, not in 30 s and not with an abort.
|
||||
//
|
||||
// THE RED (recorded in the F2 package note): put ReserveEventOrBlock back to its two 30000 ms
|
||||
// rounds and this child is killed by StartRunAheadSession's alarm(15) inside the first wait -
|
||||
// before that change it would have aborted at 30 s with Fatal{EventRingOverflow} "waited 30000 ms".
|
||||
TEST(RemoteRunAhead, AStonewallingClientForfeitsTheReverseChannelWithinTheWaitKnob) {
|
||||
const auto child = RunInChild([] {
|
||||
MG_Config::Ipc.EventWaitMs = 300;
|
||||
StartRunAheadSession();
|
||||
const auto start = std::chrono::steady_clock::now();
|
||||
Srv::ServerLoopInstance().RunProbeOnApplyThreadForTesting(
|
||||
+[](void*) -> MobileGLResult {
|
||||
MG_Pipe::MGPRange ranges[64];
|
||||
for (Uint32 r = 0; r < 64; ++r) ranges[r] = MG_Pipe::MGPRange{r * 64ull, 64ull};
|
||||
for (Uint32 i = 0; i < 3000; ++i) {
|
||||
MG_Pipe::gMGPipeCallbacks.OnGpuWritten(MG_Pipe::MGPipeHandle{7, 1}, 64, ranges);
|
||||
}
|
||||
return MOBILEGL_OK;
|
||||
},
|
||||
nullptr);
|
||||
const Uint64 elapsedMs = MillisecondsSince(start);
|
||||
auto& server = Srv::ServerSessionInstance();
|
||||
if (!server.ReverseChannelForfeited()) ::_exit(101);
|
||||
const Uint64 drops = server.ForfeitDrops();
|
||||
// Some fitted (the ring was empty) and some did not (it holds ~250 of 3000).
|
||||
if (drops == 0 || drops >= 3000u) ::_exit(102);
|
||||
// The shared page's counter is the one a client can read; the server's own tally is the
|
||||
// one a peer cannot write. They agree when nothing else counts.
|
||||
if (ClientSessionInstance().Events().DroppedEvents() != drops) ::_exit(103);
|
||||
// The knob, spent once: at least most of it, and nowhere near the old 30 s.
|
||||
if (elapsedMs < 250u || elapsedMs > 5000u) ::_exit(104);
|
||||
if (!ApplyThreadLeavesWithin(std::chrono::milliseconds(3000))) ::_exit(105);
|
||||
ClientSessionInstance().Stop();
|
||||
});
|
||||
ExpectChildSuccess(child);
|
||||
EXPECT_NE(child.Log.find("ReverseChannelForfeit{NotDraining} - kEventGpuWritten"), std::string::npos)
|
||||
<< child.Log;
|
||||
EXPECT_NE(child.Log.find("mgl-srv-apply stops on ReverseChannelForfeit"), std::string::npos) << child.Log;
|
||||
EXPECT_EQ(child.Log.find("Fatal{EventRingOverflow"), std::string::npos) << child.Log;
|
||||
}
|
||||
|
||||
// TRICKLE: the client drains, one record per 50 ms, and the record the server is waiting to place
|
||||
// is twenty times bigger than what each drain frees. Every drain clears the latch and rings the
|
||||
// server; every retry still does not fit. The old shape gave up after TWO such rounds and called
|
||||
// it "a drained ring that still refuses a record it fits is a corrupt cursor set" - ~100 ms into a
|
||||
// session whose only fault was a slow reader, and a Fatal. Now the rounds share one deadline:
|
||||
// the server keeps retrying for the whole knob (600 ms here) and then forfeits by name, TooSlow,
|
||||
// with the short drains counted.
|
||||
//
|
||||
// THE RED (recorded in the F2 package note): the old two-round ReserveEventOrBlock aborts this
|
||||
// child with Fatal{EventRingOverflow} "could not reserve ... on an emptied SEG_EVENT".
|
||||
TEST(RemoteRunAhead, ATricklingClientGetsTheWholeWaitThenForfeitsByNameInsteadOfAborting) {
|
||||
const auto child = RunInChild([] {
|
||||
MG_Config::Ipc.EventWaitMs = 600;
|
||||
StartRunAheadSession();
|
||||
std::atomic<bool> stop{false};
|
||||
std::atomic<Uint32> popped{0};
|
||||
std::thread trickler([&stop, &popped] {
|
||||
Transport::EventRingConsumer& events = ClientSessionInstance().Events();
|
||||
while (!stop.load(std::memory_order_acquire)) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(50));
|
||||
Transport::RingRecordView view;
|
||||
if (events.Pop(view)) popped.fetch_add(1);
|
||||
// Releases what was popped, clears the latch and rings the server: a real drain,
|
||||
// just a stingy one.
|
||||
events.Drained();
|
||||
}
|
||||
});
|
||||
const auto start = std::chrono::steady_clock::now();
|
||||
Srv::ServerLoopInstance().RunProbeOnApplyThreadForTesting(
|
||||
+[](void*) -> MobileGLResult {
|
||||
// Fill with ONE-range marks (48 bytes a record) until under 4 KiB is left, without
|
||||
// ever blocking - so the fill is not paced by the trickle - then ask for 64-range
|
||||
// marks (~1 KiB each). The first of those that does not fit is the one the server
|
||||
// waits for, and one popped 48-byte record never makes room for it.
|
||||
auto& ring = Srv::ServerSessionInstance().Events().Ring();
|
||||
const MG_Pipe::MGPRange one{0, 64};
|
||||
while (ring.FreeBytes() > 4096u) {
|
||||
MG_Pipe::gMGPipeCallbacks.OnGpuWritten(MG_Pipe::MGPipeHandle{7, 1}, 1, &one);
|
||||
}
|
||||
MG_Pipe::MGPRange ranges[64];
|
||||
for (Uint32 r = 0; r < 64; ++r) ranges[r] = MG_Pipe::MGPRange{r * 64ull, 64ull};
|
||||
for (Uint32 i = 0; i < 16; ++i) {
|
||||
MG_Pipe::gMGPipeCallbacks.OnGpuWritten(MG_Pipe::MGPipeHandle{7, 1}, 64, ranges);
|
||||
}
|
||||
return MOBILEGL_OK;
|
||||
},
|
||||
nullptr);
|
||||
const Uint64 elapsedMs = MillisecondsSince(start);
|
||||
stop.store(true, std::memory_order_release);
|
||||
trickler.join();
|
||||
auto& server = Srv::ServerSessionInstance();
|
||||
if (!server.ReverseChannelForfeited()) ::_exit(101);
|
||||
if (server.ForfeitDrops() == 0) ::_exit(102);
|
||||
// The trickle really happened while the server waited - otherwise this is the stonewall
|
||||
// case under another name.
|
||||
if (popped.load() < 2u) ::_exit(103);
|
||||
// The WHOLE budget was spent retrying, not two rounds of it.
|
||||
if (elapsedMs < 540u || elapsedMs > 5000u) ::_exit(104);
|
||||
if (!ApplyThreadLeavesWithin(std::chrono::milliseconds(3000))) ::_exit(105);
|
||||
ClientSessionInstance().Stop();
|
||||
});
|
||||
ExpectChildSuccess(child);
|
||||
EXPECT_NE(child.Log.find("ReverseChannelForfeit{TooSlow} - kEventGpuWritten"), std::string::npos)
|
||||
<< child.Log;
|
||||
EXPECT_EQ(child.Log.find("Fatal{EventRingOverflow"), std::string::npos) << child.Log;
|
||||
}
|
||||
|
||||
// §2.7 / ruling 13: the deferred-destroy queue stays, and an enqueue from an unbarriered
|
||||
// apply is the finding. Under strict it is the abort; this is the arm the lane owns.
|
||||
TEST(RemoteGuards, ADeferredDestroyFromAnUnbarrieredApplyIsFatalUnderStrict) {
|
||||
|
||||
@@ -653,6 +653,7 @@ CMake:
|
||||
| `MOBILEGL_IPC_RING_MB` | 8 | `SEG_CMD`;单条记录至多一半 |
|
||||
| `MOBILEGL_IPC_STAGE_MB` | 32 | `SEG_STAGE`;目标负载 profile 显式 256 |
|
||||
| `MOBILEGL_IPC_SPIN_US` | 50 | park 前自旋;P5d 三轮起自旋按一次性校准的迭代预算走、稳态不读时钟,`0` = 不自旋直接 park |
|
||||
| `MOBILEGL_IPC_EVENT_WAIT_MS` | 2000(1–600000) | PH-6(ID-P7-2),只由 server 读(`ServerSpawn.cpp` 放行给 spawn 子进程):run-ahead 下一条反向事件遇到满的 `SEG_EVENT` 时,server 等 client 腾出空间的**整笔预算**(不是每次 park 的)。到期、client 只涓流不腾够、或 client 的 bell 已死(先判死,不误报为超时)→ `ReverseChannelForfeit{NotDraining\|TooSlow\|PeerGone}`:该事件及其后全部计数丢弃(`eventDropped`),apply 循环按正常 Stop 退出,会话 exit 0,supervisor 照常欢迎下一连接。取代原先 30000 ms × 两轮后的 `Fatal{EventRingOverflow}`。lockstep 臂(无 `kCapRunAheadApply`,即 Magma)不等待,仍是 P5C 的 Fatal |
|
||||
| `MOBILEGL_IPC_PERSISTENT_BLOCK_KB` | 64 | persistent-map 推送块粒度;`0` 是 E3(a) 阴性对照 |
|
||||
| `MOBILEGL_IPC_PERSISTENT_HASH_SUPPRESS` | 1 | 推送只发内容变了的块(追踪 buffer 走 mprotect 位图,未追踪走内容哈希);`0` 恢复全范围推送(A/B 对照) |
|
||||
| `MOBILEGL_IPC_BATCH_WAITS` | 1 | 值类记录(无 reply slot 的 kCtxState/kCtxCso/kCtxObject)发布即返回,barrier 推迟到下一个拉取类 verb;**`generate_mipmap` 例外**(它的 apply 读 `MGB_CTX->GetActiveTextureUnit()`,规则:只有 apply 不读残余填充字段的记录才可免等);`0` 恢复 R-1 逐条 barrier(`MOBILEGL_PIPE_VERIFY` 强制 0) |
|
||||
|
||||
Reference in New Issue
Block a user