[Fix] (Client): recover a fresh server session after a device loss so applications can recreate their contexts

This commit is contained in:
2026-10-02 23:34:21 -04:00
parent b64c2de3c8
commit 7823bcd987
32 changed files with 1570 additions and 99 deletions
@@ -227,11 +227,20 @@ namespace MobileGL::MG_Remote::Client {
// depth24+stencil8 surface, and every buffer allocated from that answer is
// blit-incompatible with the real thing).
if (ClientSession* session = ClientSession::Active()) session->DrainPublishedEvents();
return MG_Backend::BackendObject::CreateEGLWindowSurface(surface, handle);
if (!MG_Backend::BackendObject::CreateEGLWindowSurface(surface, handle)) return false;
NoteHomed(surface);
return true;
}
Bool BackendObject_Remote::ResizeEGLWindowSurface(EGLSurface surface, Uint32 width, Uint32 height) {
WaitForApplyBeforeEglForwarder("ResizeEGLWindowSurface");
// A surface with no server side on this session (its session was lost) takes the size
// into its registration only; it is re-created at that size when it is next used.
{
const std::lock_guard<std::recursive_mutex> lock(m_eglStateMutex);
if (GetRegisteredEGLSurface(surface) != nullptr && m_homedSurfaces.count(surface) == 0)
return MG_Backend::BackendObject::ResizeEGLWindowSurface(surface, width, height);
}
// P12 review fix: a surface on the SERVER's window is resized by resizing that window. The
// server asks its display for the size and answers with the extent the window really took,
// and that - not the size asked for, which EGLImpl already wrote into the EGL state - is what
@@ -296,7 +305,9 @@ namespace MobileGL::MG_Remote::Client {
// Same drain as the window surface: InitPbufferSurface publishes the default
// framebuffer's depth/stencil format on SEG_EVENT from inside this very RPC.
if (ClientSession* session = ClientSession::Active()) session->DrainPublishedEvents();
return MG_Backend::BackendObject::CreateEGLPbufferSurface(surface, width, height);
if (!MG_Backend::BackendObject::CreateEGLPbufferSurface(surface, width, height)) return false;
NoteHomed(surface);
return true;
}
namespace {
@@ -368,7 +379,9 @@ namespace MobileGL::MG_Remote::Client {
if (local.Handle == nullptr) local.Handle = &g_serverOwnedWindowPlaceholder;
local.Width = reply.width;
local.Height = reply.height;
return MG_Backend::BackendObject::CreateEGLWindowSurface(surface, local);
if (!MG_Backend::BackendObject::CreateEGLWindowSurface(surface, local)) return false;
NoteHomed(surface);
return true;
}
Bool BackendObject_Remote::MakeEGLCurrent(EGLDisplay dpy, EGLSurface draw, EGLSurface read,
@@ -378,6 +391,15 @@ namespace MobileGL::MG_Remote::Client {
// InitCapabilities' answer comes from a snapshot the server can only publish once its
// own InitCapabilities has run - and ServerMakeEGLCurrent is what publishes it.
WaitForApplyBeforeEglForwarder("MakeEGLCurrent");
const Bool release = draw == EGL_NO_SURFACE && read == EGL_NO_SURFACE && ctx == EGL_NO_CONTEXT;
// A RELEASE ALWAYS SUCCEEDS ON A LOST SESSION. Its server has nothing left to unbind, and
// an application releasing a lost context before it destroys it - the order EGL asks for -
// must not be told EGL_BAD_ACCESS and left with the lost context still current.
if (release && ClientSession::SessionLatchedLost()) {
return MG_Backend::BackendObject::MakeEGLCurrent(dpy, draw, read, ctx);
}
// A surface a lost session had gets its server side back before anything is bound to it.
if (!release && (!EnsureSurfaceHomed(draw) || (read != draw && !EnsureSurfaceHomed(read)))) return false;
// The frame carries the context's CLIENT TOKEN rather than its handle (the server's
// MakeCurrent arm binds the session to it before the native bind). Identity is all the
// server reads from the field otherwise, and a token is as unique as a handle.
@@ -419,6 +441,7 @@ namespace MobileGL::MG_Remote::Client {
// on the server directly and put no record on the wire, which is the shape every gate
// in this phase exists to catch. The forwarder exists for a spawned P6 client whose
// Present record cannot carry the swap; in P5 it has no caller and that is deliberate.
if (!EnsureSurfaceHomed(draw)) return false;
return MG_Backend::BackendObject::SwapEGLBuffers(dpy, draw);
}
@@ -440,18 +463,92 @@ namespace MobileGL::MG_Remote::Client {
void BackendObject_Remote::ReleaseEGLSurface(EGLSurface surface) {
// P12: a released server-owned surface stops taking the server window's geometry.
if (ClientSession* session = ClientSession::Active()) session->ForgetServerOwnedWindowSurface(surface);
Server::ServerReleaseEGLSurface(surface);
// Only a surface the current session's server has: one a lost session had is gone with it.
Bool homed = false;
{
const std::lock_guard<std::recursive_mutex> lock(m_eglStateMutex);
homed = m_homedSurfaces.erase(surface) != 0;
}
if (homed && !ClientSession::SessionLatchedLost()) Server::ServerReleaseEGLSurface(surface);
MG_Backend::BackendObject::ReleaseEGLSurface(surface);
}
void BackendObject_Remote::ReleaseEGLResources() {
// BLOCKING BY CONTRACT (ServerLoop.h's header note): MobileGL::Destroy()
// (MobileGL/Init.cpp:68) walks on the moment this returns, and the server still holds
// the context until the apply thread has run it.
Server::ServerReleaseEGLResources();
// the context until the apply thread has run it. A lost session's server holds nothing.
if (!ClientSession::SessionLatchedLost()) Server::ServerReleaseEGLResources();
{
const std::lock_guard<std::recursive_mutex> lock(m_eglStateMutex);
m_homedSurfaces.clear();
}
MG_Backend::BackendObject::ReleaseEGLResources();
}
void BackendObject_Remote::OnSessionReplaced() {
const std::lock_guard<std::recursive_mutex> lock(m_eglStateMutex);
ResetEGLRuntimeState();
m_homedSurfaces.clear();
for (const auto& [id, references] : m_sessionSharedImages) m_lostSessionSharedImages.insert(id);
m_sessionSharedImages.clear();
// The display the application initialized is initialized on the new server too: its
// surfaces and contexts are created against it from here on.
if (m_eglDisplayInitialized) {
EGLint major = 0;
EGLint minor = 0;
if (!Server::ServerInitializeEGLDisplay(m_eglDisplay, &major, &minor)) {
MGLOG_E("MG_Remote client: the fresh session's server did not initialize the display; "
"surfaces and contexts on it will fail");
}
}
if (ClientSession* session = ClientSession::Active()) session->PumpControlPlane();
RefreshFormatCapabilities();
}
Bool BackendObject_Remote::EnsureSurfaceHomed(EGLSurface surface) {
if (surface == EGL_NO_SURFACE) return true;
const std::lock_guard<std::recursive_mutex> lock(m_eglStateMutex);
if (m_homedSurfaces.count(surface) != 0) return true;
const EGLSurfaceState* registered = GetRegisteredEGLSurface(surface);
// Unknown to this object: the base class's own checks answer for it.
if (registered == nullptr) return true;
const EGLSurfaceState state = *registered;
Bool recreated = false;
const char* kind = "pbuffer";
if (state.Kind == SurfaceKind::Pbuffer) {
recreated = Server::ServerCreateEGLPbufferSurface(surface, state.Width, state.Height);
} else if (ClientSession::IsServerOwnedWindowSurface(surface)) {
kind = "server-owned window";
const Server::ServerOwnedWindowReply reply =
Server::ServerCreateServerOwnedWindowSurface(surface, state.Window.Width, state.Window.Height);
recreated = reply.ok;
if (recreated && reply.width != 0 && reply.height != 0) {
if (MG_State::pEGLContext) {
(void)MG_State::pEGLContext->SetSurfaceExtent(surface, static_cast<EGLint>(reply.width),
static_cast<EGLint>(reply.height));
}
if (ClientSession* session = ClientSession::Active())
session->NoteServerOwnedWindowSurface(surface, reply.width, reply.height);
}
} else {
kind = "window";
Server::ServerSetWindowHandle(state.Window);
recreated = Server::ServerCreateEGLWindowSurface(surface, state.Window);
}
if (!recreated) {
MGLOG_E("MG_Remote client: the %s surface %p (%dx%d) of a lost session could not be re-created on "
"the fresh one",
kind, static_cast<void*>(surface), state.Width, state.Height);
return false;
}
if (ClientSession* session = ClientSession::Active()) session->DrainPublishedEvents();
m_homedSurfaces.insert(surface);
MGLOG_I("MG_Remote client: the %s surface %p (%dx%d) of a lost session is re-created on the fresh one "
"(EGL keeps surfaces across a context loss; their contents are undefined)",
kind, static_cast<void*>(surface), state.Width, state.Height);
return true;
}
// ---- shared images ----------------------------------------------------------------------
//
// One shared_image record each (ClientSession::EmitSharedImage). Allocate/Import/Release are
@@ -461,6 +558,9 @@ namespace MobileGL::MG_Remote::Client {
MG_Backend::SharedImageExport* out) {
ClientSession* session = ClientSession::Active();
if (session == nullptr || out == nullptr) return false;
// A table operation, not a draw: the calling thread's GL state (a lost context, or none) has
// no say in whether the server allocates.
const ClientSession::ScopedCurrentSessionWork tableOp;
MG_Pipe::MGPSharedImageOp op{};
op.Op = MG_Pipe::kMGPSharedImageAllocate;
op.Format = fourcc;
@@ -477,12 +577,17 @@ namespace MobileGL::MG_Remote::Client {
out->Stride = reply.Stride;
out->Offset = reply.Offset;
out->Modifier = reply.Modifier;
{
const std::lock_guard<std::recursive_mutex> lock(m_eglStateMutex);
++m_sessionSharedImages[reply.ImageId];
}
return true;
}
Bool BackendObject_Remote::ImportSharedImage(int fd, Uint32 width, Uint32 height, Uint32 fourcc, Uint64* outId) {
ClientSession* session = ClientSession::Active();
if (session == nullptr || fd < 0) return false;
const ClientSession::ScopedCurrentSessionWork tableOp;
MG_Pipe::MGPSharedImageOp op{};
op.Op = MG_Pipe::kMGPSharedImageImport;
op.Format = fourcc;
@@ -491,12 +596,26 @@ namespace MobileGL::MG_Remote::Client {
MG_Pipe::MGPSharedImageReply reply{};
if (!session->EmitSharedImage(op, fd, &reply, nullptr)) return false;
if (outId != nullptr) *outId = reply.ImageId;
{
const std::lock_guard<std::recursive_mutex> lock(m_eglStateMutex);
++m_sessionSharedImages[reply.ImageId];
}
return true;
}
Bool BackendObject_Remote::ReleaseSharedImage(Uint64 id) {
ClientSession* session = ClientSession::Active();
if (session == nullptr || !session->Started()) return false;
// An image of a lost session is gone with its server; the fresh one never issued its id.
{
const std::lock_guard<std::recursive_mutex> lock(m_eglStateMutex);
const auto held = m_sessionSharedImages.find(id);
// Nothing to release on the new server, and nothing that failed: the reference died with
// its session.
if (held == m_sessionSharedImages.end()) return m_lostSessionSharedImages.erase(id) != 0;
if (--held->second == 0) m_sessionSharedImages.erase(held);
}
const ClientSession::ScopedCurrentSessionWork tableOp;
MG_Pipe::MGPSharedImageOp op{};
op.Op = MG_Pipe::kMGPSharedImageRelease;
op.ImageId = id;
@@ -506,6 +625,7 @@ namespace MobileGL::MG_Remote::Client {
Bool BackendObject_Remote::PresentToSharedImage(Uint64 id, const MG_Util::Damage::Region& region) {
ClientSession* session = ClientSession::Active();
if (session == nullptr) return false;
if (!HoldsSharedImage(id)) return false; // a lost session's image: gone with its server
MG_Pipe::MGPSharedImageOp op{};
op.Op = MG_Pipe::kMGPSharedImagePresent;
op.ImageId = id;
@@ -539,6 +659,7 @@ namespace MobileGL::MG_Remote::Client {
Bool BackendObject_Remote::AttachSharedImageToTexture(Uint64 textureLifetimeId, Uint64 id) {
ClientSession* session = ClientSession::Active();
if (session == nullptr) return false;
if (!HoldsSharedImage(id)) return false; // an EGLImage of a lost session: its image is gone
MG_Pipe::MGPSharedImageOp op{};
op.Op = MG_Pipe::kMGPSharedImageAttach;
op.ImageId = id;
@@ -51,6 +51,9 @@
#include <MG_Backend/BackendObject.h>
#include <unordered_map>
#include <unordered_set>
namespace MobileGL::MG_Remote::Client {
class BackendObject_Remote final : public MG_Backend::BackendObject {
@@ -97,6 +100,14 @@ namespace MobileGL::MG_Remote::Client {
// and this is what it calls; it is the whole of trap 2's answer.
void RefreshFormatCapabilities();
// A FRESH SESSION REPLACED A LOST ONE (ClientSession::RecoverAfterDeviceLoss). The
// initialized display is initialized on the new server; every surface keeps its client
// registration (EGL keeps surfaces across a context loss, only their contents go) but
// has no server side yet, and is re-created there the next time it is made current,
// swapped or resized; no thread is current any more; the shared images of the lost
// session are forgotten without a release (their server is gone).
void OnSessionReplaced();
protected:
Bool InitPbufferSurface(EGLint width, EGLint height) override;
@@ -110,6 +121,28 @@ namespace MobileGL::MG_Remote::Client {
// through the log line on a refresh: a cache that silently stopped tracking the mirror
// is exactly the shape trap 2 exists to prevent.
Uint64 m_formatsGeneration = 0;
// The server side of `surface` on the CURRENT session: re-created from its client
// registration when the session that had it was replaced. True when it exists now.
Bool EnsureSurfaceHomed(EGLSurface surface);
void NoteHomed(EGLSurface surface) {
const std::lock_guard<std::recursive_mutex> lock(m_eglStateMutex);
m_homedSurfaces.insert(surface);
}
Bool HoldsSharedImage(Uint64 id) const {
const std::lock_guard<std::recursive_mutex> lock(m_eglStateMutex);
return m_sessionSharedImages.count(id) != 0;
}
// The surfaces and shared images the CURRENT session's server holds. A surface missing
// from the first is one a lost session had (EnsureSurfaceHomed); an image id missing from
// the second (counted per reference taken) is a lost session's, and is neither released
// nor presented to nor attached on the new server, which never issued it. Guarded by the
// base's m_eglStateMutex.
std::unordered_set<EGLSurface> m_homedSurfaces;
std::unordered_map<Uint64, Uint32> m_sessionSharedImages; // id -> references taken
// The ids a replaced session held: releasing one is a success with nothing sent.
std::unordered_set<Uint64> m_lostSessionSharedImages;
};
} // namespace MobileGL::MG_Remote::Client
+144 -7
View File
@@ -24,12 +24,15 @@
#include "../Transport/InProcessTransport.h"
#include "WireTables.h"
#include "EmitTables.h"
#include "PersistentMapTracker.h"
#include "ServerProbe.h"
#include <MGGitHash.h>
#include <MG_Backend/MGPipe/PipeInputs.h>
#include <MG_Impl/GLImpl/Framebuffer/GL_Framebuffer.h>
#include <MG_Impl/Pipe/ResourceTracker.h>
#include <MG_Pipe/MGPipeCallbacks.h>
#include <MG_Pipe/PipeMutation.h>
#include <MG_State/EGLState/Core.h>
#include <MG_State/GLState/Core.h>
#include <MG_State/GLState/ErrorState/ErrorInfo.h>
@@ -94,6 +97,13 @@ namespace MobileGL::MG_Remote::Client {
ClientSession* g_active = nullptr;
// THE RECOVERY'S THREE PIECES OF PROCESS STATE (ClientSession.h). The two scopes are
// per thread and nest; the count is how many sessions with a remote server this process
// has started, so the second and later know there is an ended one to forget.
thread_local Uint32 tl_wireMute = 0;
thread_local Uint32 tl_currentSessionWork = 0;
std::atomic<Uint64> g_remoteSessionsStarted{0};
// The same bounded handshake deadline the server uses. Bounded, not kWaitForever: a
// bring-up that never answers has to be a red lane rather than a wedged CI job.
constexpr Uint32 kHandshakeTimeoutMs = 5000;
@@ -593,10 +603,29 @@ namespace MobileGL::MG_Remote::Client {
ClientSession* ClientSession::Active() { return g_active; }
Bool ClientSession::DeviceLost() {
if (tl_wireMute != 0) return true;
const ClientSession* session = Active();
if (session == nullptr) return false;
if (session->m_deviceLost.load(std::memory_order_acquire)) return true;
if (tl_currentSessionWork != 0) return false;
// A thread still on a context from before a recovered loss: lost for good, whatever the
// session it would now be writing into.
const SharedPtr<MG_State::GLState::GLContext>& context = MG_State::pGLContext;
return context != nullptr && context->IsFromEndedWireSession();
}
Bool ClientSession::SessionLatchedLost() {
const ClientSession* session = Active();
return session != nullptr && session->m_deviceLost.load(std::memory_order_acquire);
}
ClientSession::ScopedWireMute::ScopedWireMute() { ++tl_wireMute; }
ClientSession::ScopedWireMute::~ScopedWireMute() { --tl_wireMute; }
ClientSession::ScopedCurrentSessionWork::ScopedCurrentSessionWork() { ++tl_currentSessionWork; }
ClientSession::ScopedCurrentSessionWork::~ScopedCurrentSessionWork() { --tl_currentSessionWork; }
Uint64 ClientSession::SessionsStarted() { return g_remoteSessionsStarted.load(std::memory_order_acquire); }
void ClientSession::LatchDeviceLost(const char* why) {
// exchange, so the log line is written EXACTLY once however many waits notice the same
// hangup at the same moment - and there will be several: the barrier, the control wait
@@ -604,13 +633,91 @@ namespace MobileGL::MG_Remote::Client {
if (m_deviceLost.exchange(true, std::memory_order_acq_rel)) {
return;
}
MGLOG_E("MGPipe: DEVICE LOST - %s. The server process is gone; every verb from here is "
"DECLINED and glGetGraphicsResetStatus reports GL_UNKNOWN_CONTEXT_RESET. This is "
"latched from the peer's HANGUP and not from any deadline (CONTRACT-P6 5.4): a "
"server one frame behind is P5e's intended steady state and must never read as a "
"dead one. There is no recovery - MOBILEGL_IPC_RESPAWN is the stage that would "
"add one and it is refused by name until then.",
why == nullptr ? "the peer hung up" : why);
MGLOG_E("MGPipe: DEVICE LOST - %s. This session's server side is gone; every verb from here "
"is DECLINED and glGetGraphicsResetStatus reports GL_UNKNOWN_CONTEXT_RESET. This is "
"latched from the peer's HANGUP (or its named SessionFault) and not from any "
"deadline (CONTRACT-P6 5.4): a server one frame behind is P5e's intended steady "
"state and must never read as a dead one. The contexts of this session stay lost; "
"the application's next eglCreateContext / eglCreate*Surface / eglInitialize brings "
"a fresh session up (MOBILEGL_IPC_RECOVER=%u).",
why == nullptr ? "the peer hung up" : why, MG_Config::Ipc.RecoverAfterLoss);
}
Bool ClientSession::ConfirmLossAfterDecline(Uint32 waitMs) {
const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(waitMs);
for (;;) {
if (m_deviceLost.load(std::memory_order_acquire)) return true;
PumpControlPlane();
if (const Transport::Doorbell* bell = m_producer.SelfDoorbell(); bell != nullptr && bell->PeerHungUp()) {
LatchDeviceLost("a readback was declined and the peer had hung up");
}
if (m_link && m_link->PeerHungUp()) LatchDeviceLost("a readback was declined and the link had hung up");
if (m_deviceLost.load(std::memory_order_acquire)) return true;
if (std::chrono::steady_clock::now() >= deadline) return false;
std::this_thread::sleep_for(std::chrono::milliseconds(2));
}
}
namespace {
// A LOST SESSION'S T0 STORES STAY MAPPED. A buffer of a lost context may still hold the
// pointer of the store it was adopted into (the application has not destroyed it yet, and
// may write through its persistent mapping), so the recovery's Stop must not unmap it.
// Bounded by what the lost session had adopted; never touched again.
Vector<Transport::AdoptT0::HeldStore>& LostSessionT0Stores() {
static auto* stores = new Vector<Transport::AdoptT0::HeldStore>();
return *stores;
}
} // namespace
MobileGLResult ClientSession::RecoverAfterDeviceLoss() {
if (!m_started || !m_deviceLost.load(std::memory_order_acquire)) return MOBILEGL_OK;
if (MG_Config::Ipc.RecoverAfterLoss == 0) {
MGLOG_E_ONCE("MG_Remote client: Refuse{Recover} the session is lost and MOBILEGL_IPC_RECOVER=0 - "
"no fresh session is dialed; every context of this process stays lost");
return MOBILEGL_ERR_UNSUPPORTED;
}
const std::string control(MG_Config::Ipc.Control.c_str());
if (MG_Config::Transport != MG_Config::TransportMode::Spawn || control.compare(0, 3, "fd:") == 0) {
MGLOG_E_ONCE("MG_Remote client: Refuse{Recover} the session is lost and this transport cannot "
"reach its server again (%s): an in-process server is this process, and a "
"pre-connected fd: pair was handed over once. Every context stays lost",
control.empty() ? "inproc" : control.c_str());
return MOBILEGL_ERR_UNSUPPORTED;
}
// THE SERVER FIRST, THEN THE TEARDOWN. A session that is stopped and cannot be replaced
// leaves no session at all, and a GL call then has nowhere honest to go; a latched session
// that is kept declines every verb, which is the answer the application already has.
if (ProbeConfiguredServer() == ServerProbeResult::Unreachable) {
MGLOG_E("MG_Remote client: the session is lost and no server answers at %s; it stays lost "
"(the next create will ask again)",
control.c_str());
return MOBILEGL_ERR_TRANSPORT_CLOSED;
}
const Uint32 lostServerPid = m_peerServerPid;
for (auto& [key, store] : m_t0Stores) LostSessionT0Stores().push_back(store);
m_t0Stores.clear();
Stop();
const MobileGLResult started = StartSpawned();
if (started != MOBILEGL_OK) {
MGLOG_E("MG_Remote client: the fresh session after a device loss did not come up (rc=%d); "
"there is no session now, and the next create will try again",
static_cast<int>(started));
return started;
}
MGLOG_I("MG_Remote client: RECOVERED - a fresh session (#%llu of this process) replaces the one "
"lost on server pid %u; it is served by pid %u. Contexts created before stay lost",
static_cast<unsigned long long>(SessionsStarted()), lostServerPid, m_peerServerPid);
return MOBILEGL_OK;
}
void ClientSession::ForgetEndedSession() {
MGLOG_I("MG_Remote client: session #%llu follows an ended one - the client's caches of that "
"session's objects (publication, CSOs, samplers, emitter latches, the create window, "
"the reset serial, persistent maps) start over",
static_cast<unsigned long long>(SessionsStarted()));
ForgetWireTablesOfEndedSession();
PersistentMapTracker::Instance().ForgetEndedSession();
MG_Pipe::MGPipeForgetEndedSession();
}
ClientSession& ClientSessionInstance() {
@@ -1690,6 +1797,17 @@ namespace MobileGL::MG_Remote::Client {
PumpControlPlane();
}
// A SESSION WITH A REMOTE SERVER STARTS FROM A CLEAN CLIENT. Every client-side cache that
// describes what a server holds (published handles, CSO and sampler caches, emitter
// latches, the create window, the applier-reset serial, tracked persistent maps) describes
// the PREVIOUS session's server once this one replaces it - after a device loss, or an
// eglTerminate/eglInitialize - and a fresh server holds none of it. And the wire epoch
// moves, which is what makes every GLContext built against the previous session lost.
// Inproc is left as it was: its server role is this process, and it never recovers.
if (!startApplyThreadHere) {
if (g_remoteSessionsStarted.fetch_add(1, std::memory_order_acq_rel) != 0) ForgetEndedSession();
MG_State::AdvanceWireEpoch();
}
// BOTH, AND m_started FIRST. `Active()` is what the integration lane's skip reads and
// `m_started` is what `EmitAndWait` reads, and c1's round-1 rewrite of this function
// set only the second of the two - so a fully-handshaken session with an apply thread
@@ -1879,6 +1997,12 @@ namespace MobileGL::MG_Remote::Client {
// The cold-start window is the next server's, not this one's (RunRemoteSurfaceControlFrame).
m_serverBackendWarm = false;
m_presentsSent = 0;
// And so are the latch and the control seq space: the next session's server counts its
// ops from 1 and has lost nothing. (The latch is read through Active(), which is null
// from here until a next session is started.)
m_deviceLost.store(false, std::memory_order_release);
m_remoteControlSeq = 0;
m_logFlushSeq = 0;
if (g_active == this) {
g_active = nullptr;
// Mirror order, and the probe goes with it: a dangling function pointer into a
@@ -2608,6 +2732,19 @@ namespace MobileGL::MG_Remote::Client {
static_cast<unsigned long long>(frame.size()));
continue;
}
if (envelope->msg_type() == ::MobileGL::Wire::CtrlMsg::Fatal) {
// The server latched this session on a named fault (a lost GPU device among them)
// and said so before declining the rest: the session is lost, from the server's
// own mouth rather than from the hangup that follows.
if (const auto* fatal = envelope->msg_as_Fatal()) {
MGLOG_E("MG_Remote client: the server ended this session on a named fault - Fatal "
"family %s: %s",
fatal->family() ? fatal->family()->c_str() : "<none>",
fatal->message() ? fatal->message()->c_str() : "");
}
LatchDeviceLost("the server published a SessionFault for this session");
continue;
}
if (envelope->msg_type() != ::MobileGL::Wire::CtrlMsg::CapsSnapshot) {
// SurfaceOp / SurfaceReply / ResyncRequest / AuxRequest / LogLine are P6's and
// P7's. Named rather than ignored, so a phase that starts sending one does not
+56 -7
View File
@@ -74,23 +74,69 @@ namespace MobileGL::MG_Remote::Client {
static ClientSession* Active();
// P6 `dl` (CONTRACT-P6 5.3). THE DEVICE-LOST LATCH, and the two words that matter are
// SESSION-SCOPED and LATCH: it is set at most once per session and never cleared, because
// a session whose server died has nothing left to recover into. MOBILEGL_IPC_RESPAWN is
// where the recovery would go and it is a named refusal until some stage writes it.
// SESSION-SCOPED and LATCH: it is set at most once per session and never cleared while
// that session lives. What recovers is the PROCESS, not the session: the application's
// next eglCreateContext / eglCreate*Surface / eglInitialize ends the lost session and
// dials a fresh one (RecoverAfterDeviceLoss), and every context built before stays lost.
//
// SET FROM Doorbell::PeerHungUp(), NEVER FROM A TIMEOUT. 5.4's whole point is that slow
// and dead are different states and must be told apart without a threshold: under P5e
// run-ahead a server one frame behind is the INTENDED steady state, so a latch armed by a
// deadline would fire on a healthy session under load. The descriptor answers instead.
// SET FROM Doorbell::PeerHungUp() OR THE SERVER'S OWN SessionFault, NEVER FROM A
// TIMEOUT. 5.4's whole point is that slow and dead are different states and must be told
// apart without a threshold: under P5e run-ahead a server one frame behind is the
// INTENDED steady state, so a latch armed by a deadline would fire on a healthy session
// under load. The descriptor (or the server, naming its fault) answers instead.
//
// WHAT THE CALLING THREAD SEES, which is more than the latch: the thread's GL state may
// belong to a session that has already been replaced (a context from before a recovered
// loss, still current on some thread), and for that thread the device is lost for good -
// nothing it does may reach the new session. ScopedWireMute below answers true as well;
// ScopedCurrentSessionWork lifts the thread's own staleness (not the latch).
//
// The static form answers false whenever no session is active, so monolith and the pull
// build get the honest answer with no branch of their own.
static Bool DeviceLost();
// The latch alone: is the ACTIVE session lost. What the EGL layer asks before recovering.
static Bool SessionLatchedLost();
// Arms the latch and says why, once. Safe to call repeatedly and from any thread; only
// the first call logs.
void LatchDeviceLost(const char* why);
// A reply-owning record came back DECLINED where only a latched server declines (a
// readback). The server publishes its SessionFault on the control plane BEFORE it declines,
// so the frame - or the hangup that follows it - is what tells "lost" from a refusal. Pumps
// the control plane and asks the bell for up to `waitMs`; true when the session is lost.
Bool ConfirmLossAfterDecline(Uint32 waitMs);
// THE RECOVERY. Ends this lost session (Stop) and dials the configured server for a fresh
// one (StartSpawned); the new session forgets every client-side cache of the old one and
// advances MG_State's wire epoch, which is what makes every older GLContext lost. The
// caller (EGLImpl) holds the EGL operation lock, which every GL entry point also takes, so
// nothing else is emitting. Refused by name - and the session stays latched - under
// MOBILEGL_IPC_RECOVER=0, for a transport whose server cannot be dialed again (inproc,
// fd:), and when the server does not answer a probe. OK when a fresh session is up.
MobileGLResult RecoverAfterDeviceLoss();
// Sessions brought up in this process so far (the recovery's arm proof for a test).
static Uint64 SessionsStarted();
// While one is open on a thread, that thread's emissions are DECLINED as on a lost
// device: the lost contexts' objects die under it, and their deaths must not reach the
// fresh session.
struct ScopedWireMute {
ScopedWireMute();
~ScopedWireMute();
ScopedWireMute(const ScopedWireMute&) = delete;
ScopedWireMute& operator=(const ScopedWireMute&) = delete;
};
// While one is open on a thread, the thread's own GL state does not make it "lost": EGL
// work that targets the current session on behalf of a thread whose current context is
// a lost one (it creates the replacement context, re-creates a surface).
struct ScopedCurrentSessionWork {
ScopedCurrentSessionWork();
~ScopedCurrentSessionWork();
ScopedCurrentSessionWork(const ScopedCurrentSessionWork&) = delete;
ScopedCurrentSessionWork& operator=(const ScopedCurrentSessionWork&) = delete;
};
// The bell this session parks on, for the `dl` red-once ONLY. A test cannot otherwise
// ask the question the latch turns on - "did the peer hang up" is a fact about a
// descriptor, and the pair of tests that pin D5c has to be able to kill a server and
@@ -632,6 +678,9 @@ namespace MobileGL::MG_Remote::Client {
Uint64 m_t0Fallbacks = 0;
std::function<void(Uint64)> m_prePublish;
void ResetT0(Bool releaseStores);
// Every client-side cache keyed to the session that just ended, dropped before the next
// one serves a record (FinishStartup, on every start after the first).
void ForgetEndedSession();
};
// One per process in P5, because P5 serves one context, and LEAKED AT EXIT like every other
+27 -1
View File
@@ -897,6 +897,11 @@ namespace MobileGL::MG_Remote::Client {
// control, so R-16's "drive the production predicate" holds rather than a second copy of
// the rule in the test.
// How long a declined readback waits for the control plane or the bell to say "lost"
// before it is judged a refusal. The SessionFault is sent before the decline is posted,
// so this is only ever spent on a server that refused for real.
constexpr Uint32 kDeclineVerdictMs = 1000;
// The same predicate at the call site, with the named Fatal each failure mode owns.
// status > expected cannot reach here: EmitAndWait already aborts Fatal{ReplyTooLarge} on
// an oversize reply, so the only failures left are a wrong status or a SHORT one.
@@ -909,6 +914,17 @@ namespace MobileGL::MG_Remote::Client {
// the caller leaves the destination untouched and returns.
Bool RequireReadbackReplyComplete(Int32 status, Uint64 replySize, Uint64 expected) {
if (status != Wire::ReplySink::kStatusOk && ClientSession::DeviceLost()) return false;
// THE DECLINE CAN OUTRUN THE NEWS. A server whose device was lost inside THIS readback
// latches the session - publishing its SessionFault on the control plane - and then
// declines the record; the client can read the declined answer before it has looked at
// the control plane or seen the hangup that follows. So a decline asks once more,
// briefly, before it is taken for a server without GL.ReadPixels.
if (status == Wire::ReplySink::kStatusDeclined) {
if (ClientSession* session = ClientSession::Active();
session != nullptr && session->ConfirmLossAfterDecline(kDeclineVerdictMs)) {
return false;
}
}
if (status == Wire::ReplySink::kStatusError) {
SessionFail(MGFatalFamily::ReplyError, "MGPipe: Fatal{ReplyError, \"ReadPixels\"} - the readback answered ERROR; "
"the destination is left untouched rather than filled with stale bytes");
@@ -1608,6 +1624,11 @@ namespace MobileGL::MG_Remote::Client {
MG_Pipe::MGPipeHandle Handle;
// P10: zero-timeout polls answered "not yet" in a row since the last escalation.
Uint32 Unanswered = 0;
// The session the fence was created on. A fence of an ENDED session (a recovered
// device loss) names nothing on the current server: it is answered as the no-op a
// lost device gives (signaled), and deleting it puts nothing on the wire.
Uint64 Epoch = MG_State::CurrentWireEpoch();
Bool FromEndedSession() const { return Epoch != MG_State::CurrentWireEpoch(); }
};
std::mutex g_fenceMutex;
UnorderedMap<MG_Backend::BackendSyncHandle, UniquePtr<RemoteFenceProxy>> g_fenceProxies;
@@ -1754,6 +1775,7 @@ namespace MobileGL::MG_Remote::Client {
ClientSession& session = RequireSession("ClientWaitSync");
const std::lock_guard<std::mutex> lock(g_fenceMutex);
RemoteFenceProxy& fence = FenceProxy(proxy);
if (fence.FromEndedSession()) return GL_ALREADY_SIGNALED;
switch (AnswerFencePollLocally(session, fence, /*mayWait=*/timeout != 0)) {
case FencePollAnswer::Signaled: return GL_ALREADY_SIGNALED;
case FencePollAnswer::NotYet: return GL_TIMEOUT_EXPIRED;
@@ -1770,6 +1792,7 @@ namespace MobileGL::MG_Remote::Client {
ClientSession& session = RequireSession("GetSyncStatus");
const std::lock_guard<std::mutex> lock(g_fenceMutex);
RemoteFenceProxy& fence = FenceProxy(proxy);
if (fence.FromEndedSession()) return true;
switch (AnswerFencePollLocally(session, fence, /*mayWait=*/false)) {
case FencePollAnswer::Signaled: return true;
case FencePollAnswer::NotYet: return false;
@@ -1786,6 +1809,7 @@ namespace MobileGL::MG_Remote::Client {
void EmitWaitSync(MG_Backend::BackendSyncHandle proxy, GLbitfield flags, GLuint64 timeout) {
ClientSession& session = RequireSession("WaitSync");
const std::lock_guard<std::mutex> lock(g_fenceMutex);
if (FenceProxy(proxy).FromEndedSession()) return;
const MG_Pipe::MGPFenceWait request{FenceHandle(proxy), timeout, flags, 0};
session.EmitAndWait(MG_Pipe::MGPWireOp::FenceWaitServer, &request, sizeof(request),
nullptr, 0, nullptr, 0, nullptr);
@@ -1796,7 +1820,9 @@ namespace MobileGL::MG_Remote::Client {
const auto handle = FenceHandle(proxy);
// MobileGL::Destroy stops the server BEFORE DestroyAllSyncObjects. Detach already
// released those native objects on the apply thread; only the local proxy remains.
if (auto* session = ClientSession::Active(); session != nullptr && session->Started()) {
// A fence of an ended session likewise: its server is gone.
if (auto* session = ClientSession::Active();
session != nullptr && session->Started() && !FenceProxy(proxy).FromEndedSession()) {
const MG_Pipe::MGPHandleOnly desc{handle, static_cast<Uint32>(MG_Pipe::MGPipeKind::Fence), 0};
session->EmitAndWait(MG_Pipe::MGPWireOp::FenceDestroy, &desc, sizeof(desc),
nullptr, 0, nullptr, 0, nullptr);
@@ -7,6 +7,7 @@
// End of Source File Header
#include "PersistentMapTracker.h"
#include "ClientSession.h"
#include <MG_Remote/FatalFunnel.h>
#include <MG_Remote/Transport/AdoptTier.h>
@@ -751,7 +752,10 @@ namespace MobileGL::MG_Remote::Client {
// member that was never large-untracked.
m_untrackedMembers.erase(key);
m_edgedMembers.erase(key);
if (IsLivePersistentMap(buffer)) {
// A buffer mapped on a LOST context (a thread still on one, or a session that is lost)
// never becomes a member: the push before every verb would carry its bytes to whichever
// session is current, and a fresh one never created the buffer.
if (IsLivePersistentMap(buffer) && !ClientSession::DeviceLost()) {
const auto range = buffer.GetMappedRange();
// The mprotect arm registers beside the hash arm: tracked buffers are pushed
// from the fault bitmap, everything else keeps the content scan. A failed
@@ -784,13 +788,21 @@ namespace MobileGL::MG_Remote::Client {
UntrackWriteMap(key);
}
void PersistentMapTracker::ClearForTest() {
void PersistentMapTracker::ForgetEndedSession() {
// Every member is a buffer of the ended session: its bytes have nowhere to go, and a push
// before the next verb would name a handle the fresh server never created. The write
// tracking goes with them (pages unprotected); a member that is unmapped or destroyed
// later finds nothing here, which Forget already tolerates.
m_livePersistentMaps.clear();
m_blockHashes.clear();
m_untrackedMembers.clear();
m_edgedMembers.clear();
UntrackAllWriteMaps();
m_lastFaultEpoch = 0;
}
void PersistentMapTracker::ClearForTest() {
ForgetEndedSession();
ResetCountersForTest();
}
@@ -158,6 +158,8 @@ namespace MobileGL::MG_Remote::Client {
// Unit tests only: clears the membership, the hash cache and the mprotect tracked
// slots (restoring every tracked page writable first), and resets the counters.
void ClearForTest();
// A fresh session replaces an ended one: drop every member (ClientSession).
void ForgetEndedSession();
// Unit tests only: the mprotect arm's white-box surface, for the alignment cases
// (SplitBufferTest) that the public GL surface cannot see - which pages a map
// protected is not observable through any push count. MprotectArmAvailableForTest
+16 -2
View File
@@ -1,8 +1,19 @@
// Opaque local proxies carry only {slot, generation, target} across the wire.
struct RemoteQueryProxy { MG_Pipe::MGPQueryDesc Desc{}; };
// Epoch: the session the query was created on. A query of an ENDED session (a recovered
// device loss) names nothing on the current server: it answers "no result yet", and its
// end and delete put nothing on the wire.
struct RemoteQueryProxy {
MG_Pipe::MGPQueryDesc Desc{};
Uint64 Epoch = MG_State::CurrentWireEpoch();
};
std::mutex g_queryProxyMutex;
UnorderedMap<MG_Backend::BackendQueryHandle, UniquePtr<RemoteQueryProxy>> g_queryProxies;
Bool QueryFromEndedSession(MG_Backend::BackendQueryHandle handle) {
const auto found = g_queryProxies.find(handle);
return found != g_queryProxies.end() && found->second->Epoch != MG_State::CurrentWireEpoch();
}
const MG_Pipe::MGPQueryDesc& QueryDesc(MG_Backend::BackendQueryHandle handle) {
const auto found = g_queryProxies.find(handle);
if (found == g_queryProxies.end())
@@ -69,6 +80,7 @@
void EmitEndQuery(MG_Backend::BackendQueryHandle handle) {
auto& session = RequireSession("QueryEnd");
const std::lock_guard<std::mutex> lock(g_queryProxyMutex);
if (QueryFromEndedSession(handle)) return;
const auto& desc = QueryDesc(handle);
BeforeReadOnlyVerb();
session.EmitAndWait(MG_Pipe::MGPWireOp::QueryEnd, &desc, sizeof(desc), nullptr, 0, nullptr, 0, nullptr);
@@ -76,6 +88,7 @@
Bool EmitIsQueryResultAvailable(MG_Backend::BackendQueryHandle handle) {
auto& session = RequireSession("IsQueryResultAvailable");
const std::lock_guard<std::mutex> lock(g_queryProxyMutex);
if (QueryFromEndedSession(handle)) return false;
const MG_Pipe::MGPHandleOnly request{QueryDesc(handle).Query, static_cast<Uint32>(MG_Pipe::MGPipeKind::Query), 0};
const auto result = ReadQueryReply<Uint32>(session, MG_Pipe::MGPWireOp::QueryAvailable, &request, sizeof(request));
if (result > 1) Wire::WireProtocolFatal("QueryAvailable.reply", "availability is not boolean");
@@ -84,6 +97,7 @@
Bool EmitGetQueryResult64(MG_Backend::BackendQueryHandle handle, Bool wait, Uint64* value) {
auto& session = RequireSession("GetQueryResult64");
const std::lock_guard<std::mutex> lock(g_queryProxyMutex);
if (QueryFromEndedSession(handle)) return false;
MG_Pipe::MGPQueryResultRequest request{};
request.Query = QueryDesc(handle).Query;
request.Wait = wait ? 1 : 0;
@@ -98,7 +112,7 @@
void EmitDeleteBackendQuery(MG_Backend::BackendQueryHandle handle) {
const std::lock_guard<std::mutex> lock(g_queryProxyMutex);
const auto query = QueryDesc(handle).Query;
if (auto* session = ClientSession::Active(); session && session->Started()) {
if (auto* session = ClientSession::Active(); session && session->Started() && !QueryFromEndedSession(handle)) {
const MG_Pipe::MGPHandleOnly request{query, static_cast<Uint32>(MG_Pipe::MGPipeKind::Query), 0};
session->EmitAndWait(MG_Pipe::MGPWireOp::QueryDestroy, &request, sizeof(request),
nullptr, 0, nullptr, 0, nullptr);
+24 -6
View File
@@ -1160,10 +1160,13 @@ namespace MobileGL::MG_Remote::Client {
if (g_clientTablesUninstalled.load(std::memory_order_acquire)) return false;
MG_Pipe::MGPBindContext record{};
record.ClientContextToken = clientContextToken;
session->EmitAndWait(MGPWireOp::BindContext, &record, sizeof(record), nullptr, 0, nullptr, 0,
nullptr);
const Uint64 seq = session->EmitAndWait(MGPWireOp::BindContext, &record, sizeof(record), nullptr, 0,
nullptr, 0, nullptr);
++g_emitted;
return true;
// A DECLINED binding moved nothing on the server (a lost session, or a thread whose
// context is from an ended one), so the caller must not believe the session is bound to
// it: the next thread that writes re-binds its own context.
return seq != Wire::kInvalidSeq;
}
Bool EmitApplierResetRecord() {
@@ -1177,13 +1180,28 @@ namespace MobileGL::MG_Remote::Client {
if (session == nullptr || !session->Started()) return false;
if (g_clientTablesUninstalled.load(std::memory_order_acquire)) return false;
MG_Pipe::MGPApplierReset record{};
record.ContextSerial = g_applierResetContextSerial++;
session->EmitAndWait(MGPWireOp::ApplierReset, &record, sizeof(record), nullptr, 0, nullptr,
0, nullptr);
record.ContextSerial = g_applierResetContextSerial;
const Uint64 seq = session->EmitAndWait(MGPWireOp::ApplierReset, &record, sizeof(record), nullptr, 0,
nullptr, 0, nullptr);
// THE SERIAL COUNTS RESETS THE SERVER WAS SENT, not resets this client primed: a declined
// record (a lost session, a thread on an ended session's context) never reached the
// server's count, and moving ours past it would latch the next session that hears one.
if (seq != Wire::kInvalidSeq) ++g_applierResetContextSerial;
++g_emitted;
return true;
}
void ForgetWireTablesOfEndedSession() {
// The ended session's deferred create answers are not coming: their seqs name a ring
// that is gone, and a drain on the new session would wait for them (or read another
// record's answer). The objects behind them are the ended session's too.
g_createWindowHead = 0;
g_createWindowCount = 0;
g_createSuspects.clear();
// A fresh session's server counts applier resets from 0.
g_applierResetContextSerial = 0;
}
// P5c (rv), CONTRACT-P5C.md §5.3. Whether set_context_values can cross RIGHT NOW: a live,
// started session whose tables are not being torn down. PipeFill.cpp gates BOTH halves of
// the row on this - the emission and the residual-fill skip - so a configured-but-wireless
+5
View File
@@ -125,6 +125,11 @@ namespace MobileGL::MG_Remote::Client {
// caller then makes the direct call the record replaced.
Bool EmitApplierResetRecord();
// A fresh session replaces an ended one: the create window's deferred answers, its suspect
// list and the applier-reset serial all belonged to the ended session (ClientSession's
// ForgetEndedSession).
void ForgetWireTablesOfEndedSession();
// ---- P14 S1: the EGL context lifecycle's control-plane half -----------------------------
//
// NOT ring records and NOT routed rows: eglCreateContext/eglDestroyContext are the EGL