[Fix] (P14): latch a lost GPU device per session on Magma and Espryt instead of aborting the whole server

MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp

@@ -34,6 +34,8 @@
 #include <MG_Remote/Server/StagedTextureStore.h>
 // Shared images: the registry the EGLImages are built from (SharedImageImpl below).
 #include <MG_Remote/Server/SharedImageRegistry.h>
+// Device loss ends the session through MG_Pipe's latch seam (LatchIfDeviceLost).
+#include <MG_Pipe/PipeSessionFail.h>
 #if defined(__ANDROID__)
 #include <android/hardware_buffer.h>
 #include <unistd.h>
@@ -15606,6 +15608,10 @@ namespace MobileGL::MG_Backend::DirectGLES {
     void ReadPixels(GLint x, GLint y, GLsizei width, GLsizei height, GLenum format, GLenum type, void* pixels) {
         MGLOG_D("ReadPixels: x=%d y=%d w=%d h=%d format=%s type=%s pixels=%p", x, y, width, height,
                 MG_Util::ConvertGLEnumToString(format).c_str(), MG_Util::ConvertGLEnumToString(type).c_str(), pixels);
+#if MOBILEGL_BUILD_DISAGGREGATED
+        // A reset context reads back nothing it rendered: the session is latched and the read declined.
+        if (LatchIfDeviceLost("read-pixels")) return;
+#endif
 
         // Combinations the ES driver has always handled directly keep the native path; other color layouts go
         // through the wide-format conversion path. Anything still uncovered degrades to a logged no-op instead
@@ -18034,7 +18040,33 @@ namespace MobileGL::MG_Backend::DirectGLES {
         return CompletedFrameSerial() >= serial;
     }
 
+#if MOBILEGL_BUILD_DISAGGREGATED
+    Bool LatchIfDeviceLost(const char* site, Bool debugKnob) {
+        if (MG_Config::Transport == MG_Config::TransportMode::Monolith) return false;
+        // Once per thread: a latched session declines everything after this verb anyway, and the
+        // apply thread is the session's own.
+        static thread_local Bool latched = false;
+        if (latched) return true;
+        GLenum status = GL_NO_ERROR;
+        if (g_GLESFuncs.glGetGraphicsResetStatus != nullptr && IsBackendContextCurrentOnThisThread()) {
+            status = g_GLESFuncs.glGetGraphicsResetStatus();
+        }
+        if (status == GL_NO_ERROR && debugKnob && MG_Pipe::MGPipeDebugDeviceLossDue()) status = GL_UNKNOWN_CONTEXT_RESET;
+        if (status == GL_NO_ERROR) return false;
+        latched = true;
+        (void)MG_Pipe::MGPipeSessionLatch(MG_Pipe::MGPipeFatalFamily::DeviceLost,
+                                          "MGPipe: Fatal{BackendDeviceLost, \"Espryt:%s\"} - this session's EGL "
+                                          "context was reset (glGetGraphicsResetStatus 0x%x); the session ends "
+                                          "and its client reads a lost context, every other session keeps running",
+                                          site != nullptr ? site : "?", static_cast<unsigned>(status));
+        return true;
+    }
+#endif
+
     void Present() {
+#if MOBILEGL_BUILD_DISAGGREGATED
+        if (LatchIfDeviceLost("present", /*debugKnob=*/false)) return;
+#endif
         // Insert one fence per frame BEFORE the swap (eglSwapBuffers' implicit flush
         // makes it reachable), then non-blocking-poll prior frames' fences AFTER to
         // advance the completed-frame watermark that gates buffer-pool recycling.
@@ -18636,6 +18668,7 @@ namespace MobileGL::MG_Backend::DirectGLES {
                 MGLOG_E_ONCE("Shared image present: no backend context is current on the calling thread");
                 return false;
             }
+            if (LatchIfDeviceLost("shared-image-present")) return false;
             const EGLSurface surface = CurrentContextDrawSurface();
             EGLint sourceWidth = 0;
             EGLint sourceHeight = 0;

MobileGL/MG_Backend/DirectGLES/DirectGLES.h

@@ -147,6 +147,19 @@ namespace MobileGL::MG_Backend::DirectGLES {
     // True when the backend ES context is current on the calling thread, i.e.
     // immediate buffer ops may issue GL calls right now.
     Bool IsBackendContextCurrentOnThisThread();
+#if MOBILEGL_BUILD_DISAGGREGATED
+    // DEVICE LOSS, Espryt's half (Magma's is VulkanRenderer::LatchWireDeviceLoss). Asks the
+    // driver whether the calling thread's context was reset (glGetGraphicsResetStatus); a reset -
+    // or the debug knob MGPipeDebugDeviceLossDue - latches THIS session through
+    // MGPipeSessionLatch and answers true, and the caller returns from its verb without issuing
+    // more GL. Asked at the frame boundaries (Present, a shared-image present) and before a
+    // readback, never per draw. The contexts keep the driver's default reset strategy: asking
+    // every context to lose itself on reset would also end the INNOCENT ones - the compositor's
+    // among them - on any other client's GPU fault.
+    // `debugKnob` false: the knob is not consulted here (the on-screen Present, so an injected loss
+    // lands on an offscreen client's shared-image present or readback, never on the compositor's).
+    Bool LatchIfDeviceLost(const char* site, Bool debugKnob = true);
+#endif
     // GL fence sync objects, backed by native ES fences. FenceSync returns null
     // (the frontend then falls back to an always-signaled sync) when the calling
     // thread does not own the ES context. Waits/queries degrade to "signaled" in

MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp

@@ -425,7 +425,10 @@ namespace MobileGL::MG_Backend::DirectVulkan {
                 uploaded = StagedWireRangeCopy(*resource, bytes, static_cast<SizeT>(offset),
                                                 static_cast<SizeT>(size));
             }
-            if (!uploaded && !WaitForWireBufferHostAccess(*resource)) WireBufferSyncFatal("host-write");
+            if (!uploaded && !WaitForWireBufferHostAccess(*resource)) {
+                if (LatchedOnDeviceLoss("buffer-host-write")) return;
+                WireBufferSyncFatal("host-write");
+            }
         }
         if (!uploaded) {
             uploaded = resource->buffer.Upload(bytes, size, offset);
@@ -434,6 +437,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
             }
         }
         if (!uploaded) {
+            if (LatchedOnDeviceLoss("buffer-upload")) return;
             MGLOG_F("Magma: Fatal{ResourceUnavailable, \"buffer-upload\"}");
             std::abort();
         }
@@ -604,6 +608,8 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         const Bool t0 = resource != nullptr && resource->imported;
         Vector<Uint8> bytes(t0 ? 0 : static_cast<SizeT>(size));
         if (t0 ? !WaitForWireBufferHostAccess(*resource) : !ReadWireBuffer(res, offset, size, bytes.data())) {
+            // A lost device: the session is latched and closing; its client reads a lost context.
+            if (LatchedOnDeviceLoss("buffer-readback")) return;
             MGLOG_F("Magma: Fatal{ResourceUnavailable, \"buffer-readback\"} {slot=%u, gen=%u}", res.Slot, res.Gen);
             std::abort();
         }
@@ -1074,6 +1080,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         // budget's worth of orphans, not one per glBufferData.
         if (pVulkanRenderer == nullptr ||
             !pVulkanRenderer->WaitForSubmitIndex(pVulkanRenderer->GetSyncPointSubmitIndex(), UINT64_MAX, true)) {
+            if (LatchedOnDeviceLoss("buffer-deferred-watermark")) return;
             WireBufferSyncFatal("deferred-watermark");
         }
         ++m_wireDeferredSyncs;
@@ -1094,6 +1101,10 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         MG_Util::PipeStats::PublishGauge(Gauge::WireDeferredSyncs, m_wireDeferredSyncs);
     }
 
+    Bool VkBufferManager::LatchedOnDeviceLoss(const char* site) {
+        return pVulkanRenderer != nullptr && pVulkanRenderer->LatchWireDeviceLoss(site);
+    }
+
     void VkBufferManager::WireBufferSyncFatal(const char* site) {
         MGLOG_F("Magma: Fatal{ResourceUnavailable, \"buffer-write-sync\"} {site=%s}", site);
         std::abort();

MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.h

@@ -362,6 +362,10 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         // for the census to count), shared by the host write that cannot wait for the GPU and
         // the watermark that cannot. `site` says which.
         [[noreturn]] static void WireBufferSyncFatal(const char* site);
+        // True when the wait or copy that just failed failed because the device is lost: the
+        // session has been latched (VulkanRenderer::LatchWireDeviceLoss) and the caller returns
+        // instead of taking its Fatal.
+        static Bool LatchedOnDeviceLoss(const char* site);
         // The wbuf[] gauges (PipeStats.h, Gauge::WireBuffers..WireDeferredSyncs). The peaks are
         // taken at the two points the numbers can rise - a park and a mint - and published when
         // the stats channel is on; MagmaWireReclaimScenario reads them off the server's line.

MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp

@@ -7123,7 +7123,7 @@ void main() {
         SplitOversizedRecording();
 #if MOBILEGL_BUILD_DISAGGREGATED
         if (MG_Config::Transport != MG_Config::TransportMode::Monolith) {
-            RewindWireDescriptorSetsIfDue();
+            if (!RewindWireDescriptorSetsIfDue()) return false;
             // The wire route returns before the monolith branch's draw-gated
             // sweep. Dead wire texture/renderbuffer records otherwise live
             // until the next frame boundary, which a long trace may not reach.
@@ -7760,7 +7760,7 @@ void main() {
         SplitOversizedRecording();
 #if MOBILEGL_BUILD_DISAGGREGATED
         if (MG_Config::Transport != MG_Config::TransportMode::Monolith) {
-            RewindWireDescriptorSetsIfDue();
+            if (!RewindWireDescriptorSetsIfDue()) return;
             m_textureManager->CollectGarbage();
             DispatchWireCompute(numGroupsX, numGroupsY, numGroupsZ);
             return;
@@ -7823,7 +7823,7 @@ void main() {
 #if MOBILEGL_BUILD_DISAGGREGATED
         // P8-SV: the wire arm's own indirect dispatch, DispatchCompute's shape one call over.
         if (MG_Config::Transport != MG_Config::TransportMode::Monolith) {
-            RewindWireDescriptorSetsIfDue();
+            if (!RewindWireDescriptorSetsIfDue()) return;
             m_textureManager->CollectGarbage();
             DispatchWireComputeIndirect(indirect);
             return;
@@ -10614,7 +10614,10 @@ void main() {
 #if MOBILEGL_BUILD_DISAGGREGATED
         const Bool wire = MG_Config::Transport != MG_Config::TransportMode::Monolith;
         MG_Pipe::MGPipeHandle dstStorageHandle = dstEndpoint.TextureHandle;
-        if (wire && HasPendingRecordedWork() && !FlushPendingCommands()) MagmaWireFatal("copy-image-flush");
+        if (wire && HasPendingRecordedWork() && !FlushPendingCommands()) {
+            if (LatchWireDeviceLoss("copy-image-flush")) return;
+            MagmaWireFatal("copy-image-flush");
+        }
 #endif
         MOBILEGL_ASSERT(srcEndpoint.Exists() && dstEndpoint.Exists(),
                         "CopyImageSubData requires valid source and destination images.");
@@ -11169,6 +11172,7 @@ void main() {
         VkResult result = vkWaitForFences(m_device, 1, &frame.imageInFlightFence, VK_TRUE, UINT64_MAX);
 #endif
         if (result != VK_SUCCESS) {
+            NoteDeviceLoss(result, "readback vkWaitForFences");
             MGLOG_E_ONCE("DirectVulkan readback: vkWaitForFences returned %d", result);
             return false;
         }
@@ -13749,6 +13753,7 @@ void main() {
         // the in-flight frame count) but never deadlocks.
         const VkResult result = vkQueueWaitIdle(m_graphicsQueue);
         if (result != VK_SUCCESS) {
+            NoteDeviceLoss(result, "WaitForFrameSerial vkQueueWaitIdle");
             MGLOG_E_ONCE("WaitForFrameSerial: vkQueueWaitIdle returned %d", result);
             return false;
         }
@@ -13784,24 +13789,29 @@ void main() {
     }
 
 #if MOBILEGL_BUILD_DISAGGREGATED
-    void VulkanRenderer::RewindWireDescriptorSetsIfDue() {
-        if (!m_uniformManager || m_frameContext.GetFrameCount() == 0) return;
+    Bool VulkanRenderer::RewindWireDescriptorSetsIfDue() {
+        if (!m_uniformManager || m_frameContext.GetFrameCount() == 0) return true;
         const Uint32 frameIndex = m_frameContext.GetCurrentFrameIndex();
-        if (!m_uniformManager->WireDescriptorSetBudgetReached(frameIndex)) return;
+        if (!m_uniformManager->WireDescriptorSetBudgetReached(frameIndex)) return true;
 
         // A set cannot be rewritten while a recorded or submitted command
         // buffer might still read it. This wire-only boundary first submits
         // the current recording, then idles the graphics queue before reuse.
-        if (HasPendingRecordedWork() && !FlushPendingCommands())
+        if (HasPendingRecordedWork() && !FlushPendingCommands()) {
+            if (LatchWireDeviceLoss("descriptor-rewind-flush")) return false;
             MagmaWireFatal("descriptor-rewind-flush");
+        }
         const Uint64 through = m_submitCounter;
         if (through > m_completedSubmitCounter) {
             // B4 has not yet made every old retirement site aggregate-safe.
             // Queue idle proves even a submission a previous single-fence path
             // prematurely removed from m_inFlightSubmits has finished; only
             // then may any layout cursor be rewound and its sets rewritten.
-            if (vkQueueWaitIdle(m_graphicsQueue) != VK_SUCCESS)
+            if (const VkResult idle = vkQueueWaitIdle(m_graphicsQueue); idle != VK_SUCCESS) {
+                NoteDeviceLoss(idle, "descriptor rewind vkQueueWaitIdle");
+                if (LatchWireDeviceLoss("descriptor-rewind-wait")) return false;
                 MagmaWireFatal("descriptor-rewind-wait");
+            }
             OnSubmitsCompletedUpTo(through);
         }
         if (HasPendingRecordedWork() || m_completedSubmitCounter < through)
@@ -13810,6 +13820,7 @@ void main() {
         MGLOG_I("Magma descriptor rewind: frame=%u retiredSubmit=%llu budget=%u cached=%zu",
                 frameIndex, static_cast<unsigned long long>(through),
                 UniformManager::kWireDescriptorSetBudget, cachedSets);
+        return true;
     }
 #endif
 
@@ -13859,6 +13870,7 @@ void main() {
             OnSubmitsCompletedUpTo(submitIndex);
             return true;
         }
+        NoteDeviceLoss(result, "WaitForSubmitsUpTo vkWaitForFences");
         if (result != VK_TIMEOUT)
             MGLOG_E_ONCE("WaitForSubmitsUpTo: vkWaitForFences returned %d", result);
         return false;
@@ -14157,6 +14169,7 @@ void main() {
 #endif
         const VkResult result = vkQueueSubmit(m_graphicsQueue, 1, &submitInfo, fence);
         if (result != VK_SUCCESS) {
+            NoteDeviceLoss(result, "SubmitPendingCommandBuffer vkQueueSubmit");
             MGLOG_E_ONCE("SubmitPendingCommandBuffer: vkQueueSubmit returned %d", result);
             return false;
         }
@@ -14248,6 +14261,45 @@ void main() {
         return true;
     }
 
+    void VulkanRenderer::NoteDeviceLoss(VkResult result, const char* where) {
+        if (result != VK_ERROR_DEVICE_LOST || m_deviceLost) return;
+        m_deviceLost = true;
+        MGLOG_E("DirectVulkan: the VkDevice is LOST - %s returned VK_ERROR_DEVICE_LOST (the driver reset a GPU "
+                "fault or hang); nothing this renderer recorded or submits from here on will execute",
+                where != nullptr ? where : "a Vulkan call");
+#if MOBILEGL_BUILD_DISAGGREGATED
+        // Latch NOW where a session latch exists, so the session stops at its next record however
+        // the verb in hand unwinds - including through a caller that only logs a failed submit.
+        if (MG_Pipe::MGPipeSessionLatchArmed()) (void)LatchWireDeviceLoss(where);
+#endif
+    }
+
+    Bool VulkanRenderer::IsDeviceLost() {
+        // Only failure paths ask, so the probe's wait never lands on a healthy frame: a device
+        // that is merely slow answers VK_SUCCESS once its work drains, a lost one answers at once.
+        if (!m_deviceLost && m_device != VK_NULL_HANDLE) {
+            NoteDeviceLoss(vkDeviceWaitIdle(m_device), "the device-loss probe's vkDeviceWaitIdle");
+        }
+        return m_deviceLost;
+    }
+
+#if MOBILEGL_BUILD_DISAGGREGATED
+    Bool VulkanRenderer::LatchWireDeviceLoss(const char* site) {
+        if (!IsDeviceLost()) return false;
+        if (!m_deviceLossLatched) {
+            m_deviceLossLatched = true;
+            // Returns (latched) on a served session; dies, as the site's own Fatal would have,
+            // where no session latch is armed.
+            (void)MG_Pipe::MGPipeSessionLatch(MG_Pipe::MGPipeFatalFamily::DeviceLost,
+                                              "MGPipe: Fatal{BackendDeviceLost, \"Magma:%s\"} - this session's "
+                                              "VkDevice was lost to a GPU fault or hang; the session ends and its "
+                                              "client reads a lost context, every other session keeps running",
+                                              site != nullptr ? site : "?");
+        }
+        return true;
+    }
+#endif
+
     Bool VulkanRenderer::FlushForSyncPoint(Uint64 submitIndex) {
         // A flush only helps a sync point whose commands are not submitted
         // yet; for an already-submitted index it would just split the frame's
@@ -14300,6 +14352,7 @@ void main() {
             }
             const VkResult waitResult = vkWaitForFences(m_device, 1, &oldest.fence, VK_TRUE, UINT64_MAX);
             if (waitResult != VK_SUCCESS) {
+                NoteDeviceLoss(waitResult, "SplitOversizedRecording vkWaitForFences");
                 MGLOG_E_ONCE("SplitOversizedRecording: vkWaitForFences returned %d", waitResult);
                 break;
             }
@@ -14465,6 +14518,8 @@ void main() {
 
     void VulkanRenderer::Present() {
 #if MOBILEGL_BUILD_DISAGGREGATED
+        // A lost device presents nothing (its session is latched and closing).
+        if (m_deviceLost && LatchWireDeviceLoss("present")) return;
         // Present has its own submit path. Tag its last wire pass before that
         // submission, rather than retaining it until a draw in the next frame.
         RetireWireDrawPass();
@@ -14507,8 +14562,10 @@ void main() {
             // free its retired command buffers and reset the slot fence.
             const Uint64 slotSubmit = m_frameContext.GetCurrent().lastSubmitIndex;
             if (slotSubmit > m_completedSubmitCounter &&
-                !WaitForSubmitsUpTo(slotSubmit, UINT64_MAX))
+                !WaitForSubmitsUpTo(slotSubmit, UINT64_MAX)) {
+                if (LatchWireDeviceLoss("deferred-acquire-submit-wait")) return;
                 MagmaWireFatal("deferred-acquire-submit-wait");
+            }
 #endif
             const VkResult acquireResult =
                 m_frameContext.WaitAndAcquireNextImage(m_device, m_swapchainObject.GetHandle(), m_imageIndexAcquired);
@@ -14591,7 +14648,12 @@ void main() {
         SharedImageSubmitSync sharedImageSync;
         AttachSharedImageSync(submitPacket.submitInfo, sharedImageSync);
 #endif
-        VK_VERIFY(vkQueueSubmit(m_graphicsQueue, 1, &submitPacket.submitInfo, frame.imageInFlightFence));
+        const VkResult presentSubmit = vkQueueSubmit(m_graphicsQueue, 1, &submitPacket.submitInfo, frame.imageInFlightFence);
+        NoteDeviceLoss(presentSubmit, "Present vkQueueSubmit");
+#if MOBILEGL_BUILD_DISAGGREGATED
+        if (presentSubmit != VK_SUCCESS && LatchWireDeviceLoss("present-submit")) return;
+#endif
+        VK_VERIFY(presentSubmit);
         RegisterSubmit(frame.imageInFlightFence, /*pooledFence=*/false);
         frame.lastSubmitIndex = m_submitCounter;
 #if MOBILEGL_BUILD_DISAGGREGATED
@@ -14627,6 +14689,10 @@ void main() {
             m_swapchainResizeRequested = false;
             result = VK_SUCCESS;
         }
+        NoteDeviceLoss(result, "vkQueuePresentKHR");
+#if MOBILEGL_BUILD_DISAGGREGATED
+        if (result != VK_SUCCESS && LatchWireDeviceLoss("present-queue")) return;
+#endif
         VK_VERIFY(result, "Present, vkQueuePresentKHR");
         // EGL swap semantics: the presented color buffer's content is undefined the
         // next time this image is acquired (EGL_BUFFER_DESTROYED, the default swap
@@ -14677,8 +14743,10 @@ void main() {
         // slot's prefix BEFORE the reset/free happens.
         const Uint64 slotSubmit = m_frameContext.GetCurrent().lastSubmitIndex;
         if (slotSubmit > m_completedSubmitCounter &&
-            !WaitForSubmitsUpTo(slotSubmit, UINT64_MAX))
+            !WaitForSubmitsUpTo(slotSubmit, UINT64_MAX)) {
+            if (LatchWireDeviceLoss("present-slot-submit-wait")) return;
             MagmaWireFatal("present-slot-submit-wait");
+        }
 #endif
         // 4) Wait/reset/acquire for next frame.
         result = m_frameContext.WaitAndAcquireNextImage(m_device, m_swapchainObject.GetHandle(), m_imageIndexAcquired);
@@ -14703,6 +14771,10 @@ void main() {
             result =
                 m_frameContext.WaitAndAcquireNextImage(m_device, m_swapchainObject.GetHandle(), m_imageIndexAcquired);
         }
+        NoteDeviceLoss(result, "vkAcquireNextImageKHR");
+#if MOBILEGL_BUILD_DISAGGREGATED
+        if (result != VK_SUCCESS && LatchWireDeviceLoss("present-acquire")) return;
+#endif
         VK_VERIFY(result, "Present, vkAcquireNextImageKHR");
         // Pull keeps its historical slot-fence inference. In a disaggregated
         // build the aggregate wait above already retired the registered prefix

MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h

@@ -376,6 +376,22 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         // one is retired until the frame slot's fence is next waited. Returns
         // true when a submission was made.
         Bool FlushPendingCommands();
+        // DEVICE LOSS. A GPU fault or hang the driver reset leaves this renderer's VkDevice lost:
+        // every later submit and wait answers VK_ERROR_DEVICE_LOST. NoteDeviceLoss records it at
+        // the submit/wait sites (cheap, any result); IsDeviceLost answers from the record and,
+        // when nothing was recorded, asks the device once (only failure paths call it, so the
+        // probe's wait costs nothing on a healthy frame). Each session owns its own VkDevice, so a
+        // loss here is this session's alone.
+        void NoteDeviceLoss(VkResult result, const char* where);
+        Bool IsDeviceLost();
+#if MOBILEGL_BUILD_DISAGGREGATED
+        // For a wire site whose submit or wait just failed: true when the failure is a lost
+        // device, in which case this session has been latched (MGPipeSessionLatch - it ends, the
+        // display server's other sessions do not) and the caller must return from its verb
+        // without recording or submitting anything more. False: an ordinary failure, the site
+        // keeps the death it had. `site` names it in the latch line.
+        Bool LatchWireDeviceLoss(const char* site);
+#endif
         // Flush gated on usefulness: only flushes when `submitIndex` is still
         // unsubmitted, so poll loops on already-submitted fences do not split
         // the frame's render pass (a full tile load/store on TBDR GPUs).
@@ -589,6 +605,9 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         };
 
         void QueueClearBufferPayload(GLenum buffer, GLint drawbuffer, const ClearAttachmentPayload& clearPayload);
+        // See NoteDeviceLoss. Set once, never cleared: a lost VkDevice stays lost.
+        Bool m_deviceLost = false;
+        Bool m_deviceLossLatched = false;
 #if MOBILEGL_BUILD_DISAGGREGATED
         struct WireImage {
             VkImage image = VK_NULL_HANDLE;
@@ -843,7 +862,8 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         // P8-SV: glDispatchComputeIndirect on the wire arm - vkCmdDispatchIndirect from the
         // dispatch-indirect store, as the monolith arm issues it (WireDraw.inc).
         void DispatchWireComputeIndirect(GLintptr offset);
-        void RewindWireDescriptorSetsIfDue();
+        // False: the device was lost and the session latched; the caller drops its draw/dispatch.
+        Bool RewindWireDescriptorSetsIfDue();
         UniquePtr<RenderPassEntry> m_wireDrawPass;
         Vector<VkImageView> m_wireDrawViews;
         VkPipeline GetOrCreatePipelineWithInput(GLenum mode, const MagmaProgramSource& program,

MobileGL/MG_Backend/DirectVulkan/Renderer/WireFramebuffer.inc

@@ -381,7 +381,13 @@
     void VulkanRenderer::ReadWirePixels(GLint x, GLint y, GLsizei width, GLsizei height,
                                         GLenum format, GLenum type, void* pixels) {
         if (width <= 0 || height <= 0) return;
-        if (HasPendingRecordedWork() && !FlushPendingCommands()) MagmaWireFatal("wire-verb-flush");
+        if (MG_Pipe::MGPipeDebugDeviceLossDue()) NoteDeviceLoss(VK_ERROR_DEVICE_LOST, "the debug knob (read-pixels)");
+        if (m_deviceLost && LatchWireDeviceLoss("read-pixels")) return;
+        if (HasPendingRecordedWork() && !FlushPendingCommands()) {
+            // A lost device: the session is latched and closing, the pixels stay unfilled.
+            if (LatchWireDeviceLoss("wire-verb-flush")) return;
+            MagmaWireFatal("wire-verb-flush");
+        }
         const auto* fbo = MG_Pipe::MGPipeApplier().ReadFramebuffer();
         if (!fbo) MagmaWireFatal("read-framebuffer-record");
         VkImageAspectFlags aspect = format == GL_DEPTH_COMPONENT ? VK_IMAGE_ASPECT_DEPTH_BIT :
@@ -446,7 +452,10 @@
         auto& frame = m_frameContext.GetCurrent();
         vkCmdCopyImageToBuffer(frame.commandBuffer,source.image,VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
                               buffer.GetHandle(),1,&region);
-        if (!SubmitReadbackCommandsAndWait(frame)) MagmaWireFatal("read-submit");
+        if (!SubmitReadbackCommandsAndWait(frame)) {
+            if (LatchWireDeviceLoss("read-submit")) return;
+            MagmaWireFatal("read-submit");
+        }
         const auto* bytes = static_cast<const Uint8*>(buffer.Map());
         if (!bytes || !buffer.Invalidate(size)) MagmaWireFatal("read-map");
         Vector<Uint8> remapped;

MobileGL/MG_Backend/DirectVulkan/Renderer/WireSharedImage.inc

@@ -72,6 +72,9 @@
 
     Bool VulkanRenderer::BlitDefaultFramebufferToSharedImage(const SharedImageView& view) {
         if (m_device == VK_NULL_HANDLE || view.Width == 0 || view.Height == 0) return false;
+        // The debug knob's device check (MGPipeDebugDeviceLossDue) rides the frame boundary here.
+        if (MG_Pipe::MGPipeDebugDeviceLossDue()) NoteDeviceLoss(VK_ERROR_DEVICE_LOST, "the debug knob (shared-image present)");
+        if (m_deviceLost && LatchWireDeviceLoss("shared-image-present")) return false;
         DestroySharedImagePresentTargets(true);
 
         // The default framebuffer's image is stored in display orientation (row 0 is the top of

MobileGL/MG_Pipe/PipeApply.cpp

@@ -53,12 +53,16 @@
 #endif
 
 #include <algorithm>
+#include <atomic>
 #include <cmath>
 #include <cstdarg>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <utility>
+#if defined(__ANDROID__)
+#include <sys/system_properties.h>
+#endif
 
 // THE VERDICT OF EVERY TRIP WIRE IN THIS FILE, IN ONE PLACE.
 //
@@ -154,6 +158,58 @@ namespace MobileGL::MG_Pipe {
         MGLOG_F("%s", line);
         std::abort();
     }
+
+    namespace {
+        MGPipeSessionLatchHook g_sessionLatchHook = nullptr;
+        MGPipeSessionLatchArmedHook g_sessionLatchArmedHook = nullptr;
+    } // namespace
+
+    void MGPipeInstallSessionLatchHook(MGPipeSessionLatchHook hook, MGPipeSessionLatchArmedHook armed) {
+        g_sessionLatchHook = hook;
+        g_sessionLatchArmedHook = armed;
+    }
+
+    MGPipeSessionLatchHook MGPipeSessionLatchHookInstalled() { return g_sessionLatchHook; }
+
+    bool MGPipeSessionLatchArmed() { return g_sessionLatchArmedHook != nullptr && g_sessionLatchArmedHook(); }
+
+    bool MGPipeDebugDeviceLossDue() {
+        long at = 0;
+        if (const char* value = std::getenv("MOBILEGL_DEBUG_INJECT_DEVICE_LOST_AT")) {
+            at = std::strtol(value, nullptr, 10);
+        }
+#if defined(__ANDROID__)
+        else {
+            char property[PROP_VALUE_MAX] = {};
+            if (__system_property_get("debug.mobilegl.inject_device_lost_at", property) > 0) {
+                at = std::strtol(property, nullptr, 10);
+            }
+        }
+#endif
+        if (at <= 0) return false;
+        static std::atomic<long> seen{0};
+        static std::atomic<bool> fired{false};
+        if (fired.load(std::memory_order_acquire)) return false;
+        if (seen.fetch_add(1, std::memory_order_acq_rel) + 1 < at) return false;
+        if (fired.exchange(true, std::memory_order_acq_rel)) return false;
+        MGLOG_E("MGPipe: debug knob inject_device_lost_at=%ld - this device check reports the device LOST", at);
+        return true;
+    }
+
+    bool MGPipeSessionLatch(MGPipeFatalFamily family, const char* fmt, ...) {
+        char line[512];
+        va_list args;
+        va_start(args, fmt);
+        const int written = std::vsnprintf(line, sizeof(line), fmt, args);
+        va_end(args);
+        if (written < 0) {
+            std::snprintf(line, sizeof(line),
+                          "MGPipe: unformattable Fatal diagnostic (format=%s)", fmt);
+        }
+        if (g_sessionLatchHook != nullptr) return g_sessionLatchHook(family, line);
+        // NO HOOK: no session to latch, so the death the fail seam would have given.
+        MGPipeSessionFail(family, "%s", line);
+    }
 #endif
 
     namespace {

MobileGL/MG_Pipe/PipeSessionFail.h

@@ -57,6 +57,9 @@ namespace MobileGL::MG_Pipe {
 #if MOBILEGL_BUILD_DISAGGREGATED
         // A malformed peer record rejected by the server-side applier.
         ProtocolCorruption,
+        // The GPU device a session's backend renders with is lost (a GPU fault or hang the driver
+        // reset). Raised only through MGPipeSessionLatch below: it ends THAT session.
+        DeviceLost,
 #endif
     };
 
@@ -82,4 +85,43 @@ namespace MobileGL::MG_Pipe {
 #endif
     void MGPipeSessionFail(MGPipeFatalFamily family, const char* fmt, ...);
 
+#if MOBILEGL_BUILD_DISAGGREGATED
+    // ---- THE LATCH TWIN: A LOST GPU DEVICE ENDS ITS SESSION, NOT THE PROCESS ----------------
+    //
+    // MGPipeSessionFail is for a verb the backend cannot honour, and it does not return. A device
+    // loss is different in kind: nothing about the peer's bytes was wrong, the GPU faulted or hung
+    // and the driver reset it, and every session the process serves except the one whose device
+    // went is still healthy. The display server runs every session in ONE process, so dying here
+    // takes the compositor and every other client down with the one that lost its device.
+    //
+    // So a backend that sees its device lost calls this instead, and then RETURNS from the verb
+    // it was applying without touching the GPU again. The hook MG_Remote installs latches the
+    // calling thread's session (SessionLatch): the session declines everything after it, closes,
+    // and its client reads a lost context. Where no latch is armed - the inproc client+server
+    // shape, where the process IS the session - the hook dies exactly as MGPipeSessionFail does,
+    // and with no hook at all this is MGPipeSessionFail. Returns false, so a Bool site can read
+    // `return MGPipeSessionLatch(...);`.
+    using MGPipeSessionLatchHook = bool (*)(MGPipeFatalFamily family, const char* line);
+    // Whether the calling thread's session HAS a latch to raise. A backend that notices its
+    // device lost outside a site that would die anyway (a submit whose failure it used to only
+    // log) latches at once when this is true - so the session stops at its next record however
+    // the current verb unwinds - and keeps its old behaviour when it is false.
+    using MGPipeSessionLatchArmedHook = bool (*)();
+    void MGPipeInstallSessionLatchHook(MGPipeSessionLatchHook hook, MGPipeSessionLatchArmedHook armed);
+    MGPipeSessionLatchHook MGPipeSessionLatchHookInstalled();
+    bool MGPipeSessionLatchArmed();
+    bool
+#if defined(__GNUC__) || defined(__clang__)
+        __attribute__((format(printf, 2, 3)))
+#endif
+        MGPipeSessionLatch(MGPipeFatalFamily family, const char* fmt, ...);
+
+    // A DEBUG KNOB THAT MAKES ONE DEVICE CHECK REPORT A LOSS, so the containment above can be
+    // driven without a GPU that faults on demand. MOBILEGL_DEBUG_INJECT_DEVICE_LOST_AT=N (or, on
+    // Android, the property debug.mobilegl.inject_device_lost_at) makes the N-th device check the
+    // backends make after the knob was first seen nonzero answer "lost", once per process. The
+    // backends ask at their frame boundaries and readbacks; unset (the default) it is one getenv.
+    bool MGPipeDebugDeviceLossDue();
+#endif
+
 } // namespace MobileGL::MG_Pipe

MobileGL/MG_Remote/FatalFamilies.def

@@ -57,6 +57,11 @@
 // server-side resource its session had to have is gone - so it reads a clean device loss. Not
 // DeviceLost: that code is the latch's own, never a family's projection (see its bucket above).
 //
+// BackendDeviceLost: the GPU device (Magma's VkDevice, Espryt's EGL context) a session renders
+// with was lost to a GPU fault or hang. It is only ever LATCHED (MGPipeSessionLatch), so the one
+// session ends and the display server's other sessions keep running; the waiting client is told
+// what ServerCrashed says and reads a lost context. Not DeviceLost, for ServerWindowLost's reason.
+//
 // Two words were NOT given rows, they were collapsed instead: `InitialBytesNotCarried`
 // (MG_Impl/Pipe/PipeFill.cpp) said the same thing as `UncarriedInitialBytes` (Client/
 // WireTables.cpp) about the same record - a respecify that crossed with no initial bytes - and
@@ -113,6 +118,7 @@
     X(ApplyThreadJoinTimeout,        ServerCrashed,      "the apply thread did not join within its bound at teardown") \
     X(ApplyThreadNotRunning,         ServerCrashed,      "an operation needed the apply thread and it was not running") \
     X(ResourceUnavailable,           ServerCrashed,      "the server could not obtain a native GL/Vulkan resource it had to have") \
-    X(ServerWindowLost,              ServerCrashed,      "the server's own display window was destroyed under a live on-screen session (P12)")
+    X(ServerWindowLost,              ServerCrashed,      "the server's own display window was destroyed under a live on-screen session (P12)") \
+    X(BackendDeviceLost,             ServerCrashed,      "the GPU device a session's backend renders with was lost (fault or hang reset)")
 
 // clang-format on

MobileGL/MG_Remote/FatalFunnel.cpp

@@ -45,28 +45,41 @@ namespace MobileGL::MG_Remote {
         // The line is forwarded through "%s" rather than re-used as a format: it has already been
         // vsnprintf'd once, and a detail string that happened to contain a `%` would otherwise be
         // read as a conversion the second time round.
-        [[noreturn]] void PipeSessionFailAdapter(MG_Pipe::MGPipeFatalFamily family,
-                                                 const char* line) {
-            MGFatalFamily mapped = MGFatalFamily::ProtocolCorruption;
+        MGFatalFamily MapPipeFamily(MG_Pipe::MGPipeFatalFamily family) {
             switch (family) {
             case MG_Pipe::MGPipeFatalFamily::UnmigratedVerb:
-                mapped = MGFatalFamily::UnmigratedVerb;
-                break;
+                return MGFatalFamily::UnmigratedVerb;
             case MG_Pipe::MGPipeFatalFamily::RoleViolation:
-                mapped = MGFatalFamily::RoleViolation;
-                break;
+                return MGFatalFamily::RoleViolation;
 #if MOBILEGL_BUILD_DISAGGREGATED
             case MG_Pipe::MGPipeFatalFamily::ProtocolCorruption:
-                mapped = MGFatalFamily::ProtocolCorruption;
-                break;
+                return MGFatalFamily::ProtocolCorruption;
+            case MG_Pipe::MGPipeFatalFamily::DeviceLost:
+                return MGFatalFamily::BackendDeviceLost;
 #endif
             }
-            SessionFail(mapped, "%s", line);
+            return MGFatalFamily::ProtocolCorruption;
         }
+
+        [[noreturn]] void PipeSessionFailAdapter(MG_Pipe::MGPipeFatalFamily family,
+                                                 const char* line) {
+            SessionFail(MapPipeFamily(family), "%s", line);
+        }
+
+#if MOBILEGL_BUILD_DISAGGREGATED
+        // The latch twin (PipeSessionFail.h): the calling thread's session latches the fault and
+        // the backend returns from its verb; unarmed, SessionLatch is SessionFail.
+        bool PipeSessionLatchAdapter(MG_Pipe::MGPipeFatalFamily family, const char* line) {
+            return SessionLatch(MapPipeFamily(family), "%s", line);
+        }
+#endif
     } // namespace
 
     void InstallPipeSessionFailHook() {
         MG_Pipe::MGPipeInstallSessionFailHook(&PipeSessionFailAdapter);
+#if MOBILEGL_BUILD_DISAGGREGATED
+        MG_Pipe::MGPipeInstallSessionLatchHook(&PipeSessionLatchAdapter, &SessionLatchArmed);
+#endif
     }
 
     namespace {

MobileGL/MG_Remote/Server/PipeApplier.cpp

@@ -578,7 +578,9 @@ namespace MobileGL::MG_Remote::Server {
         MG_Pipe::MGPPixelPackState restorePack{};
         restorePack.Pack = savedPack;
         MG_Pipe::MGPipeApplySetPixelPackState(restorePack);
-        return true;
+        // A backend that found its device lost latched this session and returned without filling
+        // the scratch: the read is declined rather than answered with stale bytes.
+        return !SessionLatched();
     }
 
     Bool ServerVerbSink::ReadTextureImageTight(const MG_Pipe::MGPReadbackInfo& image, Vector<Uint8>& bytes) {

MobileGL/MG_Test/Wire/FatalFamilyTest.cpp

@@ -203,3 +203,50 @@ TEST(FatalFamily, TheSessionFaultFrameCarriesTheFamilyAndItsCode) {
         EXPECT_EQ(decoded->code(), FatalCodeForFamily(family));
     }
 }
+
+// ---------------------------------------------------------------------------------------------
+// THE LATCH TWIN (MGPipeSessionLatch): how a backend's device loss reaches the session latch
+// instead of SessionFail. The display server runs every session in one process, so the twin must
+// RETURN on an armed session - only that session ends - and die exactly as the fail seam does
+// where nothing is armed. Red once by pointing FatalFunnel.cpp's latch adapter at SessionFail: the
+// armed case dies instead of returning.
+
+TEST(FatalFunnelSeam, ADeviceLossLatchesAnArmedSessionAndTheProcessLives) {
+#if defined(_WIN32)
+    GTEST_SKIP() << "POSIX only, as above";
+#else
+    using namespace MobileGL;
+    EXPECT_EXIT(
+        {
+            MG_Remote::InstallPipeSessionFailHook();
+            MG_Remote::ArmSessionLatch();
+            const bool answered = MG_Pipe::MGPipeSessionLatch(
+                MG_Pipe::MGPipeFatalFamily::DeviceLost,
+                "MGPipe: Fatal{BackendDeviceLost, \"Magma:%s\"} - unit probe", "wire-verb-flush");
+            const bool ok = !answered && MG_Remote::SessionLatched() &&
+                            MG_Remote::SessionLatchedFamily() == MG_Remote::MGFatalFamily::BackendDeviceLost &&
+                            MG_Pipe::MGPipeSessionLatchArmed();
+            std::fflush(nullptr);
+            std::_Exit(ok ? 0 : 3);
+        },
+        ::testing::ExitedWithCode(0), "Fatal\\{BackendDeviceLost, \"Magma:wire-verb-flush\"\\}")
+        << "an armed session's device loss did not latch and return";
+#endif
+}
+
+TEST(FatalFunnelSeam, ADeviceLossWithNoArmedSessionDiesThroughSessionFail) {
+#if defined(_WIN32)
+    GTEST_SKIP() << "POSIX only, as above";
+#else
+    using namespace MobileGL;
+    EXPECT_EXIT(
+        {
+            std::signal(SIGABRT, &ReportFaultCountAndExit);
+            MG_Remote::InstallPipeSessionFailHook();
+            (void)MG_Pipe::MGPipeSessionLatch(MG_Pipe::MGPipeFatalFamily::DeviceLost,
+                                              "MGPipe: Fatal{BackendDeviceLost, \"Magma:%s\"}", "unarmed");
+        },
+        ::testing::ExitedWithCode(kFaultExitBase + 1), "Fatal\\{BackendDeviceLost, \"Magma:unarmed\"\\}")
+        << "an unarmed device loss must keep the inproc shape's death";
+#endif
+}

MobileGL/MG_Test/Wire/MultiSessionTest.cpp

@@ -519,3 +519,62 @@ TEST_F(MultiSessionFixture, TwoSessionsInterleaveTwoContextsEachWithoutCrossTalk
     EXPECT_TRUE(WIFEXITED(server.exitStatus) && WEXITSTATUS(server.exitStatus) == 0)
         << "the in-process server did not stop cleanly: status 0x" << std::hex << server.exitStatus;
 }
+
+// A LOST GPU DEVICE ENDS ITS OWN SESSION AND NO OTHER.
+//
+// The display server serves every client in one process, so a backend that met a device loss on
+// one session's apply thread and died (Magma's wire funnels did: a failed submit took
+// Fatal{UnmigratedVerb, "Magma:wire-verb-flush"} and SIGABRT) took the compositor and every other
+// client down with it. A backend now hands a loss to MGPipeSessionLatch: that session latches
+// Fatal{BackendDeviceLost}, declines what it was doing and closes (exit 75), and its neighbours
+// keep applying. The loss is the debug knob's (MGPipeDebugDeviceLossDue) - the same device check
+// the backends make at their readbacks and frame boundaries - because no GPU here faults on
+// demand: the 2nd check in the process (session A's read-back, after B's) reports "lost".
+//
+// Red once by routing the latch seam through SessionFail (FatalFunnel.cpp's adapter): the whole
+// server process aborts at A's read-back and session B's next record is never answered.
+TEST_F(MultiSessionFixture, ADeviceLossEndsOnlyItsOwnSession) {
+    ServerProcess server;
+    ::setenv("MOBILEGL_DEBUG_INJECT_DEVICE_LOST_AT", "2", 1);
+    const bool launched = LaunchUnixInProcessServer(&server, "devlost");
+    ::unsetenv("MOBILEGL_DEBUG_INJECT_DEVICE_LOST_AT");
+    ASSERT_TRUE(launched) << "the in-process unix server did not start:\n" << server.Log();
+
+    Peer a = StartPeer(server.endpoint);
+    Peer b = StartPeer(server.endpoint);
+    ASSERT_GT(a.pid, 0);
+    ASSERT_GT(b.pid, 0);
+    ASSERT_EQ(a.ready.started, 1) << a.ready.note << "\n" << server.Log();
+    ASSERT_EQ(b.ready.started, 1) << b.ready.note << "\n" << server.Log();
+
+    // Check #1: B's read-back is answered.
+    const StepReport bFirst = b.Step(kOpHealthySession, kGreen);
+    ASSERT_EQ(bFirst.session.egl, 1) << "no headless EGL for session B: " << bFirst.session.note << "\n" << server.Log();
+    ASSERT_EQ(bFirst.session.pixel, kGreen) << bFirst.session.note << "\n" << server.Log();
+
+    // Check #2: A's device is "lost" at its read-back. A's session latches and ends.
+    const StepReport aLost = a.Step(kOpHealthySession, kRed);
+    EXPECT_NE(aLost.session.pixel, kRed) << "session A read back a frame from a lost device";
+    ASSERT_TRUE(WaitFor([&] { return server.Log().find("reaped exit=75") != std::string::npos; }, 5000))
+        << "session A did not end on a latched fault:\n" << server.Log();
+    const std::string log = server.Log();
+    EXPECT_NE(log.find("Fatal{BackendDeviceLost, \"Espryt:read-pixels\"}"), std::string::npos) << log;
+    EXPECT_NE(log.find("SessionLatch{BackendDeviceLost}"), std::string::npos) << log;
+    EXPECT_TRUE(server.pid > 0 && ::kill(server.pid, 0) == 0) << "the server process died with session A";
+
+    // B IS STILL SERVED, records and read-backs alike.
+    for (int record = 0; record < 3; ++record) {
+        const StepReport after = b.Step(kOpApplyRecord);
+        ASSERT_EQ(after.applied, 1) << "session B stopped applying after session A's device loss (record "
+                                    << record << "): " << after.note << "\n" << server.Log();
+    }
+    const StepReport bAgain = b.Step(kOpHealthySession, kGreen);
+    EXPECT_EQ(bAgain.session.pixel, kGreen) << bAgain.session.note << "\n" << server.Log();
+    EXPECT_EQ(Count(server.Log(), "SessionLatch{"), 1u) << "a session other than A latched:\n" << server.Log();
+
+    b.Close();
+    a.Close();
+    server.Stop();
+    EXPECT_TRUE(WIFEXITED(server.exitStatus) && WEXITSTATUS(server.exitStatus) == 0)
+        << "the in-process server did not stop cleanly: status 0x" << std::hex << server.exitStatus;
+}

scripts/ci/fatal_census_baseline.json

@@ -1,5 +1,5 @@
 {
-  "abort_sites": 77,
+  "abort_sites": 76,
   "abort_sites_by_file": {
     "MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp": 13,
     "MobileGL/MG_Backend/DirectGLES/Managers.cpp": 12,
@@ -7,7 +7,7 @@
     "MobileGL/MG_Backend/DirectGLES/Utils.cpp": 1,
     "MobileGL/MG_Backend/DirectVulkan/DirectVulkan.cpp": 2,
     "MobileGL/MG_Backend/DirectVulkan/Renderer/MagmaPipeArms.h": 1,
-    "MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp": 11,
+    "MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp": 10,
     "MobileGL/MG_Backend/Init.cpp": 2,
     "MobileGL/MG_Backend/MGPipe/PipeInputs.cpp": 3,
     "MobileGL/MG_Impl/Pipe/PipeFill.cpp": 5,
@@ -27,6 +27,7 @@
     "ApplyThreadNotRunning",
     "ArchiveEmpty",
     "ArtefactsMissing",
+    "BackendDeviceLost",
     "BarrierTimeout",
     "BarrierViolation",
     "BlobMissing",