[Fix] (P14): latch a lost GPU device per session on Magma and Espryt instead of aborting the whole server

This commit is contained in:
2026-10-02 13:37:58 -04:00
parent d03fe4c9dd
commit 58c3d9bde5
16 changed files with 421 additions and 30 deletions
@@ -425,7 +425,10 @@ namespace MobileGL::MG_Backend::DirectVulkan {
uploaded = StagedWireRangeCopy(*resource, bytes, static_cast<SizeT>(offset),
static_cast<SizeT>(size));
}
if (!uploaded && !WaitForWireBufferHostAccess(*resource)) WireBufferSyncFatal("host-write");
if (!uploaded && !WaitForWireBufferHostAccess(*resource)) {
if (LatchedOnDeviceLoss("buffer-host-write")) return;
WireBufferSyncFatal("host-write");
}
}
if (!uploaded) {
uploaded = resource->buffer.Upload(bytes, size, offset);
@@ -434,6 +437,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
}
}
if (!uploaded) {
if (LatchedOnDeviceLoss("buffer-upload")) return;
MGLOG_F("Magma: Fatal{ResourceUnavailable, \"buffer-upload\"}");
std::abort();
}
@@ -604,6 +608,8 @@ namespace MobileGL::MG_Backend::DirectVulkan {
const Bool t0 = resource != nullptr && resource->imported;
Vector<Uint8> bytes(t0 ? 0 : static_cast<SizeT>(size));
if (t0 ? !WaitForWireBufferHostAccess(*resource) : !ReadWireBuffer(res, offset, size, bytes.data())) {
// A lost device: the session is latched and closing; its client reads a lost context.
if (LatchedOnDeviceLoss("buffer-readback")) return;
MGLOG_F("Magma: Fatal{ResourceUnavailable, \"buffer-readback\"} {slot=%u, gen=%u}", res.Slot, res.Gen);
std::abort();
}
@@ -1074,6 +1080,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
// budget's worth of orphans, not one per glBufferData.
if (pVulkanRenderer == nullptr ||
!pVulkanRenderer->WaitForSubmitIndex(pVulkanRenderer->GetSyncPointSubmitIndex(), UINT64_MAX, true)) {
if (LatchedOnDeviceLoss("buffer-deferred-watermark")) return;
WireBufferSyncFatal("deferred-watermark");
}
++m_wireDeferredSyncs;
@@ -1094,6 +1101,10 @@ namespace MobileGL::MG_Backend::DirectVulkan {
MG_Util::PipeStats::PublishGauge(Gauge::WireDeferredSyncs, m_wireDeferredSyncs);
}
Bool VkBufferManager::LatchedOnDeviceLoss(const char* site) {
return pVulkanRenderer != nullptr && pVulkanRenderer->LatchWireDeviceLoss(site);
}
void VkBufferManager::WireBufferSyncFatal(const char* site) {
MGLOG_F("Magma: Fatal{ResourceUnavailable, \"buffer-write-sync\"} {site=%s}", site);
std::abort();
@@ -362,6 +362,10 @@ namespace MobileGL::MG_Backend::DirectVulkan {
// for the census to count), shared by the host write that cannot wait for the GPU and
// the watermark that cannot. `site` says which.
[[noreturn]] static void WireBufferSyncFatal(const char* site);
// True when the wait or copy that just failed failed because the device is lost: the
// session has been latched (VulkanRenderer::LatchWireDeviceLoss) and the caller returns
// instead of taking its Fatal.
static Bool LatchedOnDeviceLoss(const char* site);
// The wbuf[] gauges (PipeStats.h, Gauge::WireBuffers..WireDeferredSyncs). The peaks are
// taken at the two points the numbers can rise - a park and a mint - and published when
// the stats channel is on; MagmaWireReclaimScenario reads them off the server's line.
@@ -7123,7 +7123,7 @@ void main() {
SplitOversizedRecording();
#if MOBILEGL_BUILD_DISAGGREGATED
if (MG_Config::Transport != MG_Config::TransportMode::Monolith) {
RewindWireDescriptorSetsIfDue();
if (!RewindWireDescriptorSetsIfDue()) return false;
// The wire route returns before the monolith branch's draw-gated
// sweep. Dead wire texture/renderbuffer records otherwise live
// until the next frame boundary, which a long trace may not reach.
@@ -7760,7 +7760,7 @@ void main() {
SplitOversizedRecording();
#if MOBILEGL_BUILD_DISAGGREGATED
if (MG_Config::Transport != MG_Config::TransportMode::Monolith) {
RewindWireDescriptorSetsIfDue();
if (!RewindWireDescriptorSetsIfDue()) return;
m_textureManager->CollectGarbage();
DispatchWireCompute(numGroupsX, numGroupsY, numGroupsZ);
return;
@@ -7823,7 +7823,7 @@ void main() {
#if MOBILEGL_BUILD_DISAGGREGATED
// P8-SV: the wire arm's own indirect dispatch, DispatchCompute's shape one call over.
if (MG_Config::Transport != MG_Config::TransportMode::Monolith) {
RewindWireDescriptorSetsIfDue();
if (!RewindWireDescriptorSetsIfDue()) return;
m_textureManager->CollectGarbage();
DispatchWireComputeIndirect(indirect);
return;
@@ -10614,7 +10614,10 @@ void main() {
#if MOBILEGL_BUILD_DISAGGREGATED
const Bool wire = MG_Config::Transport != MG_Config::TransportMode::Monolith;
MG_Pipe::MGPipeHandle dstStorageHandle = dstEndpoint.TextureHandle;
if (wire && HasPendingRecordedWork() && !FlushPendingCommands()) MagmaWireFatal("copy-image-flush");
if (wire && HasPendingRecordedWork() && !FlushPendingCommands()) {
if (LatchWireDeviceLoss("copy-image-flush")) return;
MagmaWireFatal("copy-image-flush");
}
#endif
MOBILEGL_ASSERT(srcEndpoint.Exists() && dstEndpoint.Exists(),
"CopyImageSubData requires valid source and destination images.");
@@ -11169,6 +11172,7 @@ void main() {
VkResult result = vkWaitForFences(m_device, 1, &frame.imageInFlightFence, VK_TRUE, UINT64_MAX);
#endif
if (result != VK_SUCCESS) {
NoteDeviceLoss(result, "readback vkWaitForFences");
MGLOG_E_ONCE("DirectVulkan readback: vkWaitForFences returned %d", result);
return false;
}
@@ -13749,6 +13753,7 @@ void main() {
// the in-flight frame count) but never deadlocks.
const VkResult result = vkQueueWaitIdle(m_graphicsQueue);
if (result != VK_SUCCESS) {
NoteDeviceLoss(result, "WaitForFrameSerial vkQueueWaitIdle");
MGLOG_E_ONCE("WaitForFrameSerial: vkQueueWaitIdle returned %d", result);
return false;
}
@@ -13784,24 +13789,29 @@ void main() {
}
#if MOBILEGL_BUILD_DISAGGREGATED
void VulkanRenderer::RewindWireDescriptorSetsIfDue() {
if (!m_uniformManager || m_frameContext.GetFrameCount() == 0) return;
Bool VulkanRenderer::RewindWireDescriptorSetsIfDue() {
if (!m_uniformManager || m_frameContext.GetFrameCount() == 0) return true;
const Uint32 frameIndex = m_frameContext.GetCurrentFrameIndex();
if (!m_uniformManager->WireDescriptorSetBudgetReached(frameIndex)) return;
if (!m_uniformManager->WireDescriptorSetBudgetReached(frameIndex)) return true;
// A set cannot be rewritten while a recorded or submitted command
// buffer might still read it. This wire-only boundary first submits
// the current recording, then idles the graphics queue before reuse.
if (HasPendingRecordedWork() && !FlushPendingCommands())
if (HasPendingRecordedWork() && !FlushPendingCommands()) {
if (LatchWireDeviceLoss("descriptor-rewind-flush")) return false;
MagmaWireFatal("descriptor-rewind-flush");
}
const Uint64 through = m_submitCounter;
if (through > m_completedSubmitCounter) {
// B4 has not yet made every old retirement site aggregate-safe.
// Queue idle proves even a submission a previous single-fence path
// prematurely removed from m_inFlightSubmits has finished; only
// then may any layout cursor be rewound and its sets rewritten.
if (vkQueueWaitIdle(m_graphicsQueue) != VK_SUCCESS)
if (const VkResult idle = vkQueueWaitIdle(m_graphicsQueue); idle != VK_SUCCESS) {
NoteDeviceLoss(idle, "descriptor rewind vkQueueWaitIdle");
if (LatchWireDeviceLoss("descriptor-rewind-wait")) return false;
MagmaWireFatal("descriptor-rewind-wait");
}
OnSubmitsCompletedUpTo(through);
}
if (HasPendingRecordedWork() || m_completedSubmitCounter < through)
@@ -13810,6 +13820,7 @@ void main() {
MGLOG_I("Magma descriptor rewind: frame=%u retiredSubmit=%llu budget=%u cached=%zu",
frameIndex, static_cast<unsigned long long>(through),
UniformManager::kWireDescriptorSetBudget, cachedSets);
return true;
}
#endif
@@ -13859,6 +13870,7 @@ void main() {
OnSubmitsCompletedUpTo(submitIndex);
return true;
}
NoteDeviceLoss(result, "WaitForSubmitsUpTo vkWaitForFences");
if (result != VK_TIMEOUT)
MGLOG_E_ONCE("WaitForSubmitsUpTo: vkWaitForFences returned %d", result);
return false;
@@ -14157,6 +14169,7 @@ void main() {
#endif
const VkResult result = vkQueueSubmit(m_graphicsQueue, 1, &submitInfo, fence);
if (result != VK_SUCCESS) {
NoteDeviceLoss(result, "SubmitPendingCommandBuffer vkQueueSubmit");
MGLOG_E_ONCE("SubmitPendingCommandBuffer: vkQueueSubmit returned %d", result);
return false;
}
@@ -14248,6 +14261,45 @@ void main() {
return true;
}
void VulkanRenderer::NoteDeviceLoss(VkResult result, const char* where) {
if (result != VK_ERROR_DEVICE_LOST || m_deviceLost) return;
m_deviceLost = true;
MGLOG_E("DirectVulkan: the VkDevice is LOST - %s returned VK_ERROR_DEVICE_LOST (the driver reset a GPU "
"fault or hang); nothing this renderer recorded or submits from here on will execute",
where != nullptr ? where : "a Vulkan call");
#if MOBILEGL_BUILD_DISAGGREGATED
// Latch NOW where a session latch exists, so the session stops at its next record however
// the verb in hand unwinds - including through a caller that only logs a failed submit.
if (MG_Pipe::MGPipeSessionLatchArmed()) (void)LatchWireDeviceLoss(where);
#endif
}
Bool VulkanRenderer::IsDeviceLost() {
// Only failure paths ask, so the probe's wait never lands on a healthy frame: a device
// that is merely slow answers VK_SUCCESS once its work drains, a lost one answers at once.
if (!m_deviceLost && m_device != VK_NULL_HANDLE) {
NoteDeviceLoss(vkDeviceWaitIdle(m_device), "the device-loss probe's vkDeviceWaitIdle");
}
return m_deviceLost;
}
#if MOBILEGL_BUILD_DISAGGREGATED
Bool VulkanRenderer::LatchWireDeviceLoss(const char* site) {
if (!IsDeviceLost()) return false;
if (!m_deviceLossLatched) {
m_deviceLossLatched = true;
// Returns (latched) on a served session; dies, as the site's own Fatal would have,
// where no session latch is armed.
(void)MG_Pipe::MGPipeSessionLatch(MG_Pipe::MGPipeFatalFamily::DeviceLost,
"MGPipe: Fatal{BackendDeviceLost, \"Magma:%s\"} - this session's "
"VkDevice was lost to a GPU fault or hang; the session ends and its "
"client reads a lost context, every other session keeps running",
site != nullptr ? site : "?");
}
return true;
}
#endif
Bool VulkanRenderer::FlushForSyncPoint(Uint64 submitIndex) {
// A flush only helps a sync point whose commands are not submitted
// yet; for an already-submitted index it would just split the frame's
@@ -14300,6 +14352,7 @@ void main() {
}
const VkResult waitResult = vkWaitForFences(m_device, 1, &oldest.fence, VK_TRUE, UINT64_MAX);
if (waitResult != VK_SUCCESS) {
NoteDeviceLoss(waitResult, "SplitOversizedRecording vkWaitForFences");
MGLOG_E_ONCE("SplitOversizedRecording: vkWaitForFences returned %d", waitResult);
break;
}
@@ -14465,6 +14518,8 @@ void main() {
void VulkanRenderer::Present() {
#if MOBILEGL_BUILD_DISAGGREGATED
// A lost device presents nothing (its session is latched and closing).
if (m_deviceLost && LatchWireDeviceLoss("present")) return;
// Present has its own submit path. Tag its last wire pass before that
// submission, rather than retaining it until a draw in the next frame.
RetireWireDrawPass();
@@ -14507,8 +14562,10 @@ void main() {
// free its retired command buffers and reset the slot fence.
const Uint64 slotSubmit = m_frameContext.GetCurrent().lastSubmitIndex;
if (slotSubmit > m_completedSubmitCounter &&
!WaitForSubmitsUpTo(slotSubmit, UINT64_MAX))
!WaitForSubmitsUpTo(slotSubmit, UINT64_MAX)) {
if (LatchWireDeviceLoss("deferred-acquire-submit-wait")) return;
MagmaWireFatal("deferred-acquire-submit-wait");
}
#endif
const VkResult acquireResult =
m_frameContext.WaitAndAcquireNextImage(m_device, m_swapchainObject.GetHandle(), m_imageIndexAcquired);
@@ -14591,7 +14648,12 @@ void main() {
SharedImageSubmitSync sharedImageSync;
AttachSharedImageSync(submitPacket.submitInfo, sharedImageSync);
#endif
VK_VERIFY(vkQueueSubmit(m_graphicsQueue, 1, &submitPacket.submitInfo, frame.imageInFlightFence));
const VkResult presentSubmit = vkQueueSubmit(m_graphicsQueue, 1, &submitPacket.submitInfo, frame.imageInFlightFence);
NoteDeviceLoss(presentSubmit, "Present vkQueueSubmit");
#if MOBILEGL_BUILD_DISAGGREGATED
if (presentSubmit != VK_SUCCESS && LatchWireDeviceLoss("present-submit")) return;
#endif
VK_VERIFY(presentSubmit);
RegisterSubmit(frame.imageInFlightFence, /*pooledFence=*/false);
frame.lastSubmitIndex = m_submitCounter;
#if MOBILEGL_BUILD_DISAGGREGATED
@@ -14627,6 +14689,10 @@ void main() {
m_swapchainResizeRequested = false;
result = VK_SUCCESS;
}
NoteDeviceLoss(result, "vkQueuePresentKHR");
#if MOBILEGL_BUILD_DISAGGREGATED
if (result != VK_SUCCESS && LatchWireDeviceLoss("present-queue")) return;
#endif
VK_VERIFY(result, "Present, vkQueuePresentKHR");
// EGL swap semantics: the presented color buffer's content is undefined the
// next time this image is acquired (EGL_BUFFER_DESTROYED, the default swap
@@ -14677,8 +14743,10 @@ void main() {
// slot's prefix BEFORE the reset/free happens.
const Uint64 slotSubmit = m_frameContext.GetCurrent().lastSubmitIndex;
if (slotSubmit > m_completedSubmitCounter &&
!WaitForSubmitsUpTo(slotSubmit, UINT64_MAX))
!WaitForSubmitsUpTo(slotSubmit, UINT64_MAX)) {
if (LatchWireDeviceLoss("present-slot-submit-wait")) return;
MagmaWireFatal("present-slot-submit-wait");
}
#endif
// 4) Wait/reset/acquire for next frame.
result = m_frameContext.WaitAndAcquireNextImage(m_device, m_swapchainObject.GetHandle(), m_imageIndexAcquired);
@@ -14703,6 +14771,10 @@ void main() {
result =
m_frameContext.WaitAndAcquireNextImage(m_device, m_swapchainObject.GetHandle(), m_imageIndexAcquired);
}
NoteDeviceLoss(result, "vkAcquireNextImageKHR");
#if MOBILEGL_BUILD_DISAGGREGATED
if (result != VK_SUCCESS && LatchWireDeviceLoss("present-acquire")) return;
#endif
VK_VERIFY(result, "Present, vkAcquireNextImageKHR");
// Pull keeps its historical slot-fence inference. In a disaggregated
// build the aggregate wait above already retired the registered prefix
@@ -376,6 +376,22 @@ namespace MobileGL::MG_Backend::DirectVulkan {
// one is retired until the frame slot's fence is next waited. Returns
// true when a submission was made.
Bool FlushPendingCommands();
// DEVICE LOSS. A GPU fault or hang the driver reset leaves this renderer's VkDevice lost:
// every later submit and wait answers VK_ERROR_DEVICE_LOST. NoteDeviceLoss records it at
// the submit/wait sites (cheap, any result); IsDeviceLost answers from the record and,
// when nothing was recorded, asks the device once (only failure paths call it, so the
// probe's wait costs nothing on a healthy frame). Each session owns its own VkDevice, so a
// loss here is this session's alone.
void NoteDeviceLoss(VkResult result, const char* where);
Bool IsDeviceLost();
#if MOBILEGL_BUILD_DISAGGREGATED
// For a wire site whose submit or wait just failed: true when the failure is a lost
// device, in which case this session has been latched (MGPipeSessionLatch - it ends, the
// display server's other sessions do not) and the caller must return from its verb
// without recording or submitting anything more. False: an ordinary failure, the site
// keeps the death it had. `site` names it in the latch line.
Bool LatchWireDeviceLoss(const char* site);
#endif
// Flush gated on usefulness: only flushes when `submitIndex` is still
// unsubmitted, so poll loops on already-submitted fences do not split
// the frame's render pass (a full tile load/store on TBDR GPUs).
@@ -589,6 +605,9 @@ namespace MobileGL::MG_Backend::DirectVulkan {
};
void QueueClearBufferPayload(GLenum buffer, GLint drawbuffer, const ClearAttachmentPayload& clearPayload);
// See NoteDeviceLoss. Set once, never cleared: a lost VkDevice stays lost.
Bool m_deviceLost = false;
Bool m_deviceLossLatched = false;
#if MOBILEGL_BUILD_DISAGGREGATED
struct WireImage {
VkImage image = VK_NULL_HANDLE;
@@ -843,7 +862,8 @@ namespace MobileGL::MG_Backend::DirectVulkan {
// P8-SV: glDispatchComputeIndirect on the wire arm - vkCmdDispatchIndirect from the
// dispatch-indirect store, as the monolith arm issues it (WireDraw.inc).
void DispatchWireComputeIndirect(GLintptr offset);
void RewindWireDescriptorSetsIfDue();
// False: the device was lost and the session latched; the caller drops its draw/dispatch.
Bool RewindWireDescriptorSetsIfDue();
UniquePtr<RenderPassEntry> m_wireDrawPass;
Vector<VkImageView> m_wireDrawViews;
VkPipeline GetOrCreatePipelineWithInput(GLenum mode, const MagmaProgramSource& program,
@@ -381,7 +381,13 @@
void VulkanRenderer::ReadWirePixels(GLint x, GLint y, GLsizei width, GLsizei height,
GLenum format, GLenum type, void* pixels) {
if (width <= 0 || height <= 0) return;
if (HasPendingRecordedWork() && !FlushPendingCommands()) MagmaWireFatal("wire-verb-flush");
if (MG_Pipe::MGPipeDebugDeviceLossDue()) NoteDeviceLoss(VK_ERROR_DEVICE_LOST, "the debug knob (read-pixels)");
if (m_deviceLost && LatchWireDeviceLoss("read-pixels")) return;
if (HasPendingRecordedWork() && !FlushPendingCommands()) {
// A lost device: the session is latched and closing, the pixels stay unfilled.
if (LatchWireDeviceLoss("wire-verb-flush")) return;
MagmaWireFatal("wire-verb-flush");
}
const auto* fbo = MG_Pipe::MGPipeApplier().ReadFramebuffer();
if (!fbo) MagmaWireFatal("read-framebuffer-record");
VkImageAspectFlags aspect = format == GL_DEPTH_COMPONENT ? VK_IMAGE_ASPECT_DEPTH_BIT :
@@ -446,7 +452,10 @@
auto& frame = m_frameContext.GetCurrent();
vkCmdCopyImageToBuffer(frame.commandBuffer,source.image,VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
buffer.GetHandle(),1,&region);
if (!SubmitReadbackCommandsAndWait(frame)) MagmaWireFatal("read-submit");
if (!SubmitReadbackCommandsAndWait(frame)) {
if (LatchWireDeviceLoss("read-submit")) return;
MagmaWireFatal("read-submit");
}
const auto* bytes = static_cast<const Uint8*>(buffer.Map());
if (!bytes || !buffer.Invalidate(size)) MagmaWireFatal("read-map");
Vector<Uint8> remapped;
@@ -72,6 +72,9 @@
Bool VulkanRenderer::BlitDefaultFramebufferToSharedImage(const SharedImageView& view) {
if (m_device == VK_NULL_HANDLE || view.Width == 0 || view.Height == 0) return false;
// The debug knob's device check (MGPipeDebugDeviceLossDue) rides the frame boundary here.
if (MG_Pipe::MGPipeDebugDeviceLossDue()) NoteDeviceLoss(VK_ERROR_DEVICE_LOST, "the debug knob (shared-image present)");
if (m_deviceLost && LatchWireDeviceLoss("shared-image-present")) return false;
DestroySharedImagePresentTargets(true);
// The default framebuffer's image is stored in display orientation (row 0 is the top of