diff --git a/MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp b/MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp index db2e7335..f302f5ff 100644 --- a/MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp +++ b/MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp @@ -34,6 +34,8 @@ #include // Shared images: the registry the EGLImages are built from (SharedImageImpl below). #include +// Device loss ends the session through MG_Pipe's latch seam (LatchIfDeviceLost). +#include #if defined(__ANDROID__) #include #include @@ -15606,6 +15608,10 @@ namespace MobileGL::MG_Backend::DirectGLES { void ReadPixels(GLint x, GLint y, GLsizei width, GLsizei height, GLenum format, GLenum type, void* pixels) { MGLOG_D("ReadPixels: x=%d y=%d w=%d h=%d format=%s type=%s pixels=%p", x, y, width, height, MG_Util::ConvertGLEnumToString(format).c_str(), MG_Util::ConvertGLEnumToString(type).c_str(), pixels); +#if MOBILEGL_BUILD_DISAGGREGATED + // A reset context reads back nothing it rendered: the session is latched and the read declined. + if (LatchIfDeviceLost("read-pixels")) return; +#endif // Combinations the ES driver has always handled directly keep the native path; other color layouts go // through the wide-format conversion path. Anything still uncovered degrades to a logged no-op instead @@ -18034,7 +18040,33 @@ namespace MobileGL::MG_Backend::DirectGLES { return CompletedFrameSerial() >= serial; } +#if MOBILEGL_BUILD_DISAGGREGATED + Bool LatchIfDeviceLost(const char* site, Bool debugKnob) { + if (MG_Config::Transport == MG_Config::TransportMode::Monolith) return false; + // Once per thread: a latched session declines everything after this verb anyway, and the + // apply thread is the session's own. + static thread_local Bool latched = false; + if (latched) return true; + GLenum status = GL_NO_ERROR; + if (g_GLESFuncs.glGetGraphicsResetStatus != nullptr && IsBackendContextCurrentOnThisThread()) { + status = g_GLESFuncs.glGetGraphicsResetStatus(); + } + if (status == GL_NO_ERROR && debugKnob && MG_Pipe::MGPipeDebugDeviceLossDue()) status = GL_UNKNOWN_CONTEXT_RESET; + if (status == GL_NO_ERROR) return false; + latched = true; + (void)MG_Pipe::MGPipeSessionLatch(MG_Pipe::MGPipeFatalFamily::DeviceLost, + "MGPipe: Fatal{BackendDeviceLost, \"Espryt:%s\"} - this session's EGL " + "context was reset (glGetGraphicsResetStatus 0x%x); the session ends " + "and its client reads a lost context, every other session keeps running", + site != nullptr ? site : "?", static_cast(status)); + return true; + } +#endif + void Present() { +#if MOBILEGL_BUILD_DISAGGREGATED + if (LatchIfDeviceLost("present", /*debugKnob=*/false)) return; +#endif // Insert one fence per frame BEFORE the swap (eglSwapBuffers' implicit flush // makes it reachable), then non-blocking-poll prior frames' fences AFTER to // advance the completed-frame watermark that gates buffer-pool recycling. @@ -18636,6 +18668,7 @@ namespace MobileGL::MG_Backend::DirectGLES { MGLOG_E_ONCE("Shared image present: no backend context is current on the calling thread"); return false; } + if (LatchIfDeviceLost("shared-image-present")) return false; const EGLSurface surface = CurrentContextDrawSurface(); EGLint sourceWidth = 0; EGLint sourceHeight = 0; diff --git a/MobileGL/MG_Backend/DirectGLES/DirectGLES.h b/MobileGL/MG_Backend/DirectGLES/DirectGLES.h index 3d548840..c82e2d99 100644 --- a/MobileGL/MG_Backend/DirectGLES/DirectGLES.h +++ b/MobileGL/MG_Backend/DirectGLES/DirectGLES.h @@ -147,6 +147,19 @@ namespace MobileGL::MG_Backend::DirectGLES { // True when the backend ES context is current on the calling thread, i.e. // immediate buffer ops may issue GL calls right now. Bool IsBackendContextCurrentOnThisThread(); +#if MOBILEGL_BUILD_DISAGGREGATED + // DEVICE LOSS, Espryt's half (Magma's is VulkanRenderer::LatchWireDeviceLoss). Asks the + // driver whether the calling thread's context was reset (glGetGraphicsResetStatus); a reset - + // or the debug knob MGPipeDebugDeviceLossDue - latches THIS session through + // MGPipeSessionLatch and answers true, and the caller returns from its verb without issuing + // more GL. Asked at the frame boundaries (Present, a shared-image present) and before a + // readback, never per draw. The contexts keep the driver's default reset strategy: asking + // every context to lose itself on reset would also end the INNOCENT ones - the compositor's + // among them - on any other client's GPU fault. + // `debugKnob` false: the knob is not consulted here (the on-screen Present, so an injected loss + // lands on an offscreen client's shared-image present or readback, never on the compositor's). + Bool LatchIfDeviceLost(const char* site, Bool debugKnob = true); +#endif // GL fence sync objects, backed by native ES fences. FenceSync returns null // (the frontend then falls back to an always-signaled sync) when the calling // thread does not own the ES context. Waits/queries degrade to "signaled" in diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp index 96d96fe6..f8008728 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp @@ -425,7 +425,10 @@ namespace MobileGL::MG_Backend::DirectVulkan { uploaded = StagedWireRangeCopy(*resource, bytes, static_cast(offset), static_cast(size)); } - if (!uploaded && !WaitForWireBufferHostAccess(*resource)) WireBufferSyncFatal("host-write"); + if (!uploaded && !WaitForWireBufferHostAccess(*resource)) { + if (LatchedOnDeviceLoss("buffer-host-write")) return; + WireBufferSyncFatal("host-write"); + } } if (!uploaded) { uploaded = resource->buffer.Upload(bytes, size, offset); @@ -434,6 +437,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { } } if (!uploaded) { + if (LatchedOnDeviceLoss("buffer-upload")) return; MGLOG_F("Magma: Fatal{ResourceUnavailable, \"buffer-upload\"}"); std::abort(); } @@ -604,6 +608,8 @@ namespace MobileGL::MG_Backend::DirectVulkan { const Bool t0 = resource != nullptr && resource->imported; Vector bytes(t0 ? 0 : static_cast(size)); if (t0 ? !WaitForWireBufferHostAccess(*resource) : !ReadWireBuffer(res, offset, size, bytes.data())) { + // A lost device: the session is latched and closing; its client reads a lost context. + if (LatchedOnDeviceLoss("buffer-readback")) return; MGLOG_F("Magma: Fatal{ResourceUnavailable, \"buffer-readback\"} {slot=%u, gen=%u}", res.Slot, res.Gen); std::abort(); } @@ -1074,6 +1080,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { // budget's worth of orphans, not one per glBufferData. if (pVulkanRenderer == nullptr || !pVulkanRenderer->WaitForSubmitIndex(pVulkanRenderer->GetSyncPointSubmitIndex(), UINT64_MAX, true)) { + if (LatchedOnDeviceLoss("buffer-deferred-watermark")) return; WireBufferSyncFatal("deferred-watermark"); } ++m_wireDeferredSyncs; @@ -1094,6 +1101,10 @@ namespace MobileGL::MG_Backend::DirectVulkan { MG_Util::PipeStats::PublishGauge(Gauge::WireDeferredSyncs, m_wireDeferredSyncs); } + Bool VkBufferManager::LatchedOnDeviceLoss(const char* site) { + return pVulkanRenderer != nullptr && pVulkanRenderer->LatchWireDeviceLoss(site); + } + void VkBufferManager::WireBufferSyncFatal(const char* site) { MGLOG_F("Magma: Fatal{ResourceUnavailable, \"buffer-write-sync\"} {site=%s}", site); std::abort(); diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.h index e6ac279a..eb0cc02a 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.h @@ -362,6 +362,10 @@ namespace MobileGL::MG_Backend::DirectVulkan { // for the census to count), shared by the host write that cannot wait for the GPU and // the watermark that cannot. `site` says which. [[noreturn]] static void WireBufferSyncFatal(const char* site); + // True when the wait or copy that just failed failed because the device is lost: the + // session has been latched (VulkanRenderer::LatchWireDeviceLoss) and the caller returns + // instead of taking its Fatal. + static Bool LatchedOnDeviceLoss(const char* site); // The wbuf[] gauges (PipeStats.h, Gauge::WireBuffers..WireDeferredSyncs). The peaks are // taken at the two points the numbers can rise - a park and a mint - and published when // the stats channel is on; MagmaWireReclaimScenario reads them off the server's line. diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp index c9b22a3a..dcb809d1 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp @@ -7123,7 +7123,7 @@ void main() { SplitOversizedRecording(); #if MOBILEGL_BUILD_DISAGGREGATED if (MG_Config::Transport != MG_Config::TransportMode::Monolith) { - RewindWireDescriptorSetsIfDue(); + if (!RewindWireDescriptorSetsIfDue()) return false; // The wire route returns before the monolith branch's draw-gated // sweep. Dead wire texture/renderbuffer records otherwise live // until the next frame boundary, which a long trace may not reach. @@ -7760,7 +7760,7 @@ void main() { SplitOversizedRecording(); #if MOBILEGL_BUILD_DISAGGREGATED if (MG_Config::Transport != MG_Config::TransportMode::Monolith) { - RewindWireDescriptorSetsIfDue(); + if (!RewindWireDescriptorSetsIfDue()) return; m_textureManager->CollectGarbage(); DispatchWireCompute(numGroupsX, numGroupsY, numGroupsZ); return; @@ -7823,7 +7823,7 @@ void main() { #if MOBILEGL_BUILD_DISAGGREGATED // P8-SV: the wire arm's own indirect dispatch, DispatchCompute's shape one call over. if (MG_Config::Transport != MG_Config::TransportMode::Monolith) { - RewindWireDescriptorSetsIfDue(); + if (!RewindWireDescriptorSetsIfDue()) return; m_textureManager->CollectGarbage(); DispatchWireComputeIndirect(indirect); return; @@ -10614,7 +10614,10 @@ void main() { #if MOBILEGL_BUILD_DISAGGREGATED const Bool wire = MG_Config::Transport != MG_Config::TransportMode::Monolith; MG_Pipe::MGPipeHandle dstStorageHandle = dstEndpoint.TextureHandle; - if (wire && HasPendingRecordedWork() && !FlushPendingCommands()) MagmaWireFatal("copy-image-flush"); + if (wire && HasPendingRecordedWork() && !FlushPendingCommands()) { + if (LatchWireDeviceLoss("copy-image-flush")) return; + MagmaWireFatal("copy-image-flush"); + } #endif MOBILEGL_ASSERT(srcEndpoint.Exists() && dstEndpoint.Exists(), "CopyImageSubData requires valid source and destination images."); @@ -11169,6 +11172,7 @@ void main() { VkResult result = vkWaitForFences(m_device, 1, &frame.imageInFlightFence, VK_TRUE, UINT64_MAX); #endif if (result != VK_SUCCESS) { + NoteDeviceLoss(result, "readback vkWaitForFences"); MGLOG_E_ONCE("DirectVulkan readback: vkWaitForFences returned %d", result); return false; } @@ -13749,6 +13753,7 @@ void main() { // the in-flight frame count) but never deadlocks. const VkResult result = vkQueueWaitIdle(m_graphicsQueue); if (result != VK_SUCCESS) { + NoteDeviceLoss(result, "WaitForFrameSerial vkQueueWaitIdle"); MGLOG_E_ONCE("WaitForFrameSerial: vkQueueWaitIdle returned %d", result); return false; } @@ -13784,24 +13789,29 @@ void main() { } #if MOBILEGL_BUILD_DISAGGREGATED - void VulkanRenderer::RewindWireDescriptorSetsIfDue() { - if (!m_uniformManager || m_frameContext.GetFrameCount() == 0) return; + Bool VulkanRenderer::RewindWireDescriptorSetsIfDue() { + if (!m_uniformManager || m_frameContext.GetFrameCount() == 0) return true; const Uint32 frameIndex = m_frameContext.GetCurrentFrameIndex(); - if (!m_uniformManager->WireDescriptorSetBudgetReached(frameIndex)) return; + if (!m_uniformManager->WireDescriptorSetBudgetReached(frameIndex)) return true; // A set cannot be rewritten while a recorded or submitted command // buffer might still read it. This wire-only boundary first submits // the current recording, then idles the graphics queue before reuse. - if (HasPendingRecordedWork() && !FlushPendingCommands()) + if (HasPendingRecordedWork() && !FlushPendingCommands()) { + if (LatchWireDeviceLoss("descriptor-rewind-flush")) return false; MagmaWireFatal("descriptor-rewind-flush"); + } const Uint64 through = m_submitCounter; if (through > m_completedSubmitCounter) { // B4 has not yet made every old retirement site aggregate-safe. // Queue idle proves even a submission a previous single-fence path // prematurely removed from m_inFlightSubmits has finished; only // then may any layout cursor be rewound and its sets rewritten. - if (vkQueueWaitIdle(m_graphicsQueue) != VK_SUCCESS) + if (const VkResult idle = vkQueueWaitIdle(m_graphicsQueue); idle != VK_SUCCESS) { + NoteDeviceLoss(idle, "descriptor rewind vkQueueWaitIdle"); + if (LatchWireDeviceLoss("descriptor-rewind-wait")) return false; MagmaWireFatal("descriptor-rewind-wait"); + } OnSubmitsCompletedUpTo(through); } if (HasPendingRecordedWork() || m_completedSubmitCounter < through) @@ -13810,6 +13820,7 @@ void main() { MGLOG_I("Magma descriptor rewind: frame=%u retiredSubmit=%llu budget=%u cached=%zu", frameIndex, static_cast(through), UniformManager::kWireDescriptorSetBudget, cachedSets); + return true; } #endif @@ -13859,6 +13870,7 @@ void main() { OnSubmitsCompletedUpTo(submitIndex); return true; } + NoteDeviceLoss(result, "WaitForSubmitsUpTo vkWaitForFences"); if (result != VK_TIMEOUT) MGLOG_E_ONCE("WaitForSubmitsUpTo: vkWaitForFences returned %d", result); return false; @@ -14157,6 +14169,7 @@ void main() { #endif const VkResult result = vkQueueSubmit(m_graphicsQueue, 1, &submitInfo, fence); if (result != VK_SUCCESS) { + NoteDeviceLoss(result, "SubmitPendingCommandBuffer vkQueueSubmit"); MGLOG_E_ONCE("SubmitPendingCommandBuffer: vkQueueSubmit returned %d", result); return false; } @@ -14248,6 +14261,45 @@ void main() { return true; } + void VulkanRenderer::NoteDeviceLoss(VkResult result, const char* where) { + if (result != VK_ERROR_DEVICE_LOST || m_deviceLost) return; + m_deviceLost = true; + MGLOG_E("DirectVulkan: the VkDevice is LOST - %s returned VK_ERROR_DEVICE_LOST (the driver reset a GPU " + "fault or hang); nothing this renderer recorded or submits from here on will execute", + where != nullptr ? where : "a Vulkan call"); +#if MOBILEGL_BUILD_DISAGGREGATED + // Latch NOW where a session latch exists, so the session stops at its next record however + // the verb in hand unwinds - including through a caller that only logs a failed submit. + if (MG_Pipe::MGPipeSessionLatchArmed()) (void)LatchWireDeviceLoss(where); +#endif + } + + Bool VulkanRenderer::IsDeviceLost() { + // Only failure paths ask, so the probe's wait never lands on a healthy frame: a device + // that is merely slow answers VK_SUCCESS once its work drains, a lost one answers at once. + if (!m_deviceLost && m_device != VK_NULL_HANDLE) { + NoteDeviceLoss(vkDeviceWaitIdle(m_device), "the device-loss probe's vkDeviceWaitIdle"); + } + return m_deviceLost; + } + +#if MOBILEGL_BUILD_DISAGGREGATED + Bool VulkanRenderer::LatchWireDeviceLoss(const char* site) { + if (!IsDeviceLost()) return false; + if (!m_deviceLossLatched) { + m_deviceLossLatched = true; + // Returns (latched) on a served session; dies, as the site's own Fatal would have, + // where no session latch is armed. + (void)MG_Pipe::MGPipeSessionLatch(MG_Pipe::MGPipeFatalFamily::DeviceLost, + "MGPipe: Fatal{BackendDeviceLost, \"Magma:%s\"} - this session's " + "VkDevice was lost to a GPU fault or hang; the session ends and its " + "client reads a lost context, every other session keeps running", + site != nullptr ? site : "?"); + } + return true; + } +#endif + Bool VulkanRenderer::FlushForSyncPoint(Uint64 submitIndex) { // A flush only helps a sync point whose commands are not submitted // yet; for an already-submitted index it would just split the frame's @@ -14300,6 +14352,7 @@ void main() { } const VkResult waitResult = vkWaitForFences(m_device, 1, &oldest.fence, VK_TRUE, UINT64_MAX); if (waitResult != VK_SUCCESS) { + NoteDeviceLoss(waitResult, "SplitOversizedRecording vkWaitForFences"); MGLOG_E_ONCE("SplitOversizedRecording: vkWaitForFences returned %d", waitResult); break; } @@ -14465,6 +14518,8 @@ void main() { void VulkanRenderer::Present() { #if MOBILEGL_BUILD_DISAGGREGATED + // A lost device presents nothing (its session is latched and closing). + if (m_deviceLost && LatchWireDeviceLoss("present")) return; // Present has its own submit path. Tag its last wire pass before that // submission, rather than retaining it until a draw in the next frame. RetireWireDrawPass(); @@ -14507,8 +14562,10 @@ void main() { // free its retired command buffers and reset the slot fence. const Uint64 slotSubmit = m_frameContext.GetCurrent().lastSubmitIndex; if (slotSubmit > m_completedSubmitCounter && - !WaitForSubmitsUpTo(slotSubmit, UINT64_MAX)) + !WaitForSubmitsUpTo(slotSubmit, UINT64_MAX)) { + if (LatchWireDeviceLoss("deferred-acquire-submit-wait")) return; MagmaWireFatal("deferred-acquire-submit-wait"); + } #endif const VkResult acquireResult = m_frameContext.WaitAndAcquireNextImage(m_device, m_swapchainObject.GetHandle(), m_imageIndexAcquired); @@ -14591,7 +14648,12 @@ void main() { SharedImageSubmitSync sharedImageSync; AttachSharedImageSync(submitPacket.submitInfo, sharedImageSync); #endif - VK_VERIFY(vkQueueSubmit(m_graphicsQueue, 1, &submitPacket.submitInfo, frame.imageInFlightFence)); + const VkResult presentSubmit = vkQueueSubmit(m_graphicsQueue, 1, &submitPacket.submitInfo, frame.imageInFlightFence); + NoteDeviceLoss(presentSubmit, "Present vkQueueSubmit"); +#if MOBILEGL_BUILD_DISAGGREGATED + if (presentSubmit != VK_SUCCESS && LatchWireDeviceLoss("present-submit")) return; +#endif + VK_VERIFY(presentSubmit); RegisterSubmit(frame.imageInFlightFence, /*pooledFence=*/false); frame.lastSubmitIndex = m_submitCounter; #if MOBILEGL_BUILD_DISAGGREGATED @@ -14627,6 +14689,10 @@ void main() { m_swapchainResizeRequested = false; result = VK_SUCCESS; } + NoteDeviceLoss(result, "vkQueuePresentKHR"); +#if MOBILEGL_BUILD_DISAGGREGATED + if (result != VK_SUCCESS && LatchWireDeviceLoss("present-queue")) return; +#endif VK_VERIFY(result, "Present, vkQueuePresentKHR"); // EGL swap semantics: the presented color buffer's content is undefined the // next time this image is acquired (EGL_BUFFER_DESTROYED, the default swap @@ -14677,8 +14743,10 @@ void main() { // slot's prefix BEFORE the reset/free happens. const Uint64 slotSubmit = m_frameContext.GetCurrent().lastSubmitIndex; if (slotSubmit > m_completedSubmitCounter && - !WaitForSubmitsUpTo(slotSubmit, UINT64_MAX)) + !WaitForSubmitsUpTo(slotSubmit, UINT64_MAX)) { + if (LatchWireDeviceLoss("present-slot-submit-wait")) return; MagmaWireFatal("present-slot-submit-wait"); + } #endif // 4) Wait/reset/acquire for next frame. result = m_frameContext.WaitAndAcquireNextImage(m_device, m_swapchainObject.GetHandle(), m_imageIndexAcquired); @@ -14703,6 +14771,10 @@ void main() { result = m_frameContext.WaitAndAcquireNextImage(m_device, m_swapchainObject.GetHandle(), m_imageIndexAcquired); } + NoteDeviceLoss(result, "vkAcquireNextImageKHR"); +#if MOBILEGL_BUILD_DISAGGREGATED + if (result != VK_SUCCESS && LatchWireDeviceLoss("present-acquire")) return; +#endif VK_VERIFY(result, "Present, vkAcquireNextImageKHR"); // Pull keeps its historical slot-fence inference. In a disaggregated // build the aggregate wait above already retired the registered prefix diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h index 5e1c7062..8502f166 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h @@ -376,6 +376,22 @@ namespace MobileGL::MG_Backend::DirectVulkan { // one is retired until the frame slot's fence is next waited. Returns // true when a submission was made. Bool FlushPendingCommands(); + // DEVICE LOSS. A GPU fault or hang the driver reset leaves this renderer's VkDevice lost: + // every later submit and wait answers VK_ERROR_DEVICE_LOST. NoteDeviceLoss records it at + // the submit/wait sites (cheap, any result); IsDeviceLost answers from the record and, + // when nothing was recorded, asks the device once (only failure paths call it, so the + // probe's wait costs nothing on a healthy frame). Each session owns its own VkDevice, so a + // loss here is this session's alone. + void NoteDeviceLoss(VkResult result, const char* where); + Bool IsDeviceLost(); +#if MOBILEGL_BUILD_DISAGGREGATED + // For a wire site whose submit or wait just failed: true when the failure is a lost + // device, in which case this session has been latched (MGPipeSessionLatch - it ends, the + // display server's other sessions do not) and the caller must return from its verb + // without recording or submitting anything more. False: an ordinary failure, the site + // keeps the death it had. `site` names it in the latch line. + Bool LatchWireDeviceLoss(const char* site); +#endif // Flush gated on usefulness: only flushes when `submitIndex` is still // unsubmitted, so poll loops on already-submitted fences do not split // the frame's render pass (a full tile load/store on TBDR GPUs). @@ -589,6 +605,9 @@ namespace MobileGL::MG_Backend::DirectVulkan { }; void QueueClearBufferPayload(GLenum buffer, GLint drawbuffer, const ClearAttachmentPayload& clearPayload); + // See NoteDeviceLoss. Set once, never cleared: a lost VkDevice stays lost. + Bool m_deviceLost = false; + Bool m_deviceLossLatched = false; #if MOBILEGL_BUILD_DISAGGREGATED struct WireImage { VkImage image = VK_NULL_HANDLE; @@ -843,7 +862,8 @@ namespace MobileGL::MG_Backend::DirectVulkan { // P8-SV: glDispatchComputeIndirect on the wire arm - vkCmdDispatchIndirect from the // dispatch-indirect store, as the monolith arm issues it (WireDraw.inc). void DispatchWireComputeIndirect(GLintptr offset); - void RewindWireDescriptorSetsIfDue(); + // False: the device was lost and the session latched; the caller drops its draw/dispatch. + Bool RewindWireDescriptorSetsIfDue(); UniquePtr m_wireDrawPass; Vector m_wireDrawViews; VkPipeline GetOrCreatePipelineWithInput(GLenum mode, const MagmaProgramSource& program, diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireFramebuffer.inc b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireFramebuffer.inc index 46ba6bd4..6e5e645b 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireFramebuffer.inc +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireFramebuffer.inc @@ -381,7 +381,13 @@ void VulkanRenderer::ReadWirePixels(GLint x, GLint y, GLsizei width, GLsizei height, GLenum format, GLenum type, void* pixels) { if (width <= 0 || height <= 0) return; - if (HasPendingRecordedWork() && !FlushPendingCommands()) MagmaWireFatal("wire-verb-flush"); + if (MG_Pipe::MGPipeDebugDeviceLossDue()) NoteDeviceLoss(VK_ERROR_DEVICE_LOST, "the debug knob (read-pixels)"); + if (m_deviceLost && LatchWireDeviceLoss("read-pixels")) return; + if (HasPendingRecordedWork() && !FlushPendingCommands()) { + // A lost device: the session is latched and closing, the pixels stay unfilled. + if (LatchWireDeviceLoss("wire-verb-flush")) return; + MagmaWireFatal("wire-verb-flush"); + } const auto* fbo = MG_Pipe::MGPipeApplier().ReadFramebuffer(); if (!fbo) MagmaWireFatal("read-framebuffer-record"); VkImageAspectFlags aspect = format == GL_DEPTH_COMPONENT ? VK_IMAGE_ASPECT_DEPTH_BIT : @@ -446,7 +452,10 @@ auto& frame = m_frameContext.GetCurrent(); vkCmdCopyImageToBuffer(frame.commandBuffer,source.image,VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, buffer.GetHandle(),1,®ion); - if (!SubmitReadbackCommandsAndWait(frame)) MagmaWireFatal("read-submit"); + if (!SubmitReadbackCommandsAndWait(frame)) { + if (LatchWireDeviceLoss("read-submit")) return; + MagmaWireFatal("read-submit"); + } const auto* bytes = static_cast(buffer.Map()); if (!bytes || !buffer.Invalidate(size)) MagmaWireFatal("read-map"); Vector remapped; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireSharedImage.inc b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireSharedImage.inc index aa187e68..2eb342b5 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireSharedImage.inc +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireSharedImage.inc @@ -72,6 +72,9 @@ Bool VulkanRenderer::BlitDefaultFramebufferToSharedImage(const SharedImageView& view) { if (m_device == VK_NULL_HANDLE || view.Width == 0 || view.Height == 0) return false; + // The debug knob's device check (MGPipeDebugDeviceLossDue) rides the frame boundary here. + if (MG_Pipe::MGPipeDebugDeviceLossDue()) NoteDeviceLoss(VK_ERROR_DEVICE_LOST, "the debug knob (shared-image present)"); + if (m_deviceLost && LatchWireDeviceLoss("shared-image-present")) return false; DestroySharedImagePresentTargets(true); // The default framebuffer's image is stored in display orientation (row 0 is the top of diff --git a/MobileGL/MG_Pipe/PipeApply.cpp b/MobileGL/MG_Pipe/PipeApply.cpp index 7929da73..d6f3a439 100644 --- a/MobileGL/MG_Pipe/PipeApply.cpp +++ b/MobileGL/MG_Pipe/PipeApply.cpp @@ -53,12 +53,16 @@ #endif #include +#include #include #include #include #include #include #include +#if defined(__ANDROID__) +#include +#endif // THE VERDICT OF EVERY TRIP WIRE IN THIS FILE, IN ONE PLACE. // @@ -154,6 +158,58 @@ namespace MobileGL::MG_Pipe { MGLOG_F("%s", line); std::abort(); } + + namespace { + MGPipeSessionLatchHook g_sessionLatchHook = nullptr; + MGPipeSessionLatchArmedHook g_sessionLatchArmedHook = nullptr; + } // namespace + + void MGPipeInstallSessionLatchHook(MGPipeSessionLatchHook hook, MGPipeSessionLatchArmedHook armed) { + g_sessionLatchHook = hook; + g_sessionLatchArmedHook = armed; + } + + MGPipeSessionLatchHook MGPipeSessionLatchHookInstalled() { return g_sessionLatchHook; } + + bool MGPipeSessionLatchArmed() { return g_sessionLatchArmedHook != nullptr && g_sessionLatchArmedHook(); } + + bool MGPipeDebugDeviceLossDue() { + long at = 0; + if (const char* value = std::getenv("MOBILEGL_DEBUG_INJECT_DEVICE_LOST_AT")) { + at = std::strtol(value, nullptr, 10); + } +#if defined(__ANDROID__) + else { + char property[PROP_VALUE_MAX] = {}; + if (__system_property_get("debug.mobilegl.inject_device_lost_at", property) > 0) { + at = std::strtol(property, nullptr, 10); + } + } +#endif + if (at <= 0) return false; + static std::atomic seen{0}; + static std::atomic fired{false}; + if (fired.load(std::memory_order_acquire)) return false; + if (seen.fetch_add(1, std::memory_order_acq_rel) + 1 < at) return false; + if (fired.exchange(true, std::memory_order_acq_rel)) return false; + MGLOG_E("MGPipe: debug knob inject_device_lost_at=%ld - this device check reports the device LOST", at); + return true; + } + + bool MGPipeSessionLatch(MGPipeFatalFamily family, const char* fmt, ...) { + char line[512]; + va_list args; + va_start(args, fmt); + const int written = std::vsnprintf(line, sizeof(line), fmt, args); + va_end(args); + if (written < 0) { + std::snprintf(line, sizeof(line), + "MGPipe: unformattable Fatal diagnostic (format=%s)", fmt); + } + if (g_sessionLatchHook != nullptr) return g_sessionLatchHook(family, line); + // NO HOOK: no session to latch, so the death the fail seam would have given. + MGPipeSessionFail(family, "%s", line); + } #endif namespace { diff --git a/MobileGL/MG_Pipe/PipeSessionFail.h b/MobileGL/MG_Pipe/PipeSessionFail.h index fbd18d68..dd63dd84 100644 --- a/MobileGL/MG_Pipe/PipeSessionFail.h +++ b/MobileGL/MG_Pipe/PipeSessionFail.h @@ -57,6 +57,9 @@ namespace MobileGL::MG_Pipe { #if MOBILEGL_BUILD_DISAGGREGATED // A malformed peer record rejected by the server-side applier. ProtocolCorruption, + // The GPU device a session's backend renders with is lost (a GPU fault or hang the driver + // reset). Raised only through MGPipeSessionLatch below: it ends THAT session. + DeviceLost, #endif }; @@ -82,4 +85,43 @@ namespace MobileGL::MG_Pipe { #endif void MGPipeSessionFail(MGPipeFatalFamily family, const char* fmt, ...); +#if MOBILEGL_BUILD_DISAGGREGATED + // ---- THE LATCH TWIN: A LOST GPU DEVICE ENDS ITS SESSION, NOT THE PROCESS ---------------- + // + // MGPipeSessionFail is for a verb the backend cannot honour, and it does not return. A device + // loss is different in kind: nothing about the peer's bytes was wrong, the GPU faulted or hung + // and the driver reset it, and every session the process serves except the one whose device + // went is still healthy. The display server runs every session in ONE process, so dying here + // takes the compositor and every other client down with the one that lost its device. + // + // So a backend that sees its device lost calls this instead, and then RETURNS from the verb + // it was applying without touching the GPU again. The hook MG_Remote installs latches the + // calling thread's session (SessionLatch): the session declines everything after it, closes, + // and its client reads a lost context. Where no latch is armed - the inproc client+server + // shape, where the process IS the session - the hook dies exactly as MGPipeSessionFail does, + // and with no hook at all this is MGPipeSessionFail. Returns false, so a Bool site can read + // `return MGPipeSessionLatch(...);`. + using MGPipeSessionLatchHook = bool (*)(MGPipeFatalFamily family, const char* line); + // Whether the calling thread's session HAS a latch to raise. A backend that notices its + // device lost outside a site that would die anyway (a submit whose failure it used to only + // log) latches at once when this is true - so the session stops at its next record however + // the current verb unwinds - and keeps its old behaviour when it is false. + using MGPipeSessionLatchArmedHook = bool (*)(); + void MGPipeInstallSessionLatchHook(MGPipeSessionLatchHook hook, MGPipeSessionLatchArmedHook armed); + MGPipeSessionLatchHook MGPipeSessionLatchHookInstalled(); + bool MGPipeSessionLatchArmed(); + bool +#if defined(__GNUC__) || defined(__clang__) + __attribute__((format(printf, 2, 3))) +#endif + MGPipeSessionLatch(MGPipeFatalFamily family, const char* fmt, ...); + + // A DEBUG KNOB THAT MAKES ONE DEVICE CHECK REPORT A LOSS, so the containment above can be + // driven without a GPU that faults on demand. MOBILEGL_DEBUG_INJECT_DEVICE_LOST_AT=N (or, on + // Android, the property debug.mobilegl.inject_device_lost_at) makes the N-th device check the + // backends make after the knob was first seen nonzero answer "lost", once per process. The + // backends ask at their frame boundaries and readbacks; unset (the default) it is one getenv. + bool MGPipeDebugDeviceLossDue(); +#endif + } // namespace MobileGL::MG_Pipe diff --git a/MobileGL/MG_Remote/FatalFamilies.def b/MobileGL/MG_Remote/FatalFamilies.def index 6008a30b..6f1e2fe4 100644 --- a/MobileGL/MG_Remote/FatalFamilies.def +++ b/MobileGL/MG_Remote/FatalFamilies.def @@ -57,6 +57,11 @@ // server-side resource its session had to have is gone - so it reads a clean device loss. Not // DeviceLost: that code is the latch's own, never a family's projection (see its bucket above). // +// BackendDeviceLost: the GPU device (Magma's VkDevice, Espryt's EGL context) a session renders +// with was lost to a GPU fault or hang. It is only ever LATCHED (MGPipeSessionLatch), so the one +// session ends and the display server's other sessions keep running; the waiting client is told +// what ServerCrashed says and reads a lost context. Not DeviceLost, for ServerWindowLost's reason. +// // Two words were NOT given rows, they were collapsed instead: `InitialBytesNotCarried` // (MG_Impl/Pipe/PipeFill.cpp) said the same thing as `UncarriedInitialBytes` (Client/ // WireTables.cpp) about the same record - a respecify that crossed with no initial bytes - and @@ -113,6 +118,7 @@ X(ApplyThreadJoinTimeout, ServerCrashed, "the apply thread did not join within its bound at teardown") \ X(ApplyThreadNotRunning, ServerCrashed, "an operation needed the apply thread and it was not running") \ X(ResourceUnavailable, ServerCrashed, "the server could not obtain a native GL/Vulkan resource it had to have") \ - X(ServerWindowLost, ServerCrashed, "the server's own display window was destroyed under a live on-screen session (P12)") + X(ServerWindowLost, ServerCrashed, "the server's own display window was destroyed under a live on-screen session (P12)") \ + X(BackendDeviceLost, ServerCrashed, "the GPU device a session's backend renders with was lost (fault or hang reset)") // clang-format on diff --git a/MobileGL/MG_Remote/FatalFunnel.cpp b/MobileGL/MG_Remote/FatalFunnel.cpp index 0ed34eb8..07fecce9 100644 --- a/MobileGL/MG_Remote/FatalFunnel.cpp +++ b/MobileGL/MG_Remote/FatalFunnel.cpp @@ -45,28 +45,41 @@ namespace MobileGL::MG_Remote { // The line is forwarded through "%s" rather than re-used as a format: it has already been // vsnprintf'd once, and a detail string that happened to contain a `%` would otherwise be // read as a conversion the second time round. - [[noreturn]] void PipeSessionFailAdapter(MG_Pipe::MGPipeFatalFamily family, - const char* line) { - MGFatalFamily mapped = MGFatalFamily::ProtocolCorruption; + MGFatalFamily MapPipeFamily(MG_Pipe::MGPipeFatalFamily family) { switch (family) { case MG_Pipe::MGPipeFatalFamily::UnmigratedVerb: - mapped = MGFatalFamily::UnmigratedVerb; - break; + return MGFatalFamily::UnmigratedVerb; case MG_Pipe::MGPipeFatalFamily::RoleViolation: - mapped = MGFatalFamily::RoleViolation; - break; + return MGFatalFamily::RoleViolation; #if MOBILEGL_BUILD_DISAGGREGATED case MG_Pipe::MGPipeFatalFamily::ProtocolCorruption: - mapped = MGFatalFamily::ProtocolCorruption; - break; + return MGFatalFamily::ProtocolCorruption; + case MG_Pipe::MGPipeFatalFamily::DeviceLost: + return MGFatalFamily::BackendDeviceLost; #endif } - SessionFail(mapped, "%s", line); + return MGFatalFamily::ProtocolCorruption; } + + [[noreturn]] void PipeSessionFailAdapter(MG_Pipe::MGPipeFatalFamily family, + const char* line) { + SessionFail(MapPipeFamily(family), "%s", line); + } + +#if MOBILEGL_BUILD_DISAGGREGATED + // The latch twin (PipeSessionFail.h): the calling thread's session latches the fault and + // the backend returns from its verb; unarmed, SessionLatch is SessionFail. + bool PipeSessionLatchAdapter(MG_Pipe::MGPipeFatalFamily family, const char* line) { + return SessionLatch(MapPipeFamily(family), "%s", line); + } +#endif } // namespace void InstallPipeSessionFailHook() { MG_Pipe::MGPipeInstallSessionFailHook(&PipeSessionFailAdapter); +#if MOBILEGL_BUILD_DISAGGREGATED + MG_Pipe::MGPipeInstallSessionLatchHook(&PipeSessionLatchAdapter, &SessionLatchArmed); +#endif } namespace { diff --git a/MobileGL/MG_Remote/Server/PipeApplier.cpp b/MobileGL/MG_Remote/Server/PipeApplier.cpp index c987c544..20d6fac3 100644 --- a/MobileGL/MG_Remote/Server/PipeApplier.cpp +++ b/MobileGL/MG_Remote/Server/PipeApplier.cpp @@ -578,7 +578,9 @@ namespace MobileGL::MG_Remote::Server { MG_Pipe::MGPPixelPackState restorePack{}; restorePack.Pack = savedPack; MG_Pipe::MGPipeApplySetPixelPackState(restorePack); - return true; + // A backend that found its device lost latched this session and returned without filling + // the scratch: the read is declined rather than answered with stale bytes. + return !SessionLatched(); } Bool ServerVerbSink::ReadTextureImageTight(const MG_Pipe::MGPReadbackInfo& image, Vector& bytes) { diff --git a/MobileGL/MG_Test/Wire/FatalFamilyTest.cpp b/MobileGL/MG_Test/Wire/FatalFamilyTest.cpp index 482187a3..fd1aadee 100644 --- a/MobileGL/MG_Test/Wire/FatalFamilyTest.cpp +++ b/MobileGL/MG_Test/Wire/FatalFamilyTest.cpp @@ -203,3 +203,50 @@ TEST(FatalFamily, TheSessionFaultFrameCarriesTheFamilyAndItsCode) { EXPECT_EQ(decoded->code(), FatalCodeForFamily(family)); } } + +// --------------------------------------------------------------------------------------------- +// THE LATCH TWIN (MGPipeSessionLatch): how a backend's device loss reaches the session latch +// instead of SessionFail. The display server runs every session in one process, so the twin must +// RETURN on an armed session - only that session ends - and die exactly as the fail seam does +// where nothing is armed. Red once by pointing FatalFunnel.cpp's latch adapter at SessionFail: the +// armed case dies instead of returning. + +TEST(FatalFunnelSeam, ADeviceLossLatchesAnArmedSessionAndTheProcessLives) { +#if defined(_WIN32) + GTEST_SKIP() << "POSIX only, as above"; +#else + using namespace MobileGL; + EXPECT_EXIT( + { + MG_Remote::InstallPipeSessionFailHook(); + MG_Remote::ArmSessionLatch(); + const bool answered = MG_Pipe::MGPipeSessionLatch( + MG_Pipe::MGPipeFatalFamily::DeviceLost, + "MGPipe: Fatal{BackendDeviceLost, \"Magma:%s\"} - unit probe", "wire-verb-flush"); + const bool ok = !answered && MG_Remote::SessionLatched() && + MG_Remote::SessionLatchedFamily() == MG_Remote::MGFatalFamily::BackendDeviceLost && + MG_Pipe::MGPipeSessionLatchArmed(); + std::fflush(nullptr); + std::_Exit(ok ? 0 : 3); + }, + ::testing::ExitedWithCode(0), "Fatal\\{BackendDeviceLost, \"Magma:wire-verb-flush\"\\}") + << "an armed session's device loss did not latch and return"; +#endif +} + +TEST(FatalFunnelSeam, ADeviceLossWithNoArmedSessionDiesThroughSessionFail) { +#if defined(_WIN32) + GTEST_SKIP() << "POSIX only, as above"; +#else + using namespace MobileGL; + EXPECT_EXIT( + { + std::signal(SIGABRT, &ReportFaultCountAndExit); + MG_Remote::InstallPipeSessionFailHook(); + (void)MG_Pipe::MGPipeSessionLatch(MG_Pipe::MGPipeFatalFamily::DeviceLost, + "MGPipe: Fatal{BackendDeviceLost, \"Magma:%s\"}", "unarmed"); + }, + ::testing::ExitedWithCode(kFaultExitBase + 1), "Fatal\\{BackendDeviceLost, \"Magma:unarmed\"\\}") + << "an unarmed device loss must keep the inproc shape's death"; +#endif +} diff --git a/MobileGL/MG_Test/Wire/MultiSessionTest.cpp b/MobileGL/MG_Test/Wire/MultiSessionTest.cpp index 70242545..73af3027 100644 --- a/MobileGL/MG_Test/Wire/MultiSessionTest.cpp +++ b/MobileGL/MG_Test/Wire/MultiSessionTest.cpp @@ -519,3 +519,62 @@ TEST_F(MultiSessionFixture, TwoSessionsInterleaveTwoContextsEachWithoutCrossTalk EXPECT_TRUE(WIFEXITED(server.exitStatus) && WEXITSTATUS(server.exitStatus) == 0) << "the in-process server did not stop cleanly: status 0x" << std::hex << server.exitStatus; } + +// A LOST GPU DEVICE ENDS ITS OWN SESSION AND NO OTHER. +// +// The display server serves every client in one process, so a backend that met a device loss on +// one session's apply thread and died (Magma's wire funnels did: a failed submit took +// Fatal{UnmigratedVerb, "Magma:wire-verb-flush"} and SIGABRT) took the compositor and every other +// client down with it. A backend now hands a loss to MGPipeSessionLatch: that session latches +// Fatal{BackendDeviceLost}, declines what it was doing and closes (exit 75), and its neighbours +// keep applying. The loss is the debug knob's (MGPipeDebugDeviceLossDue) - the same device check +// the backends make at their readbacks and frame boundaries - because no GPU here faults on +// demand: the 2nd check in the process (session A's read-back, after B's) reports "lost". +// +// Red once by routing the latch seam through SessionFail (FatalFunnel.cpp's adapter): the whole +// server process aborts at A's read-back and session B's next record is never answered. +TEST_F(MultiSessionFixture, ADeviceLossEndsOnlyItsOwnSession) { + ServerProcess server; + ::setenv("MOBILEGL_DEBUG_INJECT_DEVICE_LOST_AT", "2", 1); + const bool launched = LaunchUnixInProcessServer(&server, "devlost"); + ::unsetenv("MOBILEGL_DEBUG_INJECT_DEVICE_LOST_AT"); + ASSERT_TRUE(launched) << "the in-process unix server did not start:\n" << server.Log(); + + Peer a = StartPeer(server.endpoint); + Peer b = StartPeer(server.endpoint); + ASSERT_GT(a.pid, 0); + ASSERT_GT(b.pid, 0); + ASSERT_EQ(a.ready.started, 1) << a.ready.note << "\n" << server.Log(); + ASSERT_EQ(b.ready.started, 1) << b.ready.note << "\n" << server.Log(); + + // Check #1: B's read-back is answered. + const StepReport bFirst = b.Step(kOpHealthySession, kGreen); + ASSERT_EQ(bFirst.session.egl, 1) << "no headless EGL for session B: " << bFirst.session.note << "\n" << server.Log(); + ASSERT_EQ(bFirst.session.pixel, kGreen) << bFirst.session.note << "\n" << server.Log(); + + // Check #2: A's device is "lost" at its read-back. A's session latches and ends. + const StepReport aLost = a.Step(kOpHealthySession, kRed); + EXPECT_NE(aLost.session.pixel, kRed) << "session A read back a frame from a lost device"; + ASSERT_TRUE(WaitFor([&] { return server.Log().find("reaped exit=75") != std::string::npos; }, 5000)) + << "session A did not end on a latched fault:\n" << server.Log(); + const std::string log = server.Log(); + EXPECT_NE(log.find("Fatal{BackendDeviceLost, \"Espryt:read-pixels\"}"), std::string::npos) << log; + EXPECT_NE(log.find("SessionLatch{BackendDeviceLost}"), std::string::npos) << log; + EXPECT_TRUE(server.pid > 0 && ::kill(server.pid, 0) == 0) << "the server process died with session A"; + + // B IS STILL SERVED, records and read-backs alike. + for (int record = 0; record < 3; ++record) { + const StepReport after = b.Step(kOpApplyRecord); + ASSERT_EQ(after.applied, 1) << "session B stopped applying after session A's device loss (record " + << record << "): " << after.note << "\n" << server.Log(); + } + const StepReport bAgain = b.Step(kOpHealthySession, kGreen); + EXPECT_EQ(bAgain.session.pixel, kGreen) << bAgain.session.note << "\n" << server.Log(); + EXPECT_EQ(Count(server.Log(), "SessionLatch{"), 1u) << "a session other than A latched:\n" << server.Log(); + + b.Close(); + a.Close(); + server.Stop(); + EXPECT_TRUE(WIFEXITED(server.exitStatus) && WEXITSTATUS(server.exitStatus) == 0) + << "the in-process server did not stop cleanly: status 0x" << std::hex << server.exitStatus; +} diff --git a/scripts/ci/fatal_census_baseline.json b/scripts/ci/fatal_census_baseline.json index a703435c..95746b66 100644 --- a/scripts/ci/fatal_census_baseline.json +++ b/scripts/ci/fatal_census_baseline.json @@ -1,5 +1,5 @@ { - "abort_sites": 77, + "abort_sites": 76, "abort_sites_by_file": { "MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp": 13, "MobileGL/MG_Backend/DirectGLES/Managers.cpp": 12, @@ -7,7 +7,7 @@ "MobileGL/MG_Backend/DirectGLES/Utils.cpp": 1, "MobileGL/MG_Backend/DirectVulkan/DirectVulkan.cpp": 2, "MobileGL/MG_Backend/DirectVulkan/Renderer/MagmaPipeArms.h": 1, - "MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp": 11, + "MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp": 10, "MobileGL/MG_Backend/Init.cpp": 2, "MobileGL/MG_Backend/MGPipe/PipeInputs.cpp": 3, "MobileGL/MG_Impl/Pipe/PipeFill.cpp": 5, @@ -27,6 +27,7 @@ "ApplyThreadNotRunning", "ArchiveEmpty", "ArtefactsMissing", + "BackendDeviceLost", "BarrierTimeout", "BarrierViolation", "BlobMissing",