From eb090c61701736d31db7965a04047bd5ed9eb8ab Mon Sep 17 00:00:00 2001 From: Swung0x48 Date: Fri, 10 Jul 2026 10:33:34 +0000 Subject: [PATCH] [Fix] (MG_Backend/DirectVulkan): fence-backed GL sync objects and per-frame descriptor rewind - Track every graphics-queue submission with a real fence: pooled fences for mid-frame flushes, the frame slot's fence for Present and readback. Completion advances a submit counter via vkGetFenceStatus polls, slot-fence waits, and device-idle points, and raises the buffer-manager serial floor from the frame serial each submission carried. - GL sync objects now capture the submission index that will carry the commands recorded so far; ClientWaitSync honors GL_SYNC_FLUSH_COMMANDS_BIT with a mid-frame submit (gated on the index still being unsubmitted so poll loops cannot split the render pass), and blocking waits flush then vkWaitForFences with the caller timeout. - FlushPendingCommands retires the submitted command buffer and restarts recording on a fresh one; retired buffers are freed once the slot fence is next waited, so an executing buffer is never reset. - Rewind descriptor-set cursors exactly once per frame in Present (after the slot-fence wait), plus after the synchronous readback drain, replacing the ten lazy per-draw-path rewinds. Verified: host tests 168/168, trace-replay 70/70. --- .../MG_Backend/DirectVulkan/DirectVulkan.cpp | 49 +-- .../DirectVulkan/Renderer/FrameContext.cpp | 43 +++ .../DirectVulkan/Renderer/FrameContext.h | 17 + .../DirectVulkan/Renderer/VkBufferManager.cpp | 9 + .../DirectVulkan/Renderer/VkBufferManager.h | 5 + .../DirectVulkan/Renderer/VulkanRenderer.cpp | 290 +++++++++++++++--- .../DirectVulkan/Renderer/VulkanRenderer.h | 81 ++++- 7 files changed, 436 insertions(+), 58 deletions(-) diff --git a/MobileGL/MG_Backend/DirectVulkan/DirectVulkan.cpp b/MobileGL/MG_Backend/DirectVulkan/DirectVulkan.cpp index b4977251..7c78cdbd 100644 --- a/MobileGL/MG_Backend/DirectVulkan/DirectVulkan.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/DirectVulkan.cpp @@ -1366,16 +1366,18 @@ namespace MobileGL::MG_Backend::DirectVulkan { } namespace { - // Backend fence handle: the VkBufferManager frame serial captured at - // fence creation. The fence is signaled once every command recorded - // under that serial has completed on the GPU (the same busy-tracking - // horizon used to recycle buffer resources). + // Backend fence handle: the queue-submission index captured at fence + // creation (see VulkanRenderer::GetSyncPointSubmitIndex). The fence is + // signaled once that submission's VkFence has been observed signaled, + // so completion tracks the GPU itself rather than the frame-count + // inference; MC 1.21.5's fence-paced ring buffers depend on this to + // recycle their space instead of growing without bound. struct VulkanSyncObject { - Uint64 frameSerial = 0; - // Renderer generation the serial was issued under (see + Uint64 submitIndex = 0; + // Renderer generation the index was issued under (see // g_rendererGeneration). A stale generation reports the fence // signaled: renderer destruction waits for device idle, so the - // old renderer's GPU work is long complete, and the serial must + // old renderer's GPU work is long complete, and the index must // not be compared against the new renderer's restarted counter. Uint64 rendererGeneration = 0; }; @@ -1385,26 +1387,36 @@ namespace MobileGL::MG_Backend::DirectVulkan { if (!pVulkanRenderer) { return nullptr; } - return new VulkanSyncObject{pVulkanRenderer->GetCurrentFrameSerial(), GetRendererGeneration()}; + return new VulkanSyncObject{pVulkanRenderer->GetSyncPointSubmitIndex(), GetRendererGeneration()}; } GLenum ClientWaitSync(BackendSyncHandle handle, GLbitfield flags, GLuint64 timeout) { - // Commands are only submitted at Present, so GL_SYNC_FLUSH_COMMANDS_BIT - // cannot force progress mid-frame; WaitForFrameSerial reports whether - // waiting can succeed at all. - (void)flags; const auto* sync = static_cast(handle); if (sync == nullptr || !pVulkanRenderer || sync->rendererGeneration != GetRendererGeneration()) { return GL_ALREADY_SIGNALED; } - if (pVulkanRenderer->IsFrameSerialComplete(sync->frameSerial)) { + if (pVulkanRenderer->IsSubmitIndexComplete(sync->submitIndex)) { return GL_ALREADY_SIGNALED; } - if (timeout == 0) { - return GL_TIMEOUT_EXPIRED; + // GL_SYNC_FLUSH_COMMANDS_BIT: flush regardless of timeout, so a + // zero-timeout poll loop makes progress across calls - but only when + // the sync's batch is still unsubmitted; flushing for an already + // submitted fence cannot advance it and would split the frame's + // render pass on every poll. + if ((flags & GL_SYNC_FLUSH_COMMANDS_BIT) != 0) { + pVulkanRenderer->FlushForSyncPoint(sync->submitIndex); } - return pVulkanRenderer->WaitForFrameSerial(sync->frameSerial, timeout) ? GL_CONDITION_SATISFIED - : GL_TIMEOUT_EXPIRED; + if (timeout == 0) { + return pVulkanRenderer->IsSubmitIndexComplete(sync->submitIndex) ? GL_ALREADY_SIGNALED + : GL_TIMEOUT_EXPIRED; + } + // Blocking wait: flush even without the flush bit - the sync's batch + // can only be submitted from this thread, so waiting on an unflushed + // fence would otherwise burn the full timeout with no chance of + // success. + return pVulkanRenderer->WaitForSubmitIndex(sync->submitIndex, timeout, /*flushIfPending=*/true) + ? GL_CONDITION_SATISFIED + : GL_TIMEOUT_EXPIRED; } void WaitSync(BackendSyncHandle handle, GLbitfield flags, GLuint64 timeout) { @@ -1425,7 +1437,8 @@ namespace MobileGL::MG_Backend::DirectVulkan { if (sync == nullptr || !pVulkanRenderer || sync->rendererGeneration != GetRendererGeneration()) { return true; } - return pVulkanRenderer->IsFrameSerialComplete(sync->frameSerial); + // Pure status read (glGetSynciv must not flush). + return pVulkanRenderer->IsSubmitIndexComplete(sync->submitIndex); } namespace { diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/FrameContext.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/FrameContext.cpp index 440740b5..1f72dbcb 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/FrameContext.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/FrameContext.cpp @@ -13,6 +13,8 @@ namespace MobileGL::MG_Backend::DirectVulkan { Destroy(device, commandPool); m_frames.assign(frameCount, {}); currentFrameIndex = 0; + m_device = device; + m_commandPool = commandPool; Vector commandBuffers(frameCount, VK_NULL_HANDLE); VkCommandBufferAllocateInfo allocInfo{}; @@ -55,10 +57,15 @@ namespace MobileGL::MG_Backend::DirectVulkan { } DestroySwapchainSemaphores(device); if (device != VK_NULL_HANDLE && commandPool != VK_NULL_HANDLE && !m_frames.empty()) { + for (auto& frame : m_frames) { + FreeRetiredCommandBuffers(frame); + } vkFreeCommandBuffers(device, commandPool, frameCount, commandBuffers.data()); } m_frames.clear(); currentFrameIndex = 0; + m_device = VK_NULL_HANDLE; + m_commandPool = VK_NULL_HANDLE; } FrameContext::FrameData& FrameContext::GetCurrent() { @@ -214,6 +221,9 @@ namespace MobileGL::MG_Backend::DirectVulkan { if (result != VK_SUCCESS) { return result; } + // The slot's fence has been waited: every command buffer this slot + // submitted (including mid-frame flushes) has finished executing. + FreeRetiredCommandBuffers(frame); result = vkAcquireNextImageKHR(device, swapchain, timeout, frame.imageAvailableSemaphore, acquireFence, &outImageIndex); @@ -237,6 +247,39 @@ namespace MobileGL::MG_Backend::DirectVulkan { m_recordingObserver = observer; } + VkResult FrameContext::RetireCurrentCommandBuffer() { + MOBILEGL_ASSERT(m_device != VK_NULL_HANDLE && m_commandPool != VK_NULL_HANDLE, + "RetireCurrentCommandBuffer requires an initialized FrameContext"); + auto& frame = GetCurrent(); + MOBILEGL_ASSERT(!frame.isCommandRecording, + "RetireCurrentCommandBuffer called while the command buffer is still recording"); + + VkCommandBufferAllocateInfo allocInfo{}; + allocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + allocInfo.commandPool = m_commandPool; + allocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + allocInfo.commandBufferCount = 1; + VkCommandBuffer replacement = VK_NULL_HANDLE; + const VkResult result = vkAllocateCommandBuffers(m_device, &allocInfo, &replacement); + if (result != VK_SUCCESS) { + return result; + } + frame.retiredCommandBuffers.push_back(frame.commandBuffer); + frame.commandBuffer = replacement; + return VK_SUCCESS; + } + + void FrameContext::FreeRetiredCommandBuffers(FrameData& frame) { + if (frame.retiredCommandBuffers.empty()) { + return; + } + if (m_device != VK_NULL_HANDLE && m_commandPool != VK_NULL_HANDLE) { + vkFreeCommandBuffers(m_device, m_commandPool, static_cast(frame.retiredCommandBuffers.size()), + frame.retiredCommandBuffers.data()); + } + frame.retiredCommandBuffers.clear(); + } + void FrameContext::AssertValidFrameIndex(Uint32 frameIndex) const { MOBILEGL_ASSERT(frameIndex < m_frames.size(), "FrameContext index out of range"); } diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/FrameContext.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/FrameContext.h index 71bb1ea5..14dd144b 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/FrameContext.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/FrameContext.h @@ -47,6 +47,13 @@ namespace MobileGL::MG_Backend::DirectVulkan { Bool isCommandRecording = false; Bool hasCommandBufferRecorded = false; Bool imageAvailableSemaphoreConsumed = false; + // Command buffers submitted mid-frame (FlushPendingCommands) whose + // execution is only known complete once this slot's fence has been + // waited again; freed at that point. + Vector retiredCommandBuffers; + // Submit-tracker index of this slot's most recent queue submission + // (written by the renderer at submit time). + Uint64 lastSubmitIndex = 0; }; VkResult Initialize(VkDevice device, VkCommandPool commandPool, Uint32 frameCount); @@ -69,6 +76,12 @@ namespace MobileGL::MG_Backend::DirectVulkan { VkResult WaitAndAcquireNextImage(VkDevice device, VkSwapchainKHR swapchain, Uint32& outImageIndex, Uint64 timeout = UINT64_MAX, VkFence acquireFence = VK_NULL_HANDLE); + // Parks the current (already ended and submitted) command buffer on the + // slot's retired list and installs a freshly allocated one, so recording + // can restart while the submitted buffer is still executing. Retired + // buffers are freed after the slot's fence is next waited. + VkResult RetireCurrentCommandBuffer(); + Uint32 GetCurrentFrameIndex() const; Uint32 GetFrameCount() const; @@ -83,10 +96,14 @@ namespace MobileGL::MG_Backend::DirectVulkan { const VkSemaphoreCreateInfo& semaphoreInfo, const VkFenceCreateInfo& fenceInfo); void DestroySyncObjectsForFrame(VkDevice device, Uint32 frameIndex); + void FreeRetiredCommandBuffers(FrameData& frame); Vector m_frames; Vector m_swapchainImageRenderFinishedSemaphores; Uint32 currentFrameIndex = 0; IRecordingObserver* m_recordingObserver = nullptr; + // Stored at Initialize for retired-command-buffer management. + VkDevice m_device = VK_NULL_HANDLE; + VkCommandPool m_commandPool = VK_NULL_HANDLE; }; } // namespace MobileGL::MG_Backend::DirectVulkan diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp index 3f1678ac..e5ab5d53 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp @@ -129,6 +129,15 @@ namespace MobileGL::MG_Backend::DirectVulkan { } } + void VkBufferManager::NotifyFrameSerialComplete(Uint64 serial) { + // The current serial's work is still being recorded; a completion + // report for it (or beyond) can only come from a stale caller. + if (serial >= m_frameSerial) { + return; + } + m_completedSerialFloor = std::max(m_completedSerialFloor, serial); + } + void VkBufferManager::SetCopyCommandProvider(IBufferCopyCommandProvider* provider) { m_copyProvider = provider; } diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.h index c7a66cdd..93661aa5 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.h @@ -75,6 +75,11 @@ namespace MobileGL::MG_Backend::DirectVulkan { void BeginFrame(Uint32 frameIndex); // All previously submitted GPU work has completed (vkDeviceWaitIdle). void NotifyDeviceIdle(); + // A frame slot's submission fence has been waited: every serial up to + // and including `serial` is complete. Raises the completed floor so + // GetCompletedSerial reflects real fence progress instead of only the + // frameSerial-minus-frameCount inference. + void NotifyFrameSerialComplete(Uint64 serial); void SetCopyCommandProvider(IBufferCopyCommandProvider* provider); Bool UploadTransient(BufferKind kind, Uint32 frameIndex, const void* data, VkDeviceSize size, diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp index 362b89e7..04a67702 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp @@ -1983,6 +1983,8 @@ void main() { if (m_device != VK_NULL_HANDLE) { VK_VERIFY(vkDeviceWaitIdle(m_device)); } + OnSubmitsCompletedUpTo(m_submitCounter); + DestroySubmitFencePool(); DestroyDeferredDepthMipmapCleanup(); DestroyComputePipelines(); @@ -3203,7 +3205,6 @@ void main() { // Begin command recording if not yet if (!frame.isCommandRecording) { m_frameContext.BeginCommandRecording(); - m_uniformManager->BeginFrame(m_frameContext.GetCurrentFrameIndex()); } auto* activeRenderPass = VkRenderPassManager::GetActiveRenderPass(); @@ -3361,7 +3362,6 @@ void main() { if (!frame.isCommandRecording) { m_frameContext.BeginCommandRecording(); - m_uniformManager->BeginFrame(m_frameContext.GetCurrentFrameIndex()); } if (VkRenderPassManager::GetActiveRenderPass() != nullptr) { @@ -3397,7 +3397,6 @@ void main() { if (!frame.isCommandRecording) { m_frameContext.BeginCommandRecording(); - m_uniformManager->BeginFrame(m_frameContext.GetCurrentFrameIndex()); } if (VkRenderPassManager::GetActiveRenderPass() != nullptr) { @@ -3463,7 +3462,6 @@ void main() { auto& frame = m_frameContext.GetCurrent(); if (!frame.isCommandRecording) { m_frameContext.BeginCommandRecording(); - m_uniformManager->BeginFrame(m_frameContext.GetCurrentFrameIndex()); } if (VkRenderPassManager::GetActiveRenderPass() != nullptr) { VkRenderPassManager::EndRenderPass(frame.commandBuffer); @@ -3939,7 +3937,6 @@ void main() { auto& frame = m_frameContext.GetCurrent(); if (!frame.isCommandRecording) { m_frameContext.BeginCommandRecording(); - m_uniformManager->BeginFrame(m_frameContext.GetCurrentFrameIndex()); } auto* activeRenderPass = VkRenderPassManager::GetActiveRenderPass(); @@ -4290,7 +4287,6 @@ void main() { auto& frame = m_frameContext.GetCurrent(); if (!frame.isCommandRecording) { m_frameContext.BeginCommandRecording(); - m_uniformManager->BeginFrame(m_frameContext.GetCurrentFrameIndex()); } if (VkRenderPassManager::GetActiveRenderPass() != nullptr) { @@ -4457,7 +4453,6 @@ void main() { auto& frame = m_frameContext.GetCurrent(); if (!frame.isCommandRecording) { m_frameContext.BeginCommandRecording(); - m_uniformManager->BeginFrame(m_frameContext.GetCurrentFrameIndex()); } if (VkRenderPassManager::GetActiveRenderPass() != nullptr) { @@ -4590,29 +4585,16 @@ void main() { return true; } - VkPipelineStageFlags waitDstStageMask = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; - VkSubmitInfo submitInfo{VK_STRUCTURE_TYPE_SUBMIT_INFO}; - VkSemaphore waitSemaphore = frame.imageAvailableSemaphore; - if (!frame.imageAvailableSemaphoreConsumed) { - submitInfo.waitSemaphoreCount = 1; - submitInfo.pWaitSemaphores = &waitSemaphore; - submitInfo.pWaitDstStageMask = &waitDstStageMask; - } - submitInfo.commandBufferCount = 1; - submitInfo.pCommandBuffers = &frame.commandBuffer; - - VkResult result = vkQueueSubmit(m_graphicsQueue, 1, &submitInfo, frame.imageInFlightFence); - if (result != VK_SUCCESS) { - MGLOG_E("DirectVulkan readback: vkQueueSubmit returned %d", result); + if (!SubmitPendingCommandBuffer(frame, frame.imageInFlightFence, /*pooledFence=*/false)) { return false; } - frame.imageAvailableSemaphoreConsumed = true; - result = vkWaitForFences(m_device, 1, &frame.imageInFlightFence, VK_TRUE, UINT64_MAX); + VkResult result = vkWaitForFences(m_device, 1, &frame.imageInFlightFence, VK_TRUE, UINT64_MAX); if (result != VK_SUCCESS) { MGLOG_E("DirectVulkan readback: vkWaitForFences returned %d", result); return false; } + OnSubmitsCompletedUpTo(frame.lastSubmitIndex); result = vkResetFences(m_device, 1, &frame.imageInFlightFence); if (result != VK_SUCCESS) { MGLOG_E("DirectVulkan readback: vkResetFences returned %d", result); @@ -4621,6 +4603,12 @@ void main() { frame.hasCommandBufferRecorded = false; frame.isCommandRecording = false; + // The wait proved every descriptor set this slot has in flight idle; + // rewind the reuse cursors so present-less readback loops stay + // bounded (Present is the only other rewind point). + if (m_uniformManager) { + m_uniformManager->BeginFrame(m_frameContext.GetCurrentFrameIndex()); + } return true; } @@ -4639,7 +4627,6 @@ void main() { auto& frame = m_frameContext.GetCurrent(); if (!frame.isCommandRecording) { m_frameContext.BeginCommandRecording(); - m_uniformManager->BeginFrame(m_frameContext.GetCurrentFrameIndex()); } if (VkRenderPassManager::GetActiveRenderPass() != nullptr) { VkRenderPassManager::EndRenderPass(frame.commandBuffer); @@ -4807,7 +4794,6 @@ void main() { auto& frame = m_frameContext.GetCurrent(); if (!frame.isCommandRecording) { m_frameContext.BeginCommandRecording(); - m_uniformManager->BeginFrame(m_frameContext.GetCurrentFrameIndex()); } if (VkRenderPassManager::GetActiveRenderPass() != nullptr) { VkRenderPassManager::EndRenderPass(frame.commandBuffer); @@ -4911,7 +4897,6 @@ void main() { auto& frame = m_frameContext.GetCurrent(); if (!frame.isCommandRecording) { m_frameContext.BeginCommandRecording(); - m_uniformManager->BeginFrame(m_frameContext.GetCurrentFrameIndex()); } if (VkRenderPassManager::GetActiveRenderPass() != nullptr) { @@ -5442,10 +5427,6 @@ void main() { return frame.commandBuffer; } - Uint64 VulkanRenderer::GetCurrentFrameSerial() const { - return m_bufferManager.GetFrameSerial(); - } - Bool VulkanRenderer::IsFrameSerialComplete(Uint64 serial) const { return serial <= m_bufferManager.GetCompletedSerial(); } @@ -5474,6 +5455,222 @@ void main() { return false; } m_bufferManager.NotifyDeviceIdle(); + OnSubmitsCompletedUpTo(m_submitCounter); + return true; + } + + Uint64 VulkanRenderer::GetSyncPointSubmitIndex() const { + // Commands recorded (or still recording) since the last submission are + // carried by the NEXT submission; a fence created now must wait for it. + return m_submitCounter + (HasPendingRecordedWork() ? 1 : 0); + } + + Bool VulkanRenderer::HasPendingRecordedWork() const { + if (m_frameContext.GetFrameCount() == 0) { + return false; + } + const auto& frame = m_frameContext.GetCurrent(); + return frame.isCommandRecording || frame.hasCommandBufferRecorded; + } + + Bool VulkanRenderer::IsSubmitIndexComplete(Uint64 submitIndex) { + if (submitIndex <= m_completedSubmitCounter) { + return true; + } + if (submitIndex > m_submitCounter) { + return false; // not even submitted; no point polling fences + } + RefreshCompletedSubmits(); + return submitIndex <= m_completedSubmitCounter; + } + + void VulkanRenderer::RegisterSubmit(VkFence fence, Bool pooledFence) { + ++m_submitCounter; + m_inFlightSubmits.push_back({m_submitCounter, m_bufferManager.GetFrameSerial(), fence, pooledFence}); + } + + void VulkanRenderer::RefreshCompletedSubmits() { + if (m_device == VK_NULL_HANDLE) { + return; + } + // Prefix-only scan: submissions to a single queue complete in order, + // and stopping at the first unsignaled fence stays conservative even + // if they did not. + while (!m_inFlightSubmits.empty()) { + // Copy before OnSubmitsCompletedUpTo erases the front record. + const Uint64 frontIndex = m_inFlightSubmits.front().submitIndex; + if (vkGetFenceStatus(m_device, m_inFlightSubmits.front().fence) != VK_SUCCESS) { + break; + } + OnSubmitsCompletedUpTo(frontIndex); + } + } + + void VulkanRenderer::OnSubmitsCompletedUpTo(Uint64 submitIndex) { + m_completedSubmitCounter = std::max(m_completedSubmitCounter, submitIndex); + while (!m_inFlightSubmits.empty() && m_inFlightSubmits.front().submitIndex <= submitIndex) { + SubmitRecord record = m_inFlightSubmits.front(); + m_inFlightSubmits.erase(m_inFlightSubmits.begin()); + // Frame-serial completion piggybacks on submission completion. + // NotifyFrameSerialComplete refuses the current (still-recording) + // serial, so mid-frame flush records do not mark it early. + m_bufferManager.NotifyFrameSerialComplete(record.frameSerial); + if (!record.pooledFence || m_device == VK_NULL_HANDLE) { + continue; // frame-slot fences are reset/destroyed by FrameContext + } + if (vkResetFences(m_device, 1, &record.fence) == VK_SUCCESS) { + m_freeSubmitFences.push_back(record.fence); + } else { + vkDestroyFence(m_device, record.fence, nullptr); + } + } + } + + VkFence VulkanRenderer::AcquirePooledSubmitFence() { + if (!m_freeSubmitFences.empty()) { + VkFence fence = m_freeSubmitFences.back(); + m_freeSubmitFences.pop_back(); + return fence; + } + VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + VkFence fence = VK_NULL_HANDLE; + const VkResult result = vkCreateFence(m_device, &fenceInfo, nullptr, &fence); + if (result != VK_SUCCESS) { + MGLOG_E("AcquirePooledSubmitFence: vkCreateFence returned %d", result); + return VK_NULL_HANDLE; + } + return fence; + } + + void VulkanRenderer::DestroySubmitFencePool() { + // Callers guarantee device idle, so in-flight fences are inert. + for (const auto& record : m_inFlightSubmits) { + if (record.pooledFence && m_device != VK_NULL_HANDLE) { + vkDestroyFence(m_device, record.fence, nullptr); + } + } + m_inFlightSubmits.clear(); + for (auto fence : m_freeSubmitFences) { + if (m_device != VK_NULL_HANDLE) { + vkDestroyFence(m_device, fence, nullptr); + } + } + m_freeSubmitFences.clear(); + m_completedSubmitCounter = m_submitCounter; + } + + Bool VulkanRenderer::SubmitPendingCommandBuffer(FrameContext::FrameData& frame, VkFence fence, Bool pooledFence) { + VkPipelineStageFlags waitDstStageMask = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; + VkSemaphore waitSemaphore = frame.imageAvailableSemaphore; + VkSubmitInfo submitInfo{VK_STRUCTURE_TYPE_SUBMIT_INFO}; + if (!frame.imageAvailableSemaphoreConsumed) { + submitInfo.waitSemaphoreCount = 1; + submitInfo.pWaitSemaphores = &waitSemaphore; + submitInfo.pWaitDstStageMask = &waitDstStageMask; + } + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = &frame.commandBuffer; + const VkResult result = vkQueueSubmit(m_graphicsQueue, 1, &submitInfo, fence); + if (result != VK_SUCCESS) { + MGLOG_E("SubmitPendingCommandBuffer: vkQueueSubmit returned %d", result); + return false; + } + frame.imageAvailableSemaphoreConsumed = true; + frame.hasCommandBufferRecorded = false; + RegisterSubmit(fence, pooledFence); + frame.lastSubmitIndex = m_submitCounter; + return true; + } + + Bool VulkanRenderer::FlushPendingCommands() { + if (m_device == VK_NULL_HANDLE || m_graphicsQueue == VK_NULL_HANDLE || m_frameContext.GetFrameCount() == 0) { + return false; + } + auto& frame = m_frameContext.GetCurrent(); + if (!frame.isCommandRecording && !frame.hasCommandBufferRecorded) { + return false; + } + // Acquire the fence while recording is still open: failing here must + // not end recording, or the next draw's BeginCommandRecording would + // reset the command buffer and silently drop the frame's commands. + VkFence fence = AcquirePooledSubmitFence(); + if (fence == VK_NULL_HANDLE) { + return false; + } + if (frame.isCommandRecording) { + if (VkRenderPassManager::GetActiveRenderPass() != nullptr) { + VkRenderPassManager::EndRenderPass(frame.commandBuffer); + } + m_frameContext.EndCommandRecording(); + } + if (!SubmitPendingCommandBuffer(frame, fence, /*pooledFence=*/true)) { + // Submit failure (device loss regime): the ended command buffer + // stays marked recorded so Present can still try to submit it. + m_freeSubmitFences.push_back(fence); // still unsignaled, reusable + return false; + } + + // The submitted command buffer may still be executing; recording must + // restart on a fresh one. If none can be allocated, fall back to + // draining this submission so reusing the buffer stays legal. + const VkResult retireResult = m_frameContext.RetireCurrentCommandBuffer(); + if (retireResult != VK_SUCCESS) { + MGLOG_E("FlushPendingCommands: RetireCurrentCommandBuffer returned %d; draining submission", retireResult); + if (vkWaitForFences(m_device, 1, &fence, VK_TRUE, UINT64_MAX) == VK_SUCCESS) { + OnSubmitsCompletedUpTo(m_submitCounter); + } else if (vkQueueWaitIdle(m_graphicsQueue) == VK_SUCCESS) { + m_bufferManager.NotifyDeviceIdle(); + OnSubmitsCompletedUpTo(m_submitCounter); + } else { + // Device is effectively lost; the command buffer may still be + // pending, but no recovery can make reuse legal. + MGLOG_E("FlushPendingCommands: drain failed; command buffer reuse is unsafe"); + } + } + return true; + } + + Bool VulkanRenderer::FlushForSyncPoint(Uint64 submitIndex) { + // A flush only helps a sync point whose commands are not submitted + // yet; for an already-submitted index it would just split the frame's + // render pass (a full tile load/store on TBDR GPUs) without advancing + // the fence. + if (submitIndex <= m_submitCounter) { + return false; + } + return FlushPendingCommands(); + } + + Bool VulkanRenderer::WaitForSubmitIndex(Uint64 submitIndex, Uint64 timeoutNs, Bool flushIfPending) { + if (IsSubmitIndexComplete(submitIndex)) { + return true; + } + if (submitIndex > m_submitCounter) { + if (!flushIfPending) { + return false; + } + FlushPendingCommands(); + if (submitIndex > m_submitCounter) { + // Nothing could be submitted (empty batch or submit failure); + // the index cannot complete yet. + return false; + } + } + for (const auto& record : m_inFlightSubmits) { + if (record.submitIndex >= submitIndex) { + const VkResult result = vkWaitForFences(m_device, 1, &record.fence, VK_TRUE, timeoutNs); + if (result == VK_SUCCESS) { + OnSubmitsCompletedUpTo(record.submitIndex); + return true; + } + if (result != VK_TIMEOUT) { + MGLOG_E("WaitForSubmitIndex: vkWaitForFences returned %d", result); + } + return false; + } + } + // No in-flight record at or beyond the index: it was already observed + // complete via a fence wait on a later submission. return true; } @@ -5516,9 +5713,10 @@ void main() { if (IsTimerQueryResultReady(record)) { return true; } - // Mirrors ClientWaitSync: WaitForFrameSerial refuses serials that - // cannot complete without further submissions (a timestamp written - // this frame only executes once Present submits the command buffer). + // WaitForFrameSerial refuses serials that cannot complete without + // further submissions (a timestamp written this frame only executes + // once Present submits the command buffer), so this returns false + // instead of deadlocking; the record resolves after a later Present. if (!WaitForFrameSerial(record.frameSerial, UINT64_MAX)) { return false; } @@ -5547,8 +5745,14 @@ void main() { const VkExtent2D presentStatsExtent = m_swapchainObject.GetExtent(); const char* presentDumpPath = PresentDumpPath(); const Bool shouldDumpPresent = presentDumpPath != nullptr && PresentDumpMatchesTargetCall(); - const Bool collectPresentStats = (PresentStatsEnabled() || shouldDumpPresent) && frame.isCommandRecording && - presentStatsExtent.width > 0 && presentStatsExtent.height > 0; + const Bool wantPresentStats = (PresentStatsEnabled() || shouldDumpPresent) && + presentStatsExtent.width > 0 && presentStatsExtent.height > 0; + if (wantPresentStats && !frame.isCommandRecording) { + // A mid-frame flush may have closed the frame's recording; the + // stats copy needs an open command buffer. + m_frameContext.BeginCommandRecording(); + } + const Bool collectPresentStats = wantPresentStats && frame.isCommandRecording; if (PresentStatsEnabled() && presentDumpPath != nullptr) { // Live getenv on purpose (not MG_Config::Features): the retrace // harness mutates these two variables at runtime via setenv. @@ -5623,9 +5827,12 @@ void main() { // 1) Submit current frame work. auto submitPacket = m_frameContext.GetSubmitInfo(shouldSubmitCommandBuffer, m_imageIndexAcquired); VK_VERIFY(vkQueueSubmit(m_graphicsQueue, 1, &submitPacket.submitInfo, frame.imageInFlightFence)); + RegisterSubmit(frame.imageInFlightFence, /*pooledFence=*/false); + frame.lastSubmitIndex = m_submitCounter; if (collectPresentStats) { VK_VERIFY(vkWaitForFences(m_device, 1, &frame.imageInFlightFence, VK_TRUE, UINT64_MAX), "Present stats, vkWaitForFences"); + OnSubmitsCompletedUpTo(frame.lastSubmitIndex); const auto* pixels = static_cast(presentStatsReadback.Map()); MOBILEGL_ASSERT(pixels != nullptr, "Present stats: failed to map readback buffer"); SizeT nonBlack = 0; @@ -5699,9 +5906,21 @@ void main() { m_frameContext.WaitAndAcquireNextImage(m_device, m_swapchainObject.GetHandle(), m_imageIndexAcquired); } VK_VERIFY(result, "Present, vkAcquireNextImageKHR"); + // The acquired slot's fence has been waited: its last submission + // (and, in queue order, everything before it) is complete. The frame + // serials those submissions carried advance the buffer-manager floor + // inside OnSubmitsCompletedUpTo. + OnSubmitsCompletedUpTo(m_frameContext.GetCurrent().lastSubmitIndex); CollectDeferredDepthMipmapCleanup(m_frameContext.GetCurrentFrameIndex()); m_textureManager->BeginFrame(m_frameContext.GetCurrentFrameIndex()); m_bufferManager.BeginFrame(m_frameContext.GetCurrentFrameIndex()); + // Descriptor-set reuse cursors rewind exactly once per frame, here, + // after the slot's fence wait proved its previous sets GPU-idle. (The + // per-draw-path lazy rewind missed frames whose recording was opened + // by a staged buffer copy or timer-query timestamp, leaking a fresh + // descriptor set per draw for the whole frame; it would also be unsafe + // after a mid-frame FlushPendingCommands, which does not wait.) + m_uniformManager->BeginFrame(m_frameContext.GetCurrentFrameIndex()); } void VulkanRenderer::CreateInstance() { @@ -6420,6 +6639,7 @@ void main() { } vkDeviceWaitIdle(m_device); + OnSubmitsCompletedUpTo(m_submitCounter); if (m_timerQueryManager) { // The in-progress command buffer is abandoned below (its recording diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h index aa5e9e82..643660f8 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h @@ -173,11 +173,35 @@ namespace MobileGL::MG_Backend::DirectVulkan { VkInstance GetInstance() const; Bool IsDrawIndirectCountExtensionEnabled() const; - // GL fence support, expressed in VkBufferManager frame serials: a fence - // captures GetCurrentFrameSerial() at creation and is signaled once - // IsFrameSerialComplete() reports that serial complete (the same - // busy-tracking horizon used to recycle buffer resources). - Uint64 GetCurrentFrameSerial() const; + // GL fence support, expressed in queue-submission indices backed by + // real VkFences. A GL fence captures GetSyncPointSubmitIndex() at + // creation: the index of the submission that will carry the commands + // recorded so far (m_submitCounter + 1 while work is pending, or + // m_submitCounter when nothing has been recorded since the last + // submit). It is signaled once that submission's fence is observed + // signaled - unlike the frame-serial heuristic, this makes fences + // signal as soon as the GPU actually finishes, which MC 1.21.5's + // fence-paced ring buffers rely on to recycle their space. + Uint64 GetSyncPointSubmitIndex() const; + // Non-blocking: polls outstanding submission fences and reports + // whether every submission up to `submitIndex` has completed. + Bool IsSubmitIndexComplete(Uint64 submitIndex); + // Submits the commands recorded so far without waiting (GL flush). + // Recording restarts lazily on a fresh command buffer; the submitted + // one is retired until the frame slot's fence is next waited. Returns + // true when a submission was made. + Bool FlushPendingCommands(); + // Flush gated on usefulness: only flushes when `submitIndex` is still + // unsubmitted, so poll loops on already-submitted fences do not split + // the frame's render pass (a full tile load/store on TBDR GPUs). + Bool FlushForSyncPoint(Uint64 submitIndex); + // Blocking wait for a submission index with a nanosecond timeout. + // When the index is still unsubmitted and flushIfPending is set, the + // pending commands are flushed first so the wait can make progress. + Bool WaitForSubmitIndex(Uint64 submitIndex, Uint64 timeoutNs, Bool flushIfPending); + + // Frame-serial completion, still used by the timer-query paths (their + // records are bucketed per frame slot). Bool IsFrameSerialComplete(Uint64 serial) const; // Blocking wait for a submitted serial. Returns false when the serial // cannot complete without further submissions (it belongs to the @@ -245,6 +269,53 @@ namespace MobileGL::MG_Backend::DirectVulkan { GLenum buffer, GLint drawbuffer, const ClearAttachmentPayload& clearPayload); + // ---- Submission fence tracking (GL sync objects) ---- + // One record per vkQueueSubmit still in flight, in ascending submit + // order. Present/readback submissions reference the frame slot's + // fence (not pool-owned); mid-frame flushes use pooled fences that are + // recycled once their submission is observed complete. + // Not thread-safe: like the rest of the renderer, the tracker relies + // on GL calls being serialized (launchers migrate the context across + // threads, but calls never run concurrently), so sync-object polls + // may mutate it without locking. + struct SubmitRecord { + Uint64 submitIndex = 0; + // Buffer-manager frame serial the submission was made under; its + // completion raises the completed-serial floor (timer queries and + // buffer busy-tracking live in frame-serial space). + Uint64 frameSerial = 0; + VkFence fence = VK_NULL_HANDLE; + Bool pooledFence = false; + }; + // Registers a submission that vkQueueSubmit just made with `fence`. + // Invariant: every graphics-queue submission that outlives its call + // site must be registered so GL fences observe it. Exempt are the + // texture-upload/preserve submits in VkTextureManager, which + // vkWaitForFences inline before returning. + void RegisterSubmit(VkFence fence, Bool pooledFence); + // Builds the submit packet for the frame's pending command buffer + // (consuming the acquire semaphore on the slot's first submission), + // submits it with `fence`, and registers the submission. On failure + // the frame state is left untouched. Shared by the mid-frame flush + // and the readback path so the semaphore-consumption invariant lives + // in one place. + Bool SubmitPendingCommandBuffer(FrameContext::FrameData& frame, VkFence fence, Bool pooledFence); + // Polls in-flight submission fences (prefix order) and advances the + // completed counter past every fence observed signaled. + void RefreshCompletedSubmits(); + // All submissions up to `submitIndex` are known complete (their fence + // was waited or the device was idled); drops their records and + // recycles pooled fences. + void OnSubmitsCompletedUpTo(Uint64 submitIndex); + VkFence AcquirePooledSubmitFence(); + void DestroySubmitFencePool(); + Bool HasPendingRecordedWork() const; + + Vector m_inFlightSubmits; + Vector m_freeSubmitFences; + Uint64 m_submitCounter = 0; + Uint64 m_completedSubmitCounter = 0; + NativeWindowType m_window = 0; void* m_platformDisplay = nullptr; void* m_platformLibrary = nullptr;