From a4c1bc8b1e82e8e267cc13f1463f2dbce76bdaab Mon Sep 17 00:00:00 2001 From: Swung0x48 Date: Wed, 23 Sep 2026 02:01:36 -0400 Subject: [PATCH] [MG_Backend, MG_Remote] (P7): draft B4 aggregate waits and staged extent reset --- .../DirectVulkan/Renderer/FrameContext.cpp | 6 +- .../DirectVulkan/Renderer/VkBufferManager.cpp | 5 +- .../DirectVulkan/Renderer/VulkanRenderer.cpp | 110 ++++++++++++++++-- .../DirectVulkan/Renderer/VulkanRenderer.h | 11 +- .../MG_Remote/Server/StagedTextureStore.h | 10 ++ .../MG_Test/Wire/StagedTextureStoreTest.cpp | 21 ++++ docs/Disaggregated/notes/p7/magma-b4.md | 21 ++++ 7 files changed, 171 insertions(+), 13 deletions(-) create mode 100644 docs/Disaggregated/notes/p7/magma-b4.md diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/FrameContext.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/FrameContext.cpp index d9eda7bd..04c6da55 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/FrameContext.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/FrameContext.cpp @@ -307,8 +307,10 @@ namespace MobileGL::MG_Backend::DirectVulkan { if (result != VK_SUCCESS) { return result; } - // The slot's fence has been waited: every command buffer this slot - // submitted (including mid-frame flushes) has finished executing. + // Under disaggregation VulkanRenderer waits every registered fence + // through this slot's last submit before calling here. That aggregate + // proof covers mid-frame pooled-fence flushes too; the slot fence alone + // would not. Initialize's first acquire has no prior submissions. FreeRetiredCommandBuffers(frame); result = vkAcquireNextImageKHR(device, swapchain, timeout, frame.imageAvailableSemaphore, acquireFence, diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp index a4b9b841..55d5d174 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp @@ -808,8 +808,9 @@ namespace MobileGL::MG_Backend::DirectVulkan { // Nothing bumps the slice epoch for those other resources, so freeing the buffer // here left the streamed memo handing a destroyed VkBuffer to vkCmdBindIndexBuffer // (llvmpipe then faulted inside the draw; the Create/Flywheel indirect retrace died - // exactly this way). Mid-frame drains do not advance m_frameSerial, so they must not - // free arena storage either: the arena's own ResetFrame/BeginFrame is the point where + // exactly this way). Most mid-frame drains do not advance m_frameSerial; every + // eighth does, but only through BeginFrame after the arena/memo boundary work. + // The drain's per-resource sweep must not free arena storage: ResetFrame/BeginFrame is where // the slot's slices stop being reachable, and that is where these releases land. for (Uint32 frameIndex = 0; frameIndex < m_deferredBufferReleases.size(); ++frameIndex) { CollectDeferredReleases(frameIndex); diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp index f287b55a..d22e58d1 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp @@ -7063,6 +7063,10 @@ void main() { #if MOBILEGL_BUILD_DISAGGREGATED if (MG_Config::Transport != MG_Config::TransportMode::Monolith) { RewindWireDescriptorSetsIfDue(); + // The wire route returns before the monolith branch's draw-gated + // sweep. Dead wire texture/renderbuffer records otherwise live + // until the next frame boundary, which a long trace may not reach. + m_textureManager->CollectGarbage(); return SetupWireDraw(frame, mode, aspects, drawParams, pIndexBufferView); } #endif @@ -7685,6 +7689,7 @@ void main() { #if MOBILEGL_BUILD_DISAGGREGATED if (MG_Config::Transport != MG_Config::TransportMode::Monolith) { RewindWireDescriptorSetsIfDue(); + m_textureManager->CollectGarbage(); DispatchWireCompute(numGroupsX, numGroupsY, numGroupsZ); return; } @@ -13527,12 +13532,32 @@ void main() { return false; } if (m_device == VK_NULL_HANDLE || m_graphicsQueue == VK_NULL_HANDLE) { +#if MOBILEGL_BUILD_DISAGGREGATED + return IsFrameSerialComplete(serial); +#else return true; +#endif } - // Every submission is recorded with the frame serial it was made under, so the wait can be - // narrowed to the first submission at or past the requested serial instead of draining the - // whole queue. OnSubmitsCompletedUpTo calls NotifyFrameSerialComplete for every record it - // retires, so the completed-serial floor still advances correctly after one fence wait. +#if MOBILEGL_BUILD_DISAGGREGATED + // More than one submit can carry the same frame serial (mid-frame flush, + // then Present). The completed floor is clamped below any serial still + // held by an in-flight record; the first matching fence is not proof. + // OnSubmitsCompletedUpTo erases records, so copy fields and restart the + // search after every wait instead of continuing an invalidated iterator. + for (;;) { + const auto record = std::find_if(m_inFlightSubmits.begin(), m_inFlightSubmits.end(), + [serial](const auto& candidate) { + return candidate.frameSerial >= serial && candidate.fence != VK_NULL_HANDLE; + }); + if (record == m_inFlightSubmits.end()) break; + const Uint64 submitIndex = record->submitIndex; + if (!WaitForSubmitsUpTo(submitIndex, UINT64_MAX)) + break; // fall through to the queue drain below + TryDrainFrameTransients(); + if (IsFrameSerialComplete(serial)) return true; + } +#else + // Pull build's historical one-fence wait stays byte-identical. for (const auto& record : m_inFlightSubmits) { if (record.frameSerial < serial || record.fence == VK_NULL_HANDLE) { continue; @@ -13546,6 +13571,7 @@ void main() { TryDrainFrameTransients(); return true; } +#endif // No usable record - fall back to draining the graphics queue. This over-waits (bounded by // the in-flight frame count) but never deadlocks. @@ -13559,7 +13585,11 @@ void main() { // The queue was just drained; take the free frame-boundary drain when // nothing is recorded (present-less timer-query loops). No-op otherwise. TryDrainFrameTransients(); +#if MOBILEGL_BUILD_DISAGGREGATED + return IsFrameSerialComplete(serial); +#else return true; +#endif } Uint64 VulkanRenderer::GetSyncPointSubmitIndex() const { @@ -13573,7 +13603,12 @@ void main() { return false; } const auto& frame = m_frameContext.GetCurrent(); +#if MOBILEGL_BUILD_DISAGGREGATED + return frame.isCommandRecording || frame.hasCommandBufferRecorded || + frame.isPreCommandRecording || frame.hasPreCommandBufferRecorded; +#else return frame.isCommandRecording || frame.hasCommandBufferRecorded; +#endif } #if MOBILEGL_BUILD_DISAGGREGATED @@ -13639,6 +13674,29 @@ void main() { } } +#if MOBILEGL_BUILD_DISAGGREGATED + Bool VulkanRenderer::WaitForSubmitsUpTo(Uint64 submitIndex, Uint64 timeoutNs) { + if (submitIndex <= m_completedSubmitCounter) return true; + if (submitIndex > m_submitCounter || m_device == VK_NULL_HANDLE) return false; + Vector fences; + for (const auto& record : m_inFlightSubmits) { + if (record.submitIndex > submitIndex) break; + if (record.fence == VK_NULL_HANDLE) return false; + fences.push_back(record.fence); + } + if (fences.empty()) return false; + const VkResult result = vkWaitForFences(m_device, static_cast(fences.size()), + fences.data(), VK_TRUE, timeoutNs); + if (result == VK_SUCCESS) { + OnSubmitsCompletedUpTo(submitIndex); + return true; + } + if (result != VK_TIMEOUT) + MGLOG_E_ONCE("WaitForSubmitsUpTo: vkWaitForFences returned %d", result); + return false; + } +#endif + void VulkanRenderer::OnSubmitsCompletedUpTo(Uint64 submitIndex) { m_completedSubmitCounter = std::max(m_completedSubmitCounter, submitIndex); #if MOBILEGL_BUILD_DISAGGREGATED @@ -13945,7 +14003,12 @@ void main() { // pooled fences and mid-frame command buffers. RefreshCompletedSubmits(); auto& frame = m_frameContext.GetCurrent(); +#if MOBILEGL_BUILD_DISAGGREGATED + if (!frame.isCommandRecording && !frame.hasCommandBufferRecorded && + !frame.isPreCommandRecording && !frame.hasPreCommandBufferRecorded) { +#else if (!frame.isCommandRecording && !frame.hasCommandBufferRecorded) { +#endif // GL flush semantics still demand batched texture uploads start // executing in finite time even when no draw was recorded. if (m_textureManager) { @@ -13988,9 +14051,15 @@ void main() { const VkResult retireResult = m_frameContext.RetireCurrentCommandBuffer(submittingPreCommandBuffer); if (retireResult != VK_SUCCESS) { MGLOG_E_ONCE("FlushPendingCommands: RetireCurrentCommandBuffer returned %d; draining submission", retireResult); +#if MOBILEGL_BUILD_DISAGGREGATED + if (WaitForSubmitsUpTo(m_submitCounter, UINT64_MAX)) { + // Every registered fence through this submission was waited. + } else if (vkQueueWaitIdle(m_graphicsQueue) == VK_SUCCESS) { +#else if (vkWaitForFences(m_device, 1, &fence, VK_TRUE, UINT64_MAX) == VK_SUCCESS) { OnSubmitsCompletedUpTo(m_submitCounter); } else if (vkQueueWaitIdle(m_graphicsQueue) == VK_SUCCESS) { +#endif m_bufferManager.NotifyDeviceIdle(); OnSubmitsCompletedUpTo(m_submitCounter); } else { @@ -14028,6 +14097,12 @@ void main() { return false; } } +#if MOBILEGL_BUILD_DISAGGREGATED + if (!WaitForSubmitsUpTo(submitIndex, timeoutNs)) return false; + // A blocking wait can drain a present-less loop's frame transients. + TryDrainFrameTransients(); + return true; +#else for (const auto& record : m_inFlightSubmits) { if (record.submitIndex >= submitIndex) { const VkResult result = vkWaitForFences(m_device, 1, &record.fence, VK_TRUE, timeoutNs); @@ -14048,6 +14123,7 @@ void main() { // No in-flight record at or beyond the index: it was already observed // complete via a fence wait on a later submission. return true; +#endif } void VulkanRenderer::OnFrameCommandRecordingBegan(VkCommandBuffer commandBuffer) { @@ -14196,6 +14272,15 @@ void main() { return; } m_presentSuspended = false; +#if MOBILEGL_BUILD_DISAGGREGATED + // A slot's Present fence alone does not prove its older pooled + // flushes retired. Wait the full prefix before FrameContext may + // free its retired command buffers and reset the slot fence. + const Uint64 slotSubmit = m_frameContext.GetCurrent().lastSubmitIndex; + if (slotSubmit > m_completedSubmitCounter && + !WaitForSubmitsUpTo(slotSubmit, UINT64_MAX)) + MagmaWireFatal("deferred-acquire-submit-wait"); +#endif const VkResult acquireResult = m_frameContext.WaitAndAcquireNextImage(m_device, m_swapchainObject.GetHandle(), m_imageIndexAcquired); if (acquireResult == VK_SUBOPTIMAL_KHR) { @@ -14326,6 +14411,16 @@ void main() { // 3) Advance frame slot. m_frameContext.AdvanceToNext(); +#if MOBILEGL_BUILD_DISAGGREGATED + // FrameContext waits then resets the slot fence and frees retired + // command buffers. A later Present fence is not aggregate proof for + // earlier pooled flushes, so wait every registered submission in this + // slot's prefix BEFORE the reset/free happens. + const Uint64 slotSubmit = m_frameContext.GetCurrent().lastSubmitIndex; + if (slotSubmit > m_completedSubmitCounter && + !WaitForSubmitsUpTo(slotSubmit, UINT64_MAX)) + MagmaWireFatal("present-slot-submit-wait"); +#endif // 4) Wait/reset/acquire for next frame. result = m_frameContext.WaitAndAcquireNextImage(m_device, m_swapchainObject.GetHandle(), m_imageIndexAcquired); if (result == VK_SUBOPTIMAL_KHR) { @@ -14350,10 +14445,9 @@ void main() { m_frameContext.WaitAndAcquireNextImage(m_device, m_swapchainObject.GetHandle(), m_imageIndexAcquired); } VK_VERIFY(result, "Present, vkAcquireNextImageKHR"); - // The acquired slot's fence has been waited: its last submission - // (and, in queue order, everything before it) is complete. The frame - // serials those submissions carried advance the buffer-manager floor - // inside OnSubmitsCompletedUpTo. + // Pull keeps its historical slot-fence inference. In a disaggregated + // build the aggregate wait above already retired the registered prefix + // before FrameContext reset the slot fence; this repeat is idempotent. OnSubmitsCompletedUpTo(m_frameContext.GetCurrent().lastSubmitIndex); CollectDeferredDepthMipmapCleanup(m_frameContext.GetCurrentFrameIndex()); #if MOBILEGL_BUILD_DISAGGREGATED diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h index 48d18027..a3093d50 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h @@ -147,7 +147,11 @@ namespace MobileGL::MG_Backend::DirectVulkan { #if MOBILEGL_BUILD_DISAGGREGATED VkBufferManager& GetWireBufferManager() { return m_bufferManager; } Bool FlushWirePendingCommandsForTextureUpdate() { - return !HasPendingRecordedWork() || FlushPendingCommands(); + // A new texture upload must not race graphics work that samples + // the old texels. No current recording does not imply no in-flight + // submission: finish the recorded work, then wait its whole prefix. + if (HasPendingRecordedWork() && !FlushPendingCommands()) return false; + return WaitForSubmitsUpTo(m_submitCounter, UINT64_MAX); } // P7 wave 2 package B3: the submission that will carry whatever is recorded NEXT. // RetireWireObjects (WireDraw.inc) already tags future objects with exactly this, and @@ -685,6 +689,11 @@ namespace MobileGL::MG_Backend::DirectVulkan { // was waited or the device was idled); drops their records and // recycles pooled fences. void OnSubmitsCompletedUpTo(Uint64 submitIndex); +#if MOBILEGL_BUILD_DISAGGREGATED + // A later fence alone cannot retire earlier submissions. Wait for the + // whole still-live prefix before advancing the completed submit floor. + Bool WaitForSubmitsUpTo(Uint64 submitIndex, Uint64 timeoutNs); +#endif VkFence AcquirePooledSubmitFence(); void DestroySubmitFencePool(); Bool HasPendingRecordedWork() const; diff --git a/MobileGL/MG_Remote/Server/StagedTextureStore.h b/MobileGL/MG_Remote/Server/StagedTextureStore.h index 0251ace1..284979e0 100644 --- a/MobileGL/MG_Remote/Server/StagedTextureStore.h +++ b/MobileGL/MG_Remote/Server/StagedTextureStore.h @@ -236,6 +236,16 @@ namespace MobileGL::MG_Remote::Server { if (!m_copies) return nullptr; const std::lock_guard lock(m_mutex); LevelShadow& shadow = m_shadows[key].Levels[PackLevel(uploadTarget, level)]; +#if MOBILEGL_BUILD_DISAGGREGATED + // A run can be the first piece of a redefined level. Old coverage + // belongs to the previous coordinate system even when the new run + // happens to touch its old range; retaining it invents bytes that + // the current definition never supplied. + if (!shadow.Defined || shadow.Extent != extent) { + shadow.Bytes.clear(); + shadow.Covered.clear(); + } +#endif shadow.Extent = extent; shadow.Defined = true; shadow.GpuDirty = false; diff --git a/MobileGL/MG_Test/Wire/StagedTextureStoreTest.cpp b/MobileGL/MG_Test/Wire/StagedTextureStoreTest.cpp index 19298add..488d32ae 100644 --- a/MobileGL/MG_Test/Wire/StagedTextureStoreTest.cpp +++ b/MobileGL/MG_Test/Wire/StagedTextureStoreTest.cpp @@ -307,6 +307,27 @@ TEST(StagedTextureStoreTest, StageChunkRunsAssembleTheLevelImageAndAGapIsNotCove EXPECT_EQ(store.LevelByteSize(key, kTex2DTarget, 0), 0u); } +#if MOBILEGL_BUILD_DISAGGREGATED +TEST(StagedTextureStoreTest, AdoptRunExtentMoveDoesNotInheritOldCoverage) { + Server::StagedTextureStore store(/*copies=*/true); + const Uint64 key = Server::StagedTextureStore::KeyForHandle(TestHandle(15, 1)); + Vector oldRun(16, 0x44); + Vector newRun(16, 0x99); + store.AdoptRun(key, kTex2DTarget, 0, IntVec3{4, 4, 1}, 0, oldRun.data(), oldRun.size()); + ASSERT_TRUE(store.IsCovered(key, kTex2DTarget, 0)); + store.AdoptRun(key, kTex2DTarget, 0, IntVec3{8, 4, 1}, 16, newRun.data(), newRun.size()); + EXPECT_FALSE(store.IsCovered(key, kTex2DTarget, 0)) + << "the old level's [0,16) run was mistaken for bytes of the new extent"; + EXPECT_EQ(store.LevelCoveredRunCount(key, kTex2DTarget, 0), 1u); + Vector newLower(16, 0xAA); + store.AdoptRun(key, kTex2DTarget, 0, IntVec3{8, 4, 1}, 0, newLower.data(), newLower.size()); + const Uint8* base = store.RequireLevelBytes(key, kTex2DTarget, 0, "unit_extent_move"); + ASSERT_NE(base, nullptr); + EXPECT_EQ(base[0], 0xAAu); + EXPECT_EQ(base[16], 0x99u); +} +#endif + // WHICH OF THE TWO STAGED-RUN SHAPES A RECORD CARRIES, read off the run's own LENGTH because the // wire layout is frozen and carries no field for it. The whole-level run (the shape this half has // always staged) is the level's complete shadow and begins at the level's first byte; a slab's run diff --git a/docs/Disaggregated/notes/p7/magma-b4.md b/docs/Disaggregated/notes/p7/magma-b4.md new file mode 100644 index 00000000..433b3332 --- /dev/null +++ b/docs/Disaggregated/notes/p7/magma-b4.md @@ -0,0 +1,21 @@ +# P7 B4：Magma 提交等待与长帧清理（草稿，未落地） + +基线：`0e16ca27`，独立 worktree `~/w7/p7-b4`。本包仍在实现／审查，以下是当前源码状态，不是集成树结论。 + +## 提交与 frame serial + +`WaitForSubmitsUpTo(index, timeout)` 收集 tracker 中所有 `submitIndex <= index` 的在飞 fence，以 `vkWaitForFences(..., VK_TRUE)` 等全部信号之后才推进完成计数。`WaitForSubmitIndex` 不再拿一条较晚 fence 代替整个前缀。`WaitForFrameSerial` 等待第一条匹配记录后重查地板；若同一 serial 仍被第二条提交持有，就从头重扫（不能在 `OnSubmitsCompletedUpTo` erase 后继续旧迭代器，也不再读 `record` 引用）。没有可用记录时才 queue idle，最终只按 `IsFrameSerialComplete` 答 true。 + +`Present` 在 `FrameContext::WaitAndAcquireNextImage` 重置槽 fence / 释放退休命令缓冲之前，先等该槽 `lastSubmitIndex` 的完整前缀；suspended 后首次 acquire 同样做。readback 原本已有全 fence 等待，保持；mid-frame flush 的退化重用分支改走聚合等待。`HasPendingRecordedWork` 在 disagg 构建下也计 pre-pass 录制，`FlushPendingCommands` 的空批判断对应扩展。这样 `TryDrainFrameTransients` 的 wire `CollectWireObjects(all=true)` 不会在 pre-pass 尚未提交时把未来标签视作空闲。pull 分支的可执行语句保持原样。 + +`FlushWirePendingCommandsForTextureUpdate` 在提交当前录制后等所有在飞 graphics 提交，才允许新的独立上传批修改旧 draw 可能采样的图像。这里可能有明显的 stall 代价，按规则只记录性能，不设门；需在长 trace 上实测。该修正还没有针对单 fence 错退的确定性负控，当前包不得以现有绿数直接落地。 + +## 长帧回收与 staged bytes + +wire draw / dispatch 入口现在调用 `VkTextureManager::CollectGarbage`，使每 256 次的 `pruneWire` 能在一帧内执行。`StagedTextureStore::AdoptRun` 遇到 extent 变化时清旧 `Bytes` 与 `Covered`：单测先送旧 [0,16)，再送新 extent 的 [16,32)，不能把两种坐标系拼成已覆盖。禁用清理时该单测在旧范围误判覆盖处红，恢复后绿。 + +## 已验证／待做 + +完整 `MobileGL`、`MobileGLServer`、`MobileGLIntegrationTest` 与 `StagedTextureStoreTest` 构建通过；新 extent 单测绿，红一次按上述覆盖断言红；Magma split/spawn/tcp 为 101/80/71、full-split 539，全绿；OpenRA DirectVulkan inproc / spawn 都 ssim 1.000000、0 Fatal、0 `unsound-serial-complete`。原始日志 `~/w7/logs/p7b4/`、`~/w7/logs/p7-gate-b4a-retrace/`。 + +待做：聚合等待的可证伪负控；G1 / 普查 / 棘轮 / parity / 完整八车道 / verify 门；集成者对 pipe 源码复审；与 M3 的 `VulkanRenderer.cpp` 冲突合并；真机回归。未完成前不推送本包。