diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/UniformManager.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/UniformManager.cpp index ae55c6b0..e0e0b26e 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/UniformManager.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/UniformManager.cpp @@ -674,6 +674,40 @@ namespace MobileGL::MG_Backend::DirectVulkan { } } +#if MOBILEGL_BUILD_DISAGGREGATED + Bool UniformManager::WireDescriptorSetBudgetReached(Uint32 frameIndex) const { + MOBILEGL_ASSERT(frameIndex < m_frames.size(), "WireDescriptorSetBudgetReached invalid frame index"); + return m_frames[frameIndex].allocatedSetsThisFrame >= kWireDescriptorSetBudget; + } + + SizeT UniformManager::RewindWireDescriptorSets(Uint32 frameIndex) { + MOBILEGL_ASSERT(frameIndex < m_frames.size(), "RewindWireDescriptorSets invalid frame index"); + auto& frame = m_frames[frameIndex]; + // The caller proved that the last graphics submit using these sets has + // retired and that no unsubmitted command buffer still references them. + // Keep the pools and sets: only the per-layout write cursors rewind. + // Image and buffer views stay alive until the regular frame boundary. + m_peakDescriptorSetsObserved = std::max(m_peakDescriptorSetsObserved, frame.peakAllocatedSetsThisFrame); + SizeT cachedSets = 0; + for (auto& entry : frame.descriptorSetCacheByLayout) { + cachedSets += entry.second.sets.size(); + entry.second.cursor = 0; + } + frame.allocatedSetsThisFrame = 0; + frame.peakAllocatedSetsThisFrame = 0; + for (auto& entry : m_descriptorReuseMemo) entry.valid = false; + m_fastRebindMemo.valid = false; + m_lastBindValid = false; + const Uint32 touchedBindings = + std::min(m_samplerResolveMemoHighWater, static_cast(m_samplerResolveMemo.size())); + for (Uint32 binding = 0; binding < touchedBindings; ++binding) { + m_samplerResolveMemo[binding].valid = false; + m_samplerResolveMemo[binding].infoValid = false; + } + return cachedSets; + } +#endif + void UniformManager::OnDescriptorSetLayoutDestroyed(VkDescriptorSetLayout descriptorSetLayout) { SizeT purgedSets = 0; for (auto& frame : m_frames) { diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/UniformManager.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/UniformManager.h index 990abb10..05beb507 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/UniformManager.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/UniformManager.h @@ -54,6 +54,14 @@ namespace MobileGL::MG_Backend::DirectVulkan { void Shutdown(); void BeginFrame(Uint32 frameIndex); +#if MOBILEGL_BUILD_DISAGGREGATED + // A present-less wire frame may run arbitrarily many draws. Before its + // descriptor cursor reaches this budget, the renderer retires the + // submission that last used the sets and rewinds every layout cursor. + static constexpr Uint32 kWireDescriptorSetBudget = 2048; + Bool WireDescriptorSetBudgetReached(Uint32 frameIndex) const; + SizeT RewindWireDescriptorSets(Uint32 frameIndex); +#endif // A command buffer (re)began recording: descriptor bindings recorded into // the previous buffer do not carry over, so drop the bind-dedup shadow. void OnCommandBufferBoundary() { m_lastBindValid = false; } diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp index 41339e71..143ffa7a 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp @@ -7061,8 +7061,10 @@ void main() { const DrawCmdParam& drawParams, const IndexBufferView* pIndexBufferView) { #if MOBILEGL_BUILD_DISAGGREGATED - if (MG_Config::Transport != MG_Config::TransportMode::Monolith) + if (MG_Config::Transport != MG_Config::TransportMode::Monolith) { + RewindWireDescriptorSetsIfDue(); return SetupWireDraw(frame, mode, aspects, drawParams, pIndexBufferView); + } #endif // Sync each sampled texture at most once across this whole draw: the layout // probe loop, the post-transition loop, and ResolveSamplerDescriptor would @@ -7682,6 +7684,7 @@ void main() { void VulkanRenderer::DispatchCompute(GLuint numGroupsX, GLuint numGroupsY, GLuint numGroupsZ) { #if MOBILEGL_BUILD_DISAGGREGATED if (MG_Config::Transport != MG_Config::TransportMode::Monolith) { + RewindWireDescriptorSetsIfDue(); DispatchWireCompute(numGroupsX, numGroupsY, numGroupsZ); return; } @@ -13573,6 +13576,36 @@ void main() { return frame.isCommandRecording || frame.hasCommandBufferRecorded; } +#if MOBILEGL_BUILD_DISAGGREGATED + void VulkanRenderer::RewindWireDescriptorSetsIfDue() { + if (!m_uniformManager || m_frameContext.GetFrameCount() == 0) return; + const Uint32 frameIndex = m_frameContext.GetCurrentFrameIndex(); + if (!m_uniformManager->WireDescriptorSetBudgetReached(frameIndex)) return; + + // A set cannot be rewritten while a recorded or submitted command + // buffer might still read it. This wire-only boundary first submits + // the current recording, then waits for that exact submit index. + if (HasPendingRecordedWork() && !FlushPendingCommands()) + MagmaWireFatal("descriptor-rewind-flush"); + const Uint64 through = m_submitCounter; + if (through > m_completedSubmitCounter) { + // B4 has not yet made every old retirement site aggregate-safe. + // Queue idle proves even a submission a previous single-fence path + // prematurely removed from m_inFlightSubmits has finished; only + // then may any layout cursor be rewound and its sets rewritten. + if (vkQueueWaitIdle(m_graphicsQueue) != VK_SUCCESS) + MagmaWireFatal("descriptor-rewind-wait"); + OnSubmitsCompletedUpTo(through); + } + if (HasPendingRecordedWork() || m_completedSubmitCounter < through) + MagmaWireFatal("descriptor-rewind-proof"); + const SizeT cachedSets = m_uniformManager->RewindWireDescriptorSets(frameIndex); + MGLOG_I("Magma descriptor rewind: frame=%u retiredSubmit=%llu budget=%u cached=%zu", + frameIndex, static_cast(through), + UniformManager::kWireDescriptorSetBudget, cachedSets); + } +#endif + Bool VulkanRenderer::IsSubmitIndexComplete(Uint64 submitIndex) { if (submitIndex <= m_completedSubmitCounter) { return true; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h index ef33e7b8..48d18027 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h @@ -628,6 +628,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { Vector m_wireRetiredObjects; Uint32 m_wirePreparationDepth = 0; void DispatchWireCompute(GLuint x, GLuint y, GLuint z); + void RewindWireDescriptorSetsIfDue(); UniquePtr m_wireDrawPass; Vector m_wireDrawViews; VkPipeline GetOrCreatePipelineWithInput(GLenum mode, const MagmaProgramSource& program, diff --git a/MobileGL/MG_IntegrationTest/CMakeLists.txt b/MobileGL/MG_IntegrationTest/CMakeLists.txt index 3095993d..b7826fc7 100644 --- a/MobileGL/MG_IntegrationTest/CMakeLists.txt +++ b/MobileGL/MG_IntegrationTest/CMakeLists.txt @@ -3469,13 +3469,14 @@ if (MOBILEGL_BUILD_DISAGGREGATED) mgl_itest_magma_arm_shape(${mglItestArm} reclaim) mgl_itest_join_environment(MGL_ITEST_MAGMA_RECLAIM_ENVIRONMENT ${MGL_ITEST_MAGMA_RECLAIM_FLAVOUR_ENV} ${mglItestArmTransportEnv}) - # Round 2 (ID-P7-43) adds the fourth: the reclaim's own memo hazard. It reads no gauge, but it - # is the same fixture, the same in-frame shape and the same two arms, so it rides this list. + # Round 2 (ID-P7-43) adds the fourth: the reclaim's own memo hazard. M3 adds the fifth: + # fixed-store descriptor churn in a long frame. Both use this private-log fixture and two arms. foreach(reclaimCase RespecifiesWithNoDrawBetweenKeepTheLiveStoreCountBounded RespecifyAndDrawEachStoreInOneFrameStaysWithinTheDeferredBudget ManySmallRespecifyAndDrawRoundsStayUnderTheStoreCountCeiling - ADrawAfterTheEarlyReclaimFollowsTheNewStoreNotTheMemoizedHandle) + ADrawAfterTheEarlyReclaimFollowsTheNewStoreNotTheMemoizedHandle + DescriptorSetsRewindInsideOneLongFrameAfterTheirSubmitRetires) gtest_discover_tests(MobileGLIntegrationTest TEST_PREFIX "DirectVulkan.${mglItestArm}.Reclaim." TEST_FILTER "MagmaWireReclaimScenario.${reclaimCase}" diff --git a/MobileGL/MG_IntegrationTest/Scenarios/MagmaWireReclaimScenario.cpp b/MobileGL/MG_IntegrationTest/Scenarios/MagmaWireReclaimScenario.cpp index 0f6d1e57..729a1e01 100644 --- a/MobileGL/MG_IntegrationTest/Scenarios/MagmaWireReclaimScenario.cpp +++ b/MobileGL/MG_IntegrationTest/Scenarios/MagmaWireReclaimScenario.cpp @@ -17,7 +17,7 @@ // split server held 25,923 dead stores against 28 live wire buffers and 41k mappings; on the // Redmi it died in scudo at 1.2 GiB while the monolith passed at 807 MiB. // -// WHAT THE THREE CASES PIN, all inside ONE frame (no swap between the first and the last +// WHAT THE CASES PIN, all inside ONE frame (no swap between the first and the last // respecify), because that is the only shape in which the bug existed: // * RespecifiesWithNoDrawBetween... - the stores no GPU command ever named. They are dead the // moment they are orphaned, so the live store count must stay a small multiple of the wire @@ -43,8 +43,10 @@ // whose environment the lane cannot set and whose log this process cannot read // (scripts/ci/spawn_lane_parity.py names the exception). +#include #include #include +#include #include #include @@ -99,6 +101,7 @@ namespace MGITest { constexpr std::size_t kAbaPinStoreBytes = static_cast(kMiB); constexpr std::size_t kAbaNewStoreBytes = 64 * 1024; constexpr GLuint64 kAbaWaitNs = 2000000000ull; // 2 s: a 64x16 draw on lavapipe retires in ms + constexpr int kDescriptorDraws = 2049; // one past UniformManager's wire epoch budget constexpr const char* kVertex = R"(#version 330 core layout(location = 0) in vec2 aPos; @@ -118,6 +121,14 @@ layout(std140) uniform Block { vec4 color; }; out vec4 fragColor; void main() { fragColor = color; })"; + constexpr const char* kStorageVertex = R"(#version 430 core +layout(location = 0) in vec2 aPos; +void main() { gl_Position = vec4(aPos, 0.0, 1.0); })"; + constexpr const char* kStorageFragment = R"(#version 430 core +layout(std430, binding = 0) readonly buffer Colors { vec4 color[]; }; +out vec4 fragColor; +void main() { fragColor = color[0]; })"; + // Two triangles covering pixel columns [x0, x1) over the whole height, in NDC. void WriteStripQuad(float* out, int x0, int x1) { const float l = -1.0f + 2.0f * static_cast(x0) / kWidth; @@ -506,5 +517,62 @@ void main() { fragColor = color; })"; static_cast(PipeStatsWindow::CounterOrAbsent(window, "wlivepk"))); } + // M3: a fixed SSBO store and one live layout, but 2049 distinct aligned + // windows force 2049 descriptor writes before a present. M2's buffer-store + // ceiling cannot help: no store is orphaned. The first 2048 sets must be + // retired by submit index before their per-layout cursor rewinds, and the + // last draw must read the last window rather than a set still in flight. + TEST_F(MagmaWireReclaimScenario, DescriptorSetsRewindInsideOneLongFrameAfterTheirSubmitRetires) { + if (!Ready() || IsSkipped()) return; + std::string error; + const GLuint program = CompileProgram(kStorageVertex, kStorageFragment, &error); + ASSERT_NE(program, 0u) << error; + const PipeStatsWindow::LogMark mark = SetupFrame(); + ClearTo(0.0f, 0.0f, 0.0f, 0.0f); + std::vector quad(kUndrawnStoreBytes, 0); + glBindBuffer(GL_ARRAY_BUFFER, m_buffer); + Respecify(quad, 0, kWidth); + + GLint alignment = 0; + glGetIntegerv(GL_SHADER_STORAGE_BUFFER_OFFSET_ALIGNMENT, &alignment); + ASSERT_GT(alignment, 0); + const std::size_t stride = static_cast(std::max(alignment, 16)); + std::vector bytes(stride * kDescriptorDraws, 0); + const Rgba8 last{60, 200, 90, 255}; + for (int i = 0; i < kDescriptorDraws; ++i) { + const float color[4] = {i + 1 == kDescriptorDraws ? last.r / 255.0f : 200.0f / 255.0f, + i + 1 == kDescriptorDraws ? last.g / 255.0f : 60.0f / 255.0f, + i + 1 == kDescriptorDraws ? last.b / 255.0f : 30.0f / 255.0f, 1.0f}; + std::memcpy(bytes.data() + stride * i, color, sizeof(color)); + } + GLuint storage = 0; + glGenBuffers(1, &storage); + glBindBuffer(GL_SHADER_STORAGE_BUFFER, storage); + glBufferData(GL_SHADER_STORAGE_BUFFER, static_cast(bytes.size()), bytes.data(), GL_STATIC_DRAW); + glUseProgram(program); + for (int i = 0; i < kDescriptorDraws; ++i) { + glBindBufferRange(GL_SHADER_STORAGE_BUFFER, 0, storage, + static_cast(stride * i), 16); + glDrawArrays(GL_TRIANGLES, 0, 6); + } + const Image image = ReadPixels(kWidth, kHeight); + EXPECT_EQ(FirstGLError(), GLenum(GL_NO_ERROR)); + EXPECT_TRUE(Near(image.At(kWidth / 2, kHeight / 2), last)) + << "the last SSBO window was not read after descriptor recycling"; + (void)CloseFrameAndReadServerWindow(mark); + const std::string serverLog = PipeStatsWindow::ReadServerLogSince(mark); + const std::string marker = "Magma descriptor rewind:"; + const std::size_t rewind = serverLog.find(marker); + ASSERT_NE(rewind, std::string::npos) << "2049 distinct descriptor writes in one frame did not rewind"; + const std::size_t cachedAt = serverLog.find(" cached=", rewind); + ASSERT_NE(cachedAt, std::string::npos) << serverLog.substr(rewind, 160); + const unsigned long long cached = std::strtoull(serverLog.c_str() + cachedAt + 8, nullptr, 10); + EXPECT_LE(cached, 2080ull) << "the fixed-layout cache grew beyond the 2048-set epoch: " + << serverLog.substr(rewind, 160); + RecordProperty("descriptor_cached_at_rewind", static_cast(cached)); + glDeleteBuffers(1, &storage); + glDeleteProgram(program); + } + } // namespace } // namespace MGITest diff --git a/docs/Disaggregated/notes/p7/magma-m3.md b/docs/Disaggregated/notes/p7/magma-m3.md new file mode 100644 index 00000000..cd49a607 --- /dev/null +++ b/docs/Disaggregated/notes/p7/magma-m3.md @@ -0,0 +1,19 @@ +# P7 M3：长帧内 descriptor set 游标回卷（草稿，待门） + +基线：集成树 `0e16ca27`。本包只在 disaggregated 构建的 Magma wire draw / compute 入口触发；pull 的语句与对象代码不变。设备 p7w6 在 M2 后已让 bsl-esc-menu 的 inproc / spawn 各三遍通过，但 M2 的 1,024 store 同步只是偶然给 descriptor 游标制造 drain，不能作为 descriptor set 自己的界。 + +## 机制 + +`UniformManager::AcquireDescriptorSet` 每次实际占用一个 layout 游标（从缓存取旧 set 或从 pool 分配新 set）都会增加当前 frame slot 的 `allocatedSetsThisFrame`。达到 2048 次时，下一笔 wire draw / dispatch 在任何新 descriptor 解析前调用 `RewindWireDescriptorSetsIfDue`： + +1. 若当前 graphics command buffer 已录制，`FlushPendingCommands` 把它作为新的 `submitIndex` 提交。 +2. 取此时的 `m_submitCounter` 为 `through`，等待 graphics queue idle 后才 `OnSubmitsCompletedUpTo(through)`；当前 B4 未落地，已有单 fence 退休站点可能从 tracker 中提前移除一条提交，单靠遍历 tracker 的 fence 不足以证明安全。队列 idle 覆盖这些已移除记录，并显式核对 `m_completedSubmitCounter >= through` 与没有尚未提交的当前录制。B4 全面修正聚合等待后可以再评估这笔保守等待的成本。 +3. `UniformManager::RewindWireDescriptorSets` 将每个仍存活 layout 的缓存 cursor 置零，保留 pool 和 set 供改写，清空 descriptor-reuse / fast-rebind / bind-dedup / sampler-resolve memo，并重置本 epoch 的计数。image/buffer view 不在此销毁，仍由原 frame 生命周期管理。 + +因此一个固定活 layout 的长帧不会因 draw 次数持续分配新 descriptor set。多个 layout 的缓存仍保存各自历史高水位；layout 销毁时沿用 `OnDescriptorSetLayoutDestroyed` 的逐 pool 释放。该界不替代 program/layout 数量本身的生命周期管理。 + +## 负控与门 + +`MagmaWireReclaimScenario.DescriptorSetsRewindInsideOneLongFrameAfterTheirSubmitRetires` 用一个 SSBO store、一个 program/layout，在一帧内把 2049 个对齐窗口逐个绑定后 draw，M2 的 orphan-store 上限不会触发。最后一个窗口颜色必须正确；server 日志须给出 submit-index 退休后的回卷，回卷时缓存数不超过 2080。禁用入口的回卷调用应红（无回卷行），恢复后 split / spawn 两臂绿。之后跑 G1 `.text` / 符号恒等、普查、棘轮、parity、八车道、OpenRA 回放与 verify 构建。性能只记录。 + +当前状态：`~/w7/p7-m3` 的完整 `MobileGL` / `MobileGLServer` / `MobileGLIntegrationTest` 构建通过；新用例 split 与 spawn 2/2 绿（各约 1.2 s，server 行均为 `retiredSubmit=2 budget=2048 cached=2048`）。红一次：仅删 draw 入口回卷调用，三目标重建后两臂都在「2049 distinct descriptor writes in one frame did not rewind」处红，像素断言没有先抢红；恢复源码与三目标重建后两臂再绿。原始日志 `~/w7/logs/p7m3/{green-test,red-test,reclaim-test}.log`。最终源码改用 graphics queue idle 作完整提交证明后，完整目标重建并再次全绿：pull G1 `.text=0xa52203`、符号 +0/−0；fatal census 79、棘轮 173、parity 0、wire-declines 审计 53/53/0 与自测 16/16；split 构建 unit 2436、integration-split 304、spawn 220、tcp 223、magma 102/81/71、full-split 540；verify 构建 unit 2436、integration-verify 1152、integration-verify-split 1086；OpenRA DirectVulkan inproc 与 spawn 两臂均 ssim 1.000000、0 `unsound-serial-complete`。门日志在 `~/w7/logs/p7m3/` 与 `~/w7/logs/p7-gate-m3-final-retrace/`。集成者在 pipe 源码上的审查、落地及设备回归仍待做。 diff --git a/scripts/ci/spawn_lane_parity.py b/scripts/ci/spawn_lane_parity.py index 9205e3aa..b2a73649 100644 --- a/scripts/ci/spawn_lane_parity.py +++ b/scripts/ci/spawn_lane_parity.py @@ -87,7 +87,8 @@ MAGMA_SERVER_ENV_KNOB_NO_TCP = (".ShaderMip1.", ".ShaderMip2.", ".DepthMip.", ".Reclaim.MagmaWireReclaimScenario.RespecifiesWithNoDrawBetweenKeepTheLiveStoreCountBounded", ".Reclaim.MagmaWireReclaimScenario.RespecifyAndDrawEachStoreInOneFrameStaysWithinTheDeferredBudget", ".Reclaim.MagmaWireReclaimScenario.ManySmallRespecifyAndDrawRoundsStayUnderTheStoreCountCeiling", - ".Reclaim.MagmaWireReclaimScenario.ADrawAfterTheEarlyReclaimFollowsTheNewStoreNotTheMemoizedHandle") + ".Reclaim.MagmaWireReclaimScenario.ADrawAfterTheEarlyReclaimFollowsTheNewStoreNotTheMemoizedHandle", + ".Reclaim.MagmaWireReclaimScenario.DescriptorSetsRewindInsideOneLongFrameAfterTheirSubmitRetires") def lane_names(build_dir, label):