diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp index 97214cba..dc32cf44 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp @@ -5308,6 +5308,22 @@ void main() { return rsp.PrimitiveRestartIndex <= MG_Util::FixedRestartIndexForGLType(pIndexBufferView->indexType); } +#if MOBILEGL_BUILD_DISAGGREGATED + // MOBILEGL_TEST_PIPELINE_CREATE_DELAY_MS - milliseconds to spend in a pipeline-creation MISS + // on the wire arm. See the call site below for why it exists: the device's divergence needs a + // COLD driver pipeline cache, and this host's is always warm, so the only way to reproduce + // what a cold cache costs is to buy the time. Read once; inert unless set; compiled only into + // the disaggregated build, so the monolith image G1 pins never carries it. + static Uint32 MagmaTestPipelineCreateDelayMs() { + static const Uint32 milliseconds = [] { + const char* value = std::getenv("MOBILEGL_TEST_PIPELINE_CREATE_DELAY_MS"); + if (!value || !*value) return 0u; + return static_cast(std::strtoul(value, nullptr, 10)); + }(); + return milliseconds; + } +#endif + VkPipeline VulkanRenderer::GetOrCreatePipeline( GLenum mode, const MagmaProgramSource& program, @@ -6114,6 +6130,22 @@ void main() { MG_Util::ConvertBlendEquationToVkEnum(alphaEquation), attachmentColorWriteMask); } +#if MOBILEGL_BUILD_DISAGGREGATED + // P7 wave 2-B: MOBILEGL_TEST_PIPELINE_CREATE_DELAY_MS. + // + // The Redmi's OpenRA divergence (exit gate 3) is timing-shaped and reproduces ONLY with a + // cold driver pipeline cache: three cleared-cache runs give a bit-identical 0.976494 / + // 14658 mismatched pixels, while a warm cache gives 1.000000 from the second run on. What + // a cold cache changes is how long vkCreateGraphicsPipelines takes on the apply thread, + // so this knob buys that time on a host whose cache is always warm. INERT unless set, and + // compiled only into the disaggregated build - the monolith image, which G1 pins, never + // sees these lines. The delay goes on the MISS path only: a memo or cache hit is not + // what is slow on the device either. + if (const Uint32 delayMs = MagmaTestPipelineCreateDelayMs(); + delayMs != 0 && MG_Config::Transport != MG_Config::TransportMode::Monolith) { + std::this_thread::sleep_for(std::chrono::milliseconds(delayMs)); + } +#endif VkPipeline pipeline = m_pipelineFactory->GetOrCreatePipeline(payload); if (pipeline != VK_NULL_HANDLE) { PipelineMemoEntry& entry = m_pipelineMemo[m_pipelineMemoNext]; @@ -10792,7 +10824,47 @@ void main() { return false; } + // EVERY OUTSTANDING SUBMISSION, NOT ONLY THE ONE JUST MADE (P7 wave 2-B, exit gate 3). + // + // This used to wait one fence and then call OnSubmitsCompletedUpTo(frame.lastSubmitIndex), + // and the comment below said the wait "proved every submission complete". On the monolith + // arm that is true - nothing flushes before a readback, so the frame's draws and the copy + // are one submission under one fence. On the WIRE arm it is not: ReadWirePixels calls + // FlushPendingCommands first, which submits the draws under a POOLED FENCE OF ITS OWN and + // returns without waiting, so the copy is a later, separately fenced submission. Two + // submissions on one queue start in order; they do not finish in order unless something + // makes them, and a fence says nothing about a submission it does not belong to. + // + // Measured, on this host, at the copy: submit=72 completed=71. The old code then called + // OnSubmitsCompletedUpTo(73), which RECORDED 72 as complete and retired its fence - so + // after the fact nothing could even observe that it had not been waited for. + // + // The cost is nothing a readback does not already pay: it is a full CPU stall by + // construction, and the fences gathered here are exactly the ones already in flight. + // + // UNDER #if, although this function serves both arms of the disaggregated build: the PULL + // build has no wire arm, cannot flush before a readback, and is the image G1 pins to + // 0xa52203. Its single-fence wait stays the statement it always was. +#if MOBILEGL_BUILD_DISAGGREGATED + Vector readbackFences; + readbackFences.reserve(m_inFlightSubmits.size() + 1); + for (const auto& record : m_inFlightSubmits) { + if (record.fence != VK_NULL_HANDLE && record.submitIndex <= frame.lastSubmitIndex) { + readbackFences.push_back(record.fence); + } + } + if (readbackFences.empty()) { + readbackFences.push_back(frame.imageInFlightFence); + } + // Measured here on the host (OpenRA, DirectVulkan, inproc, at the frame readback): + // fences=2 upTo=73 submitCounter=73 completed=71. Two fences, because submission 72 - + // the frame's draws, flushed by ReadWirePixels under a pooled fence - was still + // outstanding; the code this replaces waited on one. + VkResult result = vkWaitForFences(m_device, static_cast(readbackFences.size()), + readbackFences.data(), VK_TRUE, UINT64_MAX); +#else VkResult result = vkWaitForFences(m_device, 1, &frame.imageInFlightFence, VK_TRUE, UINT64_MAX); +#endif if (result != VK_SUCCESS) { MGLOG_E_ONCE("DirectVulkan readback: vkWaitForFences returned %d", result); return false; @@ -10806,8 +10878,8 @@ void main() { frame.hasCommandBufferRecorded = false; frame.isCommandRecording = false; - // The wait proved every submission complete, so the full frame-boundary - // drain applies: descriptor cursors, transient arenas, deferred + // The wait above covered every submission at or below this one, so the full + // frame-boundary drain applies: descriptor cursors, transient arenas, deferred // texture/buffer releases, retired command buffers and the converted // vertex-stream cache all rewind here, keeping present-less readback // loops bounded (Present is the only other drain point). diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h index 54422ee1..46412be5 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h @@ -517,6 +517,9 @@ namespace MobileGL::MG_Backend::DirectVulkan { struct WireDepthMipmapResources; WireDepthMipmapResources* m_wireDepthMipmapResources = nullptr; void DestroyWireDepthMipmapResources(); + // P7 wave 2-B (exit gate 3): the readback's own write-visibility barrier, recorded before + // the copy instead of inferred from the image's tracked layout. See WireFramebuffer.inc. + void RecordWireReadbackWriteBarrier(); void ReadWirePixels(GLint x, GLint y, GLsizei width, GLsizei height, GLenum format, GLenum type, void* pixels); void GenerateWireMipmap(); diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireFramebuffer.inc b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireFramebuffer.inc index 4d3ed87b..8f7d1fe5 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireFramebuffer.inc +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireFramebuffer.inc @@ -309,6 +309,43 @@ if (payload.mask & GL_STENCIL_BUFFER_BIT) clearSurface(fbo.Stencil, VK_IMAGE_ASPECT_STENCIL_BIT, 0); } + // P7 wave 2-B (exit gate 3). THE READBACK SAYS WHAT IT IS WAITING FOR INSTEAD OF INFERRING IT. + // + // Every other transition in this file derives its barrier's SOURCE scope from the image's + // tracked layout, through GetImageTransitionSourceState (VulkanRenderer.cpp). That is right + // for a transition - but two of its arms, VK_IMAGE_LAYOUT_UNDEFINED and + // VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, return VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT with no access + // bits, which is Vulkan's spelling of AN EMPTY FIRST SCOPE: wait for nothing. For a + // transition out of UNDEFINED that is correct (the contents are being discarded); for a + // READBACK it is not, because the pixels being copied were written by the very commands the + // barrier declines to wait for. A just-presented default framebuffer sits in exactly the + // second of those layouts. + // + // It matters on the wire arm and not on the monolith one because of where the draws are. + // ReadWirePixels' FlushPendingCommands has already put the frame's draws on the queue under a + // pooled fence and returned WITHOUT waiting, so the copy recorded below lands in a LATER + // submission and this barrier is the only thing that could order the two. The monolith + // readback never flushes first: its draws and its copy are one submission under one fence. + // + // Measured on the host (OpenRA, DirectVulkan, inproc): at this point the counters read + // submit=72 completed=71 - the draws' submission is in flight and unwaited. The host does not + // diverge because its default framebuffer's tracked layout here is GENERAL, which takes + // GetImageTransitionSourceState's `default:` arm (ALL_COMMANDS / MEMORY_WRITE) and orders the + // copy by accident. The device's is not. + void VulkanRenderer::RecordWireReadbackWriteBarrier() { + auto& frame = m_frameContext.GetCurrent(); + if (!frame.isCommandRecording) m_frameContext.BeginCommandRecording(); + if (VkRenderPassManager::GetActiveRenderPass()) VkRenderPassManager::EndRenderPass(frame.commandBuffer); + // ALL_COMMANDS as the source scope reaches back across the submission boundary: a + // barrier's first scope is every command earlier in SUBMISSION order on this queue, not + // merely earlier in this command buffer. + VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; + barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_MEMORY_READ_BIT; + vkCmdPipelineBarrier(frame.commandBuffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &barrier, 0, nullptr, 0, nullptr); + } + void VulkanRenderer::ReadWirePixels(GLint x, GLint y, GLsizei width, GLsizei height, GLenum format, GLenum type, void* pixels) { if (width <= 0 || height <= 0) return; @@ -334,6 +371,7 @@ source.swapchainImageIndex); } m_textureManager->FlushPendingUploads(); + RecordWireReadbackWriteBarrier(); TransitionWireImage(source, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); if (aspect != VK_IMAGE_ASPECT_COLOR_BIT) { ReadDepthStencilImageToClient(source.image,source.format, diff --git a/MobileGL/MG_IntegrationTest/CMakeLists.txt b/MobileGL/MG_IntegrationTest/CMakeLists.txt index b324db35..72636fce 100644 --- a/MobileGL/MG_IntegrationTest/CMakeLists.txt +++ b/MobileGL/MG_IntegrationTest/CMakeLists.txt @@ -2734,6 +2734,34 @@ if (MOBILEGL_BUILD_DISAGGREGATED) # strongest sense: the monolith arm declines every in-place copy, so there is no # DirectVulkan. entry to register these beside - the wire arm is the one that performs # them, which is what §3.2 asks for. + # ---- P7 wave 2-B, EXIT GATE 3: the readback must not depend on how long a pipeline + # takes to create. -------------------------------------------------------------------- + # + # The Redmi's OpenRA divergence reproduces only with a COLD driver pipeline cache (three + # cleared-cache runs: a bit-identical ssim 0.976494 / 14658 mismatched pixels; warm: + # 1.000000 from the second run on). What a cold cache changes is how long + # vkCreateGraphicsPipelines takes on the apply thread, and + # MOBILEGL_TEST_PIPELINE_CREATE_DELAY_MS buys that time on a host whose cache is warm. + # + # THESE ENTRIES ARE A REGRESSION GATE, NOT THE REPRODUCTION. The delay alone does NOT + # make lavapipe diverge, and the reason is in WireFramebuffer.inc's + # RecordWireReadbackWriteBarrier: the host's default framebuffer is tracked as + # VK_IMAGE_LAYOUT_GENERAL at the readback, which takes GetImageTransitionSourceState's + # `default:` arm (ALL_COMMANDS / MEMORY_WRITE) and orders the copy by accident. What these + # entries do pin is that a readback which follows a draw still returns that draw's pixels + # when every pipeline creation on the path costs 25 ms - i.e. that the answer does not + # move with timing. + foreach(pipelineDelayCase VertexIdSamplerAndScalarUniformPixels + ComputeImageStoreFramebufferPixels + GenerateMipmapPixels) + gtest_discover_tests(MobileGLIntegrationTest + TEST_PREFIX "DirectVulkan.${mglItestArm}.PipelineDelay." + TEST_FILTER "F1WireScenario.${pipelineDelayCase}" + DISCOVERY_TIMEOUT 30 + PROPERTIES LABELS "${mglItestMagmaLabels}" TIMEOUT ${MGL_ITEST_TIMEOUT} + ${mglItestArmProperties} + ENVIRONMENT "${mglItestMagmaWireEnvironment}\;MOBILEGL_TEST_PIPELINE_CREATE_DELAY_MS=25\;MOBILEGL_LOG_FILE_PATH=${CMAKE_CURRENT_BINARY_DIR}/p7-pipelinedelay${mglItestArmLogTag}-${pipelineDelayCase}.log") + endforeach() # `default-color-blit-shape@P7`'s retirement. TWO ENTRIES OVER ONE CASE, and the second # is the only way its arm is reachable here: every surface this lane creates reports # VK_SURFACE_TRANSFORM_IDENTITY, so the scratch path that exists FOR rotation would diff --git a/docs/Disaggregated/notes/p7/magma-b.md b/docs/Disaggregated/notes/p7/magma-b.md index 4dcc3236..89fb0a29 100644 --- a/docs/Disaggregated/notes/p7/magma-b.md +++ b/docs/Disaggregated/notes/p7/magma-b.md @@ -21,6 +21,7 @@ | 2 | `depth-stencil-mipmap@P7` + 烘焙 (A)(D) | DEPTH-only **退役**（烘焙深度 mip 程序）；D\|S **decline** | 见 §2 | | 3 | `copy-image-in-place@P7` | **退役**（单次 GENERAL 转换）；同级同层矩形重叠 **decline** | 见 §3 | | 4 | `default-color-blit-shape@P7` | **退役**（恒等回落 + 旋转 scratch）；四旋转之外 **decline** | 见 §4 | +| 5 | 出口门 3：OpenRA 回读竞态 | **修复**（回读自带写可见性 barrier + 等齐所有在飞提交） | 见 §5 | --- @@ -319,3 +320,91 @@ scratch 臂全对，bug 就是这么掉出来的。两条臂现在都对。 **欠设备核对**：非恒等旋转下的 scratch 臂只在恒等表面上被旋钮验证过（像素与回落臂相同）；真正的 ROTATE_90/180/270 几何要在手机上看一眼。四旋转之外的 decline 无设备可核（没有会报镜像 transform 的 设备），按 decline 收口。 + +--- + +## 5. 片 5（集成者追加，最高优先）：出口门 3 的 OpenRA 回读竞态 + +### 5.1 机制（`file:line`，按本片落地前的树） + +两句话： + +1. **`WireFramebuffer.inc` 的 `ReadWirePixels` 先 `FlushPendingCommands()`，那一步把整帧的 draw 以 + 自己的 pooled fence 提交上队列后就返回、不等**（`VulkanRenderer.cpp:13534-13598`，成功路径上没有 + `vkWaitForFences`）。随后回读的 `vkCmdCopyImageToBuffer` 被录进**另一个**命令缓冲、成为**后一次 + 提交**。 +2. **`SubmitReadbackCommandsAndWait`（`VulkanRenderer.cpp:10707-10748`）只等自己那一次提交的 fence**， + 然后 `OnSubmitsCompletedUpTo(frame.lastSubmitIndex)` **把更早那次也记成完成并退休它的 fence**。 + 原注释写着「the wait proved every submission complete」——那句话对 monolith 臂成立（它不先 flush， + draw 与 copy 同一次提交、同一个 fence），对 wire 臂不成立。 + +把两次提交绑在一起的**唯一**东西，是回读前那道 `TransitionWireImage` 的 barrier；而它的**源作用域来自 +图像被跟踪的布局**（`GetImageTransitionSourceState`，`VulkanRenderer.cpp:1784-1825`）。其中两条臂—— +`VK_IMAGE_LAYOUT_UNDEFINED` 与 **`VK_IMAGE_LAYOUT_PRESENT_SRC_KHR`**（`:1787-1791`）——返回 +`VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT` 且 access 为 0，**这是 Vulkan 里「什么都不等」的写法**。 +刚 present 过的默认帧缓冲正处在后一种布局。 + +于是：pipeline 创建越慢、draw 那次提交拖得越久，copy 越可能在它还没画完时就读走——**帧里最后画的那些 +对象缺失**，正是设备上看到的样子。 + +### 5.2 主机实测（决定性，且不依赖竞态碰运气） + +在 `ReadWirePixels` 记录 copy 的那一刻打点（OpenRA / DirectVulkan / inproc）： + +``` +MBPROBE ReadWirePixels/pre: isDefault=1 layout=1 aspect=0x1 submit=72 completed=71 +``` + +`submit=72 completed=71`——**承载整帧 draw 的第 72 次提交此刻在飞、且无人等它**。这不是概率事件，是 +代码结构，主机上确定性可复现。 + +主机**不**发散的原因也在这一行里：`layout=1` 是 `VK_IMAGE_LAYOUT_GENERAL`，走 +`GetImageTransitionSourceState` 的 `default:` 臂（`ALL_COMMANDS` / `MEMORY_WRITE`），**碰巧**把 copy 排 +在了 draw 之后。设备上的默认帧缓冲不是 GENERAL。 + +**集成者给的复现路径（pipeline 延迟旋钮）在主机上不成立，这里如实记录**：新增 +`MOBILEGL_TEST_PIPELINE_CREATE_DELAY_MS`（split-only、未设即惰性， +`VulkanRenderer.cpp` `GetOrCreatePipelineWithInput` 的 miss 路径）确实生效——OpenRA DirectVulkan SPLIT +从 1.88 s 变 4.53 s——但 **ssim 一直是 1.000000**。原因就是上一段：主机那条臂的 barrier 是满的。所以 +red-once 用的是 §5.4 的计数，而不是一张发散的图。 + +### 5.3 修复（两半，都只在 wire 臂） + +1. **`RecordWireReadbackWriteBarrier()`**（`WireFramebuffer.inc`，新）：回读在 copy 之前**自己记一道** + `ALL_COMMANDS → TRANSFER`、`MEMORY_WRITE → TRANSFER_READ|MEMORY_READ` 的内存 barrier，不再从布局 + **推断**该等谁。barrier 的第一作用域是本队列**提交序**里更早的每一条命令，所以它跨得过提交边界。 +2. **`SubmitReadbackCommandsAndWait` 等齐所有在飞提交**（`VulkanRenderer.cpp`，`#if + MOBILEGL_BUILD_DISAGGREGATED`）：收集 `m_inFlightSubmits` 里 `submitIndex <= frame.lastSubmitIndex` + 的每一个 fence 一起等。代价是零——回读本来就是一次完整的 CPU 停顿。 + **pull 构建保持单 fence 原样**：它没有 wire 臂、回读前不会 flush，而且是 G1 钉住的那个镜像。 + +### 5.4 red-once（R-16，已执行并还原） + +打点回读实际等了几个 fence： + +| | `fences` | `upTo` | `completed` | +|---|---|---|---| +| 修复后 | **2** | 73 | 71 | +| 修复前 | **1**（只有自己那一个） | 73 | 71 | + +`completed=71` 两边都一样——即第 72 次提交确实在飞。修复前只等 1 个 fence 然后把 72 记成完成；修复后 +两个都等。把 §5.3 的第 2 半还原即回到 `fences=1`，**这是确定性的、主机上的红**，不靠时序。 + +### 5.5 车道 + +`MOBILEGL_TEST_PIPELINE_CREATE_DELAY_MS=25` 下的三条回读用例（`VertexIdSamplerAndScalarUniformPixels` +draw→回读、`ComputeImageStoreFramebufferPixels` compute→回读、`GenerateMipmapPixels`）注册到三臂， +前缀 `.PipelineDelay.`。它们是**回归门而不是复现**：钉住「路径上每次 pipeline 创建都多花 25 ms 时，回读 +仍然返回那次 draw 的像素」——即答案不随时序移动。 + +`integration-magma-split` **84**（+3）、`-spawn` **63**（+3）、`-tcp` **65**（+3），三臂 0 红； +OpenRA DirectVulkan SPLIT retrace ssim 1.000000（带与不带 25 ms 延迟各两遍）。 + +### 5.6 欠设备核对 + +集成者要在 Redmi 上重跑三遍冷 cache（`pm clear` 后）。预期：0.976494 → 1.000000。若仍发散，下一个嫌疑 +按序是 `WireDraw.inc` 里十一处**静默丢弃 draw** 的 `return false`（尤其 `pipeline == VK_NULL_HANDLE` +那处，它与设备日志里那三行 `handing vkCreateShaderModule an INVALID module for program 2 stage 4` +在同一条路径上），以及 `SetImageContentDefined` 在 wire 臂是**画之前**就置位（monolith 是 +`EndRenderPass` 之后），这会让「没有写入记录」那条警告恰好在它该响的时候不响。两条都不在本片修的范围 +内，按名记在这里。