From 990e518e339edc1a798cbd5865b459a96037bd31 Mon Sep 17 00:00:00 2001 From: BZLZHH Date: Thu, 6 Aug 2026 14:07:49 -0400 Subject: [PATCH] [Perf] (MG_Backend): give DirectVulkan's draw memo a table that fits in cache lines The per-VAO resolved-bindings map probe was ~45% of UploadAndBindVertexBuffers' self time, and the aux-memo pointer chase was the single hottest instruction left in TrySetupDrawFastPath. Both die together: a fixed 2048-slot two-probe 64B-aligned VaoDrawMemo table embeds the VAO key, content-hash-validated layout facts and the bindings payload reordered hot-to-cold. Layout facts hold exactly while the slot's content hash equals the live VAO's own config-guarded hash; a recycled VAO address either misses or reproduces a byte-identical config, for which the facts are correct by construction. Bindings keep their full per-draw revalidation; recycled slots zero their frame serials so half-filled entries can never match. ComputePipelineStateHash, the depth/stencil probe and the primitive-restart probe now take one bulk GetRenderStateParameters() fetch instead of ~17 cross-TU accessor calls (verified pure field reads, identical bit packing). The EBO slice memo gained the same manager-wide epoch one-compare rescue the vertex half uses. GetShaderTransformFlags is memoized on pre-transform. Sodium's MultiDrawElementsBaseVertex hoists GetGLTypeSize out of the per-sub-draw loop, replaces the division with a shift, and skips unsupported index types loudly instead of dividing by zero. Also verified: a GL_BLEND toggle recompiles nothing in steady state - the glslang frames in earlier state_toggle profiles were startup contamination. Quiet-box load-gated 6-round A/B: sodium_multidraw -8.0%, tex_param -4.1%, use_program -3.3%; steady-state vanilla_draw CPU -20% ns/op at 4096 frames (the 80-frame matrix compresses CPU wins under GPU boost clocks; profiles confirm UploadAndBindVertexBuffers 6.3% -> 4.4% including the table probe, and the aux cold-line load gone). The one matrix flag (pass_switch +7.5%) reversed to -3.2% in 10-pair isolated re-runs. Unit tests 421/421. --- .../MG_Backend/DirectVulkan/DirectVulkan.cpp | 17 +- .../DirectVulkan/Renderer/VulkanRenderer.cpp | 264 ++++++++++++------ .../DirectVulkan/Renderer/VulkanRenderer.h | 101 +++++-- 3 files changed, 272 insertions(+), 110 deletions(-) diff --git a/MobileGL/MG_Backend/DirectVulkan/DirectVulkan.cpp b/MobileGL/MG_Backend/DirectVulkan/DirectVulkan.cpp index 5c1b99e3..54ec77fb 100644 --- a/MobileGL/MG_Backend/DirectVulkan/DirectVulkan.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/DirectVulkan.cpp @@ -16,6 +16,7 @@ #include "MG_Util/Metrics/TextureMetrics.h" #include "MG_Util/Miscellany/IndexGenerator.h" #include +#include #include #include @@ -1450,6 +1451,18 @@ namespace MobileGL::MG_Backend::DirectVulkan { payload.mode = mode; payload.indexBufferView.indexType = type; + // Loop-invariant: the index type is fixed for the whole multi-draw, so resolve + // its byte size once instead of twice per sub-draw (a cross-TU switch that + // showed up in per-frame profiles of sodium-style 132x32 multi-draws). Index + // sizes are 1/2/4, so the per-sub-draw offset division below reduces to a + // shift - the hardware divide was the hottest instruction of this loop. + const SizeT indexSize = MG_Util::GetGLTypeSize(type); + if (indexSize == 0) { + MGLOG_E("MultiDrawElementsBaseVertex skipped: unsupported index type 0x%x", type); + return; + } + const Uint32 indexSizeShift = static_cast(std::countr_zero(indexSize)); + // TODO: allocate draw cmd buf elsewhere static Vector params; params.clear(); @@ -1464,14 +1477,14 @@ namespace MobileGL::MG_Backend::DirectVulkan { payload.indexBufferView.indexByteOffset = 0; payload.indexBufferView.indexByteSize = - std::max(reinterpret_cast(indices[i]) + count[i] * MG_Util::GetGLTypeSize(type), + std::max(reinterpret_cast(indices[i]) + count[i] * indexSize, payload.indexBufferView.indexByteSize); auto& param = params[i]; param.indexCount = count[i]; param.instanceCount = 1; - param.firstIndex = reinterpret_cast(indices[i]) / MG_Util::GetGLTypeSize(type); + param.firstIndex = reinterpret_cast(indices[i]) >> indexSizeShift; param.vertexOffset = basevertex[i]; param.firstInstance = 0; } diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp index e557b69f..69f45886 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp @@ -3157,6 +3157,41 @@ void main() { return true; } + VulkanRenderer::VaoDrawMemo* VulkanRenderer::LookupVaoDrawMemo( + const MG_State::GLState::VertexArrayObject* vao) { + if (m_vaoDrawMemoTable.empty()) { + m_vaoDrawMemoTable.resize(kVaoDrawMemoSlotCount); + } + // Multiplicative mix of the (16-byte-aligned) address; take high bits, they + // carry the most entropy of a multiply. + const Uint64 mixed = static_cast(reinterpret_cast(vao) >> 4) * 0x9E3779B97F4A7C15ull; + const Uint32 index = static_cast(mixed >> 32) & (kVaoDrawMemoSlotCount - 1); + VaoDrawMemo& first = m_vaoDrawMemoTable[index]; + if (first.vaoKey == vao) { + return &first; + } + VaoDrawMemo& second = m_vaoDrawMemoTable[index ^ 1u]; + if (second.vaoKey == vao) { + return &second; + } + // Miss: recycle a slot. Prefer an empty one; otherwise evict the entry whose + // bindings memo is older (its VAO is the one drawn less recently). + VaoDrawMemo* victim = &first; + if (first.vaoKey != nullptr && + (second.vaoKey == nullptr || second.bindings.frameSerial < first.bindings.frameSerial)) { + victim = &second; + } + victim->vaoKey = vao; + victim->contentHash = 0; + victim->layoutFactsValid = false; + // Unmatchable until a resolve completes (same rule as before: a bailed-out + // resolve must never leave stale contents matchable). + victim->bindings.frameSerial = 0; + victim->bindings.indexFrameSerial = 0; + victim->bindings.indexBuffer = nullptr; + return victim; + } + Bool VulkanRenderer::UploadAndBindVertexBuffers( VkCommandBuffer commandBuffer, const MG_State::GLState::VertexArrayObject& vao, const ProgramFactory::VkProgramObject& programObj, const DrawCmdParam& drawParams, @@ -3188,34 +3223,46 @@ void main() { const Uint32 activeAttribMask = programObj.activeVertexInputLocationMask; const Uint64 frameSerial = m_bufferManager.GetFrameSerial(); - if (frameSerial != m_resolvedVertexBindingsFrameSerial) { - m_resolvedVertexBindingsFrameSerial = frameSerial; - if (m_resolvedVertexBindings.size() > kMaxResolvedVertexBindings) { - m_resolvedVertexBindings.clear(); - } - } // Probe the memo BEFORE resolving the vertex-input entry: a hit needs nothing // from it (the VAO's own hash memo pins layout and buffers - see // TryBindResolvedVertexBindings), and skipping the resolve also skips its - // per-draw cold chase into the factory's heap entry. + // per-draw cold chase into the factory's heap entry. The direct-mapped slot + // lookup replaces the old pointer-keyed hash-map find, whose metadata and + // key-storage probing was the dominant per-draw cost of a VAO-cycling frame. m_currentDrawResolvedEntry = nullptr; + VaoDrawMemo* slot = nullptr; ResolvedVertexBindings* memo = nullptr; Uint64 vaoContentHash = 0; const Bool vaoHashKnown = vao.GetBackendHashMemo(vaoContentHash); if (vaoHashKnown) { - if (auto found = m_resolvedVertexBindings.find(&vao); found != m_resolvedVertexBindings.end()) { - memo = &found->second; - if (TryBindResolvedVertexBindings(commandBuffer, vao, *memo, vaoContentHash, - activeAttribMask, frameSerial)) { - m_currentDrawResolvedEntry = memo; - return true; - } - // Whatever it described is stale; a resolve that bails out below must not - // leave the old contents matchable either. - memo->frameSerial = 0; + slot = LookupVaoDrawMemo(&vao); + memo = &slot->bindings; + if (TryBindResolvedVertexBindings(commandBuffer, vao, *memo, vaoContentHash, + activeAttribMask, frameSerial)) { + m_currentDrawResolvedEntry = memo; + return true; } + // Whatever it described is stale; a resolve that bails out below must not + // leave the old contents matchable either. + memo->frameSerial = 0; } auto& vertexInputState = m_vertexInputStateFactory->GetOrCreateVertexInputState(vao); + if (slot == nullptr) { + // First sight since a config change: the factory resolve just stamped the + // VAO's hash memo, so the slot can be claimed (and the facts below stored) + // for every later draw of this configuration. + slot = LookupVaoDrawMemo(&vao); + memo = &slot->bindings; + memo->frameSerial = 0; + } + // Refresh the layout facts served to TrySetupDrawFastPath. Pure values derived + // from the content hash, so this is correct even for layouts whose BINDINGS are + // not memoisable (client arrays, conversions). + slot->contentHash = vertexInputState.hash; + slot->layoutHash = vertexInputState.layoutHash; + slot->layoutAuxMasks = VertexInputStateFactory::PackVertexInputAuxMasks( + vertexInputState.unsupportedAttribMask, vertexInputState.attributeLocationMask); + slot->layoutFactsValid = true; const Uint32 vertexInputAttribMask = vertexInputState.attributeLocationMask; const Uint32 missingAttribMask = activeAttribMask & ~vertexInputAttribMask; @@ -3444,10 +3491,8 @@ void main() { // indexes the raw array, so such a binding is not memoisable. memoisable = memoisable && bindingLocation < MG_State::GLState::VertexArrayObject::MAX_VERTEX_ATTRIBS; if (memoisable) { - if (memo == nullptr) { - memo = &m_resolvedVertexBindings[&vao]; - memo->frameSerial = 0; - } + // memo is always non-null here: the slot was claimed (and its serial + // zeroed) before the resolve started. memo->attributeLocations[binding] = static_cast(bindingLocation); memo->buffers[binding] = sourceBufferShared.get(); // Read after the acquire: it is the acquire that creates the resource @@ -3587,9 +3632,11 @@ void main() { // transient copy where the application's restart index becomes the fixed one. Uint32 substituteRestartIndex = 0; Bool substituteRestart = false; - if (MG_State::pGLContext->IsCapabilityEnabled(CapabilityInput::PrimitiveRestart) && - !MG_State::pGLContext->IsCapabilityEnabled(CapabilityInput::PrimitiveRestartFixedIndex)) { - const Uint32 restartIndex = MG_State::pGLContext->GetPrimitiveRestartIndex(); + // One bulk parameters fetch instead of up to three accessor calls per indexed + // draw; all three inputs are pure reads of these fields. + const RenderStateParameters& rsp = MG_State::pGLContext->GetRenderStateParameters(); + if (rsp.PrimitiveRestartEnabled && !rsp.PrimitiveRestartFixedIndexEnabled) { + const Uint32 restartIndex = rsp.PrimitiveRestartIndex; Uint32 fixedMax = 0; switch (vkIndexType) { case VK_INDEX_TYPE_UINT8: fixedMax = 0xFFu; break; @@ -3652,15 +3699,29 @@ void main() { ResolvedVertexBindings* indexMemo = m_currentDrawResolvedEntry; if (indexMemo != nullptr && !substituteRestart && indexMemo->indexFrameSerial != 0 && indexMemo->indexBuffer == indexBuffer) { - auto* resource = static_cast( - indexBufferShared->GetBackendResource().get()); - if (resource != nullptr && resource->sliceEpoch == indexMemo->indexSliceEpoch) { - const Uint64 frameSerial = m_bufferManager.GetFrameSerial(); - if (indexMemo->indexFrameSerial != frameSerial) { - // Same busy-tracking stamp the skipped acquire would have made. + // One-compare rescue first (mirrors TryBindResolvedVertexBindings): the + // use-serial was stamped this frame and the manager-wide slice-epoch + // counter has not moved, so no buffer anywhere - this EBO included - + // changed its slice or gained a host map since the epoch was verified. + // Skips the per-draw GetBackendResource chase into a cold resource object. + Bool sliceStillValid = false; + const Uint64 frameSerial = m_bufferManager.GetFrameSerial(); + if (indexMemo->indexFrameSerial == frameSerial && + indexMemo->indexSliceEpochCounter == m_bufferManager.GetSliceEpochCounter()) { + sliceStillValid = true; + } else { + auto* resource = static_cast( + indexBufferShared->GetBackendResource().get()); + if (resource != nullptr && resource->sliceEpoch == indexMemo->indexSliceEpoch) { + sliceStillValid = true; + // Same busy-tracking stamp the skipped acquire would have made, + // then re-arm the one-compare path for the rest of the frame. resource->lastUseSerial = frameSerial; indexMemo->indexFrameSerial = frameSerial; + indexMemo->indexSliceEpochCounter = m_bufferManager.GetSliceEpochCounter(); } + } + if (sliceStillValid) { const VkDeviceSize memoBindOffset = indexMemo->indexSliceOffset + static_cast(pIndexBufferView->indexByteOffset); auto& shadow = g_dynamicStateShadow; @@ -3710,6 +3771,9 @@ void main() { if (resource != nullptr) { indexMemo->indexBuffer = indexBuffer; indexMemo->indexSliceEpoch = resource->sliceEpoch; + // Read after the acquire for the same reason as the epoch: the acquire + // may have bumped the manager-wide counter minting this very epoch. + indexMemo->indexSliceEpochCounter = m_bufferManager.GetSliceEpochCounter(); indexMemo->indexVkBuffer = slice.buffer; indexMemo->indexSliceOffset = slice.offset; indexMemo->indexFrameSerial = m_bufferManager.GetFrameSerial(); @@ -4302,50 +4366,53 @@ void main() { // FBO-derived payload inputs (attachment presence/formats/draw-buffer gating) are // pinned by the render-pass hash key, exactly as the version-keyed memo relied on. Uint64 VulkanRenderer::ComputePipelineStateHash(Uint32 colorAttachmentCount) const { - auto& ctx = *MG_State::pGLContext; + // One bulk fetch instead of ~17 per-field accessor calls into MG_State: every + // input below is a plain field of RenderStateParameters, and each accessor this + // replaces (IsCapabilityEnabled / Get*) is a verified pure read of that same + // field (RenderState.cpp), so the hashed values are bit-identical. This runs on + // every draw whose pipeline-state version moved (a per-draw GL_BLEND toggle), + // where the accessor-call overhead dominated the hash itself. + const RenderStateParameters& p = MG_State::pGLContext->GetRenderStateParameters(); Uint64 capabilityBits = 0; - capabilityBits |= ctx.IsCapabilityEnabled(CapabilityInput::CullFace) ? 1ull << 0 : 0; - capabilityBits |= ctx.IsCapabilityEnabled(CapabilityInput::DepthTest) ? 1ull << 1 : 0; - capabilityBits |= ctx.IsCapabilityEnabled(CapabilityInput::PolygonOffsetFill) ? 1ull << 2 : 0; - capabilityBits |= ctx.IsCapabilityEnabled(CapabilityInput::RasterizerDiscard) ? 1ull << 3 : 0; - capabilityBits |= ctx.IsCapabilityEnabled(CapabilityInput::ColorLogicOp) ? 1ull << 4 : 0; - capabilityBits |= ctx.IsCapabilityEnabled(CapabilityInput::StencilTest) ? 1ull << 5 : 0; - capabilityBits |= ctx.IsCapabilityEnabled(CapabilityInput::PrimitiveRestart) ? 1ull << 6 : 0; - capabilityBits |= ctx.IsCapabilityEnabled(CapabilityInput::PrimitiveRestartFixedIndex) ? 1ull << 7 : 0; - capabilityBits |= ctx.GetDepthMask() ? 1ull << 8 : 0; + capabilityBits |= p.CullFaceEnabled ? 1ull << 0 : 0; + capabilityBits |= p.DepthTestEnabled ? 1ull << 1 : 0; + capabilityBits |= p.PolygonOffsetFillEnabled ? 1ull << 2 : 0; + capabilityBits |= p.RasterizerDiscardEnabled ? 1ull << 3 : 0; + capabilityBits |= p.ColorLogicOpEnabled ? 1ull << 4 : 0; + capabilityBits |= p.StencilTestEnabled ? 1ull << 5 : 0; + capabilityBits |= p.PrimitiveRestartEnabled ? 1ull << 6 : 0; + capabilityBits |= p.PrimitiveRestartFixedIndexEnabled ? 1ull << 7 : 0; + capabilityBits |= p.DepthMask ? 1ull << 8 : 0; Uint64 hash = CombinePipelineStateWord(0x243F6A8885A308D3ull, capabilityBits); - hash = CombinePipelineStateWord(hash, static_cast(ctx.GetPatchVertices())); - hash = CombinePipelineStateWord(hash, static_cast(ctx.GetPolygonModeFront())); - hash = CombinePipelineStateWord(hash, static_cast(ctx.GetCullFaceMode())); - hash = CombinePipelineStateWord(hash, static_cast(ctx.GetDepthFunc())); - hash = CombinePipelineStateWord(hash, static_cast(ctx.GetLogicOp())); - for (const StencilFace face : {StencilFace::Front, StencilFace::Back}) { - const StencilFaceState& stencil = ctx.GetStencilState(face); + hash = CombinePipelineStateWord(hash, static_cast(p.PatchVertices)); + hash = CombinePipelineStateWord(hash, static_cast(p.PolygonModeFront)); + hash = CombinePipelineStateWord(hash, static_cast(p.CullFaceModeSetting)); + hash = CombinePipelineStateWord(hash, static_cast(p.DepthFunc)); + hash = CombinePipelineStateWord(hash, static_cast(p.LogicOp)); + // StencilStates[0] is Front, [1] is Back (RenderState::GetStencilFaceIndex) - + // the same order the two GetStencilState(face) calls used to hash in. + for (const StencilFaceState& stencil : p.StencilStates) { hash = CombinePipelineStateWord(hash, static_cast(stencil.FailOp) | (static_cast(stencil.PassDepthPassOp) << 16) | (static_cast(stencil.PassDepthFailOp) << 32) | (static_cast(stencil.Func) << 48)); } + MOBILEGL_ASSERT(colorAttachmentCount <= p.BlendStates.size(), + "ComputePipelineStateHash: colorAttachmentCount %u exceeds MAX_DRAW_BUFFERS", + colorAttachmentCount); for (Uint32 i = 0; i < colorAttachmentCount; ++i) { - BlendFactor srcRGB = BlendFactor::One; - BlendFactor dstRGB = BlendFactor::Zero; - BlendFactor srcAlpha = BlendFactor::One; - BlendFactor dstAlpha = BlendFactor::Zero; - BlendEquation colorEquation = BlendEquation::Add; - BlendEquation alphaEquation = BlendEquation::Add; - ctx.GetBlendFuncIndexed(i, srcRGB, dstRGB, srcAlpha, dstAlpha); - ctx.GetBlendEquationIndexed(i, colorEquation, alphaEquation); - const BoolVec4 mask = ctx.GetColorMaskIndexed(m_independentBlendFeatureEnabled ? i : 0); - Uint64 attachmentWord = ctx.IsCapabilityEnabledIndexed(CapabilityInput::Blend, i) ? 1ull : 0; + const PerBufferBlendState& blend = p.BlendStates[i]; + const BoolVec4 mask = p.ColorMasks[m_independentBlendFeatureEnabled ? i : 0]; + Uint64 attachmentWord = blend.Enabled ? 1ull : 0; attachmentWord |= (mask.r() ? 1ull << 1 : 0) | (mask.g() ? 1ull << 2 : 0) | (mask.b() ? 1ull << 3 : 0) | (mask.a() ? 1ull << 4 : 0); - attachmentWord |= static_cast(srcRGB) << 8; - attachmentWord |= static_cast(dstRGB) << 16; - attachmentWord |= static_cast(srcAlpha) << 24; - attachmentWord |= static_cast(dstAlpha) << 32; - attachmentWord |= static_cast(colorEquation) << 40; - attachmentWord |= static_cast(alphaEquation) << 48; + attachmentWord |= static_cast(blend.SrcFactorRGB) << 8; + attachmentWord |= static_cast(blend.DstFactorRGB) << 16; + attachmentWord |= static_cast(blend.SrcFactorAlpha) << 24; + attachmentWord |= static_cast(blend.DstFactorAlpha) << 32; + attachmentWord |= static_cast(blend.ColorEquation) << 40; + attachmentWord |= static_cast(blend.AlphaEquation) << 48; hash = CombinePipelineStateWord(hash, attachmentWord); } return hash; @@ -5045,6 +5112,18 @@ void main() { shadow.dynamicTailIsDefaultFbo = isDefaultFbo; } + Uint32 VulkanRenderer::GetBaseTransformFlagsRaw() { + // GetShaderTransformFlags is a pure function of the pre-transform, which only + // changes on surface rotation - memoised so the per-draw path pays one field + // compare instead of the call + switch. + const VkSurfaceTransformFlagBitsKHR preTransform = m_swapchainObject.GetPreTransform(); + if (preTransform != m_baseTransformFlagsPreTransform) { + m_baseTransformFlagsCache = GetShaderTransformFlags(preTransform).GetRaw(); + m_baseTransformFlagsPreTransform = preTransform; + } + return m_baseTransformFlagsCache; + } + Bool VulkanRenderer::TrySetupDrawFastPath(FrameContext::FrameData& frame, GLenum mode, Flags aspects, const DrawCmdParam& drawParams, const IndexBufferView* pIndexBufferView) { @@ -5095,15 +5174,15 @@ void main() { if (renderStateMoved) { // Only the pipeline depends on the moved state - except the render-pass // flavor input (depth/stencil participation); a flip of that must take - // the full path's pass selection. - const Bool drawUsesDepthStencil = - MG_State::pGLContext->IsCapabilityEnabled(CapabilityInput::DepthTest) || - MG_State::pGLContext->IsCapabilityEnabled(CapabilityInput::StencilTest); + // the full path's pass selection. One bulk parameters fetch instead of + // two capability-accessor calls; both are pure reads of the same fields. + const RenderStateParameters& rsp = MG_State::pGLContext->GetRenderStateParameters(); + const Bool drawUsesDepthStencil = rsp.DepthTestEnabled || rsp.StencilTestEnabled; if (drawUsesDepthStencil != snap.drawUsesDepthStencil) { return false; } } - if (GetShaderTransformFlags(m_swapchainObject.GetPreTransform()).GetRaw() != snap.baseTransformFlags) { + if (GetBaseTransformFlagsRaw() != snap.baseTransformFlags) { return false; } if (m_textureManager->GetResourceEraseEpoch() != snap.textureEraseEpoch || @@ -5138,14 +5217,40 @@ void main() { Uint64 vaoLayoutHash = snap.vaoLayoutHash; Bool vaoLayoutMoved = false; if (vaoMoved) { + // Read the layout facts through the flat per-VAO memo table, keyed by the + // VAO's content-hash memo. The hash memo shares the cache line this compare + // chain already loaded (the config version), and the table slot is compact + // and hot - unlike the VAO's aux-memo words, which start a second cold line + // of every object in a VAO-cycling frame. The facts are pure functions of + // the content hash, so a slot whose contentHash equals the live memoised + // hash serves them for ANY VAO object, recycled addresses included. Uint64 auxMasks = 0; - if (!vao.GetBackendAuxMemo(vaoLayoutHash, auxMasks)) { - // First sight of this VAO configuration: resolve (which stamps the aux - // memo for every later draw) and read the same facts from the entry. + Bool factsKnown = false; + Uint64 contentHash = 0; + if (vao.GetBackendHashMemo(contentHash)) { + const VaoDrawMemo* vaoMemo = LookupVaoDrawMemo(&vao); + if (vaoMemo->layoutFactsValid && vaoMemo->contentHash == contentHash) { + vaoLayoutHash = vaoMemo->layoutHash; + auxMasks = vaoMemo->layoutAuxMasks; + factsKnown = true; + } + } + if (!factsKnown) { + // First sight of this VAO configuration: resolve (which stamps the + // VAO's hash memo) and read the same facts from the entry, then stamp + // the table slot for every later draw. const auto& vertexInputState = m_vertexInputStateFactory->GetOrCreateVertexInputState(vao); vaoLayoutHash = vertexInputState.layoutHash; auxMasks = VertexInputStateFactory::PackVertexInputAuxMasks( vertexInputState.unsupportedAttribMask, vertexInputState.attributeLocationMask); + Uint64 stampedHash = 0; + if (vao.GetBackendHashMemo(stampedHash)) { + VaoDrawMemo* vaoMemo = LookupVaoDrawMemo(&vao); + vaoMemo->contentHash = stampedHash; + vaoMemo->layoutHash = vaoLayoutHash; + vaoMemo->layoutAuxMasks = auxMasks; + vaoMemo->layoutFactsValid = true; + } } vaoLayoutMoved = vaoLayoutHash != snap.vaoLayoutHash; if (vaoLayoutMoved) { @@ -5208,7 +5313,7 @@ void main() { if (sampledTexture == nullptr) { continue; } - const auto* resource = sampledResources[i]; + auto* resource = sampledResources[i]; if (resource == nullptr || !IsValidSampledImageLayout(resource->layout)) { return false; } @@ -5218,6 +5323,11 @@ void main() { } contentSum += sampledTexture->GetContentVersion(); paramsSum += sampledTexture->GetTextureParamsVersion(); + // Folded into this walk (was a second loop): the stamp is a plain recency + // store. Stamping ahead of the sum compare below is benign - a declined + // draw re-runs the full path, which stamps the same resources, and an + // over-stamp only delays garbage collection by one generation. + m_textureManager->StampResourceRecordingUse(*resource); } if (contentSum != snap.sampledContentSum || paramsSum != snap.sampledParamsSum) { return false; @@ -5227,11 +5337,6 @@ void main() { snap.samplingResolutionGeneration = samplingResolutionGeneration; samplerDescriptorsUnchanged = false; } - for (SizeT i = 0; i < sampledTextures.size(); ++i) { - if (sampledTextures[i] != nullptr && sampledResources[i] != nullptr) { - m_textureManager->StampResourceRecordingUse(*sampledResources[i]); - } - } // Everything the full path would re-resolve is provably unchanged - or, for // a moved pipeline-state version or a changed vertex-input LAYOUT, reduces to @@ -5343,7 +5448,8 @@ void main() { } const auto& vao = *MG_State::pGLContext->GetBoundVertexArray(); const auto& program = *MG_State::pGLContext->GetProgramForDraw(); - ProgramFactory::CompileOptionFlags transformFlags = GetShaderTransformFlags(m_swapchainObject.GetPreTransform()); + ProgramFactory::CompileOptionFlags transformFlags = + ProgramFactory::CompileOptionFlags(GetBaseTransformFlagsRaw()); // Captured draws take the xfb-decorated program variant. if (m_transformFeedbackFeatureEnabled && MG_State::pGLContext->IsTransformFeedbackActive() && program.GetTransformFeedbackVaryingCount() > 0) { @@ -5682,7 +5788,7 @@ void main() { snap.drawFboIsDefault = drawFbo->IsDefaultFramebuffer(); snap.renderStateVersion = MG_State::pGLContext->GetPipelineStateVersion(); snap.bindGeneration = MG_State::pGLContext->GetTextureBindGeneration(); - snap.baseTransformFlags = GetShaderTransformFlags(m_swapchainObject.GetPreTransform()).GetRaw(); + snap.baseTransformFlags = GetBaseTransformFlagsRaw(); snap.resolvedTransformFlags = transformFlags.GetRaw(); snap.renderPassHash = nowActiveRenderPass->hash; snap.imageIndex = m_imageIndexAcquired; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h index b589d2c4..f824177e 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h @@ -648,6 +648,13 @@ namespace MobileGL::MG_Backend::DirectVulkan { Uint32 m_pipelineStateHashColorCount = 0; Uint64 m_pipelineStateHash = 0; Bool m_pipelineStateHashValid = false; + // GetShaderTransformFlags(preTransform) memo: a pure function of the swapchain + // pre-transform, re-evaluated only when that value changes (surface rotation). + // No other invalidation input exists. + VkSurfaceTransformFlagBitsKHR m_baseTransformFlagsPreTransform = + VK_SURFACE_TRANSFORM_FLAG_BITS_MAX_ENUM_KHR; + Uint32 m_baseTransformFlagsCache = 0; + Uint32 GetBaseTransformFlagsRaw(); // Drops every memoized pipeline handle. Required at command-buffer // boundaries and whenever any pipeline may have been destroyed. Also drops // the cached pipeline-state hash: the same boundaries can retire the GL @@ -851,6 +858,10 @@ namespace MobileGL::MG_Backend::DirectVulkan { // streams re-upload from a range that depends on the draw's own vertex/index // range, and synthetic bindings carry glVertexAttrib* values that are not part // of any key here; a layout using any of them is never stored. + // Field order is hit-path cache locality, hot to cold: the per-draw validate + // reads the scalars and the EBO memo head, then only the first bindingCount + // elements of vkBuffers/vkOffsets; the per-binding revalidation arrays at the + // tail are touched once per frame at most. struct ResolvedVertexBindings { // Must equal DynamicStateShadow::kMaxShadowedVertexBindings (static_assert in // the .cpp): past that width the bind shadow cannot skip a redundant bind @@ -883,47 +894,79 @@ namespace MobileGL::MG_Backend::DirectVulkan { // Such a buffer can mutate its shadow with no API call, so it has to be // re-pushed per draw and the one-compare path above cannot apply. Bool anyBufferMapped = true; - // Per binding: the VAO attribute location its buffer comes from, that buffer, - // and the buffer's VkBufferManager slice epoch when the slice was resolved. - Uint8 attributeLocations[kMaxBindings] = {}; - const MG_State::GLState::BufferObject* buffers[kMaxBindings] = {}; - Uint64 sliceEpochs[kMaxBindings] = {}; - VkBuffer vkBuffers[kMaxBindings] = {}; - VkDeviceSize vkOffsets[kMaxBindings] = {}; // Resident element-buffer slice memo (skips the per-draw AcquireResidentSlice // for the VAO's EBO, which cold-chases 500+ distinct resources in a // chunk-cycling frame). Self-validating exactly like the bindings above: a hit - // requires the LIVE bound EBO pointer to equal indexBuffer AND that buffer's - // resource to still carry indexSliceEpoch (epochs are minted from a - // process-lifetime counter, so a recycled address can never revalidate). - // Restart-substituted and streamed EBOs are never stored. indexFrameSerial - // tracks the last frame the resource's GPU-use serial was stamped through this - // memo; 0 means no index memo. Independent of the vertex half: both are - // (pointer, epoch)-validated, so neither can serve stale state for the other. + // requires the LIVE bound EBO pointer to equal indexBuffer AND either an + // unmoved manager-wide slice-epoch counter (nothing anywhere changed slices + // or gained a host map, the same one-compare rescue the vertex half uses) or + // that buffer's resource still carrying indexSliceEpoch (epochs are minted + // from a process-lifetime counter, so a recycled address can never + // revalidate). Restart-substituted and streamed EBOs are never stored. + // indexFrameSerial tracks the last frame the resource's GPU-use serial was + // stamped through this memo; 0 means no index memo. Independent of the + // vertex half: both are (pointer, epoch)-validated, so neither can serve + // stale state for the other. const MG_State::GLState::BufferObject* indexBuffer = nullptr; Uint64 indexSliceEpoch = 0; + // GetSliceEpochCounter() when the resource's epoch was last verified; only + // meaningful while indexFrameSerial matches the current frame serial. + Uint64 indexSliceEpochCounter = 0; VkBuffer indexVkBuffer = VK_NULL_HANDLE; VkDeviceSize indexSliceOffset = 0; Uint64 indexFrameSerial = 0; + + // Bound per draw (first bindingCount elements). + VkBuffer vkBuffers[kMaxBindings] = {}; + VkDeviceSize vkOffsets[kMaxBindings] = {}; + // Per binding: the VAO attribute location its buffer comes from, that buffer, + // and the buffer's VkBufferManager slice epoch when the slice was resolved. + // Only read by the per-frame revalidation and the something-moved fallback. + Uint8 attributeLocations[kMaxBindings] = {}; + const MG_State::GLState::BufferObject* buffers[kMaxBindings] = {}; + Uint64 sliceEpochs[kMaxBindings] = {}; }; - // Keyed on the VAO address purely as a lookup hint - an entry is only ever - // compared against, never dereferenced through, so a recycled address cannot - // produce a wrong bind: every input the resolve depends on is re-read from live - // state and compared before the entry is used. - UnorderedMap - m_resolvedVertexBindings; - // Frame serial the map was last aged on; entries are frame-scoped, so this only - // drives the size sweep below. - Uint64 m_resolvedVertexBindingsFrameSerial = 0; - // Entries of deleted VAOs are never hit again but still occupy the map; drop the - // lot at a frame boundary once they could outweigh a large frame's working set. - static constexpr SizeT kMaxResolvedVertexBindings = 4096; + // One direct-mapped slot of the per-VAO draw-memo table below. The key is a + // lookup hint only - a slot is never dereferenced through vaoKey; every fact it + // carries is validated against live state before use: + // - layoutHash/layoutAuxMasks are valid only while contentHash equals the LIVE + // VAO's own hash memo (which the VAO's config version guards), so a config + // change, a buffer rebind, or a recycled VAO address with a different + // configuration all miss. A recycled address with a byte-identical + // configuration AND identical bound buffers reproduces the content hash, and + // then the facts are correct by construction (they are a pure function of it). + // - bindings revalidates per draw exactly as before (frame serial, content + // hash, per-binding live buffer pointers and slice epochs). + struct alignas(64) VaoDrawMemo { + const MG_State::GLState::VertexArrayObject* vaoKey = nullptr; + // The VAO content hash (VertexInputStateFactory::GetOrComputeHash) the two + // layout facts below were derived from; 0 while nothing valid is stored. + Uint64 contentHash = 0; + Bool layoutFactsValid = false; + // The resolved layout identity + packed (unsupported, location) masks - + // the exact values GetBackendAuxMemo used to serve, moved here so the + // per-draw probe stays inside this table's one hot line instead of + // touching a second cold line of every cycled VAO object. + Uint64 layoutHash = 0; + Uint64 layoutAuxMasks = 0; + ResolvedVertexBindings bindings; + }; + // Fixed-size, allocated on first use, never rehashed or swept: entries are + // recycled in place on slot collisions (two-slot probe, older frame serial + // evicted), and stale entries self-invalidate through the compares above. A + // fixed table also makes every VaoDrawMemo/ResolvedVertexBindings pointer + // stable for the duration of a draw, which the EBO memo handoff + // (m_currentDrawResolvedEntry) relies on. + static constexpr Uint32 kVaoDrawMemoSlotCount = 2048; // power of two + Vector m_vaoDrawMemoTable; + // Finds the slot holding `vao`, or recycles the older of its two candidate + // slots into an empty memo keyed on `vao`. Never returns null. + VaoDrawMemo* LookupVaoDrawMemo(const MG_State::GLState::VertexArrayObject* vao); // The current draw's memo entry, set by UploadAndBindVertexBuffers and consumed // by the same draw's UploadAndBindIndexBuffer (the EBO memo lives in the same - // entry). Valid ONLY within that window: the map is open-addressing, so the next - // insert (i.e. the next draw's resolve of a new VAO) can move it. Null when the - // draw's layout is not memoisable. + // entry). Valid ONLY within that window: the next draw's lookup can recycle the + // slot. Null when the draw's layout is not memoisable. ResolvedVertexBindings* m_currentDrawResolvedEntry = nullptr; void CreateInstance();