diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp index f3780ede..cffdb4a0 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp @@ -258,6 +258,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { void VkBufferManager::ReleaseAllLiveResources() { for (auto& weak : m_liveResources) { if (auto resource = weak.lock()) { + BumpSliceEpoch(*resource); resource->buffer.Destroy(); resource->storageSize = 0; resource->usageFlags = 0; @@ -272,6 +273,9 @@ namespace MobileGL::MG_Backend::DirectVulkan { Bool VkBufferManager::CreateResidentStorage(VkBufferResource& resource, VkDeviceSize size, VkBufferUsageFlags usage, VkMemoryPropertyFlags requiredFlags) { + // The only place a resident VkBuffer handle is minted, so every resident slice + // change funnels through here (callers release the old handle first). + BumpSliceEpoch(resource); // Staged range copies write resident storage with vkCmdCopyBuffer. usage |= VK_BUFFER_USAGE_TRANSFER_DST_BIT; const Bool created = resource.buffer.Create({ @@ -358,6 +362,10 @@ namespace MobileGL::MG_Backend::DirectVulkan { if (!resource) { return; // lazy: AcquireResidentSlice performs a full upload on creation } + // A respecify can change the size, the usage hint (so the resident/streamed + // route), and the contents at once; retire every memo before deciding what to + // do about the storage. + BumpSliceEpoch(*resource); // Any cached streaming slice refers to the previous contents. resource->transientFrameSerial = 0; if (!resource->buffer.IsValid()) { @@ -390,6 +398,9 @@ namespace MobileGL::MG_Backend::DirectVulkan { if (!resource) { return; } + // Drops the streaming memo below and may end in a storage swap or a deferred + // full re-upload, so no memoised slice survives this. + BumpSliceEpoch(*resource); resource->transientFrameSerial = 0; if (!resource->buffer.IsValid() || resource->pendingFullUpload) { return; @@ -422,6 +433,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { if (!resource) { return; } + BumpSliceEpoch(*resource); resource->transientFrameSerial = 0; if (!resource->buffer.IsValid() || resource->pendingFullUpload) { return; @@ -481,6 +493,13 @@ namespace MobileGL::MG_Backend::DirectVulkan { TrackLiveResource(resource); } + // Bumped for the request, not just for the storage it may create. This is the + // one call the frontend makes when a buffer becomes persistently mapped for + // writing (BufferObject::AcquireMemoryRange), and a map the backend declines + // keeps mutating its shadow with no further API call - so it is what lets + // GetSliceEpochCounter stand for "no buffer needs a persistent-map range push". + BumpSliceEpoch(*resource); + // Idempotent: an already-backed buffer returns the same mapped base. if (resource->persistentMapped && resource->buffer.IsValid() && resource->storageSize == size) { return resource->buffer.GetMappedData(); @@ -608,8 +627,10 @@ namespace MobileGL::MG_Backend::DirectVulkan { } else if (resource->transientChangeSerial == changeSerial && resource->transientSize == size && resource->transientFrameSerial != 0) { if (++resource->unchangedStreak >= kStreamedPromotionStreak) { + // Promotion moves the buffer off the arena and onto resident storage. resource->promotedResident = true; resource->promotedChangeSerial = changeSerial; + BumpSliceEpoch(*resource); if (AcquireResidentSlice(kind, bufferObject, outSlice)) { return true; } @@ -619,6 +640,11 @@ namespace MobileGL::MG_Backend::DirectVulkan { resource->unchangedStreak = 0; } + // A fresh arena allocation: a different slice than the last call handed back, + // and (below) the point where a promoted buffer's resident storage is released. + // The stable-promotion exit above returns before this, so a buffer the app has + // stopped touching keeps one slice for as long as it keeps its resident storage. + BumpSliceEpoch(*resource); if (!m_transientUploadArena.Upload(m_currentFrameIndex, bufferObject->MappedData(), size, 16, outSlice)) { return false; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.h index 951e7fce..4ec5de50 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.h @@ -57,6 +57,17 @@ namespace MobileGL::MG_Backend::DirectVulkan { // never orphaned or recreated. Draw-time acquire binds it directly, no re-upload. Bool persistentMapped = false; + // Bumped from a manager-wide counter every time anything that decides which + // BufferSlice an Acquire*Slice call hands back changes: storage created or + // released, a full re-upload becoming due, a promotion/demotion between + // resident and streamed storage, or a new per-frame arena slice. Callers that + // memoise a resolved slice compare this to prove the memo still describes the + // buffer. The counter is manager-wide (never per-resource) so a freshly + // created resource - including one that replaces a destroyed resource at the + // same address - can never reproduce a value some memo already holds. 0 means + // "no slice has ever been handed out", which no memo can match. + Uint64 sliceEpoch = 0; + // Cached transient (streaming) slice for the current frame. BufferSlice transientSlice{}; Uint64 transientFrameSerial = 0; @@ -132,6 +143,11 @@ namespace MobileGL::MG_Backend::DirectVulkan { void OnResourceDestroyed(SharedPtr&& resource); Uint64 GetFrameSerial() const { return m_frameSerial; } + // Highest value handed to any VkBufferResource::sliceEpoch. Unchanged since a + // memo was taken means no buffer this manager owns changed which slice it hands + // back, and none was persistently mapped, in between - so a memo of resolved + // slices needs no per-buffer re-check. See AcquirePersistentMap for the mapping half. + Uint64 GetSliceEpochCounter() const { return m_sliceEpochCounter; } // Highest frame serial whose GPU work is known complete; serials at or // below it may be considered signaled. Drives IsResourceBusy and the // backend GL fence objects. @@ -158,6 +174,8 @@ namespace MobileGL::MG_Backend::DirectVulkan { void DestroyAllDeferredReleases(); void TrackLiveResource(const SharedPtr& resource); void ReleaseAllLiveResources(); + // See VkBufferResource::sliceEpoch. + void BumpSliceEpoch(VkBufferResource& resource) { resource.sliceEpoch = ++m_sliceEpochCounter; } VkBufferManagerInitInfo m_initInfo{}; BufferArena m_transientUploadArena; @@ -170,5 +188,9 @@ namespace MobileGL::MG_Backend::DirectVulkan { Uint32 m_currentFrameIndex = 0; Uint64 m_frameSerial = 1; Uint64 m_completedSerialFloor = 0; + // Never reset (not even by Shutdown): a value handed to a resource must stay + // unique for the process, or a memo taken before a re-initialize could match + // a different resource's state after it. + Uint64 m_sliceEpochCounter = 0; }; } // namespace MobileGL::MG_Backend::DirectVulkan diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp index e6664efd..7caacbf2 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp @@ -271,6 +271,35 @@ namespace MobileGL::MG_Backend::DirectVulkan { g_dynamicStateShadow = {}; } + // vkCmdBindVertexBuffers, skipped when this command buffer already holds these + // buffers and offsets at binding 0. + static void ShadowedBindVertexBuffers(VkCommandBuffer commandBuffer, const VkBuffer* buffers, + const VkDeviceSize* offsets, Uint32 count) { + auto& shadow = g_dynamicStateShadow; + Bool identical = shadow.vertexBindValid && shadow.vertexBindingCount == count && + count <= DynamicStateShadow::kMaxShadowedVertexBindings; + if (identical) { + for (Uint32 i = 0; i < count; ++i) { + if (shadow.vertexBuffers[i] != buffers[i] || shadow.vertexOffsets[i] != offsets[i]) { + identical = false; + break; + } + } + } + if (identical) { + return; + } + vkCmdBindVertexBuffers(commandBuffer, 0, count, buffers, offsets); + if (count <= DynamicStateShadow::kMaxShadowedVertexBindings) { + shadow.vertexBindValid = true; + shadow.vertexBindingCount = count; + std::copy_n(buffers, count, shadow.vertexBuffers); + std::copy_n(offsets, count, shadow.vertexOffsets); + } else { + shadow.vertexBindValid = false; + } + } + static void ShadowedSetScissor(VkCommandBuffer commandBuffer, const VkRect2D& scissor) { auto& shadow = g_dynamicStateShadow; if (shadow.scissorValid && shadow.scissor.offset.x == scissor.offset.x && @@ -3015,10 +3044,66 @@ void main() { return true; } + Bool VulkanRenderer::TryBindResolvedVertexBindings( + VkCommandBuffer commandBuffer, const MG_State::GLState::VertexArrayObject& vao, + const ResolvedVertexBindings& entry, + const VertexInputStateFactory::BackendVertexInputState& vertexInputState, Uint32 activeAttribMask, + Uint32 bindingCount, Uint64 frameSerial) { + // Frame-scoped: a streamed binding's slice moves to a new arena block every + // frame by design, and the frame's first resolve is also what stamps every + // bound buffer's GPU-use serial, which the buffer manager's busy tracking (and + // therefore glBufferSubData's choice between a host write and a staged copy) + // depends on. + if (entry.frameSerial != frameSerial || entry.vertexInputState != &vertexInputState || + entry.vertexInputHash != vertexInputState.hash || entry.activeAttribMask != activeAttribMask || + entry.bindingCount != bindingCount) { + return false; + } + + // The layout identity above already fixes which buffer each binding reads: the + // factory entry is reached through the VAO's config-version-gated memo and its + // hash mixes the bound buffers' addresses, so rebinding a buffer resolves to a + // different entry. What is left to establish is that those buffers still hand + // back the slices recorded here, and that none of them is a host map whose + // shadow needs pushing down. An unmoved manager-wide epoch counter says both. + if (!entry.anyBufferMapped && entry.sliceEpochCounter == m_bufferManager.GetSliceEpochCounter()) { + ShadowedBindVertexBuffers(commandBuffer, entry.vkBuffers, entry.vkOffsets, bindingCount); + return true; + } + + // Something moved somewhere; ask the buffers themselves. + const auto& attributes = vao.GetAllAttributes(); + const MG_State::GLState::BufferObject* synced = nullptr; + for (Uint32 binding = 0; binding < bindingCount; ++binding) { + auto* bufferObject = attributes[entry.attributeLocations[binding]].Buffer.get(); + if (bufferObject != entry.buffers[binding]) { + return false; + } + // What the resolving path does before every acquire: a persistent map the + // backend could not adopt into coherent GPU storage mutates its shadow with + // no API call, so the write range has to be pushed down here too. It is a + // no-op for every buffer that is not such a map; when it is not, it dispatches + // a SubData that retires the epoch below, and this draw resolves in full. + if (bufferObject != synced) { + bufferObject->SyncPersistentMappedRange(); + synced = bufferObject; + } + const auto* resource = static_cast(bufferObject->GetBackendResource().get()); + if (resource == nullptr || resource->sliceEpoch != entry.sliceEpochs[binding]) { + return false; + } + } + + ShadowedBindVertexBuffers(commandBuffer, entry.vkBuffers, entry.vkOffsets, bindingCount); + return true; + } + Bool VulkanRenderer::UploadAndBindVertexBuffers( VkCommandBuffer commandBuffer, const MG_State::GLState::VertexArrayObject& vao, const ProgramFactory::VkProgramObject& programObj, const DrawCmdParam& drawParams, const IndexBufferView* pIndexBufferView) { + static_assert(ResolvedVertexBindings::kMaxBindings == DynamicStateShadow::kMaxShadowedVertexBindings, + "the resolved-binding memo is sized to what the bind shadow can compare"); const Bool indexedDraw = pIndexBufferView != nullptr; // Exclusive upper bound on the vertex-stream elements this draw can fetch through // vertex-rate bindings, or 0 when unbounded (indirect/multi draws). Computed lazily @@ -3050,6 +3135,30 @@ void main() { const auto bindingCount = vertexInputState.bindings.size() + static_cast(std::popcount(missingAttribMask)); + const Uint64 frameSerial = m_bufferManager.GetFrameSerial(); + if (frameSerial != m_resolvedVertexBindingsFrameSerial) { + m_resolvedVertexBindingsFrameSerial = frameSerial; + if (m_resolvedVertexBindings.size() > kMaxResolvedVertexBindings) { + m_resolvedVertexBindings.clear(); + } + } + ResolvedVertexBindings* memo = nullptr; + if (auto found = m_resolvedVertexBindings.find(&vao); found != m_resolvedVertexBindings.end()) { + memo = &found->second; + if (TryBindResolvedVertexBindings(commandBuffer, vao, *memo, vertexInputState, activeAttribMask, + static_cast(bindingCount), frameSerial)) { + return true; + } + // Whatever it described is stale; a resolve that bails out below must not + // leave the old contents matchable either. + memo->frameSerial = 0; + } + // Anything the memo cannot key on (see ResolvedVertexBindings) clears this as + // the resolve below discovers it. + Bool memoisable = missingAttribMask == 0 && bindingCount > 0 && + bindingCount <= ResolvedVertexBindings::kMaxBindings; + Bool anyBufferMapped = false; + auto& vkBuffers = m_vertexBuffersScratch; auto& vkOffsets = m_vertexOffsetsScratch; vkBuffers.assign(bindingCount, VK_NULL_HANDLE); @@ -3101,6 +3210,7 @@ void main() { ? vertexInputState.bindingConversions[binding] : VertexInputStateFactory::VertexStreamConversion::None; if (usesClientMemory) { + memoisable = false; const Uint32 location = bindingLocation; MOBILEGL_ASSERT(location < MG_State::GLState::VertexArrayObject::MAX_VERTEX_ATTRIBS, "UploadAndBindVertexStreams failed to resolve client attribute location"); @@ -3174,6 +3284,7 @@ void main() { binding, baseOffset, sourceSize); if (conversion != VertexInputStateFactory::VertexStreamConversion::None) { + memoisable = false; MOBILEGL_ASSERT(bindingLocation < MG_State::GLState::VertexArrayObject::MAX_VERTEX_ATTRIBS, "UploadAndBindVertexStreams failed to resolve converted attribute location"); const auto& attr = vao.GetAttribute(bindingLocation); @@ -3261,6 +3372,24 @@ void main() { } vkBuffers[binding] = slice.buffer; vkOffsets[binding] = slice.offset + static_cast(baseOffset); + // bindingAttributeLocations carries MAX_VERTEX_ATTRIBS as its "no location" + // sentinel; GetAttribute() folds that to an empty attribute but the memo + // indexes the raw array, so such a binding is not memoisable. + memoisable = memoisable && bindingLocation < MG_State::GLState::VertexArrayObject::MAX_VERTEX_ATTRIBS; + if (memoisable) { + if (memo == nullptr) { + memo = &m_resolvedVertexBindings[&vao]; + memo->frameSerial = 0; + } + memo->attributeLocations[binding] = static_cast(bindingLocation); + memo->buffers[binding] = sourceBufferShared.get(); + // Read after the acquire: it is the acquire that creates the resource + // and mints the epoch this slice belongs to. + const auto* resource = + static_cast(sourceBufferShared->GetBackendResource().get()); + memo->sliceEpochs[binding] = resource != nullptr ? resource->sliceEpoch : 0; + anyBufferMapped = anyBufferMapped || sourceBufferShared->IsMapped(); + } } SizeT syntheticBinding = vertexInputState.bindings.size(); @@ -3299,29 +3428,22 @@ void main() { } if (bindingCount > 0) { - auto& shadow = g_dynamicStateShadow; const Uint32 count = static_cast(bindingCount); - Bool identical = shadow.vertexBindValid && shadow.vertexBindingCount == count && - count <= DynamicStateShadow::kMaxShadowedVertexBindings; - if (identical) { - for (Uint32 i = 0; i < count; ++i) { - if (shadow.vertexBuffers[i] != vkBuffers[i] || shadow.vertexOffsets[i] != vkOffsets[i]) { - identical = false; - break; - } - } - } - if (!identical) { - vkCmdBindVertexBuffers(commandBuffer, 0, count, vkBuffers.data(), vkOffsets.data()); - if (count <= DynamicStateShadow::kMaxShadowedVertexBindings) { - shadow.vertexBindValid = true; - shadow.vertexBindingCount = count; - std::copy_n(vkBuffers.data(), count, shadow.vertexBuffers); - std::copy_n(vkOffsets.data(), count, shadow.vertexOffsets); - } else { - shadow.vertexBindValid = false; - } + if (memoisable && memo != nullptr) { + std::copy_n(vkBuffers.data(), count, memo->vkBuffers); + std::copy_n(vkOffsets.data(), count, memo->vkOffsets); + memo->vertexInputState = &vertexInputState; + memo->vertexInputHash = vertexInputState.hash; + memo->activeAttribMask = activeAttribMask; + memo->bindingCount = count; + memo->anyBufferMapped = anyBufferMapped; + // Read after every acquire above, so it covers the epochs they minted. + memo->sliceEpochCounter = m_bufferManager.GetSliceEpochCounter(); + // Published last: the entry is only matchable once every field above is + // the one this completed resolve produced. + memo->frameSerial = frameSerial; } + ShadowedBindVertexBuffers(commandBuffer, vkBuffers.data(), vkOffsets.data(), count); } return true; } diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h index 03edfa24..5887db7a 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h @@ -779,6 +779,63 @@ namespace MobileGL::MG_Backend::DirectVulkan { UnorderedMap m_convertedVertexStreams; + // One VAO's resolved vkCmdBindVertexBuffers arguments, reusable by a later draw + // that would resolve them to the same thing. Consecutive draws in a chunk-renderer + // frame keep the program and the vertex layout and only swap the VAO, so a + // per-VAO memo turns the second and later draws through each VAO into a validate + // plus (usually skipped) rebind. + // + // Only whole-buffer bindings are memoised. Client-memory and format-converted + // streams re-upload from a range that depends on the draw's own vertex/index + // range, and synthetic bindings carry glVertexAttrib* values that are not part + // of any key here; a layout using any of them is never stored. + struct ResolvedVertexBindings { + // Must equal DynamicStateShadow::kMaxShadowedVertexBindings (static_assert in + // the .cpp): past that width the bind shadow cannot skip a redundant bind + // either, so a wider layout resolves per draw. Minecraft-shaped layouts use four. + static constexpr Uint32 kMaxBindings = 8; + + // Zero until a resolve completes, and reset to zero before one starts, so a + // resolve that bails out midway cannot leave a half-filled entry matchable. + Uint64 frameSerial = 0; + // Identity of the resolved Vulkan layout: fixes bindings.size(), each + // binding's base offset, which bindings are client/converted, and (through + // the hash, which mixes the bound buffers' addresses) the VAO configuration. + const VertexInputStateFactory::BackendVertexInputState* vertexInputState = nullptr; + VertexInputStateFactory::HashType vertexInputHash = 0; + // The program's vertex input layout: decides the synthetic-binding set and + // hence the total binding count. + Uint32 activeAttribMask = 0; + Uint32 bindingCount = 0; + // VkBufferManager::GetSliceEpochCounter() at resolve time. Still equal means + // no buffer anywhere changed its slice or was persistently mapped since, which + // settles every per-binding question below in one compare. + Uint64 sliceEpochCounter = 0; + // Any bound buffer already carrying a host map when the slice was resolved. + // Such a buffer can mutate its shadow with no API call, so it has to be + // re-pushed per draw and the one-compare path above cannot apply. + Bool anyBufferMapped = true; + // Per binding: the VAO attribute location its buffer comes from, that buffer, + // and the buffer's VkBufferManager slice epoch when the slice was resolved. + Uint8 attributeLocations[kMaxBindings] = {}; + const MG_State::GLState::BufferObject* buffers[kMaxBindings] = {}; + Uint64 sliceEpochs[kMaxBindings] = {}; + VkBuffer vkBuffers[kMaxBindings] = {}; + VkDeviceSize vkOffsets[kMaxBindings] = {}; + }; + // Keyed on the VAO address purely as a lookup hint - an entry is only ever + // compared against, never dereferenced through, so a recycled address cannot + // produce a wrong bind: every input the resolve depends on is re-read from live + // state and compared before the entry is used. + UnorderedMap + m_resolvedVertexBindings; + // Frame serial the map was last aged on; entries are frame-scoped, so this only + // drives the size sweep below. + Uint64 m_resolvedVertexBindingsFrameSerial = 0; + // Entries of deleted VAOs are never hit again but still occupy the map; drop the + // lot at a frame boundary once they could outweigh a large frame's working set. + static constexpr SizeT kMaxResolvedVertexBindings = 4096; + void CreateInstance(); VkResult SetupDebugMessenger(); VkResult DestroyDebugMessenger(); @@ -813,6 +870,13 @@ namespace MobileGL::MG_Backend::DirectVulkan { const ProgramFactory::VkProgramObject& programObj, const DrawCmdParam& drawParams, const IndexBufferView* pIndexBufferView); + // Binds `entry`'s memoised buffers when every input it was resolved from is + // still live and unchanged, else returns false and leaves nothing bound. + Bool TryBindResolvedVertexBindings(VkCommandBuffer commandBuffer, + const MG_State::GLState::VertexArrayObject& vao, + const ResolvedVertexBindings& entry, + const VertexInputStateFactory::BackendVertexInputState& vertexInputState, + Uint32 activeAttribMask, Uint32 bindingCount, Uint64 frameSerial); Bool UploadAndBindIndexBuffer(FrameContext::FrameData& frame, const MG_State::GLState::VertexArrayObject& vao, const IndexBufferView* pIndexBufferView = nullptr);