mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-11-07 01:53:24 +09:00
1638 lines
92 KiB
C++
1638 lines
92 KiB
C++
// MobileGL - MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferManager.cpp
|
|
// Copyright (c) 2025-2026 MobileGL-Dev
|
|
// Licensed under the GNU Lesser General Public License v3.0:
|
|
// https://www.gnu.org/licenses/gpl-3.0.txt
|
|
// https://www.gnu.org/licenses/lgpl-3.0.txt
|
|
// SPDX-License-Identifier: LGPL-3.0-only
|
|
// End of Source File Header
|
|
|
|
#include "RenderPassGuard.h"
|
|
#include "VkBufferManager.h"
|
|
#include "../DirectVulkan.h"
|
|
#include "VulkanRenderer.h"
|
|
|
|
#include "MG_Util/Metrics/PipeStats.h"
|
|
#include "MG_Pipe/MGPipeCallbacks.h"
|
|
#include "MG_Pipe/PipeApply.h"
|
|
#include <MG_Backend/Record/StagedShadow.h>
|
|
#if MOBILEGL_BUILD_DISAGGREGATED
|
|
#include "MG_Remote/Transport/AdoptT0.h"
|
|
#endif
|
|
#if MOBILEGL_BUILD_DISAGGREGATED && defined(__ANDROID__)
|
|
#include <android/hardware_buffer.h>
|
|
#endif
|
|
#include <Config.h>
|
|
#include <chrono>
|
|
#include <cstdlib>
|
|
|
|
namespace MobileGL::MG_Backend::DirectVulkan {
|
|
namespace {
|
|
// P7 wave 2 package B3, FORCING KNOB (split-only, read once, not in ConfigLoader's
|
|
// accepted-env table - package B's MGITEST_MAGMA_FORCE_SHADER_MIPMAP shape).
|
|
//
|
|
// WriteWireBuffer has exactly one unsafe shape: taking the immediate host memcpy
|
|
// because `lastUseSerial > GetCompletedSerial()` said the buffer is idle. That
|
|
// predicate's first term is PURE COUNTING (`m_frameSerial - frameCount`, no fence),
|
|
// so on any arm where the frame serial advances without the GPU having proved the
|
|
// frame's submissions complete, the answer is a guess. Host lanes never reach that
|
|
// state on their own (measured: 0 mid-frame frame-boundary drains over the whole
|
|
// OpenRA replay), so the lane cannot arm against it without forcing it.
|
|
//
|
|
// 0/unset = off. 1 = the serial term answers "idle" for every write it would have
|
|
// ordered. N>1 = only the Nth such write (1-based), which is how one
|
|
// `glBufferSubData -> draw` pair is poisoned in isolation and the device's "one draw
|
|
// contributed nothing" signature is reproduced rather than the whole frame destroyed.
|
|
//
|
|
// The knob forces ONLY the counting term to lie. It does NOT bypass the ordered copy,
|
|
// so it stays a valid red-once across the fix: before the submission term existed the
|
|
// lie reached the host memcpy (red); with the submission term the lie is caught and
|
|
// the copy is still ordered (green).
|
|
Uint64 ForceStaleSerialSelector() {
|
|
static const Uint64 selector = [] {
|
|
const char* value = std::getenv("MGITEST_MAGMA_FORCE_STALE_BUFFER_SERIAL");
|
|
if (!value || !value[0]) return Uint64{0};
|
|
return static_cast<Uint64>(std::strtoull(value, nullptr, 10));
|
|
}();
|
|
return selector;
|
|
}
|
|
// Counts only the writes the busy predicate wanted to ORDER, so the selector indexes
|
|
// the streamed subdata -> draw pairs and nothing else.
|
|
Uint64 g_orderedWireWriteCounter = 0;
|
|
constexpr VmaAllocationCreateFlags kResidentBufferAllocationFlags =
|
|
VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT;
|
|
constexpr SizeT kLiveResourcePruneThreshold = 256;
|
|
|
|
// See VkBufferManager::AcquireUnboundStorageDescriptor. 256 bytes: comfortably past
|
|
// every minStorageBufferOffsetAlignment in the wild, and free.
|
|
constexpr VkDeviceSize kUnboundStorageDescriptorBytes = 256;
|
|
// See VkBufferManager::AcquireUnboundTexelBufferDescriptor. The same 256 bytes, for the
|
|
// same reason plus one: a texel buffer view's range must be a whole number of texels of
|
|
// whatever format the placeholder is asked for, and 256 divides by every texel size in
|
|
// the GL image-format table (1, 2, 4, 8 and 16 bytes).
|
|
constexpr VkDeviceSize kUnboundTexelBufferDescriptorBytes = 256;
|
|
|
|
// A zero-copy persistent buffer is created once and never recreated (the app holds
|
|
// its mapped pointer), and may be bound to any role, so it carries every usage.
|
|
// TRANSFER_DST is added by CreateResidentStorage.
|
|
constexpr VkBufferUsageFlags kPersistentBackedUsage =
|
|
VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT |
|
|
// "Every usage" has to mean every usage: a buffer texture reached through an IMAGE
|
|
// unit takes a VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER descriptor, and the write is
|
|
// invalid unless the buffer was created with this bit. Nothing asked for it until
|
|
// imageBuffer support existed, so the omission was invisible.
|
|
VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
|
|
// Appended to kPersistentBackedUsage when VK_EXT_transform_feedback is enabled
|
|
// (see VkBufferManagerInitInfo::transformFeedbackUsageEnabled).
|
|
constexpr VkBufferUsageFlags kTransformFeedbackUsage = VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT;
|
|
// The app writes into the persistent map with no explicit flush, so its memory must
|
|
// be host-coherent (Adreno host-visible memory is; requiring it keeps us portable).
|
|
constexpr VkMemoryPropertyFlags kPersistentBackedRequiredFlags =
|
|
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
|
|
|
|
using MG_State::GLState::BufferObject;
|
|
|
|
// The manager owned by the active VulkanRenderer; immediate ops route here.
|
|
SessionLocal<VkBufferManager*> g_activeBufferManager; // per Magma session (MagmaSession.h)
|
|
|
|
// The session's manager, or null once its renderer is gone - a client's records can still
|
|
// arrive after its eglTerminate released it (or, surfaceless, before its first surface built
|
|
// one). Such a record is declined with one line rather than taking the whole server down.
|
|
VkBufferManager* WireManager(const char* op) {
|
|
VkBufferManager* manager = g_activeBufferManager.Get();
|
|
if (manager == nullptr) {
|
|
MGLOG_E_ONCE("Magma: a wire buffer %s reached a session with no renderer; declined", op);
|
|
}
|
|
return manager;
|
|
}
|
|
|
|
const MG_Pipe::MGPipeResourceOps g_vulkanWireResourceOps = {
|
|
.Create =
|
|
[](auto res, const auto& desc) {
|
|
if (auto* m = WireManager("create")) m->CreateWireBuffer(res, desc);
|
|
},
|
|
.Respecify =
|
|
[](auto res, const auto& desc, const void* bytes) {
|
|
if (auto* m = WireManager("respecify")) m->RespecifyWireBuffer(res, desc, bytes);
|
|
},
|
|
.SubData =
|
|
[](auto res, const auto& record, const void* bytes) {
|
|
if (auto* m = WireManager("write"))
|
|
m->WriteWireBuffer(res, MG_Pipe::MGPipeSubDataBufferOffset(record),
|
|
MG_Pipe::MGPipeSubDataBufferSize(record), bytes);
|
|
},
|
|
.SubDataResident =
|
|
[](auto res, const auto& record, const void* bytes) {
|
|
if (auto* m = WireManager("write"))
|
|
m->WriteWireBuffer(res, MG_Pipe::MGPipeSubDataBufferOffset(record),
|
|
MG_Pipe::MGPipeSubDataBufferSize(record), bytes);
|
|
},
|
|
.FlushRange =
|
|
[](auto res, const auto& record, const void* bytes) {
|
|
if (auto* m = WireManager("flush")) m->FlushWireBuffer(res, record.Offset, record.Size, bytes);
|
|
},
|
|
.Readback =
|
|
[](auto res, const auto& record) {
|
|
if (auto* m = WireManager("readback")) m->ReadbackWireBuffer(res, record.Offset, record.Size);
|
|
},
|
|
.Destroy =
|
|
[](auto res) {
|
|
if (auto* m = WireManager("destroy")) m->DestroyWireBuffer(res);
|
|
},
|
|
// Split mapping remains T2: the client owns its map and pushes exact
|
|
// modified ranges. A server pointer is never donated across the wire. Monolith's record
|
|
// arm (P13 W4) is one process: there the store IS handed over, as the legacy arm's
|
|
// AcquirePersistentMap always did.
|
|
.MapPersistent = [](MG_Pipe::MGPipeHandle res, Uint64 size, const void* seed) -> void* {
|
|
if (!MG_Config::RecordArmAliasesFrontend()) return nullptr;
|
|
auto* m = WireManager("map_persistent");
|
|
return m != nullptr ? m->DonateWireBuffer(res, size, seed) : nullptr;
|
|
},
|
|
.UnmapPersistent = [](MG_Pipe::MGPipeHandle) {},
|
|
#if defined(__ANDROID__)
|
|
// P11 B2 (T0): the one door a server pointer is still never donated through stays shut;
|
|
// what T0 adds is the other direction - the CLIENT's pages imported as the store.
|
|
.ImportExternal = [](MG_Pipe::MGPipeHandle res, void* ahb, Uint64 size) -> Bool {
|
|
auto* m = WireManager("import");
|
|
return m != nullptr && m->ImportWireBuffer(res, ahb, size);
|
|
},
|
|
.SelfTestExternal = [](char* why, Uint64 whyBytes) -> Bool {
|
|
return VkBufferManager::SelfTestWireImport(why, whyBytes);
|
|
},
|
|
#endif
|
|
// T0 in one process: the client gets the store's own mapping, as monolith's
|
|
// donation does; the client seeds it from its shadow once this returns.
|
|
.DonateInProcess = [](MG_Pipe::MGPipeHandle res, Uint64 size) -> void* {
|
|
auto* m = WireManager("donate");
|
|
return m != nullptr ? m->DonateWireBuffer(res, size, nullptr) : nullptr;
|
|
},
|
|
};
|
|
} // namespace
|
|
|
|
void VkBufferManager::RegisterWireResourceOps() {
|
|
MG_Pipe::MGPipeSetResourceOps(&g_vulkanWireResourceOps);
|
|
}
|
|
|
|
namespace {
|
|
// Diagnostic memory for DescribeWireBufferHistory: the last handles created and destroyed.
|
|
struct WireBufferHistory {
|
|
static constexpr SizeT kDepth = 256;
|
|
MG_Pipe::MGPipeHandle created[kDepth]{};
|
|
MG_Pipe::MGPipeHandle destroyed[kDepth]{};
|
|
SizeT createdCount = 0;
|
|
SizeT destroyedCount = 0;
|
|
};
|
|
SessionLocal<WireBufferHistory> g_wireBufferHistory;
|
|
} // namespace
|
|
|
|
VkBufferManager::WireBufferResource* VkBufferManager::FindWireBuffer(MG_Pipe::MGPipeHandle res) {
|
|
const SizeT slot = res.Slot;
|
|
if (slot < m_wireBufferLookup.size()) {
|
|
const auto& lookup = m_wireBufferLookup[slot];
|
|
if (lookup.resource != nullptr && lookup.gen == res.Gen) return lookup.resource;
|
|
}
|
|
const auto found = m_wireBuffers.find(WireBufferKey(res));
|
|
if (found == m_wireBuffers.end()) return nullptr;
|
|
if (slot < kWireBufferLookupMaxSlots) {
|
|
if (slot >= m_wireBufferLookup.size()) m_wireBufferLookup.resize(slot + 1);
|
|
m_wireBufferLookup[slot] = {res.Gen, &found->second};
|
|
}
|
|
return &found->second;
|
|
}
|
|
|
|
void VkBufferManager::CreateWireBuffer(MG_Pipe::MGPipeHandle res, const MG_Pipe::MGPResourceDesc& desc) {
|
|
if (FindWireBuffer(res) != nullptr) {
|
|
MGLOG_F("Magma: Fatal{ProtocolCorruption, \"duplicate-buffer-create\"} {slot=%u, gen=%u}", res.Slot,
|
|
res.Gen);
|
|
std::abort();
|
|
}
|
|
m_wireBuffers.try_emplace(WireBufferKey(res));
|
|
{
|
|
auto& history = *g_wireBufferHistory;
|
|
history.created[history.createdCount++ % 256] = res;
|
|
}
|
|
RespecifyWireBuffer(res, desc, nullptr);
|
|
}
|
|
|
|
void VkBufferManager::RespecifyWireBuffer(MG_Pipe::MGPipeHandle res, const MG_Pipe::MGPResourceDesc& desc,
|
|
const void* initialBytes) {
|
|
auto* resource = FindWireBuffer(res);
|
|
if (!resource) {
|
|
MGLOG_F("Magma: Fatal{ProtocolCorruption, \"unknown-buffer-respecify\"} {slot=%u, gen=%u}", res.Slot,
|
|
res.Gen);
|
|
std::abort();
|
|
}
|
|
// THE SERIAL IS READ BEFORE THE RESET BELOW ZEROES IT. Whether it is zero - whether any
|
|
// GPU command has named this store - decides whether the orphan can go at once or has
|
|
// to wait for a submission (DeferredWireRelease), and three lines from now the record
|
|
// no longer carries it.
|
|
const Uint64 orphanedUseSerial = resource->lastUseSerial;
|
|
resource->contentSerial = ++m_wireContentSerial;
|
|
// P11 B2: a T0 store is ALWAYS released through the serial-gated list (never "at once"),
|
|
// so the client's AHardwareBuffer outlives every submission that can still name it however
|
|
// the client's own release (at this respecify) raced the frames in flight.
|
|
DeferWireRelease(std::move(resource->buffer),
|
|
resource->imported ? T0ReleaseSerial(orphanedUseSerial) : orphanedUseSerial);
|
|
resource->imported = false;
|
|
resource->size = desc.Width;
|
|
resource->lastUseSerial = 0;
|
|
resource->lastUseSubmitIndex = 0; // M2 r2: the new store carries no submission yet (B3's stamp is per store)
|
|
resource->gpuWritesPending = false;
|
|
resource->indirectReadBarrierPending = false;
|
|
resource->stagedCoverage.clear();
|
|
DropWireShadow(*resource);
|
|
ReleaseRenameSpares(*resource);
|
|
if (resource->size == 0) return;
|
|
|
|
if (!CreateWireStoreBuffer(*resource)) {
|
|
MGLOG_F("Magma: Fatal{ResourceUnavailable, \"buffer-storage\"} {slot=%u, gen=%u, size=%llu}", res.Slot,
|
|
res.Gen, static_cast<unsigned long long>(resource->size));
|
|
std::abort();
|
|
}
|
|
++m_wireStoreCount;
|
|
NoteWireStorePeaks();
|
|
PublishWireReclaimGauges();
|
|
// P15 S0 (M4): a small store starts a host shadow. Its bytes are undefined until written
|
|
// (glBufferData without data), so zeros are as good a starting copy as the new memory's.
|
|
constexpr SizeT kWireShadowMaxStore = 64u << 10;
|
|
constexpr SizeT kWireShadowBudget = 64u << 20;
|
|
if (MG_Config::Features.MagmaWriteRename != MG_Config::QuirkOverride::ForceOff &&
|
|
resource->size <= kWireShadowMaxStore && m_wireShadowBytes + resource->size <= kWireShadowBudget) {
|
|
resource->shadow.assign(static_cast<SizeT>(resource->size), 0);
|
|
m_wireShadowBytes += resource->shadow.size();
|
|
}
|
|
// Under transport, defined bytes follow as resource_subdata records. An
|
|
// undefined store has no shadow to upload and no implicit zero snapshot.
|
|
if (initialBytes != nullptr && desc.HasDefinedContent) {
|
|
WriteWireBuffer(res, 0, resource->size, initialBytes);
|
|
}
|
|
}
|
|
|
|
Bool VkBufferManager::CreateWireStoreBuffer(WireBufferResource& resource) {
|
|
VkBufferUsageFlags usage = kPersistentBackedUsage | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
|
|
if (m_initInfo.transformFeedbackUsageEnabled) usage |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT;
|
|
return resource.buffer.Create({
|
|
.allocator = m_initInfo.allocator,
|
|
.size = resource.size,
|
|
.usage = usage,
|
|
.memoryUsage = VMA_MEMORY_USAGE_AUTO,
|
|
.allocationFlags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT,
|
|
.requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
|
|
}) &&
|
|
resource.buffer.Map() != nullptr;
|
|
}
|
|
|
|
void VkBufferManager::ReleaseRenameSpares(WireBufferResource& resource) {
|
|
for (auto& spare : resource.renameSpares)
|
|
DeferWireRelease(std::move(spare.buffer), spare.lastUseSerial != 0 ? spare.lastUseSerial : m_frameSerial);
|
|
resource.renameSpares.clear();
|
|
}
|
|
|
|
void VkBufferManager::DropWireShadow(WireBufferResource& resource) {
|
|
if (resource.shadow.empty()) return;
|
|
m_wireShadowBytes -= resource.shadow.size();
|
|
resource.shadow.clear();
|
|
resource.shadow.shrink_to_fit();
|
|
}
|
|
|
|
// P15 S0 (M4): WRITE RENAMING. A busy store (the GPU may still read it: an earlier draw of
|
|
// this frame, or one in flight) gets a fresh VkBuffer holding the shadow - which already has
|
|
// this write in it - and the old one is released once everything that can name it completes,
|
|
// exactly as a glBufferData orphan is. Draws recorded before keep the old buffer and its old
|
|
// bytes; draws after bind the new one (every bind site compares VkBuffer handles, and the
|
|
// release goes through the destroy epoch the handle-keyed memos fold in). No copy is recorded,
|
|
// so the render pass stays open and no barrier is needed.
|
|
Bool VkBufferManager::RenameBusyWireStore(WireBufferResource& resource) {
|
|
// A shadow outlives only GPU writes whose bytes it has (this manager's own staged copies):
|
|
// every other GPU write drops it. So a pending GPU write does not bar the rename.
|
|
if (resource.shadow.size() != resource.size || resource.imported ||
|
|
MG_Config::Features.MagmaWriteRename == MG_Config::QuirkOverride::ForceOff)
|
|
return false;
|
|
VkBufferObject old = std::move(resource.buffer);
|
|
Bool reused = false;
|
|
// Spares are queued in rename order, so their sync points only grow: when the oldest is not
|
|
// complete, no later one is (one fence question instead of one per spare).
|
|
if (pVulkanRenderer != nullptr && !resource.renameSpares.empty() &&
|
|
pVulkanRenderer->IsSubmitIndexComplete(resource.renameSpares.front().submitIndex)) {
|
|
resource.buffer = std::move(resource.renameSpares.front().buffer);
|
|
resource.renameSpares.erase(resource.renameSpares.begin());
|
|
reused = true;
|
|
}
|
|
if (!reused) {
|
|
if (!CreateWireStoreBuffer(resource)) {
|
|
resource.buffer.Destroy();
|
|
resource.buffer = std::move(old);
|
|
return false;
|
|
}
|
|
++m_wireStoreCount;
|
|
}
|
|
if (!resource.buffer.Upload(resource.shadow.data(), static_cast<VkDeviceSize>(resource.size), 0)) {
|
|
// Keep the store on its old buffer; the new one goes the deferred way (it is idle).
|
|
DeferWireRelease(std::move(resource.buffer), 0);
|
|
resource.buffer = std::move(old);
|
|
return false;
|
|
}
|
|
// Busy means a GPU command recorded so far may name the old buffer: it waits behind this
|
|
// frame's serial and the current sync point, whatever its own stamps said - as a spare
|
|
// for the next rename, or (past the bound) through DeferWireRelease.
|
|
// P15: 4 spares recycled too few for frames in flight x renames per frame (FCL MC 1.21.5:
|
|
// VkBufferObject::Create 0.031 + the retired stores' Destroy 0.014 ms/frame). Only shadowed
|
|
// stores rename, and those are at most kWireShadowMaxStore (64 KiB), so 12 spares bound a
|
|
// resource's extra memory at 768 KiB.
|
|
constexpr SizeT kRenameSpares = 12;
|
|
if (resource.renameSpares.size() >= kRenameSpares) {
|
|
auto& oldest = resource.renameSpares.front();
|
|
DeferWireRelease(std::move(oldest.buffer), oldest.lastUseSerial);
|
|
resource.renameSpares.erase(resource.renameSpares.begin());
|
|
}
|
|
WireBufferResource::RenameSpare spare;
|
|
spare.buffer = std::move(old);
|
|
spare.submitIndex = pVulkanRenderer != nullptr ? pVulkanRenderer->GetSyncPointSubmitIndex() : 0;
|
|
spare.lastUseSerial = m_frameSerial;
|
|
resource.renameSpares.push_back(std::move(spare));
|
|
resource.lastUseSerial = 0;
|
|
resource.lastUseSubmitIndex = 0;
|
|
resource.gpuWritesPending = false;
|
|
++m_wireRenames;
|
|
NoteWireStorePeaks();
|
|
PublishWireReclaimGauges();
|
|
return true;
|
|
}
|
|
|
|
Bool VkBufferManager::WaitForWireBufferHostAccess(WireBufferResource& resource) {
|
|
// B3 review: the early return must consult the same submission term WriteWireBuffer
|
|
// does, or a caller that reaches here with the serial term idle skips the wait the
|
|
// term would have demanded.
|
|
if (resource.lastUseSerial <= GetCompletedSerial() && !resource.gpuWritesPending &&
|
|
(pVulkanRenderer == nullptr || pVulkanRenderer->IsSubmitIndexComplete(resource.lastUseSubmitIndex))) {
|
|
return true;
|
|
}
|
|
if (!m_copyProvider || !pVulkanRenderer) return false;
|
|
const VkCommandBuffer commands = m_copyProvider->AcquireBufferCopyCommandBuffer();
|
|
if (commands == VK_NULL_HANDLE) return false;
|
|
VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
|
|
barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
|
|
barrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT | VK_ACCESS_HOST_WRITE_BIT;
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdPipelineBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &barrier,
|
|
0, nullptr, 0, nullptr);
|
|
// P8-D: counted here, past the idle early-out, so `whw` is waits TAKEN, not calls.
|
|
++m_wireHostWaits;
|
|
const Bool timed = MG_Util::PipeStats::Enabled();
|
|
const auto waitStart = timed ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
|
|
const Bool waited =
|
|
pVulkanRenderer->WaitForSubmitIndex(pVulkanRenderer->GetSyncPointSubmitIndex(), UINT64_MAX, true);
|
|
if (timed) {
|
|
MG_Util::PipeStats::AddCalls(MG_Util::PipeStats::CallClass::WireHostWaits, 1);
|
|
MG_Util::PipeStats::AddCalls(MG_Util::PipeStats::CallClass::WireHostWaitMicros,
|
|
static_cast<Uint64>(std::chrono::duration_cast<std::chrono::microseconds>(
|
|
std::chrono::steady_clock::now() - waitStart)
|
|
.count()));
|
|
}
|
|
if (!waited) {
|
|
return false;
|
|
}
|
|
resource.lastUseSerial = 0;
|
|
// B3: the wait above proved every submission at or below the sync point complete, so
|
|
// the submission term must be cleared with the serial or it would keep answering busy.
|
|
resource.lastUseSubmitIndex = 0;
|
|
resource.gpuWritesPending = false;
|
|
return true;
|
|
}
|
|
|
|
void VkBufferManager::WriteWireBuffer(MG_Pipe::MGPipeHandle res, Uint64 offset, Uint64 size, const void* bytes) {
|
|
auto* resource = FindWireBuffer(res);
|
|
if (!resource || offset > resource->size || size > resource->size - offset || (size && !bytes)) {
|
|
MGLOG_F("Magma: Fatal{ProtocolCorruption, \"buffer-write-range\"} {slot=%u, gen=%u}", res.Slot, res.Gen);
|
|
std::abort();
|
|
}
|
|
if (size == 0) return;
|
|
resource->contentSerial = ++m_wireContentSerial;
|
|
if (!resource->shadow.empty()) Memcpy(resource->shadow.data() + offset, bytes, static_cast<SizeT>(size));
|
|
Bool uploaded = false;
|
|
|
|
// ---- P7 wave 2 package B3: WHAT EACH TERM OF THIS PREDICATE IS FOR -----------------
|
|
//
|
|
// A streamed `glBufferSubData -> draw` pair is only correct if the write is ordered
|
|
// against every draw that still reads the bytes it overwrites; a write this predicate
|
|
// answers "idle" is applied as an unsynchronized host memcpy, and a draw still reading
|
|
// the range then renders bytes that are not its own - silently, with no log line, no GL
|
|
// error and no Fatal. That was the OpenRA device divergence (magma-b3.md §2).
|
|
//
|
|
// THE GUARANTEE IS THE SERIAL TERM, and it is sound because of the FLOOR, not because
|
|
// of anything here. GetCompletedSerial() is max(m_frameSerial - frameCount,
|
|
// m_completedSerialFloor). The floor used to be raised to a retired submission's frame
|
|
// serial even while another submission carrying the SAME serial was still executing -
|
|
// on the wire arm a serial has two (a mid-frame FlushPendingCommands under a pooled
|
|
// fence, then Present) - so "frame N-1 complete" was asserted on a fence nobody waited.
|
|
// VulkanRenderer::OnSubmitsCompletedUpTo now raises it only to a serial no in-flight
|
|
// submission still carries, and run_trace_case.cmake reds a split retrace on any
|
|
// "MGWIRE-FLOOR unsound-serial-complete" line, which is that fix's lane.
|
|
//
|
|
// THE SUBMISSION TERM IS A PROBE AND A DEFENSE, NOT A SECOND GUARANTEE. It is not sound
|
|
// on its own: lastUseSubmitIndex is stamped at AcquireWireSlice with the next submission
|
|
// index, but the draw that acquired the slice can still be pushed into a LATER
|
|
// submission - BindProgramUniformBuffers -> SyncWireTextureShape's preserve path ->
|
|
// FlushWirePendingCommandsForTextureUpdate submits the stamped index without the draw -
|
|
// so the stamp can name S1 while the draw rides S2. What it does do: with the floor
|
|
// sound it should never be the term that says "busy" (a write it alone orders points to a
|
|
// floor hole), and it is the only thing the MGITEST_MAGMA_FORCE_STALE_BUFFER_SERIAL knob
|
|
// leaves standing, which is what the StaleSerial. entries pin.
|
|
Bool busyBySerial = resource->lastUseSerial > GetCompletedSerial();
|
|
if (busyBySerial && ForceStaleSerialSelector() != 0) {
|
|
const Uint64 index = ++g_orderedWireWriteCounter;
|
|
const Uint64 selector = ForceStaleSerialSelector();
|
|
if (selector == 1 || selector == index) {
|
|
busyBySerial = false;
|
|
MGLOG_W("MGITEST_MAGMA_FORCE_STALE_BUFFER_SERIAL: write #%llu {slot=%u, gen=%u} off=%llu "
|
|
"size=%llu - the counting term is forced to answer \"idle\"",
|
|
static_cast<unsigned long long>(index), res.Slot, res.Gen,
|
|
static_cast<unsigned long long>(offset), static_cast<unsigned long long>(size));
|
|
}
|
|
}
|
|
const Bool busyBySubmission =
|
|
pVulkanRenderer != nullptr && !pVulkanRenderer->IsSubmitIndexComplete(resource->lastUseSubmitIndex);
|
|
if ((busyBySerial || busyBySubmission) && !resource->shadow.empty() && RenameBusyWireStore(*resource)) {
|
|
uploaded = true; // the new buffer was filled from the shadow, this write included
|
|
} else if (busyBySerial || busyBySubmission) {
|
|
// Vulkan buffer copies require four-byte aligned ranges. An odd GL
|
|
// byte update waits, then writes only the exact range; rounding it
|
|
// from a CPU shadow could overwrite neighboring GPU-written bytes.
|
|
if (((offset | size) & 3u) == 0) {
|
|
uploaded = StagedWireRangeCopy(*resource, bytes, static_cast<SizeT>(offset), static_cast<SizeT>(size));
|
|
}
|
|
if (!uploaded && !WaitForWireBufferHostAccess(*resource)) {
|
|
if (LatchedOnDeviceLoss("buffer-host-write")) return;
|
|
WireBufferSyncFatal("host-write");
|
|
}
|
|
}
|
|
if (!uploaded) {
|
|
uploaded = resource->buffer.Upload(bytes, size, offset);
|
|
if (uploaded && MG_Util::PipeStats::Enabled()) {
|
|
MG_Util::PipeStats::AddBytes(MG_Util::PipeStats::ByteClass::StageBuffer, size);
|
|
}
|
|
}
|
|
if (!uploaded) {
|
|
if (LatchedOnDeviceLoss("buffer-upload")) return;
|
|
MGLOG_F("Magma: Fatal{ResourceUnavailable, \"buffer-upload\"}");
|
|
std::abort();
|
|
}
|
|
MG_Record::StagedShadowStore::CoverageAdd(resource->stagedCoverage, static_cast<SizeT>(offset),
|
|
static_cast<SizeT>(offset + size));
|
|
}
|
|
|
|
void VkBufferManager::FlushWireBuffer(MG_Pipe::MGPipeHandle res, Uint64 offset, Uint64 size, const void* bytes) {
|
|
if (bytes != nullptr) {
|
|
WriteWireBuffer(res, offset, size, bytes);
|
|
return;
|
|
}
|
|
const auto* resource = FindWireBuffer(res);
|
|
if (!resource || offset > resource->size || size > resource->size - offset ||
|
|
!MG_Record::StagedShadowStore::CoverageHas(resource->stagedCoverage, static_cast<SizeT>(offset),
|
|
static_cast<SizeT>(offset + size))) {
|
|
MGLOG_F("Magma: Fatal{StageSnapshotTooNarrow, \"buffer-flush\"} {slot=%u, gen=%u}", res.Slot, res.Gen);
|
|
std::abort();
|
|
}
|
|
// SubData has already copied the owned record bytes into GPU storage,
|
|
// ordered after earlier uses. There is no stale shadow to replay here.
|
|
}
|
|
|
|
Bool VkBufferManager::LookupConvertedStream(const ConvertedStreamKey& key, SizeT elementCount,
|
|
BufferSlice& outSlice) {
|
|
for (auto& entry : m_convertedStreams) {
|
|
if (!(entry.key == key) || entry.elementCount < elementCount) continue;
|
|
entry.lastUseFrameSerial = m_frameSerial;
|
|
outSlice = entry.buffer.GetSlice();
|
|
outSlice.mapped = nullptr;
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
void VkBufferManager::RetireConvertedStream(SizeT index) {
|
|
auto& entry = m_convertedStreams[index];
|
|
m_convertedStreamBytes -= static_cast<SizeT>(entry.buffer.GetSize());
|
|
// Its last use may be this frame: the slot's next BeginFrame is after this frame completed.
|
|
if (m_currentFrameIndex < m_deferredBufferReleases.size())
|
|
m_deferredBufferReleases[m_currentFrameIndex].push_back(std::move(entry.buffer));
|
|
m_convertedStreams.erase(m_convertedStreams.begin() + static_cast<std::ptrdiff_t>(index));
|
|
}
|
|
|
|
Bool VkBufferManager::StoreConvertedStream(const ConvertedStreamKey& key, MG_Pipe::MGPipeHandle source,
|
|
SizeT elementCount, const void* bytes, SizeT size,
|
|
BufferSlice& outSlice) {
|
|
constexpr SizeT kConvertedStreamBudget = 192ull << 20;
|
|
if (size == 0 || size > kConvertedStreamBudget / 2 || m_initInfo.allocator == nullptr ||
|
|
m_deferredBufferReleases.empty())
|
|
return false;
|
|
// A newer conversion of the same store and layout supersedes the old one (its content
|
|
// serial can never come back); an entry unused for many frames (a deleted store's, a
|
|
// layout no longer drawn) goes too.
|
|
constexpr Uint64 kConvertedStreamIdleFrames = 256;
|
|
for (SizeT i = m_convertedStreams.size(); i-- > 0;) {
|
|
const auto& entry = m_convertedStreams[i];
|
|
if (entry.lastUseFrameSerial + kConvertedStreamIdleFrames < m_frameSerial) {
|
|
RetireConvertedStream(i);
|
|
continue;
|
|
}
|
|
if (entry.source == source && entry.key.baseOffset == key.baseOffset && entry.key.stride == key.stride &&
|
|
entry.key.type == key.type && entry.key.size == key.size && entry.key.conversion == key.conversion)
|
|
RetireConvertedStream(i);
|
|
}
|
|
while (!m_convertedStreams.empty() && m_convertedStreamBytes + size > kConvertedStreamBudget) {
|
|
SizeT oldest = 0;
|
|
for (SizeT i = 1; i < m_convertedStreams.size(); ++i)
|
|
if (m_convertedStreams[i].lastUseFrameSerial < m_convertedStreams[oldest].lastUseFrameSerial)
|
|
oldest = i;
|
|
RetireConvertedStream(oldest);
|
|
}
|
|
ConvertedStream entry;
|
|
entry.key = key;
|
|
entry.source = source;
|
|
entry.elementCount = elementCount;
|
|
entry.lastUseFrameSerial = m_frameSerial;
|
|
if (!entry.buffer.Create({
|
|
.allocator = m_initInfo.allocator,
|
|
.size = static_cast<VkDeviceSize>(size),
|
|
.usage = VK_BUFFER_USAGE_VERTEX_BUFFER_BIT,
|
|
.memoryUsage = VMA_MEMORY_USAGE_AUTO,
|
|
.allocationFlags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT,
|
|
.requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
|
|
}) ||
|
|
entry.buffer.Map() == nullptr || !entry.buffer.Upload(bytes, static_cast<VkDeviceSize>(size), 0))
|
|
return false;
|
|
m_convertedStreamBytes += size;
|
|
outSlice = entry.buffer.GetSlice();
|
|
outSlice.mapped = nullptr;
|
|
m_convertedStreams.push_back(std::move(entry));
|
|
return true;
|
|
}
|
|
|
|
Bool VkBufferManager::GetWireBufferContentSerial(MG_Pipe::MGPipeHandle res, Uint64& serial) {
|
|
auto* resource = FindWireBuffer(res);
|
|
if (!resource || resource->imported || !resource->buffer.IsValid()) return false;
|
|
serial = resource->contentSerial;
|
|
return true;
|
|
}
|
|
|
|
Bool VkBufferManager::AcquireWireSlice(BufferKind kind, MG_Pipe::MGPipeHandle res, BufferSlice& outSlice) {
|
|
// The store carries every buffer usage, so changing roles never orphans it.
|
|
outSlice = {};
|
|
auto* resource = FindWireBuffer(res);
|
|
if (!resource || !resource->buffer.IsValid() || resource->size == 0) return false;
|
|
resource->lastUseSerial = m_frameSerial;
|
|
// A slice a shader may write through: whatever it writes is a new content state.
|
|
if (kind == BufferKind::ShaderStorage || kind == BufferKind::TextureBuffer) {
|
|
resource->contentSerial = ++m_wireContentSerial;
|
|
DropWireShadow(*resource);
|
|
}
|
|
// B3: the draw this slice is being acquired for is normally recorded into the NEXT
|
|
// submission - normally, not always: a mid-draw flush (SyncWireTextureShape's preserve
|
|
// path) can submit this index without the draw. This stamp is the defense term's input
|
|
// (see WriteWireBuffer); the floor behind lastUseSerial is the guarantee. Read off the
|
|
// owning renderer's counter (its GetWireNextSubmitIndex()) when it was handed one.
|
|
if (m_submitCounter != nullptr)
|
|
resource->lastUseSubmitIndex = *m_submitCounter + 1;
|
|
else if (pVulkanRenderer)
|
|
resource->lastUseSubmitIndex = pVulkanRenderer->GetWireNextSubmitIndex();
|
|
outSlice = resource->buffer.GetSlice(0, resource->size);
|
|
outSlice.mapped = nullptr;
|
|
return true;
|
|
}
|
|
|
|
Bool VkBufferManager::ReadWireBuffer(MG_Pipe::MGPipeHandle res, Uint64 offset, Uint64 size, void* dst) {
|
|
auto* resource = FindWireBuffer(res);
|
|
if (!resource || offset > resource->size || size > resource->size - offset || (size && !dst)) return false;
|
|
if (size == 0) return true;
|
|
// A shadowed store's host copy is current, staged copies included (see RenameBusyWireStore).
|
|
if (resource->shadow.size() == resource->size) {
|
|
Memcpy(dst, resource->shadow.data() + offset, static_cast<SizeT>(size));
|
|
return true;
|
|
}
|
|
// Concurrent GPU reads do not prevent a CPU read. Only staged copies or
|
|
// shader writes need a host visibility barrier and a submission wait;
|
|
// ordinary index/vertex inspection must not stall once per draw.
|
|
if (resource->gpuWritesPending) {
|
|
if (!WaitForWireBufferHostAccess(*resource)) return false;
|
|
// A draw may already hold a native slice of this store while another
|
|
// attribute needs a CPU conversion. The wait completes previous GPU
|
|
// work, not the draw that will bind that previously acquired slice.
|
|
// Preserve its reservation even if the idle drain advanced the frame.
|
|
resource->lastUseSerial = m_frameSerial;
|
|
// B3 review: and the submission half of the same reservation - the wait cleared it,
|
|
// and the draw holding the slice has not been submitted yet.
|
|
if (pVulkanRenderer) resource->lastUseSubmitIndex = pVulkanRenderer->GetWireNextSubmitIndex();
|
|
}
|
|
if (!resource->buffer.Invalidate(size, offset)) return false;
|
|
Memcpy(dst, static_cast<const Uint8*>(resource->buffer.GetMappedData()) + offset, static_cast<SizeT>(size));
|
|
return true;
|
|
}
|
|
|
|
Bool VkBufferManager::CopyWireBufferRangeToSlice(MG_Pipe::MGPipeHandle res, Uint64 offset, Uint64 size,
|
|
const BufferSlice& dst) {
|
|
auto* resource = FindWireBuffer(res);
|
|
if (!resource || offset > resource->size || size > resource->size - offset || size > dst.size) return false;
|
|
if (size == 0) return true;
|
|
if (!m_copyProvider || dst.buffer == VK_NULL_HANDLE || ((offset | size | dst.offset) & 3u) != 0) return false;
|
|
const VkCommandBuffer commands = m_copyProvider->AcquireBufferCopyCommandBuffer();
|
|
if (commands == VK_NULL_HANDLE) return false;
|
|
VkMemoryBarrier before{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
|
|
before.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT;
|
|
before.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdPipelineBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT,
|
|
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &before, 0, nullptr, 0, nullptr);
|
|
const VkBufferCopy copy{offset, dst.offset, size};
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdCopyBuffer(commands, resource->buffer.GetHandle(), dst.buffer, 1, ©);
|
|
VkMemoryBarrier after{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
|
|
after.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
|
|
after.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdPipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &after,
|
|
0, nullptr, 0, nullptr);
|
|
resource->lastUseSerial = m_frameSerial;
|
|
return true;
|
|
}
|
|
|
|
Bool VkBufferManager::CopyWireBufferSubWordRangeToSlice(MG_Pipe::MGPipeHandle res, Uint64 offset, Uint64 size,
|
|
Uint32 frameIndex, const BufferSlice& dst, Uint64 dstSkip) {
|
|
auto* resource = FindWireBuffer(res);
|
|
if (!resource || offset > resource->size || size > resource->size - offset) return false;
|
|
if (dstSkip > dst.size || size > dst.size - dstSkip) return false;
|
|
if (size == 0) return true;
|
|
if (!m_copyProvider || dst.buffer == VK_NULL_HANDLE) return false;
|
|
|
|
// Round the read of the application's store OUT to whole words. The tail can only be
|
|
// clipped by the end of the store itself, and [offset, offset + size) already fits
|
|
// inside it, so the widened window still covers every byte the caller asked for.
|
|
const Uint64 alignedOffset = offset & ~Uint64{3};
|
|
const Uint64 stagedEnd = std::min<Uint64>((offset + size + 3) & ~Uint64{3}, resource->size);
|
|
const Uint64 headPad = offset - alignedOffset;
|
|
const Uint64 stagedSize = stagedEnd - alignedOffset;
|
|
if (stagedSize < headPad + size) return false;
|
|
|
|
// Uninitialized on purpose: the widening copy below writes every byte of it. The
|
|
// arena can grow to satisfy this, which parks (never frees) the buffer `dst` may
|
|
// already name - see BufferArena::EnsureCapacity - so the two slices are allowed to
|
|
// sit in different VkBuffers and the copy names each slice's own handle.
|
|
BufferSlice staging{};
|
|
if (!m_transientUploadArena.Allocate(frameIndex, stagedSize, 4, staging) || !staging.IsValid()) return false;
|
|
|
|
const VkCommandBuffer commands = m_copyProvider->AcquireBufferCopyCommandBuffer();
|
|
if (commands == VK_NULL_HANDLE) return false;
|
|
VkMemoryBarrier before{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
|
|
before.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT;
|
|
before.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdPipelineBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT,
|
|
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &before, 0, nullptr, 0, nullptr);
|
|
const VkBufferCopy widen{alignedOffset, staging.offset, stagedSize};
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdCopyBuffer(commands, resource->buffer.GetHandle(), staging.buffer, 1, &widen);
|
|
// The shift reads what the widening copy just wrote, so it needs its own edge even
|
|
// though both halves are transfers on one command buffer.
|
|
VkMemoryBarrier staged{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
|
|
staged.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
|
|
staged.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdPipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &staged, 0,
|
|
nullptr, 0, nullptr);
|
|
const VkBufferCopy shift{staging.offset + headPad, dst.offset + dstSkip, size};
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdCopyBuffer(commands, staging.buffer, dst.buffer, 1, &shift);
|
|
VkMemoryBarrier after{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
|
|
after.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
|
|
after.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdPipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &after,
|
|
0, nullptr, 0, nullptr);
|
|
resource->lastUseSerial = m_frameSerial;
|
|
return true;
|
|
}
|
|
|
|
void VkBufferManager::ReadbackWireBuffer(MG_Pipe::MGPipeHandle res, Uint64 offset, Uint64 size) {
|
|
// P11 B2: A T0 STORE'S READBACK IS "DONE, NO BYTES". Its pages are the client's
|
|
// AHardwareBuffer, so what the client needs before it reads them is this server's GPU
|
|
// having finished every write to them - the resident copies WriteWireBuffer recorded and
|
|
// any shader write - and nothing crossing SEG_EVENT. The wait is the store's own host-access
|
|
// wait (a barrier to host, then the sync point's submission); the reply then says "done".
|
|
// One death for both shapes: the store could not be made readable.
|
|
auto* resource = FindWireBuffer(res);
|
|
const Bool t0 = resource != nullptr && resource->imported;
|
|
Vector<Uint8> bytes(t0 ? 0 : static_cast<SizeT>(size));
|
|
if (t0 ? !WaitForWireBufferHostAccess(*resource) : !ReadWireBuffer(res, offset, size, bytes.data())) {
|
|
// A lost device: the session is latched and closing; its client reads a lost context.
|
|
if (LatchedOnDeviceLoss("buffer-readback")) return;
|
|
MGLOG_F("Magma: Fatal{ResourceUnavailable, \"buffer-readback\"} {slot=%u, gen=%u}", res.Slot, res.Gen);
|
|
std::abort();
|
|
}
|
|
if (t0) {
|
|
return;
|
|
}
|
|
if (MG_Pipe::gMGPipeCallbacks.OnBufferWriteback == nullptr) {
|
|
MGLOG_F("Magma: Fatal{ResourceUnavailable, \"buffer-writeback-callback\"}");
|
|
std::abort();
|
|
}
|
|
// The reverse producer synchronously copies into SEG_EVENT before
|
|
// this owned vector is released; no mapped Vulkan pointer crosses roles.
|
|
MG_Pipe::gMGPipeCallbacks.OnBufferWriteback(
|
|
res, offset, {reinterpret_cast<Uint64>(bytes.data()), size, MG_Pipe::kMGHostSpanSegNone, 0});
|
|
}
|
|
|
|
void VkBufferManager::MarkWireBufferGpuWritten(MG_Pipe::MGPipeHandle res, Uint64 offset, Uint64 size) {
|
|
auto* resource = FindWireBuffer(res);
|
|
if (!resource || offset > resource->size) return;
|
|
if (size == MG_Pipe::kMGPipeWholeBuffer) size = resource->size - offset;
|
|
if (size == 0 || size > resource->size - offset) return;
|
|
resource->contentSerial = ++m_wireContentSerial;
|
|
DropWireShadow(*resource);
|
|
resource->lastUseSerial = m_frameSerial;
|
|
resource->stagedCoverage.clear();
|
|
resource->gpuWritesPending = true;
|
|
resource->indirectReadBarrierPending = true;
|
|
// The existing client dirty-state protocol accepts one whole-store
|
|
// notification. The store remains on the GPU and exact later SubData
|
|
// only overwrites its own range, preserving every other GPU-written byte.
|
|
const MG_Pipe::MGPRange range{0, MG_Pipe::kMGPipeWholeBuffer};
|
|
if (MG_Pipe::gMGPipeCallbacks.OnGpuWritten == nullptr) {
|
|
MGLOG_F("Magma: Fatal{ResourceUnavailable, \"buffer-gpu-written-callback\"}");
|
|
std::abort();
|
|
}
|
|
MG_Pipe::gMGPipeCallbacks.OnGpuWritten(res, 1, &range);
|
|
}
|
|
|
|
Bool VkBufferManager::TakeWireIndirectReadBarrier(MG_Pipe::MGPipeHandle res) {
|
|
auto* resource = FindWireBuffer(res);
|
|
if (!resource || !resource->indirectReadBarrierPending) return false;
|
|
resource->indirectReadBarrierPending = false;
|
|
return true;
|
|
}
|
|
|
|
// ---- P11 B2: T0 - THE CLIENT'S AHardwareBuffer AS A WIRE STORE -----------------------------
|
|
|
|
// The serial a T0 store is released under: its last use, or - when no GPU command has named it
|
|
// since a host-access wait - the current frame's, so the release still waits for the sync
|
|
// point rather than taking DeferWireRelease's "never named, destroy now" arm. Conservative by
|
|
// at most one submission.
|
|
Uint64 VkBufferManager::T0ReleaseSerial(Uint64 lastUseSerial) const {
|
|
if (lastUseSerial != 0) return lastUseSerial;
|
|
return m_frameSerial != 0 ? m_frameSerial : 1;
|
|
}
|
|
|
|
#if defined(__ANDROID__)
|
|
namespace {
|
|
Int32 T0PickMemoryType(const VkPhysicalDeviceMemoryProperties& props, Uint32 bits, VkMemoryPropertyFlags want) {
|
|
for (Uint32 i = 0; i < props.memoryTypeCount; ++i) {
|
|
if ((bits & (1u << i)) != 0 && (props.memoryTypes[i].propertyFlags & want) == want)
|
|
return static_cast<Int32>(i);
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
struct T0ImportedBuffer {
|
|
VkBuffer buffer = VK_NULL_HANDLE;
|
|
VkDeviceMemory memory = VK_NULL_HANDLE;
|
|
void* mapped = nullptr;
|
|
Bool coherent = true;
|
|
};
|
|
|
|
void T0DestroyImported(VkDevice device, T0ImportedBuffer& imported) {
|
|
if (imported.mapped != nullptr) vkUnmapMemory(device, imported.memory);
|
|
if (imported.buffer != VK_NULL_HANDLE) vkDestroyBuffer(device, imported.buffer, nullptr);
|
|
if (imported.memory != VK_NULL_HANDLE) vkFreeMemory(device, imported.memory, nullptr);
|
|
imported = T0ImportedBuffer{};
|
|
}
|
|
|
|
// A VkBuffer on a dedicated allocation IMPORTED from `ahb`, in a host-visible memory type
|
|
// (coherent preferred), mapped whole. The server's own host map is what WriteWireBuffer's
|
|
// idle path and every CPU reader of a wire store use, so a type without HOST_VISIBLE is a
|
|
// refusal, not a degraded import. Nothing is left behind on failure.
|
|
Bool T0ImportAhb(const VulkanRenderer::WireAhbImport& ctx, void* ahb, Uint64 size, VkBufferUsageFlags usage,
|
|
T0ImportedBuffer& out, String& why) {
|
|
out = T0ImportedBuffer{};
|
|
const auto getProperties =
|
|
reinterpret_cast<PFN_vkGetAndroidHardwareBufferPropertiesANDROID>(ctx.getAhbProperties);
|
|
VkAndroidHardwareBufferPropertiesANDROID properties{
|
|
VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID};
|
|
VkResult result = getProperties(ctx.device, static_cast<const AHardwareBuffer*>(ahb), &properties);
|
|
if (result != VK_SUCCESS) {
|
|
why = "vkGetAndroidHardwareBufferPropertiesANDROID -> " + std::to_string(static_cast<int>(result));
|
|
return false;
|
|
}
|
|
VkExternalMemoryBufferCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
|
|
external.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID;
|
|
VkBufferCreateInfo bufferInfo{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
|
|
bufferInfo.pNext = &external;
|
|
bufferInfo.size = size;
|
|
bufferInfo.usage = usage;
|
|
bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
|
|
result = vkCreateBuffer(ctx.device, &bufferInfo, nullptr, &out.buffer);
|
|
if (result != VK_SUCCESS) {
|
|
out.buffer = VK_NULL_HANDLE;
|
|
why = "vkCreateBuffer(external AHB) -> " + std::to_string(static_cast<int>(result));
|
|
return false;
|
|
}
|
|
Int32 type = T0PickMemoryType(ctx.memory, properties.memoryTypeBits,
|
|
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
|
|
out.coherent = type >= 0;
|
|
if (type < 0)
|
|
type = T0PickMemoryType(ctx.memory, properties.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT);
|
|
if (type < 0) {
|
|
why = "no host-visible memory type can import it (memoryTypeBits 0x" +
|
|
std::to_string(properties.memoryTypeBits) + ")";
|
|
T0DestroyImported(ctx.device, out);
|
|
return false;
|
|
}
|
|
VkMemoryDedicatedAllocateInfo dedicated{VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO};
|
|
dedicated.buffer = out.buffer;
|
|
VkImportAndroidHardwareBufferInfoANDROID import{
|
|
VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID};
|
|
import.buffer = static_cast<AHardwareBuffer*>(ahb);
|
|
import.pNext = &dedicated;
|
|
VkMemoryAllocateInfo allocateInfo{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
|
|
allocateInfo.pNext = &import;
|
|
allocateInfo.allocationSize = properties.allocationSize;
|
|
allocateInfo.memoryTypeIndex = static_cast<Uint32>(type);
|
|
result = vkAllocateMemory(ctx.device, &allocateInfo, nullptr, &out.memory);
|
|
if (result != VK_SUCCESS) {
|
|
out.memory = VK_NULL_HANDLE;
|
|
why = "vkAllocateMemory(import AHB) -> " + std::to_string(static_cast<int>(result));
|
|
T0DestroyImported(ctx.device, out);
|
|
return false;
|
|
}
|
|
result = vkBindBufferMemory(ctx.device, out.buffer, out.memory, 0);
|
|
if (result == VK_SUCCESS) result = vkMapMemory(ctx.device, out.memory, 0, VK_WHOLE_SIZE, 0, &out.mapped);
|
|
if (result != VK_SUCCESS || out.mapped == nullptr) {
|
|
out.mapped = nullptr;
|
|
why = "vkBindBufferMemory/vkMapMemory -> " + std::to_string(static_cast<int>(result));
|
|
T0DestroyImported(ctx.device, out);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
} // namespace
|
|
#endif
|
|
|
|
// P13 W4: THE IN-PROCESS DONATION. Every wire store is already host-visible, coherent and
|
|
// persistently mapped (RespecifyWireBuffer), so in one process the client can write straight
|
|
// into it, exactly as the legacy arm's persistent maps did. The store is then the client's
|
|
// pages - T0's shape without an AHardwareBuffer - so it takes T0's flag: a readback is "done, no
|
|
// bytes" (the client reads its own pointer once this side's GPU is through with it) and a
|
|
// respecify retires the store behind the serial. It is seeded from the client's shadow, after
|
|
// any GPU use of the store has drained, because the shadow is what the application sees.
|
|
void* VkBufferManager::DonateWireBuffer(MG_Pipe::MGPipeHandle res, Uint64 size, const void* seedBytes) {
|
|
auto* resource = FindWireBuffer(res);
|
|
if (resource == nullptr || size == 0 || size != resource->size) return nullptr;
|
|
void* mapped = resource->buffer.GetMappedData();
|
|
if (mapped == nullptr) return nullptr;
|
|
if (resource->imported) return mapped; // already donated: the same pointer
|
|
resource->contentSerial = ++m_wireContentSerial;
|
|
DropWireShadow(*resource);
|
|
if (!WaitForWireBufferHostAccess(*resource)) return nullptr;
|
|
if (seedBytes != nullptr) std::memcpy(mapped, seedBytes, static_cast<SizeT>(size));
|
|
resource->imported = true;
|
|
resource->gpuWritesPending = false;
|
|
resource->stagedCoverage.clear();
|
|
MG_Record::StagedShadowStore::CoverageAdd(resource->stagedCoverage, 0, static_cast<SizeT>(size));
|
|
return mapped;
|
|
}
|
|
|
|
Bool VkBufferManager::ImportWireBuffer(MG_Pipe::MGPipeHandle res, void* ahb, Uint64 size) {
|
|
#if defined(__ANDROID__) && MOBILEGL_BUILD_DISAGGREGATED // T0 is the transport's adoption tier
|
|
auto* resource = FindWireBuffer(res);
|
|
if (resource == nullptr || ahb == nullptr || size == 0 || size != resource->size) return false;
|
|
VulkanRenderer::WireAhbImport ctx;
|
|
if (pVulkanRenderer == nullptr || !pVulkanRenderer->GetWireAhbImport(ctx)) {
|
|
MGLOG_W_ONCE("Magma T0: this device did not take VK_ANDROID_external_memory_android_hardware_buffer");
|
|
return false;
|
|
}
|
|
VkBufferUsageFlags usage = kPersistentBackedUsage | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
|
|
if (m_initInfo.transformFeedbackUsageEnabled) usage |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT;
|
|
T0ImportedBuffer imported;
|
|
String why;
|
|
if (!T0ImportAhb(ctx, ahb, size, usage, imported, why)) {
|
|
MGLOG_W("Magma T0: store {slot=%u, gen=%u} (%llu bytes) not imported - %s", res.Slot, res.Gen,
|
|
static_cast<unsigned long long>(size), why.c_str());
|
|
return false;
|
|
}
|
|
// The T2 store it replaces goes the way a respecify's does (and any draw still naming it
|
|
// keeps it until its submission completes).
|
|
DeferWireRelease(std::move(resource->buffer), resource->lastUseSerial);
|
|
MG_Remote::Transport::AdoptT0::AcquireImported(ahb);
|
|
resource->buffer.AdoptExternal(ctx.device, imported.buffer, imported.memory, size, imported.mapped,
|
|
imported.coherent, ahb);
|
|
resource->imported = true;
|
|
DropWireShadow(*resource);
|
|
resource->lastUseSerial = 0;
|
|
resource->lastUseSubmitIndex = 0;
|
|
resource->gpuWritesPending = false;
|
|
// The whole store holds bytes the client supplied (its AHB, seeded from its shadow).
|
|
resource->stagedCoverage.clear();
|
|
MG_Record::StagedShadowStore::CoverageAdd(resource->stagedCoverage, 0, static_cast<SizeT>(size));
|
|
++m_wireStoreCount;
|
|
NoteWireStorePeaks();
|
|
PublishWireReclaimGauges();
|
|
MGLOG_D("Magma T0: imported {slot=%u, gen=%u} (%llu bytes, %s memory)", res.Slot, res.Gen,
|
|
static_cast<unsigned long long>(size), imported.coherent ? "coherent" : "non-coherent");
|
|
return true;
|
|
#else
|
|
(void)res;
|
|
(void)ahb;
|
|
(void)size;
|
|
return false;
|
|
#endif
|
|
}
|
|
|
|
// THE POST (B2; decided by probe, never by driver name). The AHB API promises CPU coherence only
|
|
// across an unlock/lock, and T0 holds the client's lock for the store's life, so this exercises
|
|
// that pattern on THIS device before kCapAdoptT0 may be published:
|
|
// hold - a 64 KiB BLOB allocated and locked by the test, never unlocked while it runs;
|
|
// read - this server's own host map of the import reads what the CPU wrote through the lock;
|
|
// gpu - one submission copies that pattern out of the import (the GPU read) and fills another
|
|
// region of it (the GPU write), then a transfer->host barrier and a fence wait;
|
|
// back - the fill is read through the HELD pointer, the copy from a host-visible staging buffer;
|
|
// steady - the same again with bytes the CPU wrote AFTER the import and its first GPU use (the
|
|
// block at the end says why the first round alone is not enough).
|
|
Bool VkBufferManager::SelfTestWireImport(char* whyOut, Uint64 whyBytes) {
|
|
const auto answer = [&](Bool ok, const String& text) {
|
|
std::snprintf(whyOut, static_cast<SizeT>(whyBytes), "%s", text.c_str());
|
|
return ok;
|
|
};
|
|
#if defined(__ANDROID__) && MOBILEGL_BUILD_DISAGGREGATED // T0 is the transport's adoption tier
|
|
VulkanRenderer::WireAhbImport ctx;
|
|
if (pVulkanRenderer == nullptr || !pVulkanRenderer->GetWireAhbImport(ctx))
|
|
return answer(false, "the device did not take VK_ANDROID_external_memory_android_hardware_buffer");
|
|
constexpr Uint64 kBytes = 64 * 1024;
|
|
constexpr Uint32 kWords = 1024; // one 4 KiB region
|
|
constexpr Uint64 kFillAt = 4 * 4096; // region 4
|
|
constexpr Uint32 kFillWord = 0x6C0FFEE0u;
|
|
Vector<Uint32> pattern(kBytes / 4);
|
|
for (SizeT i = 0; i < pattern.size(); ++i)
|
|
pattern[i] = 0x5A000000u ^ static_cast<Uint32>(i * 2654435761u);
|
|
MG_Remote::Transport::AdoptT0::HeldStore held;
|
|
String why;
|
|
if (!MG_Remote::Transport::AdoptT0::AllocateHeld(kBytes, pattern.data(), held, why))
|
|
return answer(false, "hold: " + why);
|
|
T0ImportedBuffer imported, staging;
|
|
VkCommandPool pool = VK_NULL_HANDLE;
|
|
VkFence fence = VK_NULL_HANDLE;
|
|
String verdict;
|
|
Bool ok = false;
|
|
do {
|
|
if (!T0ImportAhb(ctx, held.ahb, kBytes,
|
|
VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT |
|
|
VK_BUFFER_USAGE_STORAGE_BUFFER_BIT,
|
|
imported, why)) {
|
|
verdict = "import: " + why;
|
|
break;
|
|
}
|
|
if (!imported.coherent) {
|
|
VkMappedMemoryRange range{VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE};
|
|
range.memory = imported.memory;
|
|
range.size = VK_WHOLE_SIZE;
|
|
vkInvalidateMappedMemoryRanges(ctx.device, 1, &range);
|
|
}
|
|
if (std::memcmp(imported.mapped, pattern.data(), kWords * 4) != 0) {
|
|
verdict = "read: the server's host map of the import does not hold the CPU's pattern";
|
|
break;
|
|
}
|
|
// A host-visible staging buffer for the GPU read's destination.
|
|
VkBufferCreateInfo stagingInfo{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
|
|
stagingInfo.size = kWords * 4;
|
|
stagingInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT;
|
|
stagingInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
|
|
if (vkCreateBuffer(ctx.device, &stagingInfo, nullptr, &staging.buffer) != VK_SUCCESS) {
|
|
staging.buffer = VK_NULL_HANDLE;
|
|
verdict = "staging buffer";
|
|
break;
|
|
}
|
|
VkMemoryRequirements requirements{};
|
|
vkGetBufferMemoryRequirements(ctx.device, staging.buffer, &requirements);
|
|
const Int32 stagingType =
|
|
T0PickMemoryType(ctx.memory, requirements.memoryTypeBits,
|
|
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
|
|
VkMemoryAllocateInfo stagingAlloc{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
|
|
stagingAlloc.allocationSize = requirements.size;
|
|
stagingAlloc.memoryTypeIndex = static_cast<Uint32>(stagingType < 0 ? 0 : stagingType);
|
|
if (stagingType < 0 ||
|
|
vkAllocateMemory(ctx.device, &stagingAlloc, nullptr, &staging.memory) != VK_SUCCESS ||
|
|
vkBindBufferMemory(ctx.device, staging.buffer, staging.memory, 0) != VK_SUCCESS ||
|
|
vkMapMemory(ctx.device, staging.memory, 0, VK_WHOLE_SIZE, 0, &staging.mapped) != VK_SUCCESS) {
|
|
verdict = "staging memory";
|
|
break;
|
|
}
|
|
std::memset(staging.mapped, 0, kWords * 4);
|
|
VkCommandPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};
|
|
poolInfo.queueFamilyIndex = ctx.queueFamily;
|
|
poolInfo.flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT;
|
|
VkCommandBuffer commands = VK_NULL_HANDLE;
|
|
VkCommandBufferAllocateInfo commandInfo{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};
|
|
commandInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
|
|
commandInfo.commandBufferCount = 1;
|
|
VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
|
|
if (vkCreateCommandPool(ctx.device, &poolInfo, nullptr, &pool) != VK_SUCCESS) {
|
|
pool = VK_NULL_HANDLE;
|
|
verdict = "command pool";
|
|
break;
|
|
}
|
|
commandInfo.commandPool = pool;
|
|
if (vkAllocateCommandBuffers(ctx.device, &commandInfo, &commands) != VK_SUCCESS ||
|
|
vkCreateFence(ctx.device, &fenceInfo, nullptr, &fence) != VK_SUCCESS) {
|
|
verdict = "command buffer / fence";
|
|
break;
|
|
}
|
|
VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
|
|
begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
|
|
vkBeginCommandBuffer(commands, &begin);
|
|
VkMemoryBarrier hostWrites{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
|
|
hostWrites.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT;
|
|
hostWrites.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdPipelineBarrier(commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1,
|
|
&hostWrites, 0, nullptr, 0, nullptr);
|
|
VkBufferCopy region{0, 0, kWords * 4};
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdCopyBuffer(commands, imported.buffer, staging.buffer, 1, ®ion);
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdFillBuffer(commands, imported.buffer, kFillAt, kWords * 4, kFillWord);
|
|
VkMemoryBarrier toHost{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
|
|
toHost.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
|
|
toHost.dstAccessMask = VK_ACCESS_HOST_READ_BIT;
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdPipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &toHost, 0,
|
|
nullptr, 0, nullptr);
|
|
vkEndCommandBuffer(commands);
|
|
VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO};
|
|
submit.commandBufferCount = 1;
|
|
submit.pCommandBuffers = &commands;
|
|
if (vkQueueSubmit(ctx.queue, 1, &submit, fence) != VK_SUCCESS ||
|
|
vkWaitForFences(ctx.device, 1, &fence, VK_TRUE, 2'000'000'000ull) != VK_SUCCESS) {
|
|
verdict = "gpu: the submission did not complete within 2 s";
|
|
break;
|
|
}
|
|
Uint32 gpuRead = 0, heldSaw = 0;
|
|
const auto* copied = static_cast<const Uint32*>(staging.mapped);
|
|
const auto* heldWords = static_cast<const Uint32*>(held.ptr);
|
|
for (Uint32 i = 0; i < kWords; ++i) {
|
|
if (copied[i] == pattern[i]) ++gpuRead;
|
|
if (heldWords[kFillAt / 4 + i] == kFillWord) ++heldSaw;
|
|
}
|
|
ok = gpuRead == kWords && heldSaw == kWords;
|
|
verdict = "64 KiB AHB, lock held throughout: host map read ok (" +
|
|
String(imported.coherent ? "coherent" : "non-coherent") + " memory), GPU read " +
|
|
std::to_string(gpuRead) +
|
|
"/1024 words of the CPU's pattern, GPU write seen through the "
|
|
"held lock " +
|
|
std::to_string(heldSaw) + "/1024 words";
|
|
if (!ok) break;
|
|
|
|
// STEADY STATE. The round above reads bytes the CPU wrote BEFORE the import, and an import
|
|
// can clean the CPU's caches on its way in, so it passes on a device whose GPU never sees a
|
|
// write made through the held lock afterwards - which is all of T0's real traffic (Mali-G1
|
|
// on a Dimensity: the first round passed, then Flywheel's staging scatter read stale bytes
|
|
// and the create-indirect trace lost every instanced object). So the CPU now writes a fresh
|
|
// pattern into region 1 and reads region 4 back into its cache, and one more submission
|
|
// copies region 1 out and overwrites region 4.
|
|
constexpr Uint64 kSteadyAt = 1 * 4096; // region 1
|
|
constexpr Uint32 kSteadyFillWord = 0x3D5EA11Eu;
|
|
auto* heldWrite = static_cast<volatile Uint32*>(held.ptr);
|
|
Uint32 cached = 0;
|
|
for (Uint32 i = 0; i < kWords; ++i) {
|
|
heldWrite[kSteadyAt / 4 + i] = 0xA5000000u ^ (i * 40503u);
|
|
cached ^= heldWrite[kFillAt / 4 + i];
|
|
}
|
|
(void)cached;
|
|
std::memset(staging.mapped, 0, kWords * 4);
|
|
if (vkResetFences(ctx.device, 1, &fence) != VK_SUCCESS ||
|
|
vkResetCommandPool(ctx.device, pool, 0) != VK_SUCCESS ||
|
|
vkBeginCommandBuffer(commands, &begin) != VK_SUCCESS) {
|
|
ok = false;
|
|
verdict = "steady: the second submission could not be recorded";
|
|
break;
|
|
}
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdPipelineBarrier(commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1,
|
|
&hostWrites, 0, nullptr, 0, nullptr);
|
|
VkBufferCopy steadyRegion{kSteadyAt, 0, kWords * 4};
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdCopyBuffer(commands, imported.buffer, staging.buffer, 1, &steadyRegion);
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdFillBuffer(commands, imported.buffer, kFillAt, kWords * 4, kSteadyFillWord);
|
|
EndActiveRenderPassOn(commands);
|
|
vkCmdPipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &toHost, 0,
|
|
nullptr, 0, nullptr);
|
|
vkEndCommandBuffer(commands);
|
|
if (vkQueueSubmit(ctx.queue, 1, &submit, fence) != VK_SUCCESS ||
|
|
vkWaitForFences(ctx.device, 1, &fence, VK_TRUE, 2'000'000'000ull) != VK_SUCCESS) {
|
|
ok = false;
|
|
verdict = "steady: the second submission did not complete within 2 s";
|
|
break;
|
|
}
|
|
Uint32 steadyRead = 0, steadySaw = 0;
|
|
for (Uint32 i = 0; i < kWords; ++i) {
|
|
if (copied[i] == (0xA5000000u ^ (i * 40503u))) ++steadyRead;
|
|
if (heldWords[kFillAt / 4 + i] == kSteadyFillWord) ++steadySaw;
|
|
}
|
|
ok = steadyRead == kWords && steadySaw == kWords;
|
|
verdict += "; with the import in use, GPU read " + std::to_string(steadyRead) +
|
|
"/1024 words the CPU wrote after it, GPU write seen through the held lock " +
|
|
std::to_string(steadySaw) + "/1024 words";
|
|
} while (false);
|
|
if (fence != VK_NULL_HANDLE) vkDestroyFence(ctx.device, fence, nullptr);
|
|
if (pool != VK_NULL_HANDLE) vkDestroyCommandPool(ctx.device, pool, nullptr);
|
|
T0DestroyImported(ctx.device, staging);
|
|
T0DestroyImported(ctx.device, imported);
|
|
MG_Remote::Transport::AdoptT0::ReleaseHeld(held);
|
|
return answer(ok, verdict);
|
|
#else
|
|
return answer(false, "no AHardwareBuffer on this platform");
|
|
#endif
|
|
}
|
|
|
|
void VkBufferManager::DescribeWireBufferHistory(MG_Pipe::MGPipeHandle res) const {
|
|
const auto& history = *g_wireBufferHistory;
|
|
Int createdAgo = -1, destroyedAgo = -1;
|
|
for (SizeT i = 0; i < std::min(history.createdCount, WireBufferHistory::kDepth); ++i) {
|
|
const auto& h = history.created[(history.createdCount - 1 - i) % WireBufferHistory::kDepth];
|
|
if (h.Slot == res.Slot && h.Gen == res.Gen) {
|
|
createdAgo = static_cast<Int>(i);
|
|
break;
|
|
}
|
|
}
|
|
for (SizeT i = 0; i < std::min(history.destroyedCount, WireBufferHistory::kDepth); ++i) {
|
|
const auto& h = history.destroyed[(history.destroyedCount - 1 - i) % WireBufferHistory::kDepth];
|
|
if (h.Slot == res.Slot && h.Gen == res.Gen) {
|
|
destroyedAgo = static_cast<Int>(i);
|
|
break;
|
|
}
|
|
}
|
|
MGLOG_E("Magma wire buffer {slot=%u, gen=%u}: live stores %zu; created %d creates ago, destroyed %d "
|
|
"destroys ago (-1 = not in the last %zu); %zu creates, %zu destroys in this session",
|
|
res.Slot, res.Gen, m_wireBuffers.size(), createdAgo, destroyedAgo, WireBufferHistory::kDepth,
|
|
history.createdCount, history.destroyedCount);
|
|
}
|
|
|
|
void VkBufferManager::DestroyWireBuffer(MG_Pipe::MGPipeHandle res) {
|
|
{
|
|
auto& history = *g_wireBufferHistory;
|
|
history.destroyed[history.destroyedCount++ % WireBufferHistory::kDepth] = res;
|
|
}
|
|
const auto found = m_wireBuffers.find(WireBufferKey(res));
|
|
if (found == m_wireBuffers.end()) return;
|
|
DropWireShadow(found->second);
|
|
ReleaseRenameSpares(found->second);
|
|
DeferWireRelease(std::move(found->second.buffer),
|
|
found->second.imported ? T0ReleaseSerial(found->second.lastUseSerial)
|
|
: found->second.lastUseSerial); // P11 B2, as Respecify
|
|
ForgetWireBufferLookup(found->first);
|
|
m_wireBuffers.erase(found);
|
|
PublishWireReclaimGauges();
|
|
}
|
|
|
|
void VkBufferManager::DeferWireRelease(VkBufferObject&& buffer, Uint64 lastUseSerial) {
|
|
if (!buffer.IsValid()) return;
|
|
// Two stores go NOW rather than into the list. With no frame slots (DeferRelease's own
|
|
// teardown guard, and for its reason) there is no manager left to reclaim anything
|
|
// later. And a store whose last-use serial is 0 was never named by a GPU command since
|
|
// it was minted or since a host-access wait retired everything recorded before it - see
|
|
// DeferredWireRelease - so there is nothing to wait for.
|
|
if (m_deferredBufferReleases.empty() || lastUseSerial == 0) {
|
|
buffer.Destroy();
|
|
++m_wireStoreDestroyEpoch;
|
|
if (m_wireStoreCount > 0) --m_wireStoreCount;
|
|
return;
|
|
}
|
|
DeferredWireRelease entry;
|
|
entry.lastUseSerial = lastUseSerial;
|
|
// The last submission that can name this store: see DeferredWireRelease. Zero when there
|
|
// is no renderer, which is also when there is no queue and nothing to wait for.
|
|
entry.submitIndex = pVulkanRenderer != nullptr ? pVulkanRenderer->GetSyncPointSubmitIndex() : 0;
|
|
entry.bytes = static_cast<Uint64>(buffer.GetSize());
|
|
entry.buffer = std::move(buffer);
|
|
m_deferredWireBytes += entry.bytes;
|
|
m_deferredWireReleases.push_back(std::move(entry));
|
|
NoteWireStorePeaks();
|
|
// EVERY PARK SWEEPS. The frame boundary is not a reclaim point this arm can lean on - a
|
|
// pbuffer replay that snapshot-exits delivers ONE present for 1.3 M calls (ID-P7-32) -
|
|
// so the only cadence that tracks the workload is the workload itself.
|
|
SweepDeferredWireReleases();
|
|
EnforceWireDeferredWatermark();
|
|
}
|
|
|
|
void VkBufferManager::EnforceWireDeferredWatermark() {
|
|
const Uint64 budget = static_cast<Uint64>(MG_Config::RecordArm.WireDeferredMb) * 1024u * 1024u;
|
|
// TWO TRIGGERS, ONE SWITCH. Bytes are what the knob names, but a VkBuffer costs per
|
|
// OBJECT as well as per byte, and small orphans never reach a byte budget: measured on
|
|
// bsl-esc-menu-854 (spawn, lavapipe), one stretch with no submission parked 12,498 stores
|
|
// in 39.5 MB against 64 MiB. So a fixed count ceiling rides the same sync point; the
|
|
// knob's 0 turns both off, which keeps it one negative control.
|
|
if (budget == 0) return;
|
|
if (m_deferredWireBytes <= budget && m_deferredWireReleases.size() <= kWireDeferredCountCeiling) return;
|
|
// What is left after the sweep is named by work that is recorded but unsubmitted, or
|
|
// submitted but unretired. Every entry is tagged at or below the sync point taken HERE,
|
|
// so waiting it out - the host-access wait's own call, which flushes the recording first
|
|
// - proves all of them dead. A mid-frame flush is what WaitForWireBufferHostAccess and a
|
|
// glClientWaitSync already do to this command stream; it costs one submission per
|
|
// budget's worth of orphans, not one per glBufferData.
|
|
if (pVulkanRenderer == nullptr ||
|
|
!pVulkanRenderer->WaitForSubmitIndex(pVulkanRenderer->GetSyncPointSubmitIndex(), UINT64_MAX, true)) {
|
|
if (LatchedOnDeviceLoss("buffer-deferred-watermark")) return;
|
|
WireBufferSyncFatal("deferred-watermark");
|
|
}
|
|
++m_wireDeferredSyncs;
|
|
SweepDeferredWireReleases();
|
|
}
|
|
|
|
void VkBufferManager::NoteWireStorePeaks() {
|
|
m_wireStoreCountPeak = std::max(m_wireStoreCountPeak, m_wireStoreCount);
|
|
m_deferredWireBytesPeak = std::max(m_deferredWireBytesPeak, m_deferredWireBytes);
|
|
}
|
|
|
|
void VkBufferManager::PublishWireReclaimGauges() {
|
|
if (!MG_Util::PipeStats::Enabled()) return;
|
|
using MG_Util::PipeStats::Gauge;
|
|
MG_Util::PipeStats::PublishGauge(Gauge::WireBuffers, static_cast<Uint64>(m_wireBuffers.size()));
|
|
MG_Util::PipeStats::PublishGauge(Gauge::WireStoresPeak, m_wireStoreCountPeak);
|
|
MG_Util::PipeStats::PublishGauge(Gauge::WireDeferredBytesPeak, m_deferredWireBytesPeak);
|
|
MG_Util::PipeStats::PublishGauge(Gauge::WireDeferredSyncs, m_wireDeferredSyncs);
|
|
}
|
|
|
|
Bool VkBufferManager::LatchedOnDeviceLoss(const char* site) {
|
|
return pVulkanRenderer != nullptr && pVulkanRenderer->LatchWireDeviceLoss(site);
|
|
}
|
|
|
|
void VkBufferManager::WireBufferSyncFatal(const char* site) {
|
|
MGLOG_F("Magma: Fatal{ResourceUnavailable, \"buffer-write-sync\"} {site=%s}", site);
|
|
std::abort();
|
|
}
|
|
|
|
SizeT VkBufferManager::SweepDeferredWireReleases() {
|
|
if (m_deferredWireReleases.empty()) return 0;
|
|
SizeT dead = 0;
|
|
if (pVulkanRenderer == nullptr ||
|
|
pVulkanRenderer->IsSubmitIndexComplete(pVulkanRenderer->GetSyncPointSubmitIndex())) {
|
|
// Idle: nothing recorded, every submission retired - or no renderer, so no queue.
|
|
// Every parked store is dead, including one tagged for a batch that was abandoned
|
|
// instead of submitted.
|
|
dead = m_deferredWireReleases.size();
|
|
} else {
|
|
// The list is in park order and GetSyncPointSubmitIndex() steps back only when a
|
|
// pending recording is abandoned instead of submitted (RecreateSwapchain and the
|
|
// minimized Present force-clear the recording flags, so `m_submitCounter + 1`
|
|
// becomes `m_submitCounter`). Everywhere else the submit indices are non-decreasing
|
|
// and the dead entries are a PREFIX: the first entry the renderer will not call
|
|
// complete proves none after it is either. Across an abandonment a later entry can
|
|
// carry the LOWER index; the walk then stops at the earlier, higher one and holds
|
|
// both until the next submission takes that index and retires - conservative, never
|
|
// early, and the idle rule above retires them regardless. That bounds the walk to a
|
|
// couple of fence polls, which matters because this runs on every glBufferData the
|
|
// wire carries.
|
|
while (dead < m_deferredWireReleases.size() &&
|
|
pVulkanRenderer->IsSubmitIndexComplete(m_deferredWireReleases[dead].submitIndex)) {
|
|
++dead;
|
|
}
|
|
}
|
|
if (dead == 0) return 0;
|
|
for (SizeT i = 0; i < dead; ++i) {
|
|
DeferredWireRelease& entry = m_deferredWireReleases[i];
|
|
m_deferredWireBytes -= entry.bytes;
|
|
entry.buffer.Destroy();
|
|
}
|
|
// One bump per sweep that destroyed anything is enough: what a memo needs to know
|
|
// is "some handle it may name has been freed since", not how many.
|
|
++m_wireStoreDestroyEpoch;
|
|
m_deferredWireReleases.erase(m_deferredWireReleases.begin(),
|
|
m_deferredWireReleases.begin() + static_cast<std::ptrdiff_t>(dead));
|
|
const Uint64 destroyedCount = static_cast<Uint64>(dead);
|
|
m_wireStoreCount = destroyedCount <= m_wireStoreCount ? m_wireStoreCount - destroyedCount : 0;
|
|
return dead;
|
|
}
|
|
|
|
void VkBufferManager::DestroyAllDeferredWireReleases() {
|
|
if (!m_deferredWireReleases.empty()) ++m_wireStoreDestroyEpoch;
|
|
for (auto& entry : m_deferredWireReleases) {
|
|
entry.buffer.Destroy();
|
|
if (m_wireStoreCount > 0) --m_wireStoreCount;
|
|
}
|
|
m_deferredWireReleases.clear();
|
|
m_deferredWireBytes = 0;
|
|
}
|
|
|
|
Bool VkBufferManager::Initialize(const VkBufferManagerInitInfo& initInfo) {
|
|
Shutdown();
|
|
|
|
MOBILEGL_ASSERT(initInfo.allocator != nullptr, "VkBufferManager::Initialize requires valid allocator");
|
|
MOBILEGL_ASSERT(initInfo.frameCount > 0, "VkBufferManager::Initialize requires non-zero frame count");
|
|
|
|
m_initInfo = initInfo;
|
|
m_deferredBufferReleases.resize(initInfo.frameCount);
|
|
m_currentFrameIndex = 0;
|
|
m_frameSerial = 1;
|
|
m_completedSerialFloor = 0;
|
|
if (!InitializeTransientArenas()) {
|
|
return false;
|
|
}
|
|
g_activeBufferManager.Get() = this;
|
|
RegisterWireResourceOps();
|
|
return true;
|
|
}
|
|
|
|
void VkBufferManager::Shutdown() {
|
|
if (g_activeBufferManager.Get() == this) {
|
|
g_activeBufferManager.Get() = nullptr;
|
|
// Under a transport the table is the PROCESS's and serves every session: it dispatches
|
|
// to the calling session's own manager (WireManager), so one session's renderer going
|
|
// away must not take it from the others - their buffer creates would be dropped from
|
|
// then on, and every buffer born after that would be a handle the server never saw.
|
|
if (MG_Config::Transport == MG_Config::TransportMode::Monolith &&
|
|
MG_Pipe::MGPipeGetResourceOps() == &g_vulkanWireResourceOps) {
|
|
MG_Pipe::MGPipeSetResourceOps(nullptr);
|
|
}
|
|
}
|
|
m_transientUploadArena.Shutdown();
|
|
m_unboundStorageBuffer.Destroy();
|
|
m_unboundTexelBuffer.Destroy();
|
|
DestroyAllDeferredReleases();
|
|
// DestroyAllDeferredReleases above emptied the parked list; this destroys the stores the
|
|
// records still hold, so nothing this arm minted outlives the count.
|
|
if (!m_wireBuffers.empty()) ++m_wireStoreDestroyEpoch;
|
|
m_wireBufferLookup.clear();
|
|
m_wireBuffers.clear();
|
|
m_wireShadowBytes = 0;
|
|
m_wireStoreCount = 0;
|
|
m_wireStoreCountPeak = 0;
|
|
m_deferredWireBytesPeak = 0;
|
|
m_wireDeferredSyncs = 0;
|
|
m_copyProvider = nullptr;
|
|
m_initInfo = {};
|
|
m_currentFrameIndex = 0;
|
|
m_frameSerial = 1;
|
|
m_completedSerialFloor = 0;
|
|
}
|
|
|
|
Bool VkBufferManager::RecreateTransientArenas(Uint32 frameCount) {
|
|
MOBILEGL_ASSERT(m_initInfo.allocator != nullptr,
|
|
"VkBufferManager::RecreateTransientArenas requires initialized manager");
|
|
MOBILEGL_ASSERT(frameCount > 0, "VkBufferManager::RecreateTransientArenas requires non-zero frame count");
|
|
|
|
// Callers guarantee the device is idle around arena recreation.
|
|
NotifyDeviceIdle();
|
|
m_transientUploadArena.Shutdown();
|
|
m_initInfo.frameCount = frameCount;
|
|
DestroyAllDeferredReleases();
|
|
m_deferredBufferReleases.resize(frameCount);
|
|
m_currentFrameIndex = 0;
|
|
return InitializeTransientArenas();
|
|
}
|
|
|
|
void VkBufferManager::BeginFrame(Uint32 frameIndex) {
|
|
MOBILEGL_ASSERT(frameIndex < m_deferredBufferReleases.size(),
|
|
"VkBufferManager::BeginFrame frame index out of range");
|
|
m_currentFrameIndex = frameIndex;
|
|
++m_frameSerial;
|
|
CollectDeferredReleases(frameIndex);
|
|
m_transientUploadArena.BeginFrame(frameIndex);
|
|
}
|
|
|
|
void VkBufferManager::CollectAllDeferredReleases() {
|
|
// Per-resource releases only. Every one of them was deferred behind a BumpSliceEpoch,
|
|
// so no memo can still name the handle, and the caller has proved the GPU is idle.
|
|
//
|
|
// The transient arena's releases are deliberately NOT collected here. A buffer lands
|
|
// there when the arena outgrows it mid-frame (BufferArena::EnsureCapacity), and at
|
|
// that moment every slice already handed out from this frame's arena still names it -
|
|
// VkBufferResource::transientSlice above all, which AcquireStreamedSlice keeps
|
|
// serving for the whole frame serial on the strength of transientFrameSerial alone.
|
|
// Nothing bumps the slice epoch for those other resources, so freeing the buffer
|
|
// here left the streamed memo handing a destroyed VkBuffer to vkCmdBindIndexBuffer
|
|
// (llvmpipe then faulted inside the draw; the Create/Flywheel indirect retrace died
|
|
// exactly this way). Most mid-frame drains do not advance m_frameSerial; every
|
|
// eighth does, but only through BeginFrame after the arena/memo boundary work.
|
|
// The drain's per-resource sweep must not free arena storage: ResetFrame/BeginFrame is where
|
|
// the slot's slices stop being reachable, and that is where these releases land.
|
|
for (Uint32 frameIndex = 0; frameIndex < m_deferredBufferReleases.size(); ++frameIndex) {
|
|
CollectDeferredReleases(frameIndex);
|
|
}
|
|
}
|
|
|
|
void VkBufferManager::NotifyDeviceIdle() {
|
|
// Everything submitted so far has completed. Work recorded for the
|
|
// current frame has not been submitted yet, so the current serial
|
|
// remains busy.
|
|
if (m_frameSerial > 0) {
|
|
m_completedSerialFloor = m_frameSerial - 1;
|
|
}
|
|
}
|
|
|
|
void VkBufferManager::NotifyFrameSerialComplete(Uint64 serial) {
|
|
// The current serial's work is still being recorded; a completion
|
|
// report for it (or beyond) can only come from a stale caller.
|
|
if (serial >= m_frameSerial) {
|
|
return;
|
|
}
|
|
m_completedSerialFloor = std::max(m_completedSerialFloor, serial);
|
|
}
|
|
|
|
void VkBufferManager::SetCopyCommandProvider(IBufferCopyCommandProvider* provider) {
|
|
m_copyProvider = provider;
|
|
}
|
|
|
|
Uint64 VkBufferManager::GetCompletedSerial() const {
|
|
const Uint64 frameCount = m_initInfo.frameCount > 0 ? m_initInfo.frameCount : 1;
|
|
const Uint64 completed = m_frameSerial > frameCount ? m_frameSerial - frameCount : 0;
|
|
return std::max(completed, m_completedSerialFloor);
|
|
}
|
|
|
|
Bool VkBufferManager::UploadTransient(BufferKind kind, Uint32 frameIndex, const void* data, VkDeviceSize size,
|
|
VkDeviceSize alignment, BufferSlice& outSlice) {
|
|
if (!m_transientUploadArena.Upload(frameIndex, data, size, alignment, outSlice)) {
|
|
return false;
|
|
}
|
|
if (MG_Util::PipeStats::Enabled()) {
|
|
// The single chokepoint for Magma's per-draw staging. Uniform is deliberately
|
|
// absent: its bytes are counted by the caller, which is the only place that
|
|
// knows whether the payload is the default block (stage-ubo-global) or a named
|
|
// one repacked into the ring (stage-ubo-named), and counting here as well would
|
|
// double every uniform byte.
|
|
switch (kind) {
|
|
case BufferKind::Vertex:
|
|
MG_Util::PipeStats::AddBytes(MG_Util::PipeStats::ByteClass::StageVertexClient,
|
|
static_cast<Uint64>(size));
|
|
break;
|
|
case BufferKind::Index:
|
|
MG_Util::PipeStats::AddBytes(MG_Util::PipeStats::ByteClass::StageIndexClient,
|
|
static_cast<Uint64>(size));
|
|
break;
|
|
case BufferKind::Indirect:
|
|
MG_Util::PipeStats::AddBytes(MG_Util::PipeStats::ByteClass::StageIndirectCmd,
|
|
static_cast<Uint64>(size));
|
|
break;
|
|
case BufferKind::TextureBuffer:
|
|
case BufferKind::ShaderStorage:
|
|
MG_Util::PipeStats::AddBytes(MG_Util::PipeStats::ByteClass::StageBuffer, static_cast<Uint64>(size));
|
|
break;
|
|
case BufferKind::Uniform:
|
|
break;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
Bool VkBufferManager::InitializeTransientArenas() {
|
|
return m_transientUploadArena.Initialize({
|
|
.allocator = m_initInfo.allocator,
|
|
.frameCount = m_initInfo.frameCount,
|
|
.usage = VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT |
|
|
(VkBufferUsageFlags{VK_BUFFER_USAGE_TRANSFER_DST_BIT}) | VK_BUFFER_USAGE_TRANSFER_SRC_BIT,
|
|
.memoryUsage = m_initInfo.transientMemoryUsage,
|
|
.allocationFlags = m_initInfo.transientAllocationFlags,
|
|
.minBufferSize = m_initInfo.minUploadBytes,
|
|
.persistentlyMapped = m_initInfo.transientPersistentMapping,
|
|
});
|
|
}
|
|
|
|
Bool VkBufferManager::StagedWireRangeCopy(WireBufferResource& resource, const void* data, SizeT offset,
|
|
SizeT size) {
|
|
if (!m_copyProvider) {
|
|
return false;
|
|
}
|
|
BufferSlice staging{};
|
|
if (!m_transientUploadArena.Upload(m_currentFrameIndex, data, static_cast<VkDeviceSize>(size), 16, staging)) {
|
|
return false;
|
|
}
|
|
if (MG_Util::PipeStats::Enabled()) {
|
|
// The staging fill is the host copy; the vkCmdCopyBuffer below is the device
|
|
// half of the same bytes and is not counted twice.
|
|
MG_Util::PipeStats::AddBytes(MG_Util::PipeStats::ByteClass::StageBuffer, static_cast<Uint64>(size));
|
|
}
|
|
VkCommandBuffer commandBuffer = m_copyProvider->AcquireBufferCopyCommandBuffer();
|
|
if (commandBuffer == VK_NULL_HANDLE) {
|
|
return false;
|
|
}
|
|
|
|
// Order the copy after every prior read/write of this buffer, both from
|
|
// in-flight frames (submission order) and from commands already recorded
|
|
// in this frame's command buffer.
|
|
VkMemoryBarrier beforeBarrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
|
|
beforeBarrier.srcAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
|
|
beforeBarrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
|
|
EndActiveRenderPassOn(commandBuffer);
|
|
vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1,
|
|
&beforeBarrier, 0, nullptr, 0, nullptr);
|
|
|
|
VkBufferCopy region{};
|
|
region.srcOffset = staging.offset;
|
|
region.dstOffset = static_cast<VkDeviceSize>(offset);
|
|
region.size = static_cast<VkDeviceSize>(size);
|
|
EndActiveRenderPassOn(commandBuffer);
|
|
vkCmdCopyBuffer(commandBuffer, staging.buffer, resource.buffer.GetHandle(), 1, ®ion);
|
|
|
|
VkMemoryBarrier afterBarrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
|
|
afterBarrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
|
|
afterBarrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
|
|
EndActiveRenderPassOn(commandBuffer);
|
|
vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1,
|
|
&afterBarrier, 0, nullptr, 0, nullptr);
|
|
|
|
resource.lastUseSerial = m_frameSerial;
|
|
// B3: the copy just recorded rides the next submission too, so a later host write to
|
|
// the same bytes must wait for it exactly as a draw's read must.
|
|
if (pVulkanRenderer) resource.lastUseSubmitIndex = pVulkanRenderer->GetWireNextSubmitIndex();
|
|
resource.gpuWritesPending = true;
|
|
return true;
|
|
}
|
|
|
|
void VkBufferManager::CollectDeferredReleases(Uint32 frameIndex) {
|
|
MOBILEGL_ASSERT(frameIndex < m_deferredBufferReleases.size(),
|
|
"VkBufferManager::CollectDeferredReleases frame index out of range");
|
|
m_deferredBufferReleases[frameIndex].clear();
|
|
// The wire list is not per-slot and does not wait for a frame boundary; a boundary is
|
|
// just one more point to sweep at (BeginFrame after its slot fence, and every slot of
|
|
// the idle drain, where the renderer-idle rule empties the list).
|
|
SweepDeferredWireReleases();
|
|
}
|
|
|
|
BufferSlice VkBufferManager::AcquireUnboundStorageDescriptor() {
|
|
if (!m_unboundStorageBuffer.IsValid()) {
|
|
if (m_initInfo.allocator == nullptr) {
|
|
return {};
|
|
}
|
|
// Host-visible so the zero fill needs no command buffer: this can be reached from
|
|
// descriptor resolution, which runs inside an already-open recording and must not
|
|
// start a copy of its own. The size is a whole minStorageBufferOffsetAlignment-safe
|
|
// block rather than 4 bytes so that a shader which does read the block gets a
|
|
// plausible unsized-array length instead of one that rounds to zero.
|
|
const Bool created = m_unboundStorageBuffer.Create({
|
|
.allocator = m_initInfo.allocator,
|
|
.size = kUnboundStorageDescriptorBytes,
|
|
.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
|
|
.memoryUsage = VMA_MEMORY_USAGE_AUTO,
|
|
.allocationFlags =
|
|
VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT,
|
|
.requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
|
|
});
|
|
if (!created) {
|
|
MGLOG_E_ONCE("VkBufferManager::AcquireUnboundStorageDescriptor: placeholder creation failed");
|
|
m_unboundStorageBuffer.Destroy();
|
|
return {};
|
|
}
|
|
if (void* mapped = m_unboundStorageBuffer.GetMappedData()) {
|
|
Memset(mapped, 0, static_cast<SizeT>(kUnboundStorageDescriptorBytes));
|
|
}
|
|
}
|
|
return m_unboundStorageBuffer.GetSlice();
|
|
}
|
|
|
|
BufferSlice VkBufferManager::AcquireUnboundTexelBufferDescriptor() {
|
|
if (!m_unboundTexelBuffer.IsValid()) {
|
|
if (m_initInfo.allocator == nullptr) {
|
|
return {};
|
|
}
|
|
// A SECOND placeholder rather than more usage bits on the storage-block one. The two
|
|
// are independent failure domains: a device that refuses this allocation must not
|
|
// take the storage-block placeholder - and with it the fix this one is a sibling of -
|
|
// down with it. Host-visible and zero-filled for the same reason as that one: this is
|
|
// reached from descriptor resolution, inside an already-open recording, which must
|
|
// not start a copy of its own.
|
|
const Bool created = m_unboundTexelBuffer.Create({
|
|
.allocator = m_initInfo.allocator,
|
|
.size = kUnboundTexelBufferDescriptorBytes,
|
|
.usage = VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT |
|
|
VK_BUFFER_USAGE_TRANSFER_DST_BIT,
|
|
.memoryUsage = VMA_MEMORY_USAGE_AUTO,
|
|
.allocationFlags =
|
|
VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT,
|
|
.requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
|
|
});
|
|
if (!created) {
|
|
MGLOG_E_ONCE("VkBufferManager::AcquireUnboundTexelBufferDescriptor: placeholder creation failed");
|
|
m_unboundTexelBuffer.Destroy();
|
|
return {};
|
|
}
|
|
if (void* mapped = m_unboundTexelBuffer.GetMappedData()) {
|
|
Memset(mapped, 0, static_cast<SizeT>(kUnboundTexelBufferDescriptorBytes));
|
|
}
|
|
}
|
|
return m_unboundTexelBuffer.GetSlice();
|
|
}
|
|
|
|
void VkBufferManager::DestroyAllDeferredReleases() {
|
|
// Both callers (Shutdown, RecreateTransientArenas) have proven the device idle.
|
|
DestroyAllDeferredWireReleases();
|
|
for (auto& stream : m_convertedStreams)
|
|
stream.buffer.Destroy();
|
|
m_convertedStreams.clear();
|
|
m_convertedStreamBytes = 0;
|
|
for (auto& releases : m_deferredBufferReleases) {
|
|
for (auto& buffer : releases) {
|
|
buffer.Destroy();
|
|
}
|
|
releases.clear();
|
|
}
|
|
m_deferredBufferReleases.clear();
|
|
}
|
|
} // namespace MobileGL::MG_Backend::DirectVulkan
|