[Fix] (Magma): probe and work around Adreno E5 transfer reads and cross-format slice copies

This commit is contained in:
2026-10-12 21:00:18 -04:00
parent 065d1582ba
commit 4eb0dbc8a9
17 changed files with 2565 additions and 20 deletions
+2
View File
@@ -733,6 +733,8 @@ list(APPEND SOURCE_FILES
# arms and the probe that measures the first against the second, shared by the
# renderer's bring-up and the Vulkan POST.
MobileGL/MG_Backend/DirectVulkan/Renderer/WireDepthResolveProbe.cpp
MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.cpp
MobileGL/MG_Backend/DirectVulkan/Renderer/WireCopySliceProbe.cpp
# P5 b1: the block-granularity persistent-map push the record arm makes mandatory (T2).
MobileGL/MG_State/GLState/BufferState/PersistentMapTracker.cpp
# P13 W5: the verb emitters and the monolith verb port, and the client GPU-write set
+16
View File
@@ -381,6 +381,22 @@ namespace MobileGL::MG_Config {
// immune to the probe's verdict moving), and ForceOff is the negative control that
// replays the driver's silence.
QuirkOverride MagmaPrimGenQueryReroute = QuirkOverride::Auto;
// MOBILEGL_MAGMA_E5_RAW_READS: DirectVulkan's raw read route for E5B9G9R9 textures
// (WireE5RawCopy.h). GL moves RGB9_E5 texel blocks bit-exactly (glGetTexImage with
// UNSIGNED_INT_5_9_9_9_REV, glCopyImageSubData), but Adreno 750's transfer reads of
// an E5B9G9R9 image larger than ~16x16 decode and re-encode, canonicalizing every
// non-canonical shared exponent. Auto defers to a device probe at renderer bring-up
// (RunWireE5TransferProbe: plain transfer read vs. a shader read through an R32_UINT
// view) and arms the route only where the plain read changed words; ForceOn arms it
// on any device (the host lanes exercise the route with it); ForceOff is the negative
// control that replays the canonicalization.
QuirkOverride MagmaE5RawReads = QuirkOverride::Auto;
// MOBILEGL_MAGMA_SPLIT_SLICE_COPIES: DirectVulkan's per-slice split of a cross-format
// glCopyImageSubData between a layered texture and a 3D one (WireCopySliceProbe.h).
// Adreno 750 lands only the first slice of such a vkCmdCopyImage region. Auto defers to
// a device probe at renderer bring-up (one region vs. per-slice regions); ForceOn splits
// on any device; ForceOff is the negative control that replays the dropped slices.
QuirkOverride MagmaSplitSliceCopies = QuirkOverride::Auto;
// --- MGPipe (the disaggregation plan's explicit frontend/backend boundary) ---
// MOBILEGL_PIPE_PUSH: per-subsystem bitmask selecting which state the frontend
// PUSHES over MGPipe instead of leaving the backend to pull it out of GLContext.
+2
View File
@@ -420,6 +420,8 @@ namespace MobileGL::MG_ConfigLoader {
features.EsprytWidenPacked16Storage =
QueryEnvQuirkOverride("MOBILEGL_ESPRYT_WIDEN_PACKED16_STORAGE");
features.MagmaPrimGenQueryReroute = QueryEnvQuirkOverride("MOBILEGL_MAGMA_PRIMGEN_QUERY_REROUTE");
features.MagmaE5RawReads = QueryEnvQuirkOverride("MOBILEGL_MAGMA_E5_RAW_READS");
features.MagmaSplitSliceCopies = QueryEnvQuirkOverride("MOBILEGL_MAGMA_SPLIT_SLICE_COPIES");
// MGPipe. Nothing here needs adding to an allow-list: InitializeAcceptedEnvVariables
// accepts every MOBILEGL_ / LIBGL_ prefixed variable in the environment, so a name
// that starts with MOBILEGL_ is visible to these queries by construction.
@@ -274,6 +274,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
const VkTextureManager::TextureResource& oldResource,
VkTextureManager::TextureResource& newResource
, Bool splitAspects = false
, VkTextureManager* rawE5Manager = nullptr
) {
MOBILEGL_ASSERT(device != VK_NULL_HANDLE, "PreserveTextureContentsOnRecreate: device is null");
MOBILEGL_ASSERT(commandPool != VK_NULL_HANDLE, "PreserveTextureContentsOnRecreate: commandPool is null");
@@ -307,19 +308,74 @@ namespace MobileGL::MG_Backend::DirectVulkan {
0, VK_ACCESS_TRANSFER_WRITE_BIT, newResource.aspect, 0, newResource.mipLevels);
MOBILEGL_ASSERT(ok, "PreserveTextureContentsOnRecreate: failed to prepare destination image");
// E5B9G9R9 on a device whose transfer reads canonicalize it: each level leaves through the
// raw route (WireE5RawCopy.h) into a twin, and the twins are copied instead. The old image
// and the new one must agree on the shape for the twin's single-level copy to stand in.
const Bool rawE5 = rawE5Manager != nullptr && rawE5Manager->WireE5RawReadApplies(oldResource) &&
newResource.format == oldResource.format && newResource.viewType == oldResource.viewType;
VkImageLayout srcTrackedLayout = oldResource.layout;
VkPipelineStageFlags srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT;
VkAccessFlags srcAccessMask = 0;
GetImageTransitionSourceState(srcTrackedLayout, srcStageMask, srcAccessMask);
ok = VkTextureManager::TransitionImageLayout(
commandBuffer, oldResource.image, srcTrackedLayout, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
srcStageMask, VK_PIPELINE_STAGE_TRANSFER_BIT,
srcAccessMask, VK_ACCESS_TRANSFER_READ_BIT, oldResource.aspect, 0, preservedMipLevels);
commandBuffer, oldResource.image, srcTrackedLayout,
rawE5 ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
srcStageMask, rawE5 ? VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT : VK_PIPELINE_STAGE_TRANSFER_BIT,
srcAccessMask, rawE5 ? VK_ACCESS_SHADER_READ_BIT : VK_ACCESS_TRANSFER_READ_BIT,
oldResource.aspect, 0, preservedMipLevels);
MOBILEGL_ASSERT(ok, "PreserveTextureContentsOnRecreate: failed to prepare source image");
// Destroyed after the fence wait below, which is the twins' last use.
Vector<VkTextureManager::TextureResource> rawTwins;
Vector<WireE5RawCopyObjects> rawObjects;
if (rawE5) {
rawTwins.reserve(preservedMipLevels);
const Bool volume = oldResource.viewType == VK_IMAGE_VIEW_TYPE_3D;
for (Uint32 level = 0; level < preservedMipLevels; ++level) {
WireE5RawCopyRegion region;
region.source = oldResource.image;
region.volume = volume;
region.level = level;
region.layerCount = volume ? 1u : std::min(oldResource.arrayLayers, newResource.arrayLayers);
region.extent = {std::max(std::min(oldResource.extent.width, newResource.extent.width) >> level, 1u),
std::max(std::min(oldResource.extent.height, newResource.extent.height) >> level, 1u),
volume ? std::max(std::min(oldResource.depth, newResource.depth) >> level, 1u) : 1u};
VkTextureManager::TextureResource twin;
WireE5RawCopyObjects objects;
if (!rawE5Manager->RecordWireE5RawTwin(commandBuffer, region, VK_IMAGE_LAYOUT_GENERAL, twin, objects)) break;
VkImageCopy copy{};
copy.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, region.layerCount};
copy.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, 0, region.layerCount};
copy.extent = region.extent;
vkCmdCopyImage(commandBuffer, twin.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, newResource.image,
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &copy);
rawTwins.push_back(Move(twin));
rawObjects.push_back(objects);
}
if (rawTwins.size() != preservedMipLevels) {
// A twin could not be made: the remaining levels go by transfer (canonicalized,
// as every level was before the route) rather than not at all.
MGLOG_W_ONCE("PreserveTextureContentsOnRecreate: the raw E5B9G9R9 route failed; levels %zu.. "
"are copied by transfer", rawTwins.size());
}
}
const Uint32 firstTransferLevel = static_cast<Uint32>(rawTwins.size());
if (rawE5 && firstTransferLevel < preservedMipLevels) {
VkImageMemoryBarrier toTransfer{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
toTransfer.srcAccessMask = VK_ACCESS_SHADER_READ_BIT;
toTransfer.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
toTransfer.oldLayout = VK_IMAGE_LAYOUT_GENERAL;
toTransfer.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
toTransfer.srcQueueFamilyIndex = toTransfer.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
toTransfer.image = oldResource.image;
toTransfer.subresourceRange = {oldResource.aspect, 0, preservedMipLevels, 0, VK_REMAINING_ARRAY_LAYERS};
vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0,
0, nullptr, 0, nullptr, 1, &toTransfer);
}
Vector<VkImageCopy> copyRegions;
copyRegions.reserve(preservedMipLevels);
for (Uint32 level = 0; level < preservedMipLevels; ++level) {
for (Uint32 level = firstTransferLevel; level < preservedMipLevels; ++level) {
VkImageCopy copy{};
copy.srcSubresource.aspectMask = oldResource.aspect;
copy.srcSubresource.mipLevel = level;
@@ -346,10 +402,11 @@ namespace MobileGL::MG_Backend::DirectVulkan {
}
EndActiveRenderPassOn(commandBuffer);
vkCmdCopyImage(commandBuffer,
oldResource.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
newResource.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
static_cast<Uint32>(copyRegions.size()), copyRegions.data());
if (!copyRegions.empty())
vkCmdCopyImage(commandBuffer,
oldResource.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
newResource.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
static_cast<Uint32>(copyRegions.size()), copyRegions.data());
VkPipelineStageFlags dstStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT;
VkAccessFlags dstAccessMask = 0;
@@ -373,6 +430,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
VK_VERIFY(vkCreateFence(device, &fenceInfo, nullptr, &fence), "vkCreateFence(texture preserve)");
VK_VERIFY(vkQueueSubmit(graphicsQueue, 1, &submitInfo, fence), "vkQueueSubmit(texture preserve)");
VK_VERIFY(vkWaitForFences(device, 1, &fence, VK_TRUE, UINT64_MAX), "vkWaitForFences(texture preserve)");
for (WireE5RawCopyObjects& objects : rawObjects) DestroyWireE5RawCopyObjects(device, objects);
vkDestroyFence(device, fence, nullptr);
vkFreeCommandBuffers(device, commandPool, 1, &commandBuffer);
@@ -486,6 +544,8 @@ namespace MobileGL::MG_Backend::DirectVulkan {
m_deferredReleases.resize(initInfo.frameCount);
m_deferredViewReleases.clear();
m_deferredViewReleases.resize(initInfo.frameCount);
m_deferredE5Releases.clear();
m_deferredE5Releases.resize(initInfo.frameCount);
MOBILEGL_ASSERT(m_device != VK_NULL_HANDLE && m_physicalDevice != VK_NULL_HANDLE && m_allocator != nullptr &&
m_commandPool != VK_NULL_HANDLE && m_graphicsQueue != VK_NULL_HANDLE,
@@ -1252,7 +1312,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
if (preserve) {
FlushPendingUploads();
if (!PreserveTextureContentsOnRecreate(m_device, m_commandPool, m_graphicsQueue,
resource, replacement, true)) {
resource, replacement, true, this)) {
MGL_WIRE_DECLINE_AT(ShapePreserveCopyFailed,
"texture {slot=%u, gen=%u}: its texels could not be copied into the "
"reallocated image",
@@ -1953,6 +2013,45 @@ namespace MobileGL::MG_Backend::DirectVulkan {
static_cast<Uint16>(target), static_cast<Uint16>(mipLevel), true);
}
Bool VkTextureManager::WireE5RawReadApplies(const TextureResource& resource) const {
return m_wireE5RawCopyKit != nullptr && resource.format == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 &&
resource.sampleCount == VK_SAMPLE_COUNT_1_BIT &&
(resource.imageCreateFlags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) != 0 &&
resource.viewType != VK_IMAGE_VIEW_TYPE_1D && resource.viewType != VK_IMAGE_VIEW_TYPE_1D_ARRAY;
}
Bool VkTextureManager::RecordWireE5RawTwin(VkCommandBuffer commandBuffer, const WireE5RawCopyRegion& region,
VkImageLayout sourceLayout, TextureResource& twin,
WireE5RawCopyObjects& objects) {
if (m_wireE5RawCopyKit == nullptr) return false;
const VkImageCreateInfo info = WireE5RawCopyTwinInfo(region);
VmaAllocationCreateInfo allocation{};
allocation.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
allocation.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
if (vmaCreateImage(m_allocator, &info, &allocation, &twin.image, &twin.allocation, nullptr) != VK_SUCCESS) {
MGLOG_E_ONCE("RecordWireE5RawTwin: vmaCreateImage failed");
return false;
}
twin.format = info.format;
twin.extent = {info.extent.width, info.extent.height};
twin.depth = info.extent.depth;
twin.arrayLayers = info.arrayLayers;
twin.mipLevels = 1;
twin.aspect = VK_IMAGE_ASPECT_COLOR_BIT;
twin.viewType = region.volume ? VK_IMAGE_VIEW_TYPE_3D : VK_IMAGE_VIEW_TYPE_2D_ARRAY;
EndActiveRenderPassOn(commandBuffer);
const VkResult result = RecordWireE5RawCopy(m_device, commandBuffer, *m_wireE5RawCopyKit, region,
sourceLayout, twin.image, objects);
if (result != VK_SUCCESS) {
MGLOG_E_ONCE("RecordWireE5RawTwin: recording the raw copy failed (%d)", static_cast<Int>(result));
DestroyWireE5RawCopyObjects(m_device, objects);
twin.Reset();
return false;
}
twin.layout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
return true;
}
Bool VkTextureManager::GrowWireTextureMipChain(MG_Pipe::MGPipeHandle handle, Uint32 requiredMipLevels,
VkCommandBuffer commandBuffer) {
if (MG_Pipe::MGPipeHandleIsNull(handle)) {
@@ -2006,13 +2105,18 @@ namespace MobileGL::MG_Backend::DirectVulkan {
const VkImageAspectFlags aspect = old.aspect;
const VkImageLayout oldLayout = old.layout;
// E5B9G9R9 on a device whose transfer reads canonicalize it: every level leaves through the
// raw route (WireE5RawCopy.h), so the old image is read by a shader, from GENERAL.
const Bool rawE5 = WireE5RawReadApplies(old);
VkPipelineStageFlags srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT;
VkAccessFlags srcAccessMask = 0;
GetImageTransitionSourceState(oldLayout, srcStageMask, srcAccessMask);
Bool ok = TransitionImageLayout(commandBuffer, old.image, old.layout,
VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, srcStageMask,
VK_PIPELINE_STAGE_TRANSFER_BIT, srcAccessMask,
VK_ACCESS_TRANSFER_READ_BIT, aspect, 0, old.mipLevels);
rawE5 ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
srcStageMask,
rawE5 ? VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT : VK_PIPELINE_STAGE_TRANSFER_BIT,
srcAccessMask, rawE5 ? VK_ACCESS_SHADER_READ_BIT : VK_ACCESS_TRANSFER_READ_BIT,
aspect, 0, old.mipLevels);
MOBILEGL_ASSERT(ok, "GrowWireTextureMipChain: source transition failed");
VkImageLayout grownLayout = VK_IMAGE_LAYOUT_UNDEFINED;
ok = TransitionImageLayout(commandBuffer, grown.image, grownLayout,
@@ -2029,6 +2133,44 @@ namespace MobileGL::MG_Backend::DirectVulkan {
copy.dstOffset = {0, 0, 0};
copy.extent = {std::max(old.extent.width >> level, 1u), std::max(old.extent.height >> level, 1u),
std::max(old.depth >> level, 1u)};
if (rawE5) {
const Bool volume = old.viewType == VK_IMAGE_VIEW_TYPE_3D;
WireE5RawCopyRegion region;
region.source = old.image;
region.volume = volume;
region.level = level;
region.layerCount = volume ? 1u : old.arrayLayers;
region.extent = copy.extent;
TextureResource twin;
WireE5RawCopyObjects objects;
if (!RecordWireE5RawTwin(commandBuffer, region, VK_IMAGE_LAYOUT_GENERAL, twin, objects)) {
// Nothing was recorded for this level; the plain copy below reads the
// canonicalized words, which is what every level got before the route.
MGLOG_W_ONCE("GrowWireTextureMipChain: the raw E5B9G9R9 route failed; level %u is copied "
"by transfer", level);
VkImageMemoryBarrier toTransfer{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
toTransfer.srcAccessMask = VK_ACCESS_SHADER_READ_BIT;
toTransfer.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
toTransfer.oldLayout = toTransfer.newLayout = VK_IMAGE_LAYOUT_GENERAL;
toTransfer.srcQueueFamilyIndex = toTransfer.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
toTransfer.image = old.image;
toTransfer.subresourceRange = {aspect, level, 1, 0, VK_REMAINING_ARRAY_LAYERS};
vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toTransfer);
EndActiveRenderPassOn(commandBuffer);
vkCmdCopyImage(commandBuffer, old.image, VK_IMAGE_LAYOUT_GENERAL, grown.image,
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &copy);
continue;
}
VkImageCopy twinCopy = copy;
twinCopy.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, region.layerCount};
vkCmdCopyImage(commandBuffer, twin.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, grown.image,
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &twinCopy);
// Both name the frame's command buffer, so they ride the old image's ring slot.
m_deferredE5Releases[m_currentFrameIndex].push_back(objects);
DeferResourceRelease(Move(twin));
continue;
}
for (const VkImageAspectFlags copyAspect : {VK_IMAGE_ASPECT_COLOR_BIT,
VK_IMAGE_ASPECT_DEPTH_BIT,
VK_IMAGE_ASPECT_STENCIL_BIT}) {
@@ -2107,6 +2249,11 @@ namespace MobileGL::MG_Backend::DirectVulkan {
MOBILEGL_ASSERT(frameIndex < m_deferredReleases.size(),
"VkTextureManager::CollectDeferredReleases invalid frame index %u (size=%zu)",
frameIndex, m_deferredReleases.size());
if (frameIndex < m_deferredE5Releases.size()) {
for (WireE5RawCopyObjects& objects : m_deferredE5Releases[frameIndex])
DestroyWireE5RawCopyObjects(m_device, objects);
m_deferredE5Releases[frameIndex].clear();
}
m_deferredReleases[frameIndex].clear();
MOBILEGL_ASSERT(frameIndex < m_deferredViewReleases.size(),
@@ -2348,6 +2495,10 @@ namespace MobileGL::MG_Backend::DirectVulkan {
}
void VkTextureManager::DestroyDeferredReleases() {
for (auto& deferredE5 : m_deferredE5Releases) {
for (WireE5RawCopyObjects& objects : deferredE5) DestroyWireE5RawCopyObjects(m_device, objects);
}
m_deferredE5Releases.clear();
for (auto& deferredReleases : m_deferredReleases) {
deferredReleases.clear();
}
@@ -22,6 +22,7 @@
// The handle-keyed arm's key type. Header-only POD, disagg-only, exactly the shape
// MagmaPipeArms.h already gives this directory.
#include <MG_Pipe/MGPipeHandles.h>
#include "WireE5RawCopy.h"
namespace MobileGL::MG_State::GLState {
class ITextureObject;
@@ -575,6 +576,18 @@ public:
// stays as it was).
Bool GrowWireTextureMipChain(MG_Pipe::MGPipeHandle handle, Uint32 requiredMipLevels,
VkCommandBuffer commandBuffer);
// THE RAW E5B9G9R9 READ ROUTE (WireE5RawCopy.h). The renderer hands its kit over once
// ArmWireE5RawReads has armed the route; nullptr keeps every transfer read plain.
void SetWireE5RawCopyKit(const WireE5RawCopyKit* kit) { m_wireE5RawCopyKit = kit; }
// True when a transfer read of `resource` must go through the route: armed, and a
// single-sample E5B9G9R9 MUTABLE_FORMAT image that is not 1D.
Bool WireE5RawReadApplies(const TextureResource& resource) const;
// Records `region` of such an image (in GENERAL or SHADER_READ_ONLY_OPTIMAL) into a fresh
// R32_UINT twin, left in TRANSFER_SRC_OPTIMAL. On success `twin` owns the image and `objects`
// the views and the pool, and both must outlive the recording's submission. On failure
// nothing was recorded and everything created was destroyed.
Bool RecordWireE5RawTwin(VkCommandBuffer commandBuffer, const WireE5RawCopyRegion& region,
VkImageLayout sourceLayout, TextureResource& twin, WireE5RawCopyObjects& objects);
// Records that GPU writes superseded the staged CPU snapshot of this level. Layer
// ranges select cube faces; array and 3D levels share one staged level shadow.
void MarkWireTextureGpuWritten(MG_Pipe::MGPipeHandle handle, Uint32 mipLevel,
@@ -709,6 +722,7 @@ private:
VkCommandPool m_uploadCommandPool = VK_NULL_HANDLE;
VkQueue m_graphicsQueue = VK_NULL_HANDLE;
Bool m_imageFormatListSupported = false;
const WireE5RawCopyKit* m_wireE5RawCopyKit = nullptr;
Uint32 m_currentFrameIndex = 0;
Uint8 m_gcCounter = 0;
@@ -741,6 +755,8 @@ private:
std::unordered_map<VkFormat, VkSampleCountFlags> m_multisampleCountsByFormat;
Vector<Vector<TextureResource>> m_deferredReleases;
Vector<Vector<VkImageView>> m_deferredViewReleases;
// The raw E5B9G9R9 route's views and descriptor pools, on the same ring as the images they name.
Vector<Vector<WireE5RawCopyObjects>> m_deferredE5Releases;
// --- Batched upload machinery ---
// Uploads within a frame are recorded into ONE shared command buffer and
@@ -51,6 +51,7 @@
#include "WireDeclineTally.h"
#include "WireColorBlitFilter.h"
#include "WireDepthResolveArm.h"
#include "WireCopySliceProbe.h"
#include "WireDepthResolveProbe.h"
#if MOBILEGL_BUILD_DISAGGREGATED
#include <MG_Remote/Server/ServerLoop.h>
@@ -2284,6 +2285,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
m_sampledReadStageMask,
static_cast<Uint32>(m_physicalDevice.queueFamilies.graphicsFamily)});
MOBILEGL_ASSERT(succeeded, "VkTextureManager initialization failed.");
m_textureManager->SetWireE5RawCopyKit(GetWireE5RawCopyKit());
m_renderPassManager =
MakeUnique<VkRenderPassManager>(m_device, m_physicalDevice.handle, m_allocator, m_config,
*m_textureManager, m_swapchainObject);
@@ -2443,6 +2445,9 @@ namespace MobileGL::MG_Backend::DirectVulkan {
#endif
DestroyWireDepthMipmapResources();
DestroyWireMultisampleResolveResources();
if (m_textureManager) m_textureManager->SetWireE5RawCopyKit(nullptr);
DestroyWireE5RawCopyKit(m_device, m_wireE5RawCopyKit);
m_wireE5RawReads = false;
#if MOBILEGL_BUILD_DISAGGREGATED
DestroySharedImagePresentTargets(false);
DestroySharedImageSync();
@@ -3997,6 +4002,9 @@ namespace MobileGL::MG_Backend::DirectVulkan {
// formats whenever they differ (VUID-vkCmdCopyImage-srcImage-01548) and there is no
// downstream check - a mismatched pair is a promise the driver takes at face value.
VkFormat format = VK_FORMAT_UNDEFINED;
// A transfer read of this image canonicalizes E5B9G9R9 words: the copy reads it through
// the raw route (WireE5RawCopy.h) instead.
Bool rawE5 = false;
};
Bool TryResolveCopyImageSliceMapping(TextureTarget target, const CopyImageVkImage& image, Uint32 mipLevel,
@@ -4102,6 +4110,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
out.depth = resource->depth;
out.arrayLayers = resource->arrayLayers;
out.format = resource->format;
out.rawE5 = !renderbuffer && m_textureManager->WireE5RawReadApplies(*resource);
return out.image != VK_NULL_HANDLE;
};
CopyImageVkImage srcImage{};
@@ -4346,8 +4355,9 @@ namespace MobileGL::MG_Backend::DirectVulkan {
// of vkCmdCopyImage accept at once, the barrier covers the WHOLE image (both the source
// level and the destination level are in it), and the access mask carries both directions
// because this one barrier is the edge for both.
// An E5B9G9R9 source the raw route reads is read by a shader, which GENERAL also allows.
const VkImageLayout copySourceLayout =
inPlace ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
(inPlace || srcImage.rawE5) ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
const VkImageLayout copyDestinationLayout =
inPlace ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
const VkAccessFlags copySourceAccess =
@@ -4426,10 +4436,58 @@ namespace MobileGL::MG_Backend::DirectVulkan {
copyRegion.srcOffset.z, MG_Util::ConvertTextureTargetToString(dstTextureTarget).c_str(), dstMipLevel,
copyRegion.dstSubresource.baseArrayLayer, copyRegion.dstSubresource.layerCount,
copyRegion.dstOffset.z, srcWidth, srcHeight, copyRegion.extent.depth);
vkCmdCopyImage(frame.commandBuffer,
srcImage.image, copySourceLayout,
dstImage.image, copyDestinationLayout,
1, &copyRegion);
VkImage copySourceImage = srcImage.image;
VkImageLayout copySourceImageLayout = copySourceLayout;
if (srcImage.rawE5) {
// The words leave through a shader into a twin, and the twin is what is copied. A
// twin that cannot be made leaves the plain copy (canonicalized words) in place.
WireE5RawCopyRegion region;
region.source = srcImage.image;
region.volume = srcSlices.slicesAreDepth;
region.level = srcMipLevel;
region.baseLayer = copyRegion.srcSubresource.baseArrayLayer;
region.layerCount = copyRegion.srcSubresource.layerCount;
region.offset = copyRegion.srcOffset;
region.extent = {copyRegion.extent.width, copyRegion.extent.height,
srcSlices.slicesAreDepth ? copySliceCount : 1u};
auto twin = MakeUnique<VkTextureManager::TextureResource>();
WireE5RawCopyObjects objects;
if (m_textureManager->RecordWireE5RawTwin(frame.commandBuffer, region, copySourceLayout, *twin, objects)) {
copySourceImage = twin->image;
copySourceImageLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
copyRegion.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, region.layerCount};
copyRegion.srcOffset = {0, 0, 0};
WireRetiredObjects retired{};
retired.textures.push_back(Move(twin));
retired.imageViews = {objects.sourceView, objects.twinView};
retired.descriptorPools = {objects.descriptorPool};
RetireWireObjects(Move(retired));
}
}
// Cross-format copies between a layered image and a 3D one land only their first slice on
// some drivers (WireCopySliceProbe.h); there, the copy goes one slice per region.
const VkFormat copySourceFormat = copySourceImage == srcImage.image ? srcImage.format : VK_FORMAT_R32_UINT;
if (m_wireSplitCrossFormatSliceCopies && copySourceFormat != dstImage.format &&
srcSlices.slicesAreDepth != dstSlices.slicesAreDepth && copySliceCount > 1) {
Vector<VkImageCopy> slices(copySliceCount, copyRegion);
for (Uint32 slice = 0; slice < copySliceCount; ++slice) {
VkImageCopy& one = slices[slice];
one.extent.depth = 1;
if (srcSlices.slicesAreDepth) one.srcOffset.z += static_cast<Int32>(slice);
else one.srcSubresource = {copyRegion.srcSubresource.aspectMask, copyRegion.srcSubresource.mipLevel,
copyRegion.srcSubresource.baseArrayLayer + slice, 1};
if (dstSlices.slicesAreDepth) one.dstOffset.z += static_cast<Int32>(slice);
else one.dstSubresource = {copyRegion.dstSubresource.aspectMask, copyRegion.dstSubresource.mipLevel,
copyRegion.dstSubresource.baseArrayLayer + slice, 1};
}
vkCmdCopyImage(frame.commandBuffer, copySourceImage, copySourceImageLayout, dstImage.image,
copyDestinationLayout, copySliceCount, slices.data());
} else {
vkCmdCopyImage(frame.commandBuffer,
copySourceImage, copySourceImageLayout,
dstImage.image, copyDestinationLayout,
1, &copyRegion);
}
// The copy coordinates above are already storage-relative. Name that
// storage here too, so a texture view's offsets are not applied twice.
if (wire && !dstImage.isRenderbuffer) m_textureManager->MarkWireTextureGpuWritten(dstStorageHandle,
@@ -8467,6 +8525,9 @@ namespace MobileGL::MG_Backend::DirectVulkan {
ArmPrimGenReroute();
// Same terms: it records on m_graphicsQueue, which nothing but the probe above has used.
ArmWireDepthResolveOrder();
// Same terms again.
ArmWireE5RawReads();
ArmWireCopySliceSplit();
}
// P7 gate 5 (g5-msprobe): THE ARM ORDER OF THE MULTISAMPLE DEPTH/STENCIL RESOLVE IS A MEASURED
@@ -8548,6 +8609,106 @@ namespace MobileGL::MG_Backend::DirectVulkan {
m_wirePreferShaderDepthResolve = resolved.preferShader;
}
// THE RAW E5B9G9R9 READ ROUTE IS ARMED BY MEASUREMENT (WireE5RawCopy.h): Auto runs
// RunWireE5TransferProbe once per device identity and arms the route only where a plain transfer
// read changed words a shader read back exactly; MOBILEGL_MAGMA_E5_RAW_READS=1/0 forces it.
void VulkanRenderer::ArmWireE5RawReads() {
m_wireE5RawReads = false;
const MG_Config::QuirkOverride setting = MG_Config::Features.MagmaE5RawReads;
if (setting == MG_Config::QuirkOverride::ForceOff) {
MGLOG_I("DirectVulkan: E5B9G9R9 raw reads off (MOBILEGL_MAGMA_E5_RAW_READS)");
return;
}
if (!WireE5RawCopySupported(m_physicalDevice.handle)) {
MGLOG_I("DirectVulkan: E5B9G9R9 raw reads unavailable (R32_UINT is not sampled + storage here)");
return;
}
Bool armed = setting == MG_Config::QuirkOverride::ForceOn;
if (setting == MG_Config::QuirkOverride::Auto) {
struct Entry {
WireDepthResolveDeviceIdentity identity;
WireE5TransferProbeVerdict verdict;
};
static std::mutex s_mutex;
static Vector<Entry> s_entries;
const WireDepthResolveDeviceIdentity identity = MakeWireDepthResolveDeviceIdentity(
m_physicalDevice.properties, false, WireDepthResolveProbeKnob::Measure);
std::lock_guard<std::mutex> lock(s_mutex);
const Entry* found = nullptr;
for (const Entry& entry : s_entries)
if (entry.identity == identity) found = &entry;
if (found == nullptr) {
WireE5TransferProbeContext context;
context.physicalDevice = m_physicalDevice.handle;
context.device = m_device;
context.queue = m_graphicsQueue;
context.queueFamilyIndex = static_cast<Uint32>(m_physicalDevice.queueFamilies.graphicsFamily);
const WireE5TransferProbeMeasurement measurement = RunWireE5TransferProbe(context);
const WireE5TransferProbeVerdict verdict = EvaluateWireE5TransferProbe(measurement);
if (measurement.fenceWaitTimedOut)
MGLOG_W("DirectVulkan: the E5B9G9R9 transfer probe timed out waiting on its own submission; "
"its Vulkan objects are deliberately leaked");
MGLOG_I("DirectVulkan: E5B9G9R9 transfer probe verdict=%s: %s", WireE5TransferProbeVerdictName(verdict),
DescribeWireE5TransferProbe(measurement).c_str());
s_entries.push_back({identity, verdict});
found = &s_entries.back();
}
armed = found->verdict == WireE5TransferProbeVerdict::TransferReadNormalizes;
}
if (armed) {
if (const VkResult result = CreateWireE5RawCopyKit(m_device, m_wireE5RawCopyKit); result != VK_SUCCESS) {
MGLOG_W("DirectVulkan: E5B9G9R9 raw route kit creation failed (%d); transfer reads stay plain",
static_cast<Int>(result));
DestroyWireE5RawCopyKit(m_device, m_wireE5RawCopyKit);
armed = false;
}
}
m_wireE5RawReads = armed;
MGLOG_I("DirectVulkan: E5B9G9R9 reads %s",
armed ? "go through the raw shader route" : "use plain transfers");
}
// THE PER-SLICE SPLIT IS ARMED BY MEASUREMENT (WireCopySliceProbe.h), once per device identity,
// or forced by MOBILEGL_MAGMA_SPLIT_SLICE_COPIES.
void VulkanRenderer::ArmWireCopySliceSplit() {
const MG_Config::QuirkOverride setting = MG_Config::Features.MagmaSplitSliceCopies;
Bool armed = setting == MG_Config::QuirkOverride::ForceOn;
if (setting == MG_Config::QuirkOverride::Auto) {
struct Entry {
WireDepthResolveDeviceIdentity identity;
WireCopySliceProbeVerdict verdict;
};
static std::mutex s_mutex;
static Vector<Entry> s_entries;
const WireDepthResolveDeviceIdentity identity = MakeWireDepthResolveDeviceIdentity(
m_physicalDevice.properties, false, WireDepthResolveProbeKnob::Measure);
std::lock_guard<std::mutex> lock(s_mutex);
const Entry* found = nullptr;
for (const Entry& entry : s_entries)
if (entry.identity == identity) found = &entry;
if (found == nullptr) {
WireCopySliceProbeContext context;
context.physicalDevice = m_physicalDevice.handle;
context.device = m_device;
context.queue = m_graphicsQueue;
context.queueFamilyIndex = static_cast<Uint32>(m_physicalDevice.queueFamilies.graphicsFamily);
const WireCopySliceProbeMeasurement measurement = RunWireCopySliceProbe(context);
const WireCopySliceProbeVerdict verdict = EvaluateWireCopySliceProbe(measurement);
if (measurement.fenceWaitTimedOut)
MGLOG_W("DirectVulkan: the slice copy probe timed out waiting on its own submission; its Vulkan "
"objects are deliberately leaked");
MGLOG_I("DirectVulkan: slice copy probe verdict=%s: %s", WireCopySliceProbeVerdictName(verdict),
DescribeWireCopySliceProbe(measurement).c_str());
s_entries.push_back({identity, verdict});
found = &s_entries.back();
}
armed = found->verdict == WireCopySliceProbeVerdict::CrossFormatSlicesDropped;
}
m_wireSplitCrossFormatSliceCopies = armed;
MGLOG_I("DirectVulkan: cross-format layered <-> 3D copies %s",
armed ? "are split one slice per region" : "go as one region");
}
void VulkanRenderer::ArmPrimGenReroute() {
using namespace MG_Util::SelfTest;
m_primGenRerouteKind = PrimGenRerouteKind::None;
@@ -24,6 +24,7 @@
#include "VkTimerQueryManager.h"
#include "GpuProgressMarkers.h"
#include "WireRenderPassCompatibility.h"
#include "WireE5RawCopy.h"
#if MOBILEGL_BUILD_DISAGGREGATED
#include <MG_Remote/Server/SharedImageRegistry.h>
#endif
@@ -787,6 +788,24 @@ namespace MobileGL::MG_Backend::DirectVulkan {
// Runs the probe (memoized per device identity) and sets the member above. Called at the end of
// device creation, after ArmPrimGenReroute: it records on m_graphicsQueue.
void ArmWireDepthResolveOrder();
// Adreno's E5B9G9R9 transfer-read canonicalization (WireE5RawCopy.h): when armed, every
// transfer read of an E5B9G9R9 texture (readback, glCopyImageSubData, a storage
// recreate's level copy) first copies the words out through the raw route's compute pass.
// Decided by ArmWireE5RawReads at bring-up (probe memoized per device identity, or
// MOBILEGL_MAGMA_E5_RAW_READS).
Bool m_wireE5RawReads = false;
WireE5RawCopyKit m_wireE5RawCopyKit;
// Cross-format 2D-array <-> 3D vkCmdCopyImage lands only its first slice (WireCopySliceProbe.h):
// CopyImageSubData then records one region per slice. Set by ArmWireCopySliceSplit at bring-up.
Bool m_wireSplitCrossFormatSliceCopies = false;
void ArmWireCopySliceSplit();
void ArmWireE5RawReads();
public:
// The kit when the route is armed, else nullptr. VkTextureManager asks before a level copy.
const WireE5RawCopyKit* GetWireE5RawCopyKit() const {
return m_wireE5RawReads ? &m_wireE5RawCopyKit : nullptr;
}
private:
Bool BlitWireColorToDefault(WireImage source, WireImage destination,
GLint sx0, GLint sy0, GLint sx1, GLint sy1,
GLint dx0, GLint dy0, GLint dx1, GLint dy1, GLenum filter);
@@ -0,0 +1,356 @@
// MobileGL - MobileGL/MG_Backend/DirectVulkan/Renderer/WireCopySliceProbe.cpp
// Copyright (c) 2026 MobileGL-Dev
// Licensed under the GNU Lesser General Public License v3.0:
// https://www.gnu.org/licenses/gpl-3.0.txt
// https://www.gnu.org/licenses/lgpl-3.0.txt
// SPDX-License-Identifier: LGPL-3.0-only
// End of Source File Header
//
// See WireCopySliceProbe.h. Plumbing as in WireDepthResolveProbe.cpp: one throwaway command
// buffer, a bounded fence wait that leaks rather than idle-waits a hung GPU.
#include "WireCopySliceProbe.h"
#include <vulkan/vulkan_core.h>
namespace MobileGL::MG_Backend::DirectVulkan {
String DescribeWireCopySliceProbe(const WireCopySliceProbeMeasurement& measurement) {
if (!measurement.ran)
return format("not run ({})", measurement.failureReason.empty() ? "?" : measurement.failureReason);
return format("one-region 2D-array A2B10G10R10_UINT -> 3D A2B10G10R10_UNORM copy: {}/{} texels wrong, "
"{}/{} slices exact; per-slice regions: {} wrong",
measurement.subjectMismatches, measurement.texels, measurement.subjectExactSlices,
measurement.slices, measurement.controlMismatches);
}
namespace {
constexpr Uint32 kExtent = 7;
constexpr Uint32 kSlices = 8;
constexpr Uint32 kSliceTexels = kExtent * kExtent;
constexpr Uint32 kTexels = kSliceTexels * kSlices;
// The destination is larger than the region, as in the copy_image cases that drop slices.
constexpr Uint32 kDestinationExtent = 30;
constexpr Uint32 kDestinationDepth = 24;
constexpr Uint32 kDestinationTexels = kDestinationExtent * kDestinationExtent * kDestinationDepth;
constexpr VkFormat kSourceFormat = VK_FORMAT_A2B10G10R10_UINT_PACK32;
constexpr VkFormat kDestinationFormat = VK_FORMAT_A2B10G10R10_UNORM_PACK32;
constexpr Uint64 kFenceTimeoutNs = 5ull * 1000ull * 1000ull * 1000ull;
Uint32 Word(Uint32 index, Uint32 salt) {
Uint32 state = (index + salt * 0x10001u) * 2654435761u + 0x9E3779B9u;
state ^= state >> 15;
state *= 2246822519u;
state ^= state >> 13;
return state;
}
struct ProbeObjects {
VkDevice device = VK_NULL_HANDLE;
Vector<VkImage> images;
Vector<VkBuffer> buffers;
Vector<VkDeviceMemory> memories;
VkCommandPool commandPool = VK_NULL_HANDLE;
VkFence fence = VK_NULL_HANDLE;
void Destroy() {
for (VkImage image : images) vkDestroyImage(device, image, nullptr);
for (VkBuffer buffer : buffers) vkDestroyBuffer(device, buffer, nullptr);
for (VkDeviceMemory memory : memories) vkFreeMemory(device, memory, nullptr);
vkDestroyCommandPool(device, commandPool, nullptr);
vkDestroyFence(device, fence, nullptr);
}
};
Optional<Uint32> PickMemoryType(const VkPhysicalDeviceMemoryProperties& properties, Uint32 allowed,
VkMemoryPropertyFlags wanted, VkMemoryPropertyFlags required) {
Optional<Uint32> fallback;
for (Uint32 i = 0; i < properties.memoryTypeCount; ++i) {
if (!(allowed & (1u << i))) continue;
const VkMemoryPropertyFlags flags = properties.memoryTypes[i].propertyFlags;
if (flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) continue;
if ((flags & required) != required) continue;
if ((flags & wanted) == wanted) return i;
if (!fallback) fallback = i;
}
return fallback;
}
void ImageBarrier(VkCommandBuffer commandBuffer, VkImage image, VkImageLayout oldLayout,
VkImageLayout newLayout, VkAccessFlags srcAccess, VkAccessFlags dstAccess) {
VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
barrier.srcAccessMask = srcAccess;
barrier.dstAccessMask = dstAccess;
barrier.oldLayout = oldLayout;
barrier.newLayout = newLayout;
barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.image = image;
barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, VK_REMAINING_ARRAY_LAYERS};
vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
}
} // namespace
WireCopySliceProbeMeasurement RunWireCopySliceProbe(const WireCopySliceProbeContext& context) {
WireCopySliceProbeMeasurement measurement;
const VkDevice device = context.device;
if (device == VK_NULL_HANDLE || context.queue == VK_NULL_HANDLE) {
measurement.failureReason = "no device or queue";
return measurement;
}
constexpr VkFormatFeatureFlags kRequired = VK_FORMAT_FEATURE_TRANSFER_SRC_BIT | VK_FORMAT_FEATURE_TRANSFER_DST_BIT;
for (const VkFormat format : {kSourceFormat, kDestinationFormat}) {
VkFormatProperties properties{};
vkGetPhysicalDeviceFormatProperties(context.physicalDevice, format, &properties);
if ((properties.optimalTilingFeatures & kRequired) != kRequired) {
measurement.failureReason = "A2B10G10R10 UINT/UNORM are not transfer formats here";
return measurement;
}
}
ProbeObjects objects;
objects.device = device;
Bool leak = false;
struct Teardown {
ProbeObjects& objects;
Bool& leak;
~Teardown() {
if (!leak) objects.Destroy();
}
} teardown{objects, leak};
const auto fail = [&](String reason) {
measurement.ran = false;
measurement.failureReason = Move(reason);
return measurement;
};
VkPhysicalDeviceMemoryProperties memoryProperties{};
vkGetPhysicalDeviceMemoryProperties(context.physicalDevice, &memoryProperties);
// The wire's images (VkTextureManager): MUTABLE_FORMAT (a storage-compatible colour class),
// SAMPLED | TRANSFER_SRC | TRANSFER_DST | COLOR_ATTACHMENT, a 3D one 2D_ARRAY_COMPATIBLE.
const auto makeImage = [&](Bool volume, VkFormat format, VkImage& out) -> Bool {
VkFormatProperties formatProperties{};
vkGetPhysicalDeviceFormatProperties(context.physicalDevice, format, &formatProperties);
const Bool attachable =
(formatProperties.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT) != 0;
VkImageCreateInfo info{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
info.flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | (volume ? VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT : 0);
info.imageType = volume ? VK_IMAGE_TYPE_3D : VK_IMAGE_TYPE_2D;
info.format = format;
info.extent = volume ? VkExtent3D{kDestinationExtent, kDestinationExtent, kDestinationDepth}
: VkExtent3D{kExtent, kExtent, 1};
info.mipLevels = 1;
info.arrayLayers = volume ? 1u : kSlices;
info.samples = VK_SAMPLE_COUNT_1_BIT;
info.tiling = VK_IMAGE_TILING_OPTIMAL;
info.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT |
(attachable ? VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT : 0u);
info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
if (vkCreateImage(device, &info, nullptr, &out) != VK_SUCCESS) return false;
objects.images.push_back(out);
VkMemoryRequirements requirements{};
vkGetImageMemoryRequirements(device, out, &requirements);
const Optional<Uint32> type = PickMemoryType(memoryProperties, requirements.memoryTypeBits,
VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, 0);
if (!type) return false;
VkMemoryAllocateInfo allocate{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocate.allocationSize = requirements.size;
allocate.memoryTypeIndex = *type;
VkDeviceMemory memory = VK_NULL_HANDLE;
if (vkAllocateMemory(device, &allocate, nullptr, &memory) != VK_SUCCESS) return false;
objects.memories.push_back(memory);
return vkBindImageMemory(device, out, memory, 0) == VK_SUCCESS;
};
struct HostBuffer {
VkBuffer buffer = VK_NULL_HANDLE;
VkDeviceMemory memory = VK_NULL_HANDLE;
Bool coherent = false;
};
const auto makeBuffer = [&](VkDeviceSize bytes, VkBufferUsageFlags usage, HostBuffer& out) -> Bool {
VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
info.size = bytes;
info.usage = usage;
info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
if (vkCreateBuffer(device, &info, nullptr, &out.buffer) != VK_SUCCESS) return false;
objects.buffers.push_back(out.buffer);
VkMemoryRequirements requirements{};
vkGetBufferMemoryRequirements(device, out.buffer, &requirements);
const Optional<Uint32> type =
PickMemoryType(memoryProperties, requirements.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT);
if (!type) return false;
out.coherent =
(memoryProperties.memoryTypes[*type].propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0;
VkMemoryAllocateInfo allocate{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocate.allocationSize = requirements.size;
allocate.memoryTypeIndex = *type;
if (vkAllocateMemory(device, &allocate, nullptr, &out.memory) != VK_SUCCESS) return false;
objects.memories.push_back(out.memory);
return vkBindBufferMemory(device, out.buffer, out.memory, 0) == VK_SUCCESS;
};
const auto access = [&](const HostBuffer& buffer, Bool write, auto&& body) -> Bool {
void* mapped = nullptr;
if (vkMapMemory(device, buffer.memory, 0, VK_WHOLE_SIZE, 0, &mapped) != VK_SUCCESS || mapped == nullptr)
return false;
VkMappedMemoryRange range{VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE};
range.memory = buffer.memory;
range.size = VK_WHOLE_SIZE;
if (!write && !buffer.coherent) vkInvalidateMappedMemoryRanges(device, 1, &range);
body(static_cast<Uint32*>(mapped));
if (write && !buffer.coherent) vkFlushMappedMemoryRanges(device, 1, &range);
vkUnmapMemory(device, buffer.memory);
return true;
};
VkImage source = VK_NULL_HANDLE, subject = VK_NULL_HANDLE, control = VK_NULL_HANDLE;
if (!makeImage(false, kSourceFormat, source) || !makeImage(true, kDestinationFormat, subject) ||
!makeImage(true, kDestinationFormat, control))
return fail("image creation failed");
HostBuffer sourceWords, sentinelWords, subjectOut, controlOut;
if (!makeBuffer(kTexels * 4, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, sourceWords) ||
!makeBuffer(kDestinationTexels * 4, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, sentinelWords) ||
!makeBuffer(kTexels * 4, VK_BUFFER_USAGE_TRANSFER_DST_BIT, subjectOut) ||
!makeBuffer(kTexels * 4, VK_BUFFER_USAGE_TRANSFER_DST_BIT, controlOut))
return fail("buffer creation failed");
if (!access(sourceWords, true, [](Uint32* words) { for (Uint32 i = 0; i < kTexels; ++i) words[i] = Word(i, 1); }) ||
!access(sentinelWords, true, [](Uint32* words) { for (Uint32 i = 0; i < kDestinationTexels; ++i) words[i] = Word(i, 2); }))
return fail("vkMapMemory (upload) failed");
VkCommandPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};
poolInfo.flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT;
poolInfo.queueFamilyIndex = context.queueFamilyIndex;
if (vkCreateCommandPool(device, &poolInfo, nullptr, &objects.commandPool) != VK_SUCCESS)
return fail("vkCreateCommandPool failed");
VkCommandBufferAllocateInfo commandInfo{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};
commandInfo.commandPool = objects.commandPool;
commandInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
commandInfo.commandBufferCount = 1;
VkCommandBuffer cmd = VK_NULL_HANDLE;
if (vkAllocateCommandBuffers(device, &commandInfo, &cmd) != VK_SUCCESS)
return fail("vkAllocateCommandBuffers failed");
VkCommandBufferBeginInfo beginInfo{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
if (vkBeginCommandBuffer(cmd, &beginInfo) != VK_SUCCESS) return fail("vkBeginCommandBuffer failed");
// (1) The source's words, and a sentinel in both destinations so an untouched slice shows.
for (const VkImage image : {source, subject, control})
ImageBarrier(cmd, image, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 0,
VK_ACCESS_TRANSFER_WRITE_BIT);
VkBufferImageCopy layered{};
layered.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, kSlices};
layered.imageExtent = {kExtent, kExtent, 1};
VkBufferImageCopy volume{};
volume.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
volume.imageExtent = {kExtent, kExtent, kSlices};
VkBufferImageCopy wholeVolume = volume;
wholeVolume.imageExtent = {kDestinationExtent, kDestinationExtent, kDestinationDepth};
vkCmdCopyBufferToImage(cmd, sourceWords.buffer, source, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &layered);
vkCmdCopyBufferToImage(cmd, sentinelWords.buffer, subject, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &wholeVolume);
vkCmdCopyBufferToImage(cmd, sentinelWords.buffer, control, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &wholeVolume);
ImageBarrier(cmd, source, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
for (const VkImage image : {subject, control})
ImageBarrier(cmd, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT);
// (2) SUBJECT: the one region CopyImageSubData records. CONTROL: one region per slice.
VkImageCopy whole{};
whole.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, kSlices};
whole.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
whole.extent = {kExtent, kExtent, kSlices};
vkCmdCopyImage(cmd, source, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, subject, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
1, &whole);
VkImageCopy perSlice[kSlices]{};
for (Uint32 slice = 0; slice < kSlices; ++slice) {
perSlice[slice].srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, slice, 1};
perSlice[slice].dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
perSlice[slice].dstOffset = {0, 0, static_cast<Int32>(slice)};
perSlice[slice].extent = {kExtent, kExtent, 1};
}
vkCmdCopyImage(cmd, source, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, control, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
kSlices, perSlice);
// (3) Both destinations out: same-format reads of a 3D image, which the defect does not touch.
for (const VkImage image : {subject, control})
ImageBarrier(cmd, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
vkCmdCopyImageToBuffer(cmd, subject, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, subjectOut.buffer, 1, &volume);
vkCmdCopyImageToBuffer(cmd, control, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, controlOut.buffer, 1, &volume);
VkMemoryBarrier hostBarrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
hostBarrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
hostBarrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT;
vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &hostBarrier, 0,
nullptr, 0, nullptr);
if (vkEndCommandBuffer(cmd) != VK_SUCCESS) return fail("vkEndCommandBuffer failed");
VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
if (vkCreateFence(device, &fenceInfo, nullptr, &objects.fence) != VK_SUCCESS)
return fail("vkCreateFence failed");
VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO};
submit.commandBufferCount = 1;
submit.pCommandBuffers = &cmd;
if (const VkResult result = vkQueueSubmit(context.queue, 1, &submit, objects.fence); result != VK_SUCCESS)
return fail(format("vkQueueSubmit failed (VkResult {})", static_cast<Int>(result)));
const VkResult waited = vkWaitForFences(device, 1, &objects.fence, VK_TRUE, kFenceTimeoutNs);
if (waited == VK_TIMEOUT) {
leak = true;
measurement.fenceWaitTimedOut = true;
return fail("the probe's fence did not signal within 5 s");
}
if (waited != VK_SUCCESS) return fail(format("vkWaitForFences failed (VkResult {})", static_cast<Int>(waited)));
const auto tally = [&](const Uint32* words, Uint32& mismatches, Uint32* exactSlices) {
for (Uint32 slice = 0; slice < kSlices; ++slice) {
Uint32 sliceMismatches = 0;
for (Uint32 i = slice * kSliceTexels; i < (slice + 1) * kSliceTexels; ++i)
if (words[i] != Word(i, 1)) ++sliceMismatches;
mismatches += sliceMismatches;
if (exactSlices && sliceMismatches == 0) ++*exactSlices;
}
};
if (!access(subjectOut, false, [&](Uint32* words) {
tally(words, measurement.subjectMismatches, &measurement.subjectExactSlices);
}) ||
!access(controlOut, false, [&](Uint32* words) { tally(words, measurement.controlMismatches, nullptr); }))
return fail("vkMapMemory (readback) failed");
measurement.texels = kTexels;
measurement.slices = kSlices;
measurement.ran = true;
return measurement;
}
WireCopySliceProbeMeasurement RunWireCopySliceProbeOnThrowawayDevice(PFN_vkGetInstanceProcAddr getInstanceProcAddr,
VkInstance instance,
VkPhysicalDevice physicalDevice,
Uint32 graphicsQueueFamilyIndex) {
WireCopySliceProbeMeasurement measurement;
const auto createDevice =
reinterpret_cast<PFN_vkCreateDevice>(getInstanceProcAddr(instance, "vkCreateDevice"));
if (createDevice == nullptr) {
measurement.failureReason = "no vkCreateDevice";
return measurement;
}
const Float priority = 1.0f;
VkDeviceQueueCreateInfo queueInfo{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO};
queueInfo.queueFamilyIndex = graphicsQueueFamilyIndex;
queueInfo.queueCount = 1;
queueInfo.pQueuePriorities = &priority;
VkDeviceCreateInfo deviceInfo{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO};
deviceInfo.queueCreateInfoCount = 1;
deviceInfo.pQueueCreateInfos = &queueInfo;
VkDevice device = VK_NULL_HANDLE;
if (const VkResult result = createDevice(physicalDevice, &deviceInfo, nullptr, &device);
result != VK_SUCCESS || device == VK_NULL_HANDLE) {
measurement.failureReason = format("vkCreateDevice failed (VkResult {})", static_cast<Int>(result));
return measurement;
}
WireCopySliceProbeContext context;
context.physicalDevice = physicalDevice;
context.device = device;
vkGetDeviceQueue(device, graphicsQueueFamilyIndex, 0, &context.queue);
context.queueFamilyIndex = graphicsQueueFamilyIndex;
measurement = RunWireCopySliceProbe(context);
if (!measurement.fenceWaitTimedOut) vkDestroyDevice(device, nullptr);
return measurement;
}
} // namespace MobileGL::MG_Backend::DirectVulkan
@@ -0,0 +1,93 @@
// MobileGL - MobileGL/MG_Backend/DirectVulkan/Renderer/WireCopySliceProbe.h
// Copyright (c) 2026 MobileGL-Dev
// Licensed under the GNU Lesser General Public License v3.0:
// https://www.gnu.org/licenses/gpl-3.0.txt
// https://www.gnu.org/licenses/lgpl-3.0.txt
// SPDX-License-Identifier: LGPL-3.0-only
// End of Source File Header
#pragma once
#include <Config.h>
#include <Includes.h>
#include <vulkan/vulkan_core.h>
// CROSS-FORMAT SLICE COPIES BETWEEN A LAYERED IMAGE AND A 3D ONE, and the probe that decides
// whether a device needs them split.
//
// glCopyImageSubData between a 2D array and a 3D texture maps layers onto depth slices (GL 4.6
// core 18.3.2), and Vulkan expresses the whole move as ONE VkImageCopy: layerCount on the layered
// side, extent.depth on the 3D side (VUID-VkImageCopy-srcImage-01790/01791). Adreno 750 performs
// that region correctly when the two formats are the same, but when they differ (size-compatible,
// e.g. A2B10G10R10_UINT into A2B10G10R10_UNORM - KHR-GL4x.copy_image's cross-format cases) it
// lands only the FIRST slice and leaves the rest of the destination untouched. The same copy
// recorded as one single-slice region per slice is exact. The probe uses the wire's image shape
// (MUTABLE_FORMAT, attachment usage, a 30x30x24 destination): a non-mutable 7x7x8 pair without
// attachment usage copied cleanly on the same device.
//
// VulkanRenderer::CopyImageSubData therefore splits such a copy per slice where the probe below
// measured the defect (VulkanRenderer::ArmWireCopySliceSplit at bring-up, the POST's "Known Driver
// Bugs" section in MG_Util/SelfTest/DriverBugProbes.cpp); MOBILEGL_MAGMA_SPLIT_SLICE_COPIES forces it.
namespace MobileGL::MG_Backend::DirectVulkan {
struct WireCopySliceProbeMeasurement {
// Recorded, submitted and read back.
Bool ran = false;
// The bounded fence wait expired: every object was deliberately leaked.
Bool fenceWaitTimedOut = false;
String failureReason;
// SUBJECT: one region, A2B10G10R10_UINT 2D array -> A2B10G10R10_UNORM 3D, every slice.
Uint32 texels = 0;
Uint32 slices = 0;
Uint32 subjectMismatches = 0;
// The slices the subject got entirely right (the defect leaves exactly the first).
Uint32 subjectExactSlices = 0;
// CONTROL: the same copy recorded one slice per region. Must be exact, or nothing is concluded.
Uint32 controlMismatches = 0;
};
enum class WireCopySliceProbeVerdict : Uint8 {
NotRun,
Inconclusive,
Clean,
// The defect: the single region missed texels the per-slice regions moved exactly.
CrossFormatSlicesDropped,
};
inline const char* WireCopySliceProbeVerdictName(WireCopySliceProbeVerdict verdict) {
switch (verdict) {
case WireCopySliceProbeVerdict::NotRun: return "not-run";
case WireCopySliceProbeVerdict::Inconclusive: return "inconclusive";
case WireCopySliceProbeVerdict::Clean: return "clean";
case WireCopySliceProbeVerdict::CrossFormatSlicesDropped: return "cross-format-slices-dropped";
}
return "?";
}
inline WireCopySliceProbeVerdict EvaluateWireCopySliceProbe(const WireCopySliceProbeMeasurement& measurement) {
if (!measurement.ran || measurement.fenceWaitTimedOut || measurement.texels == 0)
return WireCopySliceProbeVerdict::NotRun;
if (measurement.controlMismatches != 0) return WireCopySliceProbeVerdict::Inconclusive;
return measurement.subjectMismatches != 0 ? WireCopySliceProbeVerdict::CrossFormatSlicesDropped
: WireCopySliceProbeVerdict::Clean;
}
String DescribeWireCopySliceProbe(const WireCopySliceProbeMeasurement& measurement);
struct WireCopySliceProbeContext {
VkPhysicalDevice physicalDevice = VK_NULL_HANDLE;
VkDevice device = VK_NULL_HANDLE;
// Must be otherwise idle for the probe's duration.
VkQueue queue = VK_NULL_HANDLE;
Uint32 queueFamilyIndex = 0;
};
// One submission on its own command pool, a bounded fence wait (on timeout every object is
// leaked and the measurement says so), teardown on every other path.
WireCopySliceProbeMeasurement RunWireCopySliceProbe(const WireCopySliceProbeContext& context);
// The POST's entry: a throwaway device with one graphics queue.
WireCopySliceProbeMeasurement RunWireCopySliceProbeOnThrowawayDevice(PFN_vkGetInstanceProcAddr getInstanceProcAddr,
VkInstance instance,
VkPhysicalDevice physicalDevice,
Uint32 graphicsQueueFamilyIndex);
} // namespace MobileGL::MG_Backend::DirectVulkan
@@ -0,0 +1,22 @@
#version 450
// Raw texel-block copy out of an E5B9G9R9 image (WireE5RawCopy.h). The source is read through an
// R32_UINT view, so its words are never decoded, and stored into an R32_UINT twin whose transfer
// reads move bits rather than values. Layered and plain 2D sources; WireE5RawCopy3D.comp is the
// volume twin of this shader.
layout(local_size_x = 8, local_size_y = 8, local_size_z = 1) in;
layout(set = 0, binding = 0) uniform usampler2DArray sourceImage;
layout(set = 0, binding = 1, r32ui) uniform writeonly uimage2DArray twinImage;
layout(push_constant) uniform Region {
// xy: the region's texel origin in the source level; z unused (the view starts at the
// region's first layer).
ivec4 sourceOffset;
// xyz: texels by layers to copy.
ivec4 extent;
} region;
void main() {
ivec3 id = ivec3(gl_GlobalInvocationID);
if (any(greaterThanEqual(id, region.extent.xyz))) return;
uint word = texelFetch(sourceImage, ivec3(region.sourceOffset.xy + id.xy, id.z), 0).r;
imageStore(twinImage, id, uvec4(word, 0u, 0u, 0u));
}
@@ -0,0 +1,588 @@
// MobileGL - MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.cpp
// Copyright (c) 2026 MobileGL-Dev
// Licensed under the GNU Lesser General Public License v3.0:
// https://www.gnu.org/licenses/gpl-3.0.txt
// https://www.gnu.org/licenses/lgpl-3.0.txt
// SPDX-License-Identifier: LGPL-3.0-only
// End of Source File Header
//
// See WireE5RawCopy.h. The probe's plumbing is WireDepthResolveProbe.cpp's: one throwaway command
// buffer, a bounded fence wait that leaks rather than idle-waits a hung GPU, teardown on every
// other path.
#include "WireE5RawCopy.h"
#include <MG_Util/Debug/Log.h>
#include <cstring>
#include <vulkan/vulkan_core.h>
namespace MobileGL::MG_Backend::DirectVulkan {
#include "WireE5RawCopySpirv.h"
namespace {
// The shaders' push-constant block: ivec4 sourceOffset, ivec4 extent.
struct RegionPush {
Int32 sourceOffset[4];
Int32 extent[4];
};
constexpr Uint32 kGroupSize = 8;
} // namespace
VkResult CreateWireE5RawCopyKit(VkDevice device, WireE5RawCopyKit& kit) {
const auto shader = [&](const Uint32* code, SizeT bytes, VkShaderModule& module) {
VkShaderModuleCreateInfo info{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO};
info.codeSize = bytes;
info.pCode = code;
return vkCreateShaderModule(device, &info, nullptr, &module);
};
VkResult result = shader(kWireE5RawCopySpirv, sizeof(kWireE5RawCopySpirv), kit.arrayModule);
if (result != VK_SUCCESS) return result;
result = shader(kWireE5RawCopy3DSpirv, sizeof(kWireE5RawCopy3DSpirv), kit.volumeModule);
if (result != VK_SUCCESS) return result;
const VkDescriptorSetLayoutBinding bindings[2]{
{0, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr},
{1, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr},
};
VkDescriptorSetLayoutCreateInfo layout{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO};
layout.bindingCount = 2;
layout.pBindings = bindings;
result = vkCreateDescriptorSetLayout(device, &layout, nullptr, &kit.descriptorLayout);
if (result != VK_SUCCESS) return result;
const VkPushConstantRange push{VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(RegionPush)};
VkPipelineLayoutCreateInfo pipelineLayout{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
pipelineLayout.setLayoutCount = 1;
pipelineLayout.pSetLayouts = &kit.descriptorLayout;
pipelineLayout.pushConstantRangeCount = 1;
pipelineLayout.pPushConstantRanges = &push;
result = vkCreatePipelineLayout(device, &pipelineLayout, nullptr, &kit.pipelineLayout);
if (result != VK_SUCCESS) return result;
const auto pipeline = [&](VkShaderModule module, VkPipeline& out) {
VkComputePipelineCreateInfo info{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO};
info.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
info.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;
info.stage.module = module;
info.stage.pName = "main";
info.layout = kit.pipelineLayout;
return vkCreateComputePipelines(device, VK_NULL_HANDLE, 1, &info, nullptr, &out);
};
result = pipeline(kit.arrayModule, kit.arrayPipeline);
if (result != VK_SUCCESS) return result;
result = pipeline(kit.volumeModule, kit.volumePipeline);
if (result != VK_SUCCESS) return result;
// texelFetch ignores the filter; the sampler exists only because a GLSL usampler is a
// COMBINED image sampler. An integer format must not be linearly filtered anyway.
VkSamplerCreateInfo sampler{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO};
sampler.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST;
sampler.addressModeU = sampler.addressModeV = sampler.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
sampler.magFilter = sampler.minFilter = VK_FILTER_NEAREST;
sampler.maxLod = 0.0f;
return vkCreateSampler(device, &sampler, nullptr, &kit.nearest);
}
void DestroyWireE5RawCopyKit(VkDevice device, WireE5RawCopyKit& kit) {
vkDestroySampler(device, kit.nearest, nullptr);
vkDestroyPipeline(device, kit.arrayPipeline, nullptr);
vkDestroyPipeline(device, kit.volumePipeline, nullptr);
vkDestroyPipelineLayout(device, kit.pipelineLayout, nullptr);
vkDestroyDescriptorSetLayout(device, kit.descriptorLayout, nullptr);
vkDestroyShaderModule(device, kit.arrayModule, nullptr);
vkDestroyShaderModule(device, kit.volumeModule, nullptr);
kit = {};
}
Bool WireE5RawCopySupported(VkPhysicalDevice physicalDevice) {
VkFormatProperties properties{};
vkGetPhysicalDeviceFormatProperties(physicalDevice, VK_FORMAT_R32_UINT, &properties);
constexpr VkFormatFeatureFlags kRequired = VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT |
VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT |
VK_FORMAT_FEATURE_TRANSFER_SRC_BIT;
return (properties.optimalTilingFeatures & kRequired) == kRequired;
}
VkImageCreateInfo WireE5RawCopyTwinInfo(const WireE5RawCopyRegion& region) {
VkImageCreateInfo info{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
info.imageType = region.volume ? VK_IMAGE_TYPE_3D : VK_IMAGE_TYPE_2D;
info.format = VK_FORMAT_R32_UINT;
info.extent = {region.extent.width, region.extent.height, region.volume ? region.extent.depth : 1u};
info.mipLevels = 1;
info.arrayLayers = region.volume ? 1u : region.layerCount;
info.samples = VK_SAMPLE_COUNT_1_BIT;
info.tiling = VK_IMAGE_TILING_OPTIMAL;
info.usage = VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
return info;
}
void DestroyWireE5RawCopyObjects(VkDevice device, WireE5RawCopyObjects& objects) {
vkDestroyImageView(device, objects.sourceView, nullptr);
vkDestroyImageView(device, objects.twinView, nullptr);
vkDestroyDescriptorPool(device, objects.descriptorPool, nullptr);
objects = {};
}
VkResult RecordWireE5RawCopy(VkDevice device, VkCommandBuffer commandBuffer, const WireE5RawCopyKit& kit,
const WireE5RawCopyRegion& region, VkImageLayout sourceLayout, VkImage twin,
WireE5RawCopyObjects& objects) {
if (kit.arrayPipeline == VK_NULL_HANDLE || kit.volumePipeline == VK_NULL_HANDLE)
return VK_ERROR_INITIALIZATION_FAILED;
const Uint32 slices = region.volume ? region.extent.depth : region.layerCount;
if (region.extent.width == 0 || region.extent.height == 0 || slices == 0) return VK_SUCCESS;
// The source through R32_UINT: same 32-bit compatibility class, so a plain view of a
// MUTABLE_FORMAT image. A 3D view covers every slice; the shader offsets into it.
VkImageViewCreateInfo sourceInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
sourceInfo.image = region.source;
sourceInfo.viewType = region.volume ? VK_IMAGE_VIEW_TYPE_3D : VK_IMAGE_VIEW_TYPE_2D_ARRAY;
sourceInfo.format = VK_FORMAT_R32_UINT;
sourceInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, region.level, 1,
region.volume ? 0u : region.baseLayer, region.volume ? 1u : region.layerCount};
VkResult result = vkCreateImageView(device, &sourceInfo, nullptr, &objects.sourceView);
if (result != VK_SUCCESS) return result;
VkImageViewCreateInfo twinInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
twinInfo.image = twin;
twinInfo.viewType = region.volume ? VK_IMAGE_VIEW_TYPE_3D : VK_IMAGE_VIEW_TYPE_2D_ARRAY;
twinInfo.format = VK_FORMAT_R32_UINT;
twinInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, region.volume ? 1u : region.layerCount};
result = vkCreateImageView(device, &twinInfo, nullptr, &objects.twinView);
if (result != VK_SUCCESS) return result;
const VkDescriptorPoolSize poolSizes[2]{{VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1},
{VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1}};
VkDescriptorPoolCreateInfo pool{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO};
pool.maxSets = 1;
pool.poolSizeCount = 2;
pool.pPoolSizes = poolSizes;
result = vkCreateDescriptorPool(device, &pool, nullptr, &objects.descriptorPool);
if (result != VK_SUCCESS) return result;
VkDescriptorSetAllocateInfo alloc{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO};
alloc.descriptorPool = objects.descriptorPool;
alloc.descriptorSetCount = 1;
alloc.pSetLayouts = &kit.descriptorLayout;
VkDescriptorSet set = VK_NULL_HANDLE;
result = vkAllocateDescriptorSets(device, &alloc, &set);
if (result != VK_SUCCESS) return result;
const VkDescriptorImageInfo sampled{kit.nearest, objects.sourceView, sourceLayout};
const VkDescriptorImageInfo storage{VK_NULL_HANDLE, objects.twinView, VK_IMAGE_LAYOUT_GENERAL};
VkWriteDescriptorSet writes[2]{};
for (Uint32 i = 0; i < 2; ++i) {
writes[i].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
writes[i].dstSet = set;
writes[i].dstBinding = i;
writes[i].descriptorCount = 1;
}
writes[0].descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
writes[0].pImageInfo = &sampled;
writes[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
writes[1].pImageInfo = &storage;
vkUpdateDescriptorSets(device, 2, writes, 0, nullptr);
// Every earlier write - transfer, attachment or shader - visible to the fetch, and the
// fresh twin ready for storage writes.
VkMemoryBarrier memory{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
memory.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
memory.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
VkImageMemoryBarrier twinBarrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
twinBarrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT;
twinBarrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
twinBarrier.newLayout = VK_IMAGE_LAYOUT_GENERAL;
twinBarrier.srcQueueFamilyIndex = twinBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
twinBarrier.image = twin;
twinBarrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, VK_REMAINING_ARRAY_LAYERS};
vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
0, 1, &memory, 0, nullptr, 1, &twinBarrier);
vkCmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE,
region.volume ? kit.volumePipeline : kit.arrayPipeline);
vkCmdBindDescriptorSets(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, kit.pipelineLayout, 0, 1, &set, 0,
nullptr);
const RegionPush push{{region.offset.x, region.offset.y, region.volume ? region.offset.z : 0, 0},
{static_cast<Int32>(region.extent.width), static_cast<Int32>(region.extent.height),
static_cast<Int32>(slices), 0}};
vkCmdPushConstants(commandBuffer, kit.pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), &push);
vkCmdDispatch(commandBuffer, (region.extent.width + kGroupSize - 1) / kGroupSize,
(region.extent.height + kGroupSize - 1) / kGroupSize, slices);
twinBarrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT;
twinBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
twinBarrier.oldLayout = VK_IMAGE_LAYOUT_GENERAL;
twinBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0,
nullptr, 0, nullptr, 1, &twinBarrier);
return VK_SUCCESS;
}
String DescribeWireE5TransferProbe(const WireE5TransferProbeMeasurement& measurement) {
if (!measurement.ran)
return format("not run ({})", measurement.failureReason.empty() ? "?" : measurement.failureReason);
return format("30x30 transfer read: {}/{} words changed (first {:#010x} -> {:#010x}); shader read: {} "
"changed; 16x16 transfer read: {}/{} changed",
measurement.subjectMismatches, measurement.texels, measurement.firstExpected,
measurement.firstObserved, measurement.controlMismatches, measurement.smallMismatches,
measurement.smallTexels);
}
namespace {
constexpr Uint32 kSubjectExtent = 30;
constexpr Uint32 kSmallExtent = 16;
constexpr Uint64 kFenceTimeoutNs = 5ull * 1000ull * 1000ull * 1000ull;
// A non-canonical RGB9_E5 word: exponent 1..31 and every mantissa below 256, so the same
// value also encodes with the exponent one lower and every mantissa doubled. A path that
// decodes and re-encodes changes it; a bit move does not.
Uint32 NonCanonicalWord(Uint32 index) {
Uint32 state = index * 2654435761u + 0x9E3779B9u;
state ^= state >> 15;
state *= 2246822519u;
state ^= state >> 13;
const Uint32 r = state & 0xFFu;
const Uint32 g = (state >> 8) & 0xFFu;
const Uint32 b = ((state >> 16) & 0xFFu) | ((r | g) == 0 ? 1u : 0u);
const Uint32 exponent = 1u + ((state >> 24) % 31u);
return r | (g << 9) | (b << 18) | (exponent << 27);
}
struct ProbeObjects {
VkDevice device = VK_NULL_HANDLE;
Vector<VkImage> images;
Vector<VkDeviceMemory> memories;
Vector<VkBuffer> buffers;
VkCommandPool commandPool = VK_NULL_HANDLE;
VkFence fence = VK_NULL_HANDLE;
WireE5RawCopyKit kit;
WireE5RawCopyObjects copy;
void Destroy() {
DestroyWireE5RawCopyObjects(device, copy);
for (VkImage image : images) vkDestroyImage(device, image, nullptr);
for (VkBuffer buffer : buffers) vkDestroyBuffer(device, buffer, nullptr);
for (VkDeviceMemory memory : memories) vkFreeMemory(device, memory, nullptr);
vkDestroyCommandPool(device, commandPool, nullptr);
vkDestroyFence(device, fence, nullptr);
DestroyWireE5RawCopyKit(device, kit);
}
};
Optional<Uint32> PickMemoryType(const VkPhysicalDeviceMemoryProperties& properties, Uint32 allowed,
VkMemoryPropertyFlags wanted, VkMemoryPropertyFlags required) {
Optional<Uint32> fallback;
for (Uint32 i = 0; i < properties.memoryTypeCount; ++i) {
if (!(allowed & (1u << i))) continue;
const VkMemoryPropertyFlags flags = properties.memoryTypes[i].propertyFlags;
if (flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) continue;
if ((flags & required) != required) continue;
if ((flags & wanted) == wanted) return i;
if (!fallback) fallback = i;
}
return fallback;
}
void ImageBarrier(VkCommandBuffer commandBuffer, VkImage image, VkImageLayout oldLayout,
VkImageLayout newLayout, VkPipelineStageFlags srcStage, VkAccessFlags srcAccess,
VkPipelineStageFlags dstStage, VkAccessFlags dstAccess) {
VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
barrier.srcAccessMask = srcAccess;
barrier.dstAccessMask = dstAccess;
barrier.oldLayout = oldLayout;
barrier.newLayout = newLayout;
barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.image = image;
barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
vkCmdPipelineBarrier(commandBuffer, srcStage, dstStage, 0, 0, nullptr, 0, nullptr, 1, &barrier);
}
} // namespace
WireE5TransferProbeMeasurement RunWireE5TransferProbe(const WireE5TransferProbeContext& context) {
WireE5TransferProbeMeasurement measurement;
const VkDevice device = context.device;
if (device == VK_NULL_HANDLE || context.queue == VK_NULL_HANDLE) {
measurement.failureReason = "no device or queue";
return measurement;
}
if (!WireE5RawCopySupported(context.physicalDevice)) {
measurement.failureReason = "R32_UINT is not a sampled + storage + transfer-source format here";
return measurement;
}
// The usage production gives a wire E5B9G9R9 texture: SAMPLED | TRANSFER_SRC | TRANSFER_DST,
// plus COLOR_ATTACHMENT where the format can be one.
VkFormatProperties e5Properties{};
vkGetPhysicalDeviceFormatProperties(context.physicalDevice, VK_FORMAT_E5B9G9R9_UFLOAT_PACK32, &e5Properties);
constexpr VkFormatFeatureFlags kE5Required = VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT |
VK_FORMAT_FEATURE_TRANSFER_SRC_BIT |
VK_FORMAT_FEATURE_TRANSFER_DST_BIT;
if ((e5Properties.optimalTilingFeatures & kE5Required) != kE5Required) {
measurement.failureReason = "E5B9G9R9 is not a sampled transfer format here";
return measurement;
}
VkImageUsageFlags e5Usage =
VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
if (e5Properties.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT)
e5Usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
ProbeObjects objects;
objects.device = device;
Bool leak = false;
struct Teardown {
ProbeObjects& objects;
Bool& leak;
~Teardown() {
if (!leak) objects.Destroy();
}
} teardown{objects, leak};
const auto fail = [&](String reason) {
measurement.ran = false;
measurement.failureReason = Move(reason);
return measurement;
};
VkPhysicalDeviceMemoryProperties memoryProperties{};
vkGetPhysicalDeviceMemoryProperties(context.physicalDevice, &memoryProperties);
const auto bindImage = [&](VkImage image) -> Bool {
VkMemoryRequirements requirements{};
vkGetImageMemoryRequirements(device, image, &requirements);
const Optional<Uint32> type = PickMemoryType(memoryProperties, requirements.memoryTypeBits,
VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, 0);
if (!type) return false;
VkMemoryAllocateInfo allocate{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocate.allocationSize = requirements.size;
allocate.memoryTypeIndex = *type;
VkDeviceMemory memory = VK_NULL_HANDLE;
if (vkAllocateMemory(device, &allocate, nullptr, &memory) != VK_SUCCESS) return false;
objects.memories.push_back(memory);
return vkBindImageMemory(device, image, memory, 0) == VK_SUCCESS;
};
const auto makeImage = [&](const VkImageCreateInfo& info, VkImage& out) -> Bool {
if (vkCreateImage(device, &info, nullptr, &out) != VK_SUCCESS) return false;
objects.images.push_back(out);
return bindImage(out);
};
const auto makeE5 = [&](Uint32 extent, VkImage& out) {
VkImageCreateInfo info{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
info.flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
info.imageType = VK_IMAGE_TYPE_2D;
info.format = VK_FORMAT_E5B9G9R9_UFLOAT_PACK32;
info.extent = {extent, extent, 1};
info.mipLevels = info.arrayLayers = 1;
info.samples = VK_SAMPLE_COUNT_1_BIT;
info.tiling = VK_IMAGE_TILING_OPTIMAL;
info.usage = e5Usage;
info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
return makeImage(info, out);
};
// Host-visible buffers: the upload source and the three readbacks, one allocation each.
struct HostBuffer {
VkBuffer buffer = VK_NULL_HANDLE;
VkDeviceMemory memory = VK_NULL_HANDLE;
Bool coherent = false;
};
const auto makeBuffer = [&](VkDeviceSize bytes, VkBufferUsageFlags usage, HostBuffer& out) -> Bool {
VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
info.size = bytes;
info.usage = usage;
info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
if (vkCreateBuffer(device, &info, nullptr, &out.buffer) != VK_SUCCESS) return false;
objects.buffers.push_back(out.buffer);
VkMemoryRequirements requirements{};
vkGetBufferMemoryRequirements(device, out.buffer, &requirements);
const Optional<Uint32> type =
PickMemoryType(memoryProperties, requirements.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT);
if (!type) return false;
out.coherent =
(memoryProperties.memoryTypes[*type].propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0;
VkMemoryAllocateInfo allocate{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocate.allocationSize = requirements.size;
allocate.memoryTypeIndex = *type;
if (vkAllocateMemory(device, &allocate, nullptr, &out.memory) != VK_SUCCESS) return false;
objects.memories.push_back(out.memory);
return vkBindBufferMemory(device, out.buffer, out.memory, 0) == VK_SUCCESS;
};
constexpr Uint32 kTexels = kSubjectExtent * kSubjectExtent;
constexpr Uint32 kSmallTexels = kSmallExtent * kSmallExtent;
constexpr VkDeviceSize kBytes = kTexels * sizeof(Uint32);
constexpr VkDeviceSize kSmallBytes = kSmallTexels * sizeof(Uint32);
VkImage subject = VK_NULL_HANDLE, small = VK_NULL_HANDLE, twin = VK_NULL_HANDLE;
if (!makeE5(kSubjectExtent, subject) || !makeE5(kSmallExtent, small))
return fail("E5B9G9R9 image creation failed");
WireE5RawCopyRegion region;
region.source = subject;
region.extent = {kSubjectExtent, kSubjectExtent, 1};
if (!makeImage(WireE5RawCopyTwinInfo(region), twin)) return fail("R32_UINT twin creation failed");
HostBuffer upload, subjectOut, controlOut, smallOut;
if (!makeBuffer(kBytes, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, upload) ||
!makeBuffer(kBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT, subjectOut) ||
!makeBuffer(kBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT, controlOut) ||
!makeBuffer(kSmallBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT, smallOut))
return fail("buffer creation failed");
if (const VkResult result = CreateWireE5RawCopyKit(device, objects.kit); result != VK_SUCCESS)
return fail(format("raw copy kit creation failed (VkResult {})", static_cast<Int>(result)));
{
void* mapped = nullptr;
if (vkMapMemory(device, upload.memory, 0, VK_WHOLE_SIZE, 0, &mapped) != VK_SUCCESS || mapped == nullptr)
return fail("vkMapMemory (upload) failed");
Uint32* words = static_cast<Uint32*>(mapped);
for (Uint32 i = 0; i < kTexels; ++i) words[i] = NonCanonicalWord(i);
if (!upload.coherent) {
VkMappedMemoryRange range{VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE};
range.memory = upload.memory;
range.size = VK_WHOLE_SIZE;
vkFlushMappedMemoryRanges(device, 1, &range);
}
vkUnmapMemory(device, upload.memory);
}
VkCommandPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};
poolInfo.flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT;
poolInfo.queueFamilyIndex = context.queueFamilyIndex;
if (vkCreateCommandPool(device, &poolInfo, nullptr, &objects.commandPool) != VK_SUCCESS)
return fail("vkCreateCommandPool failed");
VkCommandBufferAllocateInfo commandInfo{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};
commandInfo.commandPool = objects.commandPool;
commandInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
commandInfo.commandBufferCount = 1;
VkCommandBuffer cmd = VK_NULL_HANDLE;
if (vkAllocateCommandBuffers(device, &commandInfo, &cmd) != VK_SUCCESS)
return fail("vkAllocateCommandBuffers failed");
VkCommandBufferBeginInfo beginInfo{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
if (vkBeginCommandBuffer(cmd, &beginInfo) != VK_SUCCESS) return fail("vkBeginCommandBuffer failed");
// (1) Upload: transfer writes into E5B9G9R9 move bits (measured), so both images hold
// exactly the generated words; the small one gets the first 8x8 of them.
for (const VkImage image : {subject, small}) {
ImageBarrier(cmd, image, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0, VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_ACCESS_TRANSFER_WRITE_BIT);
}
VkBufferImageCopy copy{};
copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
copy.imageExtent = {kSubjectExtent, kSubjectExtent, 1};
vkCmdCopyBufferToImage(cmd, upload.buffer, subject, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &copy);
VkBufferImageCopy smallCopy = copy;
smallCopy.bufferRowLength = kSubjectExtent;
smallCopy.imageExtent = {kSmallExtent, kSmallExtent, 1};
vkCmdCopyBufferToImage(cmd, upload.buffer, small, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &smallCopy);
// (2) SUBJECT and the small reading: plain transfer reads.
for (const VkImage image : {subject, small}) {
ImageBarrier(cmd, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_ACCESS_TRANSFER_READ_BIT);
}
vkCmdCopyImageToBuffer(cmd, subject, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, subjectOut.buffer, 1, &copy);
VkBufferImageCopy smallOutCopy = copy;
smallOutCopy.imageExtent = {kSmallExtent, kSmallExtent, 1};
vkCmdCopyImageToBuffer(cmd, small, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, smallOut.buffer, 1, &smallOutCopy);
// (3) CONTROL: the route, exactly as production records it.
ImageBarrier(cmd, subject, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL,
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
VK_ACCESS_SHADER_READ_BIT);
if (const VkResult result =
RecordWireE5RawCopy(device, cmd, objects.kit, region, VK_IMAGE_LAYOUT_GENERAL, twin, objects.copy);
result != VK_SUCCESS)
return fail(format("recording the raw copy failed (VkResult {})", static_cast<Int>(result)));
vkCmdCopyImageToBuffer(cmd, twin, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, controlOut.buffer, 1, &copy);
VkMemoryBarrier hostBarrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
hostBarrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
hostBarrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT;
vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &hostBarrier, 0,
nullptr, 0, nullptr);
if (vkEndCommandBuffer(cmd) != VK_SUCCESS) return fail("vkEndCommandBuffer failed");
VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
if (vkCreateFence(device, &fenceInfo, nullptr, &objects.fence) != VK_SUCCESS)
return fail("vkCreateFence failed");
VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO};
submit.commandBufferCount = 1;
submit.pCommandBuffers = &cmd;
if (const VkResult result = vkQueueSubmit(context.queue, 1, &submit, objects.fence); result != VK_SUCCESS)
return fail(format("vkQueueSubmit failed (VkResult {})", static_cast<Int>(result)));
const VkResult waited = vkWaitForFences(device, 1, &objects.fence, VK_TRUE, kFenceTimeoutNs);
if (waited == VK_TIMEOUT) {
leak = true;
measurement.fenceWaitTimedOut = true;
return fail("the probe's fence did not signal within 5 s");
}
if (waited != VK_SUCCESS) return fail(format("vkWaitForFences failed (VkResult {})", static_cast<Int>(waited)));
const auto read = [&](const HostBuffer& buffer, const Uint32*& words) -> Bool {
void* mapped = nullptr;
if (vkMapMemory(device, buffer.memory, 0, VK_WHOLE_SIZE, 0, &mapped) != VK_SUCCESS || mapped == nullptr)
return false;
if (!buffer.coherent) {
VkMappedMemoryRange range{VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE};
range.memory = buffer.memory;
range.size = VK_WHOLE_SIZE;
vkInvalidateMappedMemoryRanges(device, 1, &range);
}
words = static_cast<const Uint32*>(mapped);
return true;
};
const Uint32 *subjectWords = nullptr, *controlWords = nullptr, *smallWords = nullptr;
if (!read(subjectOut, subjectWords) || !read(controlOut, controlWords) || !read(smallOut, smallWords))
return fail("vkMapMemory (readback) failed");
measurement.texels = kTexels;
for (Uint32 i = 0; i < kTexels; ++i) {
const Uint32 expected = NonCanonicalWord(i);
if (subjectWords[i] != expected) {
if (measurement.subjectMismatches++ == 0) {
measurement.firstExpected = expected;
measurement.firstObserved = subjectWords[i];
}
}
if (controlWords[i] != expected) ++measurement.controlMismatches;
}
measurement.smallTexels = kSmallTexels;
for (Uint32 y = 0; y < kSmallExtent; ++y)
for (Uint32 x = 0; x < kSmallExtent; ++x)
if (smallWords[y * kSmallExtent + x] != NonCanonicalWord(y * kSubjectExtent + x))
++measurement.smallMismatches;
vkUnmapMemory(device, subjectOut.memory);
vkUnmapMemory(device, controlOut.memory);
vkUnmapMemory(device, smallOut.memory);
measurement.ran = true;
return measurement;
}
WireE5TransferProbeMeasurement RunWireE5TransferProbeOnThrowawayDevice(PFN_vkGetInstanceProcAddr getInstanceProcAddr,
VkInstance instance,
VkPhysicalDevice physicalDevice,
Uint32 graphicsQueueFamilyIndex) {
WireE5TransferProbeMeasurement measurement;
const auto createDevice =
reinterpret_cast<PFN_vkCreateDevice>(getInstanceProcAddr(instance, "vkCreateDevice"));
if (createDevice == nullptr) {
measurement.failureReason = "no vkCreateDevice";
return measurement;
}
const Float priority = 1.0f;
VkDeviceQueueCreateInfo queueInfo{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO};
queueInfo.queueFamilyIndex = graphicsQueueFamilyIndex;
queueInfo.queueCount = 1;
queueInfo.pQueuePriorities = &priority;
VkDeviceCreateInfo deviceInfo{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO};
deviceInfo.queueCreateInfoCount = 1;
deviceInfo.pQueueCreateInfos = &queueInfo;
VkDevice device = VK_NULL_HANDLE;
if (const VkResult result = createDevice(physicalDevice, &deviceInfo, nullptr, &device);
result != VK_SUCCESS || device == VK_NULL_HANDLE) {
measurement.failureReason = format("vkCreateDevice failed (VkResult {})", static_cast<Int>(result));
return measurement;
}
WireE5TransferProbeContext context;
context.physicalDevice = physicalDevice;
context.device = device;
vkGetDeviceQueue(device, graphicsQueueFamilyIndex, 0, &context.queue);
context.queueFamilyIndex = graphicsQueueFamilyIndex;
measurement = RunWireE5TransferProbe(context);
if (!measurement.fenceWaitTimedOut) vkDestroyDevice(device, nullptr);
return measurement;
}
} // namespace MobileGL::MG_Backend::DirectVulkan
@@ -0,0 +1,168 @@
// MobileGL - MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.h
// Copyright (c) 2026 MobileGL-Dev
// Licensed under the GNU Lesser General Public License v3.0:
// https://www.gnu.org/licenses/gpl-3.0.txt
// https://www.gnu.org/licenses/lgpl-3.0.txt
// SPDX-License-Identifier: LGPL-3.0-only
// End of Source File Header
#pragma once
#include <Config.h>
#include <Includes.h>
#include <vulkan/vulkan_core.h>
// THE RAW E5B9G9R9 READ ROUTE, and the probe that decides whether a device needs it.
//
// GL makes RGB9_E5 a bit-exact format wherever it moves texel blocks: glGetTexImage with
// GL_UNSIGNED_INT_5_9_9_9_REV hands back the words the application stored, and
// glCopyImageSubData is a raw block move (GL 4.6 core 18.3.2). RGB9_E5 has redundant encodings -
// a shared exponent one higher with every mantissa halved is the same value - so a path that
// decodes and re-encodes keeps every value and still changes words.
//
// Adreno 750 does exactly that on TRANSFER READS of a VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 image once
// the level is not a power-of-two size: vkCmdCopyImageToBuffer and vkCmdCopyImage out of a 30x30
// level return every non-canonical word canonicalized, while a 16x16 or 64x64 level reads back verbatim.
// The stored words are intact - a shader reading the same image through an R32_UINT view sees
// every one of them - and so are transfer WRITES into it. Measured with format lists, without
// the colour-attachment usage and with storage usage: none of them changes the answer, and the
// device has no VK_EXT_image_compression_control to ask with.
//
// The route therefore reads through a shader: a compute pass fetches the source through an
// R32_UINT view (the wire's E5B9G9R9 textures are MUTABLE_FORMAT from birth, see
// VkTextureManager) and stores the words into an R32_UINT twin, whose transfer reads move bits.
// The caller then reads the twin where it would have read the source.
//
// Whether a device needs it is MEASURED (RunWireE5TransferProbe below), at Magma bring-up
// (VulkanRenderer::ArmWireE5RawReads) and in the POST's "Known Driver Bugs" section
// (MG_Util/SelfTest/DriverBugProbes.cpp); a device that reads E5B9G9R9 back verbatim keeps its
// plain transfers.
namespace MobileGL::MG_Backend::DirectVulkan {
// Device objects that do not depend on any image.
struct WireE5RawCopyKit {
VkShaderModule arrayModule = VK_NULL_HANDLE;
VkShaderModule volumeModule = VK_NULL_HANDLE;
VkDescriptorSetLayout descriptorLayout = VK_NULL_HANDLE;
VkPipelineLayout pipelineLayout = VK_NULL_HANDLE;
VkPipeline arrayPipeline = VK_NULL_HANDLE;
VkPipeline volumePipeline = VK_NULL_HANDLE;
VkSampler nearest = VK_NULL_HANDLE;
};
// Creates every member; on failure the members made so far stay set, for DestroyWireE5RawCopyKit.
VkResult CreateWireE5RawCopyKit(VkDevice device, WireE5RawCopyKit& kit);
void DestroyWireE5RawCopyKit(VkDevice device, WireE5RawCopyKit& kit);
// Whether the device can run the route at all: R32_UINT sampled and storage on optimal tiling.
Bool WireE5RawCopySupported(VkPhysicalDevice physicalDevice);
// One region of one level of the source. `volume` for a VK_IMAGE_TYPE_3D source, whose slices
// are `offset.z` .. `offset.z + extent.depth`; any other source is a 2D (possibly layered or
// cube-compatible) image whose layers are `baseLayer` .. `baseLayer + layerCount` and whose
// `offset.z` / `extent.depth` are 0 / 1. 1D images are not supported.
struct WireE5RawCopyRegion {
VkImage source = VK_NULL_HANDLE;
Bool volume = false;
Uint32 level = 0;
Uint32 baseLayer = 0;
Uint32 layerCount = 1;
VkOffset3D offset{0, 0, 0};
VkExtent3D extent{1, 1, 1};
};
// The twin the caller creates for `region`: R32_UINT, one level, the region's extent at its
// origin - a 3D image for a volume region, a 2D image of `layerCount` layers otherwise -
// with STORAGE | TRANSFER_SRC usage. RecordWireE5RawCopy leaves it in TRANSFER_SRC_OPTIMAL.
VkImageCreateInfo WireE5RawCopyTwinInfo(const WireE5RawCopyRegion& region);
// What a recorded copy holds until its submission completes.
struct WireE5RawCopyObjects {
VkDescriptorPool descriptorPool = VK_NULL_HANDLE;
VkImageView sourceView = VK_NULL_HANDLE;
VkImageView twinView = VK_NULL_HANDLE;
};
void DestroyWireE5RawCopyObjects(VkDevice device, WireE5RawCopyObjects& objects);
// Records the copy of `region` into `twin` (fresh, created from WireE5RawCopyTwinInfo).
// The source must have been created MUTABLE_FORMAT with an E5B9G9R9 format, and be in
// `sourceLayout` (GENERAL or SHADER_READ_ONLY_OPTIMAL); the recording starts with a global
// barrier that makes every earlier write visible to the compute stage and ends with the twin
// in TRANSFER_SRC_OPTIMAL, its writes visible to transfer reads. On failure nothing has been
// recorded and `objects` holds whatever was created, for DestroyWireE5RawCopyObjects.
VkResult RecordWireE5RawCopy(VkDevice device, VkCommandBuffer commandBuffer, const WireE5RawCopyKit& kit,
const WireE5RawCopyRegion& region, VkImageLayout sourceLayout, VkImage twin,
WireE5RawCopyObjects& objects);
// ---------------------------------------------------------------------------------------
// THE PROBE.
struct WireE5TransferProbeMeasurement {
// Recorded, submitted and read back.
Bool ran = false;
// The bounded fence wait expired: every object was deliberately leaked.
Bool fenceWaitTimedOut = false;
String failureReason;
// SUBJECT: a 30x30 E5B9G9R9 level of non-canonical words read back by vkCmdCopyImageToBuffer.
Uint32 texels = 0;
Uint32 subjectMismatches = 0;
Uint32 firstExpected = 0;
Uint32 firstObserved = 0;
// CONTROL: the same image read through the route. Must be exact, or the upload, the
// comparison or the route itself is what failed, and nothing is concluded.
Uint32 controlMismatches = 0;
// Reported, not judged: the same words in a 16x16 level read back by plain transfer. Clean
// on the device this was characterised on (power-of-two extents read back verbatim, 28, 30
// and 31 do not), which is why the subject is 30x30.
Uint32 smallTexels = 0;
Uint32 smallMismatches = 0;
};
enum class WireE5TransferProbeVerdict : Uint8 {
// Nothing was measured (the route is unavailable, or setup failed): plain transfers stay.
NotRun,
// Measured, but the route's own read was not exact: nothing is concluded.
Inconclusive,
// Plain transfer reads return the stored words.
Clean,
// The defect: plain transfer reads changed words the route read back exactly.
TransferReadNormalizes,
};
inline const char* WireE5TransferProbeVerdictName(WireE5TransferProbeVerdict verdict) {
switch (verdict) {
case WireE5TransferProbeVerdict::NotRun: return "not-run";
case WireE5TransferProbeVerdict::Inconclusive: return "inconclusive";
case WireE5TransferProbeVerdict::Clean: return "clean";
case WireE5TransferProbeVerdict::TransferReadNormalizes: return "transfer-read-normalizes";
}
return "?";
}
inline WireE5TransferProbeVerdict EvaluateWireE5TransferProbe(const WireE5TransferProbeMeasurement& measurement) {
if (!measurement.ran || measurement.fenceWaitTimedOut || measurement.texels == 0)
return WireE5TransferProbeVerdict::NotRun;
if (measurement.controlMismatches != 0) return WireE5TransferProbeVerdict::Inconclusive;
return measurement.subjectMismatches != 0 ? WireE5TransferProbeVerdict::TransferReadNormalizes
: WireE5TransferProbeVerdict::Clean;
}
// One line with every reading, for the bring-up log and the POST row.
String DescribeWireE5TransferProbe(const WireE5TransferProbeMeasurement& measurement);
struct WireE5TransferProbeContext {
VkPhysicalDevice physicalDevice = VK_NULL_HANDLE;
VkDevice device = VK_NULL_HANDLE;
// Must be otherwise idle for the probe's duration.
VkQueue queue = VK_NULL_HANDLE;
Uint32 queueFamilyIndex = 0;
};
// One submission on its own command pool, a bounded fence wait (on timeout every object is
// leaked and the measurement says so - the caller must then not destroy or idle-wait the
// device), teardown on every other path. Never touches renderer state.
WireE5TransferProbeMeasurement RunWireE5TransferProbe(const WireE5TransferProbeContext& context);
// The POST's entry: a throwaway device with one graphics queue, the probe, the device
// destroyed (leaked on a fence timeout).
WireE5TransferProbeMeasurement RunWireE5TransferProbeOnThrowawayDevice(
PFN_vkGetInstanceProcAddr getInstanceProcAddr, VkInstance instance, VkPhysicalDevice physicalDevice,
Uint32 graphicsQueueFamilyIndex);
} // namespace MobileGL::MG_Backend::DirectVulkan
@@ -0,0 +1,19 @@
#version 450
// The volume twin of WireE5RawCopy.comp: the source level is a 3D image and the region's slices
// are its z range.
layout(local_size_x = 8, local_size_y = 8, local_size_z = 1) in;
layout(set = 0, binding = 0) uniform usampler3D sourceImage;
layout(set = 0, binding = 1, r32ui) uniform writeonly uimage3D twinImage;
layout(push_constant) uniform Region {
// xyz: the region's texel origin in the source level.
ivec4 sourceOffset;
// xyz: texels to copy.
ivec4 extent;
} region;
void main() {
ivec3 id = ivec3(gl_GlobalInvocationID);
if (any(greaterThanEqual(id, region.extent.xyz))) return;
uint word = texelFetch(sourceImage, region.sourceOffset.xyz + id, 0).r;
imageStore(twinImage, id, uvec4(word, 0u, 0u, 0u));
}
@@ -0,0 +1,839 @@
// Generated from WireE5RawCopy.comp / WireE5RawCopy3D.comp (Vulkan 1.0 / SPIR-V 1.0) by
// scripts/bake_internal_shaders.py, which runs MG_Test's BakedInternalShadersTest with the
// IN-TREE glslang and splices the words it emits into the arrays below. Do not edit the words
// by hand: MOBILEGL_BAKED_INTERNAL_SHADERS (the same test, run as a gate) recompiles every
// source in its table and reds on the first word that disagrees.
#pragma once
static constexpr Uint32 kWireE5RawCopySpirv[] = {
0x07230203u,
0x00010000u,
0x0008000bu,
0x00000046u,
0x00000000u,
0x00020011u,
0x00000001u,
0x0006000bu,
0x00000001u,
0x4c534c47u,
0x6474732eu,
0x3035342eu,
0x00000000u,
0x0003000eu,
0x00000000u,
0x00000001u,
0x0006000fu,
0x00000005u,
0x00000004u,
0x6e69616du,
0x00000000u,
0x0000000du,
0x00060010u,
0x00000004u,
0x00000011u,
0x00000008u,
0x00000008u,
0x00000001u,
0x00030003u,
0x00000002u,
0x000001c2u,
0x00040005u,
0x00000004u,
0x6e69616du,
0x00000000u,
0x00030005u,
0x00000009u,
0x00006469u,
0x00080005u,
0x0000000du,
0x475f6c67u,
0x61626f6cu,
0x766e496cu,
0x7461636fu,
0x496e6f69u,
0x00000044u,
0x00040005u,
0x00000012u,
0x69676552u,
0x00006e6fu,
0x00070006u,
0x00000012u,
0x00000000u,
0x72756f73u,
0x664f6563u,
0x74657366u,
0x00000000u,
0x00050006u,
0x00000012u,
0x00000001u,
0x65747865u,
0x0000746eu,
0x00040005u,
0x00000014u,
0x69676572u,
0x00006e6fu,
0x00040005u,
0x00000022u,
0x64726f77u,
0x00000000u,
0x00050005u,
0x00000026u,
0x72756f73u,
0x6d496563u,
0x00656761u,
0x00050005u,
0x0000003eu,
0x6e697774u,
0x67616d49u,
0x00000065u,
0x00040047u,
0x0000000du,
0x0000000bu,
0x0000001cu,
0x00030047u,
0x00000012u,
0x00000002u,
0x00050048u,
0x00000012u,
0x00000000u,
0x00000023u,
0x00000000u,
0x00050048u,
0x00000012u,
0x00000001u,
0x00000023u,
0x00000010u,
0x00040047u,
0x00000026u,
0x00000021u,
0x00000000u,
0x00040047u,
0x00000026u,
0x00000022u,
0x00000000u,
0x00030047u,
0x0000003eu,
0x00000019u,
0x00040047u,
0x0000003eu,
0x00000021u,
0x00000001u,
0x00040047u,
0x0000003eu,
0x00000022u,
0x00000000u,
0x00040047u,
0x00000045u,
0x0000000bu,
0x00000019u,
0x00020013u,
0x00000002u,
0x00030021u,
0x00000003u,
0x00000002u,
0x00040015u,
0x00000006u,
0x00000020u,
0x00000001u,
0x00040017u,
0x00000007u,
0x00000006u,
0x00000003u,
0x00040020u,
0x00000008u,
0x00000007u,
0x00000007u,
0x00040015u,
0x0000000au,
0x00000020u,
0x00000000u,
0x00040017u,
0x0000000bu,
0x0000000au,
0x00000003u,
0x00040020u,
0x0000000cu,
0x00000001u,
0x0000000bu,
0x0004003bu,
0x0000000cu,
0x0000000du,
0x00000001u,
0x00040017u,
0x00000011u,
0x00000006u,
0x00000004u,
0x0004001eu,
0x00000012u,
0x00000011u,
0x00000011u,
0x00040020u,
0x00000013u,
0x00000009u,
0x00000012u,
0x0004003bu,
0x00000013u,
0x00000014u,
0x00000009u,
0x0004002bu,
0x00000006u,
0x00000015u,
0x00000001u,
0x00040020u,
0x00000016u,
0x00000009u,
0x00000011u,
0x00020014u,
0x0000001au,
0x00040017u,
0x0000001bu,
0x0000001au,
0x00000003u,
0x00040020u,
0x00000021u,
0x00000007u,
0x0000000au,
0x00090019u,
0x00000023u,
0x0000000au,
0x00000001u,
0x00000000u,
0x00000001u,
0x00000000u,
0x00000001u,
0x00000000u,
0x0003001bu,
0x00000024u,
0x00000023u,
0x00040020u,
0x00000025u,
0x00000000u,
0x00000024u,
0x0004003bu,
0x00000025u,
0x00000026u,
0x00000000u,
0x0004002bu,
0x00000006u,
0x00000028u,
0x00000000u,
0x00040017u,
0x00000029u,
0x00000006u,
0x00000002u,
0x0004002bu,
0x0000000au,
0x00000030u,
0x00000002u,
0x00040020u,
0x00000031u,
0x00000007u,
0x00000006u,
0x00040017u,
0x00000038u,
0x0000000au,
0x00000004u,
0x0004002bu,
0x0000000au,
0x0000003au,
0x00000000u,
0x00090019u,
0x0000003cu,
0x0000000au,
0x00000001u,
0x00000000u,
0x00000001u,
0x00000000u,
0x00000002u,
0x00000021u,
0x00040020u,
0x0000003du,
0x00000000u,
0x0000003cu,
0x0004003bu,
0x0000003du,
0x0000003eu,
0x00000000u,
0x0004002bu,
0x0000000au,
0x00000043u,
0x00000008u,
0x0004002bu,
0x0000000au,
0x00000044u,
0x00000001u,
0x0006002cu,
0x0000000bu,
0x00000045u,
0x00000043u,
0x00000043u,
0x00000044u,
0x00050036u,
0x00000002u,
0x00000004u,
0x00000000u,
0x00000003u,
0x000200f8u,
0x00000005u,
0x0004003bu,
0x00000008u,
0x00000009u,
0x00000007u,
0x0004003bu,
0x00000021u,
0x00000022u,
0x00000007u,
0x0004003du,
0x0000000bu,
0x0000000eu,
0x0000000du,
0x0004007cu,
0x00000007u,
0x0000000fu,
0x0000000eu,
0x0003003eu,
0x00000009u,
0x0000000fu,
0x0004003du,
0x00000007u,
0x00000010u,
0x00000009u,
0x00050041u,
0x00000016u,
0x00000017u,
0x00000014u,
0x00000015u,
0x0004003du,
0x00000011u,
0x00000018u,
0x00000017u,
0x0008004fu,
0x00000007u,
0x00000019u,
0x00000018u,
0x00000018u,
0x00000000u,
0x00000001u,
0x00000002u,
0x000500afu,
0x0000001bu,
0x0000001cu,
0x00000010u,
0x00000019u,
0x0004009au,
0x0000001au,
0x0000001du,
0x0000001cu,
0x000300f7u,
0x0000001fu,
0x00000000u,
0x000400fau,
0x0000001du,
0x0000001eu,
0x0000001fu,
0x000200f8u,
0x0000001eu,
0x000100fdu,
0x000200f8u,
0x0000001fu,
0x0004003du,
0x00000024u,
0x00000027u,
0x00000026u,
0x00050041u,
0x00000016u,
0x0000002au,
0x00000014u,
0x00000028u,
0x0004003du,
0x00000011u,
0x0000002bu,
0x0000002au,
0x0007004fu,
0x00000029u,
0x0000002cu,
0x0000002bu,
0x0000002bu,
0x00000000u,
0x00000001u,
0x0004003du,
0x00000007u,
0x0000002du,
0x00000009u,
0x0007004fu,
0x00000029u,
0x0000002eu,
0x0000002du,
0x0000002du,
0x00000000u,
0x00000001u,
0x00050080u,
0x00000029u,
0x0000002fu,
0x0000002cu,
0x0000002eu,
0x00050041u,
0x00000031u,
0x00000032u,
0x00000009u,
0x00000030u,
0x0004003du,
0x00000006u,
0x00000033u,
0x00000032u,
0x00050051u,
0x00000006u,
0x00000034u,
0x0000002fu,
0x00000000u,
0x00050051u,
0x00000006u,
0x00000035u,
0x0000002fu,
0x00000001u,
0x00060050u,
0x00000007u,
0x00000036u,
0x00000034u,
0x00000035u,
0x00000033u,
0x00040064u,
0x00000023u,
0x00000037u,
0x00000027u,
0x0007005fu,
0x00000038u,
0x00000039u,
0x00000037u,
0x00000036u,
0x00000002u,
0x00000028u,
0x00050051u,
0x0000000au,
0x0000003bu,
0x00000039u,
0x00000000u,
0x0003003eu,
0x00000022u,
0x0000003bu,
0x0004003du,
0x0000003cu,
0x0000003fu,
0x0000003eu,
0x0004003du,
0x00000007u,
0x00000040u,
0x00000009u,
0x0004003du,
0x0000000au,
0x00000041u,
0x00000022u,
0x00070050u,
0x00000038u,
0x00000042u,
0x00000041u,
0x0000003au,
0x0000003au,
0x0000003au,
0x00040063u,
0x0000003fu,
0x00000040u,
0x00000042u,
0x000100fdu,
0x00010038u,
};
static constexpr Uint32 kWireE5RawCopy3DSpirv[] = {
0x07230203u,
0x00010000u,
0x0008000bu,
0x0000003du,
0x00000000u,
0x00020011u,
0x00000001u,
0x0006000bu,
0x00000001u,
0x4c534c47u,
0x6474732eu,
0x3035342eu,
0x00000000u,
0x0003000eu,
0x00000000u,
0x00000001u,
0x0006000fu,
0x00000005u,
0x00000004u,
0x6e69616du,
0x00000000u,
0x0000000du,
0x00060010u,
0x00000004u,
0x00000011u,
0x00000008u,
0x00000008u,
0x00000001u,
0x00030003u,
0x00000002u,
0x000001c2u,
0x00040005u,
0x00000004u,
0x6e69616du,
0x00000000u,
0x00030005u,
0x00000009u,
0x00006469u,
0x00080005u,
0x0000000du,
0x475f6c67u,
0x61626f6cu,
0x766e496cu,
0x7461636fu,
0x496e6f69u,
0x00000044u,
0x00040005u,
0x00000012u,
0x69676552u,
0x00006e6fu,
0x00070006u,
0x00000012u,
0x00000000u,
0x72756f73u,
0x664f6563u,
0x74657366u,
0x00000000u,
0x00050006u,
0x00000012u,
0x00000001u,
0x65747865u,
0x0000746eu,
0x00040005u,
0x00000014u,
0x69676572u,
0x00006e6fu,
0x00040005u,
0x00000022u,
0x64726f77u,
0x00000000u,
0x00050005u,
0x00000026u,
0x72756f73u,
0x6d496563u,
0x00656761u,
0x00050005u,
0x00000035u,
0x6e697774u,
0x67616d49u,
0x00000065u,
0x00040047u,
0x0000000du,
0x0000000bu,
0x0000001cu,
0x00030047u,
0x00000012u,
0x00000002u,
0x00050048u,
0x00000012u,
0x00000000u,
0x00000023u,
0x00000000u,
0x00050048u,
0x00000012u,
0x00000001u,
0x00000023u,
0x00000010u,
0x00040047u,
0x00000026u,
0x00000021u,
0x00000000u,
0x00040047u,
0x00000026u,
0x00000022u,
0x00000000u,
0x00030047u,
0x00000035u,
0x00000019u,
0x00040047u,
0x00000035u,
0x00000021u,
0x00000001u,
0x00040047u,
0x00000035u,
0x00000022u,
0x00000000u,
0x00040047u,
0x0000003cu,
0x0000000bu,
0x00000019u,
0x00020013u,
0x00000002u,
0x00030021u,
0x00000003u,
0x00000002u,
0x00040015u,
0x00000006u,
0x00000020u,
0x00000001u,
0x00040017u,
0x00000007u,
0x00000006u,
0x00000003u,
0x00040020u,
0x00000008u,
0x00000007u,
0x00000007u,
0x00040015u,
0x0000000au,
0x00000020u,
0x00000000u,
0x00040017u,
0x0000000bu,
0x0000000au,
0x00000003u,
0x00040020u,
0x0000000cu,
0x00000001u,
0x0000000bu,
0x0004003bu,
0x0000000cu,
0x0000000du,
0x00000001u,
0x00040017u,
0x00000011u,
0x00000006u,
0x00000004u,
0x0004001eu,
0x00000012u,
0x00000011u,
0x00000011u,
0x00040020u,
0x00000013u,
0x00000009u,
0x00000012u,
0x0004003bu,
0x00000013u,
0x00000014u,
0x00000009u,
0x0004002bu,
0x00000006u,
0x00000015u,
0x00000001u,
0x00040020u,
0x00000016u,
0x00000009u,
0x00000011u,
0x00020014u,
0x0000001au,
0x00040017u,
0x0000001bu,
0x0000001au,
0x00000003u,
0x00040020u,
0x00000021u,
0x00000007u,
0x0000000au,
0x00090019u,
0x00000023u,
0x0000000au,
0x00000002u,
0x00000000u,
0x00000000u,
0x00000000u,
0x00000001u,
0x00000000u,
0x0003001bu,
0x00000024u,
0x00000023u,
0x00040020u,
0x00000025u,
0x00000000u,
0x00000024u,
0x0004003bu,
0x00000025u,
0x00000026u,
0x00000000u,
0x0004002bu,
0x00000006u,
0x00000028u,
0x00000000u,
0x00040017u,
0x0000002fu,
0x0000000au,
0x00000004u,
0x0004002bu,
0x0000000au,
0x00000031u,
0x00000000u,
0x00090019u,
0x00000033u,
0x0000000au,
0x00000002u,
0x00000000u,
0x00000000u,
0x00000000u,
0x00000002u,
0x00000021u,
0x00040020u,
0x00000034u,
0x00000000u,
0x00000033u,
0x0004003bu,
0x00000034u,
0x00000035u,
0x00000000u,
0x0004002bu,
0x0000000au,
0x0000003au,
0x00000008u,
0x0004002bu,
0x0000000au,
0x0000003bu,
0x00000001u,
0x0006002cu,
0x0000000bu,
0x0000003cu,
0x0000003au,
0x0000003au,
0x0000003bu,
0x00050036u,
0x00000002u,
0x00000004u,
0x00000000u,
0x00000003u,
0x000200f8u,
0x00000005u,
0x0004003bu,
0x00000008u,
0x00000009u,
0x00000007u,
0x0004003bu,
0x00000021u,
0x00000022u,
0x00000007u,
0x0004003du,
0x0000000bu,
0x0000000eu,
0x0000000du,
0x0004007cu,
0x00000007u,
0x0000000fu,
0x0000000eu,
0x0003003eu,
0x00000009u,
0x0000000fu,
0x0004003du,
0x00000007u,
0x00000010u,
0x00000009u,
0x00050041u,
0x00000016u,
0x00000017u,
0x00000014u,
0x00000015u,
0x0004003du,
0x00000011u,
0x00000018u,
0x00000017u,
0x0008004fu,
0x00000007u,
0x00000019u,
0x00000018u,
0x00000018u,
0x00000000u,
0x00000001u,
0x00000002u,
0x000500afu,
0x0000001bu,
0x0000001cu,
0x00000010u,
0x00000019u,
0x0004009au,
0x0000001au,
0x0000001du,
0x0000001cu,
0x000300f7u,
0x0000001fu,
0x00000000u,
0x000400fau,
0x0000001du,
0x0000001eu,
0x0000001fu,
0x000200f8u,
0x0000001eu,
0x000100fdu,
0x000200f8u,
0x0000001fu,
0x0004003du,
0x00000024u,
0x00000027u,
0x00000026u,
0x00050041u,
0x00000016u,
0x00000029u,
0x00000014u,
0x00000028u,
0x0004003du,
0x00000011u,
0x0000002au,
0x00000029u,
0x0008004fu,
0x00000007u,
0x0000002bu,
0x0000002au,
0x0000002au,
0x00000000u,
0x00000001u,
0x00000002u,
0x0004003du,
0x00000007u,
0x0000002cu,
0x00000009u,
0x00050080u,
0x00000007u,
0x0000002du,
0x0000002bu,
0x0000002cu,
0x00040064u,
0x00000023u,
0x0000002eu,
0x00000027u,
0x0007005fu,
0x0000002fu,
0x00000030u,
0x0000002eu,
0x0000002du,
0x00000002u,
0x00000028u,
0x00050051u,
0x0000000au,
0x00000032u,
0x00000030u,
0x00000000u,
0x0003003eu,
0x00000022u,
0x00000032u,
0x0004003du,
0x00000033u,
0x00000036u,
0x00000035u,
0x0004003du,
0x00000007u,
0x00000037u,
0x00000009u,
0x0004003du,
0x0000000au,
0x00000038u,
0x00000022u,
0x00070050u,
0x0000002fu,
0x00000039u,
0x00000038u,
0x00000031u,
0x00000031u,
0x00000031u,
0x00040063u,
0x00000036u,
0x00000037u,
0x00000039u,
0x000100fdu,
0x00010038u,
};
@@ -147,7 +147,7 @@
image.image = resource->image; image.format = resource->format; image.aspect = resource->aspect;
image.layout = resource->layout; image.trackedLayout = &resource->layout; image.levels = resource->mipLevels;
const auto restore = image.layout == VK_IMAGE_LAYOUT_UNDEFINED ? VK_IMAGE_LAYOUT_GENERAL : image.layout;
TransitionWireImage(image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
auto& frame = m_frameContext.GetCurrent();
VkBufferImageCopy regions[2]{};
Uint32 regionCount = 0;
const auto addCopy = [&](VkImageAspectFlags aspect, VkDeviceSize offset) {
@@ -160,8 +160,35 @@
if (!depthStencil) addCopy(VK_IMAGE_ASPECT_COLOR_BIT, 0);
if (depth) addCopy(VK_IMAGE_ASPECT_DEPTH_BIT, 0);
if (stencil) addCopy(VK_IMAGE_ASPECT_STENCIL_BIT, stencilOffset);
auto& frame = m_frameContext.GetCurrent();
vkCmdCopyImageToBuffer(frame.commandBuffer, image.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
// E5B9G9R9 on a device whose transfer reads canonicalize it: the words leave through a
// shader into a twin (WireE5RawCopy.h), which is what the buffer copy then reads.
VkImage copySource = image.image;
if (!depthStencil && m_textureManager->WireE5RawReadApplies(*resource)) {
TransitionWireImage(image, VK_IMAGE_LAYOUT_GENERAL);
WireE5RawCopyRegion region;
region.source = image.image;
region.volume = nativeVolume;
region.level = level;
region.baseLayer = nativeVolume ? 0u : layer;
region.layerCount = nativeVolume ? 1u : layerCount;
region.offset = regions[0].imageOffset;
region.extent = extent;
auto twin = MakeUnique<VkTextureManager::TextureResource>();
WireE5RawCopyObjects objects;
if (m_textureManager->RecordWireE5RawTwin(frame.commandBuffer, region, VK_IMAGE_LAYOUT_GENERAL, *twin,
objects)) {
copySource = twin->image;
regions[0].imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, region.layerCount};
regions[0].imageOffset = {0, 0, 0};
WireRetiredObjects retired{};
retired.textures.push_back(Move(twin));
retired.imageViews = {objects.sourceView, objects.twinView};
retired.descriptorPools = {objects.descriptorPool};
RetireWireObjects(Move(retired));
}
}
if (copySource == image.image) TransitionWireImage(image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
vkCmdCopyImageToBuffer(frame.commandBuffer, copySource, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
readback.GetHandle(), regionCount, regions);
TransitionWireImage(image, restore);
VkMemoryBarrier host{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
@@ -61,6 +61,7 @@ using Uint32 = std::uint32_t;
#include <MG_Backend/DirectVulkan/Renderer/WireDepthMipmapSpirv.h>
#include <MG_Backend/DirectVulkan/Renderer/WireMultisampleResolveSpirv.h>
#include <MG_Backend/DirectVulkan/Renderer/WireYuvConvertSpirv.h>
#include <MG_Backend/DirectVulkan/Renderer/WireE5RawCopySpirv.h>
struct BakedShader {
const char* name; // ctest entry name, and the bake file's stem
@@ -93,6 +94,19 @@ const BakedShader kBakedShaders[] = {
"kWireColorBlitFragmentSpirv", EShLangFragment,
glslang::EShTargetVulkan_1_0, glslang::EShTargetSpv_1_0,
kWireColorBlitFragmentSpirv),
// The raw E5B9G9R9 read route (WireE5RawCopy.h): one compute program per source shape.
MGL_BAKED_ROW("WireE5RawCopy",
"MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.comp",
"MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopySpirv.h",
"kWireE5RawCopySpirv", EShLangCompute,
glslang::EShTargetVulkan_1_0, glslang::EShTargetSpv_1_0,
kWireE5RawCopySpirv),
MGL_BAKED_ROW("WireE5RawCopy3D",
"MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy3D.comp",
"MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopySpirv.h",
"kWireE5RawCopy3DSpirv", EShLangCompute,
glslang::EShTargetVulkan_1_0, glslang::EShTargetSpv_1_0,
kWireE5RawCopy3DSpirv),
MGL_BAKED_ROW("WireYuvConvertVertex",
"MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvert.vert",
"MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvertSpirv.h",
@@ -12,6 +12,8 @@
#include <Config.h>
#include <MG_Util/Debug/Log.h>
#include <MG_Backend/DirectVulkan/Renderer/WireDepthResolveProbe.h>
#include <MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.h>
#include <MG_Backend/DirectVulkan/Renderer/WireCopySliceProbe.h>
#include <algorithm>
#include <cstring>
@@ -2528,9 +2530,59 @@ namespace MobileGL::MG_Util::SelfTest {
return DescribeDepthStencilResolvePassBug(measurement, shaderStencilExport);
}
// The same probe the Magma renderer runs at bring-up (VulkanRenderer::ArmWireE5RawReads),
// on a throwaway device with one graphics queue.
Optional<DriverBugFinding> ProbeE5TransferReadNormalizeBug(const VulkanDriverBugProbeContext& context) {
using namespace MG_Backend::DirectVulkan;
if (context.getInstanceProcAddr == nullptr) return std::nullopt;
const WireE5TransferProbeMeasurement measurement = RunWireE5TransferProbeOnThrowawayDevice(
context.getInstanceProcAddr, context.instance, context.physicalDevice,
context.graphicsQueueFamilyIndex);
const WireE5TransferProbeVerdict verdict = EvaluateWireE5TransferProbe(measurement);
MGLOG_I("Driver POST: E5B9G9R9 transfer probe verdict=%s: %s", WireE5TransferProbeVerdictName(verdict),
DescribeWireE5TransferProbe(measurement).c_str());
if (verdict != WireE5TransferProbeVerdict::TransferReadNormalizes) return std::nullopt;
String detail =
"vkCmdCopyImageToBuffer out of a 30x30 VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 image returns "
"non-canonical shared-exponent words re-encoded (same value, different bits), while a "
"compute shader reading the same image through an R32_UINT view returns every stored "
"word: ";
detail += DescribeWireE5TransferProbe(measurement);
detail +=
". GL moves RGB9_E5 blocks bit-exactly (glGetTexImage with UNSIGNED_INT_5_9_9_9_REV, "
"glCopyImageSubData), so Magma reads such textures through that shader into an R32_UINT "
"twin and copies the twin instead (MOBILEGL_MAGMA_E5_RAW_READS=0 turns it off)";
return DriverBugFinding{"E5B9G9R9 transfer reads canonicalize shared exponents",
DriverBugVerdict::Fixed, Move(detail)};
}
// The same probe the Magma renderer runs at bring-up (VulkanRenderer::ArmWireCopySliceSplit).
Optional<DriverBugFinding> ProbeCrossFormatSliceCopyBug(const VulkanDriverBugProbeContext& context) {
using namespace MG_Backend::DirectVulkan;
if (context.getInstanceProcAddr == nullptr) return std::nullopt;
const WireCopySliceProbeMeasurement measurement = RunWireCopySliceProbeOnThrowawayDevice(
context.getInstanceProcAddr, context.instance, context.physicalDevice,
context.graphicsQueueFamilyIndex);
const WireCopySliceProbeVerdict verdict = EvaluateWireCopySliceProbe(measurement);
MGLOG_I("Driver POST: slice copy probe verdict=%s: %s", WireCopySliceProbeVerdictName(verdict),
DescribeWireCopySliceProbe(measurement).c_str());
if (verdict != WireCopySliceProbeVerdict::CrossFormatSlicesDropped) return std::nullopt;
String detail =
"a vkCmdCopyImage region between a 2D array and a 3D image of DIFFERENT (size-compatible) "
"formats moves only its first slice, while the same copy as one region per slice is exact: ";
detail += DescribeWireCopySliceProbe(measurement);
detail +=
". Magma splits such glCopyImageSubData calls one slice per region "
"(MOBILEGL_MAGMA_SPLIT_SLICE_COPIES=0 turns it off)";
return DriverBugFinding{"Cross-format 2D-array <-> 3D image copies drop slices",
DriverBugVerdict::Fixed, Move(detail)};
}
using VulkanDriverBugProbeFn = Optional<DriverBugFinding> (*)(const VulkanDriverBugProbeContext&);
constexpr VulkanDriverBugProbeFn kVulkanDriverBugProbes[] = {
&ProbeDepthStencilResolvePassBug,
&ProbeE5TransferReadNormalizeBug,
&ProbeCrossFormatSliceCopyBug,
};
} // namespace