diff --git a/CMakeLists.txt b/CMakeLists.txt index 25fb6ffe..c0aacfcd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -733,6 +733,8 @@ list(APPEND SOURCE_FILES # arms and the probe that measures the first against the second, shared by the # renderer's bring-up and the Vulkan POST. MobileGL/MG_Backend/DirectVulkan/Renderer/WireDepthResolveProbe.cpp + MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.cpp + MobileGL/MG_Backend/DirectVulkan/Renderer/WireCopySliceProbe.cpp # P5 b1: the block-granularity persistent-map push the record arm makes mandatory (T2). MobileGL/MG_State/GLState/BufferState/PersistentMapTracker.cpp # P13 W5: the verb emitters and the monolith verb port, and the client GPU-write set diff --git a/MobileGL/Config.h b/MobileGL/Config.h index c9a85ac8..a3443441 100644 --- a/MobileGL/Config.h +++ b/MobileGL/Config.h @@ -381,6 +381,22 @@ namespace MobileGL::MG_Config { // immune to the probe's verdict moving), and ForceOff is the negative control that // replays the driver's silence. QuirkOverride MagmaPrimGenQueryReroute = QuirkOverride::Auto; + // MOBILEGL_MAGMA_E5_RAW_READS: DirectVulkan's raw read route for E5B9G9R9 textures + // (WireE5RawCopy.h). GL moves RGB9_E5 texel blocks bit-exactly (glGetTexImage with + // UNSIGNED_INT_5_9_9_9_REV, glCopyImageSubData), but Adreno 750's transfer reads of + // an E5B9G9R9 image larger than ~16x16 decode and re-encode, canonicalizing every + // non-canonical shared exponent. Auto defers to a device probe at renderer bring-up + // (RunWireE5TransferProbe: plain transfer read vs. a shader read through an R32_UINT + // view) and arms the route only where the plain read changed words; ForceOn arms it + // on any device (the host lanes exercise the route with it); ForceOff is the negative + // control that replays the canonicalization. + QuirkOverride MagmaE5RawReads = QuirkOverride::Auto; + // MOBILEGL_MAGMA_SPLIT_SLICE_COPIES: DirectVulkan's per-slice split of a cross-format + // glCopyImageSubData between a layered texture and a 3D one (WireCopySliceProbe.h). + // Adreno 750 lands only the first slice of such a vkCmdCopyImage region. Auto defers to + // a device probe at renderer bring-up (one region vs. per-slice regions); ForceOn splits + // on any device; ForceOff is the negative control that replays the dropped slices. + QuirkOverride MagmaSplitSliceCopies = QuirkOverride::Auto; // --- MGPipe (the disaggregation plan's explicit frontend/backend boundary) --- // MOBILEGL_PIPE_PUSH: per-subsystem bitmask selecting which state the frontend // PUSHES over MGPipe instead of leaving the backend to pull it out of GLContext. diff --git a/MobileGL/ConfigLoader.cpp b/MobileGL/ConfigLoader.cpp index fb2a8074..a3315a3b 100644 --- a/MobileGL/ConfigLoader.cpp +++ b/MobileGL/ConfigLoader.cpp @@ -420,6 +420,8 @@ namespace MobileGL::MG_ConfigLoader { features.EsprytWidenPacked16Storage = QueryEnvQuirkOverride("MOBILEGL_ESPRYT_WIDEN_PACKED16_STORAGE"); features.MagmaPrimGenQueryReroute = QueryEnvQuirkOverride("MOBILEGL_MAGMA_PRIMGEN_QUERY_REROUTE"); + features.MagmaE5RawReads = QueryEnvQuirkOverride("MOBILEGL_MAGMA_E5_RAW_READS"); + features.MagmaSplitSliceCopies = QueryEnvQuirkOverride("MOBILEGL_MAGMA_SPLIT_SLICE_COPIES"); // MGPipe. Nothing here needs adding to an allow-list: InitializeAcceptedEnvVariables // accepts every MOBILEGL_ / LIBGL_ prefixed variable in the environment, so a name // that starts with MOBILEGL_ is visible to these queries by construction. diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.cpp index ffdac4b0..640693f9 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.cpp @@ -274,6 +274,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { const VkTextureManager::TextureResource& oldResource, VkTextureManager::TextureResource& newResource , Bool splitAspects = false + , VkTextureManager* rawE5Manager = nullptr ) { MOBILEGL_ASSERT(device != VK_NULL_HANDLE, "PreserveTextureContentsOnRecreate: device is null"); MOBILEGL_ASSERT(commandPool != VK_NULL_HANDLE, "PreserveTextureContentsOnRecreate: commandPool is null"); @@ -307,19 +308,74 @@ namespace MobileGL::MG_Backend::DirectVulkan { 0, VK_ACCESS_TRANSFER_WRITE_BIT, newResource.aspect, 0, newResource.mipLevels); MOBILEGL_ASSERT(ok, "PreserveTextureContentsOnRecreate: failed to prepare destination image"); + // E5B9G9R9 on a device whose transfer reads canonicalize it: each level leaves through the + // raw route (WireE5RawCopy.h) into a twin, and the twins are copied instead. The old image + // and the new one must agree on the shape for the twin's single-level copy to stand in. + const Bool rawE5 = rawE5Manager != nullptr && rawE5Manager->WireE5RawReadApplies(oldResource) && + newResource.format == oldResource.format && newResource.viewType == oldResource.viewType; VkImageLayout srcTrackedLayout = oldResource.layout; VkPipelineStageFlags srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; VkAccessFlags srcAccessMask = 0; GetImageTransitionSourceState(srcTrackedLayout, srcStageMask, srcAccessMask); ok = VkTextureManager::TransitionImageLayout( - commandBuffer, oldResource.image, srcTrackedLayout, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - srcStageMask, VK_PIPELINE_STAGE_TRANSFER_BIT, - srcAccessMask, VK_ACCESS_TRANSFER_READ_BIT, oldResource.aspect, 0, preservedMipLevels); + commandBuffer, oldResource.image, srcTrackedLayout, + rawE5 ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + srcStageMask, rawE5 ? VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT : VK_PIPELINE_STAGE_TRANSFER_BIT, + srcAccessMask, rawE5 ? VK_ACCESS_SHADER_READ_BIT : VK_ACCESS_TRANSFER_READ_BIT, + oldResource.aspect, 0, preservedMipLevels); MOBILEGL_ASSERT(ok, "PreserveTextureContentsOnRecreate: failed to prepare source image"); + // Destroyed after the fence wait below, which is the twins' last use. + Vector rawTwins; + Vector rawObjects; + if (rawE5) { + rawTwins.reserve(preservedMipLevels); + const Bool volume = oldResource.viewType == VK_IMAGE_VIEW_TYPE_3D; + for (Uint32 level = 0; level < preservedMipLevels; ++level) { + WireE5RawCopyRegion region; + region.source = oldResource.image; + region.volume = volume; + region.level = level; + region.layerCount = volume ? 1u : std::min(oldResource.arrayLayers, newResource.arrayLayers); + region.extent = {std::max(std::min(oldResource.extent.width, newResource.extent.width) >> level, 1u), + std::max(std::min(oldResource.extent.height, newResource.extent.height) >> level, 1u), + volume ? std::max(std::min(oldResource.depth, newResource.depth) >> level, 1u) : 1u}; + VkTextureManager::TextureResource twin; + WireE5RawCopyObjects objects; + if (!rawE5Manager->RecordWireE5RawTwin(commandBuffer, region, VK_IMAGE_LAYOUT_GENERAL, twin, objects)) break; + VkImageCopy copy{}; + copy.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, region.layerCount}; + copy.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, 0, region.layerCount}; + copy.extent = region.extent; + vkCmdCopyImage(commandBuffer, twin.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, newResource.image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©); + rawTwins.push_back(Move(twin)); + rawObjects.push_back(objects); + } + if (rawTwins.size() != preservedMipLevels) { + // A twin could not be made: the remaining levels go by transfer (canonicalized, + // as every level was before the route) rather than not at all. + MGLOG_W_ONCE("PreserveTextureContentsOnRecreate: the raw E5B9G9R9 route failed; levels %zu.. " + "are copied by transfer", rawTwins.size()); + } + } + const Uint32 firstTransferLevel = static_cast(rawTwins.size()); + if (rawE5 && firstTransferLevel < preservedMipLevels) { + VkImageMemoryBarrier toTransfer{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toTransfer.srcAccessMask = VK_ACCESS_SHADER_READ_BIT; + toTransfer.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + toTransfer.oldLayout = VK_IMAGE_LAYOUT_GENERAL; + toTransfer.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + toTransfer.srcQueueFamilyIndex = toTransfer.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toTransfer.image = oldResource.image; + toTransfer.subresourceRange = {oldResource.aspect, 0, preservedMipLevels, 0, VK_REMAINING_ARRAY_LAYERS}; + vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, + 0, nullptr, 0, nullptr, 1, &toTransfer); + } + Vector copyRegions; copyRegions.reserve(preservedMipLevels); - for (Uint32 level = 0; level < preservedMipLevels; ++level) { + for (Uint32 level = firstTransferLevel; level < preservedMipLevels; ++level) { VkImageCopy copy{}; copy.srcSubresource.aspectMask = oldResource.aspect; copy.srcSubresource.mipLevel = level; @@ -346,10 +402,11 @@ namespace MobileGL::MG_Backend::DirectVulkan { } EndActiveRenderPassOn(commandBuffer); - vkCmdCopyImage(commandBuffer, - oldResource.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - newResource.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - static_cast(copyRegions.size()), copyRegions.data()); + if (!copyRegions.empty()) + vkCmdCopyImage(commandBuffer, + oldResource.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + newResource.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + static_cast(copyRegions.size()), copyRegions.data()); VkPipelineStageFlags dstStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; VkAccessFlags dstAccessMask = 0; @@ -373,6 +430,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { VK_VERIFY(vkCreateFence(device, &fenceInfo, nullptr, &fence), "vkCreateFence(texture preserve)"); VK_VERIFY(vkQueueSubmit(graphicsQueue, 1, &submitInfo, fence), "vkQueueSubmit(texture preserve)"); VK_VERIFY(vkWaitForFences(device, 1, &fence, VK_TRUE, UINT64_MAX), "vkWaitForFences(texture preserve)"); + for (WireE5RawCopyObjects& objects : rawObjects) DestroyWireE5RawCopyObjects(device, objects); vkDestroyFence(device, fence, nullptr); vkFreeCommandBuffers(device, commandPool, 1, &commandBuffer); @@ -486,6 +544,8 @@ namespace MobileGL::MG_Backend::DirectVulkan { m_deferredReleases.resize(initInfo.frameCount); m_deferredViewReleases.clear(); m_deferredViewReleases.resize(initInfo.frameCount); + m_deferredE5Releases.clear(); + m_deferredE5Releases.resize(initInfo.frameCount); MOBILEGL_ASSERT(m_device != VK_NULL_HANDLE && m_physicalDevice != VK_NULL_HANDLE && m_allocator != nullptr && m_commandPool != VK_NULL_HANDLE && m_graphicsQueue != VK_NULL_HANDLE, @@ -1252,7 +1312,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { if (preserve) { FlushPendingUploads(); if (!PreserveTextureContentsOnRecreate(m_device, m_commandPool, m_graphicsQueue, - resource, replacement, true)) { + resource, replacement, true, this)) { MGL_WIRE_DECLINE_AT(ShapePreserveCopyFailed, "texture {slot=%u, gen=%u}: its texels could not be copied into the " "reallocated image", @@ -1953,6 +2013,45 @@ namespace MobileGL::MG_Backend::DirectVulkan { static_cast(target), static_cast(mipLevel), true); } + Bool VkTextureManager::WireE5RawReadApplies(const TextureResource& resource) const { + return m_wireE5RawCopyKit != nullptr && resource.format == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 && + resource.sampleCount == VK_SAMPLE_COUNT_1_BIT && + (resource.imageCreateFlags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) != 0 && + resource.viewType != VK_IMAGE_VIEW_TYPE_1D && resource.viewType != VK_IMAGE_VIEW_TYPE_1D_ARRAY; + } + + Bool VkTextureManager::RecordWireE5RawTwin(VkCommandBuffer commandBuffer, const WireE5RawCopyRegion& region, + VkImageLayout sourceLayout, TextureResource& twin, + WireE5RawCopyObjects& objects) { + if (m_wireE5RawCopyKit == nullptr) return false; + const VkImageCreateInfo info = WireE5RawCopyTwinInfo(region); + VmaAllocationCreateInfo allocation{}; + allocation.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE; + allocation.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + if (vmaCreateImage(m_allocator, &info, &allocation, &twin.image, &twin.allocation, nullptr) != VK_SUCCESS) { + MGLOG_E_ONCE("RecordWireE5RawTwin: vmaCreateImage failed"); + return false; + } + twin.format = info.format; + twin.extent = {info.extent.width, info.extent.height}; + twin.depth = info.extent.depth; + twin.arrayLayers = info.arrayLayers; + twin.mipLevels = 1; + twin.aspect = VK_IMAGE_ASPECT_COLOR_BIT; + twin.viewType = region.volume ? VK_IMAGE_VIEW_TYPE_3D : VK_IMAGE_VIEW_TYPE_2D_ARRAY; + EndActiveRenderPassOn(commandBuffer); + const VkResult result = RecordWireE5RawCopy(m_device, commandBuffer, *m_wireE5RawCopyKit, region, + sourceLayout, twin.image, objects); + if (result != VK_SUCCESS) { + MGLOG_E_ONCE("RecordWireE5RawTwin: recording the raw copy failed (%d)", static_cast(result)); + DestroyWireE5RawCopyObjects(m_device, objects); + twin.Reset(); + return false; + } + twin.layout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + return true; + } + Bool VkTextureManager::GrowWireTextureMipChain(MG_Pipe::MGPipeHandle handle, Uint32 requiredMipLevels, VkCommandBuffer commandBuffer) { if (MG_Pipe::MGPipeHandleIsNull(handle)) { @@ -2006,13 +2105,18 @@ namespace MobileGL::MG_Backend::DirectVulkan { const VkImageAspectFlags aspect = old.aspect; const VkImageLayout oldLayout = old.layout; + // E5B9G9R9 on a device whose transfer reads canonicalize it: every level leaves through the + // raw route (WireE5RawCopy.h), so the old image is read by a shader, from GENERAL. + const Bool rawE5 = WireE5RawReadApplies(old); VkPipelineStageFlags srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; VkAccessFlags srcAccessMask = 0; GetImageTransitionSourceState(oldLayout, srcStageMask, srcAccessMask); Bool ok = TransitionImageLayout(commandBuffer, old.image, old.layout, - VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, srcStageMask, - VK_PIPELINE_STAGE_TRANSFER_BIT, srcAccessMask, - VK_ACCESS_TRANSFER_READ_BIT, aspect, 0, old.mipLevels); + rawE5 ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + srcStageMask, + rawE5 ? VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT : VK_PIPELINE_STAGE_TRANSFER_BIT, + srcAccessMask, rawE5 ? VK_ACCESS_SHADER_READ_BIT : VK_ACCESS_TRANSFER_READ_BIT, + aspect, 0, old.mipLevels); MOBILEGL_ASSERT(ok, "GrowWireTextureMipChain: source transition failed"); VkImageLayout grownLayout = VK_IMAGE_LAYOUT_UNDEFINED; ok = TransitionImageLayout(commandBuffer, grown.image, grownLayout, @@ -2029,6 +2133,44 @@ namespace MobileGL::MG_Backend::DirectVulkan { copy.dstOffset = {0, 0, 0}; copy.extent = {std::max(old.extent.width >> level, 1u), std::max(old.extent.height >> level, 1u), std::max(old.depth >> level, 1u)}; + if (rawE5) { + const Bool volume = old.viewType == VK_IMAGE_VIEW_TYPE_3D; + WireE5RawCopyRegion region; + region.source = old.image; + region.volume = volume; + region.level = level; + region.layerCount = volume ? 1u : old.arrayLayers; + region.extent = copy.extent; + TextureResource twin; + WireE5RawCopyObjects objects; + if (!RecordWireE5RawTwin(commandBuffer, region, VK_IMAGE_LAYOUT_GENERAL, twin, objects)) { + // Nothing was recorded for this level; the plain copy below reads the + // canonicalized words, which is what every level got before the route. + MGLOG_W_ONCE("GrowWireTextureMipChain: the raw E5B9G9R9 route failed; level %u is copied " + "by transfer", level); + VkImageMemoryBarrier toTransfer{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toTransfer.srcAccessMask = VK_ACCESS_SHADER_READ_BIT; + toTransfer.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + toTransfer.oldLayout = toTransfer.newLayout = VK_IMAGE_LAYOUT_GENERAL; + toTransfer.srcQueueFamilyIndex = toTransfer.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toTransfer.image = old.image; + toTransfer.subresourceRange = {aspect, level, 1, 0, VK_REMAINING_ARRAY_LAYERS}; + vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toTransfer); + EndActiveRenderPassOn(commandBuffer); + vkCmdCopyImage(commandBuffer, old.image, VK_IMAGE_LAYOUT_GENERAL, grown.image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©); + continue; + } + VkImageCopy twinCopy = copy; + twinCopy.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, region.layerCount}; + vkCmdCopyImage(commandBuffer, twin.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, grown.image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &twinCopy); + // Both name the frame's command buffer, so they ride the old image's ring slot. + m_deferredE5Releases[m_currentFrameIndex].push_back(objects); + DeferResourceRelease(Move(twin)); + continue; + } for (const VkImageAspectFlags copyAspect : {VK_IMAGE_ASPECT_COLOR_BIT, VK_IMAGE_ASPECT_DEPTH_BIT, VK_IMAGE_ASPECT_STENCIL_BIT}) { @@ -2107,6 +2249,11 @@ namespace MobileGL::MG_Backend::DirectVulkan { MOBILEGL_ASSERT(frameIndex < m_deferredReleases.size(), "VkTextureManager::CollectDeferredReleases invalid frame index %u (size=%zu)", frameIndex, m_deferredReleases.size()); + if (frameIndex < m_deferredE5Releases.size()) { + for (WireE5RawCopyObjects& objects : m_deferredE5Releases[frameIndex]) + DestroyWireE5RawCopyObjects(m_device, objects); + m_deferredE5Releases[frameIndex].clear(); + } m_deferredReleases[frameIndex].clear(); MOBILEGL_ASSERT(frameIndex < m_deferredViewReleases.size(), @@ -2348,6 +2495,10 @@ namespace MobileGL::MG_Backend::DirectVulkan { } void VkTextureManager::DestroyDeferredReleases() { + for (auto& deferredE5 : m_deferredE5Releases) { + for (WireE5RawCopyObjects& objects : deferredE5) DestroyWireE5RawCopyObjects(m_device, objects); + } + m_deferredE5Releases.clear(); for (auto& deferredReleases : m_deferredReleases) { deferredReleases.clear(); } diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.h index 5aff014d..92661f6f 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.h @@ -22,6 +22,7 @@ // The handle-keyed arm's key type. Header-only POD, disagg-only, exactly the shape // MagmaPipeArms.h already gives this directory. #include +#include "WireE5RawCopy.h" namespace MobileGL::MG_State::GLState { class ITextureObject; @@ -575,6 +576,18 @@ public: // stays as it was). Bool GrowWireTextureMipChain(MG_Pipe::MGPipeHandle handle, Uint32 requiredMipLevels, VkCommandBuffer commandBuffer); + // THE RAW E5B9G9R9 READ ROUTE (WireE5RawCopy.h). The renderer hands its kit over once + // ArmWireE5RawReads has armed the route; nullptr keeps every transfer read plain. + void SetWireE5RawCopyKit(const WireE5RawCopyKit* kit) { m_wireE5RawCopyKit = kit; } + // True when a transfer read of `resource` must go through the route: armed, and a + // single-sample E5B9G9R9 MUTABLE_FORMAT image that is not 1D. + Bool WireE5RawReadApplies(const TextureResource& resource) const; + // Records `region` of such an image (in GENERAL or SHADER_READ_ONLY_OPTIMAL) into a fresh + // R32_UINT twin, left in TRANSFER_SRC_OPTIMAL. On success `twin` owns the image and `objects` + // the views and the pool, and both must outlive the recording's submission. On failure + // nothing was recorded and everything created was destroyed. + Bool RecordWireE5RawTwin(VkCommandBuffer commandBuffer, const WireE5RawCopyRegion& region, + VkImageLayout sourceLayout, TextureResource& twin, WireE5RawCopyObjects& objects); // Records that GPU writes superseded the staged CPU snapshot of this level. Layer // ranges select cube faces; array and 3D levels share one staged level shadow. void MarkWireTextureGpuWritten(MG_Pipe::MGPipeHandle handle, Uint32 mipLevel, @@ -709,6 +722,7 @@ private: VkCommandPool m_uploadCommandPool = VK_NULL_HANDLE; VkQueue m_graphicsQueue = VK_NULL_HANDLE; Bool m_imageFormatListSupported = false; + const WireE5RawCopyKit* m_wireE5RawCopyKit = nullptr; Uint32 m_currentFrameIndex = 0; Uint8 m_gcCounter = 0; @@ -741,6 +755,8 @@ private: std::unordered_map m_multisampleCountsByFormat; Vector> m_deferredReleases; Vector> m_deferredViewReleases; + // The raw E5B9G9R9 route's views and descriptor pools, on the same ring as the images they name. + Vector> m_deferredE5Releases; // --- Batched upload machinery --- // Uploads within a frame are recorded into ONE shared command buffer and diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp index eb2a4abd..010e7371 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp @@ -51,6 +51,7 @@ #include "WireDeclineTally.h" #include "WireColorBlitFilter.h" #include "WireDepthResolveArm.h" +#include "WireCopySliceProbe.h" #include "WireDepthResolveProbe.h" #if MOBILEGL_BUILD_DISAGGREGATED #include @@ -2284,6 +2285,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { m_sampledReadStageMask, static_cast(m_physicalDevice.queueFamilies.graphicsFamily)}); MOBILEGL_ASSERT(succeeded, "VkTextureManager initialization failed."); + m_textureManager->SetWireE5RawCopyKit(GetWireE5RawCopyKit()); m_renderPassManager = MakeUnique(m_device, m_physicalDevice.handle, m_allocator, m_config, *m_textureManager, m_swapchainObject); @@ -2443,6 +2445,9 @@ namespace MobileGL::MG_Backend::DirectVulkan { #endif DestroyWireDepthMipmapResources(); DestroyWireMultisampleResolveResources(); + if (m_textureManager) m_textureManager->SetWireE5RawCopyKit(nullptr); + DestroyWireE5RawCopyKit(m_device, m_wireE5RawCopyKit); + m_wireE5RawReads = false; #if MOBILEGL_BUILD_DISAGGREGATED DestroySharedImagePresentTargets(false); DestroySharedImageSync(); @@ -3997,6 +4002,9 @@ namespace MobileGL::MG_Backend::DirectVulkan { // formats whenever they differ (VUID-vkCmdCopyImage-srcImage-01548) and there is no // downstream check - a mismatched pair is a promise the driver takes at face value. VkFormat format = VK_FORMAT_UNDEFINED; + // A transfer read of this image canonicalizes E5B9G9R9 words: the copy reads it through + // the raw route (WireE5RawCopy.h) instead. + Bool rawE5 = false; }; Bool TryResolveCopyImageSliceMapping(TextureTarget target, const CopyImageVkImage& image, Uint32 mipLevel, @@ -4102,6 +4110,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { out.depth = resource->depth; out.arrayLayers = resource->arrayLayers; out.format = resource->format; + out.rawE5 = !renderbuffer && m_textureManager->WireE5RawReadApplies(*resource); return out.image != VK_NULL_HANDLE; }; CopyImageVkImage srcImage{}; @@ -4346,8 +4355,9 @@ namespace MobileGL::MG_Backend::DirectVulkan { // of vkCmdCopyImage accept at once, the barrier covers the WHOLE image (both the source // level and the destination level are in it), and the access mask carries both directions // because this one barrier is the edge for both. + // An E5B9G9R9 source the raw route reads is read by a shader, which GENERAL also allows. const VkImageLayout copySourceLayout = - inPlace ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + (inPlace || srcImage.rawE5) ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; const VkImageLayout copyDestinationLayout = inPlace ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; const VkAccessFlags copySourceAccess = @@ -4426,10 +4436,58 @@ namespace MobileGL::MG_Backend::DirectVulkan { copyRegion.srcOffset.z, MG_Util::ConvertTextureTargetToString(dstTextureTarget).c_str(), dstMipLevel, copyRegion.dstSubresource.baseArrayLayer, copyRegion.dstSubresource.layerCount, copyRegion.dstOffset.z, srcWidth, srcHeight, copyRegion.extent.depth); - vkCmdCopyImage(frame.commandBuffer, - srcImage.image, copySourceLayout, - dstImage.image, copyDestinationLayout, - 1, ©Region); + VkImage copySourceImage = srcImage.image; + VkImageLayout copySourceImageLayout = copySourceLayout; + if (srcImage.rawE5) { + // The words leave through a shader into a twin, and the twin is what is copied. A + // twin that cannot be made leaves the plain copy (canonicalized words) in place. + WireE5RawCopyRegion region; + region.source = srcImage.image; + region.volume = srcSlices.slicesAreDepth; + region.level = srcMipLevel; + region.baseLayer = copyRegion.srcSubresource.baseArrayLayer; + region.layerCount = copyRegion.srcSubresource.layerCount; + region.offset = copyRegion.srcOffset; + region.extent = {copyRegion.extent.width, copyRegion.extent.height, + srcSlices.slicesAreDepth ? copySliceCount : 1u}; + auto twin = MakeUnique(); + WireE5RawCopyObjects objects; + if (m_textureManager->RecordWireE5RawTwin(frame.commandBuffer, region, copySourceLayout, *twin, objects)) { + copySourceImage = twin->image; + copySourceImageLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + copyRegion.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, region.layerCount}; + copyRegion.srcOffset = {0, 0, 0}; + WireRetiredObjects retired{}; + retired.textures.push_back(Move(twin)); + retired.imageViews = {objects.sourceView, objects.twinView}; + retired.descriptorPools = {objects.descriptorPool}; + RetireWireObjects(Move(retired)); + } + } + // Cross-format copies between a layered image and a 3D one land only their first slice on + // some drivers (WireCopySliceProbe.h); there, the copy goes one slice per region. + const VkFormat copySourceFormat = copySourceImage == srcImage.image ? srcImage.format : VK_FORMAT_R32_UINT; + if (m_wireSplitCrossFormatSliceCopies && copySourceFormat != dstImage.format && + srcSlices.slicesAreDepth != dstSlices.slicesAreDepth && copySliceCount > 1) { + Vector slices(copySliceCount, copyRegion); + for (Uint32 slice = 0; slice < copySliceCount; ++slice) { + VkImageCopy& one = slices[slice]; + one.extent.depth = 1; + if (srcSlices.slicesAreDepth) one.srcOffset.z += static_cast(slice); + else one.srcSubresource = {copyRegion.srcSubresource.aspectMask, copyRegion.srcSubresource.mipLevel, + copyRegion.srcSubresource.baseArrayLayer + slice, 1}; + if (dstSlices.slicesAreDepth) one.dstOffset.z += static_cast(slice); + else one.dstSubresource = {copyRegion.dstSubresource.aspectMask, copyRegion.dstSubresource.mipLevel, + copyRegion.dstSubresource.baseArrayLayer + slice, 1}; + } + vkCmdCopyImage(frame.commandBuffer, copySourceImage, copySourceImageLayout, dstImage.image, + copyDestinationLayout, copySliceCount, slices.data()); + } else { + vkCmdCopyImage(frame.commandBuffer, + copySourceImage, copySourceImageLayout, + dstImage.image, copyDestinationLayout, + 1, ©Region); + } // The copy coordinates above are already storage-relative. Name that // storage here too, so a texture view's offsets are not applied twice. if (wire && !dstImage.isRenderbuffer) m_textureManager->MarkWireTextureGpuWritten(dstStorageHandle, @@ -8467,6 +8525,9 @@ namespace MobileGL::MG_Backend::DirectVulkan { ArmPrimGenReroute(); // Same terms: it records on m_graphicsQueue, which nothing but the probe above has used. ArmWireDepthResolveOrder(); + // Same terms again. + ArmWireE5RawReads(); + ArmWireCopySliceSplit(); } // P7 gate 5 (g5-msprobe): THE ARM ORDER OF THE MULTISAMPLE DEPTH/STENCIL RESOLVE IS A MEASURED @@ -8548,6 +8609,106 @@ namespace MobileGL::MG_Backend::DirectVulkan { m_wirePreferShaderDepthResolve = resolved.preferShader; } + // THE RAW E5B9G9R9 READ ROUTE IS ARMED BY MEASUREMENT (WireE5RawCopy.h): Auto runs + // RunWireE5TransferProbe once per device identity and arms the route only where a plain transfer + // read changed words a shader read back exactly; MOBILEGL_MAGMA_E5_RAW_READS=1/0 forces it. + void VulkanRenderer::ArmWireE5RawReads() { + m_wireE5RawReads = false; + const MG_Config::QuirkOverride setting = MG_Config::Features.MagmaE5RawReads; + if (setting == MG_Config::QuirkOverride::ForceOff) { + MGLOG_I("DirectVulkan: E5B9G9R9 raw reads off (MOBILEGL_MAGMA_E5_RAW_READS)"); + return; + } + if (!WireE5RawCopySupported(m_physicalDevice.handle)) { + MGLOG_I("DirectVulkan: E5B9G9R9 raw reads unavailable (R32_UINT is not sampled + storage here)"); + return; + } + Bool armed = setting == MG_Config::QuirkOverride::ForceOn; + if (setting == MG_Config::QuirkOverride::Auto) { + struct Entry { + WireDepthResolveDeviceIdentity identity; + WireE5TransferProbeVerdict verdict; + }; + static std::mutex s_mutex; + static Vector s_entries; + const WireDepthResolveDeviceIdentity identity = MakeWireDepthResolveDeviceIdentity( + m_physicalDevice.properties, false, WireDepthResolveProbeKnob::Measure); + std::lock_guard lock(s_mutex); + const Entry* found = nullptr; + for (const Entry& entry : s_entries) + if (entry.identity == identity) found = &entry; + if (found == nullptr) { + WireE5TransferProbeContext context; + context.physicalDevice = m_physicalDevice.handle; + context.device = m_device; + context.queue = m_graphicsQueue; + context.queueFamilyIndex = static_cast(m_physicalDevice.queueFamilies.graphicsFamily); + const WireE5TransferProbeMeasurement measurement = RunWireE5TransferProbe(context); + const WireE5TransferProbeVerdict verdict = EvaluateWireE5TransferProbe(measurement); + if (measurement.fenceWaitTimedOut) + MGLOG_W("DirectVulkan: the E5B9G9R9 transfer probe timed out waiting on its own submission; " + "its Vulkan objects are deliberately leaked"); + MGLOG_I("DirectVulkan: E5B9G9R9 transfer probe verdict=%s: %s", WireE5TransferProbeVerdictName(verdict), + DescribeWireE5TransferProbe(measurement).c_str()); + s_entries.push_back({identity, verdict}); + found = &s_entries.back(); + } + armed = found->verdict == WireE5TransferProbeVerdict::TransferReadNormalizes; + } + if (armed) { + if (const VkResult result = CreateWireE5RawCopyKit(m_device, m_wireE5RawCopyKit); result != VK_SUCCESS) { + MGLOG_W("DirectVulkan: E5B9G9R9 raw route kit creation failed (%d); transfer reads stay plain", + static_cast(result)); + DestroyWireE5RawCopyKit(m_device, m_wireE5RawCopyKit); + armed = false; + } + } + m_wireE5RawReads = armed; + MGLOG_I("DirectVulkan: E5B9G9R9 reads %s", + armed ? "go through the raw shader route" : "use plain transfers"); + } + + // THE PER-SLICE SPLIT IS ARMED BY MEASUREMENT (WireCopySliceProbe.h), once per device identity, + // or forced by MOBILEGL_MAGMA_SPLIT_SLICE_COPIES. + void VulkanRenderer::ArmWireCopySliceSplit() { + const MG_Config::QuirkOverride setting = MG_Config::Features.MagmaSplitSliceCopies; + Bool armed = setting == MG_Config::QuirkOverride::ForceOn; + if (setting == MG_Config::QuirkOverride::Auto) { + struct Entry { + WireDepthResolveDeviceIdentity identity; + WireCopySliceProbeVerdict verdict; + }; + static std::mutex s_mutex; + static Vector s_entries; + const WireDepthResolveDeviceIdentity identity = MakeWireDepthResolveDeviceIdentity( + m_physicalDevice.properties, false, WireDepthResolveProbeKnob::Measure); + std::lock_guard lock(s_mutex); + const Entry* found = nullptr; + for (const Entry& entry : s_entries) + if (entry.identity == identity) found = &entry; + if (found == nullptr) { + WireCopySliceProbeContext context; + context.physicalDevice = m_physicalDevice.handle; + context.device = m_device; + context.queue = m_graphicsQueue; + context.queueFamilyIndex = static_cast(m_physicalDevice.queueFamilies.graphicsFamily); + const WireCopySliceProbeMeasurement measurement = RunWireCopySliceProbe(context); + const WireCopySliceProbeVerdict verdict = EvaluateWireCopySliceProbe(measurement); + if (measurement.fenceWaitTimedOut) + MGLOG_W("DirectVulkan: the slice copy probe timed out waiting on its own submission; its Vulkan " + "objects are deliberately leaked"); + MGLOG_I("DirectVulkan: slice copy probe verdict=%s: %s", WireCopySliceProbeVerdictName(verdict), + DescribeWireCopySliceProbe(measurement).c_str()); + s_entries.push_back({identity, verdict}); + found = &s_entries.back(); + } + armed = found->verdict == WireCopySliceProbeVerdict::CrossFormatSlicesDropped; + } + m_wireSplitCrossFormatSliceCopies = armed; + MGLOG_I("DirectVulkan: cross-format layered <-> 3D copies %s", + armed ? "are split one slice per region" : "go as one region"); + } + void VulkanRenderer::ArmPrimGenReroute() { using namespace MG_Util::SelfTest; m_primGenRerouteKind = PrimGenRerouteKind::None; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h index c52f4632..b93cb9ab 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h @@ -24,6 +24,7 @@ #include "VkTimerQueryManager.h" #include "GpuProgressMarkers.h" #include "WireRenderPassCompatibility.h" +#include "WireE5RawCopy.h" #if MOBILEGL_BUILD_DISAGGREGATED #include #endif @@ -787,6 +788,24 @@ namespace MobileGL::MG_Backend::DirectVulkan { // Runs the probe (memoized per device identity) and sets the member above. Called at the end of // device creation, after ArmPrimGenReroute: it records on m_graphicsQueue. void ArmWireDepthResolveOrder(); + // Adreno's E5B9G9R9 transfer-read canonicalization (WireE5RawCopy.h): when armed, every + // transfer read of an E5B9G9R9 texture (readback, glCopyImageSubData, a storage + // recreate's level copy) first copies the words out through the raw route's compute pass. + // Decided by ArmWireE5RawReads at bring-up (probe memoized per device identity, or + // MOBILEGL_MAGMA_E5_RAW_READS). + Bool m_wireE5RawReads = false; + WireE5RawCopyKit m_wireE5RawCopyKit; + // Cross-format 2D-array <-> 3D vkCmdCopyImage lands only its first slice (WireCopySliceProbe.h): + // CopyImageSubData then records one region per slice. Set by ArmWireCopySliceSplit at bring-up. + Bool m_wireSplitCrossFormatSliceCopies = false; + void ArmWireCopySliceSplit(); + void ArmWireE5RawReads(); + public: + // The kit when the route is armed, else nullptr. VkTextureManager asks before a level copy. + const WireE5RawCopyKit* GetWireE5RawCopyKit() const { + return m_wireE5RawReads ? &m_wireE5RawCopyKit : nullptr; + } + private: Bool BlitWireColorToDefault(WireImage source, WireImage destination, GLint sx0, GLint sy0, GLint sx1, GLint sy1, GLint dx0, GLint dy0, GLint dx1, GLint dy1, GLenum filter); diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireCopySliceProbe.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireCopySliceProbe.cpp new file mode 100644 index 00000000..806f482a --- /dev/null +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireCopySliceProbe.cpp @@ -0,0 +1,356 @@ +// MobileGL - MobileGL/MG_Backend/DirectVulkan/Renderer/WireCopySliceProbe.cpp +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header +// +// See WireCopySliceProbe.h. Plumbing as in WireDepthResolveProbe.cpp: one throwaway command +// buffer, a bounded fence wait that leaks rather than idle-waits a hung GPU. + +#include "WireCopySliceProbe.h" + +#include + +namespace MobileGL::MG_Backend::DirectVulkan { + String DescribeWireCopySliceProbe(const WireCopySliceProbeMeasurement& measurement) { + if (!measurement.ran) + return format("not run ({})", measurement.failureReason.empty() ? "?" : measurement.failureReason); + return format("one-region 2D-array A2B10G10R10_UINT -> 3D A2B10G10R10_UNORM copy: {}/{} texels wrong, " + "{}/{} slices exact; per-slice regions: {} wrong", + measurement.subjectMismatches, measurement.texels, measurement.subjectExactSlices, + measurement.slices, measurement.controlMismatches); + } + + namespace { + constexpr Uint32 kExtent = 7; + constexpr Uint32 kSlices = 8; + constexpr Uint32 kSliceTexels = kExtent * kExtent; + constexpr Uint32 kTexels = kSliceTexels * kSlices; + // The destination is larger than the region, as in the copy_image cases that drop slices. + constexpr Uint32 kDestinationExtent = 30; + constexpr Uint32 kDestinationDepth = 24; + constexpr Uint32 kDestinationTexels = kDestinationExtent * kDestinationExtent * kDestinationDepth; + constexpr VkFormat kSourceFormat = VK_FORMAT_A2B10G10R10_UINT_PACK32; + constexpr VkFormat kDestinationFormat = VK_FORMAT_A2B10G10R10_UNORM_PACK32; + constexpr Uint64 kFenceTimeoutNs = 5ull * 1000ull * 1000ull * 1000ull; + + Uint32 Word(Uint32 index, Uint32 salt) { + Uint32 state = (index + salt * 0x10001u) * 2654435761u + 0x9E3779B9u; + state ^= state >> 15; + state *= 2246822519u; + state ^= state >> 13; + return state; + } + + struct ProbeObjects { + VkDevice device = VK_NULL_HANDLE; + Vector images; + Vector buffers; + Vector memories; + VkCommandPool commandPool = VK_NULL_HANDLE; + VkFence fence = VK_NULL_HANDLE; + + void Destroy() { + for (VkImage image : images) vkDestroyImage(device, image, nullptr); + for (VkBuffer buffer : buffers) vkDestroyBuffer(device, buffer, nullptr); + for (VkDeviceMemory memory : memories) vkFreeMemory(device, memory, nullptr); + vkDestroyCommandPool(device, commandPool, nullptr); + vkDestroyFence(device, fence, nullptr); + } + }; + + Optional PickMemoryType(const VkPhysicalDeviceMemoryProperties& properties, Uint32 allowed, + VkMemoryPropertyFlags wanted, VkMemoryPropertyFlags required) { + Optional fallback; + for (Uint32 i = 0; i < properties.memoryTypeCount; ++i) { + if (!(allowed & (1u << i))) continue; + const VkMemoryPropertyFlags flags = properties.memoryTypes[i].propertyFlags; + if (flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) continue; + if ((flags & required) != required) continue; + if ((flags & wanted) == wanted) return i; + if (!fallback) fallback = i; + } + return fallback; + } + + void ImageBarrier(VkCommandBuffer commandBuffer, VkImage image, VkImageLayout oldLayout, + VkImageLayout newLayout, VkAccessFlags srcAccess, VkAccessFlags dstAccess) { + VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + barrier.srcAccessMask = srcAccess; + barrier.dstAccessMask = dstAccess; + barrier.oldLayout = oldLayout; + barrier.newLayout = newLayout; + barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = image; + barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, VK_REMAINING_ARRAY_LAYERS}; + vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); + } + } // namespace + + WireCopySliceProbeMeasurement RunWireCopySliceProbe(const WireCopySliceProbeContext& context) { + WireCopySliceProbeMeasurement measurement; + const VkDevice device = context.device; + if (device == VK_NULL_HANDLE || context.queue == VK_NULL_HANDLE) { + measurement.failureReason = "no device or queue"; + return measurement; + } + constexpr VkFormatFeatureFlags kRequired = VK_FORMAT_FEATURE_TRANSFER_SRC_BIT | VK_FORMAT_FEATURE_TRANSFER_DST_BIT; + for (const VkFormat format : {kSourceFormat, kDestinationFormat}) { + VkFormatProperties properties{}; + vkGetPhysicalDeviceFormatProperties(context.physicalDevice, format, &properties); + if ((properties.optimalTilingFeatures & kRequired) != kRequired) { + measurement.failureReason = "A2B10G10R10 UINT/UNORM are not transfer formats here"; + return measurement; + } + } + + ProbeObjects objects; + objects.device = device; + Bool leak = false; + struct Teardown { + ProbeObjects& objects; + Bool& leak; + ~Teardown() { + if (!leak) objects.Destroy(); + } + } teardown{objects, leak}; + const auto fail = [&](String reason) { + measurement.ran = false; + measurement.failureReason = Move(reason); + return measurement; + }; + + VkPhysicalDeviceMemoryProperties memoryProperties{}; + vkGetPhysicalDeviceMemoryProperties(context.physicalDevice, &memoryProperties); + // The wire's images (VkTextureManager): MUTABLE_FORMAT (a storage-compatible colour class), + // SAMPLED | TRANSFER_SRC | TRANSFER_DST | COLOR_ATTACHMENT, a 3D one 2D_ARRAY_COMPATIBLE. + const auto makeImage = [&](Bool volume, VkFormat format, VkImage& out) -> Bool { + VkFormatProperties formatProperties{}; + vkGetPhysicalDeviceFormatProperties(context.physicalDevice, format, &formatProperties); + const Bool attachable = + (formatProperties.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT) != 0; + VkImageCreateInfo info{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + info.flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | (volume ? VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT : 0); + info.imageType = volume ? VK_IMAGE_TYPE_3D : VK_IMAGE_TYPE_2D; + info.format = format; + info.extent = volume ? VkExtent3D{kDestinationExtent, kDestinationExtent, kDestinationDepth} + : VkExtent3D{kExtent, kExtent, 1}; + info.mipLevels = 1; + info.arrayLayers = volume ? 1u : kSlices; + info.samples = VK_SAMPLE_COUNT_1_BIT; + info.tiling = VK_IMAGE_TILING_OPTIMAL; + info.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | + (attachable ? VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT : 0u); + info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + if (vkCreateImage(device, &info, nullptr, &out) != VK_SUCCESS) return false; + objects.images.push_back(out); + VkMemoryRequirements requirements{}; + vkGetImageMemoryRequirements(device, out, &requirements); + const Optional type = PickMemoryType(memoryProperties, requirements.memoryTypeBits, + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, 0); + if (!type) return false; + VkMemoryAllocateInfo allocate{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocate.allocationSize = requirements.size; + allocate.memoryTypeIndex = *type; + VkDeviceMemory memory = VK_NULL_HANDLE; + if (vkAllocateMemory(device, &allocate, nullptr, &memory) != VK_SUCCESS) return false; + objects.memories.push_back(memory); + return vkBindImageMemory(device, out, memory, 0) == VK_SUCCESS; + }; + struct HostBuffer { + VkBuffer buffer = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + Bool coherent = false; + }; + const auto makeBuffer = [&](VkDeviceSize bytes, VkBufferUsageFlags usage, HostBuffer& out) -> Bool { + VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + info.size = bytes; + info.usage = usage; + info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + if (vkCreateBuffer(device, &info, nullptr, &out.buffer) != VK_SUCCESS) return false; + objects.buffers.push_back(out.buffer); + VkMemoryRequirements requirements{}; + vkGetBufferMemoryRequirements(device, out.buffer, &requirements); + const Optional type = + PickMemoryType(memoryProperties, requirements.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT); + if (!type) return false; + out.coherent = + (memoryProperties.memoryTypes[*type].propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0; + VkMemoryAllocateInfo allocate{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocate.allocationSize = requirements.size; + allocate.memoryTypeIndex = *type; + if (vkAllocateMemory(device, &allocate, nullptr, &out.memory) != VK_SUCCESS) return false; + objects.memories.push_back(out.memory); + return vkBindBufferMemory(device, out.buffer, out.memory, 0) == VK_SUCCESS; + }; + const auto access = [&](const HostBuffer& buffer, Bool write, auto&& body) -> Bool { + void* mapped = nullptr; + if (vkMapMemory(device, buffer.memory, 0, VK_WHOLE_SIZE, 0, &mapped) != VK_SUCCESS || mapped == nullptr) + return false; + VkMappedMemoryRange range{VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE}; + range.memory = buffer.memory; + range.size = VK_WHOLE_SIZE; + if (!write && !buffer.coherent) vkInvalidateMappedMemoryRanges(device, 1, &range); + body(static_cast(mapped)); + if (write && !buffer.coherent) vkFlushMappedMemoryRanges(device, 1, &range); + vkUnmapMemory(device, buffer.memory); + return true; + }; + + VkImage source = VK_NULL_HANDLE, subject = VK_NULL_HANDLE, control = VK_NULL_HANDLE; + if (!makeImage(false, kSourceFormat, source) || !makeImage(true, kDestinationFormat, subject) || + !makeImage(true, kDestinationFormat, control)) + return fail("image creation failed"); + HostBuffer sourceWords, sentinelWords, subjectOut, controlOut; + if (!makeBuffer(kTexels * 4, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, sourceWords) || + !makeBuffer(kDestinationTexels * 4, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, sentinelWords) || + !makeBuffer(kTexels * 4, VK_BUFFER_USAGE_TRANSFER_DST_BIT, subjectOut) || + !makeBuffer(kTexels * 4, VK_BUFFER_USAGE_TRANSFER_DST_BIT, controlOut)) + return fail("buffer creation failed"); + if (!access(sourceWords, true, [](Uint32* words) { for (Uint32 i = 0; i < kTexels; ++i) words[i] = Word(i, 1); }) || + !access(sentinelWords, true, [](Uint32* words) { for (Uint32 i = 0; i < kDestinationTexels; ++i) words[i] = Word(i, 2); })) + return fail("vkMapMemory (upload) failed"); + + VkCommandPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + poolInfo.flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT; + poolInfo.queueFamilyIndex = context.queueFamilyIndex; + if (vkCreateCommandPool(device, &poolInfo, nullptr, &objects.commandPool) != VK_SUCCESS) + return fail("vkCreateCommandPool failed"); + VkCommandBufferAllocateInfo commandInfo{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + commandInfo.commandPool = objects.commandPool; + commandInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + commandInfo.commandBufferCount = 1; + VkCommandBuffer cmd = VK_NULL_HANDLE; + if (vkAllocateCommandBuffers(device, &commandInfo, &cmd) != VK_SUCCESS) + return fail("vkAllocateCommandBuffers failed"); + VkCommandBufferBeginInfo beginInfo{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + if (vkBeginCommandBuffer(cmd, &beginInfo) != VK_SUCCESS) return fail("vkBeginCommandBuffer failed"); + + // (1) The source's words, and a sentinel in both destinations so an untouched slice shows. + for (const VkImage image : {source, subject, control}) + ImageBarrier(cmd, image, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 0, + VK_ACCESS_TRANSFER_WRITE_BIT); + VkBufferImageCopy layered{}; + layered.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, kSlices}; + layered.imageExtent = {kExtent, kExtent, 1}; + VkBufferImageCopy volume{}; + volume.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + volume.imageExtent = {kExtent, kExtent, kSlices}; + VkBufferImageCopy wholeVolume = volume; + wholeVolume.imageExtent = {kDestinationExtent, kDestinationExtent, kDestinationDepth}; + vkCmdCopyBufferToImage(cmd, sourceWords.buffer, source, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &layered); + vkCmdCopyBufferToImage(cmd, sentinelWords.buffer, subject, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &wholeVolume); + vkCmdCopyBufferToImage(cmd, sentinelWords.buffer, control, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &wholeVolume); + ImageBarrier(cmd, source, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + for (const VkImage image : {subject, control}) + ImageBarrier(cmd, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + + // (2) SUBJECT: the one region CopyImageSubData records. CONTROL: one region per slice. + VkImageCopy whole{}; + whole.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, kSlices}; + whole.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + whole.extent = {kExtent, kExtent, kSlices}; + vkCmdCopyImage(cmd, source, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, subject, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + 1, &whole); + VkImageCopy perSlice[kSlices]{}; + for (Uint32 slice = 0; slice < kSlices; ++slice) { + perSlice[slice].srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, slice, 1}; + perSlice[slice].dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + perSlice[slice].dstOffset = {0, 0, static_cast(slice)}; + perSlice[slice].extent = {kExtent, kExtent, 1}; + } + vkCmdCopyImage(cmd, source, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, control, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + kSlices, perSlice); + + // (3) Both destinations out: same-format reads of a 3D image, which the defect does not touch. + for (const VkImage image : {subject, control}) + ImageBarrier(cmd, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + vkCmdCopyImageToBuffer(cmd, subject, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, subjectOut.buffer, 1, &volume); + vkCmdCopyImageToBuffer(cmd, control, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, controlOut.buffer, 1, &volume); + VkMemoryBarrier hostBarrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; + hostBarrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + hostBarrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT; + vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &hostBarrier, 0, + nullptr, 0, nullptr); + if (vkEndCommandBuffer(cmd) != VK_SUCCESS) return fail("vkEndCommandBuffer failed"); + + VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + if (vkCreateFence(device, &fenceInfo, nullptr, &objects.fence) != VK_SUCCESS) + return fail("vkCreateFence failed"); + VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO}; + submit.commandBufferCount = 1; + submit.pCommandBuffers = &cmd; + if (const VkResult result = vkQueueSubmit(context.queue, 1, &submit, objects.fence); result != VK_SUCCESS) + return fail(format("vkQueueSubmit failed (VkResult {})", static_cast(result))); + const VkResult waited = vkWaitForFences(device, 1, &objects.fence, VK_TRUE, kFenceTimeoutNs); + if (waited == VK_TIMEOUT) { + leak = true; + measurement.fenceWaitTimedOut = true; + return fail("the probe's fence did not signal within 5 s"); + } + if (waited != VK_SUCCESS) return fail(format("vkWaitForFences failed (VkResult {})", static_cast(waited))); + + const auto tally = [&](const Uint32* words, Uint32& mismatches, Uint32* exactSlices) { + for (Uint32 slice = 0; slice < kSlices; ++slice) { + Uint32 sliceMismatches = 0; + for (Uint32 i = slice * kSliceTexels; i < (slice + 1) * kSliceTexels; ++i) + if (words[i] != Word(i, 1)) ++sliceMismatches; + mismatches += sliceMismatches; + if (exactSlices && sliceMismatches == 0) ++*exactSlices; + } + }; + if (!access(subjectOut, false, [&](Uint32* words) { + tally(words, measurement.subjectMismatches, &measurement.subjectExactSlices); + }) || + !access(controlOut, false, [&](Uint32* words) { tally(words, measurement.controlMismatches, nullptr); })) + return fail("vkMapMemory (readback) failed"); + measurement.texels = kTexels; + measurement.slices = kSlices; + measurement.ran = true; + return measurement; + } + + WireCopySliceProbeMeasurement RunWireCopySliceProbeOnThrowawayDevice(PFN_vkGetInstanceProcAddr getInstanceProcAddr, + VkInstance instance, + VkPhysicalDevice physicalDevice, + Uint32 graphicsQueueFamilyIndex) { + WireCopySliceProbeMeasurement measurement; + const auto createDevice = + reinterpret_cast(getInstanceProcAddr(instance, "vkCreateDevice")); + if (createDevice == nullptr) { + measurement.failureReason = "no vkCreateDevice"; + return measurement; + } + const Float priority = 1.0f; + VkDeviceQueueCreateInfo queueInfo{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; + queueInfo.queueFamilyIndex = graphicsQueueFamilyIndex; + queueInfo.queueCount = 1; + queueInfo.pQueuePriorities = &priority; + VkDeviceCreateInfo deviceInfo{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; + deviceInfo.queueCreateInfoCount = 1; + deviceInfo.pQueueCreateInfos = &queueInfo; + VkDevice device = VK_NULL_HANDLE; + if (const VkResult result = createDevice(physicalDevice, &deviceInfo, nullptr, &device); + result != VK_SUCCESS || device == VK_NULL_HANDLE) { + measurement.failureReason = format("vkCreateDevice failed (VkResult {})", static_cast(result)); + return measurement; + } + WireCopySliceProbeContext context; + context.physicalDevice = physicalDevice; + context.device = device; + vkGetDeviceQueue(device, graphicsQueueFamilyIndex, 0, &context.queue); + context.queueFamilyIndex = graphicsQueueFamilyIndex; + measurement = RunWireCopySliceProbe(context); + if (!measurement.fenceWaitTimedOut) vkDestroyDevice(device, nullptr); + return measurement; + } +} // namespace MobileGL::MG_Backend::DirectVulkan diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireCopySliceProbe.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireCopySliceProbe.h new file mode 100644 index 00000000..b332ea9d --- /dev/null +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireCopySliceProbe.h @@ -0,0 +1,93 @@ +// MobileGL - MobileGL/MG_Backend/DirectVulkan/Renderer/WireCopySliceProbe.h +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header +#pragma once + +#include +#include + +#include + +// CROSS-FORMAT SLICE COPIES BETWEEN A LAYERED IMAGE AND A 3D ONE, and the probe that decides +// whether a device needs them split. +// +// glCopyImageSubData between a 2D array and a 3D texture maps layers onto depth slices (GL 4.6 +// core 18.3.2), and Vulkan expresses the whole move as ONE VkImageCopy: layerCount on the layered +// side, extent.depth on the 3D side (VUID-VkImageCopy-srcImage-01790/01791). Adreno 750 performs +// that region correctly when the two formats are the same, but when they differ (size-compatible, +// e.g. A2B10G10R10_UINT into A2B10G10R10_UNORM - KHR-GL4x.copy_image's cross-format cases) it +// lands only the FIRST slice and leaves the rest of the destination untouched. The same copy +// recorded as one single-slice region per slice is exact. The probe uses the wire's image shape +// (MUTABLE_FORMAT, attachment usage, a 30x30x24 destination): a non-mutable 7x7x8 pair without +// attachment usage copied cleanly on the same device. +// +// VulkanRenderer::CopyImageSubData therefore splits such a copy per slice where the probe below +// measured the defect (VulkanRenderer::ArmWireCopySliceSplit at bring-up, the POST's "Known Driver +// Bugs" section in MG_Util/SelfTest/DriverBugProbes.cpp); MOBILEGL_MAGMA_SPLIT_SLICE_COPIES forces it. +namespace MobileGL::MG_Backend::DirectVulkan { + struct WireCopySliceProbeMeasurement { + // Recorded, submitted and read back. + Bool ran = false; + // The bounded fence wait expired: every object was deliberately leaked. + Bool fenceWaitTimedOut = false; + String failureReason; + // SUBJECT: one region, A2B10G10R10_UINT 2D array -> A2B10G10R10_UNORM 3D, every slice. + Uint32 texels = 0; + Uint32 slices = 0; + Uint32 subjectMismatches = 0; + // The slices the subject got entirely right (the defect leaves exactly the first). + Uint32 subjectExactSlices = 0; + // CONTROL: the same copy recorded one slice per region. Must be exact, or nothing is concluded. + Uint32 controlMismatches = 0; + }; + + enum class WireCopySliceProbeVerdict : Uint8 { + NotRun, + Inconclusive, + Clean, + // The defect: the single region missed texels the per-slice regions moved exactly. + CrossFormatSlicesDropped, + }; + + inline const char* WireCopySliceProbeVerdictName(WireCopySliceProbeVerdict verdict) { + switch (verdict) { + case WireCopySliceProbeVerdict::NotRun: return "not-run"; + case WireCopySliceProbeVerdict::Inconclusive: return "inconclusive"; + case WireCopySliceProbeVerdict::Clean: return "clean"; + case WireCopySliceProbeVerdict::CrossFormatSlicesDropped: return "cross-format-slices-dropped"; + } + return "?"; + } + + inline WireCopySliceProbeVerdict EvaluateWireCopySliceProbe(const WireCopySliceProbeMeasurement& measurement) { + if (!measurement.ran || measurement.fenceWaitTimedOut || measurement.texels == 0) + return WireCopySliceProbeVerdict::NotRun; + if (measurement.controlMismatches != 0) return WireCopySliceProbeVerdict::Inconclusive; + return measurement.subjectMismatches != 0 ? WireCopySliceProbeVerdict::CrossFormatSlicesDropped + : WireCopySliceProbeVerdict::Clean; + } + + String DescribeWireCopySliceProbe(const WireCopySliceProbeMeasurement& measurement); + + struct WireCopySliceProbeContext { + VkPhysicalDevice physicalDevice = VK_NULL_HANDLE; + VkDevice device = VK_NULL_HANDLE; + // Must be otherwise idle for the probe's duration. + VkQueue queue = VK_NULL_HANDLE; + Uint32 queueFamilyIndex = 0; + }; + + // One submission on its own command pool, a bounded fence wait (on timeout every object is + // leaked and the measurement says so), teardown on every other path. + WireCopySliceProbeMeasurement RunWireCopySliceProbe(const WireCopySliceProbeContext& context); + + // The POST's entry: a throwaway device with one graphics queue. + WireCopySliceProbeMeasurement RunWireCopySliceProbeOnThrowawayDevice(PFN_vkGetInstanceProcAddr getInstanceProcAddr, + VkInstance instance, + VkPhysicalDevice physicalDevice, + Uint32 graphicsQueueFamilyIndex); +} // namespace MobileGL::MG_Backend::DirectVulkan diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.comp b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.comp new file mode 100644 index 00000000..b91ae457 --- /dev/null +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.comp @@ -0,0 +1,22 @@ +#version 450 +// Raw texel-block copy out of an E5B9G9R9 image (WireE5RawCopy.h). The source is read through an +// R32_UINT view, so its words are never decoded, and stored into an R32_UINT twin whose transfer +// reads move bits rather than values. Layered and plain 2D sources; WireE5RawCopy3D.comp is the +// volume twin of this shader. +layout(local_size_x = 8, local_size_y = 8, local_size_z = 1) in; +layout(set = 0, binding = 0) uniform usampler2DArray sourceImage; +layout(set = 0, binding = 1, r32ui) uniform writeonly uimage2DArray twinImage; +layout(push_constant) uniform Region { + // xy: the region's texel origin in the source level; z unused (the view starts at the + // region's first layer). + ivec4 sourceOffset; + // xyz: texels by layers to copy. + ivec4 extent; +} region; + +void main() { + ivec3 id = ivec3(gl_GlobalInvocationID); + if (any(greaterThanEqual(id, region.extent.xyz))) return; + uint word = texelFetch(sourceImage, ivec3(region.sourceOffset.xy + id.xy, id.z), 0).r; + imageStore(twinImage, id, uvec4(word, 0u, 0u, 0u)); +} diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.cpp new file mode 100644 index 00000000..cb2f212a --- /dev/null +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.cpp @@ -0,0 +1,588 @@ +// MobileGL - MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.cpp +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header +// +// See WireE5RawCopy.h. The probe's plumbing is WireDepthResolveProbe.cpp's: one throwaway command +// buffer, a bounded fence wait that leaks rather than idle-waits a hung GPU, teardown on every +// other path. + +#include "WireE5RawCopy.h" + +#include + +#include +#include + +namespace MobileGL::MG_Backend::DirectVulkan { +#include "WireE5RawCopySpirv.h" + + namespace { + // The shaders' push-constant block: ivec4 sourceOffset, ivec4 extent. + struct RegionPush { + Int32 sourceOffset[4]; + Int32 extent[4]; + }; + constexpr Uint32 kGroupSize = 8; + } // namespace + + VkResult CreateWireE5RawCopyKit(VkDevice device, WireE5RawCopyKit& kit) { + const auto shader = [&](const Uint32* code, SizeT bytes, VkShaderModule& module) { + VkShaderModuleCreateInfo info{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + info.codeSize = bytes; + info.pCode = code; + return vkCreateShaderModule(device, &info, nullptr, &module); + }; + VkResult result = shader(kWireE5RawCopySpirv, sizeof(kWireE5RawCopySpirv), kit.arrayModule); + if (result != VK_SUCCESS) return result; + result = shader(kWireE5RawCopy3DSpirv, sizeof(kWireE5RawCopy3DSpirv), kit.volumeModule); + if (result != VK_SUCCESS) return result; + const VkDescriptorSetLayoutBinding bindings[2]{ + {0, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr}, + {1, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr}, + }; + VkDescriptorSetLayoutCreateInfo layout{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + layout.bindingCount = 2; + layout.pBindings = bindings; + result = vkCreateDescriptorSetLayout(device, &layout, nullptr, &kit.descriptorLayout); + if (result != VK_SUCCESS) return result; + const VkPushConstantRange push{VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(RegionPush)}; + VkPipelineLayoutCreateInfo pipelineLayout{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + pipelineLayout.setLayoutCount = 1; + pipelineLayout.pSetLayouts = &kit.descriptorLayout; + pipelineLayout.pushConstantRangeCount = 1; + pipelineLayout.pPushConstantRanges = &push; + result = vkCreatePipelineLayout(device, &pipelineLayout, nullptr, &kit.pipelineLayout); + if (result != VK_SUCCESS) return result; + const auto pipeline = [&](VkShaderModule module, VkPipeline& out) { + VkComputePipelineCreateInfo info{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; + info.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + info.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + info.stage.module = module; + info.stage.pName = "main"; + info.layout = kit.pipelineLayout; + return vkCreateComputePipelines(device, VK_NULL_HANDLE, 1, &info, nullptr, &out); + }; + result = pipeline(kit.arrayModule, kit.arrayPipeline); + if (result != VK_SUCCESS) return result; + result = pipeline(kit.volumeModule, kit.volumePipeline); + if (result != VK_SUCCESS) return result; + // texelFetch ignores the filter; the sampler exists only because a GLSL usampler is a + // COMBINED image sampler. An integer format must not be linearly filtered anyway. + VkSamplerCreateInfo sampler{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; + sampler.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST; + sampler.addressModeU = sampler.addressModeV = sampler.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + sampler.magFilter = sampler.minFilter = VK_FILTER_NEAREST; + sampler.maxLod = 0.0f; + return vkCreateSampler(device, &sampler, nullptr, &kit.nearest); + } + + void DestroyWireE5RawCopyKit(VkDevice device, WireE5RawCopyKit& kit) { + vkDestroySampler(device, kit.nearest, nullptr); + vkDestroyPipeline(device, kit.arrayPipeline, nullptr); + vkDestroyPipeline(device, kit.volumePipeline, nullptr); + vkDestroyPipelineLayout(device, kit.pipelineLayout, nullptr); + vkDestroyDescriptorSetLayout(device, kit.descriptorLayout, nullptr); + vkDestroyShaderModule(device, kit.arrayModule, nullptr); + vkDestroyShaderModule(device, kit.volumeModule, nullptr); + kit = {}; + } + + Bool WireE5RawCopySupported(VkPhysicalDevice physicalDevice) { + VkFormatProperties properties{}; + vkGetPhysicalDeviceFormatProperties(physicalDevice, VK_FORMAT_R32_UINT, &properties); + constexpr VkFormatFeatureFlags kRequired = VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | + VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT | + VK_FORMAT_FEATURE_TRANSFER_SRC_BIT; + return (properties.optimalTilingFeatures & kRequired) == kRequired; + } + + VkImageCreateInfo WireE5RawCopyTwinInfo(const WireE5RawCopyRegion& region) { + VkImageCreateInfo info{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + info.imageType = region.volume ? VK_IMAGE_TYPE_3D : VK_IMAGE_TYPE_2D; + info.format = VK_FORMAT_R32_UINT; + info.extent = {region.extent.width, region.extent.height, region.volume ? region.extent.depth : 1u}; + info.mipLevels = 1; + info.arrayLayers = region.volume ? 1u : region.layerCount; + info.samples = VK_SAMPLE_COUNT_1_BIT; + info.tiling = VK_IMAGE_TILING_OPTIMAL; + info.usage = VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + return info; + } + + void DestroyWireE5RawCopyObjects(VkDevice device, WireE5RawCopyObjects& objects) { + vkDestroyImageView(device, objects.sourceView, nullptr); + vkDestroyImageView(device, objects.twinView, nullptr); + vkDestroyDescriptorPool(device, objects.descriptorPool, nullptr); + objects = {}; + } + + VkResult RecordWireE5RawCopy(VkDevice device, VkCommandBuffer commandBuffer, const WireE5RawCopyKit& kit, + const WireE5RawCopyRegion& region, VkImageLayout sourceLayout, VkImage twin, + WireE5RawCopyObjects& objects) { + if (kit.arrayPipeline == VK_NULL_HANDLE || kit.volumePipeline == VK_NULL_HANDLE) + return VK_ERROR_INITIALIZATION_FAILED; + const Uint32 slices = region.volume ? region.extent.depth : region.layerCount; + if (region.extent.width == 0 || region.extent.height == 0 || slices == 0) return VK_SUCCESS; + + // The source through R32_UINT: same 32-bit compatibility class, so a plain view of a + // MUTABLE_FORMAT image. A 3D view covers every slice; the shader offsets into it. + VkImageViewCreateInfo sourceInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + sourceInfo.image = region.source; + sourceInfo.viewType = region.volume ? VK_IMAGE_VIEW_TYPE_3D : VK_IMAGE_VIEW_TYPE_2D_ARRAY; + sourceInfo.format = VK_FORMAT_R32_UINT; + sourceInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, region.level, 1, + region.volume ? 0u : region.baseLayer, region.volume ? 1u : region.layerCount}; + VkResult result = vkCreateImageView(device, &sourceInfo, nullptr, &objects.sourceView); + if (result != VK_SUCCESS) return result; + VkImageViewCreateInfo twinInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + twinInfo.image = twin; + twinInfo.viewType = region.volume ? VK_IMAGE_VIEW_TYPE_3D : VK_IMAGE_VIEW_TYPE_2D_ARRAY; + twinInfo.format = VK_FORMAT_R32_UINT; + twinInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, region.volume ? 1u : region.layerCount}; + result = vkCreateImageView(device, &twinInfo, nullptr, &objects.twinView); + if (result != VK_SUCCESS) return result; + + const VkDescriptorPoolSize poolSizes[2]{{VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1}, + {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1}}; + VkDescriptorPoolCreateInfo pool{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + pool.maxSets = 1; + pool.poolSizeCount = 2; + pool.pPoolSizes = poolSizes; + result = vkCreateDescriptorPool(device, &pool, nullptr, &objects.descriptorPool); + if (result != VK_SUCCESS) return result; + VkDescriptorSetAllocateInfo alloc{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + alloc.descriptorPool = objects.descriptorPool; + alloc.descriptorSetCount = 1; + alloc.pSetLayouts = &kit.descriptorLayout; + VkDescriptorSet set = VK_NULL_HANDLE; + result = vkAllocateDescriptorSets(device, &alloc, &set); + if (result != VK_SUCCESS) return result; + const VkDescriptorImageInfo sampled{kit.nearest, objects.sourceView, sourceLayout}; + const VkDescriptorImageInfo storage{VK_NULL_HANDLE, objects.twinView, VK_IMAGE_LAYOUT_GENERAL}; + VkWriteDescriptorSet writes[2]{}; + for (Uint32 i = 0; i < 2; ++i) { + writes[i].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[i].dstSet = set; + writes[i].dstBinding = i; + writes[i].descriptorCount = 1; + } + writes[0].descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; + writes[0].pImageInfo = &sampled; + writes[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; + writes[1].pImageInfo = &storage; + vkUpdateDescriptorSets(device, 2, writes, 0, nullptr); + + // Every earlier write - transfer, attachment or shader - visible to the fetch, and the + // fresh twin ready for storage writes. + VkMemoryBarrier memory{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; + memory.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + memory.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + VkImageMemoryBarrier twinBarrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + twinBarrier.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + twinBarrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + twinBarrier.newLayout = VK_IMAGE_LAYOUT_GENERAL; + twinBarrier.srcQueueFamilyIndex = twinBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + twinBarrier.image = twin; + twinBarrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, VK_REMAINING_ARRAY_LAYERS}; + vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + 0, 1, &memory, 0, nullptr, 1, &twinBarrier); + + vkCmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, + region.volume ? kit.volumePipeline : kit.arrayPipeline); + vkCmdBindDescriptorSets(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, kit.pipelineLayout, 0, 1, &set, 0, + nullptr); + const RegionPush push{{region.offset.x, region.offset.y, region.volume ? region.offset.z : 0, 0}, + {static_cast(region.extent.width), static_cast(region.extent.height), + static_cast(slices), 0}}; + vkCmdPushConstants(commandBuffer, kit.pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), &push); + vkCmdDispatch(commandBuffer, (region.extent.width + kGroupSize - 1) / kGroupSize, + (region.extent.height + kGroupSize - 1) / kGroupSize, slices); + + twinBarrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + twinBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + twinBarrier.oldLayout = VK_IMAGE_LAYOUT_GENERAL; + twinBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, + nullptr, 0, nullptr, 1, &twinBarrier); + return VK_SUCCESS; + } + + String DescribeWireE5TransferProbe(const WireE5TransferProbeMeasurement& measurement) { + if (!measurement.ran) + return format("not run ({})", measurement.failureReason.empty() ? "?" : measurement.failureReason); + return format("30x30 transfer read: {}/{} words changed (first {:#010x} -> {:#010x}); shader read: {} " + "changed; 16x16 transfer read: {}/{} changed", + measurement.subjectMismatches, measurement.texels, measurement.firstExpected, + measurement.firstObserved, measurement.controlMismatches, measurement.smallMismatches, + measurement.smallTexels); + } + + namespace { + constexpr Uint32 kSubjectExtent = 30; + constexpr Uint32 kSmallExtent = 16; + constexpr Uint64 kFenceTimeoutNs = 5ull * 1000ull * 1000ull * 1000ull; + + // A non-canonical RGB9_E5 word: exponent 1..31 and every mantissa below 256, so the same + // value also encodes with the exponent one lower and every mantissa doubled. A path that + // decodes and re-encodes changes it; a bit move does not. + Uint32 NonCanonicalWord(Uint32 index) { + Uint32 state = index * 2654435761u + 0x9E3779B9u; + state ^= state >> 15; + state *= 2246822519u; + state ^= state >> 13; + const Uint32 r = state & 0xFFu; + const Uint32 g = (state >> 8) & 0xFFu; + const Uint32 b = ((state >> 16) & 0xFFu) | ((r | g) == 0 ? 1u : 0u); + const Uint32 exponent = 1u + ((state >> 24) % 31u); + return r | (g << 9) | (b << 18) | (exponent << 27); + } + + struct ProbeObjects { + VkDevice device = VK_NULL_HANDLE; + Vector images; + Vector memories; + Vector buffers; + VkCommandPool commandPool = VK_NULL_HANDLE; + VkFence fence = VK_NULL_HANDLE; + WireE5RawCopyKit kit; + WireE5RawCopyObjects copy; + + void Destroy() { + DestroyWireE5RawCopyObjects(device, copy); + for (VkImage image : images) vkDestroyImage(device, image, nullptr); + for (VkBuffer buffer : buffers) vkDestroyBuffer(device, buffer, nullptr); + for (VkDeviceMemory memory : memories) vkFreeMemory(device, memory, nullptr); + vkDestroyCommandPool(device, commandPool, nullptr); + vkDestroyFence(device, fence, nullptr); + DestroyWireE5RawCopyKit(device, kit); + } + }; + + Optional PickMemoryType(const VkPhysicalDeviceMemoryProperties& properties, Uint32 allowed, + VkMemoryPropertyFlags wanted, VkMemoryPropertyFlags required) { + Optional fallback; + for (Uint32 i = 0; i < properties.memoryTypeCount; ++i) { + if (!(allowed & (1u << i))) continue; + const VkMemoryPropertyFlags flags = properties.memoryTypes[i].propertyFlags; + if (flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) continue; + if ((flags & required) != required) continue; + if ((flags & wanted) == wanted) return i; + if (!fallback) fallback = i; + } + return fallback; + } + + void ImageBarrier(VkCommandBuffer commandBuffer, VkImage image, VkImageLayout oldLayout, + VkImageLayout newLayout, VkPipelineStageFlags srcStage, VkAccessFlags srcAccess, + VkPipelineStageFlags dstStage, VkAccessFlags dstAccess) { + VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + barrier.srcAccessMask = srcAccess; + barrier.dstAccessMask = dstAccess; + barrier.oldLayout = oldLayout; + barrier.newLayout = newLayout; + barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = image; + barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + vkCmdPipelineBarrier(commandBuffer, srcStage, dstStage, 0, 0, nullptr, 0, nullptr, 1, &barrier); + } + } // namespace + + WireE5TransferProbeMeasurement RunWireE5TransferProbe(const WireE5TransferProbeContext& context) { + WireE5TransferProbeMeasurement measurement; + const VkDevice device = context.device; + if (device == VK_NULL_HANDLE || context.queue == VK_NULL_HANDLE) { + measurement.failureReason = "no device or queue"; + return measurement; + } + if (!WireE5RawCopySupported(context.physicalDevice)) { + measurement.failureReason = "R32_UINT is not a sampled + storage + transfer-source format here"; + return measurement; + } + // The usage production gives a wire E5B9G9R9 texture: SAMPLED | TRANSFER_SRC | TRANSFER_DST, + // plus COLOR_ATTACHMENT where the format can be one. + VkFormatProperties e5Properties{}; + vkGetPhysicalDeviceFormatProperties(context.physicalDevice, VK_FORMAT_E5B9G9R9_UFLOAT_PACK32, &e5Properties); + constexpr VkFormatFeatureFlags kE5Required = VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | + VK_FORMAT_FEATURE_TRANSFER_SRC_BIT | + VK_FORMAT_FEATURE_TRANSFER_DST_BIT; + if ((e5Properties.optimalTilingFeatures & kE5Required) != kE5Required) { + measurement.failureReason = "E5B9G9R9 is not a sampled transfer format here"; + return measurement; + } + VkImageUsageFlags e5Usage = + VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; + if (e5Properties.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT) + e5Usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; + + ProbeObjects objects; + objects.device = device; + Bool leak = false; + struct Teardown { + ProbeObjects& objects; + Bool& leak; + ~Teardown() { + if (!leak) objects.Destroy(); + } + } teardown{objects, leak}; + const auto fail = [&](String reason) { + measurement.ran = false; + measurement.failureReason = Move(reason); + return measurement; + }; + + VkPhysicalDeviceMemoryProperties memoryProperties{}; + vkGetPhysicalDeviceMemoryProperties(context.physicalDevice, &memoryProperties); + const auto bindImage = [&](VkImage image) -> Bool { + VkMemoryRequirements requirements{}; + vkGetImageMemoryRequirements(device, image, &requirements); + const Optional type = PickMemoryType(memoryProperties, requirements.memoryTypeBits, + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, 0); + if (!type) return false; + VkMemoryAllocateInfo allocate{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocate.allocationSize = requirements.size; + allocate.memoryTypeIndex = *type; + VkDeviceMemory memory = VK_NULL_HANDLE; + if (vkAllocateMemory(device, &allocate, nullptr, &memory) != VK_SUCCESS) return false; + objects.memories.push_back(memory); + return vkBindImageMemory(device, image, memory, 0) == VK_SUCCESS; + }; + const auto makeImage = [&](const VkImageCreateInfo& info, VkImage& out) -> Bool { + if (vkCreateImage(device, &info, nullptr, &out) != VK_SUCCESS) return false; + objects.images.push_back(out); + return bindImage(out); + }; + const auto makeE5 = [&](Uint32 extent, VkImage& out) { + VkImageCreateInfo info{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + info.flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT; + info.imageType = VK_IMAGE_TYPE_2D; + info.format = VK_FORMAT_E5B9G9R9_UFLOAT_PACK32; + info.extent = {extent, extent, 1}; + info.mipLevels = info.arrayLayers = 1; + info.samples = VK_SAMPLE_COUNT_1_BIT; + info.tiling = VK_IMAGE_TILING_OPTIMAL; + info.usage = e5Usage; + info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + return makeImage(info, out); + }; + // Host-visible buffers: the upload source and the three readbacks, one allocation each. + struct HostBuffer { + VkBuffer buffer = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + Bool coherent = false; + }; + const auto makeBuffer = [&](VkDeviceSize bytes, VkBufferUsageFlags usage, HostBuffer& out) -> Bool { + VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + info.size = bytes; + info.usage = usage; + info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + if (vkCreateBuffer(device, &info, nullptr, &out.buffer) != VK_SUCCESS) return false; + objects.buffers.push_back(out.buffer); + VkMemoryRequirements requirements{}; + vkGetBufferMemoryRequirements(device, out.buffer, &requirements); + const Optional type = + PickMemoryType(memoryProperties, requirements.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT); + if (!type) return false; + out.coherent = + (memoryProperties.memoryTypes[*type].propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0; + VkMemoryAllocateInfo allocate{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocate.allocationSize = requirements.size; + allocate.memoryTypeIndex = *type; + if (vkAllocateMemory(device, &allocate, nullptr, &out.memory) != VK_SUCCESS) return false; + objects.memories.push_back(out.memory); + return vkBindBufferMemory(device, out.buffer, out.memory, 0) == VK_SUCCESS; + }; + + constexpr Uint32 kTexels = kSubjectExtent * kSubjectExtent; + constexpr Uint32 kSmallTexels = kSmallExtent * kSmallExtent; + constexpr VkDeviceSize kBytes = kTexels * sizeof(Uint32); + constexpr VkDeviceSize kSmallBytes = kSmallTexels * sizeof(Uint32); + + VkImage subject = VK_NULL_HANDLE, small = VK_NULL_HANDLE, twin = VK_NULL_HANDLE; + if (!makeE5(kSubjectExtent, subject) || !makeE5(kSmallExtent, small)) + return fail("E5B9G9R9 image creation failed"); + WireE5RawCopyRegion region; + region.source = subject; + region.extent = {kSubjectExtent, kSubjectExtent, 1}; + if (!makeImage(WireE5RawCopyTwinInfo(region), twin)) return fail("R32_UINT twin creation failed"); + HostBuffer upload, subjectOut, controlOut, smallOut; + if (!makeBuffer(kBytes, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, upload) || + !makeBuffer(kBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT, subjectOut) || + !makeBuffer(kBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT, controlOut) || + !makeBuffer(kSmallBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT, smallOut)) + return fail("buffer creation failed"); + if (const VkResult result = CreateWireE5RawCopyKit(device, objects.kit); result != VK_SUCCESS) + return fail(format("raw copy kit creation failed (VkResult {})", static_cast(result))); + + { + void* mapped = nullptr; + if (vkMapMemory(device, upload.memory, 0, VK_WHOLE_SIZE, 0, &mapped) != VK_SUCCESS || mapped == nullptr) + return fail("vkMapMemory (upload) failed"); + Uint32* words = static_cast(mapped); + for (Uint32 i = 0; i < kTexels; ++i) words[i] = NonCanonicalWord(i); + if (!upload.coherent) { + VkMappedMemoryRange range{VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE}; + range.memory = upload.memory; + range.size = VK_WHOLE_SIZE; + vkFlushMappedMemoryRanges(device, 1, &range); + } + vkUnmapMemory(device, upload.memory); + } + + VkCommandPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + poolInfo.flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT; + poolInfo.queueFamilyIndex = context.queueFamilyIndex; + if (vkCreateCommandPool(device, &poolInfo, nullptr, &objects.commandPool) != VK_SUCCESS) + return fail("vkCreateCommandPool failed"); + VkCommandBufferAllocateInfo commandInfo{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + commandInfo.commandPool = objects.commandPool; + commandInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + commandInfo.commandBufferCount = 1; + VkCommandBuffer cmd = VK_NULL_HANDLE; + if (vkAllocateCommandBuffers(device, &commandInfo, &cmd) != VK_SUCCESS) + return fail("vkAllocateCommandBuffers failed"); + VkCommandBufferBeginInfo beginInfo{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + if (vkBeginCommandBuffer(cmd, &beginInfo) != VK_SUCCESS) return fail("vkBeginCommandBuffer failed"); + + // (1) Upload: transfer writes into E5B9G9R9 move bits (measured), so both images hold + // exactly the generated words; the small one gets the first 8x8 of them. + for (const VkImage image : {subject, small}) { + ImageBarrier(cmd, image, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_ACCESS_TRANSFER_WRITE_BIT); + } + VkBufferImageCopy copy{}; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {kSubjectExtent, kSubjectExtent, 1}; + vkCmdCopyBufferToImage(cmd, upload.buffer, subject, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©); + VkBufferImageCopy smallCopy = copy; + smallCopy.bufferRowLength = kSubjectExtent; + smallCopy.imageExtent = {kSmallExtent, kSmallExtent, 1}; + vkCmdCopyBufferToImage(cmd, upload.buffer, small, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &smallCopy); + + // (2) SUBJECT and the small reading: plain transfer reads. + for (const VkImage image : {subject, small}) { + ImageBarrier(cmd, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_ACCESS_TRANSFER_READ_BIT); + } + vkCmdCopyImageToBuffer(cmd, subject, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, subjectOut.buffer, 1, ©); + VkBufferImageCopy smallOutCopy = copy; + smallOutCopy.imageExtent = {kSmallExtent, kSmallExtent, 1}; + vkCmdCopyImageToBuffer(cmd, small, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, smallOut.buffer, 1, &smallOutCopy); + + // (3) CONTROL: the route, exactly as production records it. + ImageBarrier(cmd, subject, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_ACCESS_SHADER_READ_BIT); + if (const VkResult result = + RecordWireE5RawCopy(device, cmd, objects.kit, region, VK_IMAGE_LAYOUT_GENERAL, twin, objects.copy); + result != VK_SUCCESS) + return fail(format("recording the raw copy failed (VkResult {})", static_cast(result))); + vkCmdCopyImageToBuffer(cmd, twin, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, controlOut.buffer, 1, ©); + + VkMemoryBarrier hostBarrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; + hostBarrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + hostBarrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT; + vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &hostBarrier, 0, + nullptr, 0, nullptr); + if (vkEndCommandBuffer(cmd) != VK_SUCCESS) return fail("vkEndCommandBuffer failed"); + + VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + if (vkCreateFence(device, &fenceInfo, nullptr, &objects.fence) != VK_SUCCESS) + return fail("vkCreateFence failed"); + VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO}; + submit.commandBufferCount = 1; + submit.pCommandBuffers = &cmd; + if (const VkResult result = vkQueueSubmit(context.queue, 1, &submit, objects.fence); result != VK_SUCCESS) + return fail(format("vkQueueSubmit failed (VkResult {})", static_cast(result))); + const VkResult waited = vkWaitForFences(device, 1, &objects.fence, VK_TRUE, kFenceTimeoutNs); + if (waited == VK_TIMEOUT) { + leak = true; + measurement.fenceWaitTimedOut = true; + return fail("the probe's fence did not signal within 5 s"); + } + if (waited != VK_SUCCESS) return fail(format("vkWaitForFences failed (VkResult {})", static_cast(waited))); + + const auto read = [&](const HostBuffer& buffer, const Uint32*& words) -> Bool { + void* mapped = nullptr; + if (vkMapMemory(device, buffer.memory, 0, VK_WHOLE_SIZE, 0, &mapped) != VK_SUCCESS || mapped == nullptr) + return false; + if (!buffer.coherent) { + VkMappedMemoryRange range{VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE}; + range.memory = buffer.memory; + range.size = VK_WHOLE_SIZE; + vkInvalidateMappedMemoryRanges(device, 1, &range); + } + words = static_cast(mapped); + return true; + }; + const Uint32 *subjectWords = nullptr, *controlWords = nullptr, *smallWords = nullptr; + if (!read(subjectOut, subjectWords) || !read(controlOut, controlWords) || !read(smallOut, smallWords)) + return fail("vkMapMemory (readback) failed"); + measurement.texels = kTexels; + for (Uint32 i = 0; i < kTexels; ++i) { + const Uint32 expected = NonCanonicalWord(i); + if (subjectWords[i] != expected) { + if (measurement.subjectMismatches++ == 0) { + measurement.firstExpected = expected; + measurement.firstObserved = subjectWords[i]; + } + } + if (controlWords[i] != expected) ++measurement.controlMismatches; + } + measurement.smallTexels = kSmallTexels; + for (Uint32 y = 0; y < kSmallExtent; ++y) + for (Uint32 x = 0; x < kSmallExtent; ++x) + if (smallWords[y * kSmallExtent + x] != NonCanonicalWord(y * kSubjectExtent + x)) + ++measurement.smallMismatches; + vkUnmapMemory(device, subjectOut.memory); + vkUnmapMemory(device, controlOut.memory); + vkUnmapMemory(device, smallOut.memory); + measurement.ran = true; + return measurement; + } + + WireE5TransferProbeMeasurement RunWireE5TransferProbeOnThrowawayDevice(PFN_vkGetInstanceProcAddr getInstanceProcAddr, + VkInstance instance, + VkPhysicalDevice physicalDevice, + Uint32 graphicsQueueFamilyIndex) { + WireE5TransferProbeMeasurement measurement; + const auto createDevice = + reinterpret_cast(getInstanceProcAddr(instance, "vkCreateDevice")); + if (createDevice == nullptr) { + measurement.failureReason = "no vkCreateDevice"; + return measurement; + } + const Float priority = 1.0f; + VkDeviceQueueCreateInfo queueInfo{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; + queueInfo.queueFamilyIndex = graphicsQueueFamilyIndex; + queueInfo.queueCount = 1; + queueInfo.pQueuePriorities = &priority; + VkDeviceCreateInfo deviceInfo{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; + deviceInfo.queueCreateInfoCount = 1; + deviceInfo.pQueueCreateInfos = &queueInfo; + VkDevice device = VK_NULL_HANDLE; + if (const VkResult result = createDevice(physicalDevice, &deviceInfo, nullptr, &device); + result != VK_SUCCESS || device == VK_NULL_HANDLE) { + measurement.failureReason = format("vkCreateDevice failed (VkResult {})", static_cast(result)); + return measurement; + } + WireE5TransferProbeContext context; + context.physicalDevice = physicalDevice; + context.device = device; + vkGetDeviceQueue(device, graphicsQueueFamilyIndex, 0, &context.queue); + context.queueFamilyIndex = graphicsQueueFamilyIndex; + measurement = RunWireE5TransferProbe(context); + if (!measurement.fenceWaitTimedOut) vkDestroyDevice(device, nullptr); + return measurement; + } +} // namespace MobileGL::MG_Backend::DirectVulkan diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.h new file mode 100644 index 00000000..30e295e9 --- /dev/null +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.h @@ -0,0 +1,168 @@ +// MobileGL - MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.h +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header +#pragma once + +#include +#include + +#include + +// THE RAW E5B9G9R9 READ ROUTE, and the probe that decides whether a device needs it. +// +// GL makes RGB9_E5 a bit-exact format wherever it moves texel blocks: glGetTexImage with +// GL_UNSIGNED_INT_5_9_9_9_REV hands back the words the application stored, and +// glCopyImageSubData is a raw block move (GL 4.6 core 18.3.2). RGB9_E5 has redundant encodings - +// a shared exponent one higher with every mantissa halved is the same value - so a path that +// decodes and re-encodes keeps every value and still changes words. +// +// Adreno 750 does exactly that on TRANSFER READS of a VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 image once +// the level is not a power-of-two size: vkCmdCopyImageToBuffer and vkCmdCopyImage out of a 30x30 +// level return every non-canonical word canonicalized, while a 16x16 or 64x64 level reads back verbatim. +// The stored words are intact - a shader reading the same image through an R32_UINT view sees +// every one of them - and so are transfer WRITES into it. Measured with format lists, without +// the colour-attachment usage and with storage usage: none of them changes the answer, and the +// device has no VK_EXT_image_compression_control to ask with. +// +// The route therefore reads through a shader: a compute pass fetches the source through an +// R32_UINT view (the wire's E5B9G9R9 textures are MUTABLE_FORMAT from birth, see +// VkTextureManager) and stores the words into an R32_UINT twin, whose transfer reads move bits. +// The caller then reads the twin where it would have read the source. +// +// Whether a device needs it is MEASURED (RunWireE5TransferProbe below), at Magma bring-up +// (VulkanRenderer::ArmWireE5RawReads) and in the POST's "Known Driver Bugs" section +// (MG_Util/SelfTest/DriverBugProbes.cpp); a device that reads E5B9G9R9 back verbatim keeps its +// plain transfers. +namespace MobileGL::MG_Backend::DirectVulkan { + // Device objects that do not depend on any image. + struct WireE5RawCopyKit { + VkShaderModule arrayModule = VK_NULL_HANDLE; + VkShaderModule volumeModule = VK_NULL_HANDLE; + VkDescriptorSetLayout descriptorLayout = VK_NULL_HANDLE; + VkPipelineLayout pipelineLayout = VK_NULL_HANDLE; + VkPipeline arrayPipeline = VK_NULL_HANDLE; + VkPipeline volumePipeline = VK_NULL_HANDLE; + VkSampler nearest = VK_NULL_HANDLE; + }; + // Creates every member; on failure the members made so far stay set, for DestroyWireE5RawCopyKit. + VkResult CreateWireE5RawCopyKit(VkDevice device, WireE5RawCopyKit& kit); + void DestroyWireE5RawCopyKit(VkDevice device, WireE5RawCopyKit& kit); + + // Whether the device can run the route at all: R32_UINT sampled and storage on optimal tiling. + Bool WireE5RawCopySupported(VkPhysicalDevice physicalDevice); + + // One region of one level of the source. `volume` for a VK_IMAGE_TYPE_3D source, whose slices + // are `offset.z` .. `offset.z + extent.depth`; any other source is a 2D (possibly layered or + // cube-compatible) image whose layers are `baseLayer` .. `baseLayer + layerCount` and whose + // `offset.z` / `extent.depth` are 0 / 1. 1D images are not supported. + struct WireE5RawCopyRegion { + VkImage source = VK_NULL_HANDLE; + Bool volume = false; + Uint32 level = 0; + Uint32 baseLayer = 0; + Uint32 layerCount = 1; + VkOffset3D offset{0, 0, 0}; + VkExtent3D extent{1, 1, 1}; + }; + + // The twin the caller creates for `region`: R32_UINT, one level, the region's extent at its + // origin - a 3D image for a volume region, a 2D image of `layerCount` layers otherwise - + // with STORAGE | TRANSFER_SRC usage. RecordWireE5RawCopy leaves it in TRANSFER_SRC_OPTIMAL. + VkImageCreateInfo WireE5RawCopyTwinInfo(const WireE5RawCopyRegion& region); + + // What a recorded copy holds until its submission completes. + struct WireE5RawCopyObjects { + VkDescriptorPool descriptorPool = VK_NULL_HANDLE; + VkImageView sourceView = VK_NULL_HANDLE; + VkImageView twinView = VK_NULL_HANDLE; + }; + void DestroyWireE5RawCopyObjects(VkDevice device, WireE5RawCopyObjects& objects); + + // Records the copy of `region` into `twin` (fresh, created from WireE5RawCopyTwinInfo). + // The source must have been created MUTABLE_FORMAT with an E5B9G9R9 format, and be in + // `sourceLayout` (GENERAL or SHADER_READ_ONLY_OPTIMAL); the recording starts with a global + // barrier that makes every earlier write visible to the compute stage and ends with the twin + // in TRANSFER_SRC_OPTIMAL, its writes visible to transfer reads. On failure nothing has been + // recorded and `objects` holds whatever was created, for DestroyWireE5RawCopyObjects. + VkResult RecordWireE5RawCopy(VkDevice device, VkCommandBuffer commandBuffer, const WireE5RawCopyKit& kit, + const WireE5RawCopyRegion& region, VkImageLayout sourceLayout, VkImage twin, + WireE5RawCopyObjects& objects); + + // --------------------------------------------------------------------------------------- + // THE PROBE. + struct WireE5TransferProbeMeasurement { + // Recorded, submitted and read back. + Bool ran = false; + // The bounded fence wait expired: every object was deliberately leaked. + Bool fenceWaitTimedOut = false; + String failureReason; + // SUBJECT: a 30x30 E5B9G9R9 level of non-canonical words read back by vkCmdCopyImageToBuffer. + Uint32 texels = 0; + Uint32 subjectMismatches = 0; + Uint32 firstExpected = 0; + Uint32 firstObserved = 0; + // CONTROL: the same image read through the route. Must be exact, or the upload, the + // comparison or the route itself is what failed, and nothing is concluded. + Uint32 controlMismatches = 0; + // Reported, not judged: the same words in a 16x16 level read back by plain transfer. Clean + // on the device this was characterised on (power-of-two extents read back verbatim, 28, 30 + // and 31 do not), which is why the subject is 30x30. + Uint32 smallTexels = 0; + Uint32 smallMismatches = 0; + }; + + enum class WireE5TransferProbeVerdict : Uint8 { + // Nothing was measured (the route is unavailable, or setup failed): plain transfers stay. + NotRun, + // Measured, but the route's own read was not exact: nothing is concluded. + Inconclusive, + // Plain transfer reads return the stored words. + Clean, + // The defect: plain transfer reads changed words the route read back exactly. + TransferReadNormalizes, + }; + + inline const char* WireE5TransferProbeVerdictName(WireE5TransferProbeVerdict verdict) { + switch (verdict) { + case WireE5TransferProbeVerdict::NotRun: return "not-run"; + case WireE5TransferProbeVerdict::Inconclusive: return "inconclusive"; + case WireE5TransferProbeVerdict::Clean: return "clean"; + case WireE5TransferProbeVerdict::TransferReadNormalizes: return "transfer-read-normalizes"; + } + return "?"; + } + + inline WireE5TransferProbeVerdict EvaluateWireE5TransferProbe(const WireE5TransferProbeMeasurement& measurement) { + if (!measurement.ran || measurement.fenceWaitTimedOut || measurement.texels == 0) + return WireE5TransferProbeVerdict::NotRun; + if (measurement.controlMismatches != 0) return WireE5TransferProbeVerdict::Inconclusive; + return measurement.subjectMismatches != 0 ? WireE5TransferProbeVerdict::TransferReadNormalizes + : WireE5TransferProbeVerdict::Clean; + } + + // One line with every reading, for the bring-up log and the POST row. + String DescribeWireE5TransferProbe(const WireE5TransferProbeMeasurement& measurement); + + struct WireE5TransferProbeContext { + VkPhysicalDevice physicalDevice = VK_NULL_HANDLE; + VkDevice device = VK_NULL_HANDLE; + // Must be otherwise idle for the probe's duration. + VkQueue queue = VK_NULL_HANDLE; + Uint32 queueFamilyIndex = 0; + }; + + // One submission on its own command pool, a bounded fence wait (on timeout every object is + // leaked and the measurement says so - the caller must then not destroy or idle-wait the + // device), teardown on every other path. Never touches renderer state. + WireE5TransferProbeMeasurement RunWireE5TransferProbe(const WireE5TransferProbeContext& context); + + // The POST's entry: a throwaway device with one graphics queue, the probe, the device + // destroyed (leaked on a fence timeout). + WireE5TransferProbeMeasurement RunWireE5TransferProbeOnThrowawayDevice( + PFN_vkGetInstanceProcAddr getInstanceProcAddr, VkInstance instance, VkPhysicalDevice physicalDevice, + Uint32 graphicsQueueFamilyIndex); +} // namespace MobileGL::MG_Backend::DirectVulkan diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy3D.comp b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy3D.comp new file mode 100644 index 00000000..8ef12a6e --- /dev/null +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy3D.comp @@ -0,0 +1,19 @@ +#version 450 +// The volume twin of WireE5RawCopy.comp: the source level is a 3D image and the region's slices +// are its z range. +layout(local_size_x = 8, local_size_y = 8, local_size_z = 1) in; +layout(set = 0, binding = 0) uniform usampler3D sourceImage; +layout(set = 0, binding = 1, r32ui) uniform writeonly uimage3D twinImage; +layout(push_constant) uniform Region { + // xyz: the region's texel origin in the source level. + ivec4 sourceOffset; + // xyz: texels to copy. + ivec4 extent; +} region; + +void main() { + ivec3 id = ivec3(gl_GlobalInvocationID); + if (any(greaterThanEqual(id, region.extent.xyz))) return; + uint word = texelFetch(sourceImage, region.sourceOffset.xyz + id, 0).r; + imageStore(twinImage, id, uvec4(word, 0u, 0u, 0u)); +} diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopySpirv.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopySpirv.h new file mode 100644 index 00000000..3076c049 --- /dev/null +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopySpirv.h @@ -0,0 +1,839 @@ +// Generated from WireE5RawCopy.comp / WireE5RawCopy3D.comp (Vulkan 1.0 / SPIR-V 1.0) by +// scripts/bake_internal_shaders.py, which runs MG_Test's BakedInternalShadersTest with the +// IN-TREE glslang and splices the words it emits into the arrays below. Do not edit the words +// by hand: MOBILEGL_BAKED_INTERNAL_SHADERS (the same test, run as a gate) recompiles every +// source in its table and reds on the first word that disagrees. +#pragma once + +static constexpr Uint32 kWireE5RawCopySpirv[] = { + 0x07230203u, + 0x00010000u, + 0x0008000bu, + 0x00000046u, + 0x00000000u, + 0x00020011u, + 0x00000001u, + 0x0006000bu, + 0x00000001u, + 0x4c534c47u, + 0x6474732eu, + 0x3035342eu, + 0x00000000u, + 0x0003000eu, + 0x00000000u, + 0x00000001u, + 0x0006000fu, + 0x00000005u, + 0x00000004u, + 0x6e69616du, + 0x00000000u, + 0x0000000du, + 0x00060010u, + 0x00000004u, + 0x00000011u, + 0x00000008u, + 0x00000008u, + 0x00000001u, + 0x00030003u, + 0x00000002u, + 0x000001c2u, + 0x00040005u, + 0x00000004u, + 0x6e69616du, + 0x00000000u, + 0x00030005u, + 0x00000009u, + 0x00006469u, + 0x00080005u, + 0x0000000du, + 0x475f6c67u, + 0x61626f6cu, + 0x766e496cu, + 0x7461636fu, + 0x496e6f69u, + 0x00000044u, + 0x00040005u, + 0x00000012u, + 0x69676552u, + 0x00006e6fu, + 0x00070006u, + 0x00000012u, + 0x00000000u, + 0x72756f73u, + 0x664f6563u, + 0x74657366u, + 0x00000000u, + 0x00050006u, + 0x00000012u, + 0x00000001u, + 0x65747865u, + 0x0000746eu, + 0x00040005u, + 0x00000014u, + 0x69676572u, + 0x00006e6fu, + 0x00040005u, + 0x00000022u, + 0x64726f77u, + 0x00000000u, + 0x00050005u, + 0x00000026u, + 0x72756f73u, + 0x6d496563u, + 0x00656761u, + 0x00050005u, + 0x0000003eu, + 0x6e697774u, + 0x67616d49u, + 0x00000065u, + 0x00040047u, + 0x0000000du, + 0x0000000bu, + 0x0000001cu, + 0x00030047u, + 0x00000012u, + 0x00000002u, + 0x00050048u, + 0x00000012u, + 0x00000000u, + 0x00000023u, + 0x00000000u, + 0x00050048u, + 0x00000012u, + 0x00000001u, + 0x00000023u, + 0x00000010u, + 0x00040047u, + 0x00000026u, + 0x00000021u, + 0x00000000u, + 0x00040047u, + 0x00000026u, + 0x00000022u, + 0x00000000u, + 0x00030047u, + 0x0000003eu, + 0x00000019u, + 0x00040047u, + 0x0000003eu, + 0x00000021u, + 0x00000001u, + 0x00040047u, + 0x0000003eu, + 0x00000022u, + 0x00000000u, + 0x00040047u, + 0x00000045u, + 0x0000000bu, + 0x00000019u, + 0x00020013u, + 0x00000002u, + 0x00030021u, + 0x00000003u, + 0x00000002u, + 0x00040015u, + 0x00000006u, + 0x00000020u, + 0x00000001u, + 0x00040017u, + 0x00000007u, + 0x00000006u, + 0x00000003u, + 0x00040020u, + 0x00000008u, + 0x00000007u, + 0x00000007u, + 0x00040015u, + 0x0000000au, + 0x00000020u, + 0x00000000u, + 0x00040017u, + 0x0000000bu, + 0x0000000au, + 0x00000003u, + 0x00040020u, + 0x0000000cu, + 0x00000001u, + 0x0000000bu, + 0x0004003bu, + 0x0000000cu, + 0x0000000du, + 0x00000001u, + 0x00040017u, + 0x00000011u, + 0x00000006u, + 0x00000004u, + 0x0004001eu, + 0x00000012u, + 0x00000011u, + 0x00000011u, + 0x00040020u, + 0x00000013u, + 0x00000009u, + 0x00000012u, + 0x0004003bu, + 0x00000013u, + 0x00000014u, + 0x00000009u, + 0x0004002bu, + 0x00000006u, + 0x00000015u, + 0x00000001u, + 0x00040020u, + 0x00000016u, + 0x00000009u, + 0x00000011u, + 0x00020014u, + 0x0000001au, + 0x00040017u, + 0x0000001bu, + 0x0000001au, + 0x00000003u, + 0x00040020u, + 0x00000021u, + 0x00000007u, + 0x0000000au, + 0x00090019u, + 0x00000023u, + 0x0000000au, + 0x00000001u, + 0x00000000u, + 0x00000001u, + 0x00000000u, + 0x00000001u, + 0x00000000u, + 0x0003001bu, + 0x00000024u, + 0x00000023u, + 0x00040020u, + 0x00000025u, + 0x00000000u, + 0x00000024u, + 0x0004003bu, + 0x00000025u, + 0x00000026u, + 0x00000000u, + 0x0004002bu, + 0x00000006u, + 0x00000028u, + 0x00000000u, + 0x00040017u, + 0x00000029u, + 0x00000006u, + 0x00000002u, + 0x0004002bu, + 0x0000000au, + 0x00000030u, + 0x00000002u, + 0x00040020u, + 0x00000031u, + 0x00000007u, + 0x00000006u, + 0x00040017u, + 0x00000038u, + 0x0000000au, + 0x00000004u, + 0x0004002bu, + 0x0000000au, + 0x0000003au, + 0x00000000u, + 0x00090019u, + 0x0000003cu, + 0x0000000au, + 0x00000001u, + 0x00000000u, + 0x00000001u, + 0x00000000u, + 0x00000002u, + 0x00000021u, + 0x00040020u, + 0x0000003du, + 0x00000000u, + 0x0000003cu, + 0x0004003bu, + 0x0000003du, + 0x0000003eu, + 0x00000000u, + 0x0004002bu, + 0x0000000au, + 0x00000043u, + 0x00000008u, + 0x0004002bu, + 0x0000000au, + 0x00000044u, + 0x00000001u, + 0x0006002cu, + 0x0000000bu, + 0x00000045u, + 0x00000043u, + 0x00000043u, + 0x00000044u, + 0x00050036u, + 0x00000002u, + 0x00000004u, + 0x00000000u, + 0x00000003u, + 0x000200f8u, + 0x00000005u, + 0x0004003bu, + 0x00000008u, + 0x00000009u, + 0x00000007u, + 0x0004003bu, + 0x00000021u, + 0x00000022u, + 0x00000007u, + 0x0004003du, + 0x0000000bu, + 0x0000000eu, + 0x0000000du, + 0x0004007cu, + 0x00000007u, + 0x0000000fu, + 0x0000000eu, + 0x0003003eu, + 0x00000009u, + 0x0000000fu, + 0x0004003du, + 0x00000007u, + 0x00000010u, + 0x00000009u, + 0x00050041u, + 0x00000016u, + 0x00000017u, + 0x00000014u, + 0x00000015u, + 0x0004003du, + 0x00000011u, + 0x00000018u, + 0x00000017u, + 0x0008004fu, + 0x00000007u, + 0x00000019u, + 0x00000018u, + 0x00000018u, + 0x00000000u, + 0x00000001u, + 0x00000002u, + 0x000500afu, + 0x0000001bu, + 0x0000001cu, + 0x00000010u, + 0x00000019u, + 0x0004009au, + 0x0000001au, + 0x0000001du, + 0x0000001cu, + 0x000300f7u, + 0x0000001fu, + 0x00000000u, + 0x000400fau, + 0x0000001du, + 0x0000001eu, + 0x0000001fu, + 0x000200f8u, + 0x0000001eu, + 0x000100fdu, + 0x000200f8u, + 0x0000001fu, + 0x0004003du, + 0x00000024u, + 0x00000027u, + 0x00000026u, + 0x00050041u, + 0x00000016u, + 0x0000002au, + 0x00000014u, + 0x00000028u, + 0x0004003du, + 0x00000011u, + 0x0000002bu, + 0x0000002au, + 0x0007004fu, + 0x00000029u, + 0x0000002cu, + 0x0000002bu, + 0x0000002bu, + 0x00000000u, + 0x00000001u, + 0x0004003du, + 0x00000007u, + 0x0000002du, + 0x00000009u, + 0x0007004fu, + 0x00000029u, + 0x0000002eu, + 0x0000002du, + 0x0000002du, + 0x00000000u, + 0x00000001u, + 0x00050080u, + 0x00000029u, + 0x0000002fu, + 0x0000002cu, + 0x0000002eu, + 0x00050041u, + 0x00000031u, + 0x00000032u, + 0x00000009u, + 0x00000030u, + 0x0004003du, + 0x00000006u, + 0x00000033u, + 0x00000032u, + 0x00050051u, + 0x00000006u, + 0x00000034u, + 0x0000002fu, + 0x00000000u, + 0x00050051u, + 0x00000006u, + 0x00000035u, + 0x0000002fu, + 0x00000001u, + 0x00060050u, + 0x00000007u, + 0x00000036u, + 0x00000034u, + 0x00000035u, + 0x00000033u, + 0x00040064u, + 0x00000023u, + 0x00000037u, + 0x00000027u, + 0x0007005fu, + 0x00000038u, + 0x00000039u, + 0x00000037u, + 0x00000036u, + 0x00000002u, + 0x00000028u, + 0x00050051u, + 0x0000000au, + 0x0000003bu, + 0x00000039u, + 0x00000000u, + 0x0003003eu, + 0x00000022u, + 0x0000003bu, + 0x0004003du, + 0x0000003cu, + 0x0000003fu, + 0x0000003eu, + 0x0004003du, + 0x00000007u, + 0x00000040u, + 0x00000009u, + 0x0004003du, + 0x0000000au, + 0x00000041u, + 0x00000022u, + 0x00070050u, + 0x00000038u, + 0x00000042u, + 0x00000041u, + 0x0000003au, + 0x0000003au, + 0x0000003au, + 0x00040063u, + 0x0000003fu, + 0x00000040u, + 0x00000042u, + 0x000100fdu, + 0x00010038u, +}; + +static constexpr Uint32 kWireE5RawCopy3DSpirv[] = { + 0x07230203u, + 0x00010000u, + 0x0008000bu, + 0x0000003du, + 0x00000000u, + 0x00020011u, + 0x00000001u, + 0x0006000bu, + 0x00000001u, + 0x4c534c47u, + 0x6474732eu, + 0x3035342eu, + 0x00000000u, + 0x0003000eu, + 0x00000000u, + 0x00000001u, + 0x0006000fu, + 0x00000005u, + 0x00000004u, + 0x6e69616du, + 0x00000000u, + 0x0000000du, + 0x00060010u, + 0x00000004u, + 0x00000011u, + 0x00000008u, + 0x00000008u, + 0x00000001u, + 0x00030003u, + 0x00000002u, + 0x000001c2u, + 0x00040005u, + 0x00000004u, + 0x6e69616du, + 0x00000000u, + 0x00030005u, + 0x00000009u, + 0x00006469u, + 0x00080005u, + 0x0000000du, + 0x475f6c67u, + 0x61626f6cu, + 0x766e496cu, + 0x7461636fu, + 0x496e6f69u, + 0x00000044u, + 0x00040005u, + 0x00000012u, + 0x69676552u, + 0x00006e6fu, + 0x00070006u, + 0x00000012u, + 0x00000000u, + 0x72756f73u, + 0x664f6563u, + 0x74657366u, + 0x00000000u, + 0x00050006u, + 0x00000012u, + 0x00000001u, + 0x65747865u, + 0x0000746eu, + 0x00040005u, + 0x00000014u, + 0x69676572u, + 0x00006e6fu, + 0x00040005u, + 0x00000022u, + 0x64726f77u, + 0x00000000u, + 0x00050005u, + 0x00000026u, + 0x72756f73u, + 0x6d496563u, + 0x00656761u, + 0x00050005u, + 0x00000035u, + 0x6e697774u, + 0x67616d49u, + 0x00000065u, + 0x00040047u, + 0x0000000du, + 0x0000000bu, + 0x0000001cu, + 0x00030047u, + 0x00000012u, + 0x00000002u, + 0x00050048u, + 0x00000012u, + 0x00000000u, + 0x00000023u, + 0x00000000u, + 0x00050048u, + 0x00000012u, + 0x00000001u, + 0x00000023u, + 0x00000010u, + 0x00040047u, + 0x00000026u, + 0x00000021u, + 0x00000000u, + 0x00040047u, + 0x00000026u, + 0x00000022u, + 0x00000000u, + 0x00030047u, + 0x00000035u, + 0x00000019u, + 0x00040047u, + 0x00000035u, + 0x00000021u, + 0x00000001u, + 0x00040047u, + 0x00000035u, + 0x00000022u, + 0x00000000u, + 0x00040047u, + 0x0000003cu, + 0x0000000bu, + 0x00000019u, + 0x00020013u, + 0x00000002u, + 0x00030021u, + 0x00000003u, + 0x00000002u, + 0x00040015u, + 0x00000006u, + 0x00000020u, + 0x00000001u, + 0x00040017u, + 0x00000007u, + 0x00000006u, + 0x00000003u, + 0x00040020u, + 0x00000008u, + 0x00000007u, + 0x00000007u, + 0x00040015u, + 0x0000000au, + 0x00000020u, + 0x00000000u, + 0x00040017u, + 0x0000000bu, + 0x0000000au, + 0x00000003u, + 0x00040020u, + 0x0000000cu, + 0x00000001u, + 0x0000000bu, + 0x0004003bu, + 0x0000000cu, + 0x0000000du, + 0x00000001u, + 0x00040017u, + 0x00000011u, + 0x00000006u, + 0x00000004u, + 0x0004001eu, + 0x00000012u, + 0x00000011u, + 0x00000011u, + 0x00040020u, + 0x00000013u, + 0x00000009u, + 0x00000012u, + 0x0004003bu, + 0x00000013u, + 0x00000014u, + 0x00000009u, + 0x0004002bu, + 0x00000006u, + 0x00000015u, + 0x00000001u, + 0x00040020u, + 0x00000016u, + 0x00000009u, + 0x00000011u, + 0x00020014u, + 0x0000001au, + 0x00040017u, + 0x0000001bu, + 0x0000001au, + 0x00000003u, + 0x00040020u, + 0x00000021u, + 0x00000007u, + 0x0000000au, + 0x00090019u, + 0x00000023u, + 0x0000000au, + 0x00000002u, + 0x00000000u, + 0x00000000u, + 0x00000000u, + 0x00000001u, + 0x00000000u, + 0x0003001bu, + 0x00000024u, + 0x00000023u, + 0x00040020u, + 0x00000025u, + 0x00000000u, + 0x00000024u, + 0x0004003bu, + 0x00000025u, + 0x00000026u, + 0x00000000u, + 0x0004002bu, + 0x00000006u, + 0x00000028u, + 0x00000000u, + 0x00040017u, + 0x0000002fu, + 0x0000000au, + 0x00000004u, + 0x0004002bu, + 0x0000000au, + 0x00000031u, + 0x00000000u, + 0x00090019u, + 0x00000033u, + 0x0000000au, + 0x00000002u, + 0x00000000u, + 0x00000000u, + 0x00000000u, + 0x00000002u, + 0x00000021u, + 0x00040020u, + 0x00000034u, + 0x00000000u, + 0x00000033u, + 0x0004003bu, + 0x00000034u, + 0x00000035u, + 0x00000000u, + 0x0004002bu, + 0x0000000au, + 0x0000003au, + 0x00000008u, + 0x0004002bu, + 0x0000000au, + 0x0000003bu, + 0x00000001u, + 0x0006002cu, + 0x0000000bu, + 0x0000003cu, + 0x0000003au, + 0x0000003au, + 0x0000003bu, + 0x00050036u, + 0x00000002u, + 0x00000004u, + 0x00000000u, + 0x00000003u, + 0x000200f8u, + 0x00000005u, + 0x0004003bu, + 0x00000008u, + 0x00000009u, + 0x00000007u, + 0x0004003bu, + 0x00000021u, + 0x00000022u, + 0x00000007u, + 0x0004003du, + 0x0000000bu, + 0x0000000eu, + 0x0000000du, + 0x0004007cu, + 0x00000007u, + 0x0000000fu, + 0x0000000eu, + 0x0003003eu, + 0x00000009u, + 0x0000000fu, + 0x0004003du, + 0x00000007u, + 0x00000010u, + 0x00000009u, + 0x00050041u, + 0x00000016u, + 0x00000017u, + 0x00000014u, + 0x00000015u, + 0x0004003du, + 0x00000011u, + 0x00000018u, + 0x00000017u, + 0x0008004fu, + 0x00000007u, + 0x00000019u, + 0x00000018u, + 0x00000018u, + 0x00000000u, + 0x00000001u, + 0x00000002u, + 0x000500afu, + 0x0000001bu, + 0x0000001cu, + 0x00000010u, + 0x00000019u, + 0x0004009au, + 0x0000001au, + 0x0000001du, + 0x0000001cu, + 0x000300f7u, + 0x0000001fu, + 0x00000000u, + 0x000400fau, + 0x0000001du, + 0x0000001eu, + 0x0000001fu, + 0x000200f8u, + 0x0000001eu, + 0x000100fdu, + 0x000200f8u, + 0x0000001fu, + 0x0004003du, + 0x00000024u, + 0x00000027u, + 0x00000026u, + 0x00050041u, + 0x00000016u, + 0x00000029u, + 0x00000014u, + 0x00000028u, + 0x0004003du, + 0x00000011u, + 0x0000002au, + 0x00000029u, + 0x0008004fu, + 0x00000007u, + 0x0000002bu, + 0x0000002au, + 0x0000002au, + 0x00000000u, + 0x00000001u, + 0x00000002u, + 0x0004003du, + 0x00000007u, + 0x0000002cu, + 0x00000009u, + 0x00050080u, + 0x00000007u, + 0x0000002du, + 0x0000002bu, + 0x0000002cu, + 0x00040064u, + 0x00000023u, + 0x0000002eu, + 0x00000027u, + 0x0007005fu, + 0x0000002fu, + 0x00000030u, + 0x0000002eu, + 0x0000002du, + 0x00000002u, + 0x00000028u, + 0x00050051u, + 0x0000000au, + 0x00000032u, + 0x00000030u, + 0x00000000u, + 0x0003003eu, + 0x00000022u, + 0x00000032u, + 0x0004003du, + 0x00000033u, + 0x00000036u, + 0x00000035u, + 0x0004003du, + 0x00000007u, + 0x00000037u, + 0x00000009u, + 0x0004003du, + 0x0000000au, + 0x00000038u, + 0x00000022u, + 0x00070050u, + 0x0000002fu, + 0x00000039u, + 0x00000038u, + 0x00000031u, + 0x00000031u, + 0x00000031u, + 0x00040063u, + 0x00000036u, + 0x00000037u, + 0x00000039u, + 0x000100fdu, + 0x00010038u, +}; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireTextureReadback.inc b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireTextureReadback.inc index 05f84d2c..235ccd78 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireTextureReadback.inc +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireTextureReadback.inc @@ -147,7 +147,7 @@ image.image = resource->image; image.format = resource->format; image.aspect = resource->aspect; image.layout = resource->layout; image.trackedLayout = &resource->layout; image.levels = resource->mipLevels; const auto restore = image.layout == VK_IMAGE_LAYOUT_UNDEFINED ? VK_IMAGE_LAYOUT_GENERAL : image.layout; - TransitionWireImage(image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + auto& frame = m_frameContext.GetCurrent(); VkBufferImageCopy regions[2]{}; Uint32 regionCount = 0; const auto addCopy = [&](VkImageAspectFlags aspect, VkDeviceSize offset) { @@ -160,8 +160,35 @@ if (!depthStencil) addCopy(VK_IMAGE_ASPECT_COLOR_BIT, 0); if (depth) addCopy(VK_IMAGE_ASPECT_DEPTH_BIT, 0); if (stencil) addCopy(VK_IMAGE_ASPECT_STENCIL_BIT, stencilOffset); - auto& frame = m_frameContext.GetCurrent(); - vkCmdCopyImageToBuffer(frame.commandBuffer, image.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + // E5B9G9R9 on a device whose transfer reads canonicalize it: the words leave through a + // shader into a twin (WireE5RawCopy.h), which is what the buffer copy then reads. + VkImage copySource = image.image; + if (!depthStencil && m_textureManager->WireE5RawReadApplies(*resource)) { + TransitionWireImage(image, VK_IMAGE_LAYOUT_GENERAL); + WireE5RawCopyRegion region; + region.source = image.image; + region.volume = nativeVolume; + region.level = level; + region.baseLayer = nativeVolume ? 0u : layer; + region.layerCount = nativeVolume ? 1u : layerCount; + region.offset = regions[0].imageOffset; + region.extent = extent; + auto twin = MakeUnique(); + WireE5RawCopyObjects objects; + if (m_textureManager->RecordWireE5RawTwin(frame.commandBuffer, region, VK_IMAGE_LAYOUT_GENERAL, *twin, + objects)) { + copySource = twin->image; + regions[0].imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, region.layerCount}; + regions[0].imageOffset = {0, 0, 0}; + WireRetiredObjects retired{}; + retired.textures.push_back(Move(twin)); + retired.imageViews = {objects.sourceView, objects.twinView}; + retired.descriptorPools = {objects.descriptorPool}; + RetireWireObjects(Move(retired)); + } + } + if (copySource == image.image) TransitionWireImage(image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + vkCmdCopyImageToBuffer(frame.commandBuffer, copySource, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, readback.GetHandle(), regionCount, regions); TransitionWireImage(image, restore); VkMemoryBarrier host{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; diff --git a/MobileGL/MG_Test/SelfTest/BakedInternalShadersTest.cpp b/MobileGL/MG_Test/SelfTest/BakedInternalShadersTest.cpp index a43cdfb4..a35ec77d 100644 --- a/MobileGL/MG_Test/SelfTest/BakedInternalShadersTest.cpp +++ b/MobileGL/MG_Test/SelfTest/BakedInternalShadersTest.cpp @@ -61,6 +61,7 @@ using Uint32 = std::uint32_t; #include #include #include +#include struct BakedShader { const char* name; // ctest entry name, and the bake file's stem @@ -93,6 +94,19 @@ const BakedShader kBakedShaders[] = { "kWireColorBlitFragmentSpirv", EShLangFragment, glslang::EShTargetVulkan_1_0, glslang::EShTargetSpv_1_0, kWireColorBlitFragmentSpirv), + // The raw E5B9G9R9 read route (WireE5RawCopy.h): one compute program per source shape. + MGL_BAKED_ROW("WireE5RawCopy", + "MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy.comp", + "MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopySpirv.h", + "kWireE5RawCopySpirv", EShLangCompute, + glslang::EShTargetVulkan_1_0, glslang::EShTargetSpv_1_0, + kWireE5RawCopySpirv), + MGL_BAKED_ROW("WireE5RawCopy3D", + "MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopy3D.comp", + "MobileGL/MG_Backend/DirectVulkan/Renderer/WireE5RawCopySpirv.h", + "kWireE5RawCopy3DSpirv", EShLangCompute, + glslang::EShTargetVulkan_1_0, glslang::EShTargetSpv_1_0, + kWireE5RawCopy3DSpirv), MGL_BAKED_ROW("WireYuvConvertVertex", "MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvert.vert", "MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvertSpirv.h", diff --git a/MobileGL/MG_Util/SelfTest/DriverBugProbes.cpp b/MobileGL/MG_Util/SelfTest/DriverBugProbes.cpp index cff298ca..bb9ff00d 100644 --- a/MobileGL/MG_Util/SelfTest/DriverBugProbes.cpp +++ b/MobileGL/MG_Util/SelfTest/DriverBugProbes.cpp @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include @@ -2528,9 +2530,59 @@ namespace MobileGL::MG_Util::SelfTest { return DescribeDepthStencilResolvePassBug(measurement, shaderStencilExport); } + + // The same probe the Magma renderer runs at bring-up (VulkanRenderer::ArmWireE5RawReads), + // on a throwaway device with one graphics queue. + Optional ProbeE5TransferReadNormalizeBug(const VulkanDriverBugProbeContext& context) { + using namespace MG_Backend::DirectVulkan; + if (context.getInstanceProcAddr == nullptr) return std::nullopt; + const WireE5TransferProbeMeasurement measurement = RunWireE5TransferProbeOnThrowawayDevice( + context.getInstanceProcAddr, context.instance, context.physicalDevice, + context.graphicsQueueFamilyIndex); + const WireE5TransferProbeVerdict verdict = EvaluateWireE5TransferProbe(measurement); + MGLOG_I("Driver POST: E5B9G9R9 transfer probe verdict=%s: %s", WireE5TransferProbeVerdictName(verdict), + DescribeWireE5TransferProbe(measurement).c_str()); + if (verdict != WireE5TransferProbeVerdict::TransferReadNormalizes) return std::nullopt; + String detail = + "vkCmdCopyImageToBuffer out of a 30x30 VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 image returns " + "non-canonical shared-exponent words re-encoded (same value, different bits), while a " + "compute shader reading the same image through an R32_UINT view returns every stored " + "word: "; + detail += DescribeWireE5TransferProbe(measurement); + detail += + ". GL moves RGB9_E5 blocks bit-exactly (glGetTexImage with UNSIGNED_INT_5_9_9_9_REV, " + "glCopyImageSubData), so Magma reads such textures through that shader into an R32_UINT " + "twin and copies the twin instead (MOBILEGL_MAGMA_E5_RAW_READS=0 turns it off)"; + return DriverBugFinding{"E5B9G9R9 transfer reads canonicalize shared exponents", + DriverBugVerdict::Fixed, Move(detail)}; + } + + // The same probe the Magma renderer runs at bring-up (VulkanRenderer::ArmWireCopySliceSplit). + Optional ProbeCrossFormatSliceCopyBug(const VulkanDriverBugProbeContext& context) { + using namespace MG_Backend::DirectVulkan; + if (context.getInstanceProcAddr == nullptr) return std::nullopt; + const WireCopySliceProbeMeasurement measurement = RunWireCopySliceProbeOnThrowawayDevice( + context.getInstanceProcAddr, context.instance, context.physicalDevice, + context.graphicsQueueFamilyIndex); + const WireCopySliceProbeVerdict verdict = EvaluateWireCopySliceProbe(measurement); + MGLOG_I("Driver POST: slice copy probe verdict=%s: %s", WireCopySliceProbeVerdictName(verdict), + DescribeWireCopySliceProbe(measurement).c_str()); + if (verdict != WireCopySliceProbeVerdict::CrossFormatSlicesDropped) return std::nullopt; + String detail = + "a vkCmdCopyImage region between a 2D array and a 3D image of DIFFERENT (size-compatible) " + "formats moves only its first slice, while the same copy as one region per slice is exact: "; + detail += DescribeWireCopySliceProbe(measurement); + detail += + ". Magma splits such glCopyImageSubData calls one slice per region " + "(MOBILEGL_MAGMA_SPLIT_SLICE_COPIES=0 turns it off)"; + return DriverBugFinding{"Cross-format 2D-array <-> 3D image copies drop slices", + DriverBugVerdict::Fixed, Move(detail)}; + } using VulkanDriverBugProbeFn = Optional (*)(const VulkanDriverBugProbeContext&); constexpr VulkanDriverBugProbeFn kVulkanDriverBugProbes[] = { &ProbeDepthStencilResolvePassBug, + &ProbeE5TransferReadNormalizeBug, + &ProbeCrossFormatSliceCopyBug, }; } // namespace