diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferObject.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferObject.cpp index 1393cbe2..3b017c3b 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferObject.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferObject.cpp @@ -157,6 +157,25 @@ namespace MobileGL::MG_Backend::DirectVulkan { return true; } + Bool VkBufferObject::Invalidate(VkDeviceSize size, VkDeviceSize offset) { + MOBILEGL_ASSERT(IsValid(), "VkBufferObject::Invalidate called on invalid buffer"); + MOBILEGL_ASSERT(IsMapped(), "VkBufferObject::Invalidate requires mapped memory"); + MOBILEGL_ASSERT(offset <= m_size, "VkBufferObject::Invalidate offset out of range"); + + const VkDeviceSize resolvedSize = size == VK_WHOLE_SIZE ? m_size - offset : size; + MOBILEGL_ASSERT(offset + resolvedSize <= m_size, "VkBufferObject::Invalidate range out of bounds"); + if (resolvedSize == 0) { + return true; + } + + const VkResult result = vmaInvalidateAllocation(m_allocator, m_allocation, offset, resolvedSize); + if (result != VK_SUCCESS) { + MGLOG_E("VkBufferObject::Invalidate failed: vmaInvalidateAllocation returned %d", result); + return false; + } + return true; + } + BufferSlice VkBufferObject::GetSlice(VkDeviceSize offset, VkDeviceSize size) const { MOBILEGL_ASSERT(offset <= m_size, "VkBufferObject::GetSlice offset out of range"); const VkDeviceSize resolvedSize = (size == VK_WHOLE_SIZE) ? (m_size - offset) : size; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferObject.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferObject.h index 24314562..47ac8521 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferObject.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkBufferObject.h @@ -44,6 +44,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { void* Map(); void Unmap(); Bool Upload(const void* data, VkDeviceSize size, VkDeviceSize offset = 0); + Bool Invalidate(VkDeviceSize size = VK_WHOLE_SIZE, VkDeviceSize offset = 0); VkBuffer GetHandle() const { return m_buffer; } VkDeviceSize GetSize() const { return m_size; } diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp index 74ec8b80..c455d32f 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp @@ -19,11 +19,13 @@ #include "MG_Util/Converters/GLToMG/TextureEnumConverter.h" #include "MG_Util/Converters/MGToVk/RenderStateEnumConverter.h" #include "MG_Util/Converters/MGToVk/TextureEnumConverter.h" +#include "MG_Util/Math/HalfFloat.h" #include "MG_Util/Metrics/TextureMetrics.h" #include #include #include #include +#include #include #if defined(__APPLE__) @@ -1123,6 +1125,7 @@ void main() { VkImage image = VK_NULL_HANDLE; VkImageLayout* trackedLayout = nullptr; VkImageAspectFlags aspectMask = VK_IMAGE_ASPECT_NONE; + VkFormat format = VK_FORMAT_UNDEFINED; IntVec2 extent = {0, 0}; Uint32 mipLevel = 0; Uint32 mipLevelCount = 1; @@ -1286,6 +1289,7 @@ void main() { outBinding.image = swapchainObject.GetImage(swapchainImageIndex); outBinding.trackedLayout = nullptr; outBinding.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + outBinding.format = swapchainObject.GetSurfaceFormat().format; const auto extent = swapchainObject.GetExtent(); outBinding.extent = {static_cast(extent.width), static_cast(extent.height)}; outBinding.mipLevel = 0; @@ -1334,6 +1338,7 @@ void main() { outBinding.image = resource->image; outBinding.trackedLayout = &resource->layout; outBinding.aspectMask = resource->aspect; + outBinding.format = resource->format; const auto attachmentExtent = attachment.GetSize(); outBinding.extent = {attachmentExtent.x(), attachmentExtent.y()}; outBinding.mipLevel = static_cast(std::max(attachment.GetTextureLevel(), 0)); @@ -1580,19 +1585,54 @@ void main() { } } - static Bool IsBgraVkFormat(VkFormat format) { - switch (format) { + static Bool DecodeReadbackPixel(const Uint8* source, VkFormat sourceFormat, Float* rgba) { + switch (sourceFormat) { + case VK_FORMAT_R8G8B8A8_UNORM: + case VK_FORMAT_R8G8B8A8_SRGB: + rgba[0] = static_cast(source[0]) / 255.0f; + rgba[1] = static_cast(source[1]) / 255.0f; + rgba[2] = static_cast(source[2]) / 255.0f; + rgba[3] = static_cast(source[3]) / 255.0f; + return true; case VK_FORMAT_B8G8R8A8_UNORM: - case VK_FORMAT_B8G8R8A8_SNORM: case VK_FORMAT_B8G8R8A8_SRGB: - case VK_FORMAT_B8G8R8A8_USCALED: - case VK_FORMAT_B8G8R8A8_SSCALED: + rgba[0] = static_cast(source[2]) / 255.0f; + rgba[1] = static_cast(source[1]) / 255.0f; + rgba[2] = static_cast(source[0]) / 255.0f; + rgba[3] = static_cast(source[3]) / 255.0f; + return true; + case VK_FORMAT_R16G16B16A16_UNORM: + for (SizeT component = 0; component < 4; ++component) { + Uint16 value = 0; + Memcpy(&value, source + component * sizeof(value), sizeof(value)); + rgba[component] = static_cast(value) / 65535.0f; + } + return true; + case VK_FORMAT_R16G16B16A16_SFLOAT: + for (SizeT component = 0; component < 4; ++component) { + Uint16 value = 0; + Memcpy(&value, source + component * sizeof(value), sizeof(value)); + rgba[component] = MG_Util::DecodeHalfBitsToFloat(value); + } + return true; + case VK_FORMAT_R32G32B32A32_SFLOAT: + Memcpy(rgba, source, sizeof(Float) * 4); return true; default: return false; } } + static Uint8 EncodeReadbackUnorm8(Float value) { + if (!(value > 0.0f)) { + return 0; + } + if (value >= 1.0f) { + return 255; + } + return static_cast(value * 255.0f + 0.5f); + } + // Remap raw swapchain pixels (top-left origin, preTransform-rotated) into // GL-oriented pixels (bottom-left origin) for the retrace snapshot path. // Mirrors the removed GetPresentedDumpPixel mapping plus the Y-origin flip @@ -1602,6 +1642,7 @@ void main() { static Bool RemapDefaultFboReadbackToGLOrientation(const Uint8* rawPixels, VkExtent2D rawExtent, VkSurfaceTransformFlagBitsKHR preTransform, + SizeT texelSize, Uint8* outPixels) { if (IsQuarterTurnPreTransform(preTransform)) { return false; @@ -1625,12 +1666,9 @@ void main() { default: break; } - const Uint8* src = rawPixels + (static_cast(rawY) * w + rawX) * 4; - Uint8* dst = outPixels + (static_cast(outY) * w + outX) * 4; - dst[0] = src[0]; - dst[1] = src[1]; - dst[2] = src[2]; - dst[3] = src[3]; + const Uint8* src = rawPixels + (static_cast(rawY) * w + rawX) * texelSize; + Uint8* dst = outPixels + (static_cast(outY) * w + outX) * texelSize; + Memcpy(dst, src, texelSize); } } return true; @@ -1654,12 +1692,11 @@ void main() { } } - static void StoreReadbackPixel(const Uint8* src, Bool srcIsBgra, GLenum dstFormat, Uint8* dst) { - const Uint8 r = srcIsBgra ? src[2] : src[0]; - const Uint8 g = src[1]; - const Uint8 b = srcIsBgra ? src[0] : src[2]; - const Uint8 a = src[3]; - + static void StoreReadbackPixel(const Float* rgba, GLenum dstFormat, Uint8* dst) { + const Uint8 r = EncodeReadbackUnorm8(rgba[0]); + const Uint8 g = EncodeReadbackUnorm8(rgba[1]); + const Uint8 b = EncodeReadbackUnorm8(rgba[2]); + const Uint8 a = EncodeReadbackUnorm8(rgba[3]); switch (dstFormat) { case GL_RGB: dst[0] = r; @@ -1688,13 +1725,11 @@ void main() { } } - static void StoreReadbackPixelFloat(const Uint8* src, Bool srcIsBgra, GLenum dstFormat, Float* dst) { - const Float r = static_cast(srcIsBgra ? src[2] : src[0]) / 255.0f; - const Float g = static_cast(src[1]) / 255.0f; - const Float b = static_cast(srcIsBgra ? src[0] : src[2]) / 255.0f; - const Float a = static_cast(src[3]) / 255.0f; - - // TODO: extend readback packing to integer/depth formats instead of only normalized color formats. + static void StoreReadbackPixelFloat(const Float* rgba, GLenum dstFormat, Float* dst) { + const Float r = rgba[0]; + const Float g = rgba[1]; + const Float b = rgba[2]; + const Float a = rgba[3]; switch (dstFormat) { case GL_RGB: dst[0] = r; @@ -1752,20 +1787,11 @@ void main() { (static_cast(width) * static_cast(dstChannels) * dstComponentBytes); Vector packed(packedSize, 0); - const Bool srcIsBgra = IsBgraVkFormat(srcFormat); - for (GLsizei row = 0; row < height; ++row) { - const Uint8* srcRow = srcPixels + static_cast(row) * static_cast(width) * 4; - Uint8* dstRow = packed.data() + dstOffset + static_cast(row) * dstRowStride; - for (GLsizei col = 0; col < width; ++col) { - const auto* src = srcRow + static_cast(col) * 4; - auto* dst = dstRow + static_cast(col) * static_cast(dstChannels) * - dstComponentBytes; - if (type == GL_FLOAT) { - StoreReadbackPixelFloat(src, srcIsBgra, format, reinterpret_cast(dst)); - } else { - StoreReadbackPixel(src, srcIsBgra, format, dst); - } - } + if (!VulkanRenderer::ConvertReadbackPixels(srcPixels, srcFormat, width, height, format, type, + dstRowStride, packed.data() + dstOffset)) { + MGLOG_E("DirectVulkan readback skipped: unsupported source format=%d", + static_cast(srcFormat)); + return false; } const auto& pixelPackBufferObject = @@ -1787,6 +1813,60 @@ void main() { } } // namespace + SizeT VulkanRenderer::GetReadbackTexelSize(VkFormat sourceFormat) { + const VKU_FORMAT_INFO formatInfo = vkuGetFormatInfo(sourceFormat); + if (formatInfo.texels_per_block != 1) { + return 0; + } + return formatInfo.texel_block_size; + } + + Bool VulkanRenderer::ConvertReadbackPixels(const Uint8* sourcePixels, VkFormat sourceFormat, + GLsizei width, GLsizei height, GLenum destinationFormat, + GLenum destinationType, SizeT destinationRowStride, + Uint8* destinationPixels) { + if (width <= 0 || height <= 0) { + return true; + } + if (sourcePixels == nullptr || destinationPixels == nullptr) { + return false; + } + + const SizeT sourceTexelSize = GetReadbackTexelSize(sourceFormat); + const Int destinationChannels = GetReadbackChannelCount(destinationFormat); + if (sourceTexelSize == 0 || destinationChannels == 0 || + (destinationType != GL_UNSIGNED_BYTE && destinationType != GL_FLOAT)) { + return false; + } + const SizeT destinationComponentSize = destinationType == GL_FLOAT ? sizeof(Float) : sizeof(Uint8); + const SizeT destinationPixelSize = static_cast(destinationChannels) * destinationComponentSize; + if (destinationRowStride < static_cast(width) * destinationPixelSize) { + return false; + } + + for (GLsizei row = 0; row < height; ++row) { + const Uint8* sourceRow = sourcePixels + + static_cast(row) * static_cast(width) * sourceTexelSize; + Uint8* destinationRow = destinationPixels + static_cast(row) * destinationRowStride; + for (GLsizei column = 0; column < width; ++column) { + const Uint8* source = sourceRow + static_cast(column) * sourceTexelSize; + Uint8* destination = destinationRow + static_cast(column) * destinationPixelSize; + Float rgba[4]{}; + if (!DecodeReadbackPixel(source, sourceFormat, rgba)) { + return false; + } + if (destinationType == GL_FLOAT) { + Float converted[4]{}; + StoreReadbackPixelFloat(rgba, destinationFormat, converted); + Memcpy(destination, converted, destinationPixelSize); + } else { + StoreReadbackPixel(rgba, destinationFormat, destination); + } + } + } + return true; + } + VkBool32 VulkanRenderer::DebugCallback(VkDebugUtilsMessageSeverityFlagBitsEXT messageSeverity, VkDebugUtilsMessageTypeFlagsEXT messageType, const VkDebugUtilsMessengerCallbackDataEXT* pCallbackData, void* pUserData) { @@ -5340,6 +5420,7 @@ void main() { VK_ACCESS_TRANSFER_WRITE_BIT, dstRestoreAccessMask, copyAspectMask, dstMipLevel, 1); MOBILEGL_ASSERT(dstRestored, "%s: failed to restore destination image layout", __func__); } + } Bool VulkanRenderer::SubmitReadbackCommandsAndWait(FrameContext::FrameData& frame) { @@ -5424,7 +5505,15 @@ void main() { return; } - const VkDeviceSize readbackSize = static_cast(width) * static_cast(height) * 4; + const VkFormat srcFormat = srcBinding.format; + const SizeT sourceTexelSize = GetReadbackTexelSize(srcFormat); + if (sourceTexelSize == 0) { + MGLOG_E("DirectVulkan::ReadPixels skipped: unsupported source format=%d", + static_cast(srcFormat)); + return; + } + const VkDeviceSize readbackSize = static_cast(width) * + static_cast(height) * sourceTexelSize; VkBufferObject readback; if (!readback.Create({ .allocator = m_allocator, @@ -5460,7 +5549,7 @@ void main() { VkBufferImageCopy copyRegion{}; copyRegion.imageSubresource.aspectMask = srcBinding.aspectMask; copyRegion.imageSubresource.mipLevel = srcBinding.mipLevel; - copyRegion.imageSubresource.baseArrayLayer = 0; + copyRegion.imageSubresource.baseArrayLayer = srcBinding.baseArrayLayer; copyRegion.imageSubresource.layerCount = 1; copyRegion.imageOffset = {x, y, 0}; copyRegion.imageExtent = {static_cast(width), static_cast(height), 1}; @@ -5495,14 +5584,18 @@ void main() { MGLOG_E("DirectVulkan::ReadPixels skipped: failed to map readback buffer"); return; } - const VkFormat srcFormat = readIsDefaultFbo ? m_swapchainObject.GetSurfaceFormat().format : VK_FORMAT_R8G8B8A8_UNORM; + if (!readback.Invalidate(readbackSize)) { + MGLOG_E("DirectVulkan::ReadPixels skipped: failed to invalidate readback buffer"); + return; + } if (readIsDefaultFbo) { const VkExtent2D swapchainExtent = m_swapchainObject.GetExtent(); const VkSurfaceTransformFlagBitsKHR preTransform = m_swapchainObject.GetPreTransform(); if (static_cast(width) == swapchainExtent.width && static_cast(height) == swapchainExtent.height) { - Vector remapped(static_cast(width) * static_cast(height) * 4); + Vector remapped(static_cast(width) * static_cast(height) * sourceTexelSize); if (RemapDefaultFboReadbackToGLOrientation(mapped, swapchainExtent, preTransform, + sourceTexelSize, remapped.data())) { PackReadbackToClientOrPbo(remapped.data(), srcFormat, width, height, format, type, pixels); return; @@ -5568,9 +5661,10 @@ void main() { } if (bufSize >= 0) { const Int dstChannels = GetReadbackChannelCount(format); - if (type == GL_UNSIGNED_BYTE && dstChannels > 0) { + if ((type == GL_UNSIGNED_BYTE || type == GL_FLOAT) && dstChannels > 0) { + const SizeT dstComponentSize = type == GL_FLOAT ? sizeof(Float) : sizeof(Uint8); const SizeT minSize = static_cast(width) * static_cast(height) * - static_cast(dstChannels); + static_cast(dstChannels) * dstComponentSize; if (static_cast(bufSize) < minSize) { MGLOG_E("DirectVulkan::GetTextureImage skipped: destination buffer is too small"); return; @@ -5578,7 +5672,14 @@ void main() { } } - const VkDeviceSize readbackSize = static_cast(width) * static_cast(height) * 4; + const SizeT sourceTexelSize = GetReadbackTexelSize(resource->format); + if (sourceTexelSize == 0) { + MGLOG_E("DirectVulkan::GetTexImage skipped: unsupported source format=%d", + static_cast(resource->format)); + return; + } + const VkDeviceSize readbackSize = static_cast(width) * + static_cast(height) * sourceTexelSize; VkBufferObject readback; if (!readback.Create({ .allocator = m_allocator, @@ -5629,6 +5730,10 @@ void main() { MGLOG_E("DirectVulkan::GetTextureImage skipped: failed to map readback buffer"); return; } + if (!readback.Invalidate(readbackSize)) { + MGLOG_E("DirectVulkan::GetTextureImage skipped: failed to invalidate readback buffer"); + return; + } PackReadbackToClientOrPbo(mapped, resource->format, width, height, format, type, pixels); } diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h index 23326434..e533b9ef 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h @@ -165,6 +165,11 @@ namespace MobileGL::MG_Backend::DirectVulkan { GLsizei srcWidth, GLsizei srcHeight, GLsizei srcDepth); void GenerateMipmap(GLenum target); void ReadPixels(GLint x, GLint y, GLsizei width, GLsizei height, GLenum format, GLenum type, void* pixels); + static SizeT GetReadbackTexelSize(VkFormat sourceFormat); + static Bool ConvertReadbackPixels(const Uint8* sourcePixels, VkFormat sourceFormat, + GLsizei width, GLsizei height, GLenum destinationFormat, + GLenum destinationType, SizeT destinationRowStride, + Uint8* destinationPixels); void GetTexImage(GLenum target, GLint level, GLenum format, GLenum type, GLvoid* pixels); void GetTextureImage(const SharedPtr& texture, TextureUploadTarget uploadTarget, GLint level, GLenum format, GLenum type, diff --git a/MobileGL/MG_Test/Program/ProgramUtilTest.cpp b/MobileGL/MG_Test/Program/ProgramUtilTest.cpp index 5ded08a2..6319b262 100644 --- a/MobileGL/MG_Test/Program/ProgramUtilTest.cpp +++ b/MobileGL/MG_Test/Program/ProgramUtilTest.cpp @@ -8,7 +8,9 @@ #include +#include #include +#include #include "Includes.h" #include "Init.h" @@ -1497,3 +1499,128 @@ void main() { EXPECT_NE(source.find("shared float sharedScratch[8];"), String::npos); EXPECT_NE(source.find("layout(std140) uniform Blk"), String::npos); } + +namespace { + String MakeLinearSubgroupPrefixScanShader() { + return R"(#version 460 core +#extension GL_KHR_shader_subgroup_arithmetic : enable +layout(local_size_x = 1024) in; +shared float prefixSumCache[64]; + +layout(std430, binding = 0) writeonly buffer OutputBuffer { + float outputValues[]; +}; + +void main() { + float importance = 1.0f; + float prefixSum = subgroupInclusiveAdd(importance); + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) prefixSumCache[gl_SubgroupID] = prefixSum; + barrier(); + uint loopLength = uint(findMSB(gl_NumSubgroups)); + loopLength += uint(gl_NumSubgroups - (1u << (loopLength - 1u)) > 0u); + for (uint i = 0; i < loopLength; i++) { + if ((gl_SubgroupID & (1u << i)) > 0u) { + prefixSum += prefixSumCache[(gl_SubgroupID >> i << i) - 1u]; + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) prefixSumCache[gl_SubgroupID] = prefixSum; + } + barrier(); + } + if (gl_LocalInvocationID.x == uint(1024 - 1)) prefixSumCache[0] = prefixSum; + barrier(); + float sum = prefixSumCache[0]; + float warp = (prefixSum - importance) / sum - float(gl_LocalInvocationID.x + 1u) / float(1024); + outputValues[gl_GlobalInvocationID.x] = warp; +} +)"; + } +} // namespace + +TEST_F(ProgramUtilTest, RewriteLinearSubgroupPrefixScanUsesSharedMemoryAndProducesValidSpirv) { + using namespace MG_Util::ShaderTranspiler; + + String source = MakeLinearSubgroupPrefixScanShader(); + ASSERT_TRUE(RewriteLinearSubgroupPrefixScanForVulkan(ShaderStage::Compute, 64, source)); + + EXPECT_NE(source.find("shared float prefixSumCache[1024]"), String::npos) << source; + EXPECT_NE(source.find("mglVirtualSubgroupInvocation"), String::npos) << source; + EXPECT_NE(source.find("for (uint mglPrefixLane"), String::npos) << source; + EXPECT_EQ(source.find("subgroupInclusiveAdd"), String::npos) << source; + EXPECT_EQ(source.find("gl_Subgroup"), String::npos) << source; + + const String onceRewritten = source; + EXPECT_FALSE(RewriteLinearSubgroupPrefixScanForVulkan(ShaderStage::Compute, 64, source)); + EXPECT_EQ(source, onceRewritten); + + ShaderAttrib shaderAttrib{.shaderType = GL_COMPUTE_SHADER, .sourceStr = source}; + auto shaderResult = ShaderCompiler::CompileShader(shaderAttrib); + ASSERT_TRUE(shaderResult) << shaderResult.error().log << "\nsource:\n" << source; + + ProgramAttrib programAttrib{.shaders = {shaderResult.value()}}; + auto programResult = ShaderCompiler::LinkProgram(programAttrib); + ASSERT_TRUE(programResult) << programResult.error().log; + + ProgramBinaryAttrib binaryAttrib{.shaderTypes = {GL_COMPUTE_SHADER}, .program = *programResult.value()}; + auto binaryResult = ShaderCompiler::GetSpirvBinaryFromProgram(binaryAttrib); + ASSERT_TRUE(binaryResult) << binaryResult.error().log; + ASSERT_EQ(binaryResult->size(), 1u); + + String validationDiagnostics; + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_1); + tools.SetMessageConsumer([&](spv_message_level_t, const char*, const spv_position_t&, const char* message) { + validationDiagnostics += message; + validationDiagnostics += '\n'; + }); + EXPECT_TRUE(tools.Validate(binaryResult->front())) << validationDiagnostics; + + String spirvText; + ASSERT_TRUE(tools.Disassemble(binaryResult->front(), &spirvText)); + EXPECT_EQ(spirvText.find("OpGroupNonUniform"), String::npos) << spirvText; +} + +TEST_F(ProgramUtilTest, RewriteLinearSubgroupPrefixScanRejectsOtherStagesAndSubgroupWidths) { + using namespace MG_Util::ShaderTranspiler; + + const String original = MakeLinearSubgroupPrefixScanShader(); + for (const auto& [stage, subgroupSize] : + {std::pair{ShaderStage::Compute, Uint32{32}}, std::pair{ShaderStage::Fragment, Uint32{64}}, + std::pair{ShaderStage::Compute, Uint32{96}}}) { + String source = original; + EXPECT_FALSE(RewriteLinearSubgroupPrefixScanForVulkan(stage, subgroupSize, source)); + EXPECT_EQ(source, original); + } +} + +TEST_F(ProgramUtilTest, RewriteLinearSubgroupPrefixScanRejectsPartialOrUnsafeTemplateMatches) { + using namespace MG_Util::ShaderTranspiler; + + const auto expectUnchanged = [](String source) { + const String original = source; + EXPECT_FALSE(RewriteLinearSubgroupPrefixScanForVulkan(ShaderStage::Compute, 64, source)); + EXPECT_EQ(source, original); + }; + + String wrongLocalSize = MakeLinearSubgroupPrefixScanShader(); + wrongLocalSize.replace(wrongLocalSize.find("local_size_x = 1024"), std::strlen("local_size_x = 1024"), + "local_size_x = 512"); + expectUnchanged(std::move(wrongLocalSize)); + + String cacheHasAnotherUse = MakeLinearSubgroupPrefixScanShader(); + cacheHasAnotherUse.insert(cacheHasAnotherUse.find("float importance"), "prefixSumCache[0] = 0.0f;\n "); + expectUnchanged(std::move(cacheHasAnotherUse)); + + String extraSubgroupBuiltin = MakeLinearSubgroupPrefixScanShader(); + extraSubgroupBuiltin.insert(extraSubgroupBuiltin.find("float importance"), + "uvec4 extraMask = gl_SubgroupEqMask;\n "); + expectUnchanged(std::move(extraSubgroupBuiltin)); + + String alteredBarrier = MakeLinearSubgroupPrefixScanShader(); + alteredBarrier.replace(alteredBarrier.find("barrier();"), std::strlen("barrier();"), "memoryBarrierShared();"); + expectUnchanged(std::move(alteredBarrier)); + + String nestedScan = MakeLinearSubgroupPrefixScanShader(); + nestedScan.insert(nestedScan.find("float prefixSum ="), "if (importance > 0.0f) {\n "); + const SizeT consumerEnd = nestedScan.find(';', nestedScan.find("float warp =")); + ASSERT_NE(consumerEnd, String::npos); + nestedScan.insert(consumerEnd + 1, "\n }"); + expectUnchanged(std::move(nestedScan)); +} diff --git a/MobileGL/MG_Test/SanityTest.cpp b/MobileGL/MG_Test/SanityTest.cpp index 9cf420d7..60b024ea 100644 --- a/MobileGL/MG_Test/SanityTest.cpp +++ b/MobileGL/MG_Test/SanityTest.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -731,6 +732,68 @@ TEST(DirectVulkanSanity, CommandMemoryBarrierMakesIndirectDrawCommandsVisible) { EXPECT_EQ(storageOnlyBarrier.dstAccessMask & VK_ACCESS_INDIRECT_COMMAND_READ_BIT, 0u); } +TEST(DirectVulkanSanity, ReadbackUsesTheSourceFormatTexelSize) { + using MobileGL::MG_Backend::DirectVulkan::VulkanRenderer; + + EXPECT_EQ(VulkanRenderer::GetReadbackTexelSize(VK_FORMAT_R8G8B8A8_UNORM), 4u); + EXPECT_EQ(VulkanRenderer::GetReadbackTexelSize(VK_FORMAT_R16G16B16A16_SFLOAT), 8u); + EXPECT_EQ(VulkanRenderer::GetReadbackTexelSize(VK_FORMAT_R32G32B32A32_SFLOAT), 16u); +} + +TEST(DirectVulkanSanity, ReadbackConvertsRgba8AndRgba16fPixels) { + using MobileGL::MG_Backend::DirectVulkan::VulkanRenderer; + using MobileGL::MG_Util::EncodeFloatToHalfBits; + + const MobileGL::Uint8 rgba8[] = {17, 34, 51, 68, 85, 102, 119, 136}; + MobileGL::Uint8 rgba8Result[sizeof(rgba8)]{}; + ASSERT_TRUE(VulkanRenderer::ConvertReadbackPixels( + rgba8, VK_FORMAT_R8G8B8A8_UNORM, 2, 1, GL_RGBA, GL_UNSIGNED_BYTE, + sizeof(rgba8Result), rgba8Result)); + EXPECT_TRUE(std::equal(std::begin(rgba8), std::end(rgba8), std::begin(rgba8Result))); + + const MobileGL::Uint8 bgra8[] = {51, 34, 17, 68}; + MobileGL::Uint8 bgra8Result[4]{}; + ASSERT_TRUE(VulkanRenderer::ConvertReadbackPixels( + bgra8, VK_FORMAT_B8G8R8A8_UNORM, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, + sizeof(bgra8Result), bgra8Result)); + const MobileGL::Uint8 expectedBgra8[] = {17, 34, 51, 68}; + EXPECT_TRUE(std::equal(std::begin(expectedBgra8), std::end(expectedBgra8), std::begin(bgra8Result))); + + const MobileGL::Uint16 rgba16f[] = { + EncodeFloatToHalfBits(-0.25f), EncodeFloatToHalfBits(0.5f), EncodeFloatToHalfBits(1.5f), + EncodeFloatToHalfBits(1.0f), EncodeFloatToHalfBits(0.25f), EncodeFloatToHalfBits(0.0f), + EncodeFloatToHalfBits(1.0f), EncodeFloatToHalfBits(0.5f), + EncodeFloatToHalfBits(0.75f), EncodeFloatToHalfBits(0.125f), EncodeFloatToHalfBits(-1.0f), + EncodeFloatToHalfBits(2.0f), EncodeFloatToHalfBits(1.0f), EncodeFloatToHalfBits(0.75f), + EncodeFloatToHalfBits(0.25f), EncodeFloatToHalfBits(0.0f), + }; + constexpr MobileGL::SizeT kDestinationRowStride = 12; + MobileGL::Uint8 rgba16fResult[kDestinationRowStride * 2]; + std::fill(std::begin(rgba16fResult), std::end(rgba16fResult), 0xCD); + ASSERT_TRUE(VulkanRenderer::ConvertReadbackPixels( + reinterpret_cast(rgba16f), VK_FORMAT_R16G16B16A16_SFLOAT, + 2, 2, GL_RGBA, GL_UNSIGNED_BYTE, kDestinationRowStride, rgba16fResult)); + const MobileGL::Uint8 expectedRgba16fRow0[] = {0, 128, 255, 255, 64, 0, 255, 128}; + const MobileGL::Uint8 expectedRgba16fRow1[] = {191, 32, 0, 255, 255, 191, 64, 0}; + EXPECT_TRUE(std::equal(std::begin(expectedRgba16fRow0), std::end(expectedRgba16fRow0), + std::begin(rgba16fResult))); + EXPECT_TRUE(std::equal(std::begin(expectedRgba16fRow1), std::end(expectedRgba16fRow1), + std::begin(rgba16fResult) + kDestinationRowStride)); + EXPECT_TRUE(std::all_of(std::begin(rgba16fResult) + 8, + std::begin(rgba16fResult) + kDestinationRowStride, + [](MobileGL::Uint8 value) { return value == 0xCD; })); + + MobileGL::Float rgba16fFloatResult[16]{}; + ASSERT_TRUE(VulkanRenderer::ConvertReadbackPixels( + reinterpret_cast(rgba16f), VK_FORMAT_R16G16B16A16_SFLOAT, + 2, 2, GL_RGBA, GL_FLOAT, sizeof(MobileGL::Float) * 8, + reinterpret_cast(rgba16fFloatResult))); + EXPECT_FLOAT_EQ(rgba16fFloatResult[0], -0.25f); + EXPECT_FLOAT_EQ(rgba16fFloatResult[1], 0.5f); + EXPECT_FLOAT_EQ(rgba16fFloatResult[2], 1.5f); + EXPECT_FLOAT_EQ(rgba16fFloatResult[3], 1.0f); +} + TEST(DirectVulkanSanity, DrawIndexedIndirectCommandMatchesGlAndVulkanLayout) { using namespace MobileGL::MG_Backend::DirectVulkan; diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.cpp b/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.cpp index 87a82d69..4bf6d0dc 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.cpp +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.cpp @@ -10,10 +10,14 @@ #include #include +#include +#include #include namespace { using MobileGL::SizeT; + using MobileGL::String; + using MobileGL::Vector; bool IsIdentifierChar(char ch) { return (ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || ch == '_'; @@ -91,6 +95,375 @@ namespace { return masked; } + struct CodeToken { + String text; + SizeT begin = 0; + SizeT end = 0; + }; + + Vector TokenizeCode(const String& source) { + const String masked = MaskCommentsAndQuotedText(source); + Vector tokens; + tokens.reserve(source.size() / 4); + + SizeT pos = 0; + while (pos < masked.size()) { + const char ch = masked[pos]; + if (std::isspace(static_cast(ch))) { + ++pos; + continue; + } + + const SizeT begin = pos; + if (IsIdentifierStart(ch)) { + ++pos; + while (pos < masked.size() && IsIdentifierChar(masked[pos])) { + ++pos; + } + } else if (std::isdigit(static_cast(ch))) { + ++pos; + while (pos < masked.size()) { + const char numberChar = masked[pos]; + if (!IsIdentifierChar(numberChar) && numberChar != '.') { + break; + } + ++pos; + } + } else { + ++pos; + if (pos < masked.size()) { + const String twoChars = masked.substr(begin, 2); + if (twoChars == "==" || twoChars == "!=" || twoChars == "<=" || twoChars == ">=" || + twoChars == "+=" || twoChars == "-=" || twoChars == "<<" || twoChars == ">>" || + twoChars == "++" || twoChars == "--" || twoChars == "&&" || twoChars == "||") { + ++pos; + } + } + } + + tokens.push_back(CodeToken{source.substr(begin, pos - begin), begin, pos}); + } + return tokens; + } + + bool IsIdentifierToken(const CodeToken& token) { + if (token.text.empty() || !IsIdentifierStart(token.text.front())) { + return false; + } + return std::all_of(token.text.begin() + 1, token.text.end(), IsIdentifierChar); + } + + class TokenCursor { + public: + TokenCursor(const Vector& tokens, SizeT position) : m_tokens(tokens), m_position(position) {} + + bool Consume(const char* expected) { + if (m_position >= m_tokens.size() || m_tokens[m_position].text != expected) { + return false; + } + ++m_position; + return true; + } + + bool ConsumeAnyIdentifier(String& identifier) { + if (m_position >= m_tokens.size() || !IsIdentifierToken(m_tokens[m_position])) { + return false; + } + identifier = m_tokens[m_position++].text; + return true; + } + + bool ConsumeAnyIdentifier() { + if (m_position >= m_tokens.size() || !IsIdentifierToken(m_tokens[m_position])) { + return false; + } + ++m_position; + return true; + } + + bool ConsumeIdentifier(const String& expected) { + if (m_position >= m_tokens.size() || !IsIdentifierToken(m_tokens[m_position]) || + m_tokens[m_position].text != expected) { + return false; + } + ++m_position; + return true; + } + + SizeT Position() const { return m_position; } + + private: + const Vector& m_tokens; + SizeT m_position; + }; + + SizeT CountToken(const Vector& tokens, const String& tokenText) { + return static_cast(std::count_if(tokens.begin(), tokens.end(), + [&](const CodeToken& token) { return token.text == tokenText; })); + } + + bool HasIdentifierWithPrefixOutsideAllowed(const Vector& tokens, const String& prefix, + std::initializer_list allowedIdentifiers) { + return std::any_of(tokens.begin(), tokens.end(), [&](const CodeToken& token) { + if (!IsIdentifierToken(token) || !token.text.starts_with(prefix)) { + return false; + } + return std::none_of(allowedIdentifiers.begin(), allowedIdentifiers.end(), + [&](const char* allowed) { return token.text == allowed; }); + }); + } + + bool MatchTokenSequence(const Vector& tokens, SizeT position, + std::initializer_list expected) { + if (position + expected.size() > tokens.size()) { + return false; + } + for (const char* token : expected) { + if (tokens[position++].text != token) { + return false; + } + } + return true; + } + + struct LinearPrefixScanMatch { + SizeT sharedArraySizeBegin = 0; + SizeT sharedArraySizeEnd = 0; + SizeT scanBegin = 0; + SizeT scanEnd = 0; + String cache; + String importance; + String prefixSum; + String loopLength; + String loopIndex; + String sum; + }; + + bool ParseLinearPrefixScanTemplate(const Vector& tokens, LinearPrefixScanMatch& match) { + // The workaround deliberately recognizes one complete algorithm, not merely the + // subgroupInclusiveAdd token. Changing scratch storage is only safe when that storage is + // private to this scan and the workgroup has exactly 1024 X invocations. + SizeT localSizeDeclarationCount = 0; + for (SizeT i = 0; i < tokens.size(); ++i) { + if (MatchTokenSequence(tokens, i, {"layout", "(", "local_size_x", "=", "1024", ")", "in", ";"})) { + ++localSizeDeclarationCount; + } + } + if (localSizeDeclarationCount != 1) { + return false; + } + + SizeT sharedDeclarationIndex = String::npos; + SizeT sharedDeclarationCount = 0; + String cacheName; + for (SizeT i = 0; i + 6 < tokens.size(); ++i) { + if (tokens[i].text != "shared" || tokens[i + 1].text != "float" || !IsIdentifierToken(tokens[i + 2]) || + tokens[i + 3].text != "[" || tokens[i + 4].text != "64" || tokens[i + 5].text != "]" || + tokens[i + 6].text != ";") { + continue; + } + ++sharedDeclarationCount; + sharedDeclarationIndex = i; + cacheName = tokens[i + 2].text; + } + if (sharedDeclarationCount != 1) { + return false; + } + + SizeT scanTokenIndex = String::npos; + SizeT scanCount = 0; + for (SizeT i = 0; i + 7 < tokens.size(); ++i) { + if (tokens[i].text == "float" && IsIdentifierToken(tokens[i + 1]) && tokens[i + 2].text == "=" && + tokens[i + 3].text == "subgroupInclusiveAdd" && tokens[i + 4].text == "(" && + IsIdentifierToken(tokens[i + 5]) && tokens[i + 6].text == ")" && tokens[i + 7].text == ";") { + ++scanCount; + scanTokenIndex = i; + } + } + if (scanCount != 1 || sharedDeclarationIndex >= scanTokenIndex) { + return false; + } + + TokenCursor cursor(tokens, scanTokenIndex); + String prefixSum; + String importance; + String loopLength; + String loopIndex; + String sum; + if (!cursor.Consume("float") || !cursor.ConsumeAnyIdentifier(prefixSum) || !cursor.Consume("=") || + !cursor.Consume("subgroupInclusiveAdd") || !cursor.Consume("(") || + !cursor.ConsumeAnyIdentifier(importance) || !cursor.Consume(")") || !cursor.Consume(";") || + !cursor.Consume("if") || !cursor.Consume("(") || !cursor.Consume("gl_SubgroupInvocationID") || + !cursor.Consume("==") || !cursor.Consume("gl_SubgroupSize") || !cursor.Consume("-") || + !cursor.Consume("1u") || !cursor.Consume(")") || !cursor.ConsumeIdentifier(cacheName) || + !cursor.Consume("[") || !cursor.Consume("gl_SubgroupID") || !cursor.Consume("]") || !cursor.Consume("=") || + !cursor.ConsumeIdentifier(prefixSum) || !cursor.Consume(";") || !cursor.Consume("barrier") || + !cursor.Consume("(") || !cursor.Consume(")") || !cursor.Consume(";") || !cursor.Consume("uint") || + !cursor.ConsumeAnyIdentifier(loopLength) || !cursor.Consume("=") || !cursor.Consume("uint") || + !cursor.Consume("(") || !cursor.Consume("findMSB") || !cursor.Consume("(") || + !cursor.Consume("gl_NumSubgroups") || !cursor.Consume(")") || !cursor.Consume(")") || + !cursor.Consume(";") || !cursor.ConsumeIdentifier(loopLength) || !cursor.Consume("+=") || + !cursor.Consume("uint") || !cursor.Consume("(") || !cursor.Consume("gl_NumSubgroups") || + !cursor.Consume("-") || !cursor.Consume("(") || !cursor.Consume("1u") || !cursor.Consume("<<") || + !cursor.Consume("(") || !cursor.ConsumeIdentifier(loopLength) || !cursor.Consume("-") || + !cursor.Consume("1u") || !cursor.Consume(")") || !cursor.Consume(")") || !cursor.Consume(">") || + !cursor.Consume("0u") || !cursor.Consume(")") || !cursor.Consume(";") || !cursor.Consume("for") || + !cursor.Consume("(") || !cursor.Consume("uint") || !cursor.ConsumeAnyIdentifier(loopIndex) || + !cursor.Consume("=") || !cursor.Consume("0") || !cursor.Consume(";") || + !cursor.ConsumeIdentifier(loopIndex) || !cursor.Consume("<") || !cursor.ConsumeIdentifier(loopLength) || + !cursor.Consume(";") || !cursor.ConsumeIdentifier(loopIndex) || !cursor.Consume("++") || + !cursor.Consume(")") || !cursor.Consume("{") || !cursor.Consume("if") || !cursor.Consume("(") || + !cursor.Consume("(") || !cursor.Consume("gl_SubgroupID") || !cursor.Consume("&") || !cursor.Consume("(") || + !cursor.Consume("1u") || !cursor.Consume("<<") || !cursor.ConsumeIdentifier(loopIndex) || + !cursor.Consume(")") || !cursor.Consume(")") || !cursor.Consume(">") || !cursor.Consume("0u") || + !cursor.Consume(")") || !cursor.Consume("{") || !cursor.ConsumeIdentifier(prefixSum) || + !cursor.Consume("+=") || !cursor.ConsumeIdentifier(cacheName) || !cursor.Consume("[") || + !cursor.Consume("(") || !cursor.Consume("gl_SubgroupID") || !cursor.Consume(">>") || + !cursor.ConsumeIdentifier(loopIndex) || !cursor.Consume("<<") || !cursor.ConsumeIdentifier(loopIndex) || + !cursor.Consume(")") || !cursor.Consume("-") || !cursor.Consume("1u") || !cursor.Consume("]") || + !cursor.Consume(";") || !cursor.Consume("if") || !cursor.Consume("(") || + !cursor.Consume("gl_SubgroupInvocationID") || !cursor.Consume("==") || !cursor.Consume("gl_SubgroupSize") || + !cursor.Consume("-") || !cursor.Consume("1u") || !cursor.Consume(")") || + !cursor.ConsumeIdentifier(cacheName) || !cursor.Consume("[") || !cursor.Consume("gl_SubgroupID") || + !cursor.Consume("]") || !cursor.Consume("=") || !cursor.ConsumeIdentifier(prefixSum) || + !cursor.Consume(";") || !cursor.Consume("}") || !cursor.Consume("barrier") || !cursor.Consume("(") || + !cursor.Consume(")") || !cursor.Consume(";") || !cursor.Consume("}") || !cursor.Consume("if") || + !cursor.Consume("(") || !cursor.Consume("gl_LocalInvocationID") || !cursor.Consume(".") || + !cursor.Consume("x") || !cursor.Consume("==") || !cursor.Consume("uint") || !cursor.Consume("(") || + !cursor.Consume("1024") || !cursor.Consume("-") || !cursor.Consume("1") || !cursor.Consume(")") || + !cursor.Consume(")") || !cursor.ConsumeIdentifier(cacheName) || !cursor.Consume("[") || + !cursor.Consume("0") || !cursor.Consume("]") || !cursor.Consume("=") || + !cursor.ConsumeIdentifier(prefixSum) || !cursor.Consume(";") || !cursor.Consume("barrier") || + !cursor.Consume("(") || !cursor.Consume(")") || !cursor.Consume(";") || !cursor.Consume("float") || + !cursor.ConsumeAnyIdentifier(sum) || !cursor.Consume("=") || !cursor.ConsumeIdentifier(cacheName) || + !cursor.Consume("[") || !cursor.Consume("0") || !cursor.Consume("]") || !cursor.Consume(";")) { + return false; + } + const SizeT scanEndToken = cursor.Position() - 1; + + // Require the scan's immediate consumer as well. This makes the match specific to a + // linear distribution warp, and avoids changing unrelated prefix scans which may rely on + // the implementation's native subgroup partitioning. + if (!cursor.Consume("float") || !cursor.ConsumeAnyIdentifier() || !cursor.Consume("=") || + !cursor.Consume("(") || !cursor.ConsumeIdentifier(prefixSum) || !cursor.Consume("-") || + !cursor.ConsumeIdentifier(importance) || !cursor.Consume(")") || !cursor.Consume("/") || + !cursor.ConsumeIdentifier(sum) || !cursor.Consume("-") || !cursor.Consume("float") || + !cursor.Consume("(") || !cursor.Consume("gl_LocalInvocationID") || !cursor.Consume(".") || + !cursor.Consume("x") || !cursor.Consume("+") || !cursor.Consume("1u") || !cursor.Consume(")") || + !cursor.Consume("/") || !cursor.Consume("float") || !cursor.Consume("(") || !cursor.Consume("1024") || + !cursor.Consume(")") || !cursor.Consume(";")) { + return false; + } + + // No other use may share the scratch array, and no additional subgroup operation or + // builtin may silently retain native-64 semantics after this module becomes virtual-32. + if (CountToken(tokens, cacheName) != 6 || CountToken(tokens, "subgroupInclusiveAdd") != 1 || + CountToken(tokens, "gl_SubgroupInvocationID") != 2 || CountToken(tokens, "gl_SubgroupSize") != 2 || + CountToken(tokens, "gl_SubgroupID") != 4 || CountToken(tokens, "gl_NumSubgroups") != 2 || + CountToken(tokens, "gl_LocalInvocationID") != 2 || CountToken(tokens, "barrier") != 3 || + CountToken(tokens, "findMSB") != 1 || + HasIdentifierWithPrefixOutsideAllowed(tokens, "subgroup", {"subgroupInclusiveAdd"}) || + HasIdentifierWithPrefixOutsideAllowed( + tokens, "gl_Subgroup", + {"gl_SubgroupInvocationID", "gl_SubgroupSize", "gl_SubgroupID", "gl_NumSubgroups"})) { + return false; + } + + // The scan must be at the top level of the sole main() body. Its existing barriers already + // require uniform control flow; this check prevents us from introducing extra barriers in + // a nested branch or loop. + SizeT mainOpenBrace = String::npos; + SizeT mainCloseBrace = String::npos; + SizeT mainCount = 0; + for (SizeT i = 0; i + 4 < tokens.size(); ++i) { + if (!MatchTokenSequence(tokens, i, {"void", "main", "(", ")", "{"})) { + continue; + } + ++mainCount; + mainOpenBrace = i + 4; + int depth = 1; + for (SizeT j = mainOpenBrace + 1; j < tokens.size(); ++j) { + if (tokens[j].text == "{") + ++depth; + else if (tokens[j].text == "}" && --depth == 0) { + mainCloseBrace = j; + break; + } + } + } + if (mainCount != 1 || mainCloseBrace == String::npos || scanTokenIndex <= mainOpenBrace || + scanEndToken >= mainCloseBrace) { + return false; + } + int depthAtScan = 1; + for (SizeT i = mainOpenBrace + 1; i < scanTokenIndex; ++i) { + if (tokens[i].text == "{") + ++depthAtScan; + else if (tokens[i].text == "}") + --depthAtScan; + } + if (depthAtScan != 1) { + return false; + } + + constexpr const char* injectedNames[] = {"mglPrefixScanLane", "mglVirtualSubgroupInvocation", + "mglVirtualSubgroup", "mglVirtualSubgroupBase", + "mglPrefixLane", "mglVirtualSubgroupCount"}; + for (const char* injectedName : injectedNames) { + if (CountToken(tokens, injectedName) != 0) { + return false; + } + } + + match.sharedArraySizeBegin = tokens[sharedDeclarationIndex + 4].begin; + match.sharedArraySizeEnd = tokens[sharedDeclarationIndex + 4].end; + match.scanBegin = tokens[scanTokenIndex].begin; + match.scanEnd = tokens[scanEndToken].end; + match.cache = std::move(cacheName); + match.importance = std::move(importance); + match.prefixSum = std::move(prefixSum); + match.loopLength = std::move(loopLength); + match.loopIndex = std::move(loopIndex); + match.sum = std::move(sum); + return true; + } + + String BuildLinearPrefixScanReplacement(const LinearPrefixScanMatch& match) { + String replacement; + replacement.reserve(1800); + replacement += "uint mglPrefixScanLane = gl_LocalInvocationID.x;\n"; + replacement += "uint mglVirtualSubgroupInvocation = mglPrefixScanLane & 31u;\n"; + replacement += "uint mglVirtualSubgroup = mglPrefixScanLane >> 5u;\n"; + replacement += "const uint mglVirtualSubgroupCount = 32u;\n"; + replacement += match.cache + "[mglPrefixScanLane] = " + match.importance + ";\n"; + replacement += "barrier();\n"; + replacement += "float " + match.prefixSum + " = 0.0f;\n"; + replacement += "uint mglVirtualSubgroupBase = mglVirtualSubgroup << 5u;\n"; + replacement += "for (uint mglPrefixLane = mglVirtualSubgroupBase; " + "mglPrefixLane <= mglPrefixScanLane; ++mglPrefixLane) {\n"; + replacement += match.prefixSum + " += " + match.cache + "[mglPrefixLane];\n"; + replacement += "}\n"; + replacement += "barrier();\n"; + replacement += "if (mglVirtualSubgroupInvocation == 31u) " + match.cache + + "[mglVirtualSubgroup] = " + match.prefixSum + ";\n"; + replacement += "barrier();\n"; + replacement += "uint " + match.loopLength + " = uint(findMSB(mglVirtualSubgroupCount));\n"; + replacement += + match.loopLength + " += uint(mglVirtualSubgroupCount - (1u << (" + match.loopLength + " - 1u)) > 0u);\n"; + replacement += "for (uint " + match.loopIndex + " = 0u; " + match.loopIndex + " < " + match.loopLength + + "; ++" + match.loopIndex + ") {\n"; + replacement += "if ((mglVirtualSubgroup & (1u << " + match.loopIndex + ")) > 0u) {\n"; + replacement += match.prefixSum + " += " + match.cache + "[(mglVirtualSubgroup >> " + match.loopIndex + " << " + + match.loopIndex + ") - 1u];\n"; + replacement += "if (mglVirtualSubgroupInvocation == 31u) " + match.cache + + "[mglVirtualSubgroup] = " + match.prefixSum + ";\n"; + replacement += "}\nbarrier();\n}\n"; + replacement += "if (mglPrefixScanLane == 1023u) " + match.cache + "[0] = " + match.prefixSum + ";\n"; + replacement += "barrier();\n"; + replacement += "float " + match.sum + " = " + match.cache + "[0];"; + return replacement; + } + void SkipDirectiveWhitespace(const MobileGL::String& source, SizeT& pos, SizeT lineEnd) { while (pos < lineEnd && std::isspace(static_cast(source[pos]))) { pos++; @@ -576,6 +949,36 @@ namespace { namespace MobileGL { namespace MG_Util { namespace ShaderTranspiler { + Bool RewriteLinearSubgroupPrefixScanForVulkan(ShaderStage stage, Uint32 nativeSubgroupSize, + String& source) { + constexpr Uint32 capturedSubgroupSize = 32; + if (stage != ShaderStage::Compute || nativeSubgroupSize <= capturedSubgroupSize || + nativeSubgroupSize % capturedSubgroupSize != 0) { + return false; + } + + // Vulkan subgroup widths are powers of two. Keep the workaround restricted to + // wider widths which are a power-of-two multiple of the captured 32-lane model. + const Uint32 subgroupScale = nativeSubgroupSize / capturedSubgroupSize; + if ((subgroupScale & (subgroupScale - 1u)) != 0u) { + return false; + } + + const Vector tokens = TokenizeCode(source); + LinearPrefixScanMatch match; + if (!ParseLinearPrefixScanTemplate(tokens, match)) { + return false; + } + + const String replacement = BuildLinearPrefixScanReplacement(match); + source.replace(match.scanBegin, match.scanEnd - match.scanBegin, replacement); + // The declaration occurs before the replaced scan, so its original offsets remain + // valid after the first replacement. + source.replace(match.sharedArraySizeBegin, match.sharedArraySizeEnd - match.sharedArraySizeBegin, + "1024"); + return true; + } + void PreprocessShaderSource(ShaderStage stage, String& source) { // Normalize while the inspector's source span still refers to the untouched input. Later passes // remove comments and directives, so any subsequent insertion re-inspects the current source. @@ -631,6 +1034,13 @@ namespace MobileGL { RenameBuiltinShadowingFunction(source, "max3", "mg_max3"); ModernizeLegacyGLSL(stage, source); InjectDepthRangeBuiltinShim(stage, source); + + const auto& activeBackend = MG_Backend::pActiveBackendObject; + if (stage == ShaderStage::Compute && activeBackend && + activeBackend->GetBackendType() == BackendType::DirectVulkan) { + RewriteLinearSubgroupPrefixScanForVulkan(stage, activeBackend->GetDynamicParameters().SubgroupSize, + source); + } } Bool RetargetLegacyVersionDirectiveTo460(String& source) { diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.h b/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.h index b86cf153..e6c318ee 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.h +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.h @@ -21,6 +21,16 @@ namespace MobileGL { namespace ShaderTranspiler { void PreprocessShaderSource(ShaderStage stage, String& source); + // Some desktop-captured compute shaders build a workgroup-wide linear prefix scan + // from subgroupInclusiveAdd plus a shared array of subgroup totals. Qualcomm's + // Vulkan driver miscompiles that exact float InclusiveScan path for native subgroups + // wider than the capture's 32 lanes. For the narrowly recognized, uniform-control- + // flow template, replace the subgroup-local scan with a shared-memory, strict + // left-fold over virtual 32-lane segments. Returns true only when the complete safe + // template was recognized and rewritten. DirectVulkan calls this through + // PreprocessShaderSource; the explicit entry point exists for deterministic tests. + Bool RewriteLinearSubgroupPrefixScanForVulkan(ShaderStage stage, Uint32 nativeSubgroupSize, String& source); + // Rewrites a "#version 330 core" directive that PreprocessShaderSource normalized down // from a legacy desktop version back up to "#version 460 core". Returns false (leaving // the source untouched) for anything else: ES, compatibility, or an already-modern @@ -30,4 +40,4 @@ namespace MobileGL { Bool RetargetLegacyVersionDirectiveTo460(String& source); } // namespace ShaderTranspiler } // namespace MG_Util -} // namespace MobileGL \ No newline at end of file +} // namespace MobileGL diff --git a/tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-iterationrp-in-world.0000202020.png b/tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-iterationrp-in-world.0000202020.png new file mode 100644 index 00000000..f13b2a8b --- /dev/null +++ b/tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-iterationrp-in-world.0000202020.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5da3908a999c6eda1bd389b13ad0a079fe0dadc20b9fcaa8da2e893d0ece0ca3 +size 603392 diff --git a/tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-iterationrp-in-world.tgz b/tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-iterationrp-in-world.tgz new file mode 100644 index 00000000..4457a1a9 --- /dev/null +++ b/tools/trace_replay/fixtures/minecraft-1.21.4-fabric-iris-iterationrp-in-world.tgz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fc6c1b98d5c8cbbe3876cdbae09812383af00e18cf4ba313596393b112fa50d +size 22894786 diff --git a/tools/trace_replay/run_android_retrace_local.py b/tools/trace_replay/run_android_retrace_local.py index 84768555..6650d307 100644 --- a/tools/trace_replay/run_android_retrace_local.py +++ b/tools/trace_replay/run_android_retrace_local.py @@ -18,17 +18,16 @@ SUMMARY_HTML = "mobilegl-android-retrace-overview.html" DEFAULT_ANGLE_VARIANT = "ec889e6ea831" BLISS_ANGLE_VARIANT = "90a62123d794" BLISS_CASE = "minecraft-1.21.4-fabric-iris-bliss-in-world" +TRACE_APK_DIR = ROOT / "android-plugin" / "app" / "build" / "outputs" / "apk" / "trace" / "debug" BACKENDS = { "DirectGLES": { - "apk": ROOT / "android-plugin" / "app" / "build" / "outputs" / "apk" / "esprytTrace" / "debug" / "MobileGL-EsprytTrace-debug.apk", - "package": "top.mobilegl.plugin.espryt.trace", - "use_angle": True, + "package": "top.mobilegl.plugin.trace", + "use_angle": False, "use_pbuffer": False, }, "DirectVulkan": { - "apk": ROOT / "android-plugin" / "app" / "build" / "outputs" / "apk" / "magmaTrace" / "debug" / "MobileGL-MagmaTrace-debug.apk", - "package": "top.mobilegl.plugin.magma.trace", + "package": "top.mobilegl.plugin.trace", "use_angle": False, "use_pbuffer": False, }, @@ -45,6 +44,11 @@ def is_lfs_pointer(path): return path.exists() and path.read_bytes()[:80].startswith(b"version https://git-lfs.github.com/spec/v1") +def find_trace_apk(): + candidates = list(TRACE_APK_DIR.glob("MobileGL-plugin-trace-release-*.apk")) + return max(candidates, key=lambda path: path.stat().st_mtime) if candidates else None + + def bash_path(path): path = Path(path).resolve() drive = path.drive.rstrip(":").lower() @@ -117,9 +121,13 @@ def render_summary(): def run_case(case, backend): backend_info = BACKENDS[backend] + apk = find_trace_apk() trace_archive = FIXTURES / case["trace_archive"] golden = FIXTURES / case["golden"] alternate = FIXTURES / case["alternate_golden"] if case.get("alternate_golden") else None + if apk is None: + mark_skipped(case, backend, f"SKIPPED_MISSING_APK: no trace APK found under {TRACE_APK_DIR}") + return 2 if not trace_archive.exists() or is_lfs_pointer(trace_archive): mark_skipped(case, backend, "SKIPPED_LFS_POINTER: trace archive is missing or still an LFS pointer") copy_goldens(case, backend) @@ -134,7 +142,7 @@ def run_case(case, backend): "C:/Program Files/Git/bin/bash.exe", "android-plugin/trace-replay-ci.sh", "--apk-file", - bash_path(backend_info["apk"]), + bash_path(apk), "--package", backend_info["package"], "--backend", diff --git a/tools/trace_replay/trace_cases.json b/tools/trace_replay/trace_cases.json index 3fe439bc..a7212ac8 100644 --- a/tools/trace_replay/trace_cases.json +++ b/tools/trace_replay/trace_cases.json @@ -282,6 +282,14 @@ "target_call": 2667619, "timeout_seconds": 1800, "coherent_as_flush": true + }, + { + "name": "minecraft-1.21.4-fabric-iris-iterationrp-in-world", + "ci": false, + "trace_archive": "minecraft-1.21.4-fabric-iris-iterationrp-in-world.tgz", + "golden": "minecraft-1.21.4-fabric-iris-iterationrp-in-world.0000202020.png", + "target_call": 202020, + "timeout_seconds": 1800 } ] }