From 562076f657b355f876f8bf036a536a4abee28ee0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 19 Aug 2026 16:59:54 +0000 Subject: [PATCH] [Fix] (ShaderTranspiler, DirectVulkan, MG_IntegrationTest): patch both of iterationRP's under-declared subgroup scratch arrays The previous commit's fingerprint was pinned to one array's incidental dimensions - workgroup exactly 32x16x1, element exactly vec2, length exactly 32 - which is the auto-exposure reduction and nothing else. The pack ships the same idiom twice: - auto-exposure: 32x16 (512 invocations), shared vec2 prefixSumCache[32] - RTW warp: 1024 invocations, shared float prefixSumCache[64] so the warp kept writing 128 subgroups into 64 entries on an 8-lane device and the retrace stayed bit-identically wrong (ssim 0.027902). Key the fingerprint on the pack's idiom instead of one array's shape: a workgroup array of 32-bit floats indexed by gl_SubgroupID, fed by a subgroup scan, whose declared length is below ceil(invocations / native width). Three properties keep that a targeted repair rather than a general array resizer: - the index must BE gl_SubgroupID (through OpCopyObject, a signedness OpBitcast, or a spill whose every store is that id), so an index masked or clamped into range is left alone; - the >= 16-lane early-out is retained, so every module on the devices the pack was written for passes through byte-identical; - growth is certified against maxComputeSharedMemorySize using a natural-alignment layout model, and declined outright when a declaration cannot be sized, so a patched module can never fail pipeline creation where the original would not have. Verified against the shaders the CI trace actually contains: of the 14 compute modules in the fixture exactly these two change, the other twelve are byte-identical, and all fourteen pass spirv-val. The integration scenario grows a second case for the 1024-invocation shape; both abort with heap corruption when the patch is disabled. Claude-Session: https://claude.ai/code/session_01EXSURVxwp8VVrWrPEQeLCm --- .../DirectVulkan/Renderer/ProgramFactory.cpp | 1 + .../IterationRPScratchFixScenario.cpp | 98 ++++- .../FixIterationRPSubgroupScratchTest.cpp | 197 +++++++-- .../ShaderTranspiler/ShaderCompiler.cpp | 5 +- .../MG_Util/ShaderTranspiler/ShaderCompiler.h | 11 +- .../FixIterationRPSubgroupScratchPass.cpp | 382 ++++++++++++++---- .../FixIterationRPSubgroupScratchPass.h | 71 ++-- 7 files changed, 589 insertions(+), 176 deletions(-) diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp index a09d683f..38291867 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp @@ -3213,6 +3213,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { Vector patchedSpirv; if (MG_Util::ShaderTranspiler::ShaderCompiler::FixIterationRPSubgroupScratchForVulkan( moduleSpirvs[i], patchedSpirv, m_subgroupPolicy.nativeSubgroupSize, + m_subgroupPolicy.maxComputeSharedMemoryBytes, enableSpirvValidation)) { moduleSpirvs[i] = std::move(patchedSpirv); } else { diff --git a/MobileGL/MG_IntegrationTest/Scenarios/IterationRPScratchFixScenario.cpp b/MobileGL/MG_IntegrationTest/Scenarios/IterationRPScratchFixScenario.cpp index 89db6321..c6f5b23e 100644 --- a/MobileGL/MG_IntegrationTest/Scenarios/IterationRPScratchFixScenario.cpp +++ b/MobileGL/MG_IntegrationTest/Scenarios/IterationRPScratchFixScenario.cpp @@ -8,11 +8,13 @@ // // Scenario - THE FIXTURE-SHAPED SUBGROUP REDUCTION, ON WHATEVER WIDTH THE DEVICE HAS. // -// iterationRP's auto-exposure pass declares `shared vec2 prefixSumCache[32]` for a -// 512-invocation workgroup and combines per-subgroup subtotals through -// prefixSumCache[gl_SubgroupID]. The algorithm is width-agnostic; only the static 32 -// bakes in "at most 32 subgroups", which every desktop capture satisfies and an 8-lane -// device (lavapipe: 64 subgroups) does not. DirectVulkan patches exactly that with +// iterationRP hard-sizes the scratch its subgroup prefix scans write through +// prefixSumCache[gl_SubgroupID], and ships that idiom twice: the auto-exposure pass +// declares `shared vec2 prefixSumCache[32]` for a 512-invocation workgroup, and the +// RTW importance warp declares `shared float prefixSumCache[64]` for a 1024-invocation +// one. Both algorithms are width-agnostic; only the static lengths bake in "at most 32 +// (respectively 64) subgroups", which every desktop capture satisfies and an 8-lane +// device (lavapipe: 64 and 128 subgroups) does not. DirectVulkan patches exactly that with // FixIterationRPSubgroupScratchPass, growing the array to ceil(invocations / native // width) on the modules that match the pack's reduction fingerprint. // @@ -47,6 +49,10 @@ namespace MGITest { constexpr std::uint32_t kInvocationCount = 512u; // sum of 0..511, exactly representable and associativity-proof in fp32. constexpr float kExpectedTotal = 130816.0f; + // The RTW warp's shape: 1024 invocations into a 64-entry float scratch. + constexpr std::uint32_t kWideInvocationCount = 1024u; + // sum of 0..1023, likewise exact in fp32. + constexpr float kWideExpectedTotal = 523776.0f; constexpr const char* kComputeSource = R"(#version 430 core #extension GL_KHR_shader_subgroup_basic : require @@ -87,6 +93,50 @@ void main() { } atomicMax(outputData.maxSubgroupId, gl_SubgroupID); } +)"; + + // The RTW importance warp's shape: a plain float scan over 1024 invocations + // into a 64-entry scratch. Same idiom, different dimensions - which is exactly + // what a fingerprint pinned to the exposure pass's shape walks past. + constexpr const char* kWideComputeSource = R"(#version 430 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require + +layout(local_size_x = 1024) in; + +layout(std430, binding = 0) buffer Output { + float total; + uint numSubgroups; + uint maxSubgroupId; +} outputData; + +shared float prefixSumCache[64]; + +void main() { + float importance = float(gl_LocalInvocationID.x); + float prefixSum = subgroupInclusiveAdd(importance); + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) + prefixSumCache[gl_SubgroupID] = prefixSum; + barrier(); + + uint loopLength = uint(findMSB(gl_NumSubgroups)); + loopLength += uint(gl_NumSubgroups - (1u << (loopLength - 1u)) > 0u); + + for (uint scanStage = 0u; scanStage < loopLength; ++scanStage) { + if ((gl_SubgroupID & (1u << scanStage)) > 0u) { + prefixSum += prefixSumCache[(gl_SubgroupID >> scanStage << scanStage) - 1u]; + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) + prefixSumCache[gl_SubgroupID] = prefixSum; + } + barrier(); + } + + if (gl_LocalInvocationID.x == 1023u) { + outputData.total = prefixSum; + outputData.numSubgroups = gl_NumSubgroups; + } + atomicMax(outputData.maxSubgroupId, gl_SubgroupID); +} )"; struct OutputBlock { @@ -130,8 +180,7 @@ void main() { "512-invocation workgroup"; } - m_program = CompileComputeProgram(kComputeSource); - ASSERT_NE(m_program, 0u) << m_buildLog; + m_maxInvocations = static_cast(invocations); glGenBuffers(1, &m_output); glBindBuffer(GL_SHADER_STORAGE_BUFFER, m_output); @@ -181,7 +230,14 @@ void main() { return program; } - OutputBlock Dispatch() { + // Re-poisons the block, compiles the shape under test and runs it once. + OutputBlock Dispatch(const char* source) { + const OutputBlock poison{-1.0f, 0xa5a5a5a5u, 0u}; + glBindBuffer(GL_SHADER_STORAGE_BUFFER, m_output); + glBufferSubData(GL_SHADER_STORAGE_BUFFER, 0, sizeof(OutputBlock), &poison); + m_program = CompileComputeProgram(source); + EXPECT_NE(m_program, 0u) << m_buildLog; + if (m_program == 0u) return OutputBlock{}; glUseProgram(m_program); glDispatchCompute(1, 1, 1); glMemoryBarrier(GL_BUFFER_UPDATE_BARRIER_BIT); @@ -193,12 +249,13 @@ void main() { GLuint m_program = 0; GLuint m_output = 0; + std::uint32_t m_maxInvocations = 0; std::string m_buildLog; }; } // namespace TEST_F(IterationRPScratchFixScenario, FixtureShapedReductionSumsEveryInvocation) { - const OutputBlock block = Dispatch(); + const OutputBlock block = Dispatch(kComputeSource); EXPECT_EQ(glGetError(), static_cast(GL_NO_ERROR)); // The topology diagnostics catch the failure modes by name before the sum does: @@ -219,4 +276,27 @@ void main() { << "workgroup reduction produced " << block.total << " with gl_NumSubgroups=" << block.numSubgroups; } + + // The pack's second instance of the same bug, and the one that kept the CI + // retrace red after the exposure pass alone was patched. + TEST_F(IterationRPScratchFixScenario, WideFixtureShapedReductionSumsEveryInvocation) { + if (m_maxInvocations < kWideInvocationCount) { + GTEST_SKIP() << "needs a " << kWideInvocationCount << "-invocation workgroup"; + } + const OutputBlock block = Dispatch(kWideComputeSource); + EXPECT_EQ(glGetError(), static_cast(GL_NO_ERROR)); + + ASSERT_NE(block.numSubgroups, 0xa5a5a5a5u) << "invocation 1023 never reached its store"; + EXPECT_GE(block.numSubgroups, 1u); + EXPECT_LE(block.numSubgroups, kWideInvocationCount); + EXPECT_LT(block.maxSubgroupId, block.numSubgroups) + << "gl_SubgroupID exceeds gl_NumSubgroups - the inconsistency " + "DeriveNumSubgroupsPass exists to repair"; + + // Without the patch an 8-lane device writes prefixSumCache[64..127] out of + // bounds and this comparison fails. + EXPECT_EQ(block.total, kWideExpectedTotal) + << "workgroup reduction produced " << block.total << " with gl_NumSubgroups=" + << block.numSubgroups; + } } // namespace MGITest diff --git a/MobileGL/MG_Test/ShaderTranspiler/FixIterationRPSubgroupScratchTest.cpp b/MobileGL/MG_Test/ShaderTranspiler/FixIterationRPSubgroupScratchTest.cpp index 195836a4..d090d4b9 100644 --- a/MobileGL/MG_Test/ShaderTranspiler/FixIterationRPSubgroupScratchTest.cpp +++ b/MobileGL/MG_Test/ShaderTranspiler/FixIterationRPSubgroupScratchTest.cpp @@ -109,10 +109,11 @@ namespace { return tools.Validate(spirv); } - // iterationRP's reduction fingerprint: 32x16x1, subgroupInclusiveAdd on a - // vec2, and the pack's own 32-entry gl_SubgroupID-indexed scratch. A second, - // plainly indexed array rides along to prove the patch is surgical. - constexpr const char* kIterationRPShapedSource = R"(#version 450 core + // iterationRP's exposure reduction, as the pack ships it: 32x16 (512 + // invocations), subgroupInclusiveAdd on a vec2, and a 32-entry + // gl_SubgroupID-indexed scratch. A second, plainly indexed array rides along + // to prove the patch is surgical. + constexpr const char* kExposureShapedSource = R"(#version 450 core #extension GL_KHR_shader_subgroup_basic : require #extension GL_KHR_shader_subgroup_arithmetic : require layout(local_size_x = 32, local_size_y = 16, local_size_z = 1) in; @@ -141,85 +142,195 @@ void main() { } )"; - // Same scratch idiom, different workgroup shape - NOT iterationRP, so the - // fingerprint must refuse it even though it would break identically. - constexpr const char* kWrongWorkgroupShapeSource = R"(#version 450 core + // The pack's OTHER instance of the same bug, which a fingerprint pinned to the + // exposure pass's dimensions walks straight past: the RTW importance warp + // scans a plain float across 1024 invocations into a 64-entry scratch. + constexpr const char* kRtwWarpShapedSource = R"(#version 450 core #extension GL_KHR_shader_subgroup_basic : require #extension GL_KHR_shader_subgroup_arithmetic : require -layout(local_size_x = 64, local_size_y = 8, local_size_z = 1) in; +layout(local_size_x = 1024) in; layout(std430, binding = 0) buffer Output { float value; } outputData; -shared vec2 prefixSumCache[32]; +shared float prefixSumCache[64]; void main() { - vec2 v = subgroupInclusiveAdd(vec2(1.0, 0.0)); + float importance = float(gl_LocalInvocationID.x) * 0.5; + float prefixSum = subgroupInclusiveAdd(importance); if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) - prefixSumCache[gl_SubgroupID] = v; + prefixSumCache[gl_SubgroupID] = prefixSum; barrier(); - if (gl_LocalInvocationIndex == 0u) - outputData.value = prefixSumCache[0].x; + uint loopLength = uint(findMSB(gl_NumSubgroups)); + loopLength += uint(gl_NumSubgroups - (1u << (loopLength - 1u)) > 0u); + for (uint scanStage = 0u; scanStage < loopLength; ++scanStage) { + if ((gl_SubgroupID & (1u << scanStage)) > 0u) { + prefixSum += prefixSumCache[(gl_SubgroupID >> scanStage << scanStage) - 1u]; + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) + prefixSumCache[gl_SubgroupID] = prefixSum; + } + barrier(); + } + if (gl_LocalInvocationID.x == 1023u) outputData.value = prefixSumCache[0]; } )"; - // Right shape, but a float scan and a float[32] scratch - not the pack's - // vec2 accumulator signature. - constexpr const char* kWrongElementTypeSource = R"(#version 450 core + // A subgroup scan, but the scratch is indexed per invocation rather than per + // subgroup: its size is not a subgroup-count assumption, so it is not ours. + constexpr const char* kInvocationIndexedSource = R"(#version 450 core #extension GL_KHR_shader_subgroup_basic : require #extension GL_KHR_shader_subgroup_arithmetic : require layout(local_size_x = 32, local_size_y = 16, local_size_z = 1) in; layout(std430, binding = 0) buffer Output { float value; } outputData; -shared float cache[32]; +shared vec2 perInvocation[32]; void main() { - float v = subgroupInclusiveAdd(float(gl_LocalInvocationIndex)); - if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) - cache[gl_SubgroupID] = v; + vec2 v = subgroupInclusiveAdd(vec2(float(gl_LocalInvocationIndex), 0.0)); + perInvocation[gl_LocalInvocationIndex & 31u] = v; barrier(); - if (gl_LocalInvocationIndex == 0u) - outputData.value = cache[0]; + if (gl_LocalInvocationIndex == 0u) outputData.value = perInvocation[0].x; +} +)"; + + // gl_SubgroupID-indexed, but no subgroup scan feeds it and the element type is + // not the pack's float accumulator. + constexpr const char* kNonFloatScratchSource = R"(#version 450 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +layout(local_size_x = 32, local_size_y = 16, local_size_z = 1) in; +layout(std430, binding = 0) buffer Output { uint value; } outputData; +shared uint tally[32]; +void main() { + float scan = subgroupInclusiveAdd(float(gl_LocalInvocationIndex)); + tally[gl_SubgroupID] = uint(scan); + barrier(); + if (gl_LocalInvocationIndex == 0u) outputData.value = tally[0]; +} +)"; + + // gl_SubgroupID-indexed, but masked into range: the declaration is bounded by + // construction, not a subgroup-count assumption, so it is not the pack's bug. + constexpr const char* kMaskedSubgroupIndexSource = R"(#version 450 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +layout(local_size_x = 32, local_size_y = 16, local_size_z = 1) in; +layout(std430, binding = 0) buffer Output { float value; } outputData; +shared vec2 bounded[8]; +void main() { + vec2 v = subgroupInclusiveAdd(vec2(float(gl_LocalInvocationIndex), 0.0)); + bounded[gl_SubgroupID & 7u] = v; + barrier(); + if (gl_LocalInvocationIndex == 0u) outputData.value = bounded[0].x; +} +)"; + + // Neither of the pack's shapes: a small per-subgroup array in a 256-invocation + // workgroup, used to prove the width gate keeps EVERY module inert at >= 16 lanes. + constexpr const char* kForeignShapeSource = R"(#version 450 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +layout(local_size_x = 256) in; +layout(std430, binding = 0) buffer Output { float value; } outputData; +shared float partial[4]; +void main() { + float v = subgroupInclusiveAdd(float(gl_LocalInvocationID.x)); + if (gl_SubgroupID < 4u) partial[gl_SubgroupID] = v; + barrier(); + if (gl_LocalInvocationID.x == 0u) outputData.value = partial[0]; +} +)"; + + // No subgroup construct at all. + constexpr const char* kSubgroupFreeSource = R"(#version 450 core +layout(local_size_x = 32, local_size_y = 16, local_size_z = 1) in; +layout(std430, binding = 0) buffer Output { float value; } outputData; +shared vec2 scratch[32]; +void main() { + scratch[gl_LocalInvocationIndex & 31u] = vec2(float(gl_LocalInvocationIndex), 0.0); + barrier(); + if (gl_LocalInvocationIndex == 0u) outputData.value = scratch[0].x; } )"; } // namespace -TEST(FixIterationRPSubgroupScratchPass, GrowsThePacksScratchForNarrowSubgroups) { - const Vector input = CompileCompute(kIterationRPShapedSource); +TEST(FixIterationRPSubgroupScratchPass, GrowsTheExposureScratchForNarrowSubgroups) { + const Vector input = CompileCompute(kExposureShapedSource); ASSERT_FALSE(input.empty()); ASSERT_EQ(WorkgroupArrayLengths(input), (std::vector{4u, 32u})); // lavapipe: 8-lane subgroups over 512 invocations need 64 entries; the // plainly indexed neighbour must keep its 4. Vector output; - ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, output, 8u, true)); + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, output, 8u, 32768u, true)); EXPECT_EQ(WorkgroupArrayLengths(output), (std::vector{4u, 64u})); EXPECT_TRUE(Validates(output)); } -TEST(FixIterationRPSubgroupScratchPass, LeavesPackWidthAssumptionsAloneOnWideDevices) { - const Vector input = CompileCompute(kIterationRPShapedSource); +// The regression the CI retrace caught: patching only the exposure pass leaves +// this one writing 128 subgroups into 64 entries, and the frame stays wrong. +TEST(FixIterationRPSubgroupScratchPass, GrowsTheRtwWarpScratchForNarrowSubgroups) { + const Vector input = CompileCompute(kRtwWarpShapedSource); ASSERT_FALSE(input.empty()); + ASSERT_EQ(WorkgroupArrayLengths(input), (std::vector{64u})); - // >= 16 lanes means at most 32 subgroups: the pack's declared size holds and - // the module must pass through byte-identical. - for (const Uint32 nativeSize : {16u, 32u, 64u, 128u}) { - Vector output; - ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, output, nativeSize, true)); - EXPECT_EQ(output, input) << "native width " << nativeSize; + Vector output; + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, output, 8u, 32768u, true)); + EXPECT_EQ(WorkgroupArrayLengths(output), (std::vector{128u})); + EXPECT_TRUE(Validates(output)); +} + +TEST(FixIterationRPSubgroupScratchPass, LeavesPackWidthAssumptionsAloneOnWideDevices) { + // Both shapes are sized for >= 16 lanes (512/16 = 32, 1024/16 = 64), so on + // every such device the modules must pass through byte-identical. + for (const char* source : {kExposureShapedSource, kRtwWarpShapedSource}) { + const Vector input = CompileCompute(source); + ASSERT_FALSE(input.empty()); + for (const Uint32 nativeSize : {16u, 32u, 64u, 128u}) { + Vector output; + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan( + input, output, nativeSize, 32768u, true)); + EXPECT_EQ(output, input) << "native width " << nativeSize; + } } } -TEST(FixIterationRPSubgroupScratchPass, RefusesAModuleOutsideTheFingerprint) { - for (const char* source : {kWrongWorkgroupShapeSource, kWrongElementTypeSource}) { +TEST(FixIterationRPSubgroupScratchPass, RefusesAModuleOutsideTheIdiom) { + for (const char* source : {kInvocationIndexedSource, kNonFloatScratchSource, + kSubgroupFreeSource, kMaskedSubgroupIndexSource}) { const Vector input = CompileCompute(source); ASSERT_FALSE(input.empty()); Vector output; - ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, output, 8u, true)); + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, output, 8u, 32768u, true)); EXPECT_EQ(output, input); } } -TEST(FixIterationRPSubgroupScratchPass, IsIdempotent) { - const Vector input = CompileCompute(kIterationRPShapedSource); +// A grown array that would not fit the device's shared memory is left alone: +// a pipeline that cannot be created is worse than the pack's own overrun. +// The width gate is what keeps unrelated shaders untouched on the devices the pack +// was written for: at >= 16 lanes nothing is rewritten, whatever its shape. +TEST(FixIterationRPSubgroupScratchPass, LeavesEveryModuleAloneAtThePacksAssumedWidth) { + const Vector input = CompileCompute(kForeignShapeSource); ASSERT_FALSE(input.empty()); - Vector once; - ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, once, 8u, true)); - Vector twice; - ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(once, twice, 8u, true)); - EXPECT_EQ(twice, once); + for (const Uint32 nativeSize : {16u, 32u, 64u}) { + Vector output; + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan( + input, output, nativeSize, 32768u, true)); + EXPECT_EQ(output, input) << "native width " << nativeSize; + } +} + +TEST(FixIterationRPSubgroupScratchPass, RefusesGrowthThatWouldNotFitSharedMemory) { + const Vector input = CompileCompute(kRtwWarpShapedSource); + ASSERT_FALSE(input.empty()); + Vector output; + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, output, 8u, 256u, true)); + EXPECT_EQ(output, input); +} + +TEST(FixIterationRPSubgroupScratchPass, IsIdempotent) { + for (const char* source : {kExposureShapedSource, kRtwWarpShapedSource}) { + const Vector input = CompileCompute(source); + ASSERT_FALSE(input.empty()); + Vector once; + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, once, 8u, 32768u, true)); + Vector twice; + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(once, twice, 8u, 32768u, true)); + EXPECT_EQ(twice, once); + } } diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp index 36b852c9..73d4f025 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp @@ -912,12 +912,13 @@ namespace MobileGL { bool ShaderCompiler::FixIterationRPSubgroupScratchForVulkan( const Vector& inputBinary, Vector& outputBinary, - const Uint32 nativeSubgroupSize, const bool enableSpirvValidation) { + const Uint32 nativeSubgroupSize, const Uint32 maxWorkgroupScratchBytes, + const bool enableSpirvValidation) { using namespace spvtools; Optimizer optimizer(SPV_ENV_VULKAN_1_1); optimizer.RegisterPass( FixIterationRPSubgroupScratchPass::CreateFixIterationRPSubgroupScratchPass( - nativeSubgroupSize)); + nativeSubgroupSize, maxWorkgroupScratchBytes)); return RunOptimizerChecked("FixIterationRPSubgroupScratchForVulkan", optimizer, inputBinary, outputBinary, true, enableSpirvValidation); diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h index 53504f00..d3578124 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h @@ -167,12 +167,17 @@ namespace MobileGL { Vector& outputBinary, Uint32 maxWorkgroupScratchBytes, bool enableSpirvValidation = false); - // Patches iterationRP's under-declared prefixSumCache[32] on sub-16-lane - // devices, fingerprint-gated to that pack's reduction; every other module - // passes through byte-identical. See FixIterationRPSubgroupScratchPass. + // Grows iterationRP's under-declared gl_SubgroupID-indexed scratch to the + // subgroup count the device actually partitions into, fingerprint-gated to + // that pack's reduction idiom; every other module - and every device whose + // width the pack already assumed - passes through byte-identical. + // maxWorkgroupScratchBytes bounds the growth (pass the device's + // maxComputeSharedMemorySize; 0 falls back to the 16384-byte Vulkan + // minimum). See FixIterationRPSubgroupScratchPass. static bool FixIterationRPSubgroupScratchForVulkan(const Vector& inputBinary, Vector& outputBinary, Uint32 nativeSubgroupSize, + Uint32 maxWorkgroupScratchBytes, bool enableSpirvValidation = false); // Re-declares 64-bit float vertex inputs as their 32-bit unsigned word pair // (double -> uvec2, dvec2 -> uvec4) and bitcasts them back to double at entry, so no diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.cpp b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.cpp index 3ca95682..dc4d8181 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.cpp +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.cpp @@ -26,13 +26,13 @@ namespace MobileGL { using spvtools::opt::Instruction; using spvtools::opt::IRContext; - // iterationRP's reduction fingerprint, spelled out. - constexpr uint32_t kIterationRPLocalSizeX = 32u; - constexpr uint32_t kIterationRPLocalSizeY = 16u; - constexpr uint32_t kIterationRPLocalSizeZ = 1u; - constexpr uint32_t kIterationRPInvocations = - kIterationRPLocalSizeX * kIterationRPLocalSizeY * kIterationRPLocalSizeZ; - constexpr uint32_t kIterationRPScratchLength = 32u; + // The Vulkan minimum for maxComputeSharedMemorySize, used when the caller + // could not tell us the device's real limit. + constexpr uint32_t kMinimumSharedMemoryBytes = 16384u; + + // The narrowest subgroup width iterationRP's declarations are sized for. + // At or above it both shipped shapes fit and nothing may be rewritten. + constexpr uint32_t kPackAssumedSubgroupWidth = 16u; Instruction* FindBuiltinDefinition(IRContext* context, spv::BuiltIn builtin) { auto* defUseMgr = context->get_def_use_mgr(); @@ -73,18 +73,121 @@ namespace MobileGL { return nullptr; } - // vec2 of 32-bit float - the type of iterationRP's luminance/exposure - // accumulator and of its prefixSumCache entries. - bool IsVec2Float32(IRContext* context, uint32_t typeId) { - const Instruction* type = context->get_def_use_mgr()->GetDef(typeId); - if (type == nullptr || type->opcode() != spv::Op::OpTypeVector || - type->GetSingleWordInOperand(1) != 2u) { + // A 32-bit float scalar or vector - the shape of every accumulator the + // pack runs through its scans (float, vec2 and vec4 all appear). Returns + // the component count, or 0 for anything else. + uint32_t Float32ComponentCount(IRContext* context, uint32_t typeId) { + auto* defUseMgr = context->get_def_use_mgr(); + const Instruction* type = defUseMgr->GetDef(typeId); + if (type == nullptr) return 0u; + uint32_t components = 1u; + if (type->opcode() == spv::Op::OpTypeVector) { + components = type->GetSingleWordInOperand(1); + if (components < 2u || components > 4u) return 0u; + type = defUseMgr->GetDef(type->GetSingleWordInOperand(0)); + if (type == nullptr) return 0u; + } + if (type->opcode() != spv::Op::OpTypeFloat || + type->GetSingleWordInOperand(0) != 32u) { + return 0u; + } + return components; + } + + uint32_t RoundUp(uint32_t value, uint32_t alignment) { + return alignment == 0u ? value : ((value + alignment - 1u) / alignment) * alignment; + } + + // Size AND alignment of a workgroup-storage type. Drivers lay shared + // memory out at natural alignment and the limit + // (VUID-RuntimeSpirv-Workgroup-06530) counts the padding that produces, + // so a model that sums unpadded sizes would under-count exactly where the + // budget check matters. Returns false for anything not modelled here, + // which the caller answers by declining to grow at all rather than by + // certifying growth against a total it knows is an underestimate. + bool WorkgroupTypeLayout(IRContext* context, uint32_t typeId, uint32_t* size, + uint32_t* alignment, uint32_t depth = 0u) { + if (depth > 8u) return false; + auto* defUseMgr = context->get_def_use_mgr(); + const Instruction* type = defUseMgr->GetDef(typeId); + if (type == nullptr) return false; + switch (type->opcode()) { + case spv::Op::OpTypeBool: + *size = 4u; + *alignment = 4u; + return true; + case spv::Op::OpTypeInt: + case spv::Op::OpTypeFloat: { + const uint32_t width = type->GetSingleWordInOperand(0) / 8u; + if (width == 0u) return false; + *size = width; + *alignment = width; + return true; + } + case spv::Op::OpTypeVector: { + uint32_t componentSize = 0u; + uint32_t componentAlignment = 0u; + if (!WorkgroupTypeLayout(context, type->GetSingleWordInOperand(0), + &componentSize, &componentAlignment, depth + 1u)) { + return false; + } + const uint32_t components = type->GetSingleWordInOperand(1); + if (components < 2u || components > 4u) return false; + *size = componentSize * components; + // A three-component vector aligns like a four-component one. + *alignment = componentSize * (components == 3u ? 4u : components); + return true; + } + case spv::Op::OpTypeMatrix: + case spv::Op::OpTypeArray: { + uint32_t elementSize = 0u; + uint32_t elementAlignment = 0u; + if (!WorkgroupTypeLayout(context, type->GetSingleWordInOperand(0), &elementSize, + &elementAlignment, depth + 1u)) { + return false; + } + uint32_t count = 0u; + if (type->opcode() == spv::Op::OpTypeMatrix) { + count = type->GetSingleWordInOperand(1); + } else { + const Instruction* length = + defUseMgr->GetDef(type->GetSingleWordInOperand(1)); + if (length == nullptr || length->opcode() != spv::Op::OpConstant) { + return false; // spec-constant length: not sizeable here + } + count = length->GetSingleWordInOperand(0); + } + *size = RoundUp(elementSize, elementAlignment) * count; + *alignment = elementAlignment; + return true; + } + case spv::Op::OpTypeStruct: { + uint32_t offset = 0u; + uint32_t structAlignment = 1u; + for (uint32_t i = 0; i < type->NumInOperands(); ++i) { + uint32_t memberSize = 0u; + uint32_t memberAlignment = 0u; + if (!WorkgroupTypeLayout(context, type->GetSingleWordInOperand(i), + &memberSize, &memberAlignment, depth + 1u)) { + return false; + } + offset = RoundUp(offset, memberAlignment) + memberSize; + if (memberAlignment > structAlignment) structAlignment = memberAlignment; + } + *size = RoundUp(offset, structAlignment); + *alignment = structAlignment; + return true; + } + default: return false; } - const Instruction* component = - context->get_def_use_mgr()->GetDef(type->GetSingleWordInOperand(0)); - return component != nullptr && component->opcode() == spv::Op::OpTypeFloat && - component->GetSingleWordInOperand(0) == 32u; + } + + // The group operations the pack's prefix scans use. + bool IsScanOrReduce(spv::GroupOperation operation) { + return operation == spv::GroupOperation::Reduce || + operation == spv::GroupOperation::InclusiveScan || + operation == spv::GroupOperation::ExclusiveScan; } } // namespace @@ -92,13 +195,15 @@ namespace MobileGL { auto* irContext = context(); auto* defUseMgr = irContext->get_def_use_mgr(); - // A device whose native width already satisfies the pack's assumption - // (>= 16 lanes -> at most 32 subgroups) needs no patch at all. - if (m_nativeSubgroupSize == 0u || m_nativeSubgroupSize >= 16u) { + // Without a known device width there is no topology to compare against; + // and a width the pack already assumed needs no patch at all. Both of + // iterationRP's shapes are sized for >= 16 lanes (512/16 = 32 entries, + // 1024/16 = 64), so every module on such a device - the pack's or anyone + // else's - must pass through byte-identical. The per-array length test + // further down is the second gate, not a replacement for this one. + if (m_nativeSubgroupSize == 0u || m_nativeSubgroupSize >= kPackAssumedSubgroupWidth) { return Status::SuccessWithoutChange; } - const uint32_t requiredLength = - (kIterationRPInvocations + m_nativeSubgroupSize - 1u) / m_nativeSubgroupSize; for (const Instruction& entryPoint : irContext->module()->entry_points()) { if (static_cast(entryPoint.GetSingleWordInOperand(0)) != @@ -107,7 +212,8 @@ namespace MobileGL { } } - // Fingerprint 1: the pack's exposure-pass workgroup shape, 32x16x1. + // Fingerprint 1: a literal workgroup size, so the subgroup count the + // dispatch actually partitions into is known here. const auto resolveUintConstant = [&](uint32_t id, uint32_t* value) { const Instruction* def = defUseMgr->GetDef(id); if (def == nullptr || def->opcode() != spv::Op::OpConstant) return false; @@ -139,26 +245,40 @@ namespace MobileGL { } } } - if (!haveLocalSize || localSize[0] != kIterationRPLocalSizeX || - localSize[1] != kIterationRPLocalSizeY || localSize[2] != kIterationRPLocalSizeZ) { + if (!haveLocalSize || localSize[0] == 0u || localSize[1] == 0u || localSize[2] == 0u) { return Status::SuccessWithoutChange; } + const uint64_t totalInvocations = + static_cast(localSize[0]) * localSize[1] * localSize[2]; + if (totalInvocations == 0u || totalInvocations > (1u << 20)) { + return Status::SuccessWithoutChange; + } + const uint32_t requiredLength = static_cast( + (totalInvocations + m_nativeSubgroupSize - 1u) / m_nativeSubgroupSize); - // Fingerprint 2: the reduction's subgroupInclusiveAdd on a vec2. - bool sawVec2InclusiveAdd = false; + // Fingerprint 2: a subgroup scan over a 32-bit float value - the pack's + // prefix-sum reduction, and the reason its scratch is indexed per subgroup. + bool sawFloatSubgroupScan = false; for (auto& function : *irContext->module()) { for (auto& block : function) { for (auto& inst : block) { - if (inst.opcode() == spv::Op::OpGroupNonUniformFAdd && - static_cast(inst.GetSingleWordInOperand(1)) == - spv::GroupOperation::InclusiveScan && - IsVec2Float32(irContext, inst.type_id())) { - sawVec2InclusiveAdd = true; + if (inst.opcode() != spv::Op::OpGroupNonUniformFAdd && + inst.opcode() != spv::Op::OpGroupNonUniformFMin && + inst.opcode() != spv::Op::OpGroupNonUniformFMax) { + continue; + } + if (inst.NumInOperands() < 2) continue; + if (!IsScanOrReduce(static_cast( + inst.GetSingleWordInOperand(1)))) { + continue; + } + if (Float32ComponentCount(irContext, inst.type_id()) != 0u) { + sawFloatSubgroupScan = true; } } } } - if (!sawVec2InclusiveAdd) { + if (!sawFloatSubgroupScan) { return Status::SuccessWithoutChange; } @@ -171,60 +291,89 @@ namespace MobileGL { } const uint32_t subgroupIdVariableId = subgroupIdVariable->result_id(); - // Conservative taint walk over values, and through Function/Private - // temporaries by variable (glslang routinely spills builtin loads into - // locals before they reach an index expression). Over-tainting is safe: - // the candidate filter below still demands the exact vec2[32] shape. - std::unordered_map valueTainted; // result id -> tainted - std::unordered_map variableTainted; // variable id -> tainted - bool changedTaint = true; - while (changedTaint) { - changedTaint = false; + // The pack indexes its scratch with gl_SubgroupID ITSELF, so only values + // that ARE that id qualify - not everything computed from it. An index + // that is masked or clamped (cache[gl_SubgroupID & 3u]) is bounded by + // construction and is none of this pass's business; accepting it would + // turn a targeted repair into a general array resizer. Identity survives + // OpCopyObject, a signedness OpBitcast, and the Function/Private spill + // glslang emits for a builtin load - and nothing else. A spill variable + // counts only when EVERY store into it is the id. + std::unordered_map subgroupIdValues; // result id IS the id + std::unordered_map subgroupIdVariables; // spill holding only it + bool changedIdentity = true; + while (changedIdentity) { + changedIdentity = false; + + std::unordered_map totalStores; + std::unordered_map idStores; for (auto& function : *irContext->module()) { for (auto& block : function) { for (auto& inst : block) { - const spv::Op opcode = inst.opcode(); - if (opcode == spv::Op::OpStore) { - if (!valueTainted.count(inst.GetSingleWordInOperand(1))) continue; - const Instruction* root = - RootVariable(irContext, inst.GetSingleWordInOperand(0)); - if (root == nullptr) continue; - if (!variableTainted.count(root->result_id())) { - variableTainted[root->result_id()] = true; - changedTaint = true; - } + if (inst.opcode() != spv::Op::OpStore) continue; + const uint32_t pointerId = inst.GetSingleWordInOperand(0); + const Instruction* target = defUseMgr->GetDef(pointerId); + if (target == nullptr || target->opcode() != spv::Op::OpVariable) { continue; } - if (inst.result_id() == 0 || valueTainted.count(inst.result_id())) { + const auto storageClass = static_cast( + target->GetSingleWordInOperand(0)); + if (storageClass != spv::StorageClass::Function && + storageClass != spv::StorageClass::Private) { continue; } - bool tainted = false; - if (opcode == spv::Op::OpLoad) { + totalStores[pointerId] += 1u; + if (subgroupIdValues.count(inst.GetSingleWordInOperand(1))) { + idStores[pointerId] += 1u; + } + } + } + } + for (const auto& entry : totalStores) { + if (entry.second != 0u && idStores[entry.first] == entry.second && + !subgroupIdVariables.count(entry.first)) { + subgroupIdVariables[entry.first] = true; + changedIdentity = true; + } + } + + for (auto& function : *irContext->module()) { + for (auto& block : function) { + for (auto& inst : block) { + if (inst.result_id() == 0 || + subgroupIdValues.count(inst.result_id())) { + continue; + } + bool isSubgroupId = false; + switch (inst.opcode()) { + case spv::Op::OpLoad: { const uint32_t pointerId = inst.GetSingleWordInOperand(0); - if (pointerId == subgroupIdVariableId) tainted = true; - const Instruction* root = RootVariable(irContext, pointerId); - if (root != nullptr && variableTainted.count(root->result_id())) { - tainted = true; - } - } else { - inst.ForEachInId([&](const uint32_t* operandId) { - if (valueTainted.count(*operandId)) tainted = true; - }); + isSubgroupId = pointerId == subgroupIdVariableId || + subgroupIdVariables.count(pointerId) != 0u; + break; } - if (tainted) { - valueTainted[inst.result_id()] = true; - changedTaint = true; + case spv::Op::OpCopyObject: + case spv::Op::OpBitcast: + isSubgroupId = + subgroupIdValues.count(inst.GetSingleWordInOperand(0)) != 0u; + break; + default: + break; + } + if (isSubgroupId) { + subgroupIdValues[inst.result_id()] = true; + changedIdentity = true; } } } } } - if (valueTainted.empty()) { + if (subgroupIdValues.empty()) { return Status::SuccessWithoutChange; } - // Fingerprint 3: workgroup-shared vec2[32] arrays whose access-chain - // index depends on gl_SubgroupID - the under-declared prefixSumCache. + // Fingerprint 3: workgroup-shared float arrays indexed by gl_SubgroupID + // itself - the under-declared prefixSumCache. std::map candidates; for (auto& function : *irContext->module()) { for (auto& block : function) { @@ -234,7 +383,7 @@ namespace MobileGL { continue; } if (inst.NumInOperands() < 2) continue; - if (!valueTainted.count(inst.GetSingleWordInOperand(1))) continue; + if (!subgroupIdValues.count(inst.GetSingleWordInOperand(1))) continue; Instruction* baseVariable = defUseMgr->GetDef(inst.GetSingleWordInOperand(0)); if (baseVariable == nullptr || @@ -252,7 +401,16 @@ namespace MobileGL { return Status::SuccessWithoutChange; } - bool changedModule = false; + // Everything that survives the filter, with the bytes each grown array + // will need. Nothing is mutated until the whole set fits the device's + // shared-memory budget, so a module is never left half-grown. + struct Growth { + Instruction* variable = nullptr; + uint32_t elementTypeId = 0; + uint32_t lengthTypeId = 0; + uint32_t addedBytes = 0; + }; + std::vector growths; for (auto& entry : candidates) { Instruction* variable = entry.second; @@ -290,18 +448,70 @@ namespace MobileGL { continue; } const uint32_t elementTypeId = arrayType->GetSingleWordInOperand(0); - if (!IsVec2Float32(irContext, elementTypeId)) continue; + const uint32_t components = Float32ComponentCount(irContext, elementTypeId); + if (components == 0u) continue; const Instruction* lengthConstant = defUseMgr->GetDef(arrayType->GetSingleWordInOperand(1)); - uint32_t currentLength = 0; - if (lengthConstant == nullptr || - lengthConstant->opcode() != spv::Op::OpConstant || - !((currentLength = lengthConstant->GetSingleWordInOperand(0), - currentLength == kIterationRPScratchLength))) { + if (lengthConstant == nullptr || lengthConstant->opcode() != spv::Op::OpConstant) { continue; } + const uint32_t currentLength = lengthConstant->GetSingleWordInOperand(0); + + // The pack's own assumption holds on this device: the declared array + // already covers every subgroup the workgroup partitions into. That is + // every >= 16-lane device for the shapes iterationRP ships, and those + // modules must pass through byte-identical. if (currentLength >= requiredLength) continue; + // vec3 strides at its 16-byte alignment, so charge the padded stride. + const uint32_t elementStride = (components == 3u ? 4u : components) * 4u; + growths.push_back(Growth{variable, elementTypeId, lengthConstant->type_id(), + (requiredLength - currentLength) * elementStride}); + } + if (growths.empty()) { + return Status::SuccessWithoutChange; + } + + // Growing must not push the module past what the device can launch: a + // pipeline that fails to create is worse than the pack's own overrun. + { + uint64_t declaredBytes = 0; + bool sawUnsizeable = false; + for (auto& global : irContext->module()->types_values()) { + if (global.opcode() != spv::Op::OpVariable || + static_cast(global.GetSingleWordInOperand(0)) != + spv::StorageClass::Workgroup) { + continue; + } + const Instruction* pointerType = defUseMgr->GetDef(global.type_id()); + uint32_t bytes = 0u; + uint32_t alignment = 0u; + if (pointerType == nullptr || + pointerType->opcode() != spv::Op::OpTypePointer || + !WorkgroupTypeLayout(irContext, pointerType->GetSingleWordInOperand(1), + &bytes, &alignment)) { + sawUnsizeable = true; + break; + } + declaredBytes = RoundUp(static_cast(declaredBytes), alignment) + bytes; + } + // A declaration this pass cannot size leaves the total an + // underestimate, so the growth cannot be certified against the device + // limit at all - decline rather than guess. + if (sawUnsizeable) { + return Status::SuccessWithoutChange; + } + for (const Growth& growth : growths) declaredBytes += growth.addedBytes; + + const uint32_t deviceBudget = m_maxWorkgroupScratchBytes != 0u + ? m_maxWorkgroupScratchBytes + : kMinimumSharedMemoryBytes; + if (declaredBytes > deviceBudget) { + return Status::SuccessWithoutChange; + } + } + + for (const Growth& growth : growths) { // Build the grown array type. All three new instructions are inserted // immediately BEFORE the variable so definition-before-use holds in the // module's global section (manager-created instructions append to its @@ -310,17 +520,17 @@ namespace MobileGL { // scalar constant is legal SPIR-V); the fresh array type makes the // pointer type unique by construction, so neither collides with an // existing declaration. - const uint32_t lengthTypeId = lengthConstant->type_id(); + Instruction* variable = growth.variable; const uint32_t newLengthId = irContext->TakeNextId(); variable->InsertBefore(spvtools::MakeUnique( - irContext, spv::Op::OpConstant, lengthTypeId, newLengthId, + irContext, spv::Op::OpConstant, growth.lengthTypeId, newLengthId, Instruction::OperandList{{SPV_OPERAND_TYPE_TYPED_LITERAL_NUMBER, {requiredLength}}})); const uint32_t newArrayTypeId = irContext->TakeNextId(); variable->InsertBefore(spvtools::MakeUnique( irContext, spv::Op::OpTypeArray, 0, newArrayTypeId, Instruction::OperandList{ - {SPV_OPERAND_TYPE_ID, {elementTypeId}}, + {SPV_OPERAND_TYPE_ID, {growth.elementTypeId}}, {SPV_OPERAND_TYPE_ID, {newLengthId}}})); const uint32_t newPointerTypeId = irContext->TakeNextId(); variable->InsertBefore(spvtools::MakeUnique( @@ -331,21 +541,17 @@ namespace MobileGL { {SPV_OPERAND_TYPE_ID, {newArrayTypeId}}})); variable->SetResultType(newPointerTypeId); - changedModule = true; } - if (!changedModule) { - return Status::SuccessWithoutChange; - } irContext->InvalidateAnalysesExceptFor(IRContext::kAnalysisNone); return Status::SuccessWithChange; } spvtools::Optimizer::PassToken FixIterationRPSubgroupScratchPass::CreateFixIterationRPSubgroupScratchPass( - const Uint32 nativeSubgroupSize) { - return spvtools::Optimizer::PassToken( - MakeUnique(nativeSubgroupSize)); + const Uint32 nativeSubgroupSize, const Uint32 maxWorkgroupScratchBytes) { + return spvtools::Optimizer::PassToken(MakeUnique( + nativeSubgroupSize, maxWorkgroupScratchBytes)); } } // namespace ShaderTranspiler } // namespace MG_Util diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.h b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.h index db3b7d6d..19857bac 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.h +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.h @@ -16,48 +16,57 @@ namespace MobileGL { namespace MG_Util { namespace ShaderTranspiler { - // Patches ONE known shader-pack defect: iterationRP's auto-exposure reduction - // declares `shared vec2 prefixSumCache[32]` for its 512-invocation workgroup - // and stores per-subgroup subtotals through prefixSumCache[gl_SubgroupID]. - // The pack hard-sized that scratch for the >=16-lane subgroups desktop GL - // drivers ship; on a narrower Vulkan device (lavapipe's 8 lanes -> 64 - // subgroups) every subgroup past entry 31 indexes shared memory out of - // bounds - on a CPU rasterizer that is literal heap corruption. The - // reduction ALGORITHM is width-agnostic (its combine loop is sized by - // gl_NumSubgroups), so the faithful repair is to grow the one under-declared - // array to ceil(512 / native width) and change nothing else. This is the - // pack author's bug, not MobileGL's; the patch is therefore deliberately - // NOT a general mechanism - it only rewrites modules that positively match - // iterationRP's reduction fingerprint: - // - GLCompute entry point with local size exactly 32x16x1; - // - a subgroupInclusiveAdd on a vec2 (OpGroupNonUniformFAdd InclusiveScan, - // the pack's luminance/exposure accumulator signature); - // - a workgroup-shared array of exactly vec2[32] whose access-chain index - // is data-dependent on gl_SubgroupID. - // Matching at the SPIR-V level keeps the recognition robust against - // whitespace/identifier-level drift that made the old source-text template - // rewrite (removed in 7769156) so brittle, while still refusing to touch - // anything that is not this pack's reduction. On devices whose native width - // already satisfies the pack's assumption (>= 16 lanes: desktop GL, Adreno), - // the grown length equals or undershoots the declared 32 and every module - // passes through byte-identical. + // Patches ONE known shader-pack defect: iterationRP hard-sizes the scratch + // its subgroup prefix scans write through prefixSumCache[gl_SubgroupID]. + // The pack ships that idiom twice, sized for the >= 16-lane subgroups + // desktop GL drivers give it: + // - the auto-exposure reduction: 32x16 (512 invocations), vec2[32]; + // - the RTW importance warp: 1024 invocations, float[64]. + // On a narrower Vulkan device (lavapipe's 8 lanes -> 64 and 128 subgroups) + // every subgroup past the last declared entry indexes shared memory out of + // bounds - on a CPU rasterizer that is literal heap corruption. Both + // reduction ALGORITHMS are width-agnostic (their combine loops are sized by + // gl_NumSubgroups), so the faithful repair is to grow the under-declared + // arrays to ceil(invocations / native width) and change nothing else. // - // The pass never fails a module: anything it cannot prove is this exact - // pattern - or cannot grow safely (a whole-array use, a spec-constant - // length, an initializer) - is left exactly as it was. + // This is the pack author's bug, not MobileGL's, so the patch is + // deliberately NOT a general "resize shared arrays" mechanism. It rewrites + // an array only when the module positively matches the pack's reduction + // idiom AND the device's own topology proves the declaration too small: + // - GLCompute entry point with a literal workgroup size; + // - a subgroup scan/reduce over a 32-bit float scalar or vector + // (OpGroupNonUniformF{Add,Min,Max}), the pack's accumulator signature; + // - a workgroup-shared array of 32-bit float scalars/vectors whose + // access-chain index is data-dependent on gl_SubgroupID; + // - a declared length strictly below ceil(invocations / native width). + // That last clause is what keeps the patch inert wherever the pack is + // correct: on any device whose width satisfies the pack's assumption + // (>= 16 lanes: desktop GL, Adreno) both shapes already fit and every + // module passes through byte-identical. Matching at the SPIR-V level keeps + // recognition robust against the whitespace/identifier drift that made the + // old source-text template rewrite (removed in 7769156) so brittle. + // + // The pass never fails a module: anything it cannot prove is this pattern - + // or cannot grow safely (a whole-array use, a spec-constant length, an + // initializer, or growth that would not fit maxWorkgroupScratchBytes) - is + // left exactly as it was. Pass the device's maxComputeSharedMemorySize as + // maxWorkgroupScratchBytes; 0 falls back to the 16384-byte Vulkan minimum. class FixIterationRPSubgroupScratchPass : public spvtools::opt::Pass { public: - explicit FixIterationRPSubgroupScratchPass(Uint32 nativeSubgroupSize) - : m_nativeSubgroupSize(nativeSubgroupSize) {} + FixIterationRPSubgroupScratchPass(Uint32 nativeSubgroupSize, + Uint32 maxWorkgroupScratchBytes) + : m_nativeSubgroupSize(nativeSubgroupSize), + m_maxWorkgroupScratchBytes(maxWorkgroupScratchBytes) {} const char* name() const override { return "fix-iterationrp-subgroup-scratch"; } Status Process() override; static spvtools::Optimizer::PassToken CreateFixIterationRPSubgroupScratchPass( - Uint32 nativeSubgroupSize); + Uint32 nativeSubgroupSize, Uint32 maxWorkgroupScratchBytes); private: Uint32 m_nativeSubgroupSize; + Uint32 m_maxWorkgroupScratchBytes; }; } // namespace ShaderTranspiler } // namespace MG_Util