From 795e08f7e6c34fe0c27c10ed3498a6cc2e954412 Mon Sep 17 00:00:00 2001 From: Swung0x48 Date: Sat, 5 Sep 2026 05:36:49 -0400 Subject: [PATCH] [Fix, Test] (ShaderTranspiler): flatten fp64 storage blocks whose last member is a runtime array - the pass declined them, so the fp64 demotion re-derived ArrayStride 4 over the application's 8/16/32-byte double buffer and every double/dvecN data[] SSBO read raw words --- .../FlattenFloat64StorageBlockTest.cpp | 746 ++++++++++++++++++ .../FlattenFloat64StorageBlockPass.cpp | 288 ++++++- .../FlattenFloat64StorageBlockPass.h | 27 +- 3 files changed, 1013 insertions(+), 48 deletions(-) diff --git a/MobileGL/MG_Test/ShaderTranspiler/FlattenFloat64StorageBlockTest.cpp b/MobileGL/MG_Test/ShaderTranspiler/FlattenFloat64StorageBlockTest.cpp index 64b09d20..8696c237 100644 --- a/MobileGL/MG_Test/ShaderTranspiler/FlattenFloat64StorageBlockTest.cpp +++ b/MobileGL/MG_Test/ShaderTranspiler/FlattenFloat64StorageBlockTest.cpp @@ -22,6 +22,8 @@ #include "Includes.h" #include "Init.h" #include +#include +#include #include @@ -38,6 +40,7 @@ namespace { constexpr Uint32 kOpTypeInt = 21; constexpr Uint32 kOpTypeFloat = 22; constexpr Uint32 kOpTypeArray = 28; + constexpr Uint32 kOpTypeRuntimeArray = 29; constexpr Uint32 kOpTypeStruct = 30; constexpr Uint32 kOpConstant = 43; constexpr Uint32 kDecorationArrayStride = 6; @@ -116,6 +119,18 @@ namespace { return {elementTypeId, length}; } + // The element type id of OpTypeRuntimeArray , or 0 when it is not one - which is + // what a BOUNDED flattened member (an OpTypeArray) answers too, so the two shapes can be told + // apart by the pair of helpers. + Uint32 RuntimeArrayElementOf(const Vector& spirv, Uint32 arrayId) { + Uint32 elementTypeId = 0; + ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) { + if (opcode != kOpTypeRuntimeArray || wordCount < 3 || words[1] != arrayId) return; + elementTypeId = words[2]; + }); + return elementTypeId; + } + Bool IsUint32Type(const Vector& spirv, Uint32 typeId) { Bool isUint = false; ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) { @@ -140,6 +155,61 @@ namespace { return text; } + // How many lines of a disassembly hold BOTH fragments - "OpIMul %uint" and "%uint_8", say - + // which is how the index arithmetic the pass emits is pinned without a host that could run it. + Uint32 CountLinesWith(const String& text, const String& first, const String& second) { + Uint32 count = 0; + SizeT lineStart = 0; + while (lineStart < text.size()) { + SizeT lineEnd = text.find('\n', lineStart); + if (lineEnd == String::npos) lineEnd = text.size(); + const String line = text.substr(lineStart, lineEnd - lineStart); + if (line.find(first) != String::npos && line.find(second) != String::npos) ++count; + lineStart = lineEnd + 1; + } + return count; + } + + // What every test of the open-ended shape asserts: the block collapsed to ONE member, which + // is a `uint[]` RUNTIME array of stride 4 rather than a bounded one, and nothing 64-bit is + // left for the demotion to find. Returns the disassembly for the arithmetic checks. + String ExpectOpenEndedWordArray(const Vector& output, const String& blockName) { + const String text = Disassemble(output); + const Uint32 structId = StructIdNamed(output, blockName); + EXPECT_NE(structId, 0u) << text; + if (structId == 0) return text; + const Vector members = MemberTypesOf(output, structId); + EXPECT_EQ(members.size(), 1u) << "the block should have collapsed to one member\n" << text; + if (members.size() != 1) return text; + EXPECT_EQ(MemberOffsetsOf(output, structId), (Vector{0})); + const Uint32 elementTypeId = RuntimeArrayElementOf(output, members[0]); + EXPECT_NE(elementTypeId, 0u) << "member 0 is not a runtime array\n" << text; + EXPECT_EQ(ArrayShapeOf(output, members[0]).first, 0u) + << "an open-ended block must not be given a bounded length\n" + << text; + EXPECT_TRUE(IsUint32Type(output, elementTypeId)) << text; + EXPECT_EQ(DecorationValueOf(output, members[0], kDecorationArrayStride), 4u) << text; + EXPECT_EQ(CountFloatTypesOfWidth(output, 64), 0u) << text; + return text; + } + + // The compute shape every failing KHR-Single-GL45.subgroups fp64 case binds: one runtime + // array of doubles, indexed by an invocation id, read whole-element. + String OpenEndedComputeSource(const String& elementType) { + return String(R"(#version 430 core +layout(local_size_x = 16) in; +layout(std430, binding = 0) buffer Sink { uint result[]; }; +layout(std430, binding = 1) buffer Data { )") + + elementType + R"( data[]; }; +void main() { + )" + elementType + + R"( value = data[gl_LocalInvocationID.x] * data[0]; + result[gl_GlobalInvocationID.x] = uint(value)" + + (elementType == "double" ? String{} : String(".x")) + R"(); +} +)"; + } + Vector CompileToSpirv(GLenum stage, const String& source) { using namespace MG_Util::ShaderTranspiler; ShaderAttrib shaderAttrib{.shaderType = stage, .sourceStr = source}; @@ -345,3 +415,679 @@ TEST_F(FlattenFloat64StorageBlockTest, TheDemotedPathIsUnchangedByTheCapabilityA EXPECT_EQ(explicitlyDemoted, defaulted); EXPECT_EQ(CountFloatTypesOfWidth(defaulted, 64), 0u) << Disassemble(defaulted); } + +// --------------------------------------------------------------------------- +// The open-ended shape: a block whose last member is a runtime array. Before this was accepted +// the pass declined it and the demotion re-derived ArrayStride 4 for the now-float element, so +// `double data[]` read the application's 8-byte-stride buffer as 32-bit words - every fp64 +// KHR-Single-GL45.subgroups case failed on exactly that. +// --------------------------------------------------------------------------- + +TEST_F(FlattenFloat64StorageBlockTest, AnOpenEndedBlockOfDoublesBecomesAWordRuntimeArray) { + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, OpenEndedComputeSource("double")); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const String text = ExpectOpenEndedWordArray(output, "Data"); + // Element i of the original array starts at word 2i, so the dynamic index is scaled by 2 ... + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_2"), 1u) << text; + // ... and the constant `data[0]` is the pair of words at 0 and 1, reached through the one + // member the block has left. + EXPECT_GE(CountLinesWith(text, "OpAccessChain %_ptr_StorageBuffer_uint", "%uint_0 %uint_0"), 1u) << text; +} + +TEST_F(FlattenFloat64StorageBlockTest, EachDoubleVectorWidthStepsByItsOwnStride) { + struct Shape { + const char* element; + // std430 strides: dvec2 16 bytes, dvec3 and dvec4 32 bytes - i.e. 4, 8 and 8 words. + const char* strideWords; + // The last component's word offset inside one element, and the first one past it. + const char* lastComponentWords; + const char* firstWordPastIt; + }; + const Shape shapes[] = {{"dvec2", "%uint_4", "%uint_2", "%uint_4"}, + {"dvec3", "%uint_8", "%uint_4", "%uint_6"}, + {"dvec4", "%uint_8", "%uint_6", "%uint_8"}}; + for (const Shape& shape : shapes) { + SCOPED_TRACE(shape.element); + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, OpenEndedComputeSource(shape.element)); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const String text = ExpectOpenEndedWordArray(output, "Data"); + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", shape.strideWords), 1u) << text; + EXPECT_GE(CountLinesWith(text, "OpIAdd %uint", shape.lastComponentWords), 1u) << text; + // A dvec3 is six words in a stride of eight: nothing may be read from the padding. + EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", shape.firstWordPastIt), 0u) << text; + } +} + +TEST_F(FlattenFloat64StorageBlockTest, AFixedPrefixBeforeTheRuntimeArrayIsAddedToEveryIndex) { + const String source = R"(#version 430 core +layout(local_size_x = 16) in; +layout(std430, binding = 0) buffer Sink { uint result[]; }; +layout(std430, binding = 1) buffer Data { + uvec4 head; + double data[]; +}; +void main() { + result[gl_GlobalInvocationID.x] = head.x + uint(data[gl_LocalInvocationID.x]); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Uint32 inputStructId = StructIdNamed(input, "Data"); + ASSERT_NE(inputStructId, 0u); + EXPECT_EQ(MemberOffsetsOf(input, inputStructId), (Vector{0, 16})) << Disassemble(input); + + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + const String text = ExpectOpenEndedWordArray(output, "Data"); + // The 16-byte prefix is 4 words: element i is at word 4 + 2i. + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_2"), 1u) << text; + EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", "%uint_4"), 1u) << text; + // And the prefix member itself is still word 0. + EXPECT_GE(CountLinesWith(text, "OpAccessChain %_ptr_StorageBuffer_uint", "%uint_0 %uint_0"), 1u) << text; +} + +// OpArrayLength on the flattened member counts WORDS. GL's `.length()` is the number of whole +// elements the bound range holds past the array's offset, so the count has to be rebased and +// divided - in unsigned arithmetic, and clamped rather than wrapped when the range is shorter +// than the prefix. +namespace { + // A prefix, an open-ended array of doubles, and a `.length()` of it - the one shape whose + // rewrite is an instruction SPIRV-Cross has to spell rather than plain arithmetic. + constexpr const char* kOpenEndedLengthSource = R"(#version 430 core +layout(local_size_x = 16) in; +layout(std430, binding = 0) buffer Sink { uint result[]; }; +layout(std430, binding = 1) buffer Data { + uvec4 head; + double data[]; +}; +void main() { + result[gl_GlobalInvocationID.x] = uint(data.length()) + head.y; +} +)"; +} // namespace + +TEST_F(FlattenFloat64StorageBlockTest, TheLengthOfAnOpenEndedBlockIsRewrittenToAnElementCount) { + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, kOpenEndedLengthSource); + ASSERT_FALSE(input.empty()); + // glslang asks for member 1's length and signs the answer. + EXPECT_EQ(CountLinesWith(Disassemble(input), "OpArrayLength %uint", " 1"), 1u) << Disassemble(input); + + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + const String text = ExpectOpenEndedWordArray(output, "Data"); + // Re-aimed at the one member left, ... + EXPECT_EQ(CountLinesWith(text, "OpArrayLength %uint", " 0"), 1u) << text; + EXPECT_EQ(CountLinesWith(text, "OpArrayLength %uint", " 1"), 0u) << text; + // ... rebased past the 4-word prefix, clamped at zero when the range does not reach it, ... + EXPECT_EQ(CountLinesWith(text, "OpISub %uint", "%uint_4"), 1u) << text; + EXPECT_EQ(CountLinesWith(text, "OpULessThan %bool", "%uint_4"), 1u) << text; + EXPECT_EQ(CountLinesWith(text, "OpSelect %uint", "%uint_0"), 1u) << text; + // ... and divided by the 2-word stride, with glslang's own sign conversion still downstream. + EXPECT_EQ(CountLinesWith(text, "OpUDiv %uint", "%uint_2"), 1u) << text; + EXPECT_EQ(CountLinesWith(text, "OpBitcast %int", ""), 1u) << text; +} + +TEST_F(FlattenFloat64StorageBlockTest, TheLengthOfABlockWithNoPrefixNeedsNoClamp) { + const String source = R"(#version 430 core +layout(local_size_x = 16) in; +layout(std430, binding = 0) buffer Sink { uint result[]; }; +layout(std430, binding = 1) buffer Data { dvec2 data[]; }; +void main() { + result[gl_GlobalInvocationID.x] = uint(data.length()); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + const String text = ExpectOpenEndedWordArray(output, "Data"); + EXPECT_EQ(CountLinesWith(text, "OpArrayLength %uint", " 0"), 1u) << text; + // Nothing to subtract, so nothing to clamp: the word count over the 4-word stride is it. + EXPECT_EQ(CountLinesWith(text, "OpISub", ""), 0u) << text; + EXPECT_EQ(CountLinesWith(text, "OpSelect", ""), 0u) << text; + EXPECT_EQ(CountLinesWith(text, "OpUDiv %uint", "%uint_4"), 1u) << text; +} + +// The graphics shape of the same CTS group: a fragment stage reading a `readonly` block. The +// NonWritable the qualifier became is a promise about the whole block, and has to be on the one +// member the flattened block keeps. +TEST_F(FlattenFloat64StorageBlockTest, AReadOnlyOpenEndedBlockKeepsNonWritable) { + const String source = R"(#version 450 core +layout(binding = 4, std430) readonly buffer Buffer4 { dvec3 data[]; }; +layout(location = 0) out vec4 o_color; +void main() { + uint index = uint(gl_FragCoord.x); + o_color = vec4(float(data[index].z)); +} +)"; + const Vector input = CompileToSpirv(GL_FRAGMENT_SHADER, source); + ASSERT_FALSE(input.empty()); + EXPECT_EQ(CountLinesWith(Disassemble(input), "OpMemberDecorate %Buffer4 0 NonWritable", ""), 1u) + << Disassemble(input); + + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + const String text = ExpectOpenEndedWordArray(output, "Buffer4"); + EXPECT_EQ(CountLinesWith(text, "OpMemberDecorate %Buffer4 0 NonWritable", ""), 1u) << text; + // dvec3: stride 8 words, .z at +4. + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_8"), 1u) << text; + EXPECT_GE(CountLinesWith(text, "OpIAdd %uint", "%uint_4"), 1u) << text; +} + +// Writing through an open-ended block, which no CTS case does but any shader may: the store +// is decomposed into the same words the load would have read, so the bytes the application +// gets back are the ones GL says it wrote. +TEST_F(FlattenFloat64StorageBlockTest, AnOpenEndedBlockIsWrittenThroughTheSameWords) { + const String source = R"(#version 430 core +layout(local_size_x = 16) in; +layout(std430, binding = 1) buffer Data { double data[]; }; +void main() { + data[gl_LocalInvocationID.x] = double(gl_LocalInvocationID.y); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const String text = ExpectOpenEndedWordArray(output, "Data"); + // One dynamic index, scaled to the 2-word element ... + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_2"), 1u) << text; + // ... and the double left as exactly two word stores, nothing wider. + EXPECT_EQ(CountLinesWith(text, "OpStore", ""), 2u) << text; + EXPECT_EQ(CountLinesWith(text, "OpAccessChain %_ptr_StorageBuffer_uint", ""), 2u) << text; +} + +// The exact compute shader KHR-Single-GL45.subgroups.arithmetic.compute.subgroupmul_double +// generates, so the CTS shape is pinned as it is and not as a paraphrase of it. +TEST_F(FlattenFloat64StorageBlockTest, TheSubgroupMulDoubleComputeShaderIsFlattened) { + const String source = R"(#version 450 +#extension GL_KHR_shader_subgroup_arithmetic: enable +#extension GL_KHR_shader_subgroup_ballot: enable +layout (local_size_x = 16, local_size_y = 1, local_size_z = 1) in; +layout(binding = 0, std430) buffer Buffer0 +{ + uint result[]; +}; +layout(binding = 1, std430) buffer Buffer1 +{ + double data[]; +}; + +void main (void) +{ + uvec3 globalSize = gl_NumWorkGroups * gl_WorkGroupSize; + highp uint offset = globalSize.x * ((globalSize.y * gl_GlobalInvocationID.z) + gl_GlobalInvocationID.y) + gl_GlobalInvocationID.x; + uvec4 mask = subgroupBallot(true); + uint start = 0u, end = gl_SubgroupSize; + double ref = double(1); + uint tempResult = 0u; + for (uint index = start; index < end; index++) + { + if (subgroupBallotBitExtract(mask, index)) + { + ref = ref * data[index]; + } + } + tempResult = (abs(ref - subgroupMul(data[gl_SubgroupInvocationID])) < 0.00001) ? 0x1u : 0u; + if (1u == (gl_SubgroupInvocationID % 2u)) + { + mask = subgroupBallot(true); + ref = double(1); + for (uint index = start; index < end; index++) + { + if (subgroupBallotBitExtract(mask, index)) + { + ref = ref * data[index]; + } + } + tempResult |= (abs(ref - subgroupMul(data[gl_SubgroupInvocationID])) < 0.00001) ? 0x2u : 0u; + } + else + { + tempResult |= 0x2u; + } + result[offset] = tempResult; +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const String text = ExpectOpenEndedWordArray(output, "Buffer1"); + // Four reads of the array, each scaled to the 2-word element. + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_2"), 4u) << text; + // The result block holds no double and is not the pass's business. + const Uint32 resultStructId = StructIdNamed(output, "Buffer0"); + ASSERT_NE(resultStructId, 0u) << text; + const Vector resultMembers = MemberTypesOf(output, resultStructId); + ASSERT_EQ(resultMembers.size(), 1u); + EXPECT_TRUE(IsUint32Type(output, RuntimeArrayElementOf(output, resultMembers[0]))) << text; +} + +// A runtime array whose element is a MATRIX. The member's own MatrixStride and RowMajor +// decorations describe those elements, so a row-major one has to be declined - its columns are +// not contiguous, and addressing it in column order against a row-major buffer would be silently +// wrong bytes rather than a refusal. The column-major twin must flatten, stepping by the +// element's stride and then by the column's. +TEST_F(FlattenFloat64StorageBlockTest, ARowMajorMatrixRuntimeArrayIsLeftToTheDemotion) { + const String source = R"(#version 430 core +layout(local_size_x = 16) in; +layout(std430, binding = 0) buffer Sink { uint result[]; }; +layout(std430, binding = 1, row_major) buffer Data { dmat4 data[]; }; +void main() { + result[gl_GlobalInvocationID.x] = uint(data[gl_LocalInvocationID.x][1][2]); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + // The premise: glslang really did mark the member row-major. + EXPECT_EQ(CountLinesWith(Disassemble(input), "OpMemberDecorate %Data 0 RowMajor", ""), 1u) + << Disassemble(input); + + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + const String text = Disassemble(output); + const Uint32 structId = StructIdNamed(output, "Data"); + ASSERT_NE(structId, 0u) << text; + const Vector members = MemberTypesOf(output, structId); + ASSERT_EQ(members.size(), 1u) << text; + // Still a runtime array of matrices - narrowed to fp32 by the demotion, not re-addressed. + EXPECT_NE(DecorationValueOf(output, members[0], kDecorationArrayStride), 4u) + << "a row-major matrix element must not have been flattened into words\n" + << text; + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", ""), 0u) + << "nothing should have been re-addressed\n" + << text; +} + +TEST_F(FlattenFloat64StorageBlockTest, AColumnMajorMatrixRuntimeArrayStepsByItsColumnStride) { + const String source = R"(#version 430 core +layout(local_size_x = 16) in; +layout(std430, binding = 0) buffer Sink { uint result[]; }; +layout(std430, binding = 1) buffer Data { dmat2x4 data[]; }; +void main() { + dvec4 column = data[gl_LocalInvocationID.x][1]; + result[gl_GlobalInvocationID.x] = uint(column.w); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const String text = ExpectOpenEndedWordArray(output, "Data"); + // dmat2x4: two columns of dvec4, column stride 32 bytes, so one element is 64 bytes - + // 16 words - and column 1 starts 8 words into it. + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_16"), 1u) << text; + // Exactly one +8: the column's own offset inside the element. A second would mean a word + // past the column was being addressed off that same base. + EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", "%uint_8"), 1u) << text; + // All eight words of that column are read - the last of its four doubles ends at +7 ... + EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", "%uint_7"), 1u) << text; + // ... and the column that was not asked for is not touched: nothing is read at +9 or past. + EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", "%uint_9"), 0u) << text; + EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", "%uint_10"), 0u) << text; +} + +// A runtime array whose element is a STRUCT: the same walk, and the same decline test, as a +// bounded array of them - a shape no other open-ended case reaches. +TEST_F(FlattenFloat64StorageBlockTest, AStructRuntimeArrayStepsByItsElementStride) { + const String source = R"(#version 430 core +layout(local_size_x = 16) in; +struct Pair { double a; float b; }; +layout(std430, binding = 0) buffer Sink { uint result[]; }; +layout(std430, binding = 1) buffer Data { Pair data[]; }; +void main() { + result[gl_GlobalInvocationID.x] = uint(data[gl_LocalInvocationID.x].a) + + uint(data[gl_LocalInvocationID.x].b); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const String text = ExpectOpenEndedWordArray(output, "Data"); + // std430 rounds `{ double a; float b; }` up to its 8-byte alignment: 16 bytes, 4 words, + // with `b` two words in. + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_4"), 2u) << text; + EXPECT_GE(CountLinesWith(text, "OpIAdd %uint", "%uint_2"), 1u) << text; +} + +// The leaf cap bounds ONE load or store, not a member's size: a block whose element is far too +// big to expand whole is still flattened while every access to it names a scalar. Declining it +// would leave the application's 8-byte-stride doubles to the demotion's re-derived stride 4 - +// the exact defect the open-ended shape exists to avoid. +TEST_F(FlattenFloat64StorageBlockTest, AHugeRuntimeArrayElementIsStillFlattenedWhenAccessesAreSmall) { + const String source = R"(#version 430 core +layout(local_size_x = 16) in; +struct Big { dvec4 v[300]; }; +layout(std430, binding = 0) buffer Sink { uint result[]; }; +layout(std430, binding = 1) buffer Data { Big data[]; }; +void main() { + result[gl_GlobalInvocationID.x] = uint(data[gl_LocalInvocationID.x].v[3].y); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const String text = ExpectOpenEndedWordArray(output, "Data"); + // 300 dvec4 of 32 bytes each: 9600 bytes, 2400 words per element - 1200 scalars, well past + // the per-access cap that a whole-element load would have to respect and this never does. + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_2400"), 1u) << text; + // v[3].y is 3 * 8 + 2 = 26 words into the element. + EXPECT_GE(CountLinesWith(text, "OpIAdd %uint", "%uint_26"), 1u) << text; +} + +// The flatten preserves a byte layout ACROSS a narrowing; where the backend consumes 64-bit +// floats itself there is nothing to preserve, and the open-ended block has to keep its runtime +// array of doubles exactly as the driver would lay it out. +TEST_F(FlattenFloat64StorageBlockTest, TheNativePathLeavesAnOpenEndedBlockAndItsDoublesAlone) { + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, OpenEndedComputeSource("double")); + ASSERT_FALSE(input.empty()); + + Vector output; + ASSERT_TRUE(ShaderCompiler::SanitizeAndOptimizeBinary(input, output, true, true, true)); + ASSERT_FALSE(output.empty()); + + const String text = Disassemble(output); + const Uint32 structId = StructIdNamed(output, "Data"); + ASSERT_NE(structId, 0u) << text; + const Vector members = MemberTypesOf(output, structId); + ASSERT_EQ(members.size(), 1u) << text; + EXPECT_NE(RuntimeArrayElementOf(output, members[0]), 0u) + << "the member should still be a runtime array\n" + << text; + EXPECT_EQ(DecorationValueOf(output, members[0], kDecorationArrayStride), 8u) + << "the array must keep the 8-byte stride the application bound\n" + << text; + EXPECT_GT(CountFloatTypesOfWidth(output, 64), 0u) + << "nothing narrows here, so the doubles must survive\n" + << text; +} + +// The other backend prints the flattened module through SPIRV-Cross: an open-ended `uint[]` +// member has to come out as ESSL that names no 64-bit type. The `.length()` shape is here too, +// because the OpArrayLength the rewrite re-issues is the one instruction in it whose ESSL +// spelling is not plain arithmetic - if that backend ever refused it on the flattened member, +// a DirectGLES shader asking an fp64 buffer its length would fail at link and nowhere else. +namespace { + String TranspileToEssl(const Vector& spirv) { + using namespace MG_Util::ShaderTranspiler; + SpvcSession session(spirv, SessionUsageBit::Transpile); + spvc_compiler_options options; + EXPECT_EQ(session.CreateOptions(&options), SPVC_SUCCESS); + spvc_compiler_options_set_uint(options, SPVC_COMPILER_OPTION_GLSL_VERSION, 320); + spvc_compiler_options_set_bool(options, SPVC_COMPILER_OPTION_GLSL_ES, SPVC_TRUE); + spvc_compiler_options_set_bool(options, SPVC_COMPILER_OPTION_GLSL_VULKAN_SEMANTICS, SPVC_FALSE); + EXPECT_EQ(session.SetOptions(options), SPVC_SUCCESS); + + auto essl = ShaderCompiler::DecompileShader(session); + EXPECT_TRUE(essl) << (essl ? String{} : essl.error().log); + return essl ? *essl : String{}; + } +} // namespace + +TEST_F(FlattenFloat64StorageBlockTest, AnOpenEndedBlockCanBeEmittedAsEssl) { + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, OpenEndedComputeSource("dvec4")); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + ExpectOpenEndedWordArray(output, "Data"); + + const String essl = TranspileToEssl(output); + ASSERT_FALSE(essl.empty()); + EXPECT_EQ(essl.find("double"), String::npos) << essl; + EXPECT_EQ(essl.find("dvec"), String::npos) << essl; + EXPECT_NE(essl.find("uint"), String::npos) << essl; +} + +TEST_F(FlattenFloat64StorageBlockTest, TheRewrittenLengthCanBeEmittedAsEssl) { + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, kOpenEndedLengthSource); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + ExpectOpenEndedWordArray(output, "Data"); + + const String essl = TranspileToEssl(output); + ASSERT_FALSE(essl.empty()); + EXPECT_EQ(essl.find("double"), String::npos) << essl; + EXPECT_EQ(essl.find("dvec"), String::npos) << essl; + // The length survived as a length - it was not folded away or dropped on the floor. + EXPECT_NE(essl.find(".length()"), String::npos) << essl; +} + +// --------------------------------------------------------------------------- +// The gate from the other side: a runtime array anywhere but the block's own last member is a +// shape GLSL cannot spell and this pass does not describe. SPIR-V can spell it, so both are +// hand-written, and both are invalid Vulkan SPIR-V - the chain runs without its validator here, +// which is also why neither can be a validation-failure count. +// --------------------------------------------------------------------------- + +namespace { + // `buffer Odd { double data[]; uint tail; }`, the runtime array FIRST. + const char* kRuntimeArrayNotLastAsm = R"( + OpCapability Shader + OpCapability Float64 + OpMemoryModel Logical GLSL450 + OpEntryPoint GLCompute %main "main" + OpExecutionMode %main LocalSize 1 1 1 + OpName %Odd "Odd" + OpName %var "" + OpDecorate %_runtimearr_double ArrayStride 8 + OpDecorate %Odd Block + OpMemberDecorate %Odd 0 Offset 0 + OpMemberDecorate %Odd 1 Offset 8 + OpDecorate %var Binding 0 + OpDecorate %var DescriptorSet 0 + %void = OpTypeVoid + %3 = OpTypeFunction %void + %uint = OpTypeInt 32 0 + %int = OpTypeInt 32 1 + %int_0 = OpConstant %int 0 + %int_1 = OpConstant %int 1 + %double = OpTypeFloat 64 + %double_2 = OpConstant %double 2 +%_runtimearr_double = OpTypeRuntimeArray %double + %Odd = OpTypeStruct %_runtimearr_double %uint +%_ptr_StorageBuffer_Odd = OpTypePointer StorageBuffer %Odd + %var = OpVariable %_ptr_StorageBuffer_Odd StorageBuffer +%_ptr_StorageBuffer_double = OpTypePointer StorageBuffer %double + %main = OpFunction %void None %3 + %5 = OpLabel + %6 = OpAccessChain %_ptr_StorageBuffer_double %var %int_0 %int_1 + OpStore %6 %double_2 + OpReturn + OpFunctionEnd +)"; + + // `struct Inner { double data[]; }; buffer Outer { uint head; Inner inner; }`: the runtime + // array IS last, but of a member rather than of the block. + const char* kRuntimeArrayNestedAsm = R"( + OpCapability Shader + OpCapability Float64 + OpMemoryModel Logical GLSL450 + OpEntryPoint GLCompute %main "main" + OpExecutionMode %main LocalSize 1 1 1 + OpName %Outer "Outer" + OpName %Inner "Inner" + OpName %var "" + OpDecorate %_runtimearr_double ArrayStride 8 + OpMemberDecorate %Inner 0 Offset 0 + OpDecorate %Outer Block + OpMemberDecorate %Outer 0 Offset 0 + OpMemberDecorate %Outer 1 Offset 8 + OpDecorate %var Binding 0 + OpDecorate %var DescriptorSet 0 + %void = OpTypeVoid + %3 = OpTypeFunction %void + %uint = OpTypeInt 32 0 + %int = OpTypeInt 32 1 + %int_0 = OpConstant %int 0 + %int_1 = OpConstant %int 1 + %double = OpTypeFloat 64 + %double_2 = OpConstant %double 2 +%_runtimearr_double = OpTypeRuntimeArray %double + %Inner = OpTypeStruct %_runtimearr_double + %Outer = OpTypeStruct %uint %Inner +%_ptr_StorageBuffer_Outer = OpTypePointer StorageBuffer %Outer + %var = OpVariable %_ptr_StorageBuffer_Outer StorageBuffer +%_ptr_StorageBuffer_double = OpTypePointer StorageBuffer %double + %main = OpFunction %void None %3 + %5 = OpLabel + %6 = OpAccessChain %_ptr_StorageBuffer_double %var %int_1 %int_0 %int_1 + OpStore %6 %double_2 + OpReturn + OpFunctionEnd +)"; + + Vector AssembleUnchecked(const char* asmText) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_1); + Vector module; + EXPECT_TRUE(tools.Assemble(asmText, &module)); + return module; + } +} // namespace + +TEST_F(FlattenFloat64StorageBlockTest, ARuntimeArrayThatIsNotTheBlocksLastMemberIsLeftToTheDemotion) { + struct Shape { + const char* asmText; + const char* blockName; + }; + const Shape shapes[] = {{kRuntimeArrayNotLastAsm, "Odd"}, {kRuntimeArrayNestedAsm, "Outer"}}; + for (const Shape& shape : shapes) { + SCOPED_TRACE(shape.blockName); + const Vector input = AssembleUnchecked(shape.asmText); + ASSERT_FALSE(input.empty()); + + Vector output; + ASSERT_TRUE(ShaderCompiler::SanitizeAndOptimizeBinary(input, output, false, false)); + ASSERT_FALSE(output.empty()); + const String text = Disassemble(output); + + // Declined: both members are still there, and the demotion narrowed them the old way. + const Uint32 structId = StructIdNamed(output, shape.blockName); + ASSERT_NE(structId, 0u) << text; + EXPECT_EQ(MemberTypesOf(output, structId).size(), 2u) << text; + EXPECT_EQ(CountFloatTypesOfWidth(output, 64), 0u) << text; + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", ""), 0u) + << "nothing should have been re-addressed\n" + << text; + } +} + +// --------------------------------------------------------------------------- +// The front end declares types in first-use order, so a block that is the first thing the +// shader touches is declared before the module's `uint` - and the flattened member is an array +// OF `uint`. For an OPEN-ENDED block the pass moves that operand-less type up in front of the +// block rather than declining, so that where a buffer of doubles stands in the shader does not +// decide whether its bytes survive. A BOUNDED block in the same position keeps the decline it +// has always had: widening that is a change to a path this fix does not need, and the pair below +// pins both halves. +// --------------------------------------------------------------------------- + +namespace { + // The position of 's declaration in instruction order, or 0 when it has none. + Uint32 DeclarationIndexOf(const Vector& spirv, Uint32 id) { + Uint32 index = 0; + Uint32 found = 0; + ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) { + ++index; + if (found != 0 || wordCount < 2) return; + // Every OpType* has its result id in word 1; that is all this is asked about. + if (opcode >= kOpTypeInt && opcode <= kOpTypeStruct && words[1] == id) found = index; + }); + return found; + } + + Uint32 Uint32TypeIdOf(const Vector& spirv) { + Uint32 typeId = 0; + ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) { + if (opcode == kOpTypeInt && wordCount >= 4 && words[2] == 32u && words[3] == 0u) typeId = words[1]; + }); + return typeId; + } +} // namespace + +TEST_F(FlattenFloat64StorageBlockTest, AnOpenEndedBlockDeclaredBeforeTheModulesUintIsStillFlattened) { + // The block is the first thing main touches, and nothing before it needs a uint - not even + // an array length, which is a uint constant and would declare one. + const String source = R"(#version 430 core +layout(local_size_x = 1) in; +layout(std430, binding = 0) buffer Data { double data[]; }; +layout(std430, binding = 1) buffer Sink { float result[]; }; +void main() { + result[0] = float(data[0] + data[1]); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Uint32 inputStructId = StructIdNamed(input, "Data"); + ASSERT_NE(inputStructId, 0u); + const Uint32 inputUintId = Uint32TypeIdOf(input); + // The premise: the module's uint really is declared after the block (or not at all). + ASSERT_TRUE(inputUintId == 0 || + DeclarationIndexOf(input, inputUintId) > DeclarationIndexOf(input, inputStructId)) + << "this shader was meant to declare the block before any uint\n" + << Disassemble(input); + + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + const String text = ExpectOpenEndedWordArray(output, "Data"); + const Uint32 structId = StructIdNamed(output, "Data"); + ASSERT_NE(structId, 0u) << text; + const Vector members = MemberTypesOf(output, structId); + ASSERT_EQ(members.size(), 1u) << text; + // And the uint now stands in front of the block it is an element of. + EXPECT_LT(DeclarationIndexOf(output, RuntimeArrayElementOf(output, members[0])), + DeclarationIndexOf(output, structId)) + << text; +} + +TEST_F(FlattenFloat64StorageBlockTest, ABoundedBlockDeclaredBeforeTheModulesUintIsLeftToTheDemotion) { + // The same position, a bounded block: this is the shape that has always been declined, and + // it stays declined - its members and the demotion's own repacking come through untouched. + const String source = R"(#version 430 core +layout(local_size_x = 1) in; +layout(std430, binding = 0) buffer Wide { + double data0; + dvec2 data1; +} g_wide; +layout(std430, binding = 1) buffer Sink { float result[]; }; +void main() { + double sum = g_wide.data0 + g_wide.data1.y; + result[0] = float(sum); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Uint32 inputStructId = StructIdNamed(input, "Wide"); + ASSERT_NE(inputStructId, 0u); + const Uint32 inputUintId = Uint32TypeIdOf(input); + ASSERT_TRUE(inputUintId == 0 || + DeclarationIndexOf(input, inputUintId) > DeclarationIndexOf(input, inputStructId)) + << "this shader was meant to declare the block before any uint\n" + << Disassemble(input); + + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + const String text = Disassemble(output); + const Uint32 structId = StructIdNamed(output, "Wide"); + ASSERT_NE(structId, 0u) << text; + EXPECT_EQ(MemberTypesOf(output, structId).size(), 2u) + << "a bounded block in this position must keep the decline it shipped with\n" + << text; + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", ""), 0u) + << "nothing should have been re-addressed\n" + << text; +} diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.cpp b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.cpp index 66a62037..7f920ccf 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.cpp +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.cpp @@ -84,8 +84,18 @@ namespace MobileGL { struct BlockPlan { Instruction* structType = nullptr; uint32_t storageClass = 0; + // A bounded block's length in words. For an open-ended block - one whose + // last member is a runtime array - the FIXED PREFIX in words, i.e. the + // runtime array's own offset, which is where its element 0 starts. uint32_t wordCount = 0; + bool openEnded = false; + // The original runtime array's stride in words; what one element of it + // steps by, and what its word count divides by to become a length. + uint32_t tailStrideWords = 0; std::vector chains; + // The OpArrayLength users of an open-ended block's variables, which count + // WORDS once the member is a `uint[]` and so have to be rewritten too. + std::vector arrayLengths; }; bool IsDoubleType(const Instruction* type) { @@ -178,8 +188,9 @@ namespace MobileGL { } // Byte size of a type as it is laid out INSIDE a block, or 0 when this pass - // cannot describe it (a runtime array, a width it does not carry, a matrix with - // no stride or a row-major one). + // cannot describe it (a runtime array - the one place a block may have one is + // its last member, which MeasureBlock handles above this - a width it does not + // carry, a matrix with no stride or a row-major one). uint32_t LaidOutByteSize(IRContext* context, const TypeCursor& cursor) { const Instruction* type = context->get_def_use_mgr()->GetDef(cursor.typeId); if (type == nullptr) return 0; @@ -231,8 +242,17 @@ namespace MobileGL { } // Whether this type decomposes into scalars the rewrite can move one word at a - // time, counting them so a whole-aggregate access can be refused before it is - // expanded. + // time. |leafCount| counts them, so a whole-aggregate access can be refused + // before it is expanded; passing NULL asks the SHAPE question alone - is this + // type addressable at all - and then identical array elements and vector + // components are walked once instead of once each, because the answer cannot + // differ between them and the walk of a big one would not be free. + // + // The two questions are separate because only a LOAD or a STORE expands into + // leaves, and the cap bounds one of those. How large a runtime array's element + // is says nothing about how many scalars a single access to it moves, so + // MeasureBlock asks for the shape and BuildPlans applies the cap where it + // belongs - per chain, to the type that chain actually names. bool CanDecompose(IRContext* context, const TypeCursor& cursor, uint32_t* leafCount) { const Instruction* type = context->get_def_use_mgr()->GetDef(cursor.typeId); if (type == nullptr) return false; @@ -240,12 +260,15 @@ namespace MobileGL { case spv::Op::OpTypeInt: case spv::Op::OpTypeFloat: if (ScalarByteSize(type) == 0) return false; + if (leafCount == nullptr) return true; ++*leafCount; return *leafCount <= kMaxLeavesPerAccess; case spv::Op::OpTypeVector: { TypeCursor component; component.typeId = type->GetSingleWordInOperand(0); - for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) { + const uint32_t repeats = + leafCount == nullptr ? 1u : type->GetSingleWordInOperand(1); + for (uint32_t i = 0; i < repeats; ++i) { if (!CanDecompose(context, component, leafCount)) return false; } return true; @@ -257,7 +280,9 @@ namespace MobileGL { } TypeCursor column; column.typeId = type->GetSingleWordInOperand(0); - for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) { + const uint32_t repeats = + leafCount == nullptr ? 1u : type->GetSingleWordInOperand(1); + for (uint32_t i = 0; i < repeats; ++i) { if (!CanDecompose(context, column, leafCount)) return false; } return true; @@ -273,10 +298,12 @@ namespace MobileGL { context->get_constant_mgr()->FindDeclaredConstant(type->GetSingleWordInOperand(1)); if (length == nullptr || length->AsIntConstant() == nullptr) return false; const uint32_t count = length->AsIntConstant()->GetU32BitValue(); - if (count == 0 || count > kMaxLeavesPerAccess) return false; + if (count == 0) return false; + if (leafCount != nullptr && count > kMaxLeavesPerAccess) return false; TypeCursor element = cursor; element.typeId = type->GetSingleWordInOperand(0); - for (uint32_t i = 0; i < count; ++i) { + const uint32_t repeats = leafCount == nullptr ? 1u : count; + for (uint32_t i = 0; i < repeats; ++i) { if (!CanDecompose(context, element, leafCount)) return false; } return true; @@ -300,6 +327,63 @@ namespace MobileGL { } } + // Measures the block struct itself. A bounded block reports its laid-out byte + // size; a block whose LAST member is a runtime array - the only place GLSL lets + // one stand, and the only place SPIR-V lets a Block have one - reports the byte + // offset that array starts at and says so through |openEnded|, with the array's + // stride alongside. A runtime array anywhere else, one without a stride the + // words can step by, or one whose element the rewrite could not take apart is a + // shape this pass does not describe, and so is a bounded block it cannot size. + bool MeasureBlock(IRContext* context, const Instruction* structType, uint32_t* bytes, + bool* openEnded, uint32_t* tailStrideBytes) { + *bytes = 0; + *openEnded = false; + *tailStrideBytes = 0; + const uint32_t structId = structType->result_id(); + const uint32_t memberCount = structType->NumInOperands(); + uint64_t end = 0; + for (uint32_t member = 0; member < memberCount; ++member) { + uint32_t offset = 0; + if (!TryGetMemberDecorationLiteral(context, structId, member, spv::Decoration::Offset, + &offset)) { + return false; + } + const TypeCursor cursor = MemberCursor(context, structType, member); + const Instruction* type = context->get_def_use_mgr()->GetDef(cursor.typeId); + if (type == nullptr) return false; + if (type->opcode() == spv::Op::OpTypeRuntimeArray) { + if (member + 1 != memberCount) return false; + uint32_t stride = 0; + if (!TryGetDecorationLiteral(context, cursor.typeId, spv::Decoration::ArrayStride, + &stride) || + stride == 0 || stride % kWordBytes != 0) { + return false; + } + // The member's own matrix decorations describe the array's ELEMENTS, + // exactly as they do for a bounded array of matrices. Only the shape + // is asked for: how big one element is decides nothing about how + // many scalars one access moves, and a leaf cap here would decline a + // block over a member the shader may never read whole. + TypeCursor element = cursor; + element.typeId = type->GetSingleWordInOperand(0); + if (!CanDecompose(context, element, nullptr)) return false; + // Element 0 has to start past every fixed member, or the words the + // prefix owns and the words the array owns would overlap. + if (offset < end) return false; + end = offset; + *openEnded = true; + *tailStrideBytes = stride; + break; + } + const uint32_t size = LaidOutByteSize(context, cursor); + if (size == 0) return false; + end = std::max(end, static_cast(offset) + size); + } + if (end > kMaxBlockBytes) return false; + *bytes = static_cast(end); + return true; + } + bool TypeContainsFloat64(IRContext* context, uint32_t typeId, std::unordered_set& visiting) { const Instruction* type = context->get_def_use_mgr()->GetDef(typeId); @@ -366,6 +450,9 @@ namespace MobileGL { switch (type->opcode()) { case spv::Op::OpTypeArray: + // A runtime array steps exactly like a bounded one; only its end is + // unknown, and a chain never needs that. + case spv::Op::OpTypeRuntimeArray: if (!TryGetDecorationLiteral(context, cursor.typeId, spv::Decoration::ArrayStride, &stride)) { return false; @@ -611,6 +698,39 @@ namespace MobileGL { } } + // Replaces an OpArrayLength of an open-ended block with the element count + // of the ORIGINAL runtime array. The instruction now counts the words of + // the flattened `uint[]`, so the length is `(words - prefix) / stride`, in + // unsigned arithmetic and clamped at zero when the bound range does not + // even reach the array's offset - a wrapped subtraction would otherwise + // report a few billion elements. The division floors, which is what GL + // defines `.length()` as for a range that is not a whole number of + // elements. A fresh OpArrayLength is issued rather than the old one re-aimed, + // so the uses being redirected are never the ones the arithmetic just made. + void RewriteArrayLength(Instruction* arrayLength, uint32_t prefixWords, uint32_t strideWords) { + InstructionBuilder builder(m_context, arrayLength, kPreservedAnalyses); + const uint32_t variableId = arrayLength->GetSingleWordInOperand(0); + const uint32_t wordsId = m_context->TakeNextId(); + builder.AddInstruction(MakeUnique( + m_context, spv::Op::OpArrayLength, m_uintTypeId, wordsId, + std::initializer_list{{SPV_OPERAND_TYPE_ID, {variableId}}, + {SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}})); + uint32_t count = wordsId; + if (prefixWords != 0) { + const uint32_t prefixId = UintConstant(prefixWords); + const uint32_t past = Binary(builder, spv::Op::OpISub, m_uintTypeId, count, prefixId); + const uint32_t tooShort = + Binary(builder, spv::Op::OpULessThan, m_boolTypeId, count, prefixId); + count = Select(builder, tooShort, UintConstant(0), past); + } + if (strideWords != 1) { + count = Binary(builder, spv::Op::OpUDiv, m_uintTypeId, count, + UintConstant(strideWords)); + } + m_context->ReplaceAllUsesWith(arrayLength->result_id(), count); + m_context->KillInst(arrayLength); + } + private: uint32_t ComponentWords(uint32_t componentTypeId) { return ScalarByteSize(m_context->get_def_use_mgr()->GetDef(componentTypeId)) / @@ -788,38 +908,72 @@ namespace MobileGL { return false; } - // A fresh `uint[length]` with ArrayStride 4, spliced in immediately BEFORE the - // block that will name it - SPIR-V has no forward references between types, so - // appending it at the end of the section would make the module invalid. A - // duplicate OpTypeArray is legal (SPIR-V 2.8 exempts aggregates from the - // uniqueness rule, and so does spirv-val), so no search for an existing one is - // needed; the LENGTH CONSTANT is not exempt, and if the module already declares - // it after the block there is nowhere legal to put the array - the block is then - // declined and keeps today's behaviour. Returns 0 for that, and for a uint type - // that is itself declared too late. + // A fresh `uint[length]` with ArrayStride 4 - or, for an open-ended block, a + // `uint[]` runtime array with the same stride and no length at all - spliced in + // immediately BEFORE the block that will name it: SPIR-V has no forward + // references between types, so appending it at the end of the section would make + // the module invalid. A duplicate OpTypeArray or OpTypeRuntimeArray is legal + // (SPIR-V 2.8 exempts aggregates from the uniqueness rule, and so does + // spirv-val), so no search for an existing one is needed; the LENGTH CONSTANT is + // not exempt, and if the module already declares it after the block there is + // nowhere legal to put the array - the block is then declined and keeps today's + // behaviour. Returns 0 for that; an open-ended block has no length constant to + // place, so that reason cannot reach it. + // + // The `uint` element type is a different matter, and only for an OPEN-ENDED + // block. The front end declares types in first-use order, so a block that is the + // first thing a shader touches sits BEFORE the module's `uint` (or the module has + // none, and the one the pass asked for was appended at the end). Declining there + // would send exactly the buffers this rewrite exists for back to the demotion on + // nothing but where they stand in the source. OpTypeInt has no operands, and + // nothing that names it can precede where it was, so moving it up in front of the + // block is always legal. A BOUNDED block keeps declining instead: that is what it + // has always done, and widening it is a change to a path this one does not need. + // + // NOTHING IS WRITTEN until every reason to decline has been ruled out, so a block + // this returns 0 for leaves the module as it found it - which is what lets + // Process() truthfully report SuccessWithoutChange for a module of only those. uint32_t CreateWordArrayTypeBefore(IRContext* context, Instruction* structType, - uint32_t uintTypeId, uint32_t length) { - if (!DeclaredBefore(context, uintTypeId, structType->result_id())) return 0; + uint32_t uintTypeId, uint32_t length, bool openEnded) { + Instruction* uintType = context->get_def_use_mgr()->GetDef(uintTypeId); + if (uintType == nullptr || uintType->opcode() != spv::Op::OpTypeInt) return 0; + const bool hoistUint = !DeclaredBefore(context, uintTypeId, structType->result_id()); + if (hoistUint && !openEnded) return 0; - auto* constantMgr = context->get_constant_mgr(); - const spvtools::opt::analysis::Type* uintType = context->get_type_mgr()->GetType(uintTypeId); - if (uintType == nullptr) return 0; - const spvtools::opt::analysis::Constant* lengthConstant = - constantMgr->GetConstant(uintType, {length}); - if (lengthConstant == nullptr) return 0; + uint32_t lengthConstantId = 0; + if (!openEnded) { + auto* constantMgr = context->get_constant_mgr(); + const spvtools::opt::analysis::Type* uintDescriptor = + context->get_type_mgr()->GetType(uintTypeId); + if (uintDescriptor == nullptr) return 0; + const spvtools::opt::analysis::Constant* lengthConstant = + constantMgr->GetConstant(uintDescriptor, {length}); + if (lengthConstant == nullptr) return 0; - Module::inst_iterator position = PositionOf(context, structType); - if (position == context->types_values_end()) return 0; - Instruction* lengthInst = constantMgr->GetDefiningInstruction(lengthConstant, 0, &position); - if (lengthInst == nullptr) return 0; - if (!DeclaredBefore(context, lengthInst->result_id(), structType->result_id())) return 0; + Module::inst_iterator position = PositionOf(context, structType); + if (position == context->types_values_end()) return 0; + // Created in front of the block when it is not there yet, so the only way + // this declines is a constant the module already declares after it. + Instruction* lengthInst = + constantMgr->GetDefiningInstruction(lengthConstant, 0, &position); + if (lengthInst == nullptr) return 0; + if (!DeclaredBefore(context, lengthInst->result_id(), structType->result_id())) return 0; + lengthConstantId = lengthInst->result_id(); + } const uint32_t arrayTypeId = context->TakeNextId(); if (arrayTypeId == 0) return 0; - auto arrayType = MakeUnique( - context, spv::Op::OpTypeArray, 0, arrayTypeId, - std::initializer_list{{SPV_OPERAND_TYPE_ID, {uintTypeId}}, - {SPV_OPERAND_TYPE_ID, {lengthInst->result_id()}}}); + + if (hoistUint) uintType->InsertBefore(structType); + std::unique_ptr arrayType = + openEnded + ? MakeUnique( + context, spv::Op::OpTypeRuntimeArray, 0, arrayTypeId, + std::initializer_list{{SPV_OPERAND_TYPE_ID, {uintTypeId}}}) + : MakeUnique( + context, spv::Op::OpTypeArray, 0, arrayTypeId, + std::initializer_list{{SPV_OPERAND_TYPE_ID, {uintTypeId}}, + {SPV_OPERAND_TYPE_ID, {lengthConstantId}}}); Instruction* inserted = structType->InsertBefore(std::move(arrayType)); context->AnalyzeDefUse(inserted); context->get_decoration_mgr()->AddDecorationVal( @@ -948,8 +1102,14 @@ namespace MobileGL { Instruction* structType = defUseMgr->GetDef(structId); TypeCursor blockCursor; blockCursor.typeId = structId; - const uint32_t blockBytes = LaidOutByteSize(context, blockCursor); - if (blockBytes == 0 || blockBytes % kWordBytes != 0) { + uint32_t blockBytes = 0; + bool openEnded = false; + uint32_t tailStrideBytes = 0; + // An open-ended block whose runtime array is its only member measures a + // prefix of 0 bytes and is perfectly describable; only a BOUNDED block of + // no bytes is not, and MeasureBlock already refuses to size one of those. + if (!MeasureBlock(context, structType, &blockBytes, &openEnded, &tailStrideBytes) || + blockBytes % kWordBytes != 0 || (!openEnded && blockBytes == 0)) { MGLOG_D("[spirv] storage block %%%u holds a double but its byte layout cannot be " "described exactly; left to the fp64 demotion", structId); @@ -960,11 +1120,15 @@ namespace MobileGL { plan.structType = structType; plan.storageClass = storageClassByStruct[structId]; plan.wordCount = blockBytes / kWordBytes; + plan.openEnded = openEnded; + plan.tailStrideWords = tailStrideBytes / kWordBytes; + const uint32_t lastMember = structType->NumInOperands() - 1; bool expressible = true; for (Instruction* variable : variablesByStruct[structId]) { std::vector chains; std::unordered_set seenChains; + std::unordered_set seenLengths; defUseMgr->ForEachUser(variable, [&](Instruction* user) { if (!expressible) return; switch (user->opcode()) { @@ -982,6 +1146,27 @@ namespace MobileGL { } expressible = false; return; + case spv::Op::OpArrayLength: { + // Only an open-ended block has a length to ask for, and + // only of its last member; the result has to be the 32-bit + // uint the rewrite's arithmetic is typed in, which is the + // only result type the instruction allows anyway. + const Instruction* resultType = defUseMgr->GetDef(user->type_id()); + const bool isUint = resultType != nullptr && + resultType->opcode() == spv::Op::OpTypeInt && + resultType->GetSingleWordInOperand(0) == 32u && + resultType->GetSingleWordInOperand(1) == 0u; + if (openEnded && isUint && user->NumInOperands() >= 2 && + user->GetSingleWordInOperand(0) == variable->result_id() && + user->GetSingleWordInOperand(1) == lastMember) { + if (seenLengths.insert(user->result_id()).second) { + plan.arrayLengths.push_back(user); + } + return; + } + expressible = false; + return; + } default: expressible = false; return; @@ -1058,16 +1243,25 @@ namespace MobileGL { Emitter emitter(irContext, uintTypeId, boolTypeId, floatTypeId); bool modified = false; + // Every block declines before anything is written for it, so |touched| only ever + // parts company with |modified| on a shape that cannot happen without the module + // running out of ids - and even then the status must not claim the bytes are + // untouched, because the caller relies on that to skip invalidating its analyses. + bool touched = false; for (BlockPlan& plan : plans) { const uint32_t structId = plan.structType->result_id(); - const uint32_t arrayTypeId = - CreateWordArrayTypeBefore(irContext, plan.structType, uintTypeId, plan.wordCount); + const uint32_t arrayTypeId = CreateWordArrayTypeBefore( + irContext, plan.structType, uintTypeId, plan.wordCount, plan.openEnded); if (arrayTypeId == 0) { + // Nothing was written for it, so the module is still the one that came in. MGLOG_D("[spirv] storage block %%%u: no legal place for the flattened word array; " "left to the fp64 demotion", structId); continue; } + // Past this point the module HAS been written to, so an abandoned block would + // leave a dead type behind - the status has to say so even then. + touched = true; const uint32_t wordPointerTypeId = irContext->get_type_mgr()->FindPointerToType( uintTypeId, static_cast(plan.storageClass)); if (wordPointerTypeId == 0) continue; @@ -1095,6 +1289,9 @@ namespace MobileGL { } irContext->KillInst(chainPlan.chain); } + for (Instruction* arrayLength : plan.arrayLengths) { + emitter.RewriteArrayLength(arrayLength, plan.wordCount, plan.tailStrideWords); + } const std::vector surviving = SurvivingAccessQualifiers( irContext, structId, plan.structType->NumInOperands()); @@ -1113,12 +1310,19 @@ namespace MobileGL { {SPV_OPERAND_TYPE_DECORATION, {static_cast(kind)}}}); } modified = true; - MGLOG_D("[spirv] storage block %%%u: flattened into %u words so its 64-bit members keep " - "the byte layout the application bound", - structId, plan.wordCount); + if (plan.openEnded) { + MGLOG_D("[spirv] storage block %%%u: flattened into an open-ended word array (%u-word " + "prefix, %u-word elements) so its 64-bit members keep the byte layout the " + "application bound", + structId, plan.wordCount, plan.tailStrideWords); + } else { + MGLOG_D("[spirv] storage block %%%u: flattened into %u words so its 64-bit members " + "keep the byte layout the application bound", + structId, plan.wordCount); + } } - if (!modified) { + if (!modified && !touched) { return Status::SuccessWithoutChange; } irContext->InvalidateAnalysesExceptFor(IRContext::kAnalysisNone); diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.h b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.h index c21dfd5f..f6c8c1fc 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.h +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.h @@ -66,17 +66,32 @@ namespace MobileGL { // fp32 promise DemoteFloat64Pass already makes - what changes is only that the // BYTES around the value stay where the application put them. // + // AN OPEN-ENDED BLOCK - one whose last member is a runtime array, the + // `buffer B { double data[]; }` every unsized storage buffer is spelled as - is + // flattened the same way: the members before the array are the fixed prefix, and + // the flattened member is itself a `uint[]` runtime array, ArrayStride 4, with no + // length for the driver to re-derive. Element i of the original array lives at + // word `prefix + i * stride` of it, which is where the application put it. The + // block's `.length()` is rewritten too, because OpArrayLength on the flattened + // member counts WORDS: it becomes `(words - prefix) / stride` in unsigned + // arithmetic, clamped at zero when the bound range is shorter than the prefix, + // which is the floor GL defines `.length()` as. + // // DECLINES, leaving the block exactly as it was for DemoteFloat64Pass to handle the // old way, whenever it meets something it cannot rewrite exactly: // - a block whose variable is used as anything but an access-chain base (loaded - // whole, handed to a function, asked its OpArrayLength); + // whole, handed to a function), or asked an OpArrayLength it is not open-ended + // for; // - an access chain that is not rooted at the variable, or whose result feeds // anything but a plain OpLoad / OpStore (an atomic, OpCopyMemory, a further - // chain); - // - a non-constant index into a struct, a runtime array anywhere in the block, a - // RowMajor matrix (its columns are not contiguous, so a whole-column access is - // not one range), a member width other than 32 or 64 bits, or an offset or - // stride that is not a multiple of 4; + // chain), or one that names a whole runtime array rather than an element of it; + // - a non-constant index into a struct, a runtime array that is not the last + // member of the block itself (nested in a member, or followed by another - + // shapes GLSL cannot spell but SPIR-V can), a runtime array without an + // ArrayStride or whose element the pass cannot decompose, a RowMajor matrix (its + // columns are not contiguous, so a whole-column access is not one range), a + // member width other than 32 or 64 bits, or an offset or stride that is not a + // multiple of 4; // - a load or store whose type decomposes into more scalars than the cap below, // so legalizing a block can never explode the module. //