diff --git a/CMakeLists.txt b/CMakeLists.txt index e6e6259c..aab1cf97 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -279,6 +279,7 @@ set(SOURCE_FILES MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DecomposeWorkgroupVec3Pass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DecoratePositionInvariantPass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DemoteFloat64Pass.cpp + MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/LowerDrawParametersPass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/LowerViewportIndexPass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/PackDoubleVertexInputsPass.cpp diff --git a/MobileGL/MG_Backend/DirectGLES/Managers.cpp b/MobileGL/MG_Backend/DirectGLES/Managers.cpp index 406b954a..74cf1cd5 100644 --- a/MobileGL/MG_Backend/DirectGLES/Managers.cpp +++ b/MobileGL/MG_Backend/DirectGLES/Managers.cpp @@ -6106,6 +6106,22 @@ namespace MobileGL::MG_Backend::DirectGLES { spvcSession.SetAtomicCounterBlockBindings(atomicCounterEsslBindingTop, outAtomicCounterGlBindings); + // `layout(index = 0)` is the GL default spelled out loud, and GLSL ES has no such + // qualifier in core - a stage that prints it is refused with "index layout + // qualifier requires EXT_blend_func_extended" and the whole program then draws + // nothing. Drop the decoration when it carries the default; a REAL dual-source + // index (1) is left alone, because that one genuinely needs the extension and the + // driver has to see it. Fragment stage only: no other stage can carry it. + if (glShaderType == GL_FRAGMENT_SHADER) { + spvcSession.DropDefaultFragmentOutputColorIndex(); + } + + // `readonly writeonly` together says the buffer variable can only be asked its + // .length(), which the frontend has already enforced - so the pair is inert, and + // printing it is not. Mesa's ES compiler refuses a block spelled that way and the + // stage never reaches the program. + spvcSession.RelaxReadWriteExclusiveStorageBuffers(); + const char* result = nullptr; spvcSession.Compile(&result); diff --git a/MobileGL/MG_IntegrationTest/Scenarios/DoublePrecisionScenario.cpp b/MobileGL/MG_IntegrationTest/Scenarios/DoublePrecisionScenario.cpp index 682719e2..4028cd7d 100644 --- a/MobileGL/MG_IntegrationTest/Scenarios/DoublePrecisionScenario.cpp +++ b/MobileGL/MG_IntegrationTest/Scenarios/DoublePrecisionScenario.cpp @@ -27,6 +27,7 @@ // which is the whole point. #include +#include #include #include @@ -153,6 +154,151 @@ void main() { std::string m_buildLog; }; + // A SHADER STORAGE BLOCK that holds doubles is the one place the narrowing is NOT free: + // demoting `double` to `float` also repacks the block, and the bytes the application + // wrote into the buffer do not move with it. Every member past the first double then + // reads and writes at the wrong offset, and the block is simply shorter than the one + // that was bound - the tail of it is never touched at all + // (KHR-GL43.shader_storage_buffer_object.basic-stdLayout-case3, whose output matched its + // input up to the first double's slot and was zero from there on). + // + // The block layout is fixed by GL 4.6 core 7.6.2.2 and is asserted here as literal byte + // offsets rather than queried, so this says what the SPEC requires and not what MobileGL + // happens to report. Both packings are covered because they differ in exactly the places + // that matter: std140 rounds an array's stride and a matrix's column stride up to 16, + // std430 does not, and only std430 packs the scalars tightly. + // + // Every value is exactly representable in binary32, so a correct implementation copies + // the block BYTE FOR BYTE even though it narrows each double on the way through. + constexpr const char* kBlockCopySource = R"(#version 430 core +layout(local_size_x = 1) in; +layout(std140, binding = 0) buffer In140 { + int data0; + float data1[3]; + mat3x2 data2; + double data3; + double data4[2]; + int data5; + dvec3 data6; +} g_in140; +layout(std430, binding = 1) buffer In430 { + int data0; + float data1[3]; + mat3x2 data2; + double data3; + double data4[2]; + int data5; + dvec3 data6; +} g_in430; +layout(std140, binding = 2) buffer Out140 { + int data0; + float data1[3]; + mat3x2 data2; + double data3; + double data4[2]; + int data5; + dvec3 data6; +} g_out140; +layout(std430, binding = 3) buffer Out430 { + int data0; + float data1[3]; + mat3x2 data2; + double data3; + double data4[2]; + int data5; + dvec3 data6; +} g_out430; +void main() { + g_out140.data0 = g_in140.data0; + for (int i = 0; i < 3; ++i) g_out140.data1[i] = g_in140.data1[i]; + g_out140.data2 = g_in140.data2; + g_out140.data3 = g_in140.data3; + for (int i = 0; i < 2; ++i) g_out140.data4[i] = g_in140.data4[i]; + g_out140.data5 = g_in140.data5; + g_out140.data6 = g_in140.data6; + + g_out430.data0 = g_in430.data0; + for (int i = 0; i < 3; ++i) g_out430.data1[i] = g_in430.data1[i]; + g_out430.data2 = g_in430.data2; + g_out430.data3 = g_in430.data3; + for (int i = 0; i < 2; ++i) g_out430.data4[i] = g_in430.data4[i]; + g_out430.data5 = g_in430.data5; + g_out430.data6 = g_in430.data6; +} +)"; + + // GL 4.6 core 7.6.2.2 rule by rule, for the block above. + // std140: an array's element stride and a matrix's column stride round up to 16, a + // double aligns to 8 and a dvec3 to 32. + // std430: the same without the rounding - so the scalars pack tightly and only the + // dvec3's 32-byte alignment leaves a hole. + struct BlockLayout { + int data0; + int data1; + int data1Stride; + int data2; + int data2ColumnStride; + int data3; + int data4; + int data4Stride; + int data5; + int data6; + int size; + }; + constexpr BlockLayout kStd140{0, 16, 16, 64, 16, 112, 128, 16, 160, 192, 216}; + constexpr BlockLayout kStd430{0, 4, 4, 16, 8, 40, 48, 8, 64, 96, 120}; + + void PokeInt(std::vector& bytes, int offset, int value) { + std::memcpy(&bytes[static_cast(offset)], &value, sizeof(value)); + } + void PokeFloat(std::vector& bytes, int offset, float value) { + std::memcpy(&bytes[static_cast(offset)], &value, sizeof(value)); + } + void PokeDouble(std::vector& bytes, int offset, double value) { + std::memcpy(&bytes[static_cast(offset)], &value, sizeof(value)); + } + + // The block's contents, at the offsets the standard puts them. Padding stays zero, which + // is what makes a byte-for-byte comparison against the (zero-initialised) output buffer + // catch a member that landed somewhere it should not have. + std::vector MakeBlockContents(const BlockLayout& layout) { + std::vector bytes(static_cast(layout.size), 0); + PokeInt(bytes, layout.data0, 1); + for (int i = 0; i < 3; ++i) { + PokeFloat(bytes, layout.data1 + i * layout.data1Stride, 2.0f + static_cast(i)); + } + // Column-major, two rows per column. + for (int column = 0; column < 3; ++column) { + for (int row = 0; row < 2; ++row) { + PokeFloat(bytes, layout.data2 + column * layout.data2ColumnStride + row * 4, + 5.0f + static_cast(column * 2 + row)); + } + } + PokeDouble(bytes, layout.data3, 11.0); + for (int i = 0; i < 2; ++i) { + PokeDouble(bytes, layout.data4 + i * layout.data4Stride, 12.0 + static_cast(i)); + } + PokeInt(bytes, layout.data5, 14); + for (int i = 0; i < 3; ++i) { + PokeDouble(bytes, layout.data6 + i * 8, 15.0 + static_cast(i)); + } + return bytes; + } + + // Names the first byte that differs, and which member owns it, so a failure is a + // diagnosis rather than "the buffer is wrong". + std::string DescribeOffset(const BlockLayout& layout, int offset) { + const std::pair members[] = { + {layout.data0, "data0"}, {layout.data1, "data1"}, {layout.data2, "data2"}, + {layout.data3, "data3"}, {layout.data4, "data4"}, {layout.data5, "data5"}, + {layout.data6, "data6"}}; + const char* owner = "(padding before data0)"; + for (const auto& [start, name] : members) { + if (offset >= start) owner = name; + } + return std::string(owner); + } + // Every double-typed uniform shape GLSL has, all thirteen of them, in one program - the // shape of KHR-GL43.compute_shader.fp64-case2. The scalar and the square matrices are // covered by the cases above; what only a set like this reaches is the NON-SQUARE @@ -819,5 +965,66 @@ void main() { o_color = vec4(0.0, 1.0, 0.0, 1.0); } EXPECT_EQ(FirstGLError(), 0u); } + TEST_F(DoublePrecisionScenario, AStorageBlockWithDoublesKeepsTheLayoutItWasBoundWith) { + if (!Ready()) return; + + GLint blocks = 0; + glGetIntegerv(GL_MAX_COMPUTE_SHADER_STORAGE_BLOCKS, &blocks); + if (blocks < 4) { + GTEST_SKIP() << "GL_MAX_COMPUTE_SHADER_STORAGE_BLOCKS is " << blocks << "; this needs 4"; + } + + const unsigned int program = CompileComputeProgram(kBlockCopySource); + ASSERT_NE(program, 0u) << m_buildLog; + + const std::vector in140 = MakeBlockContents(kStd140); + const std::vector in430 = MakeBlockContents(kStd430); + const std::vector zero140(in140.size(), 0); + const std::vector zero430(in430.size(), 0); + + GLuint buffers[4] = {}; + glGenBuffers(4, buffers); + const std::vector* contents[4] = {&in140, &in430, &zero140, &zero430}; + for (int i = 0; i < 4; ++i) { + glBindBufferBase(GL_SHADER_STORAGE_BUFFER, static_cast(i), buffers[i]); + glBufferData(GL_SHADER_STORAGE_BUFFER, static_cast(contents[i]->size()), + contents[i]->data(), GL_DYNAMIC_COPY); + } + ASSERT_EQ(FirstGLError(), 0u); + + glUseProgram(program); + glDispatchCompute(1, 1, 1); + glMemoryBarrier(GL_BUFFER_UPDATE_BARRIER_BIT); + EXPECT_EQ(FirstGLError(), 0u); + + for (int pass = 0; pass < 2; ++pass) { + const BlockLayout& layout = pass == 0 ? kStd140 : kStd430; + const std::vector& expected = pass == 0 ? in140 : in430; + const char* packing = pass == 0 ? "std140" : "std430"; + std::vector observed(expected.size(), 0xEE); + glBindBuffer(GL_SHADER_STORAGE_BUFFER, buffers[2 + pass]); + glGetBufferSubData(GL_SHADER_STORAGE_BUFFER, 0, + static_cast(observed.size()), observed.data()); + int mismatches = 0; + int firstMismatch = -1; + for (std::size_t i = 0; i < expected.size(); ++i) { + if (expected[i] == observed[i]) continue; + ++mismatches; + if (firstMismatch < 0) firstMismatch = static_cast(i); + } + EXPECT_EQ(mismatches, 0) + << packing << " block: " << mismatches << " of " << expected.size() + << " bytes differ, first at byte " << firstMismatch << " (in " + << DescribeOffset(layout, firstMismatch < 0 ? 0 : firstMismatch) + << "); a block that was repacked around its doubles reads and writes every " + "member after the first one at the wrong offset"; + } + + glUseProgram(0); + glDeleteProgram(program); + glDeleteBuffers(4, buffers); + EXPECT_EQ(FirstGLError(), 0u); + } + } // namespace } // namespace MGITest diff --git a/MobileGL/MG_IntegrationTest/Scenarios/Glsl420DeclarationScenario.cpp b/MobileGL/MG_IntegrationTest/Scenarios/Glsl420DeclarationScenario.cpp index a0f721d5..8489ceff 100644 --- a/MobileGL/MG_IntegrationTest/Scenarios/Glsl420DeclarationScenario.cpp +++ b/MobileGL/MG_IntegrationTest/Scenarios/Glsl420DeclarationScenario.cpp @@ -179,6 +179,13 @@ void main() in flat uint v_index; out vec4 o_color; void main() { o_color = vec4(0.0, 1.0, 0.0, 1.0); } +)"; + + // The colour index spelled out at its default value. Says nothing that + // `layout(location = 0)` alone does not, and must therefore cost nothing. + constexpr const char* kExplicitColorIndexFS = R"(#version 420 core +layout(location = 0, index = 0) out vec4 o_color; +void main() { o_color = vec4(0.0, 1.0, 0.0, 1.0); } )"; class Glsl420DeclarationScenario : public ScenarioTest { @@ -473,4 +480,24 @@ void main() { o_color = vec4(0.0, 1.0, 0.0, 1.0); } EXPECT_EQ(centre.g, 255) << "the atomic-counter shader linked but painted nothing"; } + // `layout(location = 0, index = 0)` is the GL default written out loud, and an application + // is entitled to write it - KHR-GL43.shader_atomic_counters.basic-program-query does. It has + // to reach the driver as an ORDINARY single-source output: GLSL ES has no `index` qualifier + // in core, so a transpiler that prints the decoration back gets "index layout qualifier + // requires EXT_blend_func_extended", the stage never compiles, the program runs with a stage + // missing and the draw paints nothing at all. Black, not red - which is why the conformance + // case looked like the atomic counters had stopped counting. + TEST_F(Glsl420DeclarationScenario, AnExplicitDefaultColorIndexStillDraws) { + if (!Ready()) return; + + const GLuint program = Build(kQuadVS, kExplicitColorIndexFS); + if (program == 0) return; + + const Rgba8 centre = DrawAndRead(program); + EXPECT_EQ(FirstGLError(), 0u); + EXPECT_EQ(centre.g, 255) << "a fragment output declared layout(location = 0, index = 0) painted " + "nothing; its stage was almost certainly refused by the driver"; + EXPECT_EQ(centre.r, 0u); + } + } // namespace MGITest diff --git a/MobileGL/MG_IntegrationTest/Scenarios/SsboArrayLengthScenario.cpp b/MobileGL/MG_IntegrationTest/Scenarios/SsboArrayLengthScenario.cpp index 9a1b6404..934891f0 100644 --- a/MobileGL/MG_IntegrationTest/Scenarios/SsboArrayLengthScenario.cpp +++ b/MobileGL/MG_IntegrationTest/Scenarios/SsboArrayLengthScenario.cpp @@ -64,6 +64,25 @@ void main() { g_length[2] = g_input23[0].data.length(); g_length[3] = g_input23[1].data.length(); } +)"; + + // GL 4.6 core 4.10 lets a buffer variable be declared readonly AND writeonly at once: + // it can then be neither read nor written, and `.length()` is the only thing left that + // may be asked of it. The pair is inert - and printing it into ESSL is not, because + // SPIRV-Cross hoists the qualifiers every member shares onto the BLOCK and Mesa's ES + // compiler refuses that spelling ("Interface block sets both readonly and writeonly"). + // Lifted from KHR-GL43.shader_storage_buffer_object.basic-readonly-writeonly. + constexpr const char* kReadonlyWriteonlyComputeSource = R"(#version 430 core +layout(local_size_x = 1) in; +layout(std430, binding = 0) buffer Input { + readonly writeonly int g_in[]; +}; +layout(std430, binding = 4) buffer Output { + int g_length[]; +}; +void main() { + g_length[0] = g_in.length(); +} )"; constexpr int kElementBytes = 16; // ivec4, std430 @@ -212,4 +231,33 @@ void main() { glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 0, input0); glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 3, input3); } + + // A buffer variable qualified readonly AND writeonly can only be asked its length, and that + // question still has to be answered. A stage the driver refused answers 0 - and refuses + // silently, because the program links without it and the dispatch is then a no-op. + TEST_F(SsboArrayLengthScenario, AReadonlyWriteonlyArrayStillReportsItsLength) { + if (!Ready() || IsSkipped()) return; + + const GLuint program = CompileComputeProgram(kReadonlyWriteonlyComputeSource); + ASSERT_NE(program, 0u) << m_buildLog; + + const GLuint input = MakeStorageBuffer(6); // 6 ivec4 = 24 ints + const GLuint output = MakeStorageBuffer(1); + glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 0, input); + glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 4, output); + ASSERT_EQ(FirstGLError(), 0u); + + glUseProgram(program); + glDispatchCompute(1, 1, 1); + glMemoryBarrier(GL_BUFFER_UPDATE_BARRIER_BIT); + int length = -1; + glBindBuffer(GL_SHADER_STORAGE_BUFFER, output); + glGetBufferSubData(GL_SHADER_STORAGE_BUFFER, 0, sizeof(length), &length); + EXPECT_EQ(FirstGLError(), 0u); + EXPECT_EQ(length, 24) << "a readonly+writeonly runtime array reported length " << length + << "; 0 means the stage never reached the program"; + + glUseProgram(m_program); + glDeleteProgram(program); + } } // namespace MGITest diff --git a/MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt b/MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt index 23e2c031..920b4608 100644 --- a/MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt +++ b/MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt @@ -8,6 +8,7 @@ add_executable( FixIterationRPSubgroupScratchTest.cpp EmulateSubgroupsTest.cpp DemoteFloat64Test.cpp + FlattenFloat64StorageBlockTest.cpp FlattenXfbInterfaceBlocksTest.cpp UniquifyIoBlockNamesTest.cpp LowerViewportIndexTest.cpp diff --git a/MobileGL/MG_Test/ShaderTranspiler/FlattenFloat64StorageBlockTest.cpp b/MobileGL/MG_Test/ShaderTranspiler/FlattenFloat64StorageBlockTest.cpp new file mode 100644 index 00000000..0a3f5574 --- /dev/null +++ b/MobileGL/MG_Test/ShaderTranspiler/FlattenFloat64StorageBlockTest.cpp @@ -0,0 +1,305 @@ +// MobileGL - MobileGL/MG_Test/ShaderTranspiler/FlattenFloat64StorageBlockTest.cpp +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header +// +// FlattenFloat64StorageBlockPass, over the module the production chain actually hands it: +// ShaderCompiler::SanitizeAndOptimizeBinary, where the pass sits immediately before the fp64 +// demotion. The behavioural half - that a block copied through the flattened words comes back +// byte for byte - is DoublePrecisionScenario's; what only a module walk can say is WHICH blocks +// were flattened, how wide, and that the ones this pass must not touch came through unchanged. + +#include + +#include +#include +#include +#include + +#include "Includes.h" +#include "Init.h" +#include + +#include + +using namespace MobileGL; +using MobileGL::MG_Util::ShaderTranspiler::ShaderCompiler; + +namespace { + // A test-side reference walker, deliberately independent of the production code: a bug in + // the pass must not be able to hide behind the same helper. + constexpr Uint32 kSpirvHeaderWordCount = 5; + constexpr Uint32 kOpName = 5; + constexpr Uint32 kOpDecorate = 71; + constexpr Uint32 kOpMemberDecorate = 72; + constexpr Uint32 kOpTypeInt = 21; + constexpr Uint32 kOpTypeFloat = 22; + constexpr Uint32 kOpTypeArray = 28; + constexpr Uint32 kOpTypeStruct = 30; + constexpr Uint32 kOpConstant = 43; + constexpr Uint32 kDecorationArrayStride = 6; + constexpr Uint32 kDecorationOffset = 35; + + template + void ForEachInstruction(const Vector& spirv, Visitor&& visit) { + for (SizeT i = kSpirvHeaderWordCount; i < spirv.size();) { + const Uint32 wordCount = spirv[i] >> 16; + const Uint32 opcode = spirv[i] & 0xFFFFu; + if (wordCount == 0 || i + wordCount > spirv.size()) break; + visit(opcode, &spirv[i], wordCount); + i += wordCount; + } + } + + Uint32 StructIdNamed(const Vector& spirv, const String& name) { + Uint32 structId = 0; + ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) { + if (opcode != kOpName || wordCount < 3 || structId != 0) return; + const char* text = reinterpret_cast(&words[2]); + const SizeT available = static_cast(wordCount - 2) * sizeof(Uint32); + // The whole name, not a prefix of it: "Wide" must not match "WideOther". + if (available <= name.size() || text[name.size()] != 0) return; + if (std::strncmp(text, name.c_str(), name.size()) == 0) structId = words[1]; + }); + return structId; + } + + // The operands of OpTypeStruct , i.e. one type id per member. + Vector MemberTypesOf(const Vector& spirv, Uint32 structId) { + Vector members; + ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) { + if (opcode != kOpTypeStruct || wordCount < 2 || words[1] != structId) return; + for (Uint32 i = 2; i < wordCount; ++i) members.push_back(words[i]); + }); + return members; + } + + Vector MemberOffsetsOf(const Vector& spirv, Uint32 structId) { + std::map byMember; + ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) { + if (opcode != kOpMemberDecorate || wordCount < 5 || words[1] != structId) return; + if (words[3] != kDecorationOffset) return; + byMember[words[2]] = words[4]; + }); + Vector offsets; + for (const auto& [member, offset] : byMember) offsets.push_back(offset); + return offsets; + } + + Uint32 DecorationValueOf(const Vector& spirv, Uint32 id, Uint32 decoration) { + Uint32 value = 0xFFFFFFFFu; + ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) { + if (opcode != kOpDecorate || wordCount < 4 || words[1] != id || words[2] != decoration) return; + value = words[3]; + }); + return value; + } + + // (element type id, declared length) of OpTypeArray , or (0, 0). + std::pair ArrayShapeOf(const Vector& spirv, Uint32 arrayId) { + Uint32 elementTypeId = 0; + Uint32 lengthConstantId = 0; + ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) { + if (opcode != kOpTypeArray || wordCount < 4 || words[1] != arrayId) return; + elementTypeId = words[2]; + lengthConstantId = words[3]; + }); + if (elementTypeId == 0) return {0, 0}; + Uint32 length = 0; + ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) { + if (opcode != kOpConstant || wordCount < 4 || words[2] != lengthConstantId) return; + length = words[3]; + }); + return {elementTypeId, length}; + } + + Bool IsUint32Type(const Vector& spirv, Uint32 typeId) { + Bool isUint = false; + ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) { + if (opcode != kOpTypeInt || wordCount < 4 || words[1] != typeId) return; + isUint = words[2] == 32u && words[3] == 0u; + }); + return isUint; + } + + Uint32 CountFloatTypesOfWidth(const Vector& spirv, Uint32 width) { + Uint32 count = 0; + ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) { + if (opcode == kOpTypeFloat && wordCount >= 3 && words[2] == width) ++count; + }); + return count; + } + + String Disassemble(const Vector& spirv) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_1); + String text; + tools.Disassemble(spirv, &text); + return text; + } + + Vector CompileToSpirv(GLenum stage, const String& source) { + using namespace MG_Util::ShaderTranspiler; + ShaderAttrib shaderAttrib{.shaderType = stage, .sourceStr = source}; + auto shaderResult = ShaderCompiler::CompileShader(shaderAttrib); + EXPECT_TRUE(shaderResult) << (shaderResult ? String{} : shaderResult.error().log); + if (!shaderResult) return {}; + + ProgramAttrib programAttrib{.shaders = {shaderResult.value()}}; + auto programResult = ShaderCompiler::LinkProgram(programAttrib); + EXPECT_TRUE(programResult) << (programResult ? String{} : programResult.error().log); + if (!programResult) return {}; + + ProgramBinaryAttrib binaryAttrib{.shaderTypes = {stage}, .program = *programResult.value()}; + auto binaryResult = ShaderCompiler::GetSpirvBinaryFromProgram(binaryAttrib); + EXPECT_TRUE(binaryResult) << (binaryResult ? String{} : binaryResult.error().log); + if (!binaryResult || binaryResult->empty()) return {}; + return binaryResult->front(); + } + + // The whole shared chain, exactly as the frontend runs it at link. + Vector Sanitize(const Vector& input) { + Vector output; + EXPECT_TRUE(ShaderCompiler::SanitizeAndOptimizeBinary(input, output, true, true)); + return output; + } + + // The block std140 lays out as data0@0, data1[3]@16 stride 16, data2@64 column stride 16, + // data3@112, data4[2]@128 stride 16, data5@160, data6@192 - 216 bytes, i.e. 54 words. + constexpr const char* kStd140BlockSource = R"(#version 430 core +layout(local_size_x = 1) in; +layout(std140, binding = 0) buffer Wide { + int data0; + float data1[3]; + mat3x2 data2; + double data3; + double data4[2]; + int data5; + dvec3 data6; +} g_wide; +void main() { + g_wide.data0 = 1; + for (int i = 0; i < 3; ++i) g_wide.data1[i] = float(i); + g_wide.data2 = mat3x2(1.0); + g_wide.data3 = 2.0lf; + for (int i = 0; i < 2; ++i) g_wide.data4[i] = double(i); + g_wide.data5 = 3; + g_wide.data6 = dvec3(4.0lf); +} +)"; +} // namespace + +class FlattenFloat64StorageBlockTest : public ::testing::Test { +protected: + void SetUp() override { + MobileGL::Initialize(); + m_validationFailuresAtStart = ShaderCompiler::SpirvValidationFailureCount(); + } + + void TearDown() override { + // The wrapper validates its output on every run, so this covers every rewrite the test + // performed without any of them having to say so. + EXPECT_EQ(ShaderCompiler::SpirvValidationFailureCount(), m_validationFailuresAtStart) + << "the flattened module did not survive spirv-val"; + } + + Uint64 m_validationFailuresAtStart = 0; +}; + +TEST_F(FlattenFloat64StorageBlockTest, AStorageBlockWithDoublesBecomesOneWordArray) { + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, kStd140BlockSource); + ASSERT_FALSE(input.empty()); + // Before: seven members, at the std140 offsets the standard requires WITH the doubles. + const Uint32 inputStructId = StructIdNamed(input, "Wide"); + ASSERT_NE(inputStructId, 0u) << Disassemble(input); + EXPECT_EQ(MemberOffsetsOf(input, inputStructId), + (Vector{0, 16, 64, 112, 128, 160, 192})) + << Disassemble(input); + + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const Uint32 structId = StructIdNamed(output, "Wide"); + ASSERT_NE(structId, 0u) << Disassemble(output); + const Vector members = MemberTypesOf(output, structId); + ASSERT_EQ(members.size(), 1u) << "the block should have collapsed to one member\n" + << Disassemble(output); + EXPECT_EQ(MemberOffsetsOf(output, structId), (Vector{0})); + + const auto [elementTypeId, length] = ArrayShapeOf(output, members[0]); + ASSERT_NE(elementTypeId, 0u) << "member 0 is not an array\n" << Disassemble(output); + EXPECT_TRUE(IsUint32Type(output, elementTypeId)) << Disassemble(output); + // 216 bytes is where the standard puts the end of this block; 216 / 4 = 54 words. + EXPECT_EQ(length, 54u) << Disassemble(output); + EXPECT_EQ(DecorationValueOf(output, members[0], kDecorationArrayStride), 4u); + + // And the demotion that runs straight afterwards still has nothing 64-bit left to find. + EXPECT_EQ(CountFloatTypesOfWidth(output, 64), 0u) << Disassemble(output); +} + +// The gate, from the other side: a storage block with no 64-bit member keeps every member and +// every offset it was compiled with. This is what makes the pass free for every shader that does +// not use doubles - which is all of them but a handful. +TEST_F(FlattenFloat64StorageBlockTest, AStorageBlockWithoutDoublesIsLeftAlone) { + const String source = R"(#version 430 core +layout(local_size_x = 1) in; +layout(std140, binding = 0) buffer Plain { + int data0; + float data1[3]; + mat3x2 data2; + int data3; +} g_plain; +void main() { + g_plain.data0 = 1; + for (int i = 0; i < 3; ++i) g_plain.data1[i] = float(i); + g_plain.data2 = mat3x2(1.0); + g_plain.data3 = 2; +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const Uint32 structId = StructIdNamed(output, "Plain"); + ASSERT_NE(structId, 0u) << Disassemble(output); + EXPECT_EQ(MemberTypesOf(output, structId).size(), 4u) << Disassemble(output); + EXPECT_EQ(MemberOffsetsOf(output, structId), (Vector{0, 16, 64, 112})) + << Disassemble(output); +} + +// A plain UNIFORM block is deliberately NOT flattened, however many doubles it holds: the +// frontend's glUniform*d routing is built by reflecting the DEMOTED module +// (ProgramSpirvTask::BuildGlobalUboRouting), so a representation change there would have to move +// with it. It keeps its members and takes the demotion's repacking, exactly as before. +TEST_F(FlattenFloat64StorageBlockTest, AUniformBlockWithDoublesIsLeftToTheDemotion) { + const String source = R"(#version 430 core +layout(local_size_x = 1) in; +layout(std140, binding = 0) uniform Params { + int data0; + double data1; + int data2; +} g_params; +layout(std430, binding = 0) buffer Sink { + float g_out[]; +}; +void main() { + g_out[0] = float(g_params.data0) + float(g_params.data1) + float(g_params.data2); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const Uint32 structId = StructIdNamed(output, "Params"); + ASSERT_NE(structId, 0u) << Disassemble(output); + EXPECT_EQ(MemberTypesOf(output, structId).size(), 3u) + << "a uniform block must not be flattened\n" + << Disassemble(output); + // The demotion's re-derived std140 layout for `int, float, int`, which is what the frontend + // reflects and what glUniform*d then writes into. + EXPECT_EQ(MemberOffsetsOf(output, structId), (Vector{0, 4, 8})) << Disassemble(output); +} diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp index 1367f9c2..9d3e0096 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp @@ -19,6 +19,7 @@ #include "SpirvPasses/DecomposeWorkgroupVec3Pass.h" #include "SpirvPasses/DecoratePositionInvariantPass.h" #include "SpirvPasses/DemoteFloat64Pass.h" +#include "SpirvPasses/FlattenFloat64StorageBlockPass.h" #include "SpirvPasses/LowerDrawParametersPass.h" #include "SpirvPasses/LowerViewportIndexPass.h" #include "SpirvPasses/PackDoubleVertexInputsPass.h" @@ -797,6 +798,17 @@ namespace MobileGL { // in particular it runs before the backends' PackDoubleVertexInputsPass, whose // OpBitcast this one would otherwise decline on. Costs one types_values() walk on // the overwhelming majority of modules, which declare no 64-bit float at all. + // ...but demoting a double that lives in a SHADER STORAGE BLOCK also repacks that + // block, and the bytes an application put in the buffer do not move with it. This + // runs first and takes those blocks out of the demotion's hands: each becomes a + // flat `uint` array whose index arithmetic carries the std140/std430 offsets + // glslang computed WITH the doubles in place, so the layout survives byte for byte + // and only the VALUES narrow. Gated on a block actually holding a 64-bit float, so + // every other module pays one types_values() walk and nothing else, and it declines + // (leaving the block for the demotion to handle the old way) on any shape it cannot + // re-address exactly. See FlattenFloat64StorageBlockPass.h. + optimizer.RegisterPass( + FlattenFloat64StorageBlockPass::CreateFlattenFloat64StorageBlockPass()); optimizer.RegisterPass(DemoteFloat64Pass::CreateDemoteFloat64Pass()); return RunOptimizerChecked("SanitizeAndOptimizeBinary", optimizer, inputBinary, diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DemoteFloat64Pass.h b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DemoteFloat64Pass.h index b095c60f..6cff9599 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DemoteFloat64Pass.h +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DemoteFloat64Pass.h @@ -50,12 +50,10 @@ namespace MobileGL { // for the same reason - writes exactly where the demoted shader reads. Blocks with no // 64-bit member anywhere are never touched. // - // THE MEASURED COST, so the next wave does not re-diagnose it. Four GL 4.3 conformance - // cases fail on BOTH backends and on both an Adreno 830 and a Mali G925 - i.e. on every - // device, because no device has shaderFloat64 and the demotion therefore always runs: + // WHAT RE-DERIVING STILL COSTS, so the next wave does not re-diagnose it. Two GL 4.3 + // conformance cases fail on BOTH backends and on every device, because no device has + // shaderFloat64 and the demotion therefore always runs: // - // KHR-GL43.shader_storage_buffer_object.basic-stdLayout-case3-cs - // KHR-GL43.shader_storage_buffer_object.basic-stdLayout-case3-vs // KHR-GL43.compute_shader.fp64-case1 // KHR-GL43.compute_shader.fp64-case3 // @@ -63,44 +61,28 @@ namespace MobileGL { // against it: it is blocked on GLSL subroutines ("FP64 support - subroutines"), which // glslang deletes when targeting SPIR-V, and is out of scope by standing instruction. // - // The other three fail in the two ways this comment predicts and in no other. - // stdLayout-case3 copies a block byte for byte: the output matches the input for - // bytes [0, 76) and is zero from there on, which is exactly the block's size once - // every double became a float and the layout repacked tightly. Re-derived byte-exactly - // in 2026-08: the block is `int data0; float data1[5]; mat3x2 data2; double data3; - // double data4[2]; int data5; dvec3 data6`, and demoting every double to float and - // repacking std430 gives data0@0, data1@4..23, data2@24..47, data3@48, data4@52..59, - // data5@60, data6@64..75 - 76 bytes. EVERY mismatching byte the QPA reports is >= 76 - // and every expected-non-zero byte below 76 matched, on both the std140 output and the - // std430 one. - // - // ONE TRAP FOR THE NEXT READER, because it reads as evidence AGAINST demotion and is - // not: in the std430 output the doubles below the boundary appear to have round-tripped - // BIT-EXACTLY, which looks like fp64 surviving. It is an artifact. The shader reads and - // writes through the SAME demoted offset, so those four bytes are copied verbatim - // whatever they are interpreted as - the copy proves nothing about the width. - // // fp64-case1 reports ceil(2.2) as 2: the uniform's double 2.0 is 0x4000000000000000, // the demoted read takes its low 32 bits (0.0), ceil(0.0 + 0.2) = 1.0f = 0x3F800000 // lands in the low half of the 8-byte output slot and the whole thing prints as 2. // Index 0 of the same case PASSES by accident, for the same reason - writing 0.0f into // the low half of 1.0 leaves it unchanged - so a partial pass here is not progress. + // Fixing it means carrying a double in the DEFAULT UNIFORM block without re-deriving + // its layout, and that block's routing is built by reflecting the module this pass + // produces, so the representation change ripples into every glUniform*d. Deliberately + // not attempted. compute_shader.fp64-case2 passes today and any attempt has to keep it + // green. // - // Both backends produce a CHARACTER-FOR-CHARACTER identical QPA byte list, which is - // the cheapest available proof that the defect is in this shared pass and in neither - // backend. A future wave that wants to re-open this should start by re-checking that - // identity rather than by re-deriving the layout. - // - // Fixing them means NOT demoting a double that lives in a buffer block, and carrying - // it as a uvec2 word pair instead - preserving the application's byte layout exactly, - // unpacking to fp32 for arithmetic and repacking on store. That is a large pass with - // the same dmat problem the paragraph above describes (a uvec2 representation cannot - // express a matrix stride either, so it would have to decline dmat types), and the - // default-uniform routing above reflects the demoted module, so a representation - // change there ripples into every glUniform*d. THREE actionable cases of 16085 (the - // fourth, fp64-case3, is subroutine-blocked and unreachable from here); deliberately - // not attempted, and re-confirmed as not worth attempting in the 2026-08 wave. - // compute_shader.fp64-case2 passes today and any attempt has to keep it green. + // SHADER STORAGE BLOCKS ARE NO LONGER IN THAT LIST, and the two cases that used to be + // (shader_storage_buffer_object.basic-stdLayout-case3-cs and -vs, which copy a block + // byte for byte and used to come back zero from the first double's slot onwards) pass + // on both backends. FlattenFloat64StorageBlockPass runs immediately before this one + // and takes every storage block holding a 64-bit float out of its hands, rewriting the + // block into a flat `uint` array whose index arithmetic carries the offsets glslang + // computed WITH the doubles in place. A flat array has no layout for SPIRV-Cross to + // re-derive, which is what makes it expressible where a padded struct is not, and an + // offset in an address computation has none of the dmat trouble the paragraph above + // describes. See that pass's header. Everything below still describes what happens to + // every OTHER block, and to the doubles in the function bodies of all of them. // // Declines (leaves the module byte-identical, so the caller's existing "this module // still declares Float64" failure path reports it) when the module contains an diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.cpp b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.cpp new file mode 100644 index 00000000..66a62037 --- /dev/null +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.cpp @@ -0,0 +1,1134 @@ +// MobileGL - MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.cpp +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#include "FlattenFloat64StorageBlockPass.h" + +#include "spirv.hpp" +#include "source/opt/basic_block.h" +#include "source/opt/constants.h" +#include "source/opt/decoration_manager.h" +#include "source/opt/def_use_manager.h" +#include "source/opt/function.h" +#include "source/opt/instruction.h" +#include "source/opt/ir_builder.h" +#include "source/opt/ir_context.h" +#include "source/opt/module.h" +#include "source/opt/type_manager.h" +#include "source/util/make_unique.h" + +#include +#include +#include +#include +#include +#include + +namespace MobileGL { + namespace MG_Util { + namespace ShaderTranspiler { + namespace { + using spvtools::MakeUnique; + using spvtools::opt::Instruction; + using spvtools::opt::InstructionBuilder; + using spvtools::opt::IRContext; + using spvtools::opt::Module; + using spvtools::opt::Operand; + + // The flattened array's element, and the granularity every offset and stride in + // the block has to land on. + constexpr uint32_t kWordBytes = 4u; + // A block wider than any GL implementation lets one binding cover is refused + // rather than expanded into an array nothing could address. GL 4.6 core table + // 23.64 puts the minimum GL_MAX_SHADER_STORAGE_BLOCK_SIZE at 2^24 bytes; this is + // a generous multiple of that and exists only to bound the rewrite. + constexpr uint64_t kMaxBlockBytes = 1ull << 27; + // The most scalars one load or store may decompose into. A whole-aggregate copy + // becomes one word access per scalar, so without a cap a `dvec4 data[4096]` + // member would turn a two-instruction copy into a 32k-instruction one. + constexpr uint32_t kMaxLeavesPerAccess = 1024u; + // The analyses every builder in this pass keeps current as it inserts. + constexpr IRContext::Analysis kPreservedAnalyses = static_cast( + static_cast(IRContext::kAnalysisDefUse) | + static_cast(IRContext::kAnalysisInstrToBlockMapping)); + + // How a type sits in memory, as the ENCLOSING struct member described it. + // MatrixStride and RowMajor are member decorations rather than type decorations, + // so a matrix type carries no layout of its own and the walk has to hand it down - + // through arrays of matrices too, which is why this rides alongside the type id + // instead of being looked up from it. + struct TypeCursor { + uint32_t typeId = 0; + uint32_t matrixStride = 0; + bool rowMajor = false; + }; + + // One access chain rooted at a flattened block's variable, and everything the + // rewrite needs so it does not have to walk the type tree a second time. + struct ChainPlan { + Instruction* chain = nullptr; + uint32_t variableId = 0; + // What the chain's CONSTANT indices contribute, in words. + uint32_t constantWords = 0; + // Its non-constant indices, as (index value id, words per step). + std::vector> dynamicTerms; + TypeCursor pointee; + std::vector loads; + std::vector stores; + }; + + struct BlockPlan { + Instruction* structType = nullptr; + uint32_t storageClass = 0; + uint32_t wordCount = 0; + std::vector chains; + }; + + bool IsDoubleType(const Instruction* type) { + return type != nullptr && type->opcode() == spv::Op::OpTypeFloat && + type->NumInOperands() >= 1 && type->GetSingleWordInOperand(0) == 64u; + } + + // Byte size of a scalar this pass can carry, or 0 for one it cannot. + uint32_t ScalarByteSize(const Instruction* type) { + if (type == nullptr) return 0; + if (type->opcode() != spv::Op::OpTypeFloat && type->opcode() != spv::Op::OpTypeInt) { + return 0; + } + const uint32_t width = type->GetSingleWordInOperand(0); + if (width == 32u) return 4u; + if (width == 64u && type->opcode() == spv::Op::OpTypeFloat) return 8u; + return 0; + } + + bool TryGetDecorationLiteral(IRContext* context, uint32_t id, spv::Decoration kind, + uint32_t* literal) { + for (Instruction* decoration : context->get_decoration_mgr()->GetDecorationsFor(id, false)) { + if (decoration->opcode() != spv::Op::OpDecorate || decoration->NumInOperands() < 3 || + static_cast(decoration->GetSingleWordInOperand(1)) != kind) { + continue; + } + *literal = decoration->GetSingleWordInOperand(2); + return true; + } + return false; + } + + bool TryGetMemberDecorationLiteral(IRContext* context, uint32_t structId, uint32_t member, + spv::Decoration kind, uint32_t* literal) { + for (Instruction* decoration : + context->get_decoration_mgr()->GetDecorationsFor(structId, false)) { + if (decoration->opcode() != spv::Op::OpMemberDecorate || + decoration->NumInOperands() < 4 || + decoration->GetSingleWordInOperand(1) != member || + static_cast(decoration->GetSingleWordInOperand(2)) != kind) { + continue; + } + *literal = decoration->GetSingleWordInOperand(3); + return true; + } + return false; + } + + bool HasMemberDecoration(IRContext* context, uint32_t structId, uint32_t member, + spv::Decoration kind) { + for (Instruction* decoration : + context->get_decoration_mgr()->GetDecorationsFor(structId, false)) { + if (decoration->opcode() != spv::Op::OpMemberDecorate || + decoration->NumInOperands() < 3 || + decoration->GetSingleWordInOperand(1) != member) { + continue; + } + if (static_cast(decoration->GetSingleWordInOperand(2)) == kind) { + return true; + } + } + return false; + } + + bool HasDecoration(IRContext* context, uint32_t id, spv::Decoration kind) { + for (Instruction* decoration : context->get_decoration_mgr()->GetDecorationsFor(id, false)) { + if (decoration->opcode() != spv::Op::OpDecorate || decoration->NumInOperands() < 2) { + continue; + } + if (static_cast(decoration->GetSingleWordInOperand(1)) == kind) { + return true; + } + } + return false; + } + + // The cursor for member `member` of a struct: its type, plus the matrix layout + // that member's own decorations describe. + TypeCursor MemberCursor(IRContext* context, const Instruction* structType, uint32_t member) { + TypeCursor cursor; + cursor.typeId = structType->GetSingleWordInOperand(member); + uint32_t stride = 0; + if (TryGetMemberDecorationLiteral(context, structType->result_id(), member, + spv::Decoration::MatrixStride, &stride)) { + cursor.matrixStride = stride; + } + cursor.rowMajor = HasMemberDecoration(context, structType->result_id(), member, + spv::Decoration::RowMajor); + return cursor; + } + + // Byte size of a type as it is laid out INSIDE a block, or 0 when this pass + // cannot describe it (a runtime array, a width it does not carry, a matrix with + // no stride or a row-major one). + uint32_t LaidOutByteSize(IRContext* context, const TypeCursor& cursor) { + const Instruction* type = context->get_def_use_mgr()->GetDef(cursor.typeId); + if (type == nullptr) return 0; + switch (type->opcode()) { + case spv::Op::OpTypeInt: + case spv::Op::OpTypeFloat: + return ScalarByteSize(type); + case spv::Op::OpTypeVector: { + const uint32_t component = ScalarByteSize( + context->get_def_use_mgr()->GetDef(type->GetSingleWordInOperand(0))); + if (component == 0) return 0; + return component * type->GetSingleWordInOperand(1); + } + case spv::Op::OpTypeMatrix: { + if (cursor.matrixStride == 0 || cursor.rowMajor) return 0; + return cursor.matrixStride * type->GetSingleWordInOperand(1); + } + case spv::Op::OpTypeArray: { + uint32_t stride = 0; + if (!TryGetDecorationLiteral(context, cursor.typeId, spv::Decoration::ArrayStride, + &stride) || + stride == 0) { + return 0; + } + const spvtools::opt::analysis::Constant* length = + context->get_constant_mgr()->FindDeclaredConstant(type->GetSingleWordInOperand(1)); + if (length == nullptr || length->AsIntConstant() == nullptr) return 0; + const uint64_t total = static_cast(stride) * + static_cast(length->AsIntConstant()->GetU32BitValue()); + return total > kMaxBlockBytes ? 0u : static_cast(total); + } + case spv::Op::OpTypeStruct: { + uint64_t end = 0; + for (uint32_t member = 0; member < type->NumInOperands(); ++member) { + uint32_t offset = 0; + if (!TryGetMemberDecorationLiteral(context, cursor.typeId, member, + spv::Decoration::Offset, &offset)) { + return 0; + } + const uint32_t size = LaidOutByteSize(context, MemberCursor(context, type, member)); + if (size == 0) return 0; + end = std::max(end, static_cast(offset) + size); + } + return end > kMaxBlockBytes ? 0u : static_cast(end); + } + default: + return 0; + } + } + + // Whether this type decomposes into scalars the rewrite can move one word at a + // time, counting them so a whole-aggregate access can be refused before it is + // expanded. + bool CanDecompose(IRContext* context, const TypeCursor& cursor, uint32_t* leafCount) { + const Instruction* type = context->get_def_use_mgr()->GetDef(cursor.typeId); + if (type == nullptr) return false; + switch (type->opcode()) { + case spv::Op::OpTypeInt: + case spv::Op::OpTypeFloat: + if (ScalarByteSize(type) == 0) return false; + ++*leafCount; + return *leafCount <= kMaxLeavesPerAccess; + case spv::Op::OpTypeVector: { + TypeCursor component; + component.typeId = type->GetSingleWordInOperand(0); + for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) { + if (!CanDecompose(context, component, leafCount)) return false; + } + return true; + } + case spv::Op::OpTypeMatrix: { + if (cursor.matrixStride == 0 || cursor.rowMajor || + cursor.matrixStride % kWordBytes != 0) { + return false; + } + TypeCursor column; + column.typeId = type->GetSingleWordInOperand(0); + for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) { + if (!CanDecompose(context, column, leafCount)) return false; + } + return true; + } + case spv::Op::OpTypeArray: { + uint32_t stride = 0; + if (!TryGetDecorationLiteral(context, cursor.typeId, spv::Decoration::ArrayStride, + &stride) || + stride == 0 || stride % kWordBytes != 0) { + return false; + } + const spvtools::opt::analysis::Constant* length = + context->get_constant_mgr()->FindDeclaredConstant(type->GetSingleWordInOperand(1)); + if (length == nullptr || length->AsIntConstant() == nullptr) return false; + const uint32_t count = length->AsIntConstant()->GetU32BitValue(); + if (count == 0 || count > kMaxLeavesPerAccess) return false; + TypeCursor element = cursor; + element.typeId = type->GetSingleWordInOperand(0); + for (uint32_t i = 0; i < count; ++i) { + if (!CanDecompose(context, element, leafCount)) return false; + } + return true; + } + case spv::Op::OpTypeStruct: { + for (uint32_t member = 0; member < type->NumInOperands(); ++member) { + uint32_t offset = 0; + if (!TryGetMemberDecorationLiteral(context, cursor.typeId, member, + spv::Decoration::Offset, &offset) || + offset % kWordBytes != 0) { + return false; + } + if (!CanDecompose(context, MemberCursor(context, type, member), leafCount)) { + return false; + } + } + return true; + } + default: + return false; + } + } + + bool TypeContainsFloat64(IRContext* context, uint32_t typeId, + std::unordered_set& visiting) { + const Instruction* type = context->get_def_use_mgr()->GetDef(typeId); + if (type == nullptr || !visiting.insert(typeId).second) return false; + switch (type->opcode()) { + case spv::Op::OpTypeFloat: + return type->GetSingleWordInOperand(0) == 64u; + case spv::Op::OpTypeVector: + case spv::Op::OpTypeMatrix: + case spv::Op::OpTypeArray: + case spv::Op::OpTypeRuntimeArray: + return TypeContainsFloat64(context, type->GetSingleWordInOperand(0), visiting); + case spv::Op::OpTypeStruct: + for (uint32_t member = 0; member < type->NumInOperands(); ++member) { + if (TypeContainsFloat64(context, type->GetSingleWordInOperand(member), visiting)) { + return true; + } + } + return false; + default: + return false; + } + } + + // The constant an index operand names, or false when it is not one. + bool TryGetConstantIndex(IRContext* context, uint32_t id, uint32_t* value) { + const spvtools::opt::analysis::Constant* constant = + context->get_constant_mgr()->FindDeclaredConstant(id); + if (constant == nullptr || constant->AsIntConstant() == nullptr) return false; + *value = constant->AsIntConstant()->GetU32BitValue(); + return true; + } + + // Walks one access chain against the block's type tree, recording the byte offset + // it names as a constant part plus a list of (index, stride) terms. False for any + // shape the rewrite cannot address exactly. + bool PlanChain(IRContext* context, Instruction* chain, const TypeCursor& blockCursor, + ChainPlan* plan) { + TypeCursor cursor = blockCursor; + uint64_t constantBytes = 0; + for (uint32_t operand = 1; operand < chain->NumInOperands(); ++operand) { + const uint32_t indexId = chain->GetSingleWordInOperand(operand); + const Instruction* type = context->get_def_use_mgr()->GetDef(cursor.typeId); + if (type == nullptr) return false; + + uint32_t stride = 0; + if (type->opcode() == spv::Op::OpTypeStruct) { + uint32_t member = 0; + if (!TryGetConstantIndex(context, indexId, &member) || + member >= type->NumInOperands()) { + return false; + } + uint32_t offset = 0; + if (!TryGetMemberDecorationLiteral(context, cursor.typeId, member, + spv::Decoration::Offset, &offset) || + offset % kWordBytes != 0) { + return false; + } + constantBytes += offset; + cursor = MemberCursor(context, type, member); + if (constantBytes > kMaxBlockBytes) return false; + continue; + } + + switch (type->opcode()) { + case spv::Op::OpTypeArray: + if (!TryGetDecorationLiteral(context, cursor.typeId, spv::Decoration::ArrayStride, + &stride)) { + return false; + } + cursor.typeId = type->GetSingleWordInOperand(0); + break; + case spv::Op::OpTypeMatrix: + if (cursor.rowMajor || cursor.matrixStride == 0) return false; + stride = cursor.matrixStride; + cursor.typeId = type->GetSingleWordInOperand(0); + cursor.matrixStride = 0; + break; + case spv::Op::OpTypeVector: + stride = ScalarByteSize( + context->get_def_use_mgr()->GetDef(type->GetSingleWordInOperand(0))); + cursor.typeId = type->GetSingleWordInOperand(0); + cursor.matrixStride = 0; + break; + default: + return false; + } + + if (stride == 0 || stride % kWordBytes != 0) return false; + uint32_t index = 0; + if (TryGetConstantIndex(context, indexId, &index)) { + constantBytes += static_cast(index) * stride; + if (constantBytes > kMaxBlockBytes) return false; + } else { + plan->dynamicTerms.emplace_back(indexId, stride / kWordBytes); + } + } + + if (constantBytes % kWordBytes != 0) return false; + plan->constantWords = static_cast(constantBytes / kWordBytes); + plan->pointee = cursor; + return true; + } + + // Everything the rewrite emits, over one module's shared scalar types. + class Emitter { + public: + // Where one block's words live: the variable holding them, the pointer type + // that reaches one, and the base index the chain resolved to. + struct Access { + uint32_t variableId = 0; + uint32_t wordPointerTypeId = 0; + uint32_t baseWordId = 0; + uint32_t memberZeroId = 0; + }; + + Emitter(IRContext* context, uint32_t uintTypeId, uint32_t boolTypeId, uint32_t floatTypeId) + : m_context(context), m_uintTypeId(uintTypeId), m_boolTypeId(boolTypeId), + m_floatTypeId(floatTypeId) {} + + uint32_t UintConstant(uint32_t value) { + const spvtools::opt::analysis::Type* type = + m_context->get_type_mgr()->GetType(m_uintTypeId); + const spvtools::opt::analysis::Constant* constant = + m_context->get_constant_mgr()->GetConstant(type, {value}); + return m_context->get_constant_mgr()->GetDefiningInstruction(constant)->result_id(); + } + + // The word index the chain names, materialised at the chain's own position so + // every load and store that uses it is dominated by it. + uint32_t WordIndexOf(const ChainPlan& plan) { + InstructionBuilder builder(m_context, plan.chain, kPreservedAnalyses); + uint32_t total = 0; + for (const auto& [indexId, wordsPerStep] : plan.dynamicTerms) { + uint32_t term = AsUint(builder, indexId); + if (wordsPerStep != 1u) { + term = Binary(builder, spv::Op::OpIMul, m_uintTypeId, term, + UintConstant(wordsPerStep)); + } + total = total == 0 ? term + : Binary(builder, spv::Op::OpIAdd, m_uintTypeId, total, term); + } + if (total == 0) return UintConstant(plan.constantWords); + if (plan.constantWords == 0) return total; + return Binary(builder, spv::Op::OpIAdd, m_uintTypeId, total, + UintConstant(plan.constantWords)); + } + + // Rebuilds the value an OpLoad of `cursor` would have produced, out of the + // words that live at `baseWordId + relWords`. + uint32_t BuildValue(InstructionBuilder& builder, const Access& access, + const TypeCursor& cursor, uint32_t relWords) { + const Instruction* type = m_context->get_def_use_mgr()->GetDef(cursor.typeId); + switch (type->opcode()) { + case spv::Op::OpTypeInt: + case spv::Op::OpTypeFloat: { + if (IsDoubleType(type)) { + const uint32_t lo = LoadWord(builder, access, relWords); + const uint32_t hi = LoadWord(builder, access, relWords + 1); + const uint32_t narrowed = builder + .AddUnaryOp(m_floatTypeId, spv::Op::OpBitcast, + NarrowDoubleBits(builder, lo, hi)) + ->result_id(); + return builder + .AddUnaryOp(cursor.typeId, spv::Op::OpFConvert, narrowed) + ->result_id(); + } + const uint32_t word = LoadWord(builder, access, relWords); + if (cursor.typeId == m_uintTypeId) return word; + return builder.AddUnaryOp(cursor.typeId, spv::Op::OpBitcast, word)->result_id(); + } + case spv::Op::OpTypeVector: { + TypeCursor component; + component.typeId = type->GetSingleWordInOperand(0); + const uint32_t step = ComponentWords(component.typeId); + std::vector parts; + for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) { + parts.push_back(BuildValue(builder, access, component, relWords + i * step)); + } + return builder.AddCompositeConstruct(cursor.typeId, parts)->result_id(); + } + case spv::Op::OpTypeMatrix: { + TypeCursor column; + column.typeId = type->GetSingleWordInOperand(0); + const uint32_t step = cursor.matrixStride / kWordBytes; + std::vector parts; + for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) { + parts.push_back(BuildValue(builder, access, column, relWords + i * step)); + } + return builder.AddCompositeConstruct(cursor.typeId, parts)->result_id(); + } + case spv::Op::OpTypeArray: { + TypeCursor element = cursor; + element.typeId = type->GetSingleWordInOperand(0); + uint32_t stride = 0; + TryGetDecorationLiteral(m_context, cursor.typeId, spv::Decoration::ArrayStride, + &stride); + const uint32_t count = m_context->get_constant_mgr() + ->FindDeclaredConstant(type->GetSingleWordInOperand(1)) + ->AsIntConstant() + ->GetU32BitValue(); + std::vector parts; + for (uint32_t i = 0; i < count; ++i) { + parts.push_back( + BuildValue(builder, access, element, relWords + i * (stride / kWordBytes))); + } + return builder.AddCompositeConstruct(cursor.typeId, parts)->result_id(); + } + case spv::Op::OpTypeStruct: { + std::vector parts; + for (uint32_t member = 0; member < type->NumInOperands(); ++member) { + uint32_t offset = 0; + TryGetMemberDecorationLiteral(m_context, cursor.typeId, member, + spv::Decoration::Offset, &offset); + parts.push_back(BuildValue(builder, access, MemberCursor(m_context, type, member), + relWords + offset / kWordBytes)); + } + return builder.AddCompositeConstruct(cursor.typeId, parts)->result_id(); + } + default: + return 0; + } + } + + // The mirror image: writes `valueId` into the words at + // `baseWordId + relWords`. `path` is the composite-extract index list that + // reaches the part being written, empty at the root. + void StoreValue(InstructionBuilder& builder, const Access& access, + const TypeCursor& cursor, uint32_t relWords, uint32_t rootValueId, + std::vector& path) { + const Instruction* type = m_context->get_def_use_mgr()->GetDef(cursor.typeId); + switch (type->opcode()) { + case spv::Op::OpTypeInt: + case spv::Op::OpTypeFloat: { + const uint32_t leaf = Extract(builder, cursor.typeId, rootValueId, path); + if (IsDoubleType(type)) { + const uint32_t narrowed = + builder.AddUnaryOp(m_floatTypeId, spv::Op::OpFConvert, leaf)->result_id(); + const uint32_t bits = + builder.AddUnaryOp(m_uintTypeId, spv::Op::OpBitcast, narrowed)->result_id(); + uint32_t lo = 0; + uint32_t hi = 0; + WidenFloatBits(builder, bits, &lo, &hi); + StoreWord(builder, access, relWords, lo); + StoreWord(builder, access, relWords + 1, hi); + return; + } + const uint32_t word = + cursor.typeId == m_uintTypeId + ? leaf + : builder.AddUnaryOp(m_uintTypeId, spv::Op::OpBitcast, leaf)->result_id(); + StoreWord(builder, access, relWords, word); + return; + } + case spv::Op::OpTypeVector: { + TypeCursor component; + component.typeId = type->GetSingleWordInOperand(0); + const uint32_t step = ComponentWords(component.typeId); + for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) { + path.push_back(i); + StoreValue(builder, access, component, relWords + i * step, rootValueId, path); + path.pop_back(); + } + return; + } + case spv::Op::OpTypeMatrix: { + TypeCursor column; + column.typeId = type->GetSingleWordInOperand(0); + const uint32_t step = cursor.matrixStride / kWordBytes; + for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) { + path.push_back(i); + StoreValue(builder, access, column, relWords + i * step, rootValueId, path); + path.pop_back(); + } + return; + } + case spv::Op::OpTypeArray: { + TypeCursor element = cursor; + element.typeId = type->GetSingleWordInOperand(0); + uint32_t stride = 0; + TryGetDecorationLiteral(m_context, cursor.typeId, spv::Decoration::ArrayStride, + &stride); + const uint32_t count = m_context->get_constant_mgr() + ->FindDeclaredConstant(type->GetSingleWordInOperand(1)) + ->AsIntConstant() + ->GetU32BitValue(); + for (uint32_t i = 0; i < count; ++i) { + path.push_back(i); + StoreValue(builder, access, element, relWords + i * (stride / kWordBytes), + rootValueId, path); + path.pop_back(); + } + return; + } + case spv::Op::OpTypeStruct: { + for (uint32_t member = 0; member < type->NumInOperands(); ++member) { + uint32_t offset = 0; + TryGetMemberDecorationLiteral(m_context, cursor.typeId, member, + spv::Decoration::Offset, &offset); + path.push_back(member); + StoreValue(builder, access, MemberCursor(m_context, type, member), + relWords + offset / kWordBytes, rootValueId, path); + path.pop_back(); + } + return; + } + default: + return; + } + } + + private: + uint32_t ComponentWords(uint32_t componentTypeId) { + return ScalarByteSize(m_context->get_def_use_mgr()->GetDef(componentTypeId)) / + kWordBytes; + } + + uint32_t Binary(InstructionBuilder& builder, spv::Op opcode, uint32_t typeId, uint32_t a, + uint32_t b) { + return builder.AddBinaryOp(typeId, opcode, a, b)->result_id(); + } + + uint32_t Select(InstructionBuilder& builder, uint32_t condition, uint32_t whenTrue, + uint32_t whenFalse) { + return builder + .AddTernaryOp(m_uintTypeId, spv::Op::OpSelect, condition, whenTrue, whenFalse) + ->result_id(); + } + + uint32_t AsUint(InstructionBuilder& builder, uint32_t valueId) { + const Instruction* def = m_context->get_def_use_mgr()->GetDef(valueId); + if (def != nullptr && def->type_id() == m_uintTypeId) return valueId; + return builder.AddUnaryOp(m_uintTypeId, spv::Op::OpBitcast, valueId)->result_id(); + } + + // `base + words`, folding away the add when there is nothing to add. + uint32_t Offset(InstructionBuilder& builder, uint32_t baseWordId, uint32_t words) { + if (words == 0) return baseWordId; + return Binary(builder, spv::Op::OpIAdd, m_uintTypeId, baseWordId, UintConstant(words)); + } + + uint32_t LoadWord(InstructionBuilder& builder, const Access& access, uint32_t words) { + const uint32_t indexId = Offset(builder, access.baseWordId, words); + Instruction* pointer = builder.AddAccessChain( + access.wordPointerTypeId, access.variableId, {access.memberZeroId, indexId}); + return builder.AddLoad(m_uintTypeId, pointer->result_id())->result_id(); + } + + void StoreWord(InstructionBuilder& builder, const Access& access, uint32_t words, + uint32_t valueId) { + const uint32_t indexId = Offset(builder, access.baseWordId, words); + Instruction* pointer = builder.AddAccessChain( + access.wordPointerTypeId, access.variableId, {access.memberZeroId, indexId}); + builder.AddStore(pointer->result_id(), valueId); + } + + // One scalar of the value being stored. An empty path IS the value. + uint32_t Extract(InstructionBuilder& builder, uint32_t typeId, uint32_t rootValueId, + const std::vector& path) { + if (path.empty()) return rootValueId; + return builder.AddCompositeExtract(typeId, rootValueId, path)->result_id(); + } + + // binary64 word pair -> the binary32 bit pattern nearest it, truncating the + // mantissa bits binary32 cannot hold. Straight-line by construction: every + // case is an OpSelect, so this needs no control flow and never splits a block. + uint32_t NarrowDoubleBits(InstructionBuilder& builder, uint32_t lo, uint32_t hi) { + const uint32_t sign = Binary(builder, spv::Op::OpBitwiseAnd, m_uintTypeId, hi, + UintConstant(0x80000000u)); + const uint32_t exponent = Binary( + builder, spv::Op::OpBitwiseAnd, m_uintTypeId, + Binary(builder, spv::Op::OpShiftRightLogical, m_uintTypeId, hi, UintConstant(20)), + UintConstant(0x7FFu)); + const uint32_t significandHigh = Binary(builder, spv::Op::OpBitwiseAnd, m_uintTypeId, + hi, UintConstant(0xFFFFFu)); + // The 23 bits binary32 keeps: 20 from the high word, 3 from the low one. + const uint32_t significand = Binary( + builder, spv::Op::OpBitwiseOr, m_uintTypeId, + Binary(builder, spv::Op::OpShiftLeftLogical, m_uintTypeId, significandHigh, + UintConstant(3)), + Binary(builder, spv::Op::OpShiftRightLogical, m_uintTypeId, lo, UintConstant(29))); + + // 1023 - 127 = 896, so the binary32 exponent field is `exponent - 896` and + // every bound on it is an unsigned comparison against that bias. + const uint32_t normalBits = Binary( + builder, spv::Op::OpBitwiseOr, m_uintTypeId, sign, + Binary(builder, spv::Op::OpBitwiseOr, m_uintTypeId, + Binary(builder, spv::Op::OpShiftLeftLogical, m_uintTypeId, + Binary(builder, spv::Op::OpISub, m_uintTypeId, exponent, + UintConstant(896)), + UintConstant(23)), + significand)); + const uint32_t infinityBits = Binary(builder, spv::Op::OpBitwiseOr, m_uintTypeId, sign, + UintConstant(0x7F800000u)); + // A quiet NaN that stays one even when every significant bit sat in the 29 + // low bits binary32 discards. + const uint32_t nanBits = + Binary(builder, spv::Op::OpBitwiseOr, m_uintTypeId, infinityBits, + Binary(builder, spv::Op::OpBitwiseOr, m_uintTypeId, + UintConstant(0x400000u), significand)); + + const uint32_t significandIsZero = + Binary(builder, spv::Op::OpIEqual, m_boolTypeId, + Binary(builder, spv::Op::OpBitwiseOr, m_uintTypeId, significandHigh, lo), + UintConstant(0)); + const uint32_t maxExponentBits = + Select(builder, significandIsZero, infinityBits, nanBits); + + const uint32_t isZeroExponent = + Binary(builder, spv::Op::OpIEqual, m_boolTypeId, exponent, UintConstant(0)); + const uint32_t isMaxExponent = + Binary(builder, spv::Op::OpIEqual, m_boolTypeId, exponent, UintConstant(0x7FFu)); + // <= 896 is every magnitude binary32 could hold only as a subnormal; + // >= 1151 is every one it cannot hold at all. + const uint32_t underflows = Binary(builder, spv::Op::OpULessThanEqual, m_boolTypeId, + exponent, UintConstant(896)); + const uint32_t overflows = Binary(builder, spv::Op::OpUGreaterThanEqual, m_boolTypeId, + exponent, UintConstant(1151)); + + uint32_t bits = Select(builder, overflows, infinityBits, normalBits); + bits = Select(builder, underflows, sign, bits); + bits = Select(builder, isMaxExponent, maxExponentBits, bits); + return Select(builder, isZeroExponent, sign, bits); + } + + // The inverse: a binary32 bit pattern -> the binary64 word pair for it. + void WidenFloatBits(InstructionBuilder& builder, uint32_t bits, uint32_t* lo, + uint32_t* hi) { + const uint32_t sign = Binary(builder, spv::Op::OpBitwiseAnd, m_uintTypeId, bits, + UintConstant(0x80000000u)); + const uint32_t exponent = Binary( + builder, spv::Op::OpBitwiseAnd, m_uintTypeId, + Binary(builder, spv::Op::OpShiftRightLogical, m_uintTypeId, bits, UintConstant(23)), + UintConstant(0xFFu)); + const uint32_t significand = Binary(builder, spv::Op::OpBitwiseAnd, m_uintTypeId, bits, + UintConstant(0x7FFFFFu)); + const uint32_t significandHigh = Binary(builder, spv::Op::OpShiftRightLogical, + m_uintTypeId, significand, UintConstant(3)); + const uint32_t significandLow = Binary(builder, spv::Op::OpShiftLeftLogical, + m_uintTypeId, significand, UintConstant(29)); + + const uint32_t normalHigh = Binary( + builder, spv::Op::OpBitwiseOr, m_uintTypeId, sign, + Binary(builder, spv::Op::OpBitwiseOr, m_uintTypeId, + Binary(builder, spv::Op::OpShiftLeftLogical, m_uintTypeId, + Binary(builder, spv::Op::OpIAdd, m_uintTypeId, exponent, + UintConstant(896)), + UintConstant(20)), + significandHigh)); + const uint32_t maxHigh = Binary(builder, spv::Op::OpBitwiseOr, m_uintTypeId, sign, + Binary(builder, spv::Op::OpBitwiseOr, m_uintTypeId, + UintConstant(0x7FF00000u), significandHigh)); + + const uint32_t isZeroExponent = + Binary(builder, spv::Op::OpIEqual, m_boolTypeId, exponent, UintConstant(0)); + const uint32_t isMaxExponent = + Binary(builder, spv::Op::OpIEqual, m_boolTypeId, exponent, UintConstant(0xFFu)); + + // A binary32 subnormal is below every binary64 this can name without a + // normalising loop, so it becomes the signed zero it is nearest to. + uint32_t high = Select(builder, isMaxExponent, maxHigh, normalHigh); + *hi = Select(builder, isZeroExponent, sign, high); + *lo = Select(builder, isZeroExponent, UintConstant(0), significandLow); + } + + IRContext* m_context; + uint32_t m_uintTypeId; + uint32_t m_boolTypeId; + uint32_t m_floatTypeId; + }; + // --- the module-level rewrite ------------------------------------------------ + + Module::inst_iterator PositionOf(IRContext* context, const Instruction* target) { + for (auto it = context->types_values_begin(); it != context->types_values_end(); ++it) { + if (&*it == target) return it; + } + return context->types_values_end(); + } + + // Whether |firstId| is declared before |secondId| in the types/constants section. + bool DeclaredBefore(IRContext* context, uint32_t firstId, uint32_t secondId) { + for (const Instruction& inst : context->module()->types_values()) { + if (inst.result_id() == firstId) return true; + if (inst.result_id() == secondId) return false; + } + return false; + } + + // A fresh `uint[length]` with ArrayStride 4, spliced in immediately BEFORE the + // block that will name it - SPIR-V has no forward references between types, so + // appending it at the end of the section would make the module invalid. A + // duplicate OpTypeArray is legal (SPIR-V 2.8 exempts aggregates from the + // uniqueness rule, and so does spirv-val), so no search for an existing one is + // needed; the LENGTH CONSTANT is not exempt, and if the module already declares + // it after the block there is nowhere legal to put the array - the block is then + // declined and keeps today's behaviour. Returns 0 for that, and for a uint type + // that is itself declared too late. + uint32_t CreateWordArrayTypeBefore(IRContext* context, Instruction* structType, + uint32_t uintTypeId, uint32_t length) { + if (!DeclaredBefore(context, uintTypeId, structType->result_id())) return 0; + + auto* constantMgr = context->get_constant_mgr(); + const spvtools::opt::analysis::Type* uintType = context->get_type_mgr()->GetType(uintTypeId); + if (uintType == nullptr) return 0; + const spvtools::opt::analysis::Constant* lengthConstant = + constantMgr->GetConstant(uintType, {length}); + if (lengthConstant == nullptr) return 0; + + Module::inst_iterator position = PositionOf(context, structType); + if (position == context->types_values_end()) return 0; + Instruction* lengthInst = constantMgr->GetDefiningInstruction(lengthConstant, 0, &position); + if (lengthInst == nullptr) return 0; + if (!DeclaredBefore(context, lengthInst->result_id(), structType->result_id())) return 0; + + const uint32_t arrayTypeId = context->TakeNextId(); + if (arrayTypeId == 0) return 0; + auto arrayType = MakeUnique( + context, spv::Op::OpTypeArray, 0, arrayTypeId, + std::initializer_list{{SPV_OPERAND_TYPE_ID, {uintTypeId}}, + {SPV_OPERAND_TYPE_ID, {lengthInst->result_id()}}}); + Instruction* inserted = structType->InsertBefore(std::move(arrayType)); + context->AnalyzeDefUse(inserted); + context->get_decoration_mgr()->AddDecorationVal( + arrayTypeId, static_cast(spv::Decoration::ArrayStride), kWordBytes); + return arrayTypeId; + } + + // The access qualifiers the collapsed member has to keep. Coherent and Volatile + // are taken from ANY member that had them - dropping one could lose a write + // another invocation has to see - while Restrict, NonWritable and NonReadable are + // taken only from ALL of them, because each is a promise the shader would break + // if one member never made it. + std::vector SurvivingAccessQualifiers(IRContext* context, uint32_t structId, + uint32_t memberCount) { + static constexpr spv::Decoration kAny[] = {spv::Decoration::Coherent, + spv::Decoration::Volatile}; + static constexpr spv::Decoration kAll[] = {spv::Decoration::Restrict, + spv::Decoration::NonWritable, + spv::Decoration::NonReadable}; + std::vector surviving; + for (const spv::Decoration kind : kAny) { + for (uint32_t member = 0; member < memberCount; ++member) { + if (HasMemberDecoration(context, structId, member, kind)) { + surviving.push_back(kind); + break; + } + } + } + for (const spv::Decoration kind : kAll) { + bool all = memberCount > 0; + for (uint32_t member = 0; member < memberCount && all; ++member) { + all = HasMemberDecoration(context, structId, member, kind); + } + if (all) surviving.push_back(kind); + } + return surviving; + } + + // Drops every OpMemberDecorate and OpMemberName the collapsed struct no longer has + // a member for - which is all of them, since the one member it keeps is a + // different thing entirely from the one that used to be member 0. + void StripMemberAnnotations(IRContext* context, uint32_t structId) { + std::vector doomed; + for (Instruction* decoration : + context->get_decoration_mgr()->GetDecorationsFor(structId, false)) { + if (decoration->opcode() == spv::Op::OpMemberDecorate) doomed.push_back(decoration); + } + for (Instruction& debug : context->module()->debugs2()) { + if (debug.opcode() != spv::Op::OpMemberName || debug.NumInOperands() < 2) continue; + if (debug.GetSingleWordInOperand(0) != structId) continue; + doomed.push_back(&debug); + } + for (Instruction* inst : doomed) context->KillInst(inst); + } + + // Plans every shader storage block in the module that holds a 64-bit float and + // that this pass can rewrite exactly. Reads the module and never writes it. + std::vector BuildPlans(IRContext* context) { + std::vector plans; + auto* defUseMgr = context->get_def_use_mgr(); + + // Declaration order, so a module with two candidate blocks is rewritten the + // same way every time it is compiled. + std::vector structOrder; + std::unordered_map> variablesByStruct; + std::unordered_map storageClassByStruct; + for (Instruction& inst : context->module()->types_values()) { + if (inst.opcode() != spv::Op::OpVariable || inst.NumInOperands() < 1) continue; + const uint32_t storageClass = inst.GetSingleWordInOperand(0); + const bool isStorageBufferClass = + storageClass == static_cast(spv::StorageClass::StorageBuffer); + const bool isUniformClass = + storageClass == static_cast(spv::StorageClass::Uniform); + if (!isStorageBufferClass && !isUniformClass) continue; + + Instruction* pointerType = defUseMgr->GetDef(inst.type_id()); + if (pointerType == nullptr || pointerType->opcode() != spv::Op::OpTypePointer) continue; + const uint32_t structId = pointerType->GetSingleWordInOperand(1); + Instruction* structType = defUseMgr->GetDef(structId); + if (structType == nullptr || structType->opcode() != spv::Op::OpTypeStruct || + structType->NumInOperands() == 0) { + continue; + } + // A shader storage block is spelled Block + StorageBuffer from SPIR-V 1.3 + // and BufferBlock + Uniform before it; a plain UNIFORM block is neither, + // and is deliberately left to the demotion - the frontend's own uniform + // routing reflects the module that pass produces. + const bool isStorageBlock = + (isStorageBufferClass && HasDecoration(context, structId, spv::Decoration::Block)) || + (isUniformClass && HasDecoration(context, structId, spv::Decoration::BufferBlock)); + if (!isStorageBlock) continue; + + std::unordered_set visiting; + if (!TypeContainsFloat64(context, structId, visiting)) continue; + + if (variablesByStruct.find(structId) == variablesByStruct.end()) { + structOrder.push_back(structId); + storageClassByStruct[structId] = storageClass; + } + variablesByStruct[structId].push_back(&inst); + } + + // A struct type is not owned by the variables that happen to be storage blocks: + // if ANY other variable points at the same one, collapsing it would rewrite a + // declaration this pass never looked at. Count every variable of every struct + // and require the two counts to agree. + std::unordered_map variableCountByStruct; + for (Instruction& inst : context->module()->types_values()) { + if (inst.opcode() != spv::Op::OpVariable) continue; + Instruction* pointerType = defUseMgr->GetDef(inst.type_id()); + if (pointerType == nullptr || pointerType->opcode() != spv::Op::OpTypePointer) continue; + ++variableCountByStruct[pointerType->GetSingleWordInOperand(1)]; + } + for (auto it = structOrder.begin(); it != structOrder.end();) { + if (variableCountByStruct[*it] == variablesByStruct[*it].size()) { + ++it; + continue; + } + MGLOG_D("[spirv] storage block %%%u holds a double but its struct type is shared " + "with a declaration that is not one; left to the fp64 demotion", + *it); + it = structOrder.erase(it); + } + + for (const uint32_t structId : structOrder) { + Instruction* structType = defUseMgr->GetDef(structId); + TypeCursor blockCursor; + blockCursor.typeId = structId; + const uint32_t blockBytes = LaidOutByteSize(context, blockCursor); + if (blockBytes == 0 || blockBytes % kWordBytes != 0) { + MGLOG_D("[spirv] storage block %%%u holds a double but its byte layout cannot be " + "described exactly; left to the fp64 demotion", + structId); + continue; + } + + BlockPlan plan; + plan.structType = structType; + plan.storageClass = storageClassByStruct[structId]; + plan.wordCount = blockBytes / kWordBytes; + + bool expressible = true; + for (Instruction* variable : variablesByStruct[structId]) { + std::vector chains; + std::unordered_set seenChains; + defUseMgr->ForEachUser(variable, [&](Instruction* user) { + if (!expressible) return; + switch (user->opcode()) { + case spv::Op::OpName: + case spv::Op::OpDecorate: + case spv::Op::OpDecorateId: + case spv::Op::OpEntryPoint: + return; + case spv::Op::OpAccessChain: + case spv::Op::OpInBoundsAccessChain: + if (user->NumInOperands() >= 1 && + user->GetSingleWordInOperand(0) == variable->result_id()) { + if (seenChains.insert(user->result_id()).second) chains.push_back(user); + return; + } + expressible = false; + return; + default: + expressible = false; + return; + } + }); + if (!expressible) break; + + for (Instruction* chain : chains) { + ChainPlan chainPlan; + chainPlan.chain = chain; + chainPlan.variableId = variable->result_id(); + if (!PlanChain(context, chain, blockCursor, &chainPlan)) { + expressible = false; + break; + } + uint32_t leafCount = 0; + if (!CanDecompose(context, chainPlan.pointee, &leafCount)) { + expressible = false; + break; + } + std::unordered_set seenUses; + defUseMgr->ForEachUser(chain, [&](Instruction* user) { + if (!expressible) return; + if (user->opcode() == spv::Op::OpLoad && + user->GetSingleWordInOperand(0) == chain->result_id()) { + if (seenUses.insert(user->unique_id()).second) { + chainPlan.loads.push_back(user); + } + return; + } + if (user->opcode() == spv::Op::OpStore && user->NumInOperands() >= 2 && + user->GetSingleWordInOperand(0) == chain->result_id() && + user->GetSingleWordInOperand(1) != chain->result_id()) { + if (seenUses.insert(user->unique_id()).second) { + chainPlan.stores.push_back(user); + } + return; + } + expressible = false; + }); + if (!expressible) break; + plan.chains.push_back(std::move(chainPlan)); + } + if (!expressible) break; + } + if (!expressible) { + MGLOG_D("[spirv] storage block %%%u holds a double but is reached in a way this " + "pass cannot re-address; left to the fp64 demotion", + structId); + continue; + } + plans.push_back(std::move(plan)); + } + return plans; + } + } // namespace + + spvtools::opt::Pass::Status FlattenFloat64StorageBlockPass::Process() { + auto* irContext = context(); + std::vector plans = BuildPlans(irContext); + if (plans.empty()) { + return Status::SuccessWithoutChange; + } + + spvtools::opt::analysis::Integer uintDescriptor(32, false); + spvtools::opt::analysis::Bool boolDescriptor; + spvtools::opt::analysis::Float floatDescriptor(32); + const uint32_t uintTypeId = irContext->get_type_mgr()->GetTypeInstruction(&uintDescriptor); + const uint32_t boolTypeId = irContext->get_type_mgr()->GetTypeInstruction(&boolDescriptor); + const uint32_t floatTypeId = irContext->get_type_mgr()->GetTypeInstruction(&floatDescriptor); + if (uintTypeId == 0 || boolTypeId == 0 || floatTypeId == 0) { + return Status::SuccessWithoutChange; + } + + Emitter emitter(irContext, uintTypeId, boolTypeId, floatTypeId); + bool modified = false; + for (BlockPlan& plan : plans) { + const uint32_t structId = plan.structType->result_id(); + const uint32_t arrayTypeId = + CreateWordArrayTypeBefore(irContext, plan.structType, uintTypeId, plan.wordCount); + if (arrayTypeId == 0) { + MGLOG_D("[spirv] storage block %%%u: no legal place for the flattened word array; " + "left to the fp64 demotion", + structId); + continue; + } + const uint32_t wordPointerTypeId = irContext->get_type_mgr()->FindPointerToType( + uintTypeId, static_cast(plan.storageClass)); + if (wordPointerTypeId == 0) continue; + const uint32_t memberZeroId = emitter.UintConstant(0); + + for (ChainPlan& chainPlan : plan.chains) { + Emitter::Access access; + access.variableId = chainPlan.variableId; + access.wordPointerTypeId = wordPointerTypeId; + access.memberZeroId = memberZeroId; + access.baseWordId = emitter.WordIndexOf(chainPlan); + + for (Instruction* load : chainPlan.loads) { + InstructionBuilder builder(irContext, load, kPreservedAnalyses); + const uint32_t rebuilt = emitter.BuildValue(builder, access, chainPlan.pointee, 0); + irContext->ReplaceAllUsesWith(load->result_id(), rebuilt); + irContext->KillInst(load); + } + for (Instruction* store : chainPlan.stores) { + InstructionBuilder builder(irContext, store, kPreservedAnalyses); + std::vector path; + emitter.StoreValue(builder, access, chainPlan.pointee, 0, + store->GetSingleWordInOperand(1), path); + irContext->KillInst(store); + } + irContext->KillInst(chainPlan.chain); + } + + const std::vector surviving = SurvivingAccessQualifiers( + irContext, structId, plan.structType->NumInOperands()); + StripMemberAnnotations(irContext, structId); + plan.structType->SetInOperands({{SPV_OPERAND_TYPE_ID, {arrayTypeId}}}); + irContext->UpdateDefUse(plan.structType); + irContext->get_decoration_mgr()->AddMemberDecoration( + structId, 0u, static_cast(spv::Decoration::Offset), 0u); + for (const spv::Decoration kind : surviving) { + // AddMemberDecoration always carries a literal value; these have none, so + // the instruction has to be spelled out. + irContext->get_decoration_mgr()->AddDecoration( + spv::Op::OpMemberDecorate, + {{SPV_OPERAND_TYPE_ID, {structId}}, + {SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}, + {SPV_OPERAND_TYPE_DECORATION, {static_cast(kind)}}}); + } + modified = true; + MGLOG_D("[spirv] storage block %%%u: flattened into %u words so its 64-bit members keep " + "the byte layout the application bound", + structId, plan.wordCount); + } + + if (!modified) { + return Status::SuccessWithoutChange; + } + irContext->InvalidateAnalysesExceptFor(IRContext::kAnalysisNone); + return Status::SuccessWithChange; + } + + spvtools::Optimizer::PassToken + FlattenFloat64StorageBlockPass::CreateFlattenFloat64StorageBlockPass() { + return spvtools::Optimizer::PassToken(MakeUnique()); + } + } // namespace ShaderTranspiler + } // namespace MG_Util +} // namespace MobileGL diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.h b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.h new file mode 100644 index 00000000..0962e719 --- /dev/null +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.h @@ -0,0 +1,91 @@ +// MobileGL - MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.h +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#pragma once +#include "source/opt/pass.h" +#include "spirv-tools/optimizer.hpp" + +#include + +namespace MobileGL { + namespace MG_Util { + namespace ShaderTranspiler { + // Rewrites a SHADER STORAGE BLOCK that contains a 64-bit float into a flat + // `uint` word array, and turns every access to it into address arithmetic over + // that array. The application's byte layout survives exactly; the VALUES are + // still narrowed to 32-bit floats, because that is all any target here has. + // + // WHY THIS EXISTS. DemoteFloat64Pass rewrites `double` to `float` in place and + // lets SPIRV-Cross re-derive the block's packing from the declared types, because + // GLSL ES has no member `layout(offset=)` and SPIRV-Cross refuses any block whose + // stated offsets it cannot express as std140 or std430. That re-derivation moves + // every member past the first double: the block a shader reads and writes stops + // being the block the application filled. Byte-for-byte, on the shape + // KHR-GL43.shader_storage_buffer_object.basic-stdLayout-case3 uses, the output + // matched the input up to the first double's slot and was zero from there on - + // the demoted block is simply shorter than the one that was bound. + // + // A flat `uint[]` has no layout to re-derive: one member, offset 0, ArrayStride 4, + // which IS std430, so SPIRV-Cross prints it unconditionally and the driver lays it + // out the only way it can. Every member's real byte offset - the std140 or std430 + // one glslang computed WITH the doubles in place - then lives in the index + // arithmetic this pass emits, not in the declaration. The two ways the earlier + // attempt at this was blocked both disappear with it: + // + // * dmat: a `uvec2`-per-double representation cannot express a MatrixStride, so + // it would have had to decline matrices of doubles. Here a stride is a number + // in an address computation and nothing else, so dmat needs no special case. + // * the default-uniform block: its routing is built by reflecting the DEMOTED + // module (ProgramSpirvTask::BuildGlobalUboRouting), so changing how a double + // is carried there would ripple into every glUniform*d. This pass touches + // StorageBuffer blocks only and never that one. + // + // WHAT GL SEES IS UNCHANGED, and becomes CORRECT rather than merely unchanged: + // glGetProgramResourceiv answers from glslang's reflection of the pre-demotion + // module (ProgramInterface.cpp reads TObjectReflection::offset), i.e. the true + // fp64 offsets. Before this pass those offsets described a layout no shader used; + // now they describe the one it does. + // + // PRECISION, stated plainly. A double still becomes a float: the load narrows the + // stored binary64 to binary32 and the store widens it back, so a value that does + // not survive a round trip through 32 bits does not survive this either. The + // narrowing truncates the discarded mantissa bits rather than rounding to nearest, + // and flushes what binary32 can only hold as a subnormal to a signed zero; NaN + // stays NaN and an out-of-range magnitude becomes an infinity. That is the same + // fp32 promise DemoteFloat64Pass already makes - what changes is only that the + // BYTES around the value stay where the application put them. + // + // DECLINES, leaving the block exactly as it was for DemoteFloat64Pass to handle the + // old way, whenever it meets something it cannot rewrite exactly: + // - a block whose variable is used as anything but an access-chain base (loaded + // whole, handed to a function, asked its OpArrayLength); + // - an access chain that is not rooted at the variable, or whose result feeds + // anything but a plain OpLoad / OpStore (an atomic, OpCopyMemory, a further + // chain); + // - a non-constant index into a struct, a runtime array anywhere in the block, a + // RowMajor matrix (its columns are not contiguous, so a whole-column access is + // not one range), a member width other than 32 or 64 bits, or an offset or + // stride that is not a multiple of 4; + // - a load or store whose type decomposes into more scalars than the cap below, + // so legalizing a block can never explode the module. + // + // ORDERING: must run BEFORE DemoteFloat64Pass, which is what turns the doubles this + // pass leaves in the function body into floats - the OpFConvert pairs emitted here + // are width-preserving by then and collapse to their operands. It emits only 32-bit + // OpBitcasts, so it never trips that pass's "bitcast across the 64-bit boundary" + // decline. + class FlattenFloat64StorageBlockPass final : public spvtools::opt::Pass { + public: + const char* name() const override { return "mobilegl-flatten-float64-storage-block"; } + Status Process() override; + + static spvtools::Optimizer::PassToken CreateFlattenFloat64StorageBlockPass(); + }; + } // namespace ShaderTranspiler + } // namespace MG_Util +} // namespace MobileGL diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpvcSession.cpp b/MobileGL/MG_Util/ShaderTranspiler/SpvcSession.cpp index 29b32225..095149c9 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/SpvcSession.cpp +++ b/MobileGL/MG_Util/ShaderTranspiler/SpvcSession.cpp @@ -442,6 +442,60 @@ namespace MobileGL { SPVC_CHK_RETURN } + spvc_result SpvcSession::DropDefaultFragmentOutputColorIndex() { + if (!(usage & SessionUsageBit::Transpile)) return SPVC_ERROR_INVALID_ARGUMENT; + + SPVC_CHK_INIT + const spvc_reflected_resource* list = nullptr; + size_t count = 0; + SPVC_CHK_RESULT(spvc_resources_get_resource_list_for_type( + resources, SPVC_RESOURCE_TYPE_STAGE_OUTPUT, &list, &count)); + for (size_t i = 0; i < count; ++i) { + const spvc_reflected_resource& resource = list[i]; + if (!spvc_compiler_has_decoration(compiler, resource.id, SpvDecorationIndex)) continue; + if (spvc_compiler_get_decoration(compiler, resource.id, SpvDecorationIndex) != 0u) continue; + spvc_compiler_unset_decoration(compiler, resource.id, SpvDecorationIndex); + } + SPVC_CHK_RETURN + } + + spvc_result SpvcSession::RelaxReadWriteExclusiveStorageBuffers() { + if (!(usage & SessionUsageBit::Transpile)) return SPVC_ERROR_INVALID_ARGUMENT; + + SPVC_CHK_INIT + const spvc_reflected_resource* list = nullptr; + size_t count = 0; + SPVC_CHK_RESULT(spvc_resources_get_resource_list_for_type( + resources, SPVC_RESOURCE_TYPE_STORAGE_BUFFER, &list, &count)); + for (size_t i = 0; i < count; ++i) { + const spvc_reflected_resource& resource = list[i]; + // The variable itself, for a block the application qualified as a whole. + if (spvc_compiler_has_decoration(compiler, resource.id, SpvDecorationNonReadable) && + spvc_compiler_has_decoration(compiler, resource.id, SpvDecorationNonWritable)) { + spvc_compiler_unset_decoration(compiler, resource.id, SpvDecorationNonReadable); + spvc_compiler_unset_decoration(compiler, resource.id, SpvDecorationNonWritable); + } + // ...and each member, which is where the qualifiers usually sit and where + // SPIRV-Cross reads them from before hoisting the ones every member shares. + const spvc_type blockType = spvc_compiler_get_type_handle(compiler, resource.base_type_id); + if (blockType == nullptr) continue; + const unsigned memberCount = spvc_type_get_num_member_types(blockType); + for (unsigned member = 0; member < memberCount; ++member) { + if (!spvc_compiler_has_member_decoration(compiler, resource.base_type_id, member, + SpvDecorationNonReadable) || + !spvc_compiler_has_member_decoration(compiler, resource.base_type_id, member, + SpvDecorationNonWritable)) { + continue; + } + spvc_compiler_unset_member_decoration(compiler, resource.base_type_id, member, + SpvDecorationNonReadable); + spvc_compiler_unset_member_decoration(compiler, resource.base_type_id, member, + SpvDecorationNonWritable); + } + } + SPVC_CHK_RETURN + } + spvc_result SpvcSession::Compile(const char** result) { if (!(usage & SessionUsageBit::Transpile)) return SPVC_ERROR_INVALID_ARGUMENT; SPVC_CHK_INIT diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpvcSession.h b/MobileGL/MG_Util/ShaderTranspiler/SpvcSession.h index f602514a..c1d7114a 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/SpvcSession.h +++ b/MobileGL/MG_Util/ShaderTranspiler/SpvcSession.h @@ -120,6 +120,46 @@ namespace MobileGL { // `outGlBindings` is appended to, so one vector can collect a whole program's // stages; it may repeat a binding declared by several of them. spvc_result SetAtomicCounterBlockBindings(Int topBinding, Vector& outGlBindings); + // Drops the Index decoration from every fragment output that carries the DEFAULT + // colour index 0, so the emitted ESSL does not print `index = 0`. + // + // Index 0 is what every single-source fragment output already is, in GL and in + // ESSL alike, and SPIR-V carries the decoration only because the application + // spelled the qualifier out - `layout(location = 0, index = 0) out vec4 c;` is + // legal desktop GLSL and says nothing. Printing it back into ESSL is NOT + // harmless: GLSL ES has no `index` layout qualifier in core, so the driver + // answers "index layout qualifier requires EXT_blend_func_extended" and refuses + // the stage. The program then links nothing and every draw with it renders + // NOTHING - verified on Mesa 26.1.4 llvmpipe with no MobileGL in the process, + // and it is why KHR-GL43.shader_atomic_counters.basic-program-query read back a + // black render target. + // + // A NON-zero index is left exactly as it is: that one really does select the + // second dual-source input and cannot be expressed without the extension, so it + // must keep reaching the driver (the frontend's own glBindFragDataLocationIndexed + // path already emits only non-zero indices for the same reason). + spvc_result DropDefaultFragmentOutputColorIndex(); + // Drops `readonly` and `writeonly` from every shader storage block - and every + // block member - that carries BOTH of them. + // + // GL 4.6 core 4.10 lets a buffer variable be declared readonly AND writeonly at + // once: it then cannot be read or written at all, and the only thing left that + // it can be used for is `.length()`. The pair is therefore inert by + // construction - the frontend has already rejected any access to it - so + // dropping it cannot change what the shader does. + // + // Emitting it does change whether the shader EXISTS. SPIRV-Cross hoists the + // qualifiers every member shares onto the block, and Mesa's ES compiler rejects + // that spelling outright ("Interface block sets both readonly and writeonly", + // verified on Mesa 26.1.4 llvmpipe with no MobileGL in the process, against the + // exact source this transpiler emitted). The stage then never compiles, the + // program links without it, and every dispatch or draw is a silent no-op - + // which is how KHR-GL43.shader_storage_buffer_object.basic-readonly-writeonly + // read back 0 instead of the array length. + // + // A block carrying only ONE of the two is left exactly as it is: those really do + // constrain the accesses the shader makes, and the driver is entitled to know. + spvc_result RelaxReadWriteExclusiveStorageBuffers(); spvc_result Compile(const char** result); const SpvcMetadata& GetMetadata() const; const char* GetLastErrorString() const;