diff --git a/CMakeLists.txt b/CMakeLists.txt index 17b53424..1df503d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -301,6 +301,7 @@ set(SOURCE_FILES MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateNoPerspectivePass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/LegalizeFragmentOutputIndexPass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/LegalizeStorageBlockArrayIndexPass.cpp + MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenAtomicCounterBlockPass.cpp MobileGL/MG_Util/BackendLoaders/OpenGL/Loader.cpp MobileGL/MG_Util/BackendLoaders/Vulkan/Loader.cpp diff --git a/MobileGL/MG_Backend/DirectGLES/Managers.cpp b/MobileGL/MG_Backend/DirectGLES/Managers.cpp index 76d7d494..e66ce27d 100644 --- a/MobileGL/MG_Backend/DirectGLES/Managers.cpp +++ b/MobileGL/MG_Backend/DirectGLES/Managers.cpp @@ -5458,6 +5458,22 @@ namespace MobileGL::MG_Backend::DirectGLES { effectiveSpirv = &blockArrayIndexSpirv; } + // glslang kept the application's layout(offset = N) on the atomic counters it + // lowered onto gl_AtomicCounterBlock_, and no std140/std430 layout can put + // member 0 anywhere but offset 0 - so SPIRV-Cross throws ("cannot be expressed as + // neither std430 nor std140") and the stage never reaches the driver. Collapse the + // block into one uint array at offset 0 and re-index each counter to the element + // that used to be at its byte offset; the buffer then stays bound whole, which it + // has to (GL_SHADER_STORAGE_BUFFER_OFFSET_ALIGNMENT is 32 on this device, so an + // 8-byte bind offset is not expressible). BEFORE SetAtomicCounterBlockBindings + // below, which only moves the block's BINDING and needs the block intact. + Vector atomicCounterSpirv; + if (MG_Util::ShaderTranspiler::ShaderCompiler::FlattenAtomicCounterBlockOffsetsForEssl( + *effectiveSpirv, atomicCounterSpirv, enableSpirvValidation) && + !atomicCounterSpirv.empty()) { + effectiveSpirv = &atomicCounterSpirv; + } + MG_Util::ShaderTranspiler::SpvcSession spvcSession(*effectiveSpirv, MG_Util::ShaderTranspiler::SessionUsageBit::Transpile); diff --git a/MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt b/MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt index 335fdabb..3fed6320 100644 --- a/MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt +++ b/MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt @@ -13,6 +13,7 @@ add_executable( LowerViewportIndexTest.cpp ClampMultisampleFetchTest.cpp LegalizeStorageBlockArrayIndexTest.cpp + FlattenAtomicCounterBlockTest.cpp ) target_include_directories(SpirvPassTest PRIVATE diff --git a/MobileGL/MG_Test/ShaderTranspiler/FlattenAtomicCounterBlockTest.cpp b/MobileGL/MG_Test/ShaderTranspiler/FlattenAtomicCounterBlockTest.cpp new file mode 100644 index 00000000..05daba97 --- /dev/null +++ b/MobileGL/MG_Test/ShaderTranspiler/FlattenAtomicCounterBlockTest.cpp @@ -0,0 +1,214 @@ +// MobileGL - MobileGL/MG_Test/ShaderTranspiler/FlattenAtomicCounterBlockTest.cpp +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#include + +#define SPV_ENABLE_UTILITY_CODE +#include "glslang/SPIRV/spirv.hpp11" +#undef SPV_ENABLE_UTILITY_CODE + +#include "Includes.h" +#include +#include + +#include + +#include +#include +#include +#include + +using namespace MobileGL; +using MobileGL::MG_Util::ShaderTranspiler::ShaderCompiler; + +namespace { + constexpr SizeT kSpirvHeaderWordCount = 5u; + + template + void ForEachInstruction(const Vector& spirv, Visitor&& visit) { + for (SizeT offset = kSpirvHeaderWordCount; offset < spirv.size();) { + const Uint32 wordCount = spirv[offset] >> 16u; + if (wordCount == 0u || offset + wordCount > spirv.size()) break; + visit(static_cast(spirv[offset] & 0xffffu), &spirv[offset], wordCount); + offset += wordCount; + } + } + + Vector CompileCompute(const String& source) { + using namespace MobileGL::MG_Util::ShaderTranspiler; + ShaderAttrib shaderAttrib{.shaderType = GL_COMPUTE_SHADER, .sourceStr = source}; + auto shaderResult = ShaderCompiler::CompileShader(shaderAttrib); + EXPECT_TRUE(shaderResult) << (shaderResult ? String{} : shaderResult.error().log); + if (!shaderResult) return {}; + + ProgramAttrib programAttrib{.shaders = {shaderResult.value()}}; + auto programResult = ShaderCompiler::LinkProgram(programAttrib); + EXPECT_TRUE(programResult) << (programResult ? String{} : programResult.error().log); + if (!programResult) return {}; + + ProgramBinaryAttrib binaryAttrib{.shaderTypes = {GL_COMPUTE_SHADER}, .program = *programResult.value()}; + auto binaryResult = ShaderCompiler::GetSpirvBinaryFromProgram(binaryAttrib); + EXPECT_TRUE(binaryResult) << (binaryResult ? String{} : binaryResult.error().log); + if (!binaryResult || binaryResult->empty()) return {}; + return binaryResult->front(); + } + + bool Validates(const Vector& spirv) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_1); + tools.SetMessageConsumer( + [](spv_message_level_t, const char*, const spv_position_t& position, const char* message) { + ADD_FAILURE() << "spirv-val at word " << position.index << ": " << message; + }); + return tools.Validate(spirv); + } + + // Test-side reference walker, deliberately independent of the production code. + Uint32 FindAtomicCounterBlockStructId(const Vector& spirv) { + const String prefix = MG_Util::ShaderTranspiler::ATOMIC_COUNTER_BLOCK_PREFIX; + Uint32 structId = 0; + ForEachInstruction(spirv, [&](spv::Op opcode, const Uint32* words, Uint32 wordCount) { + if (opcode != spv::Op::OpName || wordCount < 3u || structId != 0u) return; + const char* text = reinterpret_cast(&words[2]); + const SizeT available = static_cast(wordCount - 2u) * sizeof(Uint32); + if (available < prefix.size()) return; + if (std::strncmp(text, prefix.c_str(), prefix.size()) != 0) return; + structId = words[1]; + }); + return structId; + } + + // The Offset of member `member` on struct `structId`, or -1. + Int64 MemberOffsetOf(const Vector& spirv, Uint32 structId, Uint32 member) { + Int64 offset = -1; + ForEachInstruction(spirv, [&](spv::Op opcode, const Uint32* words, Uint32 wordCount) { + if (opcode != spv::Op::OpMemberDecorate || wordCount < 5u) return; + if (words[1] != structId || words[2] != member) return; + if (static_cast(words[3]) != spv::Decoration::Offset) return; + offset = words[4]; + }); + return offset; + } + + Uint32 MemberCountOf(const Vector& spirv, Uint32 structId) { + Uint32 count = 0; + ForEachInstruction(spirv, [&](spv::Op opcode, const Uint32* words, Uint32 wordCount) { + if (opcode != spv::Op::OpTypeStruct || wordCount < 2u || words[1] != structId) return; + count = wordCount - 2u; + }); + return count; + } + + Uint32 MemberTypeOf(const Vector& spirv, Uint32 structId, Uint32 member) { + Uint32 typeId = 0; + ForEachInstruction(spirv, [&](spv::Op opcode, const Uint32* words, Uint32 wordCount) { + if (opcode != spv::Op::OpTypeStruct || wordCount < 3u + member || words[1] != structId) return; + typeId = words[2 + member]; + }); + return typeId; + } + + // The declared length of an OpTypeArray, resolved through the uint constants in the module. + Int64 ArrayLengthOf(const Vector& spirv, Uint32 arrayTypeId) { + std::map constants; + Int64 length = -1; + ForEachInstruction(spirv, [&](spv::Op opcode, const Uint32* words, Uint32 wordCount) { + if (opcode == spv::Op::OpConstant && wordCount >= 4u) constants[words[2]] = words[3]; + if (opcode == spv::Op::OpTypeArray && wordCount >= 4u && words[1] == arrayTypeId) { + const auto it = constants.find(words[3]); + if (it != constants.end()) length = it->second; + } + }); + return length; + } + + // KHR-GL43.compute_shader.resources-atomic-counter's non-zero-offset shape: two counters + // declared eight bytes into the buffer, which glslang lowers to one block member at Offset 8. + constexpr const char* kOffsetCounters = R"(#version 450 core +layout(local_size_x = 1) in; +layout(binding = 1, offset = 8) uniform atomic_uint g_counter[2]; +layout(std430, binding = 0) buffer Output { uint value[]; } g_out; +void main() { + g_out.value[0] = atomicCounterIncrement(g_counter[0]); + g_out.value[1] = atomicCounterIncrement(g_counter[1]); +} +)"; + + // The latch: offset 0 is what nearly every shader declares, and it transpiles today. + constexpr const char* kNaturalCounters = R"(#version 450 core +layout(local_size_x = 1) in; +layout(binding = 1, offset = 0) uniform atomic_uint g_counter[2]; +layout(std430, binding = 0) buffer Output { uint value[]; } g_out; +void main() { + g_out.value[0] = atomicCounterIncrement(g_counter[0]); + g_out.value[1] = atomicCounterIncrement(g_counter[1]); +} +)"; + + constexpr const char* kNoCounters = R"(#version 450 core +layout(local_size_x = 1) in; +layout(std430, binding = 0) buffer Output { uint value[]; } g_out; +void main() { + g_out.value[0] = 1u; +} +)"; +} // namespace + +TEST(FlattenAtomicCounterBlockPass, MovesTheBlockToOffsetZeroAndGrowsTheArray) { + const Vector input = CompileCompute(kOffsetCounters); + ASSERT_FALSE(input.empty()); + const Uint32 structId = FindAtomicCounterBlockStructId(input); + ASSERT_NE(structId, 0u) << "glslang did not lower the counters onto a gl_AtomicCounterBlock_*"; + ASSERT_EQ(MemberOffsetOf(input, structId, 0u), 8) << "the input's member 0 is not at the declared offset"; + + Vector output; + ASSERT_TRUE(ShaderCompiler::FlattenAtomicCounterBlockOffsetsForEssl(input, output, true)); + ASSERT_FALSE(output.empty()); + + const Uint32 outStructId = FindAtomicCounterBlockStructId(output); + ASSERT_EQ(outStructId, structId) << "the block's id must not move; SetAtomicCounterBlockBindings " + "still finds it by name"; + EXPECT_EQ(MemberCountOf(output, outStructId), 1u); + EXPECT_EQ(MemberOffsetOf(output, outStructId, 0u), 0) + << "member 0 must sit at offset 0 or no std140/std430 layout can express the block"; + // Two counters eight bytes in: the flattened array has to cover bytes [0, 16), i.e. 4 uints, + // so counter k lands on element 2 + k and therefore on byte 8 + 4k - where it was declared. + EXPECT_EQ(ArrayLengthOf(output, MemberTypeOf(output, outStructId, 0u)), 4); + EXPECT_TRUE(Validates(output)); +} + +TEST(FlattenAtomicCounterBlockPass, LeavesANaturallyPackedBlockByteIdentical) { + const Vector input = CompileCompute(kNaturalCounters); + ASSERT_FALSE(input.empty()); + ASSERT_NE(FindAtomicCounterBlockStructId(input), 0u); + + Vector output; + ASSERT_TRUE(ShaderCompiler::FlattenAtomicCounterBlockOffsetsForEssl(input, output, true)); + EXPECT_EQ(output, input); +} + +TEST(FlattenAtomicCounterBlockPass, LeavesAShaderWithoutCountersByteIdentical) { + const Vector input = CompileCompute(kNoCounters); + ASSERT_FALSE(input.empty()); + + Vector output; + ASSERT_TRUE(ShaderCompiler::FlattenAtomicCounterBlockOffsetsForEssl(input, output, true)); + EXPECT_EQ(output, input); +} + +TEST(FlattenAtomicCounterBlockPass, IsIdempotent) { + const Vector input = CompileCompute(kOffsetCounters); + ASSERT_FALSE(input.empty()); + + Vector once; + ASSERT_TRUE(ShaderCompiler::FlattenAtomicCounterBlockOffsetsForEssl(input, once, true)); + ASSERT_FALSE(once.empty()); + + Vector twice; + ASSERT_TRUE(ShaderCompiler::FlattenAtomicCounterBlockOffsetsForEssl(once, twice, true)); + EXPECT_EQ(twice, once); +} diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp index 54a1743b..497f2365 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp @@ -42,6 +42,7 @@ #include "SpirvPasses/EmulateNoPerspectivePass.h" #include "SpirvPasses/LegalizeFragmentOutputIndexPass.h" #include "SpirvPasses/LegalizeStorageBlockArrayIndexPass.h" +#include "SpirvPasses/FlattenAtomicCounterBlockPass.h" #include "spirv-tools/libspirv.h" #include "spirv-tools/optimizer.hpp" #include "source/opt/build_module.h" @@ -1008,6 +1009,26 @@ namespace MobileGL { return true; } + bool ShaderCompiler::FlattenAtomicCounterBlockOffsetsForEssl( + const Vector& inputBinary, Vector& outputBinary, + const bool enableSpirvValidation) { + using namespace spvtools; + + // Detection gates everything: a module with no atomic counter, or one whose + // counters sit at their natural std430 offsets - which is every shader that omits + // the offset qualifier - pays one BuildModule and is handed back byte for byte. + if (!FlattenAtomicCounterBlockPass::BinaryHasOffsetAtomicCounterBlock(inputBinary)) { + outputBinary = inputBinary; + return true; + } + + Optimizer optimizer(SPV_ENV_VULKAN_1_1); + optimizer.RegisterPass(FlattenAtomicCounterBlockPass::CreateFlattenAtomicCounterBlockPass()); + + return RunOptimizerChecked("FlattenAtomicCounterBlockOffsetsForEssl", optimizer, inputBinary, + outputBinary, true, enableSpirvValidation); + } + bool ShaderCompiler::LowerRectImages(const Vector& inputBinary, Vector& outputBinary, const bool enableSpirvValidation) { diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h index c55c8275..f9d49220 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h @@ -170,6 +170,18 @@ namespace MobileGL { static bool LegalizeStorageBlockArrayIndexingForEssl(const Vector& inputBinary, Vector& outputBinary, bool enableSpirvValidation = false); + // Collapses each synthesized gl_AtomicCounterBlock_ into one uint array at + // offset 0, re-indexing every counter access to the element that used to sit at + // its byte offset. glslang preserves the application's layout(offset = N) as the + // member's Offset decoration, no std140/std430 layout can express a first member + // at a non-zero offset, and GLSL ES has no member layout(offset=) - so SPIRV-Cross + // throws and takes the whole stage with it. DirectGLES transpile path only. + // Copies the input through untouched when every counter block is already packed + // naturally, which is every shader that omits the offset qualifier. See + // FlattenAtomicCounterBlockPass. + static bool FlattenAtomicCounterBlockOffsetsForEssl(const Vector& inputBinary, + Vector& outputBinary, + bool enableSpirvValidation = false); // Rebases loads of the InstanceIndex builtin to (InstanceIndex - BaseInstance) so // shaders see GL's zero-based gl_InstanceID. Vertex shaders only; DirectVulkan // backend only (glslang's relaxed mode aliases gl_InstanceID to gl_InstanceIndex, diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenAtomicCounterBlockPass.cpp b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenAtomicCounterBlockPass.cpp new file mode 100644 index 00000000..359adb23 --- /dev/null +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenAtomicCounterBlockPass.cpp @@ -0,0 +1,492 @@ +// MobileGL - MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenAtomicCounterBlockPass.cpp +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#include "FlattenAtomicCounterBlockPass.h" + +#include "spirv.hpp" +#include "source/opt/basic_block.h" +#include "source/opt/build_module.h" +#include "source/opt/constants.h" +#include "source/opt/decoration_manager.h" +#include "source/opt/def_use_manager.h" +#include "source/opt/function.h" +#include "source/opt/instruction.h" +#include "source/opt/ir_builder.h" +#include "source/opt/ir_context.h" +#include "source/opt/module.h" +#include "source/opt/type_manager.h" +#include "source/util/make_unique.h" + +#include +#include +#include +#include +#include +#include + +namespace MobileGL { + namespace MG_Util { + namespace ShaderTranspiler { + namespace { + using spvtools::MakeUnique; + using spvtools::opt::BasicBlock; + using spvtools::opt::Function; + using spvtools::opt::Instruction; + using spvtools::opt::InstructionBuilder; + using spvtools::opt::IRContext; + using spvtools::opt::Module; + using spvtools::opt::Operand; + + // Kept in step with MG_Util/ShaderTranspiler/Types.h's + // MAX_ATOMIC_COUNTER_BUFFER_SIZE (16384 bytes), expressed in uint elements. A + // block whose declared byte window is wider than GL will ever let an application + // bind is refused rather than expanded into a huge array. + constexpr uint32_t kMaxCounterElements = 16384u / 4u; + // The lowered block's name always starts with this; the spelling lives in + // Types.h as ATOMIC_COUNTER_BLOCK_PREFIX, which is what the rest of MobileGL + // matches on. Repeated rather than included because that header pulls the whole + // backend-parameter surface into a pass that needs one string. + constexpr const char* kAtomicCounterBlockPrefix = "gl_AtomicCounterBlock"; + // The stride the flattened array is laid out with, and the size of one counter. + constexpr uint32_t kCounterBytes = 4; + + struct MemberPlan { + // Where this member starts, in uint elements from the block's byte 0. + uint32_t elementOffset = 0; + // How many uints it occupies: 1 for a scalar counter, N for `atomic_uint c[N]`. + uint32_t elementCount = 1; + bool isArray = false; + }; + + struct BlockPlan { + Instruction* structType = nullptr; + uint32_t uintTypeId = 0; + std::vector members; + // Access chains rooted at a variable of this block, in the order found. + std::vector chains; + uint32_t totalElements = 0; + }; + + bool NameStartsWithAtomicCounterBlockPrefix(IRContext* context, uint32_t id) { + for (const Instruction& debug : context->module()->debugs2()) { + if (debug.opcode() != spv::Op::OpName || debug.NumInOperands() < 2) continue; + if (debug.GetSingleWordInOperand(0) != id) continue; + const std::string name = debug.GetInOperand(1).AsString(); + return name.compare(0, std::strlen(kAtomicCounterBlockPrefix), + kAtomicCounterBlockPrefix) == 0; + } + return false; + } + + // The literal of the first OpMemberDecorate , or none. + bool TryGetMemberDecorationLiteral(IRContext* context, uint32_t structId, uint32_t member, + spv::Decoration kind, uint32_t* literal) { + for (Instruction* decoration : + context->get_decoration_mgr()->GetDecorationsFor(structId, false)) { + if (decoration->opcode() != spv::Op::OpMemberDecorate || + decoration->NumInOperands() < 4 || + decoration->GetSingleWordInOperand(1) != member || + static_cast(decoration->GetSingleWordInOperand(2)) != kind) { + continue; + } + *literal = decoration->GetSingleWordInOperand(3); + return true; + } + return false; + } + + bool TryGetDecorationLiteral(IRContext* context, uint32_t id, spv::Decoration kind, + uint32_t* literal) { + for (Instruction* decoration : context->get_decoration_mgr()->GetDecorationsFor(id, false)) { + if (decoration->opcode() != spv::Op::OpDecorate || decoration->NumInOperands() < 3 || + static_cast(decoration->GetSingleWordInOperand(1)) != kind) { + continue; + } + *literal = decoration->GetSingleWordInOperand(2); + return true; + } + return false; + } + + bool IsUint32Type(const Instruction* type) { + return type != nullptr && type->opcode() == spv::Op::OpTypeInt && + type->NumInOperands() >= 2 && type->GetSingleWordInOperand(0) == 32u && + type->GetSingleWordInOperand(1) == 0u; + } + + // The member's shape as this pass needs it, or false when it is one the pass + // cannot re-index. + bool DescribeMember(IRContext* context, uint32_t memberTypeId, uint32_t* uintTypeId, + MemberPlan* plan) { + auto* defUseMgr = context->get_def_use_mgr(); + Instruction* memberType = defUseMgr->GetDef(memberTypeId); + if (memberType == nullptr) return false; + + if (IsUint32Type(memberType)) { + plan->isArray = false; + plan->elementCount = 1; + *uintTypeId = memberTypeId; + return true; + } + if (memberType->opcode() != spv::Op::OpTypeArray || memberType->NumInOperands() < 2) { + return false; + } + const uint32_t elementTypeId = memberType->GetSingleWordInOperand(0); + if (!IsUint32Type(defUseMgr->GetDef(elementTypeId))) return false; + // The array's stride must be the tight 4 for the flattening to keep every + // counter on the byte it was declared at. + uint32_t stride = 0; + if (!TryGetDecorationLiteral(context, memberTypeId, spv::Decoration::ArrayStride, &stride) || + stride != kCounterBytes) { + return false; + } + const spvtools::opt::analysis::Constant* length = + context->get_constant_mgr()->FindDeclaredConstant(memberType->GetSingleWordInOperand(1)); + if (length == nullptr || length->AsIntConstant() == nullptr) return false; + const uint32_t count = length->AsIntConstant()->GetU32BitValue(); + if (count == 0u) return false; + plan->isArray = true; + plan->elementCount = count; + *uintTypeId = elementTypeId; + return true; + } + + // Whether the members' offsets already ARE the natural std430 packing, i.e. + // whether the block transpiles as it stands and this pass must leave it alone. + bool IsNaturallyPacked(const std::vector& members) { + uint32_t natural = 0; + for (const MemberPlan& member : members) { + if (member.elementOffset != natural) return false; + natural += member.elementCount; + } + return true; + } + + // Plans every atomic-counter block the module declares that is NOT already + // naturally packed and that this pass can re-index exactly. Reads the module; + // never rewrites it, so the same walk serves both the detection probe and phase 1 + // of the rewrite. + std::vector BuildPlans(IRContext* context) { + std::vector plans; + auto* defUseMgr = context->get_def_use_mgr(); + + std::unordered_map candidateStructs; + for (Instruction& inst : context->module()->types_values()) { + if (inst.opcode() != spv::Op::OpTypeStruct || inst.NumInOperands() == 0) continue; + if (!NameStartsWithAtomicCounterBlockPrefix(context, inst.result_id())) continue; + candidateStructs.emplace(inst.result_id(), &inst); + } + if (candidateStructs.empty()) return plans; + + std::unordered_map variableToStruct; + for (Instruction& inst : context->module()->types_values()) { + if (inst.opcode() != spv::Op::OpVariable) continue; + Instruction* pointerType = defUseMgr->GetDef(inst.type_id()); + if (pointerType == nullptr || pointerType->opcode() != spv::Op::OpTypePointer) continue; + if (candidateStructs.count(pointerType->GetSingleWordInOperand(1)) == 0) continue; + variableToStruct.emplace(inst.result_id(), pointerType->GetSingleWordInOperand(1)); + } + if (variableToStruct.empty()) return plans; + + // A block whose variable is used as anything but an access-chain base (loaded + // whole, handed to a function) cannot be re-indexed; a partially re-indexed + // block would address the wrong counters, so the whole block is refused. + std::unordered_map> chainsByStruct; + std::unordered_set undoableStructs; + for (const auto& [variableId, structId] : variableToStruct) { + defUseMgr->ForEachUser(defUseMgr->GetDef(variableId), [&](Instruction* user) { + switch (user->opcode()) { + case spv::Op::OpName: + case spv::Op::OpDecorate: + case spv::Op::OpDecorateId: + case spv::Op::OpEntryPoint: + return; + case spv::Op::OpAccessChain: + case spv::Op::OpInBoundsAccessChain: + if (user->NumInOperands() >= 2 && + user->GetSingleWordInOperand(0) == variableId) { + chainsByStruct[structId].push_back(user); + return; + } + undoableStructs.insert(structId); + return; + default: + undoableStructs.insert(structId); + return; + } + }); + } + + for (const auto& [structId, structType] : candidateStructs) { + if (undoableStructs.count(structId) != 0) continue; + + BlockPlan plan; + plan.structType = structType; + const uint32_t memberCount = structType->NumInOperands(); + bool expressible = true; + for (uint32_t member = 0; member < memberCount; ++member) { + uint32_t byteOffset = 0; + if (!TryGetMemberDecorationLiteral(context, structId, member, + spv::Decoration::Offset, &byteOffset) || + byteOffset % kCounterBytes != 0u) { + expressible = false; + break; + } + MemberPlan memberPlan; + uint32_t uintTypeId = 0; + if (!DescribeMember(context, structType->GetSingleWordInOperand(member), &uintTypeId, + &memberPlan)) { + expressible = false; + break; + } + if (plan.uintTypeId != 0 && plan.uintTypeId != uintTypeId) { + expressible = false; + break; + } + plan.uintTypeId = uintTypeId; + memberPlan.elementOffset = byteOffset / kCounterBytes; + const uint64_t end = static_cast(memberPlan.elementOffset) + + static_cast(memberPlan.elementCount); + if (end > kMaxCounterElements) { + expressible = false; + break; + } + if (end > plan.totalElements) plan.totalElements = static_cast(end); + plan.members.push_back(memberPlan); + } + if (!expressible || plan.members.empty() || plan.totalElements == 0) continue; + // Already std430: leave it exactly as it is. This is the overwhelmingly + // common answer and the reason the pass can be gated on a cheap probe. + if (IsNaturallyPacked(plan.members)) continue; + + // Every chain must be one of the two shapes the re-index understands: a + // scalar counter reached by (variable, member) or an array element + // reached by (variable, member, index). One that stops at the member, or + // reaches deeper, is not a counter access this pass can move. + const auto chains = chainsByStruct.find(structId); + if (chains != chainsByStruct.end()) { + for (Instruction* chain : chains->second) { + const spvtools::opt::analysis::Constant* memberIndex = + context->get_constant_mgr()->FindDeclaredConstant( + chain->GetSingleWordInOperand(1)); + if (memberIndex == nullptr || memberIndex->AsIntConstant() == nullptr) { + expressible = false; + break; + } + const uint32_t member = memberIndex->AsIntConstant()->GetU32BitValue(); + if (member >= plan.members.size() || + chain->NumInOperands() != (plan.members[member].isArray ? 3u : 2u)) { + expressible = false; + break; + } + plan.chains.push_back(chain); + } + } + if (!expressible) continue; + + plans.push_back(std::move(plan)); + } + return plans; + } + + // The id of |value| as a constant of the same integer type as |likeId|. + uint32_t ConstantLike(IRContext* context, uint32_t likeId, uint32_t value) { + Instruction* likeDef = context->get_def_use_mgr()->GetDef(likeId); + const spvtools::opt::analysis::Type* type = + context->get_type_mgr()->GetType(likeDef->type_id()); + const spvtools::opt::analysis::Constant* constant = + context->get_constant_mgr()->GetConstant(type, {value}); + return context->get_constant_mgr()->GetDefiningInstruction(constant)->result_id(); + } + + Module::inst_iterator PositionOf(IRContext* context, const Instruction* target) { + for (auto it = context->types_values_begin(); it != context->types_values_end(); ++it) { + if (&*it == target) return it; + } + return context->types_values_end(); + } + + // Whether |firstId| is declared before |secondId| in the types/constants section. + bool DeclaredBefore(IRContext* context, uint32_t firstId, uint32_t secondId) { + for (const Instruction& inst : context->module()->types_values()) { + if (inst.result_id() == firstId) return true; + if (inst.result_id() == secondId) return false; + } + return false; + } + + // A fresh `uint[length]` with ArrayStride 4, spliced in immediately BEFORE the + // block that will name it - SPIR-V has no forward references between types, so + // appending it at the end of the section would make the module invalid. A + // duplicate OpTypeArray is legal (SPIR-V 2.8 exempts aggregates from the + // uniqueness rule, and so does spirv-val), so no search for an existing one is + // needed; the LENGTH CONSTANT is not exempt, and if the module already declares + // it after the block there is nowhere legal to put the array - the block is then + // declined and keeps today's behaviour. Returns 0 for that. + uint32_t CreateCounterArrayTypeBefore(IRContext* context, Instruction* structType, + uint32_t uintTypeId, uint32_t length) { + auto* constantMgr = context->get_constant_mgr(); + const spvtools::opt::analysis::Type* uintType = context->get_type_mgr()->GetType(uintTypeId); + if (uintType == nullptr) return 0; + const spvtools::opt::analysis::Constant* lengthConstant = + constantMgr->GetConstant(uintType, {length}); + if (lengthConstant == nullptr) return 0; + + Module::inst_iterator position = PositionOf(context, structType); + if (position == context->types_values_end()) return 0; + Instruction* lengthInst = constantMgr->GetDefiningInstruction(lengthConstant, 0, &position); + if (lengthInst == nullptr) return 0; + if (!DeclaredBefore(context, lengthInst->result_id(), structType->result_id())) return 0; + + const uint32_t arrayTypeId = context->TakeNextId(); + if (arrayTypeId == 0) return 0; + auto arrayType = MakeUnique( + context, spv::Op::OpTypeArray, 0, arrayTypeId, + std::initializer_list{{SPV_OPERAND_TYPE_ID, {uintTypeId}}, + {SPV_OPERAND_TYPE_ID, {lengthInst->result_id()}}}); + Instruction* inserted = structType->InsertBefore(std::move(arrayType)); + context->AnalyzeDefUse(inserted); + context->get_decoration_mgr()->AddDecorationVal( + arrayTypeId, static_cast(spv::Decoration::ArrayStride), kCounterBytes); + return arrayTypeId; + } + + // Drops the annotations the collapsed struct no longer has a member for: every + // OpMemberDecorate and OpMemberName past member 0, plus member 0's own Offset + // (the caller re-adds it as 0). Member 0's OTHER decorations - Coherent, + // Volatile, Restrict and the like, which describe how the counters are accessed + // rather than where they sit - are deliberately kept. + void StripMemberAnnotations(IRContext* context, uint32_t structId) { + std::vector doomed; + for (Instruction* decoration : + context->get_decoration_mgr()->GetDecorationsFor(structId, false)) { + if (decoration->opcode() != spv::Op::OpMemberDecorate || + decoration->NumInOperands() < 3) { + continue; + } + const bool pastMemberZero = decoration->GetSingleWordInOperand(1) != 0u; + const bool isOffset = + static_cast(decoration->GetSingleWordInOperand(2)) == + spv::Decoration::Offset; + if (pastMemberZero || isOffset) doomed.push_back(decoration); + } + for (Instruction& debug : context->module()->debugs2()) { + if (debug.opcode() != spv::Op::OpMemberName || debug.NumInOperands() < 2) continue; + if (debug.GetSingleWordInOperand(0) != structId) continue; + if (debug.GetSingleWordInOperand(1) == 0u) continue; // member 0 keeps its name + doomed.push_back(&debug); + } + for (Instruction* inst : doomed) context->KillInst(inst); + } + } // namespace + + bool FlattenAtomicCounterBlockPass::BinaryHasOffsetAtomicCounterBlock(const Vector& binary) { + if (binary.empty()) { + return false; + } + std::unique_ptr context = spvtools::BuildModule( + SPV_ENV_VULKAN_1_1, + [](spv_message_level_t, const char*, const spv_position_t&, const char*) {}, + binary.data(), binary.size()); + if (!context) { + // Unparseable here means unusable downstream too; let the ordinary transpile + // path produce the error rather than inventing a verdict from it. + return false; + } + return !BuildPlans(context.get()).empty(); + } + + spvtools::opt::Pass::Status FlattenAtomicCounterBlockPass::Process() { + auto* irContext = context(); + const std::vector plans = BuildPlans(irContext); + if (plans.empty()) { + return Status::SuccessWithoutChange; + } + + bool modified = false; + for (const BlockPlan& plan : plans) { + const uint32_t structId = plan.structType->result_id(); + const uint32_t arrayTypeId = CreateCounterArrayTypeBefore( + irContext, plan.structType, plan.uintTypeId, plan.totalElements); + if (arrayTypeId == 0) { + MGLOG_D("[spirv] atomic-counter block %%%u: no legal place for the flattened array " + "type; leaving the block alone", + structId); + continue; + } + + // Re-index BEFORE the struct is collapsed, so the member index each chain + // carries still names the member the plan was built from. + for (Instruction* chain : plan.chains) { + const uint32_t memberIndexId = chain->GetSingleWordInOperand(1); + const uint32_t member = irContext->get_constant_mgr() + ->FindDeclaredConstant(memberIndexId) + ->AsIntConstant() + ->GetU32BitValue(); + const MemberPlan& memberPlan = plan.members[member]; + + std::vector operands; + operands.push_back(chain->GetInOperand(0)); + operands.push_back({SPV_OPERAND_TYPE_ID, {ConstantLike(irContext, memberIndexId, 0u)}}); + if (!memberPlan.isArray) { + operands.push_back( + {SPV_OPERAND_TYPE_ID, + {ConstantLike(irContext, memberIndexId, memberPlan.elementOffset)}}); + } else { + const uint32_t elementId = chain->GetSingleWordInOperand(2); + uint32_t shiftedId = elementId; + if (memberPlan.elementOffset != 0u) { + const spvtools::opt::analysis::Constant* elementConstant = + irContext->get_constant_mgr()->FindDeclaredConstant(elementId); + if (elementConstant != nullptr && elementConstant->AsIntConstant() != nullptr) { + shiftedId = ConstantLike(irContext, elementId, + elementConstant->AsIntConstant()->GetU32BitValue() + + memberPlan.elementOffset); + } else { + InstructionBuilder builder( + irContext, chain, + IRContext::kAnalysisDefUse | IRContext::kAnalysisInstrToBlockMapping); + Instruction* elementDef = irContext->get_def_use_mgr()->GetDef(elementId); + shiftedId = builder + .AddBinaryOp(elementDef->type_id(), spv::Op::OpIAdd, + elementId, + ConstantLike(irContext, elementId, + memberPlan.elementOffset)) + ->result_id(); + } + } + operands.push_back({SPV_OPERAND_TYPE_ID, {shiftedId}}); + } + chain->SetInOperands(std::move(operands)); + irContext->UpdateDefUse(chain); + } + + StripMemberAnnotations(irContext, structId); + plan.structType->SetInOperands({{SPV_OPERAND_TYPE_ID, {arrayTypeId}}}); + irContext->UpdateDefUse(plan.structType); + irContext->get_decoration_mgr()->AddMemberDecoration( + structId, 0u, static_cast(spv::Decoration::Offset), 0u); + modified = true; + MGLOG_D("[spirv] atomic-counter block %%%u: collapsed %zu offset member(s) into one " + "%u-element array so std430 can express it", + structId, plan.members.size(), plan.totalElements); + } + + if (!modified) { + return Status::SuccessWithoutChange; + } + irContext->InvalidateAnalysesExceptFor(IRContext::kAnalysisNone); + return Status::SuccessWithChange; + } + + spvtools::Optimizer::PassToken FlattenAtomicCounterBlockPass::CreateFlattenAtomicCounterBlockPass() { + return spvtools::Optimizer::PassToken(MakeUnique()); + } + } // namespace ShaderTranspiler + } // namespace MG_Util +} // namespace MobileGL diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenAtomicCounterBlockPass.h b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenAtomicCounterBlockPass.h new file mode 100644 index 00000000..effb3782 --- /dev/null +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenAtomicCounterBlockPass.h @@ -0,0 +1,81 @@ +// MobileGL - MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenAtomicCounterBlockPass.h +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#pragma once + +#include "spirv-tools/optimizer.hpp" +#include "source/opt/pass.h" + +#include +#include + +namespace MobileGL { + namespace MG_Util { + namespace ShaderTranspiler { + // glslang's relaxed parse lowers every atomic_uint onto a synthesized storage block + // named gl_AtomicCounterBlock_, and it PRESERVES the application's + // `layout(offset = N)` as the member's SPIR-V Offset decoration. A block whose first + // member sits at offset 8 is not expressible in std140 or std430 - both put member 0 + // at offset 0 - and GLSL ES has no member layout(offset=), so SPIRV-Cross refuses the + // whole stage rather than emit something wrong: + // + // Push constant block cannot be expressed as neither std430 nor std140. + // ES-targets do not support GL_ARB_enhanced_layouts. + // + // (The message says "push constant"; the variable is StorageClass Uniform. Do not + // chase push constants.) The stage never reaches the driver, the program links short + // of it with an EMPTY driver info log, and the dispatch no-ops while the frontend + // still reports the link status glslang published - KHR-GL43.compute_shader.resources + // -atomic-counter's non-zero-offset sibling, and the shape every conformance case + // that declares `layout(binding = B, offset = N)` takes. + // + // The repair is to make the offsets DISAPPEAR rather than to move the buffer. Each + // atomic-counter block is collapsed into ONE `uint` array covering the same byte + // window, member 0 at offset 0 with ArrayStride 4 - a layout std430 expresses + // exactly - and every access is re-indexed to the element that used to be at its + // byte offset. `counters[k]` declared at offset 8 becomes element (2 + k) of the + // array, i.e. byte 8 + 4k, which is the byte the application's counter buffer really + // holds. + // + // Why not simply rebase the offsets to zero and bind the buffer 8 bytes in: because + // glBindBufferRange's offset must be a multiple of + // GL_SHADER_STORAGE_BUFFER_OFFSET_ALIGNMENT, which the target device reports as 32. + // A byte offset of 8 cannot be expressed as a binding at all, so the correction has + // to live in the shader's indexing, where it costs nothing. + // + // A block that is ALREADY laid out naturally - which is every shader that omits the + // offset qualifier, and so very nearly all of them - is left byte-identical: the + // detection below is the gate, and KHR-GL43.compute_shader.resource-atomic-counter + // (offset 0) is the latch that the no-op case stays a no-op. + // + // Declines the whole block, leaving it untouched, on any shape it cannot re-index + // exactly: a member that is not `uint` or an array of `uint` with stride 4, an offset + // that is not a multiple of 4, a byte window past what GL_MAX_ATOMIC_COUNTER_BUFFER + // _SIZE allows, a member with no Offset decoration at all, or an access chain that + // stops at the member (a pointer handed to a function) rather than reaching a + // counter. + // + // DirectGLES transpile path only. DirectVulkan takes the block's declared offsets + // natively through an explicitly-laid-out descriptor and must see them unchanged. + class FlattenAtomicCounterBlockPass final : public spvtools::opt::Pass { + public: + const char* name() const override { return "mobilegl-flatten-atomic-counter-block"; } + Status Process() override; + + // The detection half, on a serialized module: true when the module declares an + // atomic-counter block whose member offsets are not already the natural std430 + // packing, i.e. whether this pass could change anything. One BuildModule, no + // serialization, so the ~every shader that declares no counter (or declares one + // at offset 0) pays no optimizer round trip. + static bool BinaryHasOffsetAtomicCounterBlock(const Vector& binary); + + static spvtools::Optimizer::PassToken CreateFlattenAtomicCounterBlockPass(); + }; + } // namespace ShaderTranspiler + } // namespace MG_Util +} // namespace MobileGL