// MobileGL - MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.cpp
// Copyright (c) 2026 MobileGL-Dev
// Licensed under the GNU Lesser General Public License v3.0:
//   https://www.gnu.org/licenses/gpl-3.0.txt
//   https://www.gnu.org/licenses/lgpl-3.0.txt
// SPDX-License-Identifier: LGPL-3.0-only
// End of Source File Header

#include "FlattenFloat64StorageBlockPass.h"

#include "spirv.hpp"
#include "source/opt/basic_block.h"
#include "source/opt/constants.h"
#include "source/opt/decoration_manager.h"
#include "source/opt/def_use_manager.h"
#include "source/opt/function.h"
#include "source/opt/instruction.h"
#include "source/opt/ir_builder.h"
#include "source/opt/ir_context.h"
#include "source/opt/module.h"
#include "source/opt/type_manager.h"
#include "source/util/make_unique.h"

#include <algorithm>
#include <memory>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>

namespace MobileGL {
    namespace MG_Util {
        namespace ShaderTranspiler {
            namespace {
                using spvtools::MakeUnique;
                using spvtools::opt::Instruction;
                using spvtools::opt::InstructionBuilder;
                using spvtools::opt::IRContext;
                using spvtools::opt::Module;
                using spvtools::opt::Operand;

                // The flattened array's element, and the granularity every offset and stride in
                // the block has to land on.
                constexpr uint32_t kWordBytes = 4u;
                // A block wider than any GL implementation lets one binding cover is refused
                // rather than expanded into an array nothing could address. GL 4.6 core table
                // 23.64 puts the minimum GL_MAX_SHADER_STORAGE_BLOCK_SIZE at 2^24 bytes; this is
                // a generous multiple of that and exists only to bound the rewrite.
                constexpr uint64_t kMaxBlockBytes = 1ull << 27;
                // The most scalars one load or store may decompose into. A whole-aggregate copy
                // becomes one word access per scalar, so without a cap a `dvec4 data[4096]`
                // member would turn a two-instruction copy into a 32k-instruction one.
                constexpr uint32_t kMaxLeavesPerAccess = 1024u;
                // The analyses every builder in this pass keeps current as it inserts.
                constexpr IRContext::Analysis kPreservedAnalyses = static_cast<IRContext::Analysis>(
                    static_cast<uint32_t>(IRContext::kAnalysisDefUse) |
                    static_cast<uint32_t>(IRContext::kAnalysisInstrToBlockMapping));

                // How a type sits in memory, as the ENCLOSING struct member described it.
                // MatrixStride and RowMajor are member decorations rather than type decorations,
                // so a matrix type carries no layout of its own and the walk has to hand it down -
                // through arrays of matrices too, which is why this rides alongside the type id
                // instead of being looked up from it.
                struct TypeCursor {
                    uint32_t typeId = 0;
                    uint32_t matrixStride = 0;
                    bool rowMajor = false;
                };

                // One access chain rooted at a flattened block's variable, and everything the
                // rewrite needs so it does not have to walk the type tree a second time.
                struct ChainPlan {
                    Instruction* chain = nullptr;
                    uint32_t variableId = 0;
                    // What the chain's CONSTANT indices contribute, in words.
                    uint32_t constantWords = 0;
                    // Its non-constant indices, as (index value id, words per step).
                    std::vector<std::pair<uint32_t, uint32_t>> dynamicTerms;
                    TypeCursor pointee;
                    std::vector<Instruction*> loads;
                    std::vector<Instruction*> stores;
                };

                struct BlockPlan {
                    Instruction* structType = nullptr;
                    uint32_t storageClass = 0;
                    uint32_t wordCount = 0;
                    std::vector<ChainPlan> chains;
                };

                bool IsDoubleType(const Instruction* type) {
                    return type != nullptr && type->opcode() == spv::Op::OpTypeFloat &&
                           type->NumInOperands() >= 1 && type->GetSingleWordInOperand(0) == 64u;
                }

                // Byte size of a scalar this pass can carry, or 0 for one it cannot.
                uint32_t ScalarByteSize(const Instruction* type) {
                    if (type == nullptr) return 0;
                    if (type->opcode() != spv::Op::OpTypeFloat && type->opcode() != spv::Op::OpTypeInt) {
                        return 0;
                    }
                    const uint32_t width = type->GetSingleWordInOperand(0);
                    if (width == 32u) return 4u;
                    if (width == 64u && type->opcode() == spv::Op::OpTypeFloat) return 8u;
                    return 0;
                }

                bool TryGetDecorationLiteral(IRContext* context, uint32_t id, spv::Decoration kind,
                                             uint32_t* literal) {
                    for (Instruction* decoration : context->get_decoration_mgr()->GetDecorationsFor(id, false)) {
                        if (decoration->opcode() != spv::Op::OpDecorate || decoration->NumInOperands() < 3 ||
                            static_cast<spv::Decoration>(decoration->GetSingleWordInOperand(1)) != kind) {
                            continue;
                        }
                        *literal = decoration->GetSingleWordInOperand(2);
                        return true;
                    }
                    return false;
                }

                bool TryGetMemberDecorationLiteral(IRContext* context, uint32_t structId, uint32_t member,
                                                   spv::Decoration kind, uint32_t* literal) {
                    for (Instruction* decoration :
                         context->get_decoration_mgr()->GetDecorationsFor(structId, false)) {
                        if (decoration->opcode() != spv::Op::OpMemberDecorate ||
                            decoration->NumInOperands() < 4 ||
                            decoration->GetSingleWordInOperand(1) != member ||
                            static_cast<spv::Decoration>(decoration->GetSingleWordInOperand(2)) != kind) {
                            continue;
                        }
                        *literal = decoration->GetSingleWordInOperand(3);
                        return true;
                    }
                    return false;
                }

                bool HasMemberDecoration(IRContext* context, uint32_t structId, uint32_t member,
                                         spv::Decoration kind) {
                    for (Instruction* decoration :
                         context->get_decoration_mgr()->GetDecorationsFor(structId, false)) {
                        if (decoration->opcode() != spv::Op::OpMemberDecorate ||
                            decoration->NumInOperands() < 3 ||
                            decoration->GetSingleWordInOperand(1) != member) {
                            continue;
                        }
                        if (static_cast<spv::Decoration>(decoration->GetSingleWordInOperand(2)) == kind) {
                            return true;
                        }
                    }
                    return false;
                }

                bool HasDecoration(IRContext* context, uint32_t id, spv::Decoration kind) {
                    for (Instruction* decoration : context->get_decoration_mgr()->GetDecorationsFor(id, false)) {
                        if (decoration->opcode() != spv::Op::OpDecorate || decoration->NumInOperands() < 2) {
                            continue;
                        }
                        if (static_cast<spv::Decoration>(decoration->GetSingleWordInOperand(1)) == kind) {
                            return true;
                        }
                    }
                    return false;
                }

                // The cursor for member `member` of a struct: its type, plus the matrix layout
                // that member's own decorations describe.
                TypeCursor MemberCursor(IRContext* context, const Instruction* structType, uint32_t member) {
                    TypeCursor cursor;
                    cursor.typeId = structType->GetSingleWordInOperand(member);
                    uint32_t stride = 0;
                    if (TryGetMemberDecorationLiteral(context, structType->result_id(), member,
                                                      spv::Decoration::MatrixStride, &stride)) {
                        cursor.matrixStride = stride;
                    }
                    cursor.rowMajor = HasMemberDecoration(context, structType->result_id(), member,
                                                          spv::Decoration::RowMajor);
                    return cursor;
                }

                // Byte size of a type as it is laid out INSIDE a block, or 0 when this pass
                // cannot describe it (a runtime array, a width it does not carry, a matrix with
                // no stride or a row-major one).
                uint32_t LaidOutByteSize(IRContext* context, const TypeCursor& cursor) {
                    const Instruction* type = context->get_def_use_mgr()->GetDef(cursor.typeId);
                    if (type == nullptr) return 0;
                    switch (type->opcode()) {
                    case spv::Op::OpTypeInt:
                    case spv::Op::OpTypeFloat:
                        return ScalarByteSize(type);
                    case spv::Op::OpTypeVector: {
                        const uint32_t component = ScalarByteSize(
                            context->get_def_use_mgr()->GetDef(type->GetSingleWordInOperand(0)));
                        if (component == 0) return 0;
                        return component * type->GetSingleWordInOperand(1);
                    }
                    case spv::Op::OpTypeMatrix: {
                        if (cursor.matrixStride == 0 || cursor.rowMajor) return 0;
                        return cursor.matrixStride * type->GetSingleWordInOperand(1);
                    }
                    case spv::Op::OpTypeArray: {
                        uint32_t stride = 0;
                        if (!TryGetDecorationLiteral(context, cursor.typeId, spv::Decoration::ArrayStride,
                                                     &stride) ||
                            stride == 0) {
                            return 0;
                        }
                        const spvtools::opt::analysis::Constant* length =
                            context->get_constant_mgr()->FindDeclaredConstant(type->GetSingleWordInOperand(1));
                        if (length == nullptr || length->AsIntConstant() == nullptr) return 0;
                        const uint64_t total = static_cast<uint64_t>(stride) *
                                               static_cast<uint64_t>(length->AsIntConstant()->GetU32BitValue());
                        return total > kMaxBlockBytes ? 0u : static_cast<uint32_t>(total);
                    }
                    case spv::Op::OpTypeStruct: {
                        uint64_t end = 0;
                        for (uint32_t member = 0; member < type->NumInOperands(); ++member) {
                            uint32_t offset = 0;
                            if (!TryGetMemberDecorationLiteral(context, cursor.typeId, member,
                                                               spv::Decoration::Offset, &offset)) {
                                return 0;
                            }
                            const uint32_t size = LaidOutByteSize(context, MemberCursor(context, type, member));
                            if (size == 0) return 0;
                            end = std::max<uint64_t>(end, static_cast<uint64_t>(offset) + size);
                        }
                        return end > kMaxBlockBytes ? 0u : static_cast<uint32_t>(end);
                    }
                    default:
                        return 0;
                    }
                }

                // Whether this type decomposes into scalars the rewrite can move one word at a
                // time, counting them so a whole-aggregate access can be refused before it is
                // expanded.
                bool CanDecompose(IRContext* context, const TypeCursor& cursor, uint32_t* leafCount) {
                    const Instruction* type = context->get_def_use_mgr()->GetDef(cursor.typeId);
                    if (type == nullptr) return false;
                    switch (type->opcode()) {
                    case spv::Op::OpTypeInt:
                    case spv::Op::OpTypeFloat:
                        if (ScalarByteSize(type) == 0) return false;
                        ++*leafCount;
                        return *leafCount <= kMaxLeavesPerAccess;
                    case spv::Op::OpTypeVector: {
                        TypeCursor component;
                        component.typeId = type->GetSingleWordInOperand(0);
                        for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) {
                            if (!CanDecompose(context, component, leafCount)) return false;
                        }
                        return true;
                    }
                    case spv::Op::OpTypeMatrix: {
                        if (cursor.matrixStride == 0 || cursor.rowMajor ||
                            cursor.matrixStride % kWordBytes != 0) {
                            return false;
                        }
                        TypeCursor column;
                        column.typeId = type->GetSingleWordInOperand(0);
                        for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) {
                            if (!CanDecompose(context, column, leafCount)) return false;
                        }
                        return true;
                    }
                    case spv::Op::OpTypeArray: {
                        uint32_t stride = 0;
                        if (!TryGetDecorationLiteral(context, cursor.typeId, spv::Decoration::ArrayStride,
                                                     &stride) ||
                            stride == 0 || stride % kWordBytes != 0) {
                            return false;
                        }
                        const spvtools::opt::analysis::Constant* length =
                            context->get_constant_mgr()->FindDeclaredConstant(type->GetSingleWordInOperand(1));
                        if (length == nullptr || length->AsIntConstant() == nullptr) return false;
                        const uint32_t count = length->AsIntConstant()->GetU32BitValue();
                        if (count == 0 || count > kMaxLeavesPerAccess) return false;
                        TypeCursor element = cursor;
                        element.typeId = type->GetSingleWordInOperand(0);
                        for (uint32_t i = 0; i < count; ++i) {
                            if (!CanDecompose(context, element, leafCount)) return false;
                        }
                        return true;
                    }
                    case spv::Op::OpTypeStruct: {
                        for (uint32_t member = 0; member < type->NumInOperands(); ++member) {
                            uint32_t offset = 0;
                            if (!TryGetMemberDecorationLiteral(context, cursor.typeId, member,
                                                               spv::Decoration::Offset, &offset) ||
                                offset % kWordBytes != 0) {
                                return false;
                            }
                            if (!CanDecompose(context, MemberCursor(context, type, member), leafCount)) {
                                return false;
                            }
                        }
                        return true;
                    }
                    default:
                        return false;
                    }
                }

                bool TypeContainsFloat64(IRContext* context, uint32_t typeId,
                                         std::unordered_set<uint32_t>& visiting) {
                    const Instruction* type = context->get_def_use_mgr()->GetDef(typeId);
                    if (type == nullptr || !visiting.insert(typeId).second) return false;
                    switch (type->opcode()) {
                    case spv::Op::OpTypeFloat:
                        return type->GetSingleWordInOperand(0) == 64u;
                    case spv::Op::OpTypeVector:
                    case spv::Op::OpTypeMatrix:
                    case spv::Op::OpTypeArray:
                    case spv::Op::OpTypeRuntimeArray:
                        return TypeContainsFloat64(context, type->GetSingleWordInOperand(0), visiting);
                    case spv::Op::OpTypeStruct:
                        for (uint32_t member = 0; member < type->NumInOperands(); ++member) {
                            if (TypeContainsFloat64(context, type->GetSingleWordInOperand(member), visiting)) {
                                return true;
                            }
                        }
                        return false;
                    default:
                        return false;
                    }
                }

                // The constant an index operand names, or false when it is not one.
                bool TryGetConstantIndex(IRContext* context, uint32_t id, uint32_t* value) {
                    const spvtools::opt::analysis::Constant* constant =
                        context->get_constant_mgr()->FindDeclaredConstant(id);
                    if (constant == nullptr || constant->AsIntConstant() == nullptr) return false;
                    *value = constant->AsIntConstant()->GetU32BitValue();
                    return true;
                }

                // Walks one access chain against the block's type tree, recording the byte offset
                // it names as a constant part plus a list of (index, stride) terms. False for any
                // shape the rewrite cannot address exactly.
                bool PlanChain(IRContext* context, Instruction* chain, const TypeCursor& blockCursor,
                               ChainPlan* plan) {
                    TypeCursor cursor = blockCursor;
                    uint64_t constantBytes = 0;
                    for (uint32_t operand = 1; operand < chain->NumInOperands(); ++operand) {
                        const uint32_t indexId = chain->GetSingleWordInOperand(operand);
                        const Instruction* type = context->get_def_use_mgr()->GetDef(cursor.typeId);
                        if (type == nullptr) return false;

                        uint32_t stride = 0;
                        if (type->opcode() == spv::Op::OpTypeStruct) {
                            uint32_t member = 0;
                            if (!TryGetConstantIndex(context, indexId, &member) ||
                                member >= type->NumInOperands()) {
                                return false;
                            }
                            uint32_t offset = 0;
                            if (!TryGetMemberDecorationLiteral(context, cursor.typeId, member,
                                                               spv::Decoration::Offset, &offset) ||
                                offset % kWordBytes != 0) {
                                return false;
                            }
                            constantBytes += offset;
                            cursor = MemberCursor(context, type, member);
                            if (constantBytes > kMaxBlockBytes) return false;
                            continue;
                        }

                        switch (type->opcode()) {
                        case spv::Op::OpTypeArray:
                            if (!TryGetDecorationLiteral(context, cursor.typeId, spv::Decoration::ArrayStride,
                                                         &stride)) {
                                return false;
                            }
                            cursor.typeId = type->GetSingleWordInOperand(0);
                            break;
                        case spv::Op::OpTypeMatrix:
                            if (cursor.rowMajor || cursor.matrixStride == 0) return false;
                            stride = cursor.matrixStride;
                            cursor.typeId = type->GetSingleWordInOperand(0);
                            cursor.matrixStride = 0;
                            break;
                        case spv::Op::OpTypeVector:
                            stride = ScalarByteSize(
                                context->get_def_use_mgr()->GetDef(type->GetSingleWordInOperand(0)));
                            cursor.typeId = type->GetSingleWordInOperand(0);
                            cursor.matrixStride = 0;
                            break;
                        default:
                            return false;
                        }

                        if (stride == 0 || stride % kWordBytes != 0) return false;
                        uint32_t index = 0;
                        if (TryGetConstantIndex(context, indexId, &index)) {
                            constantBytes += static_cast<uint64_t>(index) * stride;
                            if (constantBytes > kMaxBlockBytes) return false;
                        } else {
                            plan->dynamicTerms.emplace_back(indexId, stride / kWordBytes);
                        }
                    }

                    if (constantBytes % kWordBytes != 0) return false;
                    plan->constantWords = static_cast<uint32_t>(constantBytes / kWordBytes);
                    plan->pointee = cursor;
                    return true;
                }

                // Everything the rewrite emits, over one module's shared scalar types.
                class Emitter {
                public:
                    // Where one block's words live: the variable holding them, the pointer type
                    // that reaches one, and the base index the chain resolved to.
                    struct Access {
                        uint32_t variableId = 0;
                        uint32_t wordPointerTypeId = 0;
                        uint32_t baseWordId = 0;
                        uint32_t memberZeroId = 0;
                    };

                    Emitter(IRContext* context, uint32_t uintTypeId, uint32_t boolTypeId, uint32_t floatTypeId)
                        : m_context(context), m_uintTypeId(uintTypeId), m_boolTypeId(boolTypeId),
                          m_floatTypeId(floatTypeId) {}

                    uint32_t UintConstant(uint32_t value) {
                        const spvtools::opt::analysis::Type* type =
                            m_context->get_type_mgr()->GetType(m_uintTypeId);
                        const spvtools::opt::analysis::Constant* constant =
                            m_context->get_constant_mgr()->GetConstant(type, {value});
                        return m_context->get_constant_mgr()->GetDefiningInstruction(constant)->result_id();
                    }

                    // The word index the chain names, materialised at the chain's own position so
                    // every load and store that uses it is dominated by it.
                    uint32_t WordIndexOf(const ChainPlan& plan) {
                        InstructionBuilder builder(m_context, plan.chain, kPreservedAnalyses);
                        uint32_t total = 0;
                        for (const auto& [indexId, wordsPerStep] : plan.dynamicTerms) {
                            uint32_t term = AsUint(builder, indexId);
                            if (wordsPerStep != 1u) {
                                term = Binary(builder, spv::Op::OpIMul, m_uintTypeId, term,
                                              UintConstant(wordsPerStep));
                            }
                            total = total == 0 ? term
                                               : Binary(builder, spv::Op::OpIAdd, m_uintTypeId, total, term);
                        }
                        if (total == 0) return UintConstant(plan.constantWords);
                        if (plan.constantWords == 0) return total;
                        return Binary(builder, spv::Op::OpIAdd, m_uintTypeId, total,
                                      UintConstant(plan.constantWords));
                    }

                    // Rebuilds the value an OpLoad of `cursor` would have produced, out of the
                    // words that live at `baseWordId + relWords`.
                    uint32_t BuildValue(InstructionBuilder& builder, const Access& access,
                                        const TypeCursor& cursor, uint32_t relWords) {
                        const Instruction* type = m_context->get_def_use_mgr()->GetDef(cursor.typeId);
                        switch (type->opcode()) {
                        case spv::Op::OpTypeInt:
                        case spv::Op::OpTypeFloat: {
                            if (IsDoubleType(type)) {
                                const uint32_t lo = LoadWord(builder, access, relWords);
                                const uint32_t hi = LoadWord(builder, access, relWords + 1);
                                const uint32_t narrowed = builder
                                                              .AddUnaryOp(m_floatTypeId, spv::Op::OpBitcast,
                                                                          NarrowDoubleBits(builder, lo, hi))
                                                              ->result_id();
                                return builder
                                    .AddUnaryOp(cursor.typeId, spv::Op::OpFConvert, narrowed)
                                    ->result_id();
                            }
                            const uint32_t word = LoadWord(builder, access, relWords);
                            if (cursor.typeId == m_uintTypeId) return word;
                            return builder.AddUnaryOp(cursor.typeId, spv::Op::OpBitcast, word)->result_id();
                        }
                        case spv::Op::OpTypeVector: {
                            TypeCursor component;
                            component.typeId = type->GetSingleWordInOperand(0);
                            const uint32_t step = ComponentWords(component.typeId);
                            std::vector<uint32_t> parts;
                            for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) {
                                parts.push_back(BuildValue(builder, access, component, relWords + i * step));
                            }
                            return builder.AddCompositeConstruct(cursor.typeId, parts)->result_id();
                        }
                        case spv::Op::OpTypeMatrix: {
                            TypeCursor column;
                            column.typeId = type->GetSingleWordInOperand(0);
                            const uint32_t step = cursor.matrixStride / kWordBytes;
                            std::vector<uint32_t> parts;
                            for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) {
                                parts.push_back(BuildValue(builder, access, column, relWords + i * step));
                            }
                            return builder.AddCompositeConstruct(cursor.typeId, parts)->result_id();
                        }
                        case spv::Op::OpTypeArray: {
                            TypeCursor element = cursor;
                            element.typeId = type->GetSingleWordInOperand(0);
                            uint32_t stride = 0;
                            TryGetDecorationLiteral(m_context, cursor.typeId, spv::Decoration::ArrayStride,
                                                    &stride);
                            const uint32_t count = m_context->get_constant_mgr()
                                                       ->FindDeclaredConstant(type->GetSingleWordInOperand(1))
                                                       ->AsIntConstant()
                                                       ->GetU32BitValue();
                            std::vector<uint32_t> parts;
                            for (uint32_t i = 0; i < count; ++i) {
                                parts.push_back(
                                    BuildValue(builder, access, element, relWords + i * (stride / kWordBytes)));
                            }
                            return builder.AddCompositeConstruct(cursor.typeId, parts)->result_id();
                        }
                        case spv::Op::OpTypeStruct: {
                            std::vector<uint32_t> parts;
                            for (uint32_t member = 0; member < type->NumInOperands(); ++member) {
                                uint32_t offset = 0;
                                TryGetMemberDecorationLiteral(m_context, cursor.typeId, member,
                                                              spv::Decoration::Offset, &offset);
                                parts.push_back(BuildValue(builder, access, MemberCursor(m_context, type, member),
                                                           relWords + offset / kWordBytes));
                            }
                            return builder.AddCompositeConstruct(cursor.typeId, parts)->result_id();
                        }
                        default:
                            return 0;
                        }
                    }

                    // The mirror image: writes `valueId` into the words at
                    // `baseWordId + relWords`. `path` is the composite-extract index list that
                    // reaches the part being written, empty at the root.
                    void StoreValue(InstructionBuilder& builder, const Access& access,
                                    const TypeCursor& cursor, uint32_t relWords, uint32_t rootValueId,
                                    std::vector<uint32_t>& path) {
                        const Instruction* type = m_context->get_def_use_mgr()->GetDef(cursor.typeId);
                        switch (type->opcode()) {
                        case spv::Op::OpTypeInt:
                        case spv::Op::OpTypeFloat: {
                            const uint32_t leaf = Extract(builder, cursor.typeId, rootValueId, path);
                            if (IsDoubleType(type)) {
                                const uint32_t narrowed =
                                    builder.AddUnaryOp(m_floatTypeId, spv::Op::OpFConvert, leaf)->result_id();
                                const uint32_t bits =
                                    builder.AddUnaryOp(m_uintTypeId, spv::Op::OpBitcast, narrowed)->result_id();
                                uint32_t lo = 0;
                                uint32_t hi = 0;
                                WidenFloatBits(builder, bits, &lo, &hi);
                                StoreWord(builder, access, relWords, lo);
                                StoreWord(builder, access, relWords + 1, hi);
                                return;
                            }
                            const uint32_t word =
                                cursor.typeId == m_uintTypeId
                                    ? leaf
                                    : builder.AddUnaryOp(m_uintTypeId, spv::Op::OpBitcast, leaf)->result_id();
                            StoreWord(builder, access, relWords, word);
                            return;
                        }
                        case spv::Op::OpTypeVector: {
                            TypeCursor component;
                            component.typeId = type->GetSingleWordInOperand(0);
                            const uint32_t step = ComponentWords(component.typeId);
                            for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) {
                                path.push_back(i);
                                StoreValue(builder, access, component, relWords + i * step, rootValueId, path);
                                path.pop_back();
                            }
                            return;
                        }
                        case spv::Op::OpTypeMatrix: {
                            TypeCursor column;
                            column.typeId = type->GetSingleWordInOperand(0);
                            const uint32_t step = cursor.matrixStride / kWordBytes;
                            for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) {
                                path.push_back(i);
                                StoreValue(builder, access, column, relWords + i * step, rootValueId, path);
                                path.pop_back();
                            }
                            return;
                        }
                        case spv::Op::OpTypeArray: {
                            TypeCursor element = cursor;
                            element.typeId = type->GetSingleWordInOperand(0);
                            uint32_t stride = 0;
                            TryGetDecorationLiteral(m_context, cursor.typeId, spv::Decoration::ArrayStride,
                                                    &stride);
                            const uint32_t count = m_context->get_constant_mgr()
                                                       ->FindDeclaredConstant(type->GetSingleWordInOperand(1))
                                                       ->AsIntConstant()
                                                       ->GetU32BitValue();
                            for (uint32_t i = 0; i < count; ++i) {
                                path.push_back(i);
                                StoreValue(builder, access, element, relWords + i * (stride / kWordBytes),
                                           rootValueId, path);
                                path.pop_back();
                            }
                            return;
                        }
                        case spv::Op::OpTypeStruct: {
                            for (uint32_t member = 0; member < type->NumInOperands(); ++member) {
                                uint32_t offset = 0;
                                TryGetMemberDecorationLiteral(m_context, cursor.typeId, member,
                                                              spv::Decoration::Offset, &offset);
                                path.push_back(member);
                                StoreValue(builder, access, MemberCursor(m_context, type, member),
                                           relWords + offset / kWordBytes, rootValueId, path);
                                path.pop_back();
                            }
                            return;
                        }
                        default:
                            return;
                        }
                    }

                private:
                    uint32_t ComponentWords(uint32_t componentTypeId) {
                        return ScalarByteSize(m_context->get_def_use_mgr()->GetDef(componentTypeId)) /
                               kWordBytes;
                    }

                    uint32_t Binary(InstructionBuilder& builder, spv::Op opcode, uint32_t typeId, uint32_t a,
                                    uint32_t b) {
                        return builder.AddBinaryOp(typeId, opcode, a, b)->result_id();
                    }

                    uint32_t Select(InstructionBuilder& builder, uint32_t condition, uint32_t whenTrue,
                                    uint32_t whenFalse) {
                        return builder
                            .AddTernaryOp(m_uintTypeId, spv::Op::OpSelect, condition, whenTrue, whenFalse)
                            ->result_id();
                    }

                    uint32_t AsUint(InstructionBuilder& builder, uint32_t valueId) {
                        const Instruction* def = m_context->get_def_use_mgr()->GetDef(valueId);
                        if (def != nullptr && def->type_id() == m_uintTypeId) return valueId;
                        return builder.AddUnaryOp(m_uintTypeId, spv::Op::OpBitcast, valueId)->result_id();
                    }

                    // `base + words`, folding away the add when there is nothing to add.
                    uint32_t Offset(InstructionBuilder& builder, uint32_t baseWordId, uint32_t words) {
                        if (words == 0) return baseWordId;
                        return Binary(builder, spv::Op::OpIAdd, m_uintTypeId, baseWordId, UintConstant(words));
                    }

                    uint32_t LoadWord(InstructionBuilder& builder, const Access& access, uint32_t words) {
                        const uint32_t indexId = Offset(builder, access.baseWordId, words);
                        Instruction* pointer = builder.AddAccessChain(
                            access.wordPointerTypeId, access.variableId, {access.memberZeroId, indexId});
                        return builder.AddLoad(m_uintTypeId, pointer->result_id())->result_id();
                    }

                    void StoreWord(InstructionBuilder& builder, const Access& access, uint32_t words,
                                   uint32_t valueId) {
                        const uint32_t indexId = Offset(builder, access.baseWordId, words);
                        Instruction* pointer = builder.AddAccessChain(
                            access.wordPointerTypeId, access.variableId, {access.memberZeroId, indexId});
                        builder.AddStore(pointer->result_id(), valueId);
                    }

                    // One scalar of the value being stored. An empty path IS the value.
                    uint32_t Extract(InstructionBuilder& builder, uint32_t typeId, uint32_t rootValueId,
                                     const std::vector<uint32_t>& path) {
                        if (path.empty()) return rootValueId;
                        return builder.AddCompositeExtract(typeId, rootValueId, path)->result_id();
                    }

                    // binary64 word pair -> the binary32 bit pattern nearest it, truncating the
                    // mantissa bits binary32 cannot hold. Straight-line by construction: every
                    // case is an OpSelect, so this needs no control flow and never splits a block.
                    uint32_t NarrowDoubleBits(InstructionBuilder& builder, uint32_t lo, uint32_t hi) {
                        const uint32_t sign = Binary(builder, spv::Op::OpBitwiseAnd, m_uintTypeId, hi,
                                                     UintConstant(0x80000000u));
                        const uint32_t exponent = Binary(
                            builder, spv::Op::OpBitwiseAnd, m_uintTypeId,
                            Binary(builder, spv::Op::OpShiftRightLogical, m_uintTypeId, hi, UintConstant(20)),
                            UintConstant(0x7FFu));
                        const uint32_t significandHigh = Binary(builder, spv::Op::OpBitwiseAnd, m_uintTypeId,
                                                                hi, UintConstant(0xFFFFFu));
                        // The 23 bits binary32 keeps: 20 from the high word, 3 from the low one.
                        const uint32_t significand = Binary(
                            builder, spv::Op::OpBitwiseOr, m_uintTypeId,
                            Binary(builder, spv::Op::OpShiftLeftLogical, m_uintTypeId, significandHigh,
                                   UintConstant(3)),
                            Binary(builder, spv::Op::OpShiftRightLogical, m_uintTypeId, lo, UintConstant(29)));

                        // 1023 - 127 = 896, so the binary32 exponent field is `exponent - 896` and
                        // every bound on it is an unsigned comparison against that bias.
                        const uint32_t normalBits = Binary(
                            builder, spv::Op::OpBitwiseOr, m_uintTypeId, sign,
                            Binary(builder, spv::Op::OpBitwiseOr, m_uintTypeId,
                                   Binary(builder, spv::Op::OpShiftLeftLogical, m_uintTypeId,
                                          Binary(builder, spv::Op::OpISub, m_uintTypeId, exponent,
                                                 UintConstant(896)),
                                          UintConstant(23)),
                                   significand));
                        const uint32_t infinityBits = Binary(builder, spv::Op::OpBitwiseOr, m_uintTypeId, sign,
                                                             UintConstant(0x7F800000u));
                        // A quiet NaN that stays one even when every significant bit sat in the 29
                        // low bits binary32 discards.
                        const uint32_t nanBits =
                            Binary(builder, spv::Op::OpBitwiseOr, m_uintTypeId, infinityBits,
                                   Binary(builder, spv::Op::OpBitwiseOr, m_uintTypeId,
                                          UintConstant(0x400000u), significand));

                        const uint32_t significandIsZero =
                            Binary(builder, spv::Op::OpIEqual, m_boolTypeId,
                                   Binary(builder, spv::Op::OpBitwiseOr, m_uintTypeId, significandHigh, lo),
                                   UintConstant(0));
                        const uint32_t maxExponentBits =
                            Select(builder, significandIsZero, infinityBits, nanBits);

                        const uint32_t isZeroExponent =
                            Binary(builder, spv::Op::OpIEqual, m_boolTypeId, exponent, UintConstant(0));
                        const uint32_t isMaxExponent =
                            Binary(builder, spv::Op::OpIEqual, m_boolTypeId, exponent, UintConstant(0x7FFu));
                        // <= 896 is every magnitude binary32 could hold only as a subnormal;
                        // >= 1151 is every one it cannot hold at all.
                        const uint32_t underflows = Binary(builder, spv::Op::OpULessThanEqual, m_boolTypeId,
                                                           exponent, UintConstant(896));
                        const uint32_t overflows = Binary(builder, spv::Op::OpUGreaterThanEqual, m_boolTypeId,
                                                          exponent, UintConstant(1151));

                        uint32_t bits = Select(builder, overflows, infinityBits, normalBits);
                        bits = Select(builder, underflows, sign, bits);
                        bits = Select(builder, isMaxExponent, maxExponentBits, bits);
                        return Select(builder, isZeroExponent, sign, bits);
                    }

                    // The inverse: a binary32 bit pattern -> the binary64 word pair for it.
                    void WidenFloatBits(InstructionBuilder& builder, uint32_t bits, uint32_t* lo,
                                        uint32_t* hi) {
                        const uint32_t sign = Binary(builder, spv::Op::OpBitwiseAnd, m_uintTypeId, bits,
                                                     UintConstant(0x80000000u));
                        const uint32_t exponent = Binary(
                            builder, spv::Op::OpBitwiseAnd, m_uintTypeId,
                            Binary(builder, spv::Op::OpShiftRightLogical, m_uintTypeId, bits, UintConstant(23)),
                            UintConstant(0xFFu));
                        const uint32_t significand = Binary(builder, spv::Op::OpBitwiseAnd, m_uintTypeId, bits,
                                                            UintConstant(0x7FFFFFu));
                        const uint32_t significandHigh = Binary(builder, spv::Op::OpShiftRightLogical,
                                                                m_uintTypeId, significand, UintConstant(3));
                        const uint32_t significandLow = Binary(builder, spv::Op::OpShiftLeftLogical,
                                                               m_uintTypeId, significand, UintConstant(29));

                        const uint32_t normalHigh = Binary(
                            builder, spv::Op::OpBitwiseOr, m_uintTypeId, sign,
                            Binary(builder, spv::Op::OpBitwiseOr, m_uintTypeId,
                                   Binary(builder, spv::Op::OpShiftLeftLogical, m_uintTypeId,
                                          Binary(builder, spv::Op::OpIAdd, m_uintTypeId, exponent,
                                                 UintConstant(896)),
                                          UintConstant(20)),
                                   significandHigh));
                        const uint32_t maxHigh = Binary(builder, spv::Op::OpBitwiseOr, m_uintTypeId, sign,
                                                        Binary(builder, spv::Op::OpBitwiseOr, m_uintTypeId,
                                                               UintConstant(0x7FF00000u), significandHigh));

                        const uint32_t isZeroExponent =
                            Binary(builder, spv::Op::OpIEqual, m_boolTypeId, exponent, UintConstant(0));
                        const uint32_t isMaxExponent =
                            Binary(builder, spv::Op::OpIEqual, m_boolTypeId, exponent, UintConstant(0xFFu));

                        // A binary32 subnormal is below every binary64 this can name without a
                        // normalising loop, so it becomes the signed zero it is nearest to.
                        uint32_t high = Select(builder, isMaxExponent, maxHigh, normalHigh);
                        *hi = Select(builder, isZeroExponent, sign, high);
                        *lo = Select(builder, isZeroExponent, UintConstant(0), significandLow);
                    }

                    IRContext* m_context;
                    uint32_t m_uintTypeId;
                    uint32_t m_boolTypeId;
                    uint32_t m_floatTypeId;
                };
                // --- the module-level rewrite ------------------------------------------------

                Module::inst_iterator PositionOf(IRContext* context, const Instruction* target) {
                    for (auto it = context->types_values_begin(); it != context->types_values_end(); ++it) {
                        if (&*it == target) return it;
                    }
                    return context->types_values_end();
                }

                // Whether |firstId| is declared before |secondId| in the types/constants section.
                bool DeclaredBefore(IRContext* context, uint32_t firstId, uint32_t secondId) {
                    for (const Instruction& inst : context->module()->types_values()) {
                        if (inst.result_id() == firstId) return true;
                        if (inst.result_id() == secondId) return false;
                    }
                    return false;
                }

                // A fresh `uint[length]` with ArrayStride 4, spliced in immediately BEFORE the
                // block that will name it - SPIR-V has no forward references between types, so
                // appending it at the end of the section would make the module invalid. A
                // duplicate OpTypeArray is legal (SPIR-V 2.8 exempts aggregates from the
                // uniqueness rule, and so does spirv-val), so no search for an existing one is
                // needed; the LENGTH CONSTANT is not exempt, and if the module already declares
                // it after the block there is nowhere legal to put the array - the block is then
                // declined and keeps today's behaviour. Returns 0 for that, and for a uint type
                // that is itself declared too late.
                uint32_t CreateWordArrayTypeBefore(IRContext* context, Instruction* structType,
                                                   uint32_t uintTypeId, uint32_t length) {
                    if (!DeclaredBefore(context, uintTypeId, structType->result_id())) return 0;

                    auto* constantMgr = context->get_constant_mgr();
                    const spvtools::opt::analysis::Type* uintType = context->get_type_mgr()->GetType(uintTypeId);
                    if (uintType == nullptr) return 0;
                    const spvtools::opt::analysis::Constant* lengthConstant =
                        constantMgr->GetConstant(uintType, {length});
                    if (lengthConstant == nullptr) return 0;

                    Module::inst_iterator position = PositionOf(context, structType);
                    if (position == context->types_values_end()) return 0;
                    Instruction* lengthInst = constantMgr->GetDefiningInstruction(lengthConstant, 0, &position);
                    if (lengthInst == nullptr) return 0;
                    if (!DeclaredBefore(context, lengthInst->result_id(), structType->result_id())) return 0;

                    const uint32_t arrayTypeId = context->TakeNextId();
                    if (arrayTypeId == 0) return 0;
                    auto arrayType = MakeUnique<Instruction>(
                        context, spv::Op::OpTypeArray, 0, arrayTypeId,
                        std::initializer_list<Operand>{{SPV_OPERAND_TYPE_ID, {uintTypeId}},
                                                       {SPV_OPERAND_TYPE_ID, {lengthInst->result_id()}}});
                    Instruction* inserted = structType->InsertBefore(std::move(arrayType));
                    context->AnalyzeDefUse(inserted);
                    context->get_decoration_mgr()->AddDecorationVal(
                        arrayTypeId, static_cast<uint32_t>(spv::Decoration::ArrayStride), kWordBytes);
                    return arrayTypeId;
                }

                // The access qualifiers the collapsed member has to keep. Coherent and Volatile
                // are taken from ANY member that had them - dropping one could lose a write
                // another invocation has to see - while Restrict, NonWritable and NonReadable are
                // taken only from ALL of them, because each is a promise the shader would break
                // if one member never made it.
                std::vector<spv::Decoration> SurvivingAccessQualifiers(IRContext* context, uint32_t structId,
                                                                      uint32_t memberCount) {
                    static constexpr spv::Decoration kAny[] = {spv::Decoration::Coherent,
                                                               spv::Decoration::Volatile};
                    static constexpr spv::Decoration kAll[] = {spv::Decoration::Restrict,
                                                              spv::Decoration::NonWritable,
                                                              spv::Decoration::NonReadable};
                    std::vector<spv::Decoration> surviving;
                    for (const spv::Decoration kind : kAny) {
                        for (uint32_t member = 0; member < memberCount; ++member) {
                            if (HasMemberDecoration(context, structId, member, kind)) {
                                surviving.push_back(kind);
                                break;
                            }
                        }
                    }
                    for (const spv::Decoration kind : kAll) {
                        bool all = memberCount > 0;
                        for (uint32_t member = 0; member < memberCount && all; ++member) {
                            all = HasMemberDecoration(context, structId, member, kind);
                        }
                        if (all) surviving.push_back(kind);
                    }
                    return surviving;
                }

                // Drops every OpMemberDecorate and OpMemberName the collapsed struct no longer has
                // a member for - which is all of them, since the one member it keeps is a
                // different thing entirely from the one that used to be member 0.
                void StripMemberAnnotations(IRContext* context, uint32_t structId) {
                    std::vector<Instruction*> doomed;
                    for (Instruction* decoration :
                         context->get_decoration_mgr()->GetDecorationsFor(structId, false)) {
                        if (decoration->opcode() == spv::Op::OpMemberDecorate) doomed.push_back(decoration);
                    }
                    for (Instruction& debug : context->module()->debugs2()) {
                        if (debug.opcode() != spv::Op::OpMemberName || debug.NumInOperands() < 2) continue;
                        if (debug.GetSingleWordInOperand(0) != structId) continue;
                        doomed.push_back(&debug);
                    }
                    for (Instruction* inst : doomed) context->KillInst(inst);
                }

                // Plans every shader storage block in the module that holds a 64-bit float and
                // that this pass can rewrite exactly. Reads the module and never writes it.
                std::vector<BlockPlan> BuildPlans(IRContext* context) {
                    std::vector<BlockPlan> plans;
                    auto* defUseMgr = context->get_def_use_mgr();

                    // Declaration order, so a module with two candidate blocks is rewritten the
                    // same way every time it is compiled.
                    std::vector<uint32_t> structOrder;
                    std::unordered_map<uint32_t, std::vector<Instruction*>> variablesByStruct;
                    std::unordered_map<uint32_t, uint32_t> storageClassByStruct;
                    for (Instruction& inst : context->module()->types_values()) {
                        if (inst.opcode() != spv::Op::OpVariable || inst.NumInOperands() < 1) continue;
                        const uint32_t storageClass = inst.GetSingleWordInOperand(0);
                        const bool isStorageBufferClass =
                            storageClass == static_cast<uint32_t>(spv::StorageClass::StorageBuffer);
                        const bool isUniformClass =
                            storageClass == static_cast<uint32_t>(spv::StorageClass::Uniform);
                        if (!isStorageBufferClass && !isUniformClass) continue;

                        Instruction* pointerType = defUseMgr->GetDef(inst.type_id());
                        if (pointerType == nullptr || pointerType->opcode() != spv::Op::OpTypePointer) continue;
                        const uint32_t structId = pointerType->GetSingleWordInOperand(1);
                        Instruction* structType = defUseMgr->GetDef(structId);
                        if (structType == nullptr || structType->opcode() != spv::Op::OpTypeStruct ||
                            structType->NumInOperands() == 0) {
                            continue;
                        }
                        // A shader storage block is spelled Block + StorageBuffer from SPIR-V 1.3
                        // and BufferBlock + Uniform before it; a plain UNIFORM block is neither,
                        // and is deliberately left to the demotion - the frontend's own uniform
                        // routing reflects the module that pass produces.
                        const bool isStorageBlock =
                            (isStorageBufferClass && HasDecoration(context, structId, spv::Decoration::Block)) ||
                            (isUniformClass && HasDecoration(context, structId, spv::Decoration::BufferBlock));
                        if (!isStorageBlock) continue;

                        std::unordered_set<uint32_t> visiting;
                        if (!TypeContainsFloat64(context, structId, visiting)) continue;

                        if (variablesByStruct.find(structId) == variablesByStruct.end()) {
                            structOrder.push_back(structId);
                            storageClassByStruct[structId] = storageClass;
                        }
                        variablesByStruct[structId].push_back(&inst);
                    }

                    // A struct type is not owned by the variables that happen to be storage blocks:
                    // if ANY other variable points at the same one, collapsing it would rewrite a
                    // declaration this pass never looked at. Count every variable of every struct
                    // and require the two counts to agree.
                    std::unordered_map<uint32_t, uint32_t> variableCountByStruct;
                    for (Instruction& inst : context->module()->types_values()) {
                        if (inst.opcode() != spv::Op::OpVariable) continue;
                        Instruction* pointerType = defUseMgr->GetDef(inst.type_id());
                        if (pointerType == nullptr || pointerType->opcode() != spv::Op::OpTypePointer) continue;
                        ++variableCountByStruct[pointerType->GetSingleWordInOperand(1)];
                    }
                    for (auto it = structOrder.begin(); it != structOrder.end();) {
                        if (variableCountByStruct[*it] == variablesByStruct[*it].size()) {
                            ++it;
                            continue;
                        }
                        MGLOG_D("[spirv] storage block %%%u holds a double but its struct type is shared "
                                "with a declaration that is not one; left to the fp64 demotion",
                                *it);
                        it = structOrder.erase(it);
                    }

                    for (const uint32_t structId : structOrder) {
                        Instruction* structType = defUseMgr->GetDef(structId);
                        TypeCursor blockCursor;
                        blockCursor.typeId = structId;
                        const uint32_t blockBytes = LaidOutByteSize(context, blockCursor);
                        if (blockBytes == 0 || blockBytes % kWordBytes != 0) {
                            MGLOG_D("[spirv] storage block %%%u holds a double but its byte layout cannot be "
                                    "described exactly; left to the fp64 demotion",
                                    structId);
                            continue;
                        }

                        BlockPlan plan;
                        plan.structType = structType;
                        plan.storageClass = storageClassByStruct[structId];
                        plan.wordCount = blockBytes / kWordBytes;

                        bool expressible = true;
                        for (Instruction* variable : variablesByStruct[structId]) {
                            std::vector<Instruction*> chains;
                            std::unordered_set<uint32_t> seenChains;
                            defUseMgr->ForEachUser(variable, [&](Instruction* user) {
                                if (!expressible) return;
                                switch (user->opcode()) {
                                case spv::Op::OpName:
                                case spv::Op::OpDecorate:
                                case spv::Op::OpDecorateId:
                                case spv::Op::OpEntryPoint:
                                    return;
                                case spv::Op::OpAccessChain:
                                case spv::Op::OpInBoundsAccessChain:
                                    if (user->NumInOperands() >= 1 &&
                                        user->GetSingleWordInOperand(0) == variable->result_id()) {
                                        if (seenChains.insert(user->result_id()).second) chains.push_back(user);
                                        return;
                                    }
                                    expressible = false;
                                    return;
                                default:
                                    expressible = false;
                                    return;
                                }
                            });
                            if (!expressible) break;

                            for (Instruction* chain : chains) {
                                ChainPlan chainPlan;
                                chainPlan.chain = chain;
                                chainPlan.variableId = variable->result_id();
                                if (!PlanChain(context, chain, blockCursor, &chainPlan)) {
                                    expressible = false;
                                    break;
                                }
                                uint32_t leafCount = 0;
                                if (!CanDecompose(context, chainPlan.pointee, &leafCount)) {
                                    expressible = false;
                                    break;
                                }
                                std::unordered_set<uint32_t> seenUses;
                                defUseMgr->ForEachUser(chain, [&](Instruction* user) {
                                    if (!expressible) return;
                                    if (user->opcode() == spv::Op::OpLoad &&
                                        user->GetSingleWordInOperand(0) == chain->result_id()) {
                                        if (seenUses.insert(user->unique_id()).second) {
                                            chainPlan.loads.push_back(user);
                                        }
                                        return;
                                    }
                                    if (user->opcode() == spv::Op::OpStore && user->NumInOperands() >= 2 &&
                                        user->GetSingleWordInOperand(0) == chain->result_id() &&
                                        user->GetSingleWordInOperand(1) != chain->result_id()) {
                                        if (seenUses.insert(user->unique_id()).second) {
                                            chainPlan.stores.push_back(user);
                                        }
                                        return;
                                    }
                                    expressible = false;
                                });
                                if (!expressible) break;
                                plan.chains.push_back(std::move(chainPlan));
                            }
                            if (!expressible) break;
                        }
                        if (!expressible) {
                            MGLOG_D("[spirv] storage block %%%u holds a double but is reached in a way this "
                                    "pass cannot re-address; left to the fp64 demotion",
                                    structId);
                            continue;
                        }
                        plans.push_back(std::move(plan));
                    }
                    return plans;
                }
            } // namespace

            spvtools::opt::Pass::Status FlattenFloat64StorageBlockPass::Process() {
                auto* irContext = context();
                std::vector<BlockPlan> plans = BuildPlans(irContext);
                if (plans.empty()) {
                    return Status::SuccessWithoutChange;
                }

                spvtools::opt::analysis::Integer uintDescriptor(32, false);
                spvtools::opt::analysis::Bool boolDescriptor;
                spvtools::opt::analysis::Float floatDescriptor(32);
                const uint32_t uintTypeId = irContext->get_type_mgr()->GetTypeInstruction(&uintDescriptor);
                const uint32_t boolTypeId = irContext->get_type_mgr()->GetTypeInstruction(&boolDescriptor);
                const uint32_t floatTypeId = irContext->get_type_mgr()->GetTypeInstruction(&floatDescriptor);
                if (uintTypeId == 0 || boolTypeId == 0 || floatTypeId == 0) {
                    return Status::SuccessWithoutChange;
                }

                Emitter emitter(irContext, uintTypeId, boolTypeId, floatTypeId);
                bool modified = false;
                for (BlockPlan& plan : plans) {
                    const uint32_t structId = plan.structType->result_id();
                    const uint32_t arrayTypeId =
                        CreateWordArrayTypeBefore(irContext, plan.structType, uintTypeId, plan.wordCount);
                    if (arrayTypeId == 0) {
                        MGLOG_D("[spirv] storage block %%%u: no legal place for the flattened word array; "
                                "left to the fp64 demotion",
                                structId);
                        continue;
                    }
                    const uint32_t wordPointerTypeId = irContext->get_type_mgr()->FindPointerToType(
                        uintTypeId, static_cast<spv::StorageClass>(plan.storageClass));
                    if (wordPointerTypeId == 0) continue;
                    const uint32_t memberZeroId = emitter.UintConstant(0);

                    for (ChainPlan& chainPlan : plan.chains) {
                        Emitter::Access access;
                        access.variableId = chainPlan.variableId;
                        access.wordPointerTypeId = wordPointerTypeId;
                        access.memberZeroId = memberZeroId;
                        access.baseWordId = emitter.WordIndexOf(chainPlan);

                        for (Instruction* load : chainPlan.loads) {
                            InstructionBuilder builder(irContext, load, kPreservedAnalyses);
                            const uint32_t rebuilt = emitter.BuildValue(builder, access, chainPlan.pointee, 0);
                            irContext->ReplaceAllUsesWith(load->result_id(), rebuilt);
                            irContext->KillInst(load);
                        }
                        for (Instruction* store : chainPlan.stores) {
                            InstructionBuilder builder(irContext, store, kPreservedAnalyses);
                            std::vector<uint32_t> path;
                            emitter.StoreValue(builder, access, chainPlan.pointee, 0,
                                               store->GetSingleWordInOperand(1), path);
                            irContext->KillInst(store);
                        }
                        irContext->KillInst(chainPlan.chain);
                    }

                    const std::vector<spv::Decoration> surviving = SurvivingAccessQualifiers(
                        irContext, structId, plan.structType->NumInOperands());
                    StripMemberAnnotations(irContext, structId);
                    plan.structType->SetInOperands({{SPV_OPERAND_TYPE_ID, {arrayTypeId}}});
                    irContext->UpdateDefUse(plan.structType);
                    irContext->get_decoration_mgr()->AddMemberDecoration(
                        structId, 0u, static_cast<uint32_t>(spv::Decoration::Offset), 0u);
                    for (const spv::Decoration kind : surviving) {
                        // AddMemberDecoration always carries a literal value; these have none, so
                        // the instruction has to be spelled out.
                        irContext->get_decoration_mgr()->AddDecoration(
                            spv::Op::OpMemberDecorate,
                            {{SPV_OPERAND_TYPE_ID, {structId}},
                             {SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}},
                             {SPV_OPERAND_TYPE_DECORATION, {static_cast<uint32_t>(kind)}}});
                    }
                    modified = true;
                    MGLOG_D("[spirv] storage block %%%u: flattened into %u words so its 64-bit members keep "
                            "the byte layout the application bound",
                            structId, plan.wordCount);
                }

                if (!modified) {
                    return Status::SuccessWithoutChange;
                }
                irContext->InvalidateAnalysesExceptFor(IRContext::kAnalysisNone);
                return Status::SuccessWithChange;
            }

            spvtools::Optimizer::PassToken
            FlattenFloat64StorageBlockPass::CreateFlattenFloat64StorageBlockPass() {
                return spvtools::Optimizer::PassToken(MakeUnique<FlattenFloat64StorageBlockPass>());
            }
        } // namespace ShaderTranspiler
    } // namespace MG_Util
} // namespace MobileGL
