Files
MobileGL/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.cpp
T

1452 lines
86 KiB
C++

// MobileGL - MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.cpp
// Copyright (c) 2026 MobileGL-Dev
// Licensed under the GNU Lesser General Public License v3.0:
// https://www.gnu.org/licenses/gpl-3.0.txt
// https://www.gnu.org/licenses/lgpl-3.0.txt
// SPDX-License-Identifier: LGPL-3.0-only
// End of Source File Header
#include "EmulateSubgroupsPass.h"
#include "spirv.hpp"
#include "source/opt/constants.h"
#include "source/opt/def_use_manager.h"
#include "source/opt/instruction.h"
#include "source/opt/ir_context.h"
#include "source/opt/module.h"
#include "source/opt/type_manager.h"
#include "source/util/make_unique.h"
#include "source/util/string_utils.h"
#include <array>
#include <map>
#include <set>
#include <vector>
namespace MobileGL {
namespace MG_Util {
namespace ShaderTranspiler {
namespace {
using spvtools::opt::Instruction;
using spvtools::opt::IRContext;
using spvtools::opt::Operand;
namespace analysis = spvtools::opt::analysis;
// The GL-visible subgroup this pass implements. Keep in sync with
// SubgroupSupportPolicy.h (kEmulatedSubgroupSize).
constexpr uint32_t kWidth = 32u;
constexpr uint32_t kLaneMask = kWidth - 1u;
constexpr uint32_t kIdShift = 5u;
// GLSL.std.450 instruction numbers (see 3rdparty/glslang/SPIRV/GLSL.std.450.h).
constexpr uint32_t kGlslFMin = 37u;
constexpr uint32_t kGlslUMin = 38u;
constexpr uint32_t kGlslSMin = 39u;
constexpr uint32_t kGlslFMax = 40u;
constexpr uint32_t kGlslUMax = 41u;
constexpr uint32_t kGlslSMax = 42u;
constexpr uint32_t kGlslFindILsb = 73u;
constexpr uint32_t kGlslFindUMsb = 75u;
Operand IdOp(uint32_t id) { return {SPV_OPERAND_TYPE_ID, {id}}; }
Instruction* FindBuiltinDefinition(IRContext* context, spv::BuiltIn builtin) {
auto* defUseMgr = context->get_def_use_mgr();
for (auto& annotation : context->annotations()) {
if (annotation.opcode() != spv::Op::OpDecorate || annotation.NumInOperands() < 3) {
continue;
}
if (static_cast<spv::Decoration>(annotation.GetSingleWordInOperand(1)) !=
spv::Decoration::BuiltIn) {
continue;
}
if (static_cast<spv::BuiltIn>(annotation.GetSingleWordInOperand(2)) != builtin) {
continue;
}
return defUseMgr->GetDef(annotation.GetSingleWordInOperand(0));
}
return nullptr;
}
// The scalar shapes a value participating in an emulated exchange may have.
enum class ScalarKind { Float, SInt, UInt, Bool };
struct ValueTypeInfo {
ScalarKind kind = ScalarKind::UInt;
uint32_t componentCount = 1; // 1 for scalars, 2..4 for vectors
uint32_t typeId = 0; // the value's own type
uint32_t scalarTypeId = 0; // the component type
};
bool ClassifyScalar(const Instruction* type, ScalarKind* kind) {
switch (type->opcode()) {
case spv::Op::OpTypeBool:
*kind = ScalarKind::Bool;
return true;
case spv::Op::OpTypeInt:
if (type->GetSingleWordInOperand(0) != 32u) return false;
*kind = type->GetSingleWordInOperand(1) == 0u ? ScalarKind::UInt : ScalarKind::SInt;
return true;
case spv::Op::OpTypeFloat:
if (type->GetSingleWordInOperand(0) != 32u) return false;
*kind = ScalarKind::Float;
return true;
default:
return false;
}
}
bool ClassifyValueType(IRContext* context, uint32_t typeId, ValueTypeInfo* info) {
const Instruction* type = context->get_def_use_mgr()->GetDef(typeId);
if (type == nullptr) return false;
info->typeId = typeId;
if (type->opcode() == spv::Op::OpTypeVector) {
info->componentCount = type->GetSingleWordInOperand(1);
if (info->componentCount < 2u || info->componentCount > 4u) return false;
info->scalarTypeId = type->GetSingleWordInOperand(0);
const Instruction* component = context->get_def_use_mgr()->GetDef(info->scalarTypeId);
return component != nullptr && ClassifyScalar(component, &info->kind);
}
info->componentCount = 1u;
info->scalarTypeId = typeId;
return ClassifyScalar(type, &info->kind);
}
// Emits instructions immediately before a fixed anchor instruction. Every
// lowering sequence is straight-line code in the anchor's own block, so
// dominance is trivially preserved.
class Emitter {
public:
Emitter(IRContext* context, Instruction* anchor) : m_context(context), m_anchor(anchor) {}
uint32_t Emit(spv::Op opcode, uint32_t typeId, Instruction::OperandList operands) {
const uint32_t resultId = m_context->TakeNextId();
m_anchor->InsertBefore(spvtools::MakeUnique<Instruction>(
m_context, opcode, typeId, resultId, std::move(operands)));
return resultId;
}
void EmitNoResult(spv::Op opcode, Instruction::OperandList operands) {
m_anchor->InsertBefore(spvtools::MakeUnique<Instruction>(
m_context, opcode, 0, 0, std::move(operands)));
}
private:
IRContext* m_context;
Instruction* m_anchor;
};
// One recognized OpGroupNonUniform* site with everything Phase A resolved
// about it, so Phase B can rewrite without def-use queries.
struct GroupOpSite {
Instruction* inst = nullptr;
spv::Op opcode = spv::Op::OpNop;
spv::GroupOperation groupOperation = spv::GroupOperation::Reduce;
uint32_t clusterSize = 0; // ClusteredReduce only
uint32_t quadDirection = 0; // QuadSwap only
ValueTypeInfo valueType{}; // participating value (where applicable)
uint32_t valueId = 0;
uint32_t indexId = 0; // broadcast/shuffle/extract index operand
};
struct BarrierSite {
Instruction* inst = nullptr;
bool executionScopeIsSubgroup = false;
bool memoryScopeIsSubgroup = false;
};
bool IsGroupNonUniformOpcode(spv::Op opcode) {
return opcode >= spv::Op::OpGroupNonUniformElect &&
opcode <= spv::Op::OpGroupNonUniformQuadSwap;
}
} // namespace
spvtools::opt::Pass::Status EmulateSubgroupsPass::Process() {
auto* irContext = context();
auto* defUseMgr = irContext->get_def_use_mgr();
auto* constMgr = irContext->get_constant_mgr();
auto* typeMgr = irContext->get_type_mgr();
// ---------------------------------------------------------------- Phase A
// Pure analysis; nothing is mutated until every site has been vetted, so a
// Failure return leaves the module untouched.
bool allEntryPointsCompute = true;
for (const Instruction& entryPoint : irContext->module()->entry_points()) {
if (static_cast<spv::ExecutionModel>(entryPoint.GetSingleWordInOperand(0)) !=
spv::ExecutionModel::GLCompute) {
allEntryPointsCompute = false;
}
}
const auto resolveUintConstant = [&](uint32_t id, uint32_t* value) {
const Instruction* def = defUseMgr->GetDef(id);
if (def == nullptr || def->opcode() != spv::Op::OpConstant) return false;
*value = def->GetSingleWordInOperand(0);
return true;
};
// Subgroup builtin variables and their loads.
static constexpr std::array<spv::BuiltIn, 9> kSubgroupBuiltins = {
spv::BuiltIn::SubgroupSize, spv::BuiltIn::SubgroupLocalInvocationId,
spv::BuiltIn::SubgroupId, spv::BuiltIn::NumSubgroups,
spv::BuiltIn::SubgroupEqMask, spv::BuiltIn::SubgroupGeMask,
spv::BuiltIn::SubgroupGtMask, spv::BuiltIn::SubgroupLeMask,
spv::BuiltIn::SubgroupLtMask};
const auto isMaskBuiltin = [](spv::BuiltIn builtin) {
return builtin == spv::BuiltIn::SubgroupEqMask ||
builtin == spv::BuiltIn::SubgroupGeMask ||
builtin == spv::BuiltIn::SubgroupGtMask ||
builtin == spv::BuiltIn::SubgroupLeMask ||
builtin == spv::BuiltIn::SubgroupLtMask;
};
struct BuiltinUse {
spv::BuiltIn builtin;
Instruction* variable;
std::vector<Instruction*> loads;
// Mask builtins are uvec4; glslang reaches a single component (the
// ubiquitous gl_Subgroup*Mask.x) through an access chain plus a scalar
// load. The chain dies with the variable once its loads are rewritten.
std::vector<Instruction*> accessChains;
std::vector<std::pair<Instruction*, uint32_t>> componentLoads;
};
std::vector<BuiltinUse> builtinUses;
bool sawUnexpectedUser = false;
for (const spv::BuiltIn builtin : kSubgroupBuiltins) {
Instruction* variable = FindBuiltinDefinition(irContext, builtin);
if (variable == nullptr || variable->opcode() != spv::Op::OpVariable) continue;
BuiltinUse use{builtin, variable, {}, {}, {}};
const uint32_t variableId = variable->result_id();
defUseMgr->ForEachUser(variable, [&](Instruction* user) {
switch (user->opcode()) {
case spv::Op::OpLoad:
if (user->NumInOperands() >= 1 &&
user->GetSingleWordInOperand(0) == variableId) {
use.loads.push_back(user);
} else {
sawUnexpectedUser = true;
}
return;
case spv::Op::OpAccessChain:
case spv::Op::OpInBoundsAccessChain: {
uint32_t component = 0;
if (!isMaskBuiltin(builtin) || user->NumInOperands() != 2 ||
user->GetSingleWordInOperand(0) != variableId) {
sawUnexpectedUser = true;
return;
}
const Instruction* index =
defUseMgr->GetDef(user->GetSingleWordInOperand(1));
if (index == nullptr || index->opcode() != spv::Op::OpConstant ||
(component = index->GetSingleWordInOperand(0)) > 3u) {
sawUnexpectedUser = true;
return;
}
const uint32_t chainId = user->result_id();
defUseMgr->ForEachUser(user, [&](Instruction* chainUser) {
if (chainUser->opcode() == spv::Op::OpLoad &&
chainUser->NumInOperands() >= 1 &&
chainUser->GetSingleWordInOperand(0) == chainId) {
use.componentLoads.emplace_back(chainUser, component);
} else {
sawUnexpectedUser = true;
}
});
use.accessChains.push_back(user);
return;
}
case spv::Op::OpDecorate:
case spv::Op::OpDecorateId:
case spv::Op::OpDecorateString:
case spv::Op::OpName:
case spv::Op::OpEntryPoint:
return;
default:
sawUnexpectedUser = true;
return;
}
});
builtinUses.push_back(std::move(use));
}
// Group-nonuniform operations and subgroup-scoped barriers.
std::vector<GroupOpSite> groupOps;
std::vector<BarrierSite> barriers;
bool sawUnsupported = false;
for (auto& function : *irContext->module()) {
for (auto& block : function) {
for (auto& inst : block) {
const spv::Op opcode = inst.opcode();
// Extended subgroup instructions live outside the core
// [Elect..QuadSwap] range IsGroupNonUniformOpcode covers; they
// must fail the pass rather than survive into the "subgroup
// free" output.
if (opcode == spv::Op::OpGroupNonUniformPartitionNV ||
opcode == spv::Op::OpGroupNonUniformRotateKHR ||
opcode == spv::Op::OpGroupNonUniformQuadAllKHR ||
opcode == spv::Op::OpGroupNonUniformQuadAnyKHR) {
sawUnsupported = true;
continue;
}
if (opcode == spv::Op::OpControlBarrier) {
BarrierSite site{&inst, false, false};
uint32_t scope = 0;
if (resolveUintConstant(inst.GetSingleWordInOperand(0), &scope) &&
scope == static_cast<uint32_t>(spv::Scope::Subgroup)) {
site.executionScopeIsSubgroup = true;
}
if (resolveUintConstant(inst.GetSingleWordInOperand(1), &scope) &&
scope == static_cast<uint32_t>(spv::Scope::Subgroup)) {
site.memoryScopeIsSubgroup = true;
}
if (site.executionScopeIsSubgroup || site.memoryScopeIsSubgroup) {
barriers.push_back(site);
}
continue;
}
if (opcode == spv::Op::OpMemoryBarrier) {
uint32_t scope = 0;
if (resolveUintConstant(inst.GetSingleWordInOperand(0), &scope) &&
scope == static_cast<uint32_t>(spv::Scope::Subgroup)) {
barriers.push_back(BarrierSite{&inst, false, true});
}
continue;
}
if (!IsGroupNonUniformOpcode(opcode)) continue;
GroupOpSite site;
site.inst = &inst;
site.opcode = opcode;
uint32_t scope = 0;
if (!resolveUintConstant(inst.GetSingleWordInOperand(0), &scope) ||
scope != static_cast<uint32_t>(spv::Scope::Subgroup)) {
sawUnsupported = true;
continue;
}
switch (opcode) {
case spv::Op::OpGroupNonUniformElect:
break;
case spv::Op::OpGroupNonUniformAll:
case spv::Op::OpGroupNonUniformAny:
case spv::Op::OpGroupNonUniformAllEqual:
case spv::Op::OpGroupNonUniformBroadcastFirst:
case spv::Op::OpGroupNonUniformBallot:
case spv::Op::OpGroupNonUniformInverseBallot:
case spv::Op::OpGroupNonUniformBallotFindLSB:
case spv::Op::OpGroupNonUniformBallotFindMSB:
site.valueId = inst.GetSingleWordInOperand(1);
break;
case spv::Op::OpGroupNonUniformBroadcast:
case spv::Op::OpGroupNonUniformShuffle:
case spv::Op::OpGroupNonUniformShuffleXor:
case spv::Op::OpGroupNonUniformShuffleUp:
case spv::Op::OpGroupNonUniformShuffleDown:
case spv::Op::OpGroupNonUniformQuadBroadcast:
case spv::Op::OpGroupNonUniformBallotBitExtract:
site.valueId = inst.GetSingleWordInOperand(1);
site.indexId = inst.GetSingleWordInOperand(2);
break;
case spv::Op::OpGroupNonUniformQuadSwap:
site.valueId = inst.GetSingleWordInOperand(1);
if (!resolveUintConstant(inst.GetSingleWordInOperand(2),
&site.quadDirection) ||
site.quadDirection > 2u) {
sawUnsupported = true;
}
break;
case spv::Op::OpGroupNonUniformBallotBitCount:
site.groupOperation = static_cast<spv::GroupOperation>(
inst.GetSingleWordInOperand(1));
site.valueId = inst.GetSingleWordInOperand(2);
break;
case spv::Op::OpGroupNonUniformIAdd:
case spv::Op::OpGroupNonUniformFAdd:
case spv::Op::OpGroupNonUniformIMul:
case spv::Op::OpGroupNonUniformFMul:
case spv::Op::OpGroupNonUniformSMin:
case spv::Op::OpGroupNonUniformUMin:
case spv::Op::OpGroupNonUniformFMin:
case spv::Op::OpGroupNonUniformSMax:
case spv::Op::OpGroupNonUniformUMax:
case spv::Op::OpGroupNonUniformFMax:
case spv::Op::OpGroupNonUniformBitwiseAnd:
case spv::Op::OpGroupNonUniformBitwiseOr:
case spv::Op::OpGroupNonUniformBitwiseXor:
case spv::Op::OpGroupNonUniformLogicalAnd:
case spv::Op::OpGroupNonUniformLogicalOr:
case spv::Op::OpGroupNonUniformLogicalXor:
site.groupOperation = static_cast<spv::GroupOperation>(
inst.GetSingleWordInOperand(1));
site.valueId = inst.GetSingleWordInOperand(2);
if (site.groupOperation == spv::GroupOperation::ClusteredReduce) {
if (inst.NumInOperands() < 4 ||
!resolveUintConstant(inst.GetSingleWordInOperand(3),
&site.clusterSize) ||
site.clusterSize == 0u ||
(site.clusterSize & (site.clusterSize - 1u)) != 0u) {
sawUnsupported = true;
}
} else if (site.groupOperation != spv::GroupOperation::Reduce &&
site.groupOperation != spv::GroupOperation::InclusiveScan &&
site.groupOperation != spv::GroupOperation::ExclusiveScan) {
sawUnsupported = true;
}
break;
default:
sawUnsupported = true;
continue;
}
// Classify the participating value where the lowering exchanges
// it through shared memory (everything except Elect and the pure
// ballot-math ops, whose operand is the uvec4 ballot itself).
switch (opcode) {
case spv::Op::OpGroupNonUniformElect:
case spv::Op::OpGroupNonUniformInverseBallot:
case spv::Op::OpGroupNonUniformBallotBitExtract:
case spv::Op::OpGroupNonUniformBallotBitCount:
case spv::Op::OpGroupNonUniformBallotFindLSB:
case spv::Op::OpGroupNonUniformBallotFindMSB:
break;
case spv::Op::OpGroupNonUniformAll:
case spv::Op::OpGroupNonUniformAny:
case spv::Op::OpGroupNonUniformBallot: {
const Instruction* value = defUseMgr->GetDef(site.valueId);
if (value == nullptr ||
!ClassifyValueType(irContext, value->type_id(), &site.valueType) ||
site.valueType.kind != ScalarKind::Bool ||
site.valueType.componentCount != 1u) {
sawUnsupported = true;
}
break;
}
default: {
const Instruction* value = defUseMgr->GetDef(site.valueId);
if (value == nullptr ||
!ClassifyValueType(irContext, value->type_id(), &site.valueType)) {
sawUnsupported = true;
}
break;
}
}
groupOps.push_back(site);
}
}
}
bool anyBuiltinLoads = false;
for (const BuiltinUse& use : builtinUses) {
if (!use.loads.empty() || !use.componentLoads.empty()) anyBuiltinLoads = true;
}
if (!anyBuiltinLoads && groupOps.empty() && barriers.empty() && !sawUnsupported) {
return Status::SuccessWithoutChange;
}
if (!allEntryPointsCompute || sawUnexpectedUser || sawUnsupported) {
return Status::Failure;
}
// Workgroup size. The compile chain pins SPIR-V 1.3, where a literal
// local_size always reaches the module as OpExecutionMode LocalSize and -
// through glslang - as the WorkgroupSize builtin constant as well. Either
// is accepted; spec-constant sizes are not (GL has no specialization).
uint32_t localSize[3] = {0, 0, 0};
bool haveLocalSize = false;
if (Instruction* workgroupSize =
FindBuiltinDefinition(irContext, spv::BuiltIn::WorkgroupSize)) {
if (workgroupSize->opcode() == spv::Op::OpConstantComposite &&
workgroupSize->NumInOperands() == 3) {
haveLocalSize =
resolveUintConstant(workgroupSize->GetSingleWordInOperand(0), &localSize[0]) &&
resolveUintConstant(workgroupSize->GetSingleWordInOperand(1), &localSize[1]) &&
resolveUintConstant(workgroupSize->GetSingleWordInOperand(2), &localSize[2]);
}
}
if (!haveLocalSize) {
for (const Instruction& mode : irContext->module()->execution_modes()) {
if (mode.opcode() == spv::Op::OpExecutionMode &&
static_cast<spv::ExecutionMode>(mode.GetSingleWordInOperand(1)) ==
spv::ExecutionMode::LocalSize) {
localSize[0] = mode.GetSingleWordInOperand(2);
localSize[1] = mode.GetSingleWordInOperand(3);
localSize[2] = mode.GetSingleWordInOperand(4);
haveLocalSize = true;
break;
}
}
}
if (!haveLocalSize || localSize[0] == 0u || localSize[1] == 0u || localSize[2] == 0u) {
return Status::Failure;
}
const uint64_t total64 =
static_cast<uint64_t>(localSize[0]) * localSize[1] * localSize[2];
if (total64 == 0u || total64 > (1u << 20)) {
return Status::Failure;
}
const uint32_t totalInvocations = static_cast<uint32_t>(total64);
const uint32_t virtualSubgroups = (totalInvocations + kWidth - 1u) / kWidth;
const uint32_t paddedSlots = virtualSubgroups * kWidth;
// Phase C creates one Workgroup array of paddedSlots elements per distinct
// participating element type (bools travel as uint words), 4 bytes per
// component. Refuse up front any module whose added scratch would exceed
// the caller's budget - the device's maxComputeSharedMemorySize, or the
// 16384-byte Vulkan minimum when the caller passed 0 - since spirv-val
// cannot catch this and the pipeline would fail at creation instead. The
// estimate over-counts slightly (a bool type and its uint twin are keyed
// separately) and does not subtract the module's own shared declarations.
{
const uint64_t scratchBudgetBytes =
m_maxWorkgroupScratchBytes != 0u ? m_maxWorkgroupScratchBytes : 16384u;
std::set<uint32_t> scratchTypeKeys;
uint64_t scratchBytes = 0;
for (const GroupOpSite& site : groupOps) {
if (site.valueType.typeId == 0u) continue; // no shared-memory exchange
const uint32_t key = site.valueType.kind == ScalarKind::Bool
? (0x80000000u | site.valueType.componentCount)
: site.valueType.typeId;
if (!scratchTypeKeys.insert(key).second) continue;
scratchBytes += static_cast<uint64_t>(paddedSlots) * 4u *
site.valueType.componentCount;
}
if (scratchBytes > scratchBudgetBytes) {
return Status::Failure;
}
}
// ---------------------------------------------------------------- Phase B
// Cached types and constants.
analysis::Bool boolTypeCandidate;
const uint32_t boolTypeId = typeMgr->GetTypeInstruction(&boolTypeCandidate);
analysis::Integer uintTypeCandidate(32, false);
const uint32_t uintTypeId = typeMgr->GetTypeInstruction(&uintTypeCandidate);
if (boolTypeId == 0u || uintTypeId == 0u) {
return Status::Failure;
}
const auto uintConst = [&](uint32_t value) -> uint32_t {
const analysis::Type* type = typeMgr->GetType(uintTypeId);
const analysis::Constant* constant = constMgr->GetConstant(type, {value});
const Instruction* inst =
constant != nullptr ? constMgr->GetDefiningInstruction(constant, uintTypeId)
: nullptr;
return inst != nullptr ? inst->result_id() : 0u;
};
const auto boolConst = [&](bool value) -> uint32_t {
const analysis::Type* type = typeMgr->GetType(boolTypeId);
const analysis::Constant* constant =
constMgr->GetConstant(type, {value ? 1u : 0u});
const Instruction* inst =
constant != nullptr ? constMgr->GetDefiningInstruction(constant, boolTypeId)
: nullptr;
return inst != nullptr ? inst->result_id() : 0u;
};
// A constant of an arbitrary 32-bit scalar type from its bit pattern, splat
// to the value type when it is a vector.
const auto typedConst = [&](const ValueTypeInfo& info, uint32_t scalarBits,
uint32_t scratchScalarTypeId,
uint32_t scratchTypeId) -> uint32_t {
const analysis::Type* scalarType = typeMgr->GetType(scratchScalarTypeId);
const analysis::Constant* scalar = constMgr->GetConstant(scalarType, {scalarBits});
if (scalar == nullptr) return 0u;
const Instruction* scalarInst =
constMgr->GetDefiningInstruction(scalar, scratchScalarTypeId);
if (scalarInst == nullptr) return 0u;
if (info.componentCount == 1u) return scalarInst->result_id();
const analysis::Type* vectorType = typeMgr->GetType(scratchTypeId);
std::vector<uint32_t> componentIds(info.componentCount,
scalarInst->result_id());
const analysis::Constant* vector = constMgr->GetConstant(vectorType, componentIds);
if (vector == nullptr) return 0u;
const Instruction* vectorInst =
constMgr->GetDefiningInstruction(vector, scratchTypeId);
return vectorInst != nullptr ? vectorInst->result_id() : 0u;
};
const uint32_t c0 = uintConst(0u);
const uint32_t c1 = uintConst(1u);
const uint32_t c31 = uintConst(kLaneMask);
const uint32_t c32 = uintConst(kWidth);
const uint32_t cNotLane = uintConst(~kLaneMask);
const uint32_t cTotal = uintConst(totalInvocations);
const uint32_t cAllOnes = uintConst(0xffffffffu);
const uint32_t cTrue = boolConst(true);
if (c0 == 0u || c1 == 0u || c31 == 0u || c32 == 0u || cNotLane == 0u ||
cTotal == 0u || cAllOnes == 0u || cTrue == 0u) {
return Status::Failure;
}
// GLSL.std.450 import (for min/max combines and ballot find ops).
uint32_t glslStd450Id = 0;
for (const Instruction& import : irContext->module()->ext_inst_imports()) {
const std::string importName =
spvtools::utils::MakeString(import.GetInOperand(0).words);
if (importName == "GLSL.std.450") {
glslStd450Id = import.result_id();
break;
}
}
const auto ensureGlslStd450 = [&]() -> uint32_t {
if (glslStd450Id != 0u) return glslStd450Id;
glslStd450Id = irContext->TakeNextId();
irContext->module()->AddExtInstImport(spvtools::MakeUnique<Instruction>(
irContext, spv::Op::OpExtInstImport, 0, glslStd450Id,
Instruction::OperandList{
{SPV_OPERAND_TYPE_LITERAL_STRING,
spvtools::utils::MakeVector("GLSL.std.450")}}));
return glslStd450Id;
};
// gl_LocalInvocationIndex drives the whole virtual topology; synthesize the
// builtin when the module never declared it.
Instruction* liiVariable =
FindBuiltinDefinition(irContext, spv::BuiltIn::LocalInvocationIndex);
uint32_t liiVariableId = 0;
if (liiVariable != nullptr && liiVariable->opcode() == spv::Op::OpVariable) {
liiVariableId = liiVariable->result_id();
} else {
const uint32_t pointerTypeId =
typeMgr->FindPointerToType(uintTypeId, spv::StorageClass::Input);
if (pointerTypeId == 0u) return Status::Failure;
liiVariableId = irContext->TakeNextId();
irContext->AddGlobalValue(spvtools::MakeUnique<Instruction>(
irContext, spv::Op::OpVariable, pointerTypeId, liiVariableId,
Instruction::OperandList{
{SPV_OPERAND_TYPE_STORAGE_CLASS,
{static_cast<uint32_t>(spv::StorageClass::Input)}}}));
irContext->AddAnnotationInst(spvtools::MakeUnique<Instruction>(
irContext, spv::Op::OpDecorate, 0, 0,
Instruction::OperandList{
IdOp(liiVariableId),
{SPV_OPERAND_TYPE_DECORATION,
{static_cast<uint32_t>(spv::Decoration::BuiltIn)}},
{SPV_OPERAND_TYPE_LITERAL_INTEGER,
{static_cast<uint32_t>(spv::BuiltIn::LocalInvocationIndex)}}}));
for (Instruction& entryPoint : irContext->module()->entry_points()) {
entryPoint.AddOperand(IdOp(liiVariableId));
}
}
// Scratch arrays, one per participating element type, each padded to whole
// virtual subgroups so every guarded read stays in bounds.
struct Scratch {
uint32_t variableId = 0;
uint32_t elementTypeId = 0;
uint32_t pointerTypeId = 0;
};
std::map<uint32_t, Scratch> scratchByType;
const uint32_t cPadded = uintConst(paddedSlots);
if (cPadded == 0u) return Status::Failure;
// `elementTypeId` is the scratch element type; with atomic scratch reads its
// array is instead a flat run of uint words, `wordsPerSlot` per slot (an atomic
// load needs a scalar integer pointer). Still one array per element type, so
// the types never share slots - the same aliasing as without atomic reads.
const auto scratchFor = [&](uint32_t elementTypeId, uint32_t wordsPerSlot) -> const Scratch* {
const uint32_t key = elementTypeId;
if (m_atomicScratchReads) elementTypeId = uintTypeId;
auto it = scratchByType.find(key);
if (it != scratchByType.end()) return &it->second;
const uint32_t lengthId =
m_atomicScratchReads ? uintConst(paddedSlots * wordsPerSlot) : cPadded;
if (lengthId == 0u) return nullptr;
const uint32_t arrayTypeId = irContext->TakeNextId();
irContext->AddType(spvtools::MakeUnique<Instruction>(
irContext, spv::Op::OpTypeArray, 0, arrayTypeId,
Instruction::OperandList{IdOp(elementTypeId), IdOp(lengthId)}));
const uint32_t arrayPointerTypeId =
typeMgr->FindPointerToType(arrayTypeId, spv::StorageClass::Workgroup);
const uint32_t elementPointerTypeId =
typeMgr->FindPointerToType(elementTypeId, spv::StorageClass::Workgroup);
if (arrayPointerTypeId == 0u || elementPointerTypeId == 0u) return nullptr;
const uint32_t variableId = irContext->TakeNextId();
irContext->AddGlobalValue(spvtools::MakeUnique<Instruction>(
irContext, spv::Op::OpVariable, arrayPointerTypeId, variableId,
Instruction::OperandList{
{SPV_OPERAND_TYPE_STORAGE_CLASS,
{static_cast<uint32_t>(spv::StorageClass::Workgroup)}}}));
Scratch scratch{variableId, elementTypeId, elementPointerTypeId};
return &scratchByType.emplace(key, scratch).first->second;
};
// The uint-domain twin of a participating type: bools travel as 0/1 words.
struct ScratchTypeInfo {
uint32_t typeId = 0; // element type stored in scratch
uint32_t scalarTypeId = 0; // its component type
bool boolConverted = false;
};
const auto scratchTypeFor = [&](const ValueTypeInfo& info) -> ScratchTypeInfo {
if (info.kind != ScalarKind::Bool) {
return {info.typeId, info.scalarTypeId, false};
}
if (info.componentCount == 1u) {
return {uintTypeId, uintTypeId, true};
}
const analysis::Type* registeredUint = typeMgr->GetType(uintTypeId);
analysis::Vector vectorType(registeredUint, info.componentCount);
const uint32_t vectorTypeId = typeMgr->GetTypeInstruction(&vectorType);
return {vectorTypeId, uintTypeId, true};
};
const auto boolVectorTypeId = [&](uint32_t componentCount) -> uint32_t {
if (componentCount == 1u) return boolTypeId;
const analysis::Type* registeredBool = typeMgr->GetType(boolTypeId);
analysis::Vector vectorType(registeredBool, componentCount);
return typeMgr->GetTypeInstruction(&vectorType);
};
// Rewrite the builtin loads. Every rewrite keeps the original result id, so
// downstream uses see the derived value with no further surgery.
const uint32_t numVirtualConstId = uintConst(virtualSubgroups);
if (numVirtualConstId == 0u) return Status::Failure;
// The x word of one of the five ballot-mask builtins for the calling
// invocation's virtual subgroup (words y..w are zero at 32 lanes). Defined
// over gl_SubgroupSize bits, NOT clipped to the lanes a partial last subgroup
// actually has: the masks name lane positions, and GL/Vulkan define them for
// every position below the subgroup size.
const auto emitMaskWord = [&](Emitter& em, spv::BuiltIn builtin) -> uint32_t {
const uint32_t index =
em.Emit(spv::Op::OpLoad, uintTypeId, {IdOp(liiVariableId)});
const uint32_t lane =
em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(index), IdOp(c31)});
const auto lowMaskThroughLane = [&](bool inclusive) -> uint32_t {
if (!inclusive) {
const uint32_t bit = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId,
{IdOp(c1), IdOp(lane)});
return em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(bit), IdOp(c1)});
}
const uint32_t lanePlusOne =
em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(lane), IdOp(c1)});
const uint32_t atTop = em.Emit(spv::Op::OpUGreaterThanEqual, boolTypeId,
{IdOp(lanePlusOne), IdOp(c32)});
const uint32_t shifted = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId,
{IdOp(c1), IdOp(lanePlusOne)});
const uint32_t below =
em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(shifted), IdOp(c1)});
return em.Emit(spv::Op::OpSelect, uintTypeId,
{IdOp(atTop), IdOp(cAllOnes), IdOp(below)});
};
uint32_t word = 0;
switch (builtin) {
case spv::BuiltIn::SubgroupEqMask:
word = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId,
{IdOp(c1), IdOp(lane)});
break;
case spv::BuiltIn::SubgroupLtMask:
word = lowMaskThroughLane(false);
break;
case spv::BuiltIn::SubgroupLeMask:
word = lowMaskThroughLane(true);
break;
case spv::BuiltIn::SubgroupGeMask:
word = em.Emit(spv::Op::OpNot, uintTypeId, {IdOp(lowMaskThroughLane(false))});
break;
case spv::BuiltIn::SubgroupGtMask:
word = em.Emit(spv::Op::OpNot, uintTypeId, {IdOp(lowMaskThroughLane(true))});
break;
default:
return 0u;
}
return word;
};
for (const BuiltinUse& use : builtinUses) {
for (Instruction* load : use.loads) {
Emitter em(irContext, load);
switch (use.builtin) {
case spv::BuiltIn::SubgroupSize:
load->SetOpcode(spv::Op::OpCopyObject);
load->SetInOperands({IdOp(c32)});
break;
case spv::BuiltIn::NumSubgroups:
load->SetOpcode(spv::Op::OpCopyObject);
load->SetInOperands({IdOp(numVirtualConstId)});
break;
case spv::BuiltIn::SubgroupLocalInvocationId: {
const uint32_t index =
em.Emit(spv::Op::OpLoad, uintTypeId, {IdOp(liiVariableId)});
load->SetOpcode(spv::Op::OpBitwiseAnd);
load->SetInOperands({IdOp(index), IdOp(c31)});
break;
}
case spv::BuiltIn::SubgroupId: {
const uint32_t index =
em.Emit(spv::Op::OpLoad, uintTypeId, {IdOp(liiVariableId)});
load->SetOpcode(spv::Op::OpShiftRightLogical);
load->SetInOperands({IdOp(index), IdOp(uintConst(kIdShift))});
break;
}
default: {
const uint32_t word = emitMaskWord(em, use.builtin);
if (word == 0u) return Status::Failure;
load->SetOpcode(spv::Op::OpCompositeConstruct);
load->SetInOperands(
{IdOp(word), IdOp(c0), IdOp(c0), IdOp(c0)});
break;
}
}
}
for (const auto& [load, component] : use.componentLoads) {
// gl_Subgroup*Mask.x reached through an access chain: only word 0
// carries bits at 32 lanes.
Emitter em(irContext, load);
uint32_t value = c0;
if (component == 0u) {
value = emitMaskWord(em, use.builtin);
if (value == 0u) return Status::Failure;
}
load->SetOpcode(spv::Op::OpCopyObject);
load->SetInOperands({IdOp(value)});
}
}
// Rewrite the group operations.
for (const GroupOpSite& site : groupOps) {
Instruction* inst = site.inst;
Emitter em(irContext, inst);
const uint32_t lii =
em.Emit(spv::Op::OpLoad, uintTypeId, {IdOp(liiVariableId)});
const uint32_t lane =
em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(lii), IdOp(c31)});
const uint32_t base =
em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(lii), IdOp(cNotLane)});
const auto barrier = [&]() {
const uint32_t scopeId = uintConst(static_cast<uint32_t>(spv::Scope::Workgroup));
const uint32_t semanticsId = uintConst(
static_cast<uint32_t>(spv::MemorySemanticsMask::AcquireRelease) |
static_cast<uint32_t>(spv::MemorySemanticsMask::WorkgroupMemory));
em.EmitNoResult(spv::Op::OpControlBarrier,
{IdOp(scopeId), IdOp(scopeId), IdOp(semanticsId)});
};
const ScratchTypeInfo scratchType = scratchTypeFor(site.valueType);
const auto toScratch = [&](uint32_t valueId) -> uint32_t {
if (!scratchType.boolConverted) return valueId;
const uint32_t ones =
typedConst(site.valueType, 1u, uintTypeId, scratchType.typeId);
const uint32_t zeros =
typedConst(site.valueType, 0u, uintTypeId, scratchType.typeId);
return em.Emit(spv::Op::OpSelect, scratchType.typeId,
{IdOp(valueId), IdOp(ones), IdOp(zeros)});
};
const auto fromScratch = [&](uint32_t valueId) -> uint32_t {
if (!scratchType.boolConverted) return valueId;
const uint32_t zeros =
typedConst(site.valueType, 0u, uintTypeId, scratchType.typeId);
return em.Emit(spv::Op::OpINotEqual, site.valueType.typeId,
{IdOp(valueId), IdOp(zeros)});
};
// Atomic scratch reads keep each slot as componentCount uint words; the
// value crosses into that domain by bitcast (a no-op for uint data).
const uint32_t wordsPerSlot = site.valueType.componentCount;
const uint32_t wordTypeId = [&]() -> uint32_t {
if (!m_atomicScratchReads || wordsPerSlot == 1u) return uintTypeId;
const analysis::Type* registeredUint = typeMgr->GetType(uintTypeId);
analysis::Vector vectorType(registeredUint, wordsPerSlot);
return typeMgr->GetTypeInstruction(&vectorType);
}();
const bool wordsNeedBitcast = scratchType.scalarTypeId != uintTypeId;
const auto wordIndex = [&](uint32_t slotId, uint32_t component) -> uint32_t {
if (wordsPerSlot == 1u) return slotId;
const uint32_t scaled = em.Emit(spv::Op::OpIMul, uintTypeId,
{IdOp(slotId), IdOp(uintConst(wordsPerSlot))});
return em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(scaled), IdOp(uintConst(component))});
};
const Scratch* scratch = nullptr;
const auto storeOwnSlot = [&](uint32_t valueId) -> bool {
scratch = scratchFor(scratchType.typeId, wordsPerSlot);
if (scratch == nullptr) return false;
if (!m_atomicScratchReads) {
const uint32_t pointer =
em.Emit(spv::Op::OpAccessChain, scratch->pointerTypeId,
{IdOp(scratch->variableId), IdOp(lii)});
em.EmitNoResult(spv::Op::OpStore, {IdOp(pointer), IdOp(valueId)});
} else {
const uint32_t words =
wordsNeedBitcast ? em.Emit(spv::Op::OpBitcast, wordTypeId, {IdOp(valueId)}) : valueId;
for (uint32_t c = 0; c < wordsPerSlot; ++c) {
const uint32_t word =
wordsPerSlot == 1u
? words
: em.Emit(spv::Op::OpCompositeExtract, uintTypeId,
{IdOp(words), {SPV_OPERAND_TYPE_LITERAL_INTEGER, {c}}});
const uint32_t pointer =
em.Emit(spv::Op::OpAccessChain, scratch->pointerTypeId,
{IdOp(scratch->variableId), IdOp(wordIndex(lii, c))});
em.EmitNoResult(spv::Op::OpStore, {IdOp(pointer), IdOp(word)});
}
}
barrier();
return true;
};
const auto readSlot = [&](uint32_t indexId) -> uint32_t {
if (!m_atomicScratchReads) {
const uint32_t pointer =
em.Emit(spv::Op::OpAccessChain, scratch->pointerTypeId,
{IdOp(scratch->variableId), IdOp(indexId)});
return em.Emit(spv::Op::OpLoad, scratchType.typeId, {IdOp(pointer)});
}
const uint32_t scopeId = uintConst(static_cast<uint32_t>(spv::Scope::Workgroup));
Instruction::OperandList words;
uint32_t packed = 0;
for (uint32_t c = 0; c < wordsPerSlot; ++c) {
const uint32_t pointer =
em.Emit(spv::Op::OpAccessChain, scratch->pointerTypeId,
{IdOp(scratch->variableId), IdOp(wordIndex(indexId, c))});
packed = em.Emit(spv::Op::OpAtomicLoad, uintTypeId,
{IdOp(pointer), IdOp(scopeId), IdOp(c0)});
words.push_back(IdOp(packed));
}
if (wordsPerSlot > 1u) {
packed = em.Emit(spv::Op::OpCompositeConstruct, wordTypeId, std::move(words));
}
return wordsNeedBitcast ? em.Emit(spv::Op::OpBitcast, scratchType.typeId, {IdOp(packed)})
: packed;
};
// A select whose condition is a scalar bool but whose values may be
// vectors; pre-SPIR-V-1.4 needs the condition splat to a bvec.
const auto guardedSelect = [&](uint32_t condId, uint32_t thenId,
uint32_t elseId) -> uint32_t {
const uint32_t componentCount = site.valueType.componentCount;
uint32_t conditionId = condId;
if (componentCount > 1u) {
const uint32_t conditionTypeId = boolVectorTypeId(componentCount);
Instruction::OperandList splat;
for (uint32_t i = 0; i < componentCount; ++i) splat.push_back(IdOp(condId));
conditionId =
em.Emit(spv::Op::OpCompositeConstruct, conditionTypeId, std::move(splat));
}
return em.Emit(spv::Op::OpSelect, scratchType.typeId,
{IdOp(conditionId), IdOp(thenId), IdOp(elseId)});
};
const auto extInst2 = [&](uint32_t typeId, uint32_t instNumber, uint32_t a,
uint32_t b) -> uint32_t {
const uint32_t setId = ensureGlslStd450();
return em.Emit(spv::Op::OpExtInst, typeId,
{IdOp(setId),
{SPV_OPERAND_TYPE_EXTENSION_INSTRUCTION_NUMBER, {instNumber}},
IdOp(a), IdOp(b)});
};
const auto extInst1 = [&](uint32_t typeId, uint32_t instNumber,
uint32_t a) -> uint32_t {
const uint32_t setId = ensureGlslStd450();
return em.Emit(spv::Op::OpExtInst, typeId,
{IdOp(setId),
{SPV_OPERAND_TYPE_EXTENSION_INSTRUCTION_NUMBER, {instNumber}},
IdOp(a)});
};
// Combine two scratch-domain values with the site's operation.
const auto combine = [&](uint32_t a, uint32_t b) -> uint32_t {
const uint32_t typeId = scratchType.typeId;
switch (site.opcode) {
case spv::Op::OpGroupNonUniformIAdd:
return em.Emit(spv::Op::OpIAdd, typeId, {IdOp(a), IdOp(b)});
case spv::Op::OpGroupNonUniformFAdd:
return em.Emit(spv::Op::OpFAdd, typeId, {IdOp(a), IdOp(b)});
case spv::Op::OpGroupNonUniformIMul:
return em.Emit(spv::Op::OpIMul, typeId, {IdOp(a), IdOp(b)});
case spv::Op::OpGroupNonUniformFMul:
return em.Emit(spv::Op::OpFMul, typeId, {IdOp(a), IdOp(b)});
case spv::Op::OpGroupNonUniformSMin:
return extInst2(typeId, kGlslSMin, a, b);
case spv::Op::OpGroupNonUniformUMin:
return extInst2(typeId, kGlslUMin, a, b);
case spv::Op::OpGroupNonUniformFMin:
return extInst2(typeId, kGlslFMin, a, b);
case spv::Op::OpGroupNonUniformSMax:
return extInst2(typeId, kGlslSMax, a, b);
case spv::Op::OpGroupNonUniformUMax:
return extInst2(typeId, kGlslUMax, a, b);
case spv::Op::OpGroupNonUniformFMax:
return extInst2(typeId, kGlslFMax, a, b);
case spv::Op::OpGroupNonUniformBitwiseAnd:
case spv::Op::OpGroupNonUniformLogicalAnd:
return em.Emit(spv::Op::OpBitwiseAnd, typeId, {IdOp(a), IdOp(b)});
case spv::Op::OpGroupNonUniformBitwiseOr:
case spv::Op::OpGroupNonUniformLogicalOr:
return em.Emit(spv::Op::OpBitwiseOr, typeId, {IdOp(a), IdOp(b)});
case spv::Op::OpGroupNonUniformBitwiseXor:
case spv::Op::OpGroupNonUniformLogicalXor:
return em.Emit(spv::Op::OpBitwiseXor, typeId, {IdOp(a), IdOp(b)});
default:
return 0u;
}
};
// The identity element for the site's operation, in the scratch domain.
const auto identity = [&]() -> uint32_t {
const uint32_t typeId = scratchType.typeId;
const uint32_t scalarId = scratchType.scalarTypeId;
switch (site.opcode) {
case spv::Op::OpGroupNonUniformIAdd:
case spv::Op::OpGroupNonUniformBitwiseOr:
case spv::Op::OpGroupNonUniformBitwiseXor:
case spv::Op::OpGroupNonUniformLogicalOr:
case spv::Op::OpGroupNonUniformLogicalXor:
case spv::Op::OpGroupNonUniformFAdd:
case spv::Op::OpGroupNonUniformUMax:
// 0 and +0.0f share the bit pattern; exclusive-scan lane 0 of an
// FAdd therefore returns +0.0 even for a -0.0 input, matching
// the identity the SPIR-V spec assigns.
return typedConst(site.valueType, 0u, scalarId, typeId);
case spv::Op::OpGroupNonUniformIMul:
return typedConst(site.valueType, 1u, scalarId, typeId);
case spv::Op::OpGroupNonUniformFMul:
return typedConst(site.valueType, 0x3f800000u, scalarId, typeId);
case spv::Op::OpGroupNonUniformSMin:
return typedConst(site.valueType, 0x7fffffffu, scalarId, typeId);
case spv::Op::OpGroupNonUniformUMin:
return typedConst(site.valueType, 0xffffffffu, scalarId, typeId);
case spv::Op::OpGroupNonUniformFMin:
return typedConst(site.valueType, 0x7f800000u, scalarId, typeId);
case spv::Op::OpGroupNonUniformSMax:
return typedConst(site.valueType, 0x80000000u, scalarId, typeId);
case spv::Op::OpGroupNonUniformFMax:
return typedConst(site.valueType, 0xff800000u, scalarId, typeId);
case spv::Op::OpGroupNonUniformBitwiseAnd:
return typedConst(site.valueType, 0xffffffffu, scalarId, typeId);
case spv::Op::OpGroupNonUniformLogicalAnd:
return typedConst(site.valueType, 1u, scalarId, typeId);
default:
return 0u;
}
};
const auto finish = [&](uint32_t finalId) {
inst->SetOpcode(spv::Op::OpCopyObject);
inst->SetInOperands({IdOp(finalId)});
};
bool ok = true;
switch (site.opcode) {
case spv::Op::OpGroupNonUniformElect: {
finish(em.Emit(spv::Op::OpIEqual, boolTypeId, {IdOp(lane), IdOp(c0)}));
break;
}
case spv::Op::OpGroupNonUniformAll:
case spv::Op::OpGroupNonUniformAny: {
const bool isAll = site.opcode == spv::Op::OpGroupNonUniformAll;
ok = storeOwnSlot(toScratch(site.valueId));
if (!ok) break;
const uint32_t neutral = uintConst(isAll ? 1u : 0u);
uint32_t acc = 0;
for (uint32_t k = 0; k < kWidth; ++k) {
const uint32_t slot =
em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(base), IdOp(uintConst(k))});
const uint32_t exists = em.Emit(spv::Op::OpULessThan, boolTypeId,
{IdOp(slot), IdOp(cTotal)});
const uint32_t value = readSlot(slot);
const uint32_t guarded = em.Emit(
spv::Op::OpSelect, uintTypeId,
{IdOp(exists), IdOp(value), IdOp(neutral)});
acc = k == 0 ? guarded
: em.Emit(isAll ? spv::Op::OpBitwiseAnd : spv::Op::OpBitwiseOr,
uintTypeId, {IdOp(acc), IdOp(guarded)});
}
barrier();
finish(em.Emit(spv::Op::OpINotEqual, boolTypeId, {IdOp(acc), IdOp(c0)}));
break;
}
case spv::Op::OpGroupNonUniformAllEqual: {
ok = storeOwnSlot(toScratch(site.valueId));
if (!ok) break;
const uint32_t reference = readSlot(base);
uint32_t acc = cTrue;
for (uint32_t k = 0; k < kWidth; ++k) {
const uint32_t slot =
em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(base), IdOp(uintConst(k))});
const uint32_t exists = em.Emit(spv::Op::OpULessThan, boolTypeId,
{IdOp(slot), IdOp(cTotal)});
const uint32_t value = readSlot(slot);
const spv::Op compareOpcode = site.valueType.kind == ScalarKind::Float
? spv::Op::OpFOrdEqual
: spv::Op::OpIEqual;
uint32_t equal = 0;
if (site.valueType.componentCount == 1u) {
equal = em.Emit(compareOpcode, boolTypeId,
{IdOp(value), IdOp(reference)});
} else {
const uint32_t comparisonTypeId =
boolVectorTypeId(site.valueType.componentCount);
const uint32_t componentsEqual = em.Emit(
compareOpcode, comparisonTypeId, {IdOp(value), IdOp(reference)});
equal = em.Emit(spv::Op::OpAll, boolTypeId, {IdOp(componentsEqual)});
}
const uint32_t guarded = em.Emit(
spv::Op::OpSelect, boolTypeId,
{IdOp(exists), IdOp(equal), IdOp(cTrue)});
acc = em.Emit(spv::Op::OpLogicalAnd, boolTypeId,
{IdOp(acc), IdOp(guarded)});
}
barrier();
finish(acc);
break;
}
case spv::Op::OpGroupNonUniformBroadcast:
case spv::Op::OpGroupNonUniformBroadcastFirst:
case spv::Op::OpGroupNonUniformShuffle:
case spv::Op::OpGroupNonUniformShuffleXor:
case spv::Op::OpGroupNonUniformShuffleUp:
case spv::Op::OpGroupNonUniformShuffleDown:
case spv::Op::OpGroupNonUniformQuadBroadcast:
case spv::Op::OpGroupNonUniformQuadSwap: {
ok = storeOwnSlot(toScratch(site.valueId));
if (!ok) break;
uint32_t sourceLane = 0;
switch (site.opcode) {
case spv::Op::OpGroupNonUniformBroadcastFirst:
sourceLane = c0;
break;
case spv::Op::OpGroupNonUniformBroadcast:
case spv::Op::OpGroupNonUniformShuffle:
sourceLane = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId,
{IdOp(site.indexId), IdOp(c31)});
break;
case spv::Op::OpGroupNonUniformShuffleXor: {
const uint32_t flipped = em.Emit(
spv::Op::OpBitwiseXor, uintTypeId, {IdOp(lane), IdOp(site.indexId)});
sourceLane = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId,
{IdOp(flipped), IdOp(c31)});
break;
}
case spv::Op::OpGroupNonUniformShuffleUp: {
const uint32_t shifted =
em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(lane), IdOp(site.indexId)});
sourceLane = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId,
{IdOp(shifted), IdOp(c31)});
break;
}
case spv::Op::OpGroupNonUniformShuffleDown: {
const uint32_t shifted =
em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(lane), IdOp(site.indexId)});
sourceLane = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId,
{IdOp(shifted), IdOp(c31)});
break;
}
case spv::Op::OpGroupNonUniformQuadBroadcast: {
const uint32_t quadBase =
em.Emit(spv::Op::OpBitwiseAnd, uintTypeId,
{IdOp(lane), IdOp(uintConst(~3u & kLaneMask))});
const uint32_t withinQuad =
em.Emit(spv::Op::OpBitwiseAnd, uintTypeId,
{IdOp(site.indexId), IdOp(uintConst(3u))});
sourceLane = em.Emit(spv::Op::OpIAdd, uintTypeId,
{IdOp(quadBase), IdOp(withinQuad)});
break;
}
case spv::Op::OpGroupNonUniformQuadSwap: {
const uint32_t partnerMask = uintConst(
site.quadDirection == 0u ? 1u : site.quadDirection == 1u ? 2u : 3u);
sourceLane = em.Emit(spv::Op::OpBitwiseXor, uintTypeId,
{IdOp(lane), IdOp(partnerMask)});
break;
}
default:
break;
}
const uint32_t sourceSlot =
em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(base), IdOp(sourceLane)});
const uint32_t value = readSlot(sourceSlot);
barrier();
finish(fromScratch(value));
break;
}
case spv::Op::OpGroupNonUniformBallot: {
ok = storeOwnSlot(toScratch(site.valueId));
if (!ok) break;
uint32_t word = c0;
for (uint32_t k = 0; k < kWidth; ++k) {
const uint32_t slot =
em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(base), IdOp(uintConst(k))});
const uint32_t exists = em.Emit(spv::Op::OpULessThan, boolTypeId,
{IdOp(slot), IdOp(cTotal)});
const uint32_t value = readSlot(slot);
const uint32_t guarded = em.Emit(
spv::Op::OpSelect, uintTypeId, {IdOp(exists), IdOp(value), IdOp(c0)});
const uint32_t bit = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId,
{IdOp(guarded), IdOp(uintConst(k))});
word = em.Emit(spv::Op::OpBitwiseOr, uintTypeId, {IdOp(word), IdOp(bit)});
}
barrier();
inst->SetOpcode(spv::Op::OpCompositeConstruct);
inst->SetInOperands({IdOp(word), IdOp(c0), IdOp(c0), IdOp(c0)});
break;
}
case spv::Op::OpGroupNonUniformInverseBallot: {
const uint32_t word = em.Emit(spv::Op::OpCompositeExtract, uintTypeId,
{IdOp(site.valueId),
{SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}});
const uint32_t shifted = em.Emit(spv::Op::OpShiftRightLogical, uintTypeId,
{IdOp(word), IdOp(lane)});
const uint32_t bit =
em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(shifted), IdOp(c1)});
finish(em.Emit(spv::Op::OpINotEqual, boolTypeId, {IdOp(bit), IdOp(c0)}));
break;
}
case spv::Op::OpGroupNonUniformBallotBitExtract: {
const uint32_t words[4] = {
em.Emit(spv::Op::OpCompositeExtract, uintTypeId,
{IdOp(site.valueId), {SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}}),
em.Emit(spv::Op::OpCompositeExtract, uintTypeId,
{IdOp(site.valueId), {SPV_OPERAND_TYPE_LITERAL_INTEGER, {1u}}}),
em.Emit(spv::Op::OpCompositeExtract, uintTypeId,
{IdOp(site.valueId), {SPV_OPERAND_TYPE_LITERAL_INTEGER, {2u}}}),
em.Emit(spv::Op::OpCompositeExtract, uintTypeId,
{IdOp(site.valueId), {SPV_OPERAND_TYPE_LITERAL_INTEGER, {3u}}})};
const uint32_t inFirstPair = em.Emit(
spv::Op::OpULessThan, boolTypeId, {IdOp(site.indexId), IdOp(uintConst(64u))});
const uint32_t inWord0 = em.Emit(
spv::Op::OpULessThan, boolTypeId, {IdOp(site.indexId), IdOp(c32)});
const uint32_t inWord2 = em.Emit(
spv::Op::OpULessThan, boolTypeId, {IdOp(site.indexId), IdOp(uintConst(96u))});
const uint32_t firstPair = em.Emit(
spv::Op::OpSelect, uintTypeId,
{IdOp(inWord0), IdOp(words[0]), IdOp(words[1])});
const uint32_t secondPair = em.Emit(
spv::Op::OpSelect, uintTypeId,
{IdOp(inWord2), IdOp(words[2]), IdOp(words[3])});
const uint32_t word = em.Emit(
spv::Op::OpSelect, uintTypeId,
{IdOp(inFirstPair), IdOp(firstPair), IdOp(secondPair)});
const uint32_t bitIndex = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId,
{IdOp(site.indexId), IdOp(c31)});
const uint32_t shifted = em.Emit(spv::Op::OpShiftRightLogical, uintTypeId,
{IdOp(word), IdOp(bitIndex)});
const uint32_t bit =
em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(shifted), IdOp(c1)});
finish(em.Emit(spv::Op::OpINotEqual, boolTypeId, {IdOp(bit), IdOp(c0)}));
break;
}
case spv::Op::OpGroupNonUniformBallotBitCount: {
const uint32_t word = em.Emit(spv::Op::OpCompositeExtract, uintTypeId,
{IdOp(site.valueId),
{SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}});
uint32_t masked = 0;
// Counted over the gl_SubgroupSize (32) bits of the x word - the value is
// an arbitrary uvec4, and bits of lanes a partial subgroup lacks still
// count (subgroupBallotBitCount(uvec4(~0u)) == gl_SubgroupSize).
if (site.groupOperation == spv::GroupOperation::Reduce) {
masked = word;
} else {
const bool inclusive =
site.groupOperation == spv::GroupOperation::InclusiveScan;
uint32_t laneMask = 0;
if (inclusive) {
const uint32_t lanePlusOne =
em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(lane), IdOp(c1)});
const uint32_t atTop =
em.Emit(spv::Op::OpUGreaterThanEqual, boolTypeId,
{IdOp(lanePlusOne), IdOp(c32)});
const uint32_t shifted =
em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId,
{IdOp(c1), IdOp(lanePlusOne)});
const uint32_t below =
em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(shifted), IdOp(c1)});
laneMask = em.Emit(spv::Op::OpSelect, uintTypeId,
{IdOp(atTop), IdOp(cAllOnes), IdOp(below)});
} else {
const uint32_t bit = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId,
{IdOp(c1), IdOp(lane)});
laneMask =
em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(bit), IdOp(c1)});
}
masked = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId,
{IdOp(word), IdOp(laneMask)});
}
finish(em.Emit(spv::Op::OpBitCount, uintTypeId, {IdOp(masked)}));
break;
}
case spv::Op::OpGroupNonUniformBallotFindLSB:
case spv::Op::OpGroupNonUniformBallotFindMSB: {
const uint32_t word = em.Emit(spv::Op::OpCompositeExtract, uintTypeId,
{IdOp(site.valueId),
{SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}});
// Over the gl_SubgroupSize (32) bits of the x word, like the bit count.
const uint32_t masked = word;
const uint32_t instNumber =
site.opcode == spv::Op::OpGroupNonUniformBallotFindLSB ? kGlslFindILsb
: kGlslFindUMsb;
finish(extInst1(uintTypeId, instNumber, masked));
break;
}
default: {
// The arithmetic family. Serial, ascending-lane combines: the same
// left-associated order a host-side reference computes, and exact
// for every integer-valued input regardless of native topology.
ok = storeOwnSlot(toScratch(site.valueId));
if (!ok) break;
uint32_t result = 0;
switch (site.groupOperation) {
case spv::GroupOperation::Reduce: {
uint32_t acc = readSlot(base);
for (uint32_t k = 1; k < kWidth; ++k) {
const uint32_t slot = em.Emit(spv::Op::OpIAdd, uintTypeId,
{IdOp(base), IdOp(uintConst(k))});
const uint32_t exists = em.Emit(spv::Op::OpULessThan, boolTypeId,
{IdOp(slot), IdOp(cTotal)});
const uint32_t value = readSlot(slot);
const uint32_t combined = combine(acc, value);
if (combined == 0u) { ok = false; break; }
acc = guardedSelect(exists, combined, acc);
}
result = acc;
break;
}
case spv::GroupOperation::InclusiveScan: {
// Slots at or below the own lane always exist, so the guard is
// purely the lane comparison.
uint32_t acc = readSlot(base);
for (uint32_t k = 1; k < kWidth; ++k) {
const uint32_t included =
em.Emit(spv::Op::OpULessThanEqual, boolTypeId,
{IdOp(uintConst(k)), IdOp(lane)});
const uint32_t slot = em.Emit(spv::Op::OpIAdd, uintTypeId,
{IdOp(base), IdOp(uintConst(k))});
const uint32_t value = readSlot(slot);
const uint32_t combined = combine(acc, value);
if (combined == 0u) { ok = false; break; }
acc = guardedSelect(included, combined, acc);
}
result = acc;
break;
}
case spv::GroupOperation::ExclusiveScan: {
uint32_t acc = identity();
if (acc == 0u) { ok = false; break; }
for (uint32_t k = 0; k < kWidth; ++k) {
const uint32_t included = em.Emit(
spv::Op::OpULessThan, boolTypeId, {IdOp(uintConst(k)), IdOp(lane)});
const uint32_t slot = em.Emit(spv::Op::OpIAdd, uintTypeId,
{IdOp(base), IdOp(uintConst(k))});
const uint32_t value = readSlot(slot);
const uint32_t combined = combine(acc, value);
if (combined == 0u) { ok = false; break; }
acc = guardedSelect(included, combined, acc);
}
result = acc;
break;
}
case spv::GroupOperation::ClusteredReduce: {
const uint32_t cluster =
site.clusterSize >= kWidth ? kWidth : site.clusterSize;
const uint32_t clusterBase = em.Emit(
spv::Op::OpBitwiseAnd, uintTypeId,
{IdOp(lane), IdOp(uintConst(~(cluster - 1u) & kLaneMask))});
const uint32_t firstSlot = em.Emit(spv::Op::OpIAdd, uintTypeId,
{IdOp(base), IdOp(clusterBase)});
uint32_t acc = readSlot(firstSlot);
for (uint32_t k = 1; k < cluster; ++k) {
const uint32_t slot = em.Emit(spv::Op::OpIAdd, uintTypeId,
{IdOp(firstSlot), IdOp(uintConst(k))});
const uint32_t exists = em.Emit(spv::Op::OpULessThan, boolTypeId,
{IdOp(slot), IdOp(cTotal)});
const uint32_t value = readSlot(slot);
const uint32_t combined = combine(acc, value);
if (combined == 0u) { ok = false; break; }
acc = guardedSelect(exists, combined, acc);
}
result = acc;
break;
}
default:
ok = false;
break;
}
if (!ok) break;
barrier();
finish(fromScratch(result));
break;
}
}
if (!ok) {
// Phase A vetted every site, so a build failure here means an
// internal inconsistency; the module is already partially rewritten
// and must not be used.
return Status::Failure;
}
}
// Widen subgroup-scoped barriers to the workgroup scope the emulation
// synchronizes at anyway; strictly stronger, so always safe.
const uint32_t workgroupScopeId =
uintConst(static_cast<uint32_t>(spv::Scope::Workgroup));
if (workgroupScopeId == 0u) return Status::Failure;
for (const BarrierSite& site : barriers) {
if (site.inst->opcode() == spv::Op::OpControlBarrier) {
if (site.executionScopeIsSubgroup) {
site.inst->SetInOperand(0, {workgroupScopeId});
}
if (site.memoryScopeIsSubgroup) {
site.inst->SetInOperand(1, {workgroupScopeId});
}
} else if (site.memoryScopeIsSubgroup) {
site.inst->SetInOperand(0, {workgroupScopeId});
}
}
// ---------------------------------------------------------------- Phase C
// The replaced builtin variables are dead now; remove them together with
// their decorations and interface entries, then drop the GroupNonUniform*
// capabilities nothing references any more.
irContext->InvalidateAnalysesExceptFor(IRContext::kAnalysisNone);
for (const BuiltinUse& use : builtinUses) {
const uint32_t variableId = use.variable->result_id();
for (Instruction& entryPoint : irContext->module()->entry_points()) {
for (uint32_t operandIndex = entryPoint.NumInOperands(); operandIndex > 3;) {
--operandIndex;
if (entryPoint.GetInOperand(operandIndex).type == SPV_OPERAND_TYPE_ID &&
entryPoint.GetSingleWordInOperand(operandIndex) == variableId) {
entryPoint.RemoveInOperand(operandIndex);
}
}
}
for (Instruction* chain : use.accessChains) {
irContext->KillNamesAndDecorates(chain);
irContext->KillInst(chain);
}
irContext->KillNamesAndDecorates(use.variable);
irContext->KillInst(use.variable);
}
static constexpr std::array<spv::Capability, 9> kSubgroupCapabilities = {
spv::Capability::GroupNonUniform,
spv::Capability::GroupNonUniformVote,
spv::Capability::GroupNonUniformArithmetic,
spv::Capability::GroupNonUniformBallot,
spv::Capability::GroupNonUniformShuffle,
spv::Capability::GroupNonUniformShuffleRelative,
spv::Capability::GroupNonUniformClustered,
spv::Capability::GroupNonUniformQuad,
spv::Capability::GroupNonUniformPartitionedNV};
std::vector<Instruction*> deadCapabilities;
for (Instruction& capability : irContext->module()->capabilities()) {
const auto value =
static_cast<spv::Capability>(capability.GetSingleWordInOperand(0));
for (const spv::Capability candidate : kSubgroupCapabilities) {
if (value == candidate) {
deadCapabilities.push_back(&capability);
break;
}
}
}
for (Instruction* capability : deadCapabilities) {
irContext->KillInst(capability);
}
irContext->InvalidateAnalysesExceptFor(IRContext::kAnalysisNone);
return Status::SuccessWithChange;
}
spvtools::Optimizer::PassToken EmulateSubgroupsPass::CreateEmulateSubgroupsPass(
const Uint32 maxWorkgroupScratchBytes, const bool atomicScratchReads) {
return spvtools::Optimizer::PassToken(
MakeUnique<EmulateSubgroupsPass>(maxWorkgroupScratchBytes, atomicScratchReads));
}
} // namespace ShaderTranspiler
} // namespace MG_Util
} // namespace MobileGL