[Feat] (Subgroups): use native subgroups or emulate them on both backends

This commit is contained in:
2026-10-13 06:03:57 -04:00
parent cfbaffd757
commit d6b0eba8fe
50 changed files with 909 additions and 360 deletions
@@ -1171,6 +1171,9 @@ namespace MobileGL::MG_Util::BackendLoader {
if (std::strcmp(extension, "GL_EXT_fragment_shading_rate") == 0) {
caps.SupportsFragmentShadingRate = true;
}
if (std::strcmp(extension, "GL_KHR_shader_subgroup") == 0) {
caps.SupportsShaderSubgroup = true;
}
if (std::strcmp(extension, "GL_EXT_fragment_shading_rate_primitive") == 0) {
caps.SupportsFragmentShadingRatePrimitive = true;
}
@@ -1292,6 +1295,27 @@ namespace MobileGL::MG_Util::BackendLoader {
if (caps.SupportsFragmentShadingRate) {
FillInFragmentShadingRateCapabilities(caps, glesFuncs);
}
if (caps.SupportsShaderSubgroup) {
// Through glesFuncs, never the bare entry point (that is MobileGL's own getter).
GLint size = 0, stages = 0, features = 0;
GLboolean quad = GL_FALSE;
while (glesFuncs.glGetError() != GL_NO_ERROR) {}
glesFuncs.glGetIntegerv(GL_SUBGROUP_SIZE_KHR, &size);
glesFuncs.glGetIntegerv(GL_SUBGROUP_SUPPORTED_STAGES_KHR, &stages);
glesFuncs.glGetIntegerv(GL_SUBGROUP_SUPPORTED_FEATURES_KHR, &features);
glesFuncs.glGetBooleanv(GL_SUBGROUP_QUAD_ALL_STAGES_KHR, &quad);
const Bool answered = glesFuncs.glGetError() == GL_NO_ERROR;
caps.SubgroupSize = size > 0 ? static_cast<Uint32>(size) : 0u;
caps.SubgroupSupportedStages = static_cast<Uint32>(stages);
caps.SubgroupSupportedFeatures = static_cast<Uint32>(features);
caps.SubgroupQuadOperationsInAllStages = quad == GL_TRUE;
caps.SupportsShaderSubgroup = answered && caps.SubgroupSize > 0 &&
(caps.SubgroupSupportedStages & GL_COMPUTE_SHADER_BIT) != 0 &&
(caps.SubgroupSupportedFeatures & GL_SUBGROUP_FEATURE_BASIC_BIT_KHR) != 0;
}
MGLOG_I(" GL_KHR_shader_subgroup (native ES): %s size=%u stages=0x%x features=0x%x",
caps.SupportsShaderSubgroup ? "usable" : "absent", caps.SubgroupSize, caps.SubgroupSupportedStages,
caps.SubgroupSupportedFeatures);
// Core from ES 3.2 on, so an extension string is not required there; below 3.2 the
// extension is, and the pointer still has to have resolved either way.
const Bool esAtLeast32 = caps.GLESVersion.Major > 3 ||
@@ -1881,6 +1905,12 @@ namespace MobileGL::MG_Util::BackendLoader {
}
caps.MaxShaderStorageBufferBindings = maxShaderStorageBufferBindings;
caps.MaxTextureBufferSize = maxTextureBufferSize;
{
GLint sharedMemory = 0;
while (glesFuncs.glGetError() != GL_NO_ERROR) {}
glesFuncs.glGetIntegerv(GL_MAX_COMPUTE_SHARED_MEMORY_SIZE, &sharedMemory);
if (glesFuncs.glGetError() == GL_NO_ERROR && sharedMemory > 0) caps.MaxComputeSharedMemorySize = sharedMemory;
}
// Through glesFuncs, like every other capability query here: a bare glGetIntegerv resolves
// to MobileGL's own exported entry point, which answers this pname from the very
// capability table being filled in - so the driver's real alignment never arrived and the
@@ -2078,6 +2108,23 @@ namespace MobileGL::MG_Util::BackendLoader {
? "true (DirectGLES defers a writing geometry stage's emits behind its writes)"
: "false");
switch (MG_Config::Features.EsprytAtomicSubgroupScratch) {
case MG_Config::QuirkOverride::ForceOn:
caps.SharedMemoryLoadsGoStale = true;
break;
case MG_Config::QuirkOverride::ForceOff:
caps.SharedMemoryLoadsGoStale = false;
break;
case MG_Config::QuirkOverride::Auto:
default:
caps.SharedMemoryLoadsGoStale = SelfTest::SharedMemoryLoadsGoStale(glesFuncs);
break;
}
MGLOG_I(" Shared-memory loads go stale across barrier(): %s",
caps.SharedMemoryLoadsGoStale
? "true (the subgroup emulation reads its scratch through atomic loads)"
: "false");
caps.IsAngleRenderer = caps.GLESRendererString.find("ANGLE") != String::npos;
caps.IsAngleLlvmpipeRenderer =
caps.IsAngleRenderer && caps.GLESRendererString.find("llvmpipe") != String::npos;
@@ -1324,6 +1324,11 @@ namespace MobileGL {
// then defers every emit of a geometry stage that writes memory behind those writes
// (DeferGeometryEmitsPass). Defaults FALSE: never assumed broken on an unmeasured driver.
Bool DropsGeometryWritesAfterEmit = false;
// True when two loads of one shared-memory word separated by a store and barrier()
// return the same (first) value (SelfTest::SharedMemoryLoadsGoStale - measured on
// Adreno). The subgroup emulation then reads its scratch through atomic loads.
// Defaults FALSE: never assumed broken on an unmeasured driver.
Bool SharedMemoryLoadsGoStale = false;
Int UniformBufferOffsetAlignment = 256;
// Its storage-buffer counterpart, queried separately because it is a separate limit:
// Adreno 830 answers 32 for GL_UNIFORM_BUFFER_OFFSET_ALIGNMENT and 64 for
@@ -1380,6 +1385,18 @@ namespace MobileGL {
// initialisers are those minimums, for a context that rejects the query.
Int MaxComputeWorkGroupCount[3] = {65535, 65535, 65535};
Int MaxComputeWorkGroupSize[3] = {1024, 1024, 64};
// GL_MAX_COMPUTE_SHARED_MEMORY_SIZE; the initialiser is the ES 3.1 minimum. The budget
// the subgroup emulation's workgroup scratch has to fit in.
Int MaxComputeSharedMemorySize = 16384;
// GL_KHR_shader_subgroup on the ES driver, with what it answers for it. Extension-only
// in every ES version; the values are meaningful only when the flag is set, and the
// flag is only set when the driver also answers a usable size (> 0) with the compute
// stage and the basic feature - the same rule the Vulkan loader applies.
Bool SupportsShaderSubgroup = false;
Uint32 SubgroupSize = 0;
Uint32 SubgroupSupportedStages = 0; // GL_*_SHADER_BIT, as the ES query answers
Uint32 SubgroupSupportedFeatures = 0; // GL_SUBGROUP_FEATURE_*_BIT_KHR
Bool SubgroupQuadOperationsInAllStages = false;
Int MaxShaderStorageBufferBindings = 8;
Int MaxTextureBufferSize = 65536;
// GL_TEXTURE_BUFFER_OFFSET_ALIGNMENT; 1 means the offset is unconstrained.
@@ -91,9 +91,6 @@ namespace MobileGL::MG_Util::BackendLoader {
return loaded;
}
Bool IsShaderSubgroupForcedDisabled() {
return MG_Config::Features.MagmaDisableSubgroup;
}
} // namespace
inline Version DecodeApiVersion(uint32_t version) {
@@ -266,8 +263,7 @@ namespace MobileGL::MG_Util::BackendLoader {
caps.MaxShaderStorageBlockSize = static_cast<SizeT>(p.limits.maxStorageBufferRange);
const Bool supportsShaderSubgroup = vk.vkGetPhysicalDeviceProperties2 &&
HasUsableShaderSubgroupSupport(subgroupProps);
const Bool forceDisableShaderSubgroup = IsShaderSubgroupForcedDisabled();
caps.SupportsShaderSubgroup = supportsShaderSubgroup && !forceDisableShaderSubgroup;
caps.SupportsShaderSubgroup = supportsShaderSubgroup;
if (caps.SupportsShaderSubgroup) {
caps.SubgroupSize = subgroupProps.subgroupSize;
caps.SubgroupSupportedStages = subgroupProps.supportedStages;
@@ -280,12 +276,9 @@ namespace MobileGL::MG_Util::BackendLoader {
caps.SubgroupQuadOperationsInAllStages = false;
}
MGLOG_I("Vulkan shader subgroup support: detected=%s advertised=%s size=%u stages=0x%x operations=0x%x",
supportsShaderSubgroup ? "true" : "false", caps.SupportsShaderSubgroup ? "true" : "false",
subgroupProps.subgroupSize, subgroupProps.supportedStages, subgroupProps.supportedOperations);
if (supportsShaderSubgroup && forceDisableShaderSubgroup) {
MGLOG_W("Vulkan shader subgroup support forced off by MOBILEGL_MAGMA_DISABLE_SUBGROUP");
}
MGLOG_I("Vulkan shader subgroup support: native=%s size=%u stages=0x%x operations=0x%x",
supportsShaderSubgroup ? "true" : "false", subgroupProps.subgroupSize,
subgroupProps.supportedStages, subgroupProps.supportedOperations);
return true;
}
@@ -424,6 +424,112 @@ namespace MobileGL::MG_Util::SelfTest {
return dropped;
}
namespace {
constexpr const char* kStaleSharedProbeName = "stale shared-memory load";
// a * 16 + b with a = 1 (first read) and b = 7 (second read).
constexpr GLuint kStaleSharedExpected = 0x17u;
// One result word per invocation of the 32-wide workgroup.
constexpr GLsizei kStaleSharedResultBytes = 32 * sizeof(GLuint);
// The probe program around `read`, the expression that loads slot 0: a plain load (the
// subject) or an atomic one (the control). Every invocation writes its own result word,
// unconditionally: guarding the write (`if (lane == 0u)`) splits the control flow and the
// compiler then keeps the loads in place - measured, that shape reads 0x17 on the very
// driver where this one reads 0x11.
String StaleSharedComputeSource(const char* read) {
return String("#version 310 es\n"
"layout(local_size_x = 32) in;\n"
"layout(std430, binding = 0) buffer Result { highp uint data[32]; } g_result;\n"
"shared highp uint s_slots[32];\n"
"void main() {\n"
" highp uint lane = gl_LocalInvocationIndex;\n"
" s_slots[lane] = 1u;\n"
" barrier();\n"
" highp uint a = ") +
read +
";\n"
" barrier();\n"
" s_slots[lane] = 7u;\n"
" barrier();\n"
" highp uint b = " +
read +
";\n"
" barrier();\n"
" g_result.data[lane] = a * 16u + b;\n"
"}\n";
}
// Runs one variant and returns what invocation 0 wrote, or nullopt when the program
// could not be built or run.
Optional<GLuint> RunStaleSharedVariant(const GLESFunctionsTable& gl, const char* readExpression,
GLuint buffer) {
const String source = StaleSharedComputeSource(readExpression);
const GLuint shader =
CompileStage(gl, GL_COMPUTE_SHADER, source.c_str(), "compute", kStaleSharedProbeName);
if (shader == 0) return std::nullopt;
const GLuint program = gl.glCreateProgram();
Optional<GLuint> result;
if (program != 0) {
gl.glAttachShader(program, shader);
gl.glLinkProgram(program);
GLint linked = 0;
gl.glGetProgramiv(program, GL_LINK_STATUS, &linked);
if (linked != GL_FALSE) {
const GLuint zero[kStaleSharedResultBytes / sizeof(GLuint)] = {};
gl.glBindBuffer(GL_SHADER_STORAGE_BUFFER, buffer);
gl.glBufferData(GL_SHADER_STORAGE_BUFFER, kStaleSharedResultBytes, zero, GL_DYNAMIC_DRAW);
gl.glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 0, buffer);
gl.glUseProgram(program);
Drain(gl);
gl.glDispatchCompute(1, 1, 1);
if (gl.glGetError() == GL_NO_ERROR) {
gl.glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT | GL_BUFFER_UPDATE_BARRIER_BIT);
gl.glFinish();
result = ReadFirstWord(gl, buffer);
}
}
gl.glDeleteProgram(program);
}
gl.glDeleteShader(shader);
return result;
}
} // namespace
Bool ProbeSharedMemoryLoadsGoStale(const GLESFunctionsTable& gl) {
if (!HasEveryEntryPoint(gl) || gl.glDispatchCompute == nullptr) return false;
SavedState saved;
Save(gl, saved);
Drain(gl);
GLuint buffer = 0;
gl.glGenBuffers(1, &buffer);
const Optional<GLuint> plain = RunStaleSharedVariant(gl, "s_slots[0u]", buffer);
const Optional<GLuint> atomic = RunStaleSharedVariant(gl, "atomicAdd(s_slots[0u], 0u)", buffer);
if (buffer != 0) gl.glDeleteBuffers(1, &buffer);
Restore(gl, saved);
if (!plain || !atomic) {
MGLOG_I("[driver-bug] %s probe reached no verdict (a variant did not build or run)",
kStaleSharedProbeName);
return false;
}
if (*atomic != kStaleSharedExpected) {
MGLOG_I("[driver-bug] %s probe reached no verdict: the atomic control read 0x%x (expected 0x%x)",
kStaleSharedProbeName, *atomic, kStaleSharedExpected);
return false;
}
const Bool stale = *plain != kStaleSharedExpected;
MGLOG_I("[driver-bug] %s probe: plain reads 0x%x, atomic reads 0x%x (expected 0x%x)%s", kStaleSharedProbeName,
*plain, *atomic, kStaleSharedExpected, stale ? " - SHARED LOADS GO STALE ACROSS barrier()" : "");
return stale;
}
Bool SharedMemoryLoadsGoStale(const GLESFunctionsTable& gl) {
static const Bool stale = ProbeSharedMemoryLoadsGoStale(gl);
return stale;
}
namespace {
// ===================== SHARED PROGRAM BUILDING =====================
// The four probes below all decide their verdict from whether a program LINKS or from
@@ -2294,6 +2400,27 @@ namespace MobileGL::MG_Util::SelfTest {
Move(detail)};
}
Optional<DriverBugFinding> ProbeStaleSharedMemoryLoadBug(const GLESFunctionsTable& gl) {
if (!SharedMemoryLoadsGoStale(gl)) return std::nullopt;
String detail =
"the ES compiler treats ordinary loads from shared variables as pure: two reads of one "
"shared word separated by a store and barrier() (memoryBarrierShared() too) return the "
"first value, and a read can move past a later store; atomicAdd(x, 0u) reads of the same "
"words stay in place. ";
if (MG_Config::Features.EsprytAtomicSubgroupScratch == MG_Config::QuirkOverride::ForceOff) {
detail += "THE REPAIR IS DISABLED in this process (MOBILEGL_ESPRYT_ATOMIC_SUBGROUP_SCRATCH), so "
"an emulated subgroup operation can see the lanes of an earlier one";
return DriverBugFinding{"Shared-memory loads go stale across barrier()", DriverBugVerdict::Unfixable,
Move(detail)};
}
detail += "DirectGLES's subgroup emulation reads its workgroup scratch through atomic loads "
"(EmulateSubgroupsPass), so emulated subgroup operations see every lane's current value; "
"an application's own shared-memory code is passed through as written; override with "
"MOBILEGL_ESPRYT_ATOMIC_SUBGROUP_SCRATCH";
return DriverBugFinding{"Shared-memory loads go stale across barrier()", DriverBugVerdict::Fixed,
Move(detail)};
}
Optional<DriverBugFinding> ProbeR32FMultisampleSwizzleBug(const GLESFunctionsTable& gl) {
if (!R32FMultisampleSwizzleCorrupted(gl)) return std::nullopt;
return DriverBugFinding{
@@ -2466,6 +2593,7 @@ namespace MobileGL::MG_Util::SelfTest {
using DriverBugProbeFn = Optional<DriverBugFinding> (*)(const GLESFunctionsTable&);
constexpr DriverBugProbeFn kGlesDriverBugProbes[] = {
&ProbeGeometryWriteAfterEmitBug,
&ProbeStaleSharedMemoryLoadBug,
&ProbeR32FMultisampleSwizzleBug,
&ProbeImageLocationPerNameBug,
&ProbeCrossStageImageQualifierMergeBug,
@@ -184,6 +184,20 @@ namespace MobileGL::MG_Util::SelfTest {
// ProbeGeometryStageSsboWriteAfterEmitDropped(), evaluated at most once per process.
Bool GeometryStageSsboWriteAfterEmitDropped(const MG_External::GLESFunctionsTable& gl);
// Every invocation of a 32-wide compute workgroup stores 1 into its shared slot, barrier(),
// reads slot 0, barrier(), stores 7, barrier(), reads slot 0 again, and writes both reads
// to its own result word: the second read must see 7. The Adreno ES compiler treats ordinary shared loads as pure - it hands back the
// first read (and, with other addresses, moves a read past a later store) - while an
// atomicAdd(x, 0u) read of the same words is kept in place. That atomic twin is the
// control: the verdict is "stale" only when the plain program goes wrong while the atomic
// one stays right. Deterministic (a compiler transform), so latched; false when compute
// shaders or an entry point are missing or anything fails to set up. Restores the GL state
// it touches.
Bool ProbeSharedMemoryLoadsGoStale(const MG_External::GLESFunctionsTable& gl);
// ProbeSharedMemoryLoadsGoStale(), evaluated at most once per process.
Bool SharedMemoryLoadsGoStale(const MG_External::GLESFunctionsTable& gl);
// Samples one R32F GL_TEXTURE_2D_MULTISAMPLE texel through a swizzled alpha channel, twice,
// with a separately linked program each time. Returns true only when the swizzled read goes
// wrong while every control read stays right.
+61 -22
View File
@@ -20,6 +20,8 @@
// MG_State code: it runs standalone, before MG_State::Init().
#include <MG_State/GLState/VertexArrayState/VertexArrayObject.h>
#include <MG_Backend/DirectGLES/Utils.h>
#include <MG_Backend/DirectGLES/SubgroupPolicy.h>
#include <MG_Backend/SubgroupImplementation.h>
#include <MG_Util/Converters/GLToStr/GLEnumConverter.h>
#include <MG_Util/Converters/MGToGL/TextureEnumConverter.h>
#include <MG_Util/Converters/MGToStr/GLExtensionConverter.h>
@@ -310,6 +312,37 @@ namespace MobileGL::MG_Util::SelfTest {
driverLimitName, deviceLimit, deviceLimit));
}
// "Shader subgroups": how this backend implements GL_KHR_shader_subgroup on this driver
// (MG_Backend/SubgroupImplementation.h) - the driver's own operations at their native width,
// the 32-lane compute emulation, or nothing - and whether the iterationRP repairs are armed.
// `nativeDetail` describes what the driver offers natively, or is empty when it has nothing.
void ReportShaderSubgroups(ReportBuilder& builder, const Bool nativeSupported, const String& nativeDetail) {
using MG_Backend::SubgroupImplementation;
const String itrp = MG_Backend::ShouldApplyIterationRPFixes()
? "iterationRP repairs armed (MOBILEGL_ITRP_FIX)"
: "iterationRP repairs DISABLED (MOBILEGL_ITRP_FIX=ForceOff)";
switch (MG_Backend::ResolveSubgroupImplementation(nativeSupported)) {
case SubgroupImplementation::Native:
builder.Pass("Shader subgroups", format("native: {}; {}", nativeDetail, itrp));
break;
case SubgroupImplementation::Emulated:
builder.Pass("Shader subgroups",
format("emulated: {}-lane virtual subgroups in workgroup-shared memory, compute stage "
"only, every feature ({}); {}",
MG_Backend::kEmulatedSubgroupSize,
nativeSupported ? "forced over native " + nativeDetail +
" by MOBILEGL_SUBGROUP_EMULATION=ForceOn"
: String("the driver has no subgroup operations"),
itrp));
break;
case SubgroupImplementation::None:
builder.FailOptional("Shader subgroups",
"the driver has no subgroup operations and MOBILEGL_SUBGROUP_EMULATION=ForceOff "
"turned the emulation off, so GL_KHR_shader_subgroup is not advertised");
break;
}
}
void EvaluateGlesChecklist(ReportBuilder& builder, const MG_External::GLESCapabilities& caps,
const MG_External::GLESFunctionsTable& glesFuncs) {
const Int major = caps.GLESVersion.Major;
@@ -1239,6 +1272,14 @@ namespace MobileGL::MG_Util::SelfTest {
EvaluateGlesChecklist(builder, caps, glesFuncs);
ProbeGlesTimerQuery(builder, caps, glesFuncs);
ProbeGlesNoperspective(builder, caps, glesFuncs);
ReportShaderSubgroups(
builder, caps.SupportsShaderSubgroup,
caps.SupportsShaderSubgroup
? format("GL_KHR_shader_subgroup, subgroup size {}, stages 0x{:x}, features 0x{:x} (advertised: "
"0x{:x}, what ESSL output can spell)",
caps.SubgroupSize, caps.SubgroupSupportedStages, caps.SubgroupSupportedFeatures,
caps.SubgroupSupportedFeatures & MG_Backend::DirectGLES::kEsslPrintableSubgroupFeatures)
: String());
builder.report.formatCapabilities.emplace();
MG_Backend::DirectGLES::PopulateFormatCapabilities(
glesFuncs, caps, builder.report.formatCapabilities.value());
@@ -1266,11 +1307,16 @@ namespace MobileGL::MG_Util::SelfTest {
if (summary.capsValid) {
backendApiVersionString = MG_Backend::DirectGLES::FormatBackendAPIVersionString(
summary.caps.GLESRendererString, summary.caps.GLESVersion.Major, summary.caps.GLESVersion.Minor);
advertisedExtensions = JoinAdvertisedExtensions(MG_Backend::DirectGLES::BuildAdvertisedExtensions(
Vector<GLExtension> extensions = MG_Backend::DirectGLES::BuildAdvertisedExtensions(
summary.caps.SupportsDisjointTimerQuery, summary.caps.SupportsTextureFilterAnisotropy,
summary.caps.SupportsDrawIndirect,
summary.caps.SupportsDrawIndirect && summary.caps.SupportsBaseInstance,
summary.caps.SupportsTextureView, summary.caps.SupportsTextureCubeMapArray));
summary.caps.SupportsTextureView, summary.caps.SupportsTextureCubeMapArray);
if (MG_Backend::DirectGLES::ResolveEsprytSubgroupImplementation(summary.caps) !=
MG_Backend::SubgroupImplementation::None) {
extensions.push_back(E_GL_KHR_shader_subgroup);
}
advertisedExtensions = JoinAdvertisedExtensions(extensions);
}
AppendMobileGLReportedRows(builder, MG_Backend::DirectGLES::GetRendererIdentity(), backendApiVersionString,
advertisedExtensions);
@@ -1929,7 +1975,7 @@ namespace MobileGL::MG_Util::SelfTest {
// Native iterationRP compute witness. This deliberately uses a separate
// throwaway Vulkan device rather than the real renderer's queues, and it
// treats MOBILEGL_MAGMA_DISABLE_SUBGROUP as irrelevant: the row reports what the
// treats MOBILEGL_SUBGROUP_EMULATION as irrelevant: the row reports what the
// driver does, not what MobileGL elects to advertise to applications.
void ProbeVulkanIterationRPWitness(ReportBuilder& builder, PFN_vkGetInstanceProcAddr getInstanceProcAddr,
VkInstance instance, VkPhysicalDevice physicalDevice,
@@ -2999,26 +3045,19 @@ namespace MobileGL::MG_Util::SelfTest {
(subgroupProperties.supportedStages & VK_SHADER_STAGE_COMPUTE_BIT) != 0 &&
(subgroupProperties.supportedOperations & VK_SUBGROUP_FEATURE_BASIC_BIT) != 0;
// Same usability rule as the Vulkan capability loader's
// HasUsableShaderSubgroupSupport, which feeds the GL_KHR_shader_subgroup
// advertisement of the real backend.
summary.shaderSubgroupUsable = subgroupUsable;
if (subgroupUsable) {
builder.Pass("Compute shader subgroup",
format("basic subgroup operations in compute, subgroup size {}",
subgroupProperties.subgroupSize));
} else {
builder.FailOptional("Compute shader subgroup",
"basic subgroup operations are not usable from compute shaders, so "
"MobileGL withholds GL_KHR_shader_subgroup and the subgroup "
"iteration-render-pass path cannot run; there is no scalar rewrite "
"that stands in for a subgroup reduction");
}
// HasUsableShaderSubgroupSupport, which decides native vs emulated in the real backend.
ReportShaderSubgroups(builder, subgroupUsable,
subgroupUsable ? format("subgroup size {}, stages 0x{:x}, operations 0x{:x}",
subgroupProperties.subgroupSize,
subgroupProperties.supportedStages,
subgroupProperties.supportedOperations)
: String());
summary.shaderSubgroupUsable = MG_Backend::ResolveSubgroupImplementation(subgroupUsable) !=
MG_Backend::SubgroupImplementation::None;
} else {
builder.FailOptional("Compute shader subgroup",
"subgroup properties could not be queried (no "
"vkGetPhysicalDeviceProperties2, or a pre-1.1 device), so MobileGL "
"withholds GL_KHR_shader_subgroup and the subgroup paths are "
"unavailable whatever the hardware can actually do");
ReportShaderSubgroups(builder, false, String());
summary.shaderSubgroupUsable =
MG_Backend::ResolveSubgroupImplementation(false) != MG_Backend::SubgroupImplementation::None;
}
ProbeVulkanIterationRPWitness(builder, getInstanceProcAddr, instance, physicalDevice, computeQueueFamilyIndex,
@@ -115,10 +115,11 @@ namespace MobileGL::MG_Util::ShaderTranspiler {
// GlslangToSpv branches on which backend is active - ShaderAttrib::flags is 0 on
// both production parse paths (ShaderCompileTask::RunCompilePipeline and
// ClaimParsedShader). Backend identity steers the TRANSPILE, which is L2's key.
// * `advertisedExtensions`. Its only front-end consumer is ShaderSourceProcessor's
// FilterUnsupportedGpuShaderInt64, which REWRITES THE SOURCE TEXT - and the
// * `advertisedExtensions`, and params.SubgroupSupportedStages / SubgroupSupportedFeatures.
// Their only front-end consumer is ShaderSourceProcessor's
// FilterUnavailableShaderExtensions, which REWRITES THE SOURCE TEXT - and the
// preprocessed text is in the L1 key verbatim, a strictly finer discriminator
// than the extension list. (E_GL_ARB_gpu_shader_fp64 is never read by the front
// than the extension list or the masks. (E_GL_ARB_gpu_shader_fp64 is never read by the front
// end at all: MOBILEGL_ADVERTISE_FP64 only adds it to the extension STRING the
// application queries, and glslang parses `double` the same way either way.)
// * params.SupportsShaderFloat64, i.e. ConsumesFloat64Natively(). glslang produces
@@ -2049,11 +2049,12 @@ namespace MobileGL {
bool ShaderCompiler::EmulateSubgroupsForVulkan(const Vector<Uint32>& inputBinary,
Vector<uint32_t>& outputBinary,
const Uint32 maxWorkgroupScratchBytes,
const bool enableSpirvValidation) {
const bool enableSpirvValidation,
const bool atomicScratchReads) {
using namespace spvtools;
Optimizer optimizer(SPV_ENV_VULKAN_1_1);
optimizer.RegisterPass(
EmulateSubgroupsPass::CreateEmulateSubgroupsPass(maxWorkgroupScratchBytes));
optimizer.RegisterPass(EmulateSubgroupsPass::CreateEmulateSubgroupsPass(maxWorkgroupScratchBytes,
atomicScratchReads));
return RunOptimizerChecked("EmulateSubgroupsForVulkan", optimizer, inputBinary,
outputBinary, true, enableSpirvValidation);
@@ -383,17 +383,21 @@ namespace MobileGL {
Vector<uint32_t>& outputBinary,
bool enableSpirvValidation = false);
// Lowers every GL_KHR_shader_subgroup construct in a compute module onto a
// 32-lane virtual subgroup built from workgroup-shared memory. Last-resort
// path for devices with NO native subgroup support, opt-in via
// MOBILEGL_MAGMA_EMULATE_SUBGROUP=1; a device with native subgroup
// operations always uses them. maxWorkgroupScratchBytes bounds the shared
// 32-lane virtual subgroup built from workgroup-shared memory - what either
// backend runs on a driver with NO native subgroup support (or under
// MOBILEGL_SUBGROUP_EMULATION=ForceOn); the module is generic SPIR-V, consumed by
// DirectVulkan and transpiled to ESSL by DirectGLES alike.
// maxWorkgroupScratchBytes bounds the shared
// scratch the lowering may add (pass the device's
// maxComputeSharedMemorySize; 0 falls back to the 16384-byte Vulkan
// minimum). See EmulateSubgroupsPass.
// minimum). atomicScratchReads reads the scratch back through atomic loads,
// for a driver whose plain shared-memory loads go stale across barrier()
// (SelfTest::SharedMemoryLoadsGoStale). See EmulateSubgroupsPass.
static bool EmulateSubgroupsForVulkan(const Vector<Uint32>& inputBinary,
Vector<uint32_t>& outputBinary,
Uint32 maxWorkgroupScratchBytes,
bool enableSpirvValidation = false);
bool enableSpirvValidation = false,
bool atomicScratchReads = false);
// Grows iterationRP's under-declared gl_SubgroupID-indexed scratch to the
// subgroup count the device actually partitions into, fingerprint-gated to
// that pack's reduction idiom; every other module - and every device whose
@@ -905,17 +905,68 @@ namespace {
return token.substr(start, end - start);
}
void FilterUnsupportedGpuShaderInt64(const MobileGL::MG_Util::ShaderTranspiler::CompileEnv& env,
MobileGL::String& source) {
if (env.IsExtensionAdvertised(MobileGL::E_GL_ARB_gpu_shader_int64)) {
return;
// GL_SHADER_STAGE bit of a stage, as GL_SUBGROUP_SUPPORTED_STAGES_KHR spells it.
GLbitfield SubgroupStageBit(MobileGL::ShaderStage stage) {
switch (MobileGL::MG_Util::ConvertShaderStageToGLEnum(stage)) {
case GL_VERTEX_SHADER: return GL_VERTEX_SHADER_BIT;
case GL_TESS_CONTROL_SHADER: return GL_TESS_CONTROL_SHADER_BIT;
case GL_TESS_EVALUATION_SHADER: return GL_TESS_EVALUATION_SHADER_BIT;
case GL_GEOMETRY_SHADER: return GL_GEOMETRY_SHADER_BIT;
case GL_FRAGMENT_SHADER: return GL_FRAGMENT_SHADER_BIT;
case GL_COMPUTE_SHADER: return GL_COMPUTE_SHADER_BIT;
default: return 0;
}
}
// The GL_SUBGROUP_FEATURE_*_BIT_KHR an extension name stands for, or 0 for a name that is
// not one of the eight GL_KHR_shader_subgroup_* extensions.
GLbitfield SubgroupFeatureBit(const MobileGL::String& name) {
static const std::pair<const char*, GLbitfield> kFeatures[] = {
{"GL_KHR_shader_subgroup_basic", GL_SUBGROUP_FEATURE_BASIC_BIT_KHR},
{"GL_KHR_shader_subgroup_vote", GL_SUBGROUP_FEATURE_VOTE_BIT_KHR},
{"GL_KHR_shader_subgroup_arithmetic", GL_SUBGROUP_FEATURE_ARITHMETIC_BIT_KHR},
{"GL_KHR_shader_subgroup_ballot", GL_SUBGROUP_FEATURE_BALLOT_BIT_KHR},
{"GL_KHR_shader_subgroup_shuffle", GL_SUBGROUP_FEATURE_SHUFFLE_BIT_KHR},
{"GL_KHR_shader_subgroup_shuffle_relative", GL_SUBGROUP_FEATURE_SHUFFLE_RELATIVE_BIT_KHR},
{"GL_KHR_shader_subgroup_clustered", GL_SUBGROUP_FEATURE_CLUSTERED_BIT_KHR},
{"GL_KHR_shader_subgroup_quad", GL_SUBGROUP_FEATURE_QUAD_BIT_KHR},
};
for (const auto& [featureName, bit] : kFeatures) {
if (name == featureName) return bit;
}
return 0;
}
// Whether the shader-language extension `name` is one this backend cannot honour in this
// stage - which is what a `#extension name : require` must then fail on, and what an
// `enable` must not pretend to have turned on. glslang parses desktop GLSL 4.60, where
// these are always available, so without this a shader would compile here and then fail
// (or worse, silently do nothing) at the backend:
// * GL_ARB_gpu_shader_int64 when the extension is not advertised;
// * any GL_KHR_shader_subgroup_* when GL_KHR_shader_subgroup is not advertised, when
// this stage is not in GL_SUBGROUP_SUPPORTED_STAGES_KHR, or when that extension's
// feature is not in GL_SUBGROUP_SUPPORTED_FEATURES_KHR - the KHR spec's own terms for
// where each of the eight may be used. The stage and feature tests need a backend:
// with none, every extension reads as available (CompileEnv::IsExtensionAdvertised).
bool IsUnavailableShaderExtension(const MobileGL::MG_Util::ShaderTranspiler::CompileEnv& env,
MobileGL::ShaderStage stage, const MobileGL::String& name) {
if (name == "GL_ARB_gpu_shader_int64") return !env.IsExtensionAdvertised(MobileGL::E_GL_ARB_gpu_shader_int64);
const GLbitfield feature = SubgroupFeatureBit(name);
if (feature == 0) return false;
if (!env.IsExtensionAdvertised(MobileGL::E_GL_KHR_shader_subgroup)) return true;
if (!env.HasBackend()) return false;
return (env.params.SubgroupSupportedStages & SubgroupStageBit(stage)) == 0 ||
(env.params.SubgroupSupportedFeatures & feature) == 0;
}
void FilterUnavailableShaderExtensions(const MobileGL::MG_Util::ShaderTranspiler::CompileEnv& env,
MobileGL::ShaderStage stage, MobileGL::String& source) {
// Detect the directive on a comment/string-masked copy so a commented-out
// "#extension GL_ARB_gpu_shader_int64" is never turned into a synthesized #error. Comments are
// no longer blanked in the delivered source (glslang handles them), so this pass must mask
// locally like its siblings. Masking preserves offsets, so edits collected against the scan
// apply verbatim to `source`; they are applied back-to-front to keep earlier offsets valid.
// "#extension" is never turned into a synthesized #error. Comments are no longer
// blanked in the delivered source (glslang handles them), so this pass must mask
// locally like its siblings. Masking preserves offsets, so edits collected against the
// scan apply verbatim to `source`; they are applied back-to-front to keep earlier
// offsets valid.
const MobileGL::String scan = MaskCommentsAndQuotedText(source);
struct DirectiveEdit {
SizeT pos;
@@ -923,6 +974,7 @@ namespace {
MobileGL::String replacement;
};
Vector<DirectiveEdit> edits;
Vector<MobileGL::String> hiddenNames;
SizeT lineStart = 0;
while (lineStart < scan.size()) {
@@ -931,69 +983,56 @@ namespace {
if (!hasLineBreak) {
lineEnd = scan.size();
}
const MobileGL::String line = scan.substr(lineStart, lineEnd - lineStart);
const SizeT replaceLen = lineEnd - lineStart + (hasLineBreak ? 1 : 0);
SizeT probe = 0;
while (probe < line.size() && std::isspace(static_cast<unsigned char>(line[probe]))) {
probe++;
}
const auto skipSpace = [&] {
while (probe < line.size() && std::isspace(static_cast<unsigned char>(line[probe]))) probe++;
};
skipSpace();
if (probe < line.size() && line[probe] == '#') {
probe++;
while (probe < line.size() && std::isspace(static_cast<unsigned char>(line[probe]))) {
probe++;
}
skipSpace();
constexpr const char* extensionToken = "extension";
constexpr SizeT extensionLen = 9;
const bool hasExtensionDirective =
probe + extensionLen <= line.size() &&
line.compare(probe, extensionLen, extensionToken) == 0 &&
(probe + extensionLen == line.size() || !IsIdentifierChar(line[probe + extensionLen]));
if (hasExtensionDirective) {
if (probe + extensionLen <= line.size() && line.compare(probe, extensionLen, extensionToken) == 0 &&
(probe + extensionLen == line.size() || !IsIdentifierChar(line[probe + extensionLen]))) {
probe += extensionLen;
while (probe < line.size() && std::isspace(static_cast<unsigned char>(line[probe]))) {
probe++;
}
constexpr const char* int64Extension = "GL_ARB_gpu_shader_int64";
constexpr SizeT int64ExtensionLen = 23;
const bool hasInt64Extension =
probe + int64ExtensionLen <= line.size() &&
line.compare(probe, int64ExtensionLen, int64Extension) == 0 &&
(probe + int64ExtensionLen == line.size() ||
!IsIdentifierChar(line[probe + int64ExtensionLen]));
if (hasInt64Extension) {
probe += int64ExtensionLen;
while (probe < line.size() && std::isspace(static_cast<unsigned char>(line[probe]))) {
probe++;
skipSpace();
const SizeT nameStart = probe;
while (probe < line.size() && IsIdentifierChar(line[probe])) probe++;
const MobileGL::String name = line.substr(nameStart, probe - nameStart);
skipSpace();
if (!name.empty() && probe < line.size() && line[probe] == ':' &&
IsUnavailableShaderExtension(env, stage, name)) {
const MobileGL::String behavior = TrimDirectiveToken(line.substr(probe + 1));
if (behavior == "require") {
edits.push_back({lineStart, replaceLen,
"#error " + name + " is not available to this shader stage in MobileGL\n"});
} else if (behavior == "enable" || behavior == "warn") {
edits.push_back({lineStart, replaceLen, "\n"});
}
if (probe < line.size() && line[probe] == ':') {
probe++;
const MobileGL::String behavior = TrimDirectiveToken(line.substr(probe));
const SizeT replaceLen = lineEnd - lineStart + (hasLineBreak ? 1 : 0);
if (behavior == "require") {
edits.push_back({lineStart, replaceLen,
"#error GL_ARB_gpu_shader_int64 is not advertised by MobileGL\n"});
} else if (behavior == "enable" || behavior == "warn") {
edits.push_back({lineStart, replaceLen, "\n"});
}
lineStart = lineEnd + (hasLineBreak ? 1 : 0);
continue;
if (std::find(hiddenNames.begin(), hiddenNames.end(), name) == hiddenNames.end()) {
hiddenNames.push_back(name);
}
}
}
}
lineStart = lineEnd + (hasLineBreak ? 1 : 0);
}
for (auto it = edits.rbegin(); it != edits.rend(); ++it) {
source.replace(it->pos, it->len, it->replacement);
}
ReplaceIdentifier(source, "GL_ARB_gpu_shader_int64", "MG_DISABLED_GL_ARB_gpu_shader_int64");
// The extension's own feature-test macro, so an `#ifdef` guard takes the fallback branch.
// int64 keeps its historical treatment: hidden whether or not the source names it.
if (IsUnavailableShaderExtension(env, stage, "GL_ARB_gpu_shader_int64") &&
std::find(hiddenNames.begin(), hiddenNames.end(), "GL_ARB_gpu_shader_int64") == hiddenNames.end()) {
hiddenNames.push_back("GL_ARB_gpu_shader_int64");
}
for (const auto& name : hiddenNames) {
ReplaceIdentifier(source, name, "MG_DISABLED_" + name);
}
}
// GLSL 4.30 4.1.9 allows an interface-block member array to be left unsized when it is NOT the
@@ -1945,7 +1984,7 @@ namespace MobileGL {
// identifier that merely contained the word. The GLES fallback for devices without
// the extension lives in the backend, where device capabilities are known.
FilterUnsupportedGpuShaderInt64(env, source);
FilterUnavailableShaderExtensions(env, stage, source);
CoerceUniformBlockPackingToStd140(source);
// After the packing coercion: that one rewrites `packed`/`shared` in place and so
// cannot move an offset this pass depends on, and reading the block declarations
@@ -637,13 +637,22 @@ namespace MobileGL {
std::map<uint32_t, Scratch> scratchByType;
const uint32_t cPadded = uintConst(paddedSlots);
if (cPadded == 0u) return Status::Failure;
const auto scratchFor = [&](uint32_t elementTypeId) -> const Scratch* {
auto it = scratchByType.find(elementTypeId);
// `elementTypeId` is the scratch element type; with atomic scratch reads its
// array is instead a flat run of uint words, `wordsPerSlot` per slot (an atomic
// load needs a scalar integer pointer). Still one array per element type, so
// the types never share slots - the same aliasing as without atomic reads.
const auto scratchFor = [&](uint32_t elementTypeId, uint32_t wordsPerSlot) -> const Scratch* {
const uint32_t key = elementTypeId;
if (m_atomicScratchReads) elementTypeId = uintTypeId;
auto it = scratchByType.find(key);
if (it != scratchByType.end()) return &it->second;
const uint32_t lengthId =
m_atomicScratchReads ? uintConst(paddedSlots * wordsPerSlot) : cPadded;
if (lengthId == 0u) return nullptr;
const uint32_t arrayTypeId = irContext->TakeNextId();
irContext->AddType(spvtools::MakeUnique<Instruction>(
irContext, spv::Op::OpTypeArray, 0, arrayTypeId,
Instruction::OperandList{IdOp(elementTypeId), IdOp(cPadded)}));
Instruction::OperandList{IdOp(elementTypeId), IdOp(lengthId)}));
const uint32_t arrayPointerTypeId =
typeMgr->FindPointerToType(arrayTypeId, spv::StorageClass::Workgroup);
const uint32_t elementPointerTypeId =
@@ -656,7 +665,7 @@ namespace MobileGL {
{SPV_OPERAND_TYPE_STORAGE_CLASS,
{static_cast<uint32_t>(spv::StorageClass::Workgroup)}}}));
Scratch scratch{variableId, elementTypeId, elementPointerTypeId};
return &scratchByType.emplace(elementTypeId, scratch).first->second;
return &scratchByType.emplace(key, scratch).first->second;
};
// The uint-domain twin of a participating type: bools travel as 0/1 words.
@@ -691,25 +700,15 @@ namespace MobileGL {
if (numVirtualConstId == 0u) return Status::Failure;
// The x word of one of the five ballot-mask builtins for the calling
// invocation's virtual subgroup, bounded to lanes that exist (words y..w
// are zero at 32 lanes).
// invocation's virtual subgroup (words y..w are zero at 32 lanes). Defined
// over gl_SubgroupSize bits, NOT clipped to the lanes a partial last subgroup
// actually has: the masks name lane positions, and GL/Vulkan define them for
// every position below the subgroup size.
const auto emitMaskWord = [&](Emitter& em, spv::BuiltIn builtin) -> uint32_t {
const uint32_t index =
em.Emit(spv::Op::OpLoad, uintTypeId, {IdOp(liiVariableId)});
const uint32_t lane =
em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(index), IdOp(c31)});
const uint32_t base = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId,
{IdOp(index), IdOp(cNotLane)});
const uint32_t tailBits =
em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(cTotal), IdOp(base)});
const uint32_t full = em.Emit(spv::Op::OpUGreaterThanEqual, boolTypeId,
{IdOp(tailBits), IdOp(c32)});
const uint32_t tailShift = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId,
{IdOp(c1), IdOp(tailBits)});
const uint32_t tailMask =
em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(tailShift), IdOp(c1)});
const uint32_t exists = em.Emit(spv::Op::OpSelect, uintTypeId,
{IdOp(full), IdOp(cAllOnes), IdOp(tailMask)});
const auto lowMaskThroughLane = [&](bool inclusive) -> uint32_t {
if (!inclusive) {
const uint32_t bit = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId,
@@ -748,7 +747,7 @@ namespace MobileGL {
default:
return 0u;
}
return em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(word), IdOp(exists)});
return word;
};
for (const BuiltinUse& use : builtinUses) {
@@ -840,22 +839,73 @@ namespace MobileGL {
{IdOp(valueId), IdOp(zeros)});
};
// Atomic scratch reads keep each slot as componentCount uint words; the
// value crosses into that domain by bitcast (a no-op for uint data).
const uint32_t wordsPerSlot = site.valueType.componentCount;
const uint32_t wordTypeId = [&]() -> uint32_t {
if (!m_atomicScratchReads || wordsPerSlot == 1u) return uintTypeId;
const analysis::Type* registeredUint = typeMgr->GetType(uintTypeId);
analysis::Vector vectorType(registeredUint, wordsPerSlot);
return typeMgr->GetTypeInstruction(&vectorType);
}();
const bool wordsNeedBitcast = scratchType.scalarTypeId != uintTypeId;
const auto wordIndex = [&](uint32_t slotId, uint32_t component) -> uint32_t {
if (wordsPerSlot == 1u) return slotId;
const uint32_t scaled = em.Emit(spv::Op::OpIMul, uintTypeId,
{IdOp(slotId), IdOp(uintConst(wordsPerSlot))});
return em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(scaled), IdOp(uintConst(component))});
};
const Scratch* scratch = nullptr;
const auto storeOwnSlot = [&](uint32_t valueId) -> bool {
scratch = scratchFor(scratchType.typeId);
scratch = scratchFor(scratchType.typeId, wordsPerSlot);
if (scratch == nullptr) return false;
const uint32_t pointer =
em.Emit(spv::Op::OpAccessChain, scratch->pointerTypeId,
{IdOp(scratch->variableId), IdOp(lii)});
em.EmitNoResult(spv::Op::OpStore, {IdOp(pointer), IdOp(valueId)});
if (!m_atomicScratchReads) {
const uint32_t pointer =
em.Emit(spv::Op::OpAccessChain, scratch->pointerTypeId,
{IdOp(scratch->variableId), IdOp(lii)});
em.EmitNoResult(spv::Op::OpStore, {IdOp(pointer), IdOp(valueId)});
} else {
const uint32_t words =
wordsNeedBitcast ? em.Emit(spv::Op::OpBitcast, wordTypeId, {IdOp(valueId)}) : valueId;
for (uint32_t c = 0; c < wordsPerSlot; ++c) {
const uint32_t word =
wordsPerSlot == 1u
? words
: em.Emit(spv::Op::OpCompositeExtract, uintTypeId,
{IdOp(words), {SPV_OPERAND_TYPE_LITERAL_INTEGER, {c}}});
const uint32_t pointer =
em.Emit(spv::Op::OpAccessChain, scratch->pointerTypeId,
{IdOp(scratch->variableId), IdOp(wordIndex(lii, c))});
em.EmitNoResult(spv::Op::OpStore, {IdOp(pointer), IdOp(word)});
}
}
barrier();
return true;
};
const auto readSlot = [&](uint32_t indexId) -> uint32_t {
const uint32_t pointer =
em.Emit(spv::Op::OpAccessChain, scratch->pointerTypeId,
{IdOp(scratch->variableId), IdOp(indexId)});
return em.Emit(spv::Op::OpLoad, scratchType.typeId, {IdOp(pointer)});
if (!m_atomicScratchReads) {
const uint32_t pointer =
em.Emit(spv::Op::OpAccessChain, scratch->pointerTypeId,
{IdOp(scratch->variableId), IdOp(indexId)});
return em.Emit(spv::Op::OpLoad, scratchType.typeId, {IdOp(pointer)});
}
const uint32_t scopeId = uintConst(static_cast<uint32_t>(spv::Scope::Workgroup));
Instruction::OperandList words;
uint32_t packed = 0;
for (uint32_t c = 0; c < wordsPerSlot; ++c) {
const uint32_t pointer =
em.Emit(spv::Op::OpAccessChain, scratch->pointerTypeId,
{IdOp(scratch->variableId), IdOp(wordIndex(indexId, c))});
packed = em.Emit(spv::Op::OpAtomicLoad, uintTypeId,
{IdOp(pointer), IdOp(scopeId), IdOp(c0)});
words.push_back(IdOp(packed));
}
if (wordsPerSlot > 1u) {
packed = em.Emit(spv::Op::OpCompositeConstruct, wordTypeId, std::move(words));
}
return wordsNeedBitcast ? em.Emit(spv::Op::OpBitcast, scratchType.typeId, {IdOp(packed)})
: packed;
};
// A select whose condition is a scalar bool but whose values may be
// vectors; pre-SPIR-V-1.4 needs the condition splat to a bvec.
@@ -966,20 +1016,6 @@ namespace MobileGL {
}
};
// The lanes-that-exist word for this virtual subgroup (tail handling).
const auto existsWord = [&]() -> uint32_t {
const uint32_t tailBits =
em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(cTotal), IdOp(base)});
const uint32_t full = em.Emit(spv::Op::OpUGreaterThanEqual, boolTypeId,
{IdOp(tailBits), IdOp(c32)});
const uint32_t shifted = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId,
{IdOp(c1), IdOp(tailBits)});
const uint32_t mask =
em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(shifted), IdOp(c1)});
return em.Emit(spv::Op::OpSelect, uintTypeId,
{IdOp(full), IdOp(cAllOnes), IdOp(mask)});
};
const auto finish = [&](uint32_t finalId) {
inst->SetOpcode(spv::Op::OpCopyObject);
inst->SetInOperands({IdOp(finalId)});
@@ -1190,9 +1226,11 @@ namespace MobileGL {
{IdOp(site.valueId),
{SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}});
uint32_t masked = 0;
// Counted over the gl_SubgroupSize (32) bits of the x word - the value is
// an arbitrary uvec4, and bits of lanes a partial subgroup lacks still
// count (subgroupBallotBitCount(uvec4(~0u)) == gl_SubgroupSize).
if (site.groupOperation == spv::GroupOperation::Reduce) {
masked = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId,
{IdOp(word), IdOp(existsWord())});
masked = word;
} else {
const bool inclusive =
site.groupOperation == spv::GroupOperation::InclusiveScan;
@@ -1227,8 +1265,8 @@ namespace MobileGL {
const uint32_t word = em.Emit(spv::Op::OpCompositeExtract, uintTypeId,
{IdOp(site.valueId),
{SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}});
const uint32_t masked = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId,
{IdOp(word), IdOp(existsWord())});
// Over the gl_SubgroupSize (32) bits of the x word, like the bit count.
const uint32_t masked = word;
const uint32_t instNumber =
site.opcode == spv::Op::OpGroupNonUniformBallotFindLSB ? kGlslFindILsb
: kGlslFindUMsb;
@@ -1404,9 +1442,9 @@ namespace MobileGL {
}
spvtools::Optimizer::PassToken EmulateSubgroupsPass::CreateEmulateSubgroupsPass(
const Uint32 maxWorkgroupScratchBytes) {
const Uint32 maxWorkgroupScratchBytes, const bool atomicScratchReads) {
return spvtools::Optimizer::PassToken(
MakeUnique<EmulateSubgroupsPass>(maxWorkgroupScratchBytes));
MakeUnique<EmulateSubgroupsPass>(maxWorkgroupScratchBytes, atomicScratchReads));
}
} // namespace ShaderTranspiler
} // namespace MG_Util
@@ -21,9 +21,9 @@ namespace MobileGL {
// subgroups partition the workgroup by gl_LocalInvocationIndex:
// lane = index & 31, id = index >> 5, count = ceil(invocations / 32).
//
// This is a LAST-RESORT path, never a substitute for real subgroups: it only
// runs when MOBILEGL_MAGMA_EMULATE_SUBGROUP=1 is set explicitly and the device
// has no native subgroup support at all (SubgroupSupportPolicy.h). A device
// This is never a substitute for real subgroups: by default it only runs on a
// driver with no native subgroup support at all (MG_Backend/SubgroupImplementation.h;
// MOBILEGL_SUBGROUP_EMULATION=ForceOn runs it everywhere, for testing). A device
// with native subgroup operations - however narrow - uses them natively, with
// FixIterationRPSubgroupScratchPass patching the known pack bug instead.
//
@@ -54,19 +54,28 @@ namespace MobileGL {
// module whose lowering would add more workgroup scratch than
// maxWorkgroupScratchBytes (pass the device's maxComputeSharedMemorySize;
// 0 falls back to the 16384-byte Vulkan minimum).
//
// atomicScratchReads stores the scratch as plain 32-bit words and reads every
// slot back with OpAtomicLoad (atomicAdd(x, 0u) in ESSL), bit-casting to and from
// the participating type. It is for drivers whose compiler treats ordinary
// shared-memory loads as pure and so merges or reorders them across stores and
// barrier() (SelfTest::SharedMemoryLoadsGoStale, measured on Adreno's ES driver);
// an atomic load is a memory operation such a compiler keeps in place.
class EmulateSubgroupsPass : public spvtools::opt::Pass {
public:
explicit EmulateSubgroupsPass(Uint32 maxWorkgroupScratchBytes)
: m_maxWorkgroupScratchBytes(maxWorkgroupScratchBytes) {}
EmulateSubgroupsPass(Uint32 maxWorkgroupScratchBytes, bool atomicScratchReads)
: m_maxWorkgroupScratchBytes(maxWorkgroupScratchBytes),
m_atomicScratchReads(atomicScratchReads) {}
const char* name() const override { return "emulate-subgroups"; }
Status Process() override;
static spvtools::Optimizer::PassToken CreateEmulateSubgroupsPass(
Uint32 maxWorkgroupScratchBytes);
Uint32 maxWorkgroupScratchBytes, bool atomicScratchReads = false);
private:
Uint32 m_maxWorkgroupScratchBytes;
bool m_atomicScratchReads;
};
} // namespace ShaderTranspiler
} // namespace MG_Util
@@ -182,6 +182,10 @@ namespace MobileGL::MG_Util::ShaderTranspiler {
builder.Value(static_cast<Uint32>(inputs.shaderType));
builder.Value(static_cast<Uint8>(inputs.viewportIndexLoweringArmed));
builder.Value(static_cast<Uint8>(inputs.deferGeometryEmitsArmed));
builder.Value(static_cast<Uint8>(inputs.subgroupEmulationArmed));
builder.Value(inputs.subgroupEmulationScratchBytes);
builder.Value(static_cast<Uint8>(inputs.iterationRPBarrierFixArmed));
builder.Value(static_cast<Uint8>(inputs.subgroupScratchAtomicReads));
builder.Value(static_cast<Uint8>(inputs.supportsNoperspectiveInterpolation));
builder.Value(static_cast<Uint8>(inputs.supportsExtendedImageFormats));
builder.Value(inputs.maxColorTextureSamples);
@@ -647,6 +647,15 @@ namespace MobileGL::MG_Util::ShaderTranspiler {
// measured dropped storage writes after EmitVertex, or MOBILEGL_ESPRYT_DEFER_GEOMETRY_EMITS
// forced it.
Bool deferGeometryEmitsArmed = false;
// Whether EmulateSubgroupsPass lowers this (compute) module, and the workgroup-scratch
// budget it may spend (GL_MAX_COMPUTE_SHARED_MEMORY_SIZE, which decides whether it
// declines); plus whether FixIterationRPBarrierPass runs ahead of it.
Bool subgroupEmulationArmed = false;
Int subgroupEmulationScratchBytes = 0;
Bool iterationRPBarrierFixArmed = false;
// Whether that lowering reads its scratch through atomic loads (the driver POST measured
// stale shared-memory loads, or MOBILEGL_ESPRYT_ATOMIC_SUBGROUP_SCRATCH forced it).
Bool subgroupScratchAtomicReads = false;
Bool supportsNoperspectiveInterpolation = false;
// GL_NV_image_formats. Arms WidenImageFormatsForEssl, which re-declares every storage
// image whose format GLSL ES core cannot spell in the core format that carries it and