From d8576a2ed3b3cf41e27a29acaa568a93d9e4e8f6 Mon Sep 17 00:00:00 2001 From: Swung0x48 Date: Wed, 19 Aug 2026 13:40:10 +0000 Subject: [PATCH] [Fix] (DirectVulkan, ShaderTranspiler, MG_IntegrationTest, SelfTest, TraceReplay): use native subgroups and patch iterationRP's under-declared scratch iterationRP's Program 203 declares shared vec2 prefixSumCache[32] for a 512-invocation workgroup indexed by gl_SubgroupID; any device narrower than 16 lanes partitions into more than 32 subgroups and the pack writes shared memory out of bounds (heap corruption on lavapipe's CPU rasterizer, ssim 0.028 on the CI retrace). Fix it where the fault lies - in the fixture - and keep the GL contract sound everywhere else: - FixIterationRPSubgroupScratchPass: fingerprint-gated SPIR-V pass that grows exactly that array to ceil(invocations/width) entries on sub-16-lane devices; every other module passes through byte-identical. - DeriveNumSubgroupsPass stays default-on for the Adreno topology bug and is made spec-sound: pipelines request REQUIRE_FULL_SUBGROUPS whenever the workgroup shape makes the flag legal (computeFullSubgroups enabled, local_size_x a multiple of the native width, subgroup count within maxComputeWorkgroupSubgroups). - EmulateSubgroupsPass: 32-lane virtual-subgroup lowering kept in-tree as a last resort, enabled only by MOBILEGL_MAGMA_EMULATE_SUBGROUP=1 on devices with no native subgroup support; fails closed on extended subgroup instructions and on modules whose added scratch would exceed maxComputeSharedMemorySize. - IterationRPFirstReductionScenario skips gracefully outside the pack's 16..256-lane source domain; the new IterationRPScratchFixScenario runs the fixture-shaped reduction on any width and asserts the exact width-independent total. DriverPost keeps reporting FAIL on out-of-domain devices. - Program203 -> IterationRP rename throughout; the per-trace num_subgroups_quirk plumbing is removed from the trace replayer, JNI chain, and CI workflows. --- .github/workflows/apk.yml | 3 - CMakeLists.txt | 4 +- MobileGL/Config.h | 36 +- MobileGL/ConfigLoader.cpp | 5 +- .../BackendObject_DirectVulkan.cpp | 21 +- .../DirectVulkan/Renderer/ProgramFactory.cpp | 110 +- .../DirectVulkan/Renderer/ProgramFactory.h | 27 +- .../DirectVulkan/Renderer/VulkanRenderer.cpp | 81 +- .../DirectVulkan/Renderer/VulkanRenderer.h | 10 + .../DirectVulkan/SubgroupSupportPolicy.h | 57 + MobileGL/MG_IntegrationTest/CMakeLists.txt | 3 +- ... => IterationRPFirstReductionScenario.cpp} | 68 +- .../IterationRPScratchFixScenario.cpp | 222 +++ MobileGL/MG_Test/SelfTest/CMakeLists.txt | 10 +- ...p => DriverPostIterationRPWitnessTest.cpp} | 126 +- .../MG_Test/ShaderTranspiler/CMakeLists.txt | 2 + .../ShaderTranspiler/EmulateSubgroupsTest.cpp | 248 +++ .../FixIterationRPSubgroupScratchTest.cpp | 225 +++ MobileGL/MG_Util/SelfTest/DriverPost.cpp | 36 +- ...comp => DriverPostIterationRPWitness.comp} | 12 +- ...s.cpp => DriverPostIterationRPWitness.cpp} | 110 +- .../SelfTest/DriverPostIterationRPWitness.h | 153 ++ ...pv.h => DriverPostIterationRPWitnessSpv.h} | 18 +- .../SelfTest/DriverPostProgram203Witness.h | 151 -- .../ShaderTranspiler/ShaderCompiler.cpp | 28 + .../MG_Util/ShaderTranspiler/ShaderCompiler.h | 30 +- .../SpirvPasses/DeriveNumSubgroupsPass.cpp | 10 +- .../SpirvPasses/DeriveNumSubgroupsPass.h | 14 +- .../SpirvPasses/EmulateSubgroupsPass.cpp | 1413 +++++++++++++++++ .../SpirvPasses/EmulateSubgroupsPass.h | 73 + .../FixIterationRPSubgroupScratchPass.cpp | 352 ++++ .../FixIterationRPSubgroupScratchPass.h | 64 + .../app/src/trace/cpp/trace_replay_core.cpp | 6 - .../app/src/trace/cpp/trace_replay_core.hpp | 1 - .../app/src/trace/cpp/trace_replay_jni.cpp | 2 - .../plugin/trace/TraceReplayActivity.java | 8 +- android-plugin/trace-replay-ci.sh | 8 - tools/trace_replay/trace_cases.json | 3 +- 38 files changed, 3354 insertions(+), 396 deletions(-) create mode 100644 MobileGL/MG_Backend/DirectVulkan/SubgroupSupportPolicy.h rename MobileGL/MG_IntegrationTest/Scenarios/{Program203FirstReductionScenario.cpp => IterationRPFirstReductionScenario.cpp} (93%) create mode 100644 MobileGL/MG_IntegrationTest/Scenarios/IterationRPScratchFixScenario.cpp rename MobileGL/MG_Test/SelfTest/{DriverPostProgram203WitnessTest.cpp => DriverPostIterationRPWitnessTest.cpp} (51%) create mode 100644 MobileGL/MG_Test/ShaderTranspiler/EmulateSubgroupsTest.cpp create mode 100644 MobileGL/MG_Test/ShaderTranspiler/FixIterationRPSubgroupScratchTest.cpp rename MobileGL/MG_Util/SelfTest/{DriverPostProgram203Witness.comp => DriverPostIterationRPWitness.comp} (94%) rename MobileGL/MG_Util/SelfTest/{DriverPostProgram203Witness.cpp => DriverPostIterationRPWitness.cpp} (69%) create mode 100644 MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.h rename MobileGL/MG_Util/SelfTest/{DriverPostProgram203WitnessSpv.h => DriverPostIterationRPWitnessSpv.h} (97%) delete mode 100644 MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.h create mode 100644 MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.cpp create mode 100644 MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.h create mode 100644 MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.cpp create mode 100644 MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.h diff --git a/.github/workflows/apk.yml b/.github/workflows/apk.yml index e8801850..d8cb80e8 100644 --- a/.github/workflows/apk.yml +++ b/.github/workflows/apk.yml @@ -435,9 +435,6 @@ jobs: if [ "${{ matrix.case.coherent_as_flush || false }}" = "true" ]; then extra_retrace_args+=(--coherent-as-flush) fi - if [ "${{ matrix.case.num_subgroups_quirk || false }}" = "true" ]; then - extra_retrace_args+=(--num-subgroups-quirk) - fi run_retrace() { timeout "$(( ${{ matrix.case.timeout_seconds }} + 300 ))" sh android-plugin/trace-replay-ci.sh \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ffc7bca..2d0b9af6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -285,6 +285,8 @@ set(SOURCE_FILES MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/RebaseInstanceIndexPass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/ZeroBaseVertexPass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.cpp + MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.cpp + MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/NormalizeRectCoordinatesPass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/Lower1DArrayImagesPass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/BakeImageFormatsPass.cpp @@ -299,7 +301,7 @@ set(SOURCE_FILES MobileGL/MG_Util/BackendLoaders/Vulkan/Loader.cpp MobileGL/MG_Util/SelfTest/DriverPost.cpp - MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.cpp + MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.cpp MobileGL/MG_Util/Texture/PixelStoreProcessor.cpp MobileGL/MG_Util/Texture/TextureFormatProcessor.cpp diff --git a/MobileGL/Config.h b/MobileGL/Config.h index b0540a60..ff659bd0 100644 --- a/MobileGL/Config.h +++ b/MobileGL/Config.h @@ -78,13 +78,37 @@ namespace MobileGL::MG_Config { // MOBILEGL_TRACE_ANGLE_VARIANT: signed trace-APK ANGLE build short hash. String TraceAngleVariant; #endif - // MOBILEGL_DISABLE_SUBGROUP: force-disable Vulkan shader subgroup support. + // MOBILEGL_DISABLE_SUBGROUP: force-disable Vulkan shader subgroup support, + // including the opt-in emulated compute path below. Bool DisableSubgroup = false; - // MOBILEGL_NUM_SUBGROUPS_QUIRK: derive compute gl_NumSubgroups from the local - // workgroup dimensions and gl_SubgroupSize instead of reading Vulkan's - // NumSubgroups builtin. Off by default; enable only for drivers whose builtin - // disagrees with the SubgroupId topology emitted by the same dispatch. - Bool NumSubgroupsQuirk = false; + // MOBILEGL_MAGMA_EMULATE_SUBGROUP: implement GL_KHR_shader_subgroup's compute + // stage on a 32-lane VIRTUAL subgroup lowered to workgroup-shared memory + // (ShaderTranspiler::EmulateSubgroupsPass). Strictly a last resort: it only ever + // engages when this flag is set AND the device has no native subgroup support at + // all - a device with real subgroup operations always uses them natively, + // whatever their width (the known iterationRP defect is patched by + // FixIterationRPSubgroupScratch below instead). Off by default. + Bool MagmaEmulateSubgroup = false; + // MOBILEGL_FIX_ITERATIONRP_SUBGROUP_SCRATCH: patch iterationRP's own bug - the + // pack declares `shared vec2 prefixSumCache[32]` for a 512-invocation exposure + // reduction and indexes it by gl_SubgroupID, so any device with sub-16-lane + // subgroups (8-lane lavapipe -> 64 subgroups) writes shared memory out of + // bounds. The pass grows that one array to what the device's topology needs and + // touches nothing else; it only rewrites modules positively matching the pack's + // reduction fingerprint (ShaderTranspiler::FixIterationRPSubgroupScratchPass), + // so every other shader passes through byte-identical - as does iterationRP + // itself on >= 16-lane devices. Auto is ON; ForceOff replays the pack's bug + // verbatim. + QuirkOverride FixIterationRPSubgroupScratch = QuirkOverride::Auto; + // MOBILEGL_DERIVE_NUM_SUBGROUPS: replace compute gl_NumSubgroups loads with + // ceil(workgroup invocations / gl_SubgroupSize) on the NATIVE subgroup path + // (ShaderTranspiler::DeriveNumSubgroupsPass). Auto is ON: GL requires + // gl_SubgroupID < gl_NumSubgroups, Adreno's builtin reports 1 while the same + // dispatch emits IDs 0..7, and the derived value is the one Vulkan guarantees + // whenever the pipeline can request REQUIRE_FULL_SUBGROUPS (which the renderer + // does whenever local_size_x is a multiple of the native width). ForceOff returns + // to the raw driver builtin. + QuirkOverride DeriveNumSubgroups = QuirkOverride::Auto; // MOBILEGL_ADVERTISE_FP64: add GL_ARB_gpu_shader_fp64 to the advertised extension // string. `double` in a shader always WORKS - it is narrowed to 32 bits before any // module reaches a backend (ShaderTranspiler::DemoteFloat64Pass) - but the extension diff --git a/MobileGL/ConfigLoader.cpp b/MobileGL/ConfigLoader.cpp index 4e20e8e3..f23a43d5 100644 --- a/MobileGL/ConfigLoader.cpp +++ b/MobileGL/ConfigLoader.cpp @@ -168,7 +168,10 @@ namespace MobileGL::MG_ConfigLoader { QueryEnvVariable("MOBILEGL_TRACE_ANGLE_VARIANT", features.TraceAngleVariant, ""); #endif features.DisableSubgroup = QueryEnvFlag("MOBILEGL_DISABLE_SUBGROUP"); - features.NumSubgroupsQuirk = QueryEnvFlag("MOBILEGL_NUM_SUBGROUPS_QUIRK"); + features.MagmaEmulateSubgroup = QueryEnvFlag("MOBILEGL_MAGMA_EMULATE_SUBGROUP"); + features.FixIterationRPSubgroupScratch = + QueryEnvQuirkOverride("MOBILEGL_FIX_ITERATIONRP_SUBGROUP_SCRATCH"); + features.DeriveNumSubgroups = QueryEnvQuirkOverride("MOBILEGL_DERIVE_NUM_SUBGROUPS"); features.AdvertiseFp64 = QueryEnvFlag("MOBILEGL_ADVERTISE_FP64"); features.MagmaR11G11B10FFallback = QueryEnvFlag("MOBILEGL_MAGMA_R11G11B10F_FALLBACK"); features.MagmaFramesInFlight = QueryEnvUint32("MOBILEGL_MAGMA_FRAMESINFLIGHT", 3, 1, 64); diff --git a/MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.cpp b/MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.cpp index 37cc313c..b5c062cd 100644 --- a/MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.cpp @@ -9,6 +9,7 @@ #include "BackendObject_DirectVulkan.h" #include "MG_Backend/BackendObject.h" #include "DirectVulkan.h" +#include "SubgroupSupportPolicy.h" #include "MG_State/GLState/FramebufferState/FramebufferObject.h" #include "MG_State/GLState/Core.h" #include "MG_State/GLState/TextureState/TextureState.h" @@ -704,8 +705,14 @@ namespace MobileGL::MG_Backend::DirectVulkan { // real device timestamp support. ApplyVulkanCapabilitiesForTesting may // run without a renderer; no timer query is advertised then. Rebuilding // the whole list keeps re-runs idempotent. + // The opt-in emulated compute path (SubgroupSupportPolicy.h) carries the + // extension by itself on devices with no native subgroup support at all; a + // device with native subgroups always advertises - and uses - those. + const Bool subgroupSupportAdvertised = + m_vulkanCaps.SupportsShaderSubgroup || + ShouldEmulateSubgroups(m_vulkanCaps.SupportsShaderSubgroup); m_rendererInfo.RendererGLInfo.Extensions = BuildAdvertisedExtensions( - m_vulkanCaps.SupportsShaderSubgroup, pVulkanRenderer && pVulkanRenderer->IsTimerQuerySupported(), + subgroupSupportAdvertised, pVulkanRenderer && pVulkanRenderer->IsTimerQuerySupported(), pVulkanRenderer && pVulkanRenderer->IsSamplerAnisotropySupported(), pVulkanRenderer && pVulkanRenderer->IsNonZeroIndirectBaseInstanceSupported()); } @@ -941,6 +948,18 @@ namespace MobileGL::MG_Backend::DirectVulkan { m_dynamicParameters.SubgroupSupportedFeatures = mapSubgroupFeatures(m_vulkanCaps.SubgroupSupportedOperations); m_dynamicParameters.SubgroupQuadOperationsInAllStages = m_vulkanCaps.SubgroupQuadOperationsInAllStages; + } else if (ShouldEmulateSubgroups(m_vulkanCaps.SupportsShaderSubgroup)) { + // MOBILEGL_MAGMA_EMULATE_SUBGROUP on a device with no native subgroups: the + // advertised values describe the 32-lane virtual subgroup the compute + // lowering implements (SubgroupSupportPolicy.h / EmulateSubgroupsPass). + // GL requires the advertisement and the execution to agree, and on this + // path the emulation is what executes; only the compute stage is offered. + m_dynamicParameters.SubgroupSize = kEmulatedSubgroupSize; + m_dynamicParameters.SubgroupSupportedStages = kEmulatedSubgroupStages; + m_dynamicParameters.SubgroupSupportedFeatures = kEmulatedSubgroupFeatures; + m_dynamicParameters.SubgroupQuadOperationsInAllStages = false; + MGLOG_I("DirectVulkan: emulating 32-lane compute subgroups " + "(MOBILEGL_MAGMA_EMULATE_SUBGROUP, no native subgroup support)"); } else { m_dynamicParameters.SubgroupSize = 0; m_dynamicParameters.SubgroupSupportedStages = 0; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp index 151ffc1e..a09d683f 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp @@ -8,7 +8,6 @@ #include "ProgramFactory.h" -#include "Config.h" #include "MG_Backend/DirectVulkan/DirectVulkanResourceState.h" #include "MG_Util/ShaderTranspiler/ShaderCompiler.h" #include "MG_Util/ShaderTranspiler/SpvcSession.h" @@ -34,6 +33,32 @@ namespace MobileGL::MG_Backend::DirectVulkan { using SpvcSession = MG_Util::ShaderTranspiler::SpvcSession; using SessionUsageBit = MG_Util::ShaderTranspiler::SessionUsageBit; + // Local size of a compute module, read from OpExecutionMode LocalSize; all-zero + // when absent. The compile chain pins SPIR-V 1.3, where a literal local size + // always reaches the module as this execution mode (LocalSizeId does not exist + // yet). + struct ComputeLocalSize { + Uint32 x = 0; + Uint32 y = 0; + Uint32 z = 0; + Uint64 Total() const { return static_cast(x) * y * z; } + }; + ComputeLocalSize TryGetComputeLocalSize(const Vector& spirv) { + constexpr SizeT kHeaderWords = 5; + constexpr Uint32 kOpExecutionMode = 16; + constexpr Uint32 kModeLocalSize = 17; + for (SizeT offset = kHeaderWords; offset < spirv.size();) { + const Uint32 wordCount = spirv[offset] >> 16u; + const Uint32 opcode = spirv[offset] & 0xffffu; + if (wordCount == 0 || offset + wordCount > spirv.size()) break; + if (opcode == kOpExecutionMode && wordCount >= 6 && spirv[offset + 2] == kModeLocalSize) { + return {spirv[offset + 3], spirv[offset + 4], spirv[offset + 5]}; + } + offset += wordCount; + } + return {}; + } + struct DescriptorKey { ProgramFactory::DescriptorBindingKind kind = ProgramFactory::DescriptorBindingKind::None; String name; @@ -3164,20 +3189,56 @@ namespace MobileGL::MG_Backend::DirectVulkan { } } - // NumSubgroups is defined by the local workgroup dimensions and SubgroupSize. Derive - // it in SPIR-V instead of trusting a driver builtin that can disagree with the - // SubgroupId topology produced by the same compute dispatch (Adreno reports 1 while - // emitting IDs 0..7 for a 512-invocation, 64-wide workgroup). - if (MG_Config::Features.NumSubgroupsQuirk && shaders[i] && - shaders[i]->GetShaderStage() == ShaderStage::Compute) { - Vector derivedNumSubgroupsSpirv; - if (MG_Util::ShaderTranspiler::ShaderCompiler::DeriveNumSubgroupsForVulkan( - moduleSpirvs[i], derivedNumSubgroupsSpirv, enableSpirvValidation)) { - moduleSpirvs[i] = std::move(derivedNumSubgroupsSpirv); + // GL_KHR_shader_subgroup handling (SubgroupSupportPolicy.h). Native subgroup + // operations execute natively; two module repairs keep the GL contract intact + // around them. The opt-in emulation path replaces them only on devices with no + // subgroup support at all (MOBILEGL_MAGMA_EMULATE_SUBGROUP). + if (shaders[i] && shaders[i]->GetShaderStage() == ShaderStage::Compute) { + if (m_subgroupPolicy.emulateSubgroups) { + Vector emulatedSpirv; + if (MG_Util::ShaderTranspiler::ShaderCompiler::EmulateSubgroupsForVulkan( + moduleSpirvs[i], emulatedSpirv, + m_subgroupPolicy.maxComputeSharedMemoryBytes, enableSpirvValidation)) { + moduleSpirvs[i] = std::move(emulatedSpirv); + } else { + MGLOG_E("ProgramFactory: subgroup emulation failed for program %u; the " + "module keeps subgroup operations the device cannot execute", + program.GetExternalIndex()); + } } else { - MGLOG_E("ProgramFactory: failed to derive gl_NumSubgroups for program %u; " - "compute shaders may observe a driver-inconsistent subgroup count", - program.GetExternalIndex()); + // iterationRP under-declares its cross-subgroup scratch + // (prefixSumCache[32] for 512 invocations); on a sub-16-lane device + // grow that one fingerprinted array to what the topology needs. + if (m_subgroupPolicy.fixIterationRPSubgroupScratch) { + Vector patchedSpirv; + if (MG_Util::ShaderTranspiler::ShaderCompiler::FixIterationRPSubgroupScratchForVulkan( + moduleSpirvs[i], patchedSpirv, m_subgroupPolicy.nativeSubgroupSize, + enableSpirvValidation)) { + moduleSpirvs[i] = std::move(patchedSpirv); + } else { + MGLOG_E("ProgramFactory: iterationRP subgroup scratch patch failed for " + "program %u; the pack's declared array sizes stay in effect", + program.GetExternalIndex()); + } + } + // gl_NumSubgroups must agree with the gl_SubgroupID range GL promises; + // derive it from the workgroup dimensions and gl_SubgroupSize instead of + // trusting a driver builtin that can disagree with the topology the same + // dispatch emits (Adreno reports 1 while emitting IDs 0..7 for a + // 512-invocation, 64-wide workgroup). The ceil() partition this derives + // is pinned by REQUIRE_FULL_SUBGROUPS at pipeline creation whenever the + // workgroup shape makes that flag legal (see the stage setup below). + if (m_subgroupPolicy.deriveNumSubgroups) { + Vector derivedNumSubgroupsSpirv; + if (MG_Util::ShaderTranspiler::ShaderCompiler::DeriveNumSubgroupsForVulkan( + moduleSpirvs[i], derivedNumSubgroupsSpirv, enableSpirvValidation)) { + moduleSpirvs[i] = std::move(derivedNumSubgroupsSpirv); + } else { + MGLOG_E("ProgramFactory: failed to derive gl_NumSubgroups for program %u; " + "compute shaders may observe a driver-inconsistent subgroup count", + program.GetExternalIndex()); + } + } } } @@ -3326,6 +3387,27 @@ namespace MobileGL::MG_Backend::DirectVulkan { stage.stage = ToVkStage(shaderStage); stage.module = module; stage.pName = "main"; + // Pin the full-subgroup launch the derived gl_NumSubgroups assumes. Legal + // exactly when the computeFullSubgroups feature is enabled and local_size_x is + // a multiple of the subgroup size (VUID-VkPipelineShaderStageCreateInfo- + // flags-02759/-02785), and only worth requesting while the resulting subgroup + // count fits the device's maxComputeWorkgroupSubgroups (lavapipe caps it at + // 32, below a 512-invocation dispatch's 64). With the bit set, "Full + // Subgroups" guarantees every subgroup launches with all invocations active, + // making the subgroup count exactly invocations / size. Shapes the flag + // cannot cover (e.g. 32x16 on a 64-wide device) fall back to the driver's + // own - spec-encouraged - tight partitioning, which the DriverPost witness + // verifies per device. + if (shaderStage == ShaderStage::Compute && m_subgroupPolicy.requireFullSubgroups && + !m_subgroupPolicy.emulateSubgroups && m_subgroupPolicy.nativeSubgroupSize != 0) { + const ComputeLocalSize localSize = TryGetComputeLocalSize(moduleSpv); + const Uint64 fullSubgroupCount = + localSize.Total() / m_subgroupPolicy.nativeSubgroupSize; + if (localSize.x != 0 && localSize.x % m_subgroupPolicy.nativeSubgroupSize == 0 && + fullSubgroupCount <= m_subgroupPolicy.maxComputeWorkgroupSubgroups) { + stage.flags |= VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT; + } + } entry.modules.push_back(module); entry.stages.push_back(stage); diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.h index e64e5895..560aec89 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.h @@ -372,16 +372,38 @@ namespace MobileGL::MG_Backend::DirectVulkan { virtual void OnProgramEvicted(HashType programHash, VkDescriptorSetLayout descriptorSetLayout) = 0; }; + // How this factory's compute modules implement GL_KHR_shader_subgroup. Computed + // once at renderer initialization (SubgroupSupportPolicy.h + the device's + // subgroup properties) so lowering can never disagree with the advertised + // capabilities. Native subgroup operations always execute natively; the two + // repair passes patch modules AROUND them, and the emulation only replaces them + // on opted-in devices with no subgroup support at all. + struct SubgroupLoweringPolicy { + Bool emulateSubgroups = false; // MOBILEGL_MAGMA_EMULATE_SUBGROUP, no-native-support devices + Bool fixIterationRPSubgroupScratch = false; // patch iterationRP's under-declared scratch + Bool deriveNumSubgroups = false; // repair the NumSubgroups builtin + Bool requireFullSubgroups = false; // computeFullSubgroups enabled on the device + Uint32 nativeSubgroupSize = 0; + // Full-subgroup launches are bounded by this device limit; a dispatch whose + // workgroup needs more subgroups than this cannot request the flag. + Uint32 maxComputeWorkgroupSubgroups = 0; + // VkPhysicalDeviceLimits::maxComputeSharedMemorySize; bounds the scratch the + // emulation pass may add (0 falls back to the Vulkan minimum, 16384). + Uint32 maxComputeSharedMemoryBytes = 0; + }; + explicit ProgramFactory(VkDevice device, const VulkanRendererConfig& config, Uint32 maxBindings, Bool shaderDrawParametersEnabled, Bool unformattedFloatStorageImagesEnabled, Bool enableSpirvValidation, - UpdateAfterBindLimits updateAfterBindLimits) + UpdateAfterBindLimits updateAfterBindLimits, + SubgroupLoweringPolicy subgroupPolicy) : m_device(device), m_maxBindings(maxBindings), m_config(config), m_shaderDrawParametersEnabled(shaderDrawParametersEnabled), m_unformattedFloatStorageImagesEnabled(unformattedFloatStorageImagesEnabled), m_enableSpirvValidation(enableSpirvValidation), - m_updateAfterBindLimits(updateAfterBindLimits) { + m_updateAfterBindLimits(updateAfterBindLimits), + m_subgroupPolicy(subgroupPolicy) { VkProgramObject::s_device = device; } // Destroys the pass-through tessellation control modules. Runs while the device is @@ -511,6 +533,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { // the factory lets each reflected layout choose ordinary descriptors when its // own counts would exceed the update-after-bind budget. UpdateAfterBindLimits m_updateAfterBindLimits{}; + SubgroupLoweringPolicy m_subgroupPolicy{}; // See SetDefaultFramebufferHeight. 0 means "not known yet"; the FragCoordYFlip bit is // never set before the swapchain exists, so no variant can be compiled against it. Uint32 m_defaultFramebufferHeight = 0; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp index 85ec4a46..92b9469a 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp @@ -8,6 +8,7 @@ #include "VulkanRenderer.h" +#include "MG_Backend/DirectVulkan/SubgroupSupportPolicy.h" #include "MG_Backend/DirectGLES/Utils.h" #include "VertexInputStateFactory.h" #include "VertexInputStateBuilder.h" @@ -3058,11 +3059,22 @@ void main() { } PipelineFactory::SetSuppressBlendedDepthWrite(suppressBlendedDepthWrite); } + ProgramFactory::SubgroupLoweringPolicy subgroupPolicy{}; + subgroupPolicy.emulateSubgroups = ShouldEmulateSubgroups(m_nativeSubgroupSupported); + subgroupPolicy.fixIterationRPSubgroupScratch = + m_nativeSubgroupSupported && ShouldFixIterationRPSubgroupScratch(); + subgroupPolicy.deriveNumSubgroups = + m_nativeSubgroupSupported && ShouldDeriveNumSubgroups(); + subgroupPolicy.requireFullSubgroups = m_computeFullSubgroupsFeatureEnabled; + subgroupPolicy.nativeSubgroupSize = m_nativeSubgroupSize; + subgroupPolicy.maxComputeWorkgroupSubgroups = m_maxComputeWorkgroupSubgroups; + subgroupPolicy.maxComputeSharedMemoryBytes = + m_physicalDevice.properties.limits.maxComputeSharedMemorySize; m_programFactory = MakeUnique(m_device, m_config, maxProgramBindings, m_shaderDrawParametersFeatureEnabled, m_unformattedFloatStorageImagesEnabled, MG_Config::Features.EnableSpirvValidation, - m_updateAfterBindLimits); + m_updateAfterBindLimits, subgroupPolicy); MOBILEGL_ASSERT(m_programFactory != nullptr, "ProgramFactory creation failed."); // The swapchain already exists at this point (Initialize creates it first), so seed the // height the factory could not be told about from CreateSwapchain. @@ -12740,6 +12752,73 @@ void main() { } } + // Native subgroup topology, and VK_EXT_subgroup_size_control's + // computeFullSubgroups feature. REQUIRE_FULL_SUBGROUPS on a compute stage is what + // turns the derived gl_NumSubgroups (DeriveNumSubgroupsPass) from + // encouraged-but-unspecified driver behaviour into a spec guarantee: with the bit + // set and local_size_x a multiple of the subgroup size, every subgroup launches + // full, so the subgroup count is exactly invocations / size ("Full Subgroups", + // VUID-VkPipelineShaderStageCreateInfo-flags-02759/-02785). + m_nativeSubgroupSize = 0; + m_nativeSubgroupSupported = false; + m_computeFullSubgroupsFeatureEnabled = false; + if (getPhysicalDeviceProperties2 != nullptr) { + VkPhysicalDeviceSubgroupProperties subgroupProperties{}; + subgroupProperties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES; + VkPhysicalDeviceProperties2 subgroupPropertyQuery{}; + subgroupPropertyQuery.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; + subgroupPropertyQuery.pNext = &subgroupProperties; + getPhysicalDeviceProperties2(m_physicalDevice.handle, &subgroupPropertyQuery); + // Mirrors the loader's HasUsableShaderSubgroupSupport gate, including the + // MOBILEGL_DISABLE_SUBGROUP escape hatch, so the module lowerings can never + // disagree with the advertised capabilities. + const Bool usableSubgroups = + subgroupProperties.subgroupSize > 0 && + (subgroupProperties.supportedStages & VK_SHADER_STAGE_COMPUTE_BIT) != 0 && + (subgroupProperties.supportedOperations & VK_SUBGROUP_FEATURE_BASIC_BIT) != 0; + if (usableSubgroups && !MG_Config::Features.DisableSubgroup) { + m_nativeSubgroupSize = subgroupProperties.subgroupSize; + m_nativeSubgroupSupported = true; + } + } + VkPhysicalDeviceSubgroupSizeControlFeaturesEXT subgroupSizeControlFeatures{}; + subgroupSizeControlFeatures.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES_EXT; + m_maxComputeWorkgroupSubgroups = 0; + if (m_nativeSubgroupSupported && + IsExtensionSupported(availableExtensions, VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME) && + getPhysicalDeviceFeatures2 != nullptr) { + VkPhysicalDeviceFeatures2 featureQuery{}; + featureQuery.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; + featureQuery.pNext = &subgroupSizeControlFeatures; + getPhysicalDeviceFeatures2(m_physicalDevice.handle, &featureQuery); + if (getPhysicalDeviceProperties2 != nullptr) { + VkPhysicalDeviceSubgroupSizeControlPropertiesEXT subgroupSizeControlProperties{}; + subgroupSizeControlProperties.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_PROPERTIES_EXT; + VkPhysicalDeviceProperties2 propertyQuery{}; + propertyQuery.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; + propertyQuery.pNext = &subgroupSizeControlProperties; + getPhysicalDeviceProperties2(m_physicalDevice.handle, &propertyQuery); + m_maxComputeWorkgroupSubgroups = + subgroupSizeControlProperties.maxComputeWorkgroupSubgroups; + } + if (subgroupSizeControlFeatures.computeFullSubgroups == VK_TRUE) { + if (!IsExtensionAlreadyEnabled(enabledDeviceExtensions, + VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME)) { + enabledDeviceExtensions.push_back(VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME); + } + // Only the full-subgroups guarantee is wanted; required/varying subgroup + // sizes stay unrequested. + subgroupSizeControlFeatures.subgroupSizeControl = VK_FALSE; + subgroupSizeControlFeatures.pNext = const_cast(deviceCreateInfo.pNext); + deviceCreateInfo.pNext = &subgroupSizeControlFeatures; + m_computeFullSubgroupsFeatureEnabled = true; + MGLOG_I("Enabled optional device extension: %s (computeFullSubgroups)", + VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME); + } + } + // VK_EXT_transform_feedback backs GL transform feedback capture. m_transformFeedbackFeatureEnabled = false; VkPhysicalDeviceTransformFeedbackFeaturesEXT transformFeedbackFeatures{}; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h index 06b23605..d021df94 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h @@ -554,6 +554,16 @@ namespace MobileGL::MG_Backend::DirectVulkan { Bool m_samplerAnisotropyFeatureEnabled = false; Bool m_shaderDrawParametersExtensionEnabled = false; Bool m_shaderDrawParametersFeatureEnabled = false; + // Native subgroup topology, queried at device creation for the compute-module + // subgroup repairs (SubgroupSupportPolicy.h) and the REQUIRE_FULL_SUBGROUPS + // stage flag; 0 / false when the device has no usable compute subgroups or + // MOBILEGL_DISABLE_SUBGROUP forced them off. + Uint32 m_nativeSubgroupSize = 0; + Bool m_nativeSubgroupSupported = false; + Bool m_computeFullSubgroupsFeatureEnabled = false; + // VkPhysicalDeviceSubgroupSizeControlProperties::maxComputeWorkgroupSubgroups; + // 0 when the extension (and therefore the full-subgroups flag) is unavailable. + Uint32 m_maxComputeWorkgroupSubgroups = 0; Bool m_unformattedFloatStorageImagesEnabled = false; // Set only after descriptor-indexing feature AND property queries prove that // update-after-bind is legal for every descriptor category this renderer emits. diff --git a/MobileGL/MG_Backend/DirectVulkan/SubgroupSupportPolicy.h b/MobileGL/MG_Backend/DirectVulkan/SubgroupSupportPolicy.h new file mode 100644 index 00000000..0f06e7bc --- /dev/null +++ b/MobileGL/MG_Backend/DirectVulkan/SubgroupSupportPolicy.h @@ -0,0 +1,57 @@ +// MobileGL - MobileGL/MG_Backend/DirectVulkan/SubgroupSupportPolicy.h +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#pragma once + +#include +#include + +namespace MobileGL::MG_Backend::DirectVulkan { + // The single decision point for how DirectVulkan implements GL_KHR_shader_subgroup, + // shared by capability advertisement (BackendObject) and module lowering + // (VulkanRenderer / ProgramFactory) so the two can never disagree. + // + // Native subgroups are the implementation whenever the device has them, whatever + // their width - subgroup operations execute on the hardware paths they were made + // for. Two module-level repairs keep the GL contract intact around them: + // - FixIterationRPSubgroupScratchPass patches the one known pack bug: iterationRP's + // prefixSumCache[32], under-declared for sub-16-lane devices (8-lane lavapipe); + // - DeriveNumSubgroupsPass replaces the one builtin drivers get wrong + // (gl_NumSubgroups) with the value the rest of the topology implies. + // The 32-lane shared-memory emulation (EmulateSubgroupsPass) is a LAST RESORT for + // devices with no subgroup support at all, and only when the user opts in with + // MOBILEGL_MAGMA_EMULATE_SUBGROUP=1; it never replaces available native operations. + + inline constexpr Uint32 kEmulatedSubgroupSize = 32u; + inline constexpr Uint32 kEmulatedSubgroupStages = GL_COMPUTE_SHADER_BIT; + inline constexpr Uint32 kEmulatedSubgroupFeatures = + GL_SUBGROUP_FEATURE_BASIC_BIT_KHR | GL_SUBGROUP_FEATURE_VOTE_BIT_KHR | + GL_SUBGROUP_FEATURE_ARITHMETIC_BIT_KHR | GL_SUBGROUP_FEATURE_BALLOT_BIT_KHR | + GL_SUBGROUP_FEATURE_SHUFFLE_BIT_KHR | GL_SUBGROUP_FEATURE_SHUFFLE_RELATIVE_BIT_KHR | + GL_SUBGROUP_FEATURE_CLUSTERED_BIT_KHR | GL_SUBGROUP_FEATURE_QUAD_BIT_KHR; + + inline Bool ShouldEmulateSubgroups(const Bool nativeSubgroupSupported) { + return MG_Config::Features.MagmaEmulateSubgroup && !nativeSubgroupSupported && + !MG_Config::Features.DisableSubgroup; + } + + inline Bool ShouldFixIterationRPSubgroupScratch() { + // Auto is ON: the patch is fingerprint-gated to iterationRP's reduction and + // grows one under-declared array; every other module passes through untouched. + return MG_Config::Features.FixIterationRPSubgroupScratch != + MG_Config::QuirkOverride::ForceOff; + } + + inline Bool ShouldDeriveNumSubgroups() { + // Auto is ON: gl_NumSubgroups must agree with the gl_SubgroupID range for the GL + // contract to hold, and the derived ceil() value is the one the renderer can pin + // with REQUIRE_FULL_SUBGROUPS - the driver builtin is the value with no + // cross-driver guarantee (Adreno returns 1 for an 8-subgroup dispatch). + return MG_Config::Features.DeriveNumSubgroups != MG_Config::QuirkOverride::ForceOff; + } +} // namespace MobileGL::MG_Backend::DirectVulkan diff --git a/MobileGL/MG_IntegrationTest/CMakeLists.txt b/MobileGL/MG_IntegrationTest/CMakeLists.txt index 1c234208..4584ab73 100644 --- a/MobileGL/MG_IntegrationTest/CMakeLists.txt +++ b/MobileGL/MG_IntegrationTest/CMakeLists.txt @@ -68,7 +68,8 @@ add_executable(MobileGLIntegrationTest Scenarios/DoublePrecisionScenario.cpp Scenarios/UniformInitializerScenario.cpp Scenarios/SwizzleAccessRoutineScenario.cpp - Scenarios/Program203FirstReductionScenario.cpp + Scenarios/IterationRPFirstReductionScenario.cpp + Scenarios/IterationRPScratchFixScenario.cpp Scenarios/ProgramPipelineScenario.cpp Scenarios/ImageLoadStoreSsoScenario.cpp Scenarios/ImageTargetKindScenario.cpp diff --git a/MobileGL/MG_IntegrationTest/Scenarios/Program203FirstReductionScenario.cpp b/MobileGL/MG_IntegrationTest/Scenarios/IterationRPFirstReductionScenario.cpp similarity index 93% rename from MobileGL/MG_IntegrationTest/Scenarios/Program203FirstReductionScenario.cpp rename to MobileGL/MG_IntegrationTest/Scenarios/IterationRPFirstReductionScenario.cpp index 32539f0d..19bdb51b 100644 --- a/MobileGL/MG_IntegrationTest/Scenarios/Program203FirstReductionScenario.cpp +++ b/MobileGL/MG_IntegrationTest/Scenarios/IterationRPFirstReductionScenario.cpp @@ -1,4 +1,4 @@ -// MobileGL - MobileGL/MG_IntegrationTest/Scenarios/Program203FirstReductionScenario.cpp +// MobileGL - MobileGL/MG_IntegrationTest/Scenarios/IterationRPFirstReductionScenario.cpp // Copyright (c) 2026 MobileGL-Dev // Licensed under the GNU Lesser General Public License v3.0: // https://www.gnu.org/licenses/gpl-3.0.txt @@ -6,9 +6,9 @@ // SPDX-License-Identifier: LGPL-3.0-only // End of Source File Header // -// Scenario - PROGRAM 203'S FIRST SUBGROUP REDUCTION. +// Scenario - ITERATIONRP'S FIRST SUBGROUP REDUCTION. // -// Program 203 reduces a 32 x 16 exposure tile with a vector subgroup inclusive add, +// iterationRP reduces a 32 x 16 exposure tile with a vector subgroup inclusive add, // then a shared-memory scan of subgroup totals. The source assumes that every // subgroup has a last lane, that there are 2..32 subgroups, and that local index // 511 belongs to the last subgroup and its last lane. Those are source assumptions, @@ -133,6 +133,17 @@ namespace MGITest { std::array maxWorkGroupSize{}; bool queryHadError = false; + // iterationRP's source contract needs gl_NumSubgroups in [2, 32] for its 512 + // invocations, i.e. an advertised subgroup width in [16, 256]. A device + // outside that window (lavapipe's 8-lane subgroups give 64 subgroups) cannot + // run the fixture's verbatim reduction at all, so the scenario SKIPS there - + // the pack itself replays through the FixIterationRPSubgroupScratch patch, which + // this probe deliberately does not model. The width only gates the domain; + // lane placement and group counts still come from observed values alone. + bool SubgroupWidthInSourceDomain() const { + return subgroupSize >= 16 && subgroupSize <= 256; + } + bool SupportsProbe() const { const auto stages = static_cast(supportedStages); const auto features = static_cast(supportedFeatures); @@ -140,6 +151,7 @@ namespace MGITest { (stages & GL_COMPUTE_SHADER_BIT) != 0 && (features & (GL_SUBGROUP_FEATURE_BASIC_BIT_KHR | GL_SUBGROUP_FEATURE_ARITHMETIC_BIT_KHR)) == (GL_SUBGROUP_FEATURE_BASIC_BIT_KHR | GL_SUBGROUP_FEATURE_ARITHMETIC_BIT_KHR) && + SubgroupWidthInSourceDomain() && maxComputeStorageBlocks >= 2 && maxStorageBindings >= 2 && maxWorkGroupInvocations >= static_cast(kInvocationCount) && maxWorkGroupSize[0] >= 32 && maxWorkGroupSize[1] >= 16 && maxWorkGroupSize[2] >= 1; @@ -159,6 +171,12 @@ namespace MGITest { if ((features & requiredFeatures) != requiredFeatures) { missing.emplace_back("basic|arithmetic in GL_SUBGROUP_SUPPORTED_FEATURES_KHR"); } + if (!SubgroupWidthInSourceDomain()) { + missing.emplace_back( + "GL_SUBGROUP_SIZE_KHR in [16, 256] (iterationRP's source contract needs " + "gl_NumSubgroups in [2, 32] for 512 invocations; width " + + std::to_string(subgroupSize) + " is outside the fixture's domain)"); + } if (maxComputeStorageBlocks < 2 || maxStorageBindings < 2) { missing.emplace_back("two compute SSBO bindings"); } @@ -194,7 +212,7 @@ namespace MGITest { } void PrintMetadata(const CapabilityInfo& info, std::ostream& output) { - output << "Program203FirstReductionScenario metadata: " + output << "IterationRPFirstReductionScenario metadata: " << "GL_SUBGROUP_SIZE_KHR=" << info.subgroupSize << ", GL_SUBGROUP_SUPPORTED_STAGES_KHR=0x" << std::hex << static_cast(info.supportedStages) @@ -243,7 +261,7 @@ layout(std430, binding = 0) readonly buffer Input { )"; // Only the expression producing tileExposure differs between the two - // tests. The remainder is the program-203 first reduction, with stores + // tests. The remainder is the iterationRP first reduction, with stores // placed after its existing barriers to expose each handoff. constexpr const char* kSampledTileExposure = R"( vec2 texCoord = (vec2(gl_GlobalInvocationID.xy) + 0.5) * @@ -506,7 +524,7 @@ layout(std430, binding = 0) readonly buffer Input { if (!IsQuietNanSentinel(reduction.z) || !IsQuietNanSentinel(reduction.w) || !IsQuietNanSentinel(output.finalAverage[slot])) { std::ostringstream message; - message << "program 203 source reduction has no valid contract for gl_NumSubgroups=" + message << "iterationRP source reduction has no valid contract for gl_NumSubgroups=" << reportedNumSubgroups << "; localIndex " << localIndex << " did not preserve its qNaN source-reduction sentinel"; return Failure("source domain", message.str()); @@ -514,7 +532,7 @@ layout(std430, binding = 0) readonly buffer Input { for (std::size_t stage = 0; stage < kScanStageCount; ++stage) { if (!IsQuietNanSentinel(output.scanAfter[stage][slot])) { std::ostringstream message; - message << "program 203 source reduction has no valid contract for gl_NumSubgroups=" + message << "iterationRP source reduction has no valid contract for gl_NumSubgroups=" << reportedNumSubgroups << "; localIndex " << localIndex << ", scan stage " << stage << " did not preserve its qNaN source-reduction sentinel"; return Failure("source domain", message.str()); @@ -522,12 +540,12 @@ layout(std430, binding = 0) readonly buffer Input { } } std::ostringstream message; - message << "program 203 source reduction has no valid contract for observed gl_NumSubgroups=" + message << "iterationRP source reduction has no valid contract for observed gl_NumSubgroups=" << reportedNumSubgroups << " (requires 2..32); native subgroup results were recorded"; return Failure("source domain", message.str()); } - // 4. Program-203 source writer and first shared-memory handoff. + // 4. iterationRP source writer and first shared-memory handoff. std::vector sourceWriter(reportedNumSubgroups, kNoSlot); for (std::uint32_t subgroupID = 0; subgroupID < reportedNumSubgroups; ++subgroupID) { std::size_t writerCount = 0; @@ -541,7 +559,7 @@ layout(std430, binding = 0) readonly buffer Input { if (writerCount != 1u) { std::ostringstream message; message << "subgroupID " << subgroupID << " has " << writerCount - << " recorded lane(s) where laneID == subgroupSize - 1; program 203 leaves that " + << " recorded lane(s) where laneID == subgroupSize - 1; iterationRP leaves that " "shared-cache entry unwritten"; return Failure("source writer", message.str()); } @@ -633,7 +651,7 @@ layout(std430, binding = 0) readonly buffer Input { index511Subgroup.z == ownerResult.highestObservedSubgroup; if (!ownerResult.index511IsSourceLastLaneWriter || !ownerResult.index511IsHighestSubgroupMember) { std::ostringstream message; - message << "program 203 topology incompatibility: localIndex 511 is sourceLastLaneWriter=" + message << "iterationRP topology incompatibility: localIndex 511 is sourceLastLaneWriter=" << ownerResult.index511IsSourceLastLaneWriter << ", highestSubgroupMember=" << ownerResult.index511IsHighestSubgroupMember << " (subgroupID=" << index511Subgroup.z << ", highest observed subgroupID=" << ownerResult.highestObservedSubgroup << ')'; @@ -650,7 +668,7 @@ layout(std430, binding = 0) readonly buffer Input { const float expectedTotal = mode == InputMode::IndexedSsbo ? 131328.0f : sampledExpectedTotal; if (!SameBits(total, expectedTotal) || !SameBits(mergedPrefix[index511Slot], expectedTotal)) { std::ostringstream message; - message << "program 203 source total was " << FormatFloat(mergedPrefix[index511Slot]) + message << "iterationRP source total was " << FormatFloat(mergedPrefix[index511Slot]) << " (native total " << FormatFloat(total) << "), expected " << FormatFloat(expectedTotal); ownerResult.ok = false; ownerResult.phase = "final average"; @@ -675,9 +693,9 @@ layout(std430, binding = 0) readonly buffer Input { bool includeScanStages) { PrintMetadata(capabilities, std::cout); if (validation.ok) { - std::cout << "Program203FirstReductionScenario firstFailure=none\n"; + std::cout << "IterationRPFirstReductionScenario firstFailure=none\n"; } else { - std::cout << "Program203FirstReductionScenario firstFailure=" << validation.phase << ": " + std::cout << "IterationRPFirstReductionScenario firstFailure=" << validation.phase << ": " << validation.message << '\n'; } std::cout << "localIndex,localX,localY,localZ,subgroupSize,numSubgroups,subgroupID,laneID,input," @@ -702,17 +720,19 @@ layout(std430, binding = 0) readonly buffer Input { } } - class Program203FirstReductionScenario : public ScenarioTest { + class IterationRPFirstReductionScenario : public ScenarioTest { protected: void SetUp() override { ScenarioTest::SetUp(); if (!Ready()) return; m_capabilities = QueryCapabilities(); - // GL_SUBGROUP_SIZE_KHR is diagnostic only. It is deliberately - // never used to infer lane placement or an expected group count. + // GL_SUBGROUP_SIZE_KHR gates only whether the fixture's source contract + // can hold on this device (SubgroupWidthInSourceDomain); it is + // deliberately never used to infer lane placement or an expected group + // count - those come from observed values alone. PrintMetadata(m_capabilities, std::cout); - RecordProperty("program203_gl_subgroup_size_khr", std::to_string(m_capabilities.subgroupSize)); + RecordProperty("iterationrp_gl_subgroup_size_khr", std::to_string(m_capabilities.subgroupSize)); if (!m_capabilities.SupportsProbe()) { GTEST_SKIP() << "subgroup probe requires " << m_capabilities.MissingRequirements(); } @@ -839,13 +859,13 @@ layout(std430, binding = 0) readonly buffer Input { const ValidationResult validation = ValidateProbe(output, mode); if (validation.ownerEvaluated) { - RecordProperty("program203_index511_source_last_lane_writer", + RecordProperty("iterationrp_index511_source_last_lane_writer", validation.index511IsSourceLastLaneWriter ? "true" : "false"); - RecordProperty("program203_index511_highest_subgroup_member", + RecordProperty("iterationrp_index511_highest_subgroup_member", validation.index511IsHighestSubgroupMember ? "true" : "false"); - RecordProperty("program203_highest_observed_subgroup", + RecordProperty("iterationrp_highest_observed_subgroup", std::to_string(validation.highestObservedSubgroup)); - std::cout << "Program203FirstReductionScenario owner: localIndex511 sourceLastLaneWriter=" + std::cout << "IterationRPFirstReductionScenario owner: localIndex511 sourceLastLaneWriter=" << validation.index511IsSourceLastLaneWriter << ", highestSubgroupMember=" << validation.index511IsHighestSubgroupMember << ", highestObservedSubgroup=" << validation.highestObservedSubgroup << '\n'; @@ -865,12 +885,12 @@ layout(std430, binding = 0) readonly buffer Input { } // namespace - TEST_F(Program203FirstReductionScenario, SampledRgba32fFirstAverage) { + TEST_F(IterationRPFirstReductionScenario, SampledRgba32fFirstAverage) { if (!Ready() || IsSkipped()) return; RunAndValidate(InputMode::SampledRgba32f); } - TEST_F(Program203FirstReductionScenario, IndexedInputTopologyAndReduction) { + TEST_F(IterationRPFirstReductionScenario, IndexedInputTopologyAndReduction) { if (!Ready() || IsSkipped()) return; RunAndValidate(InputMode::IndexedSsbo); } diff --git a/MobileGL/MG_IntegrationTest/Scenarios/IterationRPScratchFixScenario.cpp b/MobileGL/MG_IntegrationTest/Scenarios/IterationRPScratchFixScenario.cpp new file mode 100644 index 00000000..89db6321 --- /dev/null +++ b/MobileGL/MG_IntegrationTest/Scenarios/IterationRPScratchFixScenario.cpp @@ -0,0 +1,222 @@ +// MobileGL - MobileGL/MG_IntegrationTest/Scenarios/IterationRPScratchFixScenario.cpp +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header +// +// Scenario - THE FIXTURE-SHAPED SUBGROUP REDUCTION, ON WHATEVER WIDTH THE DEVICE HAS. +// +// iterationRP's auto-exposure pass declares `shared vec2 prefixSumCache[32]` for a +// 512-invocation workgroup and combines per-subgroup subtotals through +// prefixSumCache[gl_SubgroupID]. The algorithm is width-agnostic; only the static 32 +// bakes in "at most 32 subgroups", which every desktop capture satisfies and an 8-lane +// device (lavapipe: 64 subgroups) does not. DirectVulkan patches exactly that with +// FixIterationRPSubgroupScratchPass, growing the array to ceil(invocations / native +// width) on the modules that match the pack's reduction fingerprint. +// +// This scenario replays the fixture's reduction shape verbatim - the same 32-entry +// declaration, the same last-lane handoff, the same findMSB combine loop, and NO +// domain guard - and asserts only the width-independent result: the workgroup total. +// The inputs are small integers, so the fp32 sum is exact under any lane order and any +// association; a correct run produces the exact constant on a 4-lane device and a +// 128-lane device alike. Without the patch, a sub-16-lane device indexes the +// 32-entry array out of bounds - on lavapipe that is literal heap corruption - and +// this scenario is the regression test that keeps the patch working, and it runs on every device that +// has basic+arithmetic compute subgroups (unlike IterationRPFirstReductionScenario, +// which probes the UNREPAIRED source contract and must skip outside [16, 256]). + +#include +#include +#include + +#include "../Harness/HeadlessGL.h" +#include "../Harness/ScenarioFixture.h" + +#ifdef GLAPI +#undef GLAPI +#endif +#define GL_GLEXT_PROTOTYPES +#include +#include +#undef GL_GLEXT_PROTOTYPES + +namespace MGITest { + namespace { + constexpr std::uint32_t kInvocationCount = 512u; + // sum of 0..511, exactly representable and associativity-proof in fp32. + constexpr float kExpectedTotal = 130816.0f; + + constexpr const char* kComputeSource = R"(#version 430 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require + +layout(local_size_x = 32, local_size_y = 16, local_size_z = 1) in; + +layout(std430, binding = 0) buffer Output { + float total; + uint numSubgroups; + uint maxSubgroupId; +} outputData; + +shared vec2 prefixSumCache[32]; + +void main() { + vec2 sampleLuminance = vec2(float(gl_LocalInvocationIndex), 0.0); + sampleLuminance = subgroupInclusiveAdd(sampleLuminance); + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) + prefixSumCache[gl_SubgroupID] = sampleLuminance; + barrier(); + + uint loopLength = uint(findMSB(gl_NumSubgroups)); + loopLength += uint(gl_NumSubgroups - (1u << (loopLength - 1u)) > 0u); + + for (uint scanStage = 0u; scanStage < loopLength; ++scanStage) { + if ((gl_SubgroupID & (1u << scanStage)) > 0u) { + sampleLuminance += prefixSumCache[(gl_SubgroupID >> scanStage << scanStage) - 1u]; + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) + prefixSumCache[gl_SubgroupID] = sampleLuminance; + } + barrier(); + } + + if (gl_LocalInvocationIndex == 511u) { + outputData.total = sampleLuminance.x; + outputData.numSubgroups = gl_NumSubgroups; + } + atomicMax(outputData.maxSubgroupId, gl_SubgroupID); +} +)"; + + struct OutputBlock { + float total = -1.0f; + std::uint32_t numSubgroups = 0; + std::uint32_t maxSubgroupId = 0; + }; + + bool HasExtension(const char* wanted) { + GLint extensionCount = 0; + glGetIntegerv(GL_NUM_EXTENSIONS, &extensionCount); + for (GLint i = 0; i < extensionCount; ++i) { + const auto* extension = + reinterpret_cast(glGetStringi(GL_EXTENSIONS, static_cast(i))); + if (extension != nullptr && std::string(extension) == wanted) return true; + } + return false; + } + + class IterationRPScratchFixScenario : public ScenarioTest { + protected: + void SetUp() override { + ScenarioTest::SetUp(); + if (!Ready()) return; + + GLint stages = 0; + GLint features = 0; + GLint invocations = 0; + const bool subgroupExtension = HasExtension("GL_KHR_shader_subgroup"); + if (subgroupExtension) { + glGetIntegerv(GL_SUBGROUP_SUPPORTED_STAGES_KHR, &stages); + glGetIntegerv(GL_SUBGROUP_SUPPORTED_FEATURES_KHR, &features); + } + glGetIntegerv(GL_MAX_COMPUTE_WORK_GROUP_INVOCATIONS, &invocations); + const GLbitfield requiredFeatures = + GL_SUBGROUP_FEATURE_BASIC_BIT_KHR | GL_SUBGROUP_FEATURE_ARITHMETIC_BIT_KHR; + if (!subgroupExtension || (static_cast(stages) & GL_COMPUTE_SHADER_BIT) == 0 || + (static_cast(features) & requiredFeatures) != requiredFeatures || + invocations < static_cast(kInvocationCount)) { + GTEST_SKIP() << "needs GL_KHR_shader_subgroup basic+arithmetic in compute and a " + "512-invocation workgroup"; + } + + m_program = CompileComputeProgram(kComputeSource); + ASSERT_NE(m_program, 0u) << m_buildLog; + + glGenBuffers(1, &m_output); + glBindBuffer(GL_SHADER_STORAGE_BUFFER, m_output); + // maxSubgroupId starts at zero HOST-side: the word is touched only by + // atomicMax during the dispatch, since a plain shader-side zeroing store + // would race the other invocations' atomics (barrier() orders shared + // memory, not SSBO stores). + const OutputBlock poison{-1.0f, 0xa5a5a5a5u, 0u}; + glBufferData(GL_SHADER_STORAGE_BUFFER, sizeof(OutputBlock), &poison, GL_DYNAMIC_READ); + glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 0, m_output); + } + + void TearDown() override { + if (!Ready()) return; + glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 0, 0); + glBindBuffer(GL_SHADER_STORAGE_BUFFER, 0); + if (m_output != 0) glDeleteBuffers(1, &m_output); + if (m_program != 0) glDeleteProgram(m_program); + } + + unsigned int CompileComputeProgram(const char* source) { + const GLuint shader = glCreateShader(GL_COMPUTE_SHADER); + glShaderSource(shader, 1, &source, nullptr); + glCompileShader(shader); + GLint compiled = 0; + glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled); + if (compiled == GL_FALSE) { + char log[2048] = {}; + glGetShaderInfoLog(shader, sizeof(log) - 1, nullptr, log); + m_buildLog = std::string("compute shader did not compile: ") + log; + glDeleteShader(shader); + return 0; + } + const GLuint program = glCreateProgram(); + glAttachShader(program, shader); + glLinkProgram(program); + glDeleteShader(shader); + GLint linked = 0; + glGetProgramiv(program, GL_LINK_STATUS, &linked); + if (linked == GL_FALSE) { + char log[2048] = {}; + glGetProgramInfoLog(program, sizeof(log) - 1, nullptr, log); + m_buildLog = std::string("compute program did not link: ") + log; + glDeleteProgram(program); + return 0; + } + return program; + } + + OutputBlock Dispatch() { + glUseProgram(m_program); + glDispatchCompute(1, 1, 1); + glMemoryBarrier(GL_BUFFER_UPDATE_BARRIER_BIT); + OutputBlock block{}; + glBindBuffer(GL_SHADER_STORAGE_BUFFER, m_output); + glGetBufferSubData(GL_SHADER_STORAGE_BUFFER, 0, sizeof(OutputBlock), &block); + return block; + } + + GLuint m_program = 0; + GLuint m_output = 0; + std::string m_buildLog; + }; + } // namespace + + TEST_F(IterationRPScratchFixScenario, FixtureShapedReductionSumsEveryInvocation) { + const OutputBlock block = Dispatch(); + EXPECT_EQ(glGetError(), static_cast(GL_NO_ERROR)); + + // The topology diagnostics catch the failure modes by name before the sum does: + // an out-of-bounds handoff corrupts the total, a wrong gl_NumSubgroups breaks + // the combine loop's length. + ASSERT_NE(block.numSubgroups, 0xa5a5a5a5u) << "invocation 511 never reached its store"; + EXPECT_GE(block.numSubgroups, 1u); + EXPECT_LE(block.numSubgroups, kInvocationCount); + EXPECT_LT(block.maxSubgroupId, block.numSubgroups) + << "gl_SubgroupID exceeds gl_NumSubgroups - the inconsistency " + "DeriveNumSubgroupsPass exists to repair"; + + // Integer-valued fp32 inputs: the workgroup total is exact under any subgroup + // width, lane order, and association. This is the value iterationRP's exposure + // average is built from; without FixIterationRPSubgroupScratchPass an 8-lane + // device writes prefixSumCache[32..63] out of bounds and this comparison fails. + EXPECT_EQ(block.total, kExpectedTotal) + << "workgroup reduction produced " << block.total << " with gl_NumSubgroups=" + << block.numSubgroups; + } +} // namespace MGITest diff --git a/MobileGL/MG_Test/SelfTest/CMakeLists.txt b/MobileGL/MG_Test/SelfTest/CMakeLists.txt index 9beb3e27..3dd1e5c0 100644 --- a/MobileGL/MG_Test/SelfTest/CMakeLists.txt +++ b/MobileGL/MG_Test/SelfTest/CMakeLists.txt @@ -1,19 +1,19 @@ # MobileGL - MobileGL/MG_Test/SelfTest/CMakeLists.txt add_executable( - DriverPostProgram203WitnessTest - DriverPostProgram203WitnessTest.cpp + DriverPostIterationRPWitnessTest + DriverPostIterationRPWitnessTest.cpp ) -target_include_directories(DriverPostProgram203WitnessTest PRIVATE +target_include_directories(DriverPostIterationRPWitnessTest PRIVATE ${MGL_ROOT}/include ${MGL_ROOT}/MobileGL ) -target_link_libraries(DriverPostProgram203WitnessTest PRIVATE +target_link_libraries(DriverPostIterationRPWitnessTest PRIVATE GTest::gtest_main ${LINK_LIBRARIES} ) include(GoogleTest) -gtest_discover_tests(DriverPostProgram203WitnessTest DISCOVERY_TIMEOUT 30 PROPERTIES LABELS unit) +gtest_discover_tests(DriverPostIterationRPWitnessTest DISCOVERY_TIMEOUT 30 PROPERTIES LABELS unit) diff --git a/MobileGL/MG_Test/SelfTest/DriverPostProgram203WitnessTest.cpp b/MobileGL/MG_Test/SelfTest/DriverPostIterationRPWitnessTest.cpp similarity index 51% rename from MobileGL/MG_Test/SelfTest/DriverPostProgram203WitnessTest.cpp rename to MobileGL/MG_Test/SelfTest/DriverPostIterationRPWitnessTest.cpp index 888228a1..1cccd884 100644 --- a/MobileGL/MG_Test/SelfTest/DriverPostProgram203WitnessTest.cpp +++ b/MobileGL/MG_Test/SelfTest/DriverPostIterationRPWitnessTest.cpp @@ -1,4 +1,4 @@ -// MobileGL - MobileGL/MG_Test/SelfTest/DriverPostProgram203WitnessTest.cpp +// MobileGL - MobileGL/MG_Test/SelfTest/DriverPostIterationRPWitnessTest.cpp // Copyright (c) 2026 MobileGL-Dev // Licensed under the GNU Lesser General Public License v3.0: // https://www.gnu.org/licenses/gpl-3.0.txt @@ -10,21 +10,21 @@ #include -#include "MG_Util/SelfTest/DriverPostProgram203Witness.h" +#include "MG_Util/SelfTest/DriverPostIterationRPWitness.h" namespace MobileGL::MG_Util::SelfTest { namespace { - Program203WitnessOutput MakeValidWitness(std::uint32_t numSubgroups) { - Program203WitnessOutput output{}; - output.magic = kProgram203WitnessMagic; + IterationRPWitnessOutput MakeValidWitness(std::uint32_t numSubgroups) { + IterationRPWitnessOutput output{}; + output.magic = kIterationRPWitnessMagic; output.numSubgroups = numSubgroups; - output.loopLength = ComputeProgram203WitnessLoopLength(numSubgroups); + output.loopLength = ComputeIterationRPWitnessLoopLength(numSubgroups); output.seenSubgroupMask = - numSubgroups == kProgram203WitnessMaxSubgroups ? 0xffffffffu : (1u << numSubgroups) - 1u; + numSubgroups == kIterationRPWitnessMaxSubgroups ? 0xffffffffu : (1u << numSubgroups) - 1u; // Valid test layouts use equal contiguous groups of the indexed // 1..512 input. The compact witness only needs their independent sums. - const std::uint32_t subgroupSize = kProgram203WitnessInvocationCount / numSubgroups; + const std::uint32_t subgroupSize = kIterationRPWitnessInvocationCount / numSubgroups; for (std::uint32_t subgroup = 0u; subgroup < numSubgroups; ++subgroup) { const std::uint32_t first = subgroup * subgroupSize + 1u; const std::uint32_t last = first + subgroupSize - 1u; @@ -50,139 +50,139 @@ namespace MobileGL::MG_Util::SelfTest { return output; } - Program203WitnessLimits MakeSufficientLimits() { - Program203WitnessLimits limits; + IterationRPWitnessLimits MakeSufficientLimits() { + IterationRPWitnessLimits limits; limits.computeStageSupported = true; limits.basicSubgroupSupported = true; limits.arithmeticSubgroupSupported = true; limits.subgroupSize = 32u; - limits.maxComputeWorkGroupInvocations = kProgram203WitnessInvocationCount; + limits.maxComputeWorkGroupInvocations = kIterationRPWitnessInvocationCount; limits.maxComputeWorkGroupSize = {32u, 16u, 1u}; - limits.maxComputeSharedMemorySize = kProgram203WitnessSharedMemoryBytes; + limits.maxComputeSharedMemorySize = kIterationRPWitnessSharedMemoryBytes; limits.maxPerStageDescriptorStorageBuffers = 1u; limits.maxDescriptorSetStorageBuffers = 1u; limits.maxBoundDescriptorSets = 1u; - limits.maxStorageBufferRange = sizeof(Program203WitnessOutput); + limits.maxStorageBufferRange = sizeof(IterationRPWitnessOutput); return limits; } } // namespace - TEST(DriverPostProgram203WitnessTest, ValidTwoSubgroupWitness) { - const Program203WitnessValidationResult validation = ValidateProgram203Witness(MakeValidWitness(2u)); + TEST(DriverPostIterationRPWitnessTest, ValidTwoSubgroupWitness) { + const IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(MakeValidWitness(2u)); ASSERT_TRUE(validation.ok) << validation.detail; EXPECT_EQ(validation.detail, "N=2, owner511=id1/lane255, 2 scan stages, average=(256.5,0)"); } - TEST(DriverPostProgram203WitnessTest, ValidThirtyTwoSubgroupWitness) { - const Program203WitnessValidationResult validation = ValidateProgram203Witness(MakeValidWitness(32u)); + TEST(DriverPostIterationRPWitnessTest, ValidThirtyTwoSubgroupWitness) { + const IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(MakeValidWitness(32u)); ASSERT_TRUE(validation.ok) << validation.detail; EXPECT_EQ(validation.detail, "N=32, owner511=id31/lane15, 6 scan stages, average=(256.5,0)"); } - TEST(DriverPostProgram203WitnessTest, RejectsNonuniformNumSubgroups) { - Program203WitnessOutput output = MakeValidWitness(16u); - output.topologyFlags |= Program203WitnessNonuniformNumSubgroups; - const Program203WitnessValidationResult validation = ValidateProgram203Witness(output); + TEST(DriverPostIterationRPWitnessTest, RejectsNonuniformNumSubgroups) { + IterationRPWitnessOutput output = MakeValidWitness(16u); + output.topologyFlags |= IterationRPWitnessNonuniformNumSubgroups; + const IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(output); EXPECT_FALSE(validation.ok); - EXPECT_EQ(validation.failure, Program203WitnessValidationFailure::Topology); + EXPECT_EQ(validation.failure, IterationRPWitnessValidationFailure::Topology); EXPECT_NE(validation.detail.find("gl_NumSubgroups differed"), std::string::npos); } - TEST(DriverPostProgram203WitnessTest, RejectsMissingAndOutOfRangeSubgroupIds) { - Program203WitnessOutput missing = MakeValidWitness(16u); + TEST(DriverPostIterationRPWitnessTest, RejectsMissingAndOutOfRangeSubgroupIds) { + IterationRPWitnessOutput missing = MakeValidWitness(16u); missing.seenSubgroupMask &= ~(1u << 7u); - Program203WitnessValidationResult validation = ValidateProgram203Witness(missing); + IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(missing); EXPECT_FALSE(validation.ok); EXPECT_NE(validation.detail.find("seen subgroup-ID mask"), std::string::npos); - Program203WitnessOutput outOfRange = MakeValidWitness(16u); - outOfRange.topologyFlags |= Program203WitnessInvalidSubgroupId; - validation = ValidateProgram203Witness(outOfRange); + IterationRPWitnessOutput outOfRange = MakeValidWitness(16u); + outOfRange.topologyFlags |= IterationRPWitnessInvalidSubgroupId; + validation = ValidateIterationRPWitness(outOfRange); EXPECT_FALSE(validation.ok); EXPECT_NE(validation.detail.find("invalid gl_SubgroupID"), std::string::npos); } - TEST(DriverPostProgram203WitnessTest, RejectsInvalidMultipleAndMissingLastLaneWriters) { - Program203WitnessOutput invalidLane = MakeValidWitness(16u); - invalidLane.topologyFlags |= Program203WitnessInvalidSubgroupLane; - Program203WitnessValidationResult validation = ValidateProgram203Witness(invalidLane); + TEST(DriverPostIterationRPWitnessTest, RejectsInvalidMultipleAndMissingLastLaneWriters) { + IterationRPWitnessOutput invalidLane = MakeValidWitness(16u); + invalidLane.topologyFlags |= IterationRPWitnessInvalidSubgroupLane; + IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(invalidLane); EXPECT_FALSE(validation.ok); EXPECT_NE(validation.detail.find("invalid subgroup lane"), std::string::npos); - Program203WitnessOutput multiple = MakeValidWitness(16u); + IterationRPWitnessOutput multiple = MakeValidWitness(16u); multiple.lastLaneWriterCount[4] = 2u; - validation = ValidateProgram203Witness(multiple); + validation = ValidateIterationRPWitness(multiple); EXPECT_FALSE(validation.ok); EXPECT_NE(validation.detail.find("subgroup 4 has 2 source last-lane writers"), std::string::npos); - Program203WitnessOutput missing = MakeValidWitness(16u); + IterationRPWitnessOutput missing = MakeValidWitness(16u); missing.lastLaneWriterCount[6] = 0u; - validation = ValidateProgram203Witness(missing); + validation = ValidateIterationRPWitness(missing); EXPECT_FALSE(validation.ok); EXPECT_NE(validation.detail.find("subgroup 6 has 0 source last-lane writers"), std::string::npos); } - TEST(DriverPostProgram203WitnessTest, ReportsEarliestCorruptSourceScanStage) { - Program203WitnessOutput output = MakeValidWitness(32u); + TEST(DriverPostIterationRPWitnessTest, ReportsEarliestCorruptSourceScanStage) { + IterationRPWitnessOutput output = MakeValidWitness(32u); output.scanCache[0][1].x += 1.0f; output.scanCache[3][5].x += 1.0f; - Program203WitnessValidationResult validation = ValidateProgram203Witness(output); + IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(output); EXPECT_FALSE(validation.ok); - EXPECT_EQ(validation.failure, Program203WitnessValidationFailure::SourceScan); + EXPECT_EQ(validation.failure, IterationRPWitnessValidationFailure::SourceScan); EXPECT_EQ(validation.scanStage, 0u); EXPECT_NE(validation.detail.find("source scan stage 0, subgroup 1"), std::string::npos); output = MakeValidWitness(32u); output.scanCache[3][5].x += 1.0f; - validation = ValidateProgram203Witness(output); + validation = ValidateIterationRPWitness(output); EXPECT_FALSE(validation.ok); - EXPECT_EQ(validation.failure, Program203WitnessValidationFailure::SourceScan); + EXPECT_EQ(validation.failure, IterationRPWitnessValidationFailure::SourceScan); EXPECT_EQ(validation.scanStage, 3u); EXPECT_NE(validation.detail.find("source scan stage 3, subgroup 5"), std::string::npos); } - TEST(DriverPostProgram203WitnessTest, RejectsOwner511OutsideHighestFinalLane) { - Program203WitnessOutput output = MakeValidWitness(16u); + TEST(DriverPostIterationRPWitnessTest, RejectsOwner511OutsideHighestFinalLane) { + IterationRPWitnessOutput output = MakeValidWitness(16u); output.owner511.z = 14u; - const Program203WitnessValidationResult validation = ValidateProgram203Witness(output); + const IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(output); EXPECT_FALSE(validation.ok); - EXPECT_EQ(validation.failure, Program203WitnessValidationFailure::FinalOwner); + EXPECT_EQ(validation.failure, IterationRPWitnessValidationFailure::FinalOwner); EXPECT_NE(validation.detail.find("not in the highest subgroup"), std::string::npos); } - TEST(DriverPostProgram203WitnessTest, RejectsIncorrectVectorFinalAverage) { - Program203WitnessOutput output = MakeValidWitness(16u); + TEST(DriverPostIterationRPWitnessTest, RejectsIncorrectVectorFinalAverage) { + IterationRPWitnessOutput output = MakeValidWitness(16u); output.finalAverage.y = 1.0f; - const Program203WitnessValidationResult validation = ValidateProgram203Witness(output); + const IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(output); EXPECT_FALSE(validation.ok); - EXPECT_EQ(validation.failure, Program203WitnessValidationFailure::FinalAverage); + EXPECT_EQ(validation.failure, IterationRPWitnessValidationFailure::FinalAverage); EXPECT_NE(validation.detail.find("final average"), std::string::npos); } - TEST(DriverPostProgram203WitnessTest, MissingNativeFeatureIsTheOnlySkipCondition) { + TEST(DriverPostIterationRPWitnessTest, MissingNativeFeatureIsTheOnlySkipCondition) { for (const auto toggleMissingFeature : {0u, 1u, 2u}) { - Program203WitnessLimits limits = MakeSufficientLimits(); + IterationRPWitnessLimits limits = MakeSufficientLimits(); if (toggleMissingFeature == 0u) limits.computeStageSupported = false; if (toggleMissingFeature == 1u) limits.basicSubgroupSupported = false; if (toggleMissingFeature == 2u) limits.arithmeticSubgroupSupported = false; - const Program203WitnessEligibilityResult eligibility = EvaluateProgram203WitnessEligibility(limits); - EXPECT_EQ(eligibility.eligibility, Program203WitnessEligibility::SkipUnsupportedNativeFeatureSet) + const IterationRPWitnessEligibilityResult eligibility = EvaluateIterationRPWitnessEligibility(limits); + EXPECT_EQ(eligibility.eligibility, IterationRPWitnessEligibility::SkipUnsupportedNativeFeatureSet) << eligibility.detail; } - Program203WitnessLimits zeroSubgroupSize = MakeSufficientLimits(); + IterationRPWitnessLimits zeroSubgroupSize = MakeSufficientLimits(); zeroSubgroupSize.subgroupSize = 0u; - Program203WitnessEligibilityResult eligibility = EvaluateProgram203WitnessEligibility(zeroSubgroupSize); - EXPECT_EQ(eligibility.eligibility, Program203WitnessEligibility::FailInadequateLimits) << eligibility.detail; + IterationRPWitnessEligibilityResult eligibility = EvaluateIterationRPWitnessEligibility(zeroSubgroupSize); + EXPECT_EQ(eligibility.eligibility, IterationRPWitnessEligibility::FailInadequateLimits) << eligibility.detail; - Program203WitnessLimits limits = MakeSufficientLimits(); + IterationRPWitnessLimits limits = MakeSufficientLimits(); limits.maxComputeWorkGroupInvocations = 511u; - eligibility = EvaluateProgram203WitnessEligibility(limits); - EXPECT_EQ(eligibility.eligibility, Program203WitnessEligibility::FailInadequateLimits) << eligibility.detail; + eligibility = EvaluateIterationRPWitnessEligibility(limits); + EXPECT_EQ(eligibility.eligibility, IterationRPWitnessEligibility::FailInadequateLimits) << eligibility.detail; limits = MakeSufficientLimits(); - limits.maxStorageBufferRange = sizeof(Program203WitnessOutput) - 1u; - eligibility = EvaluateProgram203WitnessEligibility(limits); - EXPECT_EQ(eligibility.eligibility, Program203WitnessEligibility::FailInadequateLimits) << eligibility.detail; + limits.maxStorageBufferRange = sizeof(IterationRPWitnessOutput) - 1u; + eligibility = EvaluateIterationRPWitnessEligibility(limits); + EXPECT_EQ(eligibility.eligibility, IterationRPWitnessEligibility::FailInadequateLimits) << eligibility.detail; } } // namespace MobileGL::MG_Util::SelfTest diff --git a/MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt b/MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt index 91db71dd..9f16aa6f 100644 --- a/MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt +++ b/MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt @@ -4,6 +4,8 @@ add_executable( SpirvPassTest SpirvPassTest.cpp DeriveNumSubgroupsTest.cpp + FixIterationRPSubgroupScratchTest.cpp + EmulateSubgroupsTest.cpp DemoteFloat64Test.cpp FlattenXfbInterfaceBlocksTest.cpp ) diff --git a/MobileGL/MG_Test/ShaderTranspiler/EmulateSubgroupsTest.cpp b/MobileGL/MG_Test/ShaderTranspiler/EmulateSubgroupsTest.cpp new file mode 100644 index 00000000..cfddb3df --- /dev/null +++ b/MobileGL/MG_Test/ShaderTranspiler/EmulateSubgroupsTest.cpp @@ -0,0 +1,248 @@ +// MobileGL - MobileGL/MG_Test/ShaderTranspiler/EmulateSubgroupsTest.cpp +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#include + +#define SPV_ENABLE_UTILITY_CODE +#include "glslang/SPIRV/spirv.hpp11" +#undef SPV_ENABLE_UTILITY_CODE + +#include "Includes.h" +#include +#include + +#include + +using namespace MobileGL; +using MobileGL::MG_Util::ShaderTranspiler::ShaderCompiler; + +namespace { + constexpr SizeT kSpirvHeaderWordCount = 5u; + + template + void ForEachInstruction(const Vector& spirv, Visitor&& visit) { + for (SizeT offset = kSpirvHeaderWordCount; offset < spirv.size();) { + const Uint32 wordCount = spirv[offset] >> 16u; + if (wordCount == 0u || offset + wordCount > spirv.size()) break; + visit(static_cast(spirv[offset] & 0xffffu), &spirv[offset], wordCount); + offset += wordCount; + } + } + + Vector CompileStage(GLenum stage, const String& source) { + using namespace MobileGL::MG_Util::ShaderTranspiler; + ShaderAttrib shaderAttrib{.shaderType = stage, .sourceStr = source}; + auto shaderResult = ShaderCompiler::CompileShader(shaderAttrib); + EXPECT_TRUE(shaderResult) << (shaderResult ? String{} : shaderResult.error().log); + if (!shaderResult) return {}; + + ProgramAttrib programAttrib{.shaders = {shaderResult.value()}}; + auto programResult = ShaderCompiler::LinkProgram(programAttrib); + EXPECT_TRUE(programResult) << (programResult ? String{} : programResult.error().log); + if (!programResult) return {}; + + ProgramBinaryAttrib binaryAttrib{.shaderTypes = {stage}, .program = *programResult.value()}; + auto binaryResult = ShaderCompiler::GetSpirvBinaryFromProgram(binaryAttrib); + EXPECT_TRUE(binaryResult) << (binaryResult ? String{} : binaryResult.error().log); + if (!binaryResult || binaryResult->empty()) return {}; + return binaryResult->front(); + } + + Uint32 CountGroupNonUniform(const Vector& spirv) { + Uint32 count = 0; + ForEachInstruction(spirv, [&](spv::Op opcode, const Uint32*, Uint32) { + if (opcode >= spv::Op::OpGroupNonUniformElect && opcode <= spv::Op::OpGroupNonUniformQuadSwap) { + ++count; + } + }); + return count; + } + + Uint32 CountGroupNonUniformCapabilities(const Vector& spirv) { + Uint32 count = 0; + ForEachInstruction(spirv, [&](spv::Op opcode, const Uint32* words, Uint32 wordCount) { + if (opcode != spv::Op::OpCapability || wordCount < 2u) return; + const auto capability = static_cast(words[1]); + if (capability >= spv::Capability::GroupNonUniform && + capability <= spv::Capability::GroupNonUniformQuad) { + ++count; + } + }); + return count; + } + + Uint32 CountOpcode(const Vector& spirv, spv::Op wanted) { + Uint32 count = 0; + ForEachInstruction(spirv, [&](spv::Op opcode, const Uint32*, Uint32) { + if (opcode == wanted) ++count; + }); + return count; + } + + bool HasWorkgroupVariable(const Vector& spirv) { + bool found = false; + ForEachInstruction(spirv, [&](spv::Op opcode, const Uint32* words, Uint32 wordCount) { + if (opcode == spv::Op::OpVariable && wordCount >= 4u && + static_cast(words[3]) == spv::StorageClass::Workgroup) { + found = true; + } + }); + return found; + } + + bool Validates(const Vector& spirv) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_1); + tools.SetMessageConsumer([](spv_message_level_t, const char*, const spv_position_t& position, + const char* message) { + ADD_FAILURE() << "spirv-val at word " << position.index << ": " << message; + }); + return tools.Validate(spirv); + } + + // One shader touching every lowered category: builtins, vote, arithmetic + // scans, ballot math, shuffles, clustered and quad operations. + constexpr const char* kEveryCategorySource = R"(#version 450 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_vote : require +#extension GL_KHR_shader_subgroup_arithmetic : require +#extension GL_KHR_shader_subgroup_ballot : require +#extension GL_KHR_shader_subgroup_shuffle : require +#extension GL_KHR_shader_subgroup_shuffle_relative : require +#extension GL_KHR_shader_subgroup_clustered : require +#extension GL_KHR_shader_subgroup_quad : require +layout(local_size_x = 48, local_size_y = 1, local_size_z = 1) in; +layout(std430, binding = 0) buffer Output { float value[]; } outputData; +void main() { + uint slot = gl_LocalInvocationIndex * 24u; + float v = float(gl_LocalInvocationIndex + 1u); + outputData.value[slot + 0u] = float(gl_SubgroupSize); + outputData.value[slot + 1u] = float(gl_NumSubgroups); + outputData.value[slot + 2u] = float(gl_SubgroupID); + outputData.value[slot + 3u] = float(gl_SubgroupInvocationID); + outputData.value[slot + 4u] = float(gl_SubgroupEqMask.x + gl_SubgroupLtMask.x); + outputData.value[slot + 5u] = subgroupElect() ? 1.0 : 0.0; + outputData.value[slot + 6u] = subgroupAll(v > 0.0) ? 1.0 : 0.0; + outputData.value[slot + 7u] = subgroupAny(v > 40.0) ? 1.0 : 0.0; + outputData.value[slot + 8u] = subgroupAllEqual(gl_WorkGroupID.x) ? 1.0 : 0.0; + outputData.value[slot + 9u] = subgroupAdd(v); + outputData.value[slot + 10u] = subgroupInclusiveAdd(v); + outputData.value[slot + 11u] = subgroupExclusiveMax(v); + outputData.value[slot + 12u] = float(subgroupMin(gl_LocalInvocationIndex)); + uvec4 ballot = subgroupBallot((gl_LocalInvocationIndex & 1u) == 0u); + outputData.value[slot + 13u] = float(subgroupBallotBitCount(ballot)); + outputData.value[slot + 14u] = float(subgroupBallotFindLSB(ballot)); + outputData.value[slot + 15u] = float(subgroupBallotFindMSB(ballot)); + outputData.value[slot + 16u] = subgroupInverseBallot(ballot) ? 1.0 : 0.0; + outputData.value[slot + 17u] = subgroupBallotBitExtract(ballot, 3u) ? 1.0 : 0.0; + outputData.value[slot + 18u] = subgroupBroadcast(v, 2u); + outputData.value[slot + 19u] = subgroupBroadcastFirst(v); + outputData.value[slot + 20u] = subgroupShuffle(v, gl_SubgroupInvocationID ^ 5u); + outputData.value[slot + 21u] = subgroupShuffleXor(v, 1u) + subgroupShuffleUp(v, 1u) + + subgroupShuffleDown(v, 1u); + outputData.value[slot + 22u] = subgroupClusteredAdd(v, 4u); + outputData.value[slot + 23u] = subgroupQuadBroadcast(v, 1u) + subgroupQuadSwapHorizontal(v); + subgroupBarrier(); + subgroupMemoryBarrierShared(); +} +)"; + + constexpr const char* kNoSubgroupSource = R"(#version 450 core +layout(local_size_x = 64) in; +layout(std430, binding = 0) buffer Output { uint value; } outputData; +void main() { + if (gl_LocalInvocationIndex == 0u) outputData.value = gl_WorkGroupSize.x; +} +)"; + + // An extended subgroup instruction (SPV_KHR_subgroup_rotate) alongside core + // ones: outside the lowered set, so the pass must fail rather than emit + // "subgroup-free" output that still rotates. + constexpr const char* kRotateSource = R"(#version 450 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +#extension GL_KHR_shader_subgroup_rotate : require +layout(local_size_x = 64) in; +layout(std430, binding = 0) buffer Output { float value[]; } outputData; +void main() { + float v = subgroupAdd(float(gl_SubgroupInvocationID)); + outputData.value[gl_LocalInvocationIndex] = subgroupRotate(v, 1u); +} +)"; + + // A 1024-invocation workgroup exchanging a vec4 and a float: the lowering + // would need 16 KiB + 4 KiB of scratch, past the Vulkan-minimum shared + // budget of 16384 bytes. + constexpr const char* kScratchHungrySource = R"(#version 450 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +layout(local_size_x = 1024) in; +layout(std430, binding = 0) buffer Output { vec4 value[]; } outputData; +void main() { + vec4 wide = subgroupAdd(vec4(float(gl_LocalInvocationIndex))); + wide.x += subgroupInclusiveAdd(float(gl_SubgroupInvocationID)); + outputData.value[gl_LocalInvocationIndex] = wide; +} +)"; +} // namespace + +TEST(EmulateSubgroupsPass, LowersEveryCategoryToSharedMemory) { + const Vector input = CompileStage(GL_COMPUTE_SHADER, kEveryCategorySource); + ASSERT_FALSE(input.empty()); + ASSERT_GT(CountGroupNonUniform(input), 0u); + ASSERT_GT(CountGroupNonUniformCapabilities(input), 0u); + + Vector output; + ASSERT_TRUE(ShaderCompiler::EmulateSubgroupsForVulkan(input, output, 16384u, true)); + ASSERT_TRUE(Validates(output)); + + // The whole point: nothing subgroup-shaped survives, so the module runs on a + // device with no subgroup support at all. + EXPECT_EQ(CountGroupNonUniform(output), 0u); + EXPECT_EQ(CountGroupNonUniformCapabilities(output), 0u); + // The exchanges go through workgroup-shared scratch behind control barriers. + EXPECT_TRUE(HasWorkgroupVariable(output)); + EXPECT_GT(CountOpcode(output, spv::Op::OpControlBarrier), CountOpcode(input, spv::Op::OpControlBarrier)); +} + +TEST(EmulateSubgroupsPass, IsIdempotent) { + const Vector input = CompileStage(GL_COMPUTE_SHADER, kEveryCategorySource); + ASSERT_FALSE(input.empty()); + Vector once; + ASSERT_TRUE(ShaderCompiler::EmulateSubgroupsForVulkan(input, once, 16384u, true)); + Vector twice; + ASSERT_TRUE(ShaderCompiler::EmulateSubgroupsForVulkan(once, twice, 16384u, true)); + EXPECT_EQ(twice, once); +} + +TEST(EmulateSubgroupsPass, LeavesSubgroupFreeComputeUntouched) { + const Vector input = CompileStage(GL_COMPUTE_SHADER, kNoSubgroupSource); + ASSERT_FALSE(input.empty()); + Vector output; + ASSERT_TRUE(ShaderCompiler::EmulateSubgroupsForVulkan(input, output, 16384u, true)); + EXPECT_EQ(output, input); +} + +TEST(EmulateSubgroupsPass, RefusesExtendedSubgroupInstructions) { + const Vector input = CompileStage(GL_COMPUTE_SHADER, kRotateSource); + ASSERT_FALSE(input.empty()); + Vector output; + EXPECT_FALSE(ShaderCompiler::EmulateSubgroupsForVulkan(input, output, 16384u, false)); +} + +TEST(EmulateSubgroupsPass, RefusesAModuleOverTheScratchBudget) { + const Vector input = CompileStage(GL_COMPUTE_SHADER, kScratchHungrySource); + ASSERT_FALSE(input.empty()); + // vec4 scratch (1024 slots * 16 bytes) plus float scratch (4 KiB) exceeds + // the 16 KiB Vulkan-minimum budget. + Vector output; + EXPECT_FALSE(ShaderCompiler::EmulateSubgroupsForVulkan(input, output, 16384u, false)); + // A device advertising more shared memory takes the same module fine. + Vector roomier; + EXPECT_TRUE(ShaderCompiler::EmulateSubgroupsForVulkan(input, roomier, 32768u, true)); + EXPECT_TRUE(Validates(roomier)); +} diff --git a/MobileGL/MG_Test/ShaderTranspiler/FixIterationRPSubgroupScratchTest.cpp b/MobileGL/MG_Test/ShaderTranspiler/FixIterationRPSubgroupScratchTest.cpp new file mode 100644 index 00000000..195836a4 --- /dev/null +++ b/MobileGL/MG_Test/ShaderTranspiler/FixIterationRPSubgroupScratchTest.cpp @@ -0,0 +1,225 @@ +// MobileGL - MobileGL/MG_Test/ShaderTranspiler/FixIterationRPSubgroupScratchTest.cpp +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#include + +#define SPV_ENABLE_UTILITY_CODE +#include "glslang/SPIRV/spirv.hpp11" +#undef SPV_ENABLE_UTILITY_CODE + +#include "Includes.h" +#include +#include + +#include + +#include +#include +#include + +using namespace MobileGL; +using MobileGL::MG_Util::ShaderTranspiler::ShaderCompiler; + +namespace { + constexpr SizeT kSpirvHeaderWordCount = 5u; + + template + void ForEachInstruction(const Vector& spirv, Visitor&& visit) { + for (SizeT offset = kSpirvHeaderWordCount; offset < spirv.size();) { + const Uint32 wordCount = spirv[offset] >> 16u; + if (wordCount == 0u || offset + wordCount > spirv.size()) break; + visit(static_cast(spirv[offset] & 0xffffu), &spirv[offset], wordCount); + offset += wordCount; + } + } + + Vector CompileCompute(const String& source) { + using namespace MobileGL::MG_Util::ShaderTranspiler; + ShaderAttrib shaderAttrib{.shaderType = GL_COMPUTE_SHADER, .sourceStr = source}; + auto shaderResult = ShaderCompiler::CompileShader(shaderAttrib); + EXPECT_TRUE(shaderResult) << (shaderResult ? String{} : shaderResult.error().log); + if (!shaderResult) return {}; + + ProgramAttrib programAttrib{.shaders = {shaderResult.value()}}; + auto programResult = ShaderCompiler::LinkProgram(programAttrib); + EXPECT_TRUE(programResult) << (programResult ? String{} : programResult.error().log); + if (!programResult) return {}; + + ProgramBinaryAttrib binaryAttrib{.shaderTypes = {GL_COMPUTE_SHADER}, .program = *programResult.value()}; + auto binaryResult = ShaderCompiler::GetSpirvBinaryFromProgram(binaryAttrib); + EXPECT_TRUE(binaryResult) << (binaryResult ? String{} : binaryResult.error().log); + if (!binaryResult || binaryResult->empty()) return {}; + return binaryResult->front(); + } + + // The declared lengths of every Workgroup-storage array variable, sorted. + std::vector WorkgroupArrayLengths(const Vector& spirv) { + std::map constantValues; // constant id -> value + std::map arrayLengthIds; // array type id -> length constant id + std::map pointerPointees; // pointer type id -> pointee type id + std::vector workgroupPointerTypes; // type ids of Workgroup variables + ForEachInstruction(spirv, [&](spv::Op opcode, const Uint32* words, Uint32 wordCount) { + switch (opcode) { + case spv::Op::OpConstant: + if (wordCount >= 4u) constantValues[words[2]] = words[3]; + break; + case spv::Op::OpTypeArray: + if (wordCount >= 4u) arrayLengthIds[words[1]] = words[3]; + break; + case spv::Op::OpTypePointer: + if (wordCount >= 4u && + static_cast(words[2]) == spv::StorageClass::Workgroup) { + pointerPointees[words[1]] = words[3]; + } + break; + case spv::Op::OpVariable: + if (wordCount >= 4u && + static_cast(words[3]) == spv::StorageClass::Workgroup) { + workgroupPointerTypes.push_back(words[1]); + } + break; + default: + break; + } + }); + std::vector lengths; + for (const Uint32 pointerTypeId : workgroupPointerTypes) { + const auto pointee = pointerPointees.find(pointerTypeId); + if (pointee == pointerPointees.end()) continue; + const auto lengthId = arrayLengthIds.find(pointee->second); + if (lengthId == arrayLengthIds.end()) continue; + const auto value = constantValues.find(lengthId->second); + if (value != constantValues.end()) lengths.push_back(value->second); + } + std::sort(lengths.begin(), lengths.end()); + return lengths; + } + + bool Validates(const Vector& spirv) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_1); + tools.SetMessageConsumer([](spv_message_level_t, const char*, const spv_position_t& position, + const char* message) { + ADD_FAILURE() << "spirv-val at word " << position.index << ": " << message; + }); + return tools.Validate(spirv); + } + + // iterationRP's reduction fingerprint: 32x16x1, subgroupInclusiveAdd on a + // vec2, and the pack's own 32-entry gl_SubgroupID-indexed scratch. A second, + // plainly indexed array rides along to prove the patch is surgical. + constexpr const char* kIterationRPShapedSource = R"(#version 450 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +layout(local_size_x = 32, local_size_y = 16, local_size_z = 1) in; +layout(std430, binding = 0) buffer Output { float value; } outputData; +shared vec2 prefixSumCache[32]; +shared float plainScratch[4]; +void main() { + vec2 sampleLuminance = vec2(float(gl_LocalInvocationIndex), 0.0); + sampleLuminance = subgroupInclusiveAdd(sampleLuminance); + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) + prefixSumCache[gl_SubgroupID] = sampleLuminance; + plainScratch[gl_LocalInvocationIndex & 3u] = sampleLuminance.x; + barrier(); + uint loopLength = uint(findMSB(gl_NumSubgroups)); + loopLength += uint(gl_NumSubgroups - (1u << (loopLength - 1u)) > 0u); + for (uint scanStage = 0u; scanStage < loopLength; ++scanStage) { + if ((gl_SubgroupID & (1u << scanStage)) > 0u) { + sampleLuminance += prefixSumCache[(gl_SubgroupID >> scanStage << scanStage) - 1u]; + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) + prefixSumCache[gl_SubgroupID] = sampleLuminance; + } + barrier(); + } + if (gl_LocalInvocationIndex == 511u) + outputData.value = prefixSumCache[0].x / 512.0 + plainScratch[0]; +} +)"; + + // Same scratch idiom, different workgroup shape - NOT iterationRP, so the + // fingerprint must refuse it even though it would break identically. + constexpr const char* kWrongWorkgroupShapeSource = R"(#version 450 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +layout(local_size_x = 64, local_size_y = 8, local_size_z = 1) in; +layout(std430, binding = 0) buffer Output { float value; } outputData; +shared vec2 prefixSumCache[32]; +void main() { + vec2 v = subgroupInclusiveAdd(vec2(1.0, 0.0)); + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) + prefixSumCache[gl_SubgroupID] = v; + barrier(); + if (gl_LocalInvocationIndex == 0u) + outputData.value = prefixSumCache[0].x; +} +)"; + + // Right shape, but a float scan and a float[32] scratch - not the pack's + // vec2 accumulator signature. + constexpr const char* kWrongElementTypeSource = R"(#version 450 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +layout(local_size_x = 32, local_size_y = 16, local_size_z = 1) in; +layout(std430, binding = 0) buffer Output { float value; } outputData; +shared float cache[32]; +void main() { + float v = subgroupInclusiveAdd(float(gl_LocalInvocationIndex)); + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) + cache[gl_SubgroupID] = v; + barrier(); + if (gl_LocalInvocationIndex == 0u) + outputData.value = cache[0]; +} +)"; +} // namespace + +TEST(FixIterationRPSubgroupScratchPass, GrowsThePacksScratchForNarrowSubgroups) { + const Vector input = CompileCompute(kIterationRPShapedSource); + ASSERT_FALSE(input.empty()); + ASSERT_EQ(WorkgroupArrayLengths(input), (std::vector{4u, 32u})); + + // lavapipe: 8-lane subgroups over 512 invocations need 64 entries; the + // plainly indexed neighbour must keep its 4. + Vector output; + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, output, 8u, true)); + EXPECT_EQ(WorkgroupArrayLengths(output), (std::vector{4u, 64u})); + EXPECT_TRUE(Validates(output)); +} + +TEST(FixIterationRPSubgroupScratchPass, LeavesPackWidthAssumptionsAloneOnWideDevices) { + const Vector input = CompileCompute(kIterationRPShapedSource); + ASSERT_FALSE(input.empty()); + + // >= 16 lanes means at most 32 subgroups: the pack's declared size holds and + // the module must pass through byte-identical. + for (const Uint32 nativeSize : {16u, 32u, 64u, 128u}) { + Vector output; + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, output, nativeSize, true)); + EXPECT_EQ(output, input) << "native width " << nativeSize; + } +} + +TEST(FixIterationRPSubgroupScratchPass, RefusesAModuleOutsideTheFingerprint) { + for (const char* source : {kWrongWorkgroupShapeSource, kWrongElementTypeSource}) { + const Vector input = CompileCompute(source); + ASSERT_FALSE(input.empty()); + Vector output; + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, output, 8u, true)); + EXPECT_EQ(output, input); + } +} + +TEST(FixIterationRPSubgroupScratchPass, IsIdempotent) { + const Vector input = CompileCompute(kIterationRPShapedSource); + ASSERT_FALSE(input.empty()); + Vector once; + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, once, 8u, true)); + Vector twice; + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(once, twice, 8u, true)); + EXPECT_EQ(twice, once); +} diff --git a/MobileGL/MG_Util/SelfTest/DriverPost.cpp b/MobileGL/MG_Util/SelfTest/DriverPost.cpp index 8b1070ed..4cd7c41d 100644 --- a/MobileGL/MG_Util/SelfTest/DriverPost.cpp +++ b/MobileGL/MG_Util/SelfTest/DriverPost.cpp @@ -7,8 +7,8 @@ // End of Source File Header #include "DriverPost.h" -#include "DriverPostProgram203Witness.h" -#include "DriverPostProgram203WitnessSpv.h" +#include "DriverPostIterationRPWitness.h" +#include "DriverPostIterationRPWitnessSpv.h" #include "MG_Util/BackendLoaders/OpenGL/Loader.h" #include #include @@ -1458,11 +1458,11 @@ namespace MobileGL::MG_Util::SelfTest { disabledNote); } - // Native Program-203 compute witness. This deliberately uses a separate + // Native iterationRP compute witness. This deliberately uses a separate // throwaway Vulkan device rather than the real renderer's queues, and it // treats MOBILEGL_DISABLE_SUBGROUP as irrelevant: the row reports what the // driver does, not what MobileGL elects to advertise to applications. - void ProbeVulkanProgram203Witness(ReportBuilder& builder, PFN_vkGetInstanceProcAddr getInstanceProcAddr, + void ProbeVulkanIterationRPWitness(ReportBuilder& builder, PFN_vkGetInstanceProcAddr getInstanceProcAddr, VkInstance instance, VkPhysicalDevice physicalDevice, Uint32 computeQueueFamilyIndex, const VkPhysicalDeviceProperties& properties, @@ -1476,7 +1476,7 @@ namespace MobileGL::MG_Util::SelfTest { return; } - Program203WitnessLimits limits{}; + IterationRPWitnessLimits limits{}; limits.computeStageSupported = (subgroupProperties.supportedStages & VK_SHADER_STAGE_COMPUTE_BIT) != 0; limits.basicSubgroupSupported = @@ -1494,12 +1494,12 @@ namespace MobileGL::MG_Util::SelfTest { limits.maxBoundDescriptorSets = properties.limits.maxBoundDescriptorSets; limits.maxStorageBufferRange = properties.limits.maxStorageBufferRange; - const Program203WitnessEligibilityResult eligibility = EvaluateProgram203WitnessEligibility(limits); - if (eligibility.eligibility == Program203WitnessEligibility::SkipUnsupportedNativeFeatureSet) { + const IterationRPWitnessEligibilityResult eligibility = EvaluateIterationRPWitnessEligibility(limits); + if (eligibility.eligibility == IterationRPWitnessEligibility::SkipUnsupportedNativeFeatureSet) { builder.Info(RowName, eligibility.detail); return; } - if (eligibility.eligibility == Program203WitnessEligibility::FailInadequateLimits) { + if (eligibility.eligibility == IterationRPWitnessEligibility::FailInadequateLimits) { fail(eligibility.detail); return; } @@ -1668,7 +1668,7 @@ namespace MobileGL::MG_Util::SelfTest { VkBufferCreateInfo bufferInfo{}; bufferInfo.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; - bufferInfo.size = sizeof(Program203WitnessOutput); + bufferInfo.size = sizeof(IterationRPWitnessOutput); bufferInfo.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; result = vkCreateBufferFn(device, &bufferInfo, nullptr, &outputBuffer); @@ -1710,12 +1710,12 @@ namespace MobileGL::MG_Util::SelfTest { fail(format("vkBindBufferMemory(output SSBO) failed (VkResult = {})", static_cast(result))); return; } - result = vkMapMemoryFn(device, outputMemory, 0, sizeof(Program203WitnessOutput), 0, &mappedOutput); + result = vkMapMemoryFn(device, outputMemory, 0, sizeof(IterationRPWitnessOutput), 0, &mappedOutput); if (result != VK_SUCCESS || mappedOutput == nullptr) { fail(format("vkMapMemory(output SSBO) failed (VkResult = {})", static_cast(result))); return; } - std::memset(mappedOutput, 0xa5, sizeof(Program203WitnessOutput)); + std::memset(mappedOutput, 0xa5, sizeof(IterationRPWitnessOutput)); VkDescriptorSetLayoutBinding outputBinding{}; outputBinding.binding = 0; @@ -1760,7 +1760,7 @@ namespace MobileGL::MG_Util::SelfTest { VkDescriptorBufferInfo outputDescriptor{}; outputDescriptor.buffer = outputBuffer; outputDescriptor.offset = 0; - outputDescriptor.range = sizeof(Program203WitnessOutput); + outputDescriptor.range = sizeof(IterationRPWitnessOutput); VkWriteDescriptorSet descriptorWrite{}; descriptorWrite.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; descriptorWrite.dstSet = descriptorSet; @@ -1772,8 +1772,8 @@ namespace MobileGL::MG_Util::SelfTest { VkShaderModuleCreateInfo shaderModuleInfo{}; shaderModuleInfo.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO; - shaderModuleInfo.codeSize = sizeof(kDriverPostProgram203WitnessSpv); - shaderModuleInfo.pCode = kDriverPostProgram203WitnessSpv; + shaderModuleInfo.codeSize = sizeof(kDriverPostIterationRPWitnessSpv); + shaderModuleInfo.pCode = kDriverPostIterationRPWitnessSpv; result = vkCreateShaderModuleFn(device, &shaderModuleInfo, nullptr, &shaderModule); if (result != VK_SUCCESS) { fail(format("vkCreateShaderModule failed (VkResult = {})", static_cast(result))); @@ -1845,7 +1845,7 @@ namespace MobileGL::MG_Util::SelfTest { hostReadBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; hostReadBarrier.buffer = outputBuffer; hostReadBarrier.offset = 0; - hostReadBarrier.size = sizeof(Program203WitnessOutput); + hostReadBarrier.size = sizeof(IterationRPWitnessOutput); vkCmdPipelineBarrierFn(commandBuffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 0, nullptr, 1, &hostReadBarrier, 0, nullptr); result = vkEndCommandBufferFn(commandBuffer); @@ -1879,9 +1879,9 @@ namespace MobileGL::MG_Util::SelfTest { return; } - Program203WitnessOutput output{}; + IterationRPWitnessOutput output{}; std::memcpy(&output, mappedOutput, sizeof(output)); - const Program203WitnessValidationResult validation = ValidateProgram203Witness(output); + const IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(output); if (!validation.ok) { fail(validation.detail); return; @@ -2491,7 +2491,7 @@ namespace MobileGL::MG_Util::SelfTest { builder.Warn("Compute shader subgroup", "subgroup properties could not be queried"); } - ProbeVulkanProgram203Witness(builder, getInstanceProcAddr, instance, physicalDevice, computeQueueFamilyIndex, + ProbeVulkanIterationRPWitness(builder, getInstanceProcAddr, instance, physicalDevice, computeQueueFamilyIndex, properties, subgroupPropertiesAvailable, subgroupProperties); if (HasVkExtension(deviceExtensions, VK_KHR_DRAW_INDIRECT_COUNT_EXTENSION_NAME)) { diff --git a/MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.comp b/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.comp similarity index 94% rename from MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.comp rename to MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.comp index 2576228c..3e9d95c8 100644 --- a/MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.comp +++ b/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.comp @@ -1,4 +1,4 @@ -// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.comp +// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.comp // Copyright (c) 2026 MobileGL-Dev // Licensed under the GNU Lesser General Public License v3.0: // https://www.gnu.org/licenses/gpl-3.0.txt @@ -6,9 +6,9 @@ // SPDX-License-Identifier: LGPL-3.0-only // End of Source File Header // -// Native Vulkan GLSL 450 witness for Program 203's first subgroup reduction. +// Native Vulkan GLSL 450 witness for iterationRP's first subgroup reduction. // It is intentionally independent of the GL 430 integration scenario. The body -// below preserves Program 203's source reduction; the surrounding diagnostics +// below preserves iterationRP's source reduction; the surrounding diagnostics // only observe its topology and cache handoffs. #version 450 @@ -23,7 +23,7 @@ const uint kTopologyInvalidSubgroupId = 1u << 2u; const uint kTopologyInvalidSubgroupLane = 1u << 3u; const uint kWitnessMagic = 0x50323033u; -layout(std430, set = 0, binding = 0) buffer Program203WitnessOutput { +layout(std430, set = 0, binding = 0) buffer IterationRPWitnessOutput { uint magic; uint topologyFlags; uint numSubgroups; @@ -40,7 +40,7 @@ layout(std430, set = 0, binding = 0) buffer Program203WitnessOutput { vec2 finalAverage; } outWitness; -// Program 203's cache stays separate from all diagnostic shared state. In +// iterationRP's cache stays separate from all diagnostic shared state. In // particular, no instrumentation stores through prefixSumCache except source // writes retained below. shared vec2 prefixSumCache[32]; @@ -108,7 +108,7 @@ void main() { } // This branch is uniform after collection and is solely a safety guard for - // broken topology reports. The valid side retains Program 203 verbatim. + // broken topology reports. The valid side retains iterationRP verbatim. const bool sourceDomain = canonicalDomain && topologyFlagsShared == 0u; if (sourceDomain) { vec2 sampleLuminance = vec2(float(gl_LocalInvocationIndex + 1u), 0.0); diff --git a/MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.cpp b/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.cpp similarity index 69% rename from MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.cpp rename to MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.cpp index 74a0078f..141d3b01 100644 --- a/MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.cpp +++ b/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.cpp @@ -1,4 +1,4 @@ -// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.cpp +// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.cpp // Copyright (c) 2026 MobileGL-Dev // Licensed under the GNU Lesser General Public License v3.0: // https://www.gnu.org/licenses/gpl-3.0.txt @@ -6,7 +6,7 @@ // SPDX-License-Identifier: LGPL-3.0-only // End of Source File Header -#include "DriverPostProgram203Witness.h" +#include "DriverPostIterationRPWitness.h" #include #include @@ -15,11 +15,11 @@ namespace MobileGL::MG_Util::SelfTest { namespace { - [[nodiscard]] Program203WitnessValidationResult Failure(Program203WitnessValidationFailure failure, + [[nodiscard]] IterationRPWitnessValidationResult Failure(IterationRPWitnessValidationFailure failure, std::string detail, std::uint32_t scanStage = 0u, std::uint32_t subgroup = 0u) { - Program203WitnessValidationResult result; + IterationRPWitnessValidationResult result; result.ok = false; result.failure = failure; result.scanStage = scanStage; @@ -36,18 +36,18 @@ namespace MobileGL::MG_Util::SelfTest { return FloatBits(lhs) == FloatBits(rhs); } - [[nodiscard]] bool SameBits(const Program203WitnessVec2& lhs, const Program203WitnessVec2& rhs) { + [[nodiscard]] bool SameBits(const IterationRPWitnessVec2& lhs, const IterationRPWitnessVec2& rhs) { return SameBits(lhs.x, rhs.x) && SameBits(lhs.y, rhs.y); } - [[nodiscard]] std::string Vec2String(const Program203WitnessVec2& value) { + [[nodiscard]] std::string Vec2String(const IterationRPWitnessVec2& value) { std::ostringstream output; output << '(' << value.x << ',' << value.y << ')'; return output.str(); } [[nodiscard]] std::uint32_t ExpectedSeenSubgroupMask(std::uint32_t numSubgroups) { - return numSubgroups == kProgram203WitnessMaxSubgroups ? 0xffffffffu : (1u << numSubgroups) - 1u; + return numSubgroups == kIterationRPWitnessMaxSubgroups ? 0xffffffffu : (1u << numSubgroups) - 1u; } [[nodiscard]] std::string JoinRequirements(const std::vector& requirements) { @@ -60,8 +60,8 @@ namespace MobileGL::MG_Util::SelfTest { } } // namespace - Program203WitnessEligibilityResult - EvaluateProgram203WitnessEligibility(const Program203WitnessLimits& limits) { + IterationRPWitnessEligibilityResult + EvaluateIterationRPWitnessEligibility(const IterationRPWitnessLimits& limits) { // This classification deliberately precedes numeric limits. An absent native // compute/basic/arithmetic subgroup contract means there is nothing to witness, // whereas every resource/entry-point failure on a capable device is a POST FAIL. @@ -70,7 +70,7 @@ namespace MobileGL::MG_Util::SelfTest { if (!limits.computeStageSupported) missing.emplace_back("VK_SHADER_STAGE_COMPUTE_BIT"); if (!limits.basicSubgroupSupported) missing.emplace_back("VK_SUBGROUP_FEATURE_BASIC_BIT"); if (!limits.arithmeticSubgroupSupported) missing.emplace_back("VK_SUBGROUP_FEATURE_ARITHMETIC_BIT"); - return {Program203WitnessEligibility::SkipUnsupportedNativeFeatureSet, + return {IterationRPWitnessEligibility::SkipUnsupportedNativeFeatureSet, "skipped because the native compute/basic/arithmetic subgroup feature set is unsupported (missing " + JoinRequirements(missing) + ')'}; } @@ -79,16 +79,16 @@ namespace MobileGL::MG_Util::SelfTest { if (limits.subgroupSize == 0u) { inadequate.emplace_back("subgroupSize == 0"); } - if (limits.maxComputeWorkGroupInvocations < kProgram203WitnessInvocationCount) { + if (limits.maxComputeWorkGroupInvocations < kIterationRPWitnessInvocationCount) { inadequate.emplace_back("maxComputeWorkGroupInvocations < 512"); } if (limits.maxComputeWorkGroupSize[0] < 32u || limits.maxComputeWorkGroupSize[1] < 16u || limits.maxComputeWorkGroupSize[2] < 1u) { inadequate.emplace_back("maxComputeWorkGroupSize does not cover 32x16x1"); } - if (limits.maxComputeSharedMemorySize < kProgram203WitnessSharedMemoryBytes) { + if (limits.maxComputeSharedMemorySize < kIterationRPWitnessSharedMemoryBytes) { inadequate.emplace_back("maxComputeSharedMemorySize < " + - std::to_string(kProgram203WitnessSharedMemoryBytes)); + std::to_string(kIterationRPWitnessSharedMemoryBytes)); } if (limits.maxPerStageDescriptorStorageBuffers < 1u) { inadequate.emplace_back("maxPerStageDescriptorStorageBuffers < 1"); @@ -99,21 +99,21 @@ namespace MobileGL::MG_Util::SelfTest { if (limits.maxBoundDescriptorSets < 1u) { inadequate.emplace_back("maxBoundDescriptorSets < 1"); } - if (limits.maxStorageBufferRange < sizeof(Program203WitnessOutput)) { + if (limits.maxStorageBufferRange < sizeof(IterationRPWitnessOutput)) { inadequate.emplace_back("maxStorageBufferRange < " + - std::to_string(sizeof(Program203WitnessOutput))); + std::to_string(sizeof(IterationRPWitnessOutput))); } if (!inadequate.empty()) { - return {Program203WitnessEligibility::FailInadequateLimits, + return {IterationRPWitnessEligibility::FailInadequateLimits, "insufficient Vulkan limits for a 32x16x1 workgroup, one output SSBO, and " + - std::to_string(kProgram203WitnessSharedMemoryBytes) + " bytes of shared memory: " + + std::to_string(kIterationRPWitnessSharedMemoryBytes) + " bytes of shared memory: " + JoinRequirements(inadequate)}; } - return {Program203WitnessEligibility::Execute, {}}; + return {IterationRPWitnessEligibility::Execute, {}}; } - std::uint32_t ComputeProgram203WitnessLoopLength(std::uint32_t numSubgroups) { - if (numSubgroups < 2u || numSubgroups > kProgram203WitnessMaxSubgroups) return 0u; + std::uint32_t ComputeIterationRPWitnessLoopLength(std::uint32_t numSubgroups) { + if (numSubgroups < 2u || numSubgroups > kIterationRPWitnessMaxSubgroups) return 0u; // Exact C++ spelling of the source's findMSB-based calculation. In // particular, its final iteration for powers of two is intentional. @@ -125,87 +125,87 @@ namespace MobileGL::MG_Util::SelfTest { return loopLength; } - Program203WitnessValidationResult ValidateProgram203Witness(const Program203WitnessOutput& output) { + IterationRPWitnessValidationResult ValidateIterationRPWitness(const IterationRPWitnessOutput& output) { // 1. Completion. A poisoned or unwritten result must never turn into a // topology diagnosis, because it says nothing about execution. - if (output.magic != kProgram203WitnessMagic) { + if (output.magic != kIterationRPWitnessMagic) { std::ostringstream detail; detail << "completion: magic was 0x" << std::hex << output.magic << ", expected 0x" - << kProgram203WitnessMagic; - return Failure(Program203WitnessValidationFailure::Completion, detail.str()); + << kIterationRPWitnessMagic; + return Failure(IterationRPWitnessValidationFailure::Completion, detail.str()); } // 2. Observed topology. All checks consume observations written by the // shader, rather than inferring subgroup layout from invocation indices. const std::uint32_t numSubgroups = output.numSubgroups; - if (numSubgroups < 2u || numSubgroups > kProgram203WitnessMaxSubgroups) { + if (numSubgroups < 2u || numSubgroups > kIterationRPWitnessMaxSubgroups) { std::ostringstream detail; detail << "topology: canonical gl_NumSubgroups=" << numSubgroups << " is outside [2, 32]"; - return Failure(Program203WitnessValidationFailure::Topology, detail.str()); + return Failure(IterationRPWitnessValidationFailure::Topology, detail.str()); } - if ((output.topologyFlags & Program203WitnessNonuniformNumSubgroups) != 0u) { - return Failure(Program203WitnessValidationFailure::Topology, + if ((output.topologyFlags & IterationRPWitnessNonuniformNumSubgroups) != 0u) { + return Failure(IterationRPWitnessValidationFailure::Topology, "topology: gl_NumSubgroups differed across workgroup"); } - if ((output.topologyFlags & Program203WitnessInvalidNumSubgroups) != 0u) { - return Failure(Program203WitnessValidationFailure::Topology, + if ((output.topologyFlags & IterationRPWitnessInvalidNumSubgroups) != 0u) { + return Failure(IterationRPWitnessValidationFailure::Topology, "topology: an invocation reported gl_NumSubgroups outside [2, 32]"); } - if ((output.topologyFlags & Program203WitnessInvalidSubgroupId) != 0u) { - return Failure(Program203WitnessValidationFailure::Topology, + if ((output.topologyFlags & IterationRPWitnessInvalidSubgroupId) != 0u) { + return Failure(IterationRPWitnessValidationFailure::Topology, "topology: an invocation reported an invalid gl_SubgroupID"); } - if ((output.topologyFlags & Program203WitnessInvalidSubgroupLane) != 0u) { - return Failure(Program203WitnessValidationFailure::Topology, + if ((output.topologyFlags & IterationRPWitnessInvalidSubgroupLane) != 0u) { + return Failure(IterationRPWitnessValidationFailure::Topology, "topology: an invocation reported an invalid subgroup lane"); } - if ((output.topologyFlags & ~(Program203WitnessNonuniformNumSubgroups | - Program203WitnessInvalidNumSubgroups | - Program203WitnessInvalidSubgroupId | - Program203WitnessInvalidSubgroupLane)) != 0u) { + if ((output.topologyFlags & ~(IterationRPWitnessNonuniformNumSubgroups | + IterationRPWitnessInvalidNumSubgroups | + IterationRPWitnessInvalidSubgroupId | + IterationRPWitnessInvalidSubgroupLane)) != 0u) { std::ostringstream detail; detail << "topology: unknown topology flags 0x" << std::hex << output.topologyFlags; - return Failure(Program203WitnessValidationFailure::Topology, detail.str()); + return Failure(IterationRPWitnessValidationFailure::Topology, detail.str()); } const std::uint32_t expectedMask = ExpectedSeenSubgroupMask(numSubgroups); if (output.seenSubgroupMask != expectedMask) { std::ostringstream detail; detail << "topology: seen subgroup-ID mask was 0x" << std::hex << output.seenSubgroupMask << ", expected 0x" << expectedMask; - return Failure(Program203WitnessValidationFailure::Topology, detail.str()); + return Failure(IterationRPWitnessValidationFailure::Topology, detail.str()); } - const std::uint32_t expectedLoopLength = ComputeProgram203WitnessLoopLength(numSubgroups); + const std::uint32_t expectedLoopLength = ComputeIterationRPWitnessLoopLength(numSubgroups); if (output.loopLength != expectedLoopLength) { std::ostringstream detail; detail << "topology: loopLength was " << std::dec << output.loopLength << ", expected " << expectedLoopLength; - return Failure(Program203WitnessValidationFailure::Topology, detail.str()); + return Failure(IterationRPWitnessValidationFailure::Topology, detail.str()); } for (std::uint32_t subgroup = 0u; subgroup < numSubgroups; ++subgroup) { if (output.lastLaneWriterCount[subgroup] != 1u) { std::ostringstream detail; detail << "topology: subgroup " << subgroup << " has " << output.lastLaneWriterCount[subgroup] << " source last-lane writers, expected exactly 1"; - return Failure(Program203WitnessValidationFailure::Topology, detail.str(), 0u, subgroup); + return Failure(IterationRPWitnessValidationFailure::Topology, detail.str(), 0u, subgroup); } } if (output.owner511.y != numSubgroups) { std::ostringstream detail; detail << "final owner: invocation 511 reported gl_NumSubgroups=" << output.owner511.y << ", expected " << numSubgroups; - return Failure(Program203WitnessValidationFailure::FinalOwner, detail.str()); + return Failure(IterationRPWitnessValidationFailure::FinalOwner, detail.str()); } if (output.owner511.z != numSubgroups - 1u) { std::ostringstream detail; detail << "final owner: invocation 511 is not in the highest subgroup (id" << output.owner511.z << ", expected id" << (numSubgroups - 1u) << ')'; - return Failure(Program203WitnessValidationFailure::FinalOwner, detail.str()); + return Failure(IterationRPWitnessValidationFailure::FinalOwner, detail.str()); } if (output.owner511.x == 0u || output.owner511.w != output.owner511.x - 1u) { std::ostringstream detail; detail << "final owner: invocation 511 is not the last lane of highest subgroup (size " << output.owner511.x << ", lane " << output.owner511.w << ')'; - return Failure(Program203WitnessValidationFailure::FinalOwner, detail.str()); + return Failure(IterationRPWitnessValidationFailure::FinalOwner, detail.str()); } // 3. Initial subgroup handoff. The atomic scalar totals are independent @@ -218,22 +218,22 @@ namespace MobileGL::MG_Util::SelfTest { if (indexedTotal != 131328u) { std::ostringstream detail; detail << "initial subgroup handoff: indexed input total was " << indexedTotal << ", expected 131328"; - return Failure(Program203WitnessValidationFailure::InitialSubgroupHandoff, detail.str()); + return Failure(IterationRPWitnessValidationFailure::InitialSubgroupHandoff, detail.str()); } for (std::uint32_t subgroup = 0u; subgroup < numSubgroups; ++subgroup) { - const Program203WitnessVec2 expected = {static_cast(output.indexedInputTotal[subgroup]), 0.0f}; + const IterationRPWitnessVec2 expected = {static_cast(output.indexedInputTotal[subgroup]), 0.0f}; if (!SameBits(output.rawPrefix[subgroup], expected)) { std::ostringstream detail; detail << "initial subgroup handoff: subgroup " << subgroup << " rawPrefix observed " << Vec2String(output.rawPrefix[subgroup]) << ", expected " << Vec2String(expected); - return Failure(Program203WitnessValidationFailure::InitialSubgroupHandoff, detail.str(), 0u, + return Failure(IterationRPWitnessValidationFailure::InitialSubgroupHandoff, detail.str(), 0u, subgroup); } } // 4. Source scan. Do not substitute a conventional scan: this reproduces // the source cache index expression and stage ordering word for word. - std::array expectedCache = output.rawPrefix; + std::array expectedCache = output.rawPrefix; for (std::uint32_t scanStage = 0u; scanStage < expectedLoopLength; ++scanStage) { auto cacheAfterStage = expectedCache; for (std::uint32_t subgroup = 0u; subgroup < numSubgroups; ++subgroup) { @@ -250,27 +250,27 @@ namespace MobileGL::MG_Util::SelfTest { detail << "source scan stage " << scanStage << ", subgroup " << subgroup << ": observed " << Vec2String(output.scanCache[scanStage][subgroup]) << ", expected " << Vec2String(expectedCache[subgroup]); - return Failure(Program203WitnessValidationFailure::SourceScan, detail.str(), scanStage, subgroup); + return Failure(IterationRPWitnessValidationFailure::SourceScan, detail.str(), scanStage, subgroup); } } } // 5. The owner contract was checked above with the other topology facts; // this final result remains a separate exact-vector check. - const Program203WitnessVec2 expectedAverage = {256.5f, 0.0f}; + const IterationRPWitnessVec2 expectedAverage = {256.5f, 0.0f}; if (!SameBits(output.finalAverage, expectedAverage)) { std::ostringstream detail; detail << "final average: observed " << Vec2String(output.finalAverage) << ", expected " << Vec2String(expectedAverage); - return Failure(Program203WitnessValidationFailure::FinalAverage, detail.str()); + return Failure(IterationRPWitnessValidationFailure::FinalAverage, detail.str()); } std::ostringstream detail; detail << "N=" << numSubgroups << ", owner511=id" << output.owner511.z << "/lane" << output.owner511.w << ", " << expectedLoopLength << " scan stages, average=" << Vec2String(output.finalAverage); - Program203WitnessValidationResult result; + IterationRPWitnessValidationResult result; result.ok = true; - result.failure = Program203WitnessValidationFailure::None; + result.failure = IterationRPWitnessValidationFailure::None; result.detail = detail.str(); return result; } diff --git a/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.h b/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.h new file mode 100644 index 00000000..1999ed4f --- /dev/null +++ b/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.h @@ -0,0 +1,153 @@ +// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.h +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header +// +// Compact, native-Vulkan iterationRP first-reduction witness ABI and its pure +// validator. The types below deliberately mirror DriverPostIterationRPWitness.comp's +// single std430 storage block; changing either side requires updating the static +// layout assertions here. + +#pragma once + +#include +#include +#include +#include +#include + +namespace MobileGL::MG_Util::SelfTest { + // "P203": the pack's trace program id, kept stable so the checked-in witness + // SPIR-V (DriverPostIterationRPWitnessSpv.h) needs no regeneration. + constexpr std::uint32_t kIterationRPWitnessMagic = 0x50323033u; + constexpr std::uint32_t kIterationRPWitnessInvocationCount = 512u; + constexpr std::uint32_t kIterationRPWitnessMaxSubgroups = 32u; + constexpr std::uint32_t kIterationRPWitnessMaxScanStages = 6u; + + // These bit values are shared with the GLSL source. They document failures in + // topology observations rather than guessing a topology from local IDs on the host. + enum IterationRPWitnessTopologyFlag : std::uint32_t { + IterationRPWitnessNonuniformNumSubgroups = 1u << 0u, + IterationRPWitnessInvalidNumSubgroups = 1u << 1u, + IterationRPWitnessInvalidSubgroupId = 1u << 2u, + IterationRPWitnessInvalidSubgroupLane = 1u << 3u, + }; + + struct alignas(8) IterationRPWitnessVec2 { + float x; + float y; + }; + + struct alignas(16) IterationRPWitnessUVec4 { + std::uint32_t x; + std::uint32_t y; + std::uint32_t z; + std::uint32_t w; + }; + + // std430 layout of DriverPostIterationRPWitness.comp's IterationRPWitnessOutput block. + struct alignas(16) IterationRPWitnessOutput { + std::uint32_t magic; + std::uint32_t topologyFlags; + std::uint32_t numSubgroups; + std::uint32_t loopLength; + std::uint32_t seenSubgroupMask; + + IterationRPWitnessUVec4 owner511; + + std::array lastLaneWriterCount; + std::array indexedInputTotal; + + std::array rawPrefix; + std::array, + kIterationRPWitnessMaxScanStages> + scanCache; + IterationRPWitnessVec2 finalAverage; + }; + + static_assert(std::is_standard_layout_v); + static_assert(std::is_standard_layout_v); + static_assert(std::is_standard_layout_v); + static_assert(sizeof(IterationRPWitnessVec2) == 8u); + static_assert(alignof(IterationRPWitnessVec2) == 8u); + static_assert(sizeof(IterationRPWitnessUVec4) == 16u); + static_assert(alignof(IterationRPWitnessUVec4) == 16u); + static_assert(offsetof(IterationRPWitnessOutput, magic) == 0u); + static_assert(offsetof(IterationRPWitnessOutput, topologyFlags) == 4u); + static_assert(offsetof(IterationRPWitnessOutput, numSubgroups) == 8u); + static_assert(offsetof(IterationRPWitnessOutput, loopLength) == 12u); + static_assert(offsetof(IterationRPWitnessOutput, seenSubgroupMask) == 16u); + static_assert(offsetof(IterationRPWitnessOutput, owner511) == 32u); + static_assert(offsetof(IterationRPWitnessOutput, lastLaneWriterCount) == 48u); + static_assert(offsetof(IterationRPWitnessOutput, indexedInputTotal) == 176u); + static_assert(offsetof(IterationRPWitnessOutput, rawPrefix) == 304u); + static_assert(offsetof(IterationRPWitnessOutput, scanCache) == 560u); + static_assert(offsetof(IterationRPWitnessOutput, finalAverage) == 2096u); + static_assert(sizeof(IterationRPWitnessOutput) == 2112u); + + // The witness uses prefixSumCache[32], three scalar shared diagnostics, and + // two 32-entry scalar diagnostic arrays in the GLSL source. Keep this + // independent of the output SSBO size. + constexpr std::uint32_t kIterationRPWitnessSharedMemoryBytes = + kIterationRPWitnessMaxSubgroups * sizeof(IterationRPWitnessVec2) + + 3u * sizeof(std::uint32_t) + + 2u * kIterationRPWitnessMaxSubgroups * sizeof(std::uint32_t); + + enum class IterationRPWitnessEligibility { + Execute, + SkipUnsupportedNativeFeatureSet, + FailInadequateLimits, + }; + + // The raw physical-device conditions needed by the native witness. This is + // intentionally distinct from MobileGL's advertised-extension policy. + struct IterationRPWitnessLimits { + bool computeStageSupported = false; + bool basicSubgroupSupported = false; + bool arithmeticSubgroupSupported = false; + std::uint32_t subgroupSize = 0u; + + std::uint32_t maxComputeWorkGroupInvocations = 0u; + std::array maxComputeWorkGroupSize{}; + std::uint32_t maxComputeSharedMemorySize = 0u; + std::uint32_t maxPerStageDescriptorStorageBuffers = 0u; + std::uint32_t maxDescriptorSetStorageBuffers = 0u; + std::uint32_t maxBoundDescriptorSets = 0u; + std::uint64_t maxStorageBufferRange = 0u; + }; + + struct IterationRPWitnessEligibilityResult { + IterationRPWitnessEligibility eligibility = IterationRPWitnessEligibility::FailInadequateLimits; + std::string detail; + }; + + enum class IterationRPWitnessValidationFailure { + None, + Completion, + Topology, + InitialSubgroupHandoff, + SourceScan, + FinalOwner, + FinalAverage, + }; + + struct IterationRPWitnessValidationResult { + bool ok = false; + IterationRPWitnessValidationFailure failure = IterationRPWitnessValidationFailure::Completion; + std::uint32_t scanStage = 0u; + std::uint32_t subgroup = 0u; + std::string detail; + }; + + [[nodiscard]] IterationRPWitnessEligibilityResult + EvaluateIterationRPWitnessEligibility(const IterationRPWitnessLimits& limits); + + // Mirrors the source's findMSB expression for valid N in [2, 32]. + [[nodiscard]] std::uint32_t ComputeIterationRPWitnessLoopLength(std::uint32_t numSubgroups); + + [[nodiscard]] IterationRPWitnessValidationResult + ValidateIterationRPWitness(const IterationRPWitnessOutput& output); +} // namespace MobileGL::MG_Util::SelfTest diff --git a/MobileGL/MG_Util/SelfTest/DriverPostProgram203WitnessSpv.h b/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitnessSpv.h similarity index 97% rename from MobileGL/MG_Util/SelfTest/DriverPostProgram203WitnessSpv.h rename to MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitnessSpv.h index c5bf98eb..556fdcf8 100644 --- a/MobileGL/MG_Util/SelfTest/DriverPostProgram203WitnessSpv.h +++ b/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitnessSpv.h @@ -1,4 +1,4 @@ -// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostProgram203WitnessSpv.h +// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitnessSpv.h // Copyright (c) 2026 MobileGL-Dev // Licensed under the GNU Lesser General Public License v3.0: // https://www.gnu.org/licenses/gpl-3.0.txt @@ -6,9 +6,15 @@ // SPDX-License-Identifier: LGPL-3.0-only // End of Source File Header // -// Generated from DriverPostProgram203Witness.comp with: -// glslangValidator --target-env vulkan1.1 -V DriverPostProgram203Witness.comp +// Generated from DriverPostIterationRPWitness.comp with: +// glslangValidator --target-env vulkan1.1 -V DriverPostIterationRPWitness.comp // Validated with spirv-val --target-env vulkan1.1. Do not edit words by hand. +// +// The stored words predate the Program203 -> IterationRP source rename, so their +// embedded OpName debug strings still spell the old identifiers; regeneration from +// the renamed source produces semantically identical code differing only in those +// strings. The witness magic stays 0x50323033 ("P203" - the trace's program id) so +// these words remain valid without regeneration. #pragma once @@ -16,7 +22,7 @@ #include namespace MobileGL::MG_Util::SelfTest { - inline constexpr std::uint32_t kDriverPostProgram203WitnessSpv[] = { + inline constexpr std::uint32_t kDriverPostIterationRPWitnessSpv[] = { 0x07230203u, 0x00010300u, 0x0008000bu, 0x00000145u, 0x00000000u, 0x00020011u, 0x00000001u, 0x00020011u, 0x0000003du, 0x00020011u, 0x0000003fu, 0x0006000bu, 0x00000001u, 0x4c534c47u, 0x6474732eu, 0x3035342eu, 0x00000000u, 0x0003000eu, 0x00000000u, 0x00000001u, 0x000a000fu, 0x00000005u, 0x00000004u, 0x6e69616du, @@ -286,6 +292,6 @@ namespace MobileGL::MG_Util::SelfTest { 0x00050041u, 0x0000003bu, 0x00000141u, 0x00000037u, 0x00000124u, 0x0003003eu, 0x00000141u, 0x00000140u, 0x000200f9u, 0x0000013du, 0x000200f8u, 0x0000013du, 0x000100fdu, 0x00010038u, }; - inline constexpr std::size_t kDriverPostProgram203WitnessSpvWordCount = - sizeof(kDriverPostProgram203WitnessSpv) / sizeof(kDriverPostProgram203WitnessSpv[0]); + inline constexpr std::size_t kDriverPostIterationRPWitnessSpvWordCount = + sizeof(kDriverPostIterationRPWitnessSpv) / sizeof(kDriverPostIterationRPWitnessSpv[0]); } // namespace MobileGL::MG_Util::SelfTest diff --git a/MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.h b/MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.h deleted file mode 100644 index be7af86c..00000000 --- a/MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.h +++ /dev/null @@ -1,151 +0,0 @@ -// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.h -// Copyright (c) 2026 MobileGL-Dev -// Licensed under the GNU Lesser General Public License v3.0: -// https://www.gnu.org/licenses/gpl-3.0.txt -// https://www.gnu.org/licenses/lgpl-3.0.txt -// SPDX-License-Identifier: LGPL-3.0-only -// End of Source File Header -// -// Compact, native-Vulkan Program-203 first-reduction witness ABI and its pure -// validator. The types below deliberately mirror DriverPostProgram203Witness.comp's -// single std430 storage block; changing either side requires updating the static -// layout assertions here. - -#pragma once - -#include -#include -#include -#include -#include - -namespace MobileGL::MG_Util::SelfTest { - constexpr std::uint32_t kProgram203WitnessMagic = 0x50323033u; // "P203" - constexpr std::uint32_t kProgram203WitnessInvocationCount = 512u; - constexpr std::uint32_t kProgram203WitnessMaxSubgroups = 32u; - constexpr std::uint32_t kProgram203WitnessMaxScanStages = 6u; - - // These bit values are shared with the GLSL source. They document failures in - // topology observations rather than guessing a topology from local IDs on the host. - enum Program203WitnessTopologyFlag : std::uint32_t { - Program203WitnessNonuniformNumSubgroups = 1u << 0u, - Program203WitnessInvalidNumSubgroups = 1u << 1u, - Program203WitnessInvalidSubgroupId = 1u << 2u, - Program203WitnessInvalidSubgroupLane = 1u << 3u, - }; - - struct alignas(8) Program203WitnessVec2 { - float x; - float y; - }; - - struct alignas(16) Program203WitnessUVec4 { - std::uint32_t x; - std::uint32_t y; - std::uint32_t z; - std::uint32_t w; - }; - - // std430 layout of DriverPostProgram203Witness.comp's Program203WitnessOutput block. - struct alignas(16) Program203WitnessOutput { - std::uint32_t magic; - std::uint32_t topologyFlags; - std::uint32_t numSubgroups; - std::uint32_t loopLength; - std::uint32_t seenSubgroupMask; - - Program203WitnessUVec4 owner511; - - std::array lastLaneWriterCount; - std::array indexedInputTotal; - - std::array rawPrefix; - std::array, - kProgram203WitnessMaxScanStages> - scanCache; - Program203WitnessVec2 finalAverage; - }; - - static_assert(std::is_standard_layout_v); - static_assert(std::is_standard_layout_v); - static_assert(std::is_standard_layout_v); - static_assert(sizeof(Program203WitnessVec2) == 8u); - static_assert(alignof(Program203WitnessVec2) == 8u); - static_assert(sizeof(Program203WitnessUVec4) == 16u); - static_assert(alignof(Program203WitnessUVec4) == 16u); - static_assert(offsetof(Program203WitnessOutput, magic) == 0u); - static_assert(offsetof(Program203WitnessOutput, topologyFlags) == 4u); - static_assert(offsetof(Program203WitnessOutput, numSubgroups) == 8u); - static_assert(offsetof(Program203WitnessOutput, loopLength) == 12u); - static_assert(offsetof(Program203WitnessOutput, seenSubgroupMask) == 16u); - static_assert(offsetof(Program203WitnessOutput, owner511) == 32u); - static_assert(offsetof(Program203WitnessOutput, lastLaneWriterCount) == 48u); - static_assert(offsetof(Program203WitnessOutput, indexedInputTotal) == 176u); - static_assert(offsetof(Program203WitnessOutput, rawPrefix) == 304u); - static_assert(offsetof(Program203WitnessOutput, scanCache) == 560u); - static_assert(offsetof(Program203WitnessOutput, finalAverage) == 2096u); - static_assert(sizeof(Program203WitnessOutput) == 2112u); - - // The witness uses prefixSumCache[32], three scalar shared diagnostics, and - // two 32-entry scalar diagnostic arrays in the GLSL source. Keep this - // independent of the output SSBO size. - constexpr std::uint32_t kProgram203WitnessSharedMemoryBytes = - kProgram203WitnessMaxSubgroups * sizeof(Program203WitnessVec2) + - 3u * sizeof(std::uint32_t) + - 2u * kProgram203WitnessMaxSubgroups * sizeof(std::uint32_t); - - enum class Program203WitnessEligibility { - Execute, - SkipUnsupportedNativeFeatureSet, - FailInadequateLimits, - }; - - // The raw physical-device conditions needed by the native witness. This is - // intentionally distinct from MobileGL's advertised-extension policy. - struct Program203WitnessLimits { - bool computeStageSupported = false; - bool basicSubgroupSupported = false; - bool arithmeticSubgroupSupported = false; - std::uint32_t subgroupSize = 0u; - - std::uint32_t maxComputeWorkGroupInvocations = 0u; - std::array maxComputeWorkGroupSize{}; - std::uint32_t maxComputeSharedMemorySize = 0u; - std::uint32_t maxPerStageDescriptorStorageBuffers = 0u; - std::uint32_t maxDescriptorSetStorageBuffers = 0u; - std::uint32_t maxBoundDescriptorSets = 0u; - std::uint64_t maxStorageBufferRange = 0u; - }; - - struct Program203WitnessEligibilityResult { - Program203WitnessEligibility eligibility = Program203WitnessEligibility::FailInadequateLimits; - std::string detail; - }; - - enum class Program203WitnessValidationFailure { - None, - Completion, - Topology, - InitialSubgroupHandoff, - SourceScan, - FinalOwner, - FinalAverage, - }; - - struct Program203WitnessValidationResult { - bool ok = false; - Program203WitnessValidationFailure failure = Program203WitnessValidationFailure::Completion; - std::uint32_t scanStage = 0u; - std::uint32_t subgroup = 0u; - std::string detail; - }; - - [[nodiscard]] Program203WitnessEligibilityResult - EvaluateProgram203WitnessEligibility(const Program203WitnessLimits& limits); - - // Mirrors the source's findMSB expression for valid N in [2, 32]. - [[nodiscard]] std::uint32_t ComputeProgram203WitnessLoopLength(std::uint32_t numSubgroups); - - [[nodiscard]] Program203WitnessValidationResult - ValidateProgram203Witness(const Program203WitnessOutput& output); -} // namespace MobileGL::MG_Util::SelfTest diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp index 28367501..36b852c9 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp @@ -26,6 +26,8 @@ #include "SpirvPasses/RebaseInstanceIndexPass.h" #include "SpirvPasses/ZeroBaseVertexPass.h" #include "SpirvPasses/DeriveNumSubgroupsPass.h" +#include "SpirvPasses/EmulateSubgroupsPass.h" +#include "SpirvPasses/FixIterationRPSubgroupScratchPass.h" #include "SpirvPasses/NormalizeRectCoordinatesPass.h" #include "SpirvPasses/Lower1DArrayImagesPass.h" #include "SpirvPasses/BakeImageFormatsPass.h" @@ -895,6 +897,32 @@ namespace MobileGL { outputBinary, true, enableSpirvValidation); } + bool ShaderCompiler::EmulateSubgroupsForVulkan(const Vector& inputBinary, + Vector& outputBinary, + const Uint32 maxWorkgroupScratchBytes, + const bool enableSpirvValidation) { + using namespace spvtools; + Optimizer optimizer(SPV_ENV_VULKAN_1_1); + optimizer.RegisterPass( + EmulateSubgroupsPass::CreateEmulateSubgroupsPass(maxWorkgroupScratchBytes)); + + return RunOptimizerChecked("EmulateSubgroupsForVulkan", optimizer, inputBinary, + outputBinary, true, enableSpirvValidation); + } + + bool ShaderCompiler::FixIterationRPSubgroupScratchForVulkan( + const Vector& inputBinary, Vector& outputBinary, + const Uint32 nativeSubgroupSize, const bool enableSpirvValidation) { + using namespace spvtools; + Optimizer optimizer(SPV_ENV_VULKAN_1_1); + optimizer.RegisterPass( + FixIterationRPSubgroupScratchPass::CreateFixIterationRPSubgroupScratchPass( + nativeSubgroupSize)); + + return RunOptimizerChecked("FixIterationRPSubgroupScratchForVulkan", optimizer, + inputBinary, outputBinary, true, enableSpirvValidation); + } + bool ShaderCompiler::DecoratePositionInvariantForVulkan(const Vector& inputBinary, Vector& outputBinary, const bool enableSpirvValidation) { using namespace spvtools; diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h index 5073de2b..53504f00 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h @@ -145,13 +145,35 @@ namespace MobileGL { static bool ZeroBaseVertexForVulkan(const Vector& inputBinary, Vector& outputBinary, bool enableSpirvValidation = false); - // Replaces compute gl_NumSubgroups loads with the value derived from the local - // workgroup dimensions and gl_SubgroupSize. DirectVulkan only; this avoids a - // driver builtin that can disagree with the subgroup IDs the same dispatch emits. - // See DeriveNumSubgroupsPass. + // Replaces compute gl_NumSubgroups loads with ceil(workgroup invocations / + // gl_SubgroupSize). DirectVulkan only; this repairs drivers whose builtin + // disagrees with the subgroup IDs the same dispatch emits (Adreno reports 1 + // while emitting IDs 0..7). The ceil() partition is only spec-guaranteed + // under VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT, which + // the caller requests whenever it is legal for the workgroup shape; see + // DeriveNumSubgroupsPass. static bool DeriveNumSubgroupsForVulkan(const Vector& inputBinary, Vector& outputBinary, bool enableSpirvValidation = false); + // Lowers every GL_KHR_shader_subgroup construct in a compute module onto a + // 32-lane virtual subgroup built from workgroup-shared memory. Last-resort + // path for devices with NO native subgroup support, opt-in via + // MOBILEGL_MAGMA_EMULATE_SUBGROUP=1; a device with native subgroup + // operations always uses them. maxWorkgroupScratchBytes bounds the shared + // scratch the lowering may add (pass the device's + // maxComputeSharedMemorySize; 0 falls back to the 16384-byte Vulkan + // minimum). See EmulateSubgroupsPass. + static bool EmulateSubgroupsForVulkan(const Vector& inputBinary, + Vector& outputBinary, + Uint32 maxWorkgroupScratchBytes, + bool enableSpirvValidation = false); + // Patches iterationRP's under-declared prefixSumCache[32] on sub-16-lane + // devices, fingerprint-gated to that pack's reduction; every other module + // passes through byte-identical. See FixIterationRPSubgroupScratchPass. + static bool FixIterationRPSubgroupScratchForVulkan(const Vector& inputBinary, + Vector& outputBinary, + Uint32 nativeSubgroupSize, + bool enableSpirvValidation = false); // Re-declares 64-bit float vertex inputs as their 32-bit unsigned word pair // (double -> uvec2, dvec2 -> uvec4) and bitcasts them back to double at entry, so no // VK_FORMAT_R64*_SFLOAT is needed - lavapipe advertises none of them for vertex diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.cpp b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.cpp index f3fbeacf..2bc1c8af 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.cpp +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.cpp @@ -179,8 +179,14 @@ namespace MobileGL { : SynthesizeSubgroupSizeVariable(irContext, numSubgroupsVar->type_id()); const uint32_t workgroupSizeId = workgroupSize->result_id(); - // The pipeline never enables ALLOW_VARYING_SUBGROUP_SIZE, so Vulkan's fixed - // subgroup partition is exactly ceil(local invocation count / SubgroupSize). + // ceil(local invocation count / SubgroupSize): the subgroup count of a + // full-subgroup launch. Vulkan only guarantees that partition under + // REQUIRE_FULL_SUBGROUPS - which ProgramFactory requests whenever + // local_size_x is a multiple of the subgroup size makes it legal + // (VUID-VkPipelineShaderStageCreateInfo-flags-02759) - and calls the + // tighter behaviour "encouraged" everywhere else; the DriverPost witness + // verifies it per device where the flag cannot be set. The absence of + // ALLOW_VARYING_SUBGROUP_SIZE pins only the SubgroupSize builtin itself. // `(count - 1) / size + 1` avoids an addition overflow at count + size - 1. for (Instruction* load : numSubgroupsLoads) { const uint32_t localSizeXId = irContext->TakeNextId(); diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.h b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.h index 330c17a4..e5b117e3 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.h +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.h @@ -19,12 +19,14 @@ namespace MobileGL { // Replaces compute-stage NumSubgroups builtin loads with // ceil(WorkgroupSize.x * WorkgroupSize.y * WorkgroupSize.z / SubgroupSize). // - // That is the value Vulkan defines for NumSubgroups when the pipeline does not - // enable varying subgroup sizes, which MobileGL never does. Deriving it avoids - // drivers that expose the real SubgroupId topology but return an inconsistent - // NumSubgroups value. This is a DirectVulkan semantic repair, not a source-shader - // rewrite; the application's subgroup arithmetic and shared-memory logic remain - // unchanged. + // That is the subgroup count of a full-subgroup launch - guaranteed by Vulkan + // under REQUIRE_FULL_SUBGROUPS (which ProgramFactory requests whenever the + // workgroup shape makes it legal), spec-"encouraged" and witness-verified + // (DriverPost) elsewhere. Deriving it repairs drivers that expose the real + // SubgroupId topology but return an inconsistent NumSubgroups value, breaking + // GL's gl_SubgroupID < gl_NumSubgroups contract. This is a DirectVulkan + // semantic repair, not a source-shader rewrite; the application's subgroup + // arithmetic and shared-memory logic remain unchanged. class DeriveNumSubgroupsPass : public spvtools::opt::Pass { public: const char* name() const override { return "derive-num-subgroups"; } diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.cpp b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.cpp new file mode 100644 index 00000000..7ef6df64 --- /dev/null +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.cpp @@ -0,0 +1,1413 @@ +// MobileGL - MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.cpp +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#include "EmulateSubgroupsPass.h" + +#include "spirv.hpp" +#include "source/opt/constants.h" +#include "source/opt/def_use_manager.h" +#include "source/opt/instruction.h" +#include "source/opt/ir_context.h" +#include "source/opt/module.h" +#include "source/opt/type_manager.h" +#include "source/util/make_unique.h" +#include "source/util/string_utils.h" + +#include +#include +#include +#include + +namespace MobileGL { + namespace MG_Util { + namespace ShaderTranspiler { + namespace { + using spvtools::opt::Instruction; + using spvtools::opt::IRContext; + using spvtools::opt::Operand; + namespace analysis = spvtools::opt::analysis; + + // The GL-visible subgroup this pass implements. Keep in sync with + // SubgroupSupportPolicy.h (kEmulatedSubgroupSize). + constexpr uint32_t kWidth = 32u; + constexpr uint32_t kLaneMask = kWidth - 1u; + constexpr uint32_t kIdShift = 5u; + + // GLSL.std.450 instruction numbers (see 3rdparty/glslang/SPIRV/GLSL.std.450.h). + constexpr uint32_t kGlslFMin = 37u; + constexpr uint32_t kGlslUMin = 38u; + constexpr uint32_t kGlslSMin = 39u; + constexpr uint32_t kGlslFMax = 40u; + constexpr uint32_t kGlslUMax = 41u; + constexpr uint32_t kGlslSMax = 42u; + constexpr uint32_t kGlslFindILsb = 73u; + constexpr uint32_t kGlslFindUMsb = 75u; + + Operand IdOp(uint32_t id) { return {SPV_OPERAND_TYPE_ID, {id}}; } + + Instruction* FindBuiltinDefinition(IRContext* context, spv::BuiltIn builtin) { + auto* defUseMgr = context->get_def_use_mgr(); + for (auto& annotation : context->annotations()) { + if (annotation.opcode() != spv::Op::OpDecorate || annotation.NumInOperands() < 3) { + continue; + } + if (static_cast(annotation.GetSingleWordInOperand(1)) != + spv::Decoration::BuiltIn) { + continue; + } + if (static_cast(annotation.GetSingleWordInOperand(2)) != builtin) { + continue; + } + return defUseMgr->GetDef(annotation.GetSingleWordInOperand(0)); + } + return nullptr; + } + + // The scalar shapes a value participating in an emulated exchange may have. + enum class ScalarKind { Float, SInt, UInt, Bool }; + + struct ValueTypeInfo { + ScalarKind kind = ScalarKind::UInt; + uint32_t componentCount = 1; // 1 for scalars, 2..4 for vectors + uint32_t typeId = 0; // the value's own type + uint32_t scalarTypeId = 0; // the component type + }; + + bool ClassifyScalar(const Instruction* type, ScalarKind* kind) { + switch (type->opcode()) { + case spv::Op::OpTypeBool: + *kind = ScalarKind::Bool; + return true; + case spv::Op::OpTypeInt: + if (type->GetSingleWordInOperand(0) != 32u) return false; + *kind = type->GetSingleWordInOperand(1) == 0u ? ScalarKind::UInt : ScalarKind::SInt; + return true; + case spv::Op::OpTypeFloat: + if (type->GetSingleWordInOperand(0) != 32u) return false; + *kind = ScalarKind::Float; + return true; + default: + return false; + } + } + + bool ClassifyValueType(IRContext* context, uint32_t typeId, ValueTypeInfo* info) { + const Instruction* type = context->get_def_use_mgr()->GetDef(typeId); + if (type == nullptr) return false; + info->typeId = typeId; + if (type->opcode() == spv::Op::OpTypeVector) { + info->componentCount = type->GetSingleWordInOperand(1); + if (info->componentCount < 2u || info->componentCount > 4u) return false; + info->scalarTypeId = type->GetSingleWordInOperand(0); + const Instruction* component = context->get_def_use_mgr()->GetDef(info->scalarTypeId); + return component != nullptr && ClassifyScalar(component, &info->kind); + } + info->componentCount = 1u; + info->scalarTypeId = typeId; + return ClassifyScalar(type, &info->kind); + } + + // Emits instructions immediately before a fixed anchor instruction. Every + // lowering sequence is straight-line code in the anchor's own block, so + // dominance is trivially preserved. + class Emitter { + public: + Emitter(IRContext* context, Instruction* anchor) : m_context(context), m_anchor(anchor) {} + + uint32_t Emit(spv::Op opcode, uint32_t typeId, Instruction::OperandList operands) { + const uint32_t resultId = m_context->TakeNextId(); + m_anchor->InsertBefore(spvtools::MakeUnique( + m_context, opcode, typeId, resultId, std::move(operands))); + return resultId; + } + + void EmitNoResult(spv::Op opcode, Instruction::OperandList operands) { + m_anchor->InsertBefore(spvtools::MakeUnique( + m_context, opcode, 0, 0, std::move(operands))); + } + + private: + IRContext* m_context; + Instruction* m_anchor; + }; + + // One recognized OpGroupNonUniform* site with everything Phase A resolved + // about it, so Phase B can rewrite without def-use queries. + struct GroupOpSite { + Instruction* inst = nullptr; + spv::Op opcode = spv::Op::OpNop; + spv::GroupOperation groupOperation = spv::GroupOperation::Reduce; + uint32_t clusterSize = 0; // ClusteredReduce only + uint32_t quadDirection = 0; // QuadSwap only + ValueTypeInfo valueType{}; // participating value (where applicable) + uint32_t valueId = 0; + uint32_t indexId = 0; // broadcast/shuffle/extract index operand + }; + + struct BarrierSite { + Instruction* inst = nullptr; + bool executionScopeIsSubgroup = false; + bool memoryScopeIsSubgroup = false; + }; + + bool IsGroupNonUniformOpcode(spv::Op opcode) { + return opcode >= spv::Op::OpGroupNonUniformElect && + opcode <= spv::Op::OpGroupNonUniformQuadSwap; + } + } // namespace + + spvtools::opt::Pass::Status EmulateSubgroupsPass::Process() { + auto* irContext = context(); + auto* defUseMgr = irContext->get_def_use_mgr(); + auto* constMgr = irContext->get_constant_mgr(); + auto* typeMgr = irContext->get_type_mgr(); + + // ---------------------------------------------------------------- Phase A + // Pure analysis; nothing is mutated until every site has been vetted, so a + // Failure return leaves the module untouched. + + bool allEntryPointsCompute = true; + for (const Instruction& entryPoint : irContext->module()->entry_points()) { + if (static_cast(entryPoint.GetSingleWordInOperand(0)) != + spv::ExecutionModel::GLCompute) { + allEntryPointsCompute = false; + } + } + + const auto resolveUintConstant = [&](uint32_t id, uint32_t* value) { + const Instruction* def = defUseMgr->GetDef(id); + if (def == nullptr || def->opcode() != spv::Op::OpConstant) return false; + *value = def->GetSingleWordInOperand(0); + return true; + }; + + // Subgroup builtin variables and their loads. + static constexpr std::array kSubgroupBuiltins = { + spv::BuiltIn::SubgroupSize, spv::BuiltIn::SubgroupLocalInvocationId, + spv::BuiltIn::SubgroupId, spv::BuiltIn::NumSubgroups, + spv::BuiltIn::SubgroupEqMask, spv::BuiltIn::SubgroupGeMask, + spv::BuiltIn::SubgroupGtMask, spv::BuiltIn::SubgroupLeMask, + spv::BuiltIn::SubgroupLtMask}; + + const auto isMaskBuiltin = [](spv::BuiltIn builtin) { + return builtin == spv::BuiltIn::SubgroupEqMask || + builtin == spv::BuiltIn::SubgroupGeMask || + builtin == spv::BuiltIn::SubgroupGtMask || + builtin == spv::BuiltIn::SubgroupLeMask || + builtin == spv::BuiltIn::SubgroupLtMask; + }; + struct BuiltinUse { + spv::BuiltIn builtin; + Instruction* variable; + std::vector loads; + // Mask builtins are uvec4; glslang reaches a single component (the + // ubiquitous gl_Subgroup*Mask.x) through an access chain plus a scalar + // load. The chain dies with the variable once its loads are rewritten. + std::vector accessChains; + std::vector> componentLoads; + }; + std::vector builtinUses; + bool sawUnexpectedUser = false; + for (const spv::BuiltIn builtin : kSubgroupBuiltins) { + Instruction* variable = FindBuiltinDefinition(irContext, builtin); + if (variable == nullptr || variable->opcode() != spv::Op::OpVariable) continue; + BuiltinUse use{builtin, variable, {}, {}, {}}; + const uint32_t variableId = variable->result_id(); + defUseMgr->ForEachUser(variable, [&](Instruction* user) { + switch (user->opcode()) { + case spv::Op::OpLoad: + if (user->NumInOperands() >= 1 && + user->GetSingleWordInOperand(0) == variableId) { + use.loads.push_back(user); + } else { + sawUnexpectedUser = true; + } + return; + case spv::Op::OpAccessChain: + case spv::Op::OpInBoundsAccessChain: { + uint32_t component = 0; + if (!isMaskBuiltin(builtin) || user->NumInOperands() != 2 || + user->GetSingleWordInOperand(0) != variableId) { + sawUnexpectedUser = true; + return; + } + const Instruction* index = + defUseMgr->GetDef(user->GetSingleWordInOperand(1)); + if (index == nullptr || index->opcode() != spv::Op::OpConstant || + (component = index->GetSingleWordInOperand(0)) > 3u) { + sawUnexpectedUser = true; + return; + } + const uint32_t chainId = user->result_id(); + defUseMgr->ForEachUser(user, [&](Instruction* chainUser) { + if (chainUser->opcode() == spv::Op::OpLoad && + chainUser->NumInOperands() >= 1 && + chainUser->GetSingleWordInOperand(0) == chainId) { + use.componentLoads.emplace_back(chainUser, component); + } else { + sawUnexpectedUser = true; + } + }); + use.accessChains.push_back(user); + return; + } + case spv::Op::OpDecorate: + case spv::Op::OpDecorateId: + case spv::Op::OpDecorateString: + case spv::Op::OpName: + case spv::Op::OpEntryPoint: + return; + default: + sawUnexpectedUser = true; + return; + } + }); + builtinUses.push_back(std::move(use)); + } + + // Group-nonuniform operations and subgroup-scoped barriers. + std::vector groupOps; + std::vector barriers; + bool sawUnsupported = false; + for (auto& function : *irContext->module()) { + for (auto& block : function) { + for (auto& inst : block) { + const spv::Op opcode = inst.opcode(); + // Extended subgroup instructions live outside the core + // [Elect..QuadSwap] range IsGroupNonUniformOpcode covers; they + // must fail the pass rather than survive into the "subgroup + // free" output. + if (opcode == spv::Op::OpGroupNonUniformPartitionNV || + opcode == spv::Op::OpGroupNonUniformRotateKHR || + opcode == spv::Op::OpGroupNonUniformQuadAllKHR || + opcode == spv::Op::OpGroupNonUniformQuadAnyKHR) { + sawUnsupported = true; + continue; + } + if (opcode == spv::Op::OpControlBarrier) { + BarrierSite site{&inst, false, false}; + uint32_t scope = 0; + if (resolveUintConstant(inst.GetSingleWordInOperand(0), &scope) && + scope == static_cast(spv::Scope::Subgroup)) { + site.executionScopeIsSubgroup = true; + } + if (resolveUintConstant(inst.GetSingleWordInOperand(1), &scope) && + scope == static_cast(spv::Scope::Subgroup)) { + site.memoryScopeIsSubgroup = true; + } + if (site.executionScopeIsSubgroup || site.memoryScopeIsSubgroup) { + barriers.push_back(site); + } + continue; + } + if (opcode == spv::Op::OpMemoryBarrier) { + uint32_t scope = 0; + if (resolveUintConstant(inst.GetSingleWordInOperand(0), &scope) && + scope == static_cast(spv::Scope::Subgroup)) { + barriers.push_back(BarrierSite{&inst, false, true}); + } + continue; + } + if (!IsGroupNonUniformOpcode(opcode)) continue; + + GroupOpSite site; + site.inst = &inst; + site.opcode = opcode; + uint32_t scope = 0; + if (!resolveUintConstant(inst.GetSingleWordInOperand(0), &scope) || + scope != static_cast(spv::Scope::Subgroup)) { + sawUnsupported = true; + continue; + } + switch (opcode) { + case spv::Op::OpGroupNonUniformElect: + break; + case spv::Op::OpGroupNonUniformAll: + case spv::Op::OpGroupNonUniformAny: + case spv::Op::OpGroupNonUniformAllEqual: + case spv::Op::OpGroupNonUniformBroadcastFirst: + case spv::Op::OpGroupNonUniformBallot: + case spv::Op::OpGroupNonUniformInverseBallot: + case spv::Op::OpGroupNonUniformBallotFindLSB: + case spv::Op::OpGroupNonUniformBallotFindMSB: + site.valueId = inst.GetSingleWordInOperand(1); + break; + case spv::Op::OpGroupNonUniformBroadcast: + case spv::Op::OpGroupNonUniformShuffle: + case spv::Op::OpGroupNonUniformShuffleXor: + case spv::Op::OpGroupNonUniformShuffleUp: + case spv::Op::OpGroupNonUniformShuffleDown: + case spv::Op::OpGroupNonUniformQuadBroadcast: + case spv::Op::OpGroupNonUniformBallotBitExtract: + site.valueId = inst.GetSingleWordInOperand(1); + site.indexId = inst.GetSingleWordInOperand(2); + break; + case spv::Op::OpGroupNonUniformQuadSwap: + site.valueId = inst.GetSingleWordInOperand(1); + if (!resolveUintConstant(inst.GetSingleWordInOperand(2), + &site.quadDirection) || + site.quadDirection > 2u) { + sawUnsupported = true; + } + break; + case spv::Op::OpGroupNonUniformBallotBitCount: + site.groupOperation = static_cast( + inst.GetSingleWordInOperand(1)); + site.valueId = inst.GetSingleWordInOperand(2); + break; + case spv::Op::OpGroupNonUniformIAdd: + case spv::Op::OpGroupNonUniformFAdd: + case spv::Op::OpGroupNonUniformIMul: + case spv::Op::OpGroupNonUniformFMul: + case spv::Op::OpGroupNonUniformSMin: + case spv::Op::OpGroupNonUniformUMin: + case spv::Op::OpGroupNonUniformFMin: + case spv::Op::OpGroupNonUniformSMax: + case spv::Op::OpGroupNonUniformUMax: + case spv::Op::OpGroupNonUniformFMax: + case spv::Op::OpGroupNonUniformBitwiseAnd: + case spv::Op::OpGroupNonUniformBitwiseOr: + case spv::Op::OpGroupNonUniformBitwiseXor: + case spv::Op::OpGroupNonUniformLogicalAnd: + case spv::Op::OpGroupNonUniformLogicalOr: + case spv::Op::OpGroupNonUniformLogicalXor: + site.groupOperation = static_cast( + inst.GetSingleWordInOperand(1)); + site.valueId = inst.GetSingleWordInOperand(2); + if (site.groupOperation == spv::GroupOperation::ClusteredReduce) { + if (inst.NumInOperands() < 4 || + !resolveUintConstant(inst.GetSingleWordInOperand(3), + &site.clusterSize) || + site.clusterSize == 0u || + (site.clusterSize & (site.clusterSize - 1u)) != 0u) { + sawUnsupported = true; + } + } else if (site.groupOperation != spv::GroupOperation::Reduce && + site.groupOperation != spv::GroupOperation::InclusiveScan && + site.groupOperation != spv::GroupOperation::ExclusiveScan) { + sawUnsupported = true; + } + break; + default: + sawUnsupported = true; + continue; + } + + // Classify the participating value where the lowering exchanges + // it through shared memory (everything except Elect and the pure + // ballot-math ops, whose operand is the uvec4 ballot itself). + switch (opcode) { + case spv::Op::OpGroupNonUniformElect: + case spv::Op::OpGroupNonUniformInverseBallot: + case spv::Op::OpGroupNonUniformBallotBitExtract: + case spv::Op::OpGroupNonUniformBallotBitCount: + case spv::Op::OpGroupNonUniformBallotFindLSB: + case spv::Op::OpGroupNonUniformBallotFindMSB: + break; + case spv::Op::OpGroupNonUniformAll: + case spv::Op::OpGroupNonUniformAny: + case spv::Op::OpGroupNonUniformBallot: { + const Instruction* value = defUseMgr->GetDef(site.valueId); + if (value == nullptr || + !ClassifyValueType(irContext, value->type_id(), &site.valueType) || + site.valueType.kind != ScalarKind::Bool || + site.valueType.componentCount != 1u) { + sawUnsupported = true; + } + break; + } + default: { + const Instruction* value = defUseMgr->GetDef(site.valueId); + if (value == nullptr || + !ClassifyValueType(irContext, value->type_id(), &site.valueType)) { + sawUnsupported = true; + } + break; + } + } + groupOps.push_back(site); + } + } + } + + bool anyBuiltinLoads = false; + for (const BuiltinUse& use : builtinUses) { + if (!use.loads.empty() || !use.componentLoads.empty()) anyBuiltinLoads = true; + } + if (!anyBuiltinLoads && groupOps.empty() && barriers.empty() && !sawUnsupported) { + return Status::SuccessWithoutChange; + } + if (!allEntryPointsCompute || sawUnexpectedUser || sawUnsupported) { + return Status::Failure; + } + + // Workgroup size. The compile chain pins SPIR-V 1.3, where a literal + // local_size always reaches the module as OpExecutionMode LocalSize and - + // through glslang - as the WorkgroupSize builtin constant as well. Either + // is accepted; spec-constant sizes are not (GL has no specialization). + uint32_t localSize[3] = {0, 0, 0}; + bool haveLocalSize = false; + if (Instruction* workgroupSize = + FindBuiltinDefinition(irContext, spv::BuiltIn::WorkgroupSize)) { + if (workgroupSize->opcode() == spv::Op::OpConstantComposite && + workgroupSize->NumInOperands() == 3) { + haveLocalSize = + resolveUintConstant(workgroupSize->GetSingleWordInOperand(0), &localSize[0]) && + resolveUintConstant(workgroupSize->GetSingleWordInOperand(1), &localSize[1]) && + resolveUintConstant(workgroupSize->GetSingleWordInOperand(2), &localSize[2]); + } + } + if (!haveLocalSize) { + for (const Instruction& mode : irContext->module()->execution_modes()) { + if (mode.opcode() == spv::Op::OpExecutionMode && + static_cast(mode.GetSingleWordInOperand(1)) == + spv::ExecutionMode::LocalSize) { + localSize[0] = mode.GetSingleWordInOperand(2); + localSize[1] = mode.GetSingleWordInOperand(3); + localSize[2] = mode.GetSingleWordInOperand(4); + haveLocalSize = true; + break; + } + } + } + if (!haveLocalSize || localSize[0] == 0u || localSize[1] == 0u || localSize[2] == 0u) { + return Status::Failure; + } + const uint64_t total64 = + static_cast(localSize[0]) * localSize[1] * localSize[2]; + if (total64 == 0u || total64 > (1u << 20)) { + return Status::Failure; + } + const uint32_t totalInvocations = static_cast(total64); + const uint32_t virtualSubgroups = (totalInvocations + kWidth - 1u) / kWidth; + const uint32_t paddedSlots = virtualSubgroups * kWidth; + + // Phase C creates one Workgroup array of paddedSlots elements per distinct + // participating element type (bools travel as uint words), 4 bytes per + // component. Refuse up front any module whose added scratch would exceed + // the caller's budget - the device's maxComputeSharedMemorySize, or the + // 16384-byte Vulkan minimum when the caller passed 0 - since spirv-val + // cannot catch this and the pipeline would fail at creation instead. The + // estimate over-counts slightly (a bool type and its uint twin are keyed + // separately) and does not subtract the module's own shared declarations. + { + const uint64_t scratchBudgetBytes = + m_maxWorkgroupScratchBytes != 0u ? m_maxWorkgroupScratchBytes : 16384u; + std::set scratchTypeKeys; + uint64_t scratchBytes = 0; + for (const GroupOpSite& site : groupOps) { + if (site.valueType.typeId == 0u) continue; // no shared-memory exchange + const uint32_t key = site.valueType.kind == ScalarKind::Bool + ? (0x80000000u | site.valueType.componentCount) + : site.valueType.typeId; + if (!scratchTypeKeys.insert(key).second) continue; + scratchBytes += static_cast(paddedSlots) * 4u * + site.valueType.componentCount; + } + if (scratchBytes > scratchBudgetBytes) { + return Status::Failure; + } + } + + // ---------------------------------------------------------------- Phase B + // Cached types and constants. + analysis::Bool boolTypeCandidate; + const uint32_t boolTypeId = typeMgr->GetTypeInstruction(&boolTypeCandidate); + analysis::Integer uintTypeCandidate(32, false); + const uint32_t uintTypeId = typeMgr->GetTypeInstruction(&uintTypeCandidate); + if (boolTypeId == 0u || uintTypeId == 0u) { + return Status::Failure; + } + + const auto uintConst = [&](uint32_t value) -> uint32_t { + const analysis::Type* type = typeMgr->GetType(uintTypeId); + const analysis::Constant* constant = constMgr->GetConstant(type, {value}); + const Instruction* inst = + constant != nullptr ? constMgr->GetDefiningInstruction(constant, uintTypeId) + : nullptr; + return inst != nullptr ? inst->result_id() : 0u; + }; + const auto boolConst = [&](bool value) -> uint32_t { + const analysis::Type* type = typeMgr->GetType(boolTypeId); + const analysis::Constant* constant = + constMgr->GetConstant(type, {value ? 1u : 0u}); + const Instruction* inst = + constant != nullptr ? constMgr->GetDefiningInstruction(constant, boolTypeId) + : nullptr; + return inst != nullptr ? inst->result_id() : 0u; + }; + // A constant of an arbitrary 32-bit scalar type from its bit pattern, splat + // to the value type when it is a vector. + const auto typedConst = [&](const ValueTypeInfo& info, uint32_t scalarBits, + uint32_t scratchScalarTypeId, + uint32_t scratchTypeId) -> uint32_t { + const analysis::Type* scalarType = typeMgr->GetType(scratchScalarTypeId); + const analysis::Constant* scalar = constMgr->GetConstant(scalarType, {scalarBits}); + if (scalar == nullptr) return 0u; + const Instruction* scalarInst = + constMgr->GetDefiningInstruction(scalar, scratchScalarTypeId); + if (scalarInst == nullptr) return 0u; + if (info.componentCount == 1u) return scalarInst->result_id(); + const analysis::Type* vectorType = typeMgr->GetType(scratchTypeId); + std::vector componentIds(info.componentCount, + scalarInst->result_id()); + const analysis::Constant* vector = constMgr->GetConstant(vectorType, componentIds); + if (vector == nullptr) return 0u; + const Instruction* vectorInst = + constMgr->GetDefiningInstruction(vector, scratchTypeId); + return vectorInst != nullptr ? vectorInst->result_id() : 0u; + }; + + const uint32_t c0 = uintConst(0u); + const uint32_t c1 = uintConst(1u); + const uint32_t c31 = uintConst(kLaneMask); + const uint32_t c32 = uintConst(kWidth); + const uint32_t cNotLane = uintConst(~kLaneMask); + const uint32_t cTotal = uintConst(totalInvocations); + const uint32_t cAllOnes = uintConst(0xffffffffu); + const uint32_t cTrue = boolConst(true); + if (c0 == 0u || c1 == 0u || c31 == 0u || c32 == 0u || cNotLane == 0u || + cTotal == 0u || cAllOnes == 0u || cTrue == 0u) { + return Status::Failure; + } + + // GLSL.std.450 import (for min/max combines and ballot find ops). + uint32_t glslStd450Id = 0; + for (const Instruction& import : irContext->module()->ext_inst_imports()) { + const std::string importName = + spvtools::utils::MakeString(import.GetInOperand(0).words); + if (importName == "GLSL.std.450") { + glslStd450Id = import.result_id(); + break; + } + } + const auto ensureGlslStd450 = [&]() -> uint32_t { + if (glslStd450Id != 0u) return glslStd450Id; + glslStd450Id = irContext->TakeNextId(); + irContext->module()->AddExtInstImport(spvtools::MakeUnique( + irContext, spv::Op::OpExtInstImport, 0, glslStd450Id, + Instruction::OperandList{ + {SPV_OPERAND_TYPE_LITERAL_STRING, + spvtools::utils::MakeVector("GLSL.std.450")}})); + return glslStd450Id; + }; + + // gl_LocalInvocationIndex drives the whole virtual topology; synthesize the + // builtin when the module never declared it. + Instruction* liiVariable = + FindBuiltinDefinition(irContext, spv::BuiltIn::LocalInvocationIndex); + uint32_t liiVariableId = 0; + if (liiVariable != nullptr && liiVariable->opcode() == spv::Op::OpVariable) { + liiVariableId = liiVariable->result_id(); + } else { + const uint32_t pointerTypeId = + typeMgr->FindPointerToType(uintTypeId, spv::StorageClass::Input); + if (pointerTypeId == 0u) return Status::Failure; + liiVariableId = irContext->TakeNextId(); + irContext->AddGlobalValue(spvtools::MakeUnique( + irContext, spv::Op::OpVariable, pointerTypeId, liiVariableId, + Instruction::OperandList{ + {SPV_OPERAND_TYPE_STORAGE_CLASS, + {static_cast(spv::StorageClass::Input)}}})); + irContext->AddAnnotationInst(spvtools::MakeUnique( + irContext, spv::Op::OpDecorate, 0, 0, + Instruction::OperandList{ + IdOp(liiVariableId), + {SPV_OPERAND_TYPE_DECORATION, + {static_cast(spv::Decoration::BuiltIn)}}, + {SPV_OPERAND_TYPE_LITERAL_INTEGER, + {static_cast(spv::BuiltIn::LocalInvocationIndex)}}})); + for (Instruction& entryPoint : irContext->module()->entry_points()) { + entryPoint.AddOperand(IdOp(liiVariableId)); + } + } + + // Scratch arrays, one per participating element type, each padded to whole + // virtual subgroups so every guarded read stays in bounds. + struct Scratch { + uint32_t variableId = 0; + uint32_t elementTypeId = 0; + uint32_t pointerTypeId = 0; + }; + std::map scratchByType; + const uint32_t cPadded = uintConst(paddedSlots); + if (cPadded == 0u) return Status::Failure; + const auto scratchFor = [&](uint32_t elementTypeId) -> const Scratch* { + auto it = scratchByType.find(elementTypeId); + if (it != scratchByType.end()) return &it->second; + const uint32_t arrayTypeId = irContext->TakeNextId(); + irContext->AddType(spvtools::MakeUnique( + irContext, spv::Op::OpTypeArray, 0, arrayTypeId, + Instruction::OperandList{IdOp(elementTypeId), IdOp(cPadded)})); + const uint32_t arrayPointerTypeId = + typeMgr->FindPointerToType(arrayTypeId, spv::StorageClass::Workgroup); + const uint32_t elementPointerTypeId = + typeMgr->FindPointerToType(elementTypeId, spv::StorageClass::Workgroup); + if (arrayPointerTypeId == 0u || elementPointerTypeId == 0u) return nullptr; + const uint32_t variableId = irContext->TakeNextId(); + irContext->AddGlobalValue(spvtools::MakeUnique( + irContext, spv::Op::OpVariable, arrayPointerTypeId, variableId, + Instruction::OperandList{ + {SPV_OPERAND_TYPE_STORAGE_CLASS, + {static_cast(spv::StorageClass::Workgroup)}}})); + Scratch scratch{variableId, elementTypeId, elementPointerTypeId}; + return &scratchByType.emplace(elementTypeId, scratch).first->second; + }; + + // The uint-domain twin of a participating type: bools travel as 0/1 words. + struct ScratchTypeInfo { + uint32_t typeId = 0; // element type stored in scratch + uint32_t scalarTypeId = 0; // its component type + bool boolConverted = false; + }; + const auto scratchTypeFor = [&](const ValueTypeInfo& info) -> ScratchTypeInfo { + if (info.kind != ScalarKind::Bool) { + return {info.typeId, info.scalarTypeId, false}; + } + if (info.componentCount == 1u) { + return {uintTypeId, uintTypeId, true}; + } + const analysis::Type* registeredUint = typeMgr->GetType(uintTypeId); + analysis::Vector vectorType(registeredUint, info.componentCount); + const uint32_t vectorTypeId = typeMgr->GetTypeInstruction(&vectorType); + return {vectorTypeId, uintTypeId, true}; + }; + + const auto boolVectorTypeId = [&](uint32_t componentCount) -> uint32_t { + if (componentCount == 1u) return boolTypeId; + const analysis::Type* registeredBool = typeMgr->GetType(boolTypeId); + analysis::Vector vectorType(registeredBool, componentCount); + return typeMgr->GetTypeInstruction(&vectorType); + }; + + // Rewrite the builtin loads. Every rewrite keeps the original result id, so + // downstream uses see the derived value with no further surgery. + const uint32_t numVirtualConstId = uintConst(virtualSubgroups); + if (numVirtualConstId == 0u) return Status::Failure; + + // The x word of one of the five ballot-mask builtins for the calling + // invocation's virtual subgroup, bounded to lanes that exist (words y..w + // are zero at 32 lanes). + const auto emitMaskWord = [&](Emitter& em, spv::BuiltIn builtin) -> uint32_t { + const uint32_t index = + em.Emit(spv::Op::OpLoad, uintTypeId, {IdOp(liiVariableId)}); + const uint32_t lane = + em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(index), IdOp(c31)}); + const uint32_t base = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(index), IdOp(cNotLane)}); + const uint32_t tailBits = + em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(cTotal), IdOp(base)}); + const uint32_t full = em.Emit(spv::Op::OpUGreaterThanEqual, boolTypeId, + {IdOp(tailBits), IdOp(c32)}); + const uint32_t tailShift = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId, + {IdOp(c1), IdOp(tailBits)}); + const uint32_t tailMask = + em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(tailShift), IdOp(c1)}); + const uint32_t exists = em.Emit(spv::Op::OpSelect, uintTypeId, + {IdOp(full), IdOp(cAllOnes), IdOp(tailMask)}); + const auto lowMaskThroughLane = [&](bool inclusive) -> uint32_t { + if (!inclusive) { + const uint32_t bit = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId, + {IdOp(c1), IdOp(lane)}); + return em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(bit), IdOp(c1)}); + } + const uint32_t lanePlusOne = + em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(lane), IdOp(c1)}); + const uint32_t atTop = em.Emit(spv::Op::OpUGreaterThanEqual, boolTypeId, + {IdOp(lanePlusOne), IdOp(c32)}); + const uint32_t shifted = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId, + {IdOp(c1), IdOp(lanePlusOne)}); + const uint32_t below = + em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(shifted), IdOp(c1)}); + return em.Emit(spv::Op::OpSelect, uintTypeId, + {IdOp(atTop), IdOp(cAllOnes), IdOp(below)}); + }; + uint32_t word = 0; + switch (builtin) { + case spv::BuiltIn::SubgroupEqMask: + word = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId, + {IdOp(c1), IdOp(lane)}); + break; + case spv::BuiltIn::SubgroupLtMask: + word = lowMaskThroughLane(false); + break; + case spv::BuiltIn::SubgroupLeMask: + word = lowMaskThroughLane(true); + break; + case spv::BuiltIn::SubgroupGeMask: + word = em.Emit(spv::Op::OpNot, uintTypeId, {IdOp(lowMaskThroughLane(false))}); + break; + case spv::BuiltIn::SubgroupGtMask: + word = em.Emit(spv::Op::OpNot, uintTypeId, {IdOp(lowMaskThroughLane(true))}); + break; + default: + return 0u; + } + return em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(word), IdOp(exists)}); + }; + + for (const BuiltinUse& use : builtinUses) { + for (Instruction* load : use.loads) { + Emitter em(irContext, load); + switch (use.builtin) { + case spv::BuiltIn::SubgroupSize: + load->SetOpcode(spv::Op::OpCopyObject); + load->SetInOperands({IdOp(c32)}); + break; + case spv::BuiltIn::NumSubgroups: + load->SetOpcode(spv::Op::OpCopyObject); + load->SetInOperands({IdOp(numVirtualConstId)}); + break; + case spv::BuiltIn::SubgroupLocalInvocationId: { + const uint32_t index = + em.Emit(spv::Op::OpLoad, uintTypeId, {IdOp(liiVariableId)}); + load->SetOpcode(spv::Op::OpBitwiseAnd); + load->SetInOperands({IdOp(index), IdOp(c31)}); + break; + } + case spv::BuiltIn::SubgroupId: { + const uint32_t index = + em.Emit(spv::Op::OpLoad, uintTypeId, {IdOp(liiVariableId)}); + load->SetOpcode(spv::Op::OpShiftRightLogical); + load->SetInOperands({IdOp(index), IdOp(uintConst(kIdShift))}); + break; + } + default: { + const uint32_t word = emitMaskWord(em, use.builtin); + if (word == 0u) return Status::Failure; + load->SetOpcode(spv::Op::OpCompositeConstruct); + load->SetInOperands( + {IdOp(word), IdOp(c0), IdOp(c0), IdOp(c0)}); + break; + } + } + } + for (const auto& [load, component] : use.componentLoads) { + // gl_Subgroup*Mask.x reached through an access chain: only word 0 + // carries bits at 32 lanes. + Emitter em(irContext, load); + uint32_t value = c0; + if (component == 0u) { + value = emitMaskWord(em, use.builtin); + if (value == 0u) return Status::Failure; + } + load->SetOpcode(spv::Op::OpCopyObject); + load->SetInOperands({IdOp(value)}); + } + } + + // Rewrite the group operations. + for (const GroupOpSite& site : groupOps) { + Instruction* inst = site.inst; + Emitter em(irContext, inst); + + const uint32_t lii = + em.Emit(spv::Op::OpLoad, uintTypeId, {IdOp(liiVariableId)}); + const uint32_t lane = + em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(lii), IdOp(c31)}); + const uint32_t base = + em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(lii), IdOp(cNotLane)}); + + const auto barrier = [&]() { + const uint32_t scopeId = uintConst(static_cast(spv::Scope::Workgroup)); + const uint32_t semanticsId = uintConst( + static_cast(spv::MemorySemanticsMask::AcquireRelease) | + static_cast(spv::MemorySemanticsMask::WorkgroupMemory)); + em.EmitNoResult(spv::Op::OpControlBarrier, + {IdOp(scopeId), IdOp(scopeId), IdOp(semanticsId)}); + }; + + const ScratchTypeInfo scratchType = scratchTypeFor(site.valueType); + const auto toScratch = [&](uint32_t valueId) -> uint32_t { + if (!scratchType.boolConverted) return valueId; + const uint32_t ones = + typedConst(site.valueType, 1u, uintTypeId, scratchType.typeId); + const uint32_t zeros = + typedConst(site.valueType, 0u, uintTypeId, scratchType.typeId); + return em.Emit(spv::Op::OpSelect, scratchType.typeId, + {IdOp(valueId), IdOp(ones), IdOp(zeros)}); + }; + const auto fromScratch = [&](uint32_t valueId) -> uint32_t { + if (!scratchType.boolConverted) return valueId; + const uint32_t zeros = + typedConst(site.valueType, 0u, uintTypeId, scratchType.typeId); + return em.Emit(spv::Op::OpINotEqual, site.valueType.typeId, + {IdOp(valueId), IdOp(zeros)}); + }; + + const Scratch* scratch = nullptr; + const auto storeOwnSlot = [&](uint32_t valueId) -> bool { + scratch = scratchFor(scratchType.typeId); + if (scratch == nullptr) return false; + const uint32_t pointer = + em.Emit(spv::Op::OpAccessChain, scratch->pointerTypeId, + {IdOp(scratch->variableId), IdOp(lii)}); + em.EmitNoResult(spv::Op::OpStore, {IdOp(pointer), IdOp(valueId)}); + barrier(); + return true; + }; + const auto readSlot = [&](uint32_t indexId) -> uint32_t { + const uint32_t pointer = + em.Emit(spv::Op::OpAccessChain, scratch->pointerTypeId, + {IdOp(scratch->variableId), IdOp(indexId)}); + return em.Emit(spv::Op::OpLoad, scratchType.typeId, {IdOp(pointer)}); + }; + // A select whose condition is a scalar bool but whose values may be + // vectors; pre-SPIR-V-1.4 needs the condition splat to a bvec. + const auto guardedSelect = [&](uint32_t condId, uint32_t thenId, + uint32_t elseId) -> uint32_t { + const uint32_t componentCount = site.valueType.componentCount; + uint32_t conditionId = condId; + if (componentCount > 1u) { + const uint32_t conditionTypeId = boolVectorTypeId(componentCount); + Instruction::OperandList splat; + for (uint32_t i = 0; i < componentCount; ++i) splat.push_back(IdOp(condId)); + conditionId = + em.Emit(spv::Op::OpCompositeConstruct, conditionTypeId, std::move(splat)); + } + return em.Emit(spv::Op::OpSelect, scratchType.typeId, + {IdOp(conditionId), IdOp(thenId), IdOp(elseId)}); + }; + const auto extInst2 = [&](uint32_t typeId, uint32_t instNumber, uint32_t a, + uint32_t b) -> uint32_t { + const uint32_t setId = ensureGlslStd450(); + return em.Emit(spv::Op::OpExtInst, typeId, + {IdOp(setId), + {SPV_OPERAND_TYPE_EXTENSION_INSTRUCTION_NUMBER, {instNumber}}, + IdOp(a), IdOp(b)}); + }; + const auto extInst1 = [&](uint32_t typeId, uint32_t instNumber, + uint32_t a) -> uint32_t { + const uint32_t setId = ensureGlslStd450(); + return em.Emit(spv::Op::OpExtInst, typeId, + {IdOp(setId), + {SPV_OPERAND_TYPE_EXTENSION_INSTRUCTION_NUMBER, {instNumber}}, + IdOp(a)}); + }; + + // Combine two scratch-domain values with the site's operation. + const auto combine = [&](uint32_t a, uint32_t b) -> uint32_t { + const uint32_t typeId = scratchType.typeId; + switch (site.opcode) { + case spv::Op::OpGroupNonUniformIAdd: + return em.Emit(spv::Op::OpIAdd, typeId, {IdOp(a), IdOp(b)}); + case spv::Op::OpGroupNonUniformFAdd: + return em.Emit(spv::Op::OpFAdd, typeId, {IdOp(a), IdOp(b)}); + case spv::Op::OpGroupNonUniformIMul: + return em.Emit(spv::Op::OpIMul, typeId, {IdOp(a), IdOp(b)}); + case spv::Op::OpGroupNonUniformFMul: + return em.Emit(spv::Op::OpFMul, typeId, {IdOp(a), IdOp(b)}); + case spv::Op::OpGroupNonUniformSMin: + return extInst2(typeId, kGlslSMin, a, b); + case spv::Op::OpGroupNonUniformUMin: + return extInst2(typeId, kGlslUMin, a, b); + case spv::Op::OpGroupNonUniformFMin: + return extInst2(typeId, kGlslFMin, a, b); + case spv::Op::OpGroupNonUniformSMax: + return extInst2(typeId, kGlslSMax, a, b); + case spv::Op::OpGroupNonUniformUMax: + return extInst2(typeId, kGlslUMax, a, b); + case spv::Op::OpGroupNonUniformFMax: + return extInst2(typeId, kGlslFMax, a, b); + case spv::Op::OpGroupNonUniformBitwiseAnd: + case spv::Op::OpGroupNonUniformLogicalAnd: + return em.Emit(spv::Op::OpBitwiseAnd, typeId, {IdOp(a), IdOp(b)}); + case spv::Op::OpGroupNonUniformBitwiseOr: + case spv::Op::OpGroupNonUniformLogicalOr: + return em.Emit(spv::Op::OpBitwiseOr, typeId, {IdOp(a), IdOp(b)}); + case spv::Op::OpGroupNonUniformBitwiseXor: + case spv::Op::OpGroupNonUniformLogicalXor: + return em.Emit(spv::Op::OpBitwiseXor, typeId, {IdOp(a), IdOp(b)}); + default: + return 0u; + } + }; + // The identity element for the site's operation, in the scratch domain. + const auto identity = [&]() -> uint32_t { + const uint32_t typeId = scratchType.typeId; + const uint32_t scalarId = scratchType.scalarTypeId; + switch (site.opcode) { + case spv::Op::OpGroupNonUniformIAdd: + case spv::Op::OpGroupNonUniformBitwiseOr: + case spv::Op::OpGroupNonUniformBitwiseXor: + case spv::Op::OpGroupNonUniformLogicalOr: + case spv::Op::OpGroupNonUniformLogicalXor: + case spv::Op::OpGroupNonUniformFAdd: + case spv::Op::OpGroupNonUniformUMax: + // 0 and +0.0f share the bit pattern; exclusive-scan lane 0 of an + // FAdd therefore returns +0.0 even for a -0.0 input, matching + // the identity the SPIR-V spec assigns. + return typedConst(site.valueType, 0u, scalarId, typeId); + case spv::Op::OpGroupNonUniformIMul: + return typedConst(site.valueType, 1u, scalarId, typeId); + case spv::Op::OpGroupNonUniformFMul: + return typedConst(site.valueType, 0x3f800000u, scalarId, typeId); + case spv::Op::OpGroupNonUniformSMin: + return typedConst(site.valueType, 0x7fffffffu, scalarId, typeId); + case spv::Op::OpGroupNonUniformUMin: + return typedConst(site.valueType, 0xffffffffu, scalarId, typeId); + case spv::Op::OpGroupNonUniformFMin: + return typedConst(site.valueType, 0x7f800000u, scalarId, typeId); + case spv::Op::OpGroupNonUniformSMax: + return typedConst(site.valueType, 0x80000000u, scalarId, typeId); + case spv::Op::OpGroupNonUniformFMax: + return typedConst(site.valueType, 0xff800000u, scalarId, typeId); + case spv::Op::OpGroupNonUniformBitwiseAnd: + return typedConst(site.valueType, 0xffffffffu, scalarId, typeId); + case spv::Op::OpGroupNonUniformLogicalAnd: + return typedConst(site.valueType, 1u, scalarId, typeId); + default: + return 0u; + } + }; + + // The lanes-that-exist word for this virtual subgroup (tail handling). + const auto existsWord = [&]() -> uint32_t { + const uint32_t tailBits = + em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(cTotal), IdOp(base)}); + const uint32_t full = em.Emit(spv::Op::OpUGreaterThanEqual, boolTypeId, + {IdOp(tailBits), IdOp(c32)}); + const uint32_t shifted = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId, + {IdOp(c1), IdOp(tailBits)}); + const uint32_t mask = + em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(shifted), IdOp(c1)}); + return em.Emit(spv::Op::OpSelect, uintTypeId, + {IdOp(full), IdOp(cAllOnes), IdOp(mask)}); + }; + + const auto finish = [&](uint32_t finalId) { + inst->SetOpcode(spv::Op::OpCopyObject); + inst->SetInOperands({IdOp(finalId)}); + }; + + bool ok = true; + switch (site.opcode) { + case spv::Op::OpGroupNonUniformElect: { + finish(em.Emit(spv::Op::OpIEqual, boolTypeId, {IdOp(lane), IdOp(c0)})); + break; + } + case spv::Op::OpGroupNonUniformAll: + case spv::Op::OpGroupNonUniformAny: { + const bool isAll = site.opcode == spv::Op::OpGroupNonUniformAll; + ok = storeOwnSlot(toScratch(site.valueId)); + if (!ok) break; + const uint32_t neutral = uintConst(isAll ? 1u : 0u); + uint32_t acc = 0; + for (uint32_t k = 0; k < kWidth; ++k) { + const uint32_t slot = + em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(base), IdOp(uintConst(k))}); + const uint32_t exists = em.Emit(spv::Op::OpULessThan, boolTypeId, + {IdOp(slot), IdOp(cTotal)}); + const uint32_t value = readSlot(slot); + const uint32_t guarded = em.Emit( + spv::Op::OpSelect, uintTypeId, + {IdOp(exists), IdOp(value), IdOp(neutral)}); + acc = k == 0 ? guarded + : em.Emit(isAll ? spv::Op::OpBitwiseAnd : spv::Op::OpBitwiseOr, + uintTypeId, {IdOp(acc), IdOp(guarded)}); + } + barrier(); + finish(em.Emit(spv::Op::OpINotEqual, boolTypeId, {IdOp(acc), IdOp(c0)})); + break; + } + case spv::Op::OpGroupNonUniformAllEqual: { + ok = storeOwnSlot(toScratch(site.valueId)); + if (!ok) break; + const uint32_t reference = readSlot(base); + uint32_t acc = cTrue; + for (uint32_t k = 0; k < kWidth; ++k) { + const uint32_t slot = + em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(base), IdOp(uintConst(k))}); + const uint32_t exists = em.Emit(spv::Op::OpULessThan, boolTypeId, + {IdOp(slot), IdOp(cTotal)}); + const uint32_t value = readSlot(slot); + const spv::Op compareOpcode = site.valueType.kind == ScalarKind::Float + ? spv::Op::OpFOrdEqual + : spv::Op::OpIEqual; + uint32_t equal = 0; + if (site.valueType.componentCount == 1u) { + equal = em.Emit(compareOpcode, boolTypeId, + {IdOp(value), IdOp(reference)}); + } else { + const uint32_t comparisonTypeId = + boolVectorTypeId(site.valueType.componentCount); + const uint32_t componentsEqual = em.Emit( + compareOpcode, comparisonTypeId, {IdOp(value), IdOp(reference)}); + equal = em.Emit(spv::Op::OpAll, boolTypeId, {IdOp(componentsEqual)}); + } + const uint32_t guarded = em.Emit( + spv::Op::OpSelect, boolTypeId, + {IdOp(exists), IdOp(equal), IdOp(cTrue)}); + acc = em.Emit(spv::Op::OpLogicalAnd, boolTypeId, + {IdOp(acc), IdOp(guarded)}); + } + barrier(); + finish(acc); + break; + } + case spv::Op::OpGroupNonUniformBroadcast: + case spv::Op::OpGroupNonUniformBroadcastFirst: + case spv::Op::OpGroupNonUniformShuffle: + case spv::Op::OpGroupNonUniformShuffleXor: + case spv::Op::OpGroupNonUniformShuffleUp: + case spv::Op::OpGroupNonUniformShuffleDown: + case spv::Op::OpGroupNonUniformQuadBroadcast: + case spv::Op::OpGroupNonUniformQuadSwap: { + ok = storeOwnSlot(toScratch(site.valueId)); + if (!ok) break; + uint32_t sourceLane = 0; + switch (site.opcode) { + case spv::Op::OpGroupNonUniformBroadcastFirst: + sourceLane = c0; + break; + case spv::Op::OpGroupNonUniformBroadcast: + case spv::Op::OpGroupNonUniformShuffle: + sourceLane = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(site.indexId), IdOp(c31)}); + break; + case spv::Op::OpGroupNonUniformShuffleXor: { + const uint32_t flipped = em.Emit( + spv::Op::OpBitwiseXor, uintTypeId, {IdOp(lane), IdOp(site.indexId)}); + sourceLane = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(flipped), IdOp(c31)}); + break; + } + case spv::Op::OpGroupNonUniformShuffleUp: { + const uint32_t shifted = + em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(lane), IdOp(site.indexId)}); + sourceLane = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(shifted), IdOp(c31)}); + break; + } + case spv::Op::OpGroupNonUniformShuffleDown: { + const uint32_t shifted = + em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(lane), IdOp(site.indexId)}); + sourceLane = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(shifted), IdOp(c31)}); + break; + } + case spv::Op::OpGroupNonUniformQuadBroadcast: { + const uint32_t quadBase = + em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(lane), IdOp(uintConst(~3u & kLaneMask))}); + const uint32_t withinQuad = + em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(site.indexId), IdOp(uintConst(3u))}); + sourceLane = em.Emit(spv::Op::OpIAdd, uintTypeId, + {IdOp(quadBase), IdOp(withinQuad)}); + break; + } + case spv::Op::OpGroupNonUniformQuadSwap: { + const uint32_t partnerMask = uintConst( + site.quadDirection == 0u ? 1u : site.quadDirection == 1u ? 2u : 3u); + sourceLane = em.Emit(spv::Op::OpBitwiseXor, uintTypeId, + {IdOp(lane), IdOp(partnerMask)}); + break; + } + default: + break; + } + const uint32_t sourceSlot = + em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(base), IdOp(sourceLane)}); + const uint32_t value = readSlot(sourceSlot); + barrier(); + finish(fromScratch(value)); + break; + } + case spv::Op::OpGroupNonUniformBallot: { + ok = storeOwnSlot(toScratch(site.valueId)); + if (!ok) break; + uint32_t word = c0; + for (uint32_t k = 0; k < kWidth; ++k) { + const uint32_t slot = + em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(base), IdOp(uintConst(k))}); + const uint32_t exists = em.Emit(spv::Op::OpULessThan, boolTypeId, + {IdOp(slot), IdOp(cTotal)}); + const uint32_t value = readSlot(slot); + const uint32_t guarded = em.Emit( + spv::Op::OpSelect, uintTypeId, {IdOp(exists), IdOp(value), IdOp(c0)}); + const uint32_t bit = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId, + {IdOp(guarded), IdOp(uintConst(k))}); + word = em.Emit(spv::Op::OpBitwiseOr, uintTypeId, {IdOp(word), IdOp(bit)}); + } + barrier(); + inst->SetOpcode(spv::Op::OpCompositeConstruct); + inst->SetInOperands({IdOp(word), IdOp(c0), IdOp(c0), IdOp(c0)}); + break; + } + case spv::Op::OpGroupNonUniformInverseBallot: { + const uint32_t word = em.Emit(spv::Op::OpCompositeExtract, uintTypeId, + {IdOp(site.valueId), + {SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}}); + const uint32_t shifted = em.Emit(spv::Op::OpShiftRightLogical, uintTypeId, + {IdOp(word), IdOp(lane)}); + const uint32_t bit = + em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(shifted), IdOp(c1)}); + finish(em.Emit(spv::Op::OpINotEqual, boolTypeId, {IdOp(bit), IdOp(c0)})); + break; + } + case spv::Op::OpGroupNonUniformBallotBitExtract: { + const uint32_t words[4] = { + em.Emit(spv::Op::OpCompositeExtract, uintTypeId, + {IdOp(site.valueId), {SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}}), + em.Emit(spv::Op::OpCompositeExtract, uintTypeId, + {IdOp(site.valueId), {SPV_OPERAND_TYPE_LITERAL_INTEGER, {1u}}}), + em.Emit(spv::Op::OpCompositeExtract, uintTypeId, + {IdOp(site.valueId), {SPV_OPERAND_TYPE_LITERAL_INTEGER, {2u}}}), + em.Emit(spv::Op::OpCompositeExtract, uintTypeId, + {IdOp(site.valueId), {SPV_OPERAND_TYPE_LITERAL_INTEGER, {3u}}})}; + const uint32_t inFirstPair = em.Emit( + spv::Op::OpULessThan, boolTypeId, {IdOp(site.indexId), IdOp(uintConst(64u))}); + const uint32_t inWord0 = em.Emit( + spv::Op::OpULessThan, boolTypeId, {IdOp(site.indexId), IdOp(c32)}); + const uint32_t inWord2 = em.Emit( + spv::Op::OpULessThan, boolTypeId, {IdOp(site.indexId), IdOp(uintConst(96u))}); + const uint32_t firstPair = em.Emit( + spv::Op::OpSelect, uintTypeId, + {IdOp(inWord0), IdOp(words[0]), IdOp(words[1])}); + const uint32_t secondPair = em.Emit( + spv::Op::OpSelect, uintTypeId, + {IdOp(inWord2), IdOp(words[2]), IdOp(words[3])}); + const uint32_t word = em.Emit( + spv::Op::OpSelect, uintTypeId, + {IdOp(inFirstPair), IdOp(firstPair), IdOp(secondPair)}); + const uint32_t bitIndex = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(site.indexId), IdOp(c31)}); + const uint32_t shifted = em.Emit(spv::Op::OpShiftRightLogical, uintTypeId, + {IdOp(word), IdOp(bitIndex)}); + const uint32_t bit = + em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(shifted), IdOp(c1)}); + finish(em.Emit(spv::Op::OpINotEqual, boolTypeId, {IdOp(bit), IdOp(c0)})); + break; + } + case spv::Op::OpGroupNonUniformBallotBitCount: { + const uint32_t word = em.Emit(spv::Op::OpCompositeExtract, uintTypeId, + {IdOp(site.valueId), + {SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}}); + uint32_t masked = 0; + if (site.groupOperation == spv::GroupOperation::Reduce) { + masked = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(word), IdOp(existsWord())}); + } else { + const bool inclusive = + site.groupOperation == spv::GroupOperation::InclusiveScan; + uint32_t laneMask = 0; + if (inclusive) { + const uint32_t lanePlusOne = + em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(lane), IdOp(c1)}); + const uint32_t atTop = + em.Emit(spv::Op::OpUGreaterThanEqual, boolTypeId, + {IdOp(lanePlusOne), IdOp(c32)}); + const uint32_t shifted = + em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId, + {IdOp(c1), IdOp(lanePlusOne)}); + const uint32_t below = + em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(shifted), IdOp(c1)}); + laneMask = em.Emit(spv::Op::OpSelect, uintTypeId, + {IdOp(atTop), IdOp(cAllOnes), IdOp(below)}); + } else { + const uint32_t bit = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId, + {IdOp(c1), IdOp(lane)}); + laneMask = + em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(bit), IdOp(c1)}); + } + masked = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(word), IdOp(laneMask)}); + } + finish(em.Emit(spv::Op::OpBitCount, uintTypeId, {IdOp(masked)})); + break; + } + case spv::Op::OpGroupNonUniformBallotFindLSB: + case spv::Op::OpGroupNonUniformBallotFindMSB: { + const uint32_t word = em.Emit(spv::Op::OpCompositeExtract, uintTypeId, + {IdOp(site.valueId), + {SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}}); + const uint32_t masked = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(word), IdOp(existsWord())}); + const uint32_t instNumber = + site.opcode == spv::Op::OpGroupNonUniformBallotFindLSB ? kGlslFindILsb + : kGlslFindUMsb; + finish(extInst1(uintTypeId, instNumber, masked)); + break; + } + default: { + // The arithmetic family. Serial, ascending-lane combines: the same + // left-associated order a host-side reference computes, and exact + // for every integer-valued input regardless of native topology. + ok = storeOwnSlot(toScratch(site.valueId)); + if (!ok) break; + uint32_t result = 0; + switch (site.groupOperation) { + case spv::GroupOperation::Reduce: { + uint32_t acc = readSlot(base); + for (uint32_t k = 1; k < kWidth; ++k) { + const uint32_t slot = em.Emit(spv::Op::OpIAdd, uintTypeId, + {IdOp(base), IdOp(uintConst(k))}); + const uint32_t exists = em.Emit(spv::Op::OpULessThan, boolTypeId, + {IdOp(slot), IdOp(cTotal)}); + const uint32_t value = readSlot(slot); + const uint32_t combined = combine(acc, value); + if (combined == 0u) { ok = false; break; } + acc = guardedSelect(exists, combined, acc); + } + result = acc; + break; + } + case spv::GroupOperation::InclusiveScan: { + // Slots at or below the own lane always exist, so the guard is + // purely the lane comparison. + uint32_t acc = readSlot(base); + for (uint32_t k = 1; k < kWidth; ++k) { + const uint32_t included = + em.Emit(spv::Op::OpULessThanEqual, boolTypeId, + {IdOp(uintConst(k)), IdOp(lane)}); + const uint32_t slot = em.Emit(spv::Op::OpIAdd, uintTypeId, + {IdOp(base), IdOp(uintConst(k))}); + const uint32_t value = readSlot(slot); + const uint32_t combined = combine(acc, value); + if (combined == 0u) { ok = false; break; } + acc = guardedSelect(included, combined, acc); + } + result = acc; + break; + } + case spv::GroupOperation::ExclusiveScan: { + uint32_t acc = identity(); + if (acc == 0u) { ok = false; break; } + for (uint32_t k = 0; k < kWidth; ++k) { + const uint32_t included = em.Emit( + spv::Op::OpULessThan, boolTypeId, {IdOp(uintConst(k)), IdOp(lane)}); + const uint32_t slot = em.Emit(spv::Op::OpIAdd, uintTypeId, + {IdOp(base), IdOp(uintConst(k))}); + const uint32_t value = readSlot(slot); + const uint32_t combined = combine(acc, value); + if (combined == 0u) { ok = false; break; } + acc = guardedSelect(included, combined, acc); + } + result = acc; + break; + } + case spv::GroupOperation::ClusteredReduce: { + const uint32_t cluster = + site.clusterSize >= kWidth ? kWidth : site.clusterSize; + const uint32_t clusterBase = em.Emit( + spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(lane), IdOp(uintConst(~(cluster - 1u) & kLaneMask))}); + const uint32_t firstSlot = em.Emit(spv::Op::OpIAdd, uintTypeId, + {IdOp(base), IdOp(clusterBase)}); + uint32_t acc = readSlot(firstSlot); + for (uint32_t k = 1; k < cluster; ++k) { + const uint32_t slot = em.Emit(spv::Op::OpIAdd, uintTypeId, + {IdOp(firstSlot), IdOp(uintConst(k))}); + const uint32_t exists = em.Emit(spv::Op::OpULessThan, boolTypeId, + {IdOp(slot), IdOp(cTotal)}); + const uint32_t value = readSlot(slot); + const uint32_t combined = combine(acc, value); + if (combined == 0u) { ok = false; break; } + acc = guardedSelect(exists, combined, acc); + } + result = acc; + break; + } + default: + ok = false; + break; + } + if (!ok) break; + barrier(); + finish(fromScratch(result)); + break; + } + } + if (!ok) { + // Phase A vetted every site, so a build failure here means an + // internal inconsistency; the module is already partially rewritten + // and must not be used. + return Status::Failure; + } + } + + // Widen subgroup-scoped barriers to the workgroup scope the emulation + // synchronizes at anyway; strictly stronger, so always safe. + const uint32_t workgroupScopeId = + uintConst(static_cast(spv::Scope::Workgroup)); + if (workgroupScopeId == 0u) return Status::Failure; + for (const BarrierSite& site : barriers) { + if (site.inst->opcode() == spv::Op::OpControlBarrier) { + if (site.executionScopeIsSubgroup) { + site.inst->SetInOperand(0, {workgroupScopeId}); + } + if (site.memoryScopeIsSubgroup) { + site.inst->SetInOperand(1, {workgroupScopeId}); + } + } else if (site.memoryScopeIsSubgroup) { + site.inst->SetInOperand(0, {workgroupScopeId}); + } + } + + // ---------------------------------------------------------------- Phase C + // The replaced builtin variables are dead now; remove them together with + // their decorations and interface entries, then drop the GroupNonUniform* + // capabilities nothing references any more. + irContext->InvalidateAnalysesExceptFor(IRContext::kAnalysisNone); + for (const BuiltinUse& use : builtinUses) { + const uint32_t variableId = use.variable->result_id(); + for (Instruction& entryPoint : irContext->module()->entry_points()) { + for (uint32_t operandIndex = entryPoint.NumInOperands(); operandIndex > 3;) { + --operandIndex; + if (entryPoint.GetInOperand(operandIndex).type == SPV_OPERAND_TYPE_ID && + entryPoint.GetSingleWordInOperand(operandIndex) == variableId) { + entryPoint.RemoveInOperand(operandIndex); + } + } + } + for (Instruction* chain : use.accessChains) { + irContext->KillNamesAndDecorates(chain); + irContext->KillInst(chain); + } + irContext->KillNamesAndDecorates(use.variable); + irContext->KillInst(use.variable); + } + + static constexpr std::array kSubgroupCapabilities = { + spv::Capability::GroupNonUniform, + spv::Capability::GroupNonUniformVote, + spv::Capability::GroupNonUniformArithmetic, + spv::Capability::GroupNonUniformBallot, + spv::Capability::GroupNonUniformShuffle, + spv::Capability::GroupNonUniformShuffleRelative, + spv::Capability::GroupNonUniformClustered, + spv::Capability::GroupNonUniformQuad, + spv::Capability::GroupNonUniformPartitionedNV}; + std::vector deadCapabilities; + for (Instruction& capability : irContext->module()->capabilities()) { + const auto value = + static_cast(capability.GetSingleWordInOperand(0)); + for (const spv::Capability candidate : kSubgroupCapabilities) { + if (value == candidate) { + deadCapabilities.push_back(&capability); + break; + } + } + } + for (Instruction* capability : deadCapabilities) { + irContext->KillInst(capability); + } + + irContext->InvalidateAnalysesExceptFor(IRContext::kAnalysisNone); + return Status::SuccessWithChange; + } + + spvtools::Optimizer::PassToken EmulateSubgroupsPass::CreateEmulateSubgroupsPass( + const Uint32 maxWorkgroupScratchBytes) { + return spvtools::Optimizer::PassToken( + MakeUnique(maxWorkgroupScratchBytes)); + } + } // namespace ShaderTranspiler + } // namespace MG_Util +} // namespace MobileGL diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.h b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.h new file mode 100644 index 00000000..d2868bec --- /dev/null +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.h @@ -0,0 +1,73 @@ +// MobileGL - MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.h +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#pragma once + +#include "source/opt/pass.h" +#include "spirv-tools/optimizer.hpp" + +#include + +namespace MobileGL { + namespace MG_Util { + namespace ShaderTranspiler { + // Lowers every GL_KHR_shader_subgroup construct in a compute module onto a + // 32-lane VIRTUAL subgroup implemented with workgroup-shared memory. Virtual + // subgroups partition the workgroup by gl_LocalInvocationIndex: + // lane = index & 31, id = index >> 5, count = ceil(invocations / 32). + // + // This is a LAST-RESORT path, never a substitute for real subgroups: it only + // runs when MOBILEGL_MAGMA_EMULATE_SUBGROUP=1 is set explicitly and the device + // has no native subgroup support at all (SubgroupSupportPolicy.h). A device + // with native subgroup operations - however narrow - uses them natively, with + // FixIterationRPSubgroupScratchPass patching the known pack bug instead. + // + // Lowered constructs: + // - the builtins gl_SubgroupSize / gl_SubgroupInvocationID / gl_SubgroupID / + // gl_NumSubgroups and the five gl_Subgroup*Mask ballot builtins; + // - OpGroupNonUniform{Elect,All,Any,AllEqual,Broadcast,BroadcastFirst, + // Ballot,InverseBallot,BallotBitExtract,BallotBitCount,BallotFind{L,M}SB, + // Shuffle,ShuffleXor,ShuffleUp,ShuffleDown, + // , + // QuadBroadcast,QuadSwap}; + // - subgroupBarrier()/subgroupMemoryBarrier*() (their Subgroup scopes widen + // to Workgroup, which is strictly stronger). + // The output uses no GroupNonUniform* instruction or capability at all, which + // is what lets it run on devices with no subgroup feature bits. + // + // Semantic contract, narrower than native subgroups in exactly one way: every + // emulated exchange synchronizes through OpControlBarrier, so subgroup + // operations must sit in WORKGROUP-uniform control flow (the shape every + // Iris-style pack reduction has). GLSL already imposes this for barrier(); + // a subgroup op in divergent flow - legal on native subgroups - is undefined + // here. + // + // Fails (Status::Failure, leaving the input module unchanged) on anything it + // cannot lower faithfully: extended subgroup ops (partitioned-NV, rotate, + // quad-all/any), non-32-bit participating types, spec-constant workgroup + // sizes, a subgroup builtin reached by anything but a direct OpLoad, or a + // module whose lowering would add more workgroup scratch than + // maxWorkgroupScratchBytes (pass the device's maxComputeSharedMemorySize; + // 0 falls back to the 16384-byte Vulkan minimum). + class EmulateSubgroupsPass : public spvtools::opt::Pass { + public: + explicit EmulateSubgroupsPass(Uint32 maxWorkgroupScratchBytes) + : m_maxWorkgroupScratchBytes(maxWorkgroupScratchBytes) {} + + const char* name() const override { return "emulate-subgroups"; } + Status Process() override; + + static spvtools::Optimizer::PassToken CreateEmulateSubgroupsPass( + Uint32 maxWorkgroupScratchBytes); + + private: + Uint32 m_maxWorkgroupScratchBytes; + }; + } // namespace ShaderTranspiler + } // namespace MG_Util +} // namespace MobileGL diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.cpp b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.cpp new file mode 100644 index 00000000..3ca95682 --- /dev/null +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.cpp @@ -0,0 +1,352 @@ +// MobileGL - MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.cpp +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#include "FixIterationRPSubgroupScratchPass.h" + +#include "spirv.hpp" +#include "source/opt/def_use_manager.h" +#include "source/opt/instruction.h" +#include "source/opt/ir_context.h" +#include "source/opt/module.h" +#include "source/util/make_unique.h" + +#include +#include +#include + +namespace MobileGL { + namespace MG_Util { + namespace ShaderTranspiler { + namespace { + using spvtools::opt::Instruction; + using spvtools::opt::IRContext; + + // iterationRP's reduction fingerprint, spelled out. + constexpr uint32_t kIterationRPLocalSizeX = 32u; + constexpr uint32_t kIterationRPLocalSizeY = 16u; + constexpr uint32_t kIterationRPLocalSizeZ = 1u; + constexpr uint32_t kIterationRPInvocations = + kIterationRPLocalSizeX * kIterationRPLocalSizeY * kIterationRPLocalSizeZ; + constexpr uint32_t kIterationRPScratchLength = 32u; + + Instruction* FindBuiltinDefinition(IRContext* context, spv::BuiltIn builtin) { + auto* defUseMgr = context->get_def_use_mgr(); + for (auto& annotation : context->annotations()) { + if (annotation.opcode() != spv::Op::OpDecorate || annotation.NumInOperands() < 3) { + continue; + } + if (static_cast(annotation.GetSingleWordInOperand(1)) != + spv::Decoration::BuiltIn) { + continue; + } + if (static_cast(annotation.GetSingleWordInOperand(2)) != builtin) { + continue; + } + return defUseMgr->GetDef(annotation.GetSingleWordInOperand(0)); + } + return nullptr; + } + + // Walks an access-chain pointer expression back to the variable it is + // rooted at; returns nullptr for anything that is not a plain chain. + const Instruction* RootVariable(IRContext* context, uint32_t pointerId) { + auto* defUseMgr = context->get_def_use_mgr(); + const Instruction* def = defUseMgr->GetDef(pointerId); + while (def != nullptr) { + switch (def->opcode()) { + case spv::Op::OpVariable: + return def; + case spv::Op::OpAccessChain: + case spv::Op::OpInBoundsAccessChain: + case spv::Op::OpCopyObject: + def = defUseMgr->GetDef(def->GetSingleWordInOperand(0)); + break; + default: + return nullptr; + } + } + return nullptr; + } + + // vec2 of 32-bit float - the type of iterationRP's luminance/exposure + // accumulator and of its prefixSumCache entries. + bool IsVec2Float32(IRContext* context, uint32_t typeId) { + const Instruction* type = context->get_def_use_mgr()->GetDef(typeId); + if (type == nullptr || type->opcode() != spv::Op::OpTypeVector || + type->GetSingleWordInOperand(1) != 2u) { + return false; + } + const Instruction* component = + context->get_def_use_mgr()->GetDef(type->GetSingleWordInOperand(0)); + return component != nullptr && component->opcode() == spv::Op::OpTypeFloat && + component->GetSingleWordInOperand(0) == 32u; + } + } // namespace + + spvtools::opt::Pass::Status FixIterationRPSubgroupScratchPass::Process() { + auto* irContext = context(); + auto* defUseMgr = irContext->get_def_use_mgr(); + + // A device whose native width already satisfies the pack's assumption + // (>= 16 lanes -> at most 32 subgroups) needs no patch at all. + if (m_nativeSubgroupSize == 0u || m_nativeSubgroupSize >= 16u) { + return Status::SuccessWithoutChange; + } + const uint32_t requiredLength = + (kIterationRPInvocations + m_nativeSubgroupSize - 1u) / m_nativeSubgroupSize; + + for (const Instruction& entryPoint : irContext->module()->entry_points()) { + if (static_cast(entryPoint.GetSingleWordInOperand(0)) != + spv::ExecutionModel::GLCompute) { + return Status::SuccessWithoutChange; + } + } + + // Fingerprint 1: the pack's exposure-pass workgroup shape, 32x16x1. + const auto resolveUintConstant = [&](uint32_t id, uint32_t* value) { + const Instruction* def = defUseMgr->GetDef(id); + if (def == nullptr || def->opcode() != spv::Op::OpConstant) return false; + *value = def->GetSingleWordInOperand(0); + return true; + }; + uint32_t localSize[3] = {0, 0, 0}; + bool haveLocalSize = false; + if (Instruction* workgroupSize = + FindBuiltinDefinition(irContext, spv::BuiltIn::WorkgroupSize)) { + if (workgroupSize->opcode() == spv::Op::OpConstantComposite && + workgroupSize->NumInOperands() == 3) { + haveLocalSize = + resolveUintConstant(workgroupSize->GetSingleWordInOperand(0), &localSize[0]) && + resolveUintConstant(workgroupSize->GetSingleWordInOperand(1), &localSize[1]) && + resolveUintConstant(workgroupSize->GetSingleWordInOperand(2), &localSize[2]); + } + } + if (!haveLocalSize) { + for (const Instruction& mode : irContext->module()->execution_modes()) { + if (mode.opcode() == spv::Op::OpExecutionMode && + static_cast(mode.GetSingleWordInOperand(1)) == + spv::ExecutionMode::LocalSize) { + localSize[0] = mode.GetSingleWordInOperand(2); + localSize[1] = mode.GetSingleWordInOperand(3); + localSize[2] = mode.GetSingleWordInOperand(4); + haveLocalSize = true; + break; + } + } + } + if (!haveLocalSize || localSize[0] != kIterationRPLocalSizeX || + localSize[1] != kIterationRPLocalSizeY || localSize[2] != kIterationRPLocalSizeZ) { + return Status::SuccessWithoutChange; + } + + // Fingerprint 2: the reduction's subgroupInclusiveAdd on a vec2. + bool sawVec2InclusiveAdd = false; + for (auto& function : *irContext->module()) { + for (auto& block : function) { + for (auto& inst : block) { + if (inst.opcode() == spv::Op::OpGroupNonUniformFAdd && + static_cast(inst.GetSingleWordInOperand(1)) == + spv::GroupOperation::InclusiveScan && + IsVec2Float32(irContext, inst.type_id())) { + sawVec2InclusiveAdd = true; + } + } + } + } + if (!sawVec2InclusiveAdd) { + return Status::SuccessWithoutChange; + } + + // gl_SubgroupID, whose value range the pack's scratch size bakes in. + const Instruction* subgroupIdVariable = + FindBuiltinDefinition(irContext, spv::BuiltIn::SubgroupId); + if (subgroupIdVariable == nullptr || + subgroupIdVariable->opcode() != spv::Op::OpVariable) { + return Status::SuccessWithoutChange; + } + const uint32_t subgroupIdVariableId = subgroupIdVariable->result_id(); + + // Conservative taint walk over values, and through Function/Private + // temporaries by variable (glslang routinely spills builtin loads into + // locals before they reach an index expression). Over-tainting is safe: + // the candidate filter below still demands the exact vec2[32] shape. + std::unordered_map valueTainted; // result id -> tainted + std::unordered_map variableTainted; // variable id -> tainted + bool changedTaint = true; + while (changedTaint) { + changedTaint = false; + for (auto& function : *irContext->module()) { + for (auto& block : function) { + for (auto& inst : block) { + const spv::Op opcode = inst.opcode(); + if (opcode == spv::Op::OpStore) { + if (!valueTainted.count(inst.GetSingleWordInOperand(1))) continue; + const Instruction* root = + RootVariable(irContext, inst.GetSingleWordInOperand(0)); + if (root == nullptr) continue; + if (!variableTainted.count(root->result_id())) { + variableTainted[root->result_id()] = true; + changedTaint = true; + } + continue; + } + if (inst.result_id() == 0 || valueTainted.count(inst.result_id())) { + continue; + } + bool tainted = false; + if (opcode == spv::Op::OpLoad) { + const uint32_t pointerId = inst.GetSingleWordInOperand(0); + if (pointerId == subgroupIdVariableId) tainted = true; + const Instruction* root = RootVariable(irContext, pointerId); + if (root != nullptr && variableTainted.count(root->result_id())) { + tainted = true; + } + } else { + inst.ForEachInId([&](const uint32_t* operandId) { + if (valueTainted.count(*operandId)) tainted = true; + }); + } + if (tainted) { + valueTainted[inst.result_id()] = true; + changedTaint = true; + } + } + } + } + } + if (valueTainted.empty()) { + return Status::SuccessWithoutChange; + } + + // Fingerprint 3: workgroup-shared vec2[32] arrays whose access-chain + // index depends on gl_SubgroupID - the under-declared prefixSumCache. + std::map candidates; + for (auto& function : *irContext->module()) { + for (auto& block : function) { + for (auto& inst : block) { + if (inst.opcode() != spv::Op::OpAccessChain && + inst.opcode() != spv::Op::OpInBoundsAccessChain) { + continue; + } + if (inst.NumInOperands() < 2) continue; + if (!valueTainted.count(inst.GetSingleWordInOperand(1))) continue; + Instruction* baseVariable = + defUseMgr->GetDef(inst.GetSingleWordInOperand(0)); + if (baseVariable == nullptr || + baseVariable->opcode() != spv::Op::OpVariable || + static_cast( + baseVariable->GetSingleWordInOperand(0)) != + spv::StorageClass::Workgroup) { + continue; + } + candidates.emplace(baseVariable->result_id(), baseVariable); + } + } + } + if (candidates.empty()) { + return Status::SuccessWithoutChange; + } + + bool changedModule = false; + for (auto& entry : candidates) { + Instruction* variable = entry.second; + + // The variable must be reached exclusively through access chains (plus + // debug/decoration instructions): a whole-array load, store, or copy + // would change type with the array and is left alone. + bool onlyAccessChains = true; + const uint32_t variableId = variable->result_id(); + defUseMgr->ForEachUser(variable, [&](Instruction* user) { + switch (user->opcode()) { + case spv::Op::OpAccessChain: + case spv::Op::OpInBoundsAccessChain: + if (user->GetSingleWordInOperand(0) != variableId) { + onlyAccessChains = false; + } + return; + case spv::Op::OpName: + case spv::Op::OpDecorate: + return; + default: + onlyAccessChains = false; + return; + } + }); + if (!onlyAccessChains) continue; + if (variable->NumInOperands() > 1) continue; // initializer: leave alone + + const Instruction* pointerType = defUseMgr->GetDef(variable->type_id()); + if (pointerType == nullptr || pointerType->opcode() != spv::Op::OpTypePointer) { + continue; + } + const Instruction* arrayType = + defUseMgr->GetDef(pointerType->GetSingleWordInOperand(1)); + if (arrayType == nullptr || arrayType->opcode() != spv::Op::OpTypeArray) { + continue; + } + const uint32_t elementTypeId = arrayType->GetSingleWordInOperand(0); + if (!IsVec2Float32(irContext, elementTypeId)) continue; + const Instruction* lengthConstant = + defUseMgr->GetDef(arrayType->GetSingleWordInOperand(1)); + uint32_t currentLength = 0; + if (lengthConstant == nullptr || + lengthConstant->opcode() != spv::Op::OpConstant || + !((currentLength = lengthConstant->GetSingleWordInOperand(0), + currentLength == kIterationRPScratchLength))) { + continue; + } + if (currentLength >= requiredLength) continue; + + // Build the grown array type. All three new instructions are inserted + // immediately BEFORE the variable so definition-before-use holds in the + // module's global section (manager-created instructions append to its + // end, after the variable). The new length constant reuses the old + // one's integer type, whatever signedness glslang gave it (a duplicate + // scalar constant is legal SPIR-V); the fresh array type makes the + // pointer type unique by construction, so neither collides with an + // existing declaration. + const uint32_t lengthTypeId = lengthConstant->type_id(); + const uint32_t newLengthId = irContext->TakeNextId(); + variable->InsertBefore(spvtools::MakeUnique( + irContext, spv::Op::OpConstant, lengthTypeId, newLengthId, + Instruction::OperandList{{SPV_OPERAND_TYPE_TYPED_LITERAL_NUMBER, + {requiredLength}}})); + const uint32_t newArrayTypeId = irContext->TakeNextId(); + variable->InsertBefore(spvtools::MakeUnique( + irContext, spv::Op::OpTypeArray, 0, newArrayTypeId, + Instruction::OperandList{ + {SPV_OPERAND_TYPE_ID, {elementTypeId}}, + {SPV_OPERAND_TYPE_ID, {newLengthId}}})); + const uint32_t newPointerTypeId = irContext->TakeNextId(); + variable->InsertBefore(spvtools::MakeUnique( + irContext, spv::Op::OpTypePointer, 0, newPointerTypeId, + Instruction::OperandList{ + {SPV_OPERAND_TYPE_STORAGE_CLASS, + {static_cast(spv::StorageClass::Workgroup)}}, + {SPV_OPERAND_TYPE_ID, {newArrayTypeId}}})); + + variable->SetResultType(newPointerTypeId); + changedModule = true; + } + + if (!changedModule) { + return Status::SuccessWithoutChange; + } + irContext->InvalidateAnalysesExceptFor(IRContext::kAnalysisNone); + return Status::SuccessWithChange; + } + + spvtools::Optimizer::PassToken + FixIterationRPSubgroupScratchPass::CreateFixIterationRPSubgroupScratchPass( + const Uint32 nativeSubgroupSize) { + return spvtools::Optimizer::PassToken( + MakeUnique(nativeSubgroupSize)); + } + } // namespace ShaderTranspiler + } // namespace MG_Util +} // namespace MobileGL diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.h b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.h new file mode 100644 index 00000000..db3b7d6d --- /dev/null +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.h @@ -0,0 +1,64 @@ +// MobileGL - MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.h +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#pragma once + +#include "source/opt/pass.h" +#include "spirv-tools/optimizer.hpp" + +#include + +namespace MobileGL { + namespace MG_Util { + namespace ShaderTranspiler { + // Patches ONE known shader-pack defect: iterationRP's auto-exposure reduction + // declares `shared vec2 prefixSumCache[32]` for its 512-invocation workgroup + // and stores per-subgroup subtotals through prefixSumCache[gl_SubgroupID]. + // The pack hard-sized that scratch for the >=16-lane subgroups desktop GL + // drivers ship; on a narrower Vulkan device (lavapipe's 8 lanes -> 64 + // subgroups) every subgroup past entry 31 indexes shared memory out of + // bounds - on a CPU rasterizer that is literal heap corruption. The + // reduction ALGORITHM is width-agnostic (its combine loop is sized by + // gl_NumSubgroups), so the faithful repair is to grow the one under-declared + // array to ceil(512 / native width) and change nothing else. This is the + // pack author's bug, not MobileGL's; the patch is therefore deliberately + // NOT a general mechanism - it only rewrites modules that positively match + // iterationRP's reduction fingerprint: + // - GLCompute entry point with local size exactly 32x16x1; + // - a subgroupInclusiveAdd on a vec2 (OpGroupNonUniformFAdd InclusiveScan, + // the pack's luminance/exposure accumulator signature); + // - a workgroup-shared array of exactly vec2[32] whose access-chain index + // is data-dependent on gl_SubgroupID. + // Matching at the SPIR-V level keeps the recognition robust against + // whitespace/identifier-level drift that made the old source-text template + // rewrite (removed in 7769156) so brittle, while still refusing to touch + // anything that is not this pack's reduction. On devices whose native width + // already satisfies the pack's assumption (>= 16 lanes: desktop GL, Adreno), + // the grown length equals or undershoots the declared 32 and every module + // passes through byte-identical. + // + // The pass never fails a module: anything it cannot prove is this exact + // pattern - or cannot grow safely (a whole-array use, a spec-constant + // length, an initializer) - is left exactly as it was. + class FixIterationRPSubgroupScratchPass : public spvtools::opt::Pass { + public: + explicit FixIterationRPSubgroupScratchPass(Uint32 nativeSubgroupSize) + : m_nativeSubgroupSize(nativeSubgroupSize) {} + + const char* name() const override { return "fix-iterationrp-subgroup-scratch"; } + Status Process() override; + + static spvtools::Optimizer::PassToken CreateFixIterationRPSubgroupScratchPass( + Uint32 nativeSubgroupSize); + + private: + Uint32 m_nativeSubgroupSize; + }; + } // namespace ShaderTranspiler + } // namespace MG_Util +} // namespace MobileGL diff --git a/android-plugin/app/src/trace/cpp/trace_replay_core.cpp b/android-plugin/app/src/trace/cpp/trace_replay_core.cpp index 7a700921..151fa98c 100644 --- a/android-plugin/app/src/trace/cpp/trace_replay_core.cpp +++ b/android-plugin/app/src/trace/cpp/trace_replay_core.cpp @@ -164,11 +164,6 @@ bool LoadMobileGL(const Request& request, std::string& error) { } else { unsetenv("MOBILEGL_COHERENT_AS_FLUSH"); } - if (request.numSubgroupsQuirk) { - setenv("MOBILEGL_NUM_SUBGROUPS_QUIRK", "1", 1); - } else { - unsetenv("MOBILEGL_NUM_SUBGROUPS_QUIRK"); - } if (request.fboAttachmentDumps.empty()) { unsetenv("MOBILEGL_TRACE_DUMP_FBO_ATTACHMENTS"); } else { @@ -823,7 +818,6 @@ bool WriteResultJson(const Request& request, const Result& result) { << (request.avoidAngleLlvmpipeSamplerMipmapMinFilter ? "true" : "false") << ",\n"; file << " \"avoidAngleLlvmpipeExplicitLodBias\": " << (request.avoidAngleLlvmpipeExplicitLodBias ? "true" : "false") << ",\n"; - file << " \"numSubgroupsQuirk\": " << (request.numSubgroupsQuirk ? "true" : "false") << ",\n"; file << " \"holdMs\": " << request.holdMs << ",\n"; file << " \"mismatchPixels\": " << result.mismatchPixels << "\n"; file << "}\n"; diff --git a/android-plugin/app/src/trace/cpp/trace_replay_core.hpp b/android-plugin/app/src/trace/cpp/trace_replay_core.hpp index aab23e71..cc550378 100644 --- a/android-plugin/app/src/trace/cpp/trace_replay_core.hpp +++ b/android-plugin/app/src/trace/cpp/trace_replay_core.hpp @@ -44,7 +44,6 @@ struct Request { bool avoidAngleLlvmpipeSamplerMipmapMinFilter = false; bool avoidAngleLlvmpipeExplicitLodBias = false; bool coherentAsFlush = false; - bool numSubgroupsQuirk = false; int holdMs = 0; }; diff --git a/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp b/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp index 590d15df..8c6e1a6e 100644 --- a/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp +++ b/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp @@ -122,7 +122,6 @@ Java_top_mobilegl_plugin_trace_TraceReplayActivity_nativeRunTraceReplay(JNIEnv* jboolean avoidAngleLlvmpipeSamplerMipmapMinFilter, jboolean avoidAngleLlvmpipeExplicitLodBias, jboolean coherentAsFlush, - jboolean numSubgroupsQuirk, jstring texture2dDumps) { mobilegl_trace::Request request; request.tracePath = ToString(env, tracePath); @@ -151,7 +150,6 @@ Java_top_mobilegl_plugin_trace_TraceReplayActivity_nativeRunTraceReplay(JNIEnv* avoidAngleLlvmpipeSamplerMipmapMinFilter == JNI_TRUE; request.avoidAngleLlvmpipeExplicitLodBias = avoidAngleLlvmpipeExplicitLodBias == JNI_TRUE; request.coherentAsFlush = coherentAsFlush == JNI_TRUE; - request.numSubgroupsQuirk = numSubgroupsQuirk == JNI_TRUE; ScopedTraceReplayState replayState; mobilegl_trace_set_requested_size(request.width, request.height); diff --git a/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java b/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java index b2bacd3c..e3c91aa1 100644 --- a/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java +++ b/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java @@ -116,7 +116,6 @@ public final class TraceReplayActivity extends Activity { request.avoidAngleLlvmpipeSamplerMipmapMinFilter, request.avoidAngleLlvmpipeExplicitLodBias, request.coherentAsFlush, - request.numSubgroupsQuirk, request.texture2dDumps ); Log.i(TAG, result.toString()); @@ -150,7 +149,6 @@ public final class TraceReplayActivity extends Activity { boolean avoidAngleLlvmpipeSamplerMipmapMinFilter, boolean avoidAngleLlvmpipeExplicitLodBias, boolean coherentAsFlush, - boolean numSubgroupsQuirk, String texture2dDumps ); @@ -176,7 +174,6 @@ public final class TraceReplayActivity extends Activity { final boolean avoidAngleLlvmpipeSamplerMipmapMinFilter; final boolean avoidAngleLlvmpipeExplicitLodBias; final boolean coherentAsFlush; - final boolean numSubgroupsQuirk; final String texture2dDumps; private TraceReplayRequest( @@ -201,8 +198,7 @@ public final class TraceReplayActivity extends Activity { boolean avoidAngleLlvmpipeSamplerMipmapMinFilter, boolean avoidAngleLlvmpipeExplicitLodBias, boolean coherentAsFlush, - boolean numSubgroupsQuirk, - String texture2dDumps + String texture2dDumps ) { this.tracePath = tracePath; this.goldenPath = goldenPath; @@ -225,7 +221,6 @@ public final class TraceReplayActivity extends Activity { this.avoidAngleLlvmpipeSamplerMipmapMinFilter = avoidAngleLlvmpipeSamplerMipmapMinFilter; this.avoidAngleLlvmpipeExplicitLodBias = avoidAngleLlvmpipeExplicitLodBias; this.coherentAsFlush = coherentAsFlush; - this.numSubgroupsQuirk = numSubgroupsQuirk; this.texture2dDumps = texture2dDumps; } @@ -254,7 +249,6 @@ public final class TraceReplayActivity extends Activity { intent.getBooleanExtra("avoid_angle_llvmpipe_sampler_mipmap_min_filter", false), intent.getBooleanExtra("avoid_angle_llvmpipe_explicit_lod_bias", false), intent.getBooleanExtra("coherent_as_flush", false), - intent.getBooleanExtra("num_subgroups_quirk", false), readString(intent, "texture_2d_dumps", "") ); } diff --git a/android-plugin/trace-replay-ci.sh b/android-plugin/trace-replay-ci.sh index 8f461a70..f80a79e2 100644 --- a/android-plugin/trace-replay-ci.sh +++ b/android-plugin/trace-replay-ci.sh @@ -31,7 +31,6 @@ Usage: [--avoid-angle-llvmpipe-sampler-mipmap-min-filter] \ [--avoid-angle-llvmpipe-explicit-lod-bias] \ [--coherent-as-flush] \ - [--num-subgroups-quirk] \ [--dump-texture-2d CALL,TEXTURE,LEVEL,DIR] \ --timeout-seconds N @@ -48,8 +47,6 @@ sample with an explicit LOD that ANGLE llvmpipe cannot take a LOD bias on (MOBILEGL_AVOID_EXPLICIT_LOD_BIAS=1). Pass --coherent-as-flush for traces whose engine writes persistent GL_MAP_FLUSH_EXPLICIT_BIT maps it never flushes (MOBILEGL_COHERENT_AS_FLUSH=1). -Pass --num-subgroups-quirk to derive compute gl_NumSubgroups instead of reading -the Vulkan builtin (MOBILEGL_NUM_SUBGROUPS_QUIRK=1). EOF } @@ -108,7 +105,6 @@ use_pbuffer=0 avoid_angle_llvmpipe_sampler_mipmap_min_filter=0 avoid_angle_llvmpipe_explicit_lod_bias=0 coherent_as_flush=0 -num_subgroups_quirk=0 texture_2d_dumps="" timeout_seconds="" @@ -150,7 +146,6 @@ while [ "$#" -gt 0 ]; do shift 1 ;; --coherent-as-flush) coherent_as_flush=1; shift 1 ;; - --num-subgroups-quirk) num_subgroups_quirk=1; shift 1 ;; --dump-texture-2d) texture_2d_dumps="$(next_arg "$@")"; shift 2 ;; --timeout-seconds) timeout_seconds="$(next_arg "$@")"; shift 2 ;; -h|--help) usage; exit 0 ;; @@ -368,9 +363,6 @@ run_retrace() { if [ "${coherent_as_flush}" -eq 1 ]; then set -- "$@" --ez coherent_as_flush true fi - if [ "${num_subgroups_quirk}" -eq 1 ]; then - set -- "$@" --ez num_subgroups_quirk true - fi if [ -n "${texture_2d_dumps}" ]; then set -- "$@" --es texture_2d_dumps "${texture_2d_dumps}" fi diff --git a/tools/trace_replay/trace_cases.json b/tools/trace_replay/trace_cases.json index 7cee7bc7..b8f4dc85 100644 --- a/tools/trace_replay/trace_cases.json +++ b/tools/trace_replay/trace_cases.json @@ -284,8 +284,7 @@ "golden": "minecraft-1.21.4-fabric-iris-iterationrp-in-world.0000202020.png", "target_call": 202020, "timeout_seconds": 1800, - "ssim_threshold": 0.98, - "num_subgroups_quirk": true + "ssim_threshold": 0.98 }, { "name": "minecraft-1.21.4-fabric-iris-bsl-esc-menu-854",