diff --git a/.github/workflows/apk.yml b/.github/workflows/apk.yml index e8801850..d8cb80e8 100644 --- a/.github/workflows/apk.yml +++ b/.github/workflows/apk.yml @@ -435,9 +435,6 @@ jobs: if [ "${{ matrix.case.coherent_as_flush || false }}" = "true" ]; then extra_retrace_args+=(--coherent-as-flush) fi - if [ "${{ matrix.case.num_subgroups_quirk || false }}" = "true" ]; then - extra_retrace_args+=(--num-subgroups-quirk) - fi run_retrace() { timeout "$(( ${{ matrix.case.timeout_seconds }} + 300 ))" sh android-plugin/trace-replay-ci.sh \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ffc7bca..2d0b9af6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -285,6 +285,8 @@ set(SOURCE_FILES MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/RebaseInstanceIndexPass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/ZeroBaseVertexPass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.cpp + MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.cpp + MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/NormalizeRectCoordinatesPass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/Lower1DArrayImagesPass.cpp MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/BakeImageFormatsPass.cpp @@ -299,7 +301,7 @@ set(SOURCE_FILES MobileGL/MG_Util/BackendLoaders/Vulkan/Loader.cpp MobileGL/MG_Util/SelfTest/DriverPost.cpp - MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.cpp + MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.cpp MobileGL/MG_Util/Texture/PixelStoreProcessor.cpp MobileGL/MG_Util/Texture/TextureFormatProcessor.cpp diff --git a/MobileGL/Config.h b/MobileGL/Config.h index b0540a60..ff659bd0 100644 --- a/MobileGL/Config.h +++ b/MobileGL/Config.h @@ -78,13 +78,37 @@ namespace MobileGL::MG_Config { // MOBILEGL_TRACE_ANGLE_VARIANT: signed trace-APK ANGLE build short hash. String TraceAngleVariant; #endif - // MOBILEGL_DISABLE_SUBGROUP: force-disable Vulkan shader subgroup support. + // MOBILEGL_DISABLE_SUBGROUP: force-disable Vulkan shader subgroup support, + // including the opt-in emulated compute path below. Bool DisableSubgroup = false; - // MOBILEGL_NUM_SUBGROUPS_QUIRK: derive compute gl_NumSubgroups from the local - // workgroup dimensions and gl_SubgroupSize instead of reading Vulkan's - // NumSubgroups builtin. Off by default; enable only for drivers whose builtin - // disagrees with the SubgroupId topology emitted by the same dispatch. - Bool NumSubgroupsQuirk = false; + // MOBILEGL_MAGMA_EMULATE_SUBGROUP: implement GL_KHR_shader_subgroup's compute + // stage on a 32-lane VIRTUAL subgroup lowered to workgroup-shared memory + // (ShaderTranspiler::EmulateSubgroupsPass). Strictly a last resort: it only ever + // engages when this flag is set AND the device has no native subgroup support at + // all - a device with real subgroup operations always uses them natively, + // whatever their width (the known iterationRP defect is patched by + // FixIterationRPSubgroupScratch below instead). Off by default. + Bool MagmaEmulateSubgroup = false; + // MOBILEGL_FIX_ITERATIONRP_SUBGROUP_SCRATCH: patch iterationRP's own bug - the + // pack declares `shared vec2 prefixSumCache[32]` for a 512-invocation exposure + // reduction and indexes it by gl_SubgroupID, so any device with sub-16-lane + // subgroups (8-lane lavapipe -> 64 subgroups) writes shared memory out of + // bounds. The pass grows that one array to what the device's topology needs and + // touches nothing else; it only rewrites modules positively matching the pack's + // reduction fingerprint (ShaderTranspiler::FixIterationRPSubgroupScratchPass), + // so every other shader passes through byte-identical - as does iterationRP + // itself on >= 16-lane devices. Auto is ON; ForceOff replays the pack's bug + // verbatim. + QuirkOverride FixIterationRPSubgroupScratch = QuirkOverride::Auto; + // MOBILEGL_DERIVE_NUM_SUBGROUPS: replace compute gl_NumSubgroups loads with + // ceil(workgroup invocations / gl_SubgroupSize) on the NATIVE subgroup path + // (ShaderTranspiler::DeriveNumSubgroupsPass). Auto is ON: GL requires + // gl_SubgroupID < gl_NumSubgroups, Adreno's builtin reports 1 while the same + // dispatch emits IDs 0..7, and the derived value is the one Vulkan guarantees + // whenever the pipeline can request REQUIRE_FULL_SUBGROUPS (which the renderer + // does whenever local_size_x is a multiple of the native width). ForceOff returns + // to the raw driver builtin. + QuirkOverride DeriveNumSubgroups = QuirkOverride::Auto; // MOBILEGL_ADVERTISE_FP64: add GL_ARB_gpu_shader_fp64 to the advertised extension // string. `double` in a shader always WORKS - it is narrowed to 32 bits before any // module reaches a backend (ShaderTranspiler::DemoteFloat64Pass) - but the extension diff --git a/MobileGL/ConfigLoader.cpp b/MobileGL/ConfigLoader.cpp index 4e20e8e3..f23a43d5 100644 --- a/MobileGL/ConfigLoader.cpp +++ b/MobileGL/ConfigLoader.cpp @@ -168,7 +168,10 @@ namespace MobileGL::MG_ConfigLoader { QueryEnvVariable("MOBILEGL_TRACE_ANGLE_VARIANT", features.TraceAngleVariant, ""); #endif features.DisableSubgroup = QueryEnvFlag("MOBILEGL_DISABLE_SUBGROUP"); - features.NumSubgroupsQuirk = QueryEnvFlag("MOBILEGL_NUM_SUBGROUPS_QUIRK"); + features.MagmaEmulateSubgroup = QueryEnvFlag("MOBILEGL_MAGMA_EMULATE_SUBGROUP"); + features.FixIterationRPSubgroupScratch = + QueryEnvQuirkOverride("MOBILEGL_FIX_ITERATIONRP_SUBGROUP_SCRATCH"); + features.DeriveNumSubgroups = QueryEnvQuirkOverride("MOBILEGL_DERIVE_NUM_SUBGROUPS"); features.AdvertiseFp64 = QueryEnvFlag("MOBILEGL_ADVERTISE_FP64"); features.MagmaR11G11B10FFallback = QueryEnvFlag("MOBILEGL_MAGMA_R11G11B10F_FALLBACK"); features.MagmaFramesInFlight = QueryEnvUint32("MOBILEGL_MAGMA_FRAMESINFLIGHT", 3, 1, 64); diff --git a/MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.cpp b/MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.cpp index 37cc313c..b5c062cd 100644 --- a/MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.cpp @@ -9,6 +9,7 @@ #include "BackendObject_DirectVulkan.h" #include "MG_Backend/BackendObject.h" #include "DirectVulkan.h" +#include "SubgroupSupportPolicy.h" #include "MG_State/GLState/FramebufferState/FramebufferObject.h" #include "MG_State/GLState/Core.h" #include "MG_State/GLState/TextureState/TextureState.h" @@ -704,8 +705,14 @@ namespace MobileGL::MG_Backend::DirectVulkan { // real device timestamp support. ApplyVulkanCapabilitiesForTesting may // run without a renderer; no timer query is advertised then. Rebuilding // the whole list keeps re-runs idempotent. + // The opt-in emulated compute path (SubgroupSupportPolicy.h) carries the + // extension by itself on devices with no native subgroup support at all; a + // device with native subgroups always advertises - and uses - those. + const Bool subgroupSupportAdvertised = + m_vulkanCaps.SupportsShaderSubgroup || + ShouldEmulateSubgroups(m_vulkanCaps.SupportsShaderSubgroup); m_rendererInfo.RendererGLInfo.Extensions = BuildAdvertisedExtensions( - m_vulkanCaps.SupportsShaderSubgroup, pVulkanRenderer && pVulkanRenderer->IsTimerQuerySupported(), + subgroupSupportAdvertised, pVulkanRenderer && pVulkanRenderer->IsTimerQuerySupported(), pVulkanRenderer && pVulkanRenderer->IsSamplerAnisotropySupported(), pVulkanRenderer && pVulkanRenderer->IsNonZeroIndirectBaseInstanceSupported()); } @@ -941,6 +948,18 @@ namespace MobileGL::MG_Backend::DirectVulkan { m_dynamicParameters.SubgroupSupportedFeatures = mapSubgroupFeatures(m_vulkanCaps.SubgroupSupportedOperations); m_dynamicParameters.SubgroupQuadOperationsInAllStages = m_vulkanCaps.SubgroupQuadOperationsInAllStages; + } else if (ShouldEmulateSubgroups(m_vulkanCaps.SupportsShaderSubgroup)) { + // MOBILEGL_MAGMA_EMULATE_SUBGROUP on a device with no native subgroups: the + // advertised values describe the 32-lane virtual subgroup the compute + // lowering implements (SubgroupSupportPolicy.h / EmulateSubgroupsPass). + // GL requires the advertisement and the execution to agree, and on this + // path the emulation is what executes; only the compute stage is offered. + m_dynamicParameters.SubgroupSize = kEmulatedSubgroupSize; + m_dynamicParameters.SubgroupSupportedStages = kEmulatedSubgroupStages; + m_dynamicParameters.SubgroupSupportedFeatures = kEmulatedSubgroupFeatures; + m_dynamicParameters.SubgroupQuadOperationsInAllStages = false; + MGLOG_I("DirectVulkan: emulating 32-lane compute subgroups " + "(MOBILEGL_MAGMA_EMULATE_SUBGROUP, no native subgroup support)"); } else { m_dynamicParameters.SubgroupSize = 0; m_dynamicParameters.SubgroupSupportedStages = 0; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp index 151ffc1e..a09d683f 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp @@ -8,7 +8,6 @@ #include "ProgramFactory.h" -#include "Config.h" #include "MG_Backend/DirectVulkan/DirectVulkanResourceState.h" #include "MG_Util/ShaderTranspiler/ShaderCompiler.h" #include "MG_Util/ShaderTranspiler/SpvcSession.h" @@ -34,6 +33,32 @@ namespace MobileGL::MG_Backend::DirectVulkan { using SpvcSession = MG_Util::ShaderTranspiler::SpvcSession; using SessionUsageBit = MG_Util::ShaderTranspiler::SessionUsageBit; + // Local size of a compute module, read from OpExecutionMode LocalSize; all-zero + // when absent. The compile chain pins SPIR-V 1.3, where a literal local size + // always reaches the module as this execution mode (LocalSizeId does not exist + // yet). + struct ComputeLocalSize { + Uint32 x = 0; + Uint32 y = 0; + Uint32 z = 0; + Uint64 Total() const { return static_cast(x) * y * z; } + }; + ComputeLocalSize TryGetComputeLocalSize(const Vector& spirv) { + constexpr SizeT kHeaderWords = 5; + constexpr Uint32 kOpExecutionMode = 16; + constexpr Uint32 kModeLocalSize = 17; + for (SizeT offset = kHeaderWords; offset < spirv.size();) { + const Uint32 wordCount = spirv[offset] >> 16u; + const Uint32 opcode = spirv[offset] & 0xffffu; + if (wordCount == 0 || offset + wordCount > spirv.size()) break; + if (opcode == kOpExecutionMode && wordCount >= 6 && spirv[offset + 2] == kModeLocalSize) { + return {spirv[offset + 3], spirv[offset + 4], spirv[offset + 5]}; + } + offset += wordCount; + } + return {}; + } + struct DescriptorKey { ProgramFactory::DescriptorBindingKind kind = ProgramFactory::DescriptorBindingKind::None; String name; @@ -3164,20 +3189,56 @@ namespace MobileGL::MG_Backend::DirectVulkan { } } - // NumSubgroups is defined by the local workgroup dimensions and SubgroupSize. Derive - // it in SPIR-V instead of trusting a driver builtin that can disagree with the - // SubgroupId topology produced by the same compute dispatch (Adreno reports 1 while - // emitting IDs 0..7 for a 512-invocation, 64-wide workgroup). - if (MG_Config::Features.NumSubgroupsQuirk && shaders[i] && - shaders[i]->GetShaderStage() == ShaderStage::Compute) { - Vector derivedNumSubgroupsSpirv; - if (MG_Util::ShaderTranspiler::ShaderCompiler::DeriveNumSubgroupsForVulkan( - moduleSpirvs[i], derivedNumSubgroupsSpirv, enableSpirvValidation)) { - moduleSpirvs[i] = std::move(derivedNumSubgroupsSpirv); + // GL_KHR_shader_subgroup handling (SubgroupSupportPolicy.h). Native subgroup + // operations execute natively; two module repairs keep the GL contract intact + // around them. The opt-in emulation path replaces them only on devices with no + // subgroup support at all (MOBILEGL_MAGMA_EMULATE_SUBGROUP). + if (shaders[i] && shaders[i]->GetShaderStage() == ShaderStage::Compute) { + if (m_subgroupPolicy.emulateSubgroups) { + Vector emulatedSpirv; + if (MG_Util::ShaderTranspiler::ShaderCompiler::EmulateSubgroupsForVulkan( + moduleSpirvs[i], emulatedSpirv, + m_subgroupPolicy.maxComputeSharedMemoryBytes, enableSpirvValidation)) { + moduleSpirvs[i] = std::move(emulatedSpirv); + } else { + MGLOG_E("ProgramFactory: subgroup emulation failed for program %u; the " + "module keeps subgroup operations the device cannot execute", + program.GetExternalIndex()); + } } else { - MGLOG_E("ProgramFactory: failed to derive gl_NumSubgroups for program %u; " - "compute shaders may observe a driver-inconsistent subgroup count", - program.GetExternalIndex()); + // iterationRP under-declares its cross-subgroup scratch + // (prefixSumCache[32] for 512 invocations); on a sub-16-lane device + // grow that one fingerprinted array to what the topology needs. + if (m_subgroupPolicy.fixIterationRPSubgroupScratch) { + Vector patchedSpirv; + if (MG_Util::ShaderTranspiler::ShaderCompiler::FixIterationRPSubgroupScratchForVulkan( + moduleSpirvs[i], patchedSpirv, m_subgroupPolicy.nativeSubgroupSize, + enableSpirvValidation)) { + moduleSpirvs[i] = std::move(patchedSpirv); + } else { + MGLOG_E("ProgramFactory: iterationRP subgroup scratch patch failed for " + "program %u; the pack's declared array sizes stay in effect", + program.GetExternalIndex()); + } + } + // gl_NumSubgroups must agree with the gl_SubgroupID range GL promises; + // derive it from the workgroup dimensions and gl_SubgroupSize instead of + // trusting a driver builtin that can disagree with the topology the same + // dispatch emits (Adreno reports 1 while emitting IDs 0..7 for a + // 512-invocation, 64-wide workgroup). The ceil() partition this derives + // is pinned by REQUIRE_FULL_SUBGROUPS at pipeline creation whenever the + // workgroup shape makes that flag legal (see the stage setup below). + if (m_subgroupPolicy.deriveNumSubgroups) { + Vector derivedNumSubgroupsSpirv; + if (MG_Util::ShaderTranspiler::ShaderCompiler::DeriveNumSubgroupsForVulkan( + moduleSpirvs[i], derivedNumSubgroupsSpirv, enableSpirvValidation)) { + moduleSpirvs[i] = std::move(derivedNumSubgroupsSpirv); + } else { + MGLOG_E("ProgramFactory: failed to derive gl_NumSubgroups for program %u; " + "compute shaders may observe a driver-inconsistent subgroup count", + program.GetExternalIndex()); + } + } } } @@ -3326,6 +3387,27 @@ namespace MobileGL::MG_Backend::DirectVulkan { stage.stage = ToVkStage(shaderStage); stage.module = module; stage.pName = "main"; + // Pin the full-subgroup launch the derived gl_NumSubgroups assumes. Legal + // exactly when the computeFullSubgroups feature is enabled and local_size_x is + // a multiple of the subgroup size (VUID-VkPipelineShaderStageCreateInfo- + // flags-02759/-02785), and only worth requesting while the resulting subgroup + // count fits the device's maxComputeWorkgroupSubgroups (lavapipe caps it at + // 32, below a 512-invocation dispatch's 64). With the bit set, "Full + // Subgroups" guarantees every subgroup launches with all invocations active, + // making the subgroup count exactly invocations / size. Shapes the flag + // cannot cover (e.g. 32x16 on a 64-wide device) fall back to the driver's + // own - spec-encouraged - tight partitioning, which the DriverPost witness + // verifies per device. + if (shaderStage == ShaderStage::Compute && m_subgroupPolicy.requireFullSubgroups && + !m_subgroupPolicy.emulateSubgroups && m_subgroupPolicy.nativeSubgroupSize != 0) { + const ComputeLocalSize localSize = TryGetComputeLocalSize(moduleSpv); + const Uint64 fullSubgroupCount = + localSize.Total() / m_subgroupPolicy.nativeSubgroupSize; + if (localSize.x != 0 && localSize.x % m_subgroupPolicy.nativeSubgroupSize == 0 && + fullSubgroupCount <= m_subgroupPolicy.maxComputeWorkgroupSubgroups) { + stage.flags |= VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT; + } + } entry.modules.push_back(module); entry.stages.push_back(stage); diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.h index e64e5895..560aec89 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.h @@ -372,16 +372,38 @@ namespace MobileGL::MG_Backend::DirectVulkan { virtual void OnProgramEvicted(HashType programHash, VkDescriptorSetLayout descriptorSetLayout) = 0; }; + // How this factory's compute modules implement GL_KHR_shader_subgroup. Computed + // once at renderer initialization (SubgroupSupportPolicy.h + the device's + // subgroup properties) so lowering can never disagree with the advertised + // capabilities. Native subgroup operations always execute natively; the two + // repair passes patch modules AROUND them, and the emulation only replaces them + // on opted-in devices with no subgroup support at all. + struct SubgroupLoweringPolicy { + Bool emulateSubgroups = false; // MOBILEGL_MAGMA_EMULATE_SUBGROUP, no-native-support devices + Bool fixIterationRPSubgroupScratch = false; // patch iterationRP's under-declared scratch + Bool deriveNumSubgroups = false; // repair the NumSubgroups builtin + Bool requireFullSubgroups = false; // computeFullSubgroups enabled on the device + Uint32 nativeSubgroupSize = 0; + // Full-subgroup launches are bounded by this device limit; a dispatch whose + // workgroup needs more subgroups than this cannot request the flag. + Uint32 maxComputeWorkgroupSubgroups = 0; + // VkPhysicalDeviceLimits::maxComputeSharedMemorySize; bounds the scratch the + // emulation pass may add (0 falls back to the Vulkan minimum, 16384). + Uint32 maxComputeSharedMemoryBytes = 0; + }; + explicit ProgramFactory(VkDevice device, const VulkanRendererConfig& config, Uint32 maxBindings, Bool shaderDrawParametersEnabled, Bool unformattedFloatStorageImagesEnabled, Bool enableSpirvValidation, - UpdateAfterBindLimits updateAfterBindLimits) + UpdateAfterBindLimits updateAfterBindLimits, + SubgroupLoweringPolicy subgroupPolicy) : m_device(device), m_maxBindings(maxBindings), m_config(config), m_shaderDrawParametersEnabled(shaderDrawParametersEnabled), m_unformattedFloatStorageImagesEnabled(unformattedFloatStorageImagesEnabled), m_enableSpirvValidation(enableSpirvValidation), - m_updateAfterBindLimits(updateAfterBindLimits) { + m_updateAfterBindLimits(updateAfterBindLimits), + m_subgroupPolicy(subgroupPolicy) { VkProgramObject::s_device = device; } // Destroys the pass-through tessellation control modules. Runs while the device is @@ -511,6 +533,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { // the factory lets each reflected layout choose ordinary descriptors when its // own counts would exceed the update-after-bind budget. UpdateAfterBindLimits m_updateAfterBindLimits{}; + SubgroupLoweringPolicy m_subgroupPolicy{}; // See SetDefaultFramebufferHeight. 0 means "not known yet"; the FragCoordYFlip bit is // never set before the swapchain exists, so no variant can be compiled against it. Uint32 m_defaultFramebufferHeight = 0; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp index 85ec4a46..92b9469a 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp @@ -8,6 +8,7 @@ #include "VulkanRenderer.h" +#include "MG_Backend/DirectVulkan/SubgroupSupportPolicy.h" #include "MG_Backend/DirectGLES/Utils.h" #include "VertexInputStateFactory.h" #include "VertexInputStateBuilder.h" @@ -3058,11 +3059,22 @@ void main() { } PipelineFactory::SetSuppressBlendedDepthWrite(suppressBlendedDepthWrite); } + ProgramFactory::SubgroupLoweringPolicy subgroupPolicy{}; + subgroupPolicy.emulateSubgroups = ShouldEmulateSubgroups(m_nativeSubgroupSupported); + subgroupPolicy.fixIterationRPSubgroupScratch = + m_nativeSubgroupSupported && ShouldFixIterationRPSubgroupScratch(); + subgroupPolicy.deriveNumSubgroups = + m_nativeSubgroupSupported && ShouldDeriveNumSubgroups(); + subgroupPolicy.requireFullSubgroups = m_computeFullSubgroupsFeatureEnabled; + subgroupPolicy.nativeSubgroupSize = m_nativeSubgroupSize; + subgroupPolicy.maxComputeWorkgroupSubgroups = m_maxComputeWorkgroupSubgroups; + subgroupPolicy.maxComputeSharedMemoryBytes = + m_physicalDevice.properties.limits.maxComputeSharedMemorySize; m_programFactory = MakeUnique(m_device, m_config, maxProgramBindings, m_shaderDrawParametersFeatureEnabled, m_unformattedFloatStorageImagesEnabled, MG_Config::Features.EnableSpirvValidation, - m_updateAfterBindLimits); + m_updateAfterBindLimits, subgroupPolicy); MOBILEGL_ASSERT(m_programFactory != nullptr, "ProgramFactory creation failed."); // The swapchain already exists at this point (Initialize creates it first), so seed the // height the factory could not be told about from CreateSwapchain. @@ -12740,6 +12752,73 @@ void main() { } } + // Native subgroup topology, and VK_EXT_subgroup_size_control's + // computeFullSubgroups feature. REQUIRE_FULL_SUBGROUPS on a compute stage is what + // turns the derived gl_NumSubgroups (DeriveNumSubgroupsPass) from + // encouraged-but-unspecified driver behaviour into a spec guarantee: with the bit + // set and local_size_x a multiple of the subgroup size, every subgroup launches + // full, so the subgroup count is exactly invocations / size ("Full Subgroups", + // VUID-VkPipelineShaderStageCreateInfo-flags-02759/-02785). + m_nativeSubgroupSize = 0; + m_nativeSubgroupSupported = false; + m_computeFullSubgroupsFeatureEnabled = false; + if (getPhysicalDeviceProperties2 != nullptr) { + VkPhysicalDeviceSubgroupProperties subgroupProperties{}; + subgroupProperties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES; + VkPhysicalDeviceProperties2 subgroupPropertyQuery{}; + subgroupPropertyQuery.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; + subgroupPropertyQuery.pNext = &subgroupProperties; + getPhysicalDeviceProperties2(m_physicalDevice.handle, &subgroupPropertyQuery); + // Mirrors the loader's HasUsableShaderSubgroupSupport gate, including the + // MOBILEGL_DISABLE_SUBGROUP escape hatch, so the module lowerings can never + // disagree with the advertised capabilities. + const Bool usableSubgroups = + subgroupProperties.subgroupSize > 0 && + (subgroupProperties.supportedStages & VK_SHADER_STAGE_COMPUTE_BIT) != 0 && + (subgroupProperties.supportedOperations & VK_SUBGROUP_FEATURE_BASIC_BIT) != 0; + if (usableSubgroups && !MG_Config::Features.DisableSubgroup) { + m_nativeSubgroupSize = subgroupProperties.subgroupSize; + m_nativeSubgroupSupported = true; + } + } + VkPhysicalDeviceSubgroupSizeControlFeaturesEXT subgroupSizeControlFeatures{}; + subgroupSizeControlFeatures.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES_EXT; + m_maxComputeWorkgroupSubgroups = 0; + if (m_nativeSubgroupSupported && + IsExtensionSupported(availableExtensions, VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME) && + getPhysicalDeviceFeatures2 != nullptr) { + VkPhysicalDeviceFeatures2 featureQuery{}; + featureQuery.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; + featureQuery.pNext = &subgroupSizeControlFeatures; + getPhysicalDeviceFeatures2(m_physicalDevice.handle, &featureQuery); + if (getPhysicalDeviceProperties2 != nullptr) { + VkPhysicalDeviceSubgroupSizeControlPropertiesEXT subgroupSizeControlProperties{}; + subgroupSizeControlProperties.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_PROPERTIES_EXT; + VkPhysicalDeviceProperties2 propertyQuery{}; + propertyQuery.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; + propertyQuery.pNext = &subgroupSizeControlProperties; + getPhysicalDeviceProperties2(m_physicalDevice.handle, &propertyQuery); + m_maxComputeWorkgroupSubgroups = + subgroupSizeControlProperties.maxComputeWorkgroupSubgroups; + } + if (subgroupSizeControlFeatures.computeFullSubgroups == VK_TRUE) { + if (!IsExtensionAlreadyEnabled(enabledDeviceExtensions, + VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME)) { + enabledDeviceExtensions.push_back(VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME); + } + // Only the full-subgroups guarantee is wanted; required/varying subgroup + // sizes stay unrequested. + subgroupSizeControlFeatures.subgroupSizeControl = VK_FALSE; + subgroupSizeControlFeatures.pNext = const_cast(deviceCreateInfo.pNext); + deviceCreateInfo.pNext = &subgroupSizeControlFeatures; + m_computeFullSubgroupsFeatureEnabled = true; + MGLOG_I("Enabled optional device extension: %s (computeFullSubgroups)", + VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME); + } + } + // VK_EXT_transform_feedback backs GL transform feedback capture. m_transformFeedbackFeatureEnabled = false; VkPhysicalDeviceTransformFeedbackFeaturesEXT transformFeedbackFeatures{}; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h index 06b23605..d021df94 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h @@ -554,6 +554,16 @@ namespace MobileGL::MG_Backend::DirectVulkan { Bool m_samplerAnisotropyFeatureEnabled = false; Bool m_shaderDrawParametersExtensionEnabled = false; Bool m_shaderDrawParametersFeatureEnabled = false; + // Native subgroup topology, queried at device creation for the compute-module + // subgroup repairs (SubgroupSupportPolicy.h) and the REQUIRE_FULL_SUBGROUPS + // stage flag; 0 / false when the device has no usable compute subgroups or + // MOBILEGL_DISABLE_SUBGROUP forced them off. + Uint32 m_nativeSubgroupSize = 0; + Bool m_nativeSubgroupSupported = false; + Bool m_computeFullSubgroupsFeatureEnabled = false; + // VkPhysicalDeviceSubgroupSizeControlProperties::maxComputeWorkgroupSubgroups; + // 0 when the extension (and therefore the full-subgroups flag) is unavailable. + Uint32 m_maxComputeWorkgroupSubgroups = 0; Bool m_unformattedFloatStorageImagesEnabled = false; // Set only after descriptor-indexing feature AND property queries prove that // update-after-bind is legal for every descriptor category this renderer emits. diff --git a/MobileGL/MG_Backend/DirectVulkan/SubgroupSupportPolicy.h b/MobileGL/MG_Backend/DirectVulkan/SubgroupSupportPolicy.h new file mode 100644 index 00000000..0f06e7bc --- /dev/null +++ b/MobileGL/MG_Backend/DirectVulkan/SubgroupSupportPolicy.h @@ -0,0 +1,57 @@ +// MobileGL - MobileGL/MG_Backend/DirectVulkan/SubgroupSupportPolicy.h +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#pragma once + +#include +#include + +namespace MobileGL::MG_Backend::DirectVulkan { + // The single decision point for how DirectVulkan implements GL_KHR_shader_subgroup, + // shared by capability advertisement (BackendObject) and module lowering + // (VulkanRenderer / ProgramFactory) so the two can never disagree. + // + // Native subgroups are the implementation whenever the device has them, whatever + // their width - subgroup operations execute on the hardware paths they were made + // for. Two module-level repairs keep the GL contract intact around them: + // - FixIterationRPSubgroupScratchPass patches the one known pack bug: iterationRP's + // prefixSumCache[32], under-declared for sub-16-lane devices (8-lane lavapipe); + // - DeriveNumSubgroupsPass replaces the one builtin drivers get wrong + // (gl_NumSubgroups) with the value the rest of the topology implies. + // The 32-lane shared-memory emulation (EmulateSubgroupsPass) is a LAST RESORT for + // devices with no subgroup support at all, and only when the user opts in with + // MOBILEGL_MAGMA_EMULATE_SUBGROUP=1; it never replaces available native operations. + + inline constexpr Uint32 kEmulatedSubgroupSize = 32u; + inline constexpr Uint32 kEmulatedSubgroupStages = GL_COMPUTE_SHADER_BIT; + inline constexpr Uint32 kEmulatedSubgroupFeatures = + GL_SUBGROUP_FEATURE_BASIC_BIT_KHR | GL_SUBGROUP_FEATURE_VOTE_BIT_KHR | + GL_SUBGROUP_FEATURE_ARITHMETIC_BIT_KHR | GL_SUBGROUP_FEATURE_BALLOT_BIT_KHR | + GL_SUBGROUP_FEATURE_SHUFFLE_BIT_KHR | GL_SUBGROUP_FEATURE_SHUFFLE_RELATIVE_BIT_KHR | + GL_SUBGROUP_FEATURE_CLUSTERED_BIT_KHR | GL_SUBGROUP_FEATURE_QUAD_BIT_KHR; + + inline Bool ShouldEmulateSubgroups(const Bool nativeSubgroupSupported) { + return MG_Config::Features.MagmaEmulateSubgroup && !nativeSubgroupSupported && + !MG_Config::Features.DisableSubgroup; + } + + inline Bool ShouldFixIterationRPSubgroupScratch() { + // Auto is ON: the patch is fingerprint-gated to iterationRP's reduction and + // grows one under-declared array; every other module passes through untouched. + return MG_Config::Features.FixIterationRPSubgroupScratch != + MG_Config::QuirkOverride::ForceOff; + } + + inline Bool ShouldDeriveNumSubgroups() { + // Auto is ON: gl_NumSubgroups must agree with the gl_SubgroupID range for the GL + // contract to hold, and the derived ceil() value is the one the renderer can pin + // with REQUIRE_FULL_SUBGROUPS - the driver builtin is the value with no + // cross-driver guarantee (Adreno returns 1 for an 8-subgroup dispatch). + return MG_Config::Features.DeriveNumSubgroups != MG_Config::QuirkOverride::ForceOff; + } +} // namespace MobileGL::MG_Backend::DirectVulkan diff --git a/MobileGL/MG_IntegrationTest/CMakeLists.txt b/MobileGL/MG_IntegrationTest/CMakeLists.txt index 1c234208..4584ab73 100644 --- a/MobileGL/MG_IntegrationTest/CMakeLists.txt +++ b/MobileGL/MG_IntegrationTest/CMakeLists.txt @@ -68,7 +68,8 @@ add_executable(MobileGLIntegrationTest Scenarios/DoublePrecisionScenario.cpp Scenarios/UniformInitializerScenario.cpp Scenarios/SwizzleAccessRoutineScenario.cpp - Scenarios/Program203FirstReductionScenario.cpp + Scenarios/IterationRPFirstReductionScenario.cpp + Scenarios/IterationRPScratchFixScenario.cpp Scenarios/ProgramPipelineScenario.cpp Scenarios/ImageLoadStoreSsoScenario.cpp Scenarios/ImageTargetKindScenario.cpp diff --git a/MobileGL/MG_IntegrationTest/Scenarios/Program203FirstReductionScenario.cpp b/MobileGL/MG_IntegrationTest/Scenarios/IterationRPFirstReductionScenario.cpp similarity index 93% rename from MobileGL/MG_IntegrationTest/Scenarios/Program203FirstReductionScenario.cpp rename to MobileGL/MG_IntegrationTest/Scenarios/IterationRPFirstReductionScenario.cpp index 32539f0d..19bdb51b 100644 --- a/MobileGL/MG_IntegrationTest/Scenarios/Program203FirstReductionScenario.cpp +++ b/MobileGL/MG_IntegrationTest/Scenarios/IterationRPFirstReductionScenario.cpp @@ -1,4 +1,4 @@ -// MobileGL - MobileGL/MG_IntegrationTest/Scenarios/Program203FirstReductionScenario.cpp +// MobileGL - MobileGL/MG_IntegrationTest/Scenarios/IterationRPFirstReductionScenario.cpp // Copyright (c) 2026 MobileGL-Dev // Licensed under the GNU Lesser General Public License v3.0: // https://www.gnu.org/licenses/gpl-3.0.txt @@ -6,9 +6,9 @@ // SPDX-License-Identifier: LGPL-3.0-only // End of Source File Header // -// Scenario - PROGRAM 203'S FIRST SUBGROUP REDUCTION. +// Scenario - ITERATIONRP'S FIRST SUBGROUP REDUCTION. // -// Program 203 reduces a 32 x 16 exposure tile with a vector subgroup inclusive add, +// iterationRP reduces a 32 x 16 exposure tile with a vector subgroup inclusive add, // then a shared-memory scan of subgroup totals. The source assumes that every // subgroup has a last lane, that there are 2..32 subgroups, and that local index // 511 belongs to the last subgroup and its last lane. Those are source assumptions, @@ -133,6 +133,17 @@ namespace MGITest { std::array maxWorkGroupSize{}; bool queryHadError = false; + // iterationRP's source contract needs gl_NumSubgroups in [2, 32] for its 512 + // invocations, i.e. an advertised subgroup width in [16, 256]. A device + // outside that window (lavapipe's 8-lane subgroups give 64 subgroups) cannot + // run the fixture's verbatim reduction at all, so the scenario SKIPS there - + // the pack itself replays through the FixIterationRPSubgroupScratch patch, which + // this probe deliberately does not model. The width only gates the domain; + // lane placement and group counts still come from observed values alone. + bool SubgroupWidthInSourceDomain() const { + return subgroupSize >= 16 && subgroupSize <= 256; + } + bool SupportsProbe() const { const auto stages = static_cast(supportedStages); const auto features = static_cast(supportedFeatures); @@ -140,6 +151,7 @@ namespace MGITest { (stages & GL_COMPUTE_SHADER_BIT) != 0 && (features & (GL_SUBGROUP_FEATURE_BASIC_BIT_KHR | GL_SUBGROUP_FEATURE_ARITHMETIC_BIT_KHR)) == (GL_SUBGROUP_FEATURE_BASIC_BIT_KHR | GL_SUBGROUP_FEATURE_ARITHMETIC_BIT_KHR) && + SubgroupWidthInSourceDomain() && maxComputeStorageBlocks >= 2 && maxStorageBindings >= 2 && maxWorkGroupInvocations >= static_cast(kInvocationCount) && maxWorkGroupSize[0] >= 32 && maxWorkGroupSize[1] >= 16 && maxWorkGroupSize[2] >= 1; @@ -159,6 +171,12 @@ namespace MGITest { if ((features & requiredFeatures) != requiredFeatures) { missing.emplace_back("basic|arithmetic in GL_SUBGROUP_SUPPORTED_FEATURES_KHR"); } + if (!SubgroupWidthInSourceDomain()) { + missing.emplace_back( + "GL_SUBGROUP_SIZE_KHR in [16, 256] (iterationRP's source contract needs " + "gl_NumSubgroups in [2, 32] for 512 invocations; width " + + std::to_string(subgroupSize) + " is outside the fixture's domain)"); + } if (maxComputeStorageBlocks < 2 || maxStorageBindings < 2) { missing.emplace_back("two compute SSBO bindings"); } @@ -194,7 +212,7 @@ namespace MGITest { } void PrintMetadata(const CapabilityInfo& info, std::ostream& output) { - output << "Program203FirstReductionScenario metadata: " + output << "IterationRPFirstReductionScenario metadata: " << "GL_SUBGROUP_SIZE_KHR=" << info.subgroupSize << ", GL_SUBGROUP_SUPPORTED_STAGES_KHR=0x" << std::hex << static_cast(info.supportedStages) @@ -243,7 +261,7 @@ layout(std430, binding = 0) readonly buffer Input { )"; // Only the expression producing tileExposure differs between the two - // tests. The remainder is the program-203 first reduction, with stores + // tests. The remainder is the iterationRP first reduction, with stores // placed after its existing barriers to expose each handoff. constexpr const char* kSampledTileExposure = R"( vec2 texCoord = (vec2(gl_GlobalInvocationID.xy) + 0.5) * @@ -506,7 +524,7 @@ layout(std430, binding = 0) readonly buffer Input { if (!IsQuietNanSentinel(reduction.z) || !IsQuietNanSentinel(reduction.w) || !IsQuietNanSentinel(output.finalAverage[slot])) { std::ostringstream message; - message << "program 203 source reduction has no valid contract for gl_NumSubgroups=" + message << "iterationRP source reduction has no valid contract for gl_NumSubgroups=" << reportedNumSubgroups << "; localIndex " << localIndex << " did not preserve its qNaN source-reduction sentinel"; return Failure("source domain", message.str()); @@ -514,7 +532,7 @@ layout(std430, binding = 0) readonly buffer Input { for (std::size_t stage = 0; stage < kScanStageCount; ++stage) { if (!IsQuietNanSentinel(output.scanAfter[stage][slot])) { std::ostringstream message; - message << "program 203 source reduction has no valid contract for gl_NumSubgroups=" + message << "iterationRP source reduction has no valid contract for gl_NumSubgroups=" << reportedNumSubgroups << "; localIndex " << localIndex << ", scan stage " << stage << " did not preserve its qNaN source-reduction sentinel"; return Failure("source domain", message.str()); @@ -522,12 +540,12 @@ layout(std430, binding = 0) readonly buffer Input { } } std::ostringstream message; - message << "program 203 source reduction has no valid contract for observed gl_NumSubgroups=" + message << "iterationRP source reduction has no valid contract for observed gl_NumSubgroups=" << reportedNumSubgroups << " (requires 2..32); native subgroup results were recorded"; return Failure("source domain", message.str()); } - // 4. Program-203 source writer and first shared-memory handoff. + // 4. iterationRP source writer and first shared-memory handoff. std::vector sourceWriter(reportedNumSubgroups, kNoSlot); for (std::uint32_t subgroupID = 0; subgroupID < reportedNumSubgroups; ++subgroupID) { std::size_t writerCount = 0; @@ -541,7 +559,7 @@ layout(std430, binding = 0) readonly buffer Input { if (writerCount != 1u) { std::ostringstream message; message << "subgroupID " << subgroupID << " has " << writerCount - << " recorded lane(s) where laneID == subgroupSize - 1; program 203 leaves that " + << " recorded lane(s) where laneID == subgroupSize - 1; iterationRP leaves that " "shared-cache entry unwritten"; return Failure("source writer", message.str()); } @@ -633,7 +651,7 @@ layout(std430, binding = 0) readonly buffer Input { index511Subgroup.z == ownerResult.highestObservedSubgroup; if (!ownerResult.index511IsSourceLastLaneWriter || !ownerResult.index511IsHighestSubgroupMember) { std::ostringstream message; - message << "program 203 topology incompatibility: localIndex 511 is sourceLastLaneWriter=" + message << "iterationRP topology incompatibility: localIndex 511 is sourceLastLaneWriter=" << ownerResult.index511IsSourceLastLaneWriter << ", highestSubgroupMember=" << ownerResult.index511IsHighestSubgroupMember << " (subgroupID=" << index511Subgroup.z << ", highest observed subgroupID=" << ownerResult.highestObservedSubgroup << ')'; @@ -650,7 +668,7 @@ layout(std430, binding = 0) readonly buffer Input { const float expectedTotal = mode == InputMode::IndexedSsbo ? 131328.0f : sampledExpectedTotal; if (!SameBits(total, expectedTotal) || !SameBits(mergedPrefix[index511Slot], expectedTotal)) { std::ostringstream message; - message << "program 203 source total was " << FormatFloat(mergedPrefix[index511Slot]) + message << "iterationRP source total was " << FormatFloat(mergedPrefix[index511Slot]) << " (native total " << FormatFloat(total) << "), expected " << FormatFloat(expectedTotal); ownerResult.ok = false; ownerResult.phase = "final average"; @@ -675,9 +693,9 @@ layout(std430, binding = 0) readonly buffer Input { bool includeScanStages) { PrintMetadata(capabilities, std::cout); if (validation.ok) { - std::cout << "Program203FirstReductionScenario firstFailure=none\n"; + std::cout << "IterationRPFirstReductionScenario firstFailure=none\n"; } else { - std::cout << "Program203FirstReductionScenario firstFailure=" << validation.phase << ": " + std::cout << "IterationRPFirstReductionScenario firstFailure=" << validation.phase << ": " << validation.message << '\n'; } std::cout << "localIndex,localX,localY,localZ,subgroupSize,numSubgroups,subgroupID,laneID,input," @@ -702,17 +720,19 @@ layout(std430, binding = 0) readonly buffer Input { } } - class Program203FirstReductionScenario : public ScenarioTest { + class IterationRPFirstReductionScenario : public ScenarioTest { protected: void SetUp() override { ScenarioTest::SetUp(); if (!Ready()) return; m_capabilities = QueryCapabilities(); - // GL_SUBGROUP_SIZE_KHR is diagnostic only. It is deliberately - // never used to infer lane placement or an expected group count. + // GL_SUBGROUP_SIZE_KHR gates only whether the fixture's source contract + // can hold on this device (SubgroupWidthInSourceDomain); it is + // deliberately never used to infer lane placement or an expected group + // count - those come from observed values alone. PrintMetadata(m_capabilities, std::cout); - RecordProperty("program203_gl_subgroup_size_khr", std::to_string(m_capabilities.subgroupSize)); + RecordProperty("iterationrp_gl_subgroup_size_khr", std::to_string(m_capabilities.subgroupSize)); if (!m_capabilities.SupportsProbe()) { GTEST_SKIP() << "subgroup probe requires " << m_capabilities.MissingRequirements(); } @@ -839,13 +859,13 @@ layout(std430, binding = 0) readonly buffer Input { const ValidationResult validation = ValidateProbe(output, mode); if (validation.ownerEvaluated) { - RecordProperty("program203_index511_source_last_lane_writer", + RecordProperty("iterationrp_index511_source_last_lane_writer", validation.index511IsSourceLastLaneWriter ? "true" : "false"); - RecordProperty("program203_index511_highest_subgroup_member", + RecordProperty("iterationrp_index511_highest_subgroup_member", validation.index511IsHighestSubgroupMember ? "true" : "false"); - RecordProperty("program203_highest_observed_subgroup", + RecordProperty("iterationrp_highest_observed_subgroup", std::to_string(validation.highestObservedSubgroup)); - std::cout << "Program203FirstReductionScenario owner: localIndex511 sourceLastLaneWriter=" + std::cout << "IterationRPFirstReductionScenario owner: localIndex511 sourceLastLaneWriter=" << validation.index511IsSourceLastLaneWriter << ", highestSubgroupMember=" << validation.index511IsHighestSubgroupMember << ", highestObservedSubgroup=" << validation.highestObservedSubgroup << '\n'; @@ -865,12 +885,12 @@ layout(std430, binding = 0) readonly buffer Input { } // namespace - TEST_F(Program203FirstReductionScenario, SampledRgba32fFirstAverage) { + TEST_F(IterationRPFirstReductionScenario, SampledRgba32fFirstAverage) { if (!Ready() || IsSkipped()) return; RunAndValidate(InputMode::SampledRgba32f); } - TEST_F(Program203FirstReductionScenario, IndexedInputTopologyAndReduction) { + TEST_F(IterationRPFirstReductionScenario, IndexedInputTopologyAndReduction) { if (!Ready() || IsSkipped()) return; RunAndValidate(InputMode::IndexedSsbo); } diff --git a/MobileGL/MG_IntegrationTest/Scenarios/IterationRPScratchFixScenario.cpp b/MobileGL/MG_IntegrationTest/Scenarios/IterationRPScratchFixScenario.cpp new file mode 100644 index 00000000..89db6321 --- /dev/null +++ b/MobileGL/MG_IntegrationTest/Scenarios/IterationRPScratchFixScenario.cpp @@ -0,0 +1,222 @@ +// MobileGL - MobileGL/MG_IntegrationTest/Scenarios/IterationRPScratchFixScenario.cpp +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header +// +// Scenario - THE FIXTURE-SHAPED SUBGROUP REDUCTION, ON WHATEVER WIDTH THE DEVICE HAS. +// +// iterationRP's auto-exposure pass declares `shared vec2 prefixSumCache[32]` for a +// 512-invocation workgroup and combines per-subgroup subtotals through +// prefixSumCache[gl_SubgroupID]. The algorithm is width-agnostic; only the static 32 +// bakes in "at most 32 subgroups", which every desktop capture satisfies and an 8-lane +// device (lavapipe: 64 subgroups) does not. DirectVulkan patches exactly that with +// FixIterationRPSubgroupScratchPass, growing the array to ceil(invocations / native +// width) on the modules that match the pack's reduction fingerprint. +// +// This scenario replays the fixture's reduction shape verbatim - the same 32-entry +// declaration, the same last-lane handoff, the same findMSB combine loop, and NO +// domain guard - and asserts only the width-independent result: the workgroup total. +// The inputs are small integers, so the fp32 sum is exact under any lane order and any +// association; a correct run produces the exact constant on a 4-lane device and a +// 128-lane device alike. Without the patch, a sub-16-lane device indexes the +// 32-entry array out of bounds - on lavapipe that is literal heap corruption - and +// this scenario is the regression test that keeps the patch working, and it runs on every device that +// has basic+arithmetic compute subgroups (unlike IterationRPFirstReductionScenario, +// which probes the UNREPAIRED source contract and must skip outside [16, 256]). + +#include +#include +#include + +#include "../Harness/HeadlessGL.h" +#include "../Harness/ScenarioFixture.h" + +#ifdef GLAPI +#undef GLAPI +#endif +#define GL_GLEXT_PROTOTYPES +#include +#include +#undef GL_GLEXT_PROTOTYPES + +namespace MGITest { + namespace { + constexpr std::uint32_t kInvocationCount = 512u; + // sum of 0..511, exactly representable and associativity-proof in fp32. + constexpr float kExpectedTotal = 130816.0f; + + constexpr const char* kComputeSource = R"(#version 430 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require + +layout(local_size_x = 32, local_size_y = 16, local_size_z = 1) in; + +layout(std430, binding = 0) buffer Output { + float total; + uint numSubgroups; + uint maxSubgroupId; +} outputData; + +shared vec2 prefixSumCache[32]; + +void main() { + vec2 sampleLuminance = vec2(float(gl_LocalInvocationIndex), 0.0); + sampleLuminance = subgroupInclusiveAdd(sampleLuminance); + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) + prefixSumCache[gl_SubgroupID] = sampleLuminance; + barrier(); + + uint loopLength = uint(findMSB(gl_NumSubgroups)); + loopLength += uint(gl_NumSubgroups - (1u << (loopLength - 1u)) > 0u); + + for (uint scanStage = 0u; scanStage < loopLength; ++scanStage) { + if ((gl_SubgroupID & (1u << scanStage)) > 0u) { + sampleLuminance += prefixSumCache[(gl_SubgroupID >> scanStage << scanStage) - 1u]; + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) + prefixSumCache[gl_SubgroupID] = sampleLuminance; + } + barrier(); + } + + if (gl_LocalInvocationIndex == 511u) { + outputData.total = sampleLuminance.x; + outputData.numSubgroups = gl_NumSubgroups; + } + atomicMax(outputData.maxSubgroupId, gl_SubgroupID); +} +)"; + + struct OutputBlock { + float total = -1.0f; + std::uint32_t numSubgroups = 0; + std::uint32_t maxSubgroupId = 0; + }; + + bool HasExtension(const char* wanted) { + GLint extensionCount = 0; + glGetIntegerv(GL_NUM_EXTENSIONS, &extensionCount); + for (GLint i = 0; i < extensionCount; ++i) { + const auto* extension = + reinterpret_cast(glGetStringi(GL_EXTENSIONS, static_cast(i))); + if (extension != nullptr && std::string(extension) == wanted) return true; + } + return false; + } + + class IterationRPScratchFixScenario : public ScenarioTest { + protected: + void SetUp() override { + ScenarioTest::SetUp(); + if (!Ready()) return; + + GLint stages = 0; + GLint features = 0; + GLint invocations = 0; + const bool subgroupExtension = HasExtension("GL_KHR_shader_subgroup"); + if (subgroupExtension) { + glGetIntegerv(GL_SUBGROUP_SUPPORTED_STAGES_KHR, &stages); + glGetIntegerv(GL_SUBGROUP_SUPPORTED_FEATURES_KHR, &features); + } + glGetIntegerv(GL_MAX_COMPUTE_WORK_GROUP_INVOCATIONS, &invocations); + const GLbitfield requiredFeatures = + GL_SUBGROUP_FEATURE_BASIC_BIT_KHR | GL_SUBGROUP_FEATURE_ARITHMETIC_BIT_KHR; + if (!subgroupExtension || (static_cast(stages) & GL_COMPUTE_SHADER_BIT) == 0 || + (static_cast(features) & requiredFeatures) != requiredFeatures || + invocations < static_cast(kInvocationCount)) { + GTEST_SKIP() << "needs GL_KHR_shader_subgroup basic+arithmetic in compute and a " + "512-invocation workgroup"; + } + + m_program = CompileComputeProgram(kComputeSource); + ASSERT_NE(m_program, 0u) << m_buildLog; + + glGenBuffers(1, &m_output); + glBindBuffer(GL_SHADER_STORAGE_BUFFER, m_output); + // maxSubgroupId starts at zero HOST-side: the word is touched only by + // atomicMax during the dispatch, since a plain shader-side zeroing store + // would race the other invocations' atomics (barrier() orders shared + // memory, not SSBO stores). + const OutputBlock poison{-1.0f, 0xa5a5a5a5u, 0u}; + glBufferData(GL_SHADER_STORAGE_BUFFER, sizeof(OutputBlock), &poison, GL_DYNAMIC_READ); + glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 0, m_output); + } + + void TearDown() override { + if (!Ready()) return; + glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 0, 0); + glBindBuffer(GL_SHADER_STORAGE_BUFFER, 0); + if (m_output != 0) glDeleteBuffers(1, &m_output); + if (m_program != 0) glDeleteProgram(m_program); + } + + unsigned int CompileComputeProgram(const char* source) { + const GLuint shader = glCreateShader(GL_COMPUTE_SHADER); + glShaderSource(shader, 1, &source, nullptr); + glCompileShader(shader); + GLint compiled = 0; + glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled); + if (compiled == GL_FALSE) { + char log[2048] = {}; + glGetShaderInfoLog(shader, sizeof(log) - 1, nullptr, log); + m_buildLog = std::string("compute shader did not compile: ") + log; + glDeleteShader(shader); + return 0; + } + const GLuint program = glCreateProgram(); + glAttachShader(program, shader); + glLinkProgram(program); + glDeleteShader(shader); + GLint linked = 0; + glGetProgramiv(program, GL_LINK_STATUS, &linked); + if (linked == GL_FALSE) { + char log[2048] = {}; + glGetProgramInfoLog(program, sizeof(log) - 1, nullptr, log); + m_buildLog = std::string("compute program did not link: ") + log; + glDeleteProgram(program); + return 0; + } + return program; + } + + OutputBlock Dispatch() { + glUseProgram(m_program); + glDispatchCompute(1, 1, 1); + glMemoryBarrier(GL_BUFFER_UPDATE_BARRIER_BIT); + OutputBlock block{}; + glBindBuffer(GL_SHADER_STORAGE_BUFFER, m_output); + glGetBufferSubData(GL_SHADER_STORAGE_BUFFER, 0, sizeof(OutputBlock), &block); + return block; + } + + GLuint m_program = 0; + GLuint m_output = 0; + std::string m_buildLog; + }; + } // namespace + + TEST_F(IterationRPScratchFixScenario, FixtureShapedReductionSumsEveryInvocation) { + const OutputBlock block = Dispatch(); + EXPECT_EQ(glGetError(), static_cast(GL_NO_ERROR)); + + // The topology diagnostics catch the failure modes by name before the sum does: + // an out-of-bounds handoff corrupts the total, a wrong gl_NumSubgroups breaks + // the combine loop's length. + ASSERT_NE(block.numSubgroups, 0xa5a5a5a5u) << "invocation 511 never reached its store"; + EXPECT_GE(block.numSubgroups, 1u); + EXPECT_LE(block.numSubgroups, kInvocationCount); + EXPECT_LT(block.maxSubgroupId, block.numSubgroups) + << "gl_SubgroupID exceeds gl_NumSubgroups - the inconsistency " + "DeriveNumSubgroupsPass exists to repair"; + + // Integer-valued fp32 inputs: the workgroup total is exact under any subgroup + // width, lane order, and association. This is the value iterationRP's exposure + // average is built from; without FixIterationRPSubgroupScratchPass an 8-lane + // device writes prefixSumCache[32..63] out of bounds and this comparison fails. + EXPECT_EQ(block.total, kExpectedTotal) + << "workgroup reduction produced " << block.total << " with gl_NumSubgroups=" + << block.numSubgroups; + } +} // namespace MGITest diff --git a/MobileGL/MG_Test/SelfTest/CMakeLists.txt b/MobileGL/MG_Test/SelfTest/CMakeLists.txt index 9beb3e27..3dd1e5c0 100644 --- a/MobileGL/MG_Test/SelfTest/CMakeLists.txt +++ b/MobileGL/MG_Test/SelfTest/CMakeLists.txt @@ -1,19 +1,19 @@ # MobileGL - MobileGL/MG_Test/SelfTest/CMakeLists.txt add_executable( - DriverPostProgram203WitnessTest - DriverPostProgram203WitnessTest.cpp + DriverPostIterationRPWitnessTest + DriverPostIterationRPWitnessTest.cpp ) -target_include_directories(DriverPostProgram203WitnessTest PRIVATE +target_include_directories(DriverPostIterationRPWitnessTest PRIVATE ${MGL_ROOT}/include ${MGL_ROOT}/MobileGL ) -target_link_libraries(DriverPostProgram203WitnessTest PRIVATE +target_link_libraries(DriverPostIterationRPWitnessTest PRIVATE GTest::gtest_main ${LINK_LIBRARIES} ) include(GoogleTest) -gtest_discover_tests(DriverPostProgram203WitnessTest DISCOVERY_TIMEOUT 30 PROPERTIES LABELS unit) +gtest_discover_tests(DriverPostIterationRPWitnessTest DISCOVERY_TIMEOUT 30 PROPERTIES LABELS unit) diff --git a/MobileGL/MG_Test/SelfTest/DriverPostProgram203WitnessTest.cpp b/MobileGL/MG_Test/SelfTest/DriverPostIterationRPWitnessTest.cpp similarity index 51% rename from MobileGL/MG_Test/SelfTest/DriverPostProgram203WitnessTest.cpp rename to MobileGL/MG_Test/SelfTest/DriverPostIterationRPWitnessTest.cpp index 888228a1..1cccd884 100644 --- a/MobileGL/MG_Test/SelfTest/DriverPostProgram203WitnessTest.cpp +++ b/MobileGL/MG_Test/SelfTest/DriverPostIterationRPWitnessTest.cpp @@ -1,4 +1,4 @@ -// MobileGL - MobileGL/MG_Test/SelfTest/DriverPostProgram203WitnessTest.cpp +// MobileGL - MobileGL/MG_Test/SelfTest/DriverPostIterationRPWitnessTest.cpp // Copyright (c) 2026 MobileGL-Dev // Licensed under the GNU Lesser General Public License v3.0: // https://www.gnu.org/licenses/gpl-3.0.txt @@ -10,21 +10,21 @@ #include -#include "MG_Util/SelfTest/DriverPostProgram203Witness.h" +#include "MG_Util/SelfTest/DriverPostIterationRPWitness.h" namespace MobileGL::MG_Util::SelfTest { namespace { - Program203WitnessOutput MakeValidWitness(std::uint32_t numSubgroups) { - Program203WitnessOutput output{}; - output.magic = kProgram203WitnessMagic; + IterationRPWitnessOutput MakeValidWitness(std::uint32_t numSubgroups) { + IterationRPWitnessOutput output{}; + output.magic = kIterationRPWitnessMagic; output.numSubgroups = numSubgroups; - output.loopLength = ComputeProgram203WitnessLoopLength(numSubgroups); + output.loopLength = ComputeIterationRPWitnessLoopLength(numSubgroups); output.seenSubgroupMask = - numSubgroups == kProgram203WitnessMaxSubgroups ? 0xffffffffu : (1u << numSubgroups) - 1u; + numSubgroups == kIterationRPWitnessMaxSubgroups ? 0xffffffffu : (1u << numSubgroups) - 1u; // Valid test layouts use equal contiguous groups of the indexed // 1..512 input. The compact witness only needs their independent sums. - const std::uint32_t subgroupSize = kProgram203WitnessInvocationCount / numSubgroups; + const std::uint32_t subgroupSize = kIterationRPWitnessInvocationCount / numSubgroups; for (std::uint32_t subgroup = 0u; subgroup < numSubgroups; ++subgroup) { const std::uint32_t first = subgroup * subgroupSize + 1u; const std::uint32_t last = first + subgroupSize - 1u; @@ -50,139 +50,139 @@ namespace MobileGL::MG_Util::SelfTest { return output; } - Program203WitnessLimits MakeSufficientLimits() { - Program203WitnessLimits limits; + IterationRPWitnessLimits MakeSufficientLimits() { + IterationRPWitnessLimits limits; limits.computeStageSupported = true; limits.basicSubgroupSupported = true; limits.arithmeticSubgroupSupported = true; limits.subgroupSize = 32u; - limits.maxComputeWorkGroupInvocations = kProgram203WitnessInvocationCount; + limits.maxComputeWorkGroupInvocations = kIterationRPWitnessInvocationCount; limits.maxComputeWorkGroupSize = {32u, 16u, 1u}; - limits.maxComputeSharedMemorySize = kProgram203WitnessSharedMemoryBytes; + limits.maxComputeSharedMemorySize = kIterationRPWitnessSharedMemoryBytes; limits.maxPerStageDescriptorStorageBuffers = 1u; limits.maxDescriptorSetStorageBuffers = 1u; limits.maxBoundDescriptorSets = 1u; - limits.maxStorageBufferRange = sizeof(Program203WitnessOutput); + limits.maxStorageBufferRange = sizeof(IterationRPWitnessOutput); return limits; } } // namespace - TEST(DriverPostProgram203WitnessTest, ValidTwoSubgroupWitness) { - const Program203WitnessValidationResult validation = ValidateProgram203Witness(MakeValidWitness(2u)); + TEST(DriverPostIterationRPWitnessTest, ValidTwoSubgroupWitness) { + const IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(MakeValidWitness(2u)); ASSERT_TRUE(validation.ok) << validation.detail; EXPECT_EQ(validation.detail, "N=2, owner511=id1/lane255, 2 scan stages, average=(256.5,0)"); } - TEST(DriverPostProgram203WitnessTest, ValidThirtyTwoSubgroupWitness) { - const Program203WitnessValidationResult validation = ValidateProgram203Witness(MakeValidWitness(32u)); + TEST(DriverPostIterationRPWitnessTest, ValidThirtyTwoSubgroupWitness) { + const IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(MakeValidWitness(32u)); ASSERT_TRUE(validation.ok) << validation.detail; EXPECT_EQ(validation.detail, "N=32, owner511=id31/lane15, 6 scan stages, average=(256.5,0)"); } - TEST(DriverPostProgram203WitnessTest, RejectsNonuniformNumSubgroups) { - Program203WitnessOutput output = MakeValidWitness(16u); - output.topologyFlags |= Program203WitnessNonuniformNumSubgroups; - const Program203WitnessValidationResult validation = ValidateProgram203Witness(output); + TEST(DriverPostIterationRPWitnessTest, RejectsNonuniformNumSubgroups) { + IterationRPWitnessOutput output = MakeValidWitness(16u); + output.topologyFlags |= IterationRPWitnessNonuniformNumSubgroups; + const IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(output); EXPECT_FALSE(validation.ok); - EXPECT_EQ(validation.failure, Program203WitnessValidationFailure::Topology); + EXPECT_EQ(validation.failure, IterationRPWitnessValidationFailure::Topology); EXPECT_NE(validation.detail.find("gl_NumSubgroups differed"), std::string::npos); } - TEST(DriverPostProgram203WitnessTest, RejectsMissingAndOutOfRangeSubgroupIds) { - Program203WitnessOutput missing = MakeValidWitness(16u); + TEST(DriverPostIterationRPWitnessTest, RejectsMissingAndOutOfRangeSubgroupIds) { + IterationRPWitnessOutput missing = MakeValidWitness(16u); missing.seenSubgroupMask &= ~(1u << 7u); - Program203WitnessValidationResult validation = ValidateProgram203Witness(missing); + IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(missing); EXPECT_FALSE(validation.ok); EXPECT_NE(validation.detail.find("seen subgroup-ID mask"), std::string::npos); - Program203WitnessOutput outOfRange = MakeValidWitness(16u); - outOfRange.topologyFlags |= Program203WitnessInvalidSubgroupId; - validation = ValidateProgram203Witness(outOfRange); + IterationRPWitnessOutput outOfRange = MakeValidWitness(16u); + outOfRange.topologyFlags |= IterationRPWitnessInvalidSubgroupId; + validation = ValidateIterationRPWitness(outOfRange); EXPECT_FALSE(validation.ok); EXPECT_NE(validation.detail.find("invalid gl_SubgroupID"), std::string::npos); } - TEST(DriverPostProgram203WitnessTest, RejectsInvalidMultipleAndMissingLastLaneWriters) { - Program203WitnessOutput invalidLane = MakeValidWitness(16u); - invalidLane.topologyFlags |= Program203WitnessInvalidSubgroupLane; - Program203WitnessValidationResult validation = ValidateProgram203Witness(invalidLane); + TEST(DriverPostIterationRPWitnessTest, RejectsInvalidMultipleAndMissingLastLaneWriters) { + IterationRPWitnessOutput invalidLane = MakeValidWitness(16u); + invalidLane.topologyFlags |= IterationRPWitnessInvalidSubgroupLane; + IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(invalidLane); EXPECT_FALSE(validation.ok); EXPECT_NE(validation.detail.find("invalid subgroup lane"), std::string::npos); - Program203WitnessOutput multiple = MakeValidWitness(16u); + IterationRPWitnessOutput multiple = MakeValidWitness(16u); multiple.lastLaneWriterCount[4] = 2u; - validation = ValidateProgram203Witness(multiple); + validation = ValidateIterationRPWitness(multiple); EXPECT_FALSE(validation.ok); EXPECT_NE(validation.detail.find("subgroup 4 has 2 source last-lane writers"), std::string::npos); - Program203WitnessOutput missing = MakeValidWitness(16u); + IterationRPWitnessOutput missing = MakeValidWitness(16u); missing.lastLaneWriterCount[6] = 0u; - validation = ValidateProgram203Witness(missing); + validation = ValidateIterationRPWitness(missing); EXPECT_FALSE(validation.ok); EXPECT_NE(validation.detail.find("subgroup 6 has 0 source last-lane writers"), std::string::npos); } - TEST(DriverPostProgram203WitnessTest, ReportsEarliestCorruptSourceScanStage) { - Program203WitnessOutput output = MakeValidWitness(32u); + TEST(DriverPostIterationRPWitnessTest, ReportsEarliestCorruptSourceScanStage) { + IterationRPWitnessOutput output = MakeValidWitness(32u); output.scanCache[0][1].x += 1.0f; output.scanCache[3][5].x += 1.0f; - Program203WitnessValidationResult validation = ValidateProgram203Witness(output); + IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(output); EXPECT_FALSE(validation.ok); - EXPECT_EQ(validation.failure, Program203WitnessValidationFailure::SourceScan); + EXPECT_EQ(validation.failure, IterationRPWitnessValidationFailure::SourceScan); EXPECT_EQ(validation.scanStage, 0u); EXPECT_NE(validation.detail.find("source scan stage 0, subgroup 1"), std::string::npos); output = MakeValidWitness(32u); output.scanCache[3][5].x += 1.0f; - validation = ValidateProgram203Witness(output); + validation = ValidateIterationRPWitness(output); EXPECT_FALSE(validation.ok); - EXPECT_EQ(validation.failure, Program203WitnessValidationFailure::SourceScan); + EXPECT_EQ(validation.failure, IterationRPWitnessValidationFailure::SourceScan); EXPECT_EQ(validation.scanStage, 3u); EXPECT_NE(validation.detail.find("source scan stage 3, subgroup 5"), std::string::npos); } - TEST(DriverPostProgram203WitnessTest, RejectsOwner511OutsideHighestFinalLane) { - Program203WitnessOutput output = MakeValidWitness(16u); + TEST(DriverPostIterationRPWitnessTest, RejectsOwner511OutsideHighestFinalLane) { + IterationRPWitnessOutput output = MakeValidWitness(16u); output.owner511.z = 14u; - const Program203WitnessValidationResult validation = ValidateProgram203Witness(output); + const IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(output); EXPECT_FALSE(validation.ok); - EXPECT_EQ(validation.failure, Program203WitnessValidationFailure::FinalOwner); + EXPECT_EQ(validation.failure, IterationRPWitnessValidationFailure::FinalOwner); EXPECT_NE(validation.detail.find("not in the highest subgroup"), std::string::npos); } - TEST(DriverPostProgram203WitnessTest, RejectsIncorrectVectorFinalAverage) { - Program203WitnessOutput output = MakeValidWitness(16u); + TEST(DriverPostIterationRPWitnessTest, RejectsIncorrectVectorFinalAverage) { + IterationRPWitnessOutput output = MakeValidWitness(16u); output.finalAverage.y = 1.0f; - const Program203WitnessValidationResult validation = ValidateProgram203Witness(output); + const IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(output); EXPECT_FALSE(validation.ok); - EXPECT_EQ(validation.failure, Program203WitnessValidationFailure::FinalAverage); + EXPECT_EQ(validation.failure, IterationRPWitnessValidationFailure::FinalAverage); EXPECT_NE(validation.detail.find("final average"), std::string::npos); } - TEST(DriverPostProgram203WitnessTest, MissingNativeFeatureIsTheOnlySkipCondition) { + TEST(DriverPostIterationRPWitnessTest, MissingNativeFeatureIsTheOnlySkipCondition) { for (const auto toggleMissingFeature : {0u, 1u, 2u}) { - Program203WitnessLimits limits = MakeSufficientLimits(); + IterationRPWitnessLimits limits = MakeSufficientLimits(); if (toggleMissingFeature == 0u) limits.computeStageSupported = false; if (toggleMissingFeature == 1u) limits.basicSubgroupSupported = false; if (toggleMissingFeature == 2u) limits.arithmeticSubgroupSupported = false; - const Program203WitnessEligibilityResult eligibility = EvaluateProgram203WitnessEligibility(limits); - EXPECT_EQ(eligibility.eligibility, Program203WitnessEligibility::SkipUnsupportedNativeFeatureSet) + const IterationRPWitnessEligibilityResult eligibility = EvaluateIterationRPWitnessEligibility(limits); + EXPECT_EQ(eligibility.eligibility, IterationRPWitnessEligibility::SkipUnsupportedNativeFeatureSet) << eligibility.detail; } - Program203WitnessLimits zeroSubgroupSize = MakeSufficientLimits(); + IterationRPWitnessLimits zeroSubgroupSize = MakeSufficientLimits(); zeroSubgroupSize.subgroupSize = 0u; - Program203WitnessEligibilityResult eligibility = EvaluateProgram203WitnessEligibility(zeroSubgroupSize); - EXPECT_EQ(eligibility.eligibility, Program203WitnessEligibility::FailInadequateLimits) << eligibility.detail; + IterationRPWitnessEligibilityResult eligibility = EvaluateIterationRPWitnessEligibility(zeroSubgroupSize); + EXPECT_EQ(eligibility.eligibility, IterationRPWitnessEligibility::FailInadequateLimits) << eligibility.detail; - Program203WitnessLimits limits = MakeSufficientLimits(); + IterationRPWitnessLimits limits = MakeSufficientLimits(); limits.maxComputeWorkGroupInvocations = 511u; - eligibility = EvaluateProgram203WitnessEligibility(limits); - EXPECT_EQ(eligibility.eligibility, Program203WitnessEligibility::FailInadequateLimits) << eligibility.detail; + eligibility = EvaluateIterationRPWitnessEligibility(limits); + EXPECT_EQ(eligibility.eligibility, IterationRPWitnessEligibility::FailInadequateLimits) << eligibility.detail; limits = MakeSufficientLimits(); - limits.maxStorageBufferRange = sizeof(Program203WitnessOutput) - 1u; - eligibility = EvaluateProgram203WitnessEligibility(limits); - EXPECT_EQ(eligibility.eligibility, Program203WitnessEligibility::FailInadequateLimits) << eligibility.detail; + limits.maxStorageBufferRange = sizeof(IterationRPWitnessOutput) - 1u; + eligibility = EvaluateIterationRPWitnessEligibility(limits); + EXPECT_EQ(eligibility.eligibility, IterationRPWitnessEligibility::FailInadequateLimits) << eligibility.detail; } } // namespace MobileGL::MG_Util::SelfTest diff --git a/MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt b/MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt index 91db71dd..9f16aa6f 100644 --- a/MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt +++ b/MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt @@ -4,6 +4,8 @@ add_executable( SpirvPassTest SpirvPassTest.cpp DeriveNumSubgroupsTest.cpp + FixIterationRPSubgroupScratchTest.cpp + EmulateSubgroupsTest.cpp DemoteFloat64Test.cpp FlattenXfbInterfaceBlocksTest.cpp ) diff --git a/MobileGL/MG_Test/ShaderTranspiler/EmulateSubgroupsTest.cpp b/MobileGL/MG_Test/ShaderTranspiler/EmulateSubgroupsTest.cpp new file mode 100644 index 00000000..cfddb3df --- /dev/null +++ b/MobileGL/MG_Test/ShaderTranspiler/EmulateSubgroupsTest.cpp @@ -0,0 +1,248 @@ +// MobileGL - MobileGL/MG_Test/ShaderTranspiler/EmulateSubgroupsTest.cpp +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#include + +#define SPV_ENABLE_UTILITY_CODE +#include "glslang/SPIRV/spirv.hpp11" +#undef SPV_ENABLE_UTILITY_CODE + +#include "Includes.h" +#include +#include + +#include + +using namespace MobileGL; +using MobileGL::MG_Util::ShaderTranspiler::ShaderCompiler; + +namespace { + constexpr SizeT kSpirvHeaderWordCount = 5u; + + template + void ForEachInstruction(const Vector& spirv, Visitor&& visit) { + for (SizeT offset = kSpirvHeaderWordCount; offset < spirv.size();) { + const Uint32 wordCount = spirv[offset] >> 16u; + if (wordCount == 0u || offset + wordCount > spirv.size()) break; + visit(static_cast(spirv[offset] & 0xffffu), &spirv[offset], wordCount); + offset += wordCount; + } + } + + Vector CompileStage(GLenum stage, const String& source) { + using namespace MobileGL::MG_Util::ShaderTranspiler; + ShaderAttrib shaderAttrib{.shaderType = stage, .sourceStr = source}; + auto shaderResult = ShaderCompiler::CompileShader(shaderAttrib); + EXPECT_TRUE(shaderResult) << (shaderResult ? String{} : shaderResult.error().log); + if (!shaderResult) return {}; + + ProgramAttrib programAttrib{.shaders = {shaderResult.value()}}; + auto programResult = ShaderCompiler::LinkProgram(programAttrib); + EXPECT_TRUE(programResult) << (programResult ? String{} : programResult.error().log); + if (!programResult) return {}; + + ProgramBinaryAttrib binaryAttrib{.shaderTypes = {stage}, .program = *programResult.value()}; + auto binaryResult = ShaderCompiler::GetSpirvBinaryFromProgram(binaryAttrib); + EXPECT_TRUE(binaryResult) << (binaryResult ? String{} : binaryResult.error().log); + if (!binaryResult || binaryResult->empty()) return {}; + return binaryResult->front(); + } + + Uint32 CountGroupNonUniform(const Vector& spirv) { + Uint32 count = 0; + ForEachInstruction(spirv, [&](spv::Op opcode, const Uint32*, Uint32) { + if (opcode >= spv::Op::OpGroupNonUniformElect && opcode <= spv::Op::OpGroupNonUniformQuadSwap) { + ++count; + } + }); + return count; + } + + Uint32 CountGroupNonUniformCapabilities(const Vector& spirv) { + Uint32 count = 0; + ForEachInstruction(spirv, [&](spv::Op opcode, const Uint32* words, Uint32 wordCount) { + if (opcode != spv::Op::OpCapability || wordCount < 2u) return; + const auto capability = static_cast(words[1]); + if (capability >= spv::Capability::GroupNonUniform && + capability <= spv::Capability::GroupNonUniformQuad) { + ++count; + } + }); + return count; + } + + Uint32 CountOpcode(const Vector& spirv, spv::Op wanted) { + Uint32 count = 0; + ForEachInstruction(spirv, [&](spv::Op opcode, const Uint32*, Uint32) { + if (opcode == wanted) ++count; + }); + return count; + } + + bool HasWorkgroupVariable(const Vector& spirv) { + bool found = false; + ForEachInstruction(spirv, [&](spv::Op opcode, const Uint32* words, Uint32 wordCount) { + if (opcode == spv::Op::OpVariable && wordCount >= 4u && + static_cast(words[3]) == spv::StorageClass::Workgroup) { + found = true; + } + }); + return found; + } + + bool Validates(const Vector& spirv) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_1); + tools.SetMessageConsumer([](spv_message_level_t, const char*, const spv_position_t& position, + const char* message) { + ADD_FAILURE() << "spirv-val at word " << position.index << ": " << message; + }); + return tools.Validate(spirv); + } + + // One shader touching every lowered category: builtins, vote, arithmetic + // scans, ballot math, shuffles, clustered and quad operations. + constexpr const char* kEveryCategorySource = R"(#version 450 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_vote : require +#extension GL_KHR_shader_subgroup_arithmetic : require +#extension GL_KHR_shader_subgroup_ballot : require +#extension GL_KHR_shader_subgroup_shuffle : require +#extension GL_KHR_shader_subgroup_shuffle_relative : require +#extension GL_KHR_shader_subgroup_clustered : require +#extension GL_KHR_shader_subgroup_quad : require +layout(local_size_x = 48, local_size_y = 1, local_size_z = 1) in; +layout(std430, binding = 0) buffer Output { float value[]; } outputData; +void main() { + uint slot = gl_LocalInvocationIndex * 24u; + float v = float(gl_LocalInvocationIndex + 1u); + outputData.value[slot + 0u] = float(gl_SubgroupSize); + outputData.value[slot + 1u] = float(gl_NumSubgroups); + outputData.value[slot + 2u] = float(gl_SubgroupID); + outputData.value[slot + 3u] = float(gl_SubgroupInvocationID); + outputData.value[slot + 4u] = float(gl_SubgroupEqMask.x + gl_SubgroupLtMask.x); + outputData.value[slot + 5u] = subgroupElect() ? 1.0 : 0.0; + outputData.value[slot + 6u] = subgroupAll(v > 0.0) ? 1.0 : 0.0; + outputData.value[slot + 7u] = subgroupAny(v > 40.0) ? 1.0 : 0.0; + outputData.value[slot + 8u] = subgroupAllEqual(gl_WorkGroupID.x) ? 1.0 : 0.0; + outputData.value[slot + 9u] = subgroupAdd(v); + outputData.value[slot + 10u] = subgroupInclusiveAdd(v); + outputData.value[slot + 11u] = subgroupExclusiveMax(v); + outputData.value[slot + 12u] = float(subgroupMin(gl_LocalInvocationIndex)); + uvec4 ballot = subgroupBallot((gl_LocalInvocationIndex & 1u) == 0u); + outputData.value[slot + 13u] = float(subgroupBallotBitCount(ballot)); + outputData.value[slot + 14u] = float(subgroupBallotFindLSB(ballot)); + outputData.value[slot + 15u] = float(subgroupBallotFindMSB(ballot)); + outputData.value[slot + 16u] = subgroupInverseBallot(ballot) ? 1.0 : 0.0; + outputData.value[slot + 17u] = subgroupBallotBitExtract(ballot, 3u) ? 1.0 : 0.0; + outputData.value[slot + 18u] = subgroupBroadcast(v, 2u); + outputData.value[slot + 19u] = subgroupBroadcastFirst(v); + outputData.value[slot + 20u] = subgroupShuffle(v, gl_SubgroupInvocationID ^ 5u); + outputData.value[slot + 21u] = subgroupShuffleXor(v, 1u) + subgroupShuffleUp(v, 1u) + + subgroupShuffleDown(v, 1u); + outputData.value[slot + 22u] = subgroupClusteredAdd(v, 4u); + outputData.value[slot + 23u] = subgroupQuadBroadcast(v, 1u) + subgroupQuadSwapHorizontal(v); + subgroupBarrier(); + subgroupMemoryBarrierShared(); +} +)"; + + constexpr const char* kNoSubgroupSource = R"(#version 450 core +layout(local_size_x = 64) in; +layout(std430, binding = 0) buffer Output { uint value; } outputData; +void main() { + if (gl_LocalInvocationIndex == 0u) outputData.value = gl_WorkGroupSize.x; +} +)"; + + // An extended subgroup instruction (SPV_KHR_subgroup_rotate) alongside core + // ones: outside the lowered set, so the pass must fail rather than emit + // "subgroup-free" output that still rotates. + constexpr const char* kRotateSource = R"(#version 450 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +#extension GL_KHR_shader_subgroup_rotate : require +layout(local_size_x = 64) in; +layout(std430, binding = 0) buffer Output { float value[]; } outputData; +void main() { + float v = subgroupAdd(float(gl_SubgroupInvocationID)); + outputData.value[gl_LocalInvocationIndex] = subgroupRotate(v, 1u); +} +)"; + + // A 1024-invocation workgroup exchanging a vec4 and a float: the lowering + // would need 16 KiB + 4 KiB of scratch, past the Vulkan-minimum shared + // budget of 16384 bytes. + constexpr const char* kScratchHungrySource = R"(#version 450 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +layout(local_size_x = 1024) in; +layout(std430, binding = 0) buffer Output { vec4 value[]; } outputData; +void main() { + vec4 wide = subgroupAdd(vec4(float(gl_LocalInvocationIndex))); + wide.x += subgroupInclusiveAdd(float(gl_SubgroupInvocationID)); + outputData.value[gl_LocalInvocationIndex] = wide; +} +)"; +} // namespace + +TEST(EmulateSubgroupsPass, LowersEveryCategoryToSharedMemory) { + const Vector input = CompileStage(GL_COMPUTE_SHADER, kEveryCategorySource); + ASSERT_FALSE(input.empty()); + ASSERT_GT(CountGroupNonUniform(input), 0u); + ASSERT_GT(CountGroupNonUniformCapabilities(input), 0u); + + Vector output; + ASSERT_TRUE(ShaderCompiler::EmulateSubgroupsForVulkan(input, output, 16384u, true)); + ASSERT_TRUE(Validates(output)); + + // The whole point: nothing subgroup-shaped survives, so the module runs on a + // device with no subgroup support at all. + EXPECT_EQ(CountGroupNonUniform(output), 0u); + EXPECT_EQ(CountGroupNonUniformCapabilities(output), 0u); + // The exchanges go through workgroup-shared scratch behind control barriers. + EXPECT_TRUE(HasWorkgroupVariable(output)); + EXPECT_GT(CountOpcode(output, spv::Op::OpControlBarrier), CountOpcode(input, spv::Op::OpControlBarrier)); +} + +TEST(EmulateSubgroupsPass, IsIdempotent) { + const Vector input = CompileStage(GL_COMPUTE_SHADER, kEveryCategorySource); + ASSERT_FALSE(input.empty()); + Vector once; + ASSERT_TRUE(ShaderCompiler::EmulateSubgroupsForVulkan(input, once, 16384u, true)); + Vector twice; + ASSERT_TRUE(ShaderCompiler::EmulateSubgroupsForVulkan(once, twice, 16384u, true)); + EXPECT_EQ(twice, once); +} + +TEST(EmulateSubgroupsPass, LeavesSubgroupFreeComputeUntouched) { + const Vector input = CompileStage(GL_COMPUTE_SHADER, kNoSubgroupSource); + ASSERT_FALSE(input.empty()); + Vector output; + ASSERT_TRUE(ShaderCompiler::EmulateSubgroupsForVulkan(input, output, 16384u, true)); + EXPECT_EQ(output, input); +} + +TEST(EmulateSubgroupsPass, RefusesExtendedSubgroupInstructions) { + const Vector input = CompileStage(GL_COMPUTE_SHADER, kRotateSource); + ASSERT_FALSE(input.empty()); + Vector output; + EXPECT_FALSE(ShaderCompiler::EmulateSubgroupsForVulkan(input, output, 16384u, false)); +} + +TEST(EmulateSubgroupsPass, RefusesAModuleOverTheScratchBudget) { + const Vector input = CompileStage(GL_COMPUTE_SHADER, kScratchHungrySource); + ASSERT_FALSE(input.empty()); + // vec4 scratch (1024 slots * 16 bytes) plus float scratch (4 KiB) exceeds + // the 16 KiB Vulkan-minimum budget. + Vector output; + EXPECT_FALSE(ShaderCompiler::EmulateSubgroupsForVulkan(input, output, 16384u, false)); + // A device advertising more shared memory takes the same module fine. + Vector roomier; + EXPECT_TRUE(ShaderCompiler::EmulateSubgroupsForVulkan(input, roomier, 32768u, true)); + EXPECT_TRUE(Validates(roomier)); +} diff --git a/MobileGL/MG_Test/ShaderTranspiler/FixIterationRPSubgroupScratchTest.cpp b/MobileGL/MG_Test/ShaderTranspiler/FixIterationRPSubgroupScratchTest.cpp new file mode 100644 index 00000000..195836a4 --- /dev/null +++ b/MobileGL/MG_Test/ShaderTranspiler/FixIterationRPSubgroupScratchTest.cpp @@ -0,0 +1,225 @@ +// MobileGL - MobileGL/MG_Test/ShaderTranspiler/FixIterationRPSubgroupScratchTest.cpp +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#include + +#define SPV_ENABLE_UTILITY_CODE +#include "glslang/SPIRV/spirv.hpp11" +#undef SPV_ENABLE_UTILITY_CODE + +#include "Includes.h" +#include +#include + +#include + +#include +#include +#include + +using namespace MobileGL; +using MobileGL::MG_Util::ShaderTranspiler::ShaderCompiler; + +namespace { + constexpr SizeT kSpirvHeaderWordCount = 5u; + + template + void ForEachInstruction(const Vector& spirv, Visitor&& visit) { + for (SizeT offset = kSpirvHeaderWordCount; offset < spirv.size();) { + const Uint32 wordCount = spirv[offset] >> 16u; + if (wordCount == 0u || offset + wordCount > spirv.size()) break; + visit(static_cast(spirv[offset] & 0xffffu), &spirv[offset], wordCount); + offset += wordCount; + } + } + + Vector CompileCompute(const String& source) { + using namespace MobileGL::MG_Util::ShaderTranspiler; + ShaderAttrib shaderAttrib{.shaderType = GL_COMPUTE_SHADER, .sourceStr = source}; + auto shaderResult = ShaderCompiler::CompileShader(shaderAttrib); + EXPECT_TRUE(shaderResult) << (shaderResult ? String{} : shaderResult.error().log); + if (!shaderResult) return {}; + + ProgramAttrib programAttrib{.shaders = {shaderResult.value()}}; + auto programResult = ShaderCompiler::LinkProgram(programAttrib); + EXPECT_TRUE(programResult) << (programResult ? String{} : programResult.error().log); + if (!programResult) return {}; + + ProgramBinaryAttrib binaryAttrib{.shaderTypes = {GL_COMPUTE_SHADER}, .program = *programResult.value()}; + auto binaryResult = ShaderCompiler::GetSpirvBinaryFromProgram(binaryAttrib); + EXPECT_TRUE(binaryResult) << (binaryResult ? String{} : binaryResult.error().log); + if (!binaryResult || binaryResult->empty()) return {}; + return binaryResult->front(); + } + + // The declared lengths of every Workgroup-storage array variable, sorted. + std::vector WorkgroupArrayLengths(const Vector& spirv) { + std::map constantValues; // constant id -> value + std::map arrayLengthIds; // array type id -> length constant id + std::map pointerPointees; // pointer type id -> pointee type id + std::vector workgroupPointerTypes; // type ids of Workgroup variables + ForEachInstruction(spirv, [&](spv::Op opcode, const Uint32* words, Uint32 wordCount) { + switch (opcode) { + case spv::Op::OpConstant: + if (wordCount >= 4u) constantValues[words[2]] = words[3]; + break; + case spv::Op::OpTypeArray: + if (wordCount >= 4u) arrayLengthIds[words[1]] = words[3]; + break; + case spv::Op::OpTypePointer: + if (wordCount >= 4u && + static_cast(words[2]) == spv::StorageClass::Workgroup) { + pointerPointees[words[1]] = words[3]; + } + break; + case spv::Op::OpVariable: + if (wordCount >= 4u && + static_cast(words[3]) == spv::StorageClass::Workgroup) { + workgroupPointerTypes.push_back(words[1]); + } + break; + default: + break; + } + }); + std::vector lengths; + for (const Uint32 pointerTypeId : workgroupPointerTypes) { + const auto pointee = pointerPointees.find(pointerTypeId); + if (pointee == pointerPointees.end()) continue; + const auto lengthId = arrayLengthIds.find(pointee->second); + if (lengthId == arrayLengthIds.end()) continue; + const auto value = constantValues.find(lengthId->second); + if (value != constantValues.end()) lengths.push_back(value->second); + } + std::sort(lengths.begin(), lengths.end()); + return lengths; + } + + bool Validates(const Vector& spirv) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_1); + tools.SetMessageConsumer([](spv_message_level_t, const char*, const spv_position_t& position, + const char* message) { + ADD_FAILURE() << "spirv-val at word " << position.index << ": " << message; + }); + return tools.Validate(spirv); + } + + // iterationRP's reduction fingerprint: 32x16x1, subgroupInclusiveAdd on a + // vec2, and the pack's own 32-entry gl_SubgroupID-indexed scratch. A second, + // plainly indexed array rides along to prove the patch is surgical. + constexpr const char* kIterationRPShapedSource = R"(#version 450 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +layout(local_size_x = 32, local_size_y = 16, local_size_z = 1) in; +layout(std430, binding = 0) buffer Output { float value; } outputData; +shared vec2 prefixSumCache[32]; +shared float plainScratch[4]; +void main() { + vec2 sampleLuminance = vec2(float(gl_LocalInvocationIndex), 0.0); + sampleLuminance = subgroupInclusiveAdd(sampleLuminance); + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) + prefixSumCache[gl_SubgroupID] = sampleLuminance; + plainScratch[gl_LocalInvocationIndex & 3u] = sampleLuminance.x; + barrier(); + uint loopLength = uint(findMSB(gl_NumSubgroups)); + loopLength += uint(gl_NumSubgroups - (1u << (loopLength - 1u)) > 0u); + for (uint scanStage = 0u; scanStage < loopLength; ++scanStage) { + if ((gl_SubgroupID & (1u << scanStage)) > 0u) { + sampleLuminance += prefixSumCache[(gl_SubgroupID >> scanStage << scanStage) - 1u]; + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) + prefixSumCache[gl_SubgroupID] = sampleLuminance; + } + barrier(); + } + if (gl_LocalInvocationIndex == 511u) + outputData.value = prefixSumCache[0].x / 512.0 + plainScratch[0]; +} +)"; + + // Same scratch idiom, different workgroup shape - NOT iterationRP, so the + // fingerprint must refuse it even though it would break identically. + constexpr const char* kWrongWorkgroupShapeSource = R"(#version 450 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +layout(local_size_x = 64, local_size_y = 8, local_size_z = 1) in; +layout(std430, binding = 0) buffer Output { float value; } outputData; +shared vec2 prefixSumCache[32]; +void main() { + vec2 v = subgroupInclusiveAdd(vec2(1.0, 0.0)); + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) + prefixSumCache[gl_SubgroupID] = v; + barrier(); + if (gl_LocalInvocationIndex == 0u) + outputData.value = prefixSumCache[0].x; +} +)"; + + // Right shape, but a float scan and a float[32] scratch - not the pack's + // vec2 accumulator signature. + constexpr const char* kWrongElementTypeSource = R"(#version 450 core +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +layout(local_size_x = 32, local_size_y = 16, local_size_z = 1) in; +layout(std430, binding = 0) buffer Output { float value; } outputData; +shared float cache[32]; +void main() { + float v = subgroupInclusiveAdd(float(gl_LocalInvocationIndex)); + if (gl_SubgroupInvocationID == gl_SubgroupSize - 1u) + cache[gl_SubgroupID] = v; + barrier(); + if (gl_LocalInvocationIndex == 0u) + outputData.value = cache[0]; +} +)"; +} // namespace + +TEST(FixIterationRPSubgroupScratchPass, GrowsThePacksScratchForNarrowSubgroups) { + const Vector input = CompileCompute(kIterationRPShapedSource); + ASSERT_FALSE(input.empty()); + ASSERT_EQ(WorkgroupArrayLengths(input), (std::vector{4u, 32u})); + + // lavapipe: 8-lane subgroups over 512 invocations need 64 entries; the + // plainly indexed neighbour must keep its 4. + Vector output; + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, output, 8u, true)); + EXPECT_EQ(WorkgroupArrayLengths(output), (std::vector{4u, 64u})); + EXPECT_TRUE(Validates(output)); +} + +TEST(FixIterationRPSubgroupScratchPass, LeavesPackWidthAssumptionsAloneOnWideDevices) { + const Vector input = CompileCompute(kIterationRPShapedSource); + ASSERT_FALSE(input.empty()); + + // >= 16 lanes means at most 32 subgroups: the pack's declared size holds and + // the module must pass through byte-identical. + for (const Uint32 nativeSize : {16u, 32u, 64u, 128u}) { + Vector output; + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, output, nativeSize, true)); + EXPECT_EQ(output, input) << "native width " << nativeSize; + } +} + +TEST(FixIterationRPSubgroupScratchPass, RefusesAModuleOutsideTheFingerprint) { + for (const char* source : {kWrongWorkgroupShapeSource, kWrongElementTypeSource}) { + const Vector input = CompileCompute(source); + ASSERT_FALSE(input.empty()); + Vector output; + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, output, 8u, true)); + EXPECT_EQ(output, input); + } +} + +TEST(FixIterationRPSubgroupScratchPass, IsIdempotent) { + const Vector input = CompileCompute(kIterationRPShapedSource); + ASSERT_FALSE(input.empty()); + Vector once; + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(input, once, 8u, true)); + Vector twice; + ASSERT_TRUE(ShaderCompiler::FixIterationRPSubgroupScratchForVulkan(once, twice, 8u, true)); + EXPECT_EQ(twice, once); +} diff --git a/MobileGL/MG_Util/SelfTest/DriverPost.cpp b/MobileGL/MG_Util/SelfTest/DriverPost.cpp index 8b1070ed..4cd7c41d 100644 --- a/MobileGL/MG_Util/SelfTest/DriverPost.cpp +++ b/MobileGL/MG_Util/SelfTest/DriverPost.cpp @@ -7,8 +7,8 @@ // End of Source File Header #include "DriverPost.h" -#include "DriverPostProgram203Witness.h" -#include "DriverPostProgram203WitnessSpv.h" +#include "DriverPostIterationRPWitness.h" +#include "DriverPostIterationRPWitnessSpv.h" #include "MG_Util/BackendLoaders/OpenGL/Loader.h" #include #include @@ -1458,11 +1458,11 @@ namespace MobileGL::MG_Util::SelfTest { disabledNote); } - // Native Program-203 compute witness. This deliberately uses a separate + // Native iterationRP compute witness. This deliberately uses a separate // throwaway Vulkan device rather than the real renderer's queues, and it // treats MOBILEGL_DISABLE_SUBGROUP as irrelevant: the row reports what the // driver does, not what MobileGL elects to advertise to applications. - void ProbeVulkanProgram203Witness(ReportBuilder& builder, PFN_vkGetInstanceProcAddr getInstanceProcAddr, + void ProbeVulkanIterationRPWitness(ReportBuilder& builder, PFN_vkGetInstanceProcAddr getInstanceProcAddr, VkInstance instance, VkPhysicalDevice physicalDevice, Uint32 computeQueueFamilyIndex, const VkPhysicalDeviceProperties& properties, @@ -1476,7 +1476,7 @@ namespace MobileGL::MG_Util::SelfTest { return; } - Program203WitnessLimits limits{}; + IterationRPWitnessLimits limits{}; limits.computeStageSupported = (subgroupProperties.supportedStages & VK_SHADER_STAGE_COMPUTE_BIT) != 0; limits.basicSubgroupSupported = @@ -1494,12 +1494,12 @@ namespace MobileGL::MG_Util::SelfTest { limits.maxBoundDescriptorSets = properties.limits.maxBoundDescriptorSets; limits.maxStorageBufferRange = properties.limits.maxStorageBufferRange; - const Program203WitnessEligibilityResult eligibility = EvaluateProgram203WitnessEligibility(limits); - if (eligibility.eligibility == Program203WitnessEligibility::SkipUnsupportedNativeFeatureSet) { + const IterationRPWitnessEligibilityResult eligibility = EvaluateIterationRPWitnessEligibility(limits); + if (eligibility.eligibility == IterationRPWitnessEligibility::SkipUnsupportedNativeFeatureSet) { builder.Info(RowName, eligibility.detail); return; } - if (eligibility.eligibility == Program203WitnessEligibility::FailInadequateLimits) { + if (eligibility.eligibility == IterationRPWitnessEligibility::FailInadequateLimits) { fail(eligibility.detail); return; } @@ -1668,7 +1668,7 @@ namespace MobileGL::MG_Util::SelfTest { VkBufferCreateInfo bufferInfo{}; bufferInfo.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; - bufferInfo.size = sizeof(Program203WitnessOutput); + bufferInfo.size = sizeof(IterationRPWitnessOutput); bufferInfo.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; result = vkCreateBufferFn(device, &bufferInfo, nullptr, &outputBuffer); @@ -1710,12 +1710,12 @@ namespace MobileGL::MG_Util::SelfTest { fail(format("vkBindBufferMemory(output SSBO) failed (VkResult = {})", static_cast(result))); return; } - result = vkMapMemoryFn(device, outputMemory, 0, sizeof(Program203WitnessOutput), 0, &mappedOutput); + result = vkMapMemoryFn(device, outputMemory, 0, sizeof(IterationRPWitnessOutput), 0, &mappedOutput); if (result != VK_SUCCESS || mappedOutput == nullptr) { fail(format("vkMapMemory(output SSBO) failed (VkResult = {})", static_cast(result))); return; } - std::memset(mappedOutput, 0xa5, sizeof(Program203WitnessOutput)); + std::memset(mappedOutput, 0xa5, sizeof(IterationRPWitnessOutput)); VkDescriptorSetLayoutBinding outputBinding{}; outputBinding.binding = 0; @@ -1760,7 +1760,7 @@ namespace MobileGL::MG_Util::SelfTest { VkDescriptorBufferInfo outputDescriptor{}; outputDescriptor.buffer = outputBuffer; outputDescriptor.offset = 0; - outputDescriptor.range = sizeof(Program203WitnessOutput); + outputDescriptor.range = sizeof(IterationRPWitnessOutput); VkWriteDescriptorSet descriptorWrite{}; descriptorWrite.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; descriptorWrite.dstSet = descriptorSet; @@ -1772,8 +1772,8 @@ namespace MobileGL::MG_Util::SelfTest { VkShaderModuleCreateInfo shaderModuleInfo{}; shaderModuleInfo.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO; - shaderModuleInfo.codeSize = sizeof(kDriverPostProgram203WitnessSpv); - shaderModuleInfo.pCode = kDriverPostProgram203WitnessSpv; + shaderModuleInfo.codeSize = sizeof(kDriverPostIterationRPWitnessSpv); + shaderModuleInfo.pCode = kDriverPostIterationRPWitnessSpv; result = vkCreateShaderModuleFn(device, &shaderModuleInfo, nullptr, &shaderModule); if (result != VK_SUCCESS) { fail(format("vkCreateShaderModule failed (VkResult = {})", static_cast(result))); @@ -1845,7 +1845,7 @@ namespace MobileGL::MG_Util::SelfTest { hostReadBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; hostReadBarrier.buffer = outputBuffer; hostReadBarrier.offset = 0; - hostReadBarrier.size = sizeof(Program203WitnessOutput); + hostReadBarrier.size = sizeof(IterationRPWitnessOutput); vkCmdPipelineBarrierFn(commandBuffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 0, nullptr, 1, &hostReadBarrier, 0, nullptr); result = vkEndCommandBufferFn(commandBuffer); @@ -1879,9 +1879,9 @@ namespace MobileGL::MG_Util::SelfTest { return; } - Program203WitnessOutput output{}; + IterationRPWitnessOutput output{}; std::memcpy(&output, mappedOutput, sizeof(output)); - const Program203WitnessValidationResult validation = ValidateProgram203Witness(output); + const IterationRPWitnessValidationResult validation = ValidateIterationRPWitness(output); if (!validation.ok) { fail(validation.detail); return; @@ -2491,7 +2491,7 @@ namespace MobileGL::MG_Util::SelfTest { builder.Warn("Compute shader subgroup", "subgroup properties could not be queried"); } - ProbeVulkanProgram203Witness(builder, getInstanceProcAddr, instance, physicalDevice, computeQueueFamilyIndex, + ProbeVulkanIterationRPWitness(builder, getInstanceProcAddr, instance, physicalDevice, computeQueueFamilyIndex, properties, subgroupPropertiesAvailable, subgroupProperties); if (HasVkExtension(deviceExtensions, VK_KHR_DRAW_INDIRECT_COUNT_EXTENSION_NAME)) { diff --git a/MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.comp b/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.comp similarity index 94% rename from MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.comp rename to MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.comp index 2576228c..3e9d95c8 100644 --- a/MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.comp +++ b/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.comp @@ -1,4 +1,4 @@ -// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.comp +// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.comp // Copyright (c) 2026 MobileGL-Dev // Licensed under the GNU Lesser General Public License v3.0: // https://www.gnu.org/licenses/gpl-3.0.txt @@ -6,9 +6,9 @@ // SPDX-License-Identifier: LGPL-3.0-only // End of Source File Header // -// Native Vulkan GLSL 450 witness for Program 203's first subgroup reduction. +// Native Vulkan GLSL 450 witness for iterationRP's first subgroup reduction. // It is intentionally independent of the GL 430 integration scenario. The body -// below preserves Program 203's source reduction; the surrounding diagnostics +// below preserves iterationRP's source reduction; the surrounding diagnostics // only observe its topology and cache handoffs. #version 450 @@ -23,7 +23,7 @@ const uint kTopologyInvalidSubgroupId = 1u << 2u; const uint kTopologyInvalidSubgroupLane = 1u << 3u; const uint kWitnessMagic = 0x50323033u; -layout(std430, set = 0, binding = 0) buffer Program203WitnessOutput { +layout(std430, set = 0, binding = 0) buffer IterationRPWitnessOutput { uint magic; uint topologyFlags; uint numSubgroups; @@ -40,7 +40,7 @@ layout(std430, set = 0, binding = 0) buffer Program203WitnessOutput { vec2 finalAverage; } outWitness; -// Program 203's cache stays separate from all diagnostic shared state. In +// iterationRP's cache stays separate from all diagnostic shared state. In // particular, no instrumentation stores through prefixSumCache except source // writes retained below. shared vec2 prefixSumCache[32]; @@ -108,7 +108,7 @@ void main() { } // This branch is uniform after collection and is solely a safety guard for - // broken topology reports. The valid side retains Program 203 verbatim. + // broken topology reports. The valid side retains iterationRP verbatim. const bool sourceDomain = canonicalDomain && topologyFlagsShared == 0u; if (sourceDomain) { vec2 sampleLuminance = vec2(float(gl_LocalInvocationIndex + 1u), 0.0); diff --git a/MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.cpp b/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.cpp similarity index 69% rename from MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.cpp rename to MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.cpp index 74a0078f..141d3b01 100644 --- a/MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.cpp +++ b/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.cpp @@ -1,4 +1,4 @@ -// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.cpp +// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.cpp // Copyright (c) 2026 MobileGL-Dev // Licensed under the GNU Lesser General Public License v3.0: // https://www.gnu.org/licenses/gpl-3.0.txt @@ -6,7 +6,7 @@ // SPDX-License-Identifier: LGPL-3.0-only // End of Source File Header -#include "DriverPostProgram203Witness.h" +#include "DriverPostIterationRPWitness.h" #include #include @@ -15,11 +15,11 @@ namespace MobileGL::MG_Util::SelfTest { namespace { - [[nodiscard]] Program203WitnessValidationResult Failure(Program203WitnessValidationFailure failure, + [[nodiscard]] IterationRPWitnessValidationResult Failure(IterationRPWitnessValidationFailure failure, std::string detail, std::uint32_t scanStage = 0u, std::uint32_t subgroup = 0u) { - Program203WitnessValidationResult result; + IterationRPWitnessValidationResult result; result.ok = false; result.failure = failure; result.scanStage = scanStage; @@ -36,18 +36,18 @@ namespace MobileGL::MG_Util::SelfTest { return FloatBits(lhs) == FloatBits(rhs); } - [[nodiscard]] bool SameBits(const Program203WitnessVec2& lhs, const Program203WitnessVec2& rhs) { + [[nodiscard]] bool SameBits(const IterationRPWitnessVec2& lhs, const IterationRPWitnessVec2& rhs) { return SameBits(lhs.x, rhs.x) && SameBits(lhs.y, rhs.y); } - [[nodiscard]] std::string Vec2String(const Program203WitnessVec2& value) { + [[nodiscard]] std::string Vec2String(const IterationRPWitnessVec2& value) { std::ostringstream output; output << '(' << value.x << ',' << value.y << ')'; return output.str(); } [[nodiscard]] std::uint32_t ExpectedSeenSubgroupMask(std::uint32_t numSubgroups) { - return numSubgroups == kProgram203WitnessMaxSubgroups ? 0xffffffffu : (1u << numSubgroups) - 1u; + return numSubgroups == kIterationRPWitnessMaxSubgroups ? 0xffffffffu : (1u << numSubgroups) - 1u; } [[nodiscard]] std::string JoinRequirements(const std::vector& requirements) { @@ -60,8 +60,8 @@ namespace MobileGL::MG_Util::SelfTest { } } // namespace - Program203WitnessEligibilityResult - EvaluateProgram203WitnessEligibility(const Program203WitnessLimits& limits) { + IterationRPWitnessEligibilityResult + EvaluateIterationRPWitnessEligibility(const IterationRPWitnessLimits& limits) { // This classification deliberately precedes numeric limits. An absent native // compute/basic/arithmetic subgroup contract means there is nothing to witness, // whereas every resource/entry-point failure on a capable device is a POST FAIL. @@ -70,7 +70,7 @@ namespace MobileGL::MG_Util::SelfTest { if (!limits.computeStageSupported) missing.emplace_back("VK_SHADER_STAGE_COMPUTE_BIT"); if (!limits.basicSubgroupSupported) missing.emplace_back("VK_SUBGROUP_FEATURE_BASIC_BIT"); if (!limits.arithmeticSubgroupSupported) missing.emplace_back("VK_SUBGROUP_FEATURE_ARITHMETIC_BIT"); - return {Program203WitnessEligibility::SkipUnsupportedNativeFeatureSet, + return {IterationRPWitnessEligibility::SkipUnsupportedNativeFeatureSet, "skipped because the native compute/basic/arithmetic subgroup feature set is unsupported (missing " + JoinRequirements(missing) + ')'}; } @@ -79,16 +79,16 @@ namespace MobileGL::MG_Util::SelfTest { if (limits.subgroupSize == 0u) { inadequate.emplace_back("subgroupSize == 0"); } - if (limits.maxComputeWorkGroupInvocations < kProgram203WitnessInvocationCount) { + if (limits.maxComputeWorkGroupInvocations < kIterationRPWitnessInvocationCount) { inadequate.emplace_back("maxComputeWorkGroupInvocations < 512"); } if (limits.maxComputeWorkGroupSize[0] < 32u || limits.maxComputeWorkGroupSize[1] < 16u || limits.maxComputeWorkGroupSize[2] < 1u) { inadequate.emplace_back("maxComputeWorkGroupSize does not cover 32x16x1"); } - if (limits.maxComputeSharedMemorySize < kProgram203WitnessSharedMemoryBytes) { + if (limits.maxComputeSharedMemorySize < kIterationRPWitnessSharedMemoryBytes) { inadequate.emplace_back("maxComputeSharedMemorySize < " + - std::to_string(kProgram203WitnessSharedMemoryBytes)); + std::to_string(kIterationRPWitnessSharedMemoryBytes)); } if (limits.maxPerStageDescriptorStorageBuffers < 1u) { inadequate.emplace_back("maxPerStageDescriptorStorageBuffers < 1"); @@ -99,21 +99,21 @@ namespace MobileGL::MG_Util::SelfTest { if (limits.maxBoundDescriptorSets < 1u) { inadequate.emplace_back("maxBoundDescriptorSets < 1"); } - if (limits.maxStorageBufferRange < sizeof(Program203WitnessOutput)) { + if (limits.maxStorageBufferRange < sizeof(IterationRPWitnessOutput)) { inadequate.emplace_back("maxStorageBufferRange < " + - std::to_string(sizeof(Program203WitnessOutput))); + std::to_string(sizeof(IterationRPWitnessOutput))); } if (!inadequate.empty()) { - return {Program203WitnessEligibility::FailInadequateLimits, + return {IterationRPWitnessEligibility::FailInadequateLimits, "insufficient Vulkan limits for a 32x16x1 workgroup, one output SSBO, and " + - std::to_string(kProgram203WitnessSharedMemoryBytes) + " bytes of shared memory: " + + std::to_string(kIterationRPWitnessSharedMemoryBytes) + " bytes of shared memory: " + JoinRequirements(inadequate)}; } - return {Program203WitnessEligibility::Execute, {}}; + return {IterationRPWitnessEligibility::Execute, {}}; } - std::uint32_t ComputeProgram203WitnessLoopLength(std::uint32_t numSubgroups) { - if (numSubgroups < 2u || numSubgroups > kProgram203WitnessMaxSubgroups) return 0u; + std::uint32_t ComputeIterationRPWitnessLoopLength(std::uint32_t numSubgroups) { + if (numSubgroups < 2u || numSubgroups > kIterationRPWitnessMaxSubgroups) return 0u; // Exact C++ spelling of the source's findMSB-based calculation. In // particular, its final iteration for powers of two is intentional. @@ -125,87 +125,87 @@ namespace MobileGL::MG_Util::SelfTest { return loopLength; } - Program203WitnessValidationResult ValidateProgram203Witness(const Program203WitnessOutput& output) { + IterationRPWitnessValidationResult ValidateIterationRPWitness(const IterationRPWitnessOutput& output) { // 1. Completion. A poisoned or unwritten result must never turn into a // topology diagnosis, because it says nothing about execution. - if (output.magic != kProgram203WitnessMagic) { + if (output.magic != kIterationRPWitnessMagic) { std::ostringstream detail; detail << "completion: magic was 0x" << std::hex << output.magic << ", expected 0x" - << kProgram203WitnessMagic; - return Failure(Program203WitnessValidationFailure::Completion, detail.str()); + << kIterationRPWitnessMagic; + return Failure(IterationRPWitnessValidationFailure::Completion, detail.str()); } // 2. Observed topology. All checks consume observations written by the // shader, rather than inferring subgroup layout from invocation indices. const std::uint32_t numSubgroups = output.numSubgroups; - if (numSubgroups < 2u || numSubgroups > kProgram203WitnessMaxSubgroups) { + if (numSubgroups < 2u || numSubgroups > kIterationRPWitnessMaxSubgroups) { std::ostringstream detail; detail << "topology: canonical gl_NumSubgroups=" << numSubgroups << " is outside [2, 32]"; - return Failure(Program203WitnessValidationFailure::Topology, detail.str()); + return Failure(IterationRPWitnessValidationFailure::Topology, detail.str()); } - if ((output.topologyFlags & Program203WitnessNonuniformNumSubgroups) != 0u) { - return Failure(Program203WitnessValidationFailure::Topology, + if ((output.topologyFlags & IterationRPWitnessNonuniformNumSubgroups) != 0u) { + return Failure(IterationRPWitnessValidationFailure::Topology, "topology: gl_NumSubgroups differed across workgroup"); } - if ((output.topologyFlags & Program203WitnessInvalidNumSubgroups) != 0u) { - return Failure(Program203WitnessValidationFailure::Topology, + if ((output.topologyFlags & IterationRPWitnessInvalidNumSubgroups) != 0u) { + return Failure(IterationRPWitnessValidationFailure::Topology, "topology: an invocation reported gl_NumSubgroups outside [2, 32]"); } - if ((output.topologyFlags & Program203WitnessInvalidSubgroupId) != 0u) { - return Failure(Program203WitnessValidationFailure::Topology, + if ((output.topologyFlags & IterationRPWitnessInvalidSubgroupId) != 0u) { + return Failure(IterationRPWitnessValidationFailure::Topology, "topology: an invocation reported an invalid gl_SubgroupID"); } - if ((output.topologyFlags & Program203WitnessInvalidSubgroupLane) != 0u) { - return Failure(Program203WitnessValidationFailure::Topology, + if ((output.topologyFlags & IterationRPWitnessInvalidSubgroupLane) != 0u) { + return Failure(IterationRPWitnessValidationFailure::Topology, "topology: an invocation reported an invalid subgroup lane"); } - if ((output.topologyFlags & ~(Program203WitnessNonuniformNumSubgroups | - Program203WitnessInvalidNumSubgroups | - Program203WitnessInvalidSubgroupId | - Program203WitnessInvalidSubgroupLane)) != 0u) { + if ((output.topologyFlags & ~(IterationRPWitnessNonuniformNumSubgroups | + IterationRPWitnessInvalidNumSubgroups | + IterationRPWitnessInvalidSubgroupId | + IterationRPWitnessInvalidSubgroupLane)) != 0u) { std::ostringstream detail; detail << "topology: unknown topology flags 0x" << std::hex << output.topologyFlags; - return Failure(Program203WitnessValidationFailure::Topology, detail.str()); + return Failure(IterationRPWitnessValidationFailure::Topology, detail.str()); } const std::uint32_t expectedMask = ExpectedSeenSubgroupMask(numSubgroups); if (output.seenSubgroupMask != expectedMask) { std::ostringstream detail; detail << "topology: seen subgroup-ID mask was 0x" << std::hex << output.seenSubgroupMask << ", expected 0x" << expectedMask; - return Failure(Program203WitnessValidationFailure::Topology, detail.str()); + return Failure(IterationRPWitnessValidationFailure::Topology, detail.str()); } - const std::uint32_t expectedLoopLength = ComputeProgram203WitnessLoopLength(numSubgroups); + const std::uint32_t expectedLoopLength = ComputeIterationRPWitnessLoopLength(numSubgroups); if (output.loopLength != expectedLoopLength) { std::ostringstream detail; detail << "topology: loopLength was " << std::dec << output.loopLength << ", expected " << expectedLoopLength; - return Failure(Program203WitnessValidationFailure::Topology, detail.str()); + return Failure(IterationRPWitnessValidationFailure::Topology, detail.str()); } for (std::uint32_t subgroup = 0u; subgroup < numSubgroups; ++subgroup) { if (output.lastLaneWriterCount[subgroup] != 1u) { std::ostringstream detail; detail << "topology: subgroup " << subgroup << " has " << output.lastLaneWriterCount[subgroup] << " source last-lane writers, expected exactly 1"; - return Failure(Program203WitnessValidationFailure::Topology, detail.str(), 0u, subgroup); + return Failure(IterationRPWitnessValidationFailure::Topology, detail.str(), 0u, subgroup); } } if (output.owner511.y != numSubgroups) { std::ostringstream detail; detail << "final owner: invocation 511 reported gl_NumSubgroups=" << output.owner511.y << ", expected " << numSubgroups; - return Failure(Program203WitnessValidationFailure::FinalOwner, detail.str()); + return Failure(IterationRPWitnessValidationFailure::FinalOwner, detail.str()); } if (output.owner511.z != numSubgroups - 1u) { std::ostringstream detail; detail << "final owner: invocation 511 is not in the highest subgroup (id" << output.owner511.z << ", expected id" << (numSubgroups - 1u) << ')'; - return Failure(Program203WitnessValidationFailure::FinalOwner, detail.str()); + return Failure(IterationRPWitnessValidationFailure::FinalOwner, detail.str()); } if (output.owner511.x == 0u || output.owner511.w != output.owner511.x - 1u) { std::ostringstream detail; detail << "final owner: invocation 511 is not the last lane of highest subgroup (size " << output.owner511.x << ", lane " << output.owner511.w << ')'; - return Failure(Program203WitnessValidationFailure::FinalOwner, detail.str()); + return Failure(IterationRPWitnessValidationFailure::FinalOwner, detail.str()); } // 3. Initial subgroup handoff. The atomic scalar totals are independent @@ -218,22 +218,22 @@ namespace MobileGL::MG_Util::SelfTest { if (indexedTotal != 131328u) { std::ostringstream detail; detail << "initial subgroup handoff: indexed input total was " << indexedTotal << ", expected 131328"; - return Failure(Program203WitnessValidationFailure::InitialSubgroupHandoff, detail.str()); + return Failure(IterationRPWitnessValidationFailure::InitialSubgroupHandoff, detail.str()); } for (std::uint32_t subgroup = 0u; subgroup < numSubgroups; ++subgroup) { - const Program203WitnessVec2 expected = {static_cast(output.indexedInputTotal[subgroup]), 0.0f}; + const IterationRPWitnessVec2 expected = {static_cast(output.indexedInputTotal[subgroup]), 0.0f}; if (!SameBits(output.rawPrefix[subgroup], expected)) { std::ostringstream detail; detail << "initial subgroup handoff: subgroup " << subgroup << " rawPrefix observed " << Vec2String(output.rawPrefix[subgroup]) << ", expected " << Vec2String(expected); - return Failure(Program203WitnessValidationFailure::InitialSubgroupHandoff, detail.str(), 0u, + return Failure(IterationRPWitnessValidationFailure::InitialSubgroupHandoff, detail.str(), 0u, subgroup); } } // 4. Source scan. Do not substitute a conventional scan: this reproduces // the source cache index expression and stage ordering word for word. - std::array expectedCache = output.rawPrefix; + std::array expectedCache = output.rawPrefix; for (std::uint32_t scanStage = 0u; scanStage < expectedLoopLength; ++scanStage) { auto cacheAfterStage = expectedCache; for (std::uint32_t subgroup = 0u; subgroup < numSubgroups; ++subgroup) { @@ -250,27 +250,27 @@ namespace MobileGL::MG_Util::SelfTest { detail << "source scan stage " << scanStage << ", subgroup " << subgroup << ": observed " << Vec2String(output.scanCache[scanStage][subgroup]) << ", expected " << Vec2String(expectedCache[subgroup]); - return Failure(Program203WitnessValidationFailure::SourceScan, detail.str(), scanStage, subgroup); + return Failure(IterationRPWitnessValidationFailure::SourceScan, detail.str(), scanStage, subgroup); } } } // 5. The owner contract was checked above with the other topology facts; // this final result remains a separate exact-vector check. - const Program203WitnessVec2 expectedAverage = {256.5f, 0.0f}; + const IterationRPWitnessVec2 expectedAverage = {256.5f, 0.0f}; if (!SameBits(output.finalAverage, expectedAverage)) { std::ostringstream detail; detail << "final average: observed " << Vec2String(output.finalAverage) << ", expected " << Vec2String(expectedAverage); - return Failure(Program203WitnessValidationFailure::FinalAverage, detail.str()); + return Failure(IterationRPWitnessValidationFailure::FinalAverage, detail.str()); } std::ostringstream detail; detail << "N=" << numSubgroups << ", owner511=id" << output.owner511.z << "/lane" << output.owner511.w << ", " << expectedLoopLength << " scan stages, average=" << Vec2String(output.finalAverage); - Program203WitnessValidationResult result; + IterationRPWitnessValidationResult result; result.ok = true; - result.failure = Program203WitnessValidationFailure::None; + result.failure = IterationRPWitnessValidationFailure::None; result.detail = detail.str(); return result; } diff --git a/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.h b/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.h new file mode 100644 index 00000000..1999ed4f --- /dev/null +++ b/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.h @@ -0,0 +1,153 @@ +// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitness.h +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header +// +// Compact, native-Vulkan iterationRP first-reduction witness ABI and its pure +// validator. The types below deliberately mirror DriverPostIterationRPWitness.comp's +// single std430 storage block; changing either side requires updating the static +// layout assertions here. + +#pragma once + +#include +#include +#include +#include +#include + +namespace MobileGL::MG_Util::SelfTest { + // "P203": the pack's trace program id, kept stable so the checked-in witness + // SPIR-V (DriverPostIterationRPWitnessSpv.h) needs no regeneration. + constexpr std::uint32_t kIterationRPWitnessMagic = 0x50323033u; + constexpr std::uint32_t kIterationRPWitnessInvocationCount = 512u; + constexpr std::uint32_t kIterationRPWitnessMaxSubgroups = 32u; + constexpr std::uint32_t kIterationRPWitnessMaxScanStages = 6u; + + // These bit values are shared with the GLSL source. They document failures in + // topology observations rather than guessing a topology from local IDs on the host. + enum IterationRPWitnessTopologyFlag : std::uint32_t { + IterationRPWitnessNonuniformNumSubgroups = 1u << 0u, + IterationRPWitnessInvalidNumSubgroups = 1u << 1u, + IterationRPWitnessInvalidSubgroupId = 1u << 2u, + IterationRPWitnessInvalidSubgroupLane = 1u << 3u, + }; + + struct alignas(8) IterationRPWitnessVec2 { + float x; + float y; + }; + + struct alignas(16) IterationRPWitnessUVec4 { + std::uint32_t x; + std::uint32_t y; + std::uint32_t z; + std::uint32_t w; + }; + + // std430 layout of DriverPostIterationRPWitness.comp's IterationRPWitnessOutput block. + struct alignas(16) IterationRPWitnessOutput { + std::uint32_t magic; + std::uint32_t topologyFlags; + std::uint32_t numSubgroups; + std::uint32_t loopLength; + std::uint32_t seenSubgroupMask; + + IterationRPWitnessUVec4 owner511; + + std::array lastLaneWriterCount; + std::array indexedInputTotal; + + std::array rawPrefix; + std::array, + kIterationRPWitnessMaxScanStages> + scanCache; + IterationRPWitnessVec2 finalAverage; + }; + + static_assert(std::is_standard_layout_v); + static_assert(std::is_standard_layout_v); + static_assert(std::is_standard_layout_v); + static_assert(sizeof(IterationRPWitnessVec2) == 8u); + static_assert(alignof(IterationRPWitnessVec2) == 8u); + static_assert(sizeof(IterationRPWitnessUVec4) == 16u); + static_assert(alignof(IterationRPWitnessUVec4) == 16u); + static_assert(offsetof(IterationRPWitnessOutput, magic) == 0u); + static_assert(offsetof(IterationRPWitnessOutput, topologyFlags) == 4u); + static_assert(offsetof(IterationRPWitnessOutput, numSubgroups) == 8u); + static_assert(offsetof(IterationRPWitnessOutput, loopLength) == 12u); + static_assert(offsetof(IterationRPWitnessOutput, seenSubgroupMask) == 16u); + static_assert(offsetof(IterationRPWitnessOutput, owner511) == 32u); + static_assert(offsetof(IterationRPWitnessOutput, lastLaneWriterCount) == 48u); + static_assert(offsetof(IterationRPWitnessOutput, indexedInputTotal) == 176u); + static_assert(offsetof(IterationRPWitnessOutput, rawPrefix) == 304u); + static_assert(offsetof(IterationRPWitnessOutput, scanCache) == 560u); + static_assert(offsetof(IterationRPWitnessOutput, finalAverage) == 2096u); + static_assert(sizeof(IterationRPWitnessOutput) == 2112u); + + // The witness uses prefixSumCache[32], three scalar shared diagnostics, and + // two 32-entry scalar diagnostic arrays in the GLSL source. Keep this + // independent of the output SSBO size. + constexpr std::uint32_t kIterationRPWitnessSharedMemoryBytes = + kIterationRPWitnessMaxSubgroups * sizeof(IterationRPWitnessVec2) + + 3u * sizeof(std::uint32_t) + + 2u * kIterationRPWitnessMaxSubgroups * sizeof(std::uint32_t); + + enum class IterationRPWitnessEligibility { + Execute, + SkipUnsupportedNativeFeatureSet, + FailInadequateLimits, + }; + + // The raw physical-device conditions needed by the native witness. This is + // intentionally distinct from MobileGL's advertised-extension policy. + struct IterationRPWitnessLimits { + bool computeStageSupported = false; + bool basicSubgroupSupported = false; + bool arithmeticSubgroupSupported = false; + std::uint32_t subgroupSize = 0u; + + std::uint32_t maxComputeWorkGroupInvocations = 0u; + std::array maxComputeWorkGroupSize{}; + std::uint32_t maxComputeSharedMemorySize = 0u; + std::uint32_t maxPerStageDescriptorStorageBuffers = 0u; + std::uint32_t maxDescriptorSetStorageBuffers = 0u; + std::uint32_t maxBoundDescriptorSets = 0u; + std::uint64_t maxStorageBufferRange = 0u; + }; + + struct IterationRPWitnessEligibilityResult { + IterationRPWitnessEligibility eligibility = IterationRPWitnessEligibility::FailInadequateLimits; + std::string detail; + }; + + enum class IterationRPWitnessValidationFailure { + None, + Completion, + Topology, + InitialSubgroupHandoff, + SourceScan, + FinalOwner, + FinalAverage, + }; + + struct IterationRPWitnessValidationResult { + bool ok = false; + IterationRPWitnessValidationFailure failure = IterationRPWitnessValidationFailure::Completion; + std::uint32_t scanStage = 0u; + std::uint32_t subgroup = 0u; + std::string detail; + }; + + [[nodiscard]] IterationRPWitnessEligibilityResult + EvaluateIterationRPWitnessEligibility(const IterationRPWitnessLimits& limits); + + // Mirrors the source's findMSB expression for valid N in [2, 32]. + [[nodiscard]] std::uint32_t ComputeIterationRPWitnessLoopLength(std::uint32_t numSubgroups); + + [[nodiscard]] IterationRPWitnessValidationResult + ValidateIterationRPWitness(const IterationRPWitnessOutput& output); +} // namespace MobileGL::MG_Util::SelfTest diff --git a/MobileGL/MG_Util/SelfTest/DriverPostProgram203WitnessSpv.h b/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitnessSpv.h similarity index 97% rename from MobileGL/MG_Util/SelfTest/DriverPostProgram203WitnessSpv.h rename to MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitnessSpv.h index c5bf98eb..556fdcf8 100644 --- a/MobileGL/MG_Util/SelfTest/DriverPostProgram203WitnessSpv.h +++ b/MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitnessSpv.h @@ -1,4 +1,4 @@ -// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostProgram203WitnessSpv.h +// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostIterationRPWitnessSpv.h // Copyright (c) 2026 MobileGL-Dev // Licensed under the GNU Lesser General Public License v3.0: // https://www.gnu.org/licenses/gpl-3.0.txt @@ -6,9 +6,15 @@ // SPDX-License-Identifier: LGPL-3.0-only // End of Source File Header // -// Generated from DriverPostProgram203Witness.comp with: -// glslangValidator --target-env vulkan1.1 -V DriverPostProgram203Witness.comp +// Generated from DriverPostIterationRPWitness.comp with: +// glslangValidator --target-env vulkan1.1 -V DriverPostIterationRPWitness.comp // Validated with spirv-val --target-env vulkan1.1. Do not edit words by hand. +// +// The stored words predate the Program203 -> IterationRP source rename, so their +// embedded OpName debug strings still spell the old identifiers; regeneration from +// the renamed source produces semantically identical code differing only in those +// strings. The witness magic stays 0x50323033 ("P203" - the trace's program id) so +// these words remain valid without regeneration. #pragma once @@ -16,7 +22,7 @@ #include namespace MobileGL::MG_Util::SelfTest { - inline constexpr std::uint32_t kDriverPostProgram203WitnessSpv[] = { + inline constexpr std::uint32_t kDriverPostIterationRPWitnessSpv[] = { 0x07230203u, 0x00010300u, 0x0008000bu, 0x00000145u, 0x00000000u, 0x00020011u, 0x00000001u, 0x00020011u, 0x0000003du, 0x00020011u, 0x0000003fu, 0x0006000bu, 0x00000001u, 0x4c534c47u, 0x6474732eu, 0x3035342eu, 0x00000000u, 0x0003000eu, 0x00000000u, 0x00000001u, 0x000a000fu, 0x00000005u, 0x00000004u, 0x6e69616du, @@ -286,6 +292,6 @@ namespace MobileGL::MG_Util::SelfTest { 0x00050041u, 0x0000003bu, 0x00000141u, 0x00000037u, 0x00000124u, 0x0003003eu, 0x00000141u, 0x00000140u, 0x000200f9u, 0x0000013du, 0x000200f8u, 0x0000013du, 0x000100fdu, 0x00010038u, }; - inline constexpr std::size_t kDriverPostProgram203WitnessSpvWordCount = - sizeof(kDriverPostProgram203WitnessSpv) / sizeof(kDriverPostProgram203WitnessSpv[0]); + inline constexpr std::size_t kDriverPostIterationRPWitnessSpvWordCount = + sizeof(kDriverPostIterationRPWitnessSpv) / sizeof(kDriverPostIterationRPWitnessSpv[0]); } // namespace MobileGL::MG_Util::SelfTest diff --git a/MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.h b/MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.h deleted file mode 100644 index be7af86c..00000000 --- a/MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.h +++ /dev/null @@ -1,151 +0,0 @@ -// MobileGL - MobileGL/MG_Util/SelfTest/DriverPostProgram203Witness.h -// Copyright (c) 2026 MobileGL-Dev -// Licensed under the GNU Lesser General Public License v3.0: -// https://www.gnu.org/licenses/gpl-3.0.txt -// https://www.gnu.org/licenses/lgpl-3.0.txt -// SPDX-License-Identifier: LGPL-3.0-only -// End of Source File Header -// -// Compact, native-Vulkan Program-203 first-reduction witness ABI and its pure -// validator. The types below deliberately mirror DriverPostProgram203Witness.comp's -// single std430 storage block; changing either side requires updating the static -// layout assertions here. - -#pragma once - -#include -#include -#include -#include -#include - -namespace MobileGL::MG_Util::SelfTest { - constexpr std::uint32_t kProgram203WitnessMagic = 0x50323033u; // "P203" - constexpr std::uint32_t kProgram203WitnessInvocationCount = 512u; - constexpr std::uint32_t kProgram203WitnessMaxSubgroups = 32u; - constexpr std::uint32_t kProgram203WitnessMaxScanStages = 6u; - - // These bit values are shared with the GLSL source. They document failures in - // topology observations rather than guessing a topology from local IDs on the host. - enum Program203WitnessTopologyFlag : std::uint32_t { - Program203WitnessNonuniformNumSubgroups = 1u << 0u, - Program203WitnessInvalidNumSubgroups = 1u << 1u, - Program203WitnessInvalidSubgroupId = 1u << 2u, - Program203WitnessInvalidSubgroupLane = 1u << 3u, - }; - - struct alignas(8) Program203WitnessVec2 { - float x; - float y; - }; - - struct alignas(16) Program203WitnessUVec4 { - std::uint32_t x; - std::uint32_t y; - std::uint32_t z; - std::uint32_t w; - }; - - // std430 layout of DriverPostProgram203Witness.comp's Program203WitnessOutput block. - struct alignas(16) Program203WitnessOutput { - std::uint32_t magic; - std::uint32_t topologyFlags; - std::uint32_t numSubgroups; - std::uint32_t loopLength; - std::uint32_t seenSubgroupMask; - - Program203WitnessUVec4 owner511; - - std::array lastLaneWriterCount; - std::array indexedInputTotal; - - std::array rawPrefix; - std::array, - kProgram203WitnessMaxScanStages> - scanCache; - Program203WitnessVec2 finalAverage; - }; - - static_assert(std::is_standard_layout_v); - static_assert(std::is_standard_layout_v); - static_assert(std::is_standard_layout_v); - static_assert(sizeof(Program203WitnessVec2) == 8u); - static_assert(alignof(Program203WitnessVec2) == 8u); - static_assert(sizeof(Program203WitnessUVec4) == 16u); - static_assert(alignof(Program203WitnessUVec4) == 16u); - static_assert(offsetof(Program203WitnessOutput, magic) == 0u); - static_assert(offsetof(Program203WitnessOutput, topologyFlags) == 4u); - static_assert(offsetof(Program203WitnessOutput, numSubgroups) == 8u); - static_assert(offsetof(Program203WitnessOutput, loopLength) == 12u); - static_assert(offsetof(Program203WitnessOutput, seenSubgroupMask) == 16u); - static_assert(offsetof(Program203WitnessOutput, owner511) == 32u); - static_assert(offsetof(Program203WitnessOutput, lastLaneWriterCount) == 48u); - static_assert(offsetof(Program203WitnessOutput, indexedInputTotal) == 176u); - static_assert(offsetof(Program203WitnessOutput, rawPrefix) == 304u); - static_assert(offsetof(Program203WitnessOutput, scanCache) == 560u); - static_assert(offsetof(Program203WitnessOutput, finalAverage) == 2096u); - static_assert(sizeof(Program203WitnessOutput) == 2112u); - - // The witness uses prefixSumCache[32], three scalar shared diagnostics, and - // two 32-entry scalar diagnostic arrays in the GLSL source. Keep this - // independent of the output SSBO size. - constexpr std::uint32_t kProgram203WitnessSharedMemoryBytes = - kProgram203WitnessMaxSubgroups * sizeof(Program203WitnessVec2) + - 3u * sizeof(std::uint32_t) + - 2u * kProgram203WitnessMaxSubgroups * sizeof(std::uint32_t); - - enum class Program203WitnessEligibility { - Execute, - SkipUnsupportedNativeFeatureSet, - FailInadequateLimits, - }; - - // The raw physical-device conditions needed by the native witness. This is - // intentionally distinct from MobileGL's advertised-extension policy. - struct Program203WitnessLimits { - bool computeStageSupported = false; - bool basicSubgroupSupported = false; - bool arithmeticSubgroupSupported = false; - std::uint32_t subgroupSize = 0u; - - std::uint32_t maxComputeWorkGroupInvocations = 0u; - std::array maxComputeWorkGroupSize{}; - std::uint32_t maxComputeSharedMemorySize = 0u; - std::uint32_t maxPerStageDescriptorStorageBuffers = 0u; - std::uint32_t maxDescriptorSetStorageBuffers = 0u; - std::uint32_t maxBoundDescriptorSets = 0u; - std::uint64_t maxStorageBufferRange = 0u; - }; - - struct Program203WitnessEligibilityResult { - Program203WitnessEligibility eligibility = Program203WitnessEligibility::FailInadequateLimits; - std::string detail; - }; - - enum class Program203WitnessValidationFailure { - None, - Completion, - Topology, - InitialSubgroupHandoff, - SourceScan, - FinalOwner, - FinalAverage, - }; - - struct Program203WitnessValidationResult { - bool ok = false; - Program203WitnessValidationFailure failure = Program203WitnessValidationFailure::Completion; - std::uint32_t scanStage = 0u; - std::uint32_t subgroup = 0u; - std::string detail; - }; - - [[nodiscard]] Program203WitnessEligibilityResult - EvaluateProgram203WitnessEligibility(const Program203WitnessLimits& limits); - - // Mirrors the source's findMSB expression for valid N in [2, 32]. - [[nodiscard]] std::uint32_t ComputeProgram203WitnessLoopLength(std::uint32_t numSubgroups); - - [[nodiscard]] Program203WitnessValidationResult - ValidateProgram203Witness(const Program203WitnessOutput& output); -} // namespace MobileGL::MG_Util::SelfTest diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp index 28367501..36b852c9 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp @@ -26,6 +26,8 @@ #include "SpirvPasses/RebaseInstanceIndexPass.h" #include "SpirvPasses/ZeroBaseVertexPass.h" #include "SpirvPasses/DeriveNumSubgroupsPass.h" +#include "SpirvPasses/EmulateSubgroupsPass.h" +#include "SpirvPasses/FixIterationRPSubgroupScratchPass.h" #include "SpirvPasses/NormalizeRectCoordinatesPass.h" #include "SpirvPasses/Lower1DArrayImagesPass.h" #include "SpirvPasses/BakeImageFormatsPass.h" @@ -895,6 +897,32 @@ namespace MobileGL { outputBinary, true, enableSpirvValidation); } + bool ShaderCompiler::EmulateSubgroupsForVulkan(const Vector& inputBinary, + Vector& outputBinary, + const Uint32 maxWorkgroupScratchBytes, + const bool enableSpirvValidation) { + using namespace spvtools; + Optimizer optimizer(SPV_ENV_VULKAN_1_1); + optimizer.RegisterPass( + EmulateSubgroupsPass::CreateEmulateSubgroupsPass(maxWorkgroupScratchBytes)); + + return RunOptimizerChecked("EmulateSubgroupsForVulkan", optimizer, inputBinary, + outputBinary, true, enableSpirvValidation); + } + + bool ShaderCompiler::FixIterationRPSubgroupScratchForVulkan( + const Vector& inputBinary, Vector& outputBinary, + const Uint32 nativeSubgroupSize, const bool enableSpirvValidation) { + using namespace spvtools; + Optimizer optimizer(SPV_ENV_VULKAN_1_1); + optimizer.RegisterPass( + FixIterationRPSubgroupScratchPass::CreateFixIterationRPSubgroupScratchPass( + nativeSubgroupSize)); + + return RunOptimizerChecked("FixIterationRPSubgroupScratchForVulkan", optimizer, + inputBinary, outputBinary, true, enableSpirvValidation); + } + bool ShaderCompiler::DecoratePositionInvariantForVulkan(const Vector& inputBinary, Vector& outputBinary, const bool enableSpirvValidation) { using namespace spvtools; diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h index 5073de2b..53504f00 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h @@ -145,13 +145,35 @@ namespace MobileGL { static bool ZeroBaseVertexForVulkan(const Vector& inputBinary, Vector& outputBinary, bool enableSpirvValidation = false); - // Replaces compute gl_NumSubgroups loads with the value derived from the local - // workgroup dimensions and gl_SubgroupSize. DirectVulkan only; this avoids a - // driver builtin that can disagree with the subgroup IDs the same dispatch emits. - // See DeriveNumSubgroupsPass. + // Replaces compute gl_NumSubgroups loads with ceil(workgroup invocations / + // gl_SubgroupSize). DirectVulkan only; this repairs drivers whose builtin + // disagrees with the subgroup IDs the same dispatch emits (Adreno reports 1 + // while emitting IDs 0..7). The ceil() partition is only spec-guaranteed + // under VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT, which + // the caller requests whenever it is legal for the workgroup shape; see + // DeriveNumSubgroupsPass. static bool DeriveNumSubgroupsForVulkan(const Vector& inputBinary, Vector& outputBinary, bool enableSpirvValidation = false); + // Lowers every GL_KHR_shader_subgroup construct in a compute module onto a + // 32-lane virtual subgroup built from workgroup-shared memory. Last-resort + // path for devices with NO native subgroup support, opt-in via + // MOBILEGL_MAGMA_EMULATE_SUBGROUP=1; a device with native subgroup + // operations always uses them. maxWorkgroupScratchBytes bounds the shared + // scratch the lowering may add (pass the device's + // maxComputeSharedMemorySize; 0 falls back to the 16384-byte Vulkan + // minimum). See EmulateSubgroupsPass. + static bool EmulateSubgroupsForVulkan(const Vector& inputBinary, + Vector& outputBinary, + Uint32 maxWorkgroupScratchBytes, + bool enableSpirvValidation = false); + // Patches iterationRP's under-declared prefixSumCache[32] on sub-16-lane + // devices, fingerprint-gated to that pack's reduction; every other module + // passes through byte-identical. See FixIterationRPSubgroupScratchPass. + static bool FixIterationRPSubgroupScratchForVulkan(const Vector& inputBinary, + Vector& outputBinary, + Uint32 nativeSubgroupSize, + bool enableSpirvValidation = false); // Re-declares 64-bit float vertex inputs as their 32-bit unsigned word pair // (double -> uvec2, dvec2 -> uvec4) and bitcasts them back to double at entry, so no // VK_FORMAT_R64*_SFLOAT is needed - lavapipe advertises none of them for vertex diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.cpp b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.cpp index f3fbeacf..2bc1c8af 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.cpp +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.cpp @@ -179,8 +179,14 @@ namespace MobileGL { : SynthesizeSubgroupSizeVariable(irContext, numSubgroupsVar->type_id()); const uint32_t workgroupSizeId = workgroupSize->result_id(); - // The pipeline never enables ALLOW_VARYING_SUBGROUP_SIZE, so Vulkan's fixed - // subgroup partition is exactly ceil(local invocation count / SubgroupSize). + // ceil(local invocation count / SubgroupSize): the subgroup count of a + // full-subgroup launch. Vulkan only guarantees that partition under + // REQUIRE_FULL_SUBGROUPS - which ProgramFactory requests whenever + // local_size_x is a multiple of the subgroup size makes it legal + // (VUID-VkPipelineShaderStageCreateInfo-flags-02759) - and calls the + // tighter behaviour "encouraged" everywhere else; the DriverPost witness + // verifies it per device where the flag cannot be set. The absence of + // ALLOW_VARYING_SUBGROUP_SIZE pins only the SubgroupSize builtin itself. // `(count - 1) / size + 1` avoids an addition overflow at count + size - 1. for (Instruction* load : numSubgroupsLoads) { const uint32_t localSizeXId = irContext->TakeNextId(); diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.h b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.h index 330c17a4..e5b117e3 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.h +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/DeriveNumSubgroupsPass.h @@ -19,12 +19,14 @@ namespace MobileGL { // Replaces compute-stage NumSubgroups builtin loads with // ceil(WorkgroupSize.x * WorkgroupSize.y * WorkgroupSize.z / SubgroupSize). // - // That is the value Vulkan defines for NumSubgroups when the pipeline does not - // enable varying subgroup sizes, which MobileGL never does. Deriving it avoids - // drivers that expose the real SubgroupId topology but return an inconsistent - // NumSubgroups value. This is a DirectVulkan semantic repair, not a source-shader - // rewrite; the application's subgroup arithmetic and shared-memory logic remain - // unchanged. + // That is the subgroup count of a full-subgroup launch - guaranteed by Vulkan + // under REQUIRE_FULL_SUBGROUPS (which ProgramFactory requests whenever the + // workgroup shape makes it legal), spec-"encouraged" and witness-verified + // (DriverPost) elsewhere. Deriving it repairs drivers that expose the real + // SubgroupId topology but return an inconsistent NumSubgroups value, breaking + // GL's gl_SubgroupID < gl_NumSubgroups contract. This is a DirectVulkan + // semantic repair, not a source-shader rewrite; the application's subgroup + // arithmetic and shared-memory logic remain unchanged. class DeriveNumSubgroupsPass : public spvtools::opt::Pass { public: const char* name() const override { return "derive-num-subgroups"; } diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.cpp b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.cpp new file mode 100644 index 00000000..7ef6df64 --- /dev/null +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.cpp @@ -0,0 +1,1413 @@ +// MobileGL - MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.cpp +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#include "EmulateSubgroupsPass.h" + +#include "spirv.hpp" +#include "source/opt/constants.h" +#include "source/opt/def_use_manager.h" +#include "source/opt/instruction.h" +#include "source/opt/ir_context.h" +#include "source/opt/module.h" +#include "source/opt/type_manager.h" +#include "source/util/make_unique.h" +#include "source/util/string_utils.h" + +#include +#include +#include +#include + +namespace MobileGL { + namespace MG_Util { + namespace ShaderTranspiler { + namespace { + using spvtools::opt::Instruction; + using spvtools::opt::IRContext; + using spvtools::opt::Operand; + namespace analysis = spvtools::opt::analysis; + + // The GL-visible subgroup this pass implements. Keep in sync with + // SubgroupSupportPolicy.h (kEmulatedSubgroupSize). + constexpr uint32_t kWidth = 32u; + constexpr uint32_t kLaneMask = kWidth - 1u; + constexpr uint32_t kIdShift = 5u; + + // GLSL.std.450 instruction numbers (see 3rdparty/glslang/SPIRV/GLSL.std.450.h). + constexpr uint32_t kGlslFMin = 37u; + constexpr uint32_t kGlslUMin = 38u; + constexpr uint32_t kGlslSMin = 39u; + constexpr uint32_t kGlslFMax = 40u; + constexpr uint32_t kGlslUMax = 41u; + constexpr uint32_t kGlslSMax = 42u; + constexpr uint32_t kGlslFindILsb = 73u; + constexpr uint32_t kGlslFindUMsb = 75u; + + Operand IdOp(uint32_t id) { return {SPV_OPERAND_TYPE_ID, {id}}; } + + Instruction* FindBuiltinDefinition(IRContext* context, spv::BuiltIn builtin) { + auto* defUseMgr = context->get_def_use_mgr(); + for (auto& annotation : context->annotations()) { + if (annotation.opcode() != spv::Op::OpDecorate || annotation.NumInOperands() < 3) { + continue; + } + if (static_cast(annotation.GetSingleWordInOperand(1)) != + spv::Decoration::BuiltIn) { + continue; + } + if (static_cast(annotation.GetSingleWordInOperand(2)) != builtin) { + continue; + } + return defUseMgr->GetDef(annotation.GetSingleWordInOperand(0)); + } + return nullptr; + } + + // The scalar shapes a value participating in an emulated exchange may have. + enum class ScalarKind { Float, SInt, UInt, Bool }; + + struct ValueTypeInfo { + ScalarKind kind = ScalarKind::UInt; + uint32_t componentCount = 1; // 1 for scalars, 2..4 for vectors + uint32_t typeId = 0; // the value's own type + uint32_t scalarTypeId = 0; // the component type + }; + + bool ClassifyScalar(const Instruction* type, ScalarKind* kind) { + switch (type->opcode()) { + case spv::Op::OpTypeBool: + *kind = ScalarKind::Bool; + return true; + case spv::Op::OpTypeInt: + if (type->GetSingleWordInOperand(0) != 32u) return false; + *kind = type->GetSingleWordInOperand(1) == 0u ? ScalarKind::UInt : ScalarKind::SInt; + return true; + case spv::Op::OpTypeFloat: + if (type->GetSingleWordInOperand(0) != 32u) return false; + *kind = ScalarKind::Float; + return true; + default: + return false; + } + } + + bool ClassifyValueType(IRContext* context, uint32_t typeId, ValueTypeInfo* info) { + const Instruction* type = context->get_def_use_mgr()->GetDef(typeId); + if (type == nullptr) return false; + info->typeId = typeId; + if (type->opcode() == spv::Op::OpTypeVector) { + info->componentCount = type->GetSingleWordInOperand(1); + if (info->componentCount < 2u || info->componentCount > 4u) return false; + info->scalarTypeId = type->GetSingleWordInOperand(0); + const Instruction* component = context->get_def_use_mgr()->GetDef(info->scalarTypeId); + return component != nullptr && ClassifyScalar(component, &info->kind); + } + info->componentCount = 1u; + info->scalarTypeId = typeId; + return ClassifyScalar(type, &info->kind); + } + + // Emits instructions immediately before a fixed anchor instruction. Every + // lowering sequence is straight-line code in the anchor's own block, so + // dominance is trivially preserved. + class Emitter { + public: + Emitter(IRContext* context, Instruction* anchor) : m_context(context), m_anchor(anchor) {} + + uint32_t Emit(spv::Op opcode, uint32_t typeId, Instruction::OperandList operands) { + const uint32_t resultId = m_context->TakeNextId(); + m_anchor->InsertBefore(spvtools::MakeUnique( + m_context, opcode, typeId, resultId, std::move(operands))); + return resultId; + } + + void EmitNoResult(spv::Op opcode, Instruction::OperandList operands) { + m_anchor->InsertBefore(spvtools::MakeUnique( + m_context, opcode, 0, 0, std::move(operands))); + } + + private: + IRContext* m_context; + Instruction* m_anchor; + }; + + // One recognized OpGroupNonUniform* site with everything Phase A resolved + // about it, so Phase B can rewrite without def-use queries. + struct GroupOpSite { + Instruction* inst = nullptr; + spv::Op opcode = spv::Op::OpNop; + spv::GroupOperation groupOperation = spv::GroupOperation::Reduce; + uint32_t clusterSize = 0; // ClusteredReduce only + uint32_t quadDirection = 0; // QuadSwap only + ValueTypeInfo valueType{}; // participating value (where applicable) + uint32_t valueId = 0; + uint32_t indexId = 0; // broadcast/shuffle/extract index operand + }; + + struct BarrierSite { + Instruction* inst = nullptr; + bool executionScopeIsSubgroup = false; + bool memoryScopeIsSubgroup = false; + }; + + bool IsGroupNonUniformOpcode(spv::Op opcode) { + return opcode >= spv::Op::OpGroupNonUniformElect && + opcode <= spv::Op::OpGroupNonUniformQuadSwap; + } + } // namespace + + spvtools::opt::Pass::Status EmulateSubgroupsPass::Process() { + auto* irContext = context(); + auto* defUseMgr = irContext->get_def_use_mgr(); + auto* constMgr = irContext->get_constant_mgr(); + auto* typeMgr = irContext->get_type_mgr(); + + // ---------------------------------------------------------------- Phase A + // Pure analysis; nothing is mutated until every site has been vetted, so a + // Failure return leaves the module untouched. + + bool allEntryPointsCompute = true; + for (const Instruction& entryPoint : irContext->module()->entry_points()) { + if (static_cast(entryPoint.GetSingleWordInOperand(0)) != + spv::ExecutionModel::GLCompute) { + allEntryPointsCompute = false; + } + } + + const auto resolveUintConstant = [&](uint32_t id, uint32_t* value) { + const Instruction* def = defUseMgr->GetDef(id); + if (def == nullptr || def->opcode() != spv::Op::OpConstant) return false; + *value = def->GetSingleWordInOperand(0); + return true; + }; + + // Subgroup builtin variables and their loads. + static constexpr std::array kSubgroupBuiltins = { + spv::BuiltIn::SubgroupSize, spv::BuiltIn::SubgroupLocalInvocationId, + spv::BuiltIn::SubgroupId, spv::BuiltIn::NumSubgroups, + spv::BuiltIn::SubgroupEqMask, spv::BuiltIn::SubgroupGeMask, + spv::BuiltIn::SubgroupGtMask, spv::BuiltIn::SubgroupLeMask, + spv::BuiltIn::SubgroupLtMask}; + + const auto isMaskBuiltin = [](spv::BuiltIn builtin) { + return builtin == spv::BuiltIn::SubgroupEqMask || + builtin == spv::BuiltIn::SubgroupGeMask || + builtin == spv::BuiltIn::SubgroupGtMask || + builtin == spv::BuiltIn::SubgroupLeMask || + builtin == spv::BuiltIn::SubgroupLtMask; + }; + struct BuiltinUse { + spv::BuiltIn builtin; + Instruction* variable; + std::vector loads; + // Mask builtins are uvec4; glslang reaches a single component (the + // ubiquitous gl_Subgroup*Mask.x) through an access chain plus a scalar + // load. The chain dies with the variable once its loads are rewritten. + std::vector accessChains; + std::vector> componentLoads; + }; + std::vector builtinUses; + bool sawUnexpectedUser = false; + for (const spv::BuiltIn builtin : kSubgroupBuiltins) { + Instruction* variable = FindBuiltinDefinition(irContext, builtin); + if (variable == nullptr || variable->opcode() != spv::Op::OpVariable) continue; + BuiltinUse use{builtin, variable, {}, {}, {}}; + const uint32_t variableId = variable->result_id(); + defUseMgr->ForEachUser(variable, [&](Instruction* user) { + switch (user->opcode()) { + case spv::Op::OpLoad: + if (user->NumInOperands() >= 1 && + user->GetSingleWordInOperand(0) == variableId) { + use.loads.push_back(user); + } else { + sawUnexpectedUser = true; + } + return; + case spv::Op::OpAccessChain: + case spv::Op::OpInBoundsAccessChain: { + uint32_t component = 0; + if (!isMaskBuiltin(builtin) || user->NumInOperands() != 2 || + user->GetSingleWordInOperand(0) != variableId) { + sawUnexpectedUser = true; + return; + } + const Instruction* index = + defUseMgr->GetDef(user->GetSingleWordInOperand(1)); + if (index == nullptr || index->opcode() != spv::Op::OpConstant || + (component = index->GetSingleWordInOperand(0)) > 3u) { + sawUnexpectedUser = true; + return; + } + const uint32_t chainId = user->result_id(); + defUseMgr->ForEachUser(user, [&](Instruction* chainUser) { + if (chainUser->opcode() == spv::Op::OpLoad && + chainUser->NumInOperands() >= 1 && + chainUser->GetSingleWordInOperand(0) == chainId) { + use.componentLoads.emplace_back(chainUser, component); + } else { + sawUnexpectedUser = true; + } + }); + use.accessChains.push_back(user); + return; + } + case spv::Op::OpDecorate: + case spv::Op::OpDecorateId: + case spv::Op::OpDecorateString: + case spv::Op::OpName: + case spv::Op::OpEntryPoint: + return; + default: + sawUnexpectedUser = true; + return; + } + }); + builtinUses.push_back(std::move(use)); + } + + // Group-nonuniform operations and subgroup-scoped barriers. + std::vector groupOps; + std::vector barriers; + bool sawUnsupported = false; + for (auto& function : *irContext->module()) { + for (auto& block : function) { + for (auto& inst : block) { + const spv::Op opcode = inst.opcode(); + // Extended subgroup instructions live outside the core + // [Elect..QuadSwap] range IsGroupNonUniformOpcode covers; they + // must fail the pass rather than survive into the "subgroup + // free" output. + if (opcode == spv::Op::OpGroupNonUniformPartitionNV || + opcode == spv::Op::OpGroupNonUniformRotateKHR || + opcode == spv::Op::OpGroupNonUniformQuadAllKHR || + opcode == spv::Op::OpGroupNonUniformQuadAnyKHR) { + sawUnsupported = true; + continue; + } + if (opcode == spv::Op::OpControlBarrier) { + BarrierSite site{&inst, false, false}; + uint32_t scope = 0; + if (resolveUintConstant(inst.GetSingleWordInOperand(0), &scope) && + scope == static_cast(spv::Scope::Subgroup)) { + site.executionScopeIsSubgroup = true; + } + if (resolveUintConstant(inst.GetSingleWordInOperand(1), &scope) && + scope == static_cast(spv::Scope::Subgroup)) { + site.memoryScopeIsSubgroup = true; + } + if (site.executionScopeIsSubgroup || site.memoryScopeIsSubgroup) { + barriers.push_back(site); + } + continue; + } + if (opcode == spv::Op::OpMemoryBarrier) { + uint32_t scope = 0; + if (resolveUintConstant(inst.GetSingleWordInOperand(0), &scope) && + scope == static_cast(spv::Scope::Subgroup)) { + barriers.push_back(BarrierSite{&inst, false, true}); + } + continue; + } + if (!IsGroupNonUniformOpcode(opcode)) continue; + + GroupOpSite site; + site.inst = &inst; + site.opcode = opcode; + uint32_t scope = 0; + if (!resolveUintConstant(inst.GetSingleWordInOperand(0), &scope) || + scope != static_cast(spv::Scope::Subgroup)) { + sawUnsupported = true; + continue; + } + switch (opcode) { + case spv::Op::OpGroupNonUniformElect: + break; + case spv::Op::OpGroupNonUniformAll: + case spv::Op::OpGroupNonUniformAny: + case spv::Op::OpGroupNonUniformAllEqual: + case spv::Op::OpGroupNonUniformBroadcastFirst: + case spv::Op::OpGroupNonUniformBallot: + case spv::Op::OpGroupNonUniformInverseBallot: + case spv::Op::OpGroupNonUniformBallotFindLSB: + case spv::Op::OpGroupNonUniformBallotFindMSB: + site.valueId = inst.GetSingleWordInOperand(1); + break; + case spv::Op::OpGroupNonUniformBroadcast: + case spv::Op::OpGroupNonUniformShuffle: + case spv::Op::OpGroupNonUniformShuffleXor: + case spv::Op::OpGroupNonUniformShuffleUp: + case spv::Op::OpGroupNonUniformShuffleDown: + case spv::Op::OpGroupNonUniformQuadBroadcast: + case spv::Op::OpGroupNonUniformBallotBitExtract: + site.valueId = inst.GetSingleWordInOperand(1); + site.indexId = inst.GetSingleWordInOperand(2); + break; + case spv::Op::OpGroupNonUniformQuadSwap: + site.valueId = inst.GetSingleWordInOperand(1); + if (!resolveUintConstant(inst.GetSingleWordInOperand(2), + &site.quadDirection) || + site.quadDirection > 2u) { + sawUnsupported = true; + } + break; + case spv::Op::OpGroupNonUniformBallotBitCount: + site.groupOperation = static_cast( + inst.GetSingleWordInOperand(1)); + site.valueId = inst.GetSingleWordInOperand(2); + break; + case spv::Op::OpGroupNonUniformIAdd: + case spv::Op::OpGroupNonUniformFAdd: + case spv::Op::OpGroupNonUniformIMul: + case spv::Op::OpGroupNonUniformFMul: + case spv::Op::OpGroupNonUniformSMin: + case spv::Op::OpGroupNonUniformUMin: + case spv::Op::OpGroupNonUniformFMin: + case spv::Op::OpGroupNonUniformSMax: + case spv::Op::OpGroupNonUniformUMax: + case spv::Op::OpGroupNonUniformFMax: + case spv::Op::OpGroupNonUniformBitwiseAnd: + case spv::Op::OpGroupNonUniformBitwiseOr: + case spv::Op::OpGroupNonUniformBitwiseXor: + case spv::Op::OpGroupNonUniformLogicalAnd: + case spv::Op::OpGroupNonUniformLogicalOr: + case spv::Op::OpGroupNonUniformLogicalXor: + site.groupOperation = static_cast( + inst.GetSingleWordInOperand(1)); + site.valueId = inst.GetSingleWordInOperand(2); + if (site.groupOperation == spv::GroupOperation::ClusteredReduce) { + if (inst.NumInOperands() < 4 || + !resolveUintConstant(inst.GetSingleWordInOperand(3), + &site.clusterSize) || + site.clusterSize == 0u || + (site.clusterSize & (site.clusterSize - 1u)) != 0u) { + sawUnsupported = true; + } + } else if (site.groupOperation != spv::GroupOperation::Reduce && + site.groupOperation != spv::GroupOperation::InclusiveScan && + site.groupOperation != spv::GroupOperation::ExclusiveScan) { + sawUnsupported = true; + } + break; + default: + sawUnsupported = true; + continue; + } + + // Classify the participating value where the lowering exchanges + // it through shared memory (everything except Elect and the pure + // ballot-math ops, whose operand is the uvec4 ballot itself). + switch (opcode) { + case spv::Op::OpGroupNonUniformElect: + case spv::Op::OpGroupNonUniformInverseBallot: + case spv::Op::OpGroupNonUniformBallotBitExtract: + case spv::Op::OpGroupNonUniformBallotBitCount: + case spv::Op::OpGroupNonUniformBallotFindLSB: + case spv::Op::OpGroupNonUniformBallotFindMSB: + break; + case spv::Op::OpGroupNonUniformAll: + case spv::Op::OpGroupNonUniformAny: + case spv::Op::OpGroupNonUniformBallot: { + const Instruction* value = defUseMgr->GetDef(site.valueId); + if (value == nullptr || + !ClassifyValueType(irContext, value->type_id(), &site.valueType) || + site.valueType.kind != ScalarKind::Bool || + site.valueType.componentCount != 1u) { + sawUnsupported = true; + } + break; + } + default: { + const Instruction* value = defUseMgr->GetDef(site.valueId); + if (value == nullptr || + !ClassifyValueType(irContext, value->type_id(), &site.valueType)) { + sawUnsupported = true; + } + break; + } + } + groupOps.push_back(site); + } + } + } + + bool anyBuiltinLoads = false; + for (const BuiltinUse& use : builtinUses) { + if (!use.loads.empty() || !use.componentLoads.empty()) anyBuiltinLoads = true; + } + if (!anyBuiltinLoads && groupOps.empty() && barriers.empty() && !sawUnsupported) { + return Status::SuccessWithoutChange; + } + if (!allEntryPointsCompute || sawUnexpectedUser || sawUnsupported) { + return Status::Failure; + } + + // Workgroup size. The compile chain pins SPIR-V 1.3, where a literal + // local_size always reaches the module as OpExecutionMode LocalSize and - + // through glslang - as the WorkgroupSize builtin constant as well. Either + // is accepted; spec-constant sizes are not (GL has no specialization). + uint32_t localSize[3] = {0, 0, 0}; + bool haveLocalSize = false; + if (Instruction* workgroupSize = + FindBuiltinDefinition(irContext, spv::BuiltIn::WorkgroupSize)) { + if (workgroupSize->opcode() == spv::Op::OpConstantComposite && + workgroupSize->NumInOperands() == 3) { + haveLocalSize = + resolveUintConstant(workgroupSize->GetSingleWordInOperand(0), &localSize[0]) && + resolveUintConstant(workgroupSize->GetSingleWordInOperand(1), &localSize[1]) && + resolveUintConstant(workgroupSize->GetSingleWordInOperand(2), &localSize[2]); + } + } + if (!haveLocalSize) { + for (const Instruction& mode : irContext->module()->execution_modes()) { + if (mode.opcode() == spv::Op::OpExecutionMode && + static_cast(mode.GetSingleWordInOperand(1)) == + spv::ExecutionMode::LocalSize) { + localSize[0] = mode.GetSingleWordInOperand(2); + localSize[1] = mode.GetSingleWordInOperand(3); + localSize[2] = mode.GetSingleWordInOperand(4); + haveLocalSize = true; + break; + } + } + } + if (!haveLocalSize || localSize[0] == 0u || localSize[1] == 0u || localSize[2] == 0u) { + return Status::Failure; + } + const uint64_t total64 = + static_cast(localSize[0]) * localSize[1] * localSize[2]; + if (total64 == 0u || total64 > (1u << 20)) { + return Status::Failure; + } + const uint32_t totalInvocations = static_cast(total64); + const uint32_t virtualSubgroups = (totalInvocations + kWidth - 1u) / kWidth; + const uint32_t paddedSlots = virtualSubgroups * kWidth; + + // Phase C creates one Workgroup array of paddedSlots elements per distinct + // participating element type (bools travel as uint words), 4 bytes per + // component. Refuse up front any module whose added scratch would exceed + // the caller's budget - the device's maxComputeSharedMemorySize, or the + // 16384-byte Vulkan minimum when the caller passed 0 - since spirv-val + // cannot catch this and the pipeline would fail at creation instead. The + // estimate over-counts slightly (a bool type and its uint twin are keyed + // separately) and does not subtract the module's own shared declarations. + { + const uint64_t scratchBudgetBytes = + m_maxWorkgroupScratchBytes != 0u ? m_maxWorkgroupScratchBytes : 16384u; + std::set scratchTypeKeys; + uint64_t scratchBytes = 0; + for (const GroupOpSite& site : groupOps) { + if (site.valueType.typeId == 0u) continue; // no shared-memory exchange + const uint32_t key = site.valueType.kind == ScalarKind::Bool + ? (0x80000000u | site.valueType.componentCount) + : site.valueType.typeId; + if (!scratchTypeKeys.insert(key).second) continue; + scratchBytes += static_cast(paddedSlots) * 4u * + site.valueType.componentCount; + } + if (scratchBytes > scratchBudgetBytes) { + return Status::Failure; + } + } + + // ---------------------------------------------------------------- Phase B + // Cached types and constants. + analysis::Bool boolTypeCandidate; + const uint32_t boolTypeId = typeMgr->GetTypeInstruction(&boolTypeCandidate); + analysis::Integer uintTypeCandidate(32, false); + const uint32_t uintTypeId = typeMgr->GetTypeInstruction(&uintTypeCandidate); + if (boolTypeId == 0u || uintTypeId == 0u) { + return Status::Failure; + } + + const auto uintConst = [&](uint32_t value) -> uint32_t { + const analysis::Type* type = typeMgr->GetType(uintTypeId); + const analysis::Constant* constant = constMgr->GetConstant(type, {value}); + const Instruction* inst = + constant != nullptr ? constMgr->GetDefiningInstruction(constant, uintTypeId) + : nullptr; + return inst != nullptr ? inst->result_id() : 0u; + }; + const auto boolConst = [&](bool value) -> uint32_t { + const analysis::Type* type = typeMgr->GetType(boolTypeId); + const analysis::Constant* constant = + constMgr->GetConstant(type, {value ? 1u : 0u}); + const Instruction* inst = + constant != nullptr ? constMgr->GetDefiningInstruction(constant, boolTypeId) + : nullptr; + return inst != nullptr ? inst->result_id() : 0u; + }; + // A constant of an arbitrary 32-bit scalar type from its bit pattern, splat + // to the value type when it is a vector. + const auto typedConst = [&](const ValueTypeInfo& info, uint32_t scalarBits, + uint32_t scratchScalarTypeId, + uint32_t scratchTypeId) -> uint32_t { + const analysis::Type* scalarType = typeMgr->GetType(scratchScalarTypeId); + const analysis::Constant* scalar = constMgr->GetConstant(scalarType, {scalarBits}); + if (scalar == nullptr) return 0u; + const Instruction* scalarInst = + constMgr->GetDefiningInstruction(scalar, scratchScalarTypeId); + if (scalarInst == nullptr) return 0u; + if (info.componentCount == 1u) return scalarInst->result_id(); + const analysis::Type* vectorType = typeMgr->GetType(scratchTypeId); + std::vector componentIds(info.componentCount, + scalarInst->result_id()); + const analysis::Constant* vector = constMgr->GetConstant(vectorType, componentIds); + if (vector == nullptr) return 0u; + const Instruction* vectorInst = + constMgr->GetDefiningInstruction(vector, scratchTypeId); + return vectorInst != nullptr ? vectorInst->result_id() : 0u; + }; + + const uint32_t c0 = uintConst(0u); + const uint32_t c1 = uintConst(1u); + const uint32_t c31 = uintConst(kLaneMask); + const uint32_t c32 = uintConst(kWidth); + const uint32_t cNotLane = uintConst(~kLaneMask); + const uint32_t cTotal = uintConst(totalInvocations); + const uint32_t cAllOnes = uintConst(0xffffffffu); + const uint32_t cTrue = boolConst(true); + if (c0 == 0u || c1 == 0u || c31 == 0u || c32 == 0u || cNotLane == 0u || + cTotal == 0u || cAllOnes == 0u || cTrue == 0u) { + return Status::Failure; + } + + // GLSL.std.450 import (for min/max combines and ballot find ops). + uint32_t glslStd450Id = 0; + for (const Instruction& import : irContext->module()->ext_inst_imports()) { + const std::string importName = + spvtools::utils::MakeString(import.GetInOperand(0).words); + if (importName == "GLSL.std.450") { + glslStd450Id = import.result_id(); + break; + } + } + const auto ensureGlslStd450 = [&]() -> uint32_t { + if (glslStd450Id != 0u) return glslStd450Id; + glslStd450Id = irContext->TakeNextId(); + irContext->module()->AddExtInstImport(spvtools::MakeUnique( + irContext, spv::Op::OpExtInstImport, 0, glslStd450Id, + Instruction::OperandList{ + {SPV_OPERAND_TYPE_LITERAL_STRING, + spvtools::utils::MakeVector("GLSL.std.450")}})); + return glslStd450Id; + }; + + // gl_LocalInvocationIndex drives the whole virtual topology; synthesize the + // builtin when the module never declared it. + Instruction* liiVariable = + FindBuiltinDefinition(irContext, spv::BuiltIn::LocalInvocationIndex); + uint32_t liiVariableId = 0; + if (liiVariable != nullptr && liiVariable->opcode() == spv::Op::OpVariable) { + liiVariableId = liiVariable->result_id(); + } else { + const uint32_t pointerTypeId = + typeMgr->FindPointerToType(uintTypeId, spv::StorageClass::Input); + if (pointerTypeId == 0u) return Status::Failure; + liiVariableId = irContext->TakeNextId(); + irContext->AddGlobalValue(spvtools::MakeUnique( + irContext, spv::Op::OpVariable, pointerTypeId, liiVariableId, + Instruction::OperandList{ + {SPV_OPERAND_TYPE_STORAGE_CLASS, + {static_cast(spv::StorageClass::Input)}}})); + irContext->AddAnnotationInst(spvtools::MakeUnique( + irContext, spv::Op::OpDecorate, 0, 0, + Instruction::OperandList{ + IdOp(liiVariableId), + {SPV_OPERAND_TYPE_DECORATION, + {static_cast(spv::Decoration::BuiltIn)}}, + {SPV_OPERAND_TYPE_LITERAL_INTEGER, + {static_cast(spv::BuiltIn::LocalInvocationIndex)}}})); + for (Instruction& entryPoint : irContext->module()->entry_points()) { + entryPoint.AddOperand(IdOp(liiVariableId)); + } + } + + // Scratch arrays, one per participating element type, each padded to whole + // virtual subgroups so every guarded read stays in bounds. + struct Scratch { + uint32_t variableId = 0; + uint32_t elementTypeId = 0; + uint32_t pointerTypeId = 0; + }; + std::map scratchByType; + const uint32_t cPadded = uintConst(paddedSlots); + if (cPadded == 0u) return Status::Failure; + const auto scratchFor = [&](uint32_t elementTypeId) -> const Scratch* { + auto it = scratchByType.find(elementTypeId); + if (it != scratchByType.end()) return &it->second; + const uint32_t arrayTypeId = irContext->TakeNextId(); + irContext->AddType(spvtools::MakeUnique( + irContext, spv::Op::OpTypeArray, 0, arrayTypeId, + Instruction::OperandList{IdOp(elementTypeId), IdOp(cPadded)})); + const uint32_t arrayPointerTypeId = + typeMgr->FindPointerToType(arrayTypeId, spv::StorageClass::Workgroup); + const uint32_t elementPointerTypeId = + typeMgr->FindPointerToType(elementTypeId, spv::StorageClass::Workgroup); + if (arrayPointerTypeId == 0u || elementPointerTypeId == 0u) return nullptr; + const uint32_t variableId = irContext->TakeNextId(); + irContext->AddGlobalValue(spvtools::MakeUnique( + irContext, spv::Op::OpVariable, arrayPointerTypeId, variableId, + Instruction::OperandList{ + {SPV_OPERAND_TYPE_STORAGE_CLASS, + {static_cast(spv::StorageClass::Workgroup)}}})); + Scratch scratch{variableId, elementTypeId, elementPointerTypeId}; + return &scratchByType.emplace(elementTypeId, scratch).first->second; + }; + + // The uint-domain twin of a participating type: bools travel as 0/1 words. + struct ScratchTypeInfo { + uint32_t typeId = 0; // element type stored in scratch + uint32_t scalarTypeId = 0; // its component type + bool boolConverted = false; + }; + const auto scratchTypeFor = [&](const ValueTypeInfo& info) -> ScratchTypeInfo { + if (info.kind != ScalarKind::Bool) { + return {info.typeId, info.scalarTypeId, false}; + } + if (info.componentCount == 1u) { + return {uintTypeId, uintTypeId, true}; + } + const analysis::Type* registeredUint = typeMgr->GetType(uintTypeId); + analysis::Vector vectorType(registeredUint, info.componentCount); + const uint32_t vectorTypeId = typeMgr->GetTypeInstruction(&vectorType); + return {vectorTypeId, uintTypeId, true}; + }; + + const auto boolVectorTypeId = [&](uint32_t componentCount) -> uint32_t { + if (componentCount == 1u) return boolTypeId; + const analysis::Type* registeredBool = typeMgr->GetType(boolTypeId); + analysis::Vector vectorType(registeredBool, componentCount); + return typeMgr->GetTypeInstruction(&vectorType); + }; + + // Rewrite the builtin loads. Every rewrite keeps the original result id, so + // downstream uses see the derived value with no further surgery. + const uint32_t numVirtualConstId = uintConst(virtualSubgroups); + if (numVirtualConstId == 0u) return Status::Failure; + + // The x word of one of the five ballot-mask builtins for the calling + // invocation's virtual subgroup, bounded to lanes that exist (words y..w + // are zero at 32 lanes). + const auto emitMaskWord = [&](Emitter& em, spv::BuiltIn builtin) -> uint32_t { + const uint32_t index = + em.Emit(spv::Op::OpLoad, uintTypeId, {IdOp(liiVariableId)}); + const uint32_t lane = + em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(index), IdOp(c31)}); + const uint32_t base = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(index), IdOp(cNotLane)}); + const uint32_t tailBits = + em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(cTotal), IdOp(base)}); + const uint32_t full = em.Emit(spv::Op::OpUGreaterThanEqual, boolTypeId, + {IdOp(tailBits), IdOp(c32)}); + const uint32_t tailShift = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId, + {IdOp(c1), IdOp(tailBits)}); + const uint32_t tailMask = + em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(tailShift), IdOp(c1)}); + const uint32_t exists = em.Emit(spv::Op::OpSelect, uintTypeId, + {IdOp(full), IdOp(cAllOnes), IdOp(tailMask)}); + const auto lowMaskThroughLane = [&](bool inclusive) -> uint32_t { + if (!inclusive) { + const uint32_t bit = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId, + {IdOp(c1), IdOp(lane)}); + return em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(bit), IdOp(c1)}); + } + const uint32_t lanePlusOne = + em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(lane), IdOp(c1)}); + const uint32_t atTop = em.Emit(spv::Op::OpUGreaterThanEqual, boolTypeId, + {IdOp(lanePlusOne), IdOp(c32)}); + const uint32_t shifted = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId, + {IdOp(c1), IdOp(lanePlusOne)}); + const uint32_t below = + em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(shifted), IdOp(c1)}); + return em.Emit(spv::Op::OpSelect, uintTypeId, + {IdOp(atTop), IdOp(cAllOnes), IdOp(below)}); + }; + uint32_t word = 0; + switch (builtin) { + case spv::BuiltIn::SubgroupEqMask: + word = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId, + {IdOp(c1), IdOp(lane)}); + break; + case spv::BuiltIn::SubgroupLtMask: + word = lowMaskThroughLane(false); + break; + case spv::BuiltIn::SubgroupLeMask: + word = lowMaskThroughLane(true); + break; + case spv::BuiltIn::SubgroupGeMask: + word = em.Emit(spv::Op::OpNot, uintTypeId, {IdOp(lowMaskThroughLane(false))}); + break; + case spv::BuiltIn::SubgroupGtMask: + word = em.Emit(spv::Op::OpNot, uintTypeId, {IdOp(lowMaskThroughLane(true))}); + break; + default: + return 0u; + } + return em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(word), IdOp(exists)}); + }; + + for (const BuiltinUse& use : builtinUses) { + for (Instruction* load : use.loads) { + Emitter em(irContext, load); + switch (use.builtin) { + case spv::BuiltIn::SubgroupSize: + load->SetOpcode(spv::Op::OpCopyObject); + load->SetInOperands({IdOp(c32)}); + break; + case spv::BuiltIn::NumSubgroups: + load->SetOpcode(spv::Op::OpCopyObject); + load->SetInOperands({IdOp(numVirtualConstId)}); + break; + case spv::BuiltIn::SubgroupLocalInvocationId: { + const uint32_t index = + em.Emit(spv::Op::OpLoad, uintTypeId, {IdOp(liiVariableId)}); + load->SetOpcode(spv::Op::OpBitwiseAnd); + load->SetInOperands({IdOp(index), IdOp(c31)}); + break; + } + case spv::BuiltIn::SubgroupId: { + const uint32_t index = + em.Emit(spv::Op::OpLoad, uintTypeId, {IdOp(liiVariableId)}); + load->SetOpcode(spv::Op::OpShiftRightLogical); + load->SetInOperands({IdOp(index), IdOp(uintConst(kIdShift))}); + break; + } + default: { + const uint32_t word = emitMaskWord(em, use.builtin); + if (word == 0u) return Status::Failure; + load->SetOpcode(spv::Op::OpCompositeConstruct); + load->SetInOperands( + {IdOp(word), IdOp(c0), IdOp(c0), IdOp(c0)}); + break; + } + } + } + for (const auto& [load, component] : use.componentLoads) { + // gl_Subgroup*Mask.x reached through an access chain: only word 0 + // carries bits at 32 lanes. + Emitter em(irContext, load); + uint32_t value = c0; + if (component == 0u) { + value = emitMaskWord(em, use.builtin); + if (value == 0u) return Status::Failure; + } + load->SetOpcode(spv::Op::OpCopyObject); + load->SetInOperands({IdOp(value)}); + } + } + + // Rewrite the group operations. + for (const GroupOpSite& site : groupOps) { + Instruction* inst = site.inst; + Emitter em(irContext, inst); + + const uint32_t lii = + em.Emit(spv::Op::OpLoad, uintTypeId, {IdOp(liiVariableId)}); + const uint32_t lane = + em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(lii), IdOp(c31)}); + const uint32_t base = + em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(lii), IdOp(cNotLane)}); + + const auto barrier = [&]() { + const uint32_t scopeId = uintConst(static_cast(spv::Scope::Workgroup)); + const uint32_t semanticsId = uintConst( + static_cast(spv::MemorySemanticsMask::AcquireRelease) | + static_cast(spv::MemorySemanticsMask::WorkgroupMemory)); + em.EmitNoResult(spv::Op::OpControlBarrier, + {IdOp(scopeId), IdOp(scopeId), IdOp(semanticsId)}); + }; + + const ScratchTypeInfo scratchType = scratchTypeFor(site.valueType); + const auto toScratch = [&](uint32_t valueId) -> uint32_t { + if (!scratchType.boolConverted) return valueId; + const uint32_t ones = + typedConst(site.valueType, 1u, uintTypeId, scratchType.typeId); + const uint32_t zeros = + typedConst(site.valueType, 0u, uintTypeId, scratchType.typeId); + return em.Emit(spv::Op::OpSelect, scratchType.typeId, + {IdOp(valueId), IdOp(ones), IdOp(zeros)}); + }; + const auto fromScratch = [&](uint32_t valueId) -> uint32_t { + if (!scratchType.boolConverted) return valueId; + const uint32_t zeros = + typedConst(site.valueType, 0u, uintTypeId, scratchType.typeId); + return em.Emit(spv::Op::OpINotEqual, site.valueType.typeId, + {IdOp(valueId), IdOp(zeros)}); + }; + + const Scratch* scratch = nullptr; + const auto storeOwnSlot = [&](uint32_t valueId) -> bool { + scratch = scratchFor(scratchType.typeId); + if (scratch == nullptr) return false; + const uint32_t pointer = + em.Emit(spv::Op::OpAccessChain, scratch->pointerTypeId, + {IdOp(scratch->variableId), IdOp(lii)}); + em.EmitNoResult(spv::Op::OpStore, {IdOp(pointer), IdOp(valueId)}); + barrier(); + return true; + }; + const auto readSlot = [&](uint32_t indexId) -> uint32_t { + const uint32_t pointer = + em.Emit(spv::Op::OpAccessChain, scratch->pointerTypeId, + {IdOp(scratch->variableId), IdOp(indexId)}); + return em.Emit(spv::Op::OpLoad, scratchType.typeId, {IdOp(pointer)}); + }; + // A select whose condition is a scalar bool but whose values may be + // vectors; pre-SPIR-V-1.4 needs the condition splat to a bvec. + const auto guardedSelect = [&](uint32_t condId, uint32_t thenId, + uint32_t elseId) -> uint32_t { + const uint32_t componentCount = site.valueType.componentCount; + uint32_t conditionId = condId; + if (componentCount > 1u) { + const uint32_t conditionTypeId = boolVectorTypeId(componentCount); + Instruction::OperandList splat; + for (uint32_t i = 0; i < componentCount; ++i) splat.push_back(IdOp(condId)); + conditionId = + em.Emit(spv::Op::OpCompositeConstruct, conditionTypeId, std::move(splat)); + } + return em.Emit(spv::Op::OpSelect, scratchType.typeId, + {IdOp(conditionId), IdOp(thenId), IdOp(elseId)}); + }; + const auto extInst2 = [&](uint32_t typeId, uint32_t instNumber, uint32_t a, + uint32_t b) -> uint32_t { + const uint32_t setId = ensureGlslStd450(); + return em.Emit(spv::Op::OpExtInst, typeId, + {IdOp(setId), + {SPV_OPERAND_TYPE_EXTENSION_INSTRUCTION_NUMBER, {instNumber}}, + IdOp(a), IdOp(b)}); + }; + const auto extInst1 = [&](uint32_t typeId, uint32_t instNumber, + uint32_t a) -> uint32_t { + const uint32_t setId = ensureGlslStd450(); + return em.Emit(spv::Op::OpExtInst, typeId, + {IdOp(setId), + {SPV_OPERAND_TYPE_EXTENSION_INSTRUCTION_NUMBER, {instNumber}}, + IdOp(a)}); + }; + + // Combine two scratch-domain values with the site's operation. + const auto combine = [&](uint32_t a, uint32_t b) -> uint32_t { + const uint32_t typeId = scratchType.typeId; + switch (site.opcode) { + case spv::Op::OpGroupNonUniformIAdd: + return em.Emit(spv::Op::OpIAdd, typeId, {IdOp(a), IdOp(b)}); + case spv::Op::OpGroupNonUniformFAdd: + return em.Emit(spv::Op::OpFAdd, typeId, {IdOp(a), IdOp(b)}); + case spv::Op::OpGroupNonUniformIMul: + return em.Emit(spv::Op::OpIMul, typeId, {IdOp(a), IdOp(b)}); + case spv::Op::OpGroupNonUniformFMul: + return em.Emit(spv::Op::OpFMul, typeId, {IdOp(a), IdOp(b)}); + case spv::Op::OpGroupNonUniformSMin: + return extInst2(typeId, kGlslSMin, a, b); + case spv::Op::OpGroupNonUniformUMin: + return extInst2(typeId, kGlslUMin, a, b); + case spv::Op::OpGroupNonUniformFMin: + return extInst2(typeId, kGlslFMin, a, b); + case spv::Op::OpGroupNonUniformSMax: + return extInst2(typeId, kGlslSMax, a, b); + case spv::Op::OpGroupNonUniformUMax: + return extInst2(typeId, kGlslUMax, a, b); + case spv::Op::OpGroupNonUniformFMax: + return extInst2(typeId, kGlslFMax, a, b); + case spv::Op::OpGroupNonUniformBitwiseAnd: + case spv::Op::OpGroupNonUniformLogicalAnd: + return em.Emit(spv::Op::OpBitwiseAnd, typeId, {IdOp(a), IdOp(b)}); + case spv::Op::OpGroupNonUniformBitwiseOr: + case spv::Op::OpGroupNonUniformLogicalOr: + return em.Emit(spv::Op::OpBitwiseOr, typeId, {IdOp(a), IdOp(b)}); + case spv::Op::OpGroupNonUniformBitwiseXor: + case spv::Op::OpGroupNonUniformLogicalXor: + return em.Emit(spv::Op::OpBitwiseXor, typeId, {IdOp(a), IdOp(b)}); + default: + return 0u; + } + }; + // The identity element for the site's operation, in the scratch domain. + const auto identity = [&]() -> uint32_t { + const uint32_t typeId = scratchType.typeId; + const uint32_t scalarId = scratchType.scalarTypeId; + switch (site.opcode) { + case spv::Op::OpGroupNonUniformIAdd: + case spv::Op::OpGroupNonUniformBitwiseOr: + case spv::Op::OpGroupNonUniformBitwiseXor: + case spv::Op::OpGroupNonUniformLogicalOr: + case spv::Op::OpGroupNonUniformLogicalXor: + case spv::Op::OpGroupNonUniformFAdd: + case spv::Op::OpGroupNonUniformUMax: + // 0 and +0.0f share the bit pattern; exclusive-scan lane 0 of an + // FAdd therefore returns +0.0 even for a -0.0 input, matching + // the identity the SPIR-V spec assigns. + return typedConst(site.valueType, 0u, scalarId, typeId); + case spv::Op::OpGroupNonUniformIMul: + return typedConst(site.valueType, 1u, scalarId, typeId); + case spv::Op::OpGroupNonUniformFMul: + return typedConst(site.valueType, 0x3f800000u, scalarId, typeId); + case spv::Op::OpGroupNonUniformSMin: + return typedConst(site.valueType, 0x7fffffffu, scalarId, typeId); + case spv::Op::OpGroupNonUniformUMin: + return typedConst(site.valueType, 0xffffffffu, scalarId, typeId); + case spv::Op::OpGroupNonUniformFMin: + return typedConst(site.valueType, 0x7f800000u, scalarId, typeId); + case spv::Op::OpGroupNonUniformSMax: + return typedConst(site.valueType, 0x80000000u, scalarId, typeId); + case spv::Op::OpGroupNonUniformFMax: + return typedConst(site.valueType, 0xff800000u, scalarId, typeId); + case spv::Op::OpGroupNonUniformBitwiseAnd: + return typedConst(site.valueType, 0xffffffffu, scalarId, typeId); + case spv::Op::OpGroupNonUniformLogicalAnd: + return typedConst(site.valueType, 1u, scalarId, typeId); + default: + return 0u; + } + }; + + // The lanes-that-exist word for this virtual subgroup (tail handling). + const auto existsWord = [&]() -> uint32_t { + const uint32_t tailBits = + em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(cTotal), IdOp(base)}); + const uint32_t full = em.Emit(spv::Op::OpUGreaterThanEqual, boolTypeId, + {IdOp(tailBits), IdOp(c32)}); + const uint32_t shifted = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId, + {IdOp(c1), IdOp(tailBits)}); + const uint32_t mask = + em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(shifted), IdOp(c1)}); + return em.Emit(spv::Op::OpSelect, uintTypeId, + {IdOp(full), IdOp(cAllOnes), IdOp(mask)}); + }; + + const auto finish = [&](uint32_t finalId) { + inst->SetOpcode(spv::Op::OpCopyObject); + inst->SetInOperands({IdOp(finalId)}); + }; + + bool ok = true; + switch (site.opcode) { + case spv::Op::OpGroupNonUniformElect: { + finish(em.Emit(spv::Op::OpIEqual, boolTypeId, {IdOp(lane), IdOp(c0)})); + break; + } + case spv::Op::OpGroupNonUniformAll: + case spv::Op::OpGroupNonUniformAny: { + const bool isAll = site.opcode == spv::Op::OpGroupNonUniformAll; + ok = storeOwnSlot(toScratch(site.valueId)); + if (!ok) break; + const uint32_t neutral = uintConst(isAll ? 1u : 0u); + uint32_t acc = 0; + for (uint32_t k = 0; k < kWidth; ++k) { + const uint32_t slot = + em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(base), IdOp(uintConst(k))}); + const uint32_t exists = em.Emit(spv::Op::OpULessThan, boolTypeId, + {IdOp(slot), IdOp(cTotal)}); + const uint32_t value = readSlot(slot); + const uint32_t guarded = em.Emit( + spv::Op::OpSelect, uintTypeId, + {IdOp(exists), IdOp(value), IdOp(neutral)}); + acc = k == 0 ? guarded + : em.Emit(isAll ? spv::Op::OpBitwiseAnd : spv::Op::OpBitwiseOr, + uintTypeId, {IdOp(acc), IdOp(guarded)}); + } + barrier(); + finish(em.Emit(spv::Op::OpINotEqual, boolTypeId, {IdOp(acc), IdOp(c0)})); + break; + } + case spv::Op::OpGroupNonUniformAllEqual: { + ok = storeOwnSlot(toScratch(site.valueId)); + if (!ok) break; + const uint32_t reference = readSlot(base); + uint32_t acc = cTrue; + for (uint32_t k = 0; k < kWidth; ++k) { + const uint32_t slot = + em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(base), IdOp(uintConst(k))}); + const uint32_t exists = em.Emit(spv::Op::OpULessThan, boolTypeId, + {IdOp(slot), IdOp(cTotal)}); + const uint32_t value = readSlot(slot); + const spv::Op compareOpcode = site.valueType.kind == ScalarKind::Float + ? spv::Op::OpFOrdEqual + : spv::Op::OpIEqual; + uint32_t equal = 0; + if (site.valueType.componentCount == 1u) { + equal = em.Emit(compareOpcode, boolTypeId, + {IdOp(value), IdOp(reference)}); + } else { + const uint32_t comparisonTypeId = + boolVectorTypeId(site.valueType.componentCount); + const uint32_t componentsEqual = em.Emit( + compareOpcode, comparisonTypeId, {IdOp(value), IdOp(reference)}); + equal = em.Emit(spv::Op::OpAll, boolTypeId, {IdOp(componentsEqual)}); + } + const uint32_t guarded = em.Emit( + spv::Op::OpSelect, boolTypeId, + {IdOp(exists), IdOp(equal), IdOp(cTrue)}); + acc = em.Emit(spv::Op::OpLogicalAnd, boolTypeId, + {IdOp(acc), IdOp(guarded)}); + } + barrier(); + finish(acc); + break; + } + case spv::Op::OpGroupNonUniformBroadcast: + case spv::Op::OpGroupNonUniformBroadcastFirst: + case spv::Op::OpGroupNonUniformShuffle: + case spv::Op::OpGroupNonUniformShuffleXor: + case spv::Op::OpGroupNonUniformShuffleUp: + case spv::Op::OpGroupNonUniformShuffleDown: + case spv::Op::OpGroupNonUniformQuadBroadcast: + case spv::Op::OpGroupNonUniformQuadSwap: { + ok = storeOwnSlot(toScratch(site.valueId)); + if (!ok) break; + uint32_t sourceLane = 0; + switch (site.opcode) { + case spv::Op::OpGroupNonUniformBroadcastFirst: + sourceLane = c0; + break; + case spv::Op::OpGroupNonUniformBroadcast: + case spv::Op::OpGroupNonUniformShuffle: + sourceLane = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(site.indexId), IdOp(c31)}); + break; + case spv::Op::OpGroupNonUniformShuffleXor: { + const uint32_t flipped = em.Emit( + spv::Op::OpBitwiseXor, uintTypeId, {IdOp(lane), IdOp(site.indexId)}); + sourceLane = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(flipped), IdOp(c31)}); + break; + } + case spv::Op::OpGroupNonUniformShuffleUp: { + const uint32_t shifted = + em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(lane), IdOp(site.indexId)}); + sourceLane = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(shifted), IdOp(c31)}); + break; + } + case spv::Op::OpGroupNonUniformShuffleDown: { + const uint32_t shifted = + em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(lane), IdOp(site.indexId)}); + sourceLane = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(shifted), IdOp(c31)}); + break; + } + case spv::Op::OpGroupNonUniformQuadBroadcast: { + const uint32_t quadBase = + em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(lane), IdOp(uintConst(~3u & kLaneMask))}); + const uint32_t withinQuad = + em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(site.indexId), IdOp(uintConst(3u))}); + sourceLane = em.Emit(spv::Op::OpIAdd, uintTypeId, + {IdOp(quadBase), IdOp(withinQuad)}); + break; + } + case spv::Op::OpGroupNonUniformQuadSwap: { + const uint32_t partnerMask = uintConst( + site.quadDirection == 0u ? 1u : site.quadDirection == 1u ? 2u : 3u); + sourceLane = em.Emit(spv::Op::OpBitwiseXor, uintTypeId, + {IdOp(lane), IdOp(partnerMask)}); + break; + } + default: + break; + } + const uint32_t sourceSlot = + em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(base), IdOp(sourceLane)}); + const uint32_t value = readSlot(sourceSlot); + barrier(); + finish(fromScratch(value)); + break; + } + case spv::Op::OpGroupNonUniformBallot: { + ok = storeOwnSlot(toScratch(site.valueId)); + if (!ok) break; + uint32_t word = c0; + for (uint32_t k = 0; k < kWidth; ++k) { + const uint32_t slot = + em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(base), IdOp(uintConst(k))}); + const uint32_t exists = em.Emit(spv::Op::OpULessThan, boolTypeId, + {IdOp(slot), IdOp(cTotal)}); + const uint32_t value = readSlot(slot); + const uint32_t guarded = em.Emit( + spv::Op::OpSelect, uintTypeId, {IdOp(exists), IdOp(value), IdOp(c0)}); + const uint32_t bit = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId, + {IdOp(guarded), IdOp(uintConst(k))}); + word = em.Emit(spv::Op::OpBitwiseOr, uintTypeId, {IdOp(word), IdOp(bit)}); + } + barrier(); + inst->SetOpcode(spv::Op::OpCompositeConstruct); + inst->SetInOperands({IdOp(word), IdOp(c0), IdOp(c0), IdOp(c0)}); + break; + } + case spv::Op::OpGroupNonUniformInverseBallot: { + const uint32_t word = em.Emit(spv::Op::OpCompositeExtract, uintTypeId, + {IdOp(site.valueId), + {SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}}); + const uint32_t shifted = em.Emit(spv::Op::OpShiftRightLogical, uintTypeId, + {IdOp(word), IdOp(lane)}); + const uint32_t bit = + em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(shifted), IdOp(c1)}); + finish(em.Emit(spv::Op::OpINotEqual, boolTypeId, {IdOp(bit), IdOp(c0)})); + break; + } + case spv::Op::OpGroupNonUniformBallotBitExtract: { + const uint32_t words[4] = { + em.Emit(spv::Op::OpCompositeExtract, uintTypeId, + {IdOp(site.valueId), {SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}}), + em.Emit(spv::Op::OpCompositeExtract, uintTypeId, + {IdOp(site.valueId), {SPV_OPERAND_TYPE_LITERAL_INTEGER, {1u}}}), + em.Emit(spv::Op::OpCompositeExtract, uintTypeId, + {IdOp(site.valueId), {SPV_OPERAND_TYPE_LITERAL_INTEGER, {2u}}}), + em.Emit(spv::Op::OpCompositeExtract, uintTypeId, + {IdOp(site.valueId), {SPV_OPERAND_TYPE_LITERAL_INTEGER, {3u}}})}; + const uint32_t inFirstPair = em.Emit( + spv::Op::OpULessThan, boolTypeId, {IdOp(site.indexId), IdOp(uintConst(64u))}); + const uint32_t inWord0 = em.Emit( + spv::Op::OpULessThan, boolTypeId, {IdOp(site.indexId), IdOp(c32)}); + const uint32_t inWord2 = em.Emit( + spv::Op::OpULessThan, boolTypeId, {IdOp(site.indexId), IdOp(uintConst(96u))}); + const uint32_t firstPair = em.Emit( + spv::Op::OpSelect, uintTypeId, + {IdOp(inWord0), IdOp(words[0]), IdOp(words[1])}); + const uint32_t secondPair = em.Emit( + spv::Op::OpSelect, uintTypeId, + {IdOp(inWord2), IdOp(words[2]), IdOp(words[3])}); + const uint32_t word = em.Emit( + spv::Op::OpSelect, uintTypeId, + {IdOp(inFirstPair), IdOp(firstPair), IdOp(secondPair)}); + const uint32_t bitIndex = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(site.indexId), IdOp(c31)}); + const uint32_t shifted = em.Emit(spv::Op::OpShiftRightLogical, uintTypeId, + {IdOp(word), IdOp(bitIndex)}); + const uint32_t bit = + em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, {IdOp(shifted), IdOp(c1)}); + finish(em.Emit(spv::Op::OpINotEqual, boolTypeId, {IdOp(bit), IdOp(c0)})); + break; + } + case spv::Op::OpGroupNonUniformBallotBitCount: { + const uint32_t word = em.Emit(spv::Op::OpCompositeExtract, uintTypeId, + {IdOp(site.valueId), + {SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}}); + uint32_t masked = 0; + if (site.groupOperation == spv::GroupOperation::Reduce) { + masked = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(word), IdOp(existsWord())}); + } else { + const bool inclusive = + site.groupOperation == spv::GroupOperation::InclusiveScan; + uint32_t laneMask = 0; + if (inclusive) { + const uint32_t lanePlusOne = + em.Emit(spv::Op::OpIAdd, uintTypeId, {IdOp(lane), IdOp(c1)}); + const uint32_t atTop = + em.Emit(spv::Op::OpUGreaterThanEqual, boolTypeId, + {IdOp(lanePlusOne), IdOp(c32)}); + const uint32_t shifted = + em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId, + {IdOp(c1), IdOp(lanePlusOne)}); + const uint32_t below = + em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(shifted), IdOp(c1)}); + laneMask = em.Emit(spv::Op::OpSelect, uintTypeId, + {IdOp(atTop), IdOp(cAllOnes), IdOp(below)}); + } else { + const uint32_t bit = em.Emit(spv::Op::OpShiftLeftLogical, uintTypeId, + {IdOp(c1), IdOp(lane)}); + laneMask = + em.Emit(spv::Op::OpISub, uintTypeId, {IdOp(bit), IdOp(c1)}); + } + masked = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(word), IdOp(laneMask)}); + } + finish(em.Emit(spv::Op::OpBitCount, uintTypeId, {IdOp(masked)})); + break; + } + case spv::Op::OpGroupNonUniformBallotFindLSB: + case spv::Op::OpGroupNonUniformBallotFindMSB: { + const uint32_t word = em.Emit(spv::Op::OpCompositeExtract, uintTypeId, + {IdOp(site.valueId), + {SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}}); + const uint32_t masked = em.Emit(spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(word), IdOp(existsWord())}); + const uint32_t instNumber = + site.opcode == spv::Op::OpGroupNonUniformBallotFindLSB ? kGlslFindILsb + : kGlslFindUMsb; + finish(extInst1(uintTypeId, instNumber, masked)); + break; + } + default: { + // The arithmetic family. Serial, ascending-lane combines: the same + // left-associated order a host-side reference computes, and exact + // for every integer-valued input regardless of native topology. + ok = storeOwnSlot(toScratch(site.valueId)); + if (!ok) break; + uint32_t result = 0; + switch (site.groupOperation) { + case spv::GroupOperation::Reduce: { + uint32_t acc = readSlot(base); + for (uint32_t k = 1; k < kWidth; ++k) { + const uint32_t slot = em.Emit(spv::Op::OpIAdd, uintTypeId, + {IdOp(base), IdOp(uintConst(k))}); + const uint32_t exists = em.Emit(spv::Op::OpULessThan, boolTypeId, + {IdOp(slot), IdOp(cTotal)}); + const uint32_t value = readSlot(slot); + const uint32_t combined = combine(acc, value); + if (combined == 0u) { ok = false; break; } + acc = guardedSelect(exists, combined, acc); + } + result = acc; + break; + } + case spv::GroupOperation::InclusiveScan: { + // Slots at or below the own lane always exist, so the guard is + // purely the lane comparison. + uint32_t acc = readSlot(base); + for (uint32_t k = 1; k < kWidth; ++k) { + const uint32_t included = + em.Emit(spv::Op::OpULessThanEqual, boolTypeId, + {IdOp(uintConst(k)), IdOp(lane)}); + const uint32_t slot = em.Emit(spv::Op::OpIAdd, uintTypeId, + {IdOp(base), IdOp(uintConst(k))}); + const uint32_t value = readSlot(slot); + const uint32_t combined = combine(acc, value); + if (combined == 0u) { ok = false; break; } + acc = guardedSelect(included, combined, acc); + } + result = acc; + break; + } + case spv::GroupOperation::ExclusiveScan: { + uint32_t acc = identity(); + if (acc == 0u) { ok = false; break; } + for (uint32_t k = 0; k < kWidth; ++k) { + const uint32_t included = em.Emit( + spv::Op::OpULessThan, boolTypeId, {IdOp(uintConst(k)), IdOp(lane)}); + const uint32_t slot = em.Emit(spv::Op::OpIAdd, uintTypeId, + {IdOp(base), IdOp(uintConst(k))}); + const uint32_t value = readSlot(slot); + const uint32_t combined = combine(acc, value); + if (combined == 0u) { ok = false; break; } + acc = guardedSelect(included, combined, acc); + } + result = acc; + break; + } + case spv::GroupOperation::ClusteredReduce: { + const uint32_t cluster = + site.clusterSize >= kWidth ? kWidth : site.clusterSize; + const uint32_t clusterBase = em.Emit( + spv::Op::OpBitwiseAnd, uintTypeId, + {IdOp(lane), IdOp(uintConst(~(cluster - 1u) & kLaneMask))}); + const uint32_t firstSlot = em.Emit(spv::Op::OpIAdd, uintTypeId, + {IdOp(base), IdOp(clusterBase)}); + uint32_t acc = readSlot(firstSlot); + for (uint32_t k = 1; k < cluster; ++k) { + const uint32_t slot = em.Emit(spv::Op::OpIAdd, uintTypeId, + {IdOp(firstSlot), IdOp(uintConst(k))}); + const uint32_t exists = em.Emit(spv::Op::OpULessThan, boolTypeId, + {IdOp(slot), IdOp(cTotal)}); + const uint32_t value = readSlot(slot); + const uint32_t combined = combine(acc, value); + if (combined == 0u) { ok = false; break; } + acc = guardedSelect(exists, combined, acc); + } + result = acc; + break; + } + default: + ok = false; + break; + } + if (!ok) break; + barrier(); + finish(fromScratch(result)); + break; + } + } + if (!ok) { + // Phase A vetted every site, so a build failure here means an + // internal inconsistency; the module is already partially rewritten + // and must not be used. + return Status::Failure; + } + } + + // Widen subgroup-scoped barriers to the workgroup scope the emulation + // synchronizes at anyway; strictly stronger, so always safe. + const uint32_t workgroupScopeId = + uintConst(static_cast(spv::Scope::Workgroup)); + if (workgroupScopeId == 0u) return Status::Failure; + for (const BarrierSite& site : barriers) { + if (site.inst->opcode() == spv::Op::OpControlBarrier) { + if (site.executionScopeIsSubgroup) { + site.inst->SetInOperand(0, {workgroupScopeId}); + } + if (site.memoryScopeIsSubgroup) { + site.inst->SetInOperand(1, {workgroupScopeId}); + } + } else if (site.memoryScopeIsSubgroup) { + site.inst->SetInOperand(0, {workgroupScopeId}); + } + } + + // ---------------------------------------------------------------- Phase C + // The replaced builtin variables are dead now; remove them together with + // their decorations and interface entries, then drop the GroupNonUniform* + // capabilities nothing references any more. + irContext->InvalidateAnalysesExceptFor(IRContext::kAnalysisNone); + for (const BuiltinUse& use : builtinUses) { + const uint32_t variableId = use.variable->result_id(); + for (Instruction& entryPoint : irContext->module()->entry_points()) { + for (uint32_t operandIndex = entryPoint.NumInOperands(); operandIndex > 3;) { + --operandIndex; + if (entryPoint.GetInOperand(operandIndex).type == SPV_OPERAND_TYPE_ID && + entryPoint.GetSingleWordInOperand(operandIndex) == variableId) { + entryPoint.RemoveInOperand(operandIndex); + } + } + } + for (Instruction* chain : use.accessChains) { + irContext->KillNamesAndDecorates(chain); + irContext->KillInst(chain); + } + irContext->KillNamesAndDecorates(use.variable); + irContext->KillInst(use.variable); + } + + static constexpr std::array kSubgroupCapabilities = { + spv::Capability::GroupNonUniform, + spv::Capability::GroupNonUniformVote, + spv::Capability::GroupNonUniformArithmetic, + spv::Capability::GroupNonUniformBallot, + spv::Capability::GroupNonUniformShuffle, + spv::Capability::GroupNonUniformShuffleRelative, + spv::Capability::GroupNonUniformClustered, + spv::Capability::GroupNonUniformQuad, + spv::Capability::GroupNonUniformPartitionedNV}; + std::vector deadCapabilities; + for (Instruction& capability : irContext->module()->capabilities()) { + const auto value = + static_cast(capability.GetSingleWordInOperand(0)); + for (const spv::Capability candidate : kSubgroupCapabilities) { + if (value == candidate) { + deadCapabilities.push_back(&capability); + break; + } + } + } + for (Instruction* capability : deadCapabilities) { + irContext->KillInst(capability); + } + + irContext->InvalidateAnalysesExceptFor(IRContext::kAnalysisNone); + return Status::SuccessWithChange; + } + + spvtools::Optimizer::PassToken EmulateSubgroupsPass::CreateEmulateSubgroupsPass( + const Uint32 maxWorkgroupScratchBytes) { + return spvtools::Optimizer::PassToken( + MakeUnique(maxWorkgroupScratchBytes)); + } + } // namespace ShaderTranspiler + } // namespace MG_Util +} // namespace MobileGL diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.h b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.h new file mode 100644 index 00000000..d2868bec --- /dev/null +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.h @@ -0,0 +1,73 @@ +// MobileGL - MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/EmulateSubgroupsPass.h +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#pragma once + +#include "source/opt/pass.h" +#include "spirv-tools/optimizer.hpp" + +#include + +namespace MobileGL { + namespace MG_Util { + namespace ShaderTranspiler { + // Lowers every GL_KHR_shader_subgroup construct in a compute module onto a + // 32-lane VIRTUAL subgroup implemented with workgroup-shared memory. Virtual + // subgroups partition the workgroup by gl_LocalInvocationIndex: + // lane = index & 31, id = index >> 5, count = ceil(invocations / 32). + // + // This is a LAST-RESORT path, never a substitute for real subgroups: it only + // runs when MOBILEGL_MAGMA_EMULATE_SUBGROUP=1 is set explicitly and the device + // has no native subgroup support at all (SubgroupSupportPolicy.h). A device + // with native subgroup operations - however narrow - uses them natively, with + // FixIterationRPSubgroupScratchPass patching the known pack bug instead. + // + // Lowered constructs: + // - the builtins gl_SubgroupSize / gl_SubgroupInvocationID / gl_SubgroupID / + // gl_NumSubgroups and the five gl_Subgroup*Mask ballot builtins; + // - OpGroupNonUniform{Elect,All,Any,AllEqual,Broadcast,BroadcastFirst, + // Ballot,InverseBallot,BallotBitExtract,BallotBitCount,BallotFind{L,M}SB, + // Shuffle,ShuffleXor,ShuffleUp,ShuffleDown, + // , + // QuadBroadcast,QuadSwap}; + // - subgroupBarrier()/subgroupMemoryBarrier*() (their Subgroup scopes widen + // to Workgroup, which is strictly stronger). + // The output uses no GroupNonUniform* instruction or capability at all, which + // is what lets it run on devices with no subgroup feature bits. + // + // Semantic contract, narrower than native subgroups in exactly one way: every + // emulated exchange synchronizes through OpControlBarrier, so subgroup + // operations must sit in WORKGROUP-uniform control flow (the shape every + // Iris-style pack reduction has). GLSL already imposes this for barrier(); + // a subgroup op in divergent flow - legal on native subgroups - is undefined + // here. + // + // Fails (Status::Failure, leaving the input module unchanged) on anything it + // cannot lower faithfully: extended subgroup ops (partitioned-NV, rotate, + // quad-all/any), non-32-bit participating types, spec-constant workgroup + // sizes, a subgroup builtin reached by anything but a direct OpLoad, or a + // module whose lowering would add more workgroup scratch than + // maxWorkgroupScratchBytes (pass the device's maxComputeSharedMemorySize; + // 0 falls back to the 16384-byte Vulkan minimum). + class EmulateSubgroupsPass : public spvtools::opt::Pass { + public: + explicit EmulateSubgroupsPass(Uint32 maxWorkgroupScratchBytes) + : m_maxWorkgroupScratchBytes(maxWorkgroupScratchBytes) {} + + const char* name() const override { return "emulate-subgroups"; } + Status Process() override; + + static spvtools::Optimizer::PassToken CreateEmulateSubgroupsPass( + Uint32 maxWorkgroupScratchBytes); + + private: + Uint32 m_maxWorkgroupScratchBytes; + }; + } // namespace ShaderTranspiler + } // namespace MG_Util +} // namespace MobileGL diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.cpp b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.cpp new file mode 100644 index 00000000..3ca95682 --- /dev/null +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.cpp @@ -0,0 +1,352 @@ +// MobileGL - MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.cpp +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#include "FixIterationRPSubgroupScratchPass.h" + +#include "spirv.hpp" +#include "source/opt/def_use_manager.h" +#include "source/opt/instruction.h" +#include "source/opt/ir_context.h" +#include "source/opt/module.h" +#include "source/util/make_unique.h" + +#include +#include +#include + +namespace MobileGL { + namespace MG_Util { + namespace ShaderTranspiler { + namespace { + using spvtools::opt::Instruction; + using spvtools::opt::IRContext; + + // iterationRP's reduction fingerprint, spelled out. + constexpr uint32_t kIterationRPLocalSizeX = 32u; + constexpr uint32_t kIterationRPLocalSizeY = 16u; + constexpr uint32_t kIterationRPLocalSizeZ = 1u; + constexpr uint32_t kIterationRPInvocations = + kIterationRPLocalSizeX * kIterationRPLocalSizeY * kIterationRPLocalSizeZ; + constexpr uint32_t kIterationRPScratchLength = 32u; + + Instruction* FindBuiltinDefinition(IRContext* context, spv::BuiltIn builtin) { + auto* defUseMgr = context->get_def_use_mgr(); + for (auto& annotation : context->annotations()) { + if (annotation.opcode() != spv::Op::OpDecorate || annotation.NumInOperands() < 3) { + continue; + } + if (static_cast(annotation.GetSingleWordInOperand(1)) != + spv::Decoration::BuiltIn) { + continue; + } + if (static_cast(annotation.GetSingleWordInOperand(2)) != builtin) { + continue; + } + return defUseMgr->GetDef(annotation.GetSingleWordInOperand(0)); + } + return nullptr; + } + + // Walks an access-chain pointer expression back to the variable it is + // rooted at; returns nullptr for anything that is not a plain chain. + const Instruction* RootVariable(IRContext* context, uint32_t pointerId) { + auto* defUseMgr = context->get_def_use_mgr(); + const Instruction* def = defUseMgr->GetDef(pointerId); + while (def != nullptr) { + switch (def->opcode()) { + case spv::Op::OpVariable: + return def; + case spv::Op::OpAccessChain: + case spv::Op::OpInBoundsAccessChain: + case spv::Op::OpCopyObject: + def = defUseMgr->GetDef(def->GetSingleWordInOperand(0)); + break; + default: + return nullptr; + } + } + return nullptr; + } + + // vec2 of 32-bit float - the type of iterationRP's luminance/exposure + // accumulator and of its prefixSumCache entries. + bool IsVec2Float32(IRContext* context, uint32_t typeId) { + const Instruction* type = context->get_def_use_mgr()->GetDef(typeId); + if (type == nullptr || type->opcode() != spv::Op::OpTypeVector || + type->GetSingleWordInOperand(1) != 2u) { + return false; + } + const Instruction* component = + context->get_def_use_mgr()->GetDef(type->GetSingleWordInOperand(0)); + return component != nullptr && component->opcode() == spv::Op::OpTypeFloat && + component->GetSingleWordInOperand(0) == 32u; + } + } // namespace + + spvtools::opt::Pass::Status FixIterationRPSubgroupScratchPass::Process() { + auto* irContext = context(); + auto* defUseMgr = irContext->get_def_use_mgr(); + + // A device whose native width already satisfies the pack's assumption + // (>= 16 lanes -> at most 32 subgroups) needs no patch at all. + if (m_nativeSubgroupSize == 0u || m_nativeSubgroupSize >= 16u) { + return Status::SuccessWithoutChange; + } + const uint32_t requiredLength = + (kIterationRPInvocations + m_nativeSubgroupSize - 1u) / m_nativeSubgroupSize; + + for (const Instruction& entryPoint : irContext->module()->entry_points()) { + if (static_cast(entryPoint.GetSingleWordInOperand(0)) != + spv::ExecutionModel::GLCompute) { + return Status::SuccessWithoutChange; + } + } + + // Fingerprint 1: the pack's exposure-pass workgroup shape, 32x16x1. + const auto resolveUintConstant = [&](uint32_t id, uint32_t* value) { + const Instruction* def = defUseMgr->GetDef(id); + if (def == nullptr || def->opcode() != spv::Op::OpConstant) return false; + *value = def->GetSingleWordInOperand(0); + return true; + }; + uint32_t localSize[3] = {0, 0, 0}; + bool haveLocalSize = false; + if (Instruction* workgroupSize = + FindBuiltinDefinition(irContext, spv::BuiltIn::WorkgroupSize)) { + if (workgroupSize->opcode() == spv::Op::OpConstantComposite && + workgroupSize->NumInOperands() == 3) { + haveLocalSize = + resolveUintConstant(workgroupSize->GetSingleWordInOperand(0), &localSize[0]) && + resolveUintConstant(workgroupSize->GetSingleWordInOperand(1), &localSize[1]) && + resolveUintConstant(workgroupSize->GetSingleWordInOperand(2), &localSize[2]); + } + } + if (!haveLocalSize) { + for (const Instruction& mode : irContext->module()->execution_modes()) { + if (mode.opcode() == spv::Op::OpExecutionMode && + static_cast(mode.GetSingleWordInOperand(1)) == + spv::ExecutionMode::LocalSize) { + localSize[0] = mode.GetSingleWordInOperand(2); + localSize[1] = mode.GetSingleWordInOperand(3); + localSize[2] = mode.GetSingleWordInOperand(4); + haveLocalSize = true; + break; + } + } + } + if (!haveLocalSize || localSize[0] != kIterationRPLocalSizeX || + localSize[1] != kIterationRPLocalSizeY || localSize[2] != kIterationRPLocalSizeZ) { + return Status::SuccessWithoutChange; + } + + // Fingerprint 2: the reduction's subgroupInclusiveAdd on a vec2. + bool sawVec2InclusiveAdd = false; + for (auto& function : *irContext->module()) { + for (auto& block : function) { + for (auto& inst : block) { + if (inst.opcode() == spv::Op::OpGroupNonUniformFAdd && + static_cast(inst.GetSingleWordInOperand(1)) == + spv::GroupOperation::InclusiveScan && + IsVec2Float32(irContext, inst.type_id())) { + sawVec2InclusiveAdd = true; + } + } + } + } + if (!sawVec2InclusiveAdd) { + return Status::SuccessWithoutChange; + } + + // gl_SubgroupID, whose value range the pack's scratch size bakes in. + const Instruction* subgroupIdVariable = + FindBuiltinDefinition(irContext, spv::BuiltIn::SubgroupId); + if (subgroupIdVariable == nullptr || + subgroupIdVariable->opcode() != spv::Op::OpVariable) { + return Status::SuccessWithoutChange; + } + const uint32_t subgroupIdVariableId = subgroupIdVariable->result_id(); + + // Conservative taint walk over values, and through Function/Private + // temporaries by variable (glslang routinely spills builtin loads into + // locals before they reach an index expression). Over-tainting is safe: + // the candidate filter below still demands the exact vec2[32] shape. + std::unordered_map valueTainted; // result id -> tainted + std::unordered_map variableTainted; // variable id -> tainted + bool changedTaint = true; + while (changedTaint) { + changedTaint = false; + for (auto& function : *irContext->module()) { + for (auto& block : function) { + for (auto& inst : block) { + const spv::Op opcode = inst.opcode(); + if (opcode == spv::Op::OpStore) { + if (!valueTainted.count(inst.GetSingleWordInOperand(1))) continue; + const Instruction* root = + RootVariable(irContext, inst.GetSingleWordInOperand(0)); + if (root == nullptr) continue; + if (!variableTainted.count(root->result_id())) { + variableTainted[root->result_id()] = true; + changedTaint = true; + } + continue; + } + if (inst.result_id() == 0 || valueTainted.count(inst.result_id())) { + continue; + } + bool tainted = false; + if (opcode == spv::Op::OpLoad) { + const uint32_t pointerId = inst.GetSingleWordInOperand(0); + if (pointerId == subgroupIdVariableId) tainted = true; + const Instruction* root = RootVariable(irContext, pointerId); + if (root != nullptr && variableTainted.count(root->result_id())) { + tainted = true; + } + } else { + inst.ForEachInId([&](const uint32_t* operandId) { + if (valueTainted.count(*operandId)) tainted = true; + }); + } + if (tainted) { + valueTainted[inst.result_id()] = true; + changedTaint = true; + } + } + } + } + } + if (valueTainted.empty()) { + return Status::SuccessWithoutChange; + } + + // Fingerprint 3: workgroup-shared vec2[32] arrays whose access-chain + // index depends on gl_SubgroupID - the under-declared prefixSumCache. + std::map candidates; + for (auto& function : *irContext->module()) { + for (auto& block : function) { + for (auto& inst : block) { + if (inst.opcode() != spv::Op::OpAccessChain && + inst.opcode() != spv::Op::OpInBoundsAccessChain) { + continue; + } + if (inst.NumInOperands() < 2) continue; + if (!valueTainted.count(inst.GetSingleWordInOperand(1))) continue; + Instruction* baseVariable = + defUseMgr->GetDef(inst.GetSingleWordInOperand(0)); + if (baseVariable == nullptr || + baseVariable->opcode() != spv::Op::OpVariable || + static_cast( + baseVariable->GetSingleWordInOperand(0)) != + spv::StorageClass::Workgroup) { + continue; + } + candidates.emplace(baseVariable->result_id(), baseVariable); + } + } + } + if (candidates.empty()) { + return Status::SuccessWithoutChange; + } + + bool changedModule = false; + for (auto& entry : candidates) { + Instruction* variable = entry.second; + + // The variable must be reached exclusively through access chains (plus + // debug/decoration instructions): a whole-array load, store, or copy + // would change type with the array and is left alone. + bool onlyAccessChains = true; + const uint32_t variableId = variable->result_id(); + defUseMgr->ForEachUser(variable, [&](Instruction* user) { + switch (user->opcode()) { + case spv::Op::OpAccessChain: + case spv::Op::OpInBoundsAccessChain: + if (user->GetSingleWordInOperand(0) != variableId) { + onlyAccessChains = false; + } + return; + case spv::Op::OpName: + case spv::Op::OpDecorate: + return; + default: + onlyAccessChains = false; + return; + } + }); + if (!onlyAccessChains) continue; + if (variable->NumInOperands() > 1) continue; // initializer: leave alone + + const Instruction* pointerType = defUseMgr->GetDef(variable->type_id()); + if (pointerType == nullptr || pointerType->opcode() != spv::Op::OpTypePointer) { + continue; + } + const Instruction* arrayType = + defUseMgr->GetDef(pointerType->GetSingleWordInOperand(1)); + if (arrayType == nullptr || arrayType->opcode() != spv::Op::OpTypeArray) { + continue; + } + const uint32_t elementTypeId = arrayType->GetSingleWordInOperand(0); + if (!IsVec2Float32(irContext, elementTypeId)) continue; + const Instruction* lengthConstant = + defUseMgr->GetDef(arrayType->GetSingleWordInOperand(1)); + uint32_t currentLength = 0; + if (lengthConstant == nullptr || + lengthConstant->opcode() != spv::Op::OpConstant || + !((currentLength = lengthConstant->GetSingleWordInOperand(0), + currentLength == kIterationRPScratchLength))) { + continue; + } + if (currentLength >= requiredLength) continue; + + // Build the grown array type. All three new instructions are inserted + // immediately BEFORE the variable so definition-before-use holds in the + // module's global section (manager-created instructions append to its + // end, after the variable). The new length constant reuses the old + // one's integer type, whatever signedness glslang gave it (a duplicate + // scalar constant is legal SPIR-V); the fresh array type makes the + // pointer type unique by construction, so neither collides with an + // existing declaration. + const uint32_t lengthTypeId = lengthConstant->type_id(); + const uint32_t newLengthId = irContext->TakeNextId(); + variable->InsertBefore(spvtools::MakeUnique( + irContext, spv::Op::OpConstant, lengthTypeId, newLengthId, + Instruction::OperandList{{SPV_OPERAND_TYPE_TYPED_LITERAL_NUMBER, + {requiredLength}}})); + const uint32_t newArrayTypeId = irContext->TakeNextId(); + variable->InsertBefore(spvtools::MakeUnique( + irContext, spv::Op::OpTypeArray, 0, newArrayTypeId, + Instruction::OperandList{ + {SPV_OPERAND_TYPE_ID, {elementTypeId}}, + {SPV_OPERAND_TYPE_ID, {newLengthId}}})); + const uint32_t newPointerTypeId = irContext->TakeNextId(); + variable->InsertBefore(spvtools::MakeUnique( + irContext, spv::Op::OpTypePointer, 0, newPointerTypeId, + Instruction::OperandList{ + {SPV_OPERAND_TYPE_STORAGE_CLASS, + {static_cast(spv::StorageClass::Workgroup)}}, + {SPV_OPERAND_TYPE_ID, {newArrayTypeId}}})); + + variable->SetResultType(newPointerTypeId); + changedModule = true; + } + + if (!changedModule) { + return Status::SuccessWithoutChange; + } + irContext->InvalidateAnalysesExceptFor(IRContext::kAnalysisNone); + return Status::SuccessWithChange; + } + + spvtools::Optimizer::PassToken + FixIterationRPSubgroupScratchPass::CreateFixIterationRPSubgroupScratchPass( + const Uint32 nativeSubgroupSize) { + return spvtools::Optimizer::PassToken( + MakeUnique(nativeSubgroupSize)); + } + } // namespace ShaderTranspiler + } // namespace MG_Util +} // namespace MobileGL diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.h b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.h new file mode 100644 index 00000000..db3b7d6d --- /dev/null +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.h @@ -0,0 +1,64 @@ +// MobileGL - MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FixIterationRPSubgroupScratchPass.h +// Copyright (c) 2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#pragma once + +#include "source/opt/pass.h" +#include "spirv-tools/optimizer.hpp" + +#include + +namespace MobileGL { + namespace MG_Util { + namespace ShaderTranspiler { + // Patches ONE known shader-pack defect: iterationRP's auto-exposure reduction + // declares `shared vec2 prefixSumCache[32]` for its 512-invocation workgroup + // and stores per-subgroup subtotals through prefixSumCache[gl_SubgroupID]. + // The pack hard-sized that scratch for the >=16-lane subgroups desktop GL + // drivers ship; on a narrower Vulkan device (lavapipe's 8 lanes -> 64 + // subgroups) every subgroup past entry 31 indexes shared memory out of + // bounds - on a CPU rasterizer that is literal heap corruption. The + // reduction ALGORITHM is width-agnostic (its combine loop is sized by + // gl_NumSubgroups), so the faithful repair is to grow the one under-declared + // array to ceil(512 / native width) and change nothing else. This is the + // pack author's bug, not MobileGL's; the patch is therefore deliberately + // NOT a general mechanism - it only rewrites modules that positively match + // iterationRP's reduction fingerprint: + // - GLCompute entry point with local size exactly 32x16x1; + // - a subgroupInclusiveAdd on a vec2 (OpGroupNonUniformFAdd InclusiveScan, + // the pack's luminance/exposure accumulator signature); + // - a workgroup-shared array of exactly vec2[32] whose access-chain index + // is data-dependent on gl_SubgroupID. + // Matching at the SPIR-V level keeps the recognition robust against + // whitespace/identifier-level drift that made the old source-text template + // rewrite (removed in 7769156) so brittle, while still refusing to touch + // anything that is not this pack's reduction. On devices whose native width + // already satisfies the pack's assumption (>= 16 lanes: desktop GL, Adreno), + // the grown length equals or undershoots the declared 32 and every module + // passes through byte-identical. + // + // The pass never fails a module: anything it cannot prove is this exact + // pattern - or cannot grow safely (a whole-array use, a spec-constant + // length, an initializer) - is left exactly as it was. + class FixIterationRPSubgroupScratchPass : public spvtools::opt::Pass { + public: + explicit FixIterationRPSubgroupScratchPass(Uint32 nativeSubgroupSize) + : m_nativeSubgroupSize(nativeSubgroupSize) {} + + const char* name() const override { return "fix-iterationrp-subgroup-scratch"; } + Status Process() override; + + static spvtools::Optimizer::PassToken CreateFixIterationRPSubgroupScratchPass( + Uint32 nativeSubgroupSize); + + private: + Uint32 m_nativeSubgroupSize; + }; + } // namespace ShaderTranspiler + } // namespace MG_Util +} // namespace MobileGL diff --git a/android-plugin/app/src/trace/cpp/trace_replay_core.cpp b/android-plugin/app/src/trace/cpp/trace_replay_core.cpp index 7a700921..151fa98c 100644 --- a/android-plugin/app/src/trace/cpp/trace_replay_core.cpp +++ b/android-plugin/app/src/trace/cpp/trace_replay_core.cpp @@ -164,11 +164,6 @@ bool LoadMobileGL(const Request& request, std::string& error) { } else { unsetenv("MOBILEGL_COHERENT_AS_FLUSH"); } - if (request.numSubgroupsQuirk) { - setenv("MOBILEGL_NUM_SUBGROUPS_QUIRK", "1", 1); - } else { - unsetenv("MOBILEGL_NUM_SUBGROUPS_QUIRK"); - } if (request.fboAttachmentDumps.empty()) { unsetenv("MOBILEGL_TRACE_DUMP_FBO_ATTACHMENTS"); } else { @@ -823,7 +818,6 @@ bool WriteResultJson(const Request& request, const Result& result) { << (request.avoidAngleLlvmpipeSamplerMipmapMinFilter ? "true" : "false") << ",\n"; file << " \"avoidAngleLlvmpipeExplicitLodBias\": " << (request.avoidAngleLlvmpipeExplicitLodBias ? "true" : "false") << ",\n"; - file << " \"numSubgroupsQuirk\": " << (request.numSubgroupsQuirk ? "true" : "false") << ",\n"; file << " \"holdMs\": " << request.holdMs << ",\n"; file << " \"mismatchPixels\": " << result.mismatchPixels << "\n"; file << "}\n"; diff --git a/android-plugin/app/src/trace/cpp/trace_replay_core.hpp b/android-plugin/app/src/trace/cpp/trace_replay_core.hpp index aab23e71..cc550378 100644 --- a/android-plugin/app/src/trace/cpp/trace_replay_core.hpp +++ b/android-plugin/app/src/trace/cpp/trace_replay_core.hpp @@ -44,7 +44,6 @@ struct Request { bool avoidAngleLlvmpipeSamplerMipmapMinFilter = false; bool avoidAngleLlvmpipeExplicitLodBias = false; bool coherentAsFlush = false; - bool numSubgroupsQuirk = false; int holdMs = 0; }; diff --git a/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp b/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp index 590d15df..8c6e1a6e 100644 --- a/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp +++ b/android-plugin/app/src/trace/cpp/trace_replay_jni.cpp @@ -122,7 +122,6 @@ Java_top_mobilegl_plugin_trace_TraceReplayActivity_nativeRunTraceReplay(JNIEnv* jboolean avoidAngleLlvmpipeSamplerMipmapMinFilter, jboolean avoidAngleLlvmpipeExplicitLodBias, jboolean coherentAsFlush, - jboolean numSubgroupsQuirk, jstring texture2dDumps) { mobilegl_trace::Request request; request.tracePath = ToString(env, tracePath); @@ -151,7 +150,6 @@ Java_top_mobilegl_plugin_trace_TraceReplayActivity_nativeRunTraceReplay(JNIEnv* avoidAngleLlvmpipeSamplerMipmapMinFilter == JNI_TRUE; request.avoidAngleLlvmpipeExplicitLodBias = avoidAngleLlvmpipeExplicitLodBias == JNI_TRUE; request.coherentAsFlush = coherentAsFlush == JNI_TRUE; - request.numSubgroupsQuirk = numSubgroupsQuirk == JNI_TRUE; ScopedTraceReplayState replayState; mobilegl_trace_set_requested_size(request.width, request.height); diff --git a/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java b/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java index b2bacd3c..e3c91aa1 100644 --- a/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java +++ b/android-plugin/app/src/trace/java/top/mobilegl/plugin/trace/TraceReplayActivity.java @@ -116,7 +116,6 @@ public final class TraceReplayActivity extends Activity { request.avoidAngleLlvmpipeSamplerMipmapMinFilter, request.avoidAngleLlvmpipeExplicitLodBias, request.coherentAsFlush, - request.numSubgroupsQuirk, request.texture2dDumps ); Log.i(TAG, result.toString()); @@ -150,7 +149,6 @@ public final class TraceReplayActivity extends Activity { boolean avoidAngleLlvmpipeSamplerMipmapMinFilter, boolean avoidAngleLlvmpipeExplicitLodBias, boolean coherentAsFlush, - boolean numSubgroupsQuirk, String texture2dDumps ); @@ -176,7 +174,6 @@ public final class TraceReplayActivity extends Activity { final boolean avoidAngleLlvmpipeSamplerMipmapMinFilter; final boolean avoidAngleLlvmpipeExplicitLodBias; final boolean coherentAsFlush; - final boolean numSubgroupsQuirk; final String texture2dDumps; private TraceReplayRequest( @@ -201,8 +198,7 @@ public final class TraceReplayActivity extends Activity { boolean avoidAngleLlvmpipeSamplerMipmapMinFilter, boolean avoidAngleLlvmpipeExplicitLodBias, boolean coherentAsFlush, - boolean numSubgroupsQuirk, - String texture2dDumps + String texture2dDumps ) { this.tracePath = tracePath; this.goldenPath = goldenPath; @@ -225,7 +221,6 @@ public final class TraceReplayActivity extends Activity { this.avoidAngleLlvmpipeSamplerMipmapMinFilter = avoidAngleLlvmpipeSamplerMipmapMinFilter; this.avoidAngleLlvmpipeExplicitLodBias = avoidAngleLlvmpipeExplicitLodBias; this.coherentAsFlush = coherentAsFlush; - this.numSubgroupsQuirk = numSubgroupsQuirk; this.texture2dDumps = texture2dDumps; } @@ -254,7 +249,6 @@ public final class TraceReplayActivity extends Activity { intent.getBooleanExtra("avoid_angle_llvmpipe_sampler_mipmap_min_filter", false), intent.getBooleanExtra("avoid_angle_llvmpipe_explicit_lod_bias", false), intent.getBooleanExtra("coherent_as_flush", false), - intent.getBooleanExtra("num_subgroups_quirk", false), readString(intent, "texture_2d_dumps", "") ); } diff --git a/android-plugin/trace-replay-ci.sh b/android-plugin/trace-replay-ci.sh index 8f461a70..f80a79e2 100644 --- a/android-plugin/trace-replay-ci.sh +++ b/android-plugin/trace-replay-ci.sh @@ -31,7 +31,6 @@ Usage: [--avoid-angle-llvmpipe-sampler-mipmap-min-filter] \ [--avoid-angle-llvmpipe-explicit-lod-bias] \ [--coherent-as-flush] \ - [--num-subgroups-quirk] \ [--dump-texture-2d CALL,TEXTURE,LEVEL,DIR] \ --timeout-seconds N @@ -48,8 +47,6 @@ sample with an explicit LOD that ANGLE llvmpipe cannot take a LOD bias on (MOBILEGL_AVOID_EXPLICIT_LOD_BIAS=1). Pass --coherent-as-flush for traces whose engine writes persistent GL_MAP_FLUSH_EXPLICIT_BIT maps it never flushes (MOBILEGL_COHERENT_AS_FLUSH=1). -Pass --num-subgroups-quirk to derive compute gl_NumSubgroups instead of reading -the Vulkan builtin (MOBILEGL_NUM_SUBGROUPS_QUIRK=1). EOF } @@ -108,7 +105,6 @@ use_pbuffer=0 avoid_angle_llvmpipe_sampler_mipmap_min_filter=0 avoid_angle_llvmpipe_explicit_lod_bias=0 coherent_as_flush=0 -num_subgroups_quirk=0 texture_2d_dumps="" timeout_seconds="" @@ -150,7 +146,6 @@ while [ "$#" -gt 0 ]; do shift 1 ;; --coherent-as-flush) coherent_as_flush=1; shift 1 ;; - --num-subgroups-quirk) num_subgroups_quirk=1; shift 1 ;; --dump-texture-2d) texture_2d_dumps="$(next_arg "$@")"; shift 2 ;; --timeout-seconds) timeout_seconds="$(next_arg "$@")"; shift 2 ;; -h|--help) usage; exit 0 ;; @@ -368,9 +363,6 @@ run_retrace() { if [ "${coherent_as_flush}" -eq 1 ]; then set -- "$@" --ez coherent_as_flush true fi - if [ "${num_subgroups_quirk}" -eq 1 ]; then - set -- "$@" --ez num_subgroups_quirk true - fi if [ -n "${texture_2d_dumps}" ]; then set -- "$@" --es texture_2d_dumps "${texture_2d_dumps}" fi diff --git a/tools/trace_replay/trace_cases.json b/tools/trace_replay/trace_cases.json index 7cee7bc7..b8f4dc85 100644 --- a/tools/trace_replay/trace_cases.json +++ b/tools/trace_replay/trace_cases.json @@ -284,8 +284,7 @@ "golden": "minecraft-1.21.4-fabric-iris-iterationrp-in-world.0000202020.png", "target_call": 202020, "timeout_seconds": 1800, - "ssim_threshold": 0.98, - "num_subgroups_quirk": true + "ssim_threshold": 0.98 }, { "name": "minecraft-1.21.4-fabric-iris-bsl-esc-menu-854",