swung0x48/MobileGL
1
0
Fork 0
mirror of https://github.com/MobileGL-Dev/MobileGL synced 2026-09-07 19:58:32 +09:00
Code Issues Packages Projects Releases Wiki Activity

[Fix, Test] (ShaderTranspiler): flatten fp64 storage blocks whose last member is a runtime array - the pass declined them, so the fp64 demotion re-derived ArrayStride 4 over the application's 8/16/32-byte double buffer and every double/dvecN data[] SSBO read raw words

Browse Source
This commit is contained in:
swung0x48
2026-09-05 05:36:49 -04:00
parent 1e7ecab4db
commit 795e08f7e6
3 changed files with 1013 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
MobileGL/MG_Test/ShaderTranspiler/FlattenFloat64StorageBlockTest.cpp
+746
View File
@@ -22,6 +22,8 @@
#include "Includes.h"
#include "Init.h"
#include <MG_Util/ShaderTranspiler/ShaderCompiler.h>
#include <MG_Util/ShaderTranspiler/SpvcSession.h>
#include <MG_Util/ShaderTranspiler/Types.h>
#include <spirv-tools/libspirv.hpp>
@@ -38,6 +40,7 @@ namespace {
constexpr Uint32 kOpTypeInt = 21;
constexpr Uint32 kOpTypeFloat = 22;
constexpr Uint32 kOpTypeArray = 28;
constexpr Uint32 kOpTypeRuntimeArray = 29;
constexpr Uint32 kOpTypeStruct = 30;
constexpr Uint32 kOpConstant = 43;
constexpr Uint32 kDecorationArrayStride = 6;
@@ -116,6 +119,18 @@ namespace {
return {elementTypeId, length};
}
// The element type id of OpTypeRuntimeArray <arrayId>, or 0 when it is not one - which is
// what a BOUNDED flattened member (an OpTypeArray) answers too, so the two shapes can be told
// apart by the pair of helpers.
Uint32 RuntimeArrayElementOf(const Vector<Uint32>& spirv, Uint32 arrayId) {
Uint32 elementTypeId = 0;
ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) {
if (opcode != kOpTypeRuntimeArray || wordCount < 3 || words[1] != arrayId) return;
elementTypeId = words[2];
});
return elementTypeId;
}
Bool IsUint32Type(const Vector<Uint32>& spirv, Uint32 typeId) {
Bool isUint = false;
ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) {
@@ -140,6 +155,61 @@ namespace {
return text;
}
// How many lines of a disassembly hold BOTH fragments - "OpIMul %uint" and "%uint_8", say -
// which is how the index arithmetic the pass emits is pinned without a host that could run it.
Uint32 CountLinesWith(const String& text, const String& first, const String& second) {
Uint32 count = 0;
SizeT lineStart = 0;
while (lineStart < text.size()) {
SizeT lineEnd = text.find('\n', lineStart);
if (lineEnd == String::npos) lineEnd = text.size();
const String line = text.substr(lineStart, lineEnd - lineStart);
if (line.find(first) != String::npos && line.find(second) != String::npos) ++count;
lineStart = lineEnd + 1;
}
return count;
}
// What every test of the open-ended shape asserts: the block collapsed to ONE member, which
// is a `uint[]` RUNTIME array of stride 4 rather than a bounded one, and nothing 64-bit is
// left for the demotion to find. Returns the disassembly for the arithmetic checks.
String ExpectOpenEndedWordArray(const Vector<Uint32>& output, const String& blockName) {
const String text = Disassemble(output);
const Uint32 structId = StructIdNamed(output, blockName);
EXPECT_NE(structId, 0u) << text;
if (structId == 0) return text;
const Vector<Uint32> members = MemberTypesOf(output, structId);
EXPECT_EQ(members.size(), 1u) << "the block should have collapsed to one member\n" << text;
if (members.size() != 1) return text;
EXPECT_EQ(MemberOffsetsOf(output, structId), (Vector<Uint32>{0}));
const Uint32 elementTypeId = RuntimeArrayElementOf(output, members[0]);
EXPECT_NE(elementTypeId, 0u) << "member 0 is not a runtime array\n" << text;
EXPECT_EQ(ArrayShapeOf(output, members[0]).first, 0u)
<< "an open-ended block must not be given a bounded length\n"
<< text;
EXPECT_TRUE(IsUint32Type(output, elementTypeId)) << text;
EXPECT_EQ(DecorationValueOf(output, members[0], kDecorationArrayStride), 4u) << text;
EXPECT_EQ(CountFloatTypesOfWidth(output, 64), 0u) << text;
return text;
}
// The compute shape every failing KHR-Single-GL45.subgroups fp64 case binds: one runtime
// array of doubles, indexed by an invocation id, read whole-element.
String OpenEndedComputeSource(const String& elementType) {
return String(R"(#version 430 core
layout(local_size_x = 16) in;
layout(std430, binding = 0) buffer Sink { uint result[]; };
layout(std430, binding = 1) buffer Data { )") +
elementType + R"( data[]; };
void main() {
)" + elementType +
R"( value = data[gl_LocalInvocationID.x] * data[0];
result[gl_GlobalInvocationID.x] = uint(value)" +
(elementType == "double" ? String{} : String(".x")) + R"();
}
)";
}
Vector<Uint32> CompileToSpirv(GLenum stage, const String& source) {
using namespace MG_Util::ShaderTranspiler;
ShaderAttrib shaderAttrib{.shaderType = stage, .sourceStr = source};
@@ -345,3 +415,679 @@ TEST_F(FlattenFloat64StorageBlockTest, TheDemotedPathIsUnchangedByTheCapabilityA
EXPECT_EQ(explicitlyDemoted, defaulted);
EXPECT_EQ(CountFloatTypesOfWidth(defaulted, 64), 0u) << Disassemble(defaulted);
}
// ---------------------------------------------------------------------------
// The open-ended shape: a block whose last member is a runtime array. Before this was accepted
// the pass declined it and the demotion re-derived ArrayStride 4 for the now-float element, so
// `double data[]` read the application's 8-byte-stride buffer as 32-bit words - every fp64
// KHR-Single-GL45.subgroups case failed on exactly that.
// ---------------------------------------------------------------------------
TEST_F(FlattenFloat64StorageBlockTest, AnOpenEndedBlockOfDoublesBecomesAWordRuntimeArray) {
const Vector<Uint32> input = CompileToSpirv(GL_COMPUTE_SHADER, OpenEndedComputeSource("double"));
ASSERT_FALSE(input.empty());
const Vector<Uint32> output = Sanitize(input);
ASSERT_FALSE(output.empty());
const String text = ExpectOpenEndedWordArray(output, "Data");
// Element i of the original array starts at word 2i, so the dynamic index is scaled by 2 ...
EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_2"), 1u) << text;
// ... and the constant `data[0]` is the pair of words at 0 and 1, reached through the one
// member the block has left.
EXPECT_GE(CountLinesWith(text, "OpAccessChain %_ptr_StorageBuffer_uint", "%uint_0 %uint_0"), 1u) << text;
}
TEST_F(FlattenFloat64StorageBlockTest, EachDoubleVectorWidthStepsByItsOwnStride) {
struct Shape {
const char* element;
// std430 strides: dvec2 16 bytes, dvec3 and dvec4 32 bytes - i.e. 4, 8 and 8 words.
const char* strideWords;
// The last component's word offset inside one element, and the first one past it.
const char* lastComponentWords;
const char* firstWordPastIt;
};
const Shape shapes[] = {{"dvec2", "%uint_4", "%uint_2", "%uint_4"},
{"dvec3", "%uint_8", "%uint_4", "%uint_6"},
{"dvec4", "%uint_8", "%uint_6", "%uint_8"}};
for (const Shape& shape : shapes) {
SCOPED_TRACE(shape.element);
const Vector<Uint32> input = CompileToSpirv(GL_COMPUTE_SHADER, OpenEndedComputeSource(shape.element));
ASSERT_FALSE(input.empty());
const Vector<Uint32> output = Sanitize(input);
ASSERT_FALSE(output.empty());
const String text = ExpectOpenEndedWordArray(output, "Data");
EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", shape.strideWords), 1u) << text;
EXPECT_GE(CountLinesWith(text, "OpIAdd %uint", shape.lastComponentWords), 1u) << text;
// A dvec3 is six words in a stride of eight: nothing may be read from the padding.
EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", shape.firstWordPastIt), 0u) << text;
}
}
TEST_F(FlattenFloat64StorageBlockTest, AFixedPrefixBeforeTheRuntimeArrayIsAddedToEveryIndex) {
const String source = R"(#version 430 core
layout(local_size_x = 16) in;
layout(std430, binding = 0) buffer Sink { uint result[]; };
layout(std430, binding = 1) buffer Data {
uvec4 head;
double data[];
};
void main() {
result[gl_GlobalInvocationID.x] = head.x + uint(data[gl_LocalInvocationID.x]);
}
)";
const Vector<Uint32> input = CompileToSpirv(GL_COMPUTE_SHADER, source);
ASSERT_FALSE(input.empty());
const Uint32 inputStructId = StructIdNamed(input, "Data");
ASSERT_NE(inputStructId, 0u);
EXPECT_EQ(MemberOffsetsOf(input, inputStructId), (Vector<Uint32>{0, 16})) << Disassemble(input);
const Vector<Uint32> output = Sanitize(input);
ASSERT_FALSE(output.empty());
const String text = ExpectOpenEndedWordArray(output, "Data");
// The 16-byte prefix is 4 words: element i is at word 4 + 2i.
EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_2"), 1u) << text;
EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", "%uint_4"), 1u) << text;
// And the prefix member itself is still word 0.
EXPECT_GE(CountLinesWith(text, "OpAccessChain %_ptr_StorageBuffer_uint", "%uint_0 %uint_0"), 1u) << text;
}
// OpArrayLength on the flattened member counts WORDS. GL's `.length()` is the number of whole
// elements the bound range holds past the array's offset, so the count has to be rebased and
// divided - in unsigned arithmetic, and clamped rather than wrapped when the range is shorter
// than the prefix.
namespace {
// A prefix, an open-ended array of doubles, and a `.length()` of it - the one shape whose
// rewrite is an instruction SPIRV-Cross has to spell rather than plain arithmetic.
constexpr const char* kOpenEndedLengthSource = R"(#version 430 core
layout(local_size_x = 16) in;
layout(std430, binding = 0) buffer Sink { uint result[]; };
layout(std430, binding = 1) buffer Data {
uvec4 head;
double data[];
};
void main() {
result[gl_GlobalInvocationID.x] = uint(data.length()) + head.y;
}
)";
} // namespace
TEST_F(FlattenFloat64StorageBlockTest, TheLengthOfAnOpenEndedBlockIsRewrittenToAnElementCount) {
const Vector<Uint32> input = CompileToSpirv(GL_COMPUTE_SHADER, kOpenEndedLengthSource);
ASSERT_FALSE(input.empty());
// glslang asks for member 1's length and signs the answer.
EXPECT_EQ(CountLinesWith(Disassemble(input), "OpArrayLength %uint", " 1"), 1u) << Disassemble(input);
const Vector<Uint32> output = Sanitize(input);
ASSERT_FALSE(output.empty());
const String text = ExpectOpenEndedWordArray(output, "Data");
// Re-aimed at the one member left, ...
EXPECT_EQ(CountLinesWith(text, "OpArrayLength %uint", " 0"), 1u) << text;
EXPECT_EQ(CountLinesWith(text, "OpArrayLength %uint", " 1"), 0u) << text;
// ... rebased past the 4-word prefix, clamped at zero when the range does not reach it, ...
EXPECT_EQ(CountLinesWith(text, "OpISub %uint", "%uint_4"), 1u) << text;
EXPECT_EQ(CountLinesWith(text, "OpULessThan %bool", "%uint_4"), 1u) << text;
EXPECT_EQ(CountLinesWith(text, "OpSelect %uint", "%uint_0"), 1u) << text;
// ... and divided by the 2-word stride, with glslang's own sign conversion still downstream.
EXPECT_EQ(CountLinesWith(text, "OpUDiv %uint", "%uint_2"), 1u) << text;
EXPECT_EQ(CountLinesWith(text, "OpBitcast %int", ""), 1u) << text;
}
TEST_F(FlattenFloat64StorageBlockTest, TheLengthOfABlockWithNoPrefixNeedsNoClamp) {
const String source = R"(#version 430 core
layout(local_size_x = 16) in;
layout(std430, binding = 0) buffer Sink { uint result[]; };
layout(std430, binding = 1) buffer Data { dvec2 data[]; };
void main() {
result[gl_GlobalInvocationID.x] = uint(data.length());
}
)";
const Vector<Uint32> input = CompileToSpirv(GL_COMPUTE_SHADER, source);
ASSERT_FALSE(input.empty());
const Vector<Uint32> output = Sanitize(input);
ASSERT_FALSE(output.empty());
const String text = ExpectOpenEndedWordArray(output, "Data");
EXPECT_EQ(CountLinesWith(text, "OpArrayLength %uint", " 0"), 1u) << text;
// Nothing to subtract, so nothing to clamp: the word count over the 4-word stride is it.
EXPECT_EQ(CountLinesWith(text, "OpISub", ""), 0u) << text;
EXPECT_EQ(CountLinesWith(text, "OpSelect", ""), 0u) << text;
EXPECT_EQ(CountLinesWith(text, "OpUDiv %uint", "%uint_4"), 1u) << text;
}
// The graphics shape of the same CTS group: a fragment stage reading a `readonly` block. The
// NonWritable the qualifier became is a promise about the whole block, and has to be on the one
// member the flattened block keeps.
TEST_F(FlattenFloat64StorageBlockTest, AReadOnlyOpenEndedBlockKeepsNonWritable) {
const String source = R"(#version 450 core
layout(binding = 4, std430) readonly buffer Buffer4 { dvec3 data[]; };
layout(location = 0) out vec4 o_color;
void main() {
uint index = uint(gl_FragCoord.x);
o_color = vec4(float(data[index].z));
}
)";
const Vector<Uint32> input = CompileToSpirv(GL_FRAGMENT_SHADER, source);
ASSERT_FALSE(input.empty());
EXPECT_EQ(CountLinesWith(Disassemble(input), "OpMemberDecorate %Buffer4 0 NonWritable", ""), 1u)
<< Disassemble(input);
const Vector<Uint32> output = Sanitize(input);
ASSERT_FALSE(output.empty());
const String text = ExpectOpenEndedWordArray(output, "Buffer4");
EXPECT_EQ(CountLinesWith(text, "OpMemberDecorate %Buffer4 0 NonWritable", ""), 1u) << text;
// dvec3: stride 8 words, .z at +4.
EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_8"), 1u) << text;
EXPECT_GE(CountLinesWith(text, "OpIAdd %uint", "%uint_4"), 1u) << text;
}
// Writing through an open-ended block, which no CTS case does but any shader may: the store
// is decomposed into the same words the load would have read, so the bytes the application
// gets back are the ones GL says it wrote.
TEST_F(FlattenFloat64StorageBlockTest, AnOpenEndedBlockIsWrittenThroughTheSameWords) {
const String source = R"(#version 430 core
layout(local_size_x = 16) in;
layout(std430, binding = 1) buffer Data { double data[]; };
void main() {
data[gl_LocalInvocationID.x] = double(gl_LocalInvocationID.y);
}
)";
const Vector<Uint32> input = CompileToSpirv(GL_COMPUTE_SHADER, source);
ASSERT_FALSE(input.empty());
const Vector<Uint32> output = Sanitize(input);
ASSERT_FALSE(output.empty());
const String text = ExpectOpenEndedWordArray(output, "Data");
// One dynamic index, scaled to the 2-word element ...
EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_2"), 1u) << text;
// ... and the double left as exactly two word stores, nothing wider.
EXPECT_EQ(CountLinesWith(text, "OpStore", ""), 2u) << text;
EXPECT_EQ(CountLinesWith(text, "OpAccessChain %_ptr_StorageBuffer_uint", ""), 2u) << text;
}
// The exact compute shader KHR-Single-GL45.subgroups.arithmetic.compute.subgroupmul_double
// generates, so the CTS shape is pinned as it is and not as a paraphrase of it.
TEST_F(FlattenFloat64StorageBlockTest, TheSubgroupMulDoubleComputeShaderIsFlattened) {
const String source = R"(#version 450
#extension GL_KHR_shader_subgroup_arithmetic: enable
#extension GL_KHR_shader_subgroup_ballot: enable
layout (local_size_x = 16, local_size_y = 1, local_size_z = 1) in;
layout(binding = 0, std430) buffer Buffer0
{
uint result[];
};
layout(binding = 1, std430) buffer Buffer1
{
double data[];
};
void main (void)
{
uvec3 globalSize = gl_NumWorkGroups * gl_WorkGroupSize;
highp uint offset = globalSize.x * ((globalSize.y * gl_GlobalInvocationID.z) + gl_GlobalInvocationID.y) + gl_GlobalInvocationID.x;
uvec4 mask = subgroupBallot(true);
uint start = 0u, end = gl_SubgroupSize;
double ref = double(1);
uint tempResult = 0u;
for (uint index = start; index < end; index++)
{
if (subgroupBallotBitExtract(mask, index))
{
ref = ref * data[index];
}
}
tempResult = (abs(ref - subgroupMul(data[gl_SubgroupInvocationID])) < 0.00001) ? 0x1u : 0u;
if (1u == (gl_SubgroupInvocationID % 2u))
{
mask = subgroupBallot(true);
ref = double(1);
for (uint index = start; index < end; index++)
{
if (subgroupBallotBitExtract(mask, index))
{
ref = ref * data[index];
}
}
tempResult |= (abs(ref - subgroupMul(data[gl_SubgroupInvocationID])) < 0.00001) ? 0x2u : 0u;
}
else
{
tempResult |= 0x2u;
}
result[offset] = tempResult;
}
)";
const Vector<Uint32> input = CompileToSpirv(GL_COMPUTE_SHADER, source);
ASSERT_FALSE(input.empty());
const Vector<Uint32> output = Sanitize(input);
ASSERT_FALSE(output.empty());
const String text = ExpectOpenEndedWordArray(output, "Buffer1");
// Four reads of the array, each scaled to the 2-word element.
EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_2"), 4u) << text;
// The result block holds no double and is not the pass's business.
const Uint32 resultStructId = StructIdNamed(output, "Buffer0");
ASSERT_NE(resultStructId, 0u) << text;
const Vector<Uint32> resultMembers = MemberTypesOf(output, resultStructId);
ASSERT_EQ(resultMembers.size(), 1u);
EXPECT_TRUE(IsUint32Type(output, RuntimeArrayElementOf(output, resultMembers[0]))) << text;
}
// A runtime array whose element is a MATRIX. The member's own MatrixStride and RowMajor
// decorations describe those elements, so a row-major one has to be declined - its columns are
// not contiguous, and addressing it in column order against a row-major buffer would be silently
// wrong bytes rather than a refusal. The column-major twin must flatten, stepping by the
// element's stride and then by the column's.
TEST_F(FlattenFloat64StorageBlockTest, ARowMajorMatrixRuntimeArrayIsLeftToTheDemotion) {
const String source = R"(#version 430 core
layout(local_size_x = 16) in;
layout(std430, binding = 0) buffer Sink { uint result[]; };
layout(std430, binding = 1, row_major) buffer Data { dmat4 data[]; };
void main() {
result[gl_GlobalInvocationID.x] = uint(data[gl_LocalInvocationID.x][1][2]);
}
)";
const Vector<Uint32> input = CompileToSpirv(GL_COMPUTE_SHADER, source);
ASSERT_FALSE(input.empty());
// The premise: glslang really did mark the member row-major.
EXPECT_EQ(CountLinesWith(Disassemble(input), "OpMemberDecorate %Data 0 RowMajor", ""), 1u)
<< Disassemble(input);
const Vector<Uint32> output = Sanitize(input);
ASSERT_FALSE(output.empty());
const String text = Disassemble(output);
const Uint32 structId = StructIdNamed(output, "Data");
ASSERT_NE(structId, 0u) << text;
const Vector<Uint32> members = MemberTypesOf(output, structId);
ASSERT_EQ(members.size(), 1u) << text;
// Still a runtime array of matrices - narrowed to fp32 by the demotion, not re-addressed.
EXPECT_NE(DecorationValueOf(output, members[0], kDecorationArrayStride), 4u)
<< "a row-major matrix element must not have been flattened into words\n"
<< text;
EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", ""), 0u)
<< "nothing should have been re-addressed\n"
<< text;
}
TEST_F(FlattenFloat64StorageBlockTest, AColumnMajorMatrixRuntimeArrayStepsByItsColumnStride) {
const String source = R"(#version 430 core
layout(local_size_x = 16) in;
layout(std430, binding = 0) buffer Sink { uint result[]; };
layout(std430, binding = 1) buffer Data { dmat2x4 data[]; };
void main() {
dvec4 column = data[gl_LocalInvocationID.x][1];
result[gl_GlobalInvocationID.x] = uint(column.w);
}
)";
const Vector<Uint32> input = CompileToSpirv(GL_COMPUTE_SHADER, source);
ASSERT_FALSE(input.empty());
const Vector<Uint32> output = Sanitize(input);
ASSERT_FALSE(output.empty());
const String text = ExpectOpenEndedWordArray(output, "Data");
// dmat2x4: two columns of dvec4, column stride 32 bytes, so one element is 64 bytes -
// 16 words - and column 1 starts 8 words into it.
EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_16"), 1u) << text;
// Exactly one +8: the column's own offset inside the element. A second would mean a word
// past the column was being addressed off that same base.
EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", "%uint_8"), 1u) << text;
// All eight words of that column are read - the last of its four doubles ends at +7 ...
EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", "%uint_7"), 1u) << text;
// ... and the column that was not asked for is not touched: nothing is read at +9 or past.
EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", "%uint_9"), 0u) << text;
EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", "%uint_10"), 0u) << text;
}
// A runtime array whose element is a STRUCT: the same walk, and the same decline test, as a
// bounded array of them - a shape no other open-ended case reaches.
TEST_F(FlattenFloat64StorageBlockTest, AStructRuntimeArrayStepsByItsElementStride) {
const String source = R"(#version 430 core
layout(local_size_x = 16) in;
struct Pair { double a; float b; };
layout(std430, binding = 0) buffer Sink { uint result[]; };
layout(std430, binding = 1) buffer Data { Pair data[]; };
void main() {
result[gl_GlobalInvocationID.x] = uint(data[gl_LocalInvocationID.x].a) +
uint(data[gl_LocalInvocationID.x].b);
}
)";
const Vector<Uint32> input = CompileToSpirv(GL_COMPUTE_SHADER, source);
ASSERT_FALSE(input.empty());
const Vector<Uint32> output = Sanitize(input);
ASSERT_FALSE(output.empty());
const String text = ExpectOpenEndedWordArray(output, "Data");
// std430 rounds `{ double a; float b; }` up to its 8-byte alignment: 16 bytes, 4 words,
// with `b` two words in.
EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_4"), 2u) << text;
EXPECT_GE(CountLinesWith(text, "OpIAdd %uint", "%uint_2"), 1u) << text;
}
// The leaf cap bounds ONE load or store, not a member's size: a block whose element is far too
// big to expand whole is still flattened while every access to it names a scalar. Declining it
// would leave the application's 8-byte-stride doubles to the demotion's re-derived stride 4 -
// the exact defect the open-ended shape exists to avoid.
TEST_F(FlattenFloat64StorageBlockTest, AHugeRuntimeArrayElementIsStillFlattenedWhenAccessesAreSmall) {
const String source = R"(#version 430 core
layout(local_size_x = 16) in;
struct Big { dvec4 v[300]; };
layout(std430, binding = 0) buffer Sink { uint result[]; };
layout(std430, binding = 1) buffer Data { Big data[]; };
void main() {
result[gl_GlobalInvocationID.x] = uint(data[gl_LocalInvocationID.x].v[3].y);
}
)";
const Vector<Uint32> input = CompileToSpirv(GL_COMPUTE_SHADER, source);
ASSERT_FALSE(input.empty());
const Vector<Uint32> output = Sanitize(input);
ASSERT_FALSE(output.empty());
const String text = ExpectOpenEndedWordArray(output, "Data");
// 300 dvec4 of 32 bytes each: 9600 bytes, 2400 words per element - 1200 scalars, well past
// the per-access cap that a whole-element load would have to respect and this never does.
EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_2400"), 1u) << text;
// v[3].y is 3 * 8 + 2 = 26 words into the element.
EXPECT_GE(CountLinesWith(text, "OpIAdd %uint", "%uint_26"), 1u) << text;
}
// The flatten preserves a byte layout ACROSS a narrowing; where the backend consumes 64-bit
// floats itself there is nothing to preserve, and the open-ended block has to keep its runtime
// array of doubles exactly as the driver would lay it out.
TEST_F(FlattenFloat64StorageBlockTest, TheNativePathLeavesAnOpenEndedBlockAndItsDoublesAlone) {
const Vector<Uint32> input = CompileToSpirv(GL_COMPUTE_SHADER, OpenEndedComputeSource("double"));
ASSERT_FALSE(input.empty());
Vector<Uint32> output;
ASSERT_TRUE(ShaderCompiler::SanitizeAndOptimizeBinary(input, output, true, true, true));
ASSERT_FALSE(output.empty());
const String text = Disassemble(output);
const Uint32 structId = StructIdNamed(output, "Data");
ASSERT_NE(structId, 0u) << text;
const Vector<Uint32> members = MemberTypesOf(output, structId);
ASSERT_EQ(members.size(), 1u) << text;
EXPECT_NE(RuntimeArrayElementOf(output, members[0]), 0u)
<< "the member should still be a runtime array\n"
<< text;
EXPECT_EQ(DecorationValueOf(output, members[0], kDecorationArrayStride), 8u)
<< "the array must keep the 8-byte stride the application bound\n"
<< text;
EXPECT_GT(CountFloatTypesOfWidth(output, 64), 0u)
<< "nothing narrows here, so the doubles must survive\n"
<< text;
}
// The other backend prints the flattened module through SPIRV-Cross: an open-ended `uint[]`
// member has to come out as ESSL that names no 64-bit type. The `.length()` shape is here too,
// because the OpArrayLength the rewrite re-issues is the one instruction in it whose ESSL
// spelling is not plain arithmetic - if that backend ever refused it on the flattened member,
// a DirectGLES shader asking an fp64 buffer its length would fail at link and nowhere else.
namespace {
String TranspileToEssl(const Vector<Uint32>& spirv) {
using namespace MG_Util::ShaderTranspiler;
SpvcSession session(spirv, SessionUsageBit::Transpile);
spvc_compiler_options options;
EXPECT_EQ(session.CreateOptions(&options), SPVC_SUCCESS);
spvc_compiler_options_set_uint(options, SPVC_COMPILER_OPTION_GLSL_VERSION, 320);
spvc_compiler_options_set_bool(options, SPVC_COMPILER_OPTION_GLSL_ES, SPVC_TRUE);
spvc_compiler_options_set_bool(options, SPVC_COMPILER_OPTION_GLSL_VULKAN_SEMANTICS, SPVC_FALSE);
EXPECT_EQ(session.SetOptions(options), SPVC_SUCCESS);
auto essl = ShaderCompiler::DecompileShader(session);
EXPECT_TRUE(essl) << (essl ? String{} : essl.error().log);
return essl ? *essl : String{};
}
} // namespace
TEST_F(FlattenFloat64StorageBlockTest, AnOpenEndedBlockCanBeEmittedAsEssl) {
const Vector<Uint32> input = CompileToSpirv(GL_COMPUTE_SHADER, OpenEndedComputeSource("dvec4"));
ASSERT_FALSE(input.empty());
const Vector<Uint32> output = Sanitize(input);
ASSERT_FALSE(output.empty());
ExpectOpenEndedWordArray(output, "Data");
const String essl = TranspileToEssl(output);
ASSERT_FALSE(essl.empty());
EXPECT_EQ(essl.find("double"), String::npos) << essl;
EXPECT_EQ(essl.find("dvec"), String::npos) << essl;
EXPECT_NE(essl.find("uint"), String::npos) << essl;
}
TEST_F(FlattenFloat64StorageBlockTest, TheRewrittenLengthCanBeEmittedAsEssl) {
const Vector<Uint32> input = CompileToSpirv(GL_COMPUTE_SHADER, kOpenEndedLengthSource);
ASSERT_FALSE(input.empty());
const Vector<Uint32> output = Sanitize(input);
ASSERT_FALSE(output.empty());
ExpectOpenEndedWordArray(output, "Data");
const String essl = TranspileToEssl(output);
ASSERT_FALSE(essl.empty());
EXPECT_EQ(essl.find("double"), String::npos) << essl;
EXPECT_EQ(essl.find("dvec"), String::npos) << essl;
// The length survived as a length - it was not folded away or dropped on the floor.
EXPECT_NE(essl.find(".length()"), String::npos) << essl;
}
// ---------------------------------------------------------------------------
// The gate from the other side: a runtime array anywhere but the block's own last member is a
// shape GLSL cannot spell and this pass does not describe. SPIR-V can spell it, so both are
// hand-written, and both are invalid Vulkan SPIR-V - the chain runs without its validator here,
// which is also why neither can be a validation-failure count.
// ---------------------------------------------------------------------------
namespace {
// `buffer Odd { double data[]; uint tail; }`, the runtime array FIRST.
const char* kRuntimeArrayNotLastAsm = R"(
OpCapability Shader
OpCapability Float64
OpMemoryModel Logical GLSL450
OpEntryPoint GLCompute %main "main"
OpExecutionMode %main LocalSize 1 1 1
OpName %Odd "Odd"
OpName %var ""
OpDecorate %_runtimearr_double ArrayStride 8
OpDecorate %Odd Block
OpMemberDecorate %Odd 0 Offset 0
OpMemberDecorate %Odd 1 Offset 8
OpDecorate %var Binding 0
OpDecorate %var DescriptorSet 0
%void = OpTypeVoid
%3 = OpTypeFunction %void
%uint = OpTypeInt 32 0
%int = OpTypeInt 32 1
%int_0 = OpConstant %int 0
%int_1 = OpConstant %int 1
%double = OpTypeFloat 64
%double_2 = OpConstant %double 2
%_runtimearr_double = OpTypeRuntimeArray %double
%Odd = OpTypeStruct %_runtimearr_double %uint
%_ptr_StorageBuffer_Odd = OpTypePointer StorageBuffer %Odd
%var = OpVariable %_ptr_StorageBuffer_Odd StorageBuffer
%_ptr_StorageBuffer_double = OpTypePointer StorageBuffer %double
%main = OpFunction %void None %3
%5 = OpLabel
%6 = OpAccessChain %_ptr_StorageBuffer_double %var %int_0 %int_1
OpStore %6 %double_2
OpReturn
OpFunctionEnd
)";
// `struct Inner { double data[]; }; buffer Outer { uint head; Inner inner; }`: the runtime
// array IS last, but of a member rather than of the block.
const char* kRuntimeArrayNestedAsm = R"(
OpCapability Shader
OpCapability Float64
OpMemoryModel Logical GLSL450
OpEntryPoint GLCompute %main "main"
OpExecutionMode %main LocalSize 1 1 1
OpName %Outer "Outer"
OpName %Inner "Inner"
OpName %var ""
OpDecorate %_runtimearr_double ArrayStride 8
OpMemberDecorate %Inner 0 Offset 0
OpDecorate %Outer Block
OpMemberDecorate %Outer 0 Offset 0
OpMemberDecorate %Outer 1 Offset 8
OpDecorate %var Binding 0
OpDecorate %var DescriptorSet 0
%void = OpTypeVoid
%3 = OpTypeFunction %void
%uint = OpTypeInt 32 0
%int = OpTypeInt 32 1
%int_0 = OpConstant %int 0
%int_1 = OpConstant %int 1
%double = OpTypeFloat 64
%double_2 = OpConstant %double 2
%_runtimearr_double = OpTypeRuntimeArray %double
%Inner = OpTypeStruct %_runtimearr_double
%Outer = OpTypeStruct %uint %Inner
%_ptr_StorageBuffer_Outer = OpTypePointer StorageBuffer %Outer
%var = OpVariable %_ptr_StorageBuffer_Outer StorageBuffer
%_ptr_StorageBuffer_double = OpTypePointer StorageBuffer %double
%main = OpFunction %void None %3
%5 = OpLabel
%6 = OpAccessChain %_ptr_StorageBuffer_double %var %int_1 %int_0 %int_1
OpStore %6 %double_2
OpReturn
OpFunctionEnd
)";
Vector<Uint32> AssembleUnchecked(const char* asmText) {
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_1);
Vector<Uint32> module;
EXPECT_TRUE(tools.Assemble(asmText, &module));
return module;
}
} // namespace
TEST_F(FlattenFloat64StorageBlockTest, ARuntimeArrayThatIsNotTheBlocksLastMemberIsLeftToTheDemotion) {
struct Shape {
const char* asmText;
const char* blockName;
};
const Shape shapes[] = {{kRuntimeArrayNotLastAsm, "Odd"}, {kRuntimeArrayNestedAsm, "Outer"}};
for (const Shape& shape : shapes) {
SCOPED_TRACE(shape.blockName);
const Vector<Uint32> input = AssembleUnchecked(shape.asmText);
ASSERT_FALSE(input.empty());
Vector<Uint32> output;
ASSERT_TRUE(ShaderCompiler::SanitizeAndOptimizeBinary(input, output, false, false));
ASSERT_FALSE(output.empty());
const String text = Disassemble(output);
// Declined: both members are still there, and the demotion narrowed them the old way.
const Uint32 structId = StructIdNamed(output, shape.blockName);
ASSERT_NE(structId, 0u) << text;
EXPECT_EQ(MemberTypesOf(output, structId).size(), 2u) << text;
EXPECT_EQ(CountFloatTypesOfWidth(output, 64), 0u) << text;
EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", ""), 0u)
<< "nothing should have been re-addressed\n"
<< text;
}
}
// ---------------------------------------------------------------------------
// The front end declares types in first-use order, so a block that is the first thing the
// shader touches is declared before the module's `uint` - and the flattened member is an array
// OF `uint`. For an OPEN-ENDED block the pass moves that operand-less type up in front of the
// block rather than declining, so that where a buffer of doubles stands in the shader does not
// decide whether its bytes survive. A BOUNDED block in the same position keeps the decline it
// has always had: widening that is a change to a path this fix does not need, and the pair below
// pins both halves.
// ---------------------------------------------------------------------------
namespace {
// The position of <id>'s declaration in instruction order, or 0 when it has none.
Uint32 DeclarationIndexOf(const Vector<Uint32>& spirv, Uint32 id) {
Uint32 index = 0;
Uint32 found = 0;
ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) {
++index;
if (found != 0 || wordCount < 2) return;
// Every OpType* has its result id in word 1; that is all this is asked about.
if (opcode >= kOpTypeInt && opcode <= kOpTypeStruct && words[1] == id) found = index;
});
return found;
}
Uint32 Uint32TypeIdOf(const Vector<Uint32>& spirv) {
Uint32 typeId = 0;
ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) {
if (opcode == kOpTypeInt && wordCount >= 4 && words[2] == 32u && words[3] == 0u) typeId = words[1];
});
return typeId;
}
} // namespace
TEST_F(FlattenFloat64StorageBlockTest, AnOpenEndedBlockDeclaredBeforeTheModulesUintIsStillFlattened) {
// The block is the first thing main touches, and nothing before it needs a uint - not even
// an array length, which is a uint constant and would declare one.
const String source = R"(#version 430 core
layout(local_size_x = 1) in;
layout(std430, binding = 0) buffer Data { double data[]; };
layout(std430, binding = 1) buffer Sink { float result[]; };
void main() {
result[0] = float(data[0] + data[1]);
}
)";
const Vector<Uint32> input = CompileToSpirv(GL_COMPUTE_SHADER, source);
ASSERT_FALSE(input.empty());
const Uint32 inputStructId = StructIdNamed(input, "Data");
ASSERT_NE(inputStructId, 0u);
const Uint32 inputUintId = Uint32TypeIdOf(input);
// The premise: the module's uint really is declared after the block (or not at all).
ASSERT_TRUE(inputUintId == 0 ||
DeclarationIndexOf(input, inputUintId) > DeclarationIndexOf(input, inputStructId))
<< "this shader was meant to declare the block before any uint\n"
<< Disassemble(input);
const Vector<Uint32> output = Sanitize(input);
ASSERT_FALSE(output.empty());
const String text = ExpectOpenEndedWordArray(output, "Data");
const Uint32 structId = StructIdNamed(output, "Data");
ASSERT_NE(structId, 0u) << text;
const Vector<Uint32> members = MemberTypesOf(output, structId);
ASSERT_EQ(members.size(), 1u) << text;
// And the uint now stands in front of the block it is an element of.
EXPECT_LT(DeclarationIndexOf(output, RuntimeArrayElementOf(output, members[0])),
DeclarationIndexOf(output, structId))
<< text;
}
TEST_F(FlattenFloat64StorageBlockTest, ABoundedBlockDeclaredBeforeTheModulesUintIsLeftToTheDemotion) {
// The same position, a bounded block: this is the shape that has always been declined, and
// it stays declined - its members and the demotion's own repacking come through untouched.
const String source = R"(#version 430 core
layout(local_size_x = 1) in;
layout(std430, binding = 0) buffer Wide {
double data0;
dvec2 data1;
} g_wide;
layout(std430, binding = 1) buffer Sink { float result[]; };
void main() {
double sum = g_wide.data0 + g_wide.data1.y;
result[0] = float(sum);
}
)";
const Vector<Uint32> input = CompileToSpirv(GL_COMPUTE_SHADER, source);
ASSERT_FALSE(input.empty());
const Uint32 inputStructId = StructIdNamed(input, "Wide");
ASSERT_NE(inputStructId, 0u);
const Uint32 inputUintId = Uint32TypeIdOf(input);
ASSERT_TRUE(inputUintId == 0 ||
DeclarationIndexOf(input, inputUintId) > DeclarationIndexOf(input, inputStructId))
<< "this shader was meant to declare the block before any uint\n"
<< Disassemble(input);
const Vector<Uint32> output = Sanitize(input);
ASSERT_FALSE(output.empty());
const String text = Disassemble(output);
const Uint32 structId = StructIdNamed(output, "Wide");
ASSERT_NE(structId, 0u) << text;
EXPECT_EQ(MemberTypesOf(output, structId).size(), 2u)
<< "a bounded block in this position must keep the decline it shipped with\n"
<< text;
EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", ""), 0u)
<< "nothing should have been re-addressed\n"
<< text;
}
MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.cpp
+246 -42
View File
@@ -84,8 +84,18 @@ namespace MobileGL {
struct BlockPlan {
Instruction* structType = nullptr;
uint32_t storageClass = 0;
// A bounded block's length in words. For an open-ended block - one whose
// last member is a runtime array - the FIXED PREFIX in words, i.e. the
// runtime array's own offset, which is where its element 0 starts.
uint32_t wordCount = 0;
bool openEnded = false;
// The original runtime array's stride in words; what one element of it
// steps by, and what its word count divides by to become a length.
uint32_t tailStrideWords = 0;
std::vector<ChainPlan> chains;
// The OpArrayLength users of an open-ended block's variables, which count
// WORDS once the member is a `uint[]` and so have to be rewritten too.
std::vector<Instruction*> arrayLengths;
};
bool IsDoubleType(const Instruction* type) {
@@ -178,8 +188,9 @@ namespace MobileGL {
}
// Byte size of a type as it is laid out INSIDE a block, or 0 when this pass
// cannot describe it (a runtime array, a width it does not carry, a matrix with
// no stride or a row-major one).
// cannot describe it (a runtime array - the one place a block may have one is
// its last member, which MeasureBlock handles above this - a width it does not
// carry, a matrix with no stride or a row-major one).
uint32_t LaidOutByteSize(IRContext* context, const TypeCursor& cursor) {
const Instruction* type = context->get_def_use_mgr()->GetDef(cursor.typeId);
if (type == nullptr) return 0;
@@ -231,8 +242,17 @@ namespace MobileGL {
}
// Whether this type decomposes into scalars the rewrite can move one word at a
// time, counting them so a whole-aggregate access can be refused before it is
// expanded.
// time. |leafCount| counts them, so a whole-aggregate access can be refused
// before it is expanded; passing NULL asks the SHAPE question alone - is this
// type addressable at all - and then identical array elements and vector
// components are walked once instead of once each, because the answer cannot
// differ between them and the walk of a big one would not be free.
//
// The two questions are separate because only a LOAD or a STORE expands into
// leaves, and the cap bounds one of those. How large a runtime array's element
// is says nothing about how many scalars a single access to it moves, so
// MeasureBlock asks for the shape and BuildPlans applies the cap where it
// belongs - per chain, to the type that chain actually names.
bool CanDecompose(IRContext* context, const TypeCursor& cursor, uint32_t* leafCount) {
const Instruction* type = context->get_def_use_mgr()->GetDef(cursor.typeId);
if (type == nullptr) return false;
@@ -240,12 +260,15 @@ namespace MobileGL {
case spv::Op::OpTypeInt:
case spv::Op::OpTypeFloat:
if (ScalarByteSize(type) == 0) return false;
if (leafCount == nullptr) return true;
++*leafCount;
return *leafCount <= kMaxLeavesPerAccess;
case spv::Op::OpTypeVector: {
TypeCursor component;
component.typeId = type->GetSingleWordInOperand(0);
for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) {
const uint32_t repeats =
leafCount == nullptr ? 1u : type->GetSingleWordInOperand(1);
for (uint32_t i = 0; i < repeats; ++i) {
if (!CanDecompose(context, component, leafCount)) return false;
}
return true;
@@ -257,7 +280,9 @@ namespace MobileGL {
}
TypeCursor column;
column.typeId = type->GetSingleWordInOperand(0);
for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) {
const uint32_t repeats =
leafCount == nullptr ? 1u : type->GetSingleWordInOperand(1);
for (uint32_t i = 0; i < repeats; ++i) {
if (!CanDecompose(context, column, leafCount)) return false;
}
return true;
@@ -273,10 +298,12 @@ namespace MobileGL {
context->get_constant_mgr()->FindDeclaredConstant(type->GetSingleWordInOperand(1));
if (length == nullptr || length->AsIntConstant() == nullptr) return false;
const uint32_t count = length->AsIntConstant()->GetU32BitValue();
if (count == 0 || count > kMaxLeavesPerAccess) return false;
if (count == 0) return false;
if (leafCount != nullptr && count > kMaxLeavesPerAccess) return false;
TypeCursor element = cursor;
element.typeId = type->GetSingleWordInOperand(0);
for (uint32_t i = 0; i < count; ++i) {
const uint32_t repeats = leafCount == nullptr ? 1u : count;
for (uint32_t i = 0; i < repeats; ++i) {
if (!CanDecompose(context, element, leafCount)) return false;
}
return true;
@@ -300,6 +327,63 @@ namespace MobileGL {
}
}
// Measures the block struct itself. A bounded block reports its laid-out byte
// size; a block whose LAST member is a runtime array - the only place GLSL lets
// one stand, and the only place SPIR-V lets a Block have one - reports the byte
// offset that array starts at and says so through |openEnded|, with the array's
// stride alongside. A runtime array anywhere else, one without a stride the
// words can step by, or one whose element the rewrite could not take apart is a
// shape this pass does not describe, and so is a bounded block it cannot size.
bool MeasureBlock(IRContext* context, const Instruction* structType, uint32_t* bytes,
bool* openEnded, uint32_t* tailStrideBytes) {
*bytes = 0;
*openEnded = false;
*tailStrideBytes = 0;
const uint32_t structId = structType->result_id();
const uint32_t memberCount = structType->NumInOperands();
uint64_t end = 0;
for (uint32_t member = 0; member < memberCount; ++member) {
uint32_t offset = 0;
if (!TryGetMemberDecorationLiteral(context, structId, member, spv::Decoration::Offset,
&offset)) {
return false;
}
const TypeCursor cursor = MemberCursor(context, structType, member);
const Instruction* type = context->get_def_use_mgr()->GetDef(cursor.typeId);
if (type == nullptr) return false;
if (type->opcode() == spv::Op::OpTypeRuntimeArray) {
if (member + 1 != memberCount) return false;
uint32_t stride = 0;
if (!TryGetDecorationLiteral(context, cursor.typeId, spv::Decoration::ArrayStride,
&stride) ||
stride == 0 || stride % kWordBytes != 0) {
return false;
}
// The member's own matrix decorations describe the array's ELEMENTS,
// exactly as they do for a bounded array of matrices. Only the shape
// is asked for: how big one element is decides nothing about how
// many scalars one access moves, and a leaf cap here would decline a
// block over a member the shader may never read whole.
TypeCursor element = cursor;
element.typeId = type->GetSingleWordInOperand(0);
if (!CanDecompose(context, element, nullptr)) return false;
// Element 0 has to start past every fixed member, or the words the
// prefix owns and the words the array owns would overlap.
if (offset < end) return false;
end = offset;
*openEnded = true;
*tailStrideBytes = stride;
break;
}
const uint32_t size = LaidOutByteSize(context, cursor);
if (size == 0) return false;
end = std::max<uint64_t>(end, static_cast<uint64_t>(offset) + size);
}
if (end > kMaxBlockBytes) return false;
*bytes = static_cast<uint32_t>(end);
return true;
}
bool TypeContainsFloat64(IRContext* context, uint32_t typeId,
std::unordered_set<uint32_t>& visiting) {
const Instruction* type = context->get_def_use_mgr()->GetDef(typeId);
@@ -366,6 +450,9 @@ namespace MobileGL {
switch (type->opcode()) {
case spv::Op::OpTypeArray:
// A runtime array steps exactly like a bounded one; only its end is
// unknown, and a chain never needs that.
case spv::Op::OpTypeRuntimeArray:
if (!TryGetDecorationLiteral(context, cursor.typeId, spv::Decoration::ArrayStride,
&stride)) {
return false;
@@ -611,6 +698,39 @@ namespace MobileGL {
}
}
// Replaces an OpArrayLength of an open-ended block with the element count
// of the ORIGINAL runtime array. The instruction now counts the words of
// the flattened `uint[]`, so the length is `(words - prefix) / stride`, in
// unsigned arithmetic and clamped at zero when the bound range does not
// even reach the array's offset - a wrapped subtraction would otherwise
// report a few billion elements. The division floors, which is what GL
// defines `.length()` as for a range that is not a whole number of
// elements. A fresh OpArrayLength is issued rather than the old one re-aimed,
// so the uses being redirected are never the ones the arithmetic just made.
void RewriteArrayLength(Instruction* arrayLength, uint32_t prefixWords, uint32_t strideWords) {
InstructionBuilder builder(m_context, arrayLength, kPreservedAnalyses);
const uint32_t variableId = arrayLength->GetSingleWordInOperand(0);
const uint32_t wordsId = m_context->TakeNextId();
builder.AddInstruction(MakeUnique<Instruction>(
m_context, spv::Op::OpArrayLength, m_uintTypeId, wordsId,
std::initializer_list<Operand>{{SPV_OPERAND_TYPE_ID, {variableId}},
{SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}}));
uint32_t count = wordsId;
if (prefixWords != 0) {
const uint32_t prefixId = UintConstant(prefixWords);
const uint32_t past = Binary(builder, spv::Op::OpISub, m_uintTypeId, count, prefixId);
const uint32_t tooShort =
Binary(builder, spv::Op::OpULessThan, m_boolTypeId, count, prefixId);
count = Select(builder, tooShort, UintConstant(0), past);
}
if (strideWords != 1) {
count = Binary(builder, spv::Op::OpUDiv, m_uintTypeId, count,
UintConstant(strideWords));
}
m_context->ReplaceAllUsesWith(arrayLength->result_id(), count);
m_context->KillInst(arrayLength);
}
private:
uint32_t ComponentWords(uint32_t componentTypeId) {
return ScalarByteSize(m_context->get_def_use_mgr()->GetDef(componentTypeId)) /
@@ -788,38 +908,72 @@ namespace MobileGL {
return false;
}
// A fresh `uint[length]` with ArrayStride 4, spliced in immediately BEFORE the
// block that will name it - SPIR-V has no forward references between types, so
// appending it at the end of the section would make the module invalid. A
// duplicate OpTypeArray is legal (SPIR-V 2.8 exempts aggregates from the
// uniqueness rule, and so does spirv-val), so no search for an existing one is
// needed; the LENGTH CONSTANT is not exempt, and if the module already declares
// it after the block there is nowhere legal to put the array - the block is then
// declined and keeps today's behaviour. Returns 0 for that, and for a uint type
// that is itself declared too late.
// A fresh `uint[length]` with ArrayStride 4 - or, for an open-ended block, a
// `uint[]` runtime array with the same stride and no length at all - spliced in
// immediately BEFORE the block that will name it: SPIR-V has no forward
// references between types, so appending it at the end of the section would make
// the module invalid. A duplicate OpTypeArray or OpTypeRuntimeArray is legal
// (SPIR-V 2.8 exempts aggregates from the uniqueness rule, and so does
// spirv-val), so no search for an existing one is needed; the LENGTH CONSTANT is
// not exempt, and if the module already declares it after the block there is
// nowhere legal to put the array - the block is then declined and keeps today's
// behaviour. Returns 0 for that; an open-ended block has no length constant to
// place, so that reason cannot reach it.
//
// The `uint` element type is a different matter, and only for an OPEN-ENDED
// block. The front end declares types in first-use order, so a block that is the
// first thing a shader touches sits BEFORE the module's `uint` (or the module has
// none, and the one the pass asked for was appended at the end). Declining there
// would send exactly the buffers this rewrite exists for back to the demotion on
// nothing but where they stand in the source. OpTypeInt has no operands, and
// nothing that names it can precede where it was, so moving it up in front of the
// block is always legal. A BOUNDED block keeps declining instead: that is what it
// has always done, and widening it is a change to a path this one does not need.
//
// NOTHING IS WRITTEN until every reason to decline has been ruled out, so a block
// this returns 0 for leaves the module as it found it - which is what lets
// Process() truthfully report SuccessWithoutChange for a module of only those.
uint32_t CreateWordArrayTypeBefore(IRContext* context, Instruction* structType,
uint32_t uintTypeId, uint32_t length) {
if (!DeclaredBefore(context, uintTypeId, structType->result_id())) return 0;
uint32_t uintTypeId, uint32_t length, bool openEnded) {
Instruction* uintType = context->get_def_use_mgr()->GetDef(uintTypeId);
if (uintType == nullptr || uintType->opcode() != spv::Op::OpTypeInt) return 0;
const bool hoistUint = !DeclaredBefore(context, uintTypeId, structType->result_id());
if (hoistUint && !openEnded) return 0;
auto* constantMgr = context->get_constant_mgr();
const spvtools::opt::analysis::Type* uintType = context->get_type_mgr()->GetType(uintTypeId);
if (uintType == nullptr) return 0;
const spvtools::opt::analysis::Constant* lengthConstant =
constantMgr->GetConstant(uintType, {length});
if (lengthConstant == nullptr) return 0;
uint32_t lengthConstantId = 0;
if (!openEnded) {
auto* constantMgr = context->get_constant_mgr();
const spvtools::opt::analysis::Type* uintDescriptor =
context->get_type_mgr()->GetType(uintTypeId);
if (uintDescriptor == nullptr) return 0;
const spvtools::opt::analysis::Constant* lengthConstant =
constantMgr->GetConstant(uintDescriptor, {length});
if (lengthConstant == nullptr) return 0;
Module::inst_iterator position = PositionOf(context, structType);
if (position == context->types_values_end()) return 0;
Instruction* lengthInst = constantMgr->GetDefiningInstruction(lengthConstant, 0, &position);
if (lengthInst == nullptr) return 0;
if (!DeclaredBefore(context, lengthInst->result_id(), structType->result_id())) return 0;
Module::inst_iterator position = PositionOf(context, structType);
if (position == context->types_values_end()) return 0;
// Created in front of the block when it is not there yet, so the only way
// this declines is a constant the module already declares after it.
Instruction* lengthInst =
constantMgr->GetDefiningInstruction(lengthConstant, 0, &position);
if (lengthInst == nullptr) return 0;
if (!DeclaredBefore(context, lengthInst->result_id(), structType->result_id())) return 0;
lengthConstantId = lengthInst->result_id();
}
const uint32_t arrayTypeId = context->TakeNextId();
if (arrayTypeId == 0) return 0;
auto arrayType = MakeUnique<Instruction>(
context, spv::Op::OpTypeArray, 0, arrayTypeId,
std::initializer_list<Operand>{{SPV_OPERAND_TYPE_ID, {uintTypeId}},
{SPV_OPERAND_TYPE_ID, {lengthInst->result_id()}}});
if (hoistUint) uintType->InsertBefore(structType);
std::unique_ptr<Instruction> arrayType =
openEnded
? MakeUnique<Instruction>(
context, spv::Op::OpTypeRuntimeArray, 0, arrayTypeId,
std::initializer_list<Operand>{{SPV_OPERAND_TYPE_ID, {uintTypeId}}})
: MakeUnique<Instruction>(
context, spv::Op::OpTypeArray, 0, arrayTypeId,
std::initializer_list<Operand>{{SPV_OPERAND_TYPE_ID, {uintTypeId}},
{SPV_OPERAND_TYPE_ID, {lengthConstantId}}});
Instruction* inserted = structType->InsertBefore(std::move(arrayType));
context->AnalyzeDefUse(inserted);
context->get_decoration_mgr()->AddDecorationVal(
@@ -948,8 +1102,14 @@ namespace MobileGL {
Instruction* structType = defUseMgr->GetDef(structId);
TypeCursor blockCursor;
blockCursor.typeId = structId;
const uint32_t blockBytes = LaidOutByteSize(context, blockCursor);
if (blockBytes == 0 || blockBytes % kWordBytes != 0) {
uint32_t blockBytes = 0;
bool openEnded = false;
uint32_t tailStrideBytes = 0;
// An open-ended block whose runtime array is its only member measures a
// prefix of 0 bytes and is perfectly describable; only a BOUNDED block of
// no bytes is not, and MeasureBlock already refuses to size one of those.
if (!MeasureBlock(context, structType, &blockBytes, &openEnded, &tailStrideBytes) ||
blockBytes % kWordBytes != 0 || (!openEnded && blockBytes == 0)) {
MGLOG_D("[spirv] storage block %%%u holds a double but its byte layout cannot be "
"described exactly; left to the fp64 demotion",
structId);
@@ -960,11 +1120,15 @@ namespace MobileGL {
plan.structType = structType;
plan.storageClass = storageClassByStruct[structId];
plan.wordCount = blockBytes / kWordBytes;
plan.openEnded = openEnded;
plan.tailStrideWords = tailStrideBytes / kWordBytes;
const uint32_t lastMember = structType->NumInOperands() - 1;
bool expressible = true;
for (Instruction* variable : variablesByStruct[structId]) {
std::vector<Instruction*> chains;
std::unordered_set<uint32_t> seenChains;
std::unordered_set<uint32_t> seenLengths;
defUseMgr->ForEachUser(variable, [&](Instruction* user) {
if (!expressible) return;
switch (user->opcode()) {
@@ -982,6 +1146,27 @@ namespace MobileGL {
}
expressible = false;
return;
case spv::Op::OpArrayLength: {
// Only an open-ended block has a length to ask for, and
// only of its last member; the result has to be the 32-bit
// uint the rewrite's arithmetic is typed in, which is the
// only result type the instruction allows anyway.
const Instruction* resultType = defUseMgr->GetDef(user->type_id());
const bool isUint = resultType != nullptr &&
resultType->opcode() == spv::Op::OpTypeInt &&
resultType->GetSingleWordInOperand(0) == 32u &&
resultType->GetSingleWordInOperand(1) == 0u;
if (openEnded && isUint && user->NumInOperands() >= 2 &&
user->GetSingleWordInOperand(0) == variable->result_id() &&
user->GetSingleWordInOperand(1) == lastMember) {
if (seenLengths.insert(user->result_id()).second) {
plan.arrayLengths.push_back(user);
}
return;
}
expressible = false;
return;
}
default:
expressible = false;
return;
@@ -1058,16 +1243,25 @@ namespace MobileGL {
Emitter emitter(irContext, uintTypeId, boolTypeId, floatTypeId);
bool modified = false;
// Every block declines before anything is written for it, so |touched| only ever
// parts company with |modified| on a shape that cannot happen without the module
// running out of ids - and even then the status must not claim the bytes are
// untouched, because the caller relies on that to skip invalidating its analyses.
bool touched = false;
for (BlockPlan& plan : plans) {
const uint32_t structId = plan.structType->result_id();
const uint32_t arrayTypeId =
CreateWordArrayTypeBefore(irContext, plan.structType, uintTypeId, plan.wordCount);
const uint32_t arrayTypeId = CreateWordArrayTypeBefore(
irContext, plan.structType, uintTypeId, plan.wordCount, plan.openEnded);
if (arrayTypeId == 0) {
// Nothing was written for it, so the module is still the one that came in.
MGLOG_D("[spirv] storage block %%%u: no legal place for the flattened word array; "
"left to the fp64 demotion",
structId);
continue;
}
// Past this point the module HAS been written to, so an abandoned block would
// leave a dead type behind - the status has to say so even then.
touched = true;
const uint32_t wordPointerTypeId = irContext->get_type_mgr()->FindPointerToType(
uintTypeId, static_cast<spv::StorageClass>(plan.storageClass));
if (wordPointerTypeId == 0) continue;
@@ -1095,6 +1289,9 @@ namespace MobileGL {
}
irContext->KillInst(chainPlan.chain);
}
for (Instruction* arrayLength : plan.arrayLengths) {
emitter.RewriteArrayLength(arrayLength, plan.wordCount, plan.tailStrideWords);
}
const std::vector<spv::Decoration> surviving = SurvivingAccessQualifiers(
irContext, structId, plan.structType->NumInOperands());
@@ -1113,12 +1310,19 @@ namespace MobileGL {
{SPV_OPERAND_TYPE_DECORATION, {static_cast<uint32_t>(kind)}}});
}
modified = true;
MGLOG_D("[spirv] storage block %%%u: flattened into %u words so its 64-bit members keep "
"the byte layout the application bound",
structId, plan.wordCount);
if (plan.openEnded) {
MGLOG_D("[spirv] storage block %%%u: flattened into an open-ended word array (%u-word "
"prefix, %u-word elements) so its 64-bit members keep the byte layout the "
"application bound",
structId, plan.wordCount, plan.tailStrideWords);
} else {
MGLOG_D("[spirv] storage block %%%u: flattened into %u words so its 64-bit members "
"keep the byte layout the application bound",
structId, plan.wordCount);
}
}
if (!modified) {
if (!modified && !touched) {
return Status::SuccessWithoutChange;
}
irContext->InvalidateAnalysesExceptFor(IRContext::kAnalysisNone);
MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.h
+21 -6
View File
@@ -66,17 +66,32 @@ namespace MobileGL {
// fp32 promise DemoteFloat64Pass already makes - what changes is only that the
// BYTES around the value stay where the application put them.
//
// AN OPEN-ENDED BLOCK - one whose last member is a runtime array, the
// `buffer B { double data[]; }` every unsized storage buffer is spelled as - is
// flattened the same way: the members before the array are the fixed prefix, and
// the flattened member is itself a `uint[]` runtime array, ArrayStride 4, with no
// length for the driver to re-derive. Element i of the original array lives at
// word `prefix + i * stride` of it, which is where the application put it. The
// block's `.length()` is rewritten too, because OpArrayLength on the flattened
// member counts WORDS: it becomes `(words - prefix) / stride` in unsigned
// arithmetic, clamped at zero when the bound range is shorter than the prefix,
// which is the floor GL defines `.length()` as.
//
// DECLINES, leaving the block exactly as it was for DemoteFloat64Pass to handle the
// old way, whenever it meets something it cannot rewrite exactly:
// - a block whose variable is used as anything but an access-chain base (loaded
// whole, handed to a function, asked its OpArrayLength);
// whole, handed to a function), or asked an OpArrayLength it is not open-ended
// for;
// - an access chain that is not rooted at the variable, or whose result feeds
// anything but a plain OpLoad / OpStore (an atomic, OpCopyMemory, a further
// chain);
// - a non-constant index into a struct, a runtime array anywhere in the block, a
// RowMajor matrix (its columns are not contiguous, so a whole-column access is
// not one range), a member width other than 32 or 64 bits, or an offset or
// stride that is not a multiple of 4;
// chain), or one that names a whole runtime array rather than an element of it;
// - a non-constant index into a struct, a runtime array that is not the last
// member of the block itself (nested in a member, or followed by another -
// shapes GLSL cannot spell but SPIR-V can), a runtime array without an
// ArrayStride or whose element the pass cannot decompose, a RowMajor matrix (its
// columns are not contiguous, so a whole-column access is not one range), a
// member width other than 32 or 64 bits, or an offset or stride that is not a
// multiple of 4;
// - a load or store whose type decomposes into more scalars than the cap below,
// so legalizing a block can never explode the module.
//