swung0x48/MobileGL
1
0
Fork 0
mirror of https://github.com/MobileGL-Dev/MobileGL synced 2026-09-08 12:18:30 +09:00
Code Issues Packages Projects Releases Wiki Activity

[Test] (ShaderTranspiler): cover both translation-cache key inventories, eviction and the concurrent path

Browse Source
This commit is contained in:
Swung0x48
2026-08-20 10:59:46 -04:00
parent 442cec1a15
commit 0f523db14d
2 changed files with 870 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
MobileGL/MG_Test/ShaderTranspiler/CMakeLists.txt
+28
View File
@@ -29,5 +29,33 @@ if (MSVC)
target_compile_options(SpirvPassTest PRIVATE /Zc:preprocessor)
endif()
# Its own binary: several cases flip MG_Config::Features.ShaderTranslationCache and
# MG_Config::Features.AsyncShaderCompile and read the PROCESS-GLOBAL L1 counters straight
# afterwards, so another suite's links running in the same process would show up as
# hits/misses this one did not cause.
add_executable(
TranslationCacheTest
TranslationCacheTest.cpp
)
target_include_directories(TranslationCacheTest PRIVATE
${MGL_ROOT}/include
${MGL_ROOT}/MobileGL
${MGL_ROOT}/3rdparty/SPIRV-Reflect
)
target_link_libraries(
TranslationCacheTest PRIVATE
GTest::gtest_main
${LINK_LIBRARIES}
)
if (MSVC)
target_compile_options(TranslationCacheTest PRIVATE /Zc:preprocessor)
endif()
include(GoogleTest)
gtest_discover_tests(SpirvPassTest DISCOVERY_TIMEOUT 30 PROPERTIES LABELS unit)
# Heavier than a pure-pass suite: the concurrency cases run 8 threads over thousands of
# rounds and the end-to-end cases link real programs through the compile pool.
gtest_discover_tests(TranslationCacheTest DISCOVERY_TIMEOUT 60 PROPERTIES LABELS unit TIMEOUT 300)
MobileGL/MG_Test/ShaderTranspiler/TranslationCacheTest.cpp
+842
View File
@@ -0,0 +1,842 @@
// MobileGL - MobileGL/MG_Test/ShaderTranspiler/TranslationCacheTest.cpp
// Copyright (c) 2025-2026 MobileGL-Dev
// Licensed under the GNU Lesser General Public License v3.0:
// https://www.gnu.org/licenses/gpl-3.0.txt
// https://www.gnu.org/licenses/lgpl-3.0.txt
// SPDX-License-Identifier: LGPL-3.0-only
// End of Source File Header
// The two-level shader translation memo (MG_Util/ShaderTranspiler/TranslationCache.h).
//
// A wrong hit here is a silently miscompiled shader, so the cases below are weighted
// heavily towards the KEY rather than towards the plumbing: for each level there is one
// case per input that can change the output, asserting that moving that input alone moves
// the key. That is the test that catches an under-specified key, which is the only way
// this feature can produce a wrong answer.
//
// The rest covers the memo contract itself: FIFO eviction under both budgets, a hash
// collision degrading to a miss rather than to a wrong payload, the MOBILEGL_SHADER_CACHE
// escape hatch, and concurrent lookups/inserts over overlapping keys agreeing with the
// single-threaded answer.
#include <gtest/gtest.h>
#include <atomic>
#include <set>
#include <string>
#include <thread>
#include <vector>
#include "Config.h"
#include "Includes.h"
#include "Init.h"
#include "MG_Impl/GLImpl/Program/GL_Program.h"
#include "MG_State/GLState/Core.h"
#include "MG_Util/ShaderTranspiler/CompileEnv.h"
#include "MG_Util/ShaderTranspiler/ShaderCompiler.h"
#include "MG_Util/ShaderTranspiler/SpvcSession.h"
#include "MG_Util/ShaderTranspiler/TranslationCache.h"
#include "MG_Util/ShaderTranspiler/Types.h"
using namespace MobileGL;
using namespace MobileGL::MG_Util::ShaderTranspiler;
namespace {
// Restores MOBILEGL_SHADER_CACHE's field on the way out, the same shape
// AsyncSpirvPhaseTest's AsyncModeScope uses for its own toggle.
class CacheModeScope {
public:
explicit CacheModeScope(const Bool enabled)
: m_saved(MG_Config::Features.ShaderTranslationCache) {
MG_Config::Features.ShaderTranslationCache =
enabled ? MG_Config::QuirkOverride::ForceOn : MG_Config::QuirkOverride::ForceOff;
}
~CacheModeScope() { MG_Config::Features.ShaderTranslationCache = m_saved; }
CacheModeScope(const CacheModeScope&) = delete;
CacheModeScope& operator=(const CacheModeScope&) = delete;
private:
const MG_Config::QuirkOverride m_saved;
};
// Synchronous links, so a case can read the L1 counters straight after LinkProgram
// instead of having to join a phase-B job first.
class SyncCompileScope {
public:
SyncCompileScope() : m_saved(MG_Config::Features.AsyncShaderCompile) {
MG_Config::Features.AsyncShaderCompile = MG_Config::QuirkOverride::ForceOff;
}
~SyncCompileScope() { MG_Config::Features.AsyncShaderCompile = m_saved; }
SyncCompileScope(const SyncCompileScope&) = delete;
SyncCompileScope& operator=(const SyncCompileScope&) = delete;
private:
const MG_Config::QuirkOverride m_saved;
};
struct TestPayload {
String text;
};
TranslationCacheKey KeyFromText(const String& text) {
TranslationKeyBuilder builder;
builder.Text(text);
return MakeTranslationCacheKey(builder);
}
SizeT PayloadBytes(const TestPayload& payload) { return payload.text.size(); }
// ---- L1 fixtures ----
const char* kVertexSource = R"(#version 460
layout(location = 0) in vec3 aPos;
out vec3 vPos;
void main() {
vPos = aPos;
gl_Position = vec4(aPos, 1.0);
}
)";
const char* kFragmentSource = R"(#version 460
in vec3 vPos;
layout(location = 0) out vec4 fragColor;
uniform vec3 uTint;
void main() {
fragColor = vec4(uTint * vPos, 1.0);
}
)";
// Same shape as the KHR-GL33.texture_swizzle.smoke_* template: one substituted type, so
// a case can build "the same shader again" and "a different shader" from one function.
String SwizzleLikeFragment(const String& basicType) {
return "#version 460\n"
"in vec3 vPos;\n"
"layout(location = 0) out " + basicType + "vec4 fragColor;\n"
"uniform sampler2D uTex;\n"
"void main() {\n"
" vec4 s = texture(uTex, vPos.xy);\n"
" fragColor = " + basicType + "vec4(s);\n"
"}\n";
}
SpirvTranslationKeyInputs BaselineSpirvInputs(const Vector<SpirvTranslationKeyInputs::Stage>& stages) {
SpirvTranslationKeyInputs inputs;
inputs.envFingerprint = 0x1234'5678'9abc'def0ull;
inputs.stages = stages;
inputs.shaderCompileFlags = 0;
inputs.enableSpirvValidation = false;
return inputs;
}
EsslTranslationKeyInputs BaselineEsslInputs(const Vector<Uint32>& spirv) {
EsslTranslationKeyInputs inputs;
inputs.spirv = &spirv;
inputs.shaderType = GL_FRAGMENT_SHADER;
inputs.supportsViewportArray = false;
inputs.supportsNoperspectiveInterpolation = false;
inputs.maxColorTextureSamples = 4;
inputs.maxIntegerSamples = 1;
inputs.maxDepthTextureSamples = 4;
inputs.advertisedMaxSamples = 4;
inputs.esslVersion = 320;
inputs.enableSpirvValidation = false;
return inputs;
}
Uint64 DigestOf(const Vector<Uint32>& words) {
Uint64 hash = 1469598103934665603ull;
for (const Uint32 word : words) hash = (hash ^ static_cast<Uint64>(word)) * 1099511628211ull;
return hash;
}
Vector<Uint64> ProgramSpirvDigest(const GLuint program) {
Vector<Uint64> digest;
const auto& object = MG_State::pGLContext->GetProgramObject(program);
if (!object) return digest;
for (const auto& module : object->GetGeneratedSpirv()) digest.push_back(DigestOf(module));
return digest;
}
GLuint MakeShader(const GLenum type, const String& source) {
const GLuint shader = MG_Impl::GLImpl::CreateShader(type);
const char* text = source.c_str();
MG_Impl::GLImpl::ShaderSource(shader, 1, &text, nullptr);
MG_Impl::GLImpl::CompileShader(shader);
return shader;
}
// One program per call, with FRESH shader objects every time - which is exactly the CTS
// shape this cache exists for (2592 glCreateShader/glLinkProgram pairs over a handful of
// distinct sources), and what makes the second link a genuine L1 lookup rather than a
// reuse of an already-parsed object.
GLuint LinkProgramFromSources(const String& vertexSource, const String& fragmentSource) {
const GLuint vs = MakeShader(GL_VERTEX_SHADER, vertexSource);
const GLuint fs = MakeShader(GL_FRAGMENT_SHADER, fragmentSource);
const GLuint program = MG_Impl::GLImpl::CreateProgram();
MG_Impl::GLImpl::AttachShader(program, vs);
MG_Impl::GLImpl::AttachShader(program, fs);
MG_Impl::GLImpl::LinkProgram(program);
return program;
}
// The real SPIRV-Cross emission, standing in for the DirectGLES member function the L2
// cache actually wraps. Same emitter, same options; what a unit test cannot reach is the
// capability-gated SPIR-V pass chain around it, which needs a live ES driver to be
// meaningful (and whose gates are covered exhaustively by the key cases instead).
Bool EmitEssl(const Vector<Uint32>& spirv, const Uint version, String& outEssl) {
SpvcSession session(spirv, SessionUsageBit::Transpile);
spvc_compiler_options options;
if (session.CreateOptions(&options) != SPVC_SUCCESS) return false;
spvc_compiler_options_set_uint(options, SPVC_COMPILER_OPTION_GLSL_VERSION, version);
spvc_compiler_options_set_bool(options, SPVC_COMPILER_OPTION_GLSL_ES, SPVC_TRUE);
spvc_compiler_options_set_bool(options, SPVC_COMPILER_OPTION_GLSL_VULKAN_SEMANTICS, SPVC_FALSE);
session.SetOptions(options);
const char* result = nullptr;
session.Compile(&result);
if (!result) return false;
outEssl = result;
return true;
}
Vector<Uint32> BuildFragmentSpirv() {
ShaderAttrib attrib{.shaderType = GL_FRAGMENT_SHADER, .sourceStr = kFragmentSource};
auto shader = ShaderCompiler::CompileShader(attrib);
if (!shader) return {};
ProgramAttrib programAttrib{.shaders = {shader.value()}};
auto program = ShaderCompiler::LinkProgram(programAttrib);
if (!program) return {};
ProgramBinaryAttrib binaryAttrib{.shaderTypes = {GL_FRAGMENT_SHADER}, .program = *program.value()};
auto binary = ShaderCompiler::GetSpirvBinaryFromProgram(binaryAttrib);
if (!binary || binary->empty()) return {};
Vector<Uint32> sanitized;
if (!ShaderCompiler::SanitizeAndOptimizeBinary(binary->front(), sanitized)) return {};
return sanitized;
}
class TranslationCacheTest : public ::testing::Test {
protected:
void SetUp() override { MobileGL::Initialize(); }
};
} // namespace
// =========================================================================================
// The memo contract: eviction, collisions, lifetime
// =========================================================================================
TEST_F(TranslationCacheTest, HitReturnsTheStoredPayload) {
BoundedTranslationCache<TestPayload> cache("test", 8, 4096);
const TranslationCacheKey key = KeyFromText("alpha");
EXPECT_EQ(cache.Find(key), nullptr);
auto payload = MakeShared<TestPayload>(TestPayload{"emitted"});
cache.Insert(key, SharedPtr<const TestPayload>(payload), PayloadBytes(*payload));
const auto hit = cache.Find(KeyFromText("alpha"));
ASSERT_NE(hit, nullptr);
EXPECT_EQ(hit->text, "emitted");
const TranslationCacheStats stats = cache.Stats();
EXPECT_EQ(stats.hits, 1u);
EXPECT_EQ(stats.misses, 1u);
EXPECT_EQ(stats.inserts, 1u);
}
// The single most important property in the whole file. A 64-bit hash is a bucket
// selector; if it were ever trusted on its own, two different shaders sharing a hash
// would swap payloads and one of them would be silently miscompiled.
TEST_F(TranslationCacheTest, HashCollisionDegradesToMissNotToAWrongPayload) {
BoundedTranslationCache<TestPayload> cache("test", 8, 4096);
TranslationCacheKey stored;
stored.hash = 0xdead'beef'dead'beefull;
stored.blob = MakeShared<const String>("the real key bytes");
TranslationCacheKey colliding;
colliding.hash = stored.hash; // same bucket, deliberately
colliding.blob = MakeShared<const String>("DIFFERENT key bytes");
cache.Insert(stored, MakeShared<const TestPayload>(TestPayload{"stored payload"}), 14);
EXPECT_EQ(cache.Find(colliding), nullptr);
ASSERT_NE(cache.Find(stored), nullptr);
EXPECT_EQ(cache.Find(stored)->text, "stored payload");
}
TEST_F(TranslationCacheTest, EvictionIsFifoUnderTheEntryCap) {
BoundedTranslationCache<TestPayload> cache("test", 2, 1u << 20);
for (const char* name : {"a", "b", "c"}) {
auto payload = MakeShared<TestPayload>(TestPayload{name});
cache.Insert(KeyFromText(name), SharedPtr<const TestPayload>(payload), PayloadBytes(*payload));
}
EXPECT_EQ(cache.EntryCount(), 2u);
EXPECT_EQ(cache.Find(KeyFromText("a")), nullptr) << "the oldest entry should have been evicted";
ASSERT_NE(cache.Find(KeyFromText("b")), nullptr);
ASSERT_NE(cache.Find(KeyFromText("c")), nullptr);
EXPECT_EQ(cache.Stats().evictions, 1u);
// And a re-insert after the eviction works, i.e. the index and the list stayed in step.
auto revived = MakeShared<TestPayload>(TestPayload{"a-again"});
cache.Insert(KeyFromText("a"), SharedPtr<const TestPayload>(revived), PayloadBytes(*revived));
const auto hit = cache.Find(KeyFromText("a"));
ASSERT_NE(hit, nullptr);
EXPECT_EQ(hit->text, "a-again");
}
TEST_F(TranslationCacheTest, EvictionIsFifoUnderTheByteBudget) {
// Room for two entries by bytes, but the entry cap is generous - so the byte budget is
// the one that has to bind.
const SizeT keyBytes = KeyFromText("aaaa").Bytes();
BoundedTranslationCache<TestPayload> cache("test", 64, (keyBytes + 64) * 2);
for (const char* name : {"aaaa", "bbbb", "cccc"}) {
auto payload = MakeShared<TestPayload>(TestPayload(String(64, name[0])));
cache.Insert(KeyFromText(name), SharedPtr<const TestPayload>(payload), PayloadBytes(*payload));
}
EXPECT_EQ(cache.EntryCount(), 2u);
EXPECT_EQ(cache.Find(KeyFromText("aaaa")), nullptr);
EXPECT_NE(cache.Find(KeyFromText("cccc")), nullptr);
EXPECT_LE(cache.StoredBytes(), (keyBytes + 64) * 2);
}
TEST_F(TranslationCacheTest, AnEntryLargerThanTheWholeBudgetIsNotCached) {
BoundedTranslationCache<TestPayload> cache("test", 64, 128);
auto payload = MakeShared<TestPayload>(TestPayload(String(4096, 'x')));
cache.Insert(KeyFromText("huge"), SharedPtr<const TestPayload>(payload), PayloadBytes(*payload));
EXPECT_EQ(cache.EntryCount(), 0u);
EXPECT_EQ(cache.Stats().rejectedOversize, 1u);
EXPECT_EQ(cache.Find(KeyFromText("huge")), nullptr);
}
// A hit hands out shared ownership, so a reader still holding a payload when the entry is
// evicted keeps reading valid memory. This is what makes the memo safe once compiles run
// on pool workers.
TEST_F(TranslationCacheTest, APayloadOutlivesTheEvictionOfItsEntry) {
BoundedTranslationCache<TestPayload> cache("test", 1, 1u << 20);
auto first = MakeShared<TestPayload>(TestPayload{"first"});
cache.Insert(KeyFromText("first"), SharedPtr<const TestPayload>(first), PayloadBytes(*first));
const auto held = cache.Find(KeyFromText("first"));
ASSERT_NE(held, nullptr);
auto second = MakeShared<TestPayload>(TestPayload{"second"});
cache.Insert(KeyFromText("second"), SharedPtr<const TestPayload>(second), PayloadBytes(*second));
EXPECT_EQ(cache.Find(KeyFromText("first")), nullptr);
EXPECT_EQ(held->text, "first"); // still readable
}
// ska::flat_hash_map iterates in insertion/capacity order, so a key builder that walked a
// map directly would produce different bytes for the same map depending on how it was
// filled - a pure loss (spurious misses), and one that is invisible without this case.
TEST_F(TranslationCacheTest, NameMapsSerializeCanonically) {
UnorderedMap<String, Uint> forward;
forward.emplace("aPos", 0u);
forward.emplace("aNormal", 1u);
forward.emplace("aUv", 2u);
UnorderedMap<String, Uint> reverse;
reverse.emplace("aUv", 2u);
reverse.emplace("aNormal", 1u);
reverse.emplace("aPos", 0u);
TranslationKeyBuilder a;
a.NameMap(forward);
TranslationKeyBuilder b;
b.NameMap(reverse);
EXPECT_EQ(a.Blob(), b.Blob());
// ... and a value change still moves it.
reverse["aPos"] = 7u;
TranslationKeyBuilder c;
c.NameMap(reverse);
EXPECT_NE(a.Blob(), c.Blob());
}
// Length-prefixing: "ab" + "c" must not serialize to the same bytes as "a" + "bc".
TEST_F(TranslationCacheTest, TextAppendsCannotRunIntoEachOther) {
TranslationKeyBuilder a;
a.Text("ab");
a.Text("c");
TranslationKeyBuilder b;
b.Text("a");
b.Text("bc");
EXPECT_NE(a.Blob(), b.Blob());
}
TEST_F(TranslationCacheTest, L1AndL2KeysNeverAlias) {
const Vector<Uint32> spirv{1u, 2u, 3u};
const TranslationCacheKey l2 = BuildEsslTranslationKey(BaselineEsslInputs(spirv));
const TranslationCacheKey l1 =
BuildSpirvTranslationKey(BaselineSpirvInputs({{GL_FRAGMENT_SHADER, "source"}}));
EXPECT_FALSE(l1 == l2);
}
// =========================================================================================
// L1 key composition - one case per input that can change the produced SPIR-V
// =========================================================================================
TEST_F(TranslationCacheTest, L1KeyMovesWithEveryInputThatMovesTheSpirv) {
const String vs = kVertexSource;
const String fs = kFragmentSource;
const Vector<SpirvTranslationKeyInputs::Stage> baseStages{
{GL_VERTEX_SHADER, vs}, {GL_FRAGMENT_SHADER, fs}};
const UnorderedMap<String, Uint> attribs{{"aPos", 3u}};
const UnorderedMap<String, Uint> fragOut{{"fragColor", 1u}};
const UnorderedMap<String, Uint> fragIndex{{"fragColor", 1u}};
const UnorderedMap<String, Uint> opaque{{"uTex", 5u}};
const SpirvTranslationKeyInputs base = BaselineSpirvInputs(baseStages);
const TranslationCacheKey baseKey = BuildSpirvTranslationKey(base);
// Identical inputs -> identical key. Everything below is measured against this.
EXPECT_TRUE(BuildSpirvTranslationKey(BaselineSpirvInputs(baseStages)) == baseKey);
Vector<Pair<const char*, TranslationCacheKey>> variants;
{ // the environment fingerprint (glslang resource limits, backend identity,
// advertised extension set, compute limits)
SpirvTranslationKeyInputs v = base;
v.envFingerprint ^= 1ull;
variants.emplace_back("envFingerprint", BuildSpirvTranslationKey(v));
}
{ // a stage's source text
const String otherFs = SwizzleLikeFragment("i");
SpirvTranslationKeyInputs v = BaselineSpirvInputs({{GL_VERTEX_SHADER, vs},
{GL_FRAGMENT_SHADER, otherFs}});
variants.emplace_back("stage source", BuildSpirvTranslationKey(v));
}
{ // a stage's TYPE, with the text unchanged
SpirvTranslationKeyInputs v = BaselineSpirvInputs({{GL_VERTEX_SHADER, vs},
{GL_COMPUTE_SHADER, fs}});
variants.emplace_back("stage type", BuildSpirvTranslationKey(v));
}
{ // the SET of stages - mapIO resolves a fragment stage's Locations against the
// vertex stage's outputs, which is why this key is per PROGRAM and not per stage
SpirvTranslationKeyInputs v = BaselineSpirvInputs({{GL_FRAGMENT_SHADER, fs}});
variants.emplace_back("stage set", BuildSpirvTranslationKey(v));
}
{ // stage ORDER
SpirvTranslationKeyInputs v = BaselineSpirvInputs({{GL_FRAGMENT_SHADER, fs},
{GL_VERTEX_SHADER, vs}});
variants.emplace_back("stage order", BuildSpirvTranslationKey(v));
}
{ // glBindAttribLocation
SpirvTranslationKeyInputs v = base;
v.explicitVertexInLocations = &attribs;
variants.emplace_back("explicitVertexInLocations", BuildSpirvTranslationKey(v));
}
{ // glBindFragDataLocation
SpirvTranslationKeyInputs v = base;
v.explicitFragmentOutLocations = &fragOut;
variants.emplace_back("explicitFragmentOutLocations", BuildSpirvTranslationKey(v));
}
{ // glBindFragDataLocationIndexed
SpirvTranslationKeyInputs v = base;
v.explicitFragmentOutIndices = &fragIndex;
variants.emplace_back("explicitFragmentOutIndices", BuildSpirvTranslationKey(v));
}
{ // the merged layout(binding = N) opaque units
SpirvTranslationKeyInputs v = base;
v.explicitOpaqueUniformBindings = &opaque;
variants.emplace_back("explicitOpaqueUniformBindings", BuildSpirvTranslationKey(v));
}
{ // ShaderCompileBits (0 on both production parse paths; keyed so a future value
// cannot alias a module parsed without it)
SpirvTranslationKeyInputs v = base;
v.shaderCompileFlags = 1u;
variants.emplace_back("shaderCompileFlags", BuildSpirvTranslationKey(v));
}
{ // MOBILEGL_ENABLE_SPIRV_VALIDATION
SpirvTranslationKeyInputs v = base;
v.enableSpirvValidation = true;
variants.emplace_back("enableSpirvValidation", BuildSpirvTranslationKey(v));
}
for (const auto& [name, key] : variants) {
EXPECT_FALSE(key == baseKey) << "moving " << name << " did not move the L1 key";
}
// Pairwise distinct too: two different inputs must not collapse onto one key.
for (SizeT i = 0; i < variants.size(); ++i) {
for (SizeT j = i + 1; j < variants.size(); ++j) {
EXPECT_FALSE(variants[i].second == variants[j].second)
<< variants[i].first << " and " << variants[j].first << " produce the same L1 key";
}
}
}
// =========================================================================================
// L1 end to end, through the real GL entry points
// =========================================================================================
// The headline case: the second program with byte-identical sources reuses the first
// program's modules instead of running GlslangToSpv and the 11-pass sanitize chain again,
// and the modules it gets are the same bytes.
TEST_F(TranslationCacheTest, L1MemoizesASecondProgramWithIdenticalSources) {
const SyncCompileScope sync;
const CacheModeScope cacheOn(true);
const String fs = SwizzleLikeFragment("");
const TranslationCacheStats before = GetSpirvTranslationCache().Stats();
const GLuint first = LinkProgramFromSources(kVertexSource, fs);
const TranslationCacheStats afterFirst = GetSpirvTranslationCache().Stats();
const GLuint second = LinkProgramFromSources(kVertexSource, fs);
const TranslationCacheStats afterSecond = GetSpirvTranslationCache().Stats();
GLint firstStatus = GL_FALSE;
GLint secondStatus = GL_FALSE;
MG_Impl::GLImpl::GetProgramiv(first, GL_LINK_STATUS, &firstStatus);
MG_Impl::GLImpl::GetProgramiv(second, GL_LINK_STATUS, &secondStatus);
ASSERT_EQ(firstStatus, GL_TRUE);
ASSERT_EQ(secondStatus, GL_TRUE);
EXPECT_EQ(afterFirst.misses - before.misses, 1u) << "the first link must be a miss";
EXPECT_EQ(afterFirst.hits - before.hits, 0u);
EXPECT_EQ(afterSecond.hits - afterFirst.hits, 1u) << "the second link must be a hit";
EXPECT_EQ(afterSecond.misses - afterFirst.misses, 0u);
const Vector<Uint64> firstDigest = ProgramSpirvDigest(first);
const Vector<Uint64> secondDigest = ProgramSpirvDigest(second);
ASSERT_EQ(firstDigest.size(), 2u);
EXPECT_EQ(firstDigest, secondDigest);
}
// The modules a hit hands out must be the modules a from-scratch translation would have
// produced. Without this the case above would still pass if the cache returned garbage.
TEST_F(TranslationCacheTest, L1HitsAgreeWithACacheDisabledTranslation) {
const SyncCompileScope sync;
const String fs = SwizzleLikeFragment("u");
Vector<Uint64> uncached;
{
const CacheModeScope cacheOff(false);
uncached = ProgramSpirvDigest(LinkProgramFromSources(kVertexSource, fs));
}
ASSERT_EQ(uncached.size(), 2u);
const CacheModeScope cacheOn(true);
const Vector<Uint64> primed = ProgramSpirvDigest(LinkProgramFromSources(kVertexSource, fs));
const Vector<Uint64> hit = ProgramSpirvDigest(LinkProgramFromSources(kVertexSource, fs));
EXPECT_EQ(primed, uncached);
EXPECT_EQ(hit, uncached);
}
TEST_F(TranslationCacheTest, L1DoesNotMemoizeAcrossDifferentSources) {
const SyncCompileScope sync;
const CacheModeScope cacheOn(true);
// Prime with one, then link a different one: a miss, not a hit.
(void)LinkProgramFromSources(kVertexSource, SwizzleLikeFragment(""));
const TranslationCacheStats before = GetSpirvTranslationCache().Stats();
(void)LinkProgramFromSources(kVertexSource, SwizzleLikeFragment("i"));
const TranslationCacheStats after = GetSpirvTranslationCache().Stats();
EXPECT_EQ(after.hits - before.hits, 0u);
EXPECT_EQ(after.misses - before.misses, 1u);
}
// The escape hatch. MOBILEGL_SHADER_CACHE falsy must make every link translate again -
// no lookup at all, so a field miscompile can be bisected against the feature in one run.
TEST_F(TranslationCacheTest, TheEscapeHatchDisablesL1Entirely) {
const SyncCompileScope sync;
const String fs = SwizzleLikeFragment("");
{ // prime the cache with the switch ON, so a later hit would be available
const CacheModeScope cacheOn(true);
(void)LinkProgramFromSources(kVertexSource, fs);
}
const CacheModeScope cacheOff(false);
const TranslationCacheStats before = GetSpirvTranslationCache().Stats();
const GLuint program = LinkProgramFromSources(kVertexSource, fs);
const TranslationCacheStats after = GetSpirvTranslationCache().Stats();
GLint status = GL_FALSE;
MG_Impl::GLImpl::GetProgramiv(program, GL_LINK_STATUS, &status);
EXPECT_EQ(status, GL_TRUE) << "the program must still link with the cache off";
EXPECT_EQ(after.hits, before.hits) << "no lookup may happen with the cache disabled";
EXPECT_EQ(after.misses, before.misses);
EXPECT_EQ(after.inserts, before.inserts);
}
// =========================================================================================
// L2 key composition - one case per gate the DirectGLES pass chain arms
// =========================================================================================
TEST_F(TranslationCacheTest, L2KeyMovesWithEveryGateThatSteersTheEsslChain) {
const Vector<Uint32> spirv{0x07230203u, 0x00010300u, 0u, 32u, 0u};
const Vector<Uint32> otherSpirv{0x07230203u, 0x00010300u, 0u, 33u, 0u};
const std::set<String> xfbBlocks{"StageData"};
const UnorderedMap<String, Uint> imageFormats{{"gImage", 0x8236u /*GL_R32UI*/}};
const UnorderedMap<String, Int> storageBindings{{"Data", 3}};
const EsslTranslationKeyInputs base = BaselineEsslInputs(spirv);
const TranslationCacheKey baseKey = BuildEsslTranslationKey(base);
EXPECT_TRUE(BuildEsslTranslationKey(BaselineEsslInputs(spirv)) == baseKey);
Vector<Pair<const char*, TranslationCacheKey>> variants;
{ // the module itself
EsslTranslationKeyInputs v = base;
v.spirv = &otherSpirv;
variants.emplace_back("spirv", BuildEsslTranslationKey(v));
}
{ // stage: gates LowerDrawParametersForEssl and SplitArrayVertexInputsForEssl (vertex)
// and LegalizeFragmentOutputIndexingForEssl (fragment)
EsslTranslationKeyInputs v = base;
v.shaderType = GL_VERTEX_SHADER;
variants.emplace_back("shaderType", BuildEsslTranslationKey(v));
}
{ // arms LowerViewportIndexForEssl
EsslTranslationKeyInputs v = base;
v.supportsViewportArray = true;
variants.emplace_back("supportsViewportArray", BuildEsslTranslationKey(v));
}
{ // arms EmulateNoPerspectiveForEssl
EsslTranslationKeyInputs v = base;
v.supportsNoperspectiveInterpolation = true;
variants.emplace_back("supportsNoperspectiveInterpolation", BuildEsslTranslationKey(v));
}
{ // arms AND parameterizes ClampMultisampleFetchesForEssl
EsslTranslationKeyInputs v = base;
v.maxColorTextureSamples = 1;
variants.emplace_back("maxColorTextureSamples", BuildEsslTranslationKey(v));
}
{
EsslTranslationKeyInputs v = base;
v.maxIntegerSamples = 4;
variants.emplace_back("maxIntegerSamples", BuildEsslTranslationKey(v));
}
{
EsslTranslationKeyInputs v = base;
v.maxDepthTextureSamples = 1;
variants.emplace_back("maxDepthTextureSamples", BuildEsslTranslationKey(v));
}
{ // the ceiling the three above are compared against
EsslTranslationKeyInputs v = base;
v.advertisedMaxSamples = 8;
variants.emplace_back("advertisedMaxSamples", BuildEsslTranslationKey(v));
}
{ // the argument to FlattenXfbInterfaceBlocksForEssl
EsslTranslationKeyInputs v = base;
v.xfbCaptureBlockNames = &xfbBlocks;
variants.emplace_back("xfbCaptureBlockNames", BuildEsslTranslationKey(v));
}
{ // the argument to BakeImageFormatsForEssl - live glBindImageTexture state
EsslTranslationKeyInputs v = base;
v.glFormatByUniformName = &imageFormats;
variants.emplace_back("glFormatByUniformName", BuildEsslTranslationKey(v));
}
{ // SpvcSession::SetShaderStorageBlockBinding
EsslTranslationKeyInputs v = base;
v.storageBlockBindingOverrides = &storageBindings;
variants.emplace_back("storageBlockBindingOverrides", BuildEsslTranslationKey(v));
}
{ // SPVC_COMPILER_OPTION_GLSL_VERSION (ResolveBackendEsslVersion)
EsslTranslationKeyInputs v = base;
v.esslVersion = 300;
variants.emplace_back("esslVersion", BuildEsslTranslationKey(v));
}
{
EsslTranslationKeyInputs v = base;
v.enableSpirvValidation = true;
variants.emplace_back("enableSpirvValidation", BuildEsslTranslationKey(v));
}
for (const auto& [name, key] : variants) {
EXPECT_FALSE(key == baseKey) << "moving " << name << " did not move the L2 key";
}
for (SizeT i = 0; i < variants.size(); ++i) {
for (SizeT j = i + 1; j < variants.size(); ++j) {
EXPECT_FALSE(variants[i].second == variants[j].second)
<< variants[i].first << " and " << variants[j].first << " produce the same L2 key";
}
}
}
// The value SIDE of L2: the payload has to carry the flattened-block report, not just the
// text. A payload that dropped it would silently un-rename every transform-feedback
// capture on a hit.
TEST_F(TranslationCacheTest, L2PayloadCarriesTheFlattenedXfbBlockReport) {
BoundedTranslationCache<EsslTranslationResult> cache("test", 8, 1u << 20);
const Vector<Uint32> spirv{1u, 2u, 3u};
const TranslationCacheKey key = BuildEsslTranslationKey(BaselineEsslInputs(spirv));
auto payload = MakeShared<EsslTranslationResult>();
payload->essl = "#version 320 es\nvoid main() {}\n";
payload->flattenedXfbBlockNames = {"StageData", "OtherBlock"};
cache.Insert(key, EsslTranslationResultPtr(payload), EsslTranslationResultBytes(*payload));
const auto hit = cache.Find(BuildEsslTranslationKey(BaselineEsslInputs(spirv)));
ASSERT_NE(hit, nullptr);
EXPECT_EQ(hit->essl, payload->essl);
EXPECT_EQ(hit->flattenedXfbBlockNames, payload->flattenedXfbBlockNames);
}
// The real emitter behind the real key: two lookups over the same module and the same
// capability snapshot run SPIRV-Cross once and return the same text; moving one capability
// bit runs it again.
TEST_F(TranslationCacheTest, L2RunsTheEmitterOncePerDistinctKey) {
const Vector<Uint32> spirv = BuildFragmentSpirv();
ASSERT_FALSE(spirv.empty());
BoundedTranslationCache<EsslTranslationResult> cache("test", 8, 4u << 20);
Int emitCount = 0;
const auto translate = [&](const EsslTranslationKeyInputs& inputs) -> String {
const TranslationCacheKey key = BuildEsslTranslationKey(inputs);
if (const auto hit = cache.Find(key)) return hit->essl;
auto payload = MakeShared<EsslTranslationResult>();
EXPECT_TRUE(EmitEssl(*inputs.spirv, inputs.esslVersion, payload->essl));
++emitCount;
cache.Insert(key, EsslTranslationResultPtr(payload), EsslTranslationResultBytes(*payload));
return payload->essl;
};
EsslTranslationKeyInputs inputs = BaselineEsslInputs(spirv);
const String first = translate(inputs);
const String second = translate(inputs);
EXPECT_FALSE(first.empty());
EXPECT_EQ(first, second);
EXPECT_EQ(emitCount, 1) << "the second lookup must not have reached SPIRV-Cross";
// A capability bit moves -> the emitter runs again. (esslVersion is the one this unit
// test can observe in the OUTPUT as well as in the key.)
inputs.esslVersion = 300;
const String downlevel = translate(inputs);
EXPECT_EQ(emitCount, 2);
EXPECT_NE(downlevel, first);
// ... and a gate that only steers the SPIR-V pass chain still moves the key, so the
// emitter runs again even though this stand-in ignores the bit.
inputs = BaselineEsslInputs(spirv);
inputs.supportsViewportArray = true;
(void)translate(inputs);
EXPECT_EQ(emitCount, 3);
}
// =========================================================================================
// Thread safety
// =========================================================================================
// Several threads racing Find/compute/Insert over OVERLAPPING keys. Two workers that miss
// on the same key both compute it - deliberately, because waiting on each other inside a
// job body is what deadlocks ShaderCompilePool - so the property under test is not "the
// work happened once" but "every payload handed out equals the single-threaded answer".
TEST_F(TranslationCacheTest, ConcurrentLookupsOverOverlappingKeysAgreeWithTheSerialAnswer) {
constexpr Int kDistinctKeys = 24;
constexpr Int kThreads = 8;
constexpr Int kRoundsPerThread = 200;
const auto expensive = [](const Int index) {
return String("payload-") + std::to_string(index) + String(64, static_cast<char>('a' + index % 26));
};
BoundedTranslationCache<TestPayload> cache("test", kDistinctKeys, 8u << 20);
std::atomic<Int> mismatches{0};
std::atomic<Int> nulls{0};
Vector<std::thread> threads;
threads.reserve(kThreads);
for (Int t = 0; t < kThreads; ++t) {
threads.emplace_back([&, t] {
for (Int round = 0; round < kRoundsPerThread; ++round) {
const Int index = (round * 7 + t * 3) % kDistinctKeys;
const TranslationCacheKey key = KeyFromText("key-" + std::to_string(index));
SharedPtr<const TestPayload> value = cache.Find(key);
if (!value) {
auto fresh = MakeShared<TestPayload>(TestPayload{expensive(index)});
cache.Insert(key, SharedPtr<const TestPayload>(fresh), PayloadBytes(*fresh));
value = fresh;
}
if (!value) {
nulls.fetch_add(1, std::memory_order_relaxed);
} else if (value->text != expensive(index)) {
mismatches.fetch_add(1, std::memory_order_relaxed);
}
}
});
}
for (std::thread& thread : threads) thread.join();
EXPECT_EQ(mismatches.load(), 0) << "a worker was handed a payload belonging to another key";
EXPECT_EQ(nulls.load(), 0);
EXPECT_LE(cache.EntryCount(), static_cast<SizeT>(kDistinctKeys));
const TranslationCacheStats stats = cache.Stats();
EXPECT_EQ(stats.hits + stats.misses, static_cast<Uint64>(kThreads) * kRoundsPerThread);
}
// The same race with eviction turned up so hard that entries are constantly being erased
// under the readers - the shape that would catch a Find() that handed back a pointer into
// the entry list instead of shared ownership.
TEST_F(TranslationCacheTest, ConcurrentLookupsStaySafeWhileEvictionChurns) {
constexpr Int kDistinctKeys = 32;
constexpr Int kThreads = 8;
constexpr Int kRoundsPerThread = 400;
BoundedTranslationCache<TestPayload> cache("test", 4, 1u << 20); // 4 slots for 32 keys
std::atomic<Int> mismatches{0};
Vector<std::thread> threads;
threads.reserve(kThreads);
for (Int t = 0; t < kThreads; ++t) {
threads.emplace_back([&, t] {
for (Int round = 0; round < kRoundsPerThread; ++round) {
const Int index = (round + t) % kDistinctKeys;
const String expected = "payload-" + std::to_string(index);
const TranslationCacheKey key = KeyFromText("key-" + std::to_string(index));
SharedPtr<const TestPayload> value = cache.Find(key);
if (!value) {
auto fresh = MakeShared<TestPayload>(TestPayload{expected});
cache.Insert(key, SharedPtr<const TestPayload>(fresh), PayloadBytes(*fresh));
value = fresh;
}
// Held across further cache traffic on purpose: the payload must stay valid
// even after its entry has been evicted by another thread.
std::this_thread::yield();
if (value->text != expected) mismatches.fetch_add(1, std::memory_order_relaxed);
}
});
}
for (std::thread& thread : threads) thread.join();
EXPECT_EQ(mismatches.load(), 0);
EXPECT_LE(cache.EntryCount(), 4u);
}
// And the production shape: many links of a handful of distinct sources across the real
// compile pool, with L1 live. Every program built from the same sources must end up with
// the same SPIR-V, whichever worker won the race to translate it.
TEST_F(TranslationCacheTest, ConcurrentLinksOfSharedSourcesProduceIdenticalSpirv) {
const CacheModeScope cacheOn(true);
constexpr Int kVariants = 3;
constexpr Int kProgramsPerVariant = 8;
Vector<String> sources;
for (Int v = 0; v < kVariants; ++v) sources.push_back(SwizzleLikeFragment(v == 0 ? "" : (v == 1 ? "i" : "u")));
Vector<GLuint> programs;
Vector<Int> variantOf;
for (Int round = 0; round < kProgramsPerVariant; ++round) {
for (Int v = 0; v < kVariants; ++v) {
programs.push_back(LinkProgramFromSources(kVertexSource, sources[v]));
variantOf.push_back(v);
}
}
Vector<Vector<Uint64>> expected(kVariants);
for (SizeT i = 0; i < programs.size(); ++i) {
GLint status = GL_FALSE;
MG_Impl::GLImpl::GetProgramiv(programs[i], GL_LINK_STATUS, &status);
ASSERT_EQ(status, GL_TRUE);
const Vector<Uint64> digest = ProgramSpirvDigest(programs[i]);
ASSERT_EQ(digest.size(), 2u);
const Int v = variantOf[i];
if (expected[v].empty()) {
expected[v] = digest;
} else {
EXPECT_EQ(digest, expected[v]) << "variant " << v << " program " << i;
}
}
// The three variants must not have collapsed onto one another.
EXPECT_NE(expected[0], expected[1]);
EXPECT_NE(expected[1], expected[2]);
}