diff --git a/CMakeLists.txt b/CMakeLists.txt index c2ce320c..bda2f898 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -291,6 +291,7 @@ set(SOURCE_FILES MobileGL/MG_State/GLState/TextureState/TextureState.cpp MobileGL/MG_State/GLState/ProgramState/ProgramObject.cpp MobileGL/MG_State/GLState/ProgramState/ShaderObject.cpp + MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.cpp MobileGL/MG_State/GLState/ProgramState/ProgramState.cpp MobileGL/MG_State/GLState/RenderState/RenderState.cpp MobileGL/MG_State/GLState/FramebufferState/FramebufferObject.cpp diff --git a/MobileGL/MG_State/GLState/ProgramState/ProgramState.cpp b/MobileGL/MG_State/GLState/ProgramState/ProgramState.cpp index bf2fcd48..e2d0767e 100644 --- a/MobileGL/MG_State/GLState/ProgramState/ProgramState.cpp +++ b/MobileGL/MG_State/GLState/ProgramState/ProgramState.cpp @@ -79,7 +79,7 @@ namespace MobileGL::MG_State::GLState { Uint shaderId = 0; m_programShaderNameGenerator.Generate(1, &shaderId); EnsureIndexAvail(shaderId, m_shaderObjects); - auto shaderObject = MakeShared(stage, shaderId); + auto shaderObject = MakeShared(stage, shaderId, &m_shaderPreprocessCache); if (shaderObject == nullptr) return 0; m_shaderObjects[shaderId] = shaderObject; return shaderId; diff --git a/MobileGL/MG_State/GLState/ProgramState/ProgramState.h b/MobileGL/MG_State/GLState/ProgramState/ProgramState.h index 50d580b8..766a836c 100644 --- a/MobileGL/MG_State/GLState/ProgramState/ProgramState.h +++ b/MobileGL/MG_State/GLState/ProgramState/ProgramState.h @@ -10,6 +10,7 @@ #include #include #include "ProgramObject.h" +#include "ShaderPreprocessCache.h" namespace MobileGL::MG_State::GLState { class ProgramState { @@ -33,6 +34,11 @@ namespace MobileGL::MG_State::GLState { const SharedPtr& GetCurrentProgram() const { return m_currentProgram; } + // P0b layer 2. Exposed for tests and diagnostics; the GL frontend never touches it + // directly - shader objects reach it through the pointer they are handed at + // CreateShader(). + ShaderPreprocessCache& GetShaderPreprocessCache() { return m_shaderPreprocessCache; } + private: Bool ShaderHasGLVisibleAttachment(const SharedPtr& shaderObject) const; // Frees the name slot and releases orphaned attached shaders; the immediate half @@ -57,6 +63,12 @@ namespace MobileGL::MG_State::GLState { // rejected with INVALID_OPERATION, and vice versa). One generator for both // object kinds keeps the names disjoint; the object tables stay separate. IndexGenerator m_programShaderNameGenerator; + + // P0b layer 2: every shader object created here is handed a pointer to this cache. + // Declared FIRST on purpose - members are destroyed in reverse declaration order, + // so the cache outlives every shader object holding a pointer to it. + ShaderPreprocessCache m_shaderPreprocessCache; + Vector> m_programObjects; Vector> m_shaderObjects; diff --git a/MobileGL/MG_State/GLState/ProgramState/ShaderObject.cpp b/MobileGL/MG_State/GLState/ProgramState/ShaderObject.cpp index 2fc1efff..d58dfd83 100644 --- a/MobileGL/MG_State/GLState/ProgramState/ShaderObject.cpp +++ b/MobileGL/MG_State/GLState/ProgramState/ShaderObject.cpp @@ -7,6 +7,7 @@ // End of Source File Header #include "ShaderObject.h" +#include "ShaderPreprocessCache.h" #include #include #include @@ -137,19 +138,91 @@ namespace { return std::nullopt; } + + // The half of ShaderObject::Compile() that depends on nothing but the source text and + // the stage: preprocessing, the two lexical rejections, and the two lexical + // side-channel extractions. Split out so P0b layer 2 can memoize exactly this and + // nothing else - the glslang parse stays per-object because its TShader is + // consume-once. Deliberately free of any per-object state so the memo is sound. + // + // Caveat, documented rather than defended against: the compute local-size verdict also + // reads the active backend's GL_MAX_COMPUTE_WORK_GROUP_* limits. Those are fixed for + // the lifetime of a context, and the cache is per-context, so the memo cannot outlive + // the limits it was computed against. + static MobileGL::MG_State::GLState::ShaderPreprocessResult RunSourceOnlyPipeline( + const MobileGL::ShaderStage stage, const MobileGL::String& source) { + using namespace MobileGL; + using namespace MobileGL::MG_Util::ShaderTranspiler; + using MobileGL::MG_State::GLState::ShaderPreprocessOutcome; + + MobileGL::MG_State::GLState::ShaderPreprocessResult result; + result.preprocessedSource = source; + PreprocessShaderSource(stage, result.preprocessedSource); + + if (stage == ShaderStage::Compute) { + if (const std::optional localSizeError = + ValidateComputeLocalSizeLimits(result.preprocessedSource)) { + result.outcome = ShaderPreprocessOutcome::ComputeLocalSizeRejected; + result.infoLog = *localSizeError; + return result; + } + } + + if (const std::optional reservedError = FindReservedIdentifierViolation(result.preprocessedSource)) { + result.outcome = ShaderPreprocessOutcome::ReservedIdentifierRejected; + result.infoLog = *reservedError; + return result; + } + + // The parse this feeds runs in the link-compatible configuration (Vulkan-client + // env with relaxed rules): the TShader it produces is what glLinkProgram links and + // what the backends' SPIR-V is generated from - there is no second, GL-client + // parse. The GL frontend semantics the relaxed parse cannot provide are restored + // on top: explicit default-block uniform locations through the lexical + // side-channels below, dead-uniform/global-UBO filtering in + // ProgramObject::DoReflection. + result.explicitUniformLocations = ExtractExplicitUniformLocations(result.preprocessedSource); + result.explicitOpaqueBindings = ExtractExplicitOpaqueBindings(result.preprocessedSource); + result.outcome = ShaderPreprocessOutcome::Preprocessed; + return result; + } } namespace MobileGL::MG_State::GLState { void ShaderObject::SetShaderSource(const String& source) { + // P0b layer 1. glShaderSource always REPLACES the source, but replacing it with a + // byte-identical one cannot change what a compile would produce: the whole + // pipeline below (preprocess -> lexical checks -> glslang parse) is a pure + // function of (stage, source) plus context-lifetime backend limits. So keeping the + // compiled state is not an optimization that changes observable behaviour - the + // COMPILE_STATUS, the info log and the reflection a caller can query are exactly + // what a real recompile would have rebuilt, byte for byte. + if (SourceMatchesCompiledState(source)) return; m_source = source; InvalidateCompiledState(); } void ShaderObject::SetShaderSource(String&& source) { + if (SourceMatchesCompiledState(source)) return; m_source = Move(source); InvalidateCompiledState(); } + Bool ShaderObject::SourceMatchesCompiledState(const String& candidate) const { + if (!m_hasCompiledState) return false; + if (candidate.length() != m_compiledSourceLength) return false; + if (ShaderPreprocessCache::HashSource(candidate) != m_compiledSourceHash) return false; + // The hash is a fast reject only; confirm against the actual stored text. While + // m_hasCompiledState holds, m_source IS the source that produced the state. + return candidate == m_source; + } + + void ShaderObject::RememberCompiledSource(const Uint64 sourceHash) { + m_hasCompiledState = true; + m_compiledSourceHash = sourceHash; + m_compiledSourceLength = m_source.length(); + } + void ShaderObject::InvalidateCompiledState() { m_shader.reset(); m_preprocessedSource.clear(); @@ -158,56 +231,76 @@ namespace MobileGL::MG_State::GLState { m_shaderConsumedByLink = false; m_compileStatus = false; m_infoLog.clear(); + m_hasCompiledState = false; + m_compiledSourceHash = 0; + m_compiledSourceLength = 0; } void ShaderObject::Compile() { using namespace MG_Util::ShaderTranspiler; + + // P0b layer 1: the state this object holds was produced by a previous Compile() of + // the exact source it still holds, so a recompile is a no-op. This covers the + // failure case too - the info log stays queryable because nothing is cleared. + // + // m_shaderConsumedByLink interaction: if the stored TShader already fed a link, + // the no-op leaves m_preprocessedSource and both side-channel maps intact, which + // is precisely what TakeShaderForLink's on-demand re-parse needs. A real recompile + // would have handed the next link a fresh parse; the no-op hands it a fresh + // re-parse of the identical source instead. Same result, one parse either way. + if (m_hasCompiledState) return; + InvalidateCompiledState(); - String compileSource = m_source; - MG_Util::ShaderTranspiler::PreprocessShaderSource(m_stage, compileSource); - if (m_stage == ShaderStage::Compute) { - const std::optional localSizeError = ValidateComputeLocalSizeLimits(compileSource); - if (localSizeError) { - m_infoLog = *localSizeError; - return; - } - } + const Uint64 sourceHash = ShaderPreprocessCache::HashSource(m_source); - const std::optional reservedError = - MG_Util::ShaderTranspiler::FindReservedIdentifierViolation(compileSource); - if (reservedError) { - m_infoLog = *reservedError; + // P0b layer 2: another shader object in this context may already have run the + // source-only half over byte-identical text. + const ShaderPreprocessResult* cached = + m_preprocessCache != nullptr ? m_preprocessCache->Find(m_stage, sourceHash, m_source) : nullptr; + ShaderPreprocessResult fresh; + if (cached == nullptr) fresh = RunSourceOnlyPipeline(m_stage, m_source); + const ShaderPreprocessResult& shared = cached != nullptr ? *cached : fresh; + const Bool shouldPopulateCache = cached == nullptr && m_preprocessCache != nullptr; + + if (!shared.Preprocessed()) { + // Rejected lexically, or a glslang failure this context has already seen for + // this exact source (ParseFailed) - either way the parse can be skipped. + m_infoLog = shared.infoLog; + if (shouldPopulateCache) m_preprocessCache->Insert(m_stage, sourceHash, m_source, Move(fresh)); + RememberCompiledSource(sourceHash); return; } - // Single parse, in the link-compatible configuration (Vulkan-client env with - // relaxed rules): the TShader stored here is what glLinkProgram links and what - // the backends' SPIR-V is generated from - there is no second, GL-client parse - // anymore. The GL frontend semantics the relaxed parse cannot provide are - // restored on top: explicit default-block uniform locations through the lexical - // side-channel below, dead-uniform/global-UBO filtering in - // ProgramObject::DoReflection. - m_explicitUniformLocations = ExtractExplicitUniformLocations(compileSource); - m_explicitOpaqueBindings = ExtractExplicitOpaqueBindings(compileSource); ShaderAttrib attrib{.shaderType = MG_Util::ConvertShaderStageToGLEnum(m_stage), - .sourceStr = compileSource, + .sourceStr = shared.preprocessedSource, .flags = 0}; auto result = ShaderCompiler::CompileShader(attrib); if (result) { m_compileStatus = true; m_shader = result.value(); - m_preprocessedSource = Move(compileSource); + // Copy, not move: `shared` may alias a cache entry that has to outlive us, and + // `fresh` is about to be handed to the cache. + m_preprocessedSource = shared.preprocessedSource; + m_explicitUniformLocations = shared.explicitUniformLocations; + m_explicitOpaqueBindings = shared.explicitOpaqueBindings; m_infoLog.clear(); + if (shouldPopulateCache) m_preprocessCache->Insert(m_stage, sourceHash, m_source, Move(fresh)); } else { - m_explicitUniformLocations.clear(); - m_explicitOpaqueBindings.clear(); m_infoLog = result.error().log; MGLOG_D("ShaderObject::Compile: Shader %d compilation failed.\nSource:\n%s\nInfoLog:\n%s\nSetting " "m_compileStatus = false as a result.", - m_externalIndex, compileSource.c_str(), m_infoLog.c_str()); + m_externalIndex, shared.preprocessedSource.c_str(), m_infoLog.c_str()); + if (shouldPopulateCache) { + fresh.outcome = ShaderPreprocessOutcome::ParseFailed; + fresh.infoLog = m_infoLog; + fresh.explicitUniformLocations.clear(); + fresh.explicitOpaqueBindings.clear(); + m_preprocessCache->Insert(m_stage, sourceHash, m_source, Move(fresh)); + } } + RememberCompiledSource(sourceHash); } SharedPtr ShaderObject::TakeShaderForLink(String& outReparseLog) { diff --git a/MobileGL/MG_State/GLState/ProgramState/ShaderObject.h b/MobileGL/MG_State/GLState/ProgramState/ShaderObject.h index 74ac9ebb..0843ff1c 100644 --- a/MobileGL/MG_State/GLState/ProgramState/ShaderObject.h +++ b/MobileGL/MG_State/GLState/ProgramState/ShaderObject.h @@ -22,10 +22,18 @@ namespace MobileGL { }; namespace MG_State::GLState { + // P0b layer 2. Declared, not included: the cache keys on ShaderStage, so including + // its header here would be circular. + class ShaderPreprocessCache; + class ShaderObject { public: - ShaderObject(const ShaderStage stage, Uint externalIndex) - : m_stage(stage), m_externalIndex(externalIndex) {} + // `preprocessCache` is the owning context's cross-object memo (P0b layer 2); + // null is fully supported and simply means "no sharing" - that is what the + // context-less internal shader objects (the default FS, the blit pipeline) use. + ShaderObject(const ShaderStage stage, Uint externalIndex, + ShaderPreprocessCache* preprocessCache = nullptr) + : m_stage(stage), m_externalIndex(externalIndex), m_preprocessCache(preprocessCache) {} void SetShaderSource(const String& source); void SetShaderSource(String&& source); void Compile(); @@ -59,8 +67,22 @@ namespace MobileGL { Bool GetCompileStatus() const { return m_compileStatus; } Bool GetDeleteStatus() const { return m_deleteStatus; } + // True while this object holds the outcome (success OR failure) of a previous + // Compile() of exactly the source it currently holds - i.e. while the P0b + // layer-1 memo is armed and a glCompileShader would be a no-op. Diagnostics + // and tests only; nothing in the GL frontend branches on it. + Bool HasMemoizedCompile() const { return m_hasCompiledState; } + private: void InvalidateCompiledState(); + // ---- P0b layer 1: per-object no-op recompile ---- + // True iff `candidate` is byte-identical to the source that produced the + // compiled state this object is currently holding. The stored hash and length + // are only a fast reject; the answer is always confirmed against the full + // stored text, so no behaviour rides on a 64-bit hash. + Bool SourceMatchesCompiledState(const String& candidate) const; + // Arms the layer-1 memo for the source that Compile() just processed. + void RememberCompiledSource(Uint64 sourceHash); const Uint m_externalIndex = 0; const ShaderStage m_stage; @@ -75,6 +97,15 @@ namespace MobileGL { UnorderedMap m_explicitOpaqueBindings; Bool m_shaderConsumedByLink = false; + // P0b layer 2: the owning context's cross-object memo, or null. Not owned. + ShaderPreprocessCache* const m_preprocessCache = nullptr; + // P0b layer 1. m_hasCompiledState is the invariant "m_source is byte-identical + // to the source that produced m_compileStatus/m_infoLog/m_shader"; it is armed + // at the end of every Compile() and disarmed by InvalidateCompiledState(). + Bool m_hasCompiledState = false; + Uint64 m_compiledSourceHash = 0; + SizeT m_compiledSourceLength = 0; + String m_infoLog; Bool m_deleteStatus = false; Bool m_compileStatus = false; diff --git a/MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.cpp b/MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.cpp new file mode 100644 index 00000000..ed35c837 --- /dev/null +++ b/MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.cpp @@ -0,0 +1,71 @@ +// MobileGL - MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.cpp +// Copyright (c) 2025-2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#include "ShaderPreprocessCache.h" + +namespace MobileGL::MG_State::GLState { + const ShaderPreprocessResult* ShaderPreprocessCache::Find(const ShaderStage stage, const Uint64 sourceHash, + const String& source) const { + const Key key{.stage = stage, .sourceHash = sourceHash, .sourceLength = source.length()}; + const auto it = m_index.find(key); + if (it == m_index.end()) return nullptr; + + // Never let correctness ride on a 64-bit hash: confirm the hit byte for byte. + // Lengths already matched (they are part of the key), so this is a plain memcmp. + const Entry& entry = *it->second; + if (entry.originalSource != source) return nullptr; + + return &entry.result; + } + + void ShaderPreprocessCache::Insert(const ShaderStage stage, const Uint64 sourceHash, const String& source, + ShaderPreprocessResult result) { + const SizeT entryBytes = EntryBytes(source, result); + // A single source bigger than the whole budget would evict every other entry and + // then itself; refuse it instead of thrashing the cache empty. + if (entryBytes > kMaxStoredSourceBytes) return; + + const Key key{.stage = stage, .sourceHash = sourceHash, .sourceLength = source.length()}; + if (const auto existing = m_index.find(key); existing != m_index.end()) { + // Either a re-insert of the same source (harmless) or a genuine hash collision + // with a different source. Both are resolved by letting the newcomer win: one + // entry per key keeps the index a plain map, and a collision is astronomically + // rare enough that the loser simply misses. + EraseEntry(existing->second); + } + + m_entries.push_back(Entry{.key = key, .originalSource = source, .result = Move(result)}); + m_index[key] = std::prev(m_entries.end()); + m_storedSourceBytes += entryBytes; + + EvictUntilWithinBudget(); + } + + void ShaderPreprocessCache::Clear() { + m_entries.clear(); + m_index.clear(); + m_storedSourceBytes = 0; + } + + void ShaderPreprocessCache::EraseEntry(const EntryList::iterator it) { + const SizeT bytes = EntryBytes(it->originalSource, it->result); + m_storedSourceBytes = bytes > m_storedSourceBytes ? 0 : m_storedSourceBytes - bytes; + m_index.erase(it->key); + m_entries.erase(it); + } + + void ShaderPreprocessCache::EvictUntilWithinBudget() { + // FIFO: the oldest insertion goes first. Insert() already refuses entries larger + // than the byte budget, so this loop always terminates with at least the entry + // that was just added still resident. + while (!m_entries.empty() && + (m_entries.size() > kMaxEntries || m_storedSourceBytes > kMaxStoredSourceBytes)) { + EraseEntry(m_entries.begin()); + } + } +} // namespace MobileGL::MG_State::GLState diff --git a/MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.h b/MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.h new file mode 100644 index 00000000..9ce1bff5 --- /dev/null +++ b/MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.h @@ -0,0 +1,140 @@ +// MobileGL - MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.h +// Copyright (c) 2025-2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#pragma once +#include +#include +#include + +namespace MobileGL::MG_State::GLState { + // Where the shared, source-only half of ShaderObject::Compile() stopped. The two + // rejection verdicts are kept apart (rather than collapsed into "failed") so a hit + // reproduces the original diagnosis, not just the original info log. + enum class ShaderPreprocessOutcome : Uint8 { + // The source-only half ran clean; preprocessedSource and both maps are valid. + Preprocessed, + // ValidateComputeLocalSizeLimits rejected it (compute only). + ComputeLocalSizeRejected, + // FindReservedIdentifierViolation rejected it. + ReservedIdentifierRejected, + // The source-only half was clean but glslang rejected the preprocessed source. + // Memoizing this saves the parse itself on every later object with that source. + ParseFailed, + }; + + // Everything ShaderObject::Compile() derives from the source text alone, i.e. + // everything that is identical for two shader objects holding byte-identical source. + struct ShaderPreprocessResult { + ShaderPreprocessOutcome outcome = ShaderPreprocessOutcome::Preprocessed; + // Valid unless the preprocessor itself never ran; kept even for the rejection + // outcomes because that is the text the diagnostics refer to. + String preprocessedSource; + UnorderedMap explicitUniformLocations; + UnorderedMap explicitOpaqueBindings; + // The compile info log to publish; empty when outcome == Preprocessed. + String infoLog; + + Bool Preprocessed() const { return outcome == ShaderPreprocessOutcome::Preprocessed; } + }; + + // P0b layer 2: a per-context, bounded memo of the source-only half of shader + // compilation, keyed by (stage, xxhash64(source), source length). + // + // Motivation: in the Iris shader-pack corpus ~21% of every glCompileShader in a trace + // is a *different* shader object holding byte-identical source (packs glue the same + // common/composite GLSL into many program stages), so the preprocess + reserved- + // identifier scan + explicit-location/binding extraction runs over the same megabytes + // again and again. Layer 1 (in ShaderObject) covers the same object recompiled with + // unchanged source; this covers the cross-object case. + // + // What is NOT cached: the glslang parse. glslang's TShader is consume-once (mapIO + // mutates the aliased intermediate at link), so every shader object still needs its + // own parse; only the text-processing half is shared. + // + // Correctness: the 64-bit hash is a lookup accelerator only. Every hit re-compares the + // full stored original source with memcmp before it is honored, so a hash collision + // degrades to a miss, never to a wrong answer. That is why the full original text is + // stored rather than a prefix/suffix digest - the cache is bounded, so the cost is. + // + // Eviction: FIFO (insertion order), bounded by BOTH an entry count and a stored-source + // byte budget, whichever binds first. FIFO rather than LRU because shader-pack loading + // is a burst of mostly-distinct sources whose reuse clusters around insertion time; + // LRU's extra list splice on every hit buys nothing measurable here, and FIFO keeps + // Find() a genuinely const, read-only operation. + class ShaderPreprocessCache { + public: + static constexpr SizeT kMaxEntries = 128; + static constexpr SizeT kMaxStoredSourceBytes = 8u * 1024u * 1024u; + + // Returns the memoized result for this exact source, or null on a miss. The + // returned pointer stays valid until the next Insert()/Clear() on this cache. + const ShaderPreprocessResult* Find(ShaderStage stage, Uint64 sourceHash, const String& source) const; + + // Memoizes `result` for this source. A source whose own storage cost already + // exceeds the byte budget is simply not cached (caching it would evict everything + // else and then itself). + void Insert(ShaderStage stage, Uint64 sourceHash, const String& source, ShaderPreprocessResult result); + + void Clear(); + + static Uint64 HashSource(const String& source) { + return static_cast(XXH64(source.data(), source.length(), 0)); + } + + SizeT GetEntryCount() const { return m_entries.size(); } + SizeT GetStoredSourceBytes() const { return m_storedSourceBytes; } + + private: + struct Key { + ShaderStage stage = ShaderStage::Unknown; + Uint64 sourceHash = 0; + SizeT sourceLength = 0; + + Bool operator==(const Key& other) const { + return stage == other.stage && sourceHash == other.sourceHash && sourceLength == other.sourceLength; + } + }; + + struct KeyHasher { + SizeT operator()(const Key& key) const { + // The source hash already spreads well; fold the two discriminators in so + // that same-hash-different-stage/length keys land in different buckets. + Uint64 mixed = key.sourceHash; + mixed ^= static_cast(key.sourceLength) + 0x9e3779b97f4a7c15ull + (mixed << 6) + (mixed >> 2); + mixed ^= static_cast(static_cast(key.stage)) * 0xff51afd7ed558ccdull; + return static_cast(mixed); + } + }; + + struct Entry { + Key key; + // The full original (pre-preprocess) source, kept so a hit can be confirmed by + // comparison instead of trusting the hash. + String originalSource; + ShaderPreprocessResult result; + }; + + using EntryList = std::list; + + static SizeT EntryBytes(const String& source, const ShaderPreprocessResult& result) { + return source.length() + result.preprocessedSource.length(); + } + + void EraseEntry(EntryList::iterator it); + void EvictUntilWithinBudget(); + + // P1: needs a mutex when compiles go async. Everything here is reached from + // glCompileShader on the single GL thread that owns the context, so today the + // cache is deliberately lock-free; the moment shader compilation moves onto a + // worker pool, Find/Insert/Clear all become critical sections (and Find's returned + // pointer stops being safe to hold across an Insert). + EntryList m_entries; // front = oldest (FIFO victim) + UnorderedMap m_index; + SizeT m_storedSourceBytes = 0; + }; +} // namespace MobileGL::MG_State::GLState diff --git a/MobileGL/MG_Test/Program/CMakeLists.txt b/MobileGL/MG_Test/Program/CMakeLists.txt index 48671244..95d23788 100644 --- a/MobileGL/MG_Test/Program/CMakeLists.txt +++ b/MobileGL/MG_Test/Program/CMakeLists.txt @@ -9,6 +9,7 @@ add_executable( ${MGL_ROOT}/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp ${MGL_ROOT}/MobileGL/MG_Util/ShaderTranspiler/SpvcSession.cpp ${MGL_ROOT}/MobileGL/MG_Util/ShaderTranspiler/glslang/UniformTraverser.cpp + ${MGL_ROOT}/MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.cpp ) target_include_directories(ProgramUtilTest PRIVATE diff --git a/MobileGL/MG_Test/Program/ProgramTest.cpp b/MobileGL/MG_Test/Program/ProgramTest.cpp index 6abc03cd..ed830a2f 100644 --- a/MobileGL/MG_Test/Program/ProgramTest.cpp +++ b/MobileGL/MG_Test/Program/ProgramTest.cpp @@ -19,6 +19,7 @@ #include "MG_Impl/GLImpl/Getter/GL_Getter.h" #include "MG_Impl/GLImpl/Program/GL_Program.h" #include "MG_State/GLState/Core.h" +#include "MG_State/GLState/ProgramState/ShaderPreprocessCache.h" #include "MG_Util/ShaderTranspiler/ShaderCompiler.h" using namespace MobileGL; @@ -2834,3 +2835,224 @@ void main() { fragColor = vec4(pow(uBase, 2.2), 1.0); } EXPECT_EQ(GetError(), GL_NO_ERROR); } + +// --------------------------------------------------------------------------- +// P0b: source-hash dedupe for shader recompiles. +// Layer 1 - the same shader object re-sourced with byte-identical text keeps its +// compiled state, and glCompileShader on it is a no-op. +// Layer 2 - two DIFFERENT shader objects holding byte-identical text share the +// source-only half of the pipeline (preprocess + lexical checks + +// side-channel extraction) through the context's ShaderPreprocessCache, +// while each still gets its own glslang parse. +// --------------------------------------------------------------------------- +namespace { + const char* kP0bVs = R"(#version 330 core +uniform mat4 uModel; +uniform vec4 uTint; +void main() { gl_Position = uModel * uTint; } +)"; + const char* kP0bFs = R"(#version 330 core +uniform vec4 uColor; +out vec4 fragColor; +void main() { fragColor = uColor; } +)"; + // Same stage, different declared uniform: makes "did it actually recompile?" + // observable through reflection rather than through internal state. + const char* kP0bAltFs = R"(#version 330 core +uniform vec4 uOtherColor; +out vec4 fragColor; +void main() { fragColor = uOtherColor; } +)"; + const char* kP0bBrokenFs = R"(#version 330 core +out vec4 fragColor; +void main() { fragColor = notADeclaredThing; } +)"; + + GLuint MakeShaderWithSource(GLenum type, const char* source) { + GLuint shader = CreateShader(type); + ShaderSource(shader, 1, &source, nullptr); + return shader; + } + + GLint QueryCompileStatus(GLuint shader) { + GLint status = GL_FALSE; + GetShaderiv(shader, GL_COMPILE_STATUS, &status); + return status; + } + + String QueryShaderInfoLog(GLuint shader) { + GLint length = 0; + GetShaderiv(shader, GL_INFO_LOG_LENGTH, &length); + if (length <= 0) return String(); + std::vector buffer(static_cast(length)); + GLsizei written = 0; + GetShaderInfoLog(shader, length, &written, buffer.data()); + return String(buffer.data(), static_cast(written)); + } + + Bool ShaderHasMemoizedCompile(GLuint shader) { + const auto& shaderObject = MG_State::pGLContext->GetShaderObject(shader); + EXPECT_NE(shaderObject, nullptr); + return shaderObject != nullptr && shaderObject->HasMemoizedCompile(); + } +} // namespace + +// Layer 1, success path: re-sourcing with identical text and recompiling must leave +// COMPILE_STATUS, the info log and every downstream consumer exactly as they were - +// including a program that links the shader AFTER the redundant recompile. +TEST_F(ProgramTest, RecompileWithIdenticalSourceKeepsCompiledStateAndStillLinks) { + GLuint vs = MakeShaderWithSource(GL_VERTEX_SHADER, kP0bVs); + GLuint fs = MakeShaderWithSource(GL_FRAGMENT_SHADER, kP0bFs); + CompileShader(vs); + CompileShader(fs); + ASSERT_EQ(QueryCompileStatus(vs), GL_TRUE) << QueryShaderInfoLog(vs); + ASSERT_EQ(QueryCompileStatus(fs), GL_TRUE) << QueryShaderInfoLog(fs); + const String vsLogBefore = QueryShaderInfoLog(vs); + EXPECT_TRUE(ShaderHasMemoizedCompile(vs)); + + // A first link consumes the stored TShader; the redundant recompile below must not + // disturb the preprocessed source that TakeShaderForLink re-parses from. + GLuint firstProgram = LinkVsFs(vs, fs, GL_TRUE); + EXPECT_GE(GetUniformLocation(firstProgram, "uColor"), 0); + + // glShaderSource with byte-identical text, then glCompileShader: both no-ops. + ShaderSource(vs, 1, &kP0bVs, nullptr); + EXPECT_TRUE(ShaderHasMemoizedCompile(vs)) << "identical re-source must not invalidate the compiled state"; + CompileShader(vs); + ShaderSource(fs, 1, &kP0bFs, nullptr); + CompileShader(fs); + + EXPECT_EQ(QueryCompileStatus(vs), GL_TRUE); + EXPECT_EQ(QueryCompileStatus(fs), GL_TRUE); + EXPECT_EQ(QueryShaderInfoLog(vs), vsLogBefore); + + // The original source text is still what glGetShaderSource reports. + GLint sourceLength = 0; + GetShaderiv(vs, GL_SHADER_SOURCE_LENGTH, &sourceLength); + ASSERT_GT(sourceLength, 1); + std::vector sourceBuffer(static_cast(sourceLength)); + GLsizei written = 0; + GetShaderSource(vs, sourceLength, &written, sourceBuffer.data()); + EXPECT_EQ(String(sourceBuffer.data(), static_cast(written)), String(kP0bVs)); + + // A second program built from the same, redundantly recompiled shaders links and + // reflects - i.e. TakeShaderForLink's re-parse path survived the no-op. + GLuint secondProgram = LinkVsFs(vs, fs, GL_TRUE); + EXPECT_GE(GetUniformLocation(secondProgram, "uColor"), 0); + EXPECT_GE(GetUniformLocation(secondProgram, "uModel"), 0); + EXPECT_EQ(GetError(), GL_NO_ERROR); +} + +// Layer 1 must not swallow a REAL source change: different text invalidates, and the +// change is visible in what the next link reflects. +TEST_F(ProgramTest, DifferentSourceAfterCompileInvalidatesCompiledState) { + GLuint vs = CompileShaderChecked(GL_VERTEX_SHADER, kP0bVs); + GLuint fs = MakeShaderWithSource(GL_FRAGMENT_SHADER, kP0bFs); + CompileShader(fs); + ASSERT_EQ(QueryCompileStatus(fs), GL_TRUE) << QueryShaderInfoLog(fs); + GLuint firstProgram = LinkVsFs(vs, fs, GL_TRUE); + EXPECT_GE(GetUniformLocation(firstProgram, "uColor"), 0); + EXPECT_EQ(GetUniformLocation(firstProgram, "uOtherColor"), -1); + + // New text -> compiled state gone, and glCompileShader is mandatory again. + ShaderSource(fs, 1, &kP0bAltFs, nullptr); + EXPECT_FALSE(ShaderHasMemoizedCompile(fs)); + EXPECT_EQ(QueryCompileStatus(fs), GL_FALSE); + + CompileShader(fs); + ASSERT_EQ(QueryCompileStatus(fs), GL_TRUE) << QueryShaderInfoLog(fs); + GLuint secondProgram = LinkVsFs(vs, fs, GL_TRUE); + EXPECT_GE(GetUniformLocation(secondProgram, "uOtherColor"), 0); + EXPECT_EQ(GetUniformLocation(secondProgram, "uColor"), -1); + EXPECT_EQ(GetError(), GL_NO_ERROR); +} + +// Layer 2: byte-identical source in two distinct shader objects. Both must compile, +// and each must own an independent TShader - if the parse were shared, the second +// link would be handed an intermediate that the first link's mapIO already mutated. +TEST_F(ProgramTest, TwoShaderObjectsWithIdenticalSourceLinkIndependently) { + GLuint vsA = CompileShaderChecked(GL_VERTEX_SHADER, kP0bVs); + GLuint fsA = CompileShaderChecked(GL_FRAGMENT_SHADER, kP0bFs); + GLuint vsB = CompileShaderChecked(GL_VERTEX_SHADER, kP0bVs); + GLuint fsB = CompileShaderChecked(GL_FRAGMENT_SHADER, kP0bFs); + ASSERT_NE(vsA, vsB); + ASSERT_NE(fsA, fsB); + + const auto& objectA = MG_State::pGLContext->GetShaderObject(vsA); + const auto& objectB = MG_State::pGLContext->GetShaderObject(vsB); + ASSERT_NE(objectA, nullptr); + ASSERT_NE(objectB, nullptr); + EXPECT_EQ(objectA->GetShaderSource(), objectB->GetShaderSource()); + // Independent parses despite the shared preprocess. + EXPECT_NE(objectA->GetCompiledShader(), objectB->GetCompiledShader()); + EXPECT_NE(objectA->GetCompiledShader(), nullptr); + EXPECT_NE(objectB->GetCompiledShader(), nullptr); + + GLuint programA = LinkVsFs(vsA, fsA, GL_TRUE); + GLuint programB = LinkVsFs(vsB, fsB, GL_TRUE); + for (GLuint program : {programA, programB}) { + EXPECT_GE(GetUniformLocation(program, "uColor"), 0); + EXPECT_GE(GetUniformLocation(program, "uModel"), 0); + } + EXPECT_EQ(GetError(), GL_NO_ERROR); +} + +// Failure memoization: a compile that failed stays failed, with the SAME log, when +// recompiled against the same source; a real fix to the source still takes effect. +// The second object pins the cached-ParseFailed path (layer 2), which skips the parse +// entirely and must reproduce the identical verdict. +TEST_F(ProgramTest, FailedCompileIsMemoizedAndStillRecoversOnGoodSource) { + GLuint fs = MakeShaderWithSource(GL_FRAGMENT_SHADER, kP0bBrokenFs); + CompileShader(fs); + ASSERT_EQ(QueryCompileStatus(fs), GL_FALSE); + const String failureLog = QueryShaderInfoLog(fs); + EXPECT_FALSE(failureLog.empty()); + + // Layer 1: identical re-source + recompile keeps the failure AND the log queryable. + ShaderSource(fs, 1, &kP0bBrokenFs, nullptr); + CompileShader(fs); + EXPECT_EQ(QueryCompileStatus(fs), GL_FALSE); + EXPECT_EQ(QueryShaderInfoLog(fs), failureLog); + + // Layer 2: a second object with the same broken source reports the same failure. + GLuint otherFs = MakeShaderWithSource(GL_FRAGMENT_SHADER, kP0bBrokenFs); + CompileShader(otherFs); + EXPECT_EQ(QueryCompileStatus(otherFs), GL_FALSE); + EXPECT_EQ(QueryShaderInfoLog(otherFs), failureLog); + + // A genuine fix still compiles and links. + ShaderSource(fs, 1, &kP0bFs, nullptr); + CompileShader(fs); + ASSERT_EQ(QueryCompileStatus(fs), GL_TRUE) << QueryShaderInfoLog(fs); + EXPECT_TRUE(QueryShaderInfoLog(fs).empty()); + GLuint vs = CompileShaderChecked(GL_VERTEX_SHADER, kP0bVs); + GLuint program = LinkVsFs(vs, fs, GL_TRUE); + EXPECT_GE(GetUniformLocation(program, "uColor"), 0); +} + +// Layer 2 under eviction: push more distinct sources through the context than the +// cache can hold, then confirm nothing broke and a fresh duplicate pair still works. +TEST_F(ProgramTest, PreprocessCacheOverflowKeepsCompilingCorrectly) { + const SizeT overflow = MG_State::GLState::ShaderPreprocessCache::kMaxEntries + 8; + for (SizeT i = 0; i < overflow; ++i) { + const String source = "#version 330 core\nuniform vec4 uColor" + ToString(i) + + ";\nout vec4 fragColor;\nvoid main() { fragColor = uColor" + ToString(i) + "; }\n"; + const char* sourcePtr = source.c_str(); + GLuint shader = MakeShaderWithSource(GL_FRAGMENT_SHADER, sourcePtr); + CompileShader(shader); + ASSERT_EQ(QueryCompileStatus(shader), GL_TRUE) << QueryShaderInfoLog(shader) << "\n" << source; + DeleteShader(shader); + } + + // Everything inserted above has long since been evicted; a brand-new duplicate + // pair must still take the layer-2 path and produce two working programs. + GLuint vsA = CompileShaderChecked(GL_VERTEX_SHADER, kP0bVs); + GLuint fsA = CompileShaderChecked(GL_FRAGMENT_SHADER, kP0bFs); + GLuint vsB = CompileShaderChecked(GL_VERTEX_SHADER, kP0bVs); + GLuint fsB = CompileShaderChecked(GL_FRAGMENT_SHADER, kP0bFs); + GLuint programA = LinkVsFs(vsA, fsA, GL_TRUE); + GLuint programB = LinkVsFs(vsB, fsB, GL_TRUE); + EXPECT_GE(GetUniformLocation(programA, "uColor"), 0); + EXPECT_GE(GetUniformLocation(programB, "uColor"), 0); + EXPECT_EQ(GetError(), GL_NO_ERROR); +} diff --git a/MobileGL/MG_Test/Program/ProgramUtilTest.cpp b/MobileGL/MG_Test/Program/ProgramUtilTest.cpp index 700ee23a..f0dec2f9 100644 --- a/MobileGL/MG_Test/Program/ProgramUtilTest.cpp +++ b/MobileGL/MG_Test/Program/ProgramUtilTest.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -2375,3 +2376,161 @@ void main() { } } } + +// --------------------------------------------------------------------------- +// P0b layer 2: ShaderPreprocessCache, tested directly. The GL-level behaviour it +// enables is covered end to end in ProgramTest; these pin the container itself, +// where the interesting cases (hash collisions, both eviction budgets) are hard +// to provoke through glCompileShader. +// --------------------------------------------------------------------------- +namespace { + using MobileGL::MG_State::GLState::ShaderPreprocessCache; + using MobileGL::MG_State::GLState::ShaderPreprocessOutcome; + using MobileGL::MG_State::GLState::ShaderPreprocessResult; + + ShaderPreprocessResult MakeResult(const String& preprocessed) { + ShaderPreprocessResult result; + result.outcome = ShaderPreprocessOutcome::Preprocessed; + result.preprocessedSource = preprocessed; + result.explicitUniformLocations["uMarker"] = 7; + result.explicitOpaqueBindings["sMarker"] = 3; + return result; + } +} // namespace + +TEST_F(ProgramUtilTest, ShaderPreprocessCacheRoundTripsAndSeparatesStages) { + ShaderPreprocessCache cache; + const String source = "// a shader\nvoid main() {}\n"; + const Uint64 hash = ShaderPreprocessCache::HashSource(source); + + EXPECT_EQ(cache.Find(ShaderStage::Vertex, hash, source), nullptr); + + cache.Insert(ShaderStage::Vertex, hash, source, MakeResult("vertex-preprocessed")); + const ShaderPreprocessResult* hit = cache.Find(ShaderStage::Vertex, hash, source); + ASSERT_NE(hit, nullptr); + EXPECT_TRUE(hit->Preprocessed()); + EXPECT_EQ(hit->preprocessedSource, "vertex-preprocessed"); + const auto uniformIt = hit->explicitUniformLocations.find("uMarker"); + ASSERT_NE(uniformIt, hit->explicitUniformLocations.end()); + EXPECT_EQ(uniformIt->second, 7); + const auto bindingIt = hit->explicitOpaqueBindings.find("sMarker"); + ASSERT_NE(bindingIt, hit->explicitOpaqueBindings.end()); + EXPECT_EQ(bindingIt->second, 3u); + + // Byte-identical source, different stage: a different key, so still a miss. Two + // stages sharing one entry would hand a fragment shader a vertex preprocess. + EXPECT_EQ(cache.Find(ShaderStage::Fragment, hash, source), nullptr); + cache.Insert(ShaderStage::Fragment, hash, source, MakeResult("fragment-preprocessed")); + const ShaderPreprocessResult* fragmentHit = cache.Find(ShaderStage::Fragment, hash, source); + ASSERT_NE(fragmentHit, nullptr); + EXPECT_EQ(fragmentHit->preprocessedSource, "fragment-preprocessed"); + EXPECT_EQ(cache.Find(ShaderStage::Vertex, hash, source)->preprocessedSource, "vertex-preprocessed"); + EXPECT_EQ(cache.GetEntryCount(), 2u); +} + +TEST_F(ProgramUtilTest, ShaderPreprocessCacheMemoizesRejectionVerdictsDistinctly) { + ShaderPreprocessCache cache; + const String reservedSource = "int packed;\n"; + const String localSizeSource = "layout(local_size_x = 99999) in;\n"; + + ShaderPreprocessResult reserved; + reserved.outcome = ShaderPreprocessOutcome::ReservedIdentifierRejected; + reserved.infoLog = "reserved identifier"; + ShaderPreprocessResult localSize; + localSize.outcome = ShaderPreprocessOutcome::ComputeLocalSizeRejected; + localSize.infoLog = "local_size too big"; + + cache.Insert(ShaderStage::Vertex, ShaderPreprocessCache::HashSource(reservedSource), reservedSource, + std::move(reserved)); + cache.Insert(ShaderStage::Compute, ShaderPreprocessCache::HashSource(localSizeSource), localSizeSource, + std::move(localSize)); + + const ShaderPreprocessResult* reservedHit = + cache.Find(ShaderStage::Vertex, ShaderPreprocessCache::HashSource(reservedSource), reservedSource); + ASSERT_NE(reservedHit, nullptr); + EXPECT_FALSE(reservedHit->Preprocessed()); + EXPECT_EQ(reservedHit->outcome, ShaderPreprocessOutcome::ReservedIdentifierRejected); + EXPECT_EQ(reservedHit->infoLog, "reserved identifier"); + + const ShaderPreprocessResult* localSizeHit = + cache.Find(ShaderStage::Compute, ShaderPreprocessCache::HashSource(localSizeSource), localSizeSource); + ASSERT_NE(localSizeHit, nullptr); + EXPECT_EQ(localSizeHit->outcome, ShaderPreprocessOutcome::ComputeLocalSizeRejected); + EXPECT_EQ(localSizeHit->infoLog, "local_size too big"); +} + +// Correctness must not ride on a 64-bit hash. Feed two different sources of the same +// length under a forged, identical hash: the entry stores the full original text, so +// the impostor lookup must miss instead of returning the wrong preprocess. +TEST_F(ProgramUtilTest, ShaderPreprocessCacheRejectsForgedHashCollision) { + ShaderPreprocessCache cache; + const String real = "void main() { int a = 1; }\n"; + const String impostor = "void main() { int a = 2; }\n"; + ASSERT_EQ(real.length(), impostor.length()); + ASSERT_NE(real, impostor); + const Uint64 forgedHash = 0xdeadbeefcafef00dull; + + cache.Insert(ShaderStage::Vertex, forgedHash, real, MakeResult("real-preprocessed")); + + ASSERT_NE(cache.Find(ShaderStage::Vertex, forgedHash, real), nullptr); + EXPECT_EQ(cache.Find(ShaderStage::Vertex, forgedHash, impostor), nullptr); + + // The colliding newcomer wins the slot rather than being silently dropped, so it + // is the previous occupant that degrades to a miss - never a wrong hit. + cache.Insert(ShaderStage::Vertex, forgedHash, impostor, MakeResult("impostor-preprocessed")); + const ShaderPreprocessResult* impostorHit = cache.Find(ShaderStage::Vertex, forgedHash, impostor); + ASSERT_NE(impostorHit, nullptr); + EXPECT_EQ(impostorHit->preprocessedSource, "impostor-preprocessed"); + EXPECT_EQ(cache.Find(ShaderStage::Vertex, forgedHash, real), nullptr); + EXPECT_EQ(cache.GetEntryCount(), 1u); +} + +TEST_F(ProgramUtilTest, ShaderPreprocessCacheEvictsFifoOnEntryCap) { + ShaderPreprocessCache cache; + Vector sources; + const SizeT overflow = ShaderPreprocessCache::kMaxEntries + 8; + for (SizeT i = 0; i < overflow; ++i) { + sources.push_back("void main() { int a = " + ToString(i) + "; }\n"); + cache.Insert(ShaderStage::Vertex, ShaderPreprocessCache::HashSource(sources.back()), sources.back(), + MakeResult("pp" + ToString(i))); + EXPECT_LE(cache.GetEntryCount(), ShaderPreprocessCache::kMaxEntries); + } + EXPECT_EQ(cache.GetEntryCount(), ShaderPreprocessCache::kMaxEntries); + + // FIFO: the first `overflow - kMaxEntries` insertions are gone, the rest resident. + for (SizeT i = 0; i < overflow; ++i) { + const ShaderPreprocessResult* hit = + cache.Find(ShaderStage::Vertex, ShaderPreprocessCache::HashSource(sources[i]), sources[i]); + if (i < overflow - ShaderPreprocessCache::kMaxEntries) { + EXPECT_EQ(hit, nullptr) << "entry " << i << " should have been evicted"; + } else { + ASSERT_NE(hit, nullptr) << "entry " << i << " should still be resident"; + EXPECT_EQ(hit->preprocessedSource, "pp" + ToString(i)); + } + } + + cache.Clear(); + EXPECT_EQ(cache.GetEntryCount(), 0u); + EXPECT_EQ(cache.GetStoredSourceBytes(), 0u); +} + +TEST_F(ProgramUtilTest, ShaderPreprocessCacheHonorsByteBudget) { + ShaderPreprocessCache cache; + // Well under the entry cap, well over the byte budget: the byte budget must be the + // one that binds, and the accounting must come back down as entries are evicted. + const SizeT chunk = ShaderPreprocessCache::kMaxStoredSourceBytes / 8; + for (SizeT i = 0; i < 24; ++i) { + String source(chunk, static_cast('a' + (i % 26))); + cache.Insert(ShaderStage::Vertex, ShaderPreprocessCache::HashSource(source), source, MakeResult("")); + EXPECT_LE(cache.GetStoredSourceBytes(), ShaderPreprocessCache::kMaxStoredSourceBytes); + EXPECT_LT(cache.GetEntryCount(), ShaderPreprocessCache::kMaxEntries); + } + + // A single source larger than the whole budget is refused outright: caching it + // would evict every other entry and then immediately itself. + const SizeT before = cache.GetEntryCount(); + const String oversized(ShaderPreprocessCache::kMaxStoredSourceBytes + 1, 'z'); + cache.Insert(ShaderStage::Vertex, ShaderPreprocessCache::HashSource(oversized), oversized, MakeResult("")); + EXPECT_EQ(cache.GetEntryCount(), before); + EXPECT_EQ(cache.Find(ShaderStage::Vertex, ShaderPreprocessCache::HashSource(oversized), oversized), nullptr); +}