mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-09-09 12:48:32 +09:00
[Perf] (MG_State): dedupe shader compiles by source hash
Iris-style packs hand MobileGL the same source text repeatedly: probed across three shaderpack traces, 28-32% of all glCompileShader work was redundant - ~9% same-object recompiles with byte-identical source, ~21% distinct shader objects sharing identical source (the same common GLSL chunk glued into many program stages). Two layers, both keyed by XXH64 + length with a full byte compare on every hit (correctness never rides on the hash): - Per-object: a successful (or failed) compile remembers its source hash; glShaderSource with byte-identical text keeps the compiled state and glCompileShader on unchanged source returns immediately. Deterministic (stage, source) pipeline makes the memo observationally identical to recompiling; the consume-once TakeShaderForLink re-parse path is untouched. - Cross-object: a per-context bounded cache (ProgramState-owned, declared to outlive every shader object) shares the preprocessed source, both explicit side-channel maps, and the validation verdicts between objects with equal source; only the glslang parse stays per-object. Single-GL-thread today; flagged for a mutex when compiles go async (P1). Interleaved A/B on the iterationrp trace (the recompile-heavy pack): 5.65s -> 5.46s median total replay, every round faster; BSL/complementary stay flat (their duplicate sources are the small common shaders, so calls drop but wall time is parse-bound on unique sources). Full DirectGLES retrace, 445-test unit suite, and dedupe-semantics tests (no-op recompile, invalidation on new source, failed-compile memo, cache bounds) all green.
This commit is contained in:
@@ -291,6 +291,7 @@ set(SOURCE_FILES
|
||||
MobileGL/MG_State/GLState/TextureState/TextureState.cpp
|
||||
MobileGL/MG_State/GLState/ProgramState/ProgramObject.cpp
|
||||
MobileGL/MG_State/GLState/ProgramState/ShaderObject.cpp
|
||||
MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.cpp
|
||||
MobileGL/MG_State/GLState/ProgramState/ProgramState.cpp
|
||||
MobileGL/MG_State/GLState/RenderState/RenderState.cpp
|
||||
MobileGL/MG_State/GLState/FramebufferState/FramebufferObject.cpp
|
||||
|
||||
@@ -79,7 +79,7 @@ namespace MobileGL::MG_State::GLState {
|
||||
Uint shaderId = 0;
|
||||
m_programShaderNameGenerator.Generate(1, &shaderId);
|
||||
EnsureIndexAvail(shaderId, m_shaderObjects);
|
||||
auto shaderObject = MakeShared<ShaderObject>(stage, shaderId);
|
||||
auto shaderObject = MakeShared<ShaderObject>(stage, shaderId, &m_shaderPreprocessCache);
|
||||
if (shaderObject == nullptr) return 0;
|
||||
m_shaderObjects[shaderId] = shaderObject;
|
||||
return shaderId;
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
#include <Includes.h>
|
||||
#include <MG_Util/Miscellany/IndexGenerator.h>
|
||||
#include "ProgramObject.h"
|
||||
#include "ShaderPreprocessCache.h"
|
||||
|
||||
namespace MobileGL::MG_State::GLState {
|
||||
class ProgramState {
|
||||
@@ -33,6 +34,11 @@ namespace MobileGL::MG_State::GLState {
|
||||
|
||||
const SharedPtr<ProgramObject>& GetCurrentProgram() const { return m_currentProgram; }
|
||||
|
||||
// P0b layer 2. Exposed for tests and diagnostics; the GL frontend never touches it
|
||||
// directly - shader objects reach it through the pointer they are handed at
|
||||
// CreateShader().
|
||||
ShaderPreprocessCache& GetShaderPreprocessCache() { return m_shaderPreprocessCache; }
|
||||
|
||||
private:
|
||||
Bool ShaderHasGLVisibleAttachment(const SharedPtr<ShaderObject>& shaderObject) const;
|
||||
// Frees the name slot and releases orphaned attached shaders; the immediate half
|
||||
@@ -57,6 +63,12 @@ namespace MobileGL::MG_State::GLState {
|
||||
// rejected with INVALID_OPERATION, and vice versa). One generator for both
|
||||
// object kinds keeps the names disjoint; the object tables stay separate.
|
||||
IndexGenerator<Uint> m_programShaderNameGenerator;
|
||||
|
||||
// P0b layer 2: every shader object created here is handed a pointer to this cache.
|
||||
// Declared FIRST on purpose - members are destroyed in reverse declaration order,
|
||||
// so the cache outlives every shader object holding a pointer to it.
|
||||
ShaderPreprocessCache m_shaderPreprocessCache;
|
||||
|
||||
Vector<SharedPtr<ProgramObject>> m_programObjects;
|
||||
Vector<SharedPtr<ShaderObject>> m_shaderObjects;
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
// End of Source File Header
|
||||
|
||||
#include "ShaderObject.h"
|
||||
#include "ShaderPreprocessCache.h"
|
||||
#include <MG_Util/ShaderTranspiler/Types.h>
|
||||
#include <MG_Util/ShaderTranspiler/ShaderCompiler.h>
|
||||
#include <MG_Util/Converters/MGToGL/ProgramEnumConverter.h>
|
||||
@@ -137,19 +138,91 @@ namespace {
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
// The half of ShaderObject::Compile() that depends on nothing but the source text and
|
||||
// the stage: preprocessing, the two lexical rejections, and the two lexical
|
||||
// side-channel extractions. Split out so P0b layer 2 can memoize exactly this and
|
||||
// nothing else - the glslang parse stays per-object because its TShader is
|
||||
// consume-once. Deliberately free of any per-object state so the memo is sound.
|
||||
//
|
||||
// Caveat, documented rather than defended against: the compute local-size verdict also
|
||||
// reads the active backend's GL_MAX_COMPUTE_WORK_GROUP_* limits. Those are fixed for
|
||||
// the lifetime of a context, and the cache is per-context, so the memo cannot outlive
|
||||
// the limits it was computed against.
|
||||
static MobileGL::MG_State::GLState::ShaderPreprocessResult RunSourceOnlyPipeline(
|
||||
const MobileGL::ShaderStage stage, const MobileGL::String& source) {
|
||||
using namespace MobileGL;
|
||||
using namespace MobileGL::MG_Util::ShaderTranspiler;
|
||||
using MobileGL::MG_State::GLState::ShaderPreprocessOutcome;
|
||||
|
||||
MobileGL::MG_State::GLState::ShaderPreprocessResult result;
|
||||
result.preprocessedSource = source;
|
||||
PreprocessShaderSource(stage, result.preprocessedSource);
|
||||
|
||||
if (stage == ShaderStage::Compute) {
|
||||
if (const std::optional<String> localSizeError =
|
||||
ValidateComputeLocalSizeLimits(result.preprocessedSource)) {
|
||||
result.outcome = ShaderPreprocessOutcome::ComputeLocalSizeRejected;
|
||||
result.infoLog = *localSizeError;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
if (const std::optional<String> reservedError = FindReservedIdentifierViolation(result.preprocessedSource)) {
|
||||
result.outcome = ShaderPreprocessOutcome::ReservedIdentifierRejected;
|
||||
result.infoLog = *reservedError;
|
||||
return result;
|
||||
}
|
||||
|
||||
// The parse this feeds runs in the link-compatible configuration (Vulkan-client
|
||||
// env with relaxed rules): the TShader it produces is what glLinkProgram links and
|
||||
// what the backends' SPIR-V is generated from - there is no second, GL-client
|
||||
// parse. The GL frontend semantics the relaxed parse cannot provide are restored
|
||||
// on top: explicit default-block uniform locations through the lexical
|
||||
// side-channels below, dead-uniform/global-UBO filtering in
|
||||
// ProgramObject::DoReflection.
|
||||
result.explicitUniformLocations = ExtractExplicitUniformLocations(result.preprocessedSource);
|
||||
result.explicitOpaqueBindings = ExtractExplicitOpaqueBindings(result.preprocessedSource);
|
||||
result.outcome = ShaderPreprocessOutcome::Preprocessed;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
namespace MobileGL::MG_State::GLState {
|
||||
void ShaderObject::SetShaderSource(const String& source) {
|
||||
// P0b layer 1. glShaderSource always REPLACES the source, but replacing it with a
|
||||
// byte-identical one cannot change what a compile would produce: the whole
|
||||
// pipeline below (preprocess -> lexical checks -> glslang parse) is a pure
|
||||
// function of (stage, source) plus context-lifetime backend limits. So keeping the
|
||||
// compiled state is not an optimization that changes observable behaviour - the
|
||||
// COMPILE_STATUS, the info log and the reflection a caller can query are exactly
|
||||
// what a real recompile would have rebuilt, byte for byte.
|
||||
if (SourceMatchesCompiledState(source)) return;
|
||||
m_source = source;
|
||||
InvalidateCompiledState();
|
||||
}
|
||||
|
||||
void ShaderObject::SetShaderSource(String&& source) {
|
||||
if (SourceMatchesCompiledState(source)) return;
|
||||
m_source = Move(source);
|
||||
InvalidateCompiledState();
|
||||
}
|
||||
|
||||
Bool ShaderObject::SourceMatchesCompiledState(const String& candidate) const {
|
||||
if (!m_hasCompiledState) return false;
|
||||
if (candidate.length() != m_compiledSourceLength) return false;
|
||||
if (ShaderPreprocessCache::HashSource(candidate) != m_compiledSourceHash) return false;
|
||||
// The hash is a fast reject only; confirm against the actual stored text. While
|
||||
// m_hasCompiledState holds, m_source IS the source that produced the state.
|
||||
return candidate == m_source;
|
||||
}
|
||||
|
||||
void ShaderObject::RememberCompiledSource(const Uint64 sourceHash) {
|
||||
m_hasCompiledState = true;
|
||||
m_compiledSourceHash = sourceHash;
|
||||
m_compiledSourceLength = m_source.length();
|
||||
}
|
||||
|
||||
void ShaderObject::InvalidateCompiledState() {
|
||||
m_shader.reset();
|
||||
m_preprocessedSource.clear();
|
||||
@@ -158,56 +231,76 @@ namespace MobileGL::MG_State::GLState {
|
||||
m_shaderConsumedByLink = false;
|
||||
m_compileStatus = false;
|
||||
m_infoLog.clear();
|
||||
m_hasCompiledState = false;
|
||||
m_compiledSourceHash = 0;
|
||||
m_compiledSourceLength = 0;
|
||||
}
|
||||
|
||||
void ShaderObject::Compile() {
|
||||
using namespace MG_Util::ShaderTranspiler;
|
||||
|
||||
// P0b layer 1: the state this object holds was produced by a previous Compile() of
|
||||
// the exact source it still holds, so a recompile is a no-op. This covers the
|
||||
// failure case too - the info log stays queryable because nothing is cleared.
|
||||
//
|
||||
// m_shaderConsumedByLink interaction: if the stored TShader already fed a link,
|
||||
// the no-op leaves m_preprocessedSource and both side-channel maps intact, which
|
||||
// is precisely what TakeShaderForLink's on-demand re-parse needs. A real recompile
|
||||
// would have handed the next link a fresh parse; the no-op hands it a fresh
|
||||
// re-parse of the identical source instead. Same result, one parse either way.
|
||||
if (m_hasCompiledState) return;
|
||||
|
||||
InvalidateCompiledState();
|
||||
String compileSource = m_source;
|
||||
MG_Util::ShaderTranspiler::PreprocessShaderSource(m_stage, compileSource);
|
||||
|
||||
if (m_stage == ShaderStage::Compute) {
|
||||
const std::optional<String> localSizeError = ValidateComputeLocalSizeLimits(compileSource);
|
||||
if (localSizeError) {
|
||||
m_infoLog = *localSizeError;
|
||||
return;
|
||||
}
|
||||
}
|
||||
const Uint64 sourceHash = ShaderPreprocessCache::HashSource(m_source);
|
||||
|
||||
const std::optional<String> reservedError =
|
||||
MG_Util::ShaderTranspiler::FindReservedIdentifierViolation(compileSource);
|
||||
if (reservedError) {
|
||||
m_infoLog = *reservedError;
|
||||
// P0b layer 2: another shader object in this context may already have run the
|
||||
// source-only half over byte-identical text.
|
||||
const ShaderPreprocessResult* cached =
|
||||
m_preprocessCache != nullptr ? m_preprocessCache->Find(m_stage, sourceHash, m_source) : nullptr;
|
||||
ShaderPreprocessResult fresh;
|
||||
if (cached == nullptr) fresh = RunSourceOnlyPipeline(m_stage, m_source);
|
||||
const ShaderPreprocessResult& shared = cached != nullptr ? *cached : fresh;
|
||||
const Bool shouldPopulateCache = cached == nullptr && m_preprocessCache != nullptr;
|
||||
|
||||
if (!shared.Preprocessed()) {
|
||||
// Rejected lexically, or a glslang failure this context has already seen for
|
||||
// this exact source (ParseFailed) - either way the parse can be skipped.
|
||||
m_infoLog = shared.infoLog;
|
||||
if (shouldPopulateCache) m_preprocessCache->Insert(m_stage, sourceHash, m_source, Move(fresh));
|
||||
RememberCompiledSource(sourceHash);
|
||||
return;
|
||||
}
|
||||
|
||||
// Single parse, in the link-compatible configuration (Vulkan-client env with
|
||||
// relaxed rules): the TShader stored here is what glLinkProgram links and what
|
||||
// the backends' SPIR-V is generated from - there is no second, GL-client parse
|
||||
// anymore. The GL frontend semantics the relaxed parse cannot provide are
|
||||
// restored on top: explicit default-block uniform locations through the lexical
|
||||
// side-channel below, dead-uniform/global-UBO filtering in
|
||||
// ProgramObject::DoReflection.
|
||||
m_explicitUniformLocations = ExtractExplicitUniformLocations(compileSource);
|
||||
m_explicitOpaqueBindings = ExtractExplicitOpaqueBindings(compileSource);
|
||||
ShaderAttrib attrib{.shaderType = MG_Util::ConvertShaderStageToGLEnum(m_stage),
|
||||
.sourceStr = compileSource,
|
||||
.sourceStr = shared.preprocessedSource,
|
||||
.flags = 0};
|
||||
|
||||
auto result = ShaderCompiler::CompileShader(attrib);
|
||||
if (result) {
|
||||
m_compileStatus = true;
|
||||
m_shader = result.value();
|
||||
m_preprocessedSource = Move(compileSource);
|
||||
// Copy, not move: `shared` may alias a cache entry that has to outlive us, and
|
||||
// `fresh` is about to be handed to the cache.
|
||||
m_preprocessedSource = shared.preprocessedSource;
|
||||
m_explicitUniformLocations = shared.explicitUniformLocations;
|
||||
m_explicitOpaqueBindings = shared.explicitOpaqueBindings;
|
||||
m_infoLog.clear();
|
||||
if (shouldPopulateCache) m_preprocessCache->Insert(m_stage, sourceHash, m_source, Move(fresh));
|
||||
} else {
|
||||
m_explicitUniformLocations.clear();
|
||||
m_explicitOpaqueBindings.clear();
|
||||
m_infoLog = result.error().log;
|
||||
MGLOG_D("ShaderObject::Compile: Shader %d compilation failed.\nSource:\n%s\nInfoLog:\n%s\nSetting "
|
||||
"m_compileStatus = false as a result.",
|
||||
m_externalIndex, compileSource.c_str(), m_infoLog.c_str());
|
||||
m_externalIndex, shared.preprocessedSource.c_str(), m_infoLog.c_str());
|
||||
if (shouldPopulateCache) {
|
||||
fresh.outcome = ShaderPreprocessOutcome::ParseFailed;
|
||||
fresh.infoLog = m_infoLog;
|
||||
fresh.explicitUniformLocations.clear();
|
||||
fresh.explicitOpaqueBindings.clear();
|
||||
m_preprocessCache->Insert(m_stage, sourceHash, m_source, Move(fresh));
|
||||
}
|
||||
}
|
||||
RememberCompiledSource(sourceHash);
|
||||
}
|
||||
|
||||
SharedPtr<glslang::TShader> ShaderObject::TakeShaderForLink(String& outReparseLog) {
|
||||
|
||||
@@ -22,10 +22,18 @@ namespace MobileGL {
|
||||
};
|
||||
|
||||
namespace MG_State::GLState {
|
||||
// P0b layer 2. Declared, not included: the cache keys on ShaderStage, so including
|
||||
// its header here would be circular.
|
||||
class ShaderPreprocessCache;
|
||||
|
||||
class ShaderObject {
|
||||
public:
|
||||
ShaderObject(const ShaderStage stage, Uint externalIndex)
|
||||
: m_stage(stage), m_externalIndex(externalIndex) {}
|
||||
// `preprocessCache` is the owning context's cross-object memo (P0b layer 2);
|
||||
// null is fully supported and simply means "no sharing" - that is what the
|
||||
// context-less internal shader objects (the default FS, the blit pipeline) use.
|
||||
ShaderObject(const ShaderStage stage, Uint externalIndex,
|
||||
ShaderPreprocessCache* preprocessCache = nullptr)
|
||||
: m_stage(stage), m_externalIndex(externalIndex), m_preprocessCache(preprocessCache) {}
|
||||
void SetShaderSource(const String& source);
|
||||
void SetShaderSource(String&& source);
|
||||
void Compile();
|
||||
@@ -59,8 +67,22 @@ namespace MobileGL {
|
||||
Bool GetCompileStatus() const { return m_compileStatus; }
|
||||
Bool GetDeleteStatus() const { return m_deleteStatus; }
|
||||
|
||||
// True while this object holds the outcome (success OR failure) of a previous
|
||||
// Compile() of exactly the source it currently holds - i.e. while the P0b
|
||||
// layer-1 memo is armed and a glCompileShader would be a no-op. Diagnostics
|
||||
// and tests only; nothing in the GL frontend branches on it.
|
||||
Bool HasMemoizedCompile() const { return m_hasCompiledState; }
|
||||
|
||||
private:
|
||||
void InvalidateCompiledState();
|
||||
// ---- P0b layer 1: per-object no-op recompile ----
|
||||
// True iff `candidate` is byte-identical to the source that produced the
|
||||
// compiled state this object is currently holding. The stored hash and length
|
||||
// are only a fast reject; the answer is always confirmed against the full
|
||||
// stored text, so no behaviour rides on a 64-bit hash.
|
||||
Bool SourceMatchesCompiledState(const String& candidate) const;
|
||||
// Arms the layer-1 memo for the source that Compile() just processed.
|
||||
void RememberCompiledSource(Uint64 sourceHash);
|
||||
|
||||
const Uint m_externalIndex = 0;
|
||||
const ShaderStage m_stage;
|
||||
@@ -75,6 +97,15 @@ namespace MobileGL {
|
||||
UnorderedMap<String, Uint> m_explicitOpaqueBindings;
|
||||
Bool m_shaderConsumedByLink = false;
|
||||
|
||||
// P0b layer 2: the owning context's cross-object memo, or null. Not owned.
|
||||
ShaderPreprocessCache* const m_preprocessCache = nullptr;
|
||||
// P0b layer 1. m_hasCompiledState is the invariant "m_source is byte-identical
|
||||
// to the source that produced m_compileStatus/m_infoLog/m_shader"; it is armed
|
||||
// at the end of every Compile() and disarmed by InvalidateCompiledState().
|
||||
Bool m_hasCompiledState = false;
|
||||
Uint64 m_compiledSourceHash = 0;
|
||||
SizeT m_compiledSourceLength = 0;
|
||||
|
||||
String m_infoLog;
|
||||
Bool m_deleteStatus = false;
|
||||
Bool m_compileStatus = false;
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
// MobileGL - MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.cpp
|
||||
// Copyright (c) 2025-2026 MobileGL-Dev
|
||||
// Licensed under the GNU Lesser General Public License v3.0:
|
||||
// https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
// https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
// SPDX-License-Identifier: LGPL-3.0-only
|
||||
// End of Source File Header
|
||||
|
||||
#include "ShaderPreprocessCache.h"
|
||||
|
||||
namespace MobileGL::MG_State::GLState {
|
||||
const ShaderPreprocessResult* ShaderPreprocessCache::Find(const ShaderStage stage, const Uint64 sourceHash,
|
||||
const String& source) const {
|
||||
const Key key{.stage = stage, .sourceHash = sourceHash, .sourceLength = source.length()};
|
||||
const auto it = m_index.find(key);
|
||||
if (it == m_index.end()) return nullptr;
|
||||
|
||||
// Never let correctness ride on a 64-bit hash: confirm the hit byte for byte.
|
||||
// Lengths already matched (they are part of the key), so this is a plain memcmp.
|
||||
const Entry& entry = *it->second;
|
||||
if (entry.originalSource != source) return nullptr;
|
||||
|
||||
return &entry.result;
|
||||
}
|
||||
|
||||
void ShaderPreprocessCache::Insert(const ShaderStage stage, const Uint64 sourceHash, const String& source,
|
||||
ShaderPreprocessResult result) {
|
||||
const SizeT entryBytes = EntryBytes(source, result);
|
||||
// A single source bigger than the whole budget would evict every other entry and
|
||||
// then itself; refuse it instead of thrashing the cache empty.
|
||||
if (entryBytes > kMaxStoredSourceBytes) return;
|
||||
|
||||
const Key key{.stage = stage, .sourceHash = sourceHash, .sourceLength = source.length()};
|
||||
if (const auto existing = m_index.find(key); existing != m_index.end()) {
|
||||
// Either a re-insert of the same source (harmless) or a genuine hash collision
|
||||
// with a different source. Both are resolved by letting the newcomer win: one
|
||||
// entry per key keeps the index a plain map, and a collision is astronomically
|
||||
// rare enough that the loser simply misses.
|
||||
EraseEntry(existing->second);
|
||||
}
|
||||
|
||||
m_entries.push_back(Entry{.key = key, .originalSource = source, .result = Move(result)});
|
||||
m_index[key] = std::prev(m_entries.end());
|
||||
m_storedSourceBytes += entryBytes;
|
||||
|
||||
EvictUntilWithinBudget();
|
||||
}
|
||||
|
||||
void ShaderPreprocessCache::Clear() {
|
||||
m_entries.clear();
|
||||
m_index.clear();
|
||||
m_storedSourceBytes = 0;
|
||||
}
|
||||
|
||||
void ShaderPreprocessCache::EraseEntry(const EntryList::iterator it) {
|
||||
const SizeT bytes = EntryBytes(it->originalSource, it->result);
|
||||
m_storedSourceBytes = bytes > m_storedSourceBytes ? 0 : m_storedSourceBytes - bytes;
|
||||
m_index.erase(it->key);
|
||||
m_entries.erase(it);
|
||||
}
|
||||
|
||||
void ShaderPreprocessCache::EvictUntilWithinBudget() {
|
||||
// FIFO: the oldest insertion goes first. Insert() already refuses entries larger
|
||||
// than the byte budget, so this loop always terminates with at least the entry
|
||||
// that was just added still resident.
|
||||
while (!m_entries.empty() &&
|
||||
(m_entries.size() > kMaxEntries || m_storedSourceBytes > kMaxStoredSourceBytes)) {
|
||||
EraseEntry(m_entries.begin());
|
||||
}
|
||||
}
|
||||
} // namespace MobileGL::MG_State::GLState
|
||||
@@ -0,0 +1,140 @@
|
||||
// MobileGL - MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.h
|
||||
// Copyright (c) 2025-2026 MobileGL-Dev
|
||||
// Licensed under the GNU Lesser General Public License v3.0:
|
||||
// https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
// https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
// SPDX-License-Identifier: LGPL-3.0-only
|
||||
// End of Source File Header
|
||||
|
||||
#pragma once
|
||||
#include <Includes.h>
|
||||
#include <list>
|
||||
#include <MG_State/GLState/ProgramState/ShaderObject.h>
|
||||
|
||||
namespace MobileGL::MG_State::GLState {
|
||||
// Where the shared, source-only half of ShaderObject::Compile() stopped. The two
|
||||
// rejection verdicts are kept apart (rather than collapsed into "failed") so a hit
|
||||
// reproduces the original diagnosis, not just the original info log.
|
||||
enum class ShaderPreprocessOutcome : Uint8 {
|
||||
// The source-only half ran clean; preprocessedSource and both maps are valid.
|
||||
Preprocessed,
|
||||
// ValidateComputeLocalSizeLimits rejected it (compute only).
|
||||
ComputeLocalSizeRejected,
|
||||
// FindReservedIdentifierViolation rejected it.
|
||||
ReservedIdentifierRejected,
|
||||
// The source-only half was clean but glslang rejected the preprocessed source.
|
||||
// Memoizing this saves the parse itself on every later object with that source.
|
||||
ParseFailed,
|
||||
};
|
||||
|
||||
// Everything ShaderObject::Compile() derives from the source text alone, i.e.
|
||||
// everything that is identical for two shader objects holding byte-identical source.
|
||||
struct ShaderPreprocessResult {
|
||||
ShaderPreprocessOutcome outcome = ShaderPreprocessOutcome::Preprocessed;
|
||||
// Valid unless the preprocessor itself never ran; kept even for the rejection
|
||||
// outcomes because that is the text the diagnostics refer to.
|
||||
String preprocessedSource;
|
||||
UnorderedMap<String, Int> explicitUniformLocations;
|
||||
UnorderedMap<String, Uint> explicitOpaqueBindings;
|
||||
// The compile info log to publish; empty when outcome == Preprocessed.
|
||||
String infoLog;
|
||||
|
||||
Bool Preprocessed() const { return outcome == ShaderPreprocessOutcome::Preprocessed; }
|
||||
};
|
||||
|
||||
// P0b layer 2: a per-context, bounded memo of the source-only half of shader
|
||||
// compilation, keyed by (stage, xxhash64(source), source length).
|
||||
//
|
||||
// Motivation: in the Iris shader-pack corpus ~21% of every glCompileShader in a trace
|
||||
// is a *different* shader object holding byte-identical source (packs glue the same
|
||||
// common/composite GLSL into many program stages), so the preprocess + reserved-
|
||||
// identifier scan + explicit-location/binding extraction runs over the same megabytes
|
||||
// again and again. Layer 1 (in ShaderObject) covers the same object recompiled with
|
||||
// unchanged source; this covers the cross-object case.
|
||||
//
|
||||
// What is NOT cached: the glslang parse. glslang's TShader is consume-once (mapIO
|
||||
// mutates the aliased intermediate at link), so every shader object still needs its
|
||||
// own parse; only the text-processing half is shared.
|
||||
//
|
||||
// Correctness: the 64-bit hash is a lookup accelerator only. Every hit re-compares the
|
||||
// full stored original source with memcmp before it is honored, so a hash collision
|
||||
// degrades to a miss, never to a wrong answer. That is why the full original text is
|
||||
// stored rather than a prefix/suffix digest - the cache is bounded, so the cost is.
|
||||
//
|
||||
// Eviction: FIFO (insertion order), bounded by BOTH an entry count and a stored-source
|
||||
// byte budget, whichever binds first. FIFO rather than LRU because shader-pack loading
|
||||
// is a burst of mostly-distinct sources whose reuse clusters around insertion time;
|
||||
// LRU's extra list splice on every hit buys nothing measurable here, and FIFO keeps
|
||||
// Find() a genuinely const, read-only operation.
|
||||
class ShaderPreprocessCache {
|
||||
public:
|
||||
static constexpr SizeT kMaxEntries = 128;
|
||||
static constexpr SizeT kMaxStoredSourceBytes = 8u * 1024u * 1024u;
|
||||
|
||||
// Returns the memoized result for this exact source, or null on a miss. The
|
||||
// returned pointer stays valid until the next Insert()/Clear() on this cache.
|
||||
const ShaderPreprocessResult* Find(ShaderStage stage, Uint64 sourceHash, const String& source) const;
|
||||
|
||||
// Memoizes `result` for this source. A source whose own storage cost already
|
||||
// exceeds the byte budget is simply not cached (caching it would evict everything
|
||||
// else and then itself).
|
||||
void Insert(ShaderStage stage, Uint64 sourceHash, const String& source, ShaderPreprocessResult result);
|
||||
|
||||
void Clear();
|
||||
|
||||
static Uint64 HashSource(const String& source) {
|
||||
return static_cast<Uint64>(XXH64(source.data(), source.length(), 0));
|
||||
}
|
||||
|
||||
SizeT GetEntryCount() const { return m_entries.size(); }
|
||||
SizeT GetStoredSourceBytes() const { return m_storedSourceBytes; }
|
||||
|
||||
private:
|
||||
struct Key {
|
||||
ShaderStage stage = ShaderStage::Unknown;
|
||||
Uint64 sourceHash = 0;
|
||||
SizeT sourceLength = 0;
|
||||
|
||||
Bool operator==(const Key& other) const {
|
||||
return stage == other.stage && sourceHash == other.sourceHash && sourceLength == other.sourceLength;
|
||||
}
|
||||
};
|
||||
|
||||
struct KeyHasher {
|
||||
SizeT operator()(const Key& key) const {
|
||||
// The source hash already spreads well; fold the two discriminators in so
|
||||
// that same-hash-different-stage/length keys land in different buckets.
|
||||
Uint64 mixed = key.sourceHash;
|
||||
mixed ^= static_cast<Uint64>(key.sourceLength) + 0x9e3779b97f4a7c15ull + (mixed << 6) + (mixed >> 2);
|
||||
mixed ^= static_cast<Uint64>(static_cast<Int>(key.stage)) * 0xff51afd7ed558ccdull;
|
||||
return static_cast<SizeT>(mixed);
|
||||
}
|
||||
};
|
||||
|
||||
struct Entry {
|
||||
Key key;
|
||||
// The full original (pre-preprocess) source, kept so a hit can be confirmed by
|
||||
// comparison instead of trusting the hash.
|
||||
String originalSource;
|
||||
ShaderPreprocessResult result;
|
||||
};
|
||||
|
||||
using EntryList = std::list<Entry>;
|
||||
|
||||
static SizeT EntryBytes(const String& source, const ShaderPreprocessResult& result) {
|
||||
return source.length() + result.preprocessedSource.length();
|
||||
}
|
||||
|
||||
void EraseEntry(EntryList::iterator it);
|
||||
void EvictUntilWithinBudget();
|
||||
|
||||
// P1: needs a mutex when compiles go async. Everything here is reached from
|
||||
// glCompileShader on the single GL thread that owns the context, so today the
|
||||
// cache is deliberately lock-free; the moment shader compilation moves onto a
|
||||
// worker pool, Find/Insert/Clear all become critical sections (and Find's returned
|
||||
// pointer stops being safe to hold across an Insert).
|
||||
EntryList m_entries; // front = oldest (FIFO victim)
|
||||
UnorderedMap<Key, EntryList::iterator, KeyHasher> m_index;
|
||||
SizeT m_storedSourceBytes = 0;
|
||||
};
|
||||
} // namespace MobileGL::MG_State::GLState
|
||||
@@ -9,6 +9,7 @@ add_executable(
|
||||
${MGL_ROOT}/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp
|
||||
${MGL_ROOT}/MobileGL/MG_Util/ShaderTranspiler/SpvcSession.cpp
|
||||
${MGL_ROOT}/MobileGL/MG_Util/ShaderTranspiler/glslang/UniformTraverser.cpp
|
||||
${MGL_ROOT}/MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.cpp
|
||||
)
|
||||
|
||||
target_include_directories(ProgramUtilTest PRIVATE
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
#include "MG_Impl/GLImpl/Getter/GL_Getter.h"
|
||||
#include "MG_Impl/GLImpl/Program/GL_Program.h"
|
||||
#include "MG_State/GLState/Core.h"
|
||||
#include "MG_State/GLState/ProgramState/ShaderPreprocessCache.h"
|
||||
#include "MG_Util/ShaderTranspiler/ShaderCompiler.h"
|
||||
|
||||
using namespace MobileGL;
|
||||
@@ -2834,3 +2835,224 @@ void main() { fragColor = vec4(pow(uBase, 2.2), 1.0); }
|
||||
EXPECT_EQ(GetError(), GL_NO_ERROR);
|
||||
}
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// P0b: source-hash dedupe for shader recompiles.
|
||||
// Layer 1 - the same shader object re-sourced with byte-identical text keeps its
|
||||
// compiled state, and glCompileShader on it is a no-op.
|
||||
// Layer 2 - two DIFFERENT shader objects holding byte-identical text share the
|
||||
// source-only half of the pipeline (preprocess + lexical checks +
|
||||
// side-channel extraction) through the context's ShaderPreprocessCache,
|
||||
// while each still gets its own glslang parse.
|
||||
// ---------------------------------------------------------------------------
|
||||
namespace {
|
||||
const char* kP0bVs = R"(#version 330 core
|
||||
uniform mat4 uModel;
|
||||
uniform vec4 uTint;
|
||||
void main() { gl_Position = uModel * uTint; }
|
||||
)";
|
||||
const char* kP0bFs = R"(#version 330 core
|
||||
uniform vec4 uColor;
|
||||
out vec4 fragColor;
|
||||
void main() { fragColor = uColor; }
|
||||
)";
|
||||
// Same stage, different declared uniform: makes "did it actually recompile?"
|
||||
// observable through reflection rather than through internal state.
|
||||
const char* kP0bAltFs = R"(#version 330 core
|
||||
uniform vec4 uOtherColor;
|
||||
out vec4 fragColor;
|
||||
void main() { fragColor = uOtherColor; }
|
||||
)";
|
||||
const char* kP0bBrokenFs = R"(#version 330 core
|
||||
out vec4 fragColor;
|
||||
void main() { fragColor = notADeclaredThing; }
|
||||
)";
|
||||
|
||||
GLuint MakeShaderWithSource(GLenum type, const char* source) {
|
||||
GLuint shader = CreateShader(type);
|
||||
ShaderSource(shader, 1, &source, nullptr);
|
||||
return shader;
|
||||
}
|
||||
|
||||
GLint QueryCompileStatus(GLuint shader) {
|
||||
GLint status = GL_FALSE;
|
||||
GetShaderiv(shader, GL_COMPILE_STATUS, &status);
|
||||
return status;
|
||||
}
|
||||
|
||||
String QueryShaderInfoLog(GLuint shader) {
|
||||
GLint length = 0;
|
||||
GetShaderiv(shader, GL_INFO_LOG_LENGTH, &length);
|
||||
if (length <= 0) return String();
|
||||
std::vector<GLchar> buffer(static_cast<size_t>(length));
|
||||
GLsizei written = 0;
|
||||
GetShaderInfoLog(shader, length, &written, buffer.data());
|
||||
return String(buffer.data(), static_cast<size_t>(written));
|
||||
}
|
||||
|
||||
Bool ShaderHasMemoizedCompile(GLuint shader) {
|
||||
const auto& shaderObject = MG_State::pGLContext->GetShaderObject(shader);
|
||||
EXPECT_NE(shaderObject, nullptr);
|
||||
return shaderObject != nullptr && shaderObject->HasMemoizedCompile();
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// Layer 1, success path: re-sourcing with identical text and recompiling must leave
|
||||
// COMPILE_STATUS, the info log and every downstream consumer exactly as they were -
|
||||
// including a program that links the shader AFTER the redundant recompile.
|
||||
TEST_F(ProgramTest, RecompileWithIdenticalSourceKeepsCompiledStateAndStillLinks) {
|
||||
GLuint vs = MakeShaderWithSource(GL_VERTEX_SHADER, kP0bVs);
|
||||
GLuint fs = MakeShaderWithSource(GL_FRAGMENT_SHADER, kP0bFs);
|
||||
CompileShader(vs);
|
||||
CompileShader(fs);
|
||||
ASSERT_EQ(QueryCompileStatus(vs), GL_TRUE) << QueryShaderInfoLog(vs);
|
||||
ASSERT_EQ(QueryCompileStatus(fs), GL_TRUE) << QueryShaderInfoLog(fs);
|
||||
const String vsLogBefore = QueryShaderInfoLog(vs);
|
||||
EXPECT_TRUE(ShaderHasMemoizedCompile(vs));
|
||||
|
||||
// A first link consumes the stored TShader; the redundant recompile below must not
|
||||
// disturb the preprocessed source that TakeShaderForLink re-parses from.
|
||||
GLuint firstProgram = LinkVsFs(vs, fs, GL_TRUE);
|
||||
EXPECT_GE(GetUniformLocation(firstProgram, "uColor"), 0);
|
||||
|
||||
// glShaderSource with byte-identical text, then glCompileShader: both no-ops.
|
||||
ShaderSource(vs, 1, &kP0bVs, nullptr);
|
||||
EXPECT_TRUE(ShaderHasMemoizedCompile(vs)) << "identical re-source must not invalidate the compiled state";
|
||||
CompileShader(vs);
|
||||
ShaderSource(fs, 1, &kP0bFs, nullptr);
|
||||
CompileShader(fs);
|
||||
|
||||
EXPECT_EQ(QueryCompileStatus(vs), GL_TRUE);
|
||||
EXPECT_EQ(QueryCompileStatus(fs), GL_TRUE);
|
||||
EXPECT_EQ(QueryShaderInfoLog(vs), vsLogBefore);
|
||||
|
||||
// The original source text is still what glGetShaderSource reports.
|
||||
GLint sourceLength = 0;
|
||||
GetShaderiv(vs, GL_SHADER_SOURCE_LENGTH, &sourceLength);
|
||||
ASSERT_GT(sourceLength, 1);
|
||||
std::vector<GLchar> sourceBuffer(static_cast<size_t>(sourceLength));
|
||||
GLsizei written = 0;
|
||||
GetShaderSource(vs, sourceLength, &written, sourceBuffer.data());
|
||||
EXPECT_EQ(String(sourceBuffer.data(), static_cast<size_t>(written)), String(kP0bVs));
|
||||
|
||||
// A second program built from the same, redundantly recompiled shaders links and
|
||||
// reflects - i.e. TakeShaderForLink's re-parse path survived the no-op.
|
||||
GLuint secondProgram = LinkVsFs(vs, fs, GL_TRUE);
|
||||
EXPECT_GE(GetUniformLocation(secondProgram, "uColor"), 0);
|
||||
EXPECT_GE(GetUniformLocation(secondProgram, "uModel"), 0);
|
||||
EXPECT_EQ(GetError(), GL_NO_ERROR);
|
||||
}
|
||||
|
||||
// Layer 1 must not swallow a REAL source change: different text invalidates, and the
|
||||
// change is visible in what the next link reflects.
|
||||
TEST_F(ProgramTest, DifferentSourceAfterCompileInvalidatesCompiledState) {
|
||||
GLuint vs = CompileShaderChecked(GL_VERTEX_SHADER, kP0bVs);
|
||||
GLuint fs = MakeShaderWithSource(GL_FRAGMENT_SHADER, kP0bFs);
|
||||
CompileShader(fs);
|
||||
ASSERT_EQ(QueryCompileStatus(fs), GL_TRUE) << QueryShaderInfoLog(fs);
|
||||
GLuint firstProgram = LinkVsFs(vs, fs, GL_TRUE);
|
||||
EXPECT_GE(GetUniformLocation(firstProgram, "uColor"), 0);
|
||||
EXPECT_EQ(GetUniformLocation(firstProgram, "uOtherColor"), -1);
|
||||
|
||||
// New text -> compiled state gone, and glCompileShader is mandatory again.
|
||||
ShaderSource(fs, 1, &kP0bAltFs, nullptr);
|
||||
EXPECT_FALSE(ShaderHasMemoizedCompile(fs));
|
||||
EXPECT_EQ(QueryCompileStatus(fs), GL_FALSE);
|
||||
|
||||
CompileShader(fs);
|
||||
ASSERT_EQ(QueryCompileStatus(fs), GL_TRUE) << QueryShaderInfoLog(fs);
|
||||
GLuint secondProgram = LinkVsFs(vs, fs, GL_TRUE);
|
||||
EXPECT_GE(GetUniformLocation(secondProgram, "uOtherColor"), 0);
|
||||
EXPECT_EQ(GetUniformLocation(secondProgram, "uColor"), -1);
|
||||
EXPECT_EQ(GetError(), GL_NO_ERROR);
|
||||
}
|
||||
|
||||
// Layer 2: byte-identical source in two distinct shader objects. Both must compile,
|
||||
// and each must own an independent TShader - if the parse were shared, the second
|
||||
// link would be handed an intermediate that the first link's mapIO already mutated.
|
||||
TEST_F(ProgramTest, TwoShaderObjectsWithIdenticalSourceLinkIndependently) {
|
||||
GLuint vsA = CompileShaderChecked(GL_VERTEX_SHADER, kP0bVs);
|
||||
GLuint fsA = CompileShaderChecked(GL_FRAGMENT_SHADER, kP0bFs);
|
||||
GLuint vsB = CompileShaderChecked(GL_VERTEX_SHADER, kP0bVs);
|
||||
GLuint fsB = CompileShaderChecked(GL_FRAGMENT_SHADER, kP0bFs);
|
||||
ASSERT_NE(vsA, vsB);
|
||||
ASSERT_NE(fsA, fsB);
|
||||
|
||||
const auto& objectA = MG_State::pGLContext->GetShaderObject(vsA);
|
||||
const auto& objectB = MG_State::pGLContext->GetShaderObject(vsB);
|
||||
ASSERT_NE(objectA, nullptr);
|
||||
ASSERT_NE(objectB, nullptr);
|
||||
EXPECT_EQ(objectA->GetShaderSource(), objectB->GetShaderSource());
|
||||
// Independent parses despite the shared preprocess.
|
||||
EXPECT_NE(objectA->GetCompiledShader(), objectB->GetCompiledShader());
|
||||
EXPECT_NE(objectA->GetCompiledShader(), nullptr);
|
||||
EXPECT_NE(objectB->GetCompiledShader(), nullptr);
|
||||
|
||||
GLuint programA = LinkVsFs(vsA, fsA, GL_TRUE);
|
||||
GLuint programB = LinkVsFs(vsB, fsB, GL_TRUE);
|
||||
for (GLuint program : {programA, programB}) {
|
||||
EXPECT_GE(GetUniformLocation(program, "uColor"), 0);
|
||||
EXPECT_GE(GetUniformLocation(program, "uModel"), 0);
|
||||
}
|
||||
EXPECT_EQ(GetError(), GL_NO_ERROR);
|
||||
}
|
||||
|
||||
// Failure memoization: a compile that failed stays failed, with the SAME log, when
|
||||
// recompiled against the same source; a real fix to the source still takes effect.
|
||||
// The second object pins the cached-ParseFailed path (layer 2), which skips the parse
|
||||
// entirely and must reproduce the identical verdict.
|
||||
TEST_F(ProgramTest, FailedCompileIsMemoizedAndStillRecoversOnGoodSource) {
|
||||
GLuint fs = MakeShaderWithSource(GL_FRAGMENT_SHADER, kP0bBrokenFs);
|
||||
CompileShader(fs);
|
||||
ASSERT_EQ(QueryCompileStatus(fs), GL_FALSE);
|
||||
const String failureLog = QueryShaderInfoLog(fs);
|
||||
EXPECT_FALSE(failureLog.empty());
|
||||
|
||||
// Layer 1: identical re-source + recompile keeps the failure AND the log queryable.
|
||||
ShaderSource(fs, 1, &kP0bBrokenFs, nullptr);
|
||||
CompileShader(fs);
|
||||
EXPECT_EQ(QueryCompileStatus(fs), GL_FALSE);
|
||||
EXPECT_EQ(QueryShaderInfoLog(fs), failureLog);
|
||||
|
||||
// Layer 2: a second object with the same broken source reports the same failure.
|
||||
GLuint otherFs = MakeShaderWithSource(GL_FRAGMENT_SHADER, kP0bBrokenFs);
|
||||
CompileShader(otherFs);
|
||||
EXPECT_EQ(QueryCompileStatus(otherFs), GL_FALSE);
|
||||
EXPECT_EQ(QueryShaderInfoLog(otherFs), failureLog);
|
||||
|
||||
// A genuine fix still compiles and links.
|
||||
ShaderSource(fs, 1, &kP0bFs, nullptr);
|
||||
CompileShader(fs);
|
||||
ASSERT_EQ(QueryCompileStatus(fs), GL_TRUE) << QueryShaderInfoLog(fs);
|
||||
EXPECT_TRUE(QueryShaderInfoLog(fs).empty());
|
||||
GLuint vs = CompileShaderChecked(GL_VERTEX_SHADER, kP0bVs);
|
||||
GLuint program = LinkVsFs(vs, fs, GL_TRUE);
|
||||
EXPECT_GE(GetUniformLocation(program, "uColor"), 0);
|
||||
}
|
||||
|
||||
// Layer 2 under eviction: push more distinct sources through the context than the
|
||||
// cache can hold, then confirm nothing broke and a fresh duplicate pair still works.
|
||||
TEST_F(ProgramTest, PreprocessCacheOverflowKeepsCompilingCorrectly) {
|
||||
const SizeT overflow = MG_State::GLState::ShaderPreprocessCache::kMaxEntries + 8;
|
||||
for (SizeT i = 0; i < overflow; ++i) {
|
||||
const String source = "#version 330 core\nuniform vec4 uColor" + ToString(i) +
|
||||
";\nout vec4 fragColor;\nvoid main() { fragColor = uColor" + ToString(i) + "; }\n";
|
||||
const char* sourcePtr = source.c_str();
|
||||
GLuint shader = MakeShaderWithSource(GL_FRAGMENT_SHADER, sourcePtr);
|
||||
CompileShader(shader);
|
||||
ASSERT_EQ(QueryCompileStatus(shader), GL_TRUE) << QueryShaderInfoLog(shader) << "\n" << source;
|
||||
DeleteShader(shader);
|
||||
}
|
||||
|
||||
// Everything inserted above has long since been evicted; a brand-new duplicate
|
||||
// pair must still take the layer-2 path and produce two working programs.
|
||||
GLuint vsA = CompileShaderChecked(GL_VERTEX_SHADER, kP0bVs);
|
||||
GLuint fsA = CompileShaderChecked(GL_FRAGMENT_SHADER, kP0bFs);
|
||||
GLuint vsB = CompileShaderChecked(GL_VERTEX_SHADER, kP0bVs);
|
||||
GLuint fsB = CompileShaderChecked(GL_FRAGMENT_SHADER, kP0bFs);
|
||||
GLuint programA = LinkVsFs(vsA, fsA, GL_TRUE);
|
||||
GLuint programB = LinkVsFs(vsB, fsB, GL_TRUE);
|
||||
EXPECT_GE(GetUniformLocation(programA, "uColor"), 0);
|
||||
EXPECT_GE(GetUniformLocation(programB, "uColor"), 0);
|
||||
EXPECT_EQ(GetError(), GL_NO_ERROR);
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
#include <MG_Util/ShaderTranspiler/SpirvPasses/RenameSamplerFunctionParameterPass.h>
|
||||
#include <MG_Util/ShaderTranspiler/Types.h>
|
||||
#include <MG_Util/ShaderTranspiler/glslang/UniformTraverser.h>
|
||||
#include <MG_State/GLState/ProgramState/ShaderPreprocessCache.h>
|
||||
#include <spirv-tools/libspirv.hpp>
|
||||
#include <spirv-tools/optimizer.hpp>
|
||||
|
||||
@@ -2375,3 +2376,161 @@ void main() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// P0b layer 2: ShaderPreprocessCache, tested directly. The GL-level behaviour it
|
||||
// enables is covered end to end in ProgramTest; these pin the container itself,
|
||||
// where the interesting cases (hash collisions, both eviction budgets) are hard
|
||||
// to provoke through glCompileShader.
|
||||
// ---------------------------------------------------------------------------
|
||||
namespace {
|
||||
using MobileGL::MG_State::GLState::ShaderPreprocessCache;
|
||||
using MobileGL::MG_State::GLState::ShaderPreprocessOutcome;
|
||||
using MobileGL::MG_State::GLState::ShaderPreprocessResult;
|
||||
|
||||
ShaderPreprocessResult MakeResult(const String& preprocessed) {
|
||||
ShaderPreprocessResult result;
|
||||
result.outcome = ShaderPreprocessOutcome::Preprocessed;
|
||||
result.preprocessedSource = preprocessed;
|
||||
result.explicitUniformLocations["uMarker"] = 7;
|
||||
result.explicitOpaqueBindings["sMarker"] = 3;
|
||||
return result;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST_F(ProgramUtilTest, ShaderPreprocessCacheRoundTripsAndSeparatesStages) {
|
||||
ShaderPreprocessCache cache;
|
||||
const String source = "// a shader\nvoid main() {}\n";
|
||||
const Uint64 hash = ShaderPreprocessCache::HashSource(source);
|
||||
|
||||
EXPECT_EQ(cache.Find(ShaderStage::Vertex, hash, source), nullptr);
|
||||
|
||||
cache.Insert(ShaderStage::Vertex, hash, source, MakeResult("vertex-preprocessed"));
|
||||
const ShaderPreprocessResult* hit = cache.Find(ShaderStage::Vertex, hash, source);
|
||||
ASSERT_NE(hit, nullptr);
|
||||
EXPECT_TRUE(hit->Preprocessed());
|
||||
EXPECT_EQ(hit->preprocessedSource, "vertex-preprocessed");
|
||||
const auto uniformIt = hit->explicitUniformLocations.find("uMarker");
|
||||
ASSERT_NE(uniformIt, hit->explicitUniformLocations.end());
|
||||
EXPECT_EQ(uniformIt->second, 7);
|
||||
const auto bindingIt = hit->explicitOpaqueBindings.find("sMarker");
|
||||
ASSERT_NE(bindingIt, hit->explicitOpaqueBindings.end());
|
||||
EXPECT_EQ(bindingIt->second, 3u);
|
||||
|
||||
// Byte-identical source, different stage: a different key, so still a miss. Two
|
||||
// stages sharing one entry would hand a fragment shader a vertex preprocess.
|
||||
EXPECT_EQ(cache.Find(ShaderStage::Fragment, hash, source), nullptr);
|
||||
cache.Insert(ShaderStage::Fragment, hash, source, MakeResult("fragment-preprocessed"));
|
||||
const ShaderPreprocessResult* fragmentHit = cache.Find(ShaderStage::Fragment, hash, source);
|
||||
ASSERT_NE(fragmentHit, nullptr);
|
||||
EXPECT_EQ(fragmentHit->preprocessedSource, "fragment-preprocessed");
|
||||
EXPECT_EQ(cache.Find(ShaderStage::Vertex, hash, source)->preprocessedSource, "vertex-preprocessed");
|
||||
EXPECT_EQ(cache.GetEntryCount(), 2u);
|
||||
}
|
||||
|
||||
TEST_F(ProgramUtilTest, ShaderPreprocessCacheMemoizesRejectionVerdictsDistinctly) {
|
||||
ShaderPreprocessCache cache;
|
||||
const String reservedSource = "int packed;\n";
|
||||
const String localSizeSource = "layout(local_size_x = 99999) in;\n";
|
||||
|
||||
ShaderPreprocessResult reserved;
|
||||
reserved.outcome = ShaderPreprocessOutcome::ReservedIdentifierRejected;
|
||||
reserved.infoLog = "reserved identifier";
|
||||
ShaderPreprocessResult localSize;
|
||||
localSize.outcome = ShaderPreprocessOutcome::ComputeLocalSizeRejected;
|
||||
localSize.infoLog = "local_size too big";
|
||||
|
||||
cache.Insert(ShaderStage::Vertex, ShaderPreprocessCache::HashSource(reservedSource), reservedSource,
|
||||
std::move(reserved));
|
||||
cache.Insert(ShaderStage::Compute, ShaderPreprocessCache::HashSource(localSizeSource), localSizeSource,
|
||||
std::move(localSize));
|
||||
|
||||
const ShaderPreprocessResult* reservedHit =
|
||||
cache.Find(ShaderStage::Vertex, ShaderPreprocessCache::HashSource(reservedSource), reservedSource);
|
||||
ASSERT_NE(reservedHit, nullptr);
|
||||
EXPECT_FALSE(reservedHit->Preprocessed());
|
||||
EXPECT_EQ(reservedHit->outcome, ShaderPreprocessOutcome::ReservedIdentifierRejected);
|
||||
EXPECT_EQ(reservedHit->infoLog, "reserved identifier");
|
||||
|
||||
const ShaderPreprocessResult* localSizeHit =
|
||||
cache.Find(ShaderStage::Compute, ShaderPreprocessCache::HashSource(localSizeSource), localSizeSource);
|
||||
ASSERT_NE(localSizeHit, nullptr);
|
||||
EXPECT_EQ(localSizeHit->outcome, ShaderPreprocessOutcome::ComputeLocalSizeRejected);
|
||||
EXPECT_EQ(localSizeHit->infoLog, "local_size too big");
|
||||
}
|
||||
|
||||
// Correctness must not ride on a 64-bit hash. Feed two different sources of the same
|
||||
// length under a forged, identical hash: the entry stores the full original text, so
|
||||
// the impostor lookup must miss instead of returning the wrong preprocess.
|
||||
TEST_F(ProgramUtilTest, ShaderPreprocessCacheRejectsForgedHashCollision) {
|
||||
ShaderPreprocessCache cache;
|
||||
const String real = "void main() { int a = 1; }\n";
|
||||
const String impostor = "void main() { int a = 2; }\n";
|
||||
ASSERT_EQ(real.length(), impostor.length());
|
||||
ASSERT_NE(real, impostor);
|
||||
const Uint64 forgedHash = 0xdeadbeefcafef00dull;
|
||||
|
||||
cache.Insert(ShaderStage::Vertex, forgedHash, real, MakeResult("real-preprocessed"));
|
||||
|
||||
ASSERT_NE(cache.Find(ShaderStage::Vertex, forgedHash, real), nullptr);
|
||||
EXPECT_EQ(cache.Find(ShaderStage::Vertex, forgedHash, impostor), nullptr);
|
||||
|
||||
// The colliding newcomer wins the slot rather than being silently dropped, so it
|
||||
// is the previous occupant that degrades to a miss - never a wrong hit.
|
||||
cache.Insert(ShaderStage::Vertex, forgedHash, impostor, MakeResult("impostor-preprocessed"));
|
||||
const ShaderPreprocessResult* impostorHit = cache.Find(ShaderStage::Vertex, forgedHash, impostor);
|
||||
ASSERT_NE(impostorHit, nullptr);
|
||||
EXPECT_EQ(impostorHit->preprocessedSource, "impostor-preprocessed");
|
||||
EXPECT_EQ(cache.Find(ShaderStage::Vertex, forgedHash, real), nullptr);
|
||||
EXPECT_EQ(cache.GetEntryCount(), 1u);
|
||||
}
|
||||
|
||||
TEST_F(ProgramUtilTest, ShaderPreprocessCacheEvictsFifoOnEntryCap) {
|
||||
ShaderPreprocessCache cache;
|
||||
Vector<String> sources;
|
||||
const SizeT overflow = ShaderPreprocessCache::kMaxEntries + 8;
|
||||
for (SizeT i = 0; i < overflow; ++i) {
|
||||
sources.push_back("void main() { int a = " + ToString(i) + "; }\n");
|
||||
cache.Insert(ShaderStage::Vertex, ShaderPreprocessCache::HashSource(sources.back()), sources.back(),
|
||||
MakeResult("pp" + ToString(i)));
|
||||
EXPECT_LE(cache.GetEntryCount(), ShaderPreprocessCache::kMaxEntries);
|
||||
}
|
||||
EXPECT_EQ(cache.GetEntryCount(), ShaderPreprocessCache::kMaxEntries);
|
||||
|
||||
// FIFO: the first `overflow - kMaxEntries` insertions are gone, the rest resident.
|
||||
for (SizeT i = 0; i < overflow; ++i) {
|
||||
const ShaderPreprocessResult* hit =
|
||||
cache.Find(ShaderStage::Vertex, ShaderPreprocessCache::HashSource(sources[i]), sources[i]);
|
||||
if (i < overflow - ShaderPreprocessCache::kMaxEntries) {
|
||||
EXPECT_EQ(hit, nullptr) << "entry " << i << " should have been evicted";
|
||||
} else {
|
||||
ASSERT_NE(hit, nullptr) << "entry " << i << " should still be resident";
|
||||
EXPECT_EQ(hit->preprocessedSource, "pp" + ToString(i));
|
||||
}
|
||||
}
|
||||
|
||||
cache.Clear();
|
||||
EXPECT_EQ(cache.GetEntryCount(), 0u);
|
||||
EXPECT_EQ(cache.GetStoredSourceBytes(), 0u);
|
||||
}
|
||||
|
||||
TEST_F(ProgramUtilTest, ShaderPreprocessCacheHonorsByteBudget) {
|
||||
ShaderPreprocessCache cache;
|
||||
// Well under the entry cap, well over the byte budget: the byte budget must be the
|
||||
// one that binds, and the accounting must come back down as entries are evicted.
|
||||
const SizeT chunk = ShaderPreprocessCache::kMaxStoredSourceBytes / 8;
|
||||
for (SizeT i = 0; i < 24; ++i) {
|
||||
String source(chunk, static_cast<char>('a' + (i % 26)));
|
||||
cache.Insert(ShaderStage::Vertex, ShaderPreprocessCache::HashSource(source), source, MakeResult(""));
|
||||
EXPECT_LE(cache.GetStoredSourceBytes(), ShaderPreprocessCache::kMaxStoredSourceBytes);
|
||||
EXPECT_LT(cache.GetEntryCount(), ShaderPreprocessCache::kMaxEntries);
|
||||
}
|
||||
|
||||
// A single source larger than the whole budget is refused outright: caching it
|
||||
// would evict every other entry and then immediately itself.
|
||||
const SizeT before = cache.GetEntryCount();
|
||||
const String oversized(ShaderPreprocessCache::kMaxStoredSourceBytes + 1, 'z');
|
||||
cache.Insert(ShaderStage::Vertex, ShaderPreprocessCache::HashSource(oversized), oversized, MakeResult(""));
|
||||
EXPECT_EQ(cache.GetEntryCount(), before);
|
||||
EXPECT_EQ(cache.Find(ShaderStage::Vertex, ShaderPreprocessCache::HashSource(oversized), oversized), nullptr);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user