mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-09-11 21:58:31 +09:00
[Perf] (MG_State): dedupe shader compiles by source hash
Iris-style packs hand MobileGL the same source text repeatedly: probed across three shaderpack traces, 28-32% of all glCompileShader work was redundant - ~9% same-object recompiles with byte-identical source, ~21% distinct shader objects sharing identical source (the same common GLSL chunk glued into many program stages). Two layers, both keyed by XXH64 + length with a full byte compare on every hit (correctness never rides on the hash): - Per-object: a successful (or failed) compile remembers its source hash; glShaderSource with byte-identical text keeps the compiled state and glCompileShader on unchanged source returns immediately. Deterministic (stage, source) pipeline makes the memo observationally identical to recompiling; the consume-once TakeShaderForLink re-parse path is untouched. - Cross-object: a per-context bounded cache (ProgramState-owned, declared to outlive every shader object) shares the preprocessed source, both explicit side-channel maps, and the validation verdicts between objects with equal source; only the glslang parse stays per-object. Single-GL-thread today; flagged for a mutex when compiles go async (P1). Interleaved A/B on the iterationrp trace (the recompile-heavy pack): 5.65s -> 5.46s median total replay, every round faster; BSL/complementary stay flat (their duplicate sources are the small common shaders, so calls drop but wall time is parse-bound on unique sources). Full DirectGLES retrace, 445-test unit suite, and dedupe-semantics tests (no-op recompile, invalidation on new source, failed-compile memo, cache bounds) all green.
This commit is contained in:
@@ -79,7 +79,7 @@ namespace MobileGL::MG_State::GLState {
|
||||
Uint shaderId = 0;
|
||||
m_programShaderNameGenerator.Generate(1, &shaderId);
|
||||
EnsureIndexAvail(shaderId, m_shaderObjects);
|
||||
auto shaderObject = MakeShared<ShaderObject>(stage, shaderId);
|
||||
auto shaderObject = MakeShared<ShaderObject>(stage, shaderId, &m_shaderPreprocessCache);
|
||||
if (shaderObject == nullptr) return 0;
|
||||
m_shaderObjects[shaderId] = shaderObject;
|
||||
return shaderId;
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
#include <Includes.h>
|
||||
#include <MG_Util/Miscellany/IndexGenerator.h>
|
||||
#include "ProgramObject.h"
|
||||
#include "ShaderPreprocessCache.h"
|
||||
|
||||
namespace MobileGL::MG_State::GLState {
|
||||
class ProgramState {
|
||||
@@ -33,6 +34,11 @@ namespace MobileGL::MG_State::GLState {
|
||||
|
||||
const SharedPtr<ProgramObject>& GetCurrentProgram() const { return m_currentProgram; }
|
||||
|
||||
// P0b layer 2. Exposed for tests and diagnostics; the GL frontend never touches it
|
||||
// directly - shader objects reach it through the pointer they are handed at
|
||||
// CreateShader().
|
||||
ShaderPreprocessCache& GetShaderPreprocessCache() { return m_shaderPreprocessCache; }
|
||||
|
||||
private:
|
||||
Bool ShaderHasGLVisibleAttachment(const SharedPtr<ShaderObject>& shaderObject) const;
|
||||
// Frees the name slot and releases orphaned attached shaders; the immediate half
|
||||
@@ -57,6 +63,12 @@ namespace MobileGL::MG_State::GLState {
|
||||
// rejected with INVALID_OPERATION, and vice versa). One generator for both
|
||||
// object kinds keeps the names disjoint; the object tables stay separate.
|
||||
IndexGenerator<Uint> m_programShaderNameGenerator;
|
||||
|
||||
// P0b layer 2: every shader object created here is handed a pointer to this cache.
|
||||
// Declared FIRST on purpose - members are destroyed in reverse declaration order,
|
||||
// so the cache outlives every shader object holding a pointer to it.
|
||||
ShaderPreprocessCache m_shaderPreprocessCache;
|
||||
|
||||
Vector<SharedPtr<ProgramObject>> m_programObjects;
|
||||
Vector<SharedPtr<ShaderObject>> m_shaderObjects;
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
// End of Source File Header
|
||||
|
||||
#include "ShaderObject.h"
|
||||
#include "ShaderPreprocessCache.h"
|
||||
#include <MG_Util/ShaderTranspiler/Types.h>
|
||||
#include <MG_Util/ShaderTranspiler/ShaderCompiler.h>
|
||||
#include <MG_Util/Converters/MGToGL/ProgramEnumConverter.h>
|
||||
@@ -137,19 +138,91 @@ namespace {
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
// The half of ShaderObject::Compile() that depends on nothing but the source text and
|
||||
// the stage: preprocessing, the two lexical rejections, and the two lexical
|
||||
// side-channel extractions. Split out so P0b layer 2 can memoize exactly this and
|
||||
// nothing else - the glslang parse stays per-object because its TShader is
|
||||
// consume-once. Deliberately free of any per-object state so the memo is sound.
|
||||
//
|
||||
// Caveat, documented rather than defended against: the compute local-size verdict also
|
||||
// reads the active backend's GL_MAX_COMPUTE_WORK_GROUP_* limits. Those are fixed for
|
||||
// the lifetime of a context, and the cache is per-context, so the memo cannot outlive
|
||||
// the limits it was computed against.
|
||||
static MobileGL::MG_State::GLState::ShaderPreprocessResult RunSourceOnlyPipeline(
|
||||
const MobileGL::ShaderStage stage, const MobileGL::String& source) {
|
||||
using namespace MobileGL;
|
||||
using namespace MobileGL::MG_Util::ShaderTranspiler;
|
||||
using MobileGL::MG_State::GLState::ShaderPreprocessOutcome;
|
||||
|
||||
MobileGL::MG_State::GLState::ShaderPreprocessResult result;
|
||||
result.preprocessedSource = source;
|
||||
PreprocessShaderSource(stage, result.preprocessedSource);
|
||||
|
||||
if (stage == ShaderStage::Compute) {
|
||||
if (const std::optional<String> localSizeError =
|
||||
ValidateComputeLocalSizeLimits(result.preprocessedSource)) {
|
||||
result.outcome = ShaderPreprocessOutcome::ComputeLocalSizeRejected;
|
||||
result.infoLog = *localSizeError;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
if (const std::optional<String> reservedError = FindReservedIdentifierViolation(result.preprocessedSource)) {
|
||||
result.outcome = ShaderPreprocessOutcome::ReservedIdentifierRejected;
|
||||
result.infoLog = *reservedError;
|
||||
return result;
|
||||
}
|
||||
|
||||
// The parse this feeds runs in the link-compatible configuration (Vulkan-client
|
||||
// env with relaxed rules): the TShader it produces is what glLinkProgram links and
|
||||
// what the backends' SPIR-V is generated from - there is no second, GL-client
|
||||
// parse. The GL frontend semantics the relaxed parse cannot provide are restored
|
||||
// on top: explicit default-block uniform locations through the lexical
|
||||
// side-channels below, dead-uniform/global-UBO filtering in
|
||||
// ProgramObject::DoReflection.
|
||||
result.explicitUniformLocations = ExtractExplicitUniformLocations(result.preprocessedSource);
|
||||
result.explicitOpaqueBindings = ExtractExplicitOpaqueBindings(result.preprocessedSource);
|
||||
result.outcome = ShaderPreprocessOutcome::Preprocessed;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
namespace MobileGL::MG_State::GLState {
|
||||
void ShaderObject::SetShaderSource(const String& source) {
|
||||
// P0b layer 1. glShaderSource always REPLACES the source, but replacing it with a
|
||||
// byte-identical one cannot change what a compile would produce: the whole
|
||||
// pipeline below (preprocess -> lexical checks -> glslang parse) is a pure
|
||||
// function of (stage, source) plus context-lifetime backend limits. So keeping the
|
||||
// compiled state is not an optimization that changes observable behaviour - the
|
||||
// COMPILE_STATUS, the info log and the reflection a caller can query are exactly
|
||||
// what a real recompile would have rebuilt, byte for byte.
|
||||
if (SourceMatchesCompiledState(source)) return;
|
||||
m_source = source;
|
||||
InvalidateCompiledState();
|
||||
}
|
||||
|
||||
void ShaderObject::SetShaderSource(String&& source) {
|
||||
if (SourceMatchesCompiledState(source)) return;
|
||||
m_source = Move(source);
|
||||
InvalidateCompiledState();
|
||||
}
|
||||
|
||||
Bool ShaderObject::SourceMatchesCompiledState(const String& candidate) const {
|
||||
if (!m_hasCompiledState) return false;
|
||||
if (candidate.length() != m_compiledSourceLength) return false;
|
||||
if (ShaderPreprocessCache::HashSource(candidate) != m_compiledSourceHash) return false;
|
||||
// The hash is a fast reject only; confirm against the actual stored text. While
|
||||
// m_hasCompiledState holds, m_source IS the source that produced the state.
|
||||
return candidate == m_source;
|
||||
}
|
||||
|
||||
void ShaderObject::RememberCompiledSource(const Uint64 sourceHash) {
|
||||
m_hasCompiledState = true;
|
||||
m_compiledSourceHash = sourceHash;
|
||||
m_compiledSourceLength = m_source.length();
|
||||
}
|
||||
|
||||
void ShaderObject::InvalidateCompiledState() {
|
||||
m_shader.reset();
|
||||
m_preprocessedSource.clear();
|
||||
@@ -158,56 +231,76 @@ namespace MobileGL::MG_State::GLState {
|
||||
m_shaderConsumedByLink = false;
|
||||
m_compileStatus = false;
|
||||
m_infoLog.clear();
|
||||
m_hasCompiledState = false;
|
||||
m_compiledSourceHash = 0;
|
||||
m_compiledSourceLength = 0;
|
||||
}
|
||||
|
||||
void ShaderObject::Compile() {
|
||||
using namespace MG_Util::ShaderTranspiler;
|
||||
|
||||
// P0b layer 1: the state this object holds was produced by a previous Compile() of
|
||||
// the exact source it still holds, so a recompile is a no-op. This covers the
|
||||
// failure case too - the info log stays queryable because nothing is cleared.
|
||||
//
|
||||
// m_shaderConsumedByLink interaction: if the stored TShader already fed a link,
|
||||
// the no-op leaves m_preprocessedSource and both side-channel maps intact, which
|
||||
// is precisely what TakeShaderForLink's on-demand re-parse needs. A real recompile
|
||||
// would have handed the next link a fresh parse; the no-op hands it a fresh
|
||||
// re-parse of the identical source instead. Same result, one parse either way.
|
||||
if (m_hasCompiledState) return;
|
||||
|
||||
InvalidateCompiledState();
|
||||
String compileSource = m_source;
|
||||
MG_Util::ShaderTranspiler::PreprocessShaderSource(m_stage, compileSource);
|
||||
|
||||
if (m_stage == ShaderStage::Compute) {
|
||||
const std::optional<String> localSizeError = ValidateComputeLocalSizeLimits(compileSource);
|
||||
if (localSizeError) {
|
||||
m_infoLog = *localSizeError;
|
||||
return;
|
||||
}
|
||||
}
|
||||
const Uint64 sourceHash = ShaderPreprocessCache::HashSource(m_source);
|
||||
|
||||
const std::optional<String> reservedError =
|
||||
MG_Util::ShaderTranspiler::FindReservedIdentifierViolation(compileSource);
|
||||
if (reservedError) {
|
||||
m_infoLog = *reservedError;
|
||||
// P0b layer 2: another shader object in this context may already have run the
|
||||
// source-only half over byte-identical text.
|
||||
const ShaderPreprocessResult* cached =
|
||||
m_preprocessCache != nullptr ? m_preprocessCache->Find(m_stage, sourceHash, m_source) : nullptr;
|
||||
ShaderPreprocessResult fresh;
|
||||
if (cached == nullptr) fresh = RunSourceOnlyPipeline(m_stage, m_source);
|
||||
const ShaderPreprocessResult& shared = cached != nullptr ? *cached : fresh;
|
||||
const Bool shouldPopulateCache = cached == nullptr && m_preprocessCache != nullptr;
|
||||
|
||||
if (!shared.Preprocessed()) {
|
||||
// Rejected lexically, or a glslang failure this context has already seen for
|
||||
// this exact source (ParseFailed) - either way the parse can be skipped.
|
||||
m_infoLog = shared.infoLog;
|
||||
if (shouldPopulateCache) m_preprocessCache->Insert(m_stage, sourceHash, m_source, Move(fresh));
|
||||
RememberCompiledSource(sourceHash);
|
||||
return;
|
||||
}
|
||||
|
||||
// Single parse, in the link-compatible configuration (Vulkan-client env with
|
||||
// relaxed rules): the TShader stored here is what glLinkProgram links and what
|
||||
// the backends' SPIR-V is generated from - there is no second, GL-client parse
|
||||
// anymore. The GL frontend semantics the relaxed parse cannot provide are
|
||||
// restored on top: explicit default-block uniform locations through the lexical
|
||||
// side-channel below, dead-uniform/global-UBO filtering in
|
||||
// ProgramObject::DoReflection.
|
||||
m_explicitUniformLocations = ExtractExplicitUniformLocations(compileSource);
|
||||
m_explicitOpaqueBindings = ExtractExplicitOpaqueBindings(compileSource);
|
||||
ShaderAttrib attrib{.shaderType = MG_Util::ConvertShaderStageToGLEnum(m_stage),
|
||||
.sourceStr = compileSource,
|
||||
.sourceStr = shared.preprocessedSource,
|
||||
.flags = 0};
|
||||
|
||||
auto result = ShaderCompiler::CompileShader(attrib);
|
||||
if (result) {
|
||||
m_compileStatus = true;
|
||||
m_shader = result.value();
|
||||
m_preprocessedSource = Move(compileSource);
|
||||
// Copy, not move: `shared` may alias a cache entry that has to outlive us, and
|
||||
// `fresh` is about to be handed to the cache.
|
||||
m_preprocessedSource = shared.preprocessedSource;
|
||||
m_explicitUniformLocations = shared.explicitUniformLocations;
|
||||
m_explicitOpaqueBindings = shared.explicitOpaqueBindings;
|
||||
m_infoLog.clear();
|
||||
if (shouldPopulateCache) m_preprocessCache->Insert(m_stage, sourceHash, m_source, Move(fresh));
|
||||
} else {
|
||||
m_explicitUniformLocations.clear();
|
||||
m_explicitOpaqueBindings.clear();
|
||||
m_infoLog = result.error().log;
|
||||
MGLOG_D("ShaderObject::Compile: Shader %d compilation failed.\nSource:\n%s\nInfoLog:\n%s\nSetting "
|
||||
"m_compileStatus = false as a result.",
|
||||
m_externalIndex, compileSource.c_str(), m_infoLog.c_str());
|
||||
m_externalIndex, shared.preprocessedSource.c_str(), m_infoLog.c_str());
|
||||
if (shouldPopulateCache) {
|
||||
fresh.outcome = ShaderPreprocessOutcome::ParseFailed;
|
||||
fresh.infoLog = m_infoLog;
|
||||
fresh.explicitUniformLocations.clear();
|
||||
fresh.explicitOpaqueBindings.clear();
|
||||
m_preprocessCache->Insert(m_stage, sourceHash, m_source, Move(fresh));
|
||||
}
|
||||
}
|
||||
RememberCompiledSource(sourceHash);
|
||||
}
|
||||
|
||||
SharedPtr<glslang::TShader> ShaderObject::TakeShaderForLink(String& outReparseLog) {
|
||||
|
||||
@@ -22,10 +22,18 @@ namespace MobileGL {
|
||||
};
|
||||
|
||||
namespace MG_State::GLState {
|
||||
// P0b layer 2. Declared, not included: the cache keys on ShaderStage, so including
|
||||
// its header here would be circular.
|
||||
class ShaderPreprocessCache;
|
||||
|
||||
class ShaderObject {
|
||||
public:
|
||||
ShaderObject(const ShaderStage stage, Uint externalIndex)
|
||||
: m_stage(stage), m_externalIndex(externalIndex) {}
|
||||
// `preprocessCache` is the owning context's cross-object memo (P0b layer 2);
|
||||
// null is fully supported and simply means "no sharing" - that is what the
|
||||
// context-less internal shader objects (the default FS, the blit pipeline) use.
|
||||
ShaderObject(const ShaderStage stage, Uint externalIndex,
|
||||
ShaderPreprocessCache* preprocessCache = nullptr)
|
||||
: m_stage(stage), m_externalIndex(externalIndex), m_preprocessCache(preprocessCache) {}
|
||||
void SetShaderSource(const String& source);
|
||||
void SetShaderSource(String&& source);
|
||||
void Compile();
|
||||
@@ -59,8 +67,22 @@ namespace MobileGL {
|
||||
Bool GetCompileStatus() const { return m_compileStatus; }
|
||||
Bool GetDeleteStatus() const { return m_deleteStatus; }
|
||||
|
||||
// True while this object holds the outcome (success OR failure) of a previous
|
||||
// Compile() of exactly the source it currently holds - i.e. while the P0b
|
||||
// layer-1 memo is armed and a glCompileShader would be a no-op. Diagnostics
|
||||
// and tests only; nothing in the GL frontend branches on it.
|
||||
Bool HasMemoizedCompile() const { return m_hasCompiledState; }
|
||||
|
||||
private:
|
||||
void InvalidateCompiledState();
|
||||
// ---- P0b layer 1: per-object no-op recompile ----
|
||||
// True iff `candidate` is byte-identical to the source that produced the
|
||||
// compiled state this object is currently holding. The stored hash and length
|
||||
// are only a fast reject; the answer is always confirmed against the full
|
||||
// stored text, so no behaviour rides on a 64-bit hash.
|
||||
Bool SourceMatchesCompiledState(const String& candidate) const;
|
||||
// Arms the layer-1 memo for the source that Compile() just processed.
|
||||
void RememberCompiledSource(Uint64 sourceHash);
|
||||
|
||||
const Uint m_externalIndex = 0;
|
||||
const ShaderStage m_stage;
|
||||
@@ -75,6 +97,15 @@ namespace MobileGL {
|
||||
UnorderedMap<String, Uint> m_explicitOpaqueBindings;
|
||||
Bool m_shaderConsumedByLink = false;
|
||||
|
||||
// P0b layer 2: the owning context's cross-object memo, or null. Not owned.
|
||||
ShaderPreprocessCache* const m_preprocessCache = nullptr;
|
||||
// P0b layer 1. m_hasCompiledState is the invariant "m_source is byte-identical
|
||||
// to the source that produced m_compileStatus/m_infoLog/m_shader"; it is armed
|
||||
// at the end of every Compile() and disarmed by InvalidateCompiledState().
|
||||
Bool m_hasCompiledState = false;
|
||||
Uint64 m_compiledSourceHash = 0;
|
||||
SizeT m_compiledSourceLength = 0;
|
||||
|
||||
String m_infoLog;
|
||||
Bool m_deleteStatus = false;
|
||||
Bool m_compileStatus = false;
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
// MobileGL - MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.cpp
|
||||
// Copyright (c) 2025-2026 MobileGL-Dev
|
||||
// Licensed under the GNU Lesser General Public License v3.0:
|
||||
// https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
// https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
// SPDX-License-Identifier: LGPL-3.0-only
|
||||
// End of Source File Header
|
||||
|
||||
#include "ShaderPreprocessCache.h"
|
||||
|
||||
namespace MobileGL::MG_State::GLState {
|
||||
const ShaderPreprocessResult* ShaderPreprocessCache::Find(const ShaderStage stage, const Uint64 sourceHash,
|
||||
const String& source) const {
|
||||
const Key key{.stage = stage, .sourceHash = sourceHash, .sourceLength = source.length()};
|
||||
const auto it = m_index.find(key);
|
||||
if (it == m_index.end()) return nullptr;
|
||||
|
||||
// Never let correctness ride on a 64-bit hash: confirm the hit byte for byte.
|
||||
// Lengths already matched (they are part of the key), so this is a plain memcmp.
|
||||
const Entry& entry = *it->second;
|
||||
if (entry.originalSource != source) return nullptr;
|
||||
|
||||
return &entry.result;
|
||||
}
|
||||
|
||||
void ShaderPreprocessCache::Insert(const ShaderStage stage, const Uint64 sourceHash, const String& source,
|
||||
ShaderPreprocessResult result) {
|
||||
const SizeT entryBytes = EntryBytes(source, result);
|
||||
// A single source bigger than the whole budget would evict every other entry and
|
||||
// then itself; refuse it instead of thrashing the cache empty.
|
||||
if (entryBytes > kMaxStoredSourceBytes) return;
|
||||
|
||||
const Key key{.stage = stage, .sourceHash = sourceHash, .sourceLength = source.length()};
|
||||
if (const auto existing = m_index.find(key); existing != m_index.end()) {
|
||||
// Either a re-insert of the same source (harmless) or a genuine hash collision
|
||||
// with a different source. Both are resolved by letting the newcomer win: one
|
||||
// entry per key keeps the index a plain map, and a collision is astronomically
|
||||
// rare enough that the loser simply misses.
|
||||
EraseEntry(existing->second);
|
||||
}
|
||||
|
||||
m_entries.push_back(Entry{.key = key, .originalSource = source, .result = Move(result)});
|
||||
m_index[key] = std::prev(m_entries.end());
|
||||
m_storedSourceBytes += entryBytes;
|
||||
|
||||
EvictUntilWithinBudget();
|
||||
}
|
||||
|
||||
void ShaderPreprocessCache::Clear() {
|
||||
m_entries.clear();
|
||||
m_index.clear();
|
||||
m_storedSourceBytes = 0;
|
||||
}
|
||||
|
||||
void ShaderPreprocessCache::EraseEntry(const EntryList::iterator it) {
|
||||
const SizeT bytes = EntryBytes(it->originalSource, it->result);
|
||||
m_storedSourceBytes = bytes > m_storedSourceBytes ? 0 : m_storedSourceBytes - bytes;
|
||||
m_index.erase(it->key);
|
||||
m_entries.erase(it);
|
||||
}
|
||||
|
||||
void ShaderPreprocessCache::EvictUntilWithinBudget() {
|
||||
// FIFO: the oldest insertion goes first. Insert() already refuses entries larger
|
||||
// than the byte budget, so this loop always terminates with at least the entry
|
||||
// that was just added still resident.
|
||||
while (!m_entries.empty() &&
|
||||
(m_entries.size() > kMaxEntries || m_storedSourceBytes > kMaxStoredSourceBytes)) {
|
||||
EraseEntry(m_entries.begin());
|
||||
}
|
||||
}
|
||||
} // namespace MobileGL::MG_State::GLState
|
||||
@@ -0,0 +1,140 @@
|
||||
// MobileGL - MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.h
|
||||
// Copyright (c) 2025-2026 MobileGL-Dev
|
||||
// Licensed under the GNU Lesser General Public License v3.0:
|
||||
// https://www.gnu.org/licenses/gpl-3.0.txt
|
||||
// https://www.gnu.org/licenses/lgpl-3.0.txt
|
||||
// SPDX-License-Identifier: LGPL-3.0-only
|
||||
// End of Source File Header
|
||||
|
||||
#pragma once
|
||||
#include <Includes.h>
|
||||
#include <list>
|
||||
#include <MG_State/GLState/ProgramState/ShaderObject.h>
|
||||
|
||||
namespace MobileGL::MG_State::GLState {
|
||||
// Where the shared, source-only half of ShaderObject::Compile() stopped. The two
|
||||
// rejection verdicts are kept apart (rather than collapsed into "failed") so a hit
|
||||
// reproduces the original diagnosis, not just the original info log.
|
||||
enum class ShaderPreprocessOutcome : Uint8 {
|
||||
// The source-only half ran clean; preprocessedSource and both maps are valid.
|
||||
Preprocessed,
|
||||
// ValidateComputeLocalSizeLimits rejected it (compute only).
|
||||
ComputeLocalSizeRejected,
|
||||
// FindReservedIdentifierViolation rejected it.
|
||||
ReservedIdentifierRejected,
|
||||
// The source-only half was clean but glslang rejected the preprocessed source.
|
||||
// Memoizing this saves the parse itself on every later object with that source.
|
||||
ParseFailed,
|
||||
};
|
||||
|
||||
// Everything ShaderObject::Compile() derives from the source text alone, i.e.
|
||||
// everything that is identical for two shader objects holding byte-identical source.
|
||||
struct ShaderPreprocessResult {
|
||||
ShaderPreprocessOutcome outcome = ShaderPreprocessOutcome::Preprocessed;
|
||||
// Valid unless the preprocessor itself never ran; kept even for the rejection
|
||||
// outcomes because that is the text the diagnostics refer to.
|
||||
String preprocessedSource;
|
||||
UnorderedMap<String, Int> explicitUniformLocations;
|
||||
UnorderedMap<String, Uint> explicitOpaqueBindings;
|
||||
// The compile info log to publish; empty when outcome == Preprocessed.
|
||||
String infoLog;
|
||||
|
||||
Bool Preprocessed() const { return outcome == ShaderPreprocessOutcome::Preprocessed; }
|
||||
};
|
||||
|
||||
// P0b layer 2: a per-context, bounded memo of the source-only half of shader
|
||||
// compilation, keyed by (stage, xxhash64(source), source length).
|
||||
//
|
||||
// Motivation: in the Iris shader-pack corpus ~21% of every glCompileShader in a trace
|
||||
// is a *different* shader object holding byte-identical source (packs glue the same
|
||||
// common/composite GLSL into many program stages), so the preprocess + reserved-
|
||||
// identifier scan + explicit-location/binding extraction runs over the same megabytes
|
||||
// again and again. Layer 1 (in ShaderObject) covers the same object recompiled with
|
||||
// unchanged source; this covers the cross-object case.
|
||||
//
|
||||
// What is NOT cached: the glslang parse. glslang's TShader is consume-once (mapIO
|
||||
// mutates the aliased intermediate at link), so every shader object still needs its
|
||||
// own parse; only the text-processing half is shared.
|
||||
//
|
||||
// Correctness: the 64-bit hash is a lookup accelerator only. Every hit re-compares the
|
||||
// full stored original source with memcmp before it is honored, so a hash collision
|
||||
// degrades to a miss, never to a wrong answer. That is why the full original text is
|
||||
// stored rather than a prefix/suffix digest - the cache is bounded, so the cost is.
|
||||
//
|
||||
// Eviction: FIFO (insertion order), bounded by BOTH an entry count and a stored-source
|
||||
// byte budget, whichever binds first. FIFO rather than LRU because shader-pack loading
|
||||
// is a burst of mostly-distinct sources whose reuse clusters around insertion time;
|
||||
// LRU's extra list splice on every hit buys nothing measurable here, and FIFO keeps
|
||||
// Find() a genuinely const, read-only operation.
|
||||
class ShaderPreprocessCache {
|
||||
public:
|
||||
static constexpr SizeT kMaxEntries = 128;
|
||||
static constexpr SizeT kMaxStoredSourceBytes = 8u * 1024u * 1024u;
|
||||
|
||||
// Returns the memoized result for this exact source, or null on a miss. The
|
||||
// returned pointer stays valid until the next Insert()/Clear() on this cache.
|
||||
const ShaderPreprocessResult* Find(ShaderStage stage, Uint64 sourceHash, const String& source) const;
|
||||
|
||||
// Memoizes `result` for this source. A source whose own storage cost already
|
||||
// exceeds the byte budget is simply not cached (caching it would evict everything
|
||||
// else and then itself).
|
||||
void Insert(ShaderStage stage, Uint64 sourceHash, const String& source, ShaderPreprocessResult result);
|
||||
|
||||
void Clear();
|
||||
|
||||
static Uint64 HashSource(const String& source) {
|
||||
return static_cast<Uint64>(XXH64(source.data(), source.length(), 0));
|
||||
}
|
||||
|
||||
SizeT GetEntryCount() const { return m_entries.size(); }
|
||||
SizeT GetStoredSourceBytes() const { return m_storedSourceBytes; }
|
||||
|
||||
private:
|
||||
struct Key {
|
||||
ShaderStage stage = ShaderStage::Unknown;
|
||||
Uint64 sourceHash = 0;
|
||||
SizeT sourceLength = 0;
|
||||
|
||||
Bool operator==(const Key& other) const {
|
||||
return stage == other.stage && sourceHash == other.sourceHash && sourceLength == other.sourceLength;
|
||||
}
|
||||
};
|
||||
|
||||
struct KeyHasher {
|
||||
SizeT operator()(const Key& key) const {
|
||||
// The source hash already spreads well; fold the two discriminators in so
|
||||
// that same-hash-different-stage/length keys land in different buckets.
|
||||
Uint64 mixed = key.sourceHash;
|
||||
mixed ^= static_cast<Uint64>(key.sourceLength) + 0x9e3779b97f4a7c15ull + (mixed << 6) + (mixed >> 2);
|
||||
mixed ^= static_cast<Uint64>(static_cast<Int>(key.stage)) * 0xff51afd7ed558ccdull;
|
||||
return static_cast<SizeT>(mixed);
|
||||
}
|
||||
};
|
||||
|
||||
struct Entry {
|
||||
Key key;
|
||||
// The full original (pre-preprocess) source, kept so a hit can be confirmed by
|
||||
// comparison instead of trusting the hash.
|
||||
String originalSource;
|
||||
ShaderPreprocessResult result;
|
||||
};
|
||||
|
||||
using EntryList = std::list<Entry>;
|
||||
|
||||
static SizeT EntryBytes(const String& source, const ShaderPreprocessResult& result) {
|
||||
return source.length() + result.preprocessedSource.length();
|
||||
}
|
||||
|
||||
void EraseEntry(EntryList::iterator it);
|
||||
void EvictUntilWithinBudget();
|
||||
|
||||
// P1: needs a mutex when compiles go async. Everything here is reached from
|
||||
// glCompileShader on the single GL thread that owns the context, so today the
|
||||
// cache is deliberately lock-free; the moment shader compilation moves onto a
|
||||
// worker pool, Find/Insert/Clear all become critical sections (and Find's returned
|
||||
// pointer stops being safe to hold across an Insert).
|
||||
EntryList m_entries; // front = oldest (FIFO victim)
|
||||
UnorderedMap<Key, EntryList::iterator, KeyHasher> m_index;
|
||||
SizeT m_storedSourceBytes = 0;
|
||||
};
|
||||
} // namespace MobileGL::MG_State::GLState
|
||||
Reference in New Issue
Block a user