diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/MagmaProgramSource.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/MagmaProgramSource.h index 1f91347e..2b0f3f28 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/MagmaProgramSource.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/MagmaProgramSource.h @@ -53,24 +53,26 @@ namespace MobileGL::MG_Backend::DirectVulkan { // The view has no lifetime of its own, so the memo lives in the server record it views // (MGPipeShaderCsoRecord::BackendHashMemo), never in the client's program object: an entry // answers only while the record's Serial and BindingsSerial are the ones it was taken at. - Bool GetBackendHashMemo(Uint flags, Uint64& hash) const { - for (const auto& entry : m_record->BackendHashMemo) { - if (entry.Valid && entry.Key == flags && entry.Serial == m_record->Serial && - entry.BindingsSerial == m_record->BindingsSerial) { - hash = entry.Hash; - return true; - } + using BackendHashMemoEntry = MG_Pipe::MGPipeShaderCsoRecord::BackendHashMemoEntry; + BackendHashMemoEntry* FindBackendHashMemo(Uint key) const { + for (auto& entry : m_record->BackendHashMemo) { + if (entry.Valid && entry.Key == key && entry.Serial == m_record->Serial && + entry.BindingsSerial == m_record->BindingsSerial) + return &entry; } - return false; + return nullptr; } - void SetBackendHashMemo(Uint flags, Uint64 hash) const { + // Takes the next slot round robin; its LastVariantHash starts as the plain `hash`. + BackendHashMemoEntry& SetBackendHashMemo(Uint key, Uint64 hash) const { auto& memo = m_record->BackendHashMemo; auto& entry = memo[m_record->BackendHashMemoNext++ % memo.size()]; entry.Serial = m_record->Serial; entry.BindingsSerial = m_record->BindingsSerial; entry.Hash = hash; - entry.Key = flags; + entry.LastVariantHash = hash; + entry.Key = key; entry.Valid = true; + return entry; } Uint64 GetBackendStateVersion() const { return m_record->Serial; } Uint64 GetBlockBindingVersion() const { return m_record->BindingsSerial; } diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp index 95302868..52c21ca4 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp @@ -3626,9 +3626,8 @@ namespace MobileGL::MG_Backend::DirectVulkan { } } // namespace - const ProgramFactory::VkProgramObject& ProgramFactory::GetOrCreateProgram( - const MagmaProgramSource& program, CompileOptionFlags flags, - const NarrowIntegerVertexInputs& narrowIntegerInputs) { + MagmaProgramSource::BackendHashMemoEntry& ProgramFactory::ProgramHashMemo(const MagmaProgramSource& program, + CompileOptionFlags flags) const { // Hashing the full SPIR-V of every stage is far too expensive to repeat per draw; // reuse the program's memoized hash while its backend state version is unchanged. // The memo keys on the flags word, which ComputeHash is no longer a pure function of: @@ -3644,19 +3643,46 @@ namespace MobileGL::MG_Backend::DirectVulkan { const Uint memoKey = (flags & CompileOptionBit::FragCoordYFlip) ? (flags.GetRaw() | (m_defaultFramebufferHeight << 16)) : flags.GetRaw(); - HashType hash = 0; - if (!program.GetBackendHashMemo(memoKey, hash)) { - hash = ComputeHash(program, flags); - program.SetBackendHashMemo(memoKey, hash); - } + if (auto* memo = program.FindBackendHashMemo(memoKey)) return *memo; + return program.SetBackendHashMemo(memoKey, ComputeHash(program, flags)); + } + + const ProgramFactory::VkProgramObject& ProgramFactory::GetOrCreateProgram( + const MagmaProgramSource& program, CompileOptionFlags flags, + const NarrowIntegerVertexInputs& narrowIntegerInputs) { + auto& memo = ProgramHashMemo(program, flags); // Mixed in after the memo, which keys on the flags word alone: the memo keeps answering // for the plain program, and the variant costs one 16-byte hash on top of it. + HashType hash = memo.Hash; if (narrowIntegerInputs.Any()) hash = MixNarrowIntegerVertexInputs(hash, narrowIntegerInputs); + memo.LastVariantHash = hash; + return FindOrBuildProgram(program, flags, narrowIntegerInputs, hash); + } + + const ProgramFactory::VkProgramObject& ProgramFactory::GetOrCreateDrawProgram(const MagmaProgramSource& program, + CompileOptionFlags flags) { + auto& memo = ProgramHashMemo(program, flags); + if (memo.LastVariantHash != memo.Hash) { + const auto it = m_cache.find(memo.LastVariantHash); + if (it != m_cache.end()) { + it->second.lastUsedFrame = m_frameCounter; + return it->second; + } + // The variant aged out, and its hash alone cannot rebuild it: start from the plain + // program, and the draw settles on its variant again. + memo.LastVariantHash = memo.Hash; + } + return FindOrBuildProgram(program, flags, {}, memo.Hash); + } + + const ProgramFactory::VkProgramObject& ProgramFactory::FindOrBuildProgram( + const MagmaProgramSource& program, CompileOptionFlags flags, + const NarrowIntegerVertexInputs& narrowIntegerInputs, HashType hash) { auto it = m_cache.find(hash); if (it != m_cache.end()) { - // Every draw/dispatch funnels through this lookup (the renderer memos only - // skip re-hashing, never the factory lookup), so an actively-used entry is - // stamped at least once per frame boundary and can never be aged out while + // Every draw/dispatch funnels through this lookup or GetOrCreateDrawProgram's (the + // renderer memos only skip re-hashing, never the factory lookup), so an actively-used + // entry is stamped at least once per frame boundary and can never be aged out while // any in-flight command buffer still references it. it->second.lastUsedFrame = m_frameCounter; return it->second; @@ -3667,6 +3693,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { ++m_cacheStructureEpoch; auto& entry = m_cache[hash]; entry.hash = hash; + entry.narrowIntegerInputs = narrowIntegerInputs; entry.lastUsedFrame = m_frameCounter; // The EXECUTABLE's stage list, not GetAttachedShaders(): `spirv` is a link artifact with // one module per linked stage, while the attach list is live and grows on diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.h index 743ec60b..8f1afcbf 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.h @@ -204,6 +204,9 @@ namespace MobileGL::MG_Backend::DirectVulkan { // The locations of vertexInputTypes that are int / uint, for the per-draw narrow-integer // variant decision (ResolveNarrowIntegerVertexInputs). Identical in every variant. IntegerVertexInputMasks integerVertexInputs; + // The narrow-integer correction this entry's vertex stage was built with (none for the + // plain program): what a draw compares its own resolved set against. + NarrowIntegerVertexInputs narrowIntegerInputs; Uint32 activeFragmentOutputLocationMask = 0; Array fragmentOutputTypes{}; ShaderStage rasterizationProducerStage = ShaderStage::Unknown; @@ -333,6 +336,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { activeVertexInputLocationMask = other.activeVertexInputLocationMask; vertexInputTypes = other.vertexInputTypes; integerVertexInputs = other.integerVertexInputs; + narrowIntegerInputs = other.narrowIntegerInputs; activeFragmentOutputLocationMask = other.activeFragmentOutputLocationMask; fragmentOutputTypes = other.fragmentOutputTypes; rasterizationProducerStage = other.rasterizationProducerStage; @@ -357,6 +361,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { other.globalUboBinding = -1; other.activeVertexInputLocationMask = 0; other.integerVertexInputs = {}; + other.narrowIntegerInputs = {}; other.activeFragmentOutputLocationMask = 0; other.rasterizationProducerStage = ShaderStage::Unknown; other.producerOutputComponentCount = 0; @@ -409,6 +414,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { activeVertexInputLocationMask = other.activeVertexInputLocationMask; vertexInputTypes = other.vertexInputTypes; integerVertexInputs = other.integerVertexInputs; + narrowIntegerInputs = other.narrowIntegerInputs; activeFragmentOutputLocationMask = other.activeFragmentOutputLocationMask; fragmentOutputTypes = other.fragmentOutputTypes; rasterizationProducerStage = other.rasterizationProducerStage; @@ -433,6 +439,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { other.globalUboBinding = -1; other.activeVertexInputLocationMask = 0; other.integerVertexInputs = {}; + other.narrowIntegerInputs = {}; other.activeFragmentOutputLocationMask = 0; other.rasterizationProducerStage = ShaderStage::Unknown; other.producerOutputComponentCount = 0; @@ -529,9 +536,15 @@ namespace MobileGL::MG_Backend::DirectVulkan { // `narrowIntegerInputs` selects the variant whose vertex stage widens narrow integer // attribute data GL's way (ExtendNarrowIntegerVertexInputsPass). It is draw-time state like // the flags, and the draw path asks for it only when ResolveNarrowIntegerVertexInputs finds - // a location that needs it, so every other program keeps a single entry. + // a location that needs it, so every other program keeps a single entry. The variant + // returned is remembered per flags for GetOrCreateDrawProgram. const VkProgramObject& GetOrCreateProgram(const MagmaProgramSource& program, CompileOptionFlags flags, const NarrowIntegerVertexInputs& narrowIntegerInputs = {}); + // A draw's first lookup: the variant GetOrCreateProgram last returned for these flags, plain + // or narrow-integer. Every variant reflects the same vertex inputs and builtins, which is all + // a draw reads before it knows the variant it needs, so a program whose draws keep needing + // one variant costs a single lookup and its plain entry, never drawn with, ages out. + const VkProgramObject& GetOrCreateDrawProgram(const MagmaProgramSource& program, CompileOptionFlags flags); // The cache identity of a narrow-integer variant: the hash of the same program and flags // without it, mixed with the corrected locations. Content in, content out, so the variant's // pipelines key apart from the plain program's through programHash alone. @@ -673,6 +686,13 @@ namespace MobileGL::MG_Backend::DirectVulkan { HashType hash = 0; }; + // The program's memo slot for `flags`, its plain hash computed on a miss. + MagmaProgramSource::BackendHashMemoEntry& ProgramHashMemo(const MagmaProgramSource& program, + CompileOptionFlags flags) const; + // The cache entry under `hash`, built as that variant of `program` when absent. + const VkProgramObject& FindOrBuildProgram(const MagmaProgramSource& program, CompileOptionFlags flags, + const NarrowIntegerVertexInputs& narrowIntegerInputs, HashType hash); + static TextureTarget UniformTypeToTextureTarget(GLenum glType); // `stages` is ALWAYS ProgramObject::GetLinkedShaderStages() - one entry per module of // `spirv`, at the same index. Taking the stages rather than the shader objects is what diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireDraw.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireDraw.cpp index a31af943..f79dd570 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireDraw.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireDraw.cpp @@ -734,10 +734,12 @@ namespace MobileGL::MG_Backend::DirectVulkan { if (m_transformFeedbackFeatureEnabled && MG_Pipe::gPipeInputs->IsTransformFeedbackActive() && program.GetTransformFeedbackVaryingCount() > 0) flags |= ProgramFactory::CompileOptionBit::XfbCapture; - const auto* programObject = &m_programFactory->GetOrCreateProgram(program, flags); + // The variant this program's last draw with these flags settled on: what the draw reads + // from it before the narrow-integer pick below is the same in every variant. + const auto* programObject = &m_programFactory->GetOrCreateDrawProgram(program, flags); if (!(aspects & DrawSetupAspect::IndexBuffer) && programObject->readsBaseVertexBuiltin) { flags |= ProgramFactory::CompileOptionBit::ZeroBaseVertex; - programObject = &m_programFactory->GetOrCreateProgram(program, flags); + programObject = &m_programFactory->GetOrCreateDrawProgram(program, flags); } const auto vaoHandle = state.BoundVertexElements; if (MG_Pipe::MGPipeHandleIsNull(vaoHandle)) MagmaWireFatal("vertex-elements-record"); @@ -770,11 +772,12 @@ namespace MobileGL::MG_Backend::DirectVulkan { const auto& input = *resolvedInput; // The narrow-integer variant (NarrowIntegerVertexFormats.h), chosen from two masks cached on // the program entry and on the resolved layout: a program without integer vertex inputs pays - // four ANDs here. Every variant reflects the same inputs, so the layout resolved above with - // the plain program's active mask is this variant's too. + // four ANDs and a compare here. Every variant reflects the same inputs, so the layout + // resolved above with the entry's active mask is the chosen variant's too. A second lookup + // only when this draw needs another variant than the last one with these flags. const NarrowIntegerVertexInputs narrowIntegerInputs = ResolveNarrowIntegerVertexInputs(programObject->integerVertexInputs, input.narrowIntegerAttributes); - if (narrowIntegerInputs.Any()) + if (narrowIntegerInputs != programObject->narrowIntegerInputs) programObject = &m_programFactory->GetOrCreateProgram(program, flags, narrowIntegerInputs); if (programObject->invalidSpirvDeclined) { MGL_WIRE_DECLINE_AT(ProgramInvalidSpirv, diff --git a/MobileGL/MG_Pipe/PipeApply.h b/MobileGL/MG_Pipe/PipeApply.h index 84d3ef91..1261ee78 100644 --- a/MobileGL/MG_Pipe/PipeApply.h +++ b/MobileGL/MG_Pipe/PipeApply.h @@ -396,6 +396,10 @@ namespace MobileGL::MG_Pipe { Uint64 Serial = 0; Uint64 BindingsSerial = 0; Uint64 Hash = 0; + // The hash of the variant of this program the factory last handed out for this key: + // Hash itself, or a variant keyed on draw-time vertex data. Derived from Hash, so it + // is valid exactly as long. + Uint64 LastVariantHash = 0; Uint32 Key = 0; Bool Valid = false; }; diff --git a/MobileGL/MG_Test/Pipeline/NarrowIntegerVertexFormatsTest.cpp b/MobileGL/MG_Test/Pipeline/NarrowIntegerVertexFormatsTest.cpp index 84cc7524..bdde843a 100644 --- a/MobileGL/MG_Test/Pipeline/NarrowIntegerVertexFormatsTest.cpp +++ b/MobileGL/MG_Test/Pipeline/NarrowIntegerVertexFormatsTest.cpp @@ -19,6 +19,7 @@ using namespace MobileGL; using MobileGL::MG_Backend::DirectVulkan::ClassifyNarrowIntegerVertexFormat; using MobileGL::MG_Backend::DirectVulkan::IntegerVertexInputMasks; +using MobileGL::MG_Backend::DirectVulkan::MagmaProgramSource; using MobileGL::MG_Backend::DirectVulkan::NarrowIntegerAttributeMasks; using MobileGL::MG_Backend::DirectVulkan::NarrowIntegerVertexFormat; using MobileGL::MG_Backend::DirectVulkan::NarrowIntegerVertexInputs; @@ -135,23 +136,61 @@ TEST(NarrowIntegerVertexFormats, VariantHashKeysEveryCorrectionApart) { hash); } -// The program half lives on the cache entry, and the cache's robin-hood insert moves entries -// around: a mask left behind by a move would pick another program's variant. +// The program half and the entry's own correction live on the cache entry, and the cache's +// robin-hood insert moves entries around: a mask left behind by a move would pick another +// program's variant, and a correction left behind would make a draw take a variant for the plain +// program or the reverse. TEST(NarrowIntegerVertexFormats, IntegerInputMasksTravelWithTheProgramEntry) { + const NarrowIntegerVertexInputs sodium = Only(&NarrowIntegerVertexInputs::zeroExtend8, 3); ProgramFactory::VkProgramObject source; source.integerVertexInputs.signedInputs = 1u << 3; source.integerVertexInputs.unsignedInputs = 1u << 7; + source.narrowIntegerInputs = sodium; ProgramFactory::VkProgramObject moved(std::move(source)); EXPECT_EQ(moved.integerVertexInputs.signedInputs, 1u << 3); EXPECT_EQ(moved.integerVertexInputs.unsignedInputs, 1u << 7); + EXPECT_EQ(moved.narrowIntegerInputs, sodium); EXPECT_EQ(source.integerVertexInputs.signedInputs, 0u); EXPECT_EQ(source.integerVertexInputs.unsignedInputs, 0u); + EXPECT_FALSE(source.narrowIntegerInputs.Any()); ProgramFactory::VkProgramObject assigned; assigned = std::move(moved); EXPECT_EQ(assigned.integerVertexInputs.signedInputs, 1u << 3); EXPECT_EQ(assigned.integerVertexInputs.unsignedInputs, 1u << 7); + EXPECT_EQ(assigned.narrowIntegerInputs, sodium); EXPECT_EQ(moved.integerVertexInputs.signedInputs, 0u); EXPECT_EQ(moved.integerVertexInputs.unsignedInputs, 0u); + EXPECT_FALSE(moved.narrowIntegerInputs.Any()); +} + +// A draw starts from the variant the program's last lookup with the same flags returned, which +// the program record keeps beside the plain hash it derives from. A fresh slot must point at the +// plain program, and the variant must be forgotten with the hash: after a relink or a binding +// change the same flags answer from a fresh slot. +TEST(NarrowIntegerVertexFormats, TheLastVariantIsRememberedOnlyWithItsPlainHash) { + MG_Pipe::MGPipeShaderCsoRecord record{}; + record.Archive = MakeShared(); + record.Serial = 4; + record.BindingsSerial = 9; + const MagmaProgramSource source({5, 1}, record); + constexpr Uint kFlags = 0x21; + + EXPECT_EQ(source.FindBackendHashMemo(kFlags), nullptr); + auto& slot = source.SetBackendHashMemo(kFlags, 0xabcull); + EXPECT_EQ(slot.LastVariantHash, 0xabcull) << "a fresh slot starts at the plain program"; + slot.LastVariantHash = 0xdefull; + const auto* found = source.FindBackendHashMemo(kFlags); + ASSERT_EQ(found, &slot); + EXPECT_EQ(found->Hash, 0xabcull); + EXPECT_EQ(found->LastVariantHash, 0xdefull); + EXPECT_EQ(source.FindBackendHashMemo(kFlags + 1), nullptr) << "other flags have their own slot"; + + record.Serial = 5; // a relink + EXPECT_EQ(source.FindBackendHashMemo(kFlags), nullptr); + EXPECT_EQ(source.SetBackendHashMemo(kFlags, 0x123ull).LastVariantHash, 0x123ull); + ASSERT_NE(source.FindBackendHashMemo(kFlags), nullptr); + record.BindingsSerial = 10; // glUniformBlockBinding and friends + EXPECT_EQ(source.FindBackendHashMemo(kFlags), nullptr); }