[Perf] (MG_Backend/DirectVulkan, MG_Pipe, MG_Test/Pipeline): start a draw from the variant its last draw settled on

- the program record remembers the last variant per flags, so a narrow-integer draw
  costs one lookup and the plain program it never draws with ages out

MobileGL/MG_Backend/DirectVulkan/Renderer/MagmaProgramSource.h

@@ -53,24 +53,26 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         // The view has no lifetime of its own, so the memo lives in the server record it views
         // (MGPipeShaderCsoRecord::BackendHashMemo), never in the client's program object: an entry
         // answers only while the record's Serial and BindingsSerial are the ones it was taken at.
-        Bool GetBackendHashMemo(Uint flags, Uint64& hash) const {
-            for (const auto& entry : m_record->BackendHashMemo) {
-                if (entry.Valid && entry.Key == flags && entry.Serial == m_record->Serial &&
-                    entry.BindingsSerial == m_record->BindingsSerial) {
-                    hash = entry.Hash;
-                    return true;
-                }
+        using BackendHashMemoEntry = MG_Pipe::MGPipeShaderCsoRecord::BackendHashMemoEntry;
+        BackendHashMemoEntry* FindBackendHashMemo(Uint key) const {
+            for (auto& entry : m_record->BackendHashMemo) {
+                if (entry.Valid && entry.Key == key && entry.Serial == m_record->Serial &&
+                    entry.BindingsSerial == m_record->BindingsSerial)
+                    return &entry;
             }
-            return false;
+            return nullptr;
         }
-        void SetBackendHashMemo(Uint flags, Uint64 hash) const {
+        // Takes the next slot round robin; its LastVariantHash starts as the plain `hash`.
+        BackendHashMemoEntry& SetBackendHashMemo(Uint key, Uint64 hash) const {
             auto& memo = m_record->BackendHashMemo;
             auto& entry = memo[m_record->BackendHashMemoNext++ % memo.size()];
             entry.Serial = m_record->Serial;
             entry.BindingsSerial = m_record->BindingsSerial;
             entry.Hash = hash;
-            entry.Key = flags;
+            entry.LastVariantHash = hash;
+            entry.Key = key;
             entry.Valid = true;
+            return entry;
         }
         Uint64 GetBackendStateVersion() const { return m_record->Serial; }
         Uint64 GetBlockBindingVersion() const { return m_record->BindingsSerial; }

MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp

@@ -3626,9 +3626,8 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         }
     } // namespace
 
-    const ProgramFactory::VkProgramObject& ProgramFactory::GetOrCreateProgram(
-        const MagmaProgramSource& program, CompileOptionFlags flags,
-        const NarrowIntegerVertexInputs& narrowIntegerInputs) {
+    MagmaProgramSource::BackendHashMemoEntry& ProgramFactory::ProgramHashMemo(const MagmaProgramSource& program,
+                                                                              CompileOptionFlags flags) const {
         // Hashing the full SPIR-V of every stage is far too expensive to repeat per draw;
         // reuse the program's memoized hash while its backend state version is unchanged.
         // The memo keys on the flags word, which ComputeHash is no longer a pure function of:
@@ -3644,19 +3643,46 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         const Uint memoKey = (flags & CompileOptionBit::FragCoordYFlip)
                                  ? (flags.GetRaw() | (m_defaultFramebufferHeight << 16))
                                  : flags.GetRaw();
-        HashType hash = 0;
-        if (!program.GetBackendHashMemo(memoKey, hash)) {
-            hash = ComputeHash(program, flags);
-            program.SetBackendHashMemo(memoKey, hash);
-        }
+        if (auto* memo = program.FindBackendHashMemo(memoKey)) return *memo;
+        return program.SetBackendHashMemo(memoKey, ComputeHash(program, flags));
+    }
+
+    const ProgramFactory::VkProgramObject& ProgramFactory::GetOrCreateProgram(
+        const MagmaProgramSource& program, CompileOptionFlags flags,
+        const NarrowIntegerVertexInputs& narrowIntegerInputs) {
+        auto& memo = ProgramHashMemo(program, flags);
         // Mixed in after the memo, which keys on the flags word alone: the memo keeps answering
         // for the plain program, and the variant costs one 16-byte hash on top of it.
+        HashType hash = memo.Hash;
         if (narrowIntegerInputs.Any()) hash = MixNarrowIntegerVertexInputs(hash, narrowIntegerInputs);
+        memo.LastVariantHash = hash;
+        return FindOrBuildProgram(program, flags, narrowIntegerInputs, hash);
+    }
+
+    const ProgramFactory::VkProgramObject& ProgramFactory::GetOrCreateDrawProgram(const MagmaProgramSource& program,
+                                                                                  CompileOptionFlags flags) {
+        auto& memo = ProgramHashMemo(program, flags);
+        if (memo.LastVariantHash != memo.Hash) {
+            const auto it = m_cache.find(memo.LastVariantHash);
+            if (it != m_cache.end()) {
+                it->second.lastUsedFrame = m_frameCounter;
+                return it->second;
+            }
+            // The variant aged out, and its hash alone cannot rebuild it: start from the plain
+            // program, and the draw settles on its variant again.
+            memo.LastVariantHash = memo.Hash;
+        }
+        return FindOrBuildProgram(program, flags, {}, memo.Hash);
+    }
+
+    const ProgramFactory::VkProgramObject& ProgramFactory::FindOrBuildProgram(
+        const MagmaProgramSource& program, CompileOptionFlags flags,
+        const NarrowIntegerVertexInputs& narrowIntegerInputs, HashType hash) {
         auto it = m_cache.find(hash);
         if (it != m_cache.end()) {
-            // Every draw/dispatch funnels through this lookup (the renderer memos only
-            // skip re-hashing, never the factory lookup), so an actively-used entry is
-            // stamped at least once per frame boundary and can never be aged out while
+            // Every draw/dispatch funnels through this lookup or GetOrCreateDrawProgram's (the
+            // renderer memos only skip re-hashing, never the factory lookup), so an actively-used
+            // entry is stamped at least once per frame boundary and can never be aged out while
             // any in-flight command buffer still references it.
             it->second.lastUsedFrame = m_frameCounter;
             return it->second;
@@ -3667,6 +3693,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         ++m_cacheStructureEpoch;
         auto& entry = m_cache[hash];
         entry.hash = hash;
+        entry.narrowIntegerInputs = narrowIntegerInputs;
         entry.lastUsedFrame = m_frameCounter;
         // The EXECUTABLE's stage list, not GetAttachedShaders(): `spirv` is a link artifact with
         // one module per linked stage, while the attach list is live and grows on

MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.h

@@ -204,6 +204,9 @@ namespace MobileGL::MG_Backend::DirectVulkan {
             // The locations of vertexInputTypes that are int / uint, for the per-draw narrow-integer
             // variant decision (ResolveNarrowIntegerVertexInputs). Identical in every variant.
             IntegerVertexInputMasks integerVertexInputs;
+            // The narrow-integer correction this entry's vertex stage was built with (none for the
+            // plain program): what a draw compares its own resolved set against.
+            NarrowIntegerVertexInputs narrowIntegerInputs;
             Uint32 activeFragmentOutputLocationMask = 0;
             Array<GLenum, kMaxVertexInputLocations> fragmentOutputTypes{};
             ShaderStage rasterizationProducerStage = ShaderStage::Unknown;
@@ -333,6 +336,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
                 activeVertexInputLocationMask = other.activeVertexInputLocationMask;
                 vertexInputTypes = other.vertexInputTypes;
                 integerVertexInputs = other.integerVertexInputs;
+                narrowIntegerInputs = other.narrowIntegerInputs;
                 activeFragmentOutputLocationMask = other.activeFragmentOutputLocationMask;
                 fragmentOutputTypes = other.fragmentOutputTypes;
                 rasterizationProducerStage = other.rasterizationProducerStage;
@@ -357,6 +361,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
                 other.globalUboBinding = -1;
                 other.activeVertexInputLocationMask = 0;
                 other.integerVertexInputs = {};
+                other.narrowIntegerInputs = {};
                 other.activeFragmentOutputLocationMask = 0;
                 other.rasterizationProducerStage = ShaderStage::Unknown;
                 other.producerOutputComponentCount = 0;
@@ -409,6 +414,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
                 activeVertexInputLocationMask = other.activeVertexInputLocationMask;
                 vertexInputTypes = other.vertexInputTypes;
                 integerVertexInputs = other.integerVertexInputs;
+                narrowIntegerInputs = other.narrowIntegerInputs;
                 activeFragmentOutputLocationMask = other.activeFragmentOutputLocationMask;
                 fragmentOutputTypes = other.fragmentOutputTypes;
                 rasterizationProducerStage = other.rasterizationProducerStage;
@@ -433,6 +439,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
                 other.globalUboBinding = -1;
                 other.activeVertexInputLocationMask = 0;
                 other.integerVertexInputs = {};
+                other.narrowIntegerInputs = {};
                 other.activeFragmentOutputLocationMask = 0;
                 other.rasterizationProducerStage = ShaderStage::Unknown;
                 other.producerOutputComponentCount = 0;
@@ -529,9 +536,15 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         // `narrowIntegerInputs` selects the variant whose vertex stage widens narrow integer
         // attribute data GL's way (ExtendNarrowIntegerVertexInputsPass). It is draw-time state like
         // the flags, and the draw path asks for it only when ResolveNarrowIntegerVertexInputs finds
-        // a location that needs it, so every other program keeps a single entry.
+        // a location that needs it, so every other program keeps a single entry. The variant
+        // returned is remembered per flags for GetOrCreateDrawProgram.
         const VkProgramObject& GetOrCreateProgram(const MagmaProgramSource& program, CompileOptionFlags flags,
                                                   const NarrowIntegerVertexInputs& narrowIntegerInputs = {});
+        // A draw's first lookup: the variant GetOrCreateProgram last returned for these flags, plain
+        // or narrow-integer. Every variant reflects the same vertex inputs and builtins, which is all
+        // a draw reads before it knows the variant it needs, so a program whose draws keep needing
+        // one variant costs a single lookup and its plain entry, never drawn with, ages out.
+        const VkProgramObject& GetOrCreateDrawProgram(const MagmaProgramSource& program, CompileOptionFlags flags);
         // The cache identity of a narrow-integer variant: the hash of the same program and flags
         // without it, mixed with the corrected locations. Content in, content out, so the variant's
         // pipelines key apart from the plain program's through programHash alone.
@@ -673,6 +686,13 @@ namespace MobileGL::MG_Backend::DirectVulkan {
             HashType hash = 0;
         };
 
+        // The program's memo slot for `flags`, its plain hash computed on a miss.
+        MagmaProgramSource::BackendHashMemoEntry& ProgramHashMemo(const MagmaProgramSource& program,
+                                                                  CompileOptionFlags flags) const;
+        // The cache entry under `hash`, built as that variant of `program` when absent.
+        const VkProgramObject& FindOrBuildProgram(const MagmaProgramSource& program, CompileOptionFlags flags,
+                                                  const NarrowIntegerVertexInputs& narrowIntegerInputs, HashType hash);
+
         static TextureTarget UniformTypeToTextureTarget(GLenum glType);
         // `stages` is ALWAYS ProgramObject::GetLinkedShaderStages() - one entry per module of
         // `spirv`, at the same index. Taking the stages rather than the shader objects is what

MobileGL/MG_Backend/DirectVulkan/Renderer/WireDraw.cpp

@@ -734,10 +734,12 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         if (m_transformFeedbackFeatureEnabled && MG_Pipe::gPipeInputs->IsTransformFeedbackActive() &&
             program.GetTransformFeedbackVaryingCount() > 0)
             flags |= ProgramFactory::CompileOptionBit::XfbCapture;
-        const auto* programObject = &m_programFactory->GetOrCreateProgram(program, flags);
+        // The variant this program's last draw with these flags settled on: what the draw reads
+        // from it before the narrow-integer pick below is the same in every variant.
+        const auto* programObject = &m_programFactory->GetOrCreateDrawProgram(program, flags);
         if (!(aspects & DrawSetupAspect::IndexBuffer) && programObject->readsBaseVertexBuiltin) {
             flags |= ProgramFactory::CompileOptionBit::ZeroBaseVertex;
-            programObject = &m_programFactory->GetOrCreateProgram(program, flags);
+            programObject = &m_programFactory->GetOrCreateDrawProgram(program, flags);
         }
         const auto vaoHandle = state.BoundVertexElements;
         if (MG_Pipe::MGPipeHandleIsNull(vaoHandle)) MagmaWireFatal("vertex-elements-record");
@@ -770,11 +772,12 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         const auto& input = *resolvedInput;
         // The narrow-integer variant (NarrowIntegerVertexFormats.h), chosen from two masks cached on
         // the program entry and on the resolved layout: a program without integer vertex inputs pays
-        // four ANDs here. Every variant reflects the same inputs, so the layout resolved above with
-        // the plain program's active mask is this variant's too.
+        // four ANDs and a compare here. Every variant reflects the same inputs, so the layout
+        // resolved above with the entry's active mask is the chosen variant's too. A second lookup
+        // only when this draw needs another variant than the last one with these flags.
         const NarrowIntegerVertexInputs narrowIntegerInputs =
             ResolveNarrowIntegerVertexInputs(programObject->integerVertexInputs, input.narrowIntegerAttributes);
-        if (narrowIntegerInputs.Any())
+        if (narrowIntegerInputs != programObject->narrowIntegerInputs)
             programObject = &m_programFactory->GetOrCreateProgram(program, flags, narrowIntegerInputs);
         if (programObject->invalidSpirvDeclined) {
             MGL_WIRE_DECLINE_AT(ProgramInvalidSpirv,

MobileGL/MG_Pipe/PipeApply.h

@@ -396,6 +396,10 @@ namespace MobileGL::MG_Pipe {
             Uint64 Serial = 0;
             Uint64 BindingsSerial = 0;
             Uint64 Hash = 0;
+            // The hash of the variant of this program the factory last handed out for this key:
+            // Hash itself, or a variant keyed on draw-time vertex data. Derived from Hash, so it
+            // is valid exactly as long.
+            Uint64 LastVariantHash = 0;
             Uint32 Key = 0;
             Bool Valid = false;
         };

MobileGL/MG_Test/Pipeline/NarrowIntegerVertexFormatsTest.cpp

@@ -19,6 +19,7 @@
 using namespace MobileGL;
 using MobileGL::MG_Backend::DirectVulkan::ClassifyNarrowIntegerVertexFormat;
 using MobileGL::MG_Backend::DirectVulkan::IntegerVertexInputMasks;
+using MobileGL::MG_Backend::DirectVulkan::MagmaProgramSource;
 using MobileGL::MG_Backend::DirectVulkan::NarrowIntegerAttributeMasks;
 using MobileGL::MG_Backend::DirectVulkan::NarrowIntegerVertexFormat;
 using MobileGL::MG_Backend::DirectVulkan::NarrowIntegerVertexInputs;
@@ -135,23 +136,61 @@ TEST(NarrowIntegerVertexFormats, VariantHashKeysEveryCorrectionApart) {
               hash);
 }
 
-// The program half lives on the cache entry, and the cache's robin-hood insert moves entries
-// around: a mask left behind by a move would pick another program's variant.
+// The program half and the entry's own correction live on the cache entry, and the cache's
+// robin-hood insert moves entries around: a mask left behind by a move would pick another
+// program's variant, and a correction left behind would make a draw take a variant for the plain
+// program or the reverse.
 TEST(NarrowIntegerVertexFormats, IntegerInputMasksTravelWithTheProgramEntry) {
+    const NarrowIntegerVertexInputs sodium = Only(&NarrowIntegerVertexInputs::zeroExtend8, 3);
     ProgramFactory::VkProgramObject source;
     source.integerVertexInputs.signedInputs = 1u << 3;
     source.integerVertexInputs.unsignedInputs = 1u << 7;
+    source.narrowIntegerInputs = sodium;
 
     ProgramFactory::VkProgramObject moved(std::move(source));
     EXPECT_EQ(moved.integerVertexInputs.signedInputs, 1u << 3);
     EXPECT_EQ(moved.integerVertexInputs.unsignedInputs, 1u << 7);
+    EXPECT_EQ(moved.narrowIntegerInputs, sodium);
     EXPECT_EQ(source.integerVertexInputs.signedInputs, 0u);
     EXPECT_EQ(source.integerVertexInputs.unsignedInputs, 0u);
+    EXPECT_FALSE(source.narrowIntegerInputs.Any());
 
     ProgramFactory::VkProgramObject assigned;
     assigned = std::move(moved);
     EXPECT_EQ(assigned.integerVertexInputs.signedInputs, 1u << 3);
     EXPECT_EQ(assigned.integerVertexInputs.unsignedInputs, 1u << 7);
+    EXPECT_EQ(assigned.narrowIntegerInputs, sodium);
     EXPECT_EQ(moved.integerVertexInputs.signedInputs, 0u);
     EXPECT_EQ(moved.integerVertexInputs.unsignedInputs, 0u);
+    EXPECT_FALSE(moved.narrowIntegerInputs.Any());
+}
+
+// A draw starts from the variant the program's last lookup with the same flags returned, which
+// the program record keeps beside the plain hash it derives from. A fresh slot must point at the
+// plain program, and the variant must be forgotten with the hash: after a relink or a binding
+// change the same flags answer from a fresh slot.
+TEST(NarrowIntegerVertexFormats, TheLastVariantIsRememberedOnlyWithItsPlainHash) {
+    MG_Pipe::MGPipeShaderCsoRecord record{};
+    record.Archive = MakeShared<MG_State::GLState::ProgramArchive>();
+    record.Serial = 4;
+    record.BindingsSerial = 9;
+    const MagmaProgramSource source({5, 1}, record);
+    constexpr Uint kFlags = 0x21;
+
+    EXPECT_EQ(source.FindBackendHashMemo(kFlags), nullptr);
+    auto& slot = source.SetBackendHashMemo(kFlags, 0xabcull);
+    EXPECT_EQ(slot.LastVariantHash, 0xabcull) << "a fresh slot starts at the plain program";
+    slot.LastVariantHash = 0xdefull;
+    const auto* found = source.FindBackendHashMemo(kFlags);
+    ASSERT_EQ(found, &slot);
+    EXPECT_EQ(found->Hash, 0xabcull);
+    EXPECT_EQ(found->LastVariantHash, 0xdefull);
+    EXPECT_EQ(source.FindBackendHashMemo(kFlags + 1), nullptr) << "other flags have their own slot";
+
+    record.Serial = 5; // a relink
+    EXPECT_EQ(source.FindBackendHashMemo(kFlags), nullptr);
+    EXPECT_EQ(source.SetBackendHashMemo(kFlags, 0x123ull).LastVariantHash, 0x123ull);
+    ASSERT_NE(source.FindBackendHashMemo(kFlags), nullptr);
+    record.BindingsSerial = 10; // glUniformBlockBinding and friends
+    EXPECT_EQ(source.FindBackendHashMemo(kFlags), nullptr);
 }