[Perf] (MG_Pipe): inline process applier, a precomputed residual-fill walk, word-wise uniform spans and memo-answered draw bindings (rd12 monolith Espryt 83.5 -> 88.6, Magma 67.0 -> 69.5 fps).

MobileGL/MG_Impl/Pipe/PipeFill.cpp

@@ -2921,6 +2921,17 @@ namespace MobileGL::MG_Pipe {
             // exactly as it always was, so "does this verb read this field" stays the walk's
             // business and this stays a statement about EMISSION alone.
             MGPipeFieldMask Supplied{};
+            // P14: the walk itself, per verb class, built with Supplied: the non-sticky fields the
+            // class reads (bit 7 set = copy it, clear = the emitted call supplies it), and the
+            // sticky ones. The walk then visits only the ~47 fields a draw reads instead of testing
+            // all 63 against three masks per verb.
+            static constexpr Uint8 kCopy = 0x80;
+            static_assert(kMGPipeInputFieldCount < kCopy, "field indices must leave the copy bit free");
+            static constexpr SizeT kClasses = static_cast<SizeT>(MGPipeVerbClass::kClassCount);
+            Array<Array<Uint8, kMGPipeInputFieldCount>, kClasses> Walk{};
+            Array<Uint8, kClasses> WalkCount{};
+            Array<Array<Uint8, kMGPipeInputFieldCount>, kClasses> Sticky{};
+            Array<Uint8, kClasses> StickyCount{};
         };
         ResidualFillPlan g_fillPlan;
 
@@ -2993,6 +3004,23 @@ namespace MobileGL::MG_Pipe {
                 MG_Remote::Client::CapsMirrorInstance().Generation() : 0;
 #endif
             g_fillPlan.Supplied = built;
+            for (SizeT c = 0; c < ResidualFillPlan::kClasses; ++c) {
+                const MGPipeFieldMask& classMask = kMGPipeClassFieldMask[c];
+                Uint8 walk = 0;
+                Uint8 sticky = 0;
+                for (SizeT i = 0; i < kMGPipeInputFieldCount; ++i) {
+                    const auto field = static_cast<MGPipeInputField>(i);
+                    if (!MGPipeFieldMaskHas(classMask, field)) continue;
+                    if (kMGPipeInputFieldSticky[i]) {
+                        g_fillPlan.Sticky[c][sticky++] = static_cast<Uint8>(i);
+                        continue;
+                    }
+                    g_fillPlan.Walk[c][walk++] = static_cast<Uint8>(
+                        i | (MGPipeFieldMaskHas(built, field) ? 0 : ResidualFillPlan::kCopy));
+                }
+                g_fillPlan.WalkCount[c] = walk;
+                g_fillPlan.StickyCount[c] = sticky;
+            }
             return built;
         }
 
@@ -3823,25 +3851,29 @@ namespace MobileGL::MG_Pipe {
         const Bool applierDerives = ApplierDerivesRenderStateFields();
         // BY VALUE, NOT BY REFERENCE: the memo's storage is rebuilt in place when the key moves,
         // and the walk below holds this across 63 iterations.
-        const MGPipeFieldMask supplied =
+        (void)SuppliedFieldMask(pushMask, applierDerives, contextValuesWireLive);
-            SuppliedFieldMask(pushMask, applierDerives, contextValuesWireLive);
+        // The plan the call above just validated (or rebuilt) for this key: the class's fields in
-        for (SizeT i = 0; fillOwed && i < kMGPipeInputFieldCount; ++i) {
+        // index order, sticky ones apart - the same visits, in the same order, as testing all 63.
-            const auto field = static_cast<MGPipeInputField>(i);
+        const SizeT planClass = static_cast<SizeT>(verbClass);
-            if (!MGPipeFieldMaskHas(mask, field)) continue;
+        (void)mask;
 #if MOBILEGL_PIPE_POISON
-            if (kMGPipeInputFieldSticky[i]) {
+        for (SizeT k = 0; fillOwed && k < g_fillPlan.StickyCount[planClass]; ++k) {
-                // Stamped once by the first fill that sees a live context; fresh through the
+            // Stamped once by the first fill that sees a live context; fresh through the
-                // Sticky -> FilledGen != 0 branch of MGPipeInputFieldIsFresh from then on.
+            // Sticky -> FilledGen != 0 branch of MGPipeInputFieldIsFresh from then on.
-                if (filled.FilledGen[i] == 0) filled.FilledGen[i] = 1;
+            const SizeT i = g_fillPlan.Sticky[planClass][k];
-                continue;
+            if (filled.FilledGen[i] == 0) filled.FilledGen[i] = 1;
-            }
+        }
-#else
-            if (kMGPipeInputFieldSticky[i]) continue;
 #endif
-            if (!MGPipeFieldMaskHas(supplied, field)) MGPipeFillAccess::CopyField(inputs, *ctx, field);
+        for (SizeT k = 0; fillOwed && k < g_fillPlan.WalkCount[planClass]; ++k) {
+            const Uint8 entry = g_fillPlan.Walk[planClass][k];
+            const SizeT i = entry & ~ResidualFillPlan::kCopy;
+            const auto field = static_cast<MGPipeInputField>(i);
+            if (entry & ResidualFillPlan::kCopy) MGPipeFillAccess::CopyField(inputs, *ctx, field);
 #if MOBILEGL_PIPE_POISON
             // The value is copied either way; only the stamp is withheld for the omitted pair.
             if (!IsOmitted(verb, field)) filled.FilledGen[i] = filled.CurrentVerbSerial;
+#else
+            (void)field;
 #endif
         }
         if (payloadBytes != 0 && MG_Util::PipeStats::Enabled()) {

MobileGL/MG_Impl/Pipe/ProgramEmit.h

@@ -223,10 +223,19 @@ namespace MobileGL::MG_Pipe {
             // span that moved goes out - when it is small enough to be worth it.
             const Uint8* image = static_cast<const Uint8*>(program->GetUBOData());
             if (cso == m_constantsCso && m_constantsShadow.size() == size && size <= 0xffffu) {
+                // Word-wise from both ends, then byte-wise inside the first differing word.
+                const Uint8* shadow = m_constantsShadow.data();
+                const auto word = [](const Uint8* at) {
+                    Uint64 value;
+                    std::memcpy(&value, at, sizeof(value));
+                    return value;
+                };
                 Uint32 first = 0;
-                while (first < size && image[first] == m_constantsShadow[first]) ++first;
+                while (first + 8 <= size && word(image + first) == word(shadow + first)) first += 8;
+                while (first < size && image[first] == shadow[first]) ++first;
                 Uint32 last = size;
-                while (last > first && image[last - 1] == m_constantsShadow[last - 1]) --last;
+                while (last >= first + 8 && word(image + last - 8) == word(shadow + last - 8)) last -= 8;
+                while (last > first && image[last - 1] == shadow[last - 1]) --last;
                 // Nothing moved but the version: one byte carries the new version.
                 if (first == size) {
                     first = 0;

MobileGL/MG_Impl/Pipe/Verb/VerbPort.cpp

@@ -630,9 +630,13 @@ namespace MobileGL::MG_Record {
             MG_State::GLState::GLContext* ctx = MG_State::pGLContext.get();
             if (ctx == nullptr) return b;
             if (const auto& vao = ctx->GetBoundVertexArray()) {
+                auto& emitter = MG_Pipe::MGPipeVertexInputEmitterInstance();
                 if (const auto& bound = vao->GetIndexBufferBindingSlot().GetBoundObject()) {
                     b.ElementBufferBound = true;
-                    b.ElementBuffer = MG_Pipe::MGPipeResourceTrackerInstance().Find(*bound);
+                    // The validate point's emitter resolved this element buffer already (P14).
+                    b.ElementBuffer = emitter.KnownIndexHandle(*vao, bound->GetLifetimeId());
+                    if (MG_Pipe::MGPipeHandleIsNull(b.ElementBuffer))
+                        b.ElementBuffer = MG_Pipe::MGPipeResourceTrackerInstance().Find(*bound);
                 }
                 // P5e (vi): the client-array probe, in the SAME single read of the bindings the
                 // rest of the plan is made from rather than in a second walk at the refusal -
@@ -640,11 +644,15 @@ namespace MobileGL::MG_Record {
                 // frontend pass. The test is the emitter's own: EmitVertexBuffers publishes
                 // Res == kMGPipeNullHandle for exactly `attrib.Enabled && !attrib.Buffer`, so
                 // the flag and the record agree by construction instead of by inspection.
-                const auto& attributes = vao->GetAllAttributes();
+                // A memoised buffer set has no client array in it (P14): the 32-attribute walk only
-                for (SizeT i = 0; i < attributes.size(); ++i) {
+                // runs when the memo does not know.
-                    if (attributes[i].Enabled && !attributes[i].Buffer) {
+                if (!emitter.KnownWithoutClientArrays(*vao)) {
-                        b.ClientVertexArrays = true;
+                    const auto& attributes = vao->GetAllAttributes();
-                        break;
+                    for (SizeT i = 0; i < attributes.size(); ++i) {
+                        if (attributes[i].Enabled && !attributes[i].Buffer) {
+                            b.ClientVertexArrays = true;
+                            break;
+                        }
                     }
                 }
             }

MobileGL/MG_Impl/Pipe/VertexInputEmit.h

@@ -352,6 +352,19 @@ namespace MobileGL::MG_Pipe {
             return sizeof(MGPIndexBuffer);
         }
 
+        // P14: two per-draw questions of the verb port (VerbPort.cpp's ReadDrawBindings), answered
+        // from the per-VAO memo when it knows: whether no enabled attribute is a client array (a
+        // memoised buffer set is exactly that, for the configuration version it was built at), and
+        // the handle of an element buffer it already resolved. False / null mean "ask the frontend".
+        Bool KnownWithoutClientArrays(const VertexArrayObject& vao) {
+            const VaoMemo& memo = MemoFor(vao);
+            return memo.BuffersValid && memo.ConfigVersion == vao.GetConfigVersion();
+        }
+        MGPipeHandle KnownIndexHandle(const VertexArrayObject& vao, Uint64 indexLifetimeId) {
+            const VaoMemo& memo = MemoFor(vao);
+            return memo.IndexLifetimeId == indexLifetimeId ? memo.IndexHandle : kMGPipeNullHandle;
+        }
+
         // A set_index_buffer this emitter did not send (OwnedDrawInputs' private element
         // buffer): the server's binding and the bound array's kept one are no longer known.
         void NoteIndexBufferRoutedElsewhere(GLContext& ctx) {
@@ -477,11 +490,17 @@ namespace MobileGL::MG_Pipe {
         VaoMemo& MemoFor(const VertexArrayObject& vao) {
             const Uint64 lifetimeId = vao.GetLifetimeId();
             const Uint64 epoch = MGPipeSlots().ResetEpoch();
+            // The three emitters of one validate ask for the same array: the last answer's index,
+            // believed only while that entry still holds this id (a grow or an erase moves entries).
+            if (m_lastMemoIndex < m_vaoMemo.size() && m_vaoMemo[m_lastMemoIndex].LifetimeId == lifetimeId &&
+                m_vaoMemo[m_lastMemoIndex].SlotEpoch == epoch)
+                return m_vaoMemo[m_lastMemoIndex];
             if (m_vaoMemo.empty() || (m_vaoMemoUsed + 1) * 2 > m_vaoMemo.size()) GrowVaoMemo();
             const SizeT mask = m_vaoMemo.size() - 1;
             SizeT i = VaoMemoHome(lifetimeId, mask);
             while (m_vaoMemo[i].LifetimeId != 0 && m_vaoMemo[i].LifetimeId != lifetimeId) i = (i + 1) & mask;
             VaoMemo& memo = m_vaoMemo[i];
+            m_lastMemoIndex = i;
             if (memo.LifetimeId == 0) ++m_vaoMemoUsed;
             if (memo.LifetimeId != lifetimeId || memo.SlotEpoch != epoch) {
                 memo = VaoMemo{};
@@ -509,6 +528,7 @@ namespace MobileGL::MG_Pipe {
         }
         Vector<VaoMemo> m_vaoMemo;
         SizeT m_vaoMemoUsed = 0;
+        SizeT m_lastMemoIndex = ~SizeT{0};
 
     public:
         // The VAO died (MGPipeEmitVertexElementsDestroyAndFree): its entry goes, by backward-shift

MobileGL/MG_Pipe/PipeApply.cpp

@@ -793,7 +793,12 @@ namespace MobileGL::MG_Pipe {
         if (const auto hook = g_compileEnvChangedHook.load(std::memory_order_acquire)) hook();
     }
 
-    MGPipeApplierState& MGPipeApplier() {
+    namespace Detail {
+        void* g_applierResolverInstalled = nullptr;
+        MGPipeApplierState* g_processApplier = &g_defaultApplier;
+    } // namespace Detail
+
+    MGPipeApplierState& MGPipeApplierForSession() {
         const MGPipeApplierKeyResolver resolve = g_applierKeyResolver;
         if (resolve == nullptr) return g_defaultApplier;
         MGPipeApplierKey key;
@@ -815,7 +820,10 @@ namespace MobileGL::MG_Pipe {
         return *applier;
     }
 
-    void MGPipeSetApplierKeyResolver(MGPipeApplierKeyResolver resolver) { g_applierKeyResolver = resolver; }
+    void MGPipeSetApplierKeyResolver(MGPipeApplierKeyResolver resolver) {
+        g_applierKeyResolver = resolver;
+        Detail::g_applierResolverInstalled = reinterpret_cast<void*>(resolver);
+    }
 
     void MGPipeApplierRegisterContext(Uint64 sessionKey, Uint64 contextToken, Uint64 shareGroupToken) {
         std::lock_guard<std::mutex> guard(g_applierSessionsMutex);
@@ -2979,12 +2987,21 @@ namespace MobileGL::MG_Pipe {
         MGPipeApplier().BoundVertexElements = handle.Handle;
         // The array's kept bindings come back with it, through the same two applies their
         // records take, so every serial and field moves exactly as it would for the records.
-        // (The apply writes the same entries back over themselves: harmless, and no copy.)
+        // The window copy and the serial bumps are the two applies' own (the kept set passed
+        // their gates when it arrived); only the store-back into this same record is skipped.
+        auto& applier = MGPipeApplier();
         if (record->StoredBuffersValid) {
-            const MGPVertexBuffers header = record->StoredBuffersHeader;
+            const MGPVertexBuffers& header = record->StoredBuffersHeader;
-            MGPipeApplySetVertexBuffers(header, record->StoredBuffers.data());
+            for (Uint32 i = 0; i < header.Count; ++i) applier.VertexBuffers[header.Start + i] = record->StoredBuffers[i];
+            applier.VertexBufferStart = header.Start;
+            applier.VertexBufferCount = header.Count;
+            applier.VertexFetchBaseInstance = header.BaseInstance;
+            ++applier.VertexBuffersSerial;
+        }
+        if (record->StoredIndexValid) {
+            applier.IndexBuffer = record->StoredIndex;
+            ++applier.IndexBufferSerial;
         }
-        if (record->StoredIndexValid) MGPipeApplySetIndexBuffer(record->StoredIndex);
     }
 
     void MGPipeApplyDeleteVertexElements(const MGPHandleOnly& handle) {

MobileGL/MG_Pipe/PipeApply.h

@@ -1057,7 +1057,21 @@ namespace MobileGL::MG_Pipe {
     // every other thread - the client process, the in-process client+server shape, a unit case,
     // the whole monolith - it is the process-wide applier, which is what it has always been and
     // is byte-for-byte the old behaviour.
-    MGPipeApplierState& MGPipeApplier();
+    //
+    // P14: INLINE, because the backends ask it per state read (dozens of times per draw): in a
+    // process that serves no session - the whole monolith - no resolver is installed and the
+    // answer is the process-wide applier, a load and a branch with no call.
+    namespace Detail {
+        // Mirrors of PipeApply.cpp's g_applierKeyResolver / g_defaultApplier, written only there.
+        extern void* g_applierResolverInstalled;
+        extern MGPipeApplierState* g_processApplier;
+    } // namespace Detail
+    MGPipeApplierState& MGPipeApplierForSession();
+    inline MGPipeApplierState& MGPipeApplier() {
+        if (Detail::g_applierResolverInstalled == nullptr && Detail::g_processApplier != nullptr)
+            return *Detail::g_processApplier;
+        return MGPipeApplierForSession();
+    }
 
     // P13 W6: the backend's "my capabilities changed" notice, without naming the frontend. A
     // backend whose advertised limits / extensions move after a context exists (Magma's caps