diff --git a/MobileGL/Config.h b/MobileGL/Config.h index efc13c73..f21f2b87 100644 --- a/MobileGL/Config.h +++ b/MobileGL/Config.h @@ -481,6 +481,16 @@ namespace MobileGL::MG_Config { // 0 IS A NEGATIVE CONTROL, NOT "unlimited": it disables the push, and // PersistentCoherentMapScenario must go RED under it (exit gate E3(a)). Uint32 PersistentBlockKb = 64; + // MOBILEGL_IPC_PERSISTENT_HASH_SUPPRESS: 1 = the persistent-map push ships only + // blocks whose xxHash64 changed since the last push, instead of the whole mapped + // range every verb. 0 restores the whole-range push (A/B control). + Uint32 PersistentHashSuppress = 1; + // MOBILEGL_IPC_BATCH_WAITS: 1 = value-class records (kCtxState / kCtxCso / kCtxObject + // with no reply slot) are published without waiting for their own apply; the barrier + // is taken at the next pull-reading verb (kCtxVerb syncs, queries, screen rows, and + // every reply-slot row), which is the only place BARRIER-PULLED fields are read. 0 + // restores the per-record barrier of R-1. + Uint32 BatchWaits = 1; // MOBILEGL_IPC_ADOPT_TIER: 2 = emulate (client keeps the shadow and pushes), which // is the only tier P5 implements and the reason persistent-map-push can be non-zero // at all (R-6). 0 and 1 parse and are Fatal at use with "P11"; they exist now so the diff --git a/MobileGL/ConfigLoader.cpp b/MobileGL/ConfigLoader.cpp index 2ea22f02..2344901f 100644 --- a/MobileGL/ConfigLoader.cpp +++ b/MobileGL/ConfigLoader.cpp @@ -366,6 +366,15 @@ namespace MobileGL::MG_ConfigLoader { // 0 is admitted ON PURPOSE and is the negative control of exit gate E3(a): it turns // the persistent-map push OFF, and PersistentCoherentMapScenario must go red. ipc.PersistentBlockKb = QueryEnvUint32("MOBILEGL_IPC_PERSISTENT_BLOCK_KB", 64, 0, 65536); + // Whole-range push is the 0 arm; with it on (default) only blocks whose + // xxHash64 changed since the last push are shipped. + ipc.PersistentHashSuppress = QueryEnvUint32("MOBILEGL_IPC_PERSISTENT_HASH_SUPPRESS", 1, 0, 1); + ipc.BatchWaits = QueryEnvUint32("MOBILEGL_IPC_BATCH_WAITS", 1, 0, 1); + // The verify harness compares the pushed block against the applier per verb; a + // batched queue lets the comparer read a supplied field mid-apply, which is a + // torn read rather than a divergence. The batch is therefore off whenever the + // shadow comparer is armed. + if (MG_Config::Features.PipeVerify) ipc.BatchWaits = 0; // 2 is the only tier P5 implements (R-6). 0 and 1 parse here and are refused at the // point of use, which is where the "P11" in the message belongs. ipc.AdoptTier = QueryEnvUint32("MOBILEGL_IPC_ADOPT_TIER", 2, 0, 2); diff --git a/MobileGL/MG_Impl/Pipe/PipeFill.cpp b/MobileGL/MG_Impl/Pipe/PipeFill.cpp index 9dbb02fe..5a516af4 100644 --- a/MobileGL/MG_Impl/Pipe/PipeFill.cpp +++ b/MobileGL/MG_Impl/Pipe/PipeFill.cpp @@ -57,6 +57,7 @@ #endif #include +#include #include #include @@ -971,7 +972,79 @@ namespace MobileGL::MG_Pipe { return published; } +#if MOBILEGL_BUILD_DISAGGREGATED + namespace { + struct MGPipeDeferredDestroy { + MGPipeKind kind; + Uint64 lifetimeId; + }; + // A plain mutex, not a lock-free queue: producers are destructors that lost the + // last-reference race (rare), the consumer is the GL thread's verb hook, and neither + // holds the lock past a vector push/swap. The count is the fast no-work check for + // the per-verb drain. + std::mutex g_deferredDestroyMutex; + Vector g_deferredDestroys; + std::atomic g_deferredDestroyCount{0}; + } // namespace + + Bool MGPipeDeferDestroyAndFreeIfOnApplyThread(MGPipeKind kind, Uint64 lifetimeId) { + if (MG_Config::Transport == MG_Config::TransportMode::Monolith) return false; + if (!MG_Remote::Client::RunsAsTheServerRole()) return false; + { + std::lock_guard lock(g_deferredDestroyMutex); + g_deferredDestroys.push_back(MGPipeDeferredDestroy{kind, lifetimeId}); + } + g_deferredDestroyCount.fetch_add(1, std::memory_order_release); + return true; + } + + void MGPipeDrainDeferredDestroys() { + if (g_deferredDestroyCount.load(std::memory_order_acquire) == 0) return; + Vector drained; + { + std::lock_guard lock(g_deferredDestroyMutex); + drained.swap(g_deferredDestroys); + g_deferredDestroyCount.store(0, std::memory_order_release); + } + for (const MGPipeDeferredDestroy& one : drained) { + // Each replay lands back in the helper itself, on the GL thread this time, so + // the deferral branch passes and the helper's own order runs unchanged. + switch (one.kind) { + case MGPipeKind::VertexElementsCso: + MGPipeEmitVertexElementsDestroyAndFree(one.lifetimeId); + break; + case MGPipeKind::SamplerViewCso: + MGPipeEmitSamplerViewCsoDestroyAndFree(one.lifetimeId); + break; + case MGPipeKind::Texture: + MGPipeEmitTextureDestroyAndFree(one.lifetimeId); + break; + case MGPipeKind::Renderbuffer: + MGPipeEmitRenderbufferDestroyAndFree(one.lifetimeId); + break; + case MGPipeKind::Framebuffer: + MGPipeEmitFramebufferDestroyAndFree(one.lifetimeId); + break; + case MGPipeKind::SamplerCso: + MGPipeEmitSamplerCsoDestroyAndFree(one.lifetimeId); + break; + case MGPipeKind::ShaderCso: + MGPipeEmitShaderCsoDestroyAndFree(one.lifetimeId); + break; + default: + break; + } + } + } +#endif + Bool MGPipeEmitVertexElementsDestroyAndFree(Uint64 lifetimeId) { +#if MOBILEGL_BUILD_DISAGGREGATED + if (MGPipeDeferDestroyAndFreeIfOnApplyThread(MGPipeKind::VertexElementsCso, + lifetimeId)) { + return false; + } +#endif // C-1. THE SAME SHAPE AS MGPipeEmitResourceDestroyAndFree ABOVE, and for the same // reason: whatever mints a handle owns the death of that handle, and the mint for this // kind is MGPipeVertexInputEmitter::EmitVertexElements - i.e. the client, on every @@ -1585,6 +1658,11 @@ namespace MobileGL::MG_Pipe { // none per object (its death is the cache's LRU, ID-17). Unconditional in a push build, // like the mints: the entries exist whether or not the family bit is set. Bool MGPipeEmitSamplerViewCsoDestroyAndFree(Uint64 lifetimeId) { +#if MOBILEGL_BUILD_DISAGGREGATED + if (MGPipeDeferDestroyAndFreeIfOnApplyThread(MGPipeKind::SamplerViewCso, lifetimeId)) { + return false; + } +#endif const MGPipeHandle handle = MGPipeSlots().FindByLifetimeId(MGPipeKind::SamplerViewCso, lifetimeId); const Bool published = @@ -1607,6 +1685,11 @@ namespace MobileGL::MG_Pipe { } Bool MGPipeEmitTextureDestroyAndFree(Uint64 lifetimeId) { +#if MOBILEGL_BUILD_DISAGGREGATED + if (MGPipeDeferDestroyAndFreeIfOnApplyThread(MGPipeKind::Texture, lifetimeId)) { + return false; + } +#endif const MGPipeHandle handle = MGPipeSlots().FindByLifetimeId(MGPipeKind::Texture, lifetimeId); const Bool published = EmitDeleteIfPublished(MGPipeKind::Texture, handle, &MGPipeRouteResourceDestroy); @@ -1647,6 +1730,11 @@ namespace MobileGL::MG_Pipe { } Bool MGPipeEmitRenderbufferDestroyAndFree(Uint64 lifetimeId) { +#if MOBILEGL_BUILD_DISAGGREGATED + if (MGPipeDeferDestroyAndFreeIfOnApplyThread(MGPipeKind::Renderbuffer, lifetimeId)) { + return false; + } +#endif const MGPipeHandle handle = MGPipeSlots().FindByLifetimeId(MGPipeKind::Renderbuffer, lifetimeId); const Bool published = @@ -1658,6 +1746,11 @@ namespace MobileGL::MG_Pipe { } Bool MGPipeEmitFramebufferDestroyAndFree(Uint64 lifetimeId) { +#if MOBILEGL_BUILD_DISAGGREGATED + if (MGPipeDeferDestroyAndFreeIfOnApplyThread(MGPipeKind::Framebuffer, lifetimeId)) { + return false; + } +#endif // NO WIRE DELETE EXISTS FOR THIS KIND, and none is invented: PipeCalls.def has // resource_destroy and the five delete_* rows and no framebuffer delete, because a // framebuffer is not a resource and is not a CSO - it is STATE, and @@ -1684,6 +1777,11 @@ namespace MobileGL::MG_Pipe { } Bool MGPipeEmitSamplerCsoDestroyAndFree(Uint64 lifetimeId) { +#if MOBILEGL_BUILD_DISAGGREGATED + if (MGPipeDeferDestroyAndFreeIfOnApplyThread(MGPipeKind::SamplerCso, lifetimeId)) { + return false; + } +#endif const MGPipeHandle handle = MGPipeSlots().FindByLifetimeId(MGPipeKind::SamplerCso, lifetimeId); const Bool published = @@ -1698,6 +1796,11 @@ namespace MobileGL::MG_Pipe { } Bool MGPipeEmitShaderCsoDestroyAndFree(Uint64 lifetimeId) { +#if MOBILEGL_BUILD_DISAGGREGATED + if (MGPipeDeferDestroyAndFreeIfOnApplyThread(MGPipeKind::ShaderCso, lifetimeId)) { + return false; + } +#endif // ORDINARY PROGRAMS AND PIPELINE COMPOSITES TAKE THE SAME PATH, deliberately: the // server never learns a composite is a composite, and the only difference on this side // is which band the slot came out of. A composite's slot has TWO independent release diff --git a/MobileGL/MG_IntegrationTest/Scenarios/CtWireScenario.cpp b/MobileGL/MG_IntegrationTest/Scenarios/CtWireScenario.cpp index 1ac051b0..350026ba 100644 --- a/MobileGL/MG_IntegrationTest/Scenarios/CtWireScenario.cpp +++ b/MobileGL/MG_IntegrationTest/Scenarios/CtWireScenario.cpp @@ -192,6 +192,12 @@ namespace { glBindRenderbuffer(GL_RENDERBUFFER, 0); const MobileGL::Uint64 deathsBefore = ServerVerbs().ObjectDeaths(); glDeleteFramebuffers(1, &fbo); + // THE FENCE (MOBILEGL_IPC_BATCH_WAITS, default on): object_death is a kCtxObject + // value-class record - published WITHOUT waiting for its own apply (production-safe: + // the in-order ring applies it before any record that recycles the slot). The + // server-side counter this assertion reads only moves at apply time, so it needs a + // wait boundary: a clear is kCtxVerb and still waits, and its wait covers the death. + glClear(GL_COLOR_BUFFER_BIT); EXPECT_GT(ServerVerbs().ObjectDeaths(), deathsBefore) << "the framebuffer's death produced no object_death record - and no other " "opcode can carry it"; diff --git a/MobileGL/MG_Pipe/PipeMutation.h b/MobileGL/MG_Pipe/PipeMutation.h index 098b2a40..d367a4bf 100644 --- a/MobileGL/MG_Pipe/PipeMutation.h +++ b/MobileGL/MG_Pipe/PipeMutation.h @@ -213,6 +213,28 @@ namespace MobileGL::MG_Pipe { // proven no-op. Bool MGPipeEmitShaderCsoDestroyAndFree(Uint64 lifetimeId); +#if MOBILEGL_BUILD_DISAGGREGATED + // ---- THE DEFERRED DESTROY QUEUE (the MOBILEGL_IPC_BATCH_WAITS class) ---- + // + // Every helper above speaks on the CLIENT's behalf: it resolves the handle through the + // client allocator, emits a client wire record and frees a client slot - so it may only + // ever run on the GL thread. The per-record barrier made that free: the client waited + // out every apply, so a server-side SharedPtr (an applier's endpoint local, a + // gPipeInputs entry) could never be an object's last owner. Batched waits remove the + // guarantee - the client races ahead, drops its own references at teardown, and the + // apply thread's local CopyImageEndpoint becomes a texture's last owner, whose + // destructor then touches the client allocator from the server role + // (Fatal{RoleViolation, "MGPipeSlots"}). The helpers therefore START with + // MGPipeDeferDestroyAndFreeIfOnApplyThread: on the apply thread it enqueues the + // (kind, lifetime id) pair - the only thing a death announcement needs - and the GL + // thread replays the helper at the next verb hook (EmitTables' BeforeDrawVerb / + // BeforeReadOnlyVerb) through MGPipeDrainDeferredDestroys. A delayed free is safe: + // slots are plentiful, reuse is delayed rather than corrupted, and the replay runs the + // helper itself, so every helper's own three-step order is untouched. + Bool MGPipeDeferDestroyAndFreeIfOnApplyThread(MGPipeKind kind, Uint64 lifetimeId); + void MGPipeDrainDeferredDestroys(); +#endif + void MGPipeEmitResourceCreate(MG_State::GLState::BufferObject& buffer); void MGPipeEmitResourceRespecify(MG_State::GLState::BufferObject& buffer); void MGPipeEmitResourceSubData(MG_State::GLState::BufferObject& buffer, SizeT offset, SizeT size); diff --git a/MobileGL/MG_Pipe/generated/PipeWire.inc b/MobileGL/MG_Pipe/generated/PipeWire.inc index fd9ea6a0..6f14039c 100644 --- a/MobileGL/MG_Pipe/generated/PipeWire.inc +++ b/MobileGL/MG_Pipe/generated/PipeWire.inc @@ -265,6 +265,102 @@ inline constexpr Uint32 MGPipeCallFlagsFor(MGPWireOp op) { : static_cast(kNone); } +// The catalogue class of every opcode, indexed the same way; index 0 is kCallClassCount, +// never a real class, for the flags table's reason. +inline constexpr Uint8 kMGPipeCallClasses[static_cast(MGPWireOp::kOpCount)] = { + /* 0 */ static_cast(kCallClassCount), + + /* 1 GetCaps */ static_cast(kScreen), + /* 2 ResourceCreate */ static_cast(kScreen), + /* 3 ResourceRespecify */ static_cast(kScreen), + /* 4 ResourceDestroy */ static_cast(kScreen), + /* 5 MapPersistent */ static_cast(kScreen), + /* 6 UnmapPersistent */ static_cast(kScreen), + /* 7 FenceCreate */ static_cast(kScreen), + /* 8 FenceStatus */ static_cast(kScreen), + /* 9 FenceWait */ static_cast(kScreen), + /* 10 FenceDestroy */ static_cast(kScreen), + /* 11 QueryCreate */ static_cast(kCtxQuery), + /* 12 QueryBegin */ static_cast(kCtxQuery), + /* 13 QueryEnd */ static_cast(kCtxQuery), + /* 14 QueryAvailable */ static_cast(kCtxQuery), + /* 15 QueryResult */ static_cast(kCtxQuery), + /* 16 QueryDestroy */ static_cast(kCtxQuery), + /* 17 CreateRenderState */ static_cast(kCtxCso), + /* 18 BindRenderState */ static_cast(kCtxCso), + /* 19 DeleteRenderState */ static_cast(kCtxCso), + /* 20 CreateVertexElements */ static_cast(kCtxCso), + /* 21 BindVertexElements */ static_cast(kCtxCso), + /* 22 DeleteVertexElements */ static_cast(kCtxCso), + /* 23 CreateSamplerState */ static_cast(kCtxCso), + /* 24 DeleteSamplerState */ static_cast(kCtxCso), + /* 25 CreateSamplerView */ static_cast(kCtxCso), + /* 26 DeleteSamplerView */ static_cast(kCtxCso), + /* 27 CreateShaderState */ static_cast(kCtxCso), + /* 28 BindShaderState */ static_cast(kCtxCso), + /* 29 DeleteShaderState */ static_cast(kCtxCso), + /* 30 SetDynamicState */ static_cast(kCtxState), + /* 31 SetFramebufferState */ static_cast(kCtxState), + /* 32 SetVertexBuffers */ static_cast(kCtxState), + /* 33 SetIndexBuffer */ static_cast(kCtxState), + /* 34 SetIndirectBuffers */ static_cast(kCtxState), + /* 35 SetSamplerViews */ static_cast(kCtxState), + /* 36 BindSamplerStates */ static_cast(kCtxState), + /* 37 SetShaderImages */ static_cast(kCtxState), + /* 38 SetShaderBuffers */ static_cast(kCtxState), + /* 39 SetStreamOutputTargets */ static_cast(kCtxState), + /* 40 SetGlobalConstants */ static_cast(kCtxState), + /* 41 SetVertexAttribDefaults */ static_cast(kCtxState), + /* 42 SetPixelPackState */ static_cast(kCtxState), + /* 43 SetPatchState */ static_cast(kCtxState), + /* 44 SetDrawProgram */ static_cast(kCtxState), + /* 45 SetDispatchProgram */ static_cast(kCtxState), + /* 46 SetResidualValueState */ static_cast(kCtxState), + /* 47 SetTextureParams */ static_cast(kCtxObject), + /* 48 ResourceSubData */ static_cast(kCtxObject), + /* 49 BufferSubDataResident */ static_cast(kCtxObject), + /* 50 ResourceSubDataComplete */ static_cast(kCtxObject), + /* 51 ResourceFlushRange */ static_cast(kCtxObject), + /* 52 ResourceReadback */ static_cast(kCtxObject), + /* 53 ResourceCopyRegion */ static_cast(kCtxObject), + /* 54 GenerateMipmap */ static_cast(kCtxObject), + /* 55 GetTextureImage */ static_cast(kCtxObject), + /* 56 Blit */ static_cast(kCtxVerb), + /* 57 Clear */ static_cast(kCtxVerb), + /* 58 ReadPixels */ static_cast(kCtxVerb), + /* 59 DrawVbo */ static_cast(kCtxVerb), + /* 60 LaunchGrid */ static_cast(kCtxVerb), + /* 61 MemoryBarrier */ static_cast(kCtxVerb), + /* 62 BeginStreamOutput */ static_cast(kCtxVerb), + /* 63 EndStreamOutput */ static_cast(kCtxVerb), + /* 64 PauseStreamOutput */ static_cast(kCtxVerb), + /* 65 ResumeStreamOutput */ static_cast(kCtxVerb), + /* 66 Flush */ static_cast(kCtxVerb), + /* 67 Present */ static_cast(kCtxVerb), + /* 68 SetSwapInterval */ static_cast(kCtxVerb), + /* 69 QueryTimestamp */ static_cast(kCtxQuery), + /* 70 QueryCounter */ static_cast(kCtxQuery), + /* 71 FenceWaitServer */ static_cast(kScreen), + /* 72 BindShaderImage */ static_cast(kCtxVerb), + /* 73 PatchParameter */ static_cast(kCtxVerb), + /* 74 BindStreamOutput */ static_cast(kCtxVerb), + /* 75 SetStorageBlockBinding */ static_cast(kCtxVerb), + /* 76 CopyFramebufferToTexture*/ static_cast(kCtxVerb), + /* 77 ApplierReset */ static_cast(kScreen), + /* 78 ObjectDeath */ static_cast(kCtxObject), + /* 79 SetContextValues */ static_cast(kCtxState), +}; +static_assert(sizeof(kMGPipeCallClasses) / sizeof(kMGPipeCallClasses[0]) == + static_cast(MGPWireOp::kOpCount), + "the class table and the opcode space disagree"); + +inline constexpr MGPipeCallClass MGPipeCallClassFor(MGPWireOp op) { + const SizeT index = static_cast(op); + return index < static_cast(MGPWireOp::kOpCount) + ? static_cast(kMGPipeCallClasses[index]) + : kCallClassCount; +} + // Spot checks the generator states about its own output, so that a catalogue edit that // silently drops a flag is a build break here and not a wrong decode six packages away. static_assert(MGPipeCallFlagsFor(MGPWireOp::kInvalid) == static_cast(kNone), diff --git a/MobileGL/MG_Remote/CONTRACT-P5.md b/MobileGL/MG_Remote/CONTRACT-P5.md index cdee8c4f..ddf3d319 100644 --- a/MobileGL/MG_Remote/CONTRACT-P5.md +++ b/MobileGL/MG_Remote/CONTRACT-P5.md @@ -387,6 +387,8 @@ is a pull-build symbol and G1 admits **no resize**, which is the same reason the | `MOBILEGL_IPC_STAGE_MB` | 32 | SEG_STAGE. Every blob and every var-tail's bytes. | | `MOBILEGL_IPC_SPIN_US` | 50 | spin before parking, either direction. | | `MOBILEGL_IPC_PERSISTENT_BLOCK_KB` | 64 | **0 is the E3(a) negative control, not "unlimited"**: it turns the push off and `PersistentCoherentMapScenario` must go red. | +| `MOBILEGL_IPC_PERSISTENT_HASH_SUPPRESS` | 1 | 1 = the push ships only blocks whose hash changed since the last push (tracked buffers: the mprotect fault bitmap; untracked: the content scan). 0 restores the whole-range push (A/B control). | +| `MOBILEGL_IPC_BATCH_WAITS` | 1 | 1 = value-class records (kCtxState / kCtxCso / kCtxObject with no reply slot) publish without waiting for their own apply; the barrier is taken at the next pull-reading verb, the only place BARRIER-PULLED fields are read. 0 restores R-1's per-record barrier, and `MOBILEGL_PIPE_VERIFY` forces 0 (the shadow compare reads the pulled fields at every record). | | `MOBILEGL_IPC_ADOPT_TIER` | 2 | 2 = emulate, the only tier P5 implements. 0 and 1 parse and are `Fatal` at use, naming P11. | | `MOBILEGL_IPC_VERB_BARRIER` | 1 | 0 is R-1's negative control and is **expected** to be red. | | `MOBILEGL_IPC_STRICT_ERRORS` | 0 | promotes BARRIER-PULLED reads — and, in a split build, the seven sticky forwards — to `Fatal`. | diff --git a/MobileGL/MG_Remote/Client/ClientSession.cpp b/MobileGL/MG_Remote/Client/ClientSession.cpp index e7fedaa4..b487509e 100644 --- a/MobileGL/MG_Remote/Client/ClientSession.cpp +++ b/MobileGL/MG_Remote/Client/ClientSession.cpp @@ -827,13 +827,17 @@ namespace MobileGL::MG_Remote::Client { std::abort(); } - // R-1's INVARIANT, AS A RUNTIME CHECK RATHER THAN A SENTENCE. While the barrier holds, - // at most one of {GL thread, apply thread} is runnable - and that is the whole reason a - // single process-wide gPipeInputs is legal (table 3). The client is about to publish a - // record whose fields the applier will read out of gPipeInputs, so the apply thread - // being inside the applier right now means the invariant has already been broken and - // the next record would be read against a half-written residual fill. - if (ApplyThreadIsInsideApplier()) { + // R-1's INVARIANT, AS A RUNTIME CHECK RATHER THAN A SENTENCE - but only while the + // per-record barrier is the fencing model. With MOBILEGL_IPC_BATCH_WAITS=1 the + // intended shape is exactly "the apply thread is inside the applier while the GL + // thread emits the next value record", and the check would fire on the first batch: + // its protection is already carried by the disjoint-field model (record-supplied + // fields are written only by the applier from records and are never filled by the + // client while the wire is live; BARRIER-PULLED fields are written only by the + // client's residual fill and are read only at the pull-verbs, whose own wait makes + // the fill quiescent during that apply). With the batch off, the original check + // stands as written. + if (MG_Config::Ipc.BatchWaits == 0 && ApplyThreadIsInsideApplier()) { MGLOG_F("MGPipe: Fatal{BarrierViolation, \"%s\"} - the apply thread is inside the " "applier while the GL thread is emitting. R-1 makes at most one of them " "runnable, which is what keeps one process-wide gPipeInputs legal", @@ -879,6 +883,27 @@ namespace MobileGL::MG_Remote::Client { const Bool ownsReplySlot = (MG_Pipe::MGPipeCallFlagsFor(op) & static_cast(MG_Pipe::kReplySlot)) != 0; + // MOBILEGL_IPC_BATCH_WAITS (default 1): a value-class record is published WITHOUT + // waiting for its own apply. The barrier's one load-bearing reader is the backend + // sync that pulls the object-class fields (CONTRACT-P5 table 2's remaining rows), and + // those reads happen only at the kCtxVerb applies - draw / clear / blit / dispatch / + // readback / XFB - plus the screen and query classes, and EVERY ONE OF THOSE STILL + // WAITS below (as does every reply-slot row, whose answer the client consumes). So + // the pulled fields are exactly as fresh at every read point as the per-record + // barrier made them: between two waits the residual fill may move a field, but no + // apply is reading it in between, and the next pull-verb's fill runs after the + // previous pull-verb's apply was waited - the R-1 fence posts are untouched, only + // the number of round trips between them changes. Set / CSO / object records carry + // their whole value in the payload and are consumed at the next (waited) sync, which + // the in-order ring guarantees precedes that sync's apply. + if (!ownsReplySlot && MG_Config::Ipc.BatchWaits != 0) { + const MG_Pipe::MGPipeCallClass callClass = MG_Pipe::MGPipeCallClassFor(op); + if (callClass == MG_Pipe::kCtxState || callClass == MG_Pipe::kCtxCso || + callClass == MG_Pipe::kCtxObject) { + return seq; + } + } + if (!m_barrierArmed && !ownsReplySlot) { // R-1's NEGATIVE CONTROL ARM, and the only thing it turns off is the barrier. A // reply-slot row still waits: the answer is not derivable here and R-5 forbids @@ -978,6 +1003,13 @@ namespace MobileGL::MG_Remote::Client { // by the operator's own hand there, and EmitAndWait's Fatal{BarrierViolation} owns the // red. Firing here instead would pre-empt the control's evidence line. if (!ClientSessionInstance().BarrierArmed()) return; + // MOBILEGL_IPC_BATCH_WAITS=1 makes "the fill runs while the apply thread applies an + // earlier value record" the INTENDED shape: the fill writes only fields no record + // supplies (the wire-live skips are the same answer the emitters use), the applier + // writes only record-supplied fields, and the pull-verbs' own wait keeps their + // pulled reads fenced. This guard exists for the model where that is not true, so + // it stands only while the batch is off. + if (MG_Config::Ipc.BatchWaits != 0) return; if (!ApplyThreadIsInsideApplier()) return; if (InBarrierWait()) return; MGLOG_F("MGPipe: Fatal{RoleViolation, \"gPipeInputs\"} - the GL thread touched gPipeInputs " diff --git a/MobileGL/MG_Remote/Client/EmitTables.cpp b/MobileGL/MG_Remote/Client/EmitTables.cpp index ac9fae5e..8e786848 100644 --- a/MobileGL/MG_Remote/Client/EmitTables.cpp +++ b/MobileGL/MG_Remote/Client/EmitTables.cpp @@ -62,6 +62,7 @@ #include #include #include +#include namespace MobileGL::MG_Remote::Client { @@ -214,16 +215,23 @@ namespace MobileGL::MG_Remote::Client { // The two hooks b1 wrote and deliberately left with no caller, because the call site is // this file's. ORDER: the push first (it produces resource_subdata records that must - // precede the verb on SEG_CMD), then the mark walk, then the verb record. + // precede the verb on SEG_CMD), then the mark walk, then the verb record. The deferred + // destroy drain rides the same boundary: it replays, on this GL thread, the death + // announcements whose last SharedPtr dropped on the apply thread (PipeMutation.h's + // deferred destroy queue), and its records must also precede this verb. void BeforeDrawVerb() { - PushPersistentMapsBeforeVerb(); + PersistentMapTracker::Instance().PushDrawConsumers(); + MG_Pipe::MGPipeDrainDeferredDestroys(); MarkGpuWritesForDraw(); } // A verb that reads buffers but starts no shader: clear, blit, readback, present. The // push still has to run - a coherent map is read by the GPU on any of them - but there // is no shader that could write one, so no mark walk. - void BeforeReadOnlyVerb() { PushPersistentMapsBeforeVerb(); } + void BeforeReadOnlyVerb() { + PushPersistentMapsBeforeVerb(); + MG_Pipe::MGPipeDrainDeferredDestroys(); + } // ============================================================================= // CLASS B - the five slots the verb census measured (CONTRACT-P5.md §7) @@ -1110,7 +1118,7 @@ namespace MobileGL::MG_Remote::Client { // slot would inherit a call site that was already correct. void EmitDispatchCompute(GLuint numGroupsX, GLuint numGroupsY, GLuint numGroupsZ) { ClientSession& session = RequireSession("DispatchCompute"); - PushPersistentMapsBeforeVerb(); + PersistentMapTracker::Instance().PushDrawConsumers(); MarkGpuWritesForDispatch(); MG_Pipe::MGPGridInfo record{}; @@ -1129,7 +1137,7 @@ namespace MobileGL::MG_Remote::Client { void EmitDispatchComputeIndirect(GLintptr indirect) { ClientSession& session = RequireSession("DispatchComputeIndirect"); - PushPersistentMapsBeforeVerb(); + PersistentMapTracker::Instance().PushDrawConsumers(); MarkGpuWritesForDispatch(); MG_Pipe::MGPGridInfo record{}; diff --git a/MobileGL/MG_Remote/Client/PersistentMapTracker.cpp b/MobileGL/MG_Remote/Client/PersistentMapTracker.cpp index a591e070..34834673 100644 --- a/MobileGL/MG_Remote/Client/PersistentMapTracker.cpp +++ b/MobileGL/MG_Remote/Client/PersistentMapTracker.cpp @@ -8,16 +8,382 @@ #include "PersistentMapTracker.h" +#include #include +#include +#include +#include #include #include +#include + +#include +#include #include +#include +#include +#include +#include +#include namespace MobileGL::MG_Remote::Client { using MG_State::GLState::BufferObject; - using MobileGL::BufferMappingAccessBit; + + // --------------------------------------------------------------------------- + // THE MPROTECT DIRTY TRACKER. Scanning a persistent write map for changes is + // O(range) per verb no matter how cheap the hash is (measured: a ~10 MB arena + // hashed at every draw is the whole frame at view distance 12). The MMU + // already knows what changed: after each push the range is protected + // PROT_READ, the application's next write faults, the handler marks the page + // dirty and makes it writable again, and the next push ships only the 64 KB + // blocks that contain a faulted page. The per-frame cost collapses from + // "hash every block of every arena at every draw" to "one fault per touched + // page per push-cycle plus the block pushes that always existed". + // + // THE TABLE IS A FIXED ARRAY, NOT A MAP: the SIGSEGV handler runs on ANY + // thread (chunk builders write arenas off the render thread) while + // registration runs on the GL thread, so the handler scans a lock-free array + // of atomics; the GL thread publishes an entry by writing pageBits, then end, + // then base with release ordering, and retires one by clearing base first. + // + // CHAINING IS LOAD-BEARING: the JVM uses SIGSEGV for implicit null checks, + // so a fault outside every tracked range MUST reach the previous handler or + // the first NPE after this installs is a process kill. Ownership is three + // tests, applied in the handler below: kernel-raised permission fault + // (SEGV_ACCERR - never a userspace raise, never an unmapped page), inside a + // live tracked range, and not an immediate same-instruction refault (a write + // we unprotected succeeds, so an instant refault is a non-write access and + // must crash honestly). Everything failing any test is chained verbatim. + // --------------------------------------------------------------------------- + namespace { + + constexpr SizeT kMaxTrackedMaps = 64; + constexpr SizeT kPageShift = 12; + constexpr SizeT kPageBytes = 1u << kPageShift; + // The bitmap is FIXED-CAPACITY and never reallocated: a handler mid-flight on + // another thread can hold the pointer it loaded before a retire, so a grow-and-free + // would be a use-after-free no memory ordering can close. 64 K pages = 256 MB per + // tracked map, 8 KB per slot, allocated lazily once and kept for the process. + constexpr SizeT kMaxTrackedPages = 65536; + constexpr SizeT kPageBitsWords = kMaxTrackedPages / 64; + // Slot states for base: 0 = free, 1 = being set up (handler must skip: end is 0), + // anything else = live. The claim token is the sentinel, NOT the real base, so the + // handler can never observe a half-published entry (base live, end still stale). + constexpr uintptr_t kSlotSettingUp = 1; + + TrackedWriteMap g_trackedMaps[kMaxTrackedMaps]; + struct sigaction g_prevSegvAction {}; + std::atomic g_segvInstalled{false}; + // Bumped by the handler for every write fault it answers. PushDrawConsumers + // compares it against the GL thread's last-walk value: an unmoved epoch proves + // no interior byte of any tracked map changed, because interior dirtiness can + // only ever arrive through a fault. + std::atomic g_faultEpoch{0}; + + // THE OWNERSHIP TESTS. "Ours" is a NARROW claim, and every fault that fails any + // one of these three tests is chained to the previous owner untouched, so a + // genuinely broken memory access crashes exactly as it would without us: + // + // (1) KERNEL-RAISED PERMISSION FAULT ONLY: si_code must be SEGV_ACCERR. A fault + // on an unmapped page (SEGV_MAPERR) is a wild pointer, and anything raised + // from userspace (raise/kill/tgkill/sigqueue) carries si_code <= 0 - this + // tracker never raises SIGSEGV itself, so a user-raised one is always foreign. + // (2) INSIDE A LIVE TRACKED RANGE: a fault anywhere else is the application's. + // (3) NOT A REFAULT WITHOUT RE-ARM: a write to a page this handler just + // unprotected SUCCEEDS, so a fault on a page whose dirty bit is still set + // is either a sibling thread mid-answer (answered once more, idempotently) + // or a non-write access such as an execute of the data page (chained, so it + // crashes honestly instead of refaulting forever). The page's own bit is + // the discriminator - see RefaultOfANonWriteAccess. + + uintptr_t FaultPcFrom(void* ucontext) { +#if defined(__aarch64__) + return static_cast( + reinterpret_cast(ucontext)->uc_mcontext.pc); +#elif defined(__x86_64__) + return static_cast( + reinterpret_cast(ucontext)->uc_mcontext.gregs[REG_RIP]); +#else + // No PC access on this arch: the refault test is skipped, ownership rests + // on the si_code and range tests alone. + return 0; +#endif + } + + struct FaultRepeatSlot { + std::atomic tid{0}; + uintptr_t pc = 0; + uintptr_t address = 0; + Uint32 setBitAnswers = 0; + }; + FaultRepeatSlot g_faultRepeats[8]; + + // THE REFAULT TEST, and its exact discriminator: the page's own bit. The + // tracker keeps ONE invariant - bit set <=> page writable, bit clear <=> + // page read-only (the handler sets-then-unprotects, the push + // clears-then-rearms). So at a fault, the bit the fetch_or just observed + // decides what this access can be: + // + // bit was CLEAR - the page was re-armed since the last answer, so this + // is a fresh write. A same-(thread, instruction, address) refault is + // EXPECTED here (malloc hands the same block in a re-armed shared + // page to the same free() site a frame later); it is not suspicious. + // bit was SET - the page should already be writable, so a WRITE cannot + // be what faulted. One exception: a sibling thread is mid-answer on + // the same page right now (it set the bit and has not unprotected + // yet). That thread's write succeeds once either answer lands, so it + // never comes back; a non-write access (an execute of the data page) + // comes back every time. Answer the first set-bit fault at a + // (thread, pc, address), chain the second. + Bool RefaultOfANonWriteAccess(uintptr_t pc, uintptr_t address, Bool bitWasSet) { + const pid_t tid = static_cast(syscall(SYS_gettid)); + FaultRepeatSlot* slot = nullptr; + for (auto& candidate : g_faultRepeats) { + if (candidate.tid.load(std::memory_order_acquire) == tid) { + slot = &candidate; + break; + } + } + if (slot == nullptr) { + for (auto& candidate : g_faultRepeats) { + pid_t empty = 0; + if (candidate.tid.compare_exchange_strong(empty, tid)) { + slot = &candidate; + break; + } + } + } + // Full: evict deterministically. Losing a record costs the refault test on + // that thread, never a wrong answer - the next fault re-records it. + if (slot == nullptr) slot = &g_faultRepeats[static_cast(tid) & 7]; + if (!bitWasSet) { + slot->pc = pc; + slot->address = address; + slot->setBitAnswers = 0; + return false; + } + const Bool same = slot->pc == pc && slot->address == address; + const Uint32 answers = same ? slot->setBitAnswers : 0; + if (answers >= 1) return true; + slot->pc = pc; + slot->address = address; + slot->setBitAnswers = answers + 1; + return false; + } + + void ChainToPreviousSegvHandler(int sig, siginfo_t* info, void* ucontext) { + // Chain with the disposition the previous owner actually installed - the two + // entry points share a union, so SA_SIGINFO decides which signature is live, + // and the original info/ucontext go through verbatim so the next handler can + // run its own si_code test. For SIG_DFL (and the meaningless-but-possible + // SIG_IGN) restore the default and return: the faulting instruction + // re-executes, faults again, and dies the way an untracked fault would have. + if ((g_prevSegvAction.sa_flags & SA_SIGINFO) != 0) { + if (g_prevSegvAction.sa_sigaction != nullptr) { + g_prevSegvAction.sa_sigaction(sig, info, ucontext); + return; + } + } else if (g_prevSegvAction.sa_handler != SIG_DFL && + g_prevSegvAction.sa_handler != SIG_IGN && + g_prevSegvAction.sa_handler != nullptr) { + g_prevSegvAction.sa_handler(sig); + return; + } + struct sigaction restore {}; + restore.sa_handler = SIG_DFL; + sigemptyset(&restore.sa_mask); + sigaction(SIGSEGV, &restore, nullptr); + } + + void PersistentWriteFaultHandler(int sig, siginfo_t* info, void* ucontext) { + if (info != nullptr && info->si_code == SEGV_ACCERR) { + const uintptr_t address = reinterpret_cast(info->si_addr); + // EVERY overlapping slot gets its bit, not just the first: two tracked + // shadows can share an edge page (separate heap allocations, one page + // granularity), and unprotecting after only the first mark would leave the + // second buffer's block permanently un-pushed - the page is writable now, + // so it never faults again. + Bool matched = false; + Bool bitWasSet = false; + for (SizeT i = 0; i < kMaxTrackedMaps; ++i) { + const uintptr_t base = + g_trackedMaps[i].base.load(std::memory_order_acquire); + if (base <= kSlotSettingUp || address < base) continue; + const uintptr_t end = + g_trackedMaps[i].end.load(std::memory_order_acquire); + if (address >= end) continue; + const SizeT pageIndex = (address - base) >> kPageShift; + std::atomic* bits = g_trackedMaps[i].pageBits; + if (bits != nullptr) { + const Uint64 mask = 1ull << (pageIndex & 63); + const Uint64 old = + bits[pageIndex >> 6].fetch_or(mask, std::memory_order_relaxed); + if (!matched) bitWasSet = (old & mask) != 0; + } + matched = true; + } + if (matched && + !RefaultOfANonWriteAccess(FaultPcFrom(ucontext), address, bitWasSet)) { + // The epoch moves BEFORE the unprotect: a walk that starts the + // moment the page becomes writable must still see this fault. + g_faultEpoch.fetch_add(1, std::memory_order_relaxed); + // Un-arm is a raw syscall, deliberately not the libc wrapper's + // bookkeeping: the handler's whole job is to get out of the way. + mprotect(reinterpret_cast(address & ~(kPageBytes - 1)), + kPageBytes, PROT_READ | PROT_WRITE); + return; + } + } + ChainToPreviousSegvHandler(sig, info, ucontext); + } + + void InstallWriteFaultHandler() { + Bool expected = false; + if (!g_segvInstalled.compare_exchange_strong(expected, true)) return; + struct sigaction action {}; + action.sa_sigaction = &PersistentWriteFaultHandler; + sigemptyset(&action.sa_mask); + action.sa_flags = SA_SIGINFO; + if (sigaction(SIGSEGV, &action, &g_prevSegvAction) != 0) { + MGLOG_W("MGPipe: persistent-map mprotect tracking unavailable (sigaction " + "failed); persistent write maps fall back to the hash scan"); + g_prevSegvAction = {}; + g_segvInstalled.store(false); + } + } + + void UntrackWriteMap(Uint64 lifetimeId); + + Bool TrackWriteMap(Uint64 lifetimeId, const Uint8* shadow, + SizeT rangeBegin, SizeT rangeEnd) { + if (shadow == nullptr || rangeEnd <= rangeBegin) return false; + InstallWriteFaultHandler(); + if (!g_segvInstalled.load()) return false; + // INWARD alignment, and the reason is a process-killer: with OUTWARD + // alignment the protected edge pages hold FOREIGN bytes (the shadow is a + // heap allocation; malloc metadata and other allocations share its first + // and last page). A foreign write to such a page is fine for a thread + // that runs our handler - it is marked and unprotected - but driver and + // runtime worker threads commonly block every signal, and a fault there + // with SIGSEGV blocked ends the process on the spot, undebuggable. + // Protecting only pages FULLY inside the mapped range means every byte of + // a protected page is the shadow's own, and no other allocation can ever + // be made to fault by us. The head/tail partial pages stay writable and + // their blocks are pushed unconditionally at every push (at most two). + const uintptr_t rawBegin = reinterpret_cast(shadow + rangeBegin); + const uintptr_t rawEnd = reinterpret_cast(shadow + rangeEnd); + const uintptr_t base = (rawBegin + kPageBytes - 1) & + ~static_cast(kPageBytes - 1); + const uintptr_t end = rawEnd & ~static_cast(kPageBytes - 1); + if (end <= base) return false; // sub-page range: the hash arm has it + const SizeT pageCount = (end - base) >> kPageShift; + if (pageCount > kMaxTrackedPages) { + MGLOG_W("MGPipe: persistent write map of %zu pages exceeds the mprotect " + "tracker's %zu-page capacity; this buffer falls back to the hash scan", + pageCount, kMaxTrackedPages); + return false; + } + // A remap/re-register of the same buffer must not stack a second slot on the + // same lifetimeId: retire whatever this id had first. No-op when untracked. + UntrackWriteMap(lifetimeId); + for (SizeT i = 0; i < kMaxTrackedMaps; ++i) { + uintptr_t empty = 0; + if (!g_trackedMaps[i].base.compare_exchange_strong(empty, kSlotSettingUp)) { + continue; + } + // Claimed. From here until the final base store the slot reads as + // (base=1, end=0), which the handler skips on both tests. + if (g_trackedMaps[i].pageBits == nullptr) { + g_trackedMaps[i].pageBits = new std::atomic[kPageBitsWords]; + } + g_trackedMaps[i].pageCount = pageCount; + g_trackedMaps[i].lifetimeId = lifetimeId; + g_trackedMaps[i].edgeHashHead = 0; + g_trackedMaps[i].edgeHashTail = 0; + g_trackedMaps[i].hasEdges = base != rawBegin || end != rawEnd; + // EVERY BIT SET, NOT ZEROED - the same "fresh state pushes everything + // once" rule the hash arm keeps: the server has never seen these bytes, + // so the first push must ship the whole interior, not just the pages the + // application happened to write first. The first push then clears and + // re-arms page by page, and the set-bit read the fault discriminator + // does on a pre-first-push fault is answered once and unprotected - + // harmless, and the invariant (bit set <=> writable) holds from the + // first push onward. The last word is masked: a set bit past pageCount + // would re-arm a page outside the interior - a foreign page, which is + // the process-killer the inward alignment exists to keep out. + const SizeT wordCount = (pageCount + 63) / 64; + for (SizeT w = 0; w < wordCount; ++w) { + const SizeT bitsInWord = + (w == wordCount - 1 && (pageCount & 63) != 0) ? (pageCount & 63) : 64; + g_trackedMaps[i].pageBits[w].store( + bitsInWord == 64 ? ~0ull : ((1ull << bitsInWord) - 1), + std::memory_order_relaxed); + } + if (mprotect(reinterpret_cast(base), end - base, PROT_READ) != 0) { + g_trackedMaps[i].base.store(0, std::memory_order_release); + return false; + } + // Publish order is end, then base: the handler reads base first, and a base + // it can observe is only ever one whose end is already visible. + g_trackedMaps[i].end.store(end, std::memory_order_release); + g_trackedMaps[i].base.store(base, std::memory_order_release); + return true; + } + MGLOG_W("MGPipe: the mprotect tracker is full (%zu live persistent write maps); " + "this buffer falls back to the hash scan", + kMaxTrackedMaps); + return false; + } + + void UntrackWriteMap(Uint64 lifetimeId) { + for (SizeT i = 0; i < kMaxTrackedMaps; ++i) { + if (g_trackedMaps[i].lifetimeId != lifetimeId || + g_trackedMaps[i].base.load(std::memory_order_acquire) == 0) { + continue; + } + const uintptr_t base = g_trackedMaps[i].base.load(std::memory_order_acquire); + const uintptr_t end = g_trackedMaps[i].end.load(std::memory_order_acquire); + // Retire first, then unprotect: a fault racing the retire falls through to + // the previous handler, which is the honest answer for a page we no longer + // own; a fault racing the unprotect is impossible, the page is writable. + // end goes to 0 with the retire so a later re-claim of this slot reads + // (base=1, end=0) for its whole setup, which the handler skips. + g_trackedMaps[i].base.store(0, std::memory_order_release); + g_trackedMaps[i].end.store(0, std::memory_order_release); + g_trackedMaps[i].lifetimeId = 0; + mprotect(reinterpret_cast(base), end - base, PROT_READ | PROT_WRITE); + return; + } + } + + // Every live slot, restored writable and freed. Test teardown only: without it a + // cleared fixture would leave its shadow pages PROT_READ behind, and the next + // fixture writing the recycled heap would fault into ranges whose owner is gone. + void UntrackAllWriteMaps() { + for (SizeT i = 0; i < kMaxTrackedMaps; ++i) { + const uintptr_t base = g_trackedMaps[i].base.load(std::memory_order_acquire); + if (base <= kSlotSettingUp) continue; + const uintptr_t end = g_trackedMaps[i].end.load(std::memory_order_acquire); + g_trackedMaps[i].base.store(0, std::memory_order_release); + g_trackedMaps[i].end.store(0, std::memory_order_release); + g_trackedMaps[i].lifetimeId = 0; + mprotect(reinterpret_cast(base), end - base, PROT_READ | PROT_WRITE); + } + } + + TrackedWriteMap* TrackedMapFor(Uint64 lifetimeId) { + for (SizeT i = 0; i < kMaxTrackedMaps; ++i) { + if (g_trackedMaps[i].lifetimeId == lifetimeId && + g_trackedMaps[i].base.load(std::memory_order_acquire) > kSlotSettingUp) { + return &g_trackedMaps[i]; + } + } + return nullptr; + } + + } // namespace PersistentMapTracker& PersistentMapTracker::Instance() { // Leaked on purpose, once, like every other role-local singleton (ID-8): a buffer's @@ -62,15 +428,43 @@ namespace MobileGL::MG_Remote::Client { void PersistentMapTracker::NoteMapStateChanged(BufferObject& buffer) { const Uint64 key = buffer.GetLifetimeId(); + // Unaccount the OLD state first: a re-notified member keeps its membership but + // may move arms (a resize re-registers the tracker), and the epoch skip's veto + // set must follow the arm the member actually runs on. Erase is a no-op for a + // member that was never large-untracked. + m_untrackedMembers.erase(key); if (IsLivePersistentMap(buffer)) { - m_livePersistentMaps[key] = &buffer; + const auto range = buffer.GetMappedRange(); + // The mprotect arm registers beside the hash arm: tracked buffers are pushed + // from the fault bitmap, everything else keeps the content scan. A failed + // registration (handler unavailable, table full, unaligned zero range) quietly + // leaves the buffer on the hash path. + TrackWriteMap(key, buffer.MappedData(), static_cast(range.start), + static_cast(range.end)); + TrackedWriteMap* slot = TrackedMapFor(key); + m_livePersistentMaps[key] = MemberEntry{&buffer, slot}; + if (slot == nullptr) m_untrackedMembers[key] = 1; return; } m_livePersistentMaps.erase(key); + UntrackWriteMap(key); } void PersistentMapTracker::Forget(const BufferObject& buffer) { - m_livePersistentMaps.erase(buffer.GetLifetimeId()); + const Uint64 key = buffer.GetLifetimeId(); + m_untrackedMembers.erase(key); + m_livePersistentMaps.erase(key); + m_blockHashes.erase(key); + UntrackWriteMap(key); + } + + void PersistentMapTracker::ClearForTest() { + m_livePersistentMaps.clear(); + m_blockHashes.clear(); + m_untrackedMembers.clear(); + UntrackAllWriteMaps(); + m_lastFaultEpoch = 0; + ResetCountersForTest(); } void PersistentMapTracker::PushBlocksFor(BufferObject& buffer) { @@ -80,6 +474,10 @@ namespace MobileGL::MG_Remote::Client { "producer belongs to the client; the server consumes transported bytes"); std::abort(); } + PushBlocksForChecked(buffer); + } + + void PersistentMapTracker::PushBlocksForChecked(BufferObject& buffer) { // Re-checked rather than trusted. The set is maintained at five events and a sixth // one arriving without a NoteMapStateChanged would otherwise push a buffer whose // shadow has been released - an adopted store's Bytes() is the GPU map, and reading @@ -118,8 +516,155 @@ namespace MobileGL::MG_Remote::Client { const auto range = buffer.GetMappedRange(); const Uint64 begin = static_cast(range.start); const Uint64 end = static_cast(range.end); - for (Uint64 at = begin; at < end; at += blockBytes) { + // THE MPROTECT ARM. A tracked buffer's push is driven by the fault bitmap: one + // 64 KB block per block that contains at least one faulted page, shipped with + // the whole machinery of every other arm (serial bump, HasDefinedContent, + // wire record), then the page is cleared and re-armed read-only. No scan, no + // hash, no per-draw cost for buffers the application did not touch. The two + // partial-page EDGE ranges beside the page-aligned interior (see TrackWriteMap's + // inward alignment) are never protected, so nothing tells us they changed - + // their blocks ride along on every push, at most two. + TrackedWriteMap* tracked = nullptr; + if (const auto it = m_livePersistentMaps.find(buffer.GetLifetimeId()); + it != m_livePersistentMaps.end() && it->second.tracked != nullptr && + it->second.tracked->lifetimeId == buffer.GetLifetimeId()) { + tracked = it->second.tracked; + } + if (tracked != nullptr) { + const uintptr_t trackedBase = tracked->base.load(std::memory_order_acquire); + const uintptr_t trackedEnd = tracked->end.load(std::memory_order_acquire); + const uintptr_t shadowBase = reinterpret_cast(buffer.MappedData()); + if (trackedBase != 0 && trackedEnd > trackedBase && shadowBase != 0) { + const Uint64 interiorBegin = + static_cast(trackedBase - shadowBase); // >= begin + const Uint64 interiorEnd = + static_cast(trackedEnd - shadowBase); // <= end + // Pushes are made in ASCENDING offset order, and `pushedUpTo` trims each + // span against the one before it: edge spans, duplicate pages inside one + // 64 KB block and faulted blocks overlapping an edge all collapse to one + // wire record per covered interval. The edge spans cover their WHOLE + // containing block rather than just the edge bytes, so a head edge and + // the faulted pages of the same first block merge into the one block + // push the hash arm would have made - the wire shape stays block-shaped. + // And an edge span ships ONLY when its bytes changed since the last push + // (the struct comment says why unconditional was measured and rejected): + // no faults happen on unprotected bytes, so the hash is the only change + // detector there is for them. + Uint64 pushedUpTo = begin; + const auto pushSpan = [&](Uint64 spanBegin, Uint64 spanEnd) { + if (spanBegin < pushedUpTo) spanBegin = pushedUpTo; + if (spanBegin >= spanEnd) return; + buffer.PushMappedSpanBlock(static_cast(spanBegin), + static_cast(spanEnd - spanBegin)); + ++m_blocksPushed; + m_bytesPushed += spanEnd - spanBegin; + pushedUpTo = spanEnd; + }; + if (interiorBegin > begin) { + const Uint64 edgeHash = XXH3_64bits( + reinterpret_cast(shadowBase) + begin, + static_cast(interiorBegin - begin)); + if (edgeHash != tracked->edgeHashHead) { + tracked->edgeHashHead = edgeHash; + const Uint64 headBlockEnd = (begin / blockBytes + 1) * blockBytes; + pushSpan(begin, headBlockEnd < end ? headBlockEnd : end); + } + } + for (SizeT wordIndex = 0; wordIndex < (tracked->pageCount + 63) / 64; ++wordIndex) { + Uint64 word = tracked->pageBits[wordIndex].load(std::memory_order_relaxed); + while (word != 0) { + const SizeT bit = static_cast(__builtin_ctzll(word)); + word &= word - 1; + const SizeT pageIndex = (wordIndex << 6) + bit; + // Clear the bit FIRST, per the order the correctness argument + // needs, but batch the re-arm: bits are visited in ascending + // page order, so a contiguous run of faulted pages is one + // mprotect instead of one per page. The order still holds for + // every page of the run - clear, then re-arm, then read - and a + // write in the clear-to-re-arm window lands before the run's + // reads, so it is inside the shipped bytes. + tracked->pageBits[wordIndex].fetch_and(~(1ull << bit), + std::memory_order_relaxed); + SizeT runEnd = pageIndex + 1; + while (word != 0) { + const SizeT nextBit = static_cast(__builtin_ctzll(word)); + if ((wordIndex << 6) + nextBit != runEnd) break; + word &= word - 1; + tracked->pageBits[wordIndex].fetch_and(~(1ull << nextBit), + std::memory_order_relaxed); + ++runEnd; + } + mprotect(reinterpret_cast(trackedBase + (pageIndex << kPageShift)), + (runEnd - pageIndex) << kPageShift, PROT_READ); + for (SizeT runPage = pageIndex; runPage < runEnd; ++runPage) { + // The interior was page-aligned INSIDE the mapped range at + // registration, so a faulted page's offset in buffer space is + // never negative; its containing block can still start before + // `begin` (a block is 64 KB, a page is 4 KB), and the clamp is + // what keeps the push inside [begin, end). + const Uint64 pageOffset = + (trackedBase + (runPage << kPageShift)) - shadowBase; + const Uint64 blockFloor = (pageOffset / blockBytes) * blockBytes; + const Uint64 blockStart = + blockFloor < begin ? begin : blockFloor; + const Uint64 blockEnd = + (end - blockStart) < blockBytes ? end : blockStart + blockBytes; + pushSpan(blockStart, blockEnd); + } + } + } + if (interiorEnd < end) { + const Uint64 edgeHash = XXH3_64bits( + reinterpret_cast(shadowBase) + interiorEnd, + static_cast(end - interiorEnd)); + if (edgeHash != tracked->edgeHashTail) { + tracked->edgeHashTail = edgeHash; + const Uint64 tailBlockStart = ((end - 1) / blockBytes) * blockBytes; + pushSpan(tailBlockStart > begin ? tailBlockStart : begin, end); + } + } + return; + } + } + // THE DIRTY-BLOCK PUSH (MOBILEGL_IPC_PERSISTENT_HASH_SUPPRESS). The whole-range push + // below is what the numbers made untenable on a real workload: every verb shipped the + // entire mapped range (measured ~25 MB/frame in a streamed world - one record, one + // staging copy, one server-side adoption and one barrier wait per 64 KB block). The + // app writes a small fraction of the range per frame, so each block is hashed against + // the shadow the last push saw and only changed blocks cross. Semantics are the + // push's own: a changed byte still reaches the server before the next consuming verb, + // and a hash of 0 is "unknown" (XXH64 of zero bytes is not zero), so a fresh state + // pushes everything once. API writes that bypass the map (NotifySubData) show up as a + // one-frame redundant push of the touched block, then the cache catches up - correct + // in both directions. + BlockHashState* state = nullptr; + const Uint8* shadow = nullptr; + if (MG_Config::Ipc.PersistentHashSuppress != 0) { + shadow = buffer.MappedData(); + if (shadow != nullptr) { + state = &m_blockHashes[buffer.GetLifetimeId()]; + const Uint64 blockCount = (end - begin + blockBytes - 1) / blockBytes; + // A remap, a resize or a block-size change invalidates the layout the + // hashes were taken against: reset, which forces one full push - the + // conservative answer for a new layout. + if (state->begin != begin || state->end != end || state->blockBytes != blockBytes || + state->hashes.size() != blockCount) { + *state = {}; + state->begin = begin; + state->end = end; + state->blockBytes = blockBytes; + state->hashes.assign(static_cast(blockCount), 0); + } + } + } + SizeT blockIndex = 0; + for (Uint64 at = begin; at < end; at += blockBytes, ++blockIndex) { const Uint64 length = (end - at) < blockBytes ? (end - at) : blockBytes; + if (state != nullptr) { + const Uint64 h = XXH3_64bits(shadow + at, static_cast(length)); + if (h == state->hashes[blockIndex]) continue; + state->hashes[blockIndex] = h; + } buffer.PushMappedSpanBlock(static_cast(at), static_cast(length)); ++m_blocksPushed; m_bytesPushed += length; @@ -138,9 +683,122 @@ namespace MobileGL::MG_Remote::Client { // being one), and ska::flat_hash_map invalidates on erase. Vector members; members.reserve(m_livePersistentMaps.size()); - for (const auto& entry : m_livePersistentMaps) members.push_back(entry.second); + for (const auto& entry : m_livePersistentMaps) members.push_back(entry.second.buffer); for (BufferObject* buffer : members) { - if (buffer != nullptr) PushBlocksFor(*buffer); + if (buffer != nullptr) PushBlocksForChecked(*buffer); + } + } + + void PersistentMapTracker::PushDrawConsumers() { + if (!PushIsArmed()) return; + if (OnServerRole()) { + MGLOG_F("MGPipe: Fatal{RoleViolation, \"PushDrawConsumers\"} - the persistent-map " + "producer belongs to the client; the server consumes transported bytes"); + std::abort(); + } + if (m_livePersistentMaps.empty()) return; + // THE EPOCH SKIP. Interior dirtiness can only arrive as a write fault, and the + // handler bumps the fault epoch for every one it answers before it unprotects; + // an unmoved epoch therefore proves no interior byte of any tracked map changed + // since the last walk. The skip still serves a tracked member's partial-page + // edge spans right here, per member, with no binding state read at all - they + // are never protected, so their two spans are hashed per push. An untracked + // (hash-arm) member vetoes the skip outright: only the binding walk knows + // whether this draw consumes it, and scanning every untracked member on faith + // measured strictly worse than the walk (VD12, on device). With none present + // and the epoch unmoved, nearly every draw in steady flight costs a handful of + // 4 KB edge hashes. + const Uint64 epoch = g_faultEpoch.load(std::memory_order_acquire); + if (m_untrackedMembers.empty() && epoch == m_lastFaultEpoch) { + for (const auto& entry : m_livePersistentMaps) { + BufferObject* member = entry.second.buffer; + if (member == nullptr) continue; + // The cached slot pointer, validated against the slot's own lifetimeId: + // a slot retired since the cache was written fails the test, as does one + // re-claimed by a different buffer. + const TrackedWriteMap* tracked = entry.second.tracked; + if (tracked != nullptr && tracked->lifetimeId == member->GetLifetimeId() && + tracked->hasEdges) { + PushBlocksForChecked(*member); + } + } + return; + } + m_lastFaultEpoch = epoch; + const auto& ctx = MG_State::pGLContext; + if (ctx == nullptr) return; + + // The buffers the upcoming draw or dispatch can read, and only those. Every one is + // named by the frontend's own binding state, read on this (the GL) thread - VAO + // attribute and element buffers, the indexed binding points of every indexed + // target, and the indirect / parameter slots. Client-sourced attributes (no + // BufferObject) live on this side already and need no push. + Vector consumers; + // One entry per buffer, not per binding point: an arena bound to six attribute + // slots would otherwise get its predicate re-check, bitmap walk and edge hashes + // run six times per draw. + const auto consume = [&consumers](BufferObject* buffer) { + if (buffer == nullptr) return; + for (const BufferObject* existing : consumers) { + if (existing == buffer) return; + } + consumers.push_back(buffer); + }; + if (const auto& vao = ctx->GetBoundVertexArray()) { + for (Uint i = 0; i < MG_Pipe::kMGPipeMaxVertexAttribs; ++i) { + const auto& attrib = vao->GetAttribute(i); + if (attrib.Enabled && attrib.Buffer != nullptr) { + consume(attrib.Buffer.get()); + } + } + if (const auto& element = vao->GetIndexBufferBindingSlot().GetBoundObject()) { + consume(element.get()); + } + } + static constexpr BufferTarget kIndexedTargets[] = { + BufferTarget::Uniform, BufferTarget::ShaderStorage, BufferTarget::AtomicCounter, + BufferTarget::TransformFeedback, BufferTarget::CopyRead, BufferTarget::CopyWrite, + BufferTarget::Query, BufferTarget::Texture}; + for (const BufferTarget target : kIndexedTargets) { + const SizeT count = ctx->GetBufferBindingPointCount(target); + for (SizeT i = 0; i < count; ++i) { + if (const auto& bound = ctx->GetBufferBindingPoint(target, i).GetBoundObject()) { + consume(bound.get()); + } + } + } + static constexpr BufferTarget kSlotTargets[] = {BufferTarget::DrawIndirect, + BufferTarget::DispatchIndirect, + BufferTarget::Parameter}; + for (const BufferTarget target : kSlotTargets) { + if (const auto& bound = ctx->GetBufferBindingSlot(target).GetBoundObject()) { + consume(bound.get()); + } + } + // BUFFER TEXTURES (the cloud renderer's shape, and the gap the binding-point walk + // cannot see): a texture attached to a buffer with glTexBuffer is read by the draw + // through a texture unit, never through a buffer binding point. Walking the touched + // units and following a buffer-backed texture to its backing buffer is the only way + // those bytes earn their push - without this arm a persistent-mapped TexBuffer is + // only refreshed by the read-only verbs' whole push (once per present), which is + // exactly the per-frame-cadence cloud twitch this comment is written against. + const Int maxUnit = ctx->GetMaxTouchedTextureUnit(); + for (Int unit = 0; unit <= maxUnit; ++unit) { + const auto& texture = + ctx->GetTextureUnitObject(unit).GetBindingSlot(TextureTarget::TextureBuffer).GetBoundObject(); + if (texture == nullptr || texture->GetStorageType() != TextureStorageType::Buffer) { + continue; + } + auto* bufferTexture = static_cast(texture.get()); + if (const auto& backing = bufferTexture->GetBufferBindingSlot().GetBoundObject()) { + consume(backing.get()); + } + } + if (consumers.empty()) return; + for (BufferObject* buffer : consumers) { + if (buffer != nullptr && m_livePersistentMaps.count(buffer->GetLifetimeId()) != 0) { + PushBlocksForChecked(*buffer); + } } } diff --git a/MobileGL/MG_Remote/Client/PersistentMapTracker.h b/MobileGL/MG_Remote/Client/PersistentMapTracker.h index e712016b..a7604f94 100644 --- a/MobileGL/MG_Remote/Client/PersistentMapTracker.h +++ b/MobileGL/MG_Remote/Client/PersistentMapTracker.h @@ -54,12 +54,39 @@ #include +#include +#include + namespace MobileGL::MG_State::GLState { class BufferObject; } namespace MobileGL::MG_Remote::Client { + // The mprotect tracker's per-map slot, declared at namespace scope rather than in the + // .cpp's anonymous namespace so PersistentMapTracker's membership map can CACHE a + // pointer to a member's slot (the 64-slot rescan per buffer per draw was measured on + // device). Every field's reader/writer discipline is unchanged and lives in the .cpp; + // only the type's visibility moved. + struct TrackedWriteMap { + std::atomic base{0}; + std::atomic end{0}; + std::atomic* pageBits = nullptr; // allocated once, never freed or moved + SizeT pageCount = 0; // GL thread only + Uint64 lifetimeId = 0; // GL thread only + // GL thread only. The two partial-page edge spans are never protected (see + // TrackWriteMap's inward alignment), so nothing faults for them; pushing their + // blocks unconditionally measured ~2 x 64 KB x every draw for a hot unaligned + // buffer (112 MB/frame). They are hashed per push instead - 8 KB of XXH3 at + // worst - and their containing block ships only on a change. + Uint64 edgeHashHead = 0; + Uint64 edgeHashTail = 0; + // GL thread only: whether the mapped range has partial-page edge spans at all. + // Feeds the epoch skip's per-member edge service (PushDrawConsumers) - edge + // bytes are never protected, so their dirtiness is invisible to the fault epoch. + Bool hasEdges = false; + }; + class PersistentMapTracker { public: // Client-role singleton (table 3: the MG_Impl/MG_Remote/Client singletons are @@ -100,6 +127,18 @@ namespace MobileGL::MG_Remote::Client { // THE VALIDATE-POINT HOOK. Every member, before the verb record is emitted. void PushAllMembers(); + // THE DRAW-VERB HOOK, FILTERED. A draw or dispatch can only read the buffers its + // own bindings name (the VAO's attribute and element buffers, the indexed + // uniform/storage/atomic/XFB binding points, the indirect and parameter slots), so + // pushing every live persistent map at every draw is a whole-range rescan of + // buffers the verb can never touch - the dominant cost on a real workload (the + // whole-range scan ran ~135x per frame over every arena in the process). Pushing + // only the consumers keeps the guarantee ("a changed byte reaches the server before + // the verb that reads it") and drops the rest. Read-only verbs (clear / blit / + // readback / present) keep PushAllMembers: they read no binding points and their + // rate is a few per frame, so the conservative whole push is cheap there. + void PushDrawConsumers(); + SizeT MemberCount() const { return m_livePersistentMaps.size(); } Uint64 BlocksPushed() const { return m_blocksPushed; } Uint64 BytesPushed() const { return m_bytesPushed; } @@ -109,20 +148,52 @@ namespace MobileGL::MG_Remote::Client { m_bytesPushed = 0; m_blockZeroAnnounced = false; } - void ClearForTest() { - m_livePersistentMaps.clear(); - ResetCountersForTest(); - } + // Unit tests only: clears the membership, the hash cache and the mprotect tracked + // slots (restoring every tracked page writable first), and resets the counters. + void ClearForTest(); private: + // One live member. tracked is the member's tracker slot when it is on the + // mprotect arm, nullptr on the hash arm; it is REFRESHED at every + // NoteMapStateChanged (a resize can re-claim a different slot) and validated + // against the slot's lifetimeId before every use, which is what makes a slot + // retired and re-claimed by another buffer safe to have pointed at. + struct MemberEntry { + MG_State::GLState::BufferObject* buffer = nullptr; + TrackedWriteMap* tracked = nullptr; + }; // Keyed on BufferObject::GetLifetimeId(), which is globally unique and never reused - // never the GL name (LIFO-recycled by glGenBuffers) and never the heap address // (recycled by the allocator). The raw pointer is safe because every removal path is // explicit: ~BufferObject and ReleaseMemory both call Forget/NoteMapStateChanged, and // PushBlocksFor re-checks the predicate before it dereferences anything it kept. - UnorderedMap m_livePersistentMaps; + UnorderedMap m_livePersistentMaps; + // Per-buffer per-block xxHash64 of the client shadow, for the dirty-block push + // (MOBILEGL_IPC_PERSISTENT_HASH_SUPPRESS): a block is pushed only when its content + // changed since the last push, instead of the whole mapped range every verb. Keyed + // the same way as m_livePersistentMaps and retired beside it in Forget(). Zero + // hashes mean "unknown", never "a block of zeros" (XXH64 of zero bytes is not zero). + struct BlockHashState { + Uint64 begin = 0; + Uint64 end = 0; + Uint64 blockBytes = 0; + Vector hashes; + }; + UnorderedMap m_blockHashes; Uint64 m_blocksPushed = 0; Uint64 m_bytesPushed = 0; + // THE EPOCH SKIP's state (see PushDrawConsumers). m_lastFaultEpoch is the fault + // counter's value at the last consumer walk. m_untrackedMembers holds EVERY + // live member that fell back to the hash arm, and any one of them vetoes the + // skip: the walk is what tells an untracked member's consumption, and scanning + // every untracked member unconditionally (the alternative, measured on device + // at VD12) costs more than the walk it replaced. + Uint64 m_lastFaultEpoch = 0; + UnorderedMap m_untrackedMembers; + // Everything of PushBlocksFor after the two role guards, for callers that already + // ran them once (the two Members hooks) - the guard's thread-local read was a + // measurable per-buffer-per-draw cost on the emutls path. + void PushBlocksForChecked(MG_State::GLState::BufferObject& buffer); // E3(a)'s diagnostic is emitted ONCE per process. PushBlocksFor runs at every validate // point of every member, so an unlatched MGLOG_W would be one line per draw per mapping // - and a control that has to grep a log cannot tell a message that fired from a diff --git a/MobileGL/MG_Remote/Transport/Doorbell.h b/MobileGL/MG_Remote/Transport/Doorbell.h index 641bc2c8..92b6d581 100644 --- a/MobileGL/MG_Remote/Transport/Doorbell.h +++ b/MobileGL/MG_Remote/Transport/Doorbell.h @@ -129,7 +129,14 @@ namespace MobileGL::MG_Remote::Transport { : start + std::chrono::milliseconds(timeoutMs); const auto spinEnd = start + std::chrono::microseconds(spinUs); - while (std::chrono::steady_clock::now() < spinEnd) { + // The clock is read once per 64 spins, not per spin: on the + // workload device a steady_clock read is a REAL syscall (no vDSO), + // and per-iteration reads made this loop the profile's top line + // (~20% of the whole process at ~1000 waits per frame). The spin + // duration is a hint, so the overshoot - one batch of yields at + // most - costs nothing. + std::uint32_t spinBatch = 0; + while ((++spinBatch & 63) != 0 || std::chrono::steady_clock::now() < spinEnd) { if (ready()) { return true; } diff --git a/MobileGL/MG_Test/Wire/RemoteClientTest.cpp b/MobileGL/MG_Test/Wire/RemoteClientTest.cpp index 0278b877..5da3cd20 100644 --- a/MobileGL/MG_Test/Wire/RemoteClientTest.cpp +++ b/MobileGL/MG_Test/Wire/RemoteClientTest.cpp @@ -1447,8 +1447,16 @@ TEST(RemoteF1, GenerateMipmapFieldsCross) { const auto child = RunInChild([] { StartControlSession(); F1Peer peer; peer.Install(); const auto tex = F1Texture(); RemoteEmitTable().GL.GenerateMipmap(GL_TEXTURE_2D); + // THE FENCE. Under MOBILEGL_IPC_BATCH_WAITS (default on) GenerateMipmap is kCtxObject - + // a value-class record whose emit no longer waits for its own apply (production-safe: + // the record carries its whole value and the in-order ring applies it before the next + // waited verb). The peer lives on the apply thread, so reading it needs a wait + // boundary: a following kCtxVerb still waits, and its wait covers everything emitted + // before it. When this clear returns, the peer has seen both records. + const GLfloat fence[4] = {0, 0, 0, 0}; + RemoteEmitTable().GL.ClearBufferfv(GL_COLOR, 0, fence); const auto& r = peer.mip; - if (peer.calls != 1 || r.Res != MGPipeTextureEmitterInstance().FindTexture(*tex) || + if (peer.calls != 2 || r.Res != MGPipeTextureEmitterInstance().FindTexture(*tex) || MGPipeHandleIsNull(r.Res) || r.Target != GL_TEXTURE_2D || r.BaseLevel != 1 || r.LevelCount != 3) ::_exit(101); ClientSessionInstance().Stop(); @@ -1908,8 +1916,14 @@ MGL_TEXTURE_GUARD_TEST(TextureGetStorageDirtyRectsFromTheApplyThreadIsFatalByNam // single-writer rule. The apply thread is NOT actually inside the applier in this case - the // flag is raised by hand, which is the exact overlap window the check exists to refuse: a fill // that ran while a real apply was in flight would race the applier's reads of gPipeInputs. +// +// MOBILEGL_IPC_BATCH_WAITS (default 1) makes that overlap the INTENDED shape (the fill +// writes only fields no record supplies; pulled reads stay fenced by the pull-verbs' own +// wait), so the guard fires only with the batch off. This case forces it off and keeps the +// Fatal; the case below it pins the batched arm as legal. TEST(RemoteGuards, ClientPipeInputsFillWhileTheApplierOwnsItIsFatalByName) { const auto child = RunInChild([] { + MG_Config::Ipc.BatchWaits = 0; StartControlSession(); ClientSession::NoteApplyThreadEnteredApplier(); MGPipeValidateForVerb(MGPipeVerb::Clear); @@ -1919,6 +1933,22 @@ TEST(RemoteGuards, ClientPipeInputsFillWhileTheApplierOwnsItIsFatalByName) { ExpectNamedAbort(child, "Fatal{RoleViolation, \"gPipeInputs\"}"); } +// The batched arm: with MOBILEGL_IPC_BATCH_WAITS=1 the fill is expected to run while the +// apply thread is inside the applier - that is the batch's whole point, and the disjoint- +// field model (record-supplied fields are never filled; pulled reads stay fenced) is what +// makes it safe. +TEST(RemoteGuards, ClientPipeInputsFillWhileTheApplierOwnsItIsAllowedWhenBatchingIsOn) { + const auto child = RunInChild([] { + MG_Config::Ipc.BatchWaits = 1; + StartControlSession(); + ClientSession::NoteApplyThreadEnteredApplier(); + MGPipeValidateForVerb(MGPipeVerb::Clear); + ClientSession::NoteApplyThreadLeftApplier(); + ClientSessionInstance().Stop(); + }); + ExpectChildSuccess(child); +} + // The same fill with the flag down is the legal shape - this is the control that keeps the // check from being "abort unconditionally". TEST(RemoteGuards, ClientPipeInputsFillWithTheApplierIdleIsAllowed) { diff --git a/docs/Disaggregated/ARCHITECTURE.md b/docs/Disaggregated/ARCHITECTURE.md index e3f70dbd..e7388ed5 100644 --- a/docs/Disaggregated/ARCHITECTURE.md +++ b/docs/Disaggregated/ARCHITECTURE.md @@ -607,6 +607,8 @@ CMake： | `MOBILEGL_IPC_STAGE_MB` | 32 | `SEG_STAGE`；目标负载 profile 显式 256 | | `MOBILEGL_IPC_SPIN_US` | 50 | park 前自旋 | | `MOBILEGL_IPC_PERSISTENT_BLOCK_KB` | 64 | persistent-map 推送块粒度；`0` 是 E3(a) 阴性对照 | +| `MOBILEGL_IPC_PERSISTENT_HASH_SUPPRESS` | 1 | 推送只发内容变了的块（追踪 buffer 走 mprotect 位图，未追踪走内容哈希）；`0` 恢复全范围推送（A/B 对照） | +| `MOBILEGL_IPC_BATCH_WAITS` | 1 | 值类记录（无 reply slot 的 kCtxState/kCtxCso/kCtxObject）发布即返回，barrier 推迟到下一个拉取类 verb；`0` 恢复 R-1 逐条 barrier（`MOBILEGL_PIPE_VERIFY` 强制 0） | | `MOBILEGL_IPC_ADOPT_TIER` | 2 | `auto/0/1/2`；P5 split 用 emulated（T2） | | `MOBILEGL_IPC_VERB_BARRIER` | 1 | 每 verb 等 `appliedSeq == emitSeq`；`0` 只作 E1 阴性对照 | | `MOBILEGL_IPC_STRICT_ERRORS` | 0 | BARRIER-PULLED residual input 提升为具名 Fatal | diff --git a/scripts/gen_pipe.py b/scripts/gen_pipe.py index 13557e02..af7dca4a 100644 --- a/scripts/gen_pipe.py +++ b/scripts/gen_pipe.py @@ -781,6 +781,25 @@ inline constexpr Uint32 MGPipeCallFlagsFor(MGPWireOp op) { : static_cast(kNone); } +// The catalogue class of every opcode, indexed the same way; index 0 is kCallClassCount, +// never a real class, for the flags table's reason. +inline constexpr Uint8 kMGPipeCallClasses[static_cast(MGPWireOp::kOpCount)] = { + /* 0 */ static_cast(kCallClassCount), +""") + for call in calls: + out.append(" /* %2d %-24s*/ static_cast(%s)," % (call.Index, call.Name, call.Class)) + out.append("};") + out.append("static_assert(sizeof(kMGPipeCallClasses) / sizeof(kMGPipeCallClasses[0]) ==") + out.append(" static_cast(MGPWireOp::kOpCount),") + out.append(" \"the class table and the opcode space disagree\");") + out.append(""" +inline constexpr MGPipeCallClass MGPipeCallClassFor(MGPWireOp op) { + const SizeT index = static_cast(op); + return index < static_cast(MGPWireOp::kOpCount) + ? static_cast(kMGPipeCallClasses[index]) + : kCallClassCount; +} + // Spot checks the generator states about its own output, so that a catalogue edit that // silently drops a flag is a build break here and not a wrong decode six packages away. static_assert(MGPipeCallFlagsFor(MGPWireOp::kInvalid) == static_cast(kNone),