From 2ab757792c021e07d5fe5395b732e34b389e6270 Mon Sep 17 00:00:00 2001 From: Swung0x48 Date: Wed, 23 Sep 2026 11:13:01 -0400 Subject: [PATCH] [MG_Remote, MG_IntegrationTest] (P7): surface ops that may bring the spawned server's backend up wait a cold-start reply budget --- MobileGL/Config.h | 15 +++++ MobileGL/ConfigLoader.cpp | 10 +++- MobileGL/MG_IntegrationTest/CMakeLists.txt | 42 ++++++++++++++ MobileGL/MG_Remote/CONTRACT-P5.md | 2 + MobileGL/MG_Remote/Client/ClientSession.cpp | 64 +++++++++++++++++---- MobileGL/MG_Remote/Client/ClientSession.h | 5 ++ MobileGL/MG_Remote/Server/ServerLoop.cpp | 38 ++++++++++++ 7 files changed, 162 insertions(+), 14 deletions(-) diff --git a/MobileGL/Config.h b/MobileGL/Config.h index 5ac0f989..db8f63d4 100644 --- a/MobileGL/Config.h +++ b/MobileGL/Config.h @@ -564,6 +564,21 @@ namespace MobileGL::MG_Config { // client that is already CPU-bound and costs a frame of latency, which is why the // default is 1 and not "as deep as the ring". Uint32 PresentCredit = 1; + // MOBILEGL_IPC_CONTROL_TIMEOUT_MS (CONTRACT-P6 §5.4 D5b): how long a spawn/tcp client + // waits for the SurfaceReply to one surface-control op (eglCreate*Surface, MakeCurrent, + // ...) once the server's backend is up. Its expiry is NOT fatal: the doorbell's death + // latch decides dead (device lost) from alive-but-silent (a named diagnostic). The + // contract named this knob from P6 on; the client hard-coded its default until P7. + Uint32 ControlTimeoutMs = 5000; + // MOBILEGL_IPC_COLD_START_MS (P7): the same wait for the three ops a server may bring its + // NATIVE backend up inside - CreatePbufferSurface, CreateWindowSurface, MakeCurrent - + // until the session's first MakeCurrent is answered ok. The bring-up is lazy (Espryt's + // eglInitialize, Magma's Vulkan instance and device), ~100 ms on a workstation and more + // than the steady 5 s on a loaded CI runner (retrace-split spawn legs, runs 35671704873 + // and 35706183230). Never shorter than CONTROL_TIMEOUT_MS; its expiry is the same + // non-fatal named answer. The default is the spawn connect budget, for the same reason: + // "the server has to create a backend, and a cold software rasteriser is not fast". + Uint32 ColdStartMs = 20000; // MOBILEGL_IPC_STRICT_ERRORS: promote a BARRIER-PULLED field read - and, in a split // build, the seven sticky forwards that are otherwise exempt - from "count it in // rsp" to Fatal (R-7.3). diff --git a/MobileGL/ConfigLoader.cpp b/MobileGL/ConfigLoader.cpp index 2f898ebc..5e317734 100644 --- a/MobileGL/ConfigLoader.cpp +++ b/MobileGL/ConfigLoader.cpp @@ -423,6 +423,10 @@ namespace MobileGL::MG_ConfigLoader { // deeper queue buys nothing on a CPU-bound client and pays for it in latency. 0 is NOT // admitted: a credit of zero would mean "publish no present at all". ipc.PresentCredit = QueryEnvUint32("MOBILEGL_IPC_PRESENT_CREDIT", 1, 1, 8); + // CONTRACT-P6 D5b's reply bound and P7's cold-start budget (Config.h has the semantics). + // Both are floored at 100 ms: a zero bound would declare every live server silent. + ipc.ControlTimeoutMs = QueryEnvUint32("MOBILEGL_IPC_CONTROL_TIMEOUT_MS", 5000, 100, 600000); + ipc.ColdStartMs = QueryEnvUint32("MOBILEGL_IPC_COLD_START_MS", 20000, 100, 600000); ipc.StrictErrors = QueryEnvFlag("MOBILEGL_IPC_STRICT_ERRORS"); ipc.Audit = QueryEnvFlag("MOBILEGL_IPC_AUDIT"); QueryEnvVariable("MOBILEGL_IPC_SERVER_AFFINITY", ipc.ServerAffinity, "auto"); @@ -466,10 +470,10 @@ namespace MobileGL::MG_ConfigLoader { // One line, on the arm where these numbers decide behaviour, because every one of // them is a number a bug report has to quote. MGLOG_I("Config: IPC ring=%uMiB stage=%uMiB wire-deferred=%uMiB spin=%uus persistent-block=%uKiB " - "adopt-tier=%u verb-barrier=%u run-ahead=%u present-credit=%u strict=%d " - "audit=%d role-split-state=%d affinity='%s'", + "adopt-tier=%u verb-barrier=%u run-ahead=%u present-credit=%u control-timeout=%ums " + "cold-start=%ums strict=%d audit=%d role-split-state=%d affinity='%s'", ipc.RingMb, ipc.StageMb, ipc.WireDeferredMb, ipc.SpinUs, ipc.PersistentBlockKb, ipc.AdoptTier, - ipc.VerbBarrier, ipc.RunAhead, ipc.PresentCredit, + ipc.VerbBarrier, ipc.RunAhead, ipc.PresentCredit, ipc.ControlTimeoutMs, ipc.ColdStartMs, static_cast(ipc.StrictErrors), static_cast(ipc.Audit), static_cast(ipc.RoleSplitState), ipc.ServerAffinity.c_str()); if (ipc.VerbBarrier == 0) { diff --git a/MobileGL/MG_IntegrationTest/CMakeLists.txt b/MobileGL/MG_IntegrationTest/CMakeLists.txt index 541ccc50..af8d6f7b 100644 --- a/MobileGL/MG_IntegrationTest/CMakeLists.txt +++ b/MobileGL/MG_IntegrationTest/CMakeLists.txt @@ -2748,6 +2748,48 @@ if (MOBILEGL_BUILD_DISAGGREGATED) ) endforeach() + # ---- P7 CI: THE SPAWN CLIENT'S COLD-START BUDGET ---------------------------------------- + # + # A spawned server brings its native backend up lazily, inside the first surface creation + # (Espryt) or the first MakeCurrent (Magma). On loaded CI runners that took longer than the + # client's 5 s steady reply bound, and retrace-split's spawn legs died on "no SurfaceReply for + # CreatePbufferSurface seq 2" / "for MakeCurrent seq 3" while the server was about to answer. + # ClientSession now gives those two kinds of op MOBILEGL_IPC_COLD_START_MS (default 20 s) + # until the session's first MakeCurrent is answered ok. + # + # These two entries are that rule's red-once, made FAST: the server's test lever + # MOBILEGL_TEST_DELAY_FIRST_BRINGUP_MS holds the first surface creation and the first + # MakeCurrent back by 3 s each, and the steady bound is lowered to 2 s. With the cold-start + # budget both ops land inside 20 s and the case is green; without it (every op on the steady + # bound, which is what the client did before) the first one times out and the case reds on + # "remote spawn bring-up failed" - the CI failure in miniature, without waiting 5 s. + # NOT on integration-spawn: this is not a copy of the split lane (spawn_lane_parity.py). The + # DirectVulkan entry does carry integration-gpu, unlike the Magma arms' informational tiers + # below: it is one expected-GREEN case, and Magma's bring-up is the MakeCurrent half. + set(MGL_COLDSTART_FILTER "ClearThenReadPixelsScenario.ClearWithNoDrawIsVisibleToDefaultFramebufferReadPixels") + foreach(mglColdBackend DirectGLES DirectVulkan) + if(mglColdBackend STREQUAL "DirectVulkan") + set(mglColdDriverEnv ${MGL_ITEST_VULKAN_ENV}) + else() + set(mglColdDriverEnv ${MGL_ITEST_COMMON_ENV}) + endif() + mgl_itest_join_environment(MGL_COLDSTART_ENV_${mglColdBackend} + "MOBILEGL_BACKEND_TYPE=${mglColdBackend}" ${MGL_ITEST_SPAWN_ARM_TRANSPORT_ENV} + "MGITEST_SPLIT_LANE=1" "MOBILEGL_ITEST_REQUIRE_GPU=1" + "MOBILEGL_TEST_DELAY_FIRST_BRINGUP_MS=3000" "MOBILEGL_IPC_CONTROL_TIMEOUT_MS=2000" + "MOBILEGL_LOG_FILE_PATH=${CMAKE_CURRENT_BINARY_DIR}/cold-start-spawn-${mglColdBackend}.log" + ${MGL_ITEST_CAPABILITY_ENV} ${mglColdDriverEnv}) + gtest_discover_tests(MobileGLIntegrationTest + TEST_PREFIX "${mglColdBackend}.Spawn.ColdStart." + TEST_FILTER "${MGL_COLDSTART_FILTER}" + DISCOVERY_TIMEOUT 30 + PROPERTIES + LABELS "integration-gpu\;integration-cold-start" + TIMEOUT ${MGL_ITEST_TIMEOUT} + ENVIRONMENT "${MGL_COLDSTART_ENV_${mglColdBackend}}" + ) + endforeach() + # ---- D2: THE TEXTURE / PROGRAM SPLIT-SCENARIO CENSUS ------------------------------------ # # PLAN-PH-P34B-P7.md 1.2, "门：纹理 / program 场景 split 覆盖". Each candidate was run once diff --git a/MobileGL/MG_Remote/CONTRACT-P5.md b/MobileGL/MG_Remote/CONTRACT-P5.md index ed23ac26..cac88c02 100644 --- a/MobileGL/MG_Remote/CONTRACT-P5.md +++ b/MobileGL/MG_Remote/CONTRACT-P5.md @@ -394,6 +394,8 @@ is a pull-build symbol and G1 admits **no resize**, which is the same reason the | `MOBILEGL_IPC_STRICT_ERRORS` | 0 | promotes BARRIER-PULLED reads — and, in a split build, the seven sticky forwards — to `Fatal`. | | `MOBILEGL_IPC_AUDIT` | 0 | `0xDD` over retired staging bytes (rule C's mechanical control). | | `MOBILEGL_IPC_SERVER_AFFINITY` | `auto` | kept as the raw string; whoever starts the apply thread logs the **resolved mask**, because an affinity that silently did nothing looks exactly like one that worked. | +| `MOBILEGL_IPC_CONTROL_TIMEOUT_MS` | 5000 | (P6 D5b, parsed from P7) a spawn/tcp client's wait for one surface-control `SurfaceReply` once the server's backend is up. Expiry is **not** fatal: dead latches device-lost, alive-but-silent is a named diagnostic. Range 100..600000. | +| `MOBILEGL_IPC_COLD_START_MS` | 20000 | (P7) the same wait for `CreatePbufferSurface` / `CreateWindowSurface` / `MakeCurrent` until the session's first `MakeCurrent` is answered ok - the ops a server brings its native backend up inside, lazily. Never shorter than `CONTROL_TIMEOUT_MS`. Measured need: retrace-split spawn legs outlasted 5 s on loaded CI runners. The test lever is the server's `MOBILEGL_TEST_DELAY_FIRST_BRINGUP_MS`. | **One consequence, stated so it is not rediscovered.** In a build *without* `MOBILEGL_BUILD_DISAGGREGATED`, `MOBILEGL_TRANSPORT=inproc` is accepted by the environment and diff --git a/MobileGL/MG_Remote/Client/ClientSession.cpp b/MobileGL/MG_Remote/Client/ClientSession.cpp index b98fcaf4..231414e0 100644 --- a/MobileGL/MG_Remote/Client/ClientSession.cpp +++ b/MobileGL/MG_Remote/Client/ClientSession.cpp @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -79,7 +80,9 @@ namespace MobileGL::MG_Remote::Client { constexpr Uint32 kHandshakeTimeoutMs = 5000; // cp (CONTRACT-P6 D5b): bounded, and its expiry is NOT fatal. A peer that // is gone is a different fact from a peer that is slow, and only the - // doorbell death latch may say which. + // doorbell death latch may say which. The surface-control reply wait itself reads + // MOBILEGL_IPC_CONTROL_TIMEOUT_MS / MOBILEGL_IPC_COLD_START_MS (ControlReplyBudgetMs + // below); this constant bounds the log-flush round trip, which is not a control op. constexpr Uint32 kRemoteControlTimeoutMs = 5000; // The server has to create a backend before it binds, and a cold // software rasteriser is not fast. Bounded, and a NAMED refusal at the end. @@ -102,6 +105,34 @@ namespace MobileGL::MG_Remote::Client { ::std::atomic g_bringUpsInFlight{0}; thread_local Bool tl_bringingUpHere = false; + // ---- P7 CI: THE COLD-START BUDGET OF A SURFACE-CONTROL REPLY ----------------------- + // + // A spawned (or tcp) server brings its NATIVE backend up lazily, INSIDE the first + // surface creation (Espryt: eglInitialize under CreatePbufferSurface/CreateWindowSurface) + // or the first MakeCurrent (Magma: the Vulkan instance and device) - ServerLoop.cpp's + // ApplySurfaceControlFrame. On a workstation that is ~100 ms; on a loaded CI runner it + // took longer than the 5 s steady-state bound, and the retrace-split spawn legs died on + // "no SurfaceReply for CreatePbufferSurface seq 2" / "for MakeCurrent seq 3" with the + // server still alive and about to answer (11 of 77 legs in run 35671704873, 17 of 77 in + // 35706183230, a different set each run). The server was never wedged; the client simply + // asked a cold software rasteriser for a warm one's latency. + // + // So until the session's first MakeCurrent is answered ok, those three ops wait the + // cold-start budget; every other op, and all three once the backend is warm, keep the + // steady bound. The budget is never shorter than the steady bound, and its expiry is + // the same non-fatal, NAMED answer (D5b) - a longer wait, not a different contract. + Bool MayBringTheBackendUp(Server::SurfaceControlOp kind) { + return kind == Server::SurfaceControlOp::CreatePbufferSurface || + kind == Server::SurfaceControlOp::CreateWindowSurface || + kind == Server::SurfaceControlOp::MakeCurrent; + } + + Uint32 ControlReplyBudgetMs(Server::SurfaceControlOp kind, Bool backendWarm, Bool* coldStart) { + const Uint32 steady = MG_Config::Ipc.ControlTimeoutMs; + *coldStart = !backendWarm && MayBringTheBackendUp(kind); + return *coldStart ? std::max(steady, MG_Config::Ipc.ColdStartMs) : steady; + } + struct BringUpScope { BringUpScope() { g_bringUpsInFlight.fetch_add(1, ::std::memory_order_acq_rel); @@ -1198,12 +1229,16 @@ namespace MobileGL::MG_Remote::Client { // caller-serialised on the server's m_callerMutex under inproc; the same // discipline has to hold here or two replies race for one correlation. const std::lock_guard lock(m_remoteControlMutex); + // The whole op - prefix drain, reply, event delivery - runs on one budget: the cold-start + // one while the server may still be bringing its backend up (see ControlReplyBudgetMs). + Bool coldStart = false; + const Uint32 budgetMs = ControlReplyBudgetMs(frame.kind, m_serverBackendWarm, &coldStart); // Control and data are independent ordered streams. Finish the old // command prefix before a surface operation can change its EGL tuple. if (m_started && m_link && !m_link->Capabilities().PublishIsDelivery) { const auto submitted = m_producer.LastPublishedSeq(); m_producer.PublishAndNotify(submitted); - if (WaitForAppliedBudget(m_producer, *m_events, submitted, kRemoteControlTimeoutMs) != + if (WaitForAppliedBudget(m_producer, *m_events, submitted, budgetMs) != Transport::SessionWait::Reached) return MOBILEGL_ERR_TRANSPORT_CLOSED; DrainEventRing(*m_events); } @@ -1218,8 +1253,7 @@ namespace MobileGL::MG_Remote::Client { // and only the doorbell's death latch may say which. for (;;) { std::vector reply; - const MobileGLResult received = - ReceiveControlFrame(reply, kRemoteControlTimeoutMs); + const MobileGLResult received = ReceiveControlFrame(reply, budgetMs); if (received != MOBILEGL_OK) { // `dl` (D5b): THE EXPIRY IS NOT FATAL AND IT IS NOT ONE ANSWER. The transport // already distinguishes the two states this wait can end in, and the contract @@ -1240,15 +1274,18 @@ namespace MobileGL::MG_Remote::Client { LatchDeviceLost("a control reply timed out and the peer had hung up"); } else { MGLOG_E("MG_Remote client: the server is ALIVE BUT SILENT - no SurfaceReply " - "for %s seq %llu within %u ms and the control stream is still open. " - "This is NOT device loss (CONTRACT-P6 5.4): a frozen or merely slow " - "peer produces no hangup, and only a hangup may arm the latch. If " - "this repeats, the server is wedged rather than gone.", + "for %s seq %llu within %u ms (%s) and the control stream is still " + "open. This is NOT device loss (CONTRACT-P6 5.4): a frozen or merely " + "slow peer produces no hangup, and only a hangup may arm the latch. " + "If this repeats, the server is wedged rather than gone.", Server::SurfaceControlOpName(frame.kind), - static_cast(frame.seq), kRemoteControlTimeoutMs); + static_cast(frame.seq), budgetMs, + coldStart ? "the cold-start budget, MOBILEGL_IPC_COLD_START_MS: the " + "server may still have been bringing its backend up" + : "the steady bound, MOBILEGL_IPC_CONTROL_TIMEOUT_MS"); } MGLOG_E("MG_Remote client: no SurfaceReply for seq %llu within %u ms (rc=%d)", - static_cast(frame.seq), kRemoteControlTimeoutMs, + static_cast(frame.seq), budgetMs, static_cast(received)); return received; } @@ -1297,9 +1334,12 @@ namespace MobileGL::MG_Remote::Client { } MG_Remote::DecodeWireSurfaceReply(*wire, &frame); if (m_link) { - const auto delivered = m_link->WaitForEventDelivery(frame.eventHead, kRemoteControlTimeoutMs); + const auto delivered = m_link->WaitForEventDelivery(frame.eventHead, budgetMs); if (delivered != MOBILEGL_OK) return delivered; } + // A MakeCurrent the server answered ok has brought its backend up: from here on + // every op keeps the steady bound. + if (frame.kind == Server::SurfaceControlOp::MakeCurrent && frame.ok) m_serverBackendWarm = true; return MOBILEGL_OK; } } @@ -1597,6 +1637,8 @@ namespace MobileGL::MG_Remote::Client { // for the same reason: the credit's id space belongs to the session that paces on it. m_runAheadArmed = false; m_runAheadLatched = false; + // The cold-start window is the next server's, not this one's (RunRemoteSurfaceControlFrame). + m_serverBackendWarm = false; m_presentsSent = 0; if (g_active == this) { g_active = nullptr; diff --git a/MobileGL/MG_Remote/Client/ClientSession.h b/MobileGL/MG_Remote/Client/ClientSession.h index 01ab3da4..df0d3fcf 100644 --- a/MobileGL/MG_Remote/Client/ClientSession.h +++ b/MobileGL/MG_Remote/Client/ClientSession.h @@ -445,6 +445,11 @@ namespace MobileGL::MG_Remote::Client { // cp: one control op at a time, and the client's own seq space. std::mutex m_remoteControlMutex; Uint64 m_remoteControlSeq = 0; + // P7 CI: false until this session's first MakeCurrent is answered ok. Until then the + // ops the server may bring its native backend up inside wait MOBILEGL_IPC_COLD_START_MS + // for their reply instead of MOBILEGL_IPC_CONTROL_TIMEOUT_MS (Config.h has both). + // Guarded by m_remoteControlMutex, like the seq above; reset by Stop(). + Bool m_serverBackendWarm = false; // The server we launched, if we launched one. Reaped in Stop(). Server::LaunchedServer m_spawned; diff --git a/MobileGL/MG_Remote/Server/ServerLoop.cpp b/MobileGL/MG_Remote/Server/ServerLoop.cpp index 6f6c32b5..871b1cd8 100644 --- a/MobileGL/MG_Remote/Server/ServerLoop.cpp +++ b/MobileGL/MG_Remote/Server/ServerLoop.cpp @@ -17,9 +17,11 @@ #include #include +#include #include #include #include +#include #if defined(__linux__) || defined(__ANDROID__) #include @@ -897,6 +899,41 @@ namespace MobileGL::MG_Remote::Server { return handle; } + // P7 CI. THE TEST LEVER FOR THE CLIENT'S COLD-START BUDGET (ClientSession.cpp, + // ControlReplyBudgetMs), in the style of MOBILEGL_TEST_DELAY_FIRST_CAPS_MS. + // + // MOBILEGL_TEST_DELAY_FIRST_BRINGUP_MS holds back this process's FIRST surface creation + // and its FIRST MakeCurrent by that many milliseconds each - the two dispatches a lazy + // native backend bring-up runs inside (Espryt's eglInitialize, Magma's instance and + // device). It stands in for a cold software rasteriser on a loaded runner, where that + // bring-up outlasted the client's steady reply bound and the retrace-split spawn legs died + // with the server still alive. Only the reply is late; the op itself is unchanged. + // + // Meant for a server in its OWN process (spawn / tcp), which is where the client's reply + // bound is. It is a TEST knob: it does nothing unless set, and is read once. + void DelayFirstBringUpForTest(SurfaceControlOp kind) { + static std::atomic surfaceSpent{false}; + static std::atomic makeCurrentSpent{false}; + std::atomic* spent = nullptr; + if (kind == SurfaceControlOp::CreatePbufferSurface || kind == SurfaceControlOp::CreateWindowSurface) { + spent = &surfaceSpent; + } else if (kind == SurfaceControlOp::MakeCurrent) { + spent = &makeCurrentSpent; + } else { + return; + } + static const long ms = [] { + const char* text = std::getenv("MOBILEGL_TEST_DELAY_FIRST_BRINGUP_MS"); + return (text == nullptr || *text == '\0') ? 0L : std::strtol(text, nullptr, 10); + }(); + if (ms <= 0 || spent->exchange(true, std::memory_order_acq_rel)) return; + MGLOG_W("MG_Remote server: MOBILEGL_TEST_DELAY_FIRST_BRINGUP_MS=%ld - the reply to the " + "first %s is held back by that many milliseconds. This is the cold-start " + "budget's test-only lever and must never be set in a measured run", + ms, SurfaceControlOpName(kind)); + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + } + } // namespace MobileGLResult ServerLoop::ApplySurfaceControlFrame(SurfaceControlFrame& frame, @@ -914,6 +951,7 @@ namespace MobileGL::MG_Remote::Server { // process whose split bring-up failed must see a refusal, not a crash. MG_Backend::BackendObject* backend = m_backend.get(); if (backend == nullptr) return MOBILEGL_ERR_NOT_INITIALIZED; + DelayFirstBringUpForTest(frame.kind); switch (frame.kind) { case SurfaceControlOp::InitializeDisplay: { // The old Args carried the poster's `major`/`minor` OUT-POINTERS; here they are