diff --git a/.claude/skills/anland-mobilegl-plasma/SKILL.md b/.claude/skills/anland-mobilegl-plasma/SKILL.md index b8fbaecd..73e4c782 100644 --- a/.claude/skills/anland-mobilegl-plasma/SKILL.md +++ b/.claude/skills/anland-mobilegl-plasma/SKILL.md @@ -128,6 +128,7 @@ sleep 8; grep -a "presented through" /tmp/mgl-run.client.log Chrome: `bash scripts/chrome-launch.sh [flags]` (log `/tmp/mgl-chrome-.client.log`, screenshot `ch-.png`; `CHROME_GPU=in-process` puts the GPU back in the browser, which cannot recover from a device loss). Device loss: `setprop debug.mobilegl.inject_device_lost_pid ` (as root) loses the server session of the client whose pid - as it sees itself in the container - is ``, once per value, on any backend and whether or not it presents; `debug.mobilegl.inject_device_lost_at N` instead loses whichever session makes the N-th device check (Espryt presents and read-backs only). `bash scripts/chrome-recover.sh [rounds]` checks that Chrome repaints after such a loss without a restart (battery-guarded, stops below 40%). `bash scripts/kwin-recover.sh [rounds]` loses KWin's own session (default twice) and checks that KWin restarts compositing on a fresh session - same KWin, plasmashell and client processes, the screen still animating (glmark2 on Wayland, glxgears on X11, `CHROME=1` adds Chrome) - which needs the anland backend + kwin.patch reset handling deployed (`kwin-stage.sh build`). Probes: `scripts/probes/texfmt.c` (texture formats keep alpha?), `dmahash.c` (hash a process's dma-bufs); build in the container, e.g. `gcc -O1 /root/texfmt.c -o /usr/local/bin/mgl-texfmt -lEGL`, run with `mgrun offscreen`. +GPU hangs: a shader that never ends stops the WHOLE GPU (every process, MobileGL or not) until the kernel resets it - ~2 s, or ~10 s ("Preemption Fault" in dmesg) when a higher-priority context waits; nothing in user space shortens one stall. What the server does is end the guilty session so it cannot hang the GPU again (Magma: event-bracketed submissions + `mgl-gpu-watch`, budget `MOBILEGL_GPU_HANG_BUDGET_MS` / `debug.mobilegl.gpu_hang_budget_ms`, default 2000; Espryt: the driver's GUILTY reset status, which it reports after the ~2 s reset only). Repro: `scripts/probes/hangclient.c` (Wayland + GL 3.3 core; `-H 60 -N 1 -E 60` hangs every 60 frames, `-H -1` is a frame-gap observer; build with wayland-scanner's xdg-shell files, `-lwayland-client -lwayland-egl -lEGL -lOpenGL -lm`), and `gpuprobe.c` (NDK, plain adb shell; `gpuprobe 30 high` is a high-priority GLES context outside MobileGL - it both measures the device-wide stall and forces the 10 s preemption path). A result counts only if ALL hold: - client log says `presented through linux-dmabuf shared images` (not `wl_shm`; `could not be presented through a linux-dmabuf shared image` = fell back); diff --git a/.claude/skills/anland-mobilegl-plasma/scripts/probes/gpuprobe.c b/.claude/skills/anland-mobilegl-plasma/scripts/probes/gpuprobe.c new file mode 100644 index 00000000..7fb4cc3c --- /dev/null +++ b/.claude/skills/anland-mobilegl-plasma/scripts/probes/gpuprobe.c @@ -0,0 +1,55 @@ +// gpuprobe - a plain Android process (outside MobileGL) that measures whether the GPU keeps executing +// other contexts' work: a pbuffer GLES context clears + draws and glFinish()es every ~10 ms, printing any +// round trip over 50 ms with a CLOCK_REALTIME timestamp. Usage: gpuprobe [priority: low|med|high] +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef EGL_CONTEXT_PRIORITY_LEVEL_IMG +#define EGL_CONTEXT_PRIORITY_LEVEL_IMG 0x3100 +#define EGL_CONTEXT_PRIORITY_HIGH_IMG 0x3101 +#define EGL_CONTEXT_PRIORITY_MEDIUM_IMG 0x3102 +#define EGL_CONTEXT_PRIORITY_LOW_IMG 0x3103 +#endif + +static double now_ms(void) { struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); return ts.tv_sec * 1e3 + ts.tv_nsec / 1e6; } + +int main(int argc, char** argv) { + int seconds = argc > 1 ? atoi(argv[1]) : 20; + EGLint prio = EGL_CONTEXT_PRIORITY_MEDIUM_IMG; + if (argc > 2 && !strcmp(argv[2], "high")) prio = EGL_CONTEXT_PRIORITY_HIGH_IMG; + if (argc > 2 && !strcmp(argv[2], "low")) prio = EGL_CONTEXT_PRIORITY_LOW_IMG; + setvbuf(stdout, NULL, _IOLBF, 0); + EGLDisplay d = eglGetDisplay(EGL_DEFAULT_DISPLAY); + eglInitialize(d, NULL, NULL); + const EGLint ca[] = {EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, EGL_RENDERABLE_TYPE, EGL_OPENGL_ES3_BIT, EGL_RED_SIZE, 8, EGL_NONE}; + EGLConfig c; EGLint n; + eglChooseConfig(d, ca, &c, 1, &n); + const EGLint pa[] = {EGL_WIDTH, 256, EGL_HEIGHT, 256, EGL_NONE}; + EGLSurface s = eglCreatePbufferSurface(d, c, pa); + const EGLint xa[] = {EGL_CONTEXT_CLIENT_VERSION, 3, EGL_CONTEXT_PRIORITY_LEVEL_IMG, prio, EGL_NONE}; + EGLContext x = eglCreateContext(d, c, EGL_NO_CONTEXT, xa); + if (x == EGL_NO_CONTEXT) { printf("no ctx 0x%x\n", eglGetError()); return 1; } + eglMakeCurrent(d, s, s, x); + EGLint got = 0; eglQueryContext(d, x, EGL_CONTEXT_PRIORITY_LEVEL_IMG, &got); + const double t0 = now_ms(); double maxRt = 0; long iters = 0, slow = 0; + printf("[probe] start %.0f prio=0x%x\n", t0, got); + while (now_ms() - t0 < seconds * 1000.0) { + const double a = now_ms(); + glClearColor((iters & 1) ? 1.f : 0.f, 0.f, 0.f, 1.f); + glClear(GL_COLOR_BUFFER_BIT); + glFinish(); + const double rt = now_ms() - a; + if (rt > maxRt) maxRt = rt; + if (rt > 50) { ++slow; printf("[probe] glFinish %.0f ms ending at %.0f\n", rt, now_ms()); } + ++iters; + usleep(10000); + } + printf("[probe] END iters=%ld max=%.0f ms slow=%ld\n", iters, maxRt, slow); + return 0; +} diff --git a/.claude/skills/anland-mobilegl-plasma/scripts/probes/hangclient.c b/.claude/skills/anland-mobilegl-plasma/scripts/probes/hangclient.c new file mode 100644 index 00000000..b496dae1 --- /dev/null +++ b/.claude/skills/anland-mobilegl-plasma/scripts/probes/hangclient.c @@ -0,0 +1,150 @@ +// hangclient - a Wayland xdg-toplevel, EGL + desktop GL 3.3 core client that animates and, on request, +// submits fragment work that never ends (the driver has to reset it). Prints per-frame gaps so the same +// binary measures "other app" frame pacing (frame callbacks come from the compositor) when run with -H -1. +// +// hangclient -T tag [-H hang_at_frame] [-N hang_repeat] [-I iters] [-S seconds] [-W w] [-X h] [-F] +// -H n frame at which the hanging draw is submitted (-1: never; default -1) +// -N k how many consecutive hanging frames (default 1) +// -E e re-arm a hang every e frames after the first (0: once) +// -I i loop count of the hanging shader (default 2^31-1) +// -S s run time in seconds (default 30) +// -F glFinish after every frame (puts the wait inside the client's own session) +#define GL_GLEXT_PROTOTYPES 1 +#include +#include +#include +#include +#include +#include "xdg-shell-client-protocol.h" +#include +#include +#include +#include +#include +#include + +static struct wl_compositor* g_comp; +static struct xdg_wm_base* g_wm; +static int g_configured, g_closed; +static int g_w = 480, g_h = 360; + +static void reg_global(void* d, struct wl_registry* r, uint32_t name, const char* iface, uint32_t ver) { + (void)d; + if (!strcmp(iface, "wl_compositor")) g_comp = wl_registry_bind(r, name, &wl_compositor_interface, 4); + else if (!strcmp(iface, "xdg_wm_base")) g_wm = wl_registry_bind(r, name, &xdg_wm_base_interface, 1); +} +static void reg_remove(void* d, struct wl_registry* r, uint32_t n) { (void)d; (void)r; (void)n; } +static const struct wl_registry_listener reg_l = {reg_global, reg_remove}; +static void wm_ping(void* d, struct xdg_wm_base* wm, uint32_t s) { (void)d; xdg_wm_base_pong(wm, s); } +static const struct xdg_wm_base_listener wm_l = {wm_ping}; +static void xs_conf(void* d, struct xdg_surface* s, uint32_t serial) { (void)d; xdg_surface_ack_configure(s, serial); g_configured = 1; } +static const struct xdg_surface_listener xs_l = {xs_conf}; +static void tl_conf(void* d, struct xdg_toplevel* t, int32_t w, int32_t h, struct wl_array* st) { (void)d; (void)t; (void)st; (void)w; (void)h; } +static void tl_close(void* d, struct xdg_toplevel* t) { (void)d; (void)t; g_closed = 1; } +static const struct xdg_toplevel_listener tl_l = {tl_conf, tl_close}; + +static double now_ms(void) { struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); return ts.tv_sec * 1e3 + ts.tv_nsec / 1e6; } + +static GLuint sh(GLenum type, const char* src) { + GLuint s = glCreateShader(type); + glShaderSource(s, 1, &src, NULL); + glCompileShader(s); + GLint ok = 0; glGetShaderiv(s, GL_COMPILE_STATUS, &ok); + if (!ok) { char log[2048]; glGetShaderInfoLog(s, sizeof log, NULL, log); fprintf(stderr, "shader: %s\n", log); exit(2); } + return s; +} + +int main(int argc, char** argv) { + const char* tag = "hc"; int hangAt = -1, hangN = 1, every = 0, seconds = 30, finish = 0; long iters = 2147483647L; + for (int i = 1; i < argc; ++i) { + if (!strcmp(argv[i], "-T") && i + 1 < argc) tag = argv[++i]; + else if (!strcmp(argv[i], "-H") && i + 1 < argc) hangAt = atoi(argv[++i]); + else if (!strcmp(argv[i], "-N") && i + 1 < argc) hangN = atoi(argv[++i]); + else if (!strcmp(argv[i], "-E") && i + 1 < argc) every = atoi(argv[++i]); + else if (!strcmp(argv[i], "-I") && i + 1 < argc) iters = atol(argv[++i]); + else if (!strcmp(argv[i], "-S") && i + 1 < argc) seconds = atoi(argv[++i]); + else if (!strcmp(argv[i], "-W") && i + 1 < argc) g_w = atoi(argv[++i]); + else if (!strcmp(argv[i], "-X") && i + 1 < argc) g_h = atoi(argv[++i]); + else if (!strcmp(argv[i], "-F")) finish = 1; + } + setvbuf(stdout, NULL, _IOLBF, 0); + struct wl_display* dpy = wl_display_connect(NULL); + if (!dpy) { fprintf(stderr, "no wayland\n"); return 1; } + struct wl_registry* reg = wl_display_get_registry(dpy); + wl_registry_add_listener(reg, ®_l, NULL); + wl_display_roundtrip(dpy); + if (!g_comp || !g_wm) { fprintf(stderr, "no compositor/xdg_wm_base\n"); return 1; } + xdg_wm_base_add_listener(g_wm, &wm_l, NULL); + struct wl_surface* surf = wl_compositor_create_surface(g_comp); + struct xdg_surface* xs = xdg_wm_base_get_xdg_surface(g_wm, surf); + xdg_surface_add_listener(xs, &xs_l, NULL); + struct xdg_toplevel* tl = xdg_surface_get_toplevel(xs); + xdg_toplevel_add_listener(tl, &tl_l, NULL); + xdg_toplevel_set_title(tl, tag); + wl_surface_commit(surf); + while (!g_configured) wl_display_dispatch(dpy); + + EGLDisplay ed = eglGetPlatformDisplay(EGL_PLATFORM_WAYLAND_KHR, dpy, NULL); + if (!eglInitialize(ed, NULL, NULL)) { fprintf(stderr, "eglInitialize\n"); return 1; } + eglBindAPI(EGL_OPENGL_API); + const EGLint ca[] = {EGL_SURFACE_TYPE, EGL_WINDOW_BIT, EGL_RENDERABLE_TYPE, EGL_OPENGL_BIT, EGL_RED_SIZE, 8, EGL_GREEN_SIZE, 8, EGL_BLUE_SIZE, 8, EGL_NONE}; + EGLConfig cfg; EGLint n = 0; + if (!eglChooseConfig(ed, ca, &cfg, 1, &n) || n < 1) { fprintf(stderr, "no config\n"); return 1; } + const EGLint xa[] = {EGL_CONTEXT_MAJOR_VERSION, 3, EGL_CONTEXT_MINOR_VERSION, 3, EGL_CONTEXT_OPENGL_PROFILE_MASK, EGL_CONTEXT_OPENGL_CORE_PROFILE_BIT, EGL_NONE}; + EGLContext ctx = eglCreateContext(ed, cfg, EGL_NO_CONTEXT, xa); + if (ctx == EGL_NO_CONTEXT) { fprintf(stderr, "no context 0x%x\n", eglGetError()); return 1; } + struct wl_egl_window* ew = wl_egl_window_create(surf, g_w, g_h); + EGLSurface es = eglCreatePlatformWindowSurface(ed, cfg, ew, NULL); + eglMakeCurrent(ed, es, es, ctx); + eglSwapInterval(ed, 1); + + const char* vs = "#version 330 core\nlayout(location=0) in vec2 p; uniform float t; out vec2 uv;\n" + "void main(){ uv=p*0.5+0.5; gl_Position=vec4(p*0.4+vec2(0.5*sin(t),0.0),0.0,1.0);}"; + const char* fs = "#version 330 core\nin vec2 uv; uniform int iters; uniform float t; out vec4 c;\n" + "void main(){ float a=uv.x; for(int i=0;i= 0 && frames >= hangAt) { + long rel = frames - hangAt; + if (every > 0) rel %= every; + hang = rel < hangN; + } + glViewport(0, 0, g_w, g_h); + glClearColor(0.5f + 0.5f * (float)sin(t * 3.0), 0.2f, 0.4f, 1.0f); + glClear(GL_COLOR_BUFFER_BIT); + glUseProgram(prog); + glUniform1i(uIt, hang ? (int)iters : 0); + glUniform1f(uT, (float)t); + glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + if (hang) printf("[%s] HANG submitted frame %ld at %.0f\n", tag, frames, now_ms()); + if (finish) glFinish(); + if (!eglSwapBuffers(ed, es)) printf("[%s] swap failed 0x%x at %.0f\n", tag, eglGetError(), now_ms()); + const GLenum rs = glGetError(); + if (rs != GL_NO_ERROR) printf("[%s] glGetError 0x%x at %.0f\n", tag, rs, now_ms()); + const double nowv = now_ms(), gap = nowv - last; + last = nowv; ++frames; + if (gap > maxGap) maxGap = gap; + if (gap > 100.0) { ++gaps100; printf("[%s] GAP %.0f ms ending at %.0f (frame %ld)\n", tag, gap, nowv, frames); } + if (frames % 120 == 0) printf("[%s] frame %ld t=%.1fs fps=%.1f\n", tag, frames, (nowv - t0) / 1e3, frames * 1e3 / (nowv - t0)); + } + const double total = now_ms() - t0; + printf("[%s] END frames=%ld fps=%.1f maxGap=%.0f ms gaps>100ms=%ld\n", tag, frames, frames * 1e3 / total, maxGap, gaps100); + return 0; +} diff --git a/CMakeLists.txt b/CMakeLists.txt index 92d1783c..ec52c5bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -440,6 +440,7 @@ set(SOURCE_FILES MobileGL/MG_Backend/Init.cpp MobileGL/MG_Backend/BackendObject.cpp + MobileGL/MG_Backend/GpuHangWatch.cpp MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp MobileGL/MG_Backend/DirectGLES/BackendObject_DirectGLES.cpp @@ -467,6 +468,7 @@ set(SOURCE_FILES MobileGL/MG_Backend/DirectVulkan/Renderer/VkSamplerManager.cpp MobileGL/MG_Backend/DirectVulkan/Renderer/VkClearManager.cpp MobileGL/MG_Backend/DirectVulkan/Renderer/VkRenderPassManager.cpp + MobileGL/MG_Backend/DirectVulkan/Renderer/GpuProgressMarkers.cpp MobileGL/MG_State/GLState/Core.cpp MobileGL/MG_State/GLState/ShareGroupState.cpp diff --git a/MobileGL/MG_Backend/BackendObject.h b/MobileGL/MG_Backend/BackendObject.h index c178386d..5581a8e6 100644 --- a/MobileGL/MG_Backend/BackendObject.h +++ b/MobileGL/MG_Backend/BackendObject.h @@ -749,6 +749,12 @@ namespace MobileGL { // native state is process-global by construction gives this object its own copy and binds // it to the calling thread. The default has nothing per process to separate. virtual void BindSessionStateToThisThread() {} + // A served session's apply thread asks this each time it wakes, BEFORE it applies another + // record: true when this session's own GPU work faulted or hung the GPU and the backend has + // latched the session for it (MGPipeSessionLatch, family DeviceLost), so the next frame - + // which would very likely stop the GPU again - is never applied. Cheap by contract: no + // wait, at most a status read. The default knows of no fault. + virtual Bool LatchIfGpuFaulted() { return false; } // A served session's ring moved to another of its client's contexts (bind_context). A // backend whose surface follows the native context it binds needs nothing; one that keeps // a single native surface binding re-binds the surface that context was last made current diff --git a/MobileGL/MG_Backend/DirectGLES/BackendObject_DirectGLES.cpp b/MobileGL/MG_Backend/DirectGLES/BackendObject_DirectGLES.cpp index 9a89ead1..8cbbf38a 100644 --- a/MobileGL/MG_Backend/DirectGLES/BackendObject_DirectGLES.cpp +++ b/MobileGL/MG_Backend/DirectGLES/BackendObject_DirectGLES.cpp @@ -961,6 +961,14 @@ namespace MobileGL::MG_Backend::DirectGLES { return true; } + Bool BackendObject_DirectGLES::LatchIfGpuFaulted() { +#if MOBILEGL_BUILD_DISAGGREGATED + return LatchIfGuiltyBeforeApply(); +#else + return false; +#endif + } + void BackendObject_DirectGLES::Initialize() { MG_Util::BackendLoader::AcquireEGLFunctions(m_EGLFunctions); MG_Util::BackendLoader::AcquireGLESFunctions(m_GLESFunctions, m_EGLFunctions.eglGetProcAddress); diff --git a/MobileGL/MG_Backend/DirectGLES/BackendObject_DirectGLES.h b/MobileGL/MG_Backend/DirectGLES/BackendObject_DirectGLES.h index bba59102..3e618956 100644 --- a/MobileGL/MG_Backend/DirectGLES/BackendObject_DirectGLES.h +++ b/MobileGL/MG_Backend/DirectGLES/BackendObject_DirectGLES.h @@ -32,6 +32,8 @@ namespace MobileGL::MG_Backend::DirectGLES { public: ~BackendObject_DirectGLES() override; + // The driver's verdict on this session's context (DirectGLES::LatchIfDeviceLost). + Bool LatchIfGpuFaulted() override; void Initialize() override; Bool InitCapabilities() override; Bool InitWindowSurface() override; diff --git a/MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp b/MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp index a0471f06..8a8496c5 100644 --- a/MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp +++ b/MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp @@ -16982,6 +16982,53 @@ namespace MobileGL::MG_Backend::DirectGLES { #endif } + // RESET NOTIFICATION, decided once per process by its first native context (EnsureNativeContext + // says why). MOBILEGL_ESPRYT_RESET_NOTIFICATION=0 keeps the driver's default. + namespace { + // EGL_EXT_create_context_robustness (eglext.h is not part of this file's include set). + constexpr EGLint kEglResetNotificationStrategyExt = 0x3138; + constexpr EGLint kEglLoseContextOnResetExt = 0x31BF; + enum class ResetNotification : Uint8 { Undecided, On, Off }; + std::atomic g_resetNotification{ResetNotification::Undecided}; + + Bool ResetNotificationWanted() { + const ResetNotification decided = g_resetNotification.load(std::memory_order_acquire); + if (decided != ResetNotification::Undecided) return decided == ResetNotification::On; + if (const char* value = std::getenv("MOBILEGL_ESPRYT_RESET_NOTIFICATION"); value != nullptr && value[0] == '0') + return false; + if (g_Display == EGL_NO_DISPLAY || !g_EGLFuncs.eglQueryString) return false; + const char* extensions = g_EGLFuncs.eglQueryString(g_Display, EGL_EXTENSIONS); + if (extensions == nullptr) return false; + constexpr const char kWord[] = "EGL_EXT_create_context_robustness"; + for (const char* at = std::strstr(extensions, kWord); at != nullptr; at = std::strstr(at + 1, kWord)) { + const char after = at[sizeof(kWord) - 1]; + if ((at == extensions || at[-1] == ' ') && (after == '\0' || after == ' ')) return true; + } + return false; + } + + void NoteResetNotificationRefused() { + ResetNotification expected = ResetNotification::Undecided; + if (g_resetNotification.compare_exchange_strong(expected, ResetNotification::Off, std::memory_order_acq_rel)) + MGLOG_W("DirectGLES: the driver refused EGL_LOSE_CONTEXT_ON_RESET; a context that hangs the GPU is " + "not told, and its session cannot be ended for it"); + } + + void NoteResetNotificationDecided(Bool used) { + ResetNotification expected = ResetNotification::Undecided; + const ResetNotification decided = used ? ResetNotification::On : ResetNotification::Off; + if (g_resetNotification.compare_exchange_strong(expected, decided, std::memory_order_acq_rel)) + MGLOG_I("DirectGLES: native contexts %s reset notification (EGL_LOSE_CONTEXT_ON_RESET): a context " + "whose work hangs or faults the GPU %s", + used ? "ask for" : "do not ask for", + used ? "is named by the driver and ends its own session" : "is not told"); + } + } // namespace + + Bool NativeContextsNotifyResets() { + return g_resetNotification.load(std::memory_order_acquire) == ResetNotification::On; + } + static Bool EnsureNativeContext(NativeContextTuple& tuple, EGLint surfaceBit, NativeWindowType window) { if (!EnsureDisplayInitialized()) { MGLOG_E("EnsureNativeContext: the EGL display did not initialize"); @@ -17030,21 +17077,47 @@ namespace MobileGL::MG_Backend::DirectGLES { // request yields a 3.0 context that lacks the 3.1/3.2 texture targets the // capability probes exercise; mobile drivers ignore the minor and hand out // their maximum either way. + // + // RESET NOTIFICATION. With the driver's default strategy a context whose work hung the GPU + // is never told: after Adreno's ~2 s timeout reset it silently drops everything that context + // submits (the client's window just stops), and its client never learns it should recreate. + // Asking for EGL_LOSE_CONTEXT_ON_RESET lets the driver name the guilty context + // (glGetGraphicsResetStatus answers GUILTY, once) and LatchIfDeviceLost ends that session + // alone, so the client reads a lost context and can recover. (A ~10 s reset that a failed + // preemption ends was not reported to the server's contexts in measurement - see + // GpuHangWatch.h.) Every context of the process shares one group, and contexts that share + // must agree on the strategy, so it is decided once, by the first context, and kept. for (const EGLint minorVersion : {2, 1, 0}) { + const Bool notify = ResetNotificationWanted(); const EGLint contextAttribs[] = {EGL_CONTEXT_MAJOR_VERSION, 3, EGL_CONTEXT_MINOR_VERSION, minorVersion, + notify ? kEglResetNotificationStrategyExt : EGL_NONE, + kEglLoseContextOnResetExt, EGL_NONE}; tuple.Context = g_EGLFuncs.eglCreateContext(g_Display, tuple.Config, share, contextAttribs); + if (tuple.Context == EGL_NO_CONTEXT && notify && share == EGL_NO_CONTEXT) { + // The process's first context: if the same version WITHOUT the strategy is accepted, + // it was the strategy the driver refused, and no context of the process asks for it. + const EGLint plainAttribs[] = {EGL_CONTEXT_MAJOR_VERSION, 3, EGL_CONTEXT_MINOR_VERSION, minorVersion, + EGL_NONE}; + tuple.Context = g_EGLFuncs.eglCreateContext(g_Display, tuple.Config, share, plainAttribs); + if (tuple.Context != EGL_NO_CONTEXT) NoteResetNotificationRefused(); + } if (tuple.Context != EGL_NO_CONTEXT) { + NoteResetNotificationDecided(notify); tuple.Serial = g_nativeContextSerials.fetch_add(1, std::memory_order_relaxed) + 1; ++g_displayUsers; return true; } } - const EGLint legacyContextAttribs[] = {EGL_CONTEXT_CLIENT_VERSION, 3, EGL_NONE}; + const Bool notify = ResetNotificationWanted(); + const EGLint legacyContextAttribs[] = {EGL_CONTEXT_CLIENT_VERSION, 3, + notify ? kEglResetNotificationStrategyExt : EGL_NONE, + kEglLoseContextOnResetExt, EGL_NONE}; tuple.Context = g_EGLFuncs.eglCreateContext(g_Display, tuple.Config, share, legacyContextAttribs); if (tuple.Context == EGL_NO_CONTEXT) return false; + NoteResetNotificationDecided(notify); tuple.Serial = g_nativeContextSerials.fetch_add(1, std::memory_order_relaxed) + 1; ++g_displayUsers; return true; @@ -18229,25 +18302,61 @@ namespace MobileGL::MG_Backend::DirectGLES { } #if MOBILEGL_BUILD_DISAGGREGATED - Bool LatchIfDeviceLost(const char* site, Bool debugKnob) { - if (MG_Config::Transport == MG_Config::TransportMode::Monolith) return false; + namespace { // Once per thread: a latched session declines everything after this verb anyway, and the // apply thread is the session's own. - static thread_local Bool latched = false; - if (latched) return true; + thread_local Bool t_deviceLossLatched = false; + + Bool LatchOnResetStatus(const char* site, GLenum status) { + if (status == GL_NO_ERROR) return false; + // ONLY THE GUILTY CONTEXT ENDS ITS SESSION. The contexts ask for reset notification + // (EnsureNativeContext) so the driver can name the context whose work hung or faulted the + // GPU; a reset it reports to an INNOCENT context - one whose work only waited behind the + // stall, the compositor's above all - is noted and the session goes on as it did before + // contexts were told. Adreno 750 measured: the guilty context reads GUILTY once, after + // which the driver drops everything it submits; innocent ones read NO_ERROR. + static thread_local Bool innocentNoted = false; + if (status == GL_INNOCENT_CONTEXT_RESET) { + if (!innocentNoted) { + innocentNoted = true; + MGLOG_W("Espryt:%s - the driver reports a GPU reset this session's context did not cause " + "(GL_INNOCENT_CONTEXT_RESET); the session goes on", + site != nullptr ? site : "?"); + } + return false; + } + t_deviceLossLatched = true; + (void)MG_Pipe::MGPipeSessionLatch(MG_Pipe::MGPipeFatalFamily::DeviceLost, + "MGPipe: Fatal{BackendDeviceLost, \"Espryt:%s\"} - this session's EGL " + "context was reset (glGetGraphicsResetStatus 0x%x); the session ends " + "and its client reads a lost context, every other session keeps running", + site != nullptr ? site : "?", static_cast(status)); + return true; + } + } // namespace + + Bool LatchIfDeviceLost(const char* site, Bool debugKnob) { + if (MG_Config::Transport == MG_Config::TransportMode::Monolith) return false; + if (t_deviceLossLatched) return true; GLenum status = GL_NO_ERROR; if (g_GLESFuncs.glGetGraphicsResetStatus != nullptr && IsBackendContextCurrentOnThisThread()) { status = g_GLESFuncs.glGetGraphicsResetStatus(); } if (status == GL_NO_ERROR && debugKnob && MG_Pipe::MGPipeDebugDeviceLossDue()) status = GL_UNKNOWN_CONTEXT_RESET; - if (status == GL_NO_ERROR) return false; - latched = true; - (void)MG_Pipe::MGPipeSessionLatch(MG_Pipe::MGPipeFatalFamily::DeviceLost, - "MGPipe: Fatal{BackendDeviceLost, \"Espryt:%s\"} - this session's EGL " - "context was reset (glGetGraphicsResetStatus 0x%x); the session ends " - "and its client reads a lost context, every other session keeps running", - site != nullptr ? site : "?", static_cast(status)); - return true; + return LatchOnResetStatus(site, status); + } + + Bool LatchIfGuiltyBeforeApply() { + if (MG_Config::Transport == MG_Config::TransportMode::Monolith) return false; + if (t_deviceLossLatched) return true; + // Asked every time the apply thread wakes, so it must leave nothing behind: not + // IsBackendContextCurrentOnThisThread, whose once-per-frame EGL verification stamp it would + // spend at the top of the loop (where the session may be between two of its contexts) - only + // whether this thread holds a native context and the backend's ownership. + if (g_GLESFuncs.glGetGraphicsResetStatus == nullptr || t_boundNativeContext == EGL_NO_CONTEXT || + g_backendContextOwnerThread.load(std::memory_order_acquire) != std::this_thread::get_id()) + return false; + return LatchOnResetStatus("apply", g_GLESFuncs.glGetGraphicsResetStatus()); } #endif diff --git a/MobileGL/MG_Backend/DirectGLES/DirectGLES.h b/MobileGL/MG_Backend/DirectGLES/DirectGLES.h index e70daef9..78021045 100644 --- a/MobileGL/MG_Backend/DirectGLES/DirectGLES.h +++ b/MobileGL/MG_Backend/DirectGLES/DirectGLES.h @@ -162,14 +162,21 @@ namespace MobileGL::MG_Backend::DirectGLES { // driver whether the calling thread's context was reset (glGetGraphicsResetStatus); a reset - // or the debug knob MGPipeDebugDeviceLossDue - latches THIS session through // MGPipeSessionLatch and answers true, and the caller returns from its verb without issuing - // more GL. Asked at the frame boundaries (Present, a shared-image present) and before a - // readback, never per draw. The contexts keep the driver's default reset strategy: asking - // every context to lose itself on reset would also end the INNOCENT ones - the compositor's - // among them - on any other client's GPU fault. + // more GL. Asked at the frame boundaries (Present, a shared-image present), before a readback, + // and by the apply loop each time it wakes (LatchIfGpuFaulted), never per draw. The contexts ask + // for reset notification (NativeContextsNotifyResets) so the driver can name the GUILTY one; + // only GUILTY (or a reset of unknown cause) ends the session - an INNOCENT report, a context + // whose work merely waited behind another's hang (the compositor's among them), is only noted. // `debugKnob` false: the knob is not consulted here (the on-screen Present, so an injected loss // lands on an offscreen client's shared-image present or readback, never on the compositor's). Bool LatchIfDeviceLost(const char* site, Bool debugKnob = true); + // The apply loop's form of it (BackendObject::LatchIfGpuFaulted), asked each time the apply thread + // wakes: the driver's report only, no debug knob, and no EGL ground-truth verification spent. + Bool LatchIfGuiltyBeforeApply(); #endif + // True once the process's native contexts were created with EGL_LOSE_CONTEXT_ON_RESET (decided + // by the first one; the driver must offer EGL_EXT_create_context_robustness). + Bool NativeContextsNotifyResets(); // GL fence sync objects, backed by native ES fences. FenceSync returns null // (the frontend then falls back to an always-signaled sync) when the calling // thread does not own the ES context. Waits/queries degrade to "signaled" in diff --git a/MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.cpp b/MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.cpp index 83e8f20b..a4fcc049 100644 --- a/MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.cpp @@ -422,6 +422,14 @@ namespace MobileGL::MG_Backend::DirectVulkan { static_cast(m_magmaSession->Id())); } + Bool BackendObject_DirectVulkan::LatchIfGpuFaulted() { +#if MOBILEGL_BUILD_DISAGGREGATED + return pVulkanRenderer != nullptr && pVulkanRenderer->LatchIfGpuHung("apply"); +#else + return false; +#endif + } + void BackendObject_DirectVulkan::Initialize() { #if MOBILEGL_BUILD_DISAGGREGATED VkBufferManager::RegisterWireResourceOps(); diff --git a/MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.h b/MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.h index 584b1cef..2b642161 100644 --- a/MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.h +++ b/MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.h @@ -28,6 +28,8 @@ namespace MobileGL::MG_Backend::DirectVulkan { void Initialize() override; void BindSessionStateToThisThread() override; + // The GPU hang watch's verdict on this session's device (VulkanRenderer::LatchIfGpuHung). + Bool LatchIfGpuFaulted() override; void OnClientContextBound(Uint64 token) override; Bool InitWindowSurface() override; Bool InitCapabilities() override; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/GpuProgressMarkers.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/GpuProgressMarkers.cpp new file mode 100644 index 00000000..681724fc --- /dev/null +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/GpuProgressMarkers.cpp @@ -0,0 +1,202 @@ +// MobileGL - MobileGL/MG_Backend/DirectVulkan/Renderer/GpuProgressMarkers.cpp +// Copyright (c) 2025-2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#include "GpuProgressMarkers.h" + +#include + +namespace MobileGL::MG_Backend::DirectVulkan { + + namespace Watch = MG_Backend::GpuHangWatch; + + void GpuProgressMarkers::Init(VkDevice device, Uint32 queueFamilyIndex) { + if (m_device != VK_NULL_HANDLE || device == VK_NULL_HANDLE) return; + if (Watch::BudgetMs() == 0) { + MGLOG_I("Magma: GPU hang watch off (MOBILEGL_GPU_HANG_BUDGET_MS=0)"); + return; + } + VkCommandPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + poolInfo.queueFamilyIndex = queueFamilyIndex; + if (vkCreateCommandPool(device, &poolInfo, nullptr, &m_pool) != VK_SUCCESS) { + MGLOG_W("Magma: GPU hang watch off - no command pool for its markers"); + m_pool = VK_NULL_HANDLE; + return; + } + m_device = device; + { + const std::lock_guard lock(m_mutex); + if (!AddSlot()) { + MGLOG_W("Magma: GPU hang watch off - the device cannot make its progress markers"); + vkDestroyCommandPool(m_device, m_pool, nullptr); + m_pool = VK_NULL_HANDLE; + m_device = VK_NULL_HANDLE; + return; + } + } + m_watch = Watch::Register(this, &Sample, &OnHung); + MGLOG_I("Magma: GPU hang watch on - a submission the GPU started and has not finished after %u ms ends " + "its session", + Watch::BudgetMs()); + } + + void GpuProgressMarkers::Shutdown() { + // The watch first, outside m_mutex: Unregister waits out a sample in progress, which takes it. + Watch::Unregister(m_watch); + m_watch = 0; + if (m_device == VK_NULL_HANDLE) return; + const std::lock_guard lock(m_mutex); + for (Slot& slot : m_slots) { + if (slot.Started != VK_NULL_HANDLE) vkDestroyEvent(m_device, slot.Started, nullptr); + if (slot.Finished != VK_NULL_HANDLE) vkDestroyEvent(m_device, slot.Finished, nullptr); + } + m_slots.clear(); + if (m_pool != VK_NULL_HANDLE) vkDestroyCommandPool(m_device, m_pool, nullptr); // frees the buffers + m_pool = VK_NULL_HANDLE; + m_device = VK_NULL_HANDLE; + } + + Bool GpuProgressMarkers::AddSlot() { + if (m_slots.size() >= kMaxSlots) return false; + Slot slot; + const VkEventCreateInfo eventInfo{VK_STRUCTURE_TYPE_EVENT_CREATE_INFO}; + VkCommandBuffer buffers[2] = {VK_NULL_HANDLE, VK_NULL_HANDLE}; + VkCommandBufferAllocateInfo allocInfo{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + allocInfo.commandPool = m_pool; + allocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + allocInfo.commandBufferCount = 2; + const auto fail = [&] { + if (slot.Started != VK_NULL_HANDLE) vkDestroyEvent(m_device, slot.Started, nullptr); + if (slot.Finished != VK_NULL_HANDLE) vkDestroyEvent(m_device, slot.Finished, nullptr); + if (buffers[0] != VK_NULL_HANDLE) vkFreeCommandBuffers(m_device, m_pool, 2, buffers); + return false; + }; + if (vkCreateEvent(m_device, &eventInfo, nullptr, &slot.Started) != VK_SUCCESS || + vkCreateEvent(m_device, &eventInfo, nullptr, &slot.Finished) != VK_SUCCESS || + vkAllocateCommandBuffers(m_device, &allocInfo, buffers) != VK_SUCCESS) + return fail(); + // Prerecorded once and resubmitted for every submission the slot brackets; SIMULTANEOUS_USE + // because the slot is reused once its `finished` event is set, which can be a moment before + // the submission that set it formally leaves the pending state. + VkCommandBufferBeginInfo beginInfo{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + beginInfo.flags = VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT; + if (vkBeginCommandBuffer(buffers[0], &beginInfo) != VK_SUCCESS) return fail(); + vkCmdSetEvent(buffers[0], slot.Started, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT); + if (vkEndCommandBuffer(buffers[0]) != VK_SUCCESS) return fail(); + if (vkBeginCommandBuffer(buffers[1], &beginInfo) != VK_SUCCESS) return fail(); + vkCmdSetEvent(buffers[1], slot.Finished, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); + if (vkEndCommandBuffer(buffers[1]) != VK_SUCCESS) return fail(); + slot.Begin = buffers[0]; + slot.End = buffers[1]; + m_slots.push_back(slot); + return true; + } + + GpuProgressMarkers::Bracket GpuProgressMarkers::Acquire() { + Bracket bracket; + if (m_device == VK_NULL_HANDLE) return bracket; + const std::lock_guard lock(m_mutex); + if (m_lost) return bracket; + Slot* free = nullptr; + for (Slot& slot : m_slots) { + if (slot.Claimed) continue; + // Retire what finished since the watch last looked: frames come faster than its polls. + if (slot.InFlight && vkGetEventStatus(m_device, slot.Finished) == VK_EVENT_SET) slot.InFlight = false; + if (!slot.InFlight && free == nullptr) free = &slot; + } + if (free == nullptr) { + if (!AddSlot()) return bracket; + free = &m_slots.back(); + } + if (free->Dirty) { + if (vkResetEvent(m_device, free->Started) != VK_SUCCESS || + vkResetEvent(m_device, free->Finished) != VK_SUCCESS) + return bracket; + free->Dirty = false; + } + free->Claimed = true; + bracket.Begin = free->Begin; + bracket.End = free->End; + bracket.Slot = static_cast(free - m_slots.data()); + return bracket; + } + + void GpuProgressMarkers::Submitted(const Bracket& bracket) { + if (!bracket.Valid()) return; + const std::lock_guard lock(m_mutex); + if (bracket.Slot >= m_slots.size()) return; + Slot& slot = m_slots[bracket.Slot]; + slot.Claimed = false; + slot.InFlight = true; + slot.Dirty = true; + slot.Id = m_nextId++; + } + + void GpuProgressMarkers::Abandon(const Bracket& bracket) { + if (!bracket.Valid()) return; + const std::lock_guard lock(m_mutex); + if (bracket.Slot >= m_slots.size()) return; + // A failed vkQueueSubmit may still have consumed nothing or something: reset before reuse. + m_slots[bracket.Slot].Claimed = false; + m_slots[bracket.Slot].Dirty = true; + } + + Bool GpuProgressMarkers::Sample(void* self, Vector& out) { + auto* markers = static_cast(self); + const std::lock_guard lock(markers->m_mutex); + if (markers->m_lost || markers->m_device == VK_NULL_HANDLE) return false; + for (Slot& slot : markers->m_slots) { + if (!slot.InFlight) continue; + const VkResult finished = vkGetEventStatus(markers->m_device, slot.Finished); + const VkResult started = vkGetEventStatus(markers->m_device, slot.Started); + if ((finished != VK_EVENT_SET && finished != VK_EVENT_RESET) || + (started != VK_EVENT_SET && started != VK_EVENT_RESET)) { + // VK_ERROR_DEVICE_LOST: the driver already reset the device and says so itself. + markers->m_lost = true; + return false; + } + if (finished == VK_EVENT_SET) { + slot.InFlight = false; + continue; + } + out.push_back({slot.Id, started == VK_EVENT_SET, false}); + } + return true; + } + + void GpuProgressMarkers::OnHung(void* self, Uint64 submissionId, Uint64 runningMs) { + auto* markers = static_cast(self); + markers->m_hungRunningMs.store(runningMs, std::memory_order_release); + markers->m_hung.store(true, std::memory_order_release); + MGLOG_E("Magma: submission %llu of this session's VkDevice has run %llu ms on the GPU and has not finished - " + "it is what stops the GPU for every process; the session ends at its next record and submits " + "nothing more (budget %u ms, MOBILEGL_GPU_HANG_BUDGET_MS)", + static_cast(submissionId), static_cast(runningMs), + MG_Backend::GpuHangWatch::BudgetMs()); + } + + VkResult GpuProgressMarkers::SubmitBracketed(VkQueue queue, VkSubmitInfo& info, VkFence fence, + const Bracket& bracket) { + constexpr Uint32 kMaxOwn = 6; + if (!bracket.Valid() || info.commandBufferCount == 0 || info.commandBufferCount > kMaxOwn) + return vkQueueSubmit(queue, 1, &info, fence); + VkCommandBuffer buffers[kMaxOwn + 2]; + Uint32 count = 0; + buffers[count++] = bracket.Begin; + for (Uint32 i = 0; i < info.commandBufferCount; ++i) buffers[count++] = info.pCommandBuffers[i]; + buffers[count++] = bracket.End; + const VkCommandBuffer* own = info.pCommandBuffers; + const Uint32 ownCount = info.commandBufferCount; + info.pCommandBuffers = buffers; + info.commandBufferCount = count; + const VkResult result = vkQueueSubmit(queue, 1, &info, fence); + info.pCommandBuffers = own; + info.commandBufferCount = ownCount; + return result; + } + +} // namespace MobileGL::MG_Backend::DirectVulkan diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/GpuProgressMarkers.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/GpuProgressMarkers.h new file mode 100644 index 00000000..2dcbf268 --- /dev/null +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/GpuProgressMarkers.h @@ -0,0 +1,95 @@ +// MobileGL - MobileGL/MG_Backend/DirectVulkan/Renderer/GpuProgressMarkers.h +// Copyright (c) 2025-2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#pragma once + +#include "../VkIncludes.h" +#include +#include + +#include +#include + +namespace MobileGL::MG_Backend::DirectVulkan { + + // WHICH OF THIS DEVICE'S SUBMISSIONS THE GPU HAS STARTED, AND WHICH IT HAS FINISHED + // (GpuHangWatch.h says why that is the whole attribution of a GPU hang). + // + // A frame's submission is bracketed by two prerecorded one-command buffers: the first sets a + // `started` event at TOP_OF_PIPE, the last sets a `finished` event once everything before it in + // the queue has completed. The host reads both with vkGetEventStatus from the watch thread - no + // fence is shared with the renderer's own waits and resets. On Adreno 750 a submission whose + // fragment shader never ends reads started=SET from its first poll until the kernel resets the + // GPU, and a submission queued behind it reads started=RESET for the whole stall. + // + // The renderer's thread brackets submissions (Acquire / Submitted / Abandon) and asks Hung(); the + // watch thread samples. Everything Vulkan the two share - the events - is touched under m_mutex. + class GpuProgressMarkers { + public: + struct Bracket { + VkCommandBuffer Begin = VK_NULL_HANDLE; + VkCommandBuffer End = VK_NULL_HANDLE; + Uint32 Slot = ~0u; + Bool Valid() const { return Slot != ~0u; } + }; + + GpuProgressMarkers() = default; + ~GpuProgressMarkers() { Shutdown(); } + GpuProgressMarkers(const GpuProgressMarkers&) = delete; + GpuProgressMarkers& operator=(const GpuProgressMarkers&) = delete; + + // Off (no brackets, never hung) when the budget is 0 or the device cannot make events. + void Init(VkDevice device, Uint32 queueFamilyIndex); + // Before the device goes; its queue must be idle. Stops the watch first. + void Shutdown(); + + // The two buffers to put first and last in one submission (an invalid bracket when off or + // when every slot is in flight: the submission then goes unwatched). + Bracket Acquire(); + // The submission carrying `bracket` was accepted by vkQueueSubmit / was not submitted. + void Submitted(const Bracket& bracket); + void Abandon(const Bracket& bracket); + + // The watch named one of this device's submissions as hanging the GPU. + Bool Hung() const { return m_hung.load(std::memory_order_acquire); } + Uint64 HungRunningMs() const { return m_hungRunningMs.load(std::memory_order_acquire); } + Bool Active() const { return m_watch != 0; } + + // vkQueueSubmit of `info` with `bracket` wrapped around its command buffers (which it puts + // back before returning). At most 6 buffers of the caller's. + static VkResult SubmitBracketed(VkQueue queue, VkSubmitInfo& info, VkFence fence, const Bracket& bracket); + + private: + struct Slot { + VkEvent Started = VK_NULL_HANDLE; + VkEvent Finished = VK_NULL_HANDLE; + VkCommandBuffer Begin = VK_NULL_HANDLE; + VkCommandBuffer End = VK_NULL_HANDLE; + Uint64 Id = 0; + Bool InFlight = false; // submitted and not yet seen finished + Bool Claimed = false; // handed out by Acquire, not yet submitted or abandoned + Bool Dirty = false; // its events were set by a past submission: reset before reuse + }; + static constexpr Uint32 kMaxSlots = 16; + + Bool AddSlot(); + static Bool Sample(void* self, Vector& out); + static void OnHung(void* self, Uint64 submissionId, Uint64 runningMs); + + std::mutex m_mutex; + VkDevice m_device = VK_NULL_HANDLE; + VkCommandPool m_pool = VK_NULL_HANDLE; + Vector m_slots; + Uint64 m_nextId = 1; + Uint64 m_watch = 0; + Bool m_lost = false; + std::atomic m_hung{false}; + std::atomic m_hungRunningMs{0}; + }; + +} // namespace MobileGL::MG_Backend::DirectVulkan diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp index c8058705..1b5a937b 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp @@ -3275,6 +3275,9 @@ void main() { CreateAllocator(); CreateCommandPool(); +#if MOBILEGL_BUILD_DISAGGREGATED + m_progressMarkers.Init(m_device, m_physicalDevice.queueFamilies.graphicsFamily); +#endif // Frames-in-flight is a request, not a guarantee: it also seeds the swapchain image // count (SwapchainObject clamps the hint into [minImageCount, maxImageCount]). Not every @@ -3504,6 +3507,9 @@ void main() { if (m_device != VK_NULL_HANDLE) { VK_VERIFY(vkDeviceWaitIdle(m_device)); } +#if MOBILEGL_BUILD_DISAGGREGATED + m_progressMarkers.Shutdown(); +#endif // Parked surface targets (ActivateSurfaceTarget) go before the device and instance. for (auto& [_, target] : m_parkedTargets) DestroyParkedTarget(target); m_parkedTargets.clear(); @@ -14173,10 +14179,16 @@ void main() { submitInfo.pCommandBuffers = commandBuffers; #if MOBILEGL_BUILD_DISAGGREGATED // Shared-image fences staged for the work recorded since the last submission. + if (LatchIfGpuHung("flush-submit")) return false; SharedImageSubmitSync sharedImageSync; AttachSharedImageSync(submitInfo, sharedImageSync); -#endif + const GpuProgressMarkers::Bracket bracket = m_progressMarkers.Acquire(); + const VkResult result = GpuProgressMarkers::SubmitBracketed(m_graphicsQueue, submitInfo, fence, bracket); + if (result == VK_SUCCESS) m_progressMarkers.Submitted(bracket); + else m_progressMarkers.Abandon(bracket); +#else const VkResult result = vkQueueSubmit(m_graphicsQueue, 1, &submitInfo, fence); +#endif if (result != VK_SUCCESS) { NoteDeviceLoss(result, "SubmitPendingCommandBuffer vkQueueSubmit"); MGLOG_E_ONCE("SubmitPendingCommandBuffer: vkQueueSubmit returned %d", result); @@ -14307,6 +14319,31 @@ void main() { } return true; } + + Bool VulkanRenderer::LatchIfGpuHung(const char* site) { + if (!m_progressMarkers.Hung()) return false; + if (!m_deviceLost) { + // NOT LOST TO THE DRIVER: a submission that a failed preemption leaves running is reset + // without a device loss, and the next one would stop the GPU again. Treated as lost from + // here on - nothing more is submitted - and the session ends as one whose device was. + m_deviceLost = true; + MGLOG_E("DirectVulkan: the GPU hang watch named this session's VkDevice (a submission ran %llu ms " + "without finishing, at %s); the device is treated as LOST and nothing more is submitted", + static_cast(m_progressMarkers.HungRunningMs()), + site != nullptr ? site : "?"); + } + // Where no session latch is armed (no served session) the device only stays "lost". + if (!m_deviceLossLatched && MG_Pipe::MGPipeSessionLatchArmed()) { + m_deviceLossLatched = true; + (void)MG_Pipe::MGPipeSessionLatch(MG_Pipe::MGPipeFatalFamily::DeviceLost, + "MGPipe: Fatal{BackendDeviceLost, \"Magma:gpu-hang:%s\"} - this " + "session's GPU work ran past the hang budget and stopped the GPU for " + "every process; the session ends and its client reads a lost context, " + "every other session keeps running", + site != nullptr ? site : "?"); + } + return true; + } #endif Bool VulkanRenderer::FlushForSyncPoint(Uint64 submitIndex) { @@ -14656,10 +14693,17 @@ void main() { } auto submitPacket = m_frameContext.GetSubmitInfo(shouldSubmitCommandBuffer, m_imageIndexAcquired); #if MOBILEGL_BUILD_DISAGGREGATED + if (LatchIfGpuHung("present-submit")) return; SharedImageSubmitSync sharedImageSync; AttachSharedImageSync(submitPacket.submitInfo, sharedImageSync); -#endif + const GpuProgressMarkers::Bracket bracket = m_progressMarkers.Acquire(); + const VkResult presentSubmit = GpuProgressMarkers::SubmitBracketed(m_graphicsQueue, submitPacket.submitInfo, + frame.imageInFlightFence, bracket); + if (presentSubmit == VK_SUCCESS) m_progressMarkers.Submitted(bracket); + else m_progressMarkers.Abandon(bracket); +#else const VkResult presentSubmit = vkQueueSubmit(m_graphicsQueue, 1, &submitPacket.submitInfo, frame.imageInFlightFence); +#endif NoteDeviceLoss(presentSubmit, "Present vkQueueSubmit"); #if MOBILEGL_BUILD_DISAGGREGATED if (presentSubmit != VK_SUCCESS && LatchWireDeviceLoss("present-submit")) return; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h index 4b29d86f..bd0a984f 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h @@ -23,6 +23,7 @@ #include "VkTextureManager.h" #include "VkTimerQueryManager.h" #if MOBILEGL_BUILD_DISAGGREGATED +#include "GpuProgressMarkers.h" #include "WireRenderPassCompatibility.h" #include #endif @@ -408,6 +409,11 @@ namespace MobileGL::MG_Backend::DirectVulkan { // without recording or submitting anything more. False: an ordinary failure, the site // keeps the death it had. `site` names it in the latch line. Bool LatchWireDeviceLoss(const char* site); + // THE GPU HANG WATCH (GpuProgressMarkers.h): true once the watch named one of this device's + // submissions as the one that stops the GPU. The device is then treated as lost, this session + // is latched, and the caller returns from its verb without submitting anything more. Asked + // before every frame submission and by the apply loop before it applies another record. + Bool LatchIfGpuHung(const char* site); #endif // Flush gated on usefulness: only flushes when `submitIndex` is still // unsubmitted, so poll loops on already-submitted fences do not split @@ -626,6 +632,8 @@ namespace MobileGL::MG_Backend::DirectVulkan { Bool m_deviceLost = false; Bool m_deviceLossLatched = false; #if MOBILEGL_BUILD_DISAGGREGATED + // Brackets this device's frame submissions for the GPU hang watch (LatchIfGpuHung). + GpuProgressMarkers m_progressMarkers; struct WireImage { VkImage image = VK_NULL_HANDLE; VkFormat format = VK_FORMAT_UNDEFINED; diff --git a/MobileGL/MG_Backend/GpuHangWatch.cpp b/MobileGL/MG_Backend/GpuHangWatch.cpp new file mode 100644 index 00000000..701dd991 --- /dev/null +++ b/MobileGL/MG_Backend/GpuHangWatch.cpp @@ -0,0 +1,184 @@ +// MobileGL - MobileGL/MG_Backend/GpuHangWatch.cpp +// Copyright (c) 2025-2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#include "GpuHangWatch.h" + +#include + +#include +#include +#include +#include +#include +#include + +#if defined(__ANDROID__) +#include +#endif +#if defined(__linux__) || defined(__ANDROID__) +#include +#endif + +namespace MobileGL::MG_Backend::GpuHangWatch { + + namespace { + constexpr Uint32 kDefaultPollMs = 250; + std::atomic g_pollOverrideMs{0}; + std::atomic g_budgetOverrideMs{0}; + std::atomic g_budgetOverridden{false}; + + Uint64 NowMs() { + return static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); + } + + Uint32 ReadBudgetMs() { + const char* value = std::getenv("MOBILEGL_GPU_HANG_BUDGET_MS"); +#if defined(__ANDROID__) + char property[PROP_VALUE_MAX] = {}; + if (value == nullptr && __system_property_get("debug.mobilegl.gpu_hang_budget_ms", property) > 0) + value = property; +#endif + if (value == nullptr || *value == '\0') return kDefaultBudgetMs; + char* end = nullptr; + const unsigned long parsed = std::strtoul(value, &end, 10); + if (end == value) return kDefaultBudgetMs; + return static_cast(std::min(parsed, 600000ul)); + } + + struct Entry { + Uint64 Handle = 0; + void* Self = nullptr; + SampleFn Sample = nullptr; + HungFn Hung = nullptr; + Detector Decide{0}; + Bool Done = false; // named a hang, or its device is gone: never sampled again + }; + + // Leaked at exit, with its thread: the watch holds no GPU object of its own, and a backend + // unregisters before it destroys what its sampler reads. + struct Watch { + std::mutex Mutex; + std::condition_variable Wake; + Vector Entries; + Uint64 NextHandle = 1; + Bool ThreadStarted = false; + }; + + Watch& TheWatch() { + static Watch& watch = *new Watch(); + return watch; + } + + void PollOnce(Watch& watch) { + const Uint64 now = NowMs(); + Vector sample; + for (Entry& entry : watch.Entries) { + if (entry.Done) continue; + sample.clear(); + if (!entry.Sample(entry.Self, sample)) { + entry.Done = true; + continue; + } + const Uint64 hung = entry.Decide.Observe(sample, now); + if (hung == 0) continue; + entry.Done = true; + entry.Hung(entry.Self, hung, entry.Decide.RunningMs(hung, now)); + } + } + + void WatchThreadMain() { +#if defined(__linux__) || defined(__ANDROID__) + pthread_setname_np(pthread_self(), "mgl-gpu-watch"); +#endif + Watch& watch = TheWatch(); + std::unique_lock lock(watch.Mutex); + for (;;) { + watch.Wake.wait_for(lock, std::chrono::milliseconds(PollIntervalMs())); + PollOnce(watch); + } + } + } // namespace + + Uint32 BudgetMs() { + if (g_budgetOverridden.load(std::memory_order_acquire)) return g_budgetOverrideMs.load(std::memory_order_acquire); + static const Uint32 budget = ReadBudgetMs(); + return budget; + } + + Uint32 PollIntervalMs() { + const Uint32 overrideMs = g_pollOverrideMs.load(std::memory_order_acquire); + return overrideMs != 0 ? overrideMs : kDefaultPollMs; + } + + void SetPollIntervalForTesting(Uint32 ms) { g_pollOverrideMs.store(ms, std::memory_order_release); } + + void SetBudgetForTesting(Uint32 ms) { + g_budgetOverrideMs.store(ms, std::memory_order_release); + g_budgetOverridden.store(ms != 0, std::memory_order_release); + } + + Uint64 Detector::Observe(const Vector& sample, Uint64 nowMs) { + Vector still; + still.reserve(sample.size()); + Uint64 named = 0; + for (const SubmissionProgress& progress : sample) { + if (!progress.Started || progress.Finished) continue; + Uint64 since = nowMs; + for (const Running& seen : m_running) { + if (seen.Id == progress.Id) { + since = seen.SinceMs; + break; + } + } + still.push_back({progress.Id, since}); + if (named == 0 && m_budgetMs != 0 && nowMs - since >= m_budgetMs) named = progress.Id; + } + m_running.swap(still); + return named; + } + + Uint64 Detector::RunningMs(Uint64 id, Uint64 nowMs) const { + for (const Running& seen : m_running) + if (seen.Id == id) return nowMs - seen.SinceMs; + return 0; + } + + Uint64 Register(void* self, SampleFn sample, HungFn hung) { + const Uint32 budget = BudgetMs(); + if (budget == 0 || sample == nullptr || hung == nullptr) return 0; + Watch& watch = TheWatch(); + const std::lock_guard lock(watch.Mutex); + Entry entry; + entry.Handle = watch.NextHandle++; + entry.Self = self; + entry.Sample = sample; + entry.Hung = hung; + entry.Decide = Detector(budget); + watch.Entries.push_back(entry); + if (!watch.ThreadStarted) { + watch.ThreadStarted = true; + std::thread(WatchThreadMain).detach(); + } + return entry.Handle; + } + + void Unregister(Uint64 handle) { + if (handle == 0) return; + Watch& watch = TheWatch(); + // The poll pass runs under this lock, so taking it waits out any call into the entry. + const std::lock_guard lock(watch.Mutex); + for (SizeT i = 0; i < watch.Entries.size(); ++i) { + if (watch.Entries[i].Handle != handle) continue; + watch.Entries.erase(watch.Entries.begin() + static_cast(i)); + return; + } + } + +} // namespace MobileGL::MG_Backend::GpuHangWatch diff --git a/MobileGL/MG_Backend/GpuHangWatch.h b/MobileGL/MG_Backend/GpuHangWatch.h new file mode 100644 index 00000000..9f1f1663 --- /dev/null +++ b/MobileGL/MG_Backend/GpuHangWatch.h @@ -0,0 +1,94 @@ +// MobileGL - MobileGL/MG_Backend/GpuHangWatch.h +// Copyright (c) 2025-2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#pragma once + +// A SESSION WHOSE GPU WORK NEVER ENDS IS FOUND AND NAMED, SO IT CANNOT HANG THE GPU AGAIN. +// +// A submission whose shader never ends stops the whole GPU, not one context: every process's work - +// the compositor's, other clients', processes outside MobileGL at the highest context priority - +// waits until the kernel resets it. Measured on Adreno 750: ~2 s when nothing of higher priority +// waits (the context is then lost: Vulkan answers VK_ERROR_DEVICE_LOST), ~10 s when something of +// higher priority does (a failed preemption; the context is NOT lost, and its next frame hangs the +// GPU again). Nothing in user space can shorten one such stall; what the server can do is make sure +// the session that caused it never submits again, so a page or program that hangs every frame +// costs the desktop one stall instead of one per frame. +// +// The attribution must be exact - every session's work is pending while the GPU is stuck, and the +// compositor's must never be blamed. A backend that can tell, per in-flight submission, whether the +// GPU has STARTED it and whether it has FINISHED it gives exactly that: the GPU runs one submission +// at a time on a ring, so the one that started and does not finish is the one that hangs, and the +// ones queued behind it have not started. Magma brackets each submission with two events +// (DirectVulkan/Renderer/GpuProgressMarkers.h). Espryt relies on the driver instead: its contexts ask +// for reset notification and glGetGraphicsResetStatus names the guilty one - measured after a ~2 s +// timeout reset; after a ~10 s failed-preemption reset the server's contexts were NOT told (a +// standalone process's context was), so on Espryt that path is not contained yet. +// +// The Detector is the decision, pure and testable; the watch thread samples every registered backend +// on its own clock, because the session that hangs is often not running - its apply thread waits for +// its own fence, or for the backend turn another session holds while it waits for the GPU. + +#include + +#include + +namespace MobileGL::MG_Backend::GpuHangWatch { + + // MOBILEGL_GPU_HANG_BUDGET_MS: how long one submission may run once the GPU has started it before + // its session is ended as the one that hangs the GPU. Default 2000 - the kernel resets a hung GPU + // by itself after about that when nothing preempts it, and no desktop frame runs that long. 0 = off. + Uint32 BudgetMs(); + constexpr Uint32 kDefaultBudgetMs = 2000; + + // One in-flight submission as the GPU reports progress on it. + struct SubmissionProgress { + Uint64 Id = 0; // the backend's own monotonic id for it; never reused + Bool Started = false; // the GPU has begun executing it + Bool Finished = false; // ... and completed it + }; + + // Which submission, if any, has run past the budget: started, not finished, and seen in that + // state for at least `budgetMs` (from the first sample that saw it running - the watch cannot know + // when between two samples it began, so a hang is named between budget and budget + one interval). + class Detector { + public: + explicit Detector(Uint32 budgetMs) : m_budgetMs(budgetMs) {} + // One sample of every in-flight submission at `nowMs`. Returns the id of the submission that + // ran past the budget, or 0. Submissions missing from the sample are forgotten. + Uint64 Observe(const Vector& sample, Uint64 nowMs); + // How long the named submission had been seen running when Observe named it. + Uint64 RunningMs(Uint64 id, Uint64 nowMs) const; + + private: + struct Running { + Uint64 Id = 0; + Uint64 SinceMs = 0; + }; + Uint32 m_budgetMs = 0; + Vector m_running; + }; + + // ---- the process-wide watch --------------------------------------------------------------- + // + // A backend registers a sampler and a callback. The watch thread calls the sampler every + // PollIntervalMs() on its own thread, and the callback once, when the Detector names a + // submission; the backend acts on it from its own thread (it may not latch a session from this + // one). Both are called with the watch's lock held: keep them short, and never call Unregister + // from them. Unregister returns only once no call into the registration runs or will run. + using SampleFn = Bool (*)(void* self, Vector& out); // false: stop watching it + using HungFn = void (*)(void* self, Uint64 submissionId, Uint64 runningMs); + + Uint64 Register(void* self, SampleFn sample, HungFn hung); + void Unregister(Uint64 handle); + + Uint32 PollIntervalMs(); + // Tests: the interval and the budget the next Register uses (0 restores the defaults). + void SetPollIntervalForTesting(Uint32 ms); + void SetBudgetForTesting(Uint32 ms); + +} // namespace MobileGL::MG_Backend::GpuHangWatch diff --git a/MobileGL/MG_Remote/Server/ServerLoop.cpp b/MobileGL/MG_Remote/Server/ServerLoop.cpp index 629d094c..af9098bc 100644 --- a/MobileGL/MG_Remote/Server/ServerLoop.cpp +++ b/MobileGL/MG_Remote/Server/ServerLoop.cpp @@ -681,6 +681,13 @@ namespace MobileGL::MG_Remote::Server { std::optional turn(std::in_place); PumpControlRequest(); if (stopOrForfeit()) break; + // A SESSION WHOSE OWN GPU WORK HUNG THE GPU APPLIES NOTHING MORE. A shader that never + // ends stops every process's GPU work until the kernel resets it (MG_Backend/ + // GpuHangWatch.h), and the same client's next frame usually carries the same shader. The + // backend's verdict - the driver's GUILTY report on Espryt, the hang watch on Magma - is + // asked HERE, before the drain, so that frame is never applied: the session latches + // (DeviceLost, its client reads a lost context) and leaves below. Cheap by contract. + if (m_backend != nullptr && !SessionLatched()) (void)m_backend->LatchIfGpuFaulted(); if (eventRingHasRoom()) DrainRing(); if (stopOrForfeit()) break; // The session-targeted device-loss debug knob (PipeSessionFail.h): asked here, after a diff --git a/MobileGL/MG_Test/Backend/GpuHangWatchTest.cpp b/MobileGL/MG_Test/Backend/GpuHangWatchTest.cpp new file mode 100644 index 00000000..dcd33e75 --- /dev/null +++ b/MobileGL/MG_Test/Backend/GpuHangWatchTest.cpp @@ -0,0 +1,177 @@ +// MobileGL - MobileGL/MG_Test/Backend/GpuHangWatchTest.cpp +// Copyright (c) 2025-2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +// THE GPU HANG WATCH (MG_Backend/GpuHangWatch.h): which session's submission stops the GPU. +// +// While one submission hangs, EVERY session's work is pending - the compositor's included - so the +// attribution may only rest on "the GPU started it and it did not finish". These cases pin that: +// * the submission seen running past the budget is named, and only once; +// * a submission queued behind it (never started) is never named, however long it waits - red with +// the Detector counting pending-but-not-started work as running (the compositor would be blamed); +// * a long but finishing submission is never named; +// * the budget runs from the first sample that saw it running, not from submission; +// * on the watch thread: a fake device whose submission never finishes is named within budget plus +// one interval, an innocent device waiting behind it is not, and Unregister waits out a sample. + +#include + +#include + +#include +#include +#include + +using namespace MobileGL; +namespace Watch = MobileGL::MG_Backend::GpuHangWatch; + +namespace { + Vector One(Uint64 id, Bool started, Bool finished) { + return {Watch::SubmissionProgress{id, started, finished}}; + } +} // namespace + +TEST(GpuHangDetectorTest, ASubmissionRunningPastTheBudgetIsNamed) { + Watch::Detector detector(2000); + EXPECT_EQ(detector.Observe(One(7, true, false), 10000), 0u); + EXPECT_EQ(detector.Observe(One(7, true, false), 11000), 0u); + EXPECT_EQ(detector.Observe(One(7, true, false), 11999), 0u); + EXPECT_EQ(detector.Observe(One(7, true, false), 12000), 7u); + EXPECT_EQ(detector.RunningMs(7, 12000), 2000u); +} + +TEST(GpuHangDetectorTest, ASubmissionQueuedBehindTheHangIsNeverNamed) { + // The compositor's frame waits behind a client's hang for the whole ~10 s stall: never started. + Watch::Detector detector(2000); + for (Uint64 now = 0; now <= 12000; now += 250) { + const Vector sample = {{1, true, false}, {2, false, false}}; + const Uint64 named = detector.Observe(sample, now); + if (named != 0) { + EXPECT_EQ(named, 1u) << "the queued submission was blamed at " << now << " ms"; + } + } + Watch::Detector innocentOnly(2000); + for (Uint64 now = 0; now <= 12000; now += 250) + EXPECT_EQ(innocentOnly.Observe(One(2, false, false), now), 0u) << "at " << now << " ms"; +} + +TEST(GpuHangDetectorTest, ALongSubmissionThatFinishesIsNeverNamed) { + Watch::Detector detector(2000); + EXPECT_EQ(detector.Observe(One(3, true, false), 0), 0u); + EXPECT_EQ(detector.Observe(One(3, true, false), 1900), 0u); + EXPECT_EQ(detector.Observe(One(3, true, true), 2100), 0u); + // Gone from the sample once retired: forgotten, so a new submission starts its own clock. + EXPECT_EQ(detector.Observe({}, 2200), 0u); + EXPECT_EQ(detector.Observe(One(4, true, false), 2300), 0u); + EXPECT_EQ(detector.Observe(One(4, true, false), 4200), 0u); + EXPECT_EQ(detector.Observe(One(4, true, false), 4300), 4u); +} + +TEST(GpuHangDetectorTest, TheBudgetRunsFromTheFirstSampleThatSawItRunning) { + Watch::Detector detector(2000); + // Submitted and queued for 5 s (another process's work), then started. + EXPECT_EQ(detector.Observe(One(9, false, false), 0), 0u); + EXPECT_EQ(detector.Observe(One(9, false, false), 5000), 0u); + EXPECT_EQ(detector.Observe(One(9, true, false), 5250), 0u); + EXPECT_EQ(detector.Observe(One(9, true, false), 7000), 0u); + EXPECT_EQ(detector.Observe(One(9, true, false), 7250), 9u); +} + +TEST(GpuHangDetectorTest, AZeroBudgetNamesNothing) { + Watch::Detector detector(0); + for (Uint64 now = 0; now <= 60000; now += 1000) EXPECT_EQ(detector.Observe(One(1, true, false), now), 0u); +} + +namespace { + // A device double: its one submission's progress, and what the watch told it. + struct FakeDevice { + std::atomic started{false}; + std::atomic finished{false}; + std::atomic samples{0}; + std::atomic named{0}; + std::atomic namedId{0}; + std::atomic sampleDelayMs{0}; + + static Bool Sample(void* self, Vector& out) { + auto* device = static_cast(self); + device->samples.fetch_add(1); + if (const int delay = device->sampleDelayMs.load(); delay > 0) + std::this_thread::sleep_for(std::chrono::milliseconds(delay)); + out.push_back({42, device->started.load(), device->finished.load()}); + return true; + } + static void Hung(void* self, Uint64 id, Uint64) { + auto* device = static_cast(self); + device->namedId.store(id); + device->named.fetch_add(1); + } + }; + + template + Bool PollUntil(F condition, int timeoutMs) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeoutMs); + while (std::chrono::steady_clock::now() < deadline) { + if (condition()) return true; + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return condition(); + } + + struct ScopedWatchTiming { + ScopedWatchTiming(Uint32 pollMs, Uint32 budgetMs) { + Watch::SetPollIntervalForTesting(pollMs); + Watch::SetBudgetForTesting(budgetMs); + } + ~ScopedWatchTiming() { + Watch::SetPollIntervalForTesting(0); + Watch::SetBudgetForTesting(0); + } + }; +} // namespace + +TEST(GpuHangWatchThreadTest, TheHangingDeviceIsNamedOnceAndTheOneWaitingBehindItIsNot) { + const ScopedWatchTiming timing(20, 200); + FakeDevice guilty; + FakeDevice innocent; + guilty.started = true; // the GPU runs it and it never ends + innocent.started = false; // queued behind it for the whole stall + const Uint64 a = Watch::Register(&guilty, &FakeDevice::Sample, &FakeDevice::Hung); + const Uint64 b = Watch::Register(&innocent, &FakeDevice::Sample, &FakeDevice::Hung); + ASSERT_NE(a, 0u); + ASSERT_NE(b, 0u); + const auto start = std::chrono::steady_clock::now(); + EXPECT_TRUE(PollUntil([&] { return guilty.named.load() > 0; }, 3000)); + const auto ms = std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count(); + EXPECT_GE(ms, 180) << "named before the budget ran out"; + EXPECT_LE(ms, 1500) << "named far past budget + interval"; + EXPECT_EQ(guilty.namedId.load(), 42u); + // Long after, still once, and the innocent never. + std::this_thread::sleep_for(std::chrono::milliseconds(400)); + EXPECT_EQ(guilty.named.load(), 1); + EXPECT_EQ(innocent.named.load(), 0); + EXPECT_GT(innocent.samples.load(), 5) << "the innocent device was not even sampled"; + Watch::Unregister(a); + Watch::Unregister(b); +} + +TEST(GpuHangWatchThreadTest, UnregisterWaitsOutASampleInProgressAndNothingRunsAfterIt) { + const ScopedWatchTiming timing(10, 5000); + FakeDevice device; + device.sampleDelayMs = 100; + const Uint64 handle = Watch::Register(&device, &FakeDevice::Sample, &FakeDevice::Hung); + ASSERT_NE(handle, 0u); + ASSERT_TRUE(PollUntil([&] { return device.samples.load() > 0; }, 2000)); + Watch::Unregister(handle); + const int after = device.samples.load(); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + EXPECT_EQ(device.samples.load(), after) << "the watch sampled a device after it unregistered"; +} + +TEST(GpuHangWatchThreadTest, ARegistrationWithoutASamplerIsRefused) { + EXPECT_EQ(Watch::Register(nullptr, nullptr, nullptr), 0u); + Watch::Unregister(0); // and unregistering "nothing" is harmless +} diff --git a/MobileGL/MG_Test/CMakeLists.txt b/MobileGL/MG_Test/CMakeLists.txt index b058fd4d..1df8035f 100644 --- a/MobileGL/MG_Test/CMakeLists.txt +++ b/MobileGL/MG_Test/CMakeLists.txt @@ -98,6 +98,11 @@ add_subdirectory(SelfTest) # The DirectGLES post-transpile ESSL passes are pure String -> String, so unlike the # DirectVulkan suite below this one needs no device and always builds. add_subdirectory(Backend/DirectGLES) +# Which session's submission stops the GPU (MG_Backend/GpuHangWatch.h): no device, no driver. +add_executable(GpuHangWatchTest Backend/GpuHangWatchTest.cpp) +target_include_directories(GpuHangWatchTest PRIVATE ${MGL_ROOT}/include ${MGL_ROOT}/MobileGL) +target_link_libraries(GpuHangWatchTest PRIVATE GTest::gtest_main ${LINK_LIBRARIES}) +gtest_discover_tests(GpuHangWatchTest DISCOVERY_TIMEOUT 30 PROPERTIES LABELS unit) if (ENABLE_INTEGRATION_TESTS) add_subdirectory(Backend/DirectVulkan) endif() diff --git a/MobileGL/MG_Test/Wire/ServerLoopTest.cpp b/MobileGL/MG_Test/Wire/ServerLoopTest.cpp index c6acab5c..8224c4a9 100644 --- a/MobileGL/MG_Test/Wire/ServerLoopTest.cpp +++ b/MobileGL/MG_Test/Wire/ServerLoopTest.cpp @@ -1986,6 +1986,96 @@ TEST(ServerLoopEglTest, TheInitializeDisplayReplyArrivesThroughTheFrame) { fixture.TearDown(); } +// ESPRYT'S HALF OF HANG CONTAINMENT (DirectGLES EnsureNativeContext / LatchIfDeviceLost). With the +// driver's default strategy a context whose shader hung the GPU is never told - measured on Adreno +// 750, the guilty context reads NO_ERROR, and after a reset that a failed preemption ended (~10 s) its +// next frame stops the GPU again, so a client that hangs every frame freezes the device for good. +// The server's native contexts must ask for EGL_LOSE_CONTEXT_ON_RESET. Red with EnsureNativeContext's +// strategy attribute removed: the context answers GL_NO_RESET_NOTIFICATION (0x8261). +TEST(ServerLoopEglTest, TheServersNativeContextAsksTheDriverToReportResets) { + EglServerFixture fixture; + MGL_EGL_BRING_UP_OR_BAIL(fixture); + // On the apply thread: the server's display and context are current there, not here. + Bool robustness = false; + GLint strategy = 0; + ASSERT_EQ(OnApply([&] { + const auto& egl = MG_Backend::DirectGLES::g_EGLFuncs; + const char* extensions = egl.eglQueryString(egl.eglGetCurrentDisplay(), EGL_EXTENSIONS); + robustness = extensions != nullptr && + std::strstr(extensions, "EGL_EXT_create_context_robustness") != nullptr; + MG_Backend::DirectGLES::g_GLESFuncs.glGetIntegerv(0x8256 /*GL_RESET_NOTIFICATION_STRATEGY*/, + &strategy); + }), + MOBILEGL_OK); + if (!robustness) { + fixture.TearDown(); + GTEST_SKIP() << "this EGL has no EGL_EXT_create_context_robustness"; + } + EXPECT_EQ(strategy, 0x8252 /*GL_LOSE_CONTEXT_ON_RESET*/) << std::hex << "strategy 0x" << strategy; + EXPECT_TRUE(MG_Backend::DirectGLES::NativeContextsNotifyResets()); + fixture.TearDown(); +} + +namespace { + // The driver's reset status, as the backend sees it through its function table. + std::atomic g_fakeResetStatus{GL_NO_ERROR}; + GLenum FakeGetGraphicsResetStatus() { return g_fakeResetStatus.exchange(GL_NO_ERROR); } + + enum : int { + kResetInnocentEnded = 1, // an INNOCENT report latched the session (the compositor would die with a client) + kResetGuiltyApplied = 2, // a record published after the GUILTY report was applied + kResetGuiltyNotLatched = 4, // GUILTY did not latch BackendDeviceLost through Espryt's apply-loop site + kResetHealthyNotApplied = 8, // a record before the GUILTY report was not applied + }; + + [[noreturn]] void ResetReportsAndExit() { + MG_Remote::ArmSessionLatch(); + MG_Remote::InstallPipeSessionFailHook(); // the MG_Pipe latch seam LatchIfDeviceLost latches through + EglServerFixture fixture; + const std::string why = fixture.BringUp(); + if (!why.empty()) { + std::fprintf(stderr, "[reset] SKIPPED - no headless EGL: %s\n", why.c_str()); + ::_exit(RequireGpuFromEnvironment() ? 98 : 0); + } + MG_External::GLESFunctionsTable gles = MG_Backend::DirectGLES::g_GLESFuncs; + gles.glGetGraphicsResetStatus = &FakeGetGraphicsResetStatus; + MG_Backend::DirectGLES::SetGLESFuncsTable(gles); + int failed = 0; + const auto applyOne = [&] { + MG_Pipe::MGPMemoryBarrier barrier{}; + barrier.Bits = 0x2000u; + const Uint64 seq = fixture.encoder.EncodeRecord(MG_Pipe::MGPWireOp::MemoryBarrier, &barrier, sizeof(barrier)); + fixture.encoder.Publish(); + fixture.producer.PublishAndNotify(seq); + return fixture.producer.WaitForApplied(seq, 2000) == Transport::SessionWait::Reached; + }; + // Another context hung the GPU; this one only waited behind it. + g_fakeResetStatus = GL_INNOCENT_CONTEXT_RESET; + if (!applyOne()) failed |= kResetHealthyNotApplied; + if (!applyOne() || MG_Remote::SessionLatched()) failed |= kResetInnocentEnded; + // This context's own work hung the GPU. + g_fakeResetStatus = GL_GUILTY_CONTEXT_RESET; + if (applyOne()) failed |= kResetGuiltyApplied; + if (!MG_Remote::SessionLatched() || MG_Remote::SessionLatchedFamily() != MG_Remote::MGFatalFamily::BackendDeviceLost || + std::strstr(MG_Remote::SessionLatchedLine(), "Espryt:apply") == nullptr) + failed |= kResetGuiltyNotLatched; + std::fprintf(stderr, "[reset] latched=%d line=%s\n", MG_Remote::SessionLatched() ? 1 : 0, + MG_Remote::SessionLatchedLine()); + ::_exit(failed); + } +} // namespace + +// A GUILTY report ends the session before its next record; an INNOCENT one does not. Red with the +// apply loop's LatchIfGpuFaulted call deleted (the record after GUILTY is applied, nothing latches: +// bits 2|4), and with LatchIfDeviceLost latching every non-NO_ERROR status (bit 1). +TEST(ServerLoopEglLatchTest, AGuiltyResetEndsTheSessionBeforeItsNextRecordAndAnInnocentOneDoesNot) { + PinHeadlessEglEnvironment(); + EXPECT_EXIT(ResetReportsAndExit(), ::testing::ExitedWithCode(0), + ".*") + << "bits: 1 = INNOCENT latched; 2 = a record after GUILTY was applied; 4 = GUILTY did not latch " + "BackendDeviceLost at Espryt:apply; 8 = a healthy record was not applied (98 = no headless EGL while MOBILEGL_ITEST_REQUIRE_GPU is set)"; +} + // ID-49's tight-size half, gated (review v2 N-5: "nothing, unit and joint"). The client's DstSize is // deliberately WRONG - 80, the size a ROW_LENGTH=8 / SKIP_* client would compute for a 4x3 RGBA8 // read whose tight extent is 48 - and the reply must still be the tight 48 bytes: posted at 48, @@ -3506,6 +3596,118 @@ TEST(ServerLoopLatchTest, ALostServerWindowIsReleasedMidBatchWhileTheClientStrea "latch; 32 = the apply thread stayed (64+ = setup)"; } +// ===================================================================================== +// A SESSION WHOSE OWN GPU WORK HUNG THE GPU APPLIES NOTHING MORE (BackendObject::LatchIfGpuFaulted) +// ===================================================================================== + +namespace { + // The backend's verdict arrives from elsewhere - the driver's GUILTY report (Espryt) or the hang + // watch's thread (Magma) - while the session's next frame is already queued. The double counts + // the frames that reach it and, once `faulted` is set, latches its session from the apply thread + // as both real backends do. + class GpuFaultBackendDouble final : public MG_Backend::BackendObject { + public: + GpuFaultBackendDouble() { m_functions.Present = &Present; } + Bool LatchIfGpuFaulted() override { + asked.fetch_add(1, std::memory_order_acq_rel); + if (!faulted.load(std::memory_order_acquire)) return false; + (void)MG_Pipe::MGPipeSessionLatch(MG_Pipe::MGPipeFatalFamily::DeviceLost, + "MGPipe: Fatal{BackendDeviceLost, \"unit:gpu-hang\"} - the double's " + "own GPU work hung the GPU"); + return true; + } + void Initialize() override {} + Bool InitCapabilities() override { return true; } + Bool InitWindowSurface() override { return true; } + const RendererInfo& GetRendererInfo() const override { return m_info; } + String GetBackendAPIVersionString() const override { return "gpu fault double"; } + const MG_Backend::GlobalBackendFunctionsTable& GetBackendFunctions() const override { return m_functions; } + const MG_Backend::DynamicBackendParameters& GetDynamicParameters() const override { return m_params; } + BackendType GetBackendType() const override { return BackendType::Unknown; } + + static void Present() { presents.fetch_add(1, std::memory_order_acq_rel); } + + static inline std::atomic presents{0}; + static inline std::atomic asked{0}; + static inline std::atomic faulted{false}; + + private: + MG_Backend::GlobalBackendFunctionsTable m_functions{}; + MG_Backend::DynamicBackendParameters m_params{}; + RendererInfo m_info{.RendererName = "Test", + .BackendName = "GpuFaultBackendDouble", + .ExtraVendor = Nullopt, + .RendererGLInfo = {.TargetGLVersion = {3, 3, 0}, + .TargetGLSLVersion = {4, 6, 0}, + .Extensions = {}, + .IsCompatibilityProfile = false}, + .StaticBackendCapability = {.AllowVSOnlyPrograms = false}}; + }; + + enum : int { + kFaultFirstFrame = 1, // the healthy frame before the fault was not applied + kFaultFrameApplied = 2, // a frame published after the verdict reached the backend + kFaultNotLatched = 4, // the session did not latch DeviceLost with the backend's line + kFaultThreadStayed = 8, // the apply thread was still running 3 s after the verdict + kFaultNotAsked = 16, // the loop never asked the backend + }; + + [[noreturn]] void ApplyNothingAfterAGpuFaultAndExit() { + Remote::ArmSessionLatch(); + Remote::InstallPipeSessionFailHook(); // the MG_Pipe latch seam the backends latch through (InitServerRoleCommon) + ServerFixture fixture; + if (!fixture.Handshake()) ::_exit(64); + if (Server::ServerLoopInstance().InstallBackendForTesting(MakeUnique()) != MOBILEGL_OK) + ::_exit(65); + if (!fixture.StartLoop() || !fixture.WaitUntilTrulyParked()) ::_exit(66); + Server::ServerLoop& loop = Server::ServerLoopInstance(); + int failed = 0; + + // One healthy frame. + MG_Pipe::MGPPresent present{}; + present.FrameSerial = 1; + Uint64 seq = fixture.encoder.EncodeRecord(MG_Pipe::MGPWireOp::Present, &present, sizeof(present)); + if (seq == Codec::kInvalidSeq) ::_exit(67); + fixture.encoder.Publish(); + fixture.producer.PublishAndNotify(seq); + if (fixture.producer.WaitForApplied(seq, 3000) != Transport::SessionWait::Reached || + GpuFaultBackendDouble::presents.load() != 1) + failed |= kFaultFirstFrame; + if (!fixture.WaitUntilTrulyParked()) ::_exit(68); + + // The verdict lands while the session is parked; then its next frames arrive. + GpuFaultBackendDouble::faulted.store(true, std::memory_order_release); + for (Uint64 serial = 2; serial <= 4; ++serial) { + present.FrameSerial = serial; + seq = fixture.encoder.EncodeRecord(MG_Pipe::MGPWireOp::Present, &present, sizeof(present)); + if (seq == Codec::kInvalidSeq) ::_exit(69); + } + fixture.encoder.Publish(); + fixture.producer.PublishAndNotify(seq); + + if (!PollUntil([&] { return !loop.Running(); }, 3000)) failed |= kFaultThreadStayed; + if (GpuFaultBackendDouble::presents.load() != 1) failed |= kFaultFrameApplied; + if (!Remote::SessionLatched() || Remote::SessionLatchedFamily() != Remote::MGFatalFamily::BackendDeviceLost || + std::strstr(Remote::SessionLatchedLine(), "\"unit:gpu-hang\"") == nullptr) + failed |= kFaultNotLatched; + if (GpuFaultBackendDouble::asked.load() == 0) failed |= kFaultNotAsked; + std::fprintf(stderr, "[gpu-fault] presents %d, asked %d, drained %llu\n", GpuFaultBackendDouble::presents.load(), + GpuFaultBackendDouble::asked.load(), static_cast(loop.DrainedRecords())); + fixture.Stop(); + ::_exit(failed); + } +} // namespace + +// Red with ServerLoop's per-wakeup LatchIfGpuFaulted call deleted: the three frames behind the +// verdict are applied - on a device each would stop the GPU again - and nothing latches (bits +// 2|4|8|16). +TEST(ServerLoopLatchTest, ASessionWhoseGpuWorkHungAppliesNoFrameAfterTheVerdict) { + EXPECT_EXIT(ApplyNothingAfterAGpuFaultAndExit(), ::testing::ExitedWithCode(0), ".*") + << "bits: 1 = the healthy frame was not applied; 2 = a frame behind the verdict was applied; 4 = no " + "DeviceLost latch with the backend's line; 8 = the apply thread stayed; 16 = the loop never asked " + "(64+ = setup)"; +} + // ===================================================================================== // SUSPEND, NOT END: the server window going away (screen off, lock) and coming back // =====================================================================================