[Fix] (Server, Magma, Espryt): end the session whose GPU work hangs the GPU so it cannot stall the desktop again.

.claude/skills/anland-mobilegl-plasma/SKILL.md

@@ -128,6 +128,7 @@ sleep 8; grep -a "presented through" /tmp/mgl-run.client.log
 Chrome: `bash scripts/chrome-launch.sh <tag> [flags]` (log `/tmp/mgl-chrome-<tag>.client.log`, screenshot `ch-<tag>.png`; `CHROME_GPU=in-process` puts the GPU back in the browser, which cannot recover from a device loss).
 Device loss: `setprop debug.mobilegl.inject_device_lost_pid <pid>` (as root) loses the server session of the client whose pid - as it sees itself in the container - is `<pid>`, once per value, on any backend and whether or not it presents; `debug.mobilegl.inject_device_lost_at N` instead loses whichever session makes the N-th device check (Espryt presents and read-backs only). `bash scripts/chrome-recover.sh <in-process|process> [rounds]` checks that Chrome repaints after such a loss without a restart (battery-guarded, stops below 40%). `bash scripts/kwin-recover.sh [rounds]` loses KWin's own session (default twice) and checks that KWin restarts compositing on a fresh session - same KWin, plasmashell and client processes, the screen still animating (glmark2 on Wayland, glxgears on X11, `CHROME=1` adds Chrome) - which needs the anland backend + kwin.patch reset handling deployed (`kwin-stage.sh build`).
 Probes: `scripts/probes/texfmt.c` (texture formats keep alpha?), `dmahash.c` (hash a process's dma-bufs); build in the container, e.g. `gcc -O1 /root/texfmt.c -o /usr/local/bin/mgl-texfmt -lEGL`, run with `mgrun offscreen`.
+GPU hangs: a shader that never ends stops the WHOLE GPU (every process, MobileGL or not) until the kernel resets it - ~2 s, or ~10 s ("Preemption Fault" in dmesg) when a higher-priority context waits; nothing in user space shortens one stall. What the server does is end the guilty session so it cannot hang the GPU again (Magma: event-bracketed submissions + `mgl-gpu-watch`, budget `MOBILEGL_GPU_HANG_BUDGET_MS` / `debug.mobilegl.gpu_hang_budget_ms`, default 2000; Espryt: the driver's GUILTY reset status, which it reports after the ~2 s reset only). Repro: `scripts/probes/hangclient.c` (Wayland + GL 3.3 core; `-H 60 -N 1 -E 60` hangs every 60 frames, `-H -1` is a frame-gap observer; build with wayland-scanner's xdg-shell files, `-lwayland-client -lwayland-egl -lEGL -lOpenGL -lm`), and `gpuprobe.c` (NDK, plain adb shell; `gpuprobe 30 high` is a high-priority GLES context outside MobileGL - it both measures the device-wide stall and forces the 10 s preemption path).
 
 A result counts only if ALL hold:
 - client log says `presented through linux-dmabuf shared images` (not `wl_shm`; `could not be presented through a linux-dmabuf shared image` = fell back);

.claude/skills/anland-mobilegl-plasma/scripts/probes/gpuprobe.c

@@ -0,0 +1,55 @@
+// gpuprobe - a plain Android process (outside MobileGL) that measures whether the GPU keeps executing
+// other contexts' work: a pbuffer GLES context clears + draws and glFinish()es every ~10 ms, printing any
+// round trip over 50 ms with a CLOCK_REALTIME timestamp.  Usage: gpuprobe <seconds> [priority: low|med|high]
+#include <EGL/egl.h>
+#include <EGL/eglext.h>
+#include <GLES3/gl3.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+
+#ifndef EGL_CONTEXT_PRIORITY_LEVEL_IMG
+#define EGL_CONTEXT_PRIORITY_LEVEL_IMG 0x3100
+#define EGL_CONTEXT_PRIORITY_HIGH_IMG 0x3101
+#define EGL_CONTEXT_PRIORITY_MEDIUM_IMG 0x3102
+#define EGL_CONTEXT_PRIORITY_LOW_IMG 0x3103
+#endif
+
+static double now_ms(void) { struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); return ts.tv_sec * 1e3 + ts.tv_nsec / 1e6; }
+
+int main(int argc, char** argv) {
+    int seconds = argc > 1 ? atoi(argv[1]) : 20;
+    EGLint prio = EGL_CONTEXT_PRIORITY_MEDIUM_IMG;
+    if (argc > 2 && !strcmp(argv[2], "high")) prio = EGL_CONTEXT_PRIORITY_HIGH_IMG;
+    if (argc > 2 && !strcmp(argv[2], "low")) prio = EGL_CONTEXT_PRIORITY_LOW_IMG;
+    setvbuf(stdout, NULL, _IOLBF, 0);
+    EGLDisplay d = eglGetDisplay(EGL_DEFAULT_DISPLAY);
+    eglInitialize(d, NULL, NULL);
+    const EGLint ca[] = {EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, EGL_RENDERABLE_TYPE, EGL_OPENGL_ES3_BIT, EGL_RED_SIZE, 8, EGL_NONE};
+    EGLConfig c; EGLint n;
+    eglChooseConfig(d, ca, &c, 1, &n);
+    const EGLint pa[] = {EGL_WIDTH, 256, EGL_HEIGHT, 256, EGL_NONE};
+    EGLSurface s = eglCreatePbufferSurface(d, c, pa);
+    const EGLint xa[] = {EGL_CONTEXT_CLIENT_VERSION, 3, EGL_CONTEXT_PRIORITY_LEVEL_IMG, prio, EGL_NONE};
+    EGLContext x = eglCreateContext(d, c, EGL_NO_CONTEXT, xa);
+    if (x == EGL_NO_CONTEXT) { printf("no ctx 0x%x\n", eglGetError()); return 1; }
+    eglMakeCurrent(d, s, s, x);
+    EGLint got = 0; eglQueryContext(d, x, EGL_CONTEXT_PRIORITY_LEVEL_IMG, &got);
+    const double t0 = now_ms(); double maxRt = 0; long iters = 0, slow = 0;
+    printf("[probe] start %.0f prio=0x%x\n", t0, got);
+    while (now_ms() - t0 < seconds * 1000.0) {
+        const double a = now_ms();
+        glClearColor((iters & 1) ? 1.f : 0.f, 0.f, 0.f, 1.f);
+        glClear(GL_COLOR_BUFFER_BIT);
+        glFinish();
+        const double rt = now_ms() - a;
+        if (rt > maxRt) maxRt = rt;
+        if (rt > 50) { ++slow; printf("[probe] glFinish %.0f ms ending at %.0f\n", rt, now_ms()); }
+        ++iters;
+        usleep(10000);
+    }
+    printf("[probe] END iters=%ld max=%.0f ms slow=%ld\n", iters, maxRt, slow);
+    return 0;
+}

.claude/skills/anland-mobilegl-plasma/scripts/probes/hangclient.c

@@ -0,0 +1,150 @@
+// hangclient - a Wayland xdg-toplevel, EGL + desktop GL 3.3 core client that animates and, on request,
+// submits fragment work that never ends (the driver has to reset it).  Prints per-frame gaps so the same
+// binary measures "other app" frame pacing (frame callbacks come from the compositor) when run with -H -1.
+//
+//   hangclient -T tag [-H hang_at_frame] [-N hang_repeat] [-I iters] [-S seconds] [-W w] [-X h] [-F]
+//     -H n   frame at which the hanging draw is submitted (-1: never; default -1)
+//     -N k   how many consecutive hanging frames (default 1)
+//     -E e   re-arm a hang every e frames after the first (0: once)
+//     -I i   loop count of the hanging shader (default 2^31-1)
+//     -S s   run time in seconds (default 30)
+//     -F     glFinish after every frame (puts the wait inside the client's own session)
+#define GL_GLEXT_PROTOTYPES 1
+#include <EGL/egl.h>
+#include <EGL/eglext.h>
+#include <GL/glcorearb.h>
+#include <wayland-client.h>
+#include <wayland-egl.h>
+#include "xdg-shell-client-protocol.h"
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+
+static struct wl_compositor* g_comp;
+static struct xdg_wm_base* g_wm;
+static int g_configured, g_closed;
+static int g_w = 480, g_h = 360;
+
+static void reg_global(void* d, struct wl_registry* r, uint32_t name, const char* iface, uint32_t ver) {
+    (void)d;
+    if (!strcmp(iface, "wl_compositor")) g_comp = wl_registry_bind(r, name, &wl_compositor_interface, 4);
+    else if (!strcmp(iface, "xdg_wm_base")) g_wm = wl_registry_bind(r, name, &xdg_wm_base_interface, 1);
+}
+static void reg_remove(void* d, struct wl_registry* r, uint32_t n) { (void)d; (void)r; (void)n; }
+static const struct wl_registry_listener reg_l = {reg_global, reg_remove};
+static void wm_ping(void* d, struct xdg_wm_base* wm, uint32_t s) { (void)d; xdg_wm_base_pong(wm, s); }
+static const struct xdg_wm_base_listener wm_l = {wm_ping};
+static void xs_conf(void* d, struct xdg_surface* s, uint32_t serial) { (void)d; xdg_surface_ack_configure(s, serial); g_configured = 1; }
+static const struct xdg_surface_listener xs_l = {xs_conf};
+static void tl_conf(void* d, struct xdg_toplevel* t, int32_t w, int32_t h, struct wl_array* st) { (void)d; (void)t; (void)st; (void)w; (void)h; }
+static void tl_close(void* d, struct xdg_toplevel* t) { (void)d; (void)t; g_closed = 1; }
+static const struct xdg_toplevel_listener tl_l = {tl_conf, tl_close};
+
+static double now_ms(void) { struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); return ts.tv_sec * 1e3 + ts.tv_nsec / 1e6; }
+
+static GLuint sh(GLenum type, const char* src) {
+    GLuint s = glCreateShader(type);
+    glShaderSource(s, 1, &src, NULL);
+    glCompileShader(s);
+    GLint ok = 0; glGetShaderiv(s, GL_COMPILE_STATUS, &ok);
+    if (!ok) { char log[2048]; glGetShaderInfoLog(s, sizeof log, NULL, log); fprintf(stderr, "shader: %s\n", log); exit(2); }
+    return s;
+}
+
+int main(int argc, char** argv) {
+    const char* tag = "hc"; int hangAt = -1, hangN = 1, every = 0, seconds = 30, finish = 0; long iters = 2147483647L;
+    for (int i = 1; i < argc; ++i) {
+        if (!strcmp(argv[i], "-T") && i + 1 < argc) tag = argv[++i];
+        else if (!strcmp(argv[i], "-H") && i + 1 < argc) hangAt = atoi(argv[++i]);
+        else if (!strcmp(argv[i], "-N") && i + 1 < argc) hangN = atoi(argv[++i]);
+        else if (!strcmp(argv[i], "-E") && i + 1 < argc) every = atoi(argv[++i]);
+        else if (!strcmp(argv[i], "-I") && i + 1 < argc) iters = atol(argv[++i]);
+        else if (!strcmp(argv[i], "-S") && i + 1 < argc) seconds = atoi(argv[++i]);
+        else if (!strcmp(argv[i], "-W") && i + 1 < argc) g_w = atoi(argv[++i]);
+        else if (!strcmp(argv[i], "-X") && i + 1 < argc) g_h = atoi(argv[++i]);
+        else if (!strcmp(argv[i], "-F")) finish = 1;
+    }
+    setvbuf(stdout, NULL, _IOLBF, 0);
+    struct wl_display* dpy = wl_display_connect(NULL);
+    if (!dpy) { fprintf(stderr, "no wayland\n"); return 1; }
+    struct wl_registry* reg = wl_display_get_registry(dpy);
+    wl_registry_add_listener(reg, &reg_l, NULL);
+    wl_display_roundtrip(dpy);
+    if (!g_comp || !g_wm) { fprintf(stderr, "no compositor/xdg_wm_base\n"); return 1; }
+    xdg_wm_base_add_listener(g_wm, &wm_l, NULL);
+    struct wl_surface* surf = wl_compositor_create_surface(g_comp);
+    struct xdg_surface* xs = xdg_wm_base_get_xdg_surface(g_wm, surf);
+    xdg_surface_add_listener(xs, &xs_l, NULL);
+    struct xdg_toplevel* tl = xdg_surface_get_toplevel(xs);
+    xdg_toplevel_add_listener(tl, &tl_l, NULL);
+    xdg_toplevel_set_title(tl, tag);
+    wl_surface_commit(surf);
+    while (!g_configured) wl_display_dispatch(dpy);
+
+    EGLDisplay ed = eglGetPlatformDisplay(EGL_PLATFORM_WAYLAND_KHR, dpy, NULL);
+    if (!eglInitialize(ed, NULL, NULL)) { fprintf(stderr, "eglInitialize\n"); return 1; }
+    eglBindAPI(EGL_OPENGL_API);
+    const EGLint ca[] = {EGL_SURFACE_TYPE, EGL_WINDOW_BIT, EGL_RENDERABLE_TYPE, EGL_OPENGL_BIT, EGL_RED_SIZE, 8, EGL_GREEN_SIZE, 8, EGL_BLUE_SIZE, 8, EGL_NONE};
+    EGLConfig cfg; EGLint n = 0;
+    if (!eglChooseConfig(ed, ca, &cfg, 1, &n) || n < 1) { fprintf(stderr, "no config\n"); return 1; }
+    const EGLint xa[] = {EGL_CONTEXT_MAJOR_VERSION, 3, EGL_CONTEXT_MINOR_VERSION, 3, EGL_CONTEXT_OPENGL_PROFILE_MASK, EGL_CONTEXT_OPENGL_CORE_PROFILE_BIT, EGL_NONE};
+    EGLContext ctx = eglCreateContext(ed, cfg, EGL_NO_CONTEXT, xa);
+    if (ctx == EGL_NO_CONTEXT) { fprintf(stderr, "no context 0x%x\n", eglGetError()); return 1; }
+    struct wl_egl_window* ew = wl_egl_window_create(surf, g_w, g_h);
+    EGLSurface es = eglCreatePlatformWindowSurface(ed, cfg, ew, NULL);
+    eglMakeCurrent(ed, es, es, ctx);
+    eglSwapInterval(ed, 1);
+
+    const char* vs = "#version 330 core\nlayout(location=0) in vec2 p; uniform float t; out vec2 uv;\n"
+                     "void main(){ uv=p*0.5+0.5; gl_Position=vec4(p*0.4+vec2(0.5*sin(t),0.0),0.0,1.0);}";
+    const char* fs = "#version 330 core\nin vec2 uv; uniform int iters; uniform float t; out vec4 c;\n"
+                     "void main(){ float a=uv.x; for(int i=0;i<iters;++i){ a=fract(a*1.0001+0.37+uv.y); }\n"
+                     " c=vec4(uv, 0.5+0.5*sin(t)+a*1e-6, 1.0);}";
+    GLuint prog = glCreateProgram();
+    glAttachShader(prog, sh(GL_VERTEX_SHADER, vs));
+    glAttachShader(prog, sh(GL_FRAGMENT_SHADER, fs));
+    glLinkProgram(prog);
+    GLint ok = 0; glGetProgramiv(prog, GL_LINK_STATUS, &ok);
+    if (!ok) { fprintf(stderr, "link failed\n"); return 2; }
+    GLint uIt = glGetUniformLocation(prog, "iters"), uT = glGetUniformLocation(prog, "t");
+    GLuint vao, vbo; glGenVertexArrays(1, &vao); glBindVertexArray(vao);
+    const float q[] = {-1, -1, 1, -1, -1, 1, 1, 1};
+    glGenBuffers(1, &vbo); glBindBuffer(GL_ARRAY_BUFFER, vbo); glBufferData(GL_ARRAY_BUFFER, sizeof q, q, GL_STATIC_DRAW);
+    glEnableVertexAttribArray(0); glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 0, 0);
+
+    const double t0 = now_ms(); double last = t0, maxGap = 0; long frames = 0, gaps100 = 0;
+    printf("[%s] start %.0f renderer=%s hangAt=%d N=%d every=%d iters=%ld\n", tag, t0, glGetString(GL_RENDERER), hangAt, hangN, every, iters);
+    while (!g_closed && now_ms() - t0 < seconds * 1000.0) {
+        wl_display_dispatch_pending(dpy);
+        const double t = (now_ms() - t0) / 1000.0;
+        int hang = 0;
+        if (hangAt >= 0 && frames >= hangAt) {
+            long rel = frames - hangAt;
+            if (every > 0) rel %= every;
+            hang = rel < hangN;
+        }
+        glViewport(0, 0, g_w, g_h);
+        glClearColor(0.5f + 0.5f * (float)sin(t * 3.0), 0.2f, 0.4f, 1.0f);
+        glClear(GL_COLOR_BUFFER_BIT);
+        glUseProgram(prog);
+        glUniform1i(uIt, hang ? (int)iters : 0);
+        glUniform1f(uT, (float)t);
+        glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+        if (hang) printf("[%s] HANG submitted frame %ld at %.0f\n", tag, frames, now_ms());
+        if (finish) glFinish();
+        if (!eglSwapBuffers(ed, es)) printf("[%s] swap failed 0x%x at %.0f\n", tag, eglGetError(), now_ms());
+        const GLenum rs = glGetError();
+        if (rs != GL_NO_ERROR) printf("[%s] glGetError 0x%x at %.0f\n", tag, rs, now_ms());
+        const double nowv = now_ms(), gap = nowv - last;
+        last = nowv; ++frames;
+        if (gap > maxGap) maxGap = gap;
+        if (gap > 100.0) { ++gaps100; printf("[%s] GAP %.0f ms ending at %.0f (frame %ld)\n", tag, gap, nowv, frames); }
+        if (frames % 120 == 0) printf("[%s] frame %ld t=%.1fs fps=%.1f\n", tag, frames, (nowv - t0) / 1e3, frames * 1e3 / (nowv - t0));
+    }
+    const double total = now_ms() - t0;
+    printf("[%s] END frames=%ld fps=%.1f maxGap=%.0f ms gaps>100ms=%ld\n", tag, frames, frames * 1e3 / total, maxGap, gaps100);
+    return 0;
+}

CMakeLists.txt

@@ -440,6 +440,7 @@ set(SOURCE_FILES
 
     MobileGL/MG_Backend/Init.cpp
     MobileGL/MG_Backend/BackendObject.cpp
+    MobileGL/MG_Backend/GpuHangWatch.cpp
 
     MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp
     MobileGL/MG_Backend/DirectGLES/BackendObject_DirectGLES.cpp
@@ -467,6 +468,7 @@ set(SOURCE_FILES
     MobileGL/MG_Backend/DirectVulkan/Renderer/VkSamplerManager.cpp
     MobileGL/MG_Backend/DirectVulkan/Renderer/VkClearManager.cpp
     MobileGL/MG_Backend/DirectVulkan/Renderer/VkRenderPassManager.cpp
+    MobileGL/MG_Backend/DirectVulkan/Renderer/GpuProgressMarkers.cpp
 
     MobileGL/MG_State/GLState/Core.cpp
     MobileGL/MG_State/GLState/ShareGroupState.cpp

MobileGL/MG_Backend/BackendObject.h

@@ -749,6 +749,12 @@ namespace MobileGL {
             // native state is process-global by construction gives this object its own copy and binds
             // it to the calling thread. The default has nothing per process to separate.
             virtual void BindSessionStateToThisThread() {}
+            // A served session's apply thread asks this each time it wakes, BEFORE it applies another
+            // record: true when this session's own GPU work faulted or hung the GPU and the backend has
+            // latched the session for it (MGPipeSessionLatch, family DeviceLost), so the next frame -
+            // which would very likely stop the GPU again - is never applied. Cheap by contract: no
+            // wait, at most a status read. The default knows of no fault.
+            virtual Bool LatchIfGpuFaulted() { return false; }
             // A served session's ring moved to another of its client's contexts (bind_context). A
             // backend whose surface follows the native context it binds needs nothing; one that keeps
             // a single native surface binding re-binds the surface that context was last made current

MobileGL/MG_Backend/DirectGLES/BackendObject_DirectGLES.cpp

@@ -961,6 +961,14 @@ namespace MobileGL::MG_Backend::DirectGLES {
         return true;
     }
 
+    Bool BackendObject_DirectGLES::LatchIfGpuFaulted() {
+#if MOBILEGL_BUILD_DISAGGREGATED
+        return LatchIfGuiltyBeforeApply();
+#else
+        return false;
+#endif
+    }
+
     void BackendObject_DirectGLES::Initialize() {
         MG_Util::BackendLoader::AcquireEGLFunctions(m_EGLFunctions);
         MG_Util::BackendLoader::AcquireGLESFunctions(m_GLESFunctions, m_EGLFunctions.eglGetProcAddress);

MobileGL/MG_Backend/DirectGLES/BackendObject_DirectGLES.h

@@ -32,6 +32,8 @@ namespace MobileGL::MG_Backend::DirectGLES {
     public:
         ~BackendObject_DirectGLES() override;
 
+        // The driver's verdict on this session's context (DirectGLES::LatchIfDeviceLost).
+        Bool LatchIfGpuFaulted() override;
         void Initialize() override;
         Bool InitCapabilities() override;
         Bool InitWindowSurface() override;

MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp

@@ -16982,6 +16982,53 @@ namespace MobileGL::MG_Backend::DirectGLES {
 #endif
     }
 
+    // RESET NOTIFICATION, decided once per process by its first native context (EnsureNativeContext
+    // says why). MOBILEGL_ESPRYT_RESET_NOTIFICATION=0 keeps the driver's default.
+    namespace {
+        // EGL_EXT_create_context_robustness (eglext.h is not part of this file's include set).
+        constexpr EGLint kEglResetNotificationStrategyExt = 0x3138;
+        constexpr EGLint kEglLoseContextOnResetExt = 0x31BF;
+        enum class ResetNotification : Uint8 { Undecided, On, Off };
+        std::atomic<ResetNotification> g_resetNotification{ResetNotification::Undecided};
+
+        Bool ResetNotificationWanted() {
+            const ResetNotification decided = g_resetNotification.load(std::memory_order_acquire);
+            if (decided != ResetNotification::Undecided) return decided == ResetNotification::On;
+            if (const char* value = std::getenv("MOBILEGL_ESPRYT_RESET_NOTIFICATION"); value != nullptr && value[0] == '0')
+                return false;
+            if (g_Display == EGL_NO_DISPLAY || !g_EGLFuncs.eglQueryString) return false;
+            const char* extensions = g_EGLFuncs.eglQueryString(g_Display, EGL_EXTENSIONS);
+            if (extensions == nullptr) return false;
+            constexpr const char kWord[] = "EGL_EXT_create_context_robustness";
+            for (const char* at = std::strstr(extensions, kWord); at != nullptr; at = std::strstr(at + 1, kWord)) {
+                const char after = at[sizeof(kWord) - 1];
+                if ((at == extensions || at[-1] == ' ') && (after == '\0' || after == ' ')) return true;
+            }
+            return false;
+        }
+
+        void NoteResetNotificationRefused() {
+            ResetNotification expected = ResetNotification::Undecided;
+            if (g_resetNotification.compare_exchange_strong(expected, ResetNotification::Off, std::memory_order_acq_rel))
+                MGLOG_W("DirectGLES: the driver refused EGL_LOSE_CONTEXT_ON_RESET; a context that hangs the GPU is "
+                        "not told, and its session cannot be ended for it");
+        }
+
+        void NoteResetNotificationDecided(Bool used) {
+            ResetNotification expected = ResetNotification::Undecided;
+            const ResetNotification decided = used ? ResetNotification::On : ResetNotification::Off;
+            if (g_resetNotification.compare_exchange_strong(expected, decided, std::memory_order_acq_rel))
+                MGLOG_I("DirectGLES: native contexts %s reset notification (EGL_LOSE_CONTEXT_ON_RESET): a context "
+                        "whose work hangs or faults the GPU %s",
+                        used ? "ask for" : "do not ask for",
+                        used ? "is named by the driver and ends its own session" : "is not told");
+        }
+    } // namespace
+
+    Bool NativeContextsNotifyResets() {
+        return g_resetNotification.load(std::memory_order_acquire) == ResetNotification::On;
+    }
+
     static Bool EnsureNativeContext(NativeContextTuple& tuple, EGLint surfaceBit, NativeWindowType window) {
         if (!EnsureDisplayInitialized()) {
             MGLOG_E("EnsureNativeContext: the EGL display did not initialize");
@@ -17030,21 +17077,47 @@ namespace MobileGL::MG_Backend::DirectGLES {
         // request yields a 3.0 context that lacks the 3.1/3.2 texture targets the
         // capability probes exercise; mobile drivers ignore the minor and hand out
         // their maximum either way.
+        //
+        // RESET NOTIFICATION. With the driver's default strategy a context whose work hung the GPU
+        // is never told: after Adreno's ~2 s timeout reset it silently drops everything that context
+        // submits (the client's window just stops), and its client never learns it should recreate.
+        // Asking for EGL_LOSE_CONTEXT_ON_RESET lets the driver name the guilty context
+        // (glGetGraphicsResetStatus answers GUILTY, once) and LatchIfDeviceLost ends that session
+        // alone, so the client reads a lost context and can recover. (A ~10 s reset that a failed
+        // preemption ends was not reported to the server's contexts in measurement - see
+        // GpuHangWatch.h.) Every context of the process shares one group, and contexts that share
+        // must agree on the strategy, so it is decided once, by the first context, and kept.
         for (const EGLint minorVersion : {2, 1, 0}) {
+            const Bool notify = ResetNotificationWanted();
             const EGLint contextAttribs[] = {EGL_CONTEXT_MAJOR_VERSION, 3,
                                              EGL_CONTEXT_MINOR_VERSION, minorVersion,
+                                             notify ? kEglResetNotificationStrategyExt : EGL_NONE,
+                                             kEglLoseContextOnResetExt,
                                              EGL_NONE};
             tuple.Context = g_EGLFuncs.eglCreateContext(g_Display, tuple.Config, share, contextAttribs);
+            if (tuple.Context == EGL_NO_CONTEXT && notify && share == EGL_NO_CONTEXT) {
+                // The process's first context: if the same version WITHOUT the strategy is accepted,
+                // it was the strategy the driver refused, and no context of the process asks for it.
+                const EGLint plainAttribs[] = {EGL_CONTEXT_MAJOR_VERSION, 3, EGL_CONTEXT_MINOR_VERSION, minorVersion,
+                                               EGL_NONE};
+                tuple.Context = g_EGLFuncs.eglCreateContext(g_Display, tuple.Config, share, plainAttribs);
+                if (tuple.Context != EGL_NO_CONTEXT) NoteResetNotificationRefused();
+            }
             if (tuple.Context != EGL_NO_CONTEXT) {
+                NoteResetNotificationDecided(notify);
                 tuple.Serial = g_nativeContextSerials.fetch_add(1, std::memory_order_relaxed) + 1;
                 ++g_displayUsers;
                 return true;
             }
         }
 
-        const EGLint legacyContextAttribs[] = {EGL_CONTEXT_CLIENT_VERSION, 3, EGL_NONE};
+        const Bool notify = ResetNotificationWanted();
+        const EGLint legacyContextAttribs[] = {EGL_CONTEXT_CLIENT_VERSION, 3,
+                                               notify ? kEglResetNotificationStrategyExt : EGL_NONE,
+                                               kEglLoseContextOnResetExt, EGL_NONE};
         tuple.Context = g_EGLFuncs.eglCreateContext(g_Display, tuple.Config, share, legacyContextAttribs);
         if (tuple.Context == EGL_NO_CONTEXT) return false;
+        NoteResetNotificationDecided(notify);
         tuple.Serial = g_nativeContextSerials.fetch_add(1, std::memory_order_relaxed) + 1;
         ++g_displayUsers;
         return true;
@@ -18229,25 +18302,61 @@ namespace MobileGL::MG_Backend::DirectGLES {
     }
 
 #if MOBILEGL_BUILD_DISAGGREGATED
-    Bool LatchIfDeviceLost(const char* site, Bool debugKnob) {
-        if (MG_Config::Transport == MG_Config::TransportMode::Monolith) return false;
+    namespace {
         // Once per thread: a latched session declines everything after this verb anyway, and the
         // apply thread is the session's own.
-        static thread_local Bool latched = false;
-        if (latched) return true;
+        thread_local Bool t_deviceLossLatched = false;
+
+        Bool LatchOnResetStatus(const char* site, GLenum status) {
+            if (status == GL_NO_ERROR) return false;
+            // ONLY THE GUILTY CONTEXT ENDS ITS SESSION. The contexts ask for reset notification
+            // (EnsureNativeContext) so the driver can name the context whose work hung or faulted the
+            // GPU; a reset it reports to an INNOCENT context - one whose work only waited behind the
+            // stall, the compositor's above all - is noted and the session goes on as it did before
+            // contexts were told. Adreno 750 measured: the guilty context reads GUILTY once, after
+            // which the driver drops everything it submits; innocent ones read NO_ERROR.
+            static thread_local Bool innocentNoted = false;
+            if (status == GL_INNOCENT_CONTEXT_RESET) {
+                if (!innocentNoted) {
+                    innocentNoted = true;
+                    MGLOG_W("Espryt:%s - the driver reports a GPU reset this session's context did not cause "
+                            "(GL_INNOCENT_CONTEXT_RESET); the session goes on",
+                            site != nullptr ? site : "?");
+                }
+                return false;
+            }
+            t_deviceLossLatched = true;
+            (void)MG_Pipe::MGPipeSessionLatch(MG_Pipe::MGPipeFatalFamily::DeviceLost,
+                                              "MGPipe: Fatal{BackendDeviceLost, \"Espryt:%s\"} - this session's EGL "
+                                              "context was reset (glGetGraphicsResetStatus 0x%x); the session ends "
+                                              "and its client reads a lost context, every other session keeps running",
+                                              site != nullptr ? site : "?", static_cast<unsigned>(status));
+            return true;
+        }
+    } // namespace
+
+    Bool LatchIfDeviceLost(const char* site, Bool debugKnob) {
+        if (MG_Config::Transport == MG_Config::TransportMode::Monolith) return false;
+        if (t_deviceLossLatched) return true;
         GLenum status = GL_NO_ERROR;
         if (g_GLESFuncs.glGetGraphicsResetStatus != nullptr && IsBackendContextCurrentOnThisThread()) {
             status = g_GLESFuncs.glGetGraphicsResetStatus();
         }
         if (status == GL_NO_ERROR && debugKnob && MG_Pipe::MGPipeDebugDeviceLossDue()) status = GL_UNKNOWN_CONTEXT_RESET;
-        if (status == GL_NO_ERROR) return false;
-        latched = true;
-        (void)MG_Pipe::MGPipeSessionLatch(MG_Pipe::MGPipeFatalFamily::DeviceLost,
-                                          "MGPipe: Fatal{BackendDeviceLost, \"Espryt:%s\"} - this session's EGL "
-                                          "context was reset (glGetGraphicsResetStatus 0x%x); the session ends "
-                                          "and its client reads a lost context, every other session keeps running",
-                                          site != nullptr ? site : "?", static_cast<unsigned>(status));
-        return true;
+        return LatchOnResetStatus(site, status);
+    }
+
+    Bool LatchIfGuiltyBeforeApply() {
+        if (MG_Config::Transport == MG_Config::TransportMode::Monolith) return false;
+        if (t_deviceLossLatched) return true;
+        // Asked every time the apply thread wakes, so it must leave nothing behind: not
+        // IsBackendContextCurrentOnThisThread, whose once-per-frame EGL verification stamp it would
+        // spend at the top of the loop (where the session may be between two of its contexts) - only
+        // whether this thread holds a native context and the backend's ownership.
+        if (g_GLESFuncs.glGetGraphicsResetStatus == nullptr || t_boundNativeContext == EGL_NO_CONTEXT ||
+            g_backendContextOwnerThread.load(std::memory_order_acquire) != std::this_thread::get_id())
+            return false;
+        return LatchOnResetStatus("apply", g_GLESFuncs.glGetGraphicsResetStatus());
     }
 #endif
 

MobileGL/MG_Backend/DirectGLES/DirectGLES.h

@@ -162,14 +162,21 @@ namespace MobileGL::MG_Backend::DirectGLES {
     // driver whether the calling thread's context was reset (glGetGraphicsResetStatus); a reset -
     // or the debug knob MGPipeDebugDeviceLossDue - latches THIS session through
     // MGPipeSessionLatch and answers true, and the caller returns from its verb without issuing
-    // more GL. Asked at the frame boundaries (Present, a shared-image present) and before a
-    // readback, never per draw. The contexts keep the driver's default reset strategy: asking
-    // every context to lose itself on reset would also end the INNOCENT ones - the compositor's
-    // among them - on any other client's GPU fault.
+    // more GL. Asked at the frame boundaries (Present, a shared-image present), before a readback,
+    // and by the apply loop each time it wakes (LatchIfGpuFaulted), never per draw. The contexts ask
+    // for reset notification (NativeContextsNotifyResets) so the driver can name the GUILTY one;
+    // only GUILTY (or a reset of unknown cause) ends the session - an INNOCENT report, a context
+    // whose work merely waited behind another's hang (the compositor's among them), is only noted.
     // `debugKnob` false: the knob is not consulted here (the on-screen Present, so an injected loss
     // lands on an offscreen client's shared-image present or readback, never on the compositor's).
     Bool LatchIfDeviceLost(const char* site, Bool debugKnob = true);
+    // The apply loop's form of it (BackendObject::LatchIfGpuFaulted), asked each time the apply thread
+    // wakes: the driver's report only, no debug knob, and no EGL ground-truth verification spent.
+    Bool LatchIfGuiltyBeforeApply();
 #endif
+    // True once the process's native contexts were created with EGL_LOSE_CONTEXT_ON_RESET (decided
+    // by the first one; the driver must offer EGL_EXT_create_context_robustness).
+    Bool NativeContextsNotifyResets();
     // GL fence sync objects, backed by native ES fences. FenceSync returns null
     // (the frontend then falls back to an always-signaled sync) when the calling
     // thread does not own the ES context. Waits/queries degrade to "signaled" in

MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.cpp

@@ -422,6 +422,14 @@ namespace MobileGL::MG_Backend::DirectVulkan {
                 static_cast<unsigned long long>(m_magmaSession->Id()));
     }
 
+    Bool BackendObject_DirectVulkan::LatchIfGpuFaulted() {
+#if MOBILEGL_BUILD_DISAGGREGATED
+        return pVulkanRenderer != nullptr && pVulkanRenderer->LatchIfGpuHung("apply");
+#else
+        return false;
+#endif
+    }
+
     void BackendObject_DirectVulkan::Initialize() {
 #if MOBILEGL_BUILD_DISAGGREGATED
         VkBufferManager::RegisterWireResourceOps();

MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.h

@@ -28,6 +28,8 @@ namespace MobileGL::MG_Backend::DirectVulkan {
 
         void Initialize() override;
         void BindSessionStateToThisThread() override;
+        // The GPU hang watch's verdict on this session's device (VulkanRenderer::LatchIfGpuHung).
+        Bool LatchIfGpuFaulted() override;
         void OnClientContextBound(Uint64 token) override;
         Bool InitWindowSurface() override;
         Bool InitCapabilities() override;

MobileGL/MG_Backend/DirectVulkan/Renderer/GpuProgressMarkers.cpp

@@ -0,0 +1,202 @@
+// MobileGL - MobileGL/MG_Backend/DirectVulkan/Renderer/GpuProgressMarkers.cpp
+// Copyright (c) 2025-2026 MobileGL-Dev
+// Licensed under the GNU Lesser General Public License v3.0:
+//   https://www.gnu.org/licenses/gpl-3.0.txt
+//   https://www.gnu.org/licenses/lgpl-3.0.txt
+// SPDX-License-Identifier: LGPL-3.0-only
+// End of Source File Header
+
+#include "GpuProgressMarkers.h"
+
+#include <MG_Util/Debug/Log.h>
+
+namespace MobileGL::MG_Backend::DirectVulkan {
+
+    namespace Watch = MG_Backend::GpuHangWatch;
+
+    void GpuProgressMarkers::Init(VkDevice device, Uint32 queueFamilyIndex) {
+        if (m_device != VK_NULL_HANDLE || device == VK_NULL_HANDLE) return;
+        if (Watch::BudgetMs() == 0) {
+            MGLOG_I("Magma: GPU hang watch off (MOBILEGL_GPU_HANG_BUDGET_MS=0)");
+            return;
+        }
+        VkCommandPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};
+        poolInfo.queueFamilyIndex = queueFamilyIndex;
+        if (vkCreateCommandPool(device, &poolInfo, nullptr, &m_pool) != VK_SUCCESS) {
+            MGLOG_W("Magma: GPU hang watch off - no command pool for its markers");
+            m_pool = VK_NULL_HANDLE;
+            return;
+        }
+        m_device = device;
+        {
+            const std::lock_guard<std::mutex> lock(m_mutex);
+            if (!AddSlot()) {
+                MGLOG_W("Magma: GPU hang watch off - the device cannot make its progress markers");
+                vkDestroyCommandPool(m_device, m_pool, nullptr);
+                m_pool = VK_NULL_HANDLE;
+                m_device = VK_NULL_HANDLE;
+                return;
+            }
+        }
+        m_watch = Watch::Register(this, &Sample, &OnHung);
+        MGLOG_I("Magma: GPU hang watch on - a submission the GPU started and has not finished after %u ms ends "
+                "its session",
+                Watch::BudgetMs());
+    }
+
+    void GpuProgressMarkers::Shutdown() {
+        // The watch first, outside m_mutex: Unregister waits out a sample in progress, which takes it.
+        Watch::Unregister(m_watch);
+        m_watch = 0;
+        if (m_device == VK_NULL_HANDLE) return;
+        const std::lock_guard<std::mutex> lock(m_mutex);
+        for (Slot& slot : m_slots) {
+            if (slot.Started != VK_NULL_HANDLE) vkDestroyEvent(m_device, slot.Started, nullptr);
+            if (slot.Finished != VK_NULL_HANDLE) vkDestroyEvent(m_device, slot.Finished, nullptr);
+        }
+        m_slots.clear();
+        if (m_pool != VK_NULL_HANDLE) vkDestroyCommandPool(m_device, m_pool, nullptr); // frees the buffers
+        m_pool = VK_NULL_HANDLE;
+        m_device = VK_NULL_HANDLE;
+    }
+
+    Bool GpuProgressMarkers::AddSlot() {
+        if (m_slots.size() >= kMaxSlots) return false;
+        Slot slot;
+        const VkEventCreateInfo eventInfo{VK_STRUCTURE_TYPE_EVENT_CREATE_INFO};
+        VkCommandBuffer buffers[2] = {VK_NULL_HANDLE, VK_NULL_HANDLE};
+        VkCommandBufferAllocateInfo allocInfo{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};
+        allocInfo.commandPool = m_pool;
+        allocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
+        allocInfo.commandBufferCount = 2;
+        const auto fail = [&] {
+            if (slot.Started != VK_NULL_HANDLE) vkDestroyEvent(m_device, slot.Started, nullptr);
+            if (slot.Finished != VK_NULL_HANDLE) vkDestroyEvent(m_device, slot.Finished, nullptr);
+            if (buffers[0] != VK_NULL_HANDLE) vkFreeCommandBuffers(m_device, m_pool, 2, buffers);
+            return false;
+        };
+        if (vkCreateEvent(m_device, &eventInfo, nullptr, &slot.Started) != VK_SUCCESS ||
+            vkCreateEvent(m_device, &eventInfo, nullptr, &slot.Finished) != VK_SUCCESS ||
+            vkAllocateCommandBuffers(m_device, &allocInfo, buffers) != VK_SUCCESS)
+            return fail();
+        // Prerecorded once and resubmitted for every submission the slot brackets; SIMULTANEOUS_USE
+        // because the slot is reused once its `finished` event is set, which can be a moment before
+        // the submission that set it formally leaves the pending state.
+        VkCommandBufferBeginInfo beginInfo{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
+        beginInfo.flags = VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT;
+        if (vkBeginCommandBuffer(buffers[0], &beginInfo) != VK_SUCCESS) return fail();
+        vkCmdSetEvent(buffers[0], slot.Started, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT);
+        if (vkEndCommandBuffer(buffers[0]) != VK_SUCCESS) return fail();
+        if (vkBeginCommandBuffer(buffers[1], &beginInfo) != VK_SUCCESS) return fail();
+        vkCmdSetEvent(buffers[1], slot.Finished, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT);
+        if (vkEndCommandBuffer(buffers[1]) != VK_SUCCESS) return fail();
+        slot.Begin = buffers[0];
+        slot.End = buffers[1];
+        m_slots.push_back(slot);
+        return true;
+    }
+
+    GpuProgressMarkers::Bracket GpuProgressMarkers::Acquire() {
+        Bracket bracket;
+        if (m_device == VK_NULL_HANDLE) return bracket;
+        const std::lock_guard<std::mutex> lock(m_mutex);
+        if (m_lost) return bracket;
+        Slot* free = nullptr;
+        for (Slot& slot : m_slots) {
+            if (slot.Claimed) continue;
+            // Retire what finished since the watch last looked: frames come faster than its polls.
+            if (slot.InFlight && vkGetEventStatus(m_device, slot.Finished) == VK_EVENT_SET) slot.InFlight = false;
+            if (!slot.InFlight && free == nullptr) free = &slot;
+        }
+        if (free == nullptr) {
+            if (!AddSlot()) return bracket;
+            free = &m_slots.back();
+        }
+        if (free->Dirty) {
+            if (vkResetEvent(m_device, free->Started) != VK_SUCCESS ||
+                vkResetEvent(m_device, free->Finished) != VK_SUCCESS)
+                return bracket;
+            free->Dirty = false;
+        }
+        free->Claimed = true;
+        bracket.Begin = free->Begin;
+        bracket.End = free->End;
+        bracket.Slot = static_cast<Uint32>(free - m_slots.data());
+        return bracket;
+    }
+
+    void GpuProgressMarkers::Submitted(const Bracket& bracket) {
+        if (!bracket.Valid()) return;
+        const std::lock_guard<std::mutex> lock(m_mutex);
+        if (bracket.Slot >= m_slots.size()) return;
+        Slot& slot = m_slots[bracket.Slot];
+        slot.Claimed = false;
+        slot.InFlight = true;
+        slot.Dirty = true;
+        slot.Id = m_nextId++;
+    }
+
+    void GpuProgressMarkers::Abandon(const Bracket& bracket) {
+        if (!bracket.Valid()) return;
+        const std::lock_guard<std::mutex> lock(m_mutex);
+        if (bracket.Slot >= m_slots.size()) return;
+        // A failed vkQueueSubmit may still have consumed nothing or something: reset before reuse.
+        m_slots[bracket.Slot].Claimed = false;
+        m_slots[bracket.Slot].Dirty = true;
+    }
+
+    Bool GpuProgressMarkers::Sample(void* self, Vector<MG_Backend::GpuHangWatch::SubmissionProgress>& out) {
+        auto* markers = static_cast<GpuProgressMarkers*>(self);
+        const std::lock_guard<std::mutex> lock(markers->m_mutex);
+        if (markers->m_lost || markers->m_device == VK_NULL_HANDLE) return false;
+        for (Slot& slot : markers->m_slots) {
+            if (!slot.InFlight) continue;
+            const VkResult finished = vkGetEventStatus(markers->m_device, slot.Finished);
+            const VkResult started = vkGetEventStatus(markers->m_device, slot.Started);
+            if ((finished != VK_EVENT_SET && finished != VK_EVENT_RESET) ||
+                (started != VK_EVENT_SET && started != VK_EVENT_RESET)) {
+                // VK_ERROR_DEVICE_LOST: the driver already reset the device and says so itself.
+                markers->m_lost = true;
+                return false;
+            }
+            if (finished == VK_EVENT_SET) {
+                slot.InFlight = false;
+                continue;
+            }
+            out.push_back({slot.Id, started == VK_EVENT_SET, false});
+        }
+        return true;
+    }
+
+    void GpuProgressMarkers::OnHung(void* self, Uint64 submissionId, Uint64 runningMs) {
+        auto* markers = static_cast<GpuProgressMarkers*>(self);
+        markers->m_hungRunningMs.store(runningMs, std::memory_order_release);
+        markers->m_hung.store(true, std::memory_order_release);
+        MGLOG_E("Magma: submission %llu of this session's VkDevice has run %llu ms on the GPU and has not finished - "
+                "it is what stops the GPU for every process; the session ends at its next record and submits "
+                "nothing more (budget %u ms, MOBILEGL_GPU_HANG_BUDGET_MS)",
+                static_cast<unsigned long long>(submissionId), static_cast<unsigned long long>(runningMs),
+                MG_Backend::GpuHangWatch::BudgetMs());
+    }
+
+    VkResult GpuProgressMarkers::SubmitBracketed(VkQueue queue, VkSubmitInfo& info, VkFence fence,
+                                                 const Bracket& bracket) {
+        constexpr Uint32 kMaxOwn = 6;
+        if (!bracket.Valid() || info.commandBufferCount == 0 || info.commandBufferCount > kMaxOwn)
+            return vkQueueSubmit(queue, 1, &info, fence);
+        VkCommandBuffer buffers[kMaxOwn + 2];
+        Uint32 count = 0;
+        buffers[count++] = bracket.Begin;
+        for (Uint32 i = 0; i < info.commandBufferCount; ++i) buffers[count++] = info.pCommandBuffers[i];
+        buffers[count++] = bracket.End;
+        const VkCommandBuffer* own = info.pCommandBuffers;
+        const Uint32 ownCount = info.commandBufferCount;
+        info.pCommandBuffers = buffers;
+        info.commandBufferCount = count;
+        const VkResult result = vkQueueSubmit(queue, 1, &info, fence);
+        info.pCommandBuffers = own;
+        info.commandBufferCount = ownCount;
+        return result;
+    }
+
+} // namespace MobileGL::MG_Backend::DirectVulkan

MobileGL/MG_Backend/DirectVulkan/Renderer/GpuProgressMarkers.h

@@ -0,0 +1,95 @@
+// MobileGL - MobileGL/MG_Backend/DirectVulkan/Renderer/GpuProgressMarkers.h
+// Copyright (c) 2025-2026 MobileGL-Dev
+// Licensed under the GNU Lesser General Public License v3.0:
+//   https://www.gnu.org/licenses/gpl-3.0.txt
+//   https://www.gnu.org/licenses/lgpl-3.0.txt
+// SPDX-License-Identifier: LGPL-3.0-only
+// End of Source File Header
+
+#pragma once
+
+#include "../VkIncludes.h"
+#include <Includes.h>
+#include <MG_Backend/GpuHangWatch.h>
+
+#include <atomic>
+#include <mutex>
+
+namespace MobileGL::MG_Backend::DirectVulkan {
+
+    // WHICH OF THIS DEVICE'S SUBMISSIONS THE GPU HAS STARTED, AND WHICH IT HAS FINISHED
+    // (GpuHangWatch.h says why that is the whole attribution of a GPU hang).
+    //
+    // A frame's submission is bracketed by two prerecorded one-command buffers: the first sets a
+    // `started` event at TOP_OF_PIPE, the last sets a `finished` event once everything before it in
+    // the queue has completed. The host reads both with vkGetEventStatus from the watch thread - no
+    // fence is shared with the renderer's own waits and resets. On Adreno 750 a submission whose
+    // fragment shader never ends reads started=SET from its first poll until the kernel resets the
+    // GPU, and a submission queued behind it reads started=RESET for the whole stall.
+    //
+    // The renderer's thread brackets submissions (Acquire / Submitted / Abandon) and asks Hung(); the
+    // watch thread samples. Everything Vulkan the two share - the events - is touched under m_mutex.
+    class GpuProgressMarkers {
+    public:
+        struct Bracket {
+            VkCommandBuffer Begin = VK_NULL_HANDLE;
+            VkCommandBuffer End = VK_NULL_HANDLE;
+            Uint32 Slot = ~0u;
+            Bool Valid() const { return Slot != ~0u; }
+        };
+
+        GpuProgressMarkers() = default;
+        ~GpuProgressMarkers() { Shutdown(); }
+        GpuProgressMarkers(const GpuProgressMarkers&) = delete;
+        GpuProgressMarkers& operator=(const GpuProgressMarkers&) = delete;
+
+        // Off (no brackets, never hung) when the budget is 0 or the device cannot make events.
+        void Init(VkDevice device, Uint32 queueFamilyIndex);
+        // Before the device goes; its queue must be idle. Stops the watch first.
+        void Shutdown();
+
+        // The two buffers to put first and last in one submission (an invalid bracket when off or
+        // when every slot is in flight: the submission then goes unwatched).
+        Bracket Acquire();
+        // The submission carrying `bracket` was accepted by vkQueueSubmit / was not submitted.
+        void Submitted(const Bracket& bracket);
+        void Abandon(const Bracket& bracket);
+
+        // The watch named one of this device's submissions as hanging the GPU.
+        Bool Hung() const { return m_hung.load(std::memory_order_acquire); }
+        Uint64 HungRunningMs() const { return m_hungRunningMs.load(std::memory_order_acquire); }
+        Bool Active() const { return m_watch != 0; }
+
+        // vkQueueSubmit of `info` with `bracket` wrapped around its command buffers (which it puts
+        // back before returning). At most 6 buffers of the caller's.
+        static VkResult SubmitBracketed(VkQueue queue, VkSubmitInfo& info, VkFence fence, const Bracket& bracket);
+
+    private:
+        struct Slot {
+            VkEvent Started = VK_NULL_HANDLE;
+            VkEvent Finished = VK_NULL_HANDLE;
+            VkCommandBuffer Begin = VK_NULL_HANDLE;
+            VkCommandBuffer End = VK_NULL_HANDLE;
+            Uint64 Id = 0;
+            Bool InFlight = false; // submitted and not yet seen finished
+            Bool Claimed = false;  // handed out by Acquire, not yet submitted or abandoned
+            Bool Dirty = false;    // its events were set by a past submission: reset before reuse
+        };
+        static constexpr Uint32 kMaxSlots = 16;
+
+        Bool AddSlot();
+        static Bool Sample(void* self, Vector<MG_Backend::GpuHangWatch::SubmissionProgress>& out);
+        static void OnHung(void* self, Uint64 submissionId, Uint64 runningMs);
+
+        std::mutex m_mutex;
+        VkDevice m_device = VK_NULL_HANDLE;
+        VkCommandPool m_pool = VK_NULL_HANDLE;
+        Vector<Slot> m_slots;
+        Uint64 m_nextId = 1;
+        Uint64 m_watch = 0;
+        Bool m_lost = false;
+        std::atomic<Bool> m_hung{false};
+        std::atomic<Uint64> m_hungRunningMs{0};
+    };
+
+} // namespace MobileGL::MG_Backend::DirectVulkan

MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp

@@ -3275,6 +3275,9 @@ void main() {
         CreateAllocator();
 
         CreateCommandPool();
+#if MOBILEGL_BUILD_DISAGGREGATED
+        m_progressMarkers.Init(m_device, m_physicalDevice.queueFamilies.graphicsFamily);
+#endif
 
         // Frames-in-flight is a request, not a guarantee: it also seeds the swapchain image
         // count (SwapchainObject clamps the hint into [minImageCount, maxImageCount]). Not every
@@ -3504,6 +3507,9 @@ void main() {
         if (m_device != VK_NULL_HANDLE) {
             VK_VERIFY(vkDeviceWaitIdle(m_device));
         }
+#if MOBILEGL_BUILD_DISAGGREGATED
+        m_progressMarkers.Shutdown();
+#endif
         // Parked surface targets (ActivateSurfaceTarget) go before the device and instance.
         for (auto& [_, target] : m_parkedTargets) DestroyParkedTarget(target);
         m_parkedTargets.clear();
@@ -14173,10 +14179,16 @@ void main() {
         submitInfo.pCommandBuffers = commandBuffers;
 #if MOBILEGL_BUILD_DISAGGREGATED
         // Shared-image fences staged for the work recorded since the last submission.
+        if (LatchIfGpuHung("flush-submit")) return false;
         SharedImageSubmitSync sharedImageSync;
         AttachSharedImageSync(submitInfo, sharedImageSync);
-#endif
+        const GpuProgressMarkers::Bracket bracket = m_progressMarkers.Acquire();
+        const VkResult result = GpuProgressMarkers::SubmitBracketed(m_graphicsQueue, submitInfo, fence, bracket);
+        if (result == VK_SUCCESS) m_progressMarkers.Submitted(bracket);
+        else m_progressMarkers.Abandon(bracket);
+#else
         const VkResult result = vkQueueSubmit(m_graphicsQueue, 1, &submitInfo, fence);
+#endif
         if (result != VK_SUCCESS) {
             NoteDeviceLoss(result, "SubmitPendingCommandBuffer vkQueueSubmit");
             MGLOG_E_ONCE("SubmitPendingCommandBuffer: vkQueueSubmit returned %d", result);
@@ -14307,6 +14319,31 @@ void main() {
         }
         return true;
     }
+
+    Bool VulkanRenderer::LatchIfGpuHung(const char* site) {
+        if (!m_progressMarkers.Hung()) return false;
+        if (!m_deviceLost) {
+            // NOT LOST TO THE DRIVER: a submission that a failed preemption leaves running is reset
+            // without a device loss, and the next one would stop the GPU again. Treated as lost from
+            // here on - nothing more is submitted - and the session ends as one whose device was.
+            m_deviceLost = true;
+            MGLOG_E("DirectVulkan: the GPU hang watch named this session's VkDevice (a submission ran %llu ms "
+                    "without finishing, at %s); the device is treated as LOST and nothing more is submitted",
+                    static_cast<unsigned long long>(m_progressMarkers.HungRunningMs()),
+                    site != nullptr ? site : "?");
+        }
+        // Where no session latch is armed (no served session) the device only stays "lost".
+        if (!m_deviceLossLatched && MG_Pipe::MGPipeSessionLatchArmed()) {
+            m_deviceLossLatched = true;
+            (void)MG_Pipe::MGPipeSessionLatch(MG_Pipe::MGPipeFatalFamily::DeviceLost,
+                                              "MGPipe: Fatal{BackendDeviceLost, \"Magma:gpu-hang:%s\"} - this "
+                                              "session's GPU work ran past the hang budget and stopped the GPU for "
+                                              "every process; the session ends and its client reads a lost context, "
+                                              "every other session keeps running",
+                                              site != nullptr ? site : "?");
+        }
+        return true;
+    }
 #endif
 
     Bool VulkanRenderer::FlushForSyncPoint(Uint64 submitIndex) {
@@ -14656,10 +14693,17 @@ void main() {
         }
         auto submitPacket = m_frameContext.GetSubmitInfo(shouldSubmitCommandBuffer, m_imageIndexAcquired);
 #if MOBILEGL_BUILD_DISAGGREGATED
+        if (LatchIfGpuHung("present-submit")) return;
         SharedImageSubmitSync sharedImageSync;
         AttachSharedImageSync(submitPacket.submitInfo, sharedImageSync);
-#endif
+        const GpuProgressMarkers::Bracket bracket = m_progressMarkers.Acquire();
+        const VkResult presentSubmit = GpuProgressMarkers::SubmitBracketed(m_graphicsQueue, submitPacket.submitInfo,
+                                                                           frame.imageInFlightFence, bracket);
+        if (presentSubmit == VK_SUCCESS) m_progressMarkers.Submitted(bracket);
+        else m_progressMarkers.Abandon(bracket);
+#else
         const VkResult presentSubmit = vkQueueSubmit(m_graphicsQueue, 1, &submitPacket.submitInfo, frame.imageInFlightFence);
+#endif
         NoteDeviceLoss(presentSubmit, "Present vkQueueSubmit");
 #if MOBILEGL_BUILD_DISAGGREGATED
         if (presentSubmit != VK_SUCCESS && LatchWireDeviceLoss("present-submit")) return;

MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h

@@ -23,6 +23,7 @@
 #include "VkTextureManager.h"
 #include "VkTimerQueryManager.h"
 #if MOBILEGL_BUILD_DISAGGREGATED
+#include "GpuProgressMarkers.h"
 #include "WireRenderPassCompatibility.h"
 #include <MG_Remote/Server/SharedImageRegistry.h>
 #endif
@@ -408,6 +409,11 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         // without recording or submitting anything more. False: an ordinary failure, the site
         // keeps the death it had. `site` names it in the latch line.
         Bool LatchWireDeviceLoss(const char* site);
+        // THE GPU HANG WATCH (GpuProgressMarkers.h): true once the watch named one of this device's
+        // submissions as the one that stops the GPU. The device is then treated as lost, this session
+        // is latched, and the caller returns from its verb without submitting anything more. Asked
+        // before every frame submission and by the apply loop before it applies another record.
+        Bool LatchIfGpuHung(const char* site);
 #endif
         // Flush gated on usefulness: only flushes when `submitIndex` is still
         // unsubmitted, so poll loops on already-submitted fences do not split
@@ -626,6 +632,8 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         Bool m_deviceLost = false;
         Bool m_deviceLossLatched = false;
 #if MOBILEGL_BUILD_DISAGGREGATED
+        // Brackets this device's frame submissions for the GPU hang watch (LatchIfGpuHung).
+        GpuProgressMarkers m_progressMarkers;
         struct WireImage {
             VkImage image = VK_NULL_HANDLE;
             VkFormat format = VK_FORMAT_UNDEFINED;

MobileGL/MG_Backend/GpuHangWatch.cpp

@@ -0,0 +1,184 @@
+// MobileGL - MobileGL/MG_Backend/GpuHangWatch.cpp
+// Copyright (c) 2025-2026 MobileGL-Dev
+// Licensed under the GNU Lesser General Public License v3.0:
+//   https://www.gnu.org/licenses/gpl-3.0.txt
+//   https://www.gnu.org/licenses/lgpl-3.0.txt
+// SPDX-License-Identifier: LGPL-3.0-only
+// End of Source File Header
+
+#include "GpuHangWatch.h"
+
+#include <MG_Util/Debug/Log.h>
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <cstdlib>
+#include <thread>
+
+#if defined(__ANDROID__)
+#include <sys/system_properties.h>
+#endif
+#if defined(__linux__) || defined(__ANDROID__)
+#include <pthread.h>
+#endif
+
+namespace MobileGL::MG_Backend::GpuHangWatch {
+
+    namespace {
+        constexpr Uint32 kDefaultPollMs = 250;
+        std::atomic<Uint32> g_pollOverrideMs{0};
+        std::atomic<Uint32> g_budgetOverrideMs{0};
+        std::atomic<Bool> g_budgetOverridden{false};
+
+        Uint64 NowMs() {
+            return static_cast<Uint64>(std::chrono::duration_cast<std::chrono::milliseconds>(
+                                           std::chrono::steady_clock::now().time_since_epoch())
+                                           .count());
+        }
+
+        Uint32 ReadBudgetMs() {
+            const char* value = std::getenv("MOBILEGL_GPU_HANG_BUDGET_MS");
+#if defined(__ANDROID__)
+            char property[PROP_VALUE_MAX] = {};
+            if (value == nullptr && __system_property_get("debug.mobilegl.gpu_hang_budget_ms", property) > 0)
+                value = property;
+#endif
+            if (value == nullptr || *value == '\0') return kDefaultBudgetMs;
+            char* end = nullptr;
+            const unsigned long parsed = std::strtoul(value, &end, 10);
+            if (end == value) return kDefaultBudgetMs;
+            return static_cast<Uint32>(std::min<unsigned long>(parsed, 600000ul));
+        }
+
+        struct Entry {
+            Uint64 Handle = 0;
+            void* Self = nullptr;
+            SampleFn Sample = nullptr;
+            HungFn Hung = nullptr;
+            Detector Decide{0};
+            Bool Done = false; // named a hang, or its device is gone: never sampled again
+        };
+
+        // Leaked at exit, with its thread: the watch holds no GPU object of its own, and a backend
+        // unregisters before it destroys what its sampler reads.
+        struct Watch {
+            std::mutex Mutex;
+            std::condition_variable Wake;
+            Vector<Entry> Entries;
+            Uint64 NextHandle = 1;
+            Bool ThreadStarted = false;
+        };
+
+        Watch& TheWatch() {
+            static Watch& watch = *new Watch();
+            return watch;
+        }
+
+        void PollOnce(Watch& watch) {
+            const Uint64 now = NowMs();
+            Vector<SubmissionProgress> sample;
+            for (Entry& entry : watch.Entries) {
+                if (entry.Done) continue;
+                sample.clear();
+                if (!entry.Sample(entry.Self, sample)) {
+                    entry.Done = true;
+                    continue;
+                }
+                const Uint64 hung = entry.Decide.Observe(sample, now);
+                if (hung == 0) continue;
+                entry.Done = true;
+                entry.Hung(entry.Self, hung, entry.Decide.RunningMs(hung, now));
+            }
+        }
+
+        void WatchThreadMain() {
+#if defined(__linux__) || defined(__ANDROID__)
+            pthread_setname_np(pthread_self(), "mgl-gpu-watch");
+#endif
+            Watch& watch = TheWatch();
+            std::unique_lock<std::mutex> lock(watch.Mutex);
+            for (;;) {
+                watch.Wake.wait_for(lock, std::chrono::milliseconds(PollIntervalMs()));
+                PollOnce(watch);
+            }
+        }
+    } // namespace
+
+    Uint32 BudgetMs() {
+        if (g_budgetOverridden.load(std::memory_order_acquire)) return g_budgetOverrideMs.load(std::memory_order_acquire);
+        static const Uint32 budget = ReadBudgetMs();
+        return budget;
+    }
+
+    Uint32 PollIntervalMs() {
+        const Uint32 overrideMs = g_pollOverrideMs.load(std::memory_order_acquire);
+        return overrideMs != 0 ? overrideMs : kDefaultPollMs;
+    }
+
+    void SetPollIntervalForTesting(Uint32 ms) { g_pollOverrideMs.store(ms, std::memory_order_release); }
+
+    void SetBudgetForTesting(Uint32 ms) {
+        g_budgetOverrideMs.store(ms, std::memory_order_release);
+        g_budgetOverridden.store(ms != 0, std::memory_order_release);
+    }
+
+    Uint64 Detector::Observe(const Vector<SubmissionProgress>& sample, Uint64 nowMs) {
+        Vector<Running> still;
+        still.reserve(sample.size());
+        Uint64 named = 0;
+        for (const SubmissionProgress& progress : sample) {
+            if (!progress.Started || progress.Finished) continue;
+            Uint64 since = nowMs;
+            for (const Running& seen : m_running) {
+                if (seen.Id == progress.Id) {
+                    since = seen.SinceMs;
+                    break;
+                }
+            }
+            still.push_back({progress.Id, since});
+            if (named == 0 && m_budgetMs != 0 && nowMs - since >= m_budgetMs) named = progress.Id;
+        }
+        m_running.swap(still);
+        return named;
+    }
+
+    Uint64 Detector::RunningMs(Uint64 id, Uint64 nowMs) const {
+        for (const Running& seen : m_running)
+            if (seen.Id == id) return nowMs - seen.SinceMs;
+        return 0;
+    }
+
+    Uint64 Register(void* self, SampleFn sample, HungFn hung) {
+        const Uint32 budget = BudgetMs();
+        if (budget == 0 || sample == nullptr || hung == nullptr) return 0;
+        Watch& watch = TheWatch();
+        const std::lock_guard<std::mutex> lock(watch.Mutex);
+        Entry entry;
+        entry.Handle = watch.NextHandle++;
+        entry.Self = self;
+        entry.Sample = sample;
+        entry.Hung = hung;
+        entry.Decide = Detector(budget);
+        watch.Entries.push_back(entry);
+        if (!watch.ThreadStarted) {
+            watch.ThreadStarted = true;
+            std::thread(WatchThreadMain).detach();
+        }
+        return entry.Handle;
+    }
+
+    void Unregister(Uint64 handle) {
+        if (handle == 0) return;
+        Watch& watch = TheWatch();
+        // The poll pass runs under this lock, so taking it waits out any call into the entry.
+        const std::lock_guard<std::mutex> lock(watch.Mutex);
+        for (SizeT i = 0; i < watch.Entries.size(); ++i) {
+            if (watch.Entries[i].Handle != handle) continue;
+            watch.Entries.erase(watch.Entries.begin() + static_cast<std::ptrdiff_t>(i));
+            return;
+        }
+    }
+
+} // namespace MobileGL::MG_Backend::GpuHangWatch

MobileGL/MG_Backend/GpuHangWatch.h

@@ -0,0 +1,94 @@
+// MobileGL - MobileGL/MG_Backend/GpuHangWatch.h
+// Copyright (c) 2025-2026 MobileGL-Dev
+// Licensed under the GNU Lesser General Public License v3.0:
+//   https://www.gnu.org/licenses/gpl-3.0.txt
+//   https://www.gnu.org/licenses/lgpl-3.0.txt
+// SPDX-License-Identifier: LGPL-3.0-only
+// End of Source File Header
+
+#pragma once
+
+// A SESSION WHOSE GPU WORK NEVER ENDS IS FOUND AND NAMED, SO IT CANNOT HANG THE GPU AGAIN.
+//
+// A submission whose shader never ends stops the whole GPU, not one context: every process's work -
+// the compositor's, other clients', processes outside MobileGL at the highest context priority -
+// waits until the kernel resets it. Measured on Adreno 750: ~2 s when nothing of higher priority
+// waits (the context is then lost: Vulkan answers VK_ERROR_DEVICE_LOST), ~10 s when something of
+// higher priority does (a failed preemption; the context is NOT lost, and its next frame hangs the
+// GPU again). Nothing in user space can shorten one such stall; what the server can do is make sure
+// the session that caused it never submits again, so a page or program that hangs every frame
+// costs the desktop one stall instead of one per frame.
+//
+// The attribution must be exact - every session's work is pending while the GPU is stuck, and the
+// compositor's must never be blamed. A backend that can tell, per in-flight submission, whether the
+// GPU has STARTED it and whether it has FINISHED it gives exactly that: the GPU runs one submission
+// at a time on a ring, so the one that started and does not finish is the one that hangs, and the
+// ones queued behind it have not started. Magma brackets each submission with two events
+// (DirectVulkan/Renderer/GpuProgressMarkers.h). Espryt relies on the driver instead: its contexts ask
+// for reset notification and glGetGraphicsResetStatus names the guilty one - measured after a ~2 s
+// timeout reset; after a ~10 s failed-preemption reset the server's contexts were NOT told (a
+// standalone process's context was), so on Espryt that path is not contained yet.
+//
+// The Detector is the decision, pure and testable; the watch thread samples every registered backend
+// on its own clock, because the session that hangs is often not running - its apply thread waits for
+// its own fence, or for the backend turn another session holds while it waits for the GPU.
+
+#include <Includes.h>
+
+#include <mutex>
+
+namespace MobileGL::MG_Backend::GpuHangWatch {
+
+    // MOBILEGL_GPU_HANG_BUDGET_MS: how long one submission may run once the GPU has started it before
+    // its session is ended as the one that hangs the GPU. Default 2000 - the kernel resets a hung GPU
+    // by itself after about that when nothing preempts it, and no desktop frame runs that long. 0 = off.
+    Uint32 BudgetMs();
+    constexpr Uint32 kDefaultBudgetMs = 2000;
+
+    // One in-flight submission as the GPU reports progress on it.
+    struct SubmissionProgress {
+        Uint64 Id = 0;       // the backend's own monotonic id for it; never reused
+        Bool Started = false; // the GPU has begun executing it
+        Bool Finished = false; // ... and completed it
+    };
+
+    // Which submission, if any, has run past the budget: started, not finished, and seen in that
+    // state for at least `budgetMs` (from the first sample that saw it running - the watch cannot know
+    // when between two samples it began, so a hang is named between budget and budget + one interval).
+    class Detector {
+    public:
+        explicit Detector(Uint32 budgetMs) : m_budgetMs(budgetMs) {}
+        // One sample of every in-flight submission at `nowMs`. Returns the id of the submission that
+        // ran past the budget, or 0. Submissions missing from the sample are forgotten.
+        Uint64 Observe(const Vector<SubmissionProgress>& sample, Uint64 nowMs);
+        // How long the named submission had been seen running when Observe named it.
+        Uint64 RunningMs(Uint64 id, Uint64 nowMs) const;
+
+    private:
+        struct Running {
+            Uint64 Id = 0;
+            Uint64 SinceMs = 0;
+        };
+        Uint32 m_budgetMs = 0;
+        Vector<Running> m_running;
+    };
+
+    // ---- the process-wide watch ---------------------------------------------------------------
+    //
+    // A backend registers a sampler and a callback. The watch thread calls the sampler every
+    // PollIntervalMs() on its own thread, and the callback once, when the Detector names a
+    // submission; the backend acts on it from its own thread (it may not latch a session from this
+    // one). Both are called with the watch's lock held: keep them short, and never call Unregister
+    // from them. Unregister returns only once no call into the registration runs or will run.
+    using SampleFn = Bool (*)(void* self, Vector<SubmissionProgress>& out); // false: stop watching it
+    using HungFn = void (*)(void* self, Uint64 submissionId, Uint64 runningMs);
+
+    Uint64 Register(void* self, SampleFn sample, HungFn hung);
+    void Unregister(Uint64 handle);
+
+    Uint32 PollIntervalMs();
+    // Tests: the interval and the budget the next Register uses (0 restores the defaults).
+    void SetPollIntervalForTesting(Uint32 ms);
+    void SetBudgetForTesting(Uint32 ms);
+
+} // namespace MobileGL::MG_Backend::GpuHangWatch

MobileGL/MG_Remote/Server/ServerLoop.cpp

@@ -681,6 +681,13 @@ namespace MobileGL::MG_Remote::Server {
             std::optional<ScopedBackendTurn> turn(std::in_place);
             PumpControlRequest();
             if (stopOrForfeit()) break;
+            // A SESSION WHOSE OWN GPU WORK HUNG THE GPU APPLIES NOTHING MORE. A shader that never
+            // ends stops every process's GPU work until the kernel resets it (MG_Backend/
+            // GpuHangWatch.h), and the same client's next frame usually carries the same shader. The
+            // backend's verdict - the driver's GUILTY report on Espryt, the hang watch on Magma - is
+            // asked HERE, before the drain, so that frame is never applied: the session latches
+            // (DeviceLost, its client reads a lost context) and leaves below. Cheap by contract.
+            if (m_backend != nullptr && !SessionLatched()) (void)m_backend->LatchIfGpuFaulted();
             if (eventRingHasRoom()) DrainRing();
             if (stopOrForfeit()) break;
             // The session-targeted device-loss debug knob (PipeSessionFail.h): asked here, after a

MobileGL/MG_Test/Backend/GpuHangWatchTest.cpp

@@ -0,0 +1,177 @@
+// MobileGL - MobileGL/MG_Test/Backend/GpuHangWatchTest.cpp
+// Copyright (c) 2025-2026 MobileGL-Dev
+// Licensed under the GNU Lesser General Public License v3.0:
+//   https://www.gnu.org/licenses/gpl-3.0.txt
+//   https://www.gnu.org/licenses/lgpl-3.0.txt
+// SPDX-License-Identifier: LGPL-3.0-only
+// End of Source File Header
+
+// THE GPU HANG WATCH (MG_Backend/GpuHangWatch.h): which session's submission stops the GPU.
+//
+// While one submission hangs, EVERY session's work is pending - the compositor's included - so the
+// attribution may only rest on "the GPU started it and it did not finish". These cases pin that:
+//   * the submission seen running past the budget is named, and only once;
+//   * a submission queued behind it (never started) is never named, however long it waits - red with
+//     the Detector counting pending-but-not-started work as running (the compositor would be blamed);
+//   * a long but finishing submission is never named;
+//   * the budget runs from the first sample that saw it running, not from submission;
+//   * on the watch thread: a fake device whose submission never finishes is named within budget plus
+//     one interval, an innocent device waiting behind it is not, and Unregister waits out a sample.
+
+#include <gtest/gtest.h>
+
+#include <MG_Backend/GpuHangWatch.h>
+
+#include <atomic>
+#include <chrono>
+#include <thread>
+
+using namespace MobileGL;
+namespace Watch = MobileGL::MG_Backend::GpuHangWatch;
+
+namespace {
+    Vector<Watch::SubmissionProgress> One(Uint64 id, Bool started, Bool finished) {
+        return {Watch::SubmissionProgress{id, started, finished}};
+    }
+} // namespace
+
+TEST(GpuHangDetectorTest, ASubmissionRunningPastTheBudgetIsNamed) {
+    Watch::Detector detector(2000);
+    EXPECT_EQ(detector.Observe(One(7, true, false), 10000), 0u);
+    EXPECT_EQ(detector.Observe(One(7, true, false), 11000), 0u);
+    EXPECT_EQ(detector.Observe(One(7, true, false), 11999), 0u);
+    EXPECT_EQ(detector.Observe(One(7, true, false), 12000), 7u);
+    EXPECT_EQ(detector.RunningMs(7, 12000), 2000u);
+}
+
+TEST(GpuHangDetectorTest, ASubmissionQueuedBehindTheHangIsNeverNamed) {
+    // The compositor's frame waits behind a client's hang for the whole ~10 s stall: never started.
+    Watch::Detector detector(2000);
+    for (Uint64 now = 0; now <= 12000; now += 250) {
+        const Vector<Watch::SubmissionProgress> sample = {{1, true, false}, {2, false, false}};
+        const Uint64 named = detector.Observe(sample, now);
+        if (named != 0) {
+            EXPECT_EQ(named, 1u) << "the queued submission was blamed at " << now << " ms";
+        }
+    }
+    Watch::Detector innocentOnly(2000);
+    for (Uint64 now = 0; now <= 12000; now += 250)
+        EXPECT_EQ(innocentOnly.Observe(One(2, false, false), now), 0u) << "at " << now << " ms";
+}
+
+TEST(GpuHangDetectorTest, ALongSubmissionThatFinishesIsNeverNamed) {
+    Watch::Detector detector(2000);
+    EXPECT_EQ(detector.Observe(One(3, true, false), 0), 0u);
+    EXPECT_EQ(detector.Observe(One(3, true, false), 1900), 0u);
+    EXPECT_EQ(detector.Observe(One(3, true, true), 2100), 0u);
+    // Gone from the sample once retired: forgotten, so a new submission starts its own clock.
+    EXPECT_EQ(detector.Observe({}, 2200), 0u);
+    EXPECT_EQ(detector.Observe(One(4, true, false), 2300), 0u);
+    EXPECT_EQ(detector.Observe(One(4, true, false), 4200), 0u);
+    EXPECT_EQ(detector.Observe(One(4, true, false), 4300), 4u);
+}
+
+TEST(GpuHangDetectorTest, TheBudgetRunsFromTheFirstSampleThatSawItRunning) {
+    Watch::Detector detector(2000);
+    // Submitted and queued for 5 s (another process's work), then started.
+    EXPECT_EQ(detector.Observe(One(9, false, false), 0), 0u);
+    EXPECT_EQ(detector.Observe(One(9, false, false), 5000), 0u);
+    EXPECT_EQ(detector.Observe(One(9, true, false), 5250), 0u);
+    EXPECT_EQ(detector.Observe(One(9, true, false), 7000), 0u);
+    EXPECT_EQ(detector.Observe(One(9, true, false), 7250), 9u);
+}
+
+TEST(GpuHangDetectorTest, AZeroBudgetNamesNothing) {
+    Watch::Detector detector(0);
+    for (Uint64 now = 0; now <= 60000; now += 1000) EXPECT_EQ(detector.Observe(One(1, true, false), now), 0u);
+}
+
+namespace {
+    // A device double: its one submission's progress, and what the watch told it.
+    struct FakeDevice {
+        std::atomic<Bool> started{false};
+        std::atomic<Bool> finished{false};
+        std::atomic<int> samples{0};
+        std::atomic<int> named{0};
+        std::atomic<Uint64> namedId{0};
+        std::atomic<int> sampleDelayMs{0};
+
+        static Bool Sample(void* self, Vector<Watch::SubmissionProgress>& out) {
+            auto* device = static_cast<FakeDevice*>(self);
+            device->samples.fetch_add(1);
+            if (const int delay = device->sampleDelayMs.load(); delay > 0)
+                std::this_thread::sleep_for(std::chrono::milliseconds(delay));
+            out.push_back({42, device->started.load(), device->finished.load()});
+            return true;
+        }
+        static void Hung(void* self, Uint64 id, Uint64) {
+            auto* device = static_cast<FakeDevice*>(self);
+            device->namedId.store(id);
+            device->named.fetch_add(1);
+        }
+    };
+
+    template <typename F>
+    Bool PollUntil(F condition, int timeoutMs) {
+        const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeoutMs);
+        while (std::chrono::steady_clock::now() < deadline) {
+            if (condition()) return true;
+            std::this_thread::sleep_for(std::chrono::milliseconds(5));
+        }
+        return condition();
+    }
+
+    struct ScopedWatchTiming {
+        ScopedWatchTiming(Uint32 pollMs, Uint32 budgetMs) {
+            Watch::SetPollIntervalForTesting(pollMs);
+            Watch::SetBudgetForTesting(budgetMs);
+        }
+        ~ScopedWatchTiming() {
+            Watch::SetPollIntervalForTesting(0);
+            Watch::SetBudgetForTesting(0);
+        }
+    };
+} // namespace
+
+TEST(GpuHangWatchThreadTest, TheHangingDeviceIsNamedOnceAndTheOneWaitingBehindItIsNot) {
+    const ScopedWatchTiming timing(20, 200);
+    FakeDevice guilty;
+    FakeDevice innocent;
+    guilty.started = true;   // the GPU runs it and it never ends
+    innocent.started = false; // queued behind it for the whole stall
+    const Uint64 a = Watch::Register(&guilty, &FakeDevice::Sample, &FakeDevice::Hung);
+    const Uint64 b = Watch::Register(&innocent, &FakeDevice::Sample, &FakeDevice::Hung);
+    ASSERT_NE(a, 0u);
+    ASSERT_NE(b, 0u);
+    const auto start = std::chrono::steady_clock::now();
+    EXPECT_TRUE(PollUntil([&] { return guilty.named.load() > 0; }, 3000));
+    const auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - start).count();
+    EXPECT_GE(ms, 180) << "named before the budget ran out";
+    EXPECT_LE(ms, 1500) << "named far past budget + interval";
+    EXPECT_EQ(guilty.namedId.load(), 42u);
+    // Long after, still once, and the innocent never.
+    std::this_thread::sleep_for(std::chrono::milliseconds(400));
+    EXPECT_EQ(guilty.named.load(), 1);
+    EXPECT_EQ(innocent.named.load(), 0);
+    EXPECT_GT(innocent.samples.load(), 5) << "the innocent device was not even sampled";
+    Watch::Unregister(a);
+    Watch::Unregister(b);
+}
+
+TEST(GpuHangWatchThreadTest, UnregisterWaitsOutASampleInProgressAndNothingRunsAfterIt) {
+    const ScopedWatchTiming timing(10, 5000);
+    FakeDevice device;
+    device.sampleDelayMs = 100;
+    const Uint64 handle = Watch::Register(&device, &FakeDevice::Sample, &FakeDevice::Hung);
+    ASSERT_NE(handle, 0u);
+    ASSERT_TRUE(PollUntil([&] { return device.samples.load() > 0; }, 2000));
+    Watch::Unregister(handle);
+    const int after = device.samples.load();
+    std::this_thread::sleep_for(std::chrono::milliseconds(200));
+    EXPECT_EQ(device.samples.load(), after) << "the watch sampled a device after it unregistered";
+}
+
+TEST(GpuHangWatchThreadTest, ARegistrationWithoutASamplerIsRefused) {
+    EXPECT_EQ(Watch::Register(nullptr, nullptr, nullptr), 0u);
+    Watch::Unregister(0); // and unregistering "nothing" is harmless
+}

MobileGL/MG_Test/CMakeLists.txt

@@ -98,6 +98,11 @@ add_subdirectory(SelfTest)
 # The DirectGLES post-transpile ESSL passes are pure String -> String, so unlike the
 # DirectVulkan suite below this one needs no device and always builds.
 add_subdirectory(Backend/DirectGLES)
+# Which session's submission stops the GPU (MG_Backend/GpuHangWatch.h): no device, no driver.
+add_executable(GpuHangWatchTest Backend/GpuHangWatchTest.cpp)
+target_include_directories(GpuHangWatchTest PRIVATE ${MGL_ROOT}/include ${MGL_ROOT}/MobileGL)
+target_link_libraries(GpuHangWatchTest PRIVATE GTest::gtest_main ${LINK_LIBRARIES})
+gtest_discover_tests(GpuHangWatchTest DISCOVERY_TIMEOUT 30 PROPERTIES LABELS unit)
 if (ENABLE_INTEGRATION_TESTS)
     add_subdirectory(Backend/DirectVulkan)
 endif()

MobileGL/MG_Test/Wire/ServerLoopTest.cpp

@@ -1986,6 +1986,96 @@ TEST(ServerLoopEglTest, TheInitializeDisplayReplyArrivesThroughTheFrame) {
     fixture.TearDown();
 }
 
+// ESPRYT'S HALF OF HANG CONTAINMENT (DirectGLES EnsureNativeContext / LatchIfDeviceLost). With the
+// driver's default strategy a context whose shader hung the GPU is never told - measured on Adreno
+// 750, the guilty context reads NO_ERROR, and after a reset that a failed preemption ended (~10 s) its
+// next frame stops the GPU again, so a client that hangs every frame freezes the device for good.
+// The server's native contexts must ask for EGL_LOSE_CONTEXT_ON_RESET. Red with EnsureNativeContext's
+// strategy attribute removed: the context answers GL_NO_RESET_NOTIFICATION (0x8261).
+TEST(ServerLoopEglTest, TheServersNativeContextAsksTheDriverToReportResets) {
+    EglServerFixture fixture;
+    MGL_EGL_BRING_UP_OR_BAIL(fixture);
+    // On the apply thread: the server's display and context are current there, not here.
+    Bool robustness = false;
+    GLint strategy = 0;
+    ASSERT_EQ(OnApply([&] {
+                  const auto& egl = MG_Backend::DirectGLES::g_EGLFuncs;
+                  const char* extensions = egl.eglQueryString(egl.eglGetCurrentDisplay(), EGL_EXTENSIONS);
+                  robustness = extensions != nullptr &&
+                               std::strstr(extensions, "EGL_EXT_create_context_robustness") != nullptr;
+                  MG_Backend::DirectGLES::g_GLESFuncs.glGetIntegerv(0x8256 /*GL_RESET_NOTIFICATION_STRATEGY*/,
+                                                                    &strategy);
+              }),
+              MOBILEGL_OK);
+    if (!robustness) {
+        fixture.TearDown();
+        GTEST_SKIP() << "this EGL has no EGL_EXT_create_context_robustness";
+    }
+    EXPECT_EQ(strategy, 0x8252 /*GL_LOSE_CONTEXT_ON_RESET*/) << std::hex << "strategy 0x" << strategy;
+    EXPECT_TRUE(MG_Backend::DirectGLES::NativeContextsNotifyResets());
+    fixture.TearDown();
+}
+
+namespace {
+    // The driver's reset status, as the backend sees it through its function table.
+    std::atomic<GLenum> g_fakeResetStatus{GL_NO_ERROR};
+    GLenum FakeGetGraphicsResetStatus() { return g_fakeResetStatus.exchange(GL_NO_ERROR); }
+
+    enum : int {
+        kResetInnocentEnded = 1,  // an INNOCENT report latched the session (the compositor would die with a client)
+        kResetGuiltyApplied = 2,  // a record published after the GUILTY report was applied
+        kResetGuiltyNotLatched = 4, // GUILTY did not latch BackendDeviceLost through Espryt's apply-loop site
+        kResetHealthyNotApplied = 8, // a record before the GUILTY report was not applied
+    };
+
+    [[noreturn]] void ResetReportsAndExit() {
+        MG_Remote::ArmSessionLatch();
+        MG_Remote::InstallPipeSessionFailHook(); // the MG_Pipe latch seam LatchIfDeviceLost latches through
+        EglServerFixture fixture;
+        const std::string why = fixture.BringUp();
+        if (!why.empty()) {
+            std::fprintf(stderr, "[reset] SKIPPED - no headless EGL: %s\n", why.c_str());
+            ::_exit(RequireGpuFromEnvironment() ? 98 : 0);
+        }
+        MG_External::GLESFunctionsTable gles = MG_Backend::DirectGLES::g_GLESFuncs;
+        gles.glGetGraphicsResetStatus = &FakeGetGraphicsResetStatus;
+        MG_Backend::DirectGLES::SetGLESFuncsTable(gles);
+        int failed = 0;
+        const auto applyOne = [&] {
+            MG_Pipe::MGPMemoryBarrier barrier{};
+            barrier.Bits = 0x2000u;
+            const Uint64 seq = fixture.encoder.EncodeRecord(MG_Pipe::MGPWireOp::MemoryBarrier, &barrier, sizeof(barrier));
+            fixture.encoder.Publish();
+            fixture.producer.PublishAndNotify(seq);
+            return fixture.producer.WaitForApplied(seq, 2000) == Transport::SessionWait::Reached;
+        };
+        // Another context hung the GPU; this one only waited behind it.
+        g_fakeResetStatus = GL_INNOCENT_CONTEXT_RESET;
+        if (!applyOne()) failed |= kResetHealthyNotApplied;
+        if (!applyOne() || MG_Remote::SessionLatched()) failed |= kResetInnocentEnded;
+        // This context's own work hung the GPU.
+        g_fakeResetStatus = GL_GUILTY_CONTEXT_RESET;
+        if (applyOne()) failed |= kResetGuiltyApplied;
+        if (!MG_Remote::SessionLatched() || MG_Remote::SessionLatchedFamily() != MG_Remote::MGFatalFamily::BackendDeviceLost ||
+            std::strstr(MG_Remote::SessionLatchedLine(), "Espryt:apply") == nullptr)
+            failed |= kResetGuiltyNotLatched;
+        std::fprintf(stderr, "[reset] latched=%d line=%s\n", MG_Remote::SessionLatched() ? 1 : 0,
+                     MG_Remote::SessionLatchedLine());
+        ::_exit(failed);
+    }
+} // namespace
+
+// A GUILTY report ends the session before its next record; an INNOCENT one does not. Red with the
+// apply loop's LatchIfGpuFaulted call deleted (the record after GUILTY is applied, nothing latches:
+// bits 2|4), and with LatchIfDeviceLost latching every non-NO_ERROR status (bit 1).
+TEST(ServerLoopEglLatchTest, AGuiltyResetEndsTheSessionBeforeItsNextRecordAndAnInnocentOneDoesNot) {
+    PinHeadlessEglEnvironment();
+    EXPECT_EXIT(ResetReportsAndExit(), ::testing::ExitedWithCode(0),
+                ".*")
+        << "bits: 1 = INNOCENT latched; 2 = a record after GUILTY was applied; 4 = GUILTY did not latch "
+           "BackendDeviceLost at Espryt:apply; 8 = a healthy record was not applied (98 = no headless EGL while MOBILEGL_ITEST_REQUIRE_GPU is set)";
+}
+
 // ID-49's tight-size half, gated (review v2 N-5: "nothing, unit and joint"). The client's DstSize is
 // deliberately WRONG - 80, the size a ROW_LENGTH=8 / SKIP_* client would compute for a 4x3 RGBA8
 // read whose tight extent is 48 - and the reply must still be the tight 48 bytes: posted at 48,
@@ -3506,6 +3596,118 @@ TEST(ServerLoopLatchTest, ALostServerWindowIsReleasedMidBatchWhileTheClientStrea
            "latch; 32 = the apply thread stayed (64+ = setup)";
 }
 
+// =====================================================================================
+// A SESSION WHOSE OWN GPU WORK HUNG THE GPU APPLIES NOTHING MORE (BackendObject::LatchIfGpuFaulted)
+// =====================================================================================
+
+namespace {
+    // The backend's verdict arrives from elsewhere - the driver's GUILTY report (Espryt) or the hang
+    // watch's thread (Magma) - while the session's next frame is already queued. The double counts
+    // the frames that reach it and, once `faulted` is set, latches its session from the apply thread
+    // as both real backends do.
+    class GpuFaultBackendDouble final : public MG_Backend::BackendObject {
+    public:
+        GpuFaultBackendDouble() { m_functions.Present = &Present; }
+        Bool LatchIfGpuFaulted() override {
+            asked.fetch_add(1, std::memory_order_acq_rel);
+            if (!faulted.load(std::memory_order_acquire)) return false;
+            (void)MG_Pipe::MGPipeSessionLatch(MG_Pipe::MGPipeFatalFamily::DeviceLost,
+                                              "MGPipe: Fatal{BackendDeviceLost, \"unit:gpu-hang\"} - the double's "
+                                              "own GPU work hung the GPU");
+            return true;
+        }
+        void Initialize() override {}
+        Bool InitCapabilities() override { return true; }
+        Bool InitWindowSurface() override { return true; }
+        const RendererInfo& GetRendererInfo() const override { return m_info; }
+        String GetBackendAPIVersionString() const override { return "gpu fault double"; }
+        const MG_Backend::GlobalBackendFunctionsTable& GetBackendFunctions() const override { return m_functions; }
+        const MG_Backend::DynamicBackendParameters& GetDynamicParameters() const override { return m_params; }
+        BackendType GetBackendType() const override { return BackendType::Unknown; }
+
+        static void Present() { presents.fetch_add(1, std::memory_order_acq_rel); }
+
+        static inline std::atomic<int> presents{0};
+        static inline std::atomic<int> asked{0};
+        static inline std::atomic<Bool> faulted{false};
+
+    private:
+        MG_Backend::GlobalBackendFunctionsTable m_functions{};
+        MG_Backend::DynamicBackendParameters m_params{};
+        RendererInfo m_info{.RendererName = "Test",
+                            .BackendName = "GpuFaultBackendDouble",
+                            .ExtraVendor = Nullopt,
+                            .RendererGLInfo = {.TargetGLVersion = {3, 3, 0},
+                                               .TargetGLSLVersion = {4, 6, 0},
+                                               .Extensions = {},
+                                               .IsCompatibilityProfile = false},
+                            .StaticBackendCapability = {.AllowVSOnlyPrograms = false}};
+    };
+
+    enum : int {
+        kFaultFirstFrame = 1,   // the healthy frame before the fault was not applied
+        kFaultFrameApplied = 2, // a frame published after the verdict reached the backend
+        kFaultNotLatched = 4,   // the session did not latch DeviceLost with the backend's line
+        kFaultThreadStayed = 8, // the apply thread was still running 3 s after the verdict
+        kFaultNotAsked = 16,    // the loop never asked the backend
+    };
+
+    [[noreturn]] void ApplyNothingAfterAGpuFaultAndExit() {
+        Remote::ArmSessionLatch();
+        Remote::InstallPipeSessionFailHook(); // the MG_Pipe latch seam the backends latch through (InitServerRoleCommon)
+        ServerFixture fixture;
+        if (!fixture.Handshake()) ::_exit(64);
+        if (Server::ServerLoopInstance().InstallBackendForTesting(MakeUnique<GpuFaultBackendDouble>()) != MOBILEGL_OK)
+            ::_exit(65);
+        if (!fixture.StartLoop() || !fixture.WaitUntilTrulyParked()) ::_exit(66);
+        Server::ServerLoop& loop = Server::ServerLoopInstance();
+        int failed = 0;
+
+        // One healthy frame.
+        MG_Pipe::MGPPresent present{};
+        present.FrameSerial = 1;
+        Uint64 seq = fixture.encoder.EncodeRecord(MG_Pipe::MGPWireOp::Present, &present, sizeof(present));
+        if (seq == Codec::kInvalidSeq) ::_exit(67);
+        fixture.encoder.Publish();
+        fixture.producer.PublishAndNotify(seq);
+        if (fixture.producer.WaitForApplied(seq, 3000) != Transport::SessionWait::Reached ||
+            GpuFaultBackendDouble::presents.load() != 1)
+            failed |= kFaultFirstFrame;
+        if (!fixture.WaitUntilTrulyParked()) ::_exit(68);
+
+        // The verdict lands while the session is parked; then its next frames arrive.
+        GpuFaultBackendDouble::faulted.store(true, std::memory_order_release);
+        for (Uint64 serial = 2; serial <= 4; ++serial) {
+            present.FrameSerial = serial;
+            seq = fixture.encoder.EncodeRecord(MG_Pipe::MGPWireOp::Present, &present, sizeof(present));
+            if (seq == Codec::kInvalidSeq) ::_exit(69);
+        }
+        fixture.encoder.Publish();
+        fixture.producer.PublishAndNotify(seq);
+
+        if (!PollUntil([&] { return !loop.Running(); }, 3000)) failed |= kFaultThreadStayed;
+        if (GpuFaultBackendDouble::presents.load() != 1) failed |= kFaultFrameApplied;
+        if (!Remote::SessionLatched() || Remote::SessionLatchedFamily() != Remote::MGFatalFamily::BackendDeviceLost ||
+            std::strstr(Remote::SessionLatchedLine(), "\"unit:gpu-hang\"") == nullptr)
+            failed |= kFaultNotLatched;
+        if (GpuFaultBackendDouble::asked.load() == 0) failed |= kFaultNotAsked;
+        std::fprintf(stderr, "[gpu-fault] presents %d, asked %d, drained %llu\n", GpuFaultBackendDouble::presents.load(),
+                     GpuFaultBackendDouble::asked.load(), static_cast<unsigned long long>(loop.DrainedRecords()));
+        fixture.Stop();
+        ::_exit(failed);
+    }
+} // namespace
+
+// Red with ServerLoop's per-wakeup LatchIfGpuFaulted call deleted: the three frames behind the
+// verdict are applied - on a device each would stop the GPU again - and nothing latches (bits
+// 2|4|8|16).
+TEST(ServerLoopLatchTest, ASessionWhoseGpuWorkHungAppliesNoFrameAfterTheVerdict) {
+    EXPECT_EXIT(ApplyNothingAfterAGpuFaultAndExit(), ::testing::ExitedWithCode(0), ".*")
+        << "bits: 1 = the healthy frame was not applied; 2 = a frame behind the verdict was applied; 4 = no "
+           "DeviceLost latch with the backend's line; 8 = the apply thread stayed; 16 = the loop never asked "
+           "(64+ = setup)";
+}
+
 // =====================================================================================
 // SUSPEND, NOT END: the server window going away (screen off, lock) and coming back
 // =====================================================================================