[Fix] (DirectVulkan): transpose quarter-turn viewports and scissors, rebuild a resized pbuffer's target, and leave out-of-range default-framebuffer reads unfilled

MobileGL/MG_Backend/DirectVulkan/BackendObject_DirectVulkan.cpp

@@ -489,6 +489,22 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         if (!BackendObject::ResizeEGLWindowSurface(surface, width, height)) {
             return false;
         }
+        const EGLSurfaceState* state = GetRegisteredEGLSurface(surface);
+        if (UsesSurfaceTargets() && pVulkanRenderer && state != nullptr && state->Kind == SurfaceKind::Pbuffer) {
+            // A pbuffer-backed window (a Wayland window presented through wl_shm) whose target is an
+            // image reader of FIXED size: a swapchain rebuild cannot follow the resize, so the target
+            // is rebuilt at the new size - as DirectGLES replaces its native pbuffer. A parked one is
+            // rebuilt when it is next activated, from the size registered above.
+            const Bool active = m_eglSurfaceInitialized && m_eglSurface == surface;
+            pVulkanRenderer->DestroySurfaceTarget(SurfaceTargetKey(surface));
+            if (active) {
+                VulkanRendererConfig config;
+                config.SurfaceWidth = static_cast<Uint32>(state->Width);
+                config.SurfaceHeight = static_cast<Uint32>(state->Height);
+                pVulkanRenderer->ActivateSurfaceTarget(SurfaceTargetKey(surface), NativeWindowType{}, config);
+            }
+            return true;
+        }
         if (pVulkanRenderer && m_eglSurface == surface) {
             pVulkanRenderer->RequestSwapchainResize(width, height);
         }

MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp

@@ -243,11 +243,23 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         return rawExtent;
     }
 
-    static Int ScaleFramebufferCoordinate(Int value, Int fromExtent, Int toExtent) {
-        if (fromExtent <= 0 || toExtent <= 0) {
-            return value;
+    // A GL bottom-left-origin rectangle on a quarter-turned default framebuffer, expressed in the
+    // stored (rotated) image: the rectangle transposes. This is the mapping the vertex fixup implies
+    // (InsertPositionFixup: Y flip, then (x, y) -> (-y, x) for 90 and (y, -x) for 270) and the one
+    // MapDefaultFramebufferReadbackRect copies with, so a viewport, a scissor and a readback of the
+    // same GL rectangle all land on the same pixels. Unclamped: a viewport may extend past the image.
+    static void MapQuarterTurnRect(Int x, Int y, Int width, Int height, const IntVec2& imageExtent,
+                                   VkSurfaceTransformFlagBitsKHR preTransform, Int* outX, Int* outY, Int* outWidth,
+                                   Int* outHeight) {
+        if (preTransform == VK_SURFACE_TRANSFORM_ROTATE_90_BIT_KHR) {
+            *outX = y;
+            *outY = x;
+        } else {
+            *outX = imageExtent.x() - y - height;
+            *outY = imageExtent.y() - x - width;
         }
-        return static_cast<Int>((static_cast<Int64>(value) * toExtent + fromExtent / 2) / fromExtent);
+        *outWidth = height;
+        *outHeight = width;
     }
 
     // ---------------------------------------------------------------------------------------
@@ -596,10 +608,10 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         Int viewportHeight = viewportState.w() > 0 ? viewportState.w() : logicalExtent.y();
 
         if (isDefaultFramebuffer && IsQuarterTurnPreTransform(preTransform)) {
-            viewportX = ScaleFramebufferCoordinate(viewportX, logicalExtent.x(), framebufferExtent.x());
-            viewportY = ScaleFramebufferCoordinate(viewportY, logicalExtent.y(), framebufferExtent.y());
-            viewportWidth = ScaleFramebufferCoordinate(viewportWidth, logicalExtent.x(), framebufferExtent.x());
-            viewportHeight = ScaleFramebufferCoordinate(viewportHeight, logicalExtent.y(), framebufferExtent.y());
+            // Transposed, not scaled: scaling each axis by the other's extent only coincides with
+            // the rotation for a viewport covering the whole surface.
+            MapQuarterTurnRect(viewportState.x(), viewportState.y(), viewportWidth, viewportHeight, framebufferExtent,
+                               preTransform, &viewportX, &viewportY, &viewportWidth, &viewportHeight);
         }
 
         // The GL viewport rect, expressed against the default framebuffer's stored orientation.
@@ -766,13 +778,20 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         const Int logicalX1 = std::min<Int>(logicalExtent.x(), scissorBox.x() + std::max<Int>(0, scissorBox.z()));
         const Int logicalY1 = std::min<Int>(logicalExtent.y(), scissorBox.y() + std::max<Int>(0, scissorBox.w()));
 
-        const Int rawX0 = ScaleFramebufferCoordinate(logicalX0, logicalExtent.x(), framebufferExtent.x());
-        const Int rawY0 = ScaleFramebufferCoordinate(logicalY0, logicalExtent.y(), framebufferExtent.y());
-        const Int rawX1 = ScaleFramebufferCoordinate(logicalX1, logicalExtent.x(), framebufferExtent.x());
-        const Int rawY1 = ScaleFramebufferCoordinate(logicalY1, logicalExtent.y(), framebufferExtent.y());
+        // Transposed into the rotated image (MapQuarterTurnRect). Scaling each axis by the other's
+        // extent - what this did before - only agrees with the rotation for a full-surface box, and
+        // turned a compositor's per-window clip into an unrelated sub-rectangle.
+        Int rawX = 0, rawY = 0, rawWidth = 0, rawHeight = 0;
+        MapQuarterTurnRect(logicalX0, logicalY0, std::max<Int>(0, logicalX1 - logicalX0),
+                           std::max<Int>(0, logicalY1 - logicalY0), framebufferExtent, preTransform, &rawX, &rawY,
+                           &rawWidth, &rawHeight);
+        const Int rawX0 = std::max<Int>(0, rawX);
+        const Int rawY0 = std::max<Int>(0, rawY);
+        const Int rawX1 = std::min<Int>(framebufferExtent.x(), rawX + rawWidth);
+        const Int rawY1 = std::min<Int>(framebufferExtent.y(), rawY + rawHeight);
 
         VkRect2D scissor{};
-        scissor.offset = {std::max<Int>(0, rawX0), std::max<Int>(0, rawY0)};
+        scissor.offset = {rawX0, rawY0};
         scissor.extent = {
             static_cast<Uint32>(std::max<Int>(0, rawX1 - rawX0)),
             static_cast<Uint32>(std::max<Int>(0, rawY1 - rawY0)),

MobileGL/MG_Backend/DirectVulkan/Renderer/WireFramebuffer.inc

@@ -400,6 +400,22 @@
                          "default framebuffer; the readback returns its raw pixels",
                          source.swapchainImageIndex);
         }
+        if (source.isDefault && aspect == VK_IMAGE_ASPECT_COLOR_BIT) {
+            VkOffset2D probeOffset{};
+            VkExtent2D probeExtent{};
+            if (!MapDefaultFramebufferReadbackRect(x, y, width, height, source.extent,
+                                                   m_swapchainObject.GetPreTransform(), &probeOffset, &probeExtent)) {
+                // A rectangle reaching past the surface: GL leaves those pixels undefined rather
+                // than raising an error, and the usual cause is a client reading back at its
+                // window's new size one frame before its surface follows the resize. Nothing is
+                // copied, and nothing is recorded.
+                MGLOG_E_ONCE("Magma wire: default-framebuffer read %dx%d at (%d,%d) does not fit surface %llx's "
+                        "%ux%u (transform %u); the read is left unfilled", width, height, x, y,
+                        static_cast<unsigned long long>(m_activeTargetKey), source.extent.width,
+                        source.extent.height, static_cast<Uint32>(m_swapchainObject.GetPreTransform()));
+                return;
+            }
+        }
         m_textureManager->FlushPendingUploads();
         RecordWireReadbackWriteBarrier();
         TransitionWireImage(source, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);