[Fix] (MG_Backend/DirectVulkan): fixing some layout mismatch

MobileGL/MG_Backend/DirectVulkan/Renderer/VkRenderPassManager.cpp

@@ -15,12 +15,13 @@
 namespace MobileGL::MG_Backend::DirectVulkan {
     VkRenderPassManager::VkRenderPassManager(VkDevice device,
         const VulkanRendererConfig& config, VkClearManager& clearManager, VkTextureManager& textureManager,
-        const SwapchainObject& swapchainObject):
+        SwapchainObject& swapchainObject):
         m_device(device), m_config(config), m_clearManager(clearManager), m_textureManager(textureManager),
         m_swapchainObject(swapchainObject) {
         RenderPassEntry::s_device = m_device;
         s_clearManager = &m_clearManager;
         s_textureManager = &m_textureManager;
+        s_swapchainObject = &m_swapchainObject;
     }
 
     VkRenderPassManager::~VkRenderPassManager() {}
@@ -79,6 +80,23 @@ namespace MobileGL::MG_Backend::DirectVulkan {
                             m_hashState, &clearPayload.attachmentType, sizeof(clearPayload.attachmentType)));
                     }
                 }
+
+                VkImageLayout currentLayout = VK_IMAGE_LAYOUT_UNDEFINED;
+                if (isDefaultFbo) {
+                    if (attachment >= FramebufferAttachmentType::Color0 &&
+                        attachment <= FramebufferAttachmentType::Color31) {
+                        currentLayout = m_swapchainObject.GetImageLayout(swapchainImageIndex);
+                    } else if (attachment == FramebufferAttachmentType::Depth ||
+                               attachment == FramebufferAttachmentType::Stencil) {
+                        currentLayout = m_swapchainObject.GetDepthStencilImageLayout(swapchainImageIndex);
+                    }
+                } else {
+                    auto* resource = m_textureManager.SyncTextureAndGetDescriptor(*texture);
+                    if (resource != nullptr) {
+                        currentLayout = resource->layout;
+                    }
+                }
+                XXHASH_VERIFY(XXH64_update(m_hashState, &currentLayout, sizeof(currentLayout)));
             }
         };
 
@@ -181,7 +199,12 @@ namespace MobileGL::MG_Backend::DirectVulkan {
                                         "GetOrCreateRenderPass: swapchain image index out of range");
                         desc.initialLayout = hasClear ?
                             VK_IMAGE_LAYOUT_UNDEFINED :
-                            VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
+                            m_swapchainObject.GetImageLayout(swapchainImageIndex);
+                        trackedAttachmentLayouts.emplace_back(TrackedAttachmentLayoutInfo {
+                            .target = TrackedAttachmentTarget::SwapchainColor,
+                            .swapchainImageIndex = swapchainImageIndex,
+                            .finalLayout = desc.finalLayout,
+                        });
                         attachmentViews[i] = swapchainViews[swapchainImageIndex];
                     } else {
                         textureResources[i] = m_textureManager.SyncTextureAndGetDescriptor(*texture);
@@ -194,6 +217,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
                             VK_IMAGE_LAYOUT_UNDEFINED :
                             textureResources[i]->layout;
                         trackedAttachmentLayouts.emplace_back(TrackedAttachmentLayoutInfo {
+                            .target = TrackedAttachmentTarget::Texture,
                             .texture = texture,
                             .finalLayout = desc.finalLayout,
                         });
@@ -226,7 +250,9 @@ namespace MobileGL::MG_Backend::DirectVulkan {
             Bool hasClear = m_clearManager.GetPendingClear(&texture, clearPayload);
             Bool clearDepth = hasClear && clearPayload.attachmentType == FramebufferAttachmentType::Depth;
             Bool clearStencil = hasClear && clearPayload.attachmentType == FramebufferAttachmentType::Stencil;
-            VkImageLayout trackedDepthLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
+            VkImageLayout trackedDepthLayout = isDefaultFbo ?
+                m_swapchainObject.GetDepthStencilImageLayout(swapchainImageIndex) :
+                VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
             if (!isDefaultFbo) {
                 depthTextureResource = m_textureManager.SyncTextureAndGetDescriptor(texture);
                 MOBILEGL_ASSERT(depthTextureResource,
@@ -246,7 +272,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
                 VK_ATTACHMENT_LOAD_OP_LOAD;
             depthAttachmentDescription.stencilStoreOp = VK_ATTACHMENT_STORE_OP_STORE;
             depthAttachmentDescription.finalLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
-            if (!isDefaultFbo && trackedDepthLayout == VK_IMAGE_LAYOUT_UNDEFINED) {
+            if (trackedDepthLayout == VK_IMAGE_LAYOUT_UNDEFINED) {
                 if (!clearDepth) {
                     depthAttachmentDescription.loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
                 }
@@ -255,7 +281,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
                 }
             }
             depthAttachmentDescription.initialLayout =
-                (clearDepth && clearStencil) || (!isDefaultFbo && trackedDepthLayout == VK_IMAGE_LAYOUT_UNDEFINED) ?
+                (clearDepth && clearStencil) || trackedDepthLayout == VK_IMAGE_LAYOUT_UNDEFINED ?
                     VK_IMAGE_LAYOUT_UNDEFINED :
                     trackedDepthLayout;
             if (hasClear) {
@@ -265,12 +291,18 @@ namespace MobileGL::MG_Backend::DirectVulkan {
                 });
             }
             if (isDefaultFbo) {
+                trackedAttachmentLayouts.emplace_back(TrackedAttachmentLayoutInfo {
+                    .target = TrackedAttachmentTarget::SwapchainDepthStencil,
+                    .swapchainImageIndex = swapchainImageIndex,
+                    .finalLayout = depthAttachmentDescription.finalLayout,
+                });
                 attachmentViews.emplace_back(m_swapchainObject.GetDepthStencilImageView(swapchainImageIndex));
             } else {
                 MOBILEGL_ASSERT(clearDepth || clearStencil || depthTextureResource->layout != VK_IMAGE_LAYOUT_UNDEFINED,
                                 "GetOrCreateRenderPass: depth attachment textureId=%d has undefined tracked layout with LOAD_OP_LOAD",
                                 texture.GetExternalIndex());
                 trackedAttachmentLayouts.emplace_back(TrackedAttachmentLayoutInfo {
+                    .target = TrackedAttachmentTarget::Texture,
                     .texture = &texture,
                     .finalLayout = depthAttachmentDescription.finalLayout,
                 });
@@ -401,9 +433,26 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         auto* activeRenderPass = GetActiveRenderPass();
         vkCmdEndRenderPass(commandBuffer);
         if (activeRenderPass != nullptr && !activeRenderPass->trackedAttachmentLayouts.empty()) {
-            MOBILEGL_ASSERT(s_textureManager != nullptr, "EndRenderPass: texture manager is null");
             for (const auto& trackedAttachment : activeRenderPass->trackedAttachmentLayouts) {
+                switch (trackedAttachment.target) {
+                    case TrackedAttachmentTarget::Texture:
+                        MOBILEGL_ASSERT(s_textureManager != nullptr, "EndRenderPass: texture manager is null");
                         s_textureManager->UpdateTrackedImageLayout(trackedAttachment.texture, trackedAttachment.finalLayout);
+                        break;
+                    case TrackedAttachmentTarget::SwapchainColor:
+                        MOBILEGL_ASSERT(s_swapchainObject != nullptr, "EndRenderPass: swapchain object is null");
+                        s_swapchainObject->SetImageLayout(trackedAttachment.swapchainImageIndex, trackedAttachment.finalLayout);
+                        break;
+                    case TrackedAttachmentTarget::SwapchainDepthStencil:
+                        MOBILEGL_ASSERT(s_swapchainObject != nullptr, "EndRenderPass: swapchain object is null");
+                        s_swapchainObject->SetDepthStencilImageLayout(trackedAttachment.swapchainImageIndex,
+                                                                      trackedAttachment.finalLayout);
+                        break;
+                    default:
+                        MOBILEGL_ASSERT(false, "EndRenderPass: unsupported tracked attachment target=%d",
+                                        static_cast<Int>(trackedAttachment.target));
+                        break;
+                }
             }
         }
         s_activeRenderPass = {};

MobileGL/MG_Backend/DirectVulkan/Renderer/VkRenderPassManager.h

@@ -18,6 +18,11 @@
 #include <Includes.h>
 
 namespace MobileGL::MG_Backend::DirectVulkan {
+    enum class TrackedAttachmentTarget : Uint8 {
+        Texture,
+        SwapchainColor,
+        SwapchainDepthStencil
+    };
 
     struct PendingClearAttachmentInfo {
         Uint32 attachmentIndex = 0;
@@ -25,7 +30,9 @@ namespace MobileGL::MG_Backend::DirectVulkan {
     };
 
     struct TrackedAttachmentLayoutInfo {
+        TrackedAttachmentTarget target = TrackedAttachmentTarget::Texture;
         MG_State::GLState::ITextureObject* texture = nullptr;
+        Uint32 swapchainImageIndex = 0;
         VkImageLayout finalLayout = VK_IMAGE_LAYOUT_UNDEFINED;
     };
 
@@ -113,7 +120,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         using HashType = Uint64;
         VkRenderPassManager(VkDevice device,
             const VulkanRendererConfig& config, VkClearManager& clearManager, VkTextureManager& textureManager,
-            const SwapchainObject& swapchainObject);
+            SwapchainObject& swapchainObject);
         ~VkRenderPassManager();
 
         Bool Initialize();
@@ -132,12 +139,13 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         const VulkanRendererConfig& m_config;
         VkClearManager& m_clearManager;
         VkTextureManager& m_textureManager;
-        const SwapchainObject& m_swapchainObject;
+        SwapchainObject& m_swapchainObject;
         UnorderedMap<Uint64, RenderPassEntry> m_renderPasses;
         static inline XXH64_state_t* m_hashState = XXH64_createState();
         static inline ActiveRenderPassInfo s_activeRenderPass{};
         static inline Bool s_hasActiveRenderPass = false;
         static inline VkClearManager* s_clearManager = nullptr;
         static inline VkTextureManager* s_textureManager = nullptr;
+        static inline SwapchainObject* s_swapchainObject = nullptr;
     };
 } // namespace MobileGL::MG_Backend::DirectVulkan

MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.cpp

@@ -95,18 +95,24 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         VkPipelineStageFlags srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT;
         VkAccessFlags srcAccessMask = 0;
         if ((resource->aspect & VK_IMAGE_ASPECT_COLOR_BIT) != 0) {
-            MOBILEGL_ASSERT(resource->layout == VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
+            MOBILEGL_ASSERT(resource->layout == VK_IMAGE_LAYOUT_UNDEFINED ||
+                            resource->layout == VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
                             "TransitionTextureForSampling: unsupported color layout=%d for textureId=%d",
                             static_cast<Int>(resource->layout), texture.GetExternalIndex());
+            if (resource->layout == VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL) {
                 srcStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
                 srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
+            }
             targetLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
         } else if ((resource->aspect & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) != 0) {
-            MOBILEGL_ASSERT(resource->layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL,
+            MOBILEGL_ASSERT(resource->layout == VK_IMAGE_LAYOUT_UNDEFINED ||
+                            resource->layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL,
                             "TransitionTextureForSampling: unsupported depth/stencil layout=%d for textureId=%d",
                             static_cast<Int>(resource->layout), texture.GetExternalIndex());
+            if (resource->layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL) {
                 srcStageMask = VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT;
                 srcAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
+            }
             targetLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL;
         } else {
             MOBILEGL_ASSERT(false, "TransitionTextureForSampling: unsupported aspect mask=0x%x for textureId=%d",

MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp

@@ -335,7 +335,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
             GLenum mode,
             const MG_State::GLState::ProgramObject& program,
             const MG_State::GLState::VertexArrayObject& vao,
-            const MG_State::GLState::FramebufferObject& drawFbo) {
+            const RenderPassEntry& renderPassEntry) {
         ProgramFactory::CompileOptionFlags transformFlags = GetShaderTransformFlags(m_swapchainObject.GetPreTransform());
         Bool invertClockwise = transformFlags & ProgramFactory::CompileOptionBit::PositionYFlip;
         auto& stages = m_programFactory->GetOrCreatePipelineShaderStages(program, transformFlags);
@@ -348,7 +348,6 @@ namespace MobileGL::MG_Backend::DirectVulkan {
         auto vertexInputHash = m_vertexInputStateFactory->ComputeHash(vao);
         auto& vis = m_vertexInputStateFactory->GetOrCreateVertexInputState(vao);
         auto pipelineLayout = m_uniformDescriptorBinder->GetOrCreatePipelineLayout(program);
-        auto& renderPassEntry = m_renderPassManager->GetOrCreateRenderPass(drawFbo, m_imageIndexAcquired);
         auto cullFaceEnabled = MG_State::pGLContext->IsCapabilityEnabled(CapabilityInput::CullFace);
         auto depthTestEnabled = MG_State::pGLContext->IsCapabilityEnabled(CapabilityInput::DepthTest);
         BlendFactor srcRGB = BlendFactor::One;
@@ -389,15 +388,10 @@ namespace MobileGL::MG_Backend::DirectVulkan {
     }
 
     void VulkanRenderer::SetupDraw(FrameContext::FrameData& frame, GLenum mode, Flags<DrawSetupAspect> aspects) {
-        // Prepare render pass
         const auto& drawFbo =
                 MG_State::pGLContext->GetFramebufferBindingSlot(FramebufferTarget::Draw).GetBoundObject();
-        auto& renderPassEntry = m_renderPassManager->GetOrCreateRenderPass(*drawFbo, m_imageIndexAcquired);
-
-        // Prepare pipeline
         const auto& vao = *MG_State::pGLContext->GetBoundVertexArray();
         const auto& program = *MG_State::pGLContext->GetCurrentProgram();
-        auto pipeline = GetOrCreatePipeline(mode, program, vao, *drawFbo);
 
         // Begin command recording if not yet
         if (!frame.isCommandRecording) {
@@ -423,6 +417,10 @@ namespace MobileGL::MG_Backend::DirectVulkan {
                             __func__, sampledTexture->GetExternalIndex());
         }
 
+        auto& renderPassEntry = m_renderPassManager->GetOrCreateRenderPass(*drawFbo, m_imageIndexAcquired);
+        auto pipeline = GetOrCreatePipeline(mode, program, vao, renderPassEntry);
+        activeRenderPass = VkRenderPassManager::GetActiveRenderPass();
+
         // Begin render pass, and handle clear
 
         if (activeRenderPass && activeRenderPass->CompatibleWith(renderPassEntry)) {

MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h

@@ -153,7 +153,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
             GLenum mode,
             const MG_State::GLState::ProgramObject& program,
             const MG_State::GLState::VertexArrayObject& vao,
-            const MG_State::GLState::FramebufferObject& drawFbo);
+            const RenderPassEntry& renderPassEntry);
 
         void DeferDestroyBuffer(VkBufferObject& buffer);
         void CollectDeferredBufferReleases(Uint32 frameIndex);