[Feature] (SharedImage): GPU-side fences between sessions on Espryt and Magma, no CPU wait at present

This commit is contained in:
2026-10-02 10:31:34 -04:00
parent b5cab44100
commit 7520e53b5e
13 changed files with 845 additions and 58 deletions
@@ -42,7 +42,8 @@ namespace MobileGL::MG_Backend::DirectGLES {
Bool MakeEGLCurrent(EGLDisplay dpy, EGLSurface draw, EGLSurface read, EGLContext ctx) override;
Bool SwapEGLBuffers(EGLDisplay dpy, EGLSurface draw) override;
// Shared images: a GL blit of default framebuffer 0 into a renderbuffer whose storage is
// the image's EGLImage, then a wait on its fence (DirectGLES.h, SharedImageImpl).
// the image's EGLImage, ordered after the image's pending reads and published as a native
// fence without a CPU wait (DirectGLES.h, SharedImageImpl).
Bool BlitDefaultFramebufferToSharedImage(const SharedImageView& image) override;
void ReleaseEGLSurface(EGLSurface surface) override;
void ReleaseEGLResources() override;
+208 -4
View File
@@ -36,6 +36,7 @@
#include <MG_Remote/Server/SharedImageRegistry.h>
#if defined(__ANDROID__)
#include <android/hardware_buffer.h>
#include <unistd.h>
#endif
#endif
#include <MG_State/GLState/ErrorState/Error.h>
@@ -17025,6 +17026,10 @@ namespace MobileGL::MG_Backend::DirectGLES {
const std::lock_guard<std::mutex> lock(g_nativeRegistryMutex);
NativeContextTuple* tuple = FindNativeContext(sessionKey, contextToken);
if (tuple == nullptr || tuple->Context == EGL_NO_CONTEXT) return;
#if MOBILEGL_BUILD_DISAGGREGATED
// Its last shared-image reads, fenced while it is still current.
if (t_boundNativeContext == tuple->Context) SharedImageImpl::PublishPendingReads();
#endif
if (g_EGLFuncs.eglDestroyContext && g_Display != EGL_NO_DISPLAY) {
// The context may be current on this very thread; unbinding first is what EGL requires
// (eglDestroyContext of a current context is EGL_BAD_ACCESS).
@@ -17406,6 +17411,12 @@ namespace MobileGL::MG_Backend::DirectGLES {
MGLOG_E_ONCE("DirectGLES::MakeCurrent failed: EGL display/surface/context is not initialized");
return false;
}
#if MOBILEGL_BUILD_DISAGGREGATED
// Shared-image reads can only be fenced from the context that made them.
if (t_boundNativeContext != EGL_NO_CONTEXT && t_boundNativeContext != tuple.Context) {
SharedImageImpl::PublishPendingReads();
}
#endif
if (!g_EGLFuncs.eglMakeCurrent(g_Display, session.Draw, session.Draw, tuple.Context)) {
const EGLint error = g_EGLFuncs.eglGetError ? g_EGLFuncs.eglGetError() : EGL_SUCCESS;
MGLOG_E_ONCE("DirectGLES::MakeCurrent failed: native eglMakeCurrent returned error 0x%04x", error);
@@ -17448,6 +17459,9 @@ namespace MobileGL::MG_Backend::DirectGLES {
g_backendContextOwnerThread.store(std::thread::id{}, std::memory_order_release);
return true;
}
#if MOBILEGL_BUILD_DISAGGREGATED
if (t_boundNativeContext != EGL_NO_CONTEXT) SharedImageImpl::PublishPendingReads();
#endif
if (!g_EGLFuncs.eglMakeCurrent(g_Display, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT)) {
const EGLint error = g_EGLFuncs.eglGetError ? g_EGLFuncs.eglGetError() : EGL_SUCCESS;
MGLOG_E_ONCE("DirectGLES::ReleaseCurrent failed: native eglMakeCurrent returned error 0x%04x", error);
@@ -17964,6 +17978,12 @@ namespace MobileGL::MG_Backend::DirectGLES {
slot = {g_GLESFuncs.glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0), g_syncContextGeneration, serial};
}
#if MOBILEGL_BUILD_DISAGGREGATED
// The session's frame boundary: the shared images its frame sampled get the frame's fence,
// so their producers' next copies wait for it.
SharedImageImpl::PublishPendingReads();
#endif
// P14 S4: the session's surface, not the process's.
//
// EXT -> KHR -> plain. The loader fills eglSwapBuffersWithDamageEXT from whichever
@@ -18069,6 +18089,9 @@ namespace MobileGL::MG_Backend::DirectGLES {
// This session's native contexts and surfaces, and the display when they were the last.
static void DestroyNativeStateOfSession(Uint64 sessionKey) {
if (g_Display == EGL_NO_DISPLAY) return;
#if MOBILEGL_BUILD_DISAGGREGATED
if (t_boundNativeContext != EGL_NO_CONTEXT) SharedImageImpl::PublishPendingReads();
#endif
{
const std::lock_guard<std::mutex> lock(g_nativeRegistryMutex);
if (g_EGLFuncs.eglMakeCurrent) {
@@ -18127,6 +18150,16 @@ namespace MobileGL::MG_Backend::DirectGLES {
using DestroyImageKhrFn = EGLBoolean (*)(EGLDisplay, void*);
using ImageTargetFn = void (*)(GLenum, void*);
// EGL_KHR_fence_sync / EGL_KHR_wait_sync / EGL_ANDROID_native_fence_sync, likewise.
constexpr EGLenum kEglSyncNativeFenceAndroid = 0x3144;
constexpr EGLint kEglSyncNativeFenceFdAndroid = 0x3145;
constexpr EGLint kEglConditionSatisfiedKhr = 0x30F6;
using CreateSyncKhrFn = void* (*)(EGLDisplay, EGLenum, const EGLint*);
using DestroySyncKhrFn = EGLBoolean (*)(EGLDisplay, void*);
using WaitSyncKhrFn = EGLint (*)(EGLDisplay, void*, EGLint);
using ClientWaitSyncKhrFn = EGLint (*)(EGLDisplay, void*, EGLint, Uint64);
using DupNativeFenceFdFn = EGLint (*)(EGLDisplay, void*);
struct EntryPoints {
Bool Resolved = false;
Bool Usable = false;
@@ -18136,6 +18169,14 @@ namespace MobileGL::MG_Backend::DirectGLES {
DestroyImageKhrFn DestroyImage = nullptr;
ImageTargetFn TargetTexture2D = nullptr;
ImageTargetFn TargetRenderbuffer = nullptr;
// Fences as sync_files: what lets one session's GPU wait for another's without a
// CPU in between. Without them every wait below is a CPU wait.
Bool FencesUsable = false;
CreateSyncKhrFn CreateSync = nullptr;
DestroySyncKhrFn DestroySync = nullptr;
WaitSyncKhrFn WaitSync = nullptr;
ClientWaitSyncKhrFn ClientWaitSync = nullptr;
DupNativeFenceFdFn DupNativeFenceFd = nullptr;
};
Bool HasExtensionWord(const char* list, const char* word) {
@@ -18170,6 +18211,24 @@ namespace MobileGL::MG_Backend::DirectGLES {
return *entry; // asked again on the next call
}
entry->Resolved = true;
if (HasExtensionWord(egl, "EGL_KHR_fence_sync") && HasExtensionWord(egl, "EGL_KHR_wait_sync") &&
HasExtensionWord(egl, "EGL_ANDROID_native_fence_sync")) {
entry->CreateSync =
reinterpret_cast<CreateSyncKhrFn>(g_EGLFuncs.eglGetProcAddress("eglCreateSyncKHR"));
entry->DestroySync =
reinterpret_cast<DestroySyncKhrFn>(g_EGLFuncs.eglGetProcAddress("eglDestroySyncKHR"));
entry->WaitSync = reinterpret_cast<WaitSyncKhrFn>(g_EGLFuncs.eglGetProcAddress("eglWaitSyncKHR"));
entry->ClientWaitSync =
reinterpret_cast<ClientWaitSyncKhrFn>(g_EGLFuncs.eglGetProcAddress("eglClientWaitSyncKHR"));
entry->DupNativeFenceFd = reinterpret_cast<DupNativeFenceFdFn>(
g_EGLFuncs.eglGetProcAddress("eglDupNativeFenceFDANDROID"));
entry->FencesUsable = entry->CreateSync && entry->DestroySync && entry->WaitSync &&
entry->ClientWaitSync && entry->DupNativeFenceFd;
}
if (!entry->FencesUsable) {
MGLOG_I("Shared images: no EGL native fence sync / wait sync; sessions order their access "
"to an image with CPU waits");
}
if (!HasExtensionWord(egl, "EGL_KHR_image_base") ||
!HasExtensionWord(egl, "EGL_ANDROID_image_native_buffer") ||
!HasExtensionWord(egl, "EGL_ANDROID_get_native_client_buffer")) {
@@ -18214,6 +18273,12 @@ namespace MobileGL::MG_Backend::DirectGLES {
EGLDisplay Display = EGL_NO_DISPLAY;
Uint64 DisplayTerminations = 0;
DestroyImageKhrFn Destroy = nullptr;
// AcquireForSampling's memory, touched only by the session whose twin holds this: the
// write generation last waited for and the native context the wait was queued in (a
// GPU wait orders that context alone), and the read frame the image was last noted in.
Uint64 WaitedGeneration = 0;
Uint64 WaitedContext = 0;
Uint64 NotedFrame = 0;
~EglImage() {
// eglDestroyImageKHR needs the display, not a context, so a session's twin teardown
@@ -18354,8 +18419,8 @@ namespace MobileGL::MG_Backend::DirectGLES {
return renderbuffer;
}
// The present returns once the copy completed (the client attaches the buffer to its
// wl_surface right after and nothing else orders the compositor's read).
// Everything the current context has submitted, waited out on the CPU: the fallback for
// a driver that cannot hand a fence to another session.
Bool WaitForCompletion() {
if (g_GLESFuncs.glFenceSync && g_GLESFuncs.glClientWaitSync && g_GLESFuncs.glDeleteSync) {
if (GLsync fence = g_GLESFuncs.glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0)) {
@@ -18369,7 +18434,7 @@ namespace MobileGL::MG_Backend::DirectGLES {
g_GLESFuncs.glDeleteSync(fence);
if (status == GL_ALREADY_SIGNALED || status == GL_CONDITION_SATISFIED) return true;
if (status == GL_TIMEOUT_EXPIRED) {
MGLOG_E("Shared image present: the copy did not complete within 10 s");
MGLOG_E("Shared images: the submitted work did not complete within 10 s");
return false;
}
// GL_WAIT_FAILED: glFinish is the same promise without the object.
@@ -18378,8 +18443,117 @@ namespace MobileGL::MG_Backend::DirectGLES {
g_GLESFuncs.glFinish();
return true;
}
// How long a CPU wait on another session's fence may take before the order is given up
// on (logged): a session whose GPU work never completes must not wedge its peers.
constexpr Uint32 kFenceCpuWaitMs = 2000;
void CloseFd(int fd) {
if (fd >= 0) ::close(fd);
}
// The current context's LATER commands wait for `fence` (a sync_file) on the GPU; a CPU
// wait when the driver cannot. Takes ownership of `fence`: a native fence sync owns it
// from a successful create on, every other path closes it here.
void WaitForFenceBeforeLaterCommands(int fence) {
if (fence < 0) return;
const EntryPoints& entry = ResolvedEntryPoints();
if (entry.FencesUsable) {
const EGLint attribs[] = {kEglSyncNativeFenceFdAndroid, fence, EGL_NONE};
if (void* sync = entry.CreateSync(g_Display, kEglSyncNativeFenceAndroid, attribs)) {
if (entry.WaitSync(g_Display, sync, 0) != EGL_TRUE) {
constexpr Uint64 kWaitNs = Uint64{kFenceCpuWaitMs} * 1000 * 1000;
if (entry.ClientWaitSync(g_Display, sync, 0, kWaitNs) != kEglConditionSatisfiedKhr) {
MGLOG_E_ONCE("Shared images: a peer session's fence could not be waited for");
}
}
// A pending server wait keeps the sync alive past this (EGL_KHR_wait_sync).
entry.DestroySync(g_Display, sync);
return;
}
}
if (!SI::SyncFile::Wait(fence, kFenceCpuWaitMs)) {
MGLOG_E_ONCE("Shared images: a peer session's fence did not signal within %u ms",
static_cast<unsigned>(kFenceCpuWaitMs));
}
CloseFd(fence);
}
// A sync_file for everything the current context has submitted so far (flushed, so it
// will signal), in `*fence`. False when the driver cannot export one; the caller then
// waits on the CPU and publishes -1.
Bool ExportFence(int* fence) {
*fence = -1;
const EntryPoints& entry = ResolvedEntryPoints();
if (!entry.FencesUsable) return false;
const EGLint attribs[] = {EGL_NONE};
void* sync = entry.CreateSync(g_Display, kEglSyncNativeFenceAndroid, attribs);
if (sync == nullptr) return false;
// The fence has no descriptor until its command reaches the driver's queue.
g_GLESFuncs.glFlush();
const EGLint fd = entry.DupNativeFenceFd(g_Display, sync);
entry.DestroySync(g_Display, sync);
if (fd < 0) return false;
*fence = fd;
return true;
}
// ONE READ FRAME PER SESSION: what the session has sampled since its last frame
// boundary. A session is served by exactly one apply thread, so the thread owns it -
// nothing about it is process-wide, and a session's end (its thread's) drops it. `Id`
// is drawn from one counter across all sessions and renewed at every boundary, so an
// EglImage's NotedFrame is a once-per-frame gate no other session's frame can match.
struct ReadFrame {
Uint64 Id = 0;
SI::ReadTracker Tracker;
};
std::atomic<Uint64> g_readFrameIds{0};
ReadFrame& CurrentReadFrame() {
thread_local ReadFrame frame;
if (frame.Id == 0) frame.Id = g_readFrameIds.fetch_add(1, std::memory_order_relaxed) + 1;
return frame;
}
} // namespace
void AcquireForSampling(const EglImageRef& image) {
if (image == nullptr || image->Image == nullptr) return;
EglImage& state = *image;
// Steady state: one lock and a compare (the write fence, once signaled, is dropped by
// the registry and nothing is duplicated any more).
Uint64 generation = 0;
int fence = SI::DupWriteFence(*state.Image, &generation);
const Uint64 context = CurrentNativeContextSerial();
if (generation != state.WaitedGeneration || context != state.WaitedContext) {
state.WaitedGeneration = generation;
state.WaitedContext = context;
WaitForFenceBeforeLaterCommands(fence);
fence = -1;
}
CloseFd(fence);
ReadFrame& frame = CurrentReadFrame();
if (state.NotedFrame != frame.Id) {
frame.Tracker.NoteRead(state.Image);
state.NotedFrame = frame.Id;
}
}
void PublishPendingReads() {
ReadFrame& frame = CurrentReadFrame();
if (frame.Tracker.Pending() == 0) return;
int fence = -1;
// No context current: the reads' context is gone (and its work with it), and there is
// nothing to fence or wait from - the frame is only forgotten.
if (t_boundNativeContext != EGL_NO_CONTEXT && !ExportFence(&fence)) {
// FALLBACK: there is no fence to hand a writer, so the frame's reads are waited out
// here instead - once this session moves on (and, say, releases the buffer to its
// producer), nothing it submitted still samples the image.
(void)WaitForCompletion();
}
frame.Tracker.PublishFrame(fence);
frame.Id = g_readFrameIds.fetch_add(1, std::memory_order_relaxed) + 1;
}
Bool BlitDefaultFramebufferTo(const SharedImageView& view) {
if (view.Id == 0 || view.Width == 0 || view.Height == 0 || view.NativeBuffer == nullptr) return false;
if (!IsBackendContextCurrentOnThisThread()) {
@@ -18396,6 +18570,23 @@ namespace MobileGL::MG_Backend::DirectGLES {
MGLOG_E_ONCE("Shared image present: the current context has no default framebuffer to copy");
return false;
}
const SI::ImageRef image = SI::Find(view.Id);
if (image == nullptr) {
MGLOG_E_ONCE("Shared image present: image %llu is no longer live",
static_cast<unsigned long long>(view.Id));
return false;
}
// WRITE AFTER READ: the frames that still sample the image finish before the copy
// overwrites it - queued on this context's GPU timeline, or waited out on the CPU when
// the driver has no fence to queue. Outside the targets' lock, which every session's
// present takes.
if (ResolvedEntryPoints().FencesUsable) {
for (const int fence : SI::DupPendingReadFences(*image)) WaitForFenceBeforeLaterCommands(fence);
} else if (!SI::WaitForReads(*image, kFenceCpuWaitMs)) {
MGLOG_E_ONCE("Shared image present: image %llu's readers did not finish within %u ms",
static_cast<unsigned long long>(view.Id), static_cast<unsigned>(kFenceCpuWaitMs));
}
Bool copied = false;
{
@@ -18475,7 +18666,18 @@ namespace MobileGL::MG_Backend::DirectGLES {
}
g_GLESFuncs.glFramebufferRenderbuffer(GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, 0);
}
return copied && WaitForCompletion();
if (!copied) return false;
// The copy's completion goes to the readers as a fence and the present returns at once;
// a reader that binds the image waits for it on its own GPU. Without fence export the
// copy is waited out here and published as already complete.
int fence = -1;
if (!ExportFence(&fence) && !WaitForCompletion()) return false;
SI::PublishWrite(*image, fence);
// A session that samples images and presents into one (a nested compositor) closes its
// read frame here too.
PublishPendingReads();
return true;
}
void ForgetPresentTargetsAfterTerminate() {
@@ -18499,6 +18701,8 @@ namespace MobileGL::MG_Backend::DirectGLES {
(void)view;
return false;
}
void AcquireForSampling(const EglImageRef& image) { (void)image; }
void PublishPendingReads() {}
void ForgetPresentTargetsAfterTerminate() {}
#endif
} // namespace SharedImageImpl
@@ -485,6 +485,17 @@ namespace MobileGL::MG_Backend::DirectGLES {
Bool TargetBoundTexture2D(const EglImageRef& image);
// BackendObject::BlitDefaultFramebufferToSharedImage, for the calling thread's context.
Bool BlitDefaultFramebufferTo(const SharedImageView& image);
// THE READER'S HALF of the image's sync state (SharedImageRegistry.h). Called on every use
// of a name whose level 0 is `image`, before the command that samples it: when a write
// landed since this context last waited, the context's later commands wait for its fence
// on the GPU; and the image is noted as read by the calling session's current frame.
void AcquireForSampling(const EglImageRef& image);
// The calling session's frame boundary for what it has read: hands a fence for everything
// the current context has submitted to every image noted since the last boundary. Called
// at Present and wherever the current native context stops being current (its reads can
// only be fenced from it). Nothing noted = nothing done.
void PublishPendingReads();
} // namespace SharedImageImpl
#endif
@@ -8975,6 +8975,10 @@ namespace MobileGL::MG_Backend::DirectGLES {
// them, and the storage path would allocate fresh, empty storage over the image.
if (pushedStorage != nullptr && (pushedStorage->SharedImageId != 0 || m_sharedImageId != 0) &&
SyncSharedImageStorage(*pushedStorage)) {
// EVERY use, not only the attach (the draw-clean gates never pass a bound image):
// a write that landed since the last one is waited for on this context's GPU
// before the command this sync is for, and the session's frame notes the read.
if (m_sharedImage != nullptr) SharedImageImpl::AcquireForSampling(m_sharedImage);
return;
}
// P5e (tx2), CONTRACT-P5E §5.2 (scout G-S2-2): THE VIEW TEST MOVES BEHIND THE
@@ -16933,6 +16937,10 @@ namespace MobileGL::MG_Backend::DirectGLES {
// PipeTextureStorageRecordForRecord and CONTRACT-P5E.md §5.2.
const MG_Pipe::MGPipeResourceRecord* storage = PipeTextureStorageRecordForRecord(record);
if (storage == nullptr) return false;
#if MOBILEGL_BUILD_DISAGGREGATED
// Never clean while bound to a shared image: see IsDrawSyncClean.
if (m_sharedImage != nullptr) return false;
#endif
// SyncMipmapsToBackend's own early-out, verbatim.
if (!m_isInitialized || m_syncedResourceSerial == 0 || m_syncedResourceSerial != storage->Serial ||
!storage->PendingUploads.empty()) {
@@ -2145,6 +2145,12 @@ namespace MobileGL::MG_Backend::DirectGLES {
Bool IsDrawSyncClean(const MG_State::GLState::ITextureObject* t, Uint64 contextId,
Uint64 samplingGeneration) const {
#if MOBILEGL_BUILD_DISAGGREGATED
// A shared image's texels move with its producer's presents, which nothing here
// sees: every use goes through SyncMipmapsToBackend's shared-image arm, which waits
// for a write that landed since the last one (SharedImageImpl::AcquireForSampling).
if (m_sharedImage != nullptr) return false;
#endif
if (!m_isInitialized || m_syncedShapeContextId == 0 || m_syncedShapeContextId != contextId ||
m_syncedShapeGeneration != samplingGeneration) {
return false;
@@ -3195,7 +3195,10 @@ namespace MobileGL::MG_Backend::DirectVulkan {
// preparing the texture would decline a storage-less one and lose the pass.
if (storage && WireShaderImageNamesNoTexel(*m_textureManager, state.BoundShaderImages[unit]))
continue;
if (!m_textureManager->SyncTextureResourceByHandle(handle, false, storage)) return false;
const auto* resource = m_textureManager->SyncTextureResourceByHandle(handle, false, storage);
if (resource == nullptr) return false;
// Another session writes it: the renderer acquires it before this use.
if (resource->sharedImageId != 0) m_textureManager->NoteSharedImageUse(*resource);
}
}
}
@@ -707,6 +707,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
// Destroy images while the device/allocator still exist, including after context death.
m_wireTextureResources.clear();
m_wireRenderbufferResources.clear();
m_sharedImageUses.clear();
#endif
m_aliveObjects.clear();
m_storageImageTextures.clear();
@@ -2946,42 +2947,11 @@ namespace MobileGL::MG_Backend::DirectVulkan {
return SharedImageBind::Unresolved;
}
// ACQUIRE FROM THE FOREIGN FAMILY. The writer (a client session's Present, on its own
// device) leaves the image released to VK_QUEUE_FAMILY_FOREIGN_EXT in GENERAL, which is
// also the layout every wire descriptor samples in. Submitted on its own and waited, like
// the preserve copy: it runs once per attach, before any recorded use of the image.
VkCommandBufferAllocateInfo allocInfo{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};
allocInfo.commandPool = m_commandPool;
allocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
allocInfo.commandBufferCount = 1;
VkCommandBuffer commandBuffer = VK_NULL_HANDLE;
VK_VERIFY(vkAllocateCommandBuffers(m_device, &allocInfo, &commandBuffer),
"vkAllocateCommandBuffers(shared image acquire)");
VkCommandBufferBeginInfo beginInfo{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
VK_VERIFY(vkBeginCommandBuffer(commandBuffer, &beginInfo), "vkBeginCommandBuffer(shared image acquire)");
VkImageMemoryBarrier acquire{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
acquire.srcAccessMask = 0;
acquire.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
acquire.oldLayout = VK_IMAGE_LAYOUT_GENERAL;
acquire.newLayout = VK_IMAGE_LAYOUT_GENERAL;
acquire.srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT;
acquire.dstQueueFamilyIndex = imported.queueFamily;
acquire.image = imported.image;
acquire.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0,
0, nullptr, 0, nullptr, 1, &acquire);
VK_VERIFY(vkEndCommandBuffer(commandBuffer), "vkEndCommandBuffer(shared image acquire)");
VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
VkFence fence = VK_NULL_HANDLE;
VK_VERIFY(vkCreateFence(m_device, &fenceInfo, nullptr, &fence), "vkCreateFence(shared image acquire)");
VkSubmitInfo submitInfo{VK_STRUCTURE_TYPE_SUBMIT_INFO};
submitInfo.commandBufferCount = 1;
submitInfo.pCommandBuffers = &commandBuffer;
VK_VERIFY(vkQueueSubmit(m_graphicsQueue, 1, &submitInfo, fence), "vkQueueSubmit(shared image acquire)");
VK_VERIFY(vkWaitForFences(m_device, 1, &fence, VK_TRUE, UINT64_MAX), "vkWaitForFences(shared image acquire)");
vkDestroyFence(m_device, fence, nullptr);
vkFreeCommandBuffers(m_device, m_commandPool, 1, &commandBuffer);
// NOT ACQUIRED HERE. The writer (a client session's present, on its own device) releases
// the image to VK_QUEUE_FAMILY_FOREIGN_EXT in GENERAL - also the layout every wire
// descriptor samples in - once per frame it writes. Each use acquires it again when the
// write generation moved, waiting for that write's fence on the GPU, and the frame boundary
// releases it back (VulkanRenderer::AcquireSharedImage, WireSharedImage.inc).
TextureResource replacement;
replacement.image = imported.image;
@@ -3087,6 +3057,7 @@ namespace MobileGL::MG_Backend::DirectVulkan {
handle.Slot, handle.Gen);
return nullptr;
}
resource.sharedImageKey = resource.sharedImageId != 0 ? key : 0;
resource.syncedWireSerial = record.Serial;
return &resource;
}
@@ -365,6 +365,13 @@ public:
Uint64 sharedImageId = 0;
SharedPtr<const void> sharedImageOwner;
Bool sharedImageAlphaOne = false;
// The reading side's ownership of the image (VulkanRenderer::AcquireSharedImage): whether
// this device holds it now - acquired from the foreign family and not yet released back at
// the frame boundary - the writer's generation it last acquired, and the
// m_wireTextureResources key it lives under, by which the frame boundary finds it again.
Bool sharedImageHeld = false;
Uint64 sharedImageGeneration = 0;
Uint64 sharedImageKey = 0;
#endif
TextureResource() = default;
@@ -405,6 +412,9 @@ public:
std::swap(this->sharedImageId, that.sharedImageId);
std::swap(this->sharedImageOwner, that.sharedImageOwner);
std::swap(this->sharedImageAlphaOne, that.sharedImageAlphaOne);
std::swap(this->sharedImageHeld, that.sharedImageHeld);
std::swap(this->sharedImageGeneration, that.sharedImageGeneration);
std::swap(this->sharedImageKey, that.sharedImageKey);
#endif
}
@@ -452,6 +462,9 @@ public:
sharedImageId = 0;
sharedImageOwner.reset();
sharedImageAlphaOne = false;
sharedImageHeld = false;
sharedImageGeneration = 0;
sharedImageKey = 0;
#endif
fullView = VK_NULL_HANDLE;
sampledView = VK_NULL_HANDLE;
@@ -631,6 +644,24 @@ public:
static Bool ImportSharedImage(void* nativeBuffer, Uint32 width, Uint32 height, ImportedSharedImage& out,
String& why);
static void DestroyImportedSharedImage(VkDevice device, ImportedSharedImage& image);
// The reading side's per-use hook: PrepareWireTextureResources notes every shared-image
// texture a draw or dispatch samples, and the renderer takes the list right after
// (VulkanRenderer::AcquireNotedSharedImages) to acquire each one ahead of the sampling.
void NoteSharedImageUse(const TextureResource& resource) {
if (std::find(m_sharedImageUses.begin(), m_sharedImageUses.end(), resource.sharedImageKey) ==
m_sharedImageUses.end())
m_sharedImageUses.push_back(resource.sharedImageKey);
}
Bool HasNotedSharedImageUses() const { return !m_sharedImageUses.empty(); }
void TakeNotedSharedImageUses(Vector<Uint64>& out) {
out.clear();
std::swap(out, m_sharedImageUses);
}
// The live handle-keyed texture resource under `key`, or null.
TextureResource* FindWireTextureResource(Uint64 key) {
const auto it = m_wireTextureResources.find(key);
return it != m_wireTextureResources.end() && it->second.image != VK_NULL_HANDLE ? &it->second : nullptr;
}
#endif
VkImageView GetOrCreateViewAtMipLevel(MG_State::GLState::ITextureObject& texture, Uint32 mipLevel);
VkImageView GetOrCreateAttachmentViewAtMipLevel(MG_State::GLState::ITextureObject& texture, Uint32 mipLevel,
@@ -827,6 +858,8 @@ private:
// Node-based for the same reason m_renderbufferResources is: callers hold TextureResource*
// across map-touching calls.
std::unordered_map<Uint64, TextureResource> m_wireTextureResources;
// Shared-image textures sampled since the renderer last took the list (NoteSharedImageUse).
Vector<Uint64> m_sharedImageUses;
// Handles are per-kind; a renderbuffer and texture may have identical slot/gen.
std::unordered_map<Uint64, TextureResource> m_wireRenderbufferResources;
#endif
@@ -67,6 +67,9 @@
#ifdef __ANDROID__
#include <sys/system_properties.h>
#endif
#if MOBILEGL_BUILD_DISAGGREGATED && !defined(_WIN32)
#include <unistd.h> // close(): shared-image sync_file descriptors (WireSharedImage.inc)
#endif
#if defined(__APPLE__)
#include <CoreGraphics/CoreGraphics.h>
@@ -3513,6 +3516,7 @@ void main() {
DestroyWireDepthMipmapResources();
DestroyWireMultisampleResolveResources();
DestroySharedImagePresentTargets(false);
DestroySharedImageSync();
#endif
OnSubmitsCompletedUpTo(m_submitCounter);
DestroySubmitFencePool();
@@ -13960,6 +13964,7 @@ void main() {
m_frameContext.FreeRetiredCommandBuffersCompletedUpTo(m_completedSubmitCounter);
#if MOBILEGL_BUILD_DISAGGREGATED
CollectWireObjects(m_completedSubmitCounter);
RecycleSharedImageSemaphores(m_completedSubmitCounter);
#endif
}
@@ -14144,6 +14149,11 @@ void main() {
}
submitInfo.commandBufferCount = commandBufferCount;
submitInfo.pCommandBuffers = commandBuffers;
#if MOBILEGL_BUILD_DISAGGREGATED
// Shared-image fences staged for the work recorded since the last submission.
SharedImageSubmitSync sharedImageSync;
AttachSharedImageSync(submitInfo, sharedImageSync);
#endif
const VkResult result = vkQueueSubmit(m_graphicsQueue, 1, &submitInfo, fence);
if (result != VK_SUCCESS) {
MGLOG_E_ONCE("SubmitPendingCommandBuffer: vkQueueSubmit returned %d", result);
@@ -14154,6 +14164,9 @@ void main() {
frame.hasPreCommandBufferRecorded = false;
RegisterSubmit(fence, pooledFence);
frame.lastSubmitIndex = m_submitCounter;
#if MOBILEGL_BUILD_DISAGGREGATED
CommitSharedImageSync(sharedImageSync);
#endif
return true;
}
@@ -14473,6 +14486,9 @@ void main() {
suspendedFrame.isCommandRecording = false;
suspendedFrame.hasCommandBufferRecorded = false;
InvalidatePipelineMemo();
#if MOBILEGL_BUILD_DISAGGREGATED
AbandonSharedImageReads();
#endif
// The dropped recording is never submitted, so once the fence
// poll shows the pre-suspension submissions complete the frame
// transients (descriptor sets, transient arenas, deferred
@@ -14532,6 +14548,14 @@ void main() {
auto* activeRenderPass = VkRenderPassManager::GetActiveRenderPass();
if (activeRenderPass)
VkRenderPassManager::EndRenderPass(frame.commandBuffer);
#if MOBILEGL_BUILD_DISAGGREGATED
// The frame boundary of the shared images this frame read (WireSharedImage.inc): handed
// back to the foreign family in this recording, and this submission signals the read
// fence their next writer waits for.
ReleaseHeldSharedImages();
if (m_sharedImageSyncFd && m_sharedImageReads.Pending() > 0 && m_sharedImageSignal == VK_NULL_HANDLE)
m_sharedImageSignal = TakeSharedImageSemaphore(/*exportable=*/true);
#endif
// Transition while this frame's recording is still open. A frame that
// rendered only into FBOs has no default-framebuffer render pass, and that
@@ -14562,9 +14586,17 @@ void main() {
m_textureManager->FlushPendingUploads();
}
auto submitPacket = m_frameContext.GetSubmitInfo(shouldSubmitCommandBuffer, m_imageIndexAcquired);
#if MOBILEGL_BUILD_DISAGGREGATED
SharedImageSubmitSync sharedImageSync;
AttachSharedImageSync(submitPacket.submitInfo, sharedImageSync);
#endif
VK_VERIFY(vkQueueSubmit(m_graphicsQueue, 1, &submitPacket.submitInfo, frame.imageInFlightFence));
RegisterSubmit(frame.imageInFlightFence, /*pooledFence=*/false);
frame.lastSubmitIndex = m_submitCounter;
#if MOBILEGL_BUILD_DISAGGREGATED
CommitSharedImageSync(sharedImageSync);
PublishSharedImageReads();
#endif
frame.isCommandRecording = false;
frame.hasCommandBufferRecorded = false;
frame.hasPreCommandBufferRecorded = false;
@@ -15380,6 +15412,7 @@ void main() {
// 1.1 this device is created at, and are enabled too when advertised (the spike's set).
m_wireAhbImport = false;
m_wireGetAhbProperties = nullptr;
m_sharedImageSyncFd = false;
#if defined(__ANDROID__)
if (MG_Config::Transport != MG_Config::TransportMode::Monolith &&
m_physicalDevice.properties.apiVersion >= VK_API_VERSION_1_1 &&
@@ -15395,6 +15428,37 @@ void main() {
if (IsExtensionSupported(availableExtensions, name))
EnableOptionalDeviceExtension(availableExtensions, enabledDeviceExtensions, name);
}
// Shared-image fences (WireSharedImage.inc) cross sessions as sync_files, which this
// device waits on and signals through SYNC_FD binary semaphores - imported for a wait,
// exported for a signal. Both directions, or the CPU fallback.
if (IsExtensionSupported(availableExtensions, VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME)) {
auto getExternalSemaphoreProperties = reinterpret_cast<PFN_vkGetPhysicalDeviceExternalSemaphoreProperties>(
vkGetInstanceProcAddr(m_instance, "vkGetPhysicalDeviceExternalSemaphoreProperties"));
if (getExternalSemaphoreProperties == nullptr) {
getExternalSemaphoreProperties = reinterpret_cast<PFN_vkGetPhysicalDeviceExternalSemaphoreProperties>(
vkGetInstanceProcAddr(m_instance, "vkGetPhysicalDeviceExternalSemaphorePropertiesKHR"));
}
VkExternalSemaphoreProperties properties{VK_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_PROPERTIES};
if (getExternalSemaphoreProperties != nullptr) {
VkPhysicalDeviceExternalSemaphoreInfo query{
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_SEMAPHORE_INFO};
query.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
getExternalSemaphoreProperties(m_physicalDevice.handle, &query, &properties);
}
constexpr VkExternalSemaphoreFeatureFlags both =
VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT | VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT;
if ((properties.externalSemaphoreFeatures & both) == both) {
m_sharedImageSyncFd = true;
for (const char* name : {VK_KHR_EXTERNAL_SEMAPHORE_EXTENSION_NAME,
VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME}) {
if (IsExtensionSupported(availableExtensions, name))
EnableOptionalDeviceExtension(availableExtensions, enabledDeviceExtensions, name);
}
}
}
MGLOG_I("DirectVulkan: shared-image fences %s",
m_sharedImageSyncFd ? "wait and signal on the GPU (SYNC_FD semaphores)"
: "are waited for on the CPU (no SYNC_FD semaphore import and export)");
}
#endif
#endif
@@ -15961,6 +16025,17 @@ void main() {
reinterpret_cast<void*>(vkGetDeviceProcAddr(m_device, "vkGetAndroidHardwareBufferPropertiesANDROID"));
if (m_wireGetAhbProperties == nullptr) m_wireAhbImport = false;
}
if (m_sharedImageSyncFd) {
m_importSemaphoreFd =
reinterpret_cast<PFN_vkImportSemaphoreFdKHR>(vkGetDeviceProcAddr(m_device, "vkImportSemaphoreFdKHR"));
m_getSemaphoreFd =
reinterpret_cast<PFN_vkGetSemaphoreFdKHR>(vkGetDeviceProcAddr(m_device, "vkGetSemaphoreFdKHR"));
if (m_importSemaphoreFd == nullptr || m_getSemaphoreFd == nullptr) {
MGLOG_W("DirectVulkan: VK_KHR_external_semaphore_fd is missing an entry point; shared-image fences "
"are waited for on the CPU");
m_sharedImageSyncFd = false;
}
}
#endif
s_vkCmdDrawIndexedIndirectCount = reinterpret_cast<PFNDrawIndexedIndirectCountFunc>(
@@ -24,6 +24,7 @@
#include "VkTimerQueryManager.h"
#if MOBILEGL_BUILD_DISAGGREGATED
#include "WireRenderPassCompatibility.h"
#include <MG_Remote/Server/SharedImageRegistry.h>
#endif
#include "MG_Util/Math/VectorTypes.h"
#include <Includes.h>
@@ -181,9 +182,21 @@ namespace MobileGL::MG_Backend::DirectVulkan {
Bool GetWireAhbImport(WireAhbImport& out) const;
// SHARED IMAGES (BackendObject::BlitDefaultFramebufferToSharedImage): the active surface
// target's default framebuffer, copied into `image` top row first and scaled to its
// extent. Returns once the copy completed on the GPU. False (logged) when it cannot be
// done here: no AHardwareBuffer import, a quarter-turned surface, an unblittable format.
// extent. Returns once the copy is submitted, with its fence published to the image
// (SharedImages::PublishWrite); waits for it on the CPU only on a device without sync_file
// export. False (logged) when it cannot be done here: no AHardwareBuffer import, a
// quarter-turned surface, an unblittable format.
Bool BlitDefaultFramebufferToSharedImage(const SharedImageView& image);
// The reading side's per-use hook (WireSharedImage.inc): every shared-image texture the
// wire draw or dispatch being set up samples (VkTextureManager::NoteSharedImageUse) is
// acquired from the foreign family before it, when it moved or is not held.
void AcquireNotedSharedImages() {
if (m_textureManager && m_textureManager->HasNotedSharedImageUses()) AcquireNotedSharedImagesSlow();
}
// One shared-image texture, before a use: a new write generation, or an image released at
// the last frame boundary, is acquired here - an ownership barrier recorded ahead of the
// use, the write's fence a wait of the submission carrying it - and noted as read.
void AcquireSharedImage(VkTextureManager::TextureResource& resource);
#endif
// FrameContext::IRecordingObserver: prepares the frame's timer-query
@@ -649,13 +662,75 @@ namespace MobileGL::MG_Backend::DirectVulkan {
void* m_wireGetAhbProperties = nullptr;
// Shared images this session presents into, imported once each and keyed by the image's
// id (monotonic, never reused). `owner` watches the registry's image: an entry whose image
// died is dropped at the next present, every earlier present having completed.
// died is dropped at a later present once `lastSubmit`, its last copy, has completed.
struct SharedImagePresentTarget {
VkTextureManager::ImportedSharedImage image;
WeakPtr<const void> owner;
Uint64 lastSubmit = 0;
};
UnorderedMap<Uint64, SharedImagePresentTarget> m_sharedImagePresentTargets;
void DestroySharedImagePresentTargets(Bool onlyDead);
// SHARED-IMAGE SYNCHRONISATION (WireSharedImage.inc). The sync_file fences other sessions
// publish become binary semaphores: a temporary SYNC_FD import is a WAIT, an exportable
// semaphore a SIGNAL, of the next frame submission - whichever of SubmitPendingCommandBuffer
// and Present's submit comes first, which is the one carrying what was recorded since.
// Each semaphore is recycled once the submission that used it has completed.
// m_sharedImageSyncFd: the device imports and exports SYNC_FD binary semaphores; without
// it the waits happen on the CPU and nothing is exported (see WireSharedImage.inc).
Bool m_sharedImageSyncFd = false;
PFN_vkImportSemaphoreFdKHR m_importSemaphoreFd = nullptr;
PFN_vkGetSemaphoreFdKHR m_getSemaphoreFd = nullptr;
struct SharedImageWait {
VkSemaphore semaphore = VK_NULL_HANDLE;
VkPipelineStageFlags stages = 0;
};
Vector<SharedImageWait> m_sharedImageWaits; // staged for the next submission
VkSemaphore m_sharedImageSignal = VK_NULL_HANDLE; // staged for the next submission
VkSemaphore m_sharedImageSignalSubmitted = VK_NULL_HANDLE; // the last one submitted, to export
struct SharedImageSemaphoreInFlight {
VkSemaphore semaphore = VK_NULL_HANDLE;
Uint64 submitIndex = 0;
Bool exportable = false;
Bool exported = false; // a signal whose sync_file was taken (which unsignals it)
};
Vector<SharedImageSemaphoreInFlight> m_sharedImageSemaphoresInFlight; // submit order
Vector<VkSemaphore> m_freeSharedImageWaitSemaphores;
Vector<VkSemaphore> m_freeSharedImageSignalSemaphores;
// The reading side, per session: the shared-image textures this frame holds (acquired, not
// yet released; keyed as in VkTextureManager's handle map, with the submission that carries
// the acquire), and the frame's ReadTracker, published at Present.
struct SharedImageHold {
Uint64 key = 0;
Uint64 acquireSubmit = 0;
};
Vector<SharedImageHold> m_sharedImagesHeld;
MG_Remote::Server::SharedImages::ReadTracker m_sharedImageReads;
Vector<Uint64> m_sharedImageUseScratch;
// What one vkQueueSubmit carries beyond its own semaphores (AttachSharedImageSync).
struct SharedImageSubmitSync {
Bool attached = false;
Vector<VkSemaphore> waits;
Vector<VkPipelineStageFlags> stages;
Vector<VkSemaphore> signals;
};
void AttachSharedImageSync(VkSubmitInfo& info, SharedImageSubmitSync& storage);
// After the submission `storage` was attached to was submitted and registered.
void CommitSharedImageSync(const SharedImageSubmitSync& storage);
// Takes ownership of `fence` (a sync_file; -1 = nothing to wait for).
void StageSharedImageWait(int fence, VkPipelineStageFlags stages);
VkSemaphore TakeSharedImageSemaphore(Bool exportable);
// The submitted signal's sync_file (-1: it has already signaled). False: not exported.
Bool ExportSharedImageSignal(VkSemaphore semaphore, int& fence);
void RecycleSharedImageSemaphores(Uint64 completedSubmit);
void DestroySharedImageSync();
void AcquireNotedSharedImagesSlow();
// The reading side's frame boundary: release every held image back to the foreign family
// in the open recording, and - after the submission carrying it - publish its fence.
void ReleaseHeldSharedImages();
void PublishSharedImageReads();
// A frame dropped unsubmitted (Present with no usable swapchain).
void AbandonSharedImageReads();
// Runs the probe (memoized per device identity) and sets the member above. Called at the end of
// device creation, after ArmPrimGenReroute: it records on m_graphicsQueue.
void ArmWireDepthResolveOrder();
@@ -240,6 +240,9 @@
MGL_WIRE_DECLINE_AT(TextureResourcesUnprepared, "a sampled/storage image could not be made ready; see the Tex* tally");
return false;
}
// Shared images another session writes: acquired ahead of the recording that samples them
// (WireSharedImage.inc).
AcquireNotedSharedImages();
// Lazy image uploads/promotions can submit the old recording. Resolve
// ALL attachments before descriptor binding captures the current cmd.
for (Uint32 i = 0; i < MG_Pipe::kMGPipeMaxColorAttachments; ++i) {
@@ -776,6 +779,7 @@
MGL_WIRE_DECLINE_AT(ComputeTextureResourcesUnprepared, "a sampled/storage image could not be made ready for the dispatch");
return;
}
AcquireNotedSharedImages(); // shared images another session writes (WireSharedImage.inc)
const auto pipeline = GetOrCreateComputePipeline(programObj);
if (!pipeline) {
MGL_WIRE_DECLINE_AT(ComputePipelineUnavailable, "no compute pipeline could be created for this dispatch");
@@ -847,6 +851,7 @@
MGL_WIRE_DECLINE_AT(ComputeTextureResourcesUnprepared, "a sampled/storage image could not be made ready for the dispatch");
return;
}
AcquireNotedSharedImages(); // shared images another session writes (WireSharedImage.inc)
const auto pipeline = GetOrCreateComputePipeline(programObj);
if (!pipeline) {
MGL_WIRE_DECLINE_AT(ComputePipelineUnavailable, "no compute pipeline could be created for this dispatch");
@@ -160,6 +160,8 @@
if (MG_Pipe::MGPipeHandleIsNull(out.storage)) MagmaWireFatal("framebuffer-view-record");
auto* resource = m_textureManager->SyncTextureResourceByHandle(out.storage, renderbuffer);
if (!resource) MagmaWireFatal("framebuffer-resource");
// Another session writes it: acquired (WireSharedImage.inc) before the layout is read below.
if (resource->sharedImageId != 0) AcquireSharedImage(*resource);
out.image = resource->image;
out.format = viewFormat != VK_FORMAT_UNDEFINED ? viewFormat : resource->format;
out.extent = {std::max(resource->extent.width >> level, 1u),
@@ -6,19 +6,54 @@
// SPDX-License-Identifier: LGPL-3.0-only
// End of Source File Header
// SHARED IMAGES, the present side (docs/Disaggregated/notes/anland/plan-ahb-dmabuf.md). Included by
// VulkanRenderer.cpp inside the backend namespace, after WireFramebuffer.inc, whose readback this
// copy is modelled on: the same flush, the same write-visibility barrier, the same transition of
// the default framebuffer's image and the same submit-and-wait.
// SHARED IMAGES (docs/Disaggregated/notes/anland/plan-ahb-dmabuf.md). Included by VulkanRenderer.cpp
// inside the backend namespace, after WireFramebuffer.inc, whose readback the present side's copy
// is modelled on: the same flush, the same write-visibility barrier, the same transition of the
// default framebuffer's image.
//
// OWNERSHIP. The image lives on more than one device: this session writes it, the compositor's
// session (another VulkanRenderer, another VkDevice) samples it. Each present acquires it from
// VK_QUEUE_FAMILY_FOREIGN_EXT (UNDEFINED: the copy overwrites every texel) and releases it back in
// GENERAL, the layout the sampling side acquires it in (VkTextureManager::SyncWireSharedImage).
// OWNERSHIP. The image lives on more than one device: a client session writes it, the
// compositor's session (another VulkanRenderer, another VkDevice) samples it. Each present
// acquires it from VK_QUEUE_FAMILY_FOREIGN_EXT (UNDEFINED: the copy overwrites every texel) and
// releases it back in GENERAL. A reader acquires it (GENERAL) before the first use of a frame and
// again whenever the write generation moved, and releases it back at its frame boundary, so every
// acquire on either side matches a release on the other.
//
// ORDER (MG_Remote/Server/SharedImageRegistry.h). Every fence is a sync_file. The writer's copy
// waits on its GPU for the read fences still pending, signals a semaphore exported as the write
// fence, and the present returns without waiting. A reader's use waits on its GPU for the write
// fence of the generation it acquires; its frame's submission signals a semaphore exported as the
// read fence of every image the frame used. A device without SYNC_FD semaphore import and export
// waits on the CPU instead where it would have waited on the GPU - before the writer's copy, after
// it, and before the reader's use - and exports nothing for its reads: there is no fence to give,
// and waiting for the frame on the CPU only to say it is over would stall every frame of the
// compositor. The writer's next copy is then ordered after the reads by nothing but the client's
// own buffer rotation (a Wayland client writes a buffer only after its release) - which is all
// this path had before - on a device Android's Vulkan 1.1 requirements leave rare.
#if MOBILEGL_BUILD_DISAGGREGATED
// A CPU wait on another session's fence is a fallback, and bounded: a peer that never signals
// costs a frame's worth of stall, not a hang.
static constexpr Uint32 kSharedImageCpuWaitMs = 1000;
// Every stage a reader's use of a shared image can reach: sampling (draws and dispatches), a
// blit or readback source, an attachment. The write fence's wait blocks these, and the
// acquire's first scope is the same set, so it is ordered after that wait.
static constexpr VkPipelineStageFlags kSharedImageUseStages =
VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT |
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT |
VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
static void CloseSyncFile(int fd) {
#if !defined(_WIN32)
if (fd >= 0) ::close(fd);
#else
(void)fd;
#endif
}
void VulkanRenderer::DestroySharedImagePresentTargets(Bool onlyDead) {
if (onlyDead) RefreshCompletedSubmits();
for (auto it = m_sharedImagePresentTargets.begin(); it != m_sharedImagePresentTargets.end();) {
if (onlyDead && !it->second.owner.expired()) {
// A dead image's last copy may still be executing: the present no longer waits for it.
if (onlyDead && (!it->second.owner.expired() || it->second.lastSubmit > m_completedSubmitCounter)) {
++it;
continue;
}
@@ -37,7 +72,6 @@
Bool VulkanRenderer::BlitDefaultFramebufferToSharedImage(const SharedImageView& view) {
if (m_device == VK_NULL_HANDLE || view.Width == 0 || view.Height == 0) return false;
// Every earlier present waited for its copy, so an entry whose image has died is idle.
DestroySharedImagePresentTargets(true);
// The default framebuffer's image is stored in display orientation (row 0 is the top of
@@ -72,6 +106,15 @@
found = m_sharedImagePresentTargets.emplace(view.Id, Move(target)).first;
}
const VkTextureManager::ImportedSharedImage& destination = found->second.image;
// The registry's record carries the image's fences. The server holds a reference for as
// long as the present runs; a lapsed one is an image released under it.
const auto image =
std::static_pointer_cast<const MG_Remote::Server::SharedImages::Image>(found->second.owner.lock());
if (image == nullptr) {
MGLOG_E_ONCE("Magma: shared image %llu is not live; the present is declined",
static_cast<unsigned long long>(view.Id));
return false;
}
// The frame's draws reach the queue first; the copy below is a later submission that the
// readback barrier orders behind them.
@@ -122,6 +165,22 @@
TransitionWireImage(source, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
auto& frame = m_frameContext.GetCurrent();
// NOT OVER WHAT IS STILL READ. The reads published since this image's last write (a
// compositor frame sampling it) end before the copy starts: on the GPU, as waits of the
// submission below at the transfer stage; on the CPU, where they cannot be imported.
// The frame's draws were flushed above, so the next submission is the copy's.
namespace SI = MG_Remote::Server::SharedImages;
if (m_sharedImageSyncFd) {
for (const int fence : SI::DupPendingReadFences(*image))
StageSharedImageWait(fence, VK_PIPELINE_STAGE_TRANSFER_BIT);
} else if (!SI::WaitForReads(*image, kSharedImageCpuWaitMs)) {
MGLOG_W_ONCE("Magma: shared image %llu: a read of the previous frame did not finish within %u ms; it is "
"overwritten anyway",
static_cast<unsigned long long>(view.Id), kSharedImageCpuWaitMs);
}
// The acquire's layout transition runs after those waits: its first scope is the stage
// they block.
VkImageMemoryBarrier acquire{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
acquire.srcAccessMask = 0;
acquire.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
@@ -131,7 +190,7 @@
acquire.dstQueueFamilyIndex = destination.queueFamily;
acquire.image = destination.image;
acquire.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
vkCmdPipelineBarrier(frame.commandBuffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0,
vkCmdPipelineBarrier(frame.commandBuffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0,
0, nullptr, 0, nullptr, 1, &acquire);
const VkImageSubresourceLayers layers{VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
@@ -175,13 +234,347 @@
vkCmdPipelineBarrier(frame.commandBuffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
0, 0, nullptr, 0, nullptr, 1, &release);
// The client attaches the buffer to its surface as soon as this returns, and nothing else
// orders the compositor's read after the copy: wait for it here.
// THE WRITE FENCE. The client attaches the buffer to its surface as soon as this returns;
// what orders the compositor's read after the copy is the fence published here, which the
// compositor's session waits for on its GPU when it next acquires the image. The copy's
// submission signals a semaphore exported as that fence, and nothing waits for it here.
// The submission is an ordinary pooled-fence flush: the renderer's in-flight tracking keeps
// the command buffer and every resource it names alive until it has completed.
if (m_sharedImageSyncFd && m_sharedImageSignal == VK_NULL_HANDLE)
m_sharedImageSignal = TakeSharedImageSemaphore(/*exportable=*/true);
if (m_sharedImageSignal != VK_NULL_HANDLE) {
m_sharedImageSignalSubmitted = VK_NULL_HANDLE;
if (!FlushPendingCommands()) {
MGLOG_E("Magma: shared image %llu: the copy could not be submitted",
static_cast<unsigned long long>(view.Id));
return false;
}
found->second.lastSubmit = m_submitCounter;
const VkSemaphore signal = std::exchange(m_sharedImageSignalSubmitted, VK_NULL_HANDLE);
int fence = -1;
if (signal != VK_NULL_HANDLE && ExportSharedImageSignal(signal, fence)) {
SI::PublishWrite(*image, fence);
return true;
}
// Submitted, but no fence came out of it: the copy is waited for, as on a device that
// exports none.
if (!WaitForSubmitsUpTo(m_submitCounter, UINT64_MAX)) {
MGLOG_E("Magma: shared image %llu: the copy could not be waited for",
static_cast<unsigned long long>(view.Id));
return false;
}
SI::PublishWrite(*image, -1);
return true;
}
if (!SubmitReadbackCommandsAndWait(frame)) {
MGLOG_E("Magma: shared image %llu: the copy could not be submitted or waited for",
static_cast<unsigned long long>(view.Id));
return false;
}
found->second.lastSubmit = m_submitCounter;
SI::PublishWrite(*image, -1);
return true;
}
// ---- fences as semaphores ------------------------------------------------------------------
VkSemaphore VulkanRenderer::TakeSharedImageSemaphore(Bool exportable) {
auto& pool = exportable ? m_freeSharedImageSignalSemaphores : m_freeSharedImageWaitSemaphores;
if (!pool.empty()) {
const VkSemaphore semaphore = pool.back();
pool.pop_back();
return semaphore;
}
VkExportSemaphoreCreateInfo exportInfo{VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO};
exportInfo.handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
VkSemaphoreCreateInfo info{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO};
if (exportable) info.pNext = &exportInfo;
VkSemaphore semaphore = VK_NULL_HANDLE;
const VkResult result = vkCreateSemaphore(m_device, &info, nullptr, &semaphore);
if (result != VK_SUCCESS) {
MGLOG_E_ONCE("Magma: a shared-image semaphore could not be created (vkCreateSemaphore -> %d)",
static_cast<Int>(result));
return VK_NULL_HANDLE;
}
return semaphore;
}
void VulkanRenderer::StageSharedImageWait(int fence, VkPipelineStageFlags stages) {
if (fence < 0) return;
if (m_sharedImageSyncFd) {
const VkSemaphore semaphore = TakeSharedImageSemaphore(/*exportable=*/false);
if (semaphore != VK_NULL_HANDLE) {
// A sync_file import is always temporary: the wait consumes it, and the semaphore
// is a plain unsignaled one again once that wait has executed.
VkImportSemaphoreFdInfoKHR import{VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR};
import.semaphore = semaphore;
import.flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT;
import.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
import.fd = fence;
const VkResult result = m_importSemaphoreFd(m_device, &import);
if (result == VK_SUCCESS) {
m_sharedImageWaits.push_back({semaphore, stages}); // the import owns `fence` now
return;
}
m_freeSharedImageWaitSemaphores.push_back(semaphore);
MGLOG_W_ONCE("Magma: a shared-image fence could not be imported (vkImportSemaphoreFdKHR -> %d); it "
"is waited for on the CPU",
static_cast<Int>(result));
}
}
// Nothing recorded since reaches the queue before this returns.
if (!MG_Remote::Server::SharedImages::SyncFile::Wait(fence, kSharedImageCpuWaitMs)) {
MGLOG_W_ONCE("Magma: a shared-image fence did not signal within %u ms; the image is used anyway",
kSharedImageCpuWaitMs);
}
CloseSyncFile(fence);
}
Bool VulkanRenderer::ExportSharedImageSignal(VkSemaphore semaphore, int& fence) {
fence = -1;
VkSemaphoreGetFdInfoKHR info{VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR};
info.semaphore = semaphore;
info.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
// A sync_file export has a wait's effect on the semaphore: it is unsignaled again, and
// reusable once the submission that signaled it has completed. -1 back means it had
// already signaled.
const VkResult result = m_getSemaphoreFd(m_device, &info, &fence);
if (result != VK_SUCCESS) {
fence = -1;
MGLOG_W_ONCE("Magma: a shared-image fence could not be exported (vkGetSemaphoreFdKHR -> %d)",
static_cast<Int>(result));
return false;
}
for (auto& entry : m_sharedImageSemaphoresInFlight) {
if (entry.semaphore == semaphore) entry.exported = true;
}
return true;
}
void VulkanRenderer::AttachSharedImageSync(VkSubmitInfo& info, SharedImageSubmitSync& storage) {
storage.attached = false;
if (m_sharedImageWaits.empty() && m_sharedImageSignal == VK_NULL_HANDLE) return;
storage.waits.assign(info.pWaitSemaphores, info.pWaitSemaphores + info.waitSemaphoreCount);
storage.stages.assign(info.pWaitDstStageMask, info.pWaitDstStageMask + info.waitSemaphoreCount);
for (const auto& wait : m_sharedImageWaits) {
storage.waits.push_back(wait.semaphore);
storage.stages.push_back(wait.stages);
}
storage.signals.assign(info.pSignalSemaphores, info.pSignalSemaphores + info.signalSemaphoreCount);
if (m_sharedImageSignal != VK_NULL_HANDLE) storage.signals.push_back(m_sharedImageSignal);
info.waitSemaphoreCount = static_cast<Uint32>(storage.waits.size());
info.pWaitSemaphores = storage.waits.data();
info.pWaitDstStageMask = storage.stages.data();
info.signalSemaphoreCount = static_cast<Uint32>(storage.signals.size());
info.pSignalSemaphores = storage.signals.data();
storage.attached = true;
}
void VulkanRenderer::CommitSharedImageSync(const SharedImageSubmitSync& storage) {
if (!storage.attached) return;
for (const auto& wait : m_sharedImageWaits)
m_sharedImageSemaphoresInFlight.push_back({wait.semaphore, m_submitCounter, false, false});
m_sharedImageWaits.clear();
if (m_sharedImageSignal != VK_NULL_HANDLE) {
m_sharedImageSemaphoresInFlight.push_back({m_sharedImageSignal, m_submitCounter, true, false});
m_sharedImageSignalSubmitted = std::exchange(m_sharedImageSignal, VK_NULL_HANDLE);
}
}
void VulkanRenderer::RecycleSharedImageSemaphores(Uint64 completedSubmit) {
SizeT done = 0;
for (; done < m_sharedImageSemaphoresInFlight.size() &&
m_sharedImageSemaphoresInFlight[done].submitIndex <= completedSubmit;
++done) {
const auto& entry = m_sharedImageSemaphoresInFlight[done];
if (entry.semaphore == m_sharedImageSignalSubmitted) m_sharedImageSignalSubmitted = VK_NULL_HANDLE;
if (!entry.exportable) {
m_freeSharedImageWaitSemaphores.push_back(entry.semaphore);
} else if (entry.exported) {
m_freeSharedImageSignalSemaphores.push_back(entry.semaphore);
} else if (m_device != VK_NULL_HANDLE) {
// Signaled and never exported, so still signaled: a binary semaphore cannot be
// signaled again until something waits on it, and nothing will.
vkDestroySemaphore(m_device, entry.semaphore, nullptr);
}
}
if (done != 0) {
m_sharedImageSemaphoresInFlight.erase(
m_sharedImageSemaphoresInFlight.begin(),
m_sharedImageSemaphoresInFlight.begin() + static_cast<std::ptrdiff_t>(done));
}
}
void VulkanRenderer::DestroySharedImageSync() {
// The device is idle: every semaphore is inert.
const auto destroy = [this](VkSemaphore semaphore) {
if (semaphore != VK_NULL_HANDLE && m_device != VK_NULL_HANDLE)
vkDestroySemaphore(m_device, semaphore, nullptr);
};
for (const auto& entry : m_sharedImageSemaphoresInFlight) destroy(entry.semaphore);
m_sharedImageSemaphoresInFlight.clear();
for (const auto& wait : m_sharedImageWaits) destroy(wait.semaphore);
m_sharedImageWaits.clear();
destroy(std::exchange(m_sharedImageSignal, VK_NULL_HANDLE));
m_sharedImageSignalSubmitted = VK_NULL_HANDLE;
for (const VkSemaphore semaphore : m_freeSharedImageWaitSemaphores) destroy(semaphore);
m_freeSharedImageWaitSemaphores.clear();
for (const VkSemaphore semaphore : m_freeSharedImageSignalSemaphores) destroy(semaphore);
m_freeSharedImageSignalSemaphores.clear();
m_sharedImagesHeld.clear();
m_sharedImageReads.PublishFrame(-1); // drops the frame's references; publishes nothing
}
// ---- the reading side ------------------------------------------------------------------------
//
// REACHED ON EVERY USE. A wire draw or dispatch prepares its sampled textures through
// PrepareWireTextureResources on every call - the wire arm keeps no sampled-set cache across
// draws (the walk-skip is the monolith arm's SetupDraw) - and that notes each shared-image
// texture it resolves; SetupWireDraw and both dispatch paths acquire the noted ones straight
// after (AcquireNotedSharedImages). ResolveWireImage, through which every attachment, blit,
// copy and readback reaches a texture's image, acquires directly. The test is the write
// generation against the one last acquired, so a generation that moves between two draws of
// one frame is caught at the second.
void VulkanRenderer::AcquireNotedSharedImagesSlow() {
m_textureManager->TakeNotedSharedImageUses(m_sharedImageUseScratch);
for (const Uint64 key : m_sharedImageUseScratch) {
if (auto* resource = m_textureManager->FindWireTextureResource(key)) AcquireSharedImage(*resource);
}
m_sharedImageUseScratch.clear();
}
void VulkanRenderer::AcquireSharedImage(VkTextureManager::TextureResource& resource) {
namespace SI = MG_Remote::Server::SharedImages;
if (resource.sharedImageId == 0 || resource.image == VK_NULL_HANDLE || resource.sharedImageOwner == nullptr)
return;
const auto image = std::static_pointer_cast<const SI::Image>(resource.sharedImageOwner);
Uint64 generation = 0;
const int fence = SI::DupWriteFence(*image, &generation);
// Held, and not written since it was acquired: every use after a frame's first.
if (resource.sharedImageHeld && generation == resource.sharedImageGeneration) {
CloseSyncFile(fence);
return;
}
// Out of any render pass, in the recording the use goes into (or one ahead of it).
auto& frame = m_frameContext.GetCurrent();
if (!frame.isCommandRecording) m_frameContext.BeginCommandRecording();
if (VkRenderPassManager::GetActiveRenderPass()) VkRenderPassManager::EndRenderPass(frame.commandBuffer);
// The write's fence gates the submission carrying the acquire: this recording's, or an
// earlier one if a flush takes it first - either way ahead of every later command.
StageSharedImageWait(fence, kSharedImageUseStages);
const Uint32 family = static_cast<Uint32>(m_physicalDevice.queueFamilies.graphicsFamily);
VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
barrier.image = resource.image;
barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
VkPipelineStageFlags acquireAfter = kSharedImageUseStages;
if (resource.sharedImageHeld) {
// Written again while this frame holds it - a buffer reused before the compositor let
// it go. It is handed back first, so the acquire below matches a release.
barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
barrier.dstAccessMask = 0;
barrier.oldLayout = resource.layout;
barrier.newLayout = VK_IMAGE_LAYOUT_GENERAL;
barrier.srcQueueFamilyIndex = family;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT;
vkCmdPipelineBarrier(frame.commandBuffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
acquireAfter = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT;
}
// GENERAL: the layout the writer releases in, and the one every wire descriptor samples
// in. An image no present has written yet (generation 0) has nothing to keep.
barrier.srcAccessMask = 0;
barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
barrier.oldLayout = generation == 0 ? VK_IMAGE_LAYOUT_UNDEFINED : VK_IMAGE_LAYOUT_GENERAL;
barrier.newLayout = VK_IMAGE_LAYOUT_GENERAL;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT;
barrier.dstQueueFamilyIndex = family;
vkCmdPipelineBarrier(frame.commandBuffer, acquireAfter, kSharedImageUseStages, 0, 0, nullptr, 0, nullptr, 1,
&barrier);
// The open recording names the image now: its out-of-pass work may not move ahead of it.
m_textureManager->StampResourceRecordingUse(resource);
resource.layout = VK_IMAGE_LAYOUT_GENERAL;
resource.sharedImageGeneration = generation;
if (!resource.sharedImageHeld) {
resource.sharedImageHeld = true;
// The current recording is the next submission (RetireWireObjects' rule).
m_sharedImagesHeld.push_back({resource.sharedImageKey, m_submitCounter + 1});
}
m_sharedImageReads.NoteRead(image);
}
// THE FRAME BOUNDARY (Present). Every image this frame acquired goes back to the foreign
// family in the frame's last recording, and that recording's submission signals the semaphore
// whose sync_file is published as the frame's read fence for all of them at once.
void VulkanRenderer::ReleaseHeldSharedImages() {
if (m_sharedImagesHeld.empty()) return;
const Uint32 family = static_cast<Uint32>(m_physicalDevice.queueFamilies.graphicsFamily);
Vector<VkImageMemoryBarrier> releases;
releases.reserve(m_sharedImagesHeld.size());
for (const auto& hold : m_sharedImagesHeld) {
// Gone or re-imaged since: a destroyed image needs no release, and a replacement
// acquired this frame has an entry of its own.
auto* resource = m_textureManager->FindWireTextureResource(hold.key);
if (resource == nullptr || !resource->sharedImageHeld) continue;
VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
barrier.dstAccessMask = 0;
barrier.oldLayout = resource->layout;
barrier.newLayout = VK_IMAGE_LAYOUT_GENERAL;
barrier.srcQueueFamilyIndex = family;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT;
barrier.image = resource->image;
barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
releases.push_back(barrier);
resource->sharedImageHeld = false;
resource->layout = VK_IMAGE_LAYOUT_GENERAL;
m_textureManager->StampResourceRecordingUse(*resource);
}
m_sharedImagesHeld.clear();
if (releases.empty()) return;
auto& frame = m_frameContext.GetCurrent();
if (!frame.isCommandRecording) m_frameContext.BeginCommandRecording();
if (VkRenderPassManager::GetActiveRenderPass()) VkRenderPassManager::EndRenderPass(frame.commandBuffer);
vkCmdPipelineBarrier(frame.commandBuffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, nullptr, 0, nullptr,
static_cast<Uint32>(releases.size()), releases.data());
}
void VulkanRenderer::PublishSharedImageReads() {
const VkSemaphore signal = std::exchange(m_sharedImageSignalSubmitted, VK_NULL_HANDLE);
int fence = -1;
// An export also unsignals the semaphore, which is what lets it be signaled again: a
// submitted signal is exported even when nothing is left to publish it to.
if (signal != VK_NULL_HANDLE && !ExportSharedImageSignal(signal, fence)) fence = -1;
if (m_sharedImageReads.Pending() == 0) {
CloseSyncFile(fence);
return;
}
// No fence (no export on this device): the frame's reads go unpublished - the file
// comment says why that is the choice.
m_sharedImageReads.PublishFrame(fence);
}
void VulkanRenderer::AbandonSharedImageReads() {
// Acquires recorded into the dropped recording never ran, so those images are not held;
// ones an earlier flush submitted are, and the next frame boundary releases them.
SizeT kept = 0;
for (const auto& hold : m_sharedImagesHeld) {
if (hold.acquireSubmit <= m_submitCounter) {
m_sharedImagesHeld[kept++] = hold;
continue;
}
if (auto* resource = m_textureManager->FindWireTextureResource(hold.key))
resource->sharedImageHeld = false;
}
m_sharedImagesHeld.resize(kept);
// Its waits and signal were never submitted. A wait semaphore's unconsumed temporary
// payload is replaced by its next import; the signal semaphore was never signaled.
for (const auto& wait : m_sharedImageWaits) m_freeSharedImageWaitSemaphores.push_back(wait.semaphore);
m_sharedImageWaits.clear();
if (m_sharedImageSignal != VK_NULL_HANDLE)
m_freeSharedImageSignalSemaphores.push_back(std::exchange(m_sharedImageSignal, VK_NULL_HANDLE));
}
#endif // MOBILEGL_BUILD_DISAGGREGATED