From 7218bacb4940ccd7aba2dfe901e71131184648a5 Mon Sep 17 00:00:00 2001 From: Swung0x48 Date: Sat, 3 Oct 2026 22:45:44 -0400 Subject: [PATCH] [Feature] (SharedImage): sample NV12/P010 dma-bufs through GL_OES_EGL_image_external on both backends, copying foreign buffers into server YUV images --- MobileGL/MG_Backend/BackendObject.cpp | 2 + MobileGL/MG_Backend/BackendObject.h | 25 + MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp | 230 ++++++++ MobileGL/MG_Backend/DirectGLES/DirectGLES.h | 7 + MobileGL/MG_Backend/DirectGLES/Managers.cpp | 19 +- MobileGL/MG_Backend/DirectGLES/Managers.h | 1 + .../DirectVulkan/Renderer/ProgramFactory.cpp | 2 + .../DirectVulkan/Renderer/UniformManager.cpp | 3 +- .../Renderer/VkTextureManager.cpp | 42 +- .../DirectVulkan/Renderer/VkTextureManager.h | 19 + .../DirectVulkan/Renderer/VulkanRenderer.cpp | 20 + .../DirectVulkan/Renderer/VulkanRenderer.h | 10 + .../DirectVulkan/Renderer/WireSharedImage.inc | 10 + .../DirectVulkan/Renderer/WireYuvConvert.frag | 10 + .../DirectVulkan/Renderer/WireYuvConvert.vert | 11 + .../Renderer/WireYuvConvertSpirv.h | 78 +++ .../DirectVulkan/Renderer/WireYuvImage.inc | 513 ++++++++++++++++++ MobileGL/MG_Gbm/MobileGLGbm.cpp | 131 ++++- MobileGL/MG_Impl/EGLImpl/EGLImpl.cpp | 79 ++- MobileGL/MG_Impl/EGLImpl/EGLImpl.h | 3 + MobileGL/MG_Impl/GLImpl/Getter/GL_Getter.cpp | 45 +- .../MG_Impl/GLImpl/Texture/GL_Texture.cpp | 75 ++- MobileGL/MG_Impl/Pipe/FramebufferEmit.h | 2 +- MobileGL/MG_Impl/Pipe/SamplerEmit.h | 5 +- MobileGL/MG_Impl/SharedImageApi.cpp | 57 +- MobileGL/MG_Impl/SharedImageApi.h | 29 +- MobileGL/MG_Pipe/MGPipeTypes.h | 35 +- .../MG_Remote/Client/BackendObject_Remote.cpp | 36 ++ .../MG_Remote/Client/BackendObject_Remote.h | 2 + MobileGL/MG_Remote/Server/PipeApplier.cpp | 43 ++ .../MG_Remote/Server/SharedImageRegistry.cpp | 413 ++++++++++++-- .../MG_Remote/Server/SharedImageRegistry.h | 67 +++ MobileGL/MG_Remote/Server/SharedImageYuv.h | 158 ++++++ MobileGL/MG_State/EGLState/Core.cpp | 132 +++-- MobileGL/MG_State/EGLState/Core.h | 28 +- .../GLState/ProgramState/ProgramLinkTask.cpp | 17 + MobileGL/MG_State/GLState/ShareGroupState.cpp | 2 + .../GLState/TextureState/TextureEnum.h | 4 + .../GLState/TextureState/TextureObject.cpp | 4 +- .../GLState/TextureState/TextureObjectStubs.h | 4 + MobileGL/MG_Test/EGLState/EGLStateTest.cpp | 119 +++- .../MG_Test/Gbm/FakeSharedImageLibrary.cpp | 23 +- MobileGL/MG_Test/Gbm/GbmGlamorTest.cpp | 56 ++ MobileGL/MG_Test/Pipe/PipeCatalogueTest.cpp | 2 +- MobileGL/MG_Test/Pipe/TextureEmitTest.cpp | 7 + MobileGL/MG_Test/Program/ProgramTest.cpp | 99 ++++ .../SelfTest/BakedInternalShadersTest.cpp | 13 + MobileGL/MG_Test/Texture/TextureTest.cpp | 56 ++ MobileGL/MG_Test/Wire/SharedImageTest.cpp | 223 +++++++- .../GLToMG/TextureEnumConverter.cpp | 2 + .../MGToGL/TextureEnumConverter.cpp | 2 + .../MGToStr/TextureEnumConverter.cpp | 2 + .../MGToVk/TextureEnumConverter.cpp | 1 + .../ShaderTranspiler/ShaderCompiler.cpp | 24 + .../MG_Util/ShaderTranspiler/ShaderCompiler.h | 6 + .../ShaderSourceProcessor.cpp | 98 ++++ .../ShaderTranspiler/ShaderSourceProcessor.h | 5 + MobileGL/MG_Util/Types.h | 14 + 58 files changed, 2972 insertions(+), 153 deletions(-) create mode 100644 MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvert.frag create mode 100644 MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvert.vert create mode 100644 MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvertSpirv.h create mode 100644 MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvImage.inc create mode 100644 MobileGL/MG_Remote/Server/SharedImageYuv.h diff --git a/MobileGL/MG_Backend/BackendObject.cpp b/MobileGL/MG_Backend/BackendObject.cpp index a1949e99..18f070d1 100644 --- a/MobileGL/MG_Backend/BackendObject.cpp +++ b/MobileGL/MG_Backend/BackendObject.cpp @@ -97,6 +97,8 @@ namespace MobileGL::MG_Backend { static_cast(target) >= kFormatCapabilityTextureTargetCount) { return kFormatCapabilityTargetCount; } + // An external texture is a 2D one wherever a format can go. + if (target == TextureTarget::External) return static_cast(TextureTarget::Texture2D); return static_cast(target); } diff --git a/MobileGL/MG_Backend/BackendObject.h b/MobileGL/MG_Backend/BackendObject.h index 5581a8e6..a56a4f5a 100644 --- a/MobileGL/MG_Backend/BackendObject.h +++ b/MobileGL/MG_Backend/BackendObject.h @@ -610,6 +610,24 @@ namespace MobileGL { Uint32 Stride = 0; Uint32 Offset = 0; Uint64 Modifier = 0; + // A YUV image's Cb Cr plane: in the same descriptor, at this offset and pitch. + Uint32 Plane1Stride = 0; + Uint32 Plane1Offset = 0; + }; + + // An import of a two-plane (YUV) dma-buf: where its planes are in the one descriptor, and + // EGL_EXT_image_dma_buf_import's colour hints (0 = not given). + struct SharedImageImportLayout { + Uint32 PlaneCount = 1; + Uint32 Offset[2] = {0, 0}; + Uint32 Pitch[2] = {0, 0}; + Uint32 ColorSpace = 0; + Uint32 Range = 0; + Uint32 SitingX = 0; + Uint32 SitingY = 0; + // The DRM format modifier the import named; HasModifier false = none (the implicit one). + Bool HasModifier = false; + Uint64 Modifier = 0; }; // THE DAMAGE OF THE PRESENT BEING APPLIED on this thread: the server's apply thread sets it @@ -697,6 +715,13 @@ namespace MobileGL { (void)fd, (void)width, (void)height, (void)fourcc, (void)outId; return false; } + // The same for a YUV fourcc, whose planes and hints `layout` gives. A YUV buffer the + // server did not export is taken too (its labelled CPU-copy fallback). + virtual Bool ImportSharedImagePlanes(int fd, Uint32 width, Uint32 height, Uint32 fourcc, + const SharedImageImportLayout& layout, Uint64* outId) { + (void)fd, (void)width, (void)height, (void)fourcc, (void)layout, (void)outId; + return false; + } // Drops this client's reference (an allocation's or an import's). virtual Bool ReleaseSharedImage(Uint64 id) { (void)id; diff --git a/MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp b/MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp index 8a8496c5..cda13b7a 100644 --- a/MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp +++ b/MobileGL/MG_Backend/DirectGLES/DirectGLES.cpp @@ -427,6 +427,8 @@ namespace MobileGL::MG_Backend::DirectGLES { // targets with no mip chain map to Unknown so the lookup falls back to no bias. TextureTarget SamplerUniformTextureTarget(GLenum uniformType) { switch (uniformType) { + case GL_SAMPLER_EXTERNAL_OES: + return TextureTarget::External; case GL_SAMPLER_1D: case GL_INT_SAMPLER_1D: case GL_UNSIGNED_INT_SAMPLER_1D: @@ -18604,6 +18606,11 @@ namespace MobileGL::MG_Backend::DirectGLES { WaitSyncKhrFn WaitSync = nullptr; ClientWaitSyncKhrFn ClientWaitSync = nullptr; DupNativeFenceFdFn DupNativeFenceFd = nullptr; + // YUV images: sampled as external textures (GL_OES_EGL_image_external_essl3), and + // through GL_EXT_YUV_target's raw Y'CbCr sampler where the driver has it, so the + // conversion is the import's own (its colour hints) rather than the driver's guess. + Bool ExternalEssl3 = false; + Bool YuvTarget = false; }; Bool HasExtensionWord(const char* list, const char* word) { @@ -18667,6 +18674,8 @@ namespace MobileGL::MG_Backend::DirectGLES { entry->Why = "GL_OES_EGL_image is not advertised"; return *entry; } + entry->ExternalEssl3 = HasExtensionWord(gl, "GL_OES_EGL_image_external_essl3"); + entry->YuvTarget = HasExtensionWord(gl, "GL_EXT_YUV_target"); entry->GetNativeClientBuffer = reinterpret_cast( g_EGLFuncs.eglGetProcAddress("eglGetNativeClientBufferANDROID")); entry->CreateImage = @@ -18706,6 +18715,11 @@ namespace MobileGL::MG_Backend::DirectGLES { Uint64 WaitedGeneration = 0; Uint64 WaitedContext = 0; Uint64 NotedFrame = 0; + // A YUV image (ConvertYuvForSampling): an external texture over the EGLImage, the source + // of the conversion into the texture's own RGBA storage, and the reader's frame the + // conversion last ran in. + GLuint ExternalTexture = 0; + Uint64 ConvertedFrame = 0; ~EglImage() { // eglDestroyImageKHR needs the display, not a context, so a session's twin teardown @@ -18713,6 +18727,10 @@ namespace MobileGL::MG_Backend::DirectGLES { // display - which destroyed the image itself - skip it. if (Handle == nullptr || Destroy == nullptr || InProcessExit()) return; if (g_displayTerminations.load(std::memory_order_acquire) != DisplayTerminations) return; + // Texture names are the share group's: deleting one needs any context of it, and + // without one current the name goes with the display. + if (ExternalTexture != 0 && t_boundNativeContext != EGL_NO_CONTEXT) + g_GLESFuncs.glDeleteTextures(1, &ExternalTexture); Destroy(Display, Handle); } #endif @@ -18938,14 +18956,23 @@ namespace MobileGL::MG_Backend::DirectGLES { // The session ends frames at publishing flushes (PublishPendingAccesses): it writes // images, so its first use of one in a frame waits for that image's pending reads. Bool ImplicitSync = false; + // When a YUV image is converted again (ConvertYuvForSampling): renewed at EVERY + // boundary - present, publishing flush, and each native fence whatever the session + // is - since an offscreen producer (a browser's GPU process) may have no other. + Uint64 YuvEpoch = 0; }; std::atomic g_readFrameIds{0}; ReadFrame& CurrentReadFrame() { thread_local ReadFrame frame; if (frame.Id == 0) frame.Id = g_readFrameIds.fetch_add(1, std::memory_order_relaxed) + 1; + if (frame.YuvEpoch == 0) frame.YuvEpoch = g_readFrameIds.fetch_add(1, std::memory_order_relaxed) + 1; return frame; } + + void RenewYuvEpoch(ReadFrame& frame) { + frame.YuvEpoch = g_readFrameIds.fetch_add(1, std::memory_order_relaxed) + 1; + } } // namespace void AcquireForSampling(const EglImageRef& image) { @@ -18975,9 +19002,201 @@ namespace MobileGL::MG_Backend::DirectGLES { } } + Bool IsYuv(const EglImageRef& image) { + return image != nullptr && image->Image != nullptr && SI::FourccIsYuv(image->Image->Fourcc); + } + + namespace { + // THE CONVERSION PASS: one triangle over the texture, sampling the image through an + // external texture. Programs are the share group's and rebuilt with a new context + // generation; the framebuffer and vertex array are per context (never shared). + struct YuvConversion { + Uint ContextGeneration = ~0u; + GLuint ExternalProgram = 0; // the driver's conversion (samplerExternalOES) + GLuint TargetProgram = 0; // the import's own (GL_EXT_YUV_target) + GLint TargetMatrix = -1; + GLint TargetOffset = -1; + Bool Failed = false; + NativeContextName Framebuffer; + NativeContextName VertexArray; + }; + YuvConversion& Conversion() { + static auto* conversion = new YuvConversion(); + return *conversion; + } + + constexpr const char* kYuvVertexSource = + "#version 300 es\n" + "out vec2 vUv;\n" + "void main() {\n" + " vec2 p = vec2(float((gl_VertexID << 1) & 2), float(gl_VertexID & 2));\n" + " gl_Position = vec4(p * 2.0 - 1.0, 0.0, 1.0);\n" + " vUv = p;\n" + "}\n"; + constexpr const char* kYuvExternalFragmentSource = + "#version 300 es\n" + "#extension GL_OES_EGL_image_external_essl3 : require\n" + "precision highp float;\n" + "uniform samplerExternalOES uSource;\n" + "in vec2 vUv;\n" + "out vec4 oColor;\n" + "void main() { oColor = vec4(texture(uSource, vUv).rgb, 1.0); }\n"; + constexpr const char* kYuvTargetFragmentSource = + "#version 300 es\n" + "#extension GL_EXT_YUV_target : require\n" + "precision highp float;\n" + "uniform __samplerExternal2DY2YEXT uSource;\n" + "uniform mat3 uMatrix;\n" + "uniform vec3 uOffset;\n" + "in vec2 vUv;\n" + "out vec4 oColor;\n" + "void main() {\n" + " vec3 ycbcr = texture(uSource, vUv).xyz;\n" + " oColor = vec4(clamp(uMatrix * (ycbcr - uOffset), 0.0, 1.0), 1.0);\n" + "}\n"; + + GLuint BuildYuvProgram(const char* fragmentSource) { + const GLuint vertexShader = g_GLESFuncs.glCreateShader(GL_VERTEX_SHADER); + const GLuint fragmentShader = g_GLESFuncs.glCreateShader(GL_FRAGMENT_SHADER); + GLint linked = GL_FALSE; + GLuint program = 0; + if (vertexShader != 0 && fragmentShader != 0) { + g_GLESFuncs.glShaderSource(vertexShader, 1, &kYuvVertexSource, nullptr); + g_GLESFuncs.glCompileShader(vertexShader); + g_GLESFuncs.glShaderSource(fragmentShader, 1, &fragmentSource, nullptr); + g_GLESFuncs.glCompileShader(fragmentShader); + program = g_GLESFuncs.glCreateProgram(); + if (program != 0) { + g_GLESFuncs.glAttachShader(program, vertexShader); + g_GLESFuncs.glAttachShader(program, fragmentShader); + g_GLESFuncs.glLinkProgram(program); + g_GLESFuncs.glGetProgramiv(program, GL_LINK_STATUS, &linked); + } + } + if (vertexShader != 0) g_GLESFuncs.glDeleteShader(vertexShader); + if (fragmentShader != 0) g_GLESFuncs.glDeleteShader(fragmentShader); + if (linked != GL_TRUE) { + if (program != 0) g_GLESFuncs.glDeleteProgram(program); + return 0; + } + return program; + } + + // MOBILEGL_YUV_DRIVER_CONVERSION=1: always the driver's conversion (samplerExternalOES), + // even where GL_EXT_YUV_target would let the import's hints decide. + Bool DriverConversionForced() { + static const Bool forced = [] { + const char* value = std::getenv("MOBILEGL_YUV_DRIVER_CONVERSION"); + return value != nullptr && value[0] == '1'; + }(); + return forced; + } + } // namespace + + void ConvertYuvForSampling(const EglImageRef& image, GLuint target, Uint32 width, Uint32 height) { + if (!IsYuv(image) || target == 0 || width == 0 || height == 0) return; + EglImage& state = *image; + ReadFrame& frame = CurrentReadFrame(); + if (state.ConvertedFrame == frame.YuvEpoch) return; + // Whatever happens below, this frame does not try again. + state.ConvertedFrame = frame.YuvEpoch; + const EntryPoints& entry = ResolvedEntryPoints(); + if (!entry.Usable || !entry.ExternalEssl3) { + MGLOG_E_ONCE("Shared images: a YUV image cannot be sampled here (%s)", + entry.Usable ? "no GL_OES_EGL_image_external_essl3" : entry.Why.c_str()); + return; + } + // THE CPU-COPY FALLBACK, for a buffer this server did not allocate (SharedImageRegistry.h). + String why; + if (state.Image->IsForeign() && !SI::RefreshForeign(*state.Image, why)) { + MGLOG_E_ONCE("Shared images: foreign YUV image %llu could not be refreshed: %s", + static_cast(state.Image->Id), why.c_str()); + } + + YuvConversion& conversion = Conversion(); + if (conversion.ContextGeneration != g_backendContextGeneration) { + conversion.ExternalProgram = 0; + conversion.TargetProgram = 0; + conversion.TargetMatrix = conversion.TargetOffset = -1; + conversion.Failed = false; + conversion.ContextGeneration = g_backendContextGeneration; + } + if (conversion.Failed) return; + const Bool ownConversion = entry.YuvTarget && !DriverConversionForced(); + GLuint& program = ownConversion ? conversion.TargetProgram : conversion.ExternalProgram; + if (program == 0) { + program = BuildYuvProgram(ownConversion ? kYuvTargetFragmentSource : kYuvExternalFragmentSource); + if (program == 0) { + conversion.Failed = true; + MGLOG_E_ONCE("Shared images: the YUV conversion program could not be built"); + return; + } + if (ownConversion) { + conversion.TargetMatrix = g_GLESFuncs.glGetUniformLocation(program, "uMatrix"); + conversion.TargetOffset = g_GLESFuncs.glGetUniformLocation(program, "uOffset"); + } + MGLOG_I("Shared images: YUV images convert through %s", + ownConversion ? "GL_EXT_YUV_target with the import's colour hints" + : "the driver's external-sampler conversion"); + } + if (conversion.Framebuffer == 0) g_GLESFuncs.glGenFramebuffers(1, &conversion.Framebuffer); + if (conversion.VertexArray == 0) g_GLESFuncs.glGenVertexArrays(1, &conversion.VertexArray); + if (conversion.Framebuffer == 0 || conversion.VertexArray == 0) return; + + // Every binding goes through Espryt's shadows or a guard that puts it back. + ScopedEmulationDrawState drawState; // also activates TextureImpl::TempTextureUnit + ScopedFramebufferBinding binding(/*saveRead=*/false, /*saveDraw=*/true); + DrainErrors(); + if (state.ExternalTexture == 0) { + g_GLESFuncs.glGenTextures(1, &state.ExternalTexture); + g_GLESFuncs.glBindTexture(GL_TEXTURE_EXTERNAL_OES, state.ExternalTexture); + entry.TargetTexture2D(GL_TEXTURE_EXTERNAL_OES, state.Handle); + g_GLESFuncs.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + g_GLESFuncs.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + g_GLESFuncs.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + g_GLESFuncs.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + const GLenum error = g_GLESFuncs.glGetError(); + if (error != GL_NO_ERROR) { + MGLOG_E_ONCE("Shared images: YUV image %llu cannot be bound as an external texture (0x%04x)", + static_cast(state.Image->Id), static_cast(error)); + } + } else { + g_GLESFuncs.glBindTexture(GL_TEXTURE_EXTERNAL_OES, state.ExternalTexture); + } + FramebufferImpl::BindFramebufferId(GL_DRAW_FRAMEBUFFER, conversion.Framebuffer); + g_GLESFuncs.glFramebufferTexture2D(GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, target, 0); + const GLenum status = g_GLESFuncs.glCheckFramebufferStatus(GL_DRAW_FRAMEBUFFER); + if (status == GL_FRAMEBUFFER_COMPLETE) { + VertexArrayImpl::BindBackendVAOId(conversion.VertexArray); + g_GLESFuncs.glUseProgram(program); + g_GLESFuncs.glUniform1i(g_GLESFuncs.glGetUniformLocation(program, "uSource"), + static_cast(TextureImpl::TempTextureUnit)); + if (ownConversion) { + // A Y2Y sample is the stored code normalized as the format stores it: 8 bits, or + // P010's 10 in the high end of 16. + const double sampleMax = state.Image->Fourcc == SI::kFourccP010 ? 65535.0 / 64.0 : 255.0; + const auto c = SI::Yuv::ConversionFor(SI::GetYuvHints(*state.Image), sampleMax); + g_GLESFuncs.glUniformMatrix3fv(conversion.TargetMatrix, 1, GL_FALSE, c.Matrix); + g_GLESFuncs.glUniform3fv(conversion.TargetOffset, 1, c.Offset); + } + g_GLESFuncs.glViewport(0, 0, static_cast(width), static_cast(height)); + g_GLESFuncs.glDrawArrays(GL_TRIANGLES, 0, 3); + const GLenum error = g_GLESFuncs.glGetError(); + if (error != GL_NO_ERROR) { + MGLOG_E_ONCE("Shared images: the YUV conversion draw failed (0x%04x)", static_cast(error)); + } + } else { + MGLOG_E_ONCE("Shared images: a YUV conversion target is not renderable (0x%04x)", + static_cast(status)); + } + g_GLESFuncs.glFramebufferTexture2D(GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, 0, 0); + g_GLESFuncs.glBindTexture(GL_TEXTURE_EXTERNAL_OES, 0); + } + Bool PublishPendingAccesses() { ReadFrame& frame = CurrentReadFrame(); frame.ImplicitSync = true; + RenewYuvEpoch(frame); // Sub-data waiting for a sync that nothing of this session would otherwise trigger. if (t_boundNativeContext != EGL_NO_CONTEXT) TextureImpl::WritePendingSharedImageUploads(); if (frame.Tracker.Pending() == 0) return true; @@ -18994,6 +19213,7 @@ namespace MobileGL::MG_Backend::DirectGLES { Bool ExportNativeFence(int* fence) { *fence = -1; ReadFrame& frame = CurrentReadFrame(); + RenewYuvEpoch(frame); // No context current: nothing of this session is in flight to fence. if (t_boundNativeContext == EGL_NO_CONTEXT) return true; TextureImpl::WritePendingSharedImageUploads(); @@ -19015,6 +19235,7 @@ namespace MobileGL::MG_Backend::DirectGLES { void PublishPendingReads() { ReadFrame& frame = CurrentReadFrame(); + RenewYuvEpoch(frame); if (frame.Tracker.Pending() == 0) return; int fence = -1; // No context current: the reads' context is gone (and its work with it), and there is @@ -19207,6 +19428,15 @@ namespace MobileGL::MG_Backend::DirectGLES { return false; } void AcquireForSampling(const EglImageRef& image) { (void)image; } + + Bool IsYuv(const EglImageRef& image) { + (void)image; + return false; + } + + void ConvertYuvForSampling(const EglImageRef& image, GLuint target, Uint32 width, Uint32 height) { + (void)image, (void)target, (void)width, (void)height; + } void PublishPendingReads() {} Bool PublishPendingAccesses() { return true; } Bool ExportNativeFence(int* fence) { diff --git a/MobileGL/MG_Backend/DirectGLES/DirectGLES.h b/MobileGL/MG_Backend/DirectGLES/DirectGLES.h index 78021045..fb25ec0c 100644 --- a/MobileGL/MG_Backend/DirectGLES/DirectGLES.h +++ b/MobileGL/MG_Backend/DirectGLES/DirectGLES.h @@ -548,6 +548,13 @@ namespace MobileGL::MG_Backend::DirectGLES { // landed since this context last waited, the context's later commands wait for its fence // on the GPU; and the image is noted as read by the calling session's current frame. void AcquireForSampling(const EglImageRef& image); + // YUV IMAGES (NV12/P010, MG_Remote/Server/SharedImageYuv.h) are never a name's level 0: the + // name keeps its own RGBA8 storage `target` (width x height), into which the image is + // converted at its first use in each of the calling session's frames - the image bound as + // an external texture, one triangle drawn - after the registry's CPU copy for a foreign + // buffer. Every touched binding is put back. Called after AcquireForSampling. + Bool IsYuv(const EglImageRef& image); + void ConvertYuvForSampling(const EglImageRef& image, GLuint target, Uint32 width, Uint32 height); // The calling session's frame boundary for what it has read: hands a fence for everything // the current context has submitted to every image noted since the last boundary. Called // at Present and wherever the current native context stops being current (its reads can diff --git a/MobileGL/MG_Backend/DirectGLES/Managers.cpp b/MobileGL/MG_Backend/DirectGLES/Managers.cpp index 4a2f6662..447f5fd8 100644 --- a/MobileGL/MG_Backend/DirectGLES/Managers.cpp +++ b/MobileGL/MG_Backend/DirectGLES/Managers.cpp @@ -8893,7 +8893,20 @@ namespace MobileGL::MG_Backend::DirectGLES { return true; } Bind(GL_TEXTURE_2D); - if (!SharedImageImpl::TargetBoundTexture2D(image)) { + if (SharedImageImpl::IsYuv(image)) { + // A YUV image is CONVERTED into the name's own RGBA8 level at each frame's first use + // (SharedImageImpl::ConvertYuvForSampling), never made its storage. + while (g_GLESFuncs.glGetError() != GL_NO_ERROR) { + } + g_GLESFuncs.glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, static_cast(record.Desc.Width), + static_cast(record.Desc.Height)); + if (g_GLESFuncs.glGetError() != GL_NO_ERROR) { + MGLOG_E_ONCE("Shared image texture %u: no %ux%u RGBA8 storage for YUV image %llu; left empty", + record.Desc.GlNameForDiag, record.Desc.Width, record.Desc.Height, + static_cast(imageId)); + return true; + } + } else if (!SharedImageImpl::TargetBoundTexture2D(image)) { MGLOG_E_ONCE("Shared image texture %u: glEGLImageTargetTexture2DOES refused image %llu; left empty", record.Desc.GlNameForDiag, static_cast(imageId)); return true; @@ -9042,6 +9055,10 @@ namespace MobileGL::MG_Backend::DirectGLES { // before the command this sync is for, and the session's frame notes the read. if (m_sharedImage != nullptr) SharedImageImpl::AcquireForSampling(m_sharedImage); ReplaySharedImageUploads(*pushedStorage); + // A YUV image is converted into the name at the frame's first use. + if (m_sharedImage != nullptr && SharedImageImpl::IsYuv(m_sharedImage)) + SharedImageImpl::ConvertYuvForSampling(m_sharedImage, GetBackendTextureId(), + pushedStorage->Desc.Width, pushedStorage->Desc.Height); return; } // P5e (tx2), CONTRACT-P5E §5.2 (scout G-S2-2): THE VIEW TEST MOVES BEHIND THE diff --git a/MobileGL/MG_Backend/DirectGLES/Managers.h b/MobileGL/MG_Backend/DirectGLES/Managers.h index b9999894..1a906c1f 100644 --- a/MobileGL/MG_Backend/DirectGLES/Managers.h +++ b/MobileGL/MG_Backend/DirectGLES/Managers.h @@ -1944,6 +1944,7 @@ namespace MobileGL::MG_Backend::DirectGLES { switch (target) { case TextureTarget::Texture1D: case TextureTarget::TextureRectangle: + case TextureTarget::External: return TextureTarget::Texture2D; case TextureTarget::Texture1DArray: return TextureTarget::Texture2DArray; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp index 54a5db70..6a0293f9 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/ProgramFactory.cpp @@ -2529,6 +2529,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { return SamplerNumericDomain::UnsignedInteger; case GL_SAMPLER_1D: case GL_SAMPLER_2D: + case GL_SAMPLER_EXTERNAL_OES: case GL_SAMPLER_3D: case GL_SAMPLER_CUBE: case GL_SAMPLER_2D_RECT: @@ -2719,6 +2720,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { case GL_UNSIGNED_INT_IMAGE_CUBE_MAP_ARRAY: return TextureTarget::TextureCubeMapArray; case GL_SAMPLER_2D: + case GL_SAMPLER_EXTERNAL_OES: // a 2D texture below the frontend case GL_SAMPLER_2D_SHADOW: case GL_INT_SAMPLER_2D: case GL_UNSIGNED_INT_SAMPLER_2D: diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/UniformManager.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/UniformManager.cpp index 31fa419e..fde6b6e2 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/UniformManager.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/UniformManager.cpp @@ -3198,7 +3198,8 @@ namespace MobileGL::MG_Backend::DirectVulkan { const auto* resource = m_textureManager->SyncTextureResourceByHandle(handle, false, storage); if (resource == nullptr) return false; // Another session writes it: the renderer acquires it before this use. - if (resource->sharedImageId != 0) m_textureManager->NoteSharedImageUse(*resource); + if (resource->sharedImageId != 0 || resource->yuvImageId != 0) + m_textureManager->NoteSharedImageUse(*resource); } } } diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.cpp index 97b9f6d6..b5fb0e37 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.cpp @@ -2115,9 +2115,41 @@ namespace MobileGL::MG_Backend::DirectVulkan { Bool VkTextureManager::SyncWireTextureShape(const MG_Pipe::MGPipeResourceRecord& record, TextureResource& resource, Bool requireStorage) { const MG_Pipe::MGPResourceDesc& desc = record.Desc; + // A YUV shared image is converted INTO the texture's own RGBA8 storage, which the rest of + // this function defines as for any texture; the image is only named here, and the + // renderer converts it at its first use in each frame (WireYuvImage.inc). + if (record.SharedImageId != resource.yuvImageId && !m_syncingYuvTextureStorage) { + // Respecified away from (or to another) YUV image: its source goes with it, once the + // GPU is done with it (a released resource's Reset is deferred). + if (resource.yuvSource != nullptr) { + TextureResource retired; + retired.yuvSource = std::move(resource.yuvSource); + DeferResourceRelease(Move(retired)); + } + resource.yuvImageId = 0; + resource.yuvOwner.reset(); + resource.yuvSource.reset(); + resource.yuvConvertedFrame = 0; + } + if (record.SharedImageId != 0 && !m_syncingYuvTextureStorage) { + const auto yuv = MG_Remote::Server::SharedImages::Find(record.SharedImageId); + if (yuv != nullptr && MG_Remote::Server::SharedImages::FourccIsYuv(yuv->Fourcc)) { + m_syncingYuvTextureStorage = true; + const Bool defined = SyncWireTextureShape(record, resource, requireStorage); + m_syncingYuvTextureStorage = false; + if (!defined) return false; + if (resource.yuvImageId != record.SharedImageId) { + resource.yuvImageId = record.SharedImageId; + resource.yuvOwner = yuv; + resource.yuvSource.reset(); + resource.yuvConvertedFrame = 0; + } + return true; + } + } // Level 0 is a shared image: bind its buffer. An image that cannot be bound here leaves // the texture the storage its NULL-data definition asked for, below. - if (record.SharedImageId != 0) { + if (record.SharedImageId != 0 && !m_syncingYuvTextureStorage) { const SharedImageBind bound = SyncWireSharedImage(record, resource, requireStorage); if (bound != SharedImageBind::Unresolved) return bound == SharedImageBind::Bound; } @@ -3057,7 +3089,7 @@ namespace MobileGL::MG_Backend::DirectVulkan { handle.Slot, handle.Gen); return nullptr; } - resource.sharedImageKey = resource.sharedImageId != 0 ? key : 0; + resource.sharedImageKey = resource.sharedImageId != 0 || resource.yuvImageId != 0 ? key : 0; resource.syncedWireSerial = record.Serial; return &resource; } @@ -3774,7 +3806,11 @@ namespace MobileGL::MG_Backend::DirectVulkan { resource.sampledView == VK_NULL_HANDLE && resource.perMipViews.empty() && resource.perMipSampledViews.empty() && resource.attachmentViews.empty() && resource.alternateSampledViews.empty() && - resource.storageImageViews.empty()) { + resource.storageImageViews.empty() +#if MOBILEGL_BUILD_DISAGGREGATED + && resource.yuvSource == nullptr +#endif + ) { return; } diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.h index 36730823..e7a634f5 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VkTextureManager.h @@ -372,6 +372,15 @@ public: Bool sharedImageHeld = false; Uint64 sharedImageGeneration = 0; Uint64 sharedImageKey = 0; + // A YUV shared image (NV12/P010) is never the texture's image: the texture keeps its own + // RGBA8 level 0, into which the renderer converts the image at its first use in each frame + // (WireYuvImage.inc) - `yuvSource` being that conversion's import of the buffer, made on + // first use, and `yuvConvertedFrame` the frame it last ran in. `yuvOwner` holds the + // registry's image for as long as the texture names it. + Uint64 yuvImageId = 0; + SharedPtr yuvOwner; + SharedPtr yuvSource; + Uint64 yuvConvertedFrame = 0; #endif TextureResource() = default; @@ -415,6 +424,10 @@ public: std::swap(this->sharedImageHeld, that.sharedImageHeld); std::swap(this->sharedImageGeneration, that.sharedImageGeneration); std::swap(this->sharedImageKey, that.sharedImageKey); + std::swap(this->yuvImageId, that.yuvImageId); + std::swap(this->yuvOwner, that.yuvOwner); + std::swap(this->yuvSource, that.yuvSource); + std::swap(this->yuvConvertedFrame, that.yuvConvertedFrame); #endif } @@ -465,6 +478,10 @@ public: sharedImageHeld = false; sharedImageGeneration = 0; sharedImageKey = 0; + yuvImageId = 0; + yuvOwner.reset(); + yuvSource.reset(); + yuvConvertedFrame = 0; #endif fullView = VK_NULL_HANDLE; sampledView = VK_NULL_HANDLE; @@ -882,6 +899,8 @@ private: // Active only between BeginDrawSyncScope/EndDrawSyncScope; identities of // textures already fully synced in the current draw (small N -> flat scan). Bool m_drawSyncScopeActive = false; + // SyncWireTextureShape defining the RGBA8 storage a YUV shared image is converted into. + Bool m_syncingYuvTextureStorage = false; // Per-draw sync memo: the identity plus the resolved resource pointer. The pointer is stable // across rehash in the node-based m_textureResources and stays valid for the draw (a texture // synced this draw is alive and is not erased mid-draw), so a repeat sync of the same texture diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp index 1b5a937b..6e17b010 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp @@ -3525,6 +3525,7 @@ void main() { CollectWireObjects(m_submitCounter, true); ClearAllWireDrawPassCaches(); DestroyWireColorBlitResources(); + DestroyWireYuvResources(); DestroyWireDepthMipmapResources(); DestroyWireMultisampleResolveResources(); DestroySharedImagePresentTargets(false); @@ -8107,6 +8108,8 @@ void main() { #include "WireDraw.inc" // AFTER WireFramebuffer.inc: the shared-image present is that file's readback, into an image. #include "WireSharedImage.inc" + // AFTER WireSharedImage.inc: a YUV image is that file's reading side, converted on the way. + #include "WireYuvImage.inc" void VulkanRenderer::Clear(GLbitfield mask) { #if MOBILEGL_BUILD_DISAGGREGATED @@ -16032,6 +16035,23 @@ void main() { } } + // YUV shared images (WireYuvImage.inc) are sampled through a VkSamplerYcbcrConversion, a + // feature of its own; asked only where AHardwareBuffers are imported at all. + m_samplerYcbcrConversion = false; + VkPhysicalDeviceSamplerYcbcrConversionFeatures ycbcrFeatures{}; + ycbcrFeatures.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLER_YCBCR_CONVERSION_FEATURES; + if (m_wireAhbImport && getPhysicalDeviceFeatures2 != nullptr) { + VkPhysicalDeviceFeatures2 featureQuery{}; + featureQuery.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; + featureQuery.pNext = &ycbcrFeatures; + getPhysicalDeviceFeatures2(m_physicalDevice.handle, &featureQuery); + if (ycbcrFeatures.samplerYcbcrConversion == VK_TRUE) { + ycbcrFeatures.pNext = const_cast(deviceCreateInfo.pNext); + deviceCreateInfo.pNext = &ycbcrFeatures; + m_samplerYcbcrConversion = true; + } + } + // VK_EXT_multi_draw: tier 1 of the multi-draw dispatch - one vkCmdDrawMulti(Indexed)EXT // for a whole glMultiDraw* batch (VkMultiDrawIndexedInfoEXT carries per-draw // firstIndex/indexCount/vertexOffset, so glMultiDrawElementsBaseVertex fits natively). diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h index bd0a984f..80f48b94 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.h @@ -717,6 +717,16 @@ namespace MobileGL::MG_Backend::DirectVulkan { // one entry point the import needs (PFN_vkGetAndroidHardwareBufferPropertiesANDROID). Bool m_wireAhbImport = false; void* m_wireGetAhbProperties = nullptr; + // YUV shared images (WireYuvImage.inc): the device enabled samplerYcbcrConversion, and the + // conversion pipelines made so far (one per buffer format and colour conversion). + Bool m_samplerYcbcrConversion = false; + struct WireYuvResources; + WireYuvResources* m_wireYuvResources = nullptr; + void DestroyWireYuvResources(); + // A YUV texture's first use in a frame converts its image (yuvConvertedFrame against this, + // which every frame boundary - present, publishing flush, native fence - moves). + Uint64 m_yuvFrameSerial = 1; + Bool ConvertYuvSharedImage(VkTextureManager::TextureResource& resource); // Shared images this session presents into, imported once each and keyed by the image's // id (monotonic, never reused). `owner` watches the registry's image: an entry whose image // died is dropped at a later present once `lastSubmit`, its last copy, has completed. diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireSharedImage.inc b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireSharedImage.inc index 9077ab90..573932f3 100644 --- a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireSharedImage.inc +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireSharedImage.inc @@ -499,6 +499,11 @@ void VulkanRenderer::AcquireSharedImage(VkTextureManager::TextureResource& resource) { namespace SI = MG_Remote::Server::SharedImages; + // A YUV image is converted into the texture's own image instead (WireYuvImage.inc). + if (resource.yuvImageId != 0) { + (void)ConvertYuvSharedImage(resource); + return; + } if (resource.sharedImageId == 0 || resource.image == VK_NULL_HANDLE || resource.sharedImageOwner == nullptr) return; const auto image = std::static_pointer_cast(resource.sharedImageOwner); @@ -567,6 +572,8 @@ // family in the frame's last recording, and that recording's submission signals the semaphore // whose sync_file is published as the frame's read fence for all of them at once. void VulkanRenderer::ReleaseHeldSharedImages() { + // A frame boundary: a YUV texture's next use converts its image again. + ++m_yuvFrameSerial; if (m_sharedImagesHeld.empty()) return; const Uint32 family = static_cast(m_physicalDevice.queueFamilies.graphicsFamily); Vector releases; @@ -633,6 +640,9 @@ // submission, and it is tracked like any other (the semaphore's reuse waits for it). Bool VulkanRenderer::ExportNativeFence(int* fence) { *fence = -1; + // Every fence the application asks for ends one of its frames as far as a YUV texture is + // concerned (an offscreen producer - a browser's GPU process - has no present to do it). + ++m_yuvFrameSerial; if (m_device == VK_NULL_HANDLE || m_graphicsQueue == VK_NULL_HANDLE || m_frameContext.GetFrameCount() == 0) return true; // nothing was ever submitted: nothing to fence // A session that writes images ends its frame here, as at a flush; a compositor that only diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvert.frag b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvert.frag new file mode 100644 index 00000000..45286db8 --- /dev/null +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvert.frag @@ -0,0 +1,10 @@ +#version 450 +// The image is bound through an immutable sampler that carries its VkSamplerYcbcrConversion, so +// a plain sample already returns RGB. +layout(set = 0, binding = 0) uniform sampler2D yuvImage; +layout(location = 0) in vec2 texCoord; +layout(location = 0) out vec4 outColor; + +void main() { + outColor = vec4(textureLod(yuvImage, texCoord, 0.0).rgb, 1.0); +} diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvert.vert b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvert.vert new file mode 100644 index 00000000..a07802aa --- /dev/null +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvert.vert @@ -0,0 +1,11 @@ +#version 450 +// A YUV shared image converted into an RGBA texture (WireYuvImage.inc): one triangle over the +// whole target, its first row at the image's first row. +layout(location = 0) out vec2 texCoord; + +void main() { + const vec2 corners[3] = vec2[](vec2(-1.0, -1.0), vec2(3.0, -1.0), vec2(-1.0, 3.0)); + vec2 p = corners[gl_VertexIndex]; + gl_Position = vec4(p, 0.0, 1.0); + texCoord = (p + 1.0) * 0.5; +} diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvertSpirv.h b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvertSpirv.h new file mode 100644 index 00000000..3004ca35 --- /dev/null +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvertSpirv.h @@ -0,0 +1,78 @@ +// Generated from WireYuvConvert.vert / WireYuvConvert.frag (Vulkan 1.0 / SPIR-V 1.0) by +// scripts/bake_internal_shaders.py, which runs MG_Test's BakedInternalShadersTest with the +// IN-TREE glslang and splices the words it emits into the arrays below. Do not edit the words +// by hand: MOBILEGL_BAKED_INTERNAL_SHADERS (the same test, run as a gate) recompiles every +// source in its table and reds on the first word that disagrees. +#pragma once + +static constexpr Uint32 kWireYuvConvertVertexSpirv[] = { + 0x07230203u, 0x00010000u, 0x0008000bu, 0x00000031u, 0x00000000u, 0x00020011u, 0x00000001u, 0x0006000bu, + 0x00000001u, 0x4c534c47u, 0x6474732eu, 0x3035342eu, 0x00000000u, 0x0003000eu, 0x00000000u, 0x00000001u, + 0x0008000fu, 0x00000000u, 0x00000004u, 0x6e69616du, 0x00000000u, 0x00000015u, 0x00000020u, 0x0000002bu, + 0x00030003u, 0x00000002u, 0x000001c2u, 0x00040005u, 0x00000004u, 0x6e69616du, 0x00000000u, 0x00030005u, + 0x00000009u, 0x00000070u, 0x00060005u, 0x00000015u, 0x565f6c67u, 0x65747265u, 0x646e4978u, 0x00007865u, + 0x00050005u, 0x00000018u, 0x65646e69u, 0x6c626178u, 0x00000065u, 0x00060005u, 0x0000001eu, 0x505f6c67u, + 0x65567265u, 0x78657472u, 0x00000000u, 0x00060006u, 0x0000001eu, 0x00000000u, 0x505f6c67u, 0x7469736fu, + 0x006e6f69u, 0x00070006u, 0x0000001eu, 0x00000001u, 0x505f6c67u, 0x746e696fu, 0x657a6953u, 0x00000000u, + 0x00070006u, 0x0000001eu, 0x00000002u, 0x435f6c67u, 0x4470696cu, 0x61747369u, 0x0065636eu, 0x00070006u, + 0x0000001eu, 0x00000003u, 0x435f6c67u, 0x446c6c75u, 0x61747369u, 0x0065636eu, 0x00030005u, 0x00000020u, + 0x00000000u, 0x00050005u, 0x0000002bu, 0x43786574u, 0x64726f6fu, 0x00000000u, 0x00040047u, 0x00000015u, + 0x0000000bu, 0x0000002au, 0x00030047u, 0x0000001eu, 0x00000002u, 0x00050048u, 0x0000001eu, 0x00000000u, + 0x0000000bu, 0x00000000u, 0x00050048u, 0x0000001eu, 0x00000001u, 0x0000000bu, 0x00000001u, 0x00050048u, + 0x0000001eu, 0x00000002u, 0x0000000bu, 0x00000003u, 0x00050048u, 0x0000001eu, 0x00000003u, 0x0000000bu, + 0x00000004u, 0x00040047u, 0x0000002bu, 0x0000001eu, 0x00000000u, 0x00020013u, 0x00000002u, 0x00030021u, + 0x00000003u, 0x00000002u, 0x00030016u, 0x00000006u, 0x00000020u, 0x00040017u, 0x00000007u, 0x00000006u, + 0x00000002u, 0x00040020u, 0x00000008u, 0x00000007u, 0x00000007u, 0x00040015u, 0x0000000au, 0x00000020u, + 0x00000000u, 0x0004002bu, 0x0000000au, 0x0000000bu, 0x00000003u, 0x0004001cu, 0x0000000cu, 0x00000007u, + 0x0000000bu, 0x0004002bu, 0x00000006u, 0x0000000du, 0xbf800000u, 0x0005002cu, 0x00000007u, 0x0000000eu, + 0x0000000du, 0x0000000du, 0x0004002bu, 0x00000006u, 0x0000000fu, 0x40400000u, 0x0005002cu, 0x00000007u, + 0x00000010u, 0x0000000fu, 0x0000000du, 0x0005002cu, 0x00000007u, 0x00000011u, 0x0000000du, 0x0000000fu, + 0x0006002cu, 0x0000000cu, 0x00000012u, 0x0000000eu, 0x00000010u, 0x00000011u, 0x00040015u, 0x00000013u, + 0x00000020u, 0x00000001u, 0x00040020u, 0x00000014u, 0x00000001u, 0x00000013u, 0x0004003bu, 0x00000014u, + 0x00000015u, 0x00000001u, 0x00040020u, 0x00000017u, 0x00000007u, 0x0000000cu, 0x00040017u, 0x0000001bu, + 0x00000006u, 0x00000004u, 0x0004002bu, 0x0000000au, 0x0000001cu, 0x00000001u, 0x0004001cu, 0x0000001du, + 0x00000006u, 0x0000001cu, 0x0006001eu, 0x0000001eu, 0x0000001bu, 0x00000006u, 0x0000001du, 0x0000001du, + 0x00040020u, 0x0000001fu, 0x00000003u, 0x0000001eu, 0x0004003bu, 0x0000001fu, 0x00000020u, 0x00000003u, + 0x0004002bu, 0x00000013u, 0x00000021u, 0x00000000u, 0x0004002bu, 0x00000006u, 0x00000023u, 0x00000000u, + 0x0004002bu, 0x00000006u, 0x00000024u, 0x3f800000u, 0x00040020u, 0x00000028u, 0x00000003u, 0x0000001bu, + 0x00040020u, 0x0000002au, 0x00000003u, 0x00000007u, 0x0004003bu, 0x0000002au, 0x0000002bu, 0x00000003u, + 0x0004002bu, 0x00000006u, 0x0000002fu, 0x3f000000u, 0x00050036u, 0x00000002u, 0x00000004u, 0x00000000u, + 0x00000003u, 0x000200f8u, 0x00000005u, 0x0004003bu, 0x00000008u, 0x00000009u, 0x00000007u, 0x0004003bu, + 0x00000017u, 0x00000018u, 0x00000007u, 0x0004003du, 0x00000013u, 0x00000016u, 0x00000015u, 0x0003003eu, + 0x00000018u, 0x00000012u, 0x00050041u, 0x00000008u, 0x00000019u, 0x00000018u, 0x00000016u, 0x0004003du, + 0x00000007u, 0x0000001au, 0x00000019u, 0x0003003eu, 0x00000009u, 0x0000001au, 0x0004003du, 0x00000007u, + 0x00000022u, 0x00000009u, 0x00050051u, 0x00000006u, 0x00000025u, 0x00000022u, 0x00000000u, 0x00050051u, + 0x00000006u, 0x00000026u, 0x00000022u, 0x00000001u, 0x00070050u, 0x0000001bu, 0x00000027u, 0x00000025u, + 0x00000026u, 0x00000023u, 0x00000024u, 0x00050041u, 0x00000028u, 0x00000029u, 0x00000020u, 0x00000021u, + 0x0003003eu, 0x00000029u, 0x00000027u, 0x0004003du, 0x00000007u, 0x0000002cu, 0x00000009u, 0x00050050u, + 0x00000007u, 0x0000002du, 0x00000024u, 0x00000024u, 0x00050081u, 0x00000007u, 0x0000002eu, 0x0000002cu, + 0x0000002du, 0x0005008eu, 0x00000007u, 0x00000030u, 0x0000002eu, 0x0000002fu, 0x0003003eu, 0x0000002bu, + 0x00000030u, 0x000100fdu, 0x00010038u, +}; + +static constexpr Uint32 kWireYuvConvertFragmentSpirv[] = { + 0x07230203u, 0x00010000u, 0x0008000bu, 0x0000001cu, 0x00000000u, 0x00020011u, 0x00000001u, 0x0006000bu, + 0x00000001u, 0x4c534c47u, 0x6474732eu, 0x3035342eu, 0x00000000u, 0x0003000eu, 0x00000000u, 0x00000001u, + 0x0007000fu, 0x00000004u, 0x00000004u, 0x6e69616du, 0x00000000u, 0x00000009u, 0x00000011u, 0x00030010u, + 0x00000004u, 0x00000007u, 0x00030003u, 0x00000002u, 0x000001c2u, 0x00040005u, 0x00000004u, 0x6e69616du, + 0x00000000u, 0x00050005u, 0x00000009u, 0x4374756fu, 0x726f6c6fu, 0x00000000u, 0x00050005u, 0x0000000du, + 0x49767579u, 0x6567616du, 0x00000000u, 0x00050005u, 0x00000011u, 0x43786574u, 0x64726f6fu, 0x00000000u, + 0x00040047u, 0x00000009u, 0x0000001eu, 0x00000000u, 0x00040047u, 0x0000000du, 0x00000021u, 0x00000000u, + 0x00040047u, 0x0000000du, 0x00000022u, 0x00000000u, 0x00040047u, 0x00000011u, 0x0000001eu, 0x00000000u, + 0x00020013u, 0x00000002u, 0x00030021u, 0x00000003u, 0x00000002u, 0x00030016u, 0x00000006u, 0x00000020u, + 0x00040017u, 0x00000007u, 0x00000006u, 0x00000004u, 0x00040020u, 0x00000008u, 0x00000003u, 0x00000007u, + 0x0004003bu, 0x00000008u, 0x00000009u, 0x00000003u, 0x00090019u, 0x0000000au, 0x00000006u, 0x00000001u, + 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x0003001bu, 0x0000000bu, 0x0000000au, + 0x00040020u, 0x0000000cu, 0x00000000u, 0x0000000bu, 0x0004003bu, 0x0000000cu, 0x0000000du, 0x00000000u, + 0x00040017u, 0x0000000fu, 0x00000006u, 0x00000002u, 0x00040020u, 0x00000010u, 0x00000001u, 0x0000000fu, + 0x0004003bu, 0x00000010u, 0x00000011u, 0x00000001u, 0x0004002bu, 0x00000006u, 0x00000013u, 0x00000000u, + 0x00040017u, 0x00000015u, 0x00000006u, 0x00000003u, 0x0004002bu, 0x00000006u, 0x00000017u, 0x3f800000u, + 0x00050036u, 0x00000002u, 0x00000004u, 0x00000000u, 0x00000003u, 0x000200f8u, 0x00000005u, 0x0004003du, + 0x0000000bu, 0x0000000eu, 0x0000000du, 0x0004003du, 0x0000000fu, 0x00000012u, 0x00000011u, 0x00070058u, + 0x00000007u, 0x00000014u, 0x0000000eu, 0x00000012u, 0x00000002u, 0x00000013u, 0x0008004fu, 0x00000015u, + 0x00000016u, 0x00000014u, 0x00000014u, 0x00000000u, 0x00000001u, 0x00000002u, 0x00050051u, 0x00000006u, + 0x00000018u, 0x00000016u, 0x00000000u, 0x00050051u, 0x00000006u, 0x00000019u, 0x00000016u, 0x00000001u, + 0x00050051u, 0x00000006u, 0x0000001au, 0x00000016u, 0x00000002u, 0x00070050u, 0x00000007u, 0x0000001bu, + 0x00000018u, 0x00000019u, 0x0000001au, 0x00000017u, 0x0003003eu, 0x00000009u, 0x0000001bu, 0x000100fdu, + 0x00010038u, +}; diff --git a/MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvImage.inc b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvImage.inc new file mode 100644 index 00000000..7e4bed59 --- /dev/null +++ b/MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvImage.inc @@ -0,0 +1,513 @@ +// Included by VulkanRenderer.cpp inside the backend namespace, after WireSharedImage.inc. +// +// YUV SHARED IMAGES (NV12, P010; MG_Remote/Server/SharedImageYuv.h). A YUV image is never a +// texture's image: the texture keeps its own RGBA8 level 0 (VkTextureManager::SyncWireTextureShape) +// and, at its first use in each frame, the image is CONVERTED into it here - the image's +// AHardwareBuffer imported with its external format, sampled through an immutable sampler that +// carries a VkSamplerYcbcrConversion built from the import's colour hints, one triangle over the +// target. Zero CPU copies for an image this server allocated; a foreign one (a video decoder's +// buffer) is first brought up to date by the registry's labelled CPU copy (RefreshForeign). +// +// Why converted rather than sampled in place: a Y'CbCr conversion is an immutable sampler, which +// lives in the descriptor set layout and therefore in every application pipeline that samples the +// texture. A conversion pass keeps those pipelines exactly what they are for an RGBA texture, at +// the cost of one small pass per new frame of the video. +#if MOBILEGL_BUILD_DISAGGREGATED +#include "WireYuvConvertSpirv.h" + + struct VulkanRenderer::WireYuvResources { + PFN_vkCreateSamplerYcbcrConversion createConversion = nullptr; + PFN_vkDestroySamplerYcbcrConversion destroyConversion = nullptr; + VkShaderModule vertex = VK_NULL_HANDLE, fragment = VK_NULL_HANDLE; + // One per (buffer format, conversion, target format). Never dropped before shutdown: a + // device sees a handful of them. + struct Pipeline { + Uint64 externalFormat = 0; + VkFormat format = VK_FORMAT_UNDEFINED; + VkSamplerYcbcrModelConversion model = VK_SAMPLER_YCBCR_MODEL_CONVERSION_RGB_IDENTITY; + VkSamplerYcbcrRange range = VK_SAMPLER_YCBCR_RANGE_ITU_FULL; + VkComponentMapping components{}; + VkChromaLocation xOffset = VK_CHROMA_LOCATION_COSITED_EVEN; + VkChromaLocation yOffset = VK_CHROMA_LOCATION_COSITED_EVEN; + VkFilter chromaFilter = VK_FILTER_NEAREST; + VkFormat target = VK_FORMAT_UNDEFINED; + VkSamplerYcbcrConversion conversion = VK_NULL_HANDLE; + VkSampler sampler = VK_NULL_HANDLE; + VkDescriptorSetLayout descriptorLayout = VK_NULL_HANDLE; + VkPipelineLayout pipelineLayout = VK_NULL_HANDLE; + VkRenderPass renderPass = VK_NULL_HANDLE; + VkPipeline pipeline = VK_NULL_HANDLE; + }; + Vector pipelines; + }; + + namespace { + // One texture's import of its YUV buffer (TextureResource::yuvSource): the image on the + // buffer's memory and a view through the conversion of `pipeline`. Destroyed by the + // resource's (deferred) Reset, so never under a submission still reading it. + struct WireYuvSource { + VkDevice device = VK_NULL_HANDLE; + VkImage image = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + VkImageView view = VK_NULL_HANDLE; + SizeT pipeline = 0; + MG_Remote::Server::SharedImages::Yuv::Hints hints; + ~WireYuvSource() { + if (device == VK_NULL_HANDLE) return; + if (view != VK_NULL_HANDLE) vkDestroyImageView(device, view, nullptr); + if (image != VK_NULL_HANDLE) vkDestroyImage(device, image, nullptr); + if (memory != VK_NULL_HANDLE) vkFreeMemory(device, memory, nullptr); + } + }; + + VkSamplerYcbcrModelConversion YcbcrModelFor(const MG_Remote::Server::SharedImages::Yuv::Hints& hints) { + namespace Yuv = MG_Remote::Server::SharedImages::Yuv; + switch (hints.EffectiveColorSpace()) { + case Yuv::kEglItuRec709: + return VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709; + case Yuv::kEglItuRec2020: + return VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_2020; + default: + return VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_601; + } + } + } // namespace + + void VulkanRenderer::DestroyWireYuvResources() { + if (!m_wireYuvResources) return; + auto& cache = *m_wireYuvResources; + for (const auto& entry : cache.pipelines) { + vkDestroyPipeline(m_device, entry.pipeline, nullptr); + vkDestroyRenderPass(m_device, entry.renderPass, nullptr); + vkDestroyPipelineLayout(m_device, entry.pipelineLayout, nullptr); + vkDestroyDescriptorSetLayout(m_device, entry.descriptorLayout, nullptr); + vkDestroySampler(m_device, entry.sampler, nullptr); + if (cache.destroyConversion != nullptr) cache.destroyConversion(m_device, entry.conversion, nullptr); + } + vkDestroyShaderModule(m_device, cache.vertex, nullptr); + vkDestroyShaderModule(m_device, cache.fragment, nullptr); + delete m_wireYuvResources; + m_wireYuvResources = nullptr; + } + + Bool VulkanRenderer::ConvertYuvSharedImage(VkTextureManager::TextureResource& resource) { +#if defined(__ANDROID__) + namespace SI = MG_Remote::Server::SharedImages; + if (resource.yuvConvertedFrame == m_yuvFrameSerial) return true; + const auto image = std::static_pointer_cast(resource.yuvOwner); + if (image == nullptr || image->Native == nullptr || resource.image == VK_NULL_HANDLE) return false; + if (!m_samplerYcbcrConversion) { + MGLOG_E_ONCE("Magma: YUV shared image %llu cannot be sampled: the device has no samplerYcbcrConversion", + static_cast(image->Id)); + return false; + } + // Whatever happens below, this frame does not try again. + resource.yuvConvertedFrame = m_yuvFrameSerial; + // THE CPU-COPY FALLBACK, for a buffer this server did not allocate (SharedImageRegistry.h). + String why; + if (image->IsForeign() && !SI::RefreshForeign(*image, why)) { + MGLOG_E_ONCE("Magma: foreign YUV image %llu could not be refreshed: %s", + static_cast(image->Id), why.c_str()); + } + const SI::Yuv::Hints hints = SI::GetYuvHints(*image); + + if (!m_wireYuvResources) { + m_wireYuvResources = new WireYuvResources; + auto& cache = *m_wireYuvResources; + cache.createConversion = reinterpret_cast( + vkGetDeviceProcAddr(m_device, "vkCreateSamplerYcbcrConversion")); + if (cache.createConversion == nullptr) + cache.createConversion = reinterpret_cast( + vkGetDeviceProcAddr(m_device, "vkCreateSamplerYcbcrConversionKHR")); + cache.destroyConversion = reinterpret_cast( + vkGetDeviceProcAddr(m_device, "vkDestroySamplerYcbcrConversion")); + if (cache.destroyConversion == nullptr) + cache.destroyConversion = reinterpret_cast( + vkGetDeviceProcAddr(m_device, "vkDestroySamplerYcbcrConversionKHR")); + const auto shader = [&](const Uint32* code, SizeT bytes, VkShaderModule& module) { + VkShaderModuleCreateInfo info{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + info.codeSize = bytes; + info.pCode = code; + VK_VERIFY(vkCreateShaderModule(m_device, &info, nullptr, &module)); + }; + shader(kWireYuvConvertVertexSpirv, sizeof(kWireYuvConvertVertexSpirv), cache.vertex); + shader(kWireYuvConvertFragmentSpirv, sizeof(kWireYuvConvertFragmentSpirv), cache.fragment); + } + auto& cache = *m_wireYuvResources; + if (cache.createConversion == nullptr || cache.destroyConversion == nullptr) { + MGLOG_E_ONCE("Magma: vkCreateSamplerYcbcrConversion is not available; YUV images cannot be sampled"); + return false; + } + + // ---- the texture's import of the buffer, and the pipeline its conversion needs ---- + auto* source = static_cast(resource.yuvSource.get()); + if (source != nullptr && !(source->hints == hints)) { + // Re-imported with other colour hints: the conversion (and so the view) changes. + auto retired = MakeUnique(); + retired->yuvSource = std::move(resource.yuvSource); + WireRetiredObjects objects{}; + objects.textures.push_back(Move(retired)); + RetireWireObjects(Move(objects)); + source = nullptr; + } + if (source == nullptr) { + WireAhbImport ctx; + if (!GetWireAhbImport(ctx)) return false; + auto* ahb = static_cast(image->Native); + const auto getProperties = + reinterpret_cast(ctx.getAhbProperties); + VkAndroidHardwareBufferFormatPropertiesANDROID format{ + VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_FORMAT_PROPERTIES_ANDROID}; + VkAndroidHardwareBufferPropertiesANDROID properties{ + VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID}; + properties.pNext = &format; + if (getProperties(m_device, ahb, &properties) != VK_SUCCESS) { + MGLOG_E_ONCE("Magma: YUV image %llu: vkGetAndroidHardwareBufferPropertiesANDROID failed", + static_cast(image->Id)); + return false; + } + // The conversion: the import's hints where it gave them, else what the buffer suggests. + WireYuvResources::Pipeline key; + key.externalFormat = format.format == VK_FORMAT_UNDEFINED ? format.externalFormat : 0; + key.format = format.format; + key.model = hints.ColorSpace != 0 ? YcbcrModelFor(hints) : format.suggestedYcbcrModel; + key.range = hints.Range != 0 ? (hints.FullRange() ? VK_SAMPLER_YCBCR_RANGE_ITU_FULL + : VK_SAMPLER_YCBCR_RANGE_ITU_NARROW) + : format.suggestedYcbcrRange; + key.components = format.samplerYcbcrConversionComponents; + const auto chroma = [&format](Uint32 siting, VkChromaLocation suggested) { + const Bool midpoint = siting == SI::Yuv::kEglYuvChromaSiting0_5; + const Bool cosited = siting == SI::Yuv::kEglYuvChromaSiting0; + if (midpoint && (format.formatFeatures & VK_FORMAT_FEATURE_MIDPOINT_CHROMA_SAMPLES_BIT)) + return VK_CHROMA_LOCATION_MIDPOINT; + if (cosited && (format.formatFeatures & VK_FORMAT_FEATURE_COSITED_CHROMA_SAMPLES_BIT)) + return VK_CHROMA_LOCATION_COSITED_EVEN; + return suggested; + }; + key.xOffset = chroma(hints.SitingX, format.suggestedXChromaOffset); + key.yOffset = chroma(hints.SitingY, format.suggestedYChromaOffset); + key.chromaFilter = + (format.formatFeatures & VK_FORMAT_FEATURE_SAMPLED_IMAGE_YCBCR_CONVERSION_LINEAR_FILTER_BIT) != 0 + ? VK_FILTER_LINEAR + : VK_FILTER_NEAREST; + key.target = resource.format; + SizeT index = cache.pipelines.size(); + for (SizeT i = 0; i < cache.pipelines.size(); ++i) { + const auto& p = cache.pipelines[i]; + if (p.externalFormat == key.externalFormat && p.format == key.format && p.model == key.model && + p.range == key.range && p.components.r == key.components.r && + p.components.g == key.components.g && p.components.b == key.components.b && + p.components.a == key.components.a && p.xOffset == key.xOffset && p.yOffset == key.yOffset && + p.chromaFilter == key.chromaFilter && p.target == key.target) { + index = i; + break; + } + } + if (index == cache.pipelines.size()) { + VkExternalFormatANDROID external{VK_STRUCTURE_TYPE_EXTERNAL_FORMAT_ANDROID}; + external.externalFormat = key.externalFormat; + VkSamplerYcbcrConversionCreateInfo conversion{VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_CREATE_INFO}; + conversion.pNext = key.externalFormat != 0 ? &external : nullptr; + conversion.format = key.format; + conversion.ycbcrModel = key.model; + conversion.ycbcrRange = key.range; + conversion.components = key.components; + conversion.xChromaOffset = key.xOffset; + conversion.yChromaOffset = key.yOffset; + conversion.chromaFilter = key.chromaFilter; + conversion.forceExplicitReconstruction = VK_FALSE; + if (cache.createConversion(m_device, &conversion, nullptr, &key.conversion) != VK_SUCCESS) { + MGLOG_E_ONCE("Magma: YUV image %llu: vkCreateSamplerYcbcrConversion failed", + static_cast(image->Id)); + return false; + } + VkSamplerYcbcrConversionInfo conversionInfo{VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_INFO}; + conversionInfo.conversion = key.conversion; + VkSamplerCreateInfo sampler{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; + sampler.pNext = &conversionInfo; + // Without separate reconstruction filters the sampler filters as the conversion does. + sampler.magFilter = sampler.minFilter = key.chromaFilter; + sampler.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST; + sampler.addressModeU = sampler.addressModeV = sampler.addressModeW = + VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + sampler.maxLod = 0.0f; + VK_VERIFY(vkCreateSampler(m_device, &sampler, nullptr, &key.sampler)); + const VkDescriptorSetLayoutBinding binding{0, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1, + VK_SHADER_STAGE_FRAGMENT_BIT, &key.sampler}; + VkDescriptorSetLayoutCreateInfo layout{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + layout.bindingCount = 1; + layout.pBindings = &binding; + VK_VERIFY(vkCreateDescriptorSetLayout(m_device, &layout, nullptr, &key.descriptorLayout)); + VkPipelineLayoutCreateInfo pipelineLayout{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + pipelineLayout.setLayoutCount = 1; + pipelineLayout.pSetLayouts = &key.descriptorLayout; + VK_VERIFY(vkCreatePipelineLayout(m_device, &pipelineLayout, nullptr, &key.pipelineLayout)); + // Every texel of the target is written: nothing to load. + VkAttachmentDescription attachment{}; + attachment.format = key.target; + attachment.samples = VK_SAMPLE_COUNT_1_BIT; + attachment.loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE; + attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + attachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + attachment.initialLayout = attachment.finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + const VkAttachmentReference color{0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL}; + VkSubpassDescription subpass{}; + subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; + subpass.colorAttachmentCount = 1; + subpass.pColorAttachments = &color; + VkRenderPassCreateInfo pass{VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO}; + pass.attachmentCount = 1; + pass.pAttachments = &attachment; + pass.subpassCount = 1; + pass.pSubpasses = &subpass; + VK_VERIFY(vkCreateRenderPass(m_device, &pass, nullptr, &key.renderPass)); + VkPipelineShaderStageCreateInfo stages[2]{}; + for (auto& stage : stages) { + stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + stage.pName = "main"; + } + stages[0].stage = VK_SHADER_STAGE_VERTEX_BIT; + stages[0].module = cache.vertex; + stages[1].stage = VK_SHADER_STAGE_FRAGMENT_BIT; + stages[1].module = cache.fragment; + const VkPipelineVertexInputStateCreateInfo vertex{ + VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO}; + VkPipelineInputAssemblyStateCreateInfo assembly{ + VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO}; + assembly.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + VkPipelineViewportStateCreateInfo viewport{VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO}; + viewport.viewportCount = viewport.scissorCount = 1; + VkPipelineRasterizationStateCreateInfo raster{ + VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO}; + raster.polygonMode = VK_POLYGON_MODE_FILL; + raster.cullMode = VK_CULL_MODE_NONE; + raster.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + raster.lineWidth = 1.0f; + VkPipelineMultisampleStateCreateInfo samples{ + VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO}; + samples.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; + VkPipelineColorBlendAttachmentState colorMask{}; + colorMask.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | + VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT; + VkPipelineColorBlendStateCreateInfo blend{VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO}; + blend.attachmentCount = 1; + blend.pAttachments = &colorMask; + const VkDynamicState dynamicStates[]{VK_DYNAMIC_STATE_VIEWPORT, VK_DYNAMIC_STATE_SCISSOR}; + VkPipelineDynamicStateCreateInfo dynamic{VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO}; + dynamic.dynamicStateCount = 2; + dynamic.pDynamicStates = dynamicStates; + VkGraphicsPipelineCreateInfo pipeline{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO}; + pipeline.stageCount = 2; + pipeline.pStages = stages; + pipeline.pVertexInputState = &vertex; + pipeline.pInputAssemblyState = &assembly; + pipeline.pViewportState = &viewport; + pipeline.pRasterizationState = &raster; + pipeline.pMultisampleState = &samples; + pipeline.pColorBlendState = &blend; + pipeline.pDynamicState = &dynamic; + pipeline.layout = key.pipelineLayout; + pipeline.renderPass = key.renderPass; + VK_VERIFY(vkCreateGraphicsPipelines(m_device, VK_NULL_HANDLE, 1, &pipeline, nullptr, &key.pipeline)); + cache.pipelines.push_back(key); + MGLOG_I("Magma: YUV conversion pipeline %zu: %s format %llu, model %d, range %d, chroma %d/%d, " + "filter %d", + index, key.externalFormat != 0 ? "external" : "Vulkan", + static_cast(key.externalFormat != 0 ? key.externalFormat + : static_cast(key.format)), + static_cast(key.model), static_cast(key.range), static_cast(key.xOffset), + static_cast(key.yOffset), static_cast(key.chromaFilter)); + } + const auto& chosen = cache.pipelines[index]; + + // The buffer, imported with its external format, sampled only. + auto created = std::make_shared(); + created->device = m_device; + created->pipeline = index; + created->hints = hints; + VkExternalFormatANDROID external{VK_STRUCTURE_TYPE_EXTERNAL_FORMAT_ANDROID}; + external.externalFormat = chosen.externalFormat; + VkExternalMemoryImageCreateInfo externalMemory{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO}; + externalMemory.pNext = chosen.externalFormat != 0 ? &external : nullptr; + externalMemory.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID; + VkImageCreateInfo imageInfo{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + imageInfo.pNext = &externalMemory; + imageInfo.imageType = VK_IMAGE_TYPE_2D; + imageInfo.format = chosen.format; + imageInfo.extent = {image->Width, image->Height, 1}; + imageInfo.mipLevels = 1; + imageInfo.arrayLayers = 1; + imageInfo.samples = VK_SAMPLE_COUNT_1_BIT; + imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imageInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT; + imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + if (vkCreateImage(m_device, &imageInfo, nullptr, &created->image) != VK_SUCCESS) { + created->image = VK_NULL_HANDLE; + MGLOG_E_ONCE("Magma: YUV image %llu: vkCreateImage(external format) failed", + static_cast(image->Id)); + return false; + } + Int32 type = -1; + for (Uint32 i = 0; i < ctx.memory.memoryTypeCount && type < 0; ++i) { + if ((properties.memoryTypeBits & (1u << i)) != 0) type = static_cast(i); + } + VkMemoryDedicatedAllocateInfo dedicated{VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO}; + dedicated.image = created->image; + VkImportAndroidHardwareBufferInfoANDROID import{ + VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID}; + import.pNext = &dedicated; + import.buffer = ahb; + VkMemoryAllocateInfo allocate{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocate.pNext = &import; + allocate.allocationSize = properties.allocationSize; + allocate.memoryTypeIndex = static_cast(type < 0 ? 0 : type); + if (type < 0 || vkAllocateMemory(m_device, &allocate, nullptr, &created->memory) != VK_SUCCESS || + vkBindImageMemory(m_device, created->image, created->memory, 0) != VK_SUCCESS) { + MGLOG_E_ONCE("Magma: YUV image %llu: its buffer could not be imported", + static_cast(image->Id)); + return false; + } + VkSamplerYcbcrConversionInfo conversionInfo{VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_INFO}; + conversionInfo.conversion = chosen.conversion; + VkImageViewCreateInfo view{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + view.pNext = &conversionInfo; + view.image = created->image; + view.viewType = VK_IMAGE_VIEW_TYPE_2D; + view.format = chosen.format; + view.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; + view.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + if (vkCreateImageView(m_device, &view, nullptr, &created->view) != VK_SUCCESS) { + created->view = VK_NULL_HANDLE; + MGLOG_E_ONCE("Magma: YUV image %llu: its view could not be created", + static_cast(image->Id)); + return false; + } + MGLOG_I("Magma: YUV image %llu (%ux%u%s) sampled through conversion pipeline %zu", + static_cast(image->Id), image->Width, image->Height, + image->IsForeign() ? ", foreign: CPU-copied" : ", zero-copy", index); + resource.yuvSource = created; + source = created.get(); + } + const auto& chosen = cache.pipelines[source->pipeline]; + + // ---- the pass ---- + auto& frame = m_frameContext.GetCurrent(); + if (!frame.isCommandRecording) m_frameContext.BeginCommandRecording(); + if (VkRenderPassManager::GetActiveRenderPass()) VkRenderPassManager::EndRenderPass(frame.commandBuffer); + m_textureManager->FlushPendingUploads(); + const Uint32 family = static_cast(m_physicalDevice.queueFamilies.graphicsFamily); + // The buffer's content is its producer's (the CPU's copy, or the allocator's writer): it is + // acquired from the foreign family for the read and handed back after it. + VkImageMemoryBarrier acquire{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + acquire.srcAccessMask = 0; + acquire.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + acquire.oldLayout = VK_IMAGE_LAYOUT_GENERAL; + acquire.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + acquire.srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT; + acquire.dstQueueFamilyIndex = family; + acquire.image = source->image; + acquire.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + vkCmdPipelineBarrier(frame.commandBuffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, nullptr, 0, nullptr, 1, &acquire); + + WireImage target{}; + target.image = resource.image; + target.format = resource.format; + target.viewType = VK_IMAGE_VIEW_TYPE_2D; + target.extent = resource.extent; + target.layout = resource.layout; + target.trackedLayout = &resource.layout; + target.aspect = VK_IMAGE_ASPECT_COLOR_BIT; + target.usageFlags = resource.usageFlags; + target.levels = resource.mipLevels; + TransitionWireImage(target, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); + + VkImageViewCreateInfo targetViewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + targetViewInfo.image = resource.image; + targetViewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + targetViewInfo.format = resource.format; + targetViewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + VkImageView targetView = VK_NULL_HANDLE; + VK_VERIFY(vkCreateImageView(m_device, &targetViewInfo, nullptr, &targetView)); + VkFramebufferCreateInfo framebufferInfo{VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO}; + framebufferInfo.renderPass = chosen.renderPass; + framebufferInfo.attachmentCount = 1; + framebufferInfo.pAttachments = &targetView; + framebufferInfo.width = resource.extent.width; + framebufferInfo.height = resource.extent.height; + framebufferInfo.layers = 1; + VkFramebuffer framebuffer = VK_NULL_HANDLE; + VK_VERIFY(vkCreateFramebuffer(m_device, &framebufferInfo, nullptr, &framebuffer)); + // A multi-planar image may take more than one descriptor of its binding. + VkDescriptorPool descriptorPool = VK_NULL_HANDLE; + VkDescriptorSet descriptorSet = VK_NULL_HANDLE; + const VkDescriptorPoolSize poolSize{VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 3}; + VkDescriptorPoolCreateInfo pool{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + pool.maxSets = 1; + pool.poolSizeCount = 1; + pool.pPoolSizes = &poolSize; + VK_VERIFY(vkCreateDescriptorPool(m_device, &pool, nullptr, &descriptorPool)); + VkDescriptorSetAllocateInfo alloc{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + alloc.descriptorPool = descriptorPool; + alloc.descriptorSetCount = 1; + alloc.pSetLayouts = &chosen.descriptorLayout; + VK_VERIFY(vkAllocateDescriptorSets(m_device, &alloc, &descriptorSet)); + const VkDescriptorImageInfo sampled{chosen.sampler, source->view, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL}; + VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; + write.dstSet = descriptorSet; + write.dstBinding = 0; + write.descriptorCount = 1; + write.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; + write.pImageInfo = &sampled; + vkUpdateDescriptorSets(m_device, 1, &write, 0, nullptr); + + VkRenderPassBeginInfo begin{VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO}; + begin.renderPass = chosen.renderPass; + begin.framebuffer = framebuffer; + begin.renderArea = {{0, 0}, resource.extent}; + vkCmdBeginRenderPass(frame.commandBuffer, &begin, VK_SUBPASS_CONTENTS_INLINE); + vkCmdBindPipeline(frame.commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, chosen.pipeline); + const VkViewport viewport{0.0f, 0.0f, static_cast(resource.extent.width), + static_cast(resource.extent.height), 0.0f, 1.0f}; + const VkRect2D scissor{{0, 0}, resource.extent}; + vkCmdSetViewport(frame.commandBuffer, 0, 1, &viewport); + vkCmdSetScissor(frame.commandBuffer, 0, 1, &scissor); + vkCmdBindDescriptorSets(frame.commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, chosen.pipelineLayout, 0, 1, + &descriptorSet, 0, nullptr); + vkCmdDraw(frame.commandBuffer, 3, 1, 0, 0); + vkCmdEndRenderPass(frame.commandBuffer); + + VkImageMemoryBarrier release = acquire; + release.srcAccessMask = VK_ACCESS_SHADER_READ_BIT; + release.dstAccessMask = 0; + release.oldLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + release.newLayout = VK_IMAGE_LAYOUT_GENERAL; + release.srcQueueFamilyIndex = family; + release.dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT; + vkCmdPipelineBarrier(frame.commandBuffer, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, + VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, nullptr, 0, nullptr, 1, &release); + // GENERAL, the layout every wire descriptor samples in, with the pass's writes made visible. + target.layout = resource.layout; + TransitionWireImage(target, VK_IMAGE_LAYOUT_GENERAL); + ResetDynamicStateShadow(); + InvalidatePipelineMemo(); + if (m_uniformManager) m_uniformManager->OnCommandBufferBoundary(); + m_textureManager->StampResourceRecordingUse(resource); + m_textureManager->StampResourceRecordingWrite(resource); + WireRetiredObjects retired{}; + retired.framebuffers.push_back(framebuffer); + retired.imageViews = {targetView}; + retired.descriptorPools.push_back(descriptorPool); + RetireWireObjects(Move(retired)); + // The frame's fence orders the next refresh of a foreign buffer after this read. + m_sharedImageReads.NoteRead(image); + return true; +#else + (void)resource; + return false; +#endif + } +#endif // MOBILEGL_BUILD_DISAGGREGATED diff --git a/MobileGL/MG_Gbm/MobileGLGbm.cpp b/MobileGL/MG_Gbm/MobileGLGbm.cpp index 871aeada..537f18ec 100644 --- a/MobileGL/MG_Gbm/MobileGLGbm.cpp +++ b/MobileGL/MG_Gbm/MobileGLGbm.cpp @@ -17,9 +17,14 @@ // process's MobileGL client session. Whatever node the caller holds is therefore equally good, // and none is vendor-specific. // -// THE LAYOUT IS THE ALLOCATOR'S. Buffers report DRM_FORMAT_MOD_INVALID and a single plane; a -// request that needs a known layout (LINEAR, or a modifier list without INVALID) is refused -// rather than answered with a buffer whose layout is not what was asked for. +// THE LAYOUT IS THE ALLOCATOR'S. Buffers report DRM_FORMAT_MOD_INVALID and a single plane (two for +// the YUV formats, both in the one descriptor); a request that needs a known layout (LINEAR, or a +// modifier list without INVALID) is refused rather than answered with a buffer whose layout is not +// what was asked for. +// +// YUV (NV12, P010) buffers exist when the server's platform allocates them (the shared-image ABI's +// format list says so). They are sampled, never rendered to: a GBM_BO_USE_RENDERING request for one +// is refused. // // Buffer operations reach the server through libMobileGL.so's shared-image C ABI // (MG_Impl/SharedImageApi.h), resolved with dlsym on first use - preferably from the instance the @@ -40,6 +45,7 @@ #include #include +#include #include #define MOBILEGL_GBM_EXPORT extern "C" __attribute__((visibility("default"))) @@ -65,6 +71,10 @@ namespace { int fd; // owned; every get_fd hands out a duplicate uint32_t offset; uint64_t modifier; + // A YUV buffer's plane 1, in the same descriptor. + uint32_t planes; + uint32_t plane1Stride; + uint32_t plane1Offset; }; // ---- libMobileGL's shared-image ABI ----------------------------------------------------- @@ -74,6 +84,9 @@ namespace { PFN_mobilegl_shared_image_import import = nullptr; PFN_mobilegl_shared_image_release release = nullptr; PFN_mobilegl_server_available serverAvailable = nullptr; // optional (older libraries lack it) + // ABI 2 (YUV); null on an older library. + PFN_mobilegl_shared_image_formats formats = nullptr; + PFN_mobilegl_shared_image_import_planes importPlanes = nullptr; }; // The library the process already has (by its soname: glvnd loads it by path, which records @@ -107,6 +120,12 @@ namespace { std::fprintf(stderr, "mobilegl_gbm: this libMobileGL.so has no shared-image ABI\n"); return; } + if (version() >= 2) { + api.formats = reinterpret_cast( + dlsym(lib, "mobilegl_shared_image_formats")); + api.importPlanes = reinterpret_cast( + dlsym(lib, "mobilegl_shared_image_import_planes")); + } resolved = true; }); return resolved ? &api : nullptr; @@ -125,11 +144,41 @@ namespace { // What a shared image can be. The ARGB/XRGB orders are stored like the ABGR/XBGR ones: the // layout is the server's (modifier INVALID, no map is offered), so no reader can tell, and // every GL that touches the image addresses its channels logically. + bool FormatIsYuv(uint32_t format) { + return format == MOBILEGL_SHARED_IMAGE_FOURCC_NV12 || format == MOBILEGL_SHARED_IMAGE_FOURCC_P010; + } + + // Whether the server holds YUV images: the library's format list, asked once (it takes the + // process's session, which the first buffer would anyway). + bool YuvFormatAvailable(uint32_t format) { + static std::once_flag once; + static bool nv12 = false, p010 = false; + std::call_once(once, [] { + const SharedImageApi* api = Api(); + if (api == nullptr || api->formats == nullptr || api->importPlanes == nullptr) return; + uint32_t formats[16] = {}; + const int total = api->formats(formats, 16); + for (int i = 0; i < total && i < 16; ++i) { + nv12 |= formats[i] == MOBILEGL_SHARED_IMAGE_FOURCC_NV12; + p010 |= formats[i] == MOBILEGL_SHARED_IMAGE_FOURCC_P010; + } + }); + return format == MOBILEGL_SHARED_IMAGE_FOURCC_NV12 ? nv12 : format == MOBILEGL_SHARED_IMAGE_FOURCC_P010 && p010; + } + bool FormatSupported(uint32_t format) { + if (FormatIsYuv(format)) return YuvFormatAvailable(format); return format == MOBILEGL_SHARED_IMAGE_FOURCC_ABGR8888 || format == MOBILEGL_SHARED_IMAGE_FOURCC_XBGR8888 || format == MOBILEGL_SHARED_IMAGE_FOURCC_ARGB8888 || format == MOBILEGL_SHARED_IMAGE_FOURCC_XRGB8888; } + uint32_t PlaneCount(uint32_t format) { return FormatIsYuv(format) ? 2 : 1; } + + // A YUV buffer is sampled only. + uint32_t RefusedUsage(uint32_t format) { + return kRefusedUsage | (FormatIsYuv(format) ? static_cast(GBM_BO_USE_RENDERING) : 0u); + } + bool ModifierAcceptable(uint64_t modifier) { return modifier == kModifierInvalid || modifier == kModifierLinear; } // There is no GEM handle behind a shared image; callers that only use the handle as a key @@ -142,7 +191,8 @@ namespace { } gbm_bo* NewBo(gbm_device* gbm, uint64_t imageId, int fd, uint32_t width, uint32_t height, uint32_t format, - uint32_t stride, uint32_t offset, uint64_t modifier) { + uint32_t stride, uint32_t offset, uint64_t modifier, uint32_t plane1Stride = 0, + uint32_t plane1Offset = 0) { auto* bo = new (std::nothrow) Bo{}; if (bo == nullptr) { errno = ENOMEM; @@ -158,6 +208,9 @@ namespace { bo->fd = fd; bo->offset = offset; bo->modifier = modifier; + bo->planes = PlaneCount(format); + bo->plane1Stride = plane1Stride; + bo->plane1Offset = plane1Offset; return &bo->base; } @@ -168,18 +221,20 @@ namespace { void DeviceDestroy(gbm_device* gbm) { delete reinterpret_cast(gbm); } int IsFormatSupported(gbm_device*, uint32_t format, uint32_t usage) { - return FormatSupported(Canonical(format)) && (usage & kRefusedUsage) == 0 ? 1 : 0; + format = Canonical(format); + return FormatSupported(format) && (usage & RefusedUsage(format)) == 0 ? 1 : 0; } int GetFormatModifierPlaneCount(gbm_device*, uint32_t format, uint64_t modifier) { - if (!FormatSupported(Canonical(format)) || !ModifierAcceptable(modifier)) return -1; - return 1; + format = Canonical(format); + if (!FormatSupported(format) || !ModifierAcceptable(modifier)) return -1; + return static_cast(PlaneCount(format)); } gbm_bo* BoCreate(gbm_device* gbm, uint32_t width, uint32_t height, uint32_t format, uint32_t usage, const uint64_t* modifiers, const unsigned int count) { format = Canonical(format); - if (!FormatSupported(format) || (usage & kRefusedUsage) != 0) { + if (!FormatSupported(format) || (usage & RefusedUsage(format)) != 0) { errno = EINVAL; return nullptr; } @@ -207,7 +262,7 @@ namespace { return nullptr; } gbm_bo* bo = NewBo(gbm, image.id, image.fd, image.width, image.height, image.fourcc, image.stride, - image.offset, image.modifier); + image.offset, image.modifier, image.plane1_stride, image.plane1_offset); if (bo == nullptr) { api->release(image.id); ::close(image.fd); @@ -215,6 +270,13 @@ namespace { return bo; } + // Whether two descriptors name one dma-buf: a YUV buffer's planes must be in one. + bool SameBuffer(int a, int b) { + if (a == b) return true; + struct stat sa {}, sb {}; + return ::fstat(a, &sa) == 0 && ::fstat(b, &sb) == 0 && sa.st_dev == sb.st_dev && sa.st_ino == sb.st_ino; + } + gbm_bo* BoImport(gbm_device* gbm, uint32_t type, void* buffer, uint32_t usage) { (void)usage; if (buffer == nullptr) { @@ -223,6 +285,7 @@ namespace { } int fd = -1; uint32_t width = 0, height = 0, format = 0, stride = 0, offset = 0; + uint32_t plane1Stride = 0, plane1Offset = 0; uint64_t modifier = kModifierInvalid; switch (type) { case GBM_BO_IMPORT_FD: { @@ -232,11 +295,19 @@ namespace { height = data->height; format = data->format; stride = data->stride; + // One fd and one stride cannot say where a second plane is. + if (FormatIsYuv(Canonical(format))) { + errno = EINVAL; + return nullptr; + } break; } case GBM_BO_IMPORT_FD_MODIFIER: { const auto* data = static_cast(buffer); - if (data->num_fds != 1 || !ModifierAcceptable(data->modifier)) { + const bool yuv = FormatIsYuv(Canonical(data->format)); + const unsigned int planes = yuv ? 2u : 1u; + if (data->num_fds < 1 || data->num_fds > planes || !ModifierAcceptable(data->modifier) || + (data->num_fds == 2 && !SameBuffer(data->fds[0], data->fds[1]))) { errno = EINVAL; return nullptr; } @@ -246,6 +317,10 @@ namespace { format = data->format; stride = static_cast(data->strides[0]); offset = static_cast(data->offsets[0]); + if (yuv) { + plane1Stride = static_cast(data->strides[1]); + plane1Offset = static_cast(data->offsets[1]); + } modifier = data->modifier; break; } @@ -269,13 +344,26 @@ namespace { const int owned = ::fcntl(fd, F_DUPFD_CLOEXEC, 0); if (owned < 0) return nullptr; uint64_t id = 0; - const int rc = api->import(owned, width, height, format, &id); + int rc = 0; + if (FormatIsYuv(format)) { + mobilegl_shared_image_planes planes{}; + planes.struct_size = sizeof(planes); + planes.plane_count = 2; + planes.offset[0] = offset; + planes.offset[1] = plane1Offset; + planes.pitch[0] = stride; + planes.pitch[1] = plane1Stride; + planes.modifier = modifier; + rc = api->importPlanes(owned, width, height, format, &planes, &id); + } else { + rc = api->import(owned, width, height, format, &id); + } if (rc != 0) { ::close(owned); errno = -rc; return nullptr; } - gbm_bo* bo = NewBo(gbm, id, owned, width, height, format, stride, offset, modifier); + gbm_bo* bo = NewBo(gbm, id, owned, width, height, format, stride, offset, modifier, plane1Stride, plane1Offset); if (bo == nullptr) { api->release(id); ::close(owned); @@ -299,11 +387,14 @@ namespace { int BoGetFd(gbm_bo* bo) { return ::fcntl(AsBo(bo)->fd, F_DUPFD_CLOEXEC, 0); } - int BoGetPlanes(gbm_bo*) { return 1; } + int BoGetPlanes(gbm_bo* bo) { return static_cast(AsBo(bo)->planes); } + bool ValidPlane(gbm_bo* bo, int plane) { return plane >= 0 && static_cast(plane) < AsBo(bo)->planes; } + + // Every plane is in the one buffer, so every plane has its handle. gbm_bo_handle BoGetHandle(gbm_bo* bo, int plane) { gbm_bo_handle handle{}; - if (plane != 0) { + if (!ValidPlane(bo, plane)) { errno = EINVAL; handle.s32 = -1; return handle; @@ -312,16 +403,22 @@ namespace { } int BoGetPlaneFd(gbm_bo* bo, int plane) { - if (plane != 0) { + if (!ValidPlane(bo, plane)) { errno = EINVAL; return -1; } return BoGetFd(bo); } - uint32_t BoGetStride(gbm_bo* bo, int plane) { return plane == 0 ? bo->v0.stride : 0; } + uint32_t BoGetStride(gbm_bo* bo, int plane) { + if (plane == 0) return bo->v0.stride; + return plane == 1 && AsBo(bo)->planes == 2 ? AsBo(bo)->plane1Stride : 0; + } - uint32_t BoGetOffset(gbm_bo* bo, int plane) { return plane == 0 ? AsBo(bo)->offset : 0; } + uint32_t BoGetOffset(gbm_bo* bo, int plane) { + if (plane == 0) return AsBo(bo)->offset; + return plane == 1 && AsBo(bo)->planes == 2 ? AsBo(bo)->plane1Offset : 0; + } uint64_t BoGetModifier(gbm_bo* bo) { return AsBo(bo)->modifier; } diff --git a/MobileGL/MG_Impl/EGLImpl/EGLImpl.cpp b/MobileGL/MG_Impl/EGLImpl/EGLImpl.cpp index 9ca72f2b..63eb4e6a 100644 --- a/MobileGL/MG_Impl/EGLImpl/EGLImpl.cpp +++ b/MobileGL/MG_Impl/EGLImpl/EGLImpl.cpp @@ -38,6 +38,10 @@ #include #include #include +#if !defined(_WIN32) +#include +#include +#endif #if defined(__linux__) && !defined(__ANDROID__) #include #endif @@ -1608,6 +1612,38 @@ namespace MobileGL::MG_Impl::EGLImpl { #endif } + // Whether the server can hold YUV images (its platform allocates the formats): asked once, by + // allocating and dropping a small NV12 image, and only when a YUV question first comes up. + Bool SharedImageYuvAvailable() { + if (!SharedImagesAvailable()) return false; + static std::atomic answer{-1}; + const Int known = answer.load(std::memory_order_acquire); + if (known >= 0) return known != 0; + auto* backendObject = MG_Backend::pActiveBackendObject.get(); + if (backendObject == nullptr) return false; + MG_Backend::SharedImageExport probe{}; + const Bool allocated = backendObject->AllocateSharedImage( + 16, 16, MG_State::EGLState::EGLContext::kDrmFourccNv12, &probe); +#if !defined(_WIN32) + if (probe.Fd >= 0) ::close(probe.Fd); +#endif + if (allocated && probe.Id != 0) (void)backendObject->ReleaseSharedImage(probe.Id); + answer.store(allocated ? 1 : 0, std::memory_order_release); + MGLOG_I("Shared images: the server %s YUV (NV12) images", allocated ? "holds" : "has no"); + return allocated; + } + + // Whether two descriptors name one dma-buf (a YUV import's planes must be in one buffer). + Bool SameDmaBuf(int a, int b) { + if (a == b) return true; +#if defined(_WIN32) + return false; +#else + struct stat sa {}, sb {}; + return ::fstat(a, &sa) == 0 && ::fstat(b, &sb) == 0 && sa.st_dev == sb.st_dev && sa.st_ino == sb.st_ino; +#endif + } + Bool LookupSharedImage(EGLImage image, Uint64* id, EGLint* width, EGLint* height) { auto* state = GetState(); MG_State::EGLState::EGLContext::SharedImageInfo info; @@ -1673,10 +1709,41 @@ namespace MobileGL::MG_Impl::EGLImpl { } auto* backendObject = GetBackendObject(state); Uint64 sharedImageId = 0; - if (!backendObject || - !backendObject->ImportSharedImage(request.Fd, static_cast(request.Width), - static_cast(request.Height), request.Fourcc, &sharedImageId) || - sharedImageId == 0) { + Bool imported = false; + if (MG_State::EGLState::EGLContext::IsDmaBufFourccYuv(request.Fourcc)) { + // YUV: both planes in one buffer (what video decoders and this server's allocator hand + // out), sampled through GL_OES_EGL_image_external. Planes in two buffers are refused. + if (!SharedImageYuvAvailable() || !SameDmaBuf(request.Fd, request.Plane1Fd)) { + MGLOG_I_ONCE("eglCreateImage: a %dx%d YUV dma-buf cannot be imported (%s); refused (EGL_BAD_MATCH)", + request.Width, request.Height, + SharedImageYuvAvailable() ? "its planes are in two buffers" + : "the server has no YUV images"); + state->SetError(EGL_BAD_MATCH); + return EGL_NO_IMAGE; + } + MG_Backend::SharedImageImportLayout layout; + layout.PlaneCount = 2; + layout.Offset[0] = static_cast(request.Offset); + layout.Offset[1] = static_cast(request.Plane1Offset); + layout.Pitch[0] = static_cast(request.Pitch); + layout.Pitch[1] = static_cast(request.Plane1Pitch); + layout.ColorSpace = request.ColorSpace; + layout.Range = request.Range; + layout.SitingX = request.SitingX; + layout.SitingY = request.SitingY; + layout.HasModifier = request.HasModifier; + layout.Modifier = request.Modifier; + imported = backendObject != nullptr && + backendObject->ImportSharedImagePlanes(request.Fd, static_cast(request.Width), + static_cast(request.Height), request.Fourcc, + layout, &sharedImageId); + } else { + imported = backendObject != nullptr && + backendObject->ImportSharedImage(request.Fd, static_cast(request.Width), + static_cast(request.Height), request.Fourcc, + &sharedImageId); + } + if (!imported || sharedImageId == 0) { MGLOG_I_ONCE("eglCreateImage: a %dx%d dma-buf is not one of the server's shared images; refused " "(EGL_BAD_MATCH)", request.Width, request.Height); @@ -1731,7 +1798,7 @@ namespace MobileGL::MG_Impl::EGLImpl { if (!state) { return EGL_FALSE; } - return state->QueryDmaBufFormats(dpy, SharedImagesAvailable(), max_formats, formats, num_formats) + return state->QueryDmaBufFormats(dpy, SharedImagesAvailable(), SharedImageYuvAvailable(), max_formats, formats, num_formats) ? EGL_TRUE : EGL_FALSE; } @@ -1743,7 +1810,7 @@ namespace MobileGL::MG_Impl::EGLImpl { if (!state) { return EGL_FALSE; } - return state->QueryDmaBufModifiers(dpy, SharedImagesAvailable(), format, max_modifiers, + return state->QueryDmaBufModifiers(dpy, SharedImagesAvailable(), SharedImageYuvAvailable(), format, max_modifiers, reinterpret_cast(modifiers), external_only, num_modifiers) ? EGL_TRUE : EGL_FALSE; diff --git a/MobileGL/MG_Impl/EGLImpl/EGLImpl.h b/MobileGL/MG_Impl/EGLImpl/EGLImpl.h index 71a103b4..400cbc7f 100644 --- a/MobileGL/MG_Impl/EGLImpl/EGLImpl.h +++ b/MobileGL/MG_Impl/EGLImpl/EGLImpl.h @@ -68,6 +68,9 @@ namespace MobileGL::MG_Impl::EGLImpl { // a split client's does, a monolith's does not. Decides the dma-buf EGL extensions, the // Wayland linux-dmabuf presentation and GL_OES_EGL_image. Bool SharedImagesAvailable(); + // Whether the server also holds YUV (NV12/P010) images: decides the YUV dma-buf formats and + // GL_OES_EGL_image_external. One round trip the first time it is asked, cached after. + Bool SharedImageYuvAvailable(); // Under a dispatching libEGL the dispatcher, not this library, owns eglBindAPI's state: it does // not forward a bind of the API it already reports (EGL_OPENGL_ES_API, EGL's default), so a // context would otherwise be created for whatever this library assumed. The glvnd vendor diff --git a/MobileGL/MG_Impl/GLImpl/Getter/GL_Getter.cpp b/MobileGL/MG_Impl/GLImpl/Getter/GL_Getter.cpp index 4bf33f79..f72b3476 100644 --- a/MobileGL/MG_Impl/GLImpl/Getter/GL_Getter.cpp +++ b/MobileGL/MG_Impl/GLImpl/Getter/GL_Getter.cpp @@ -434,6 +434,7 @@ namespace MobileGL::MG_Impl::GLImpl { case GL_TEXTURE_BINDING_CUBE_MAP: outTarget = TextureTarget::TextureCubeMap; return true; case GL_TEXTURE_BINDING_CUBE_MAP_ARRAY: outTarget = TextureTarget::TextureCubeMapArray; return true; case GL_TEXTURE_BINDING_RECTANGLE: outTarget = TextureTarget::TextureRectangle; return true; + case GL_TEXTURE_BINDING_EXTERNAL_OES: outTarget = TextureTarget::External; return true; default: return false; } } @@ -606,23 +607,35 @@ namespace MobileGL::MG_Impl::GLImpl { Bool EsIdentityActive(); const Vector& ClientSideExtensions() { - static const Vector withImages = {"GL_OES_EGL_image", "GL_KHR_robustness", - "GL_ARB_robustness", "GL_EXT_robustness"}; - static const Vector withoutImages = {"GL_KHR_robustness", "GL_ARB_robustness", - "GL_EXT_robustness"}; - // Under the ES identity (EsIdentityActive) the ES names of what desktop GL has in core: - // BGRA textures, render targets and read-backs. GL_BGRA_EXT / GL_BGRA8_EXT as internal - // formats are taken as RGBA storage (TextureEnumConverter), the byte order being the - // pixel transfer format's, as on desktop GL. - static const Vector esWithImages = { - "GL_OES_EGL_image", "GL_KHR_robustness", "GL_ARB_robustness", - "GL_EXT_robustness", "GL_EXT_texture_format_BGRA8888", "GL_EXT_read_format_bgra"}; - static const Vector esWithoutImages = {"GL_KHR_robustness", "GL_ARB_robustness", - "GL_EXT_robustness", "GL_EXT_texture_format_BGRA8888", - "GL_EXT_read_format_bgra"}; + // GL_OES_EGL_image_external(_essl3) only where the server samples YUV images: an + // external texture is a 2D texture holding the image converted to RGBA, which needs + // the server to hold the YUV image in the first place (EGLImpl::SharedImageYuvAvailable). + static const auto build = [](Bool images, Bool external, Bool es) { + Vector list; + if (images) list.push_back("GL_OES_EGL_image"); + if (images && external) { + list.push_back("GL_OES_EGL_image_external"); + list.push_back("GL_OES_EGL_image_external_essl3"); + } + for (const char* name : {"GL_KHR_robustness", "GL_ARB_robustness", "GL_EXT_robustness"}) + list.push_back(name); + // Under the ES identity (EsIdentityActive) the ES names of what desktop GL has in core: + // BGRA textures, render targets and read-backs. GL_BGRA_EXT / GL_BGRA8_EXT as internal + // formats are taken as RGBA storage (TextureEnumConverter), the byte order being the + // pixel transfer format's, as on desktop GL. + if (es) { + list.push_back("GL_EXT_texture_format_BGRA8888"); + list.push_back("GL_EXT_read_format_bgra"); + } + return list; + }; + static const Vector lists[8] = { + build(false, false, false), build(false, false, true), build(false, true, false), + build(false, true, true), build(true, false, false), build(true, false, true), + build(true, true, false), build(true, true, true)}; const Bool images = MG_Impl::EGLImpl::SharedImagesAvailable(); - if (EsIdentityActive()) return images ? esWithImages : esWithoutImages; - return images ? withImages : withoutImages; + const Bool external = images && MG_Impl::EGLImpl::SharedImageYuvAvailable(); + return lists[(images ? 4 : 0) + (external ? 2 : 0) + (EsIdentityActive() ? 1 : 0)]; } // Robust buffer access as GL 4.5 / KHR_robust_buffer_access_behavior define it: the backend diff --git a/MobileGL/MG_Impl/GLImpl/Texture/GL_Texture.cpp b/MobileGL/MG_Impl/GLImpl/Texture/GL_Texture.cpp index 669e8ea5..b5aea9ee 100644 --- a/MobileGL/MG_Impl/GLImpl/Texture/GL_Texture.cpp +++ b/MobileGL/MG_Impl/GLImpl/Texture/GL_Texture.cpp @@ -1194,6 +1194,25 @@ namespace MobileGL::MG_Impl::GLImpl { return false; } + // GL_OES_EGL_image_external: base level 0, CLAMP_TO_EDGE only, NEAREST or LINEAR only. + if (target == TextureTarget::External) { + if (pname == GL_TEXTURE_BASE_LEVEL && param != 0) { + MG_State::pGLContext->RecordError( + ErrorCode::InvalidOperation, + MakeUnique("MG_Impl/GLImpl", caller, "External texture base level must be zero.")); + return false; + } + if (((pname == GL_TEXTURE_WRAP_S || pname == GL_TEXTURE_WRAP_T || pname == GL_TEXTURE_WRAP_R) && + param != GL_CLAMP_TO_EDGE) || + (pname == GL_TEXTURE_MIN_FILTER && param != GL_NEAREST && param != GL_LINEAR)) { + MG_State::pGLContext->RecordError( + ErrorCode::InvalidEnum, + MakeUnique("MG_Impl/GLImpl", caller, + "External textures clamp to edge and filter without mipmaps.")); + return false; + } + } + if (target == TextureTarget::TextureRectangle && pname == GL_TEXTURE_BASE_LEVEL && param != 0) { MG_State::pGLContext->RecordError( ErrorCode::InvalidOperation, @@ -1635,6 +1654,7 @@ namespace MobileGL::MG_Impl::GLImpl { case GL_TEXTURE_CUBE_MAP_ARRAY: case GL_TEXTURE_2D_MULTISAMPLE: case GL_TEXTURE_2D_MULTISAMPLE_ARRAY: + case GL_TEXTURE_EXTERNAL_OES: return true; default: return false; @@ -3145,6 +3165,17 @@ namespace MobileGL::MG_Impl::GLImpl { if (!TextureImpl::ValidateTextureObject(textureObject)) return false; switch (pname) { + // GL_OES_EGL_image_external: one unit per external texture (the image arrives as RGBA). + case GL_REQUIRED_TEXTURE_IMAGE_UNITS_OES: + if (textureTarget != TextureTarget::External) { + MG_State::pGLContext->RecordError( + ErrorCode::InvalidEnum, + MakeUnique("MG_Impl/GLImpl", __func__, + "GL_REQUIRED_TEXTURE_IMAGE_UNITS_OES is an external texture's.")); + return false; + } + if (params) *params = 1; + break; case GL_TEXTURE_MAG_FILTER: if (params) { *params = (GLint)MG_Util::ConvertSamplerFilterModeToGLEnum( @@ -6805,6 +6836,13 @@ namespace MobileGL::MG_Impl::GLImpl { const char* caller) { if (!textureObject) return false; const auto target = textureObject->GetTarget(); + // GL_OES_EGL_image_external: an external texture has no mip chain to generate. + if (target == TextureTarget::External) { + MG_State::pGLContext->RecordError( + ErrorCode::InvalidEnum, + MakeUnique("MG_Impl/GLImpl", caller, "External textures have no mipmaps.")); + return false; + } if ((target == TextureTarget::TextureCubeMap || target == TextureTarget::TextureCubeMapArray) && !textureObject->IsComplete()) { MG_State::pGLContext->RecordError( @@ -7138,15 +7176,20 @@ namespace MobileGL::MG_Impl::GLImpl { // GL_OES_EGL_image. The only EGLImages with storage behind them are the server's shared images // (EGL_LINUX_DMA_BUF_EXT): level 0 of the bound texture is defined at the image's size through // glTexImage2D's own path - so the respecification is recorded and announced as for any other - // - and the backend then swaps that storage for the image. The frontend has no external-image - // target, so GL_TEXTURE_2D is the only one taken. + // - and the backend then swaps that storage for the image. + // + // GL_OES_EGL_image_external's target takes it the same way: an external texture is a 2D one + // below the frontend. A YUV image (NV12/P010) is taken by either target; the backend converts it + // into the texture's RGBA level at the first use in each frame (SharedImageYuv.h). void EGLImageTargetTexture2DOES(GLenum target, GLeglImageOES image) { - if (target != GL_TEXTURE_2D) { + if (target != GL_TEXTURE_2D && target != GL_TEXTURE_EXTERNAL_OES) { MG_State::pGLContext->RecordError( ErrorCode::InvalidEnum, - MakeUnique("MG_Impl/GLImpl", __func__, "target must be GL_TEXTURE_2D.")); + MakeUnique("MG_Impl/GLImpl", __func__, + "target must be GL_TEXTURE_2D or GL_TEXTURE_EXTERNAL_OES.")); return; } + const Bool external = target == GL_TEXTURE_EXTERNAL_OES; Uint64 sharedImageId = 0; EGLint width = 0; EGLint height = 0; @@ -7158,17 +7201,27 @@ namespace MobileGL::MG_Impl::GLImpl { } auto& activeUnit = MG_State::pGLContext->GetTextureUnitObject(MG_State::pGLContext->GetActiveTextureUnit()); const SharedPtr textureObject = - activeUnit.GetBindingSlot(TextureTarget::Texture2D).GetBoundObject(); + activeUnit.GetBindingSlot(external ? TextureTarget::External : TextureTarget::Texture2D).GetBoundObject(); if (!TextureImpl::ValidateTextureObject(textureObject)) return; if (!ValidateTextureMutable(textureObject, __func__)) return; if (!TextureImpl::ValidateTextureSizeRange(width, height, 1)) return; - // NULL is an offset into a bound unpack buffer; the definition must read nothing. - GLint unpackBuffer = 0; - GetIntegerv(GL_PIXEL_UNPACK_BUFFER_BINDING, &unpackBuffer); - if (unpackBuffer != 0) BindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); - TexImage2D_State(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr); - if (unpackBuffer != 0) BindBuffer(GL_PIXEL_UNPACK_BUFFER, static_cast(unpackBuffer)); + if (external) { + // glTexImage2D does not take the external target, so its level 0 is defined here the way + // that call defines a 2D one without data: an RGBA8 base, any older chain dropped. + textureObject->SetInternalFormat(TextureInternalFormat::RGBA8); + auto* mipmapObject = static_cast(textureObject.get()); + DiscardMipmapChainOnBaseRespecification(mipmapObject, TextureUploadTarget::Texture2D, 0); + mipmapObject->AllocateStorage(TextureUploadTarget::Texture2D, 0, + {{width, height, 1}, static_cast(width) * height * 4}); + } else { + // NULL is an offset into a bound unpack buffer; the definition must read nothing. + GLint unpackBuffer = 0; + GetIntegerv(GL_PIXEL_UNPACK_BUFFER_BINDING, &unpackBuffer); + if (unpackBuffer != 0) BindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); + TexImage2D_State(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr); + if (unpackBuffer != 0) BindBuffer(GL_PIXEL_UNPACK_BUFFER, static_cast(unpackBuffer)); + } auto* backendObject = MG_Backend::pActiveBackendObject.get(); if (backendObject == nullptr || diff --git a/MobileGL/MG_Impl/Pipe/FramebufferEmit.h b/MobileGL/MG_Impl/Pipe/FramebufferEmit.h index 368cea05..ae757069 100644 --- a/MobileGL/MG_Impl/Pipe/FramebufferEmit.h +++ b/MobileGL/MG_Impl/Pipe/FramebufferEmit.h @@ -132,7 +132,7 @@ namespace MobileGL::MG_Pipe { // TextureUploadTarget -> TextureTarget inverse exists anywhere in the tree, so // without this the inline InternalFormat cannot make them fall out at push time and // the backend keeps reading the frontend attachment objects. - surface.TextureTarget = static_cast(texture->GetTarget()); + surface.TextureTarget = static_cast(MGPipeWireTextureTarget(texture->GetTarget())); return surface; } const auto& renderbuffer = attachment.GetRenderbuffer(); diff --git a/MobileGL/MG_Impl/Pipe/SamplerEmit.h b/MobileGL/MG_Impl/Pipe/SamplerEmit.h index 8cbe3782..c54f9fe6 100644 --- a/MobileGL/MG_Impl/Pipe/SamplerEmit.h +++ b/MobileGL/MG_Impl/Pipe/SamplerEmit.h @@ -629,6 +629,9 @@ namespace MobileGL::MG_Pipe { // sample this unit". inline MobileGL::TextureTarget MGPipeSamplerUniformTextureTarget(GLenum uniformType) { switch (uniformType) { + // GL_OES_EGL_image_external: the unit's external binding, whose texture is a 2D one below. + case GL_SAMPLER_EXTERNAL_OES: + return TextureTarget::External; case GL_SAMPLER_1D: case GL_INT_SAMPLER_1D: case GL_UNSIGNED_INT_SAMPLER_1D: @@ -995,7 +998,7 @@ namespace MobileGL::MG_Pipe { // and not the storage owner's, which is the whole point of the call; for an // ordinary texture it is simply its format. m_lastView.InternalFormat = static_cast(texture.GetFormat()); - m_lastView.Target = static_cast(texture.GetTarget()); + m_lastView.Target = static_cast(MGPipeWireTextureTarget(texture.GetTarget())); // THE FOUR RESTRICTIONS COME FROM ONE PLACE. TextureObjectBase leaves all four at // 0 for an ordinary texture and glTextureView writes them for a view, and a view // always has NumLevels >= 1 - so a zero here unambiguously means "no restriction, diff --git a/MobileGL/MG_Impl/SharedImageApi.cpp b/MobileGL/MG_Impl/SharedImageApi.cpp index 8a22cd9a..42b4f17d 100644 --- a/MobileGL/MG_Impl/SharedImageApi.cpp +++ b/MobileGL/MG_Impl/SharedImageApi.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -26,14 +27,19 @@ namespace { using MobileGL::MG_Backend::BackendObject; - constexpr uint32_t kFormats[] = {MOBILEGL_SHARED_IMAGE_FOURCC_ABGR8888, MOBILEGL_SHARED_IMAGE_FOURCC_XBGR8888, - MOBILEGL_SHARED_IMAGE_FOURCC_ARGB8888, MOBILEGL_SHARED_IMAGE_FOURCC_XRGB8888}; + constexpr uint32_t kRgbFormats[] = {MOBILEGL_SHARED_IMAGE_FOURCC_ABGR8888, MOBILEGL_SHARED_IMAGE_FOURCC_XBGR8888, + MOBILEGL_SHARED_IMAGE_FOURCC_ARGB8888, MOBILEGL_SHARED_IMAGE_FOURCC_XRGB8888}; + constexpr uint32_t kYuvFormats[] = {MOBILEGL_SHARED_IMAGE_FOURCC_NV12, MOBILEGL_SHARED_IMAGE_FOURCC_P010}; + + bool FourccIsYuv(uint32_t fourcc) { + return fourcc == MOBILEGL_SHARED_IMAGE_FOURCC_NV12 || fourcc == MOBILEGL_SHARED_IMAGE_FOURCC_P010; + } bool FourccSupported(uint32_t fourcc) { - for (uint32_t format : kFormats) { + for (uint32_t format : kRgbFormats) { if (format == fourcc) return true; } - return false; + return FourccIsYuv(fourcc); } // Serializes the calls against each other (not against GL calls: see the header). @@ -54,10 +60,20 @@ MOBILEGL_EXPORT uint32_t mobilegl_shared_image_abi_version(void) { return MOBILEGL_SHARED_IMAGE_ABI_VERSION; } +// The YUV formats only when the server holds them, which takes the session (and one round trip, +// once): a caller asking which formats exist is about to allocate one. MOBILEGL_EXPORT int mobilegl_shared_image_formats(uint32_t* formats, uint32_t capacity) { - constexpr uint32_t total = sizeof(kFormats) / sizeof(kFormats[0]); if (formats == nullptr && capacity != 0) return -EINVAL; - for (uint32_t i = 0; i < total && i < capacity; ++i) formats[i] = kFormats[i]; + const std::lock_guard lock(g_apiMutex); + uint32_t total = 0; + const auto put = [&](uint32_t format) { + if (total < capacity) formats[total] = format; + ++total; + }; + for (uint32_t format : kRgbFormats) put(format); + if (ActiveBackend(true) != nullptr && MobileGL::MG_Impl::EGLImpl::SharedImageYuvAvailable()) { + for (uint32_t format : kYuvFormats) put(format); + } return static_cast(total); } @@ -83,6 +99,8 @@ MOBILEGL_EXPORT int mobilegl_shared_image_allocate(uint32_t width, uint32_t heig answer.stride = image.Stride; answer.offset = image.Offset; answer.modifier = image.Modifier; + answer.plane1_stride = image.Plane1Stride; + answer.plane1_offset = image.Plane1Offset; // An older caller's struct is a prefix of this one; a newer caller's tail stays as it was. std::memcpy(out, &answer, std::min(out->struct_size, sizeof(answer))); return 0; @@ -90,7 +108,8 @@ MOBILEGL_EXPORT int mobilegl_shared_image_allocate(uint32_t width, uint32_t heig MOBILEGL_EXPORT int mobilegl_shared_image_import(int fd, uint32_t width, uint32_t height, uint32_t fourcc, uint64_t* out_id) { - if (fd < 0 || out_id == nullptr || width == 0 || height == 0 || !FourccSupported(fourcc)) return -EINVAL; + if (fd < 0 || out_id == nullptr || width == 0 || height == 0 || !FourccSupported(fourcc) || FourccIsYuv(fourcc)) + return -EINVAL; const std::lock_guard lock(g_apiMutex); BackendObject* backend = ActiveBackend(true); if (backend == nullptr) return -EIO; @@ -100,6 +119,30 @@ MOBILEGL_EXPORT int mobilegl_shared_image_import(int fd, uint32_t width, uint32_ return 0; } +MOBILEGL_EXPORT int mobilegl_shared_image_import_planes(int fd, uint32_t width, uint32_t height, uint32_t fourcc, + const struct mobilegl_shared_image_planes* planes, + uint64_t* out_id) { + if (fd < 0 || out_id == nullptr || planes == nullptr || width == 0 || height == 0 || !FourccIsYuv(fourcc) || + planes->struct_size < sizeof(mobilegl_shared_image_planes) || planes->plane_count != 2) + return -EINVAL; + const std::lock_guard lock(g_apiMutex); + BackendObject* backend = ActiveBackend(true); + if (backend == nullptr) return -EIO; + MobileGL::MG_Backend::SharedImageImportLayout layout; + layout.PlaneCount = 2; + for (int i = 0; i < 2; ++i) { + layout.Offset[i] = planes->offset[i]; + layout.Pitch[i] = planes->pitch[i]; + } + constexpr uint64_t kModifierInvalid = 0x00ffffffffffffffull; + layout.HasModifier = planes->modifier != kModifierInvalid; + layout.Modifier = planes->modifier; + uint64_t id = 0; + if (!backend->ImportSharedImagePlanes(fd, width, height, fourcc, layout, &id)) return -EIO; + *out_id = id; + return 0; +} + MOBILEGL_EXPORT int mobilegl_server_available(void) { return MobileGL::ImplementationAvailable() ? 1 : 0; } diff --git a/MobileGL/MG_Impl/SharedImageApi.h b/MobileGL/MG_Impl/SharedImageApi.h index 765682ad..4687ae5e 100644 --- a/MobileGL/MG_Impl/SharedImageApi.h +++ b/MobileGL/MG_Impl/SharedImageApi.h @@ -36,7 +36,9 @@ extern "C" { #endif -#define MOBILEGL_SHARED_IMAGE_ABI_VERSION 1u +/* 2: YUV images (NV12, P010), the plane-1 fields of mobilegl_shared_image, and + * mobilegl_shared_image_import_planes. */ +#define MOBILEGL_SHARED_IMAGE_ABI_VERSION 2u /* DRM fourccs (little-endian packed). ABGR8888 is R, G, B, A in memory - GL's RGBA8. */ #define MOBILEGL_SHARED_IMAGE_FOURCC_ABGR8888 0x34324241u /* 'AB24' */ @@ -46,6 +48,12 @@ extern "C" { * writes them, so the name an image is allocated under never changes its colours. */ #define MOBILEGL_SHARED_IMAGE_FOURCC_ARGB8888 0x34325241u /* 'AR24' */ #define MOBILEGL_SHARED_IMAGE_FOURCC_XRGB8888 0x34325258u /* 'XR24' */ +/* Two-plane YUV 4:2:0 (ABI 2), when the server's platform allocates it (listed by + * mobilegl_shared_image_formats only then): plane 0 is Y, plane 1 Cb Cr interleaved, both in the one + * descriptor at the offsets and pitches the allocation reports. They are SAMPLED only - an EGLImage + * of one binds through GL_OES_EGL_image_external - and their layout is still the allocator's. */ +#define MOBILEGL_SHARED_IMAGE_FOURCC_NV12 0x3231564Eu /* 'NV12' */ +#define MOBILEGL_SHARED_IMAGE_FOURCC_P010 0x30313050u /* 'P010' */ /* One allocated image. VERSIONED BY SIZE: the caller sets struct_size to the sizeof it was built * against and the library writes no byte past it, so fields are only ever appended. */ @@ -60,6 +68,18 @@ struct mobilegl_shared_image { uint32_t offset; /* plane 0, bytes */ uint32_t reserved; /* zero */ uint64_t modifier; /* DRM format modifier of the layout (DRM_FORMAT_MOD_INVALID today) */ + /* ABI 2: a YUV image's plane 1 (same fd); zero for the one-plane formats. */ + uint32_t plane1_stride; + uint32_t plane1_offset; +}; + +/* Where an imported two-plane buffer's planes are (ABI 2). */ +struct mobilegl_shared_image_planes { + uint32_t struct_size; + uint32_t plane_count; /* 2 */ + uint32_t offset[2]; + uint32_t pitch[2]; + uint64_t modifier; /* DRM_FORMAT_MOD_INVALID when none was given */ }; /* MOBILEGL_SHARED_IMAGE_ABI_VERSION of the library. */ @@ -78,6 +98,11 @@ int mobilegl_shared_image_allocate(uint32_t width, uint32_t height, uint32_t fou * -EIO: not one of the server's images, or the size/format disagree. */ int mobilegl_shared_image_import(int fd, uint32_t width, uint32_t height, uint32_t fourcc, uint64_t* out_id); +/* The same for a YUV fourcc whose two planes are in `fd` as `planes` says (ABI 2). A YUV buffer the + * server did not allocate is taken too, read through the server's CPU-copy fallback. */ +int mobilegl_shared_image_import_planes(int fd, uint32_t width, uint32_t height, uint32_t fourcc, + const struct mobilegl_shared_image_planes* planes, uint64_t* out_id); + /* Drops this process's reference (an allocation's or an import's). */ int mobilegl_shared_image_release(uint64_t id); @@ -95,6 +120,8 @@ typedef int (*PFN_mobilegl_shared_image_formats)(uint32_t*, uint32_t); typedef int (*PFN_mobilegl_shared_image_allocate)(uint32_t, uint32_t, uint32_t, struct mobilegl_shared_image*); typedef int (*PFN_mobilegl_shared_image_import)(int, uint32_t, uint32_t, uint32_t, uint64_t*); typedef int (*PFN_mobilegl_shared_image_release)(uint64_t); +typedef int (*PFN_mobilegl_shared_image_import_planes)(int, uint32_t, uint32_t, uint32_t, + const struct mobilegl_shared_image_planes*, uint64_t*); #ifdef __cplusplus } diff --git a/MobileGL/MG_Pipe/MGPipeTypes.h b/MobileGL/MG_Pipe/MGPipeTypes.h index 24df2d4b..7ad7feb4 100644 --- a/MobileGL/MG_Pipe/MGPipeTypes.h +++ b/MobileGL/MG_Pipe/MGPipeTypes.h @@ -263,6 +263,11 @@ namespace MobileGL::MG_Pipe { return static_cast(MGPipeResourceTarget::TexCubeArray); case MobileGL::TextureTarget::Texture2DMultisampleArray: return static_cast(MGPipeResourceTarget::Tex2DMSArray); + // THE ONE FOLD (rather than an enumerator of its own, D-A3): an external texture + // (GL_OES_EGL_image_external) is a 2D texture to every backend - a single RGBA level 0, + // sampled through a sampler2D - and only the frontend tells the two binding points apart. + case MobileGL::TextureTarget::External: + return static_cast(MGPipeResourceTarget::Tex2D); // NOT TEXTURE TARGETS. Listed rather than defaulted so the completeness assert still // sees them, and mapped to the sentinel because no descriptor may carry either: the // count is the enum's bound and Unknown is what an unresolved GL enum becomes. @@ -273,6 +278,12 @@ namespace MobileGL::MG_Pipe { return kMGPipeResourceTargetUnmapped; } + // The TextureTarget a record that carries one raw (MGPSamplerView, MGPSurface) names: the + // frontend's own, but for the external target, which is a 2D texture below the frontend. + constexpr MobileGL::TextureTarget MGPipeWireTextureTarget(MobileGL::TextureTarget target) { + return target == MobileGL::TextureTarget::External ? MobileGL::TextureTarget::Texture2D : target; + } + constexpr Bool MGPipeEveryTextureTargetIsMapped() { for (SizeT i = 0; i < static_cast(MobileGL::TextureTarget::TextureTargetCount); ++i) { if (MGPipeResourceTargetForTextureTarget(static_cast(i)) == @@ -2044,6 +2055,25 @@ namespace MobileGL::MG_Pipe { inline constexpr Uint32 kMGPBufferAgeDamageRegionFollows = 1u; inline constexpr Uint32 kMGPNativeFenceFdFollows = 1u; + // An Import of a YUV fourcc (NV12, P010) carries its planes and colour hints in Damage, with + // DamageCount = kMGPImportPlaneWords (the field is a Present's otherwise, and 0 on an RGBA + // Import): both planes live in the one descriptor the Import sends, at these byte offsets and + // row pitches. The hints are EGL_EXT_image_dma_buf_import's values, 0 where not given. + enum MGPImportPlaneWord : Uint32 { + kMGPImportPlaneCount = 0, + kMGPImportPlane0Offset = 1, + kMGPImportPlane1Offset = 2, + kMGPImportPlane0Pitch = 3, + kMGPImportPlane1Pitch = 4, + kMGPImportYuvColorSpace = 5, + kMGPImportYuvRange = 6, + kMGPImportYuvSiting = 7, // horizontal | vertical << 16 + kMGPImportModifierLo = 8, + kMGPImportModifierHi = 9, + kMGPImportHasModifier = 10, + kMGPImportPlaneWords = 11, + }; + struct MGPSharedImageOp { Uint32 Op; Uint32 Format; // DRM fourcc @@ -2067,8 +2097,11 @@ namespace MobileGL::MG_Pipe { Uint32 Stride; Uint32 Offset; Int32 BufferAge; // QueryBufferAge only + // A YUV image's Cb Cr plane (Allocate, Import): same descriptor, this offset and pitch. + Uint32 Plane1Stride; + Uint32 Plane1Offset; }; - MGP_ASSERT_POD(MGPSharedImageReply, 40); + MGP_ASSERT_POD(MGPSharedImageReply, 48); // The aux-socket sideband an Import's descriptor travels with (and an Allocate's answer). inline constexpr Uint32 kMGPSharedImageFdMagic = 0x49534D47u; // 'MGSI' diff --git a/MobileGL/MG_Remote/Client/BackendObject_Remote.cpp b/MobileGL/MG_Remote/Client/BackendObject_Remote.cpp index 358f6af1..14403a47 100644 --- a/MobileGL/MG_Remote/Client/BackendObject_Remote.cpp +++ b/MobileGL/MG_Remote/Client/BackendObject_Remote.cpp @@ -604,6 +604,8 @@ namespace MobileGL::MG_Remote::Client { out->Stride = reply.Stride; out->Offset = reply.Offset; out->Modifier = reply.Modifier; + out->Plane1Stride = reply.Plane1Stride; + out->Plane1Offset = reply.Plane1Offset; { const std::lock_guard lock(m_eglStateMutex); ++m_sessionSharedImages[reply.ImageId]; @@ -630,6 +632,40 @@ namespace MobileGL::MG_Remote::Client { return true; } + Bool BackendObject_Remote::ImportSharedImagePlanes(int fd, Uint32 width, Uint32 height, Uint32 fourcc, + const MG_Backend::SharedImageImportLayout& layout, + Uint64* outId) { + ClientSession* session = ClientSession::Active(); + if (session == nullptr || fd < 0) return false; + const ClientSession::ScopedCurrentSessionWork tableOp; + MG_Pipe::MGPSharedImageOp op{}; + op.Op = MG_Pipe::kMGPSharedImageImport; + op.Format = fourcc; + op.Width = width; + op.Height = height; + // The planes and hints ride in Damage (MGPImportPlaneWord). + op.DamageCount = MG_Pipe::kMGPImportPlaneWords; + op.Damage[MG_Pipe::kMGPImportPlaneCount] = static_cast(layout.PlaneCount); + op.Damage[MG_Pipe::kMGPImportPlane0Offset] = static_cast(layout.Offset[0]); + op.Damage[MG_Pipe::kMGPImportPlane1Offset] = static_cast(layout.Offset[1]); + op.Damage[MG_Pipe::kMGPImportPlane0Pitch] = static_cast(layout.Pitch[0]); + op.Damage[MG_Pipe::kMGPImportPlane1Pitch] = static_cast(layout.Pitch[1]); + op.Damage[MG_Pipe::kMGPImportYuvColorSpace] = static_cast(layout.ColorSpace); + op.Damage[MG_Pipe::kMGPImportYuvRange] = static_cast(layout.Range); + op.Damage[MG_Pipe::kMGPImportYuvSiting] = static_cast((layout.SitingX & 0xffffu) | (layout.SitingY << 16)); + op.Damage[MG_Pipe::kMGPImportModifierLo] = static_cast(static_cast(layout.Modifier)); + op.Damage[MG_Pipe::kMGPImportModifierHi] = static_cast(static_cast(layout.Modifier >> 32)); + op.Damage[MG_Pipe::kMGPImportHasModifier] = layout.HasModifier ? 1 : 0; + MG_Pipe::MGPSharedImageReply reply{}; + if (!session->EmitSharedImage(op, fd, &reply, nullptr)) return false; + if (outId != nullptr) *outId = reply.ImageId; + { + const std::lock_guard lock(m_eglStateMutex); + ++m_sessionSharedImages[reply.ImageId]; + } + return true; + } + Bool BackendObject_Remote::ReleaseSharedImage(Uint64 id) { ClientSession* session = ClientSession::Active(); if (session == nullptr || !session->Started()) return false; diff --git a/MobileGL/MG_Remote/Client/BackendObject_Remote.h b/MobileGL/MG_Remote/Client/BackendObject_Remote.h index 7cbae559..4184ed56 100644 --- a/MobileGL/MG_Remote/Client/BackendObject_Remote.h +++ b/MobileGL/MG_Remote/Client/BackendObject_Remote.h @@ -90,6 +90,8 @@ namespace MobileGL::MG_Remote::Client { Bool AllocateSharedImage(Uint32 width, Uint32 height, Uint32 fourcc, MG_Backend::SharedImageExport* out) override; Bool ImportSharedImage(int fd, Uint32 width, Uint32 height, Uint32 fourcc, Uint64* outId) override; + Bool ImportSharedImagePlanes(int fd, Uint32 width, Uint32 height, Uint32 fourcc, + const MG_Backend::SharedImageImportLayout& layout, Uint64* outId) override; Bool ReleaseSharedImage(Uint64 id) override; Bool PresentToSharedImage(Uint64 id, const MG_Util::Damage::Region& region) override; Bool QueryBufferAge(Bool damageRegionFollows, EGLint* age) override; diff --git a/MobileGL/MG_Remote/Server/PipeApplier.cpp b/MobileGL/MG_Remote/Server/PipeApplier.cpp index f94f41fa..47923a5b 100644 --- a/MobileGL/MG_Remote/Server/PipeApplier.cpp +++ b/MobileGL/MG_Remote/Server/PipeApplier.cpp @@ -1517,6 +1517,8 @@ namespace MobileGL::MG_Remote::Server { reply.Format = image.Fourcc; reply.Stride = image.Stride; reply.Offset = image.Offset; + reply.Plane1Stride = image.Plane1Stride; + reply.Plane1Offset = image.Plane1Offset; }; switch (op.Op) { case MG_Pipe::kMGPSharedImageAllocate: { @@ -1554,6 +1556,33 @@ namespace MobileGL::MG_Remote::Server { return false; } SI::ImageRef image = SI::Identify(fd, why); + // YUV: the planes and colour hints ride in Damage (MGPImportPlaneWord). A buffer this + // server did not allocate is taken too, through the labelled CPU-copy fallback + // (SharedImageRegistry.h, Image::ForeignSource); RGBA ones never are. + const Bool yuv = SI::FourccIsYuv(op.Format) && op.DamageCount == MG_Pipe::kMGPImportPlaneWords; + if (image == nullptr && yuv) { + SI::PlaneLayout layout; + layout.Count = static_cast(op.Damage[MG_Pipe::kMGPImportPlaneCount]); + layout.Offset[0] = static_cast(op.Damage[MG_Pipe::kMGPImportPlane0Offset]); + layout.Offset[1] = static_cast(op.Damage[MG_Pipe::kMGPImportPlane1Offset]); + layout.Pitch[0] = static_cast(op.Damage[MG_Pipe::kMGPImportPlane0Pitch]); + layout.Pitch[1] = static_cast(op.Damage[MG_Pipe::kMGPImportPlane1Pitch]); + // The copy reads rows: only a buffer said to be linear (or said nothing) is one. + const Uint64 modifier = + static_cast(static_cast(op.Damage[MG_Pipe::kMGPImportModifierLo])) | + (static_cast(static_cast(op.Damage[MG_Pipe::kMGPImportModifierHi])) << 32); + if (op.Damage[MG_Pipe::kMGPImportHasModifier] != 0 && modifier != SI::kModifierLinear && + modifier != SI::kModifierInvalid) { + char text[96]; + std::snprintf(text, sizeof(text), "; a foreign buffer of modifier 0x%016llx is not linear", + static_cast(modifier)); + why += text; + } else { + std::string foreignWhy; + image = SI::ImportForeignYuv(fd, op.Width, op.Height, op.Format, layout, foreignWhy); + if (image == nullptr) why += "; as a foreign YUV buffer: " + foreignWhy; + } + } #if !defined(_WIN32) ::close(fd); #endif @@ -1561,6 +1590,20 @@ namespace MobileGL::MG_Remote::Server { MGLOG_W_ONCE("MG_Remote server: a dma-buf import was refused: %s", why.c_str()); return false; } + if (SI::FourccIsYuv(image->Fourcc) != SI::FourccIsYuv(op.Format) || + (SI::FourccIsYuv(op.Format) && image->Fourcc != op.Format)) { + MGLOG_W("MG_Remote server: a dma-buf import of image %llu (fourcc 0x%08x) as fourcc 0x%08x was refused", + static_cast(image->Id), image->Fourcc, op.Format); + return false; + } + if (yuv) { + SI::Yuv::Hints hints; + hints.ColorSpace = static_cast(op.Damage[MG_Pipe::kMGPImportYuvColorSpace]); + hints.Range = static_cast(op.Damage[MG_Pipe::kMGPImportYuvRange]); + hints.SitingX = static_cast(op.Damage[MG_Pipe::kMGPImportYuvSiting]) & 0xffffu; + hints.SitingY = static_cast(op.Damage[MG_Pipe::kMGPImportYuvSiting]) >> 16; + SI::SetYuvHints(*image, hints); + } if (image->Width != op.Width || image->Height != op.Height || !SI::FourccSupported(op.Format)) { MGLOG_W("MG_Remote server: a dma-buf import of image %llu (%ux%u) as %ux%u fourcc 0x%08x was refused", static_cast(image->Id), image->Width, image->Height, op.Width, op.Height, diff --git a/MobileGL/MG_Remote/Server/SharedImageRegistry.cpp b/MobileGL/MG_Remote/Server/SharedImageRegistry.cpp index 051bd91f..73043dd5 100644 --- a/MobileGL/MG_Remote/Server/SharedImageRegistry.cpp +++ b/MobileGL/MG_Remote/Server/SharedImageRegistry.cpp @@ -19,10 +19,13 @@ #if defined(__ANDROID__) #include +#include #endif #if !defined(_WIN32) #include +#include #include +#include #include #include #include @@ -43,6 +46,8 @@ namespace MobileGL::MG_Remote::Server::SharedImages { std::atomic nextId{1}; UnorderedMap> byId; std::map, std::weak_ptr> byIdentity; + // Foreign YUV images, by the identity of the buffer they copy from. + std::map, std::weak_ptr> foreignByIdentity; // Set once two live images were seen with one identity: the kernel shares an inode // between dma-bufs, so a descriptor can no longer say which image it is. Bool identityAmbiguous = false; @@ -107,14 +112,93 @@ namespace MobileGL::MG_Remote::Server::SharedImages { return true; } + // The two AHardwareBuffer entry points newer than this library's minimum platform (29), + // resolved at run time. Null on an older platform: YUV images are then not allocatable. + struct AhbPlaneEntryPoints { + using LockPlanesFn = int (*)(AHardwareBuffer*, uint64_t, int32_t, const ARect*, AHardwareBuffer_Planes*); + using IsSupportedFn = int (*)(const AHardwareBuffer_Desc*); + LockPlanesFn LockPlanes = nullptr; + IsSupportedFn IsSupported = nullptr; + }; + const AhbPlaneEntryPoints& AhbPlanes() { + static const AhbPlaneEntryPoints entry = [] { + AhbPlaneEntryPoints e; + void* lib = dlopen("libnativewindow.so", RTLD_NOW | RTLD_NOLOAD); + if (lib == nullptr) lib = dlopen("libnativewindow.so", RTLD_NOW); + if (lib != nullptr) { + e.LockPlanes = + reinterpret_cast(dlsym(lib, "AHardwareBuffer_lockPlanes")); + e.IsSupported = + reinterpret_cast(dlsym(lib, "AHardwareBuffer_isSupported")); + } + return e; + }(); + return entry; + } + + // A YUV image's buffer: sampled by the GPU, written by the CPU (a foreign image's copy). The + // CPU usage also keeps the layout one the platform can lock plane by plane, which is how + // every layout question about it is answered (AHardwareBuffer_lockPlanes) - never by + // assuming one. + constexpr uint64_t kYuvUsage = AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE | + AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN | AHARDWAREBUFFER_USAGE_CPU_READ_RARELY; + + uint32_t AhbFormatOf(Uint32 fourcc) { + if (fourcc == kFourccNv12) return AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420; + if (fourcc == kFourccP010) return AHARDWAREBUFFER_FORMAT_YCbCr_P010; + return FourccIgnoresAlpha(fourcc) ? AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM + : AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM; + } + + Bool YuvAllocatableNative(Uint32 fourcc) { + const AhbPlaneEntryPoints& entry = AhbPlanes(); + if (entry.LockPlanes == nullptr || entry.IsSupported == nullptr) return false; + AHardwareBuffer_Desc desc{}; + desc.width = 64; + desc.height = 64; + desc.layers = 1; + desc.format = AhbFormatOf(fourcc); + desc.usage = kYuvUsage; + return entry.IsSupported(&desc) == 1; + } + + // The planes of a locked YUV buffer as the plane copy addresses them. + Bool LockYuvPlanes(AHardwareBuffer* ahb, Yuv::PlanarLayout& out, std::string& why) { + const AhbPlaneEntryPoints& entry = AhbPlanes(); + if (entry.LockPlanes == nullptr) { + why = "AHardwareBuffer_lockPlanes is not available"; + return false; + } + AHardwareBuffer_Planes planes{}; + const int locked = entry.LockPlanes(ahb, AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN, -1, nullptr, &planes); + if (locked != 0) { + why = Format("AHardwareBuffer_lockPlanes rc=%lld", locked); + return false; + } + if (planes.planeCount < 3) { + AHardwareBuffer_unlock(ahb, nullptr); + why = Format("the locked buffer has %lld planes, not 3", planes.planeCount); + return false; + } + const auto view = [](const AHardwareBuffer_Plane& plane) { + return Yuv::PlaneView{static_cast(plane.data), static_cast(plane.rowStride), + plane.pixelStride}; + }; + out.Y = view(planes.planes[0]); + out.Cb = view(planes.planes[1]); + out.Cr = view(planes.planes[2]); + return true; + } + Bool AllocateNative(Image& image, std::string& why) { AHardwareBuffer_Desc desc{}; desc.width = image.Width; desc.height = image.Height; desc.layers = 1; - desc.format = FourccIgnoresAlpha(image.Fourcc) ? AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM - : AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM; - desc.usage = AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE | AHARDWAREBUFFER_USAGE_GPU_COLOR_OUTPUT; + desc.format = AhbFormatOf(image.Fourcc); + desc.usage = FourccIsYuv(image.Fourcc) + ? kYuvUsage + : AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE | AHARDWAREBUFFER_USAGE_GPU_COLOR_OUTPUT; AHardwareBuffer* ahb = nullptr; const int allocated = AHardwareBuffer_allocate(&desc, &ahb); if (allocated != 0 || ahb == nullptr) { @@ -131,8 +215,32 @@ namespace MobileGL::MG_Remote::Server::SharedImages { image.Native = ahb; image.Fd = fds[0]; for (SizeT i = 1; i < fds.size(); ++i) ::close(fds[i]); - image.Stride = described.stride * 4; image.Offset = 0; + if (!FourccIsYuv(image.Fourcc)) { + image.Stride = described.stride * 4; + return true; + } + // Plane positions as the platform reports them when the buffer is locked: offsets from + // the luma plane's start, the Cb Cr plane at whichever of Cb and Cr comes first. + const Uint32 sampleBytes = FourccYuvSampleBytes(image.Fourcc); + image.Stride = described.stride * sampleBytes; + image.Plane1Stride = image.Stride; + image.Plane1Offset = image.Stride * image.Height; + Yuv::PlanarLayout planes; + std::string lockWhy; + if (LockYuvPlanes(ahb, planes, lockWhy)) { + const Uint8* chroma = std::min(planes.Cb.Data, planes.Cr.Data); + if (planes.Y.RowStride > 0 && chroma > planes.Y.Data) { + image.Stride = static_cast(planes.Y.RowStride); + image.Plane1Stride = static_cast(planes.Cb.RowStride); + image.Plane1Offset = static_cast(chroma - planes.Y.Data); + } + AHardwareBuffer_unlock(ahb, nullptr); + } else { + MGLOG_W_ONCE("MG_Remote server: a YUV shared image's planes could not be located (%s); its " + "plane offsets are nominal", + lockWhy.c_str()); + } return true; } @@ -143,15 +251,29 @@ namespace MobileGL::MG_Remote::Server::SharedImages { // A host has no AHardwareBuffer. A memfd stands in for the dma-buf so the registry, the // descriptor round trip and identification can be exercised by the unit tests; no backend // binds one. + Bool YuvAllocatableNative(Uint32) { return true; } + Bool AllocateNative(Image& image, std::string& why) { - image.Stride = image.Width * 4; + Uint64 bytes = 0; image.Offset = 0; + if (FourccIsYuv(image.Fourcc)) { + // A tight semi-planar layout stands in for the platform's. + const Uint32 sampleBytes = FourccYuvSampleBytes(image.Fourcc); + image.Stride = image.Width * sampleBytes; + image.Plane1Stride = ((image.Width + 1) / 2) * 2 * sampleBytes; + image.Plane1Offset = image.Stride * image.Height; + bytes = static_cast(image.Plane1Offset) + + static_cast(image.Plane1Stride) * ((image.Height + 1) / 2); + } else { + image.Stride = image.Width * 4; + bytes = static_cast(image.Stride) * image.Height; + } const int fd = ::memfd_create("mobilegl-shared-image", MFD_CLOEXEC); if (fd < 0) { why = Format("memfd_create errno=%lld", errno); return false; } - if (::ftruncate(fd, static_cast(image.Stride) * image.Height) != 0) { + if (::ftruncate(fd, static_cast(bytes)) != 0) { why = Format("ftruncate errno=%lld", errno); ::close(fd); return false; @@ -162,6 +284,8 @@ namespace MobileGL::MG_Remote::Server::SharedImages { void ReleaseNative(void*) {} #else + Bool YuvAllocatableNative(Uint32) { return false; } + Bool AllocateNative(Image&, std::string& why) { why = "shared images need POSIX descriptors"; return false; @@ -169,6 +293,66 @@ namespace MobileGL::MG_Remote::Server::SharedImages { void ReleaseNative(void*) {} #endif + +#if !defined(_WIN32) + // The planes of a YUV image as the CPU writes them, held until the guard goes: the locked + // AHardwareBuffer on Android, a mapping of the stand-in memfd on a host. + class YuvWriteAccess { + public: + YuvWriteAccess(const Image& image, std::string& why) : m_image(image) { +#if defined(__ANDROID__) + auto* ahb = static_cast(image.Native); + m_ok = ahb != nullptr && LockYuvPlanes(ahb, m_planes, why); +#else + const SizeT bytes = static_cast(image.Plane1Offset) + + static_cast(image.Plane1Stride) * ((image.Height + 1) / 2); + void* map = ::mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_SHARED, image.Fd, 0); + if (map == MAP_FAILED) { + why = Format("mmap of the stand-in buffer errno=%lld", errno); + return; + } + m_map = map; + m_mapSize = bytes; + const Uint32 sampleBytes = FourccYuvSampleBytes(image.Fourcc); + auto* base = static_cast(map); + m_planes.Y = {base, image.Stride, sampleBytes}; + m_planes.Cb = {base + image.Plane1Offset, image.Plane1Stride, 2 * sampleBytes}; + m_planes.Cr = {base + image.Plane1Offset + sampleBytes, image.Plane1Stride, 2 * sampleBytes}; + m_ok = true; +#endif + } + ~YuvWriteAccess() { +#if defined(__ANDROID__) + if (m_ok) AHardwareBuffer_unlock(static_cast(m_image.Native), nullptr); +#else + if (m_map != nullptr) ::munmap(m_map, m_mapSize); +#endif + } + YuvWriteAccess(const YuvWriteAccess&) = delete; + YuvWriteAccess& operator=(const YuvWriteAccess&) = delete; + Bool Ok() const { return m_ok; } + const Yuv::PlanarLayout& Planes() const { return m_planes; } + + private: + const Image& m_image; + Yuv::PlanarLayout m_planes; + Bool m_ok = false; +#if !defined(__ANDROID__) + void* m_map = nullptr; + SizeT m_mapSize = 0; +#endif + }; + + // DMA_BUF_IOCTL_SYNC around a CPU read of a foreign buffer: the producer's CPU-side writes + // are made visible (cache maintenance) and its implicit fences waited for. A buffer that + // does not take the ioctl (not a dma-buf: a host test's memfd) is read as it is. + void DmaBufSync(int fd, Bool start) { + dma_buf_sync sync{}; + sync.flags = DMA_BUF_SYNC_READ | (start ? DMA_BUF_SYNC_START : DMA_BUF_SYNC_END); + while (::ioctl(fd, DMA_BUF_IOCTL_SYNC, &sync) != 0 && (errno == EINTR || errno == EAGAIN)) { + } + } +#endif } // namespace Image::~Image() { @@ -181,8 +365,18 @@ namespace MobileGL::MG_Remote::Server::SharedImages { const auto key = std::make_pair(IdentityDev, IdentityIno); if (auto it = reg.byIdentity.find(key); it != reg.byIdentity.end() && it->second.expired()) reg.byIdentity.erase(it); + if (Foreign != nullptr) { + const auto source = std::make_pair(Foreign->IdentityDev, Foreign->IdentityIno); + if (auto it = reg.foreignByIdentity.find(source); + it != reg.foreignByIdentity.end() && it->second.expired()) + reg.foreignByIdentity.erase(it); + } } #if !defined(_WIN32) + if (Foreign != nullptr) { + if (Foreign->Map != nullptr) ::munmap(Foreign->Map, Foreign->MapSize); + if (Foreign->Fd >= 0) ::close(Foreign->Fd); + } if (Fd >= 0) ::close(Fd); if (WriteFence >= 0) ::close(WriteFence); for (const int fence : ReadFences) ::close(fence); @@ -190,49 +384,206 @@ namespace MobileGL::MG_Remote::Server::SharedImages { ReleaseNative(Native); } + Bool FourccIsYuv(Uint32 fourcc) { return fourcc == kFourccNv12 || fourcc == kFourccP010; } + + Uint32 FourccYuvSampleBytes(Uint32 fourcc) { + if (fourcc == kFourccNv12) return 1; + if (fourcc == kFourccP010) return 2; + return 0; + } + + Bool YuvFourccAllocatable(Uint32 fourcc) { + if (!FourccIsYuv(fourcc)) return false; + static std::mutex mutex; + static std::map answers; + const std::lock_guard lock(mutex); + const auto it = answers.find(fourcc); + if (it != answers.end()) return it->second; + const Bool answer = YuvAllocatableNative(fourcc); + answers.emplace(fourcc, answer); + return answer; + } + Bool FourccSupported(Uint32 fourcc) { return fourcc == kFourccAbgr8888 || fourcc == kFourccXbgr8888 || fourcc == kFourccArgb8888 || - fourcc == kFourccXrgb8888; + fourcc == kFourccXrgb8888 || YuvFourccAllocatable(fourcc); } Bool FourccIgnoresAlpha(Uint32 fourcc) { return fourcc == kFourccXbgr8888 || fourcc == kFourccXrgb8888; } + namespace { + // Allocate's body. `foreign`, when given, is on the image before anything can find it. + ImageRef AllocateImage(Uint32 width, Uint32 height, Uint32 fourcc, UniquePtr foreign, + std::string& why) { + if (width == 0 || height == 0 || width > 16384 || height > 16384) { + why = Format("%lldx%lld is not an image size", width, height); + return nullptr; + } + if (!FourccSupported(fourcc)) { + why = Format("fourcc 0x%08llx is not a shared-image format", fourcc); + return nullptr; + } + Registry& reg = Reg(); + auto image = std::make_shared(); + image->Id = reg.nextId.fetch_add(1, std::memory_order_relaxed); + image->Width = width; + image->Height = height; + image->Fourcc = fourcc; + image->Foreign = std::move(foreign); + if (!AllocateNative(*image, why)) return nullptr; +#if !defined(_WIN32) + if (!IdentityOf(image->Fd, &image->IdentityDev, &image->IdentityIno)) { + why = Format("fstat on the exported descriptor errno=%lld", errno); + return nullptr; + } +#endif + const std::lock_guard lock(reg.mutex); + reg.byId[image->Id] = image; + const auto key = std::make_pair(image->IdentityDev, image->IdentityIno); + if (auto it = reg.byIdentity.find(key); it != reg.byIdentity.end() && !it->second.expired()) { + if (!reg.identityAmbiguous) { + MGLOG_E("MG_Remote server: two live shared images have one dma-buf identity (dev %llu ino %llu); " + "this kernel does not keep dma-buf inodes unique, so a returned descriptor cannot name its " + "image and every shared-image import is refused from here on", + static_cast(key.first), static_cast(key.second)); + } + reg.identityAmbiguous = true; + } + reg.byIdentity[key] = image; + if (image->Foreign != nullptr) { + reg.foreignByIdentity[std::make_pair(image->Foreign->IdentityDev, image->Foreign->IdentityIno)] = + image; + } + return image; + } + } // namespace + ImageRef Allocate(Uint32 width, Uint32 height, Uint32 fourcc, std::string& why) { + return AllocateImage(width, height, fourcc, nullptr, why); + } + + ImageRef ImportForeignYuv(int fd, Uint32 width, Uint32 height, Uint32 fourcc, const PlaneLayout& layout, + std::string& why) { +#if defined(_WIN32) + (void)fd, (void)width, (void)height, (void)fourcc, (void)layout; + why = "shared images need POSIX descriptors"; + return nullptr; +#else + if (!FourccIsYuv(fourcc) || !YuvFourccAllocatable(fourcc)) { + why = Format("fourcc 0x%08llx is not a YUV format this server can hold", fourcc); + return nullptr; + } if (width == 0 || height == 0 || width > 16384 || height > 16384) { why = Format("%lldx%lld is not an image size", width, height); return nullptr; } - if (!FourccSupported(fourcc)) { - why = Format("fourcc 0x%08llx is not a shared-image format", fourcc); + if (layout.Count != 2) { + why = Format("a YUV import needs 2 planes, not %lld", layout.Count); + return nullptr; + } + Uint64 dev = 0, ino = 0; + if (!IdentityOf(fd, &dev, &ino)) { + why = Format("fstat on the imported descriptor errno=%lld", errno); return nullptr; } Registry& reg = Reg(); - auto image = std::make_shared(); - image->Id = reg.nextId.fetch_add(1, std::memory_order_relaxed); - image->Width = width; - image->Height = height; - image->Fourcc = fourcc; - if (!AllocateNative(*image, why)) return nullptr; -#if !defined(_WIN32) - if (!IdentityOf(image->Fd, &image->IdentityDev, &image->IdentityIno)) { - why = Format("fstat on the exported descriptor errno=%lld", errno); + { + const std::lock_guard lock(reg.mutex); + const auto it = reg.foreignByIdentity.find(std::make_pair(dev, ino)); + ImageRef existing = it != reg.foreignByIdentity.end() ? it->second.lock() : nullptr; + if (existing != nullptr && existing->Width == width && existing->Height == height && + existing->Fourcc == fourcc && existing->Foreign->Layout.Offset[0] == layout.Offset[0] && + existing->Foreign->Layout.Offset[1] == layout.Offset[1] && + existing->Foreign->Layout.Pitch[0] == layout.Pitch[0] && + existing->Foreign->Layout.Pitch[1] == layout.Pitch[1]) + return existing; + } + // The planes must fit the buffer: every row the copy reads is inside the mapping. + const Uint32 sampleBytes = FourccYuvSampleBytes(fourcc); + const off_t end = ::lseek(fd, 0, SEEK_END); + if (end <= 0) { + why = Format("the buffer's size is unknown (lseek errno=%lld)", errno); return nullptr; } -#endif - const std::lock_guard lock(reg.mutex); - reg.byId[image->Id] = image; - const auto key = std::make_pair(image->IdentityDev, image->IdentityIno); - if (auto it = reg.byIdentity.find(key); it != reg.byIdentity.end() && !it->second.expired()) { - if (!reg.identityAmbiguous) { - MGLOG_E("MG_Remote server: two live shared images have one dma-buf identity (dev %llu ino %llu); " - "this kernel does not keep dma-buf inodes unique, so a returned descriptor cannot name its " - "image and every shared-image import is refused from here on", - static_cast(key.first), static_cast(key.second)); - } - reg.identityAmbiguous = true; + const SizeT size = static_cast(end); + const Uint32 chromaHeight = (height + 1) / 2; + const Uint64 chromaRow = static_cast((width + 1) / 2) * 2 * sampleBytes; + const Uint64 lumaEnd = static_cast(layout.Offset[0]) + + static_cast(layout.Pitch[0]) * (height - 1) + + static_cast(width) * sampleBytes; + const Uint64 chromaEnd = + static_cast(layout.Offset[1]) + static_cast(layout.Pitch[1]) * (chromaHeight - 1) + chromaRow; + if (layout.Pitch[0] < width * sampleBytes || layout.Pitch[1] < chromaRow || lumaEnd > size || + chromaEnd > size) { + why = Format("its planes do not fit the %lld-byte buffer", static_cast(size)); + return nullptr; } - reg.byIdentity[key] = image; + void* map = ::mmap(nullptr, size, PROT_READ, MAP_SHARED, fd, 0); + if (map == MAP_FAILED) { + why = Format("the buffer cannot be mapped for reading (errno=%lld)", errno); + return nullptr; + } + auto foreign = MakeUnique(); + foreign->Fd = ::fcntl(fd, F_DUPFD_CLOEXEC, 0); + foreign->Layout = layout; + foreign->Map = map; + foreign->MapSize = size; + foreign->IdentityDev = dev; + foreign->IdentityIno = ino; + if (foreign->Fd < 0) { + ::munmap(map, size); + why = Format("dup errno=%lld", errno); + return nullptr; + } + // A failed allocation drops the half-built image, and ~Image unmaps and closes the source. + ImageRef image = AllocateImage(width, height, fourcc, std::move(foreign), why); + if (image == nullptr) return nullptr; + MGLOG_I("MG_Remote server: a foreign %ux%u %s dma-buf is sampled through image %llu, refreshed by a CPU " + "copy (the fallback for a buffer this platform cannot import)", + width, height, fourcc == kFourccNv12 ? "NV12" : "P010", static_cast(image->Id)); return image; +#endif + } + + Bool RefreshForeign(const Image& image, std::string& why) { +#if defined(_WIN32) + (void)image; + why = "shared images need POSIX descriptors"; + return false; +#else + if (image.Foreign == nullptr) return true; + Image::ForeignSource& source = *image.Foreign; + // WRITE AFTER READ: what a frame still samples of the old content finishes first. The copy + // is the CPU's, so this is a CPU wait - bounded, like every wait on another session. + if (!WaitForReads(image, 1000)) { + MGLOG_W_ONCE("MG_Remote server: a foreign image's readers did not finish within 1 s; refreshed anyway"); + } + const std::lock_guard lock(source.Mutex); + YuvWriteAccess access(image, why); + if (!access.Ok()) return false; + const auto* base = static_cast(source.Map); + DmaBufSync(source.Fd, true); + const Bool copied = Yuv::CopySemiPlanar(base + source.Layout.Offset[0], source.Layout.Pitch[0], + base + source.Layout.Offset[1], source.Layout.Pitch[1], image.Width, + image.Height, FourccYuvSampleBytes(image.Fourcc), access.Planes()); + DmaBufSync(source.Fd, false); + if (!copied) { + why = "the locked buffer's planes cannot hold the samples"; + return false; + } + ++source.Copies; + return true; +#endif + } + + void SetYuvHints(const Image& image, const Yuv::Hints& hints) { + const std::lock_guard lock(image.SyncMutex); + image.YuvHints = hints; + } + + Yuv::Hints GetYuvHints(const Image& image) { + const std::lock_guard lock(image.SyncMutex); + return image.YuvHints; } ImageRef Identify(int fd, std::string& why) { diff --git a/MobileGL/MG_Remote/Server/SharedImageRegistry.h b/MobileGL/MG_Remote/Server/SharedImageRegistry.h index 08c87253..0ad895a3 100644 --- a/MobileGL/MG_Remote/Server/SharedImageRegistry.h +++ b/MobileGL/MG_Remote/Server/SharedImageRegistry.h @@ -25,6 +25,8 @@ #pragma once #include +#include "SharedImageYuv.h" + #include #include @@ -44,6 +46,11 @@ namespace MobileGL::MG_Remote::Server::SharedImages { // client renders as red samples as red, whichever of the four names it allocated under. inline constexpr Uint32 kFourccArgb8888 = FourCC('A', 'R', '2', '4'); inline constexpr Uint32 kFourccXrgb8888 = FourCC('X', 'R', '2', '4'); + // YUV 4:2:0, two planes: Y, then Cb Cr interleaved at half resolution (8-bit samples; P010 the + // same with 10 significant bits in the high end of 16). A YUV image is never rendered to: it is + // SAMPLED, converted into RGBA by whoever binds it (SharedImageYuv.h). + inline constexpr Uint32 kFourccNv12 = FourCC('N', 'V', '1', '2'); + inline constexpr Uint32 kFourccP010 = FourCC('P', '0', '1', '0'); // "The layout is the allocator's": what an AHardwareBuffer is to anybody but its allocator. inline constexpr Uint64 kModifierInvalid = 0x00ffffffffffffffull; inline constexpr Uint64 kModifierLinear = 0; @@ -52,6 +59,20 @@ namespace MobileGL::MG_Remote::Server::SharedImages { Bool FourccSupported(Uint32 fourcc); // The X formats: alpha reads as 1 whatever is stored. Bool FourccIgnoresAlpha(Uint32 fourcc); + // NV12 / P010. + Bool FourccIsYuv(Uint32 fourcc); + // Bytes of one sample of a YUV fourcc (1, or 2 for P010); 0 for anything else. + Uint32 FourccYuvSampleBytes(Uint32 fourcc); + // Whether this server can allocate images of a YUV fourcc (the platform's allocator takes the + // format for sampling). Asked once per format, cached. + Bool YuvFourccAllocatable(Uint32 fourcc); + + // Where an imported dma-buf's planes are: byte offset and row pitch of each. + struct PlaneLayout { + Uint32 Count = 1; + Uint32 Offset[2] = {0, 0}; + Uint32 Pitch[2] = {0, 0}; + }; struct Image { Uint64 Id = 0; @@ -61,6 +82,11 @@ namespace MobileGL::MG_Remote::Server::SharedImages { Uint32 Stride = 0; // bytes Uint32 Offset = 0; Uint64 Modifier = kModifierInvalid; + // A YUV image's second plane (Cb Cr): same descriptor, this offset and pitch. The layout is + // still the allocator's - these are what it reported when the buffer was locked, so a + // client can name the plane, not a promise that the descriptor maps that way. + Uint32 Plane1Stride = 0; + Uint32 Plane1Offset = 0; // AHardwareBuffer* on Android; null on a host, where the memory is a memfd (unit tests). void* Native = nullptr; // The descriptor clients receive duplicates of. Owned. @@ -77,6 +103,31 @@ namespace MobileGL::MG_Remote::Server::SharedImages { mutable Uint64 WriteGeneration = 0; mutable Vector ReadFences; + // FOREIGN YUV - THE CPU-COPY FALLBACK. A YUV dma-buf this server did not allocate (a video + // decoder's frame) cannot be imported vendor-neutrally: neither the platform's EGL nor its + // Vulkan takes a raw dma-buf. Such an import becomes a YUV image of this server's (Native, + // an AHardwareBuffer the CPU can write) whose content is COPIED from a CPU mapping of the + // foreign buffer (RefreshForeign) at a reader's first use of it in each of its frames - the + // moments the producer's content can have moved for that reader. Every other image is + // zero-copy; this one costs a frame's worth of memcpy per refresh, and its log says so. + struct ForeignSource { + int Fd = -1; // a duplicate, owned + PlaneLayout Layout; + void* Map = nullptr; + SizeT MapSize = 0; + std::mutex Mutex; // one refresh at a time + Uint64 Copies = 0; + // The foreign buffer's identity, under which a later import of it finds this image. + Uint64 IdentityDev = 0; + Uint64 IdentityIno = 0; + }; + UniquePtr Foreign; + Bool IsForeign() const { return Foreign != nullptr; } + + // How the samples of a YUV image convert to RGB: the EGL_EXT_image_dma_buf_import hints + // of its last import. Guarded by SyncMutex. + mutable Yuv::Hints YuvHints; + Image() = default; Image(const Image&) = delete; Image& operator=(const Image&) = delete; @@ -91,6 +142,22 @@ namespace MobileGL::MG_Remote::Server::SharedImages { // this server never exported, or one whose identity the kernel does not keep unique. ImageRef Identify(int fd, std::string& why); + // A YUV import of a dma-buf this server did not export (Image::ForeignSource): the live + // foreign image already made from the same buffer with the same shape, or a new one, which + // keeps a duplicate of `fd`. Null with `why` when the buffer cannot be mapped, its planes do + // not fit in it, or no YUV image can be allocated to copy it into. + ImageRef ImportForeignYuv(int fd, Uint32 width, Uint32 height, Uint32 fourcc, const PlaneLayout& layout, + std::string& why); + + // A foreign image's content brought up to date from its source (the copy). Waits for the + // image's pending reads first, so a frame still sampling the old content keeps it. True for an + // image that is not foreign: there is nothing to do. + Bool RefreshForeign(const Image& image, std::string& why); + + // The colour hints a YUV image is converted with. + void SetYuvHints(const Image& image, const Yuv::Hints& hints); + Yuv::Hints GetYuvHints(const Image& image); + // The live image with this id, or null. ImageRef Find(Uint64 id); diff --git a/MobileGL/MG_Remote/Server/SharedImageYuv.h b/MobileGL/MG_Remote/Server/SharedImageYuv.h new file mode 100644 index 00000000..2c84fe12 --- /dev/null +++ b/MobileGL/MG_Remote/Server/SharedImageYuv.h @@ -0,0 +1,158 @@ +// MobileGL - MobileGL/MG_Remote/Server/SharedImageYuv.h +// Copyright (c) 2025-2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +// YUV SHARED IMAGES: the pure parts - plane copies and the YCbCr -> RGB conversion an image's +// colour hints select - kept free of Android and of any backend so the host tests can check them. +// +// A YUV image is sampled as RGBA: whoever binds it to a texture gets the image CONVERTED into the +// texture's own RGBA8 storage at its first use in each of the reader's frames (the backends do +// that on the GPU, from the image's AHardwareBuffer). The conversion is the one the EGLImage's +// EGL_EXT_image_dma_buf_import hints name - colour space, sample range and chroma siting - with +// that extension's defaults (ITU-R BT.601, narrow range, siting 0) where a hint is absent. + +#pragma once +#include + +#include + +namespace MobileGL::MG_Remote::Server::SharedImages::Yuv { + + // EGL_EXT_image_dma_buf_import's hint values. + inline constexpr Uint32 kEglItuRec601 = 0x327F; + inline constexpr Uint32 kEglItuRec709 = 0x3280; + inline constexpr Uint32 kEglItuRec2020 = 0x3281; + inline constexpr Uint32 kEglYuvFullRange = 0x3282; + inline constexpr Uint32 kEglYuvNarrowRange = 0x3283; + inline constexpr Uint32 kEglYuvChromaSiting0 = 0x3284; + inline constexpr Uint32 kEglYuvChromaSiting0_5 = 0x3285; + + // What an import said about its samples. 0 = not given (the extension's default applies). + struct Hints { + Uint32 ColorSpace = 0; + Uint32 Range = 0; + Uint32 SitingX = 0; + Uint32 SitingY = 0; + + Uint32 EffectiveColorSpace() const { + return ColorSpace == kEglItuRec709 || ColorSpace == kEglItuRec2020 ? ColorSpace : kEglItuRec601; + } + Bool FullRange() const { return Range == kEglYuvFullRange; } + Bool MidpointX() const { return SitingX == kEglYuvChromaSiting0_5; } + Bool MidpointY() const { return SitingY == kEglYuvChromaSiting0_5; } + Bool operator==(const Hints& other) const { + return ColorSpace == other.ColorSpace && Range == other.Range && SitingX == other.SitingX && + SitingY == other.SitingY; + } + }; + + // rgb = Matrix * (ycbcr - Offset), every value normalized to [0, 1] as a UNORM texture of the + // image's sample size returns it. Matrix is COLUMN-major (a GLSL mat3 uniform as is). + struct Conversion { + float Matrix[9]; + float Offset[3]; + }; + + // `sampleMax` is what a full-scale code reads as before normalization: 255 for 8-bit samples, + // 65535/64 for P010 (10 bits in the high end of 16, read as UNORM16). + inline Conversion ConversionFor(const Hints& hints, double sampleMax) { + double kr = 0.299, kb = 0.114; + if (hints.EffectiveColorSpace() == kEglItuRec709) { + kr = 0.2126; + kb = 0.0722; + } else if (hints.EffectiveColorSpace() == kEglItuRec2020) { + kr = 0.2627; + kb = 0.0593; + } + const double kg = 1.0 - kr - kb; + // Code values of an 8-bit-equivalent scale, multiplied up for deeper samples. + const double scale = sampleMax / 255.0; + double yOffset = 0.0, yScale = 1.0, cScale = 1.0; + const double cOffset = 128.0 * scale / sampleMax; + if (!hints.FullRange()) { + yOffset = 16.0 * scale / sampleMax; + yScale = sampleMax / (219.0 * scale); + cScale = sampleMax / (224.0 * scale); + } + Conversion c{}; + // Columns: Y, Cb, Cr. + const double m[9] = { + yScale, + yScale, + yScale, + 0.0, + -cScale * 2.0 * kb * (1.0 - kb) / kg, + cScale * 2.0 * (1.0 - kb), + cScale * 2.0 * (1.0 - kr), + -cScale * 2.0 * kr * (1.0 - kr) / kg, + 0.0, + }; + for (SizeT i = 0; i < 9; ++i) c.Matrix[i] = static_cast(m[i]); + c.Offset[0] = static_cast(yOffset); + c.Offset[1] = static_cast(cOffset); + c.Offset[2] = static_cast(cOffset); + return c; + } + + // One component plane of a destination: where (x, y)'s sample of it lives is + // Data + y * RowStride + x * PixelStride. + struct PlaneView { + Uint8* Data = nullptr; + Int64 RowStride = 0; + Uint32 PixelStride = 0; + }; + struct PlanarLayout { + PlaneView Y, Cb, Cr; + }; + + // A two-plane semi-planar source (NV12: Y, then Cb Cr interleaved at half resolution; P010 + // the same with 16-bit samples, `sampleBytes` = 2) into any destination layout: semi-planar + // Cb-first rows are copied as they are, everything else (Cr first, fully planar) sample by + // sample. False when the destination cannot hold the samples (null planes, a pixel stride + // smaller than a sample). + inline Bool CopySemiPlanar(const Uint8* srcY, Int64 srcYPitch, const Uint8* srcCbCr, Int64 srcCbCrPitch, + Uint32 width, Uint32 height, Uint32 sampleBytes, const PlanarLayout& dst) { + if (srcY == nullptr || srcCbCr == nullptr || dst.Y.Data == nullptr || dst.Cb.Data == nullptr || + dst.Cr.Data == nullptr || (sampleBytes != 1 && sampleBytes != 2) || dst.Y.PixelStride < sampleBytes || + dst.Cb.PixelStride < sampleBytes || dst.Cr.PixelStride < sampleBytes) + return false; + const Uint32 chromaWidth = (width + 1) / 2; + const Uint32 chromaHeight = (height + 1) / 2; + // Luma. + for (Uint32 y = 0; y < height; ++y) { + const Uint8* from = srcY + static_cast(y) * srcYPitch; + Uint8* to = dst.Y.Data + static_cast(y) * dst.Y.RowStride; + if (dst.Y.PixelStride == sampleBytes) { + std::memcpy(to, from, static_cast(width) * sampleBytes); + } else { + for (Uint32 x = 0; x < width; ++x) + std::memcpy(to + static_cast(x) * dst.Y.PixelStride, from + static_cast(x) * sampleBytes, + sampleBytes); + } + } + // Chroma. + const Bool sameInterleave = dst.Cb.PixelStride == 2 * sampleBytes && dst.Cr.PixelStride == 2 * sampleBytes && + dst.Cr.Data == dst.Cb.Data + sampleBytes && dst.Cr.RowStride == dst.Cb.RowStride; + for (Uint32 y = 0; y < chromaHeight; ++y) { + const Uint8* from = srcCbCr + static_cast(y) * srcCbCrPitch; + if (sameInterleave) { + std::memcpy(dst.Cb.Data + static_cast(y) * dst.Cb.RowStride, from, + static_cast(chromaWidth) * 2 * sampleBytes); + continue; + } + Uint8* cb = dst.Cb.Data + static_cast(y) * dst.Cb.RowStride; + Uint8* cr = dst.Cr.Data + static_cast(y) * dst.Cr.RowStride; + for (Uint32 x = 0; x < chromaWidth; ++x) { + const Uint8* pair = from + static_cast(x) * 2 * sampleBytes; + std::memcpy(cb + static_cast(x) * dst.Cb.PixelStride, pair, sampleBytes); + std::memcpy(cr + static_cast(x) * dst.Cr.PixelStride, pair + sampleBytes, sampleBytes); + } + } + return true; + } + +} // namespace MobileGL::MG_Remote::Server::SharedImages::Yuv diff --git a/MobileGL/MG_State/EGLState/Core.cpp b/MobileGL/MG_State/EGLState/Core.cpp index 3800226a..6df82949 100644 --- a/MobileGL/MG_State/EGLState/Core.cpp +++ b/MobileGL/MG_State/EGLState/Core.cpp @@ -1874,9 +1874,14 @@ namespace MobileGL { } Bool EGLContext::IsDmaBufFourccSupported(Uint32 fourcc) { - // The four orders a shared image can be allocated under (SharedImageApi.h). + // The four orders a shared image can be allocated under (SharedImageApi.h), and the + // two YUV ones that are sampled only. return fourcc == kDrmFourccAbgr8888 || fourcc == kDrmFourccXbgr8888 || fourcc == kDrmFourccArgb8888 || - fourcc == kDrmFourccXrgb8888; + fourcc == kDrmFourccXrgb8888 || IsDmaBufFourccYuv(fourcc); + } + + Bool EGLContext::IsDmaBufFourccYuv(Uint32 fourcc) { + return fourcc == kDrmFourccNv12 || fourcc == kDrmFourccP010; } Bool EGLContext::PrepareDmaBufImport(EGLDisplayHandle display, EGLContextHandle context, @@ -1899,9 +1904,11 @@ namespace MobileGL { } DmaBufImportAttribs attribs; - Bool hasWidth = false, hasHeight = false, hasFourcc = false, hasFd = false, hasOffset = false, - hasPitch = false, hasModifierLo = false, hasModifierHi = false, otherPlane = false; - Uint64 modifierLo = 0, modifierHi = 0; + Bool hasWidth = false, hasHeight = false, hasFourcc = false; + // Per plane: fd, offset, pitch, modifier lo, modifier hi. + Bool has[2][5] = {}; + Uint64 modifierLo[2] = {0, 0}, modifierHi[2] = {0, 0}; + Bool laterPlane = false; for (SizeT i = 0; attribList != nullptr && attribList[i] != EGL_NONE; i += 2) { const EGLAttrib value = attribList[i + 1]; switch (attribList[i]) { @@ -1919,29 +1926,44 @@ namespace MobileGL { break; case EGL_DMA_BUF_PLANE0_FD_EXT: attribs.Fd = static_cast(value); - hasFd = true; + has[0][0] = true; break; case EGL_DMA_BUF_PLANE0_OFFSET_EXT: attribs.Offset = static_cast(value); - hasOffset = true; + has[0][1] = true; break; case EGL_DMA_BUF_PLANE0_PITCH_EXT: attribs.Pitch = static_cast(value); - hasPitch = true; + has[0][2] = true; break; case EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT: - modifierLo = static_cast(value); - hasModifierLo = true; + modifierLo[0] = static_cast(value); + has[0][3] = true; break; case EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT: - modifierHi = static_cast(value); - hasModifierHi = true; + modifierHi[0] = static_cast(value); + has[0][4] = true; break; case EGL_DMA_BUF_PLANE1_FD_EXT: + attribs.Plane1Fd = static_cast(value); + has[1][0] = true; + break; case EGL_DMA_BUF_PLANE1_OFFSET_EXT: + attribs.Plane1Offset = static_cast(value); + has[1][1] = true; + break; case EGL_DMA_BUF_PLANE1_PITCH_EXT: + attribs.Plane1Pitch = static_cast(value); + has[1][2] = true; + break; case EGL_DMA_BUF_PLANE1_MODIFIER_LO_EXT: + modifierLo[1] = static_cast(value); + has[1][3] = true; + break; case EGL_DMA_BUF_PLANE1_MODIFIER_HI_EXT: + modifierHi[1] = static_cast(value); + has[1][4] = true; + break; case EGL_DMA_BUF_PLANE2_FD_EXT: case EGL_DMA_BUF_PLANE2_OFFSET_EXT: case EGL_DMA_BUF_PLANE2_PITCH_EXT: @@ -1952,14 +1974,22 @@ namespace MobileGL { case EGL_DMA_BUF_PLANE3_PITCH_EXT: case EGL_DMA_BUF_PLANE3_MODIFIER_LO_EXT: case EGL_DMA_BUF_PLANE3_MODIFIER_HI_EXT: - otherPlane = true; + laterPlane = true; break; - // Hints for YUV formats, and EGL_KHR_image_base's own attribute: meaningless - // for a single-plane RGB image, accepted and ignored. + // The YUV hints: kept for a YUV import, which is converted with them; meaningless + // for an RGB one and ignored there. EGL_KHR_image_base's own attribute likewise. case EGL_YUV_COLOR_SPACE_HINT_EXT: + attribs.ColorSpace = static_cast(value); + break; case EGL_SAMPLE_RANGE_HINT_EXT: + attribs.Range = static_cast(value); + break; case EGL_YUV_CHROMA_HORIZONTAL_SITING_HINT_EXT: + attribs.SitingX = static_cast(value); + break; case EGL_YUV_CHROMA_VERTICAL_SITING_HINT_EXT: + attribs.SitingY = static_cast(value); + break; case EGL_IMAGE_PRESERVED_KHR: break; default: @@ -1967,8 +1997,8 @@ namespace MobileGL { return false; } } - if (!hasWidth || !hasHeight || !hasFourcc || !hasFd || !hasOffset || !hasPitch || - hasModifierLo != hasModifierHi) { + if (!hasWidth || !hasHeight || !hasFourcc || !has[0][0] || !has[0][1] || !has[0][2] || + has[0][3] != has[0][4]) { SetError(EGL_BAD_PARAMETER); return false; } @@ -1976,23 +2006,37 @@ namespace MobileGL { SetError(EGL_BAD_MATCH); return false; } - // Every supported format has exactly one plane. - if (otherPlane) { + const Bool yuv = IsDmaBufFourccYuv(attribs.Fourcc); + attribs.PlaneCount = yuv ? 2u : 1u; + const Bool anyPlane1 = has[1][0] || has[1][1] || has[1][2] || has[1][3] || has[1][4]; + // Planes the format does not have are an attribute error; planes it has but the + // list leaves out (or half a modifier) a parameter error. + if (laterPlane || (!yuv && anyPlane1)) { SetError(EGL_BAD_ATTRIBUTE); return false; } - if (attribs.Width <= 0 || attribs.Height <= 0 || attribs.Fd < 0) { + if (yuv && (!has[1][0] || !has[1][1] || !has[1][2] || has[1][3] != has[1][4] || has[1][3] != has[0][3])) { SetError(EGL_BAD_PARAMETER); return false; } - if (attribs.Offset < 0 || attribs.Pitch <= 0) { + if (attribs.Width <= 0 || attribs.Height <= 0 || attribs.Fd < 0 || (yuv && attribs.Plane1Fd < 0)) { + SetError(EGL_BAD_PARAMETER); + return false; + } + if (attribs.Offset < 0 || attribs.Pitch <= 0 || (yuv && (attribs.Plane1Offset < 0 || attribs.Plane1Pitch <= 0))) { SetError(EGL_BAD_ACCESS); return false; } - // Any modifier is taken: the image is identified by its descriptor, never read - // through the layout a modifier describes. - attribs.HasModifier = hasModifierLo; - attribs.Modifier = (modifierHi << 32) | modifierLo; + // Every plane names the one modifier of the buffer. + if (yuv && has[0][3] && (modifierLo[0] != modifierLo[1] || modifierHi[0] != modifierHi[1])) { + SetError(EGL_BAD_MATCH); + return false; + } + // An RGBA image is identified by its descriptor, never read through the layout a + // modifier describes, so any modifier is taken; a YUV one may be a buffer the server + // copies (a foreign one), which it reads as linear rows - the server decides. + attribs.HasModifier = has[0][3]; + attribs.Modifier = (modifierHi[0] << 32) | modifierLo[0]; if (out != nullptr) *out = attribs; return true; } @@ -2034,8 +2078,8 @@ namespace MobileGL { return orphaned; } - Bool EGLContext::QueryDmaBufFormats(EGLDisplayHandle display, Bool available, EGLint maxFormats, - EGLint* formats, EGLint* numFormats) { + Bool EGLContext::QueryDmaBufFormats(EGLDisplayHandle display, Bool available, Bool yuvAvailable, + EGLint maxFormats, EGLint* formats, EGLint* numFormats) { const std::lock_guard lock(m_mutex); if (!ValidateDisplay(display)) { SetError(EGL_BAD_DISPLAY); @@ -2050,8 +2094,10 @@ namespace MobileGL { return false; } static constexpr Uint32 kFormats[] = {kDrmFourccAbgr8888, kDrmFourccXbgr8888, kDrmFourccArgb8888, - kDrmFourccXrgb8888}; - const EGLint total = available ? static_cast(std::size(kFormats)) : 0; + kDrmFourccXrgb8888, kDrmFourccNv12, kDrmFourccP010}; + constexpr EGLint kRgbCount = 4; + const EGLint total = + !available ? 0 : (yuvAvailable ? static_cast(std::size(kFormats)) : kRgbCount); if (maxFormats == 0) { *numFormats = total; return true; @@ -2062,11 +2108,10 @@ namespace MobileGL { return true; } - Bool EGLContext::QueryDmaBufModifiers(EGLDisplayHandle display, Bool available, EGLint format, - EGLint maxModifiers, Uint64* modifiers, + Bool EGLContext::QueryDmaBufModifiers(EGLDisplayHandle display, Bool available, Bool yuvAvailable, + EGLint format, EGLint maxModifiers, Uint64* modifiers, EGLBoolean* externalOnly, EGLint* numModifiers) { const std::lock_guard lock(m_mutex); - (void)externalOnly; if (!ValidateDisplay(display)) { SetError(EGL_BAD_DISPLAY); return false; @@ -2075,14 +2120,27 @@ namespace MobileGL { SetError(EGL_NOT_INITIALIZED); return false; } - if (!available || !IsDmaBufFourccSupported(static_cast(format)) || maxModifiers < 0 || - numModifiers == nullptr || (maxModifiers > 0 && modifiers == nullptr)) { + const Bool yuv = IsDmaBufFourccYuv(static_cast(format)); + if (!available || !IsDmaBufFourccSupported(static_cast(format)) || (yuv && !yuvAvailable) || + maxModifiers < 0 || numModifiers == nullptr || (maxModifiers > 0 && modifiers == nullptr)) { SetError(EGL_BAD_PARAMETER); return false; } - // No explicit modifiers: an image's layout is its allocator's, which is what an - // import without modifier attributes (the implicit modifier) means. - *numModifiers = 0; + if (!yuv) { + // No explicit modifiers: an image's layout is its allocator's, which is what an + // import without modifier attributes (the implicit modifier) means. + *numModifiers = 0; + return true; + } + // A YUV buffer is taken as linear rows when it is not one of the server's own (which + // are identified, whatever modifier is named), and only ever sampled: external only. + if (maxModifiers == 0) { + *numModifiers = 1; + return true; + } + modifiers[0] = 0; // DRM_FORMAT_MOD_LINEAR + if (externalOnly != nullptr) externalOnly[0] = EGL_TRUE; + *numModifiers = 1; return true; } } // namespace EGLState diff --git a/MobileGL/MG_State/EGLState/Core.h b/MobileGL/MG_State/EGLState/Core.h index 68c376af..ef437a1a 100644 --- a/MobileGL/MG_State/EGLState/Core.h +++ b/MobileGL/MG_State/EGLState/Core.h @@ -200,6 +200,15 @@ namespace MobileGL { EGLint Pitch = 0; Bool HasModifier = false; Uint64 Modifier = 0; + // YUV (two planes): the second plane, and the colour hints (0 = not given). + Uint32 PlaneCount = 1; + int Plane1Fd = -1; + EGLint Plane1Offset = 0; + EGLint Plane1Pitch = 0; + Uint32 ColorSpace = 0; + Uint32 Range = 0; + Uint32 SitingX = 0; + Uint32 SitingY = 0; }; struct SharedImageInfo { Uint64 Id = 0; @@ -215,11 +224,16 @@ namespace MobileGL { static constexpr Uint32 kDrmFourccXbgr8888 = 0x34324258u; // 'XB24' static constexpr Uint32 kDrmFourccArgb8888 = 0x34325241u; // 'AR24' static constexpr Uint32 kDrmFourccXrgb8888 = 0x34325258u; // 'XR24' + // Two-plane YUV 4:2:0, sampled only (through GL_OES_EGL_image_external, or as RGBA). + static constexpr Uint32 kDrmFourccNv12 = 0x3231564Eu; // 'NV12' + static constexpr Uint32 kDrmFourccP010 = 0x30313050u; // 'P010' static Bool IsDmaBufFourccSupported(Uint32 fourcc); + static Bool IsDmaBufFourccYuv(Uint32 fourcc); // The checks eglCreateImage owes an EGL_LINUX_DMA_BUF_EXT target before anything - // is imported: the display, a null context and buffer, and a complete single-plane - // attribute list of a supported format. False with the EGL error set. + // is imported: the display, a null context and buffer, and a complete attribute + // list of a supported format - one plane, or two for YUV. False with the EGL error + // set. Whether the YUV formats can be served at all is the caller's question. Bool PrepareDmaBufImport(EGLDisplayHandle display, EGLContextHandle context, EGLClientBuffer buffer, const EGLAttrib* attribList, DmaBufImportAttribs* out); // An EGLImage naming an imported shared image. @@ -230,10 +244,12 @@ namespace MobileGL { // Shared images whose EGLImages went with an eglTerminate: the caller releases them. Vector TakeOrphanedSharedImages(); // eglQueryDmaBufFormatsEXT / eglQueryDmaBufModifiersEXT. `available` is whether the - // backend has shared images at all; without them the lists are empty. - Bool QueryDmaBufFormats(EGLDisplayHandle display, Bool available, EGLint maxFormats, EGLint* formats, - EGLint* numFormats); - Bool QueryDmaBufModifiers(EGLDisplayHandle display, Bool available, EGLint format, + // backend has shared images at all; without them the lists are empty. `yuvAvailable` + // adds the YUV formats, which are external-only (OES_EGL_image_external) and LINEAR + // as far as an explicit modifier goes. + Bool QueryDmaBufFormats(EGLDisplayHandle display, Bool available, Bool yuvAvailable, + EGLint maxFormats, EGLint* formats, EGLint* numFormats); + Bool QueryDmaBufModifiers(EGLDisplayHandle display, Bool available, Bool yuvAvailable, EGLint format, EGLint maxModifiers, Uint64* modifiers, EGLBoolean* externalOnly, EGLint* numModifiers); diff --git a/MobileGL/MG_State/GLState/ProgramState/ProgramLinkTask.cpp b/MobileGL/MG_State/GLState/ProgramState/ProgramLinkTask.cpp index 5afe23f3..52ef9155 100644 --- a/MobileGL/MG_State/GLState/ProgramState/ProgramLinkTask.cpp +++ b/MobileGL/MG_State/GLState/ProgramState/ProgramLinkTask.cpp @@ -1875,8 +1875,25 @@ namespace MobileGL::MG_State::GLState { artifacts.uniformReflection.reserve(static_cast(uniformCount)); artifacts.uniformIndexByName.clear(); artifacts.uniformIndexByName.reserve(static_cast(uniformCount)); + // GL_OES_EGL_image_external: the uniforms a stage declared samplerExternalOES were + // compiled as sampler2D (LowerExternalSamplers) and are reported with their own type, + // which is what sends their units to the external binding point. + Vector externalSamplers; + for (const LinkShaderInput& shader : in.shaders) { + if (shader.compiled == nullptr) continue; + for (String& name : + MG_Util::ShaderTranspiler::ExternalSamplerUniformNames(CompiledArtifacts(shader.compiled).preprocessedSource)) + externalSamplers.push_back(Move(name)); + } + const auto isExternalSampler = [&externalSamplers](const String& reflected) { + if (externalSamplers.empty()) return false; + const String base = reflected.substr(0, reflected.find('[')); + return std::find(externalSamplers.begin(), externalSamplers.end(), base) != externalSamplers.end(); + }; for (Int i = 0; i < uniformCount; ++i) { ProgramObject::UniformReflection record = MakeResourceReflection(program.getUniform(i)); + if (record.glDefineType == GL_SAMPLER_2D && isExternalSampler(record.name)) + record.glDefineType = GL_SAMPLER_EXTERNAL_OES; // A block-level layout(row_major)/(column_major) that the member did not inherit // in its own qualifier. Resolved once HERE rather than at every GL_UNIFORM_* query, // which is what the getUniformBlock() fallback in the old accessors was doing. diff --git a/MobileGL/MG_State/GLState/ShareGroupState.cpp b/MobileGL/MG_State/GLState/ShareGroupState.cpp index 516d32a2..78528f0e 100644 --- a/MobileGL/MG_State/GLState/ShareGroupState.cpp +++ b/MobileGL/MG_State/GLState/ShareGroupState.cpp @@ -43,6 +43,8 @@ namespace MobileGL::MG_State::GLState { return MakeShared(index); case TextureTarget::Texture2DMultisampleArray: return MakeShared(index); + case TextureTarget::External: + return MakeShared(index); default: MOBILEGL_ASSERT(false, "Unimplemented texture type when creating texture object!: %d", (int)target); return nullptr; diff --git a/MobileGL/MG_State/GLState/TextureState/TextureEnum.h b/MobileGL/MG_State/GLState/TextureState/TextureEnum.h index 5934be40..3b6d8e58 100644 --- a/MobileGL/MG_State/GLState/TextureState/TextureEnum.h +++ b/MobileGL/MG_State/GLState/TextureState/TextureEnum.h @@ -21,6 +21,10 @@ namespace MobileGL { Texture2DArray, TextureCubeMapArray, Texture2DMultisampleArray, + // GL_OES_EGL_image_external: a 2D texture whose only storage is an EGLImage (a YUV one is + // converted to RGBA by the backend), with its own binding point on every unit. Below the + // frontend it IS a 2D texture: the pipe and the backends never see this target. + External, TextureTargetCount, Unknown = -1 }; diff --git a/MobileGL/MG_State/GLState/TextureState/TextureObject.cpp b/MobileGL/MG_State/GLState/TextureState/TextureObject.cpp index 98a6d0d2..d0a74b9c 100644 --- a/MobileGL/MG_State/GLState/TextureState/TextureObject.cpp +++ b/MobileGL/MG_State/GLState/TextureState/TextureObject.cpp @@ -127,7 +127,9 @@ namespace MobileGL { TextureObjectBase::TextureObjectBase(TextureTarget target, Uint externalIndex) : m_externalIndex(externalIndex), m_lifetimeId(AllocateLifetimeId()), m_target(target) { m_sampler = MakeShared(0); - if (target == TextureTarget::TextureRectangle) { + if (target == TextureTarget::TextureRectangle || target == TextureTarget::External) { + // (An external texture, GL_OES_EGL_image_external, starts the same way: LINEAR + // and CLAMP_TO_EDGE, and it has no mip chain either.) // A rectangle texture has no mip chain, so its initial sampler state is not // the shared one: TEXTURE_MIN_FILTER is LINEAR and TEXTURE_WRAP_S/T are // CLAMP_TO_EDGE (GL 4.6 core table 23.15). Leaving the 2D default of diff --git a/MobileGL/MG_State/GLState/TextureState/TextureObjectStubs.h b/MobileGL/MG_State/GLState/TextureState/TextureObjectStubs.h index 3346b095..b161e124 100644 --- a/MobileGL/MG_State/GLState/TextureState/TextureObjectStubs.h +++ b/MobileGL/MG_State/GLState/TextureState/TextureObjectStubs.h @@ -53,6 +53,10 @@ namespace MobileGL { STUB_TEXTURE_OBJECT_CLASS_DEFINITION(TextureObject2DMultisampleArray, TextureTarget::Texture2DMultisampleArray, {TextureUploadTarget::Texture2DMultisampleArray}); + // GL_OES_EGL_image_external: level 0 is defined as a 2D image (by glEGLImageTargetTexture2DOES + // only - glTexImage2D does not take the target). + STUB_TEXTURE_OBJECT_CLASS_DEFINITION(TextureObjectExternal, TextureTarget::External, + {TextureUploadTarget::Texture2D}); #undef STUB_TEXTURE_OBJECT_CLASS_DEFINITION } // namespace GLState diff --git a/MobileGL/MG_Test/EGLState/EGLStateTest.cpp b/MobileGL/MG_Test/EGLState/EGLStateTest.cpp index 1d62fdba..875c7eb2 100644 --- a/MobileGL/MG_Test/EGLState/EGLStateTest.cpp +++ b/MobileGL/MG_Test/EGLState/EGLStateTest.cpp @@ -405,40 +405,143 @@ TEST(EGLStateDmaBufImport, SharedImageLifetime) { TEST(EGLStateDmaBufImport, FormatAndModifierQueries) { auto fixture = CreateFixture(); EGLint count = -1; - ASSERT_TRUE(fixture->State.QueryDmaBufFormats(fixture->Display, true, 0, nullptr, &count)); + ASSERT_TRUE(fixture->State.QueryDmaBufFormats(fixture->Display, true, false, 0, nullptr, &count)); EXPECT_EQ(count, 4); EGLint formats[4] = {}; - ASSERT_TRUE(fixture->State.QueryDmaBufFormats(fixture->Display, true, 4, formats, &count)); + ASSERT_TRUE(fixture->State.QueryDmaBufFormats(fixture->Display, true, false, 4, formats, &count)); ASSERT_EQ(count, 4); EXPECT_EQ(formats[0], static_cast(kAbgr8888)); EXPECT_EQ(formats[1], static_cast(kXbgr8888)); EXPECT_EQ(formats[2], static_cast(kArgb8888)); EXPECT_EQ(formats[3], static_cast(kXrgb8888)); - ASSERT_TRUE(fixture->State.QueryDmaBufFormats(fixture->Display, true, 1, formats, &count)); + ASSERT_TRUE(fixture->State.QueryDmaBufFormats(fixture->Display, true, false, 1, formats, &count)); EXPECT_EQ(count, 1); // Without shared images there is nothing to import. - ASSERT_TRUE(fixture->State.QueryDmaBufFormats(fixture->Display, false, 0, nullptr, &count)); + ASSERT_TRUE(fixture->State.QueryDmaBufFormats(fixture->Display, false, false, 0, nullptr, &count)); EXPECT_EQ(count, 0); - EXPECT_FALSE(fixture->State.QueryDmaBufFormats(fixture->Display, true, -1, formats, &count)); + EXPECT_FALSE(fixture->State.QueryDmaBufFormats(fixture->Display, true, false, -1, formats, &count)); EXPECT_EQ(fixture->State.ConsumeError(), EGL_BAD_PARAMETER); // No explicit modifiers: importers fall back to the implicit one. uint64_t modifiers[2] = {}; EGLBoolean externalOnly[2] = {}; count = -1; - ASSERT_TRUE(fixture->State.QueryDmaBufModifiers(fixture->Display, true, static_cast(kAbgr8888), 2, + ASSERT_TRUE(fixture->State.QueryDmaBufModifiers(fixture->Display, true, false, static_cast(kAbgr8888), 2, modifiers, externalOnly, &count)); EXPECT_EQ(count, 0); - EXPECT_FALSE(fixture->State.QueryDmaBufModifiers(fixture->Display, true, static_cast(kRgb565), 0, + EXPECT_FALSE(fixture->State.QueryDmaBufModifiers(fixture->Display, true, false, static_cast(kRgb565), 0, nullptr, nullptr, &count)); EXPECT_EQ(fixture->State.ConsumeError(), EGL_BAD_PARAMETER); ASSERT_TRUE(fixture->State.TerminateDisplay(fixture->Display)); - EXPECT_FALSE(fixture->State.QueryDmaBufFormats(fixture->Display, true, 0, nullptr, &count)); + EXPECT_FALSE(fixture->State.QueryDmaBufFormats(fixture->Display, true, false, 0, nullptr, &count)); EXPECT_EQ(fixture->State.ConsumeError(), EGL_NOT_INITIALIZED); } +namespace { + constexpr EGLAttrib kNv12 = StateContext::kDrmFourccNv12; + constexpr EGLAttrib kP010 = StateContext::kDrmFourccP010; + + // A complete NV12 import as a video decoder exports one: both planes in one buffer. + std::vector Nv12Attribs(EGLAttrib fourcc = kNv12) { + return {EGL_WIDTH, + 1280, + EGL_HEIGHT, + 720, + EGL_LINUX_DRM_FOURCC_EXT, + fourcc, + EGL_DMA_BUF_PLANE0_FD_EXT, + 7, + EGL_DMA_BUF_PLANE0_OFFSET_EXT, + 0, + EGL_DMA_BUF_PLANE0_PITCH_EXT, + 1280, + EGL_DMA_BUF_PLANE1_FD_EXT, + 9, + EGL_DMA_BUF_PLANE1_OFFSET_EXT, + 942080, + EGL_DMA_BUF_PLANE1_PITCH_EXT, + 1280, + EGL_NONE}; + } +} // namespace + +TEST(EGLStateDmaBufImport, ParsesATwoPlaneYuvImportWithItsHints) { + auto fixture = CreateFixture(); + auto attribs = WithAttrib(Nv12Attribs(), EGL_YUV_COLOR_SPACE_HINT_EXT, EGL_ITU_REC709_EXT); + attribs = WithAttrib(attribs, EGL_SAMPLE_RANGE_HINT_EXT, EGL_YUV_NARROW_RANGE_EXT); + attribs = WithAttrib(attribs, EGL_YUV_CHROMA_HORIZONTAL_SITING_HINT_EXT, EGL_YUV_CHROMA_SITING_0_5_EXT); + StateContext::DmaBufImportAttribs parsed; + ASSERT_TRUE(fixture->State.PrepareDmaBufImport(fixture->Display, EGL_NO_CONTEXT, nullptr, attribs.data(), &parsed)); + EXPECT_EQ(parsed.PlaneCount, 2u); + EXPECT_EQ(parsed.Fd, 7); + EXPECT_EQ(parsed.Plane1Fd, 9); + EXPECT_EQ(parsed.Plane1Offset, 942080); + EXPECT_EQ(parsed.Plane1Pitch, 1280); + EXPECT_EQ(parsed.ColorSpace, static_cast(EGL_ITU_REC709_EXT)); + EXPECT_EQ(parsed.Range, static_cast(EGL_YUV_NARROW_RANGE_EXT)); + EXPECT_EQ(parsed.SitingX, static_cast(EGL_YUV_CHROMA_SITING_0_5_EXT)); + EXPECT_EQ(parsed.SitingY, 0u); + EXPECT_FALSE(parsed.HasModifier); + EXPECT_EQ(PrepareError(*fixture, Nv12Attribs(kP010)), EGL_SUCCESS); + + // One modifier for the buffer: the same on both planes, or none on either. + auto linear = WithAttrib(WithAttrib(Nv12Attribs(), EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT, 0), + EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT, 0); + EXPECT_EQ(PrepareError(*fixture, linear), EGL_BAD_PARAMETER); + linear = WithAttrib(WithAttrib(linear, EGL_DMA_BUF_PLANE1_MODIFIER_LO_EXT, 0), EGL_DMA_BUF_PLANE1_MODIFIER_HI_EXT, 0); + EXPECT_EQ(PrepareError(*fixture, linear), EGL_SUCCESS); + auto mixed = WithAttrib(WithAttrib(WithoutAttrib(WithoutAttrib(linear, EGL_DMA_BUF_PLANE1_MODIFIER_LO_EXT), + EGL_DMA_BUF_PLANE1_MODIFIER_HI_EXT), + EGL_DMA_BUF_PLANE1_MODIFIER_LO_EXT, 1), + EGL_DMA_BUF_PLANE1_MODIFIER_HI_EXT, 0); + EXPECT_EQ(PrepareError(*fixture, mixed), EGL_BAD_MATCH); +} + +TEST(EGLStateDmaBufImport, RefusesAnIncompleteYuvImport) { + auto fixture = CreateFixture(); + for (const EGLAttrib required : + {EGL_DMA_BUF_PLANE1_FD_EXT, EGL_DMA_BUF_PLANE1_OFFSET_EXT, EGL_DMA_BUF_PLANE1_PITCH_EXT}) { + EXPECT_EQ(PrepareError(*fixture, WithoutAttrib(Nv12Attribs(), required)), EGL_BAD_PARAMETER) + << "missing 0x" << std::hex << required; + } + // NV12 has two planes, not three. + EXPECT_EQ(PrepareError(*fixture, WithAttrib(Nv12Attribs(), EGL_DMA_BUF_PLANE2_FD_EXT, 7)), EGL_BAD_ATTRIBUTE); + auto badPitch = WithAttrib(WithoutAttrib(Nv12Attribs(), EGL_DMA_BUF_PLANE1_PITCH_EXT), EGL_DMA_BUF_PLANE1_PITCH_EXT, 0); + EXPECT_EQ(PrepareError(*fixture, badPitch), EGL_BAD_ACCESS); +} + +TEST(EGLStateDmaBufImport, YuvFormatsAreListedOnlyWhenTheServerHoldsThem) { + auto fixture = CreateFixture(); + EGLint count = -1; + ASSERT_TRUE(fixture->State.QueryDmaBufFormats(fixture->Display, true, false, 0, nullptr, &count)); + EXPECT_EQ(count, 4); + ASSERT_TRUE(fixture->State.QueryDmaBufFormats(fixture->Display, true, true, 0, nullptr, &count)); + EXPECT_EQ(count, 6); + EGLint formats[6] = {}; + ASSERT_TRUE(fixture->State.QueryDmaBufFormats(fixture->Display, true, true, 6, formats, &count)); + EXPECT_EQ(formats[4], static_cast(kNv12)); + EXPECT_EQ(formats[5], static_cast(kP010)); + ASSERT_TRUE(fixture->State.QueryDmaBufFormats(fixture->Display, false, true, 0, nullptr, &count)); + EXPECT_EQ(count, 0); + + // A YUV format: LINEAR, external only. + uint64_t modifiers[2] = {99, 99}; + EGLBoolean externalOnly[2] = {EGL_FALSE, EGL_FALSE}; + ASSERT_TRUE(fixture->State.QueryDmaBufModifiers(fixture->Display, true, true, static_cast(kNv12), 0, + nullptr, nullptr, &count)); + EXPECT_EQ(count, 1); + ASSERT_TRUE(fixture->State.QueryDmaBufModifiers(fixture->Display, true, true, static_cast(kNv12), 2, + modifiers, externalOnly, &count)); + EXPECT_EQ(count, 1); + EXPECT_EQ(modifiers[0], 0u); + EXPECT_EQ(externalOnly[0], EGL_TRUE); + EXPECT_FALSE(fixture->State.QueryDmaBufModifiers(fixture->Display, true, false, static_cast(kNv12), 0, + nullptr, nullptr, &count)); + EXPECT_EQ(fixture->State.ConsumeError(), EGL_BAD_PARAMETER); +} + // EGL_EXT_create_context_robustness / EGL 1.5: the reset notification behavior a context is // created with, under both names of the attribute (the extension's, which Chrome, KWin and Qt // pass, and EGL 1.5's, which ANGLE's GL-on-EGL backend passes for its own native context), the diff --git a/MobileGL/MG_Test/Gbm/FakeSharedImageLibrary.cpp b/MobileGL/MG_Test/Gbm/FakeSharedImageLibrary.cpp index 8657ede7..18b699b9 100644 --- a/MobileGL/MG_Test/Gbm/FakeSharedImageLibrary.cpp +++ b/MobileGL/MG_Test/Gbm/FakeSharedImageLibrary.cpp @@ -36,7 +36,8 @@ namespace { bool Supported(uint32_t fourcc) { return fourcc == MOBILEGL_SHARED_IMAGE_FOURCC_ABGR8888 || fourcc == MOBILEGL_SHARED_IMAGE_FOURCC_XBGR8888 || - fourcc == MOBILEGL_SHARED_IMAGE_FOURCC_ARGB8888 || fourcc == MOBILEGL_SHARED_IMAGE_FOURCC_XRGB8888; + fourcc == MOBILEGL_SHARED_IMAGE_FOURCC_ARGB8888 || fourcc == MOBILEGL_SHARED_IMAGE_FOURCC_XRGB8888 || + fourcc == MOBILEGL_SHARED_IMAGE_FOURCC_NV12; } } // namespace @@ -44,7 +45,13 @@ FAKE_EXPORT uint32_t mobilegl_shared_image_abi_version(void) { return MOBILEGL_S FAKE_EXPORT int mobilegl_server_available(void) { return 1; } -FAKE_EXPORT int mobilegl_shared_image_formats(uint32_t*, uint32_t) { return 4; } +FAKE_EXPORT int mobilegl_shared_image_formats(uint32_t* formats, uint32_t capacity) { + const uint32_t all[] = {MOBILEGL_SHARED_IMAGE_FOURCC_ABGR8888, MOBILEGL_SHARED_IMAGE_FOURCC_XBGR8888, + MOBILEGL_SHARED_IMAGE_FOURCC_ARGB8888, MOBILEGL_SHARED_IMAGE_FOURCC_XRGB8888, + MOBILEGL_SHARED_IMAGE_FOURCC_NV12}; + for (uint32_t i = 0; i < 5 && i < capacity; ++i) formats[i] = all[i]; + return 5; +} FAKE_EXPORT int mobilegl_shared_image_allocate(uint32_t width, uint32_t height, uint32_t fourcc, struct mobilegl_shared_image* out) { @@ -64,6 +71,11 @@ FAKE_EXPORT int mobilegl_shared_image_allocate(uint32_t width, uint32_t height, out->stride = width * 4; out->offset = 0; out->modifier = 0x00ffffffffffffffull; + if (fourcc == MOBILEGL_SHARED_IMAGE_FOURCC_NV12) { + out->stride = width; + out->plane1_stride = width; + out->plane1_offset = width * height; + } return 0; } @@ -103,3 +115,10 @@ FAKE_EXPORT int fake_shared_image_imports(void) { const std::lock_guard lock(g_mutex); return g_imports; } + +// A YUV import names an image by its descriptor as an RGBA one does; the planes are not looked at. +FAKE_EXPORT int mobilegl_shared_image_import_planes(int fd, uint32_t width, uint32_t height, uint32_t fourcc, + const struct mobilegl_shared_image_planes* planes, uint64_t* out_id) { + if (planes == nullptr || planes->plane_count != 2 || fourcc != MOBILEGL_SHARED_IMAGE_FOURCC_NV12) return -EINVAL; + return mobilegl_shared_image_import(fd, width, height, fourcc, out_id); +} diff --git a/MobileGL/MG_Test/Gbm/GbmGlamorTest.cpp b/MobileGL/MG_Test/Gbm/GbmGlamorTest.cpp index 99928042..eef31da0 100644 --- a/MobileGL/MG_Test/Gbm/GbmGlamorTest.cpp +++ b/MobileGL/MG_Test/Gbm/GbmGlamorTest.cpp @@ -180,3 +180,59 @@ TEST_F(GbmGlamor, AForeignDmaBufIsRefused) { EXPECT_EQ(gbm_bo_import(m_gbm, GBM_BO_IMPORT_FD, &data, GBM_BO_USE_RENDERING), nullptr); ::close(foreign); } + +// YUV buffers (ABI 2): NV12 where the library lists it, two planes in one descriptor, sampled only. +TEST_F(GbmGlamor, AnNv12BufferHasTwoPlanesInOneDescriptor) { + EXPECT_EQ(gbm_device_is_format_supported(m_gbm, GBM_FORMAT_NV12, 0), 1); + EXPECT_EQ(gbm_device_is_format_supported(m_gbm, GBM_FORMAT_NV12, GBM_BO_USE_RENDERING), 0); + EXPECT_EQ(gbm_device_is_format_supported(m_gbm, 0x30313050u /* P010 */, 0), 0) << "the library does not list P010"; + EXPECT_EQ(gbm_device_get_format_modifier_plane_count(m_gbm, GBM_FORMAT_NV12, DRM_FORMAT_MOD_INVALID), 2); + EXPECT_EQ(gbm_bo_create(m_gbm, 64, 32, GBM_FORMAT_NV12, GBM_BO_USE_RENDERING), nullptr); + + gbm_bo* bo = gbm_bo_create(m_gbm, 64, 32, GBM_FORMAT_NV12, 0); + ASSERT_NE(bo, nullptr); + EXPECT_EQ(gbm_bo_get_plane_count(bo), 2); + EXPECT_EQ(gbm_bo_get_stride_for_plane(bo, 0), 64u); + EXPECT_EQ(gbm_bo_get_stride_for_plane(bo, 1), 64u); + EXPECT_EQ(gbm_bo_get_offset(bo, 1), 64u * 32u); + EXPECT_EQ(gbm_bo_get_handle_for_plane(bo, 1).u32, gbm_bo_get_handle(bo).u32); + const int fd = gbm_bo_get_fd(bo); + const int plane1 = gbm_bo_get_fd_for_plane(bo, 1); + ASSERT_GE(fd, 0); + ASSERT_GE(plane1, 0); + + // Imported back with both planes, through one descriptor or two of the same buffer. + gbm_import_fd_modifier_data data{}; + data.width = 64; + data.height = 32; + data.format = GBM_FORMAT_NV12; + data.num_fds = 2; + data.fds[0] = fd; + data.fds[1] = plane1; + data.strides[0] = data.strides[1] = 64; + data.offsets[1] = 64 * 32; + data.modifier = DRM_FORMAT_MOD_INVALID; + gbm_bo* imported = gbm_bo_import(m_gbm, GBM_BO_IMPORT_FD_MODIFIER, &data, 0); + ASSERT_NE(imported, nullptr); + EXPECT_EQ(gbm_bo_get_plane_count(imported), 2); + EXPECT_EQ(gbm_bo_get_offset(imported, 1), 64u * 32u); + + // Planes in two different buffers, and a one-plane import that cannot say where plane 1 is. + const int other = ::memfd_create("other", MFD_CLOEXEC); + data.fds[1] = other; + EXPECT_EQ(gbm_bo_import(m_gbm, GBM_BO_IMPORT_FD_MODIFIER, &data, 0), nullptr); + gbm_import_fd_data plain{}; + plain.fd = fd; + plain.width = 64; + plain.height = 32; + plain.stride = 64; + plain.format = GBM_FORMAT_NV12; + EXPECT_EQ(gbm_bo_import(m_gbm, GBM_BO_IMPORT_FD, &plain, 0), nullptr); + + ::close(other); + gbm_bo_destroy(imported); + gbm_bo_destroy(bo); + ::close(fd); + ::close(plane1); + EXPECT_EQ(m_live(), m_liveAtStart) << "every reference was dropped"; +} diff --git a/MobileGL/MG_Test/Pipe/PipeCatalogueTest.cpp b/MobileGL/MG_Test/Pipe/PipeCatalogueTest.cpp index c6eb11f9..184ffa17 100644 --- a/MobileGL/MG_Test/Pipe/PipeCatalogueTest.cpp +++ b/MobileGL/MG_Test/Pipe/PipeCatalogueTest.cpp @@ -704,7 +704,7 @@ TEST(PipeCatalogue, LateArrivalsAreAppendedWithoutRenumbering) { // region (kMGPMaxDamageRects rectangles), as the present record carries its frame's damage. EXPECT_EQ(sizeof(MGPSharedImageOp), 296u); EXPECT_EQ(sizeof(MGPPresent), 272u); - EXPECT_EQ(sizeof(MGPSharedImageReply), 40u); + EXPECT_EQ(sizeof(MGPSharedImageReply), 48u); EXPECT_EQ(sizeof(MGPSharedImageFdOffer), 32u); EXPECT_EQ(sizeof(MGPHandleOnly), 16u); // rv's two (§5.3/§7.6): the residual-value POD - 2 + 15 Uint32s, 2 Uint8s and 2 pad bytes, diff --git a/MobileGL/MG_Test/Pipe/TextureEmitTest.cpp b/MobileGL/MG_Test/Pipe/TextureEmitTest.cpp index b861b44c..55b37a32 100644 --- a/MobileGL/MG_Test/Pipe/TextureEmitTest.cpp +++ b/MobileGL/MG_Test/Pipe/TextureEmitTest.cpp @@ -384,6 +384,13 @@ TEST(TextureEmit, EveryTextureTargetMapsToItsOwnResourceTarget) { << "TextureTarget " << i << " maps onto the BUFFER row, which the ack predicate reads"; EXPECT_NE(resourceTarget, static_cast(MGPipeResourceTarget::Renderbuffer)) << "TextureTarget " << i << " maps onto the RENDERBUFFER row"; + // The one deliberate fold: an external texture (GL_OES_EGL_image_external) is a 2D texture + // below the frontend. + if (target == TextureTarget::External) { + EXPECT_EQ(resourceTarget, static_cast(MGPipeResourceTarget::Tex2D)); + seen.push_back(resourceTarget); + continue; + } for (const Uint32 previous : seen) { EXPECT_NE(previous, resourceTarget) << "TextureTarget " << i << " shares its resource target with an earlier one"; diff --git a/MobileGL/MG_Test/Program/ProgramTest.cpp b/MobileGL/MG_Test/Program/ProgramTest.cpp index 4f5e4449..f65876f3 100644 --- a/MobileGL/MG_Test/Program/ProgramTest.cpp +++ b/MobileGL/MG_Test/Program/ProgramTest.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -5276,3 +5277,101 @@ TEST_F(ProgramTest, ASpirvModulesXfbDecorationsDoNotSurviveIntoTheTranslatedSour DrainProgramTestErrors(); } + +// GL_OES_EGL_image_external. A samplerExternalOES uniform is compiled as a sampler2D (the desktop +// profile the shader is compiled under has no external overloads; below the frontend an external +// texture is a 2D one) and reported with its own type, which is what sends its unit to the +// external binding point. +namespace { + GLuint LinkExternalSamplerProgram(const char* vertexSource, const char* fragmentSource, String& log) { + char info[2048] = ""; + const GLuint vs = CreateShader(GL_VERTEX_SHADER); + ShaderSource(vs, 1, &vertexSource, nullptr); + CompileShader(vs); + const GLuint fs = CreateShader(GL_FRAGMENT_SHADER); + ShaderSource(fs, 1, &fragmentSource, nullptr); + CompileShader(fs); + GLint compiled = GL_FALSE; + GetShaderiv(fs, GL_COMPILE_STATUS, &compiled); + GetShaderInfoLog(fs, sizeof(info), nullptr, info); + log = info; + if (compiled != GL_TRUE) return 0; + const GLuint program = CreateProgram(); + AttachShader(program, vs); + AttachShader(program, fs); + LinkProgram(program); + GLint linked = GL_FALSE; + GetProgramiv(program, GL_LINK_STATUS, &linked); + GetProgramInfoLog(program, sizeof(info), nullptr, info); + log += info; + return linked == GL_TRUE ? program : 0; + } + + GLenum UniformTypeByName(GLuint program, const char* wanted) { + GLint count = 0; + GetProgramiv(program, GL_ACTIVE_UNIFORMS, &count); + for (GLint i = 0; i < count; ++i) { + char name[64] = ""; + GLsizei length = 0; + GLint size = 0; + GLenum type = 0; + GetActiveUniform(program, static_cast(i), sizeof(name), &length, &size, &type, name); + if (String(name) == wanted) return type; + } + return GL_NONE; + } + + const char* kExternalVertexSource = "#version 300 es\n" + "out vec2 uv;\n" + "void main() { uv = vec2(0.5); gl_Position = vec4(0.0, 0.0, 0.0, 1.0); }\n"; +} // namespace + +TEST_F(ProgramTest, AnEsslExternalSamplerReportsItsOwnType) { + DrainProgramTestErrors(); + const char* fs = "#version 300 es\n" + "#extension GL_OES_EGL_image_external_essl3 : require\n" + "precision mediump float;\n" + "uniform highp samplerExternalOES videoFrame;\n" + "uniform sampler2D overlay;\n" + "in vec2 uv;\n" + "out vec4 color;\n" + "void main() { color = texture(videoFrame, uv) + texture(overlay, uv); }\n"; + String log; + const GLuint program = LinkExternalSamplerProgram(kExternalVertexSource, fs, log); + ASSERT_NE(program, 0u) << log; + EXPECT_EQ(UniformTypeByName(program, "videoFrame"), static_cast(GL_SAMPLER_EXTERNAL_OES)); + EXPECT_EQ(UniformTypeByName(program, "overlay"), static_cast(GL_SAMPLER_2D)); + DrainProgramTestErrors(); +} + +// A compositor's shader generator declares one name both ways, picking by #if: only the branch the +// preprocessor keeps says what the uniform is. +TEST_F(ProgramTest, AnExternalSamplerInAnInactiveBranchLeavesTheSampler2DAlone) { + DrainProgramTestErrors(); + const char* fsTemplate = "#version 140\n" + "%s" + "#if TRAIT_MAP_TEXTURE\n" + "uniform sampler2D sampler;\n" + "#endif\n" + "#if TRAIT_MAP_EXTERNAL_TEXTURE\n" + "#extension GL_OES_EGL_image_external : require\n" + "uniform samplerExternalOES sampler;\n" + "#endif\n" + "in vec2 uv;\n" + "out vec4 color;\n" + "void main() { color = texture(sampler, uv); }\n"; + const char* vs = "#version 140\n" + "out vec2 uv;\n" + "void main() { uv = vec2(0.5); gl_Position = vec4(0.0, 0.0, 0.0, 1.0); }\n"; + for (const auto& [traits, expected] : + {std::pair{"#define TRAIT_MAP_TEXTURE 1\n", static_cast(GL_SAMPLER_2D)}, + std::pair{"#define TRAIT_MAP_EXTERNAL_TEXTURE 1\n", static_cast(GL_SAMPLER_EXTERNAL_OES)}}) { + char fs[1024]; + std::snprintf(fs, sizeof(fs), fsTemplate, traits); + String log; + const GLuint program = LinkExternalSamplerProgram(vs, fs, log); + ASSERT_NE(program, 0u) << traits << log; + EXPECT_EQ(UniformTypeByName(program, "sampler"), expected) << traits; + } + DrainProgramTestErrors(); +} diff --git a/MobileGL/MG_Test/SelfTest/BakedInternalShadersTest.cpp b/MobileGL/MG_Test/SelfTest/BakedInternalShadersTest.cpp index 8803e5c8..a43cdfb4 100644 --- a/MobileGL/MG_Test/SelfTest/BakedInternalShadersTest.cpp +++ b/MobileGL/MG_Test/SelfTest/BakedInternalShadersTest.cpp @@ -60,6 +60,7 @@ using Uint32 = std::uint32_t; #include #include #include +#include struct BakedShader { const char* name; // ctest entry name, and the bake file's stem @@ -92,6 +93,18 @@ const BakedShader kBakedShaders[] = { "kWireColorBlitFragmentSpirv", EShLangFragment, glslang::EShTargetVulkan_1_0, glslang::EShTargetSpv_1_0, kWireColorBlitFragmentSpirv), + MGL_BAKED_ROW("WireYuvConvertVertex", + "MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvert.vert", + "MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvertSpirv.h", + "kWireYuvConvertVertexSpirv", EShLangVertex, + glslang::EShTargetVulkan_1_0, glslang::EShTargetSpv_1_0, + kWireYuvConvertVertexSpirv), + MGL_BAKED_ROW("WireYuvConvertFragment", + "MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvert.frag", + "MobileGL/MG_Backend/DirectVulkan/Renderer/WireYuvConvertSpirv.h", + "kWireYuvConvertFragmentSpirv", EShLangFragment, + glslang::EShTargetVulkan_1_0, glslang::EShTargetSpv_1_0, + kWireYuvConvertFragmentSpirv), MGL_BAKED_ROW("WireDepthMipmapVertex", "MobileGL/MG_Backend/DirectVulkan/Renderer/WireDepthMipmap.vert", "MobileGL/MG_Backend/DirectVulkan/Renderer/WireDepthMipmapSpirv.h", diff --git a/MobileGL/MG_Test/Texture/TextureTest.cpp b/MobileGL/MG_Test/Texture/TextureTest.cpp index 204481c7..8ace546c 100644 --- a/MobileGL/MG_Test/Texture/TextureTest.cpp +++ b/MobileGL/MG_Test/Texture/TextureTest.cpp @@ -6789,3 +6789,59 @@ TEST_F(TextureTest, GenerateMipmapKeepsA2DArrayLayerCountAtEveryLevel) { MG_Impl::GLImpl::BindTexture(GL_TEXTURE_2D_ARRAY, 0); DrainPendingGlErrors(); } + +// GL_OES_EGL_image_external: the external target is a binding point of its own on every unit, +// with the extension's restrictions - no data upload, no mipmaps, clamp-to-edge and non-mipmapped +// filtering, LINEAR from birth - and one image unit per texture. +TEST_F(TextureTest, TheExternalTargetIsABindingPointOfItsOwn) { + using namespace MG_Impl::GLImpl; + GLuint names[2] = {}; + GenTextures(2, names); + ActiveTexture(GL_TEXTURE3); + BindTexture(GL_TEXTURE_2D, names[0]); + BindTexture(GL_TEXTURE_EXTERNAL_OES, names[1]); + EXPECT_EQ(GetError(), GL_NO_ERROR); + GLint bound2D = 0, boundExternal = 0; + GetIntegerv(GL_TEXTURE_BINDING_2D, &bound2D); + GetIntegerv(GL_TEXTURE_BINDING_EXTERNAL_OES, &boundExternal); + EXPECT_EQ(static_cast(bound2D), names[0]); + EXPECT_EQ(static_cast(boundExternal), names[1]); + EXPECT_EQ(MG_State::pGLContext->GetTextureObject(names[1])->GetTarget(), TextureTarget::External); + + // A texture first bound as external stays external. + BindTexture(GL_TEXTURE_2D, names[1]); + ExpectSingleGlError(GL_INVALID_OPERATION); + + GLint value = 0; + GetTexParameteriv(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, &value); + EXPECT_EQ(value, GL_LINEAR); + GetTexParameteriv(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, &value); + EXPECT_EQ(value, GL_CLAMP_TO_EDGE); + GetTexParameteriv(GL_TEXTURE_EXTERNAL_OES, GL_REQUIRED_TEXTURE_IMAGE_UNITS_OES, &value); + EXPECT_EQ(value, 1); + EXPECT_EQ(GetError(), GL_NO_ERROR); + GetTexParameteriv(GL_TEXTURE_2D, GL_REQUIRED_TEXTURE_IMAGE_UNITS_OES, &value); + ExpectSingleGlError(GL_INVALID_ENUM); + + TexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + TexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + EXPECT_EQ(GetError(), GL_NO_ERROR); + TexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_REPEAT); + ExpectSingleGlError(GL_INVALID_ENUM); + TexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR); + ExpectSingleGlError(GL_INVALID_ENUM); + TexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_BASE_LEVEL, 1); + ExpectSingleGlError(GL_INVALID_OPERATION); + + // Its only storage is an EGLImage. + TexImage2D(GL_TEXTURE_EXTERNAL_OES, 0, GL_RGBA8, 4, 4, 0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr); + ExpectSingleGlError(GL_INVALID_ENUM); + GenerateMipmap(GL_TEXTURE_EXTERNAL_OES); + ExpectSingleGlError(GL_INVALID_ENUM); + + BindTexture(GL_TEXTURE_EXTERNAL_OES, 0); + BindTexture(GL_TEXTURE_2D, 0); + ActiveTexture(GL_TEXTURE0); + DeleteTextures(2, names); + EXPECT_EQ(GetError(), GL_NO_ERROR); +} diff --git a/MobileGL/MG_Test/Wire/SharedImageTest.cpp b/MobileGL/MG_Test/Wire/SharedImageTest.cpp index 502bf46b..e865ded9 100644 --- a/MobileGL/MG_Test/Wire/SharedImageTest.cpp +++ b/MobileGL/MG_Test/Wire/SharedImageTest.cpp @@ -19,7 +19,10 @@ #include #include +#include +#include #include +#include #include #include @@ -91,7 +94,7 @@ TEST(SharedImage, AnImageDiesWithItsLastReferenceAndIsNoLongerIdentified) { TEST(SharedImage, UnsupportedFormatsAndSizesAreRefused) { std::string why; - EXPECT_EQ(SI::Allocate(16, 16, SI::FourCC('N', 'V', '1', '2'), why), nullptr); + EXPECT_EQ(SI::Allocate(16, 16, SI::FourCC('R', 'G', '1', '6'), why), nullptr); // The ARGB orders are allocatable: the layout is the server's, and X ignores alpha. SI::ImageRef argb = SI::Allocate(16, 16, SI::kFourccArgb8888, why); ASSERT_NE(argb, nullptr) << why; @@ -295,3 +298,221 @@ TEST(SharedImageSync, AFlushPublishesEveryUsedImageAsAWriteAndARead) { EXPECT_EQ(SI::DupWriteFence(*windowPixmap, &generation), -1); EXPECT_EQ(generation, 2u); } + +// ---- YUV images (SharedImageYuv.h) --------------------------------------------------------------- + +namespace { + // A memfd holding a W x H NV12 frame with padded pitches and a gap before the chroma plane, the + // way a decoder lays its buffers out; sample values are a function of position. + struct ForeignFrame { + int Fd = -1; + SI::PlaneLayout Layout; + SizeT Size = 0; + }; + Uint8 LumaAt(Uint32 x, Uint32 y) { return static_cast(x * 3 + y * 7); } + Uint8 CbAt(Uint32 x, Uint32 y) { return static_cast(100 + x + y * 5); } + Uint8 CrAt(Uint32 x, Uint32 y) { return static_cast(200 - x - y * 3); } + + ForeignFrame MakeForeignNv12(Uint32 width, Uint32 height, Uint8 bias) { + ForeignFrame frame; + frame.Layout.Count = 2; + frame.Layout.Pitch[0] = width + 32; + frame.Layout.Pitch[1] = width + 64; + frame.Layout.Offset[0] = 0; + frame.Layout.Offset[1] = frame.Layout.Pitch[0] * (height + 8); + frame.Size = frame.Layout.Offset[1] + static_cast(frame.Layout.Pitch[1]) * ((height + 1) / 2); + frame.Fd = ::memfd_create("foreign-nv12", MFD_CLOEXEC); + EXPECT_GE(frame.Fd, 0); + EXPECT_EQ(::ftruncate(frame.Fd, static_cast(frame.Size)), 0); + auto* map = static_cast(::mmap(nullptr, frame.Size, PROT_READ | PROT_WRITE, MAP_SHARED, frame.Fd, 0)); + EXPECT_NE(map, MAP_FAILED); + for (Uint32 y = 0; y < height; ++y) + for (Uint32 x = 0; x < width; ++x) + map[y * frame.Layout.Pitch[0] + x] = static_cast(LumaAt(x, y) + bias); + for (Uint32 y = 0; y < (height + 1) / 2; ++y) { + for (Uint32 x = 0; x < (width + 1) / 2; ++x) { + map[frame.Layout.Offset[1] + y * frame.Layout.Pitch[1] + 2 * x] = static_cast(CbAt(x, y) + bias); + map[frame.Layout.Offset[1] + y * frame.Layout.Pitch[1] + 2 * x + 1] = + static_cast(CrAt(x, y) + bias); + } + } + ::munmap(map, frame.Size); + return frame; + } + + // The image's own samples (a host image is a tight semi-planar memfd) equal the frame's. + void ExpectImageHolds(const SI::Image& image, Uint8 bias) { + const SizeT size = image.Plane1Offset + static_cast(image.Plane1Stride) * ((image.Height + 1) / 2); + auto* map = static_cast(::mmap(nullptr, size, PROT_READ, MAP_SHARED, image.Fd, 0)); + ASSERT_NE(map, MAP_FAILED); + Uint32 wrong = 0; + for (Uint32 y = 0; y < image.Height; ++y) + for (Uint32 x = 0; x < image.Width; ++x) + wrong += map[y * image.Stride + x] != static_cast(LumaAt(x, y) + bias); + for (Uint32 y = 0; y < (image.Height + 1) / 2; ++y) { + for (Uint32 x = 0; x < (image.Width + 1) / 2; ++x) { + wrong += map[image.Plane1Offset + y * image.Plane1Stride + 2 * x] != static_cast(CbAt(x, y) + bias); + wrong += map[image.Plane1Offset + y * image.Plane1Stride + 2 * x + 1] != + static_cast(CrAt(x, y) + bias); + } + } + EXPECT_EQ(wrong, 0u); + ::munmap(map, size); + } +} // namespace + +TEST(SharedImageYuv, AnNv12ImageHasTwoPlanesInOneDescriptor) { + std::string why; + ASSERT_TRUE(SI::FourccSupported(SI::kFourccNv12)); + SI::ImageRef image = SI::Allocate(64, 32, SI::kFourccNv12, why); + ASSERT_NE(image, nullptr) << why; + EXPECT_TRUE(SI::FourccIsYuv(image->Fourcc)); + EXPECT_FALSE(image->IsForeign()); + EXPECT_GE(image->Stride, 64u); + EXPECT_GE(image->Plane1Stride, 64u); + EXPECT_GE(image->Plane1Offset, image->Stride * 32u); + // Its own descriptor names it, like any image this server exported. + const int peer = ThroughSocket(image->Fd); + EXPECT_EQ(SI::Identify(peer, why), image); + ::close(peer); +} + +// THE CPU-COPY FALLBACK: a buffer this server never allocated becomes an image of its own, filled +// from the buffer's planes at each refresh; the same buffer imported again is the same image. +TEST(SharedImageYuv, AForeignBufferIsCopiedIntoAnImageOfItsOwn) { + std::string why; + ForeignFrame frame = MakeForeignNv12(48, 30, 0); + ASSERT_GE(frame.Fd, 0); + // Not one of the server's: Identify refuses it, the foreign import takes it. + EXPECT_EQ(SI::Identify(frame.Fd, why), nullptr); + const int peer = ThroughSocket(frame.Fd); + SI::ImageRef image = SI::ImportForeignYuv(peer, 48, 30, SI::kFourccNv12, frame.Layout, why); + ASSERT_NE(image, nullptr) << why; + EXPECT_TRUE(image->IsForeign()); + ASSERT_TRUE(SI::RefreshForeign(*image, why)) << why; + ExpectImageHolds(*image, 0); + + // New content in the same buffer shows at the next refresh. + { + ForeignFrame next = MakeForeignNv12(48, 30, 9); + auto* source = static_cast(::mmap(nullptr, next.Size, PROT_READ, MAP_SHARED, next.Fd, 0)); + auto* target = static_cast(::mmap(nullptr, frame.Size, PROT_READ | PROT_WRITE, MAP_SHARED, frame.Fd, 0)); + std::memcpy(target, source, frame.Size); + ::munmap(source, next.Size); + ::munmap(target, frame.Size); + ::close(next.Fd); + } + ASSERT_TRUE(SI::RefreshForeign(*image, why)) << why; + ExpectImageHolds(*image, 9); + + // The same buffer with the same planes is the same image; other planes make another one. + EXPECT_EQ(SI::ImportForeignYuv(frame.Fd, 48, 30, SI::kFourccNv12, frame.Layout, why), image); + SI::PlaneLayout shifted = frame.Layout; + shifted.Pitch[1] -= 2; + SI::ImageRef other = SI::ImportForeignYuv(frame.Fd, 48, 30, SI::kFourccNv12, shifted, why); + ASSERT_NE(other, nullptr) << why; + EXPECT_NE(other, image); + ::close(peer); + ::close(frame.Fd); +} + +TEST(SharedImageYuv, AForeignBufferTooSmallForItsPlanesIsRefused) { + std::string why; + ForeignFrame frame = MakeForeignNv12(48, 30, 0); + SI::PlaneLayout beyond = frame.Layout; + beyond.Offset[1] = static_cast(frame.Size); + EXPECT_EQ(SI::ImportForeignYuv(frame.Fd, 48, 30, SI::kFourccNv12, beyond, why), nullptr); + EXPECT_FALSE(why.empty()); + SI::PlaneLayout narrow = frame.Layout; + narrow.Pitch[0] = 40; // shorter than a row + EXPECT_EQ(SI::ImportForeignYuv(frame.Fd, 48, 30, SI::kFourccNv12, narrow, why), nullptr); + SI::PlaneLayout onePlane = frame.Layout; + onePlane.Count = 1; + EXPECT_EQ(SI::ImportForeignYuv(frame.Fd, 48, 30, SI::kFourccNv12, onePlane, why), nullptr); + EXPECT_EQ(SI::ImportForeignYuv(frame.Fd, 48, 30, SI::kFourccAbgr8888, frame.Layout, why), nullptr); + ::close(frame.Fd); +} + +// The plane copy into the layouts a platform's lock can hand out: semi-planar Cb first (copied by +// rows), Cr first and fully planar (sample by sample), and 16-bit samples. +TEST(SharedImageYuv, ThePlaneCopyFillsEveryDestinationLayout) { + constexpr Uint32 W = 6, H = 4; + Uint8 y[H][W], cbcr[H / 2][W]; + for (Uint32 r = 0; r < H; ++r) + for (Uint32 c = 0; c < W; ++c) y[r][c] = static_cast(r * 10 + c); + for (Uint32 r = 0; r < H / 2; ++r) + for (Uint32 c = 0; c < W / 2; ++c) { + cbcr[r][2 * c] = static_cast(100 + r * 10 + c); + cbcr[r][2 * c + 1] = static_cast(200 + r * 10 + c); + } + const auto check = [&](const SI::Yuv::PlanarLayout& dst) { + for (Uint32 r = 0; r < H; ++r) + for (Uint32 c = 0; c < W; ++c) EXPECT_EQ(dst.Y.Data[r * dst.Y.RowStride + c * dst.Y.PixelStride], y[r][c]); + for (Uint32 r = 0; r < H / 2; ++r) + for (Uint32 c = 0; c < W / 2; ++c) { + EXPECT_EQ(dst.Cb.Data[r * dst.Cb.RowStride + c * dst.Cb.PixelStride], cbcr[r][2 * c]); + EXPECT_EQ(dst.Cr.Data[r * dst.Cr.RowStride + c * dst.Cr.PixelStride], cbcr[r][2 * c + 1]); + } + }; + std::vector luma(16 * H), chroma(16 * H); + SI::Yuv::PlanarLayout nv12{{luma.data(), 16, 1}, {chroma.data(), 16, 2}, {chroma.data() + 1, 16, 2}}; + ASSERT_TRUE(SI::Yuv::CopySemiPlanar(&y[0][0], W, &cbcr[0][0], W, W, H, 1, nv12)); + check(nv12); + SI::Yuv::PlanarLayout nv21{{luma.data(), 16, 1}, {chroma.data() + 1, 16, 2}, {chroma.data(), 16, 2}}; + ASSERT_TRUE(SI::Yuv::CopySemiPlanar(&y[0][0], W, &cbcr[0][0], W, W, H, 1, nv21)); + check(nv21); + std::vector cb(8 * H), cr(8 * H); + SI::Yuv::PlanarLayout planar{{luma.data(), 16, 1}, {cb.data(), 8, 1}, {cr.data(), 8, 1}}; + ASSERT_TRUE(SI::Yuv::CopySemiPlanar(&y[0][0], W, &cbcr[0][0], W, W, H, 1, planar)); + check(planar); + SI::Yuv::PlanarLayout missing{{luma.data(), 16, 1}, {nullptr, 8, 1}, {cr.data(), 8, 1}}; + EXPECT_FALSE(SI::Yuv::CopySemiPlanar(&y[0][0], W, &cbcr[0][0], W, W, H, 1, missing)); + + // 16-bit samples (P010): two bytes each, Cb Cr pairs of four. + Uint16 y16[H][W], c16[H / 2][W]; + for (Uint32 r = 0; r < H; ++r) + for (Uint32 c = 0; c < W; ++c) y16[r][c] = static_cast((r * 10 + c) << 6); + for (Uint32 r = 0; r < H / 2; ++r) + for (Uint32 c = 0; c < W; ++c) c16[r][c] = static_cast((500 + r * 10 + c) << 6); + std::vector luma16(16 * H), chroma16(16 * H); + SI::Yuv::PlanarLayout p010{{reinterpret_cast(luma16.data()), 32, 2}, + {reinterpret_cast(chroma16.data()), 32, 4}, + {reinterpret_cast(chroma16.data()) + 2, 32, 4}}; + ASSERT_TRUE(SI::Yuv::CopySemiPlanar(reinterpret_cast(&y16[0][0]), W * 2, + reinterpret_cast(&c16[0][0]), W * 2, W, H, 2, p010)); + for (Uint32 r = 0; r < H; ++r) + for (Uint32 c = 0; c < W; ++c) EXPECT_EQ(luma16[r * 16 + c], y16[r][c]); + for (Uint32 r = 0; r < H / 2; ++r) + for (Uint32 c = 0; c < W; ++c) EXPECT_EQ(chroma16[r * 16 + c], c16[r][c]); +} + +// The conversion an import's hints select: BT.709 narrow-range codes of saturated colours decode +// to those colours, and the defaults are BT.601 narrow. +TEST(SharedImageYuv, TheConversionDecodesTheHintedColourSpace) { + const auto decode = [](const SI::Yuv::Conversion& c, double y, double cb, double cr, double max) { + const double in[3] = {y / max - c.Offset[0], cb / max - c.Offset[1], cr / max - c.Offset[2]}; + std::array rgb{}; + for (int row = 0; row < 3; ++row) + for (int col = 0; col < 3; ++col) rgb[row] += c.Matrix[col * 3 + row] * in[col]; + return rgb; + }; + SI::Yuv::Hints bt709; + bt709.ColorSpace = SI::Yuv::kEglItuRec709; + bt709.Range = SI::Yuv::kEglYuvNarrowRange; + const auto c709 = SI::Yuv::ConversionFor(bt709, 255.0); + // Red, green, blue and white in BT.709 narrow range (8-bit codes). + const double codes[4][3] = {{63, 102, 240}, {173, 42, 26}, {32, 240, 118}, {235, 128, 128}}; + const double want[4][3] = {{1, 0, 0}, {0, 1, 0}, {0, 0, 1}, {1, 1, 1}}; + for (int i = 0; i < 4; ++i) { + const auto rgb = decode(c709, codes[i][0], codes[i][1], codes[i][2], 255.0); + for (int k = 0; k < 3; ++k) EXPECT_NEAR(rgb[k], want[i][k], 0.02) << "colour " << i << " channel " << k; + } + // Full range: Y passes through. + SI::Yuv::Hints full; + full.Range = SI::Yuv::kEglYuvFullRange; + const auto cFull = SI::Yuv::ConversionFor(full, 255.0); + const auto grey = decode(cFull, 128, 128, 128, 255.0); + for (int k = 0; k < 3; ++k) EXPECT_NEAR(grey[k], 128.0 / 255.0, 0.005); + EXPECT_EQ(SI::Yuv::Hints{}.EffectiveColorSpace(), SI::Yuv::kEglItuRec601); + EXPECT_FALSE(SI::Yuv::Hints{}.FullRange()); +} diff --git a/MobileGL/MG_Util/Converters/GLToMG/TextureEnumConverter.cpp b/MobileGL/MG_Util/Converters/GLToMG/TextureEnumConverter.cpp index ee99c704..8df858aa 100644 --- a/MobileGL/MG_Util/Converters/GLToMG/TextureEnumConverter.cpp +++ b/MobileGL/MG_Util/Converters/GLToMG/TextureEnumConverter.cpp @@ -51,6 +51,8 @@ namespace MobileGL { case GL_TEXTURE_2D_MULTISAMPLE_ARRAY: case GL_PROXY_TEXTURE_2D_MULTISAMPLE_ARRAY: return TextureTarget::Texture2DMultisampleArray; + case GL_TEXTURE_EXTERNAL_OES: + return TextureTarget::External; default: return TextureTarget::Unknown; } diff --git a/MobileGL/MG_Util/Converters/MGToGL/TextureEnumConverter.cpp b/MobileGL/MG_Util/Converters/MGToGL/TextureEnumConverter.cpp index daea937f..88fbb456 100644 --- a/MobileGL/MG_Util/Converters/MGToGL/TextureEnumConverter.cpp +++ b/MobileGL/MG_Util/Converters/MGToGL/TextureEnumConverter.cpp @@ -36,6 +36,8 @@ namespace MobileGL { return GL_TEXTURE_RECTANGLE; case TextureTarget::Texture2DMultisampleArray: return GL_TEXTURE_2D_MULTISAMPLE_ARRAY; + case TextureTarget::External: + return GL_TEXTURE_EXTERNAL_OES; default: return GL_UNKNOWN_MGL; } diff --git a/MobileGL/MG_Util/Converters/MGToStr/TextureEnumConverter.cpp b/MobileGL/MG_Util/Converters/MGToStr/TextureEnumConverter.cpp index 2eda2552..d1c5580b 100644 --- a/MobileGL/MG_Util/Converters/MGToStr/TextureEnumConverter.cpp +++ b/MobileGL/MG_Util/Converters/MGToStr/TextureEnumConverter.cpp @@ -36,6 +36,8 @@ namespace MobileGL { return "Texture1DArray"; case TextureTarget::Texture2DMultisampleArray: return "Texture2DMultisampleArray"; + case TextureTarget::External: + return "External"; default: return "Unknown"; } diff --git a/MobileGL/MG_Util/Converters/MGToVk/TextureEnumConverter.cpp b/MobileGL/MG_Util/Converters/MGToVk/TextureEnumConverter.cpp index c36069ec..0f601d53 100644 --- a/MobileGL/MG_Util/Converters/MGToVk/TextureEnumConverter.cpp +++ b/MobileGL/MG_Util/Converters/MGToVk/TextureEnumConverter.cpp @@ -30,6 +30,7 @@ namespace MobileGL { case TextureTarget::Texture1DArray: return VK_IMAGE_VIEW_TYPE_1D_ARRAY; case TextureTarget::TextureRectangle: + case TextureTarget::External: return VK_IMAGE_VIEW_TYPE_2D; case TextureTarget::Texture2DMultisampleArray: return VK_IMAGE_VIEW_TYPE_2D_ARRAY; diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp index 49d8877a..d8164601 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp @@ -465,6 +465,30 @@ namespace MobileGL { } } // namespace + Bool PreprocessForInspection(GLenum shaderType, const String& source, String& out) { + const EShLanguage lang = MG_Util::ConvertGLEnumToEShLanguage(shaderType); + if (lang == EShLanguage::EShLangCount) return false; + Bool ok = false; + { + glslang::TShader shader(lang); + const char* strings[] = {source.c_str()}; + shader.setStrings(strings, 1); + const String preamble = String("#undef VULKAN\n") + CollectEsPreambleMacroDefines(source); + shader.setPreamble(preamble.c_str()); + shader.setEnvInput(glslang::EShSourceGlsl, lang, glslang::EShClientVulkan, 450); + shader.setEnvClient(glslang::EShClientOpenGL, glslang::EShTargetOpenGL_450); + shader.setEnvTarget(glslang::EShTargetSpv, glslang::EShTargetSpv_1_3); + const auto resources = BuildTBuiltInResource(&*GetDefaultCompileEnv()); + glslang::TShader::ForbidIncluder includer; + std::string text; + ok = shader.preprocess(&resources, 460, ECoreProfile, false, true, EShMsgDefault, &text, includer); + if (ok) out = text; + } + // The shader's pool is gone with it; leave no dangling thread allocator behind. + glslang::SetThreadPoolAllocator(nullptr); + return ok; + } + UnorderedMap CollectExplicitUniformLocations(const glslang::TShader& shader) { UnorderedMap locations; const glslang::TIntermediate* intermediate = shader.getIntermediate(); diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h index efe3225e..f8c06643 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h @@ -666,6 +666,12 @@ namespace MobileGL { // reflection needs those spelled out. A uniform declared in several stages must // agree, which the caller enforces across stages. UnorderedMap CollectExplicitUniformLocations(const glslang::TShader& shader); + + // `source` run through glslang's PREPROCESSOR only (inactive #if branches gone, macros + // expanded), under the same preamble a parse would use, into `out`. For questions about + // the text a parse actually sees - which of two #if-alternative declarations is live. + // False (and `out` untouched) when glslang refuses it. + Bool PreprocessForInspection(GLenum shaderType, const String& source, String& out); } // namespace ShaderTranspiler } // namespace MG_Util } // namespace MobileGL diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.cpp b/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.cpp index 9434690e..ef72053a 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.cpp +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.cpp @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include #include "EsslBuiltinFunctionNames.h" @@ -1813,6 +1815,84 @@ namespace { "#define gl_NumSamples mg_NumSamples\n"; source.insert(afterVersion.Get(source), shim); } + + // The uniforms `text` declares with the external sampler type, and (into `others`) the names it + // declares with any other type - the same name can be both in a source whose #if branches pick + // one (a compositor's shader generator does exactly that). + void ScanExternalSamplerDeclarations(const MobileGL::String& text, Vector& external, + Vector* others) { + const Vector tokens = TokenizeCode(text); + for (SizeT i = 0; i < tokens.size(); ++i) { + if (tokens[i].text != "uniform") continue; + // The type is the first identifier after `uniform` that is not a precision qualifier. + SizeT type = i + 1; + while (type < tokens.size() && (tokens[type].text == "lowp" || tokens[type].text == "mediump" || + tokens[type].text == "highp")) + ++type; + if (type >= tokens.size() || !IsIdentifierToken(tokens[type])) continue; + const bool isExternal = tokens[type].text == "samplerExternalOES"; + Vector* into = isExternal ? &external : others; + // `name`, `name[N]`, and comma-separated lists of them, up to the `;` (a block's `{` + // ends the scan: its members are not samplers). + for (SizeT j = type + 1; j < tokens.size() && tokens[j].text != ";" && tokens[j].text != "{";) { + if (IsIdentifierToken(tokens[j])) { + if (into != nullptr && std::find(into->begin(), into->end(), tokens[j].text) == into->end()) + into->push_back(tokens[j].text); + ++j; + if (j < tokens.size() && tokens[j].text == "[") { + while (j < tokens.size() && tokens[j].text != "]") ++j; + ++j; + } + continue; + } + ++j; + } + } + } + + // GL_OES_EGL_image_external's sampler type, which the desktop profile the shader is compiled + // under has no texture() overloads for. Below the frontend an external texture IS a 2D one (its + // EGLImage arrives converted to RGBA), so the type becomes sampler2D, and the names of the + // uniforms declared with it ride in a marker (kExternalSamplerMarkerPrefix) to the reflection, + // which reports them as GL_SAMPLER_EXTERNAL_OES and so sends their units to the external binding. + // The marker is also what keeps such a shader's caches apart from a sampler2D twin's. + // + // THE NAMES COME FROM THE PREPROCESSED TEXT. A source may declare one name as samplerExternalOES + // in one #if branch and as sampler2D in another (a compositor's shader generator does, per trait); only + // the branch the preprocessor keeps says what the uniform is. Should glslang's preprocessor + // refuse the source, a name declared both ways is left out - a 2D binding is the safe reading. + constexpr const char* kExternalSamplerMarkerPrefix = "/*mobilegl-external-samplers:"; + + void LowerExternalSamplers(MobileGL::ShaderStage stage, MobileGL::String& source, AfterVersionAnchor& afterVersion) { + if (source.find("samplerExternalOES") == MobileGL::String::npos) return; + Vector names; + MobileGL::String preprocessed; + if (MobileGL::MG_Util::ShaderTranspiler::PreprocessForInspection( + MobileGL::MG_Util::ConvertShaderStageToGLEnum(stage), source, preprocessed)) { + ScanExternalSamplerDeclarations(preprocessed, names, nullptr); + } else { + Vector external, others; + ScanExternalSamplerDeclarations(source, external, &others); + for (const MobileGL::String& name : external) { + if (std::find(others.begin(), others.end(), name) == others.end()) names.push_back(name); + } + } + // Every spelling of the type, in any branch: what the preprocessor drops does not matter. + Vector typeOffsets; + for (const CodeToken& token : TokenizeCode(source)) { + if (token.text == "samplerExternalOES") typeOffsets.push_back(token.begin); + } + if (typeOffsets.empty()) return; + for (auto offset = typeOffsets.rbegin(); offset != typeOffsets.rend(); ++offset) + source.replace(*offset, std::strlen("samplerExternalOES"), "sampler2D"); + MobileGL::String marker = kExternalSamplerMarkerPrefix; + for (SizeT i = 0; i < names.size(); ++i) { + if (i != 0) marker += ','; + marker += names[i]; + } + marker += "*/\n"; + source.insert(afterVersion.Get(source), marker); + } } // namespace namespace MobileGL { @@ -1870,12 +1950,30 @@ namespace MobileGL { RenameBuiltinShadowingFunctions(source); + LowerExternalSamplers(stage, source, afterVersion); ModernizeLegacyGLSL(stage, source, afterVersion); InjectDepthRangeBuiltinShim(stage, source, afterVersion); InjectNumSamplesBuiltinShim(stage, originalLanguage, source, afterVersion); } + Vector ExternalSamplerUniformNames(const String& preprocessedSource) { + Vector names; + const SizeT start = preprocessedSource.find(kExternalSamplerMarkerPrefix); + if (start == String::npos) return names; + const SizeT from = start + std::strlen(kExternalSamplerMarkerPrefix); + const SizeT end = preprocessedSource.find("*/", from); + if (end == String::npos) return names; + SizeT at = from; + while (at < end) { + SizeT comma = preprocessedSource.find(',', at); + if (comma == String::npos || comma > end) comma = end; + if (comma > at) names.push_back(preprocessedSource.substr(at, comma - at)); + at = comma + 1; + } + return names; + } + String CollectEsPreambleMacroDefines(const String& preprocessedSource) { const SizeT markerStart = preprocessedSource.find(kEsPreambleMarkerPrefix); if (markerStart == String::npos) return {}; diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.h b/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.h index fb673ebe..d19ab93a 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.h +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.h @@ -61,6 +61,11 @@ namespace MobileGL { // GetNormalizedVersionDirective records why that has not happened. String CollectEsPreambleMacroDefines(const String& preprocessedSource); + // The uniforms a shader declared as samplerExternalOES (GL_OES_EGL_image_external), + // which PreprocessShaderSource turned into sampler2D and named in a marker: the + // reflection reports them as GL_SAMPLER_EXTERNAL_OES. Empty for every other shader. + Vector ExternalSamplerUniformNames(const String& preprocessedSource); + // GLSL reserves a few names glslang happily accepts as identifiers ("packed", // "row_major" outside a layout(...) list, the image*Shadow family). Returns the // compile-error text for the first violation, or nullopt for a clean source. diff --git a/MobileGL/MG_Util/Types.h b/MobileGL/MG_Util/Types.h index 40a9568f..8e7e82b4 100644 --- a/MobileGL/MG_Util/Types.h +++ b/MobileGL/MG_Util/Types.h @@ -13,6 +13,20 @@ #define GL_UNKNOWN_MGL 0 +// GL_OES_EGL_image_external, which the desktop headers do not carry. +#ifndef GL_TEXTURE_EXTERNAL_OES +#define GL_TEXTURE_EXTERNAL_OES 0x8D65 +#endif +#ifndef GL_SAMPLER_EXTERNAL_OES +#define GL_SAMPLER_EXTERNAL_OES 0x8D66 +#endif +#ifndef GL_TEXTURE_BINDING_EXTERNAL_OES +#define GL_TEXTURE_BINDING_EXTERNAL_OES 0x8D67 +#endif +#ifndef GL_REQUIRED_TEXTURE_IMAGE_UNITS_OES +#define GL_REQUIRED_TEXTURE_IMAGE_UNITS_OES 0x8D68 +#endif + #define THROW_EXCEPTION(msg) throw std::runtime_error(msg) #define THROW_UNIMPL_EXCEPTION THROW_EXCEPTION("Unimplemented function called!")