From e5fb57f7ebc5550c89d1ee5c44682a7eca363520 Mon Sep 17 00:00:00 2001 From: BZLZHH Date: Sat, 8 Aug 2026 10:33:50 -0400 Subject: [PATCH] [Feat] (MG_State, MG_Util): async shader compilation behind the default-off flag (P1 stage 3) glCompileShader with MOBILEGL_ASYNC_SHADER_COMPILE=1 snapshots its inputs on the GL thread (source SharedPtr, CompileEnv, cache handle) and runs the whole pure pipeline - preprocess, validators, extractors, glslang parse - as a ShaderCompileTask on the worker pool, returning immediately. Every read of compile-produced state joins through the single Compiled() gate; links stay synchronous this stage and join their attached shaders at the top of the body. Flag off, the path is the same code run inline. Mechanics: the job node owns all its inputs (no back-pointer, no lifetime tie to the shader object), so re-sourcing or deleting a pending shader is cancel-and-drop, never a wait; glslang worker hygiene is a TLS-allocator scope guard plus GL-thread builtin prewarm (gated on the flag, latch reset on Destroy so re-initialization re-warms); worker-side diagnostics defer through the job and replay on the GL thread at the join, enforced by IsPoolThread asserts in RecordError and an empty-deferred-errors tripwire. A body that throws publishes a COMPLETE failed compile (status false, real info log) rather than an abandoned node, and never memoizes away the retry; a failed enqueue (OOM) cancels the node instead of stranding the joiner - including inside the dispatch loop, where the in-flight slot is repaid. The pool StopAndDrains from an atexit sentinel too: workers still inside glslang parse while exit() ran static destructors was a real 2-in-5 SIGSEGV, reproduced and fixed (15/15 clean after). Backend-internal shader objects (default FS, DirectVulkan blit/mipmap) are cache-less and always compile inline - compile-and-read-in-one-breath needs no round trip. Gates: unit suite 488/488 with the flag off AND on (x5); AsyncCompileTest (12 e2e cases: pending re-source/delete/recompile, byte-identical failure logs across modes, 48-compile cache stress) x10 repeats clean both modes; full NVIDIA DirectGLES retrace identical result sets flag off/on (zero new deltas); compile-phase timing flat as designed (links still serial - the parallel win arrives with stage 4's async link + stage 5's KHR_parallel_shader_compile). --- CMakeLists.txt | 1 + MobileGL/Init.cpp | 18 + MobileGL/MG_State/GLState/Core.cpp | 8 + .../GLState/ProgramState/ProgramObject.cpp | 14 + .../GLState/ProgramState/ProgramState.cpp | 5 + .../ProgramState/ShaderCompileTask.cpp | 303 ++++++++++ .../GLState/ProgramState/ShaderCompileTask.h | 83 +++ .../GLState/ProgramState/ShaderObject.cpp | 362 +++-------- .../GLState/ProgramState/ShaderObject.h | 198 +++--- .../ProgramState/ShaderPreprocessCache.h | 4 +- .../GLState/ProgramState/ShaderStage.h | 25 + MobileGL/MG_Test/Program/AsyncCompileTest.cpp | 571 ++++++++++++++++++ MobileGL/MG_Test/Program/CMakeLists.txt | 19 + MobileGL/MG_Test/Util/JobNodeTest.cpp | 21 +- MobileGL/MG_Util/Async/JobNode.cpp | 33 + MobileGL/MG_Util/Async/JobNode.h | 11 + MobileGL/MG_Util/Async/ShaderCompilePool.cpp | 77 ++- .../ShaderTranspiler/ShaderCompiler.cpp | 50 ++ .../MG_Util/ShaderTranspiler/ShaderCompiler.h | 24 + 19 files changed, 1448 insertions(+), 379 deletions(-) create mode 100644 MobileGL/MG_State/GLState/ProgramState/ShaderCompileTask.cpp create mode 100644 MobileGL/MG_State/GLState/ProgramState/ShaderCompileTask.h create mode 100644 MobileGL/MG_State/GLState/ProgramState/ShaderStage.h create mode 100644 MobileGL/MG_Test/Program/AsyncCompileTest.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 75c548ac..5a7ece4a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -294,6 +294,7 @@ set(SOURCE_FILES MobileGL/MG_State/GLState/TextureState/TextureUnit.cpp MobileGL/MG_State/GLState/TextureState/TextureState.cpp MobileGL/MG_State/GLState/ProgramState/ProgramObject.cpp + MobileGL/MG_State/GLState/ProgramState/ShaderCompileTask.cpp MobileGL/MG_State/GLState/ProgramState/ShaderObject.cpp MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.cpp MobileGL/MG_State/GLState/ProgramState/ProgramState.cpp diff --git a/MobileGL/Init.cpp b/MobileGL/Init.cpp index c7a0b983..905be751 100644 --- a/MobileGL/Init.cpp +++ b/MobileGL/Init.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -61,6 +62,10 @@ namespace MobileGL { // still reference levels adopted from those tables. Finalizing first left live // glslang objects pointing at freed memory for the rest of the teardown. glslang::FinalizeProcess(); + // Immediately after, and never apart from it: FinalizeProcess just deleted the + // built-in symbol tables the prewarm latch stands for, so leaving it set would + // make the next Initialize() skip a prewarm it genuinely needs. + MG_Util::ShaderTranspiler::ShaderCompiler::ResetPrewarmLatch(); MG_Backend::gBackendFunctionsTable = {}; g_isInitialized = false; if (logLifecycle) { @@ -88,6 +93,19 @@ namespace MobileGL { MG_Impl::Init(); MGLOG_D("MG_Impl initialized"); glslang::InitializeProcess(); + // On the GL thread, before any worker can exist. glslang builds its built-in symbol + // tables lazily under a process-wide lock held for the whole build, so without this + // the first concurrent compiles of a shaderpack all serialize behind the very first + // parse and asynchronous compilation looks like it is doing nothing. + // + // Gated on the flag, because the problem it solves only exists when there are + // workers: with compilation synchronous, nothing ever contends for that lock and the + // three throwaway parses buy nothing - they just add to every eglInitialize. Read the + // flag here rather than inside PrewarmBuiltins so ShaderCompiler keeps no dependency + // on the async subsystem (ProgramUtilTest compiles that file without it). + if (MG_Util::Async::AsyncShaderCompileEnabled()) { + MG_Util::ShaderTranspiler::ShaderCompiler::PrewarmBuiltins(); + } MGLOG_D("glslang initialized"); g_isInitialized = true; MGLOG_I("MobileGL initialized"); diff --git a/MobileGL/MG_State/GLState/Core.cpp b/MobileGL/MG_State/GLState/Core.cpp index 2b131807..3495ef40 100644 --- a/MobileGL/MG_State/GLState/Core.cpp +++ b/MobileGL/MG_State/GLState/Core.cpp @@ -10,6 +10,7 @@ #include "MG_State/GLState/RenderbufferState/RenderbufferObject.h" #include "MG_State/EGLState/Core.h" #include +#include #include #include @@ -40,6 +41,13 @@ namespace MobileGL::MG_State { // Error void GLContext::RecordError(ErrorCode code, UniquePtr info) { + // Invariant I1, mechanically enforced: the GL error state is GL-thread-owned. + // A compile or link body that needs to raise an error must append to its node's + // JobDiagnostics and let the join replay it here (see the P1 design section 6); + // reaching this from a worker would corrupt the sticky-flag set that + // glGetError's ordering depends on. + MOBILEGL_ASSERT(!MG_Util::Async::ShaderCompilePool::IsPoolThread(), + "GLContext::RecordError() called from a shader-compile pool thread"); m_errorState.RecordError(code, Move(info)); } diff --git a/MobileGL/MG_State/GLState/ProgramState/ProgramObject.cpp b/MobileGL/MG_State/GLState/ProgramState/ProgramObject.cpp index fb6049a0..cd3505db 100644 --- a/MobileGL/MG_State/GLState/ProgramState/ProgramObject.cpp +++ b/MobileGL/MG_State/GLState/ProgramState/ProgramObject.cpp @@ -572,6 +572,20 @@ namespace MobileGL::MG_State::GLState { // below is a pure function of the snapshot taken here, which is what lets stage 4 // lift it into a ProgramLinkTask. `env` is the first piece of that snapshot: the // link's only window onto the backend. + + // P1 stage 3: linking is still synchronous, so every attached shader's compile has + // to be settled before the body below touches a single one of its artifacts. One + // loop up front rather than leaning on the per-accessor gate, deliberately: it lets + // all the outstanding compiles finish concurrently and blocks once at the end, + // instead of serializing them one join at a time down the loop below. + // + // Placed AFTER the prologue, not before it, so it joins exactly the shader set this + // link will read. Shaders removed by the detach pass above are not joined - the link + // never reads them, their objects are still alive, and whoever queries one next + // joins it then. + for (const auto& shader : m_shaders) { + shader->JoinCompile(); + } const SharedPtr envPtr = MG_Util::ShaderTranspiler::GetCurrentCompileEnv(); const MG_Util::ShaderTranspiler::CompileEnv& env = *envPtr; diff --git a/MobileGL/MG_State/GLState/ProgramState/ProgramState.cpp b/MobileGL/MG_State/GLState/ProgramState/ProgramState.cpp index eceba376..a16a1e6c 100644 --- a/MobileGL/MG_State/GLState/ProgramState/ProgramState.cpp +++ b/MobileGL/MG_State/GLState/ProgramState/ProgramState.cpp @@ -120,6 +120,11 @@ namespace MobileGL::MG_State::GLState { auto& shaderObject = m_shaderObjects[shader]; if (shaderObject == nullptr || !shaderObject->GetDeleteStatus()) return; if (ShaderHasGLVisibleAttachment(shaderObject)) return; + // The name is about to go: nothing can observe this shader's compile any more, so a + // job still in flight for it is pure waste. Cancel-not-join - the job owns its + // inputs, so dropping the object out from under it is safe and the GL thread never + // blocks on a delete. + shaderObject->CancelCompile(); shaderObject.reset(); m_programShaderNameGenerator.Delete(shader); } diff --git a/MobileGL/MG_State/GLState/ProgramState/ShaderCompileTask.cpp b/MobileGL/MG_State/GLState/ProgramState/ShaderCompileTask.cpp new file mode 100644 index 00000000..33670779 --- /dev/null +++ b/MobileGL/MG_State/GLState/ProgramState/ShaderCompileTask.cpp @@ -0,0 +1,303 @@ +// MobileGL - MobileGL/MG_State/GLState/ProgramState/ShaderCompileTask.cpp +// Copyright (c) 2025-2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#include "ShaderCompileTask.h" + +#include +#include +#include +#include + +#include + +#include + +namespace { + struct ComputeLocalSize { + MobileGL::Uint x = 1; + MobileGL::Uint y = 1; + MobileGL::Uint z = 1; + bool declared = false; + }; + + static MobileGL::String StripGlslComments(const MobileGL::String& source) { + MobileGL::String result; + result.reserve(source.length()); + + bool inLineComment = false; + bool inBlockComment = false; + for (MobileGL::SizeT i = 0; i < source.length(); ++i) { + if (inLineComment) { + if (source[i] == '\n') { + inLineComment = false; + result.push_back(source[i]); + } else { + result.push_back(' '); + } + continue; + } + + if (inBlockComment) { + if (source[i] == '*' && i + 1 < source.length() && source[i + 1] == '/') { + inBlockComment = false; + result.append(" "); + ++i; + } else { + result.push_back(source[i] == '\n' ? '\n' : ' '); + } + continue; + } + + if (source[i] == '/' && i + 1 < source.length()) { + if (source[i + 1] == '/') { + inLineComment = true; + result.append(" "); + ++i; + continue; + } + if (source[i + 1] == '*') { + inBlockComment = true; + result.append(" "); + ++i; + continue; + } + } + + result.push_back(source[i]); + } + + return result; + } + + // Hoisted out of ParseComputeLocalSize: constructing a std::regex costs far more than + // running it over a small source, and it was being rebuilt on every compute compile. A + // const regex carries no mutable state, so sharing one instance across workers is safe. + static const std::regex kComputeLocalSizePattern(R"(local_size_([xyz])\s*=\s*([0-9]+))"); + + static ComputeLocalSize ParseComputeLocalSize(const MobileGL::String& source) { + ComputeLocalSize localSize; + const MobileGL::String uncommentedSource = StripGlslComments(source); + + for (std::sregex_iterator it(uncommentedSource.begin(), uncommentedSource.end(), kComputeLocalSizePattern), + end; + it != end; ++it) { + const char axis = (*it)[1].str()[0]; + // The [0-9]+ capture is unbounded, so `local_size_x = 99999999999999999999999` + // is a legal match. std::stoull would throw std::out_of_range on it and let the + // exception escape glCompileShader; std::from_chars reports the overflow instead. + // An overflowing literal saturates to UINT_MAX, which the device-limit check + // below rejects anyway - the same verdict a non-overflowing huge value gets. + const MobileGL::String digits = (*it)[2].str(); + unsigned long long value = 0; + const std::from_chars_result parsed = + std::from_chars(digits.data(), digits.data() + digits.size(), value); + const MobileGL::Uint clampedValue = (parsed.ec != std::errc() || value > UINT_MAX) + ? UINT_MAX + : static_cast(value); + + // TODO: Replace this literal layout scanner with parser/AST-backed validation so expressions and + // specialization-id layouts are handled consistently with glslang. + localSize.declared = true; + if (axis == 'x') { + localSize.x = clampedValue; + } else if (axis == 'y') { + localSize.y = clampedValue; + } else { + localSize.z = clampedValue; + } + } + + return localSize; + } + + // The device limits come from the CompileEnv snapshot, never from a live driver query. + // GL_MAX_COMPUTE_WORK_GROUP_SIZE is a real GLES call on the DirectGLES backend: issued + // off the context thread it would silently no-op and turn a legal local_size_z into + // COMPILE_STATUS=FALSE. CaptureCompileEnv() issues it once, on the GL thread. + static std::optional ValidateComputeLocalSizeLimits( + const MobileGL::String& source, const MobileGL::MG_Util::ShaderTranspiler::CompileEnv& env) { + const ComputeLocalSize localSize = ParseComputeLocalSize(source); + if (!localSize.declared) return std::nullopt; + + if (localSize.x > env.maxComputeWorkGroupSize[0] || localSize.y > env.maxComputeWorkGroupSize[1] || + localSize.z > env.maxComputeWorkGroupSize[2]) { + return "Compute shader local_size exceeds GL_MAX_COMPUTE_WORK_GROUP_SIZE."; + } + + const unsigned long long invocations = static_cast(localSize.x) * localSize.y * localSize.z; + if (invocations > env.maxComputeWorkGroupInvocations) { + return "Compute shader local_size product exceeds GL_MAX_COMPUTE_WORK_GROUP_INVOCATIONS."; + } + + return std::nullopt; + } + + // The half of a compile that depends on nothing but the source text, the stage and the + // environment snapshot: preprocessing, the two lexical rejections, and the two lexical + // side-channel extractions. Split out so P0b layer 2 can memoize exactly this and + // nothing else - the glslang parse stays per-object because its TShader is consume-once. + // Deliberately free of any per-object state so the memo is sound. + // + // The compute local-size verdict reads `env` rather than the live backend, and + // env.fingerprint is part of the P0b cache key, so a memo can never be returned against + // limits other than the ones it was computed against. + static MobileGL::MG_State::GLState::ShaderPreprocessResult RunSourceOnlyPipeline( + const MobileGL::ShaderStage stage, const MobileGL::String& source, + const MobileGL::MG_Util::ShaderTranspiler::CompileEnv& env) { + using namespace MobileGL; + using namespace MobileGL::MG_Util::ShaderTranspiler; + using MobileGL::MG_State::GLState::ShaderPreprocessOutcome; + + MobileGL::MG_State::GLState::ShaderPreprocessResult result; + result.preprocessedSource = source; + PreprocessShaderSource(stage, result.preprocessedSource, env); + + if (stage == ShaderStage::Compute) { + if (const std::optional localSizeError = + ValidateComputeLocalSizeLimits(result.preprocessedSource, env)) { + result.outcome = ShaderPreprocessOutcome::ComputeLocalSizeRejected; + result.infoLog = *localSizeError; + return result; + } + } + + if (const std::optional reservedError = FindReservedIdentifierViolation(result.preprocessedSource)) { + result.outcome = ShaderPreprocessOutcome::ReservedIdentifierRejected; + result.infoLog = *reservedError; + return result; + } + + // The parse this feeds runs in the link-compatible configuration (Vulkan-client + // env with relaxed rules): the TShader it produces is what glLinkProgram links and + // what the backends' SPIR-V is generated from - there is no second, GL-client + // parse. The GL frontend semantics the relaxed parse cannot provide are restored + // on top: explicit default-block uniform locations through the lexical + // side-channels below, dead-uniform/global-UBO filtering in + // ProgramObject::DoReflection. + result.explicitUniformLocations = ExtractExplicitUniformLocations(result.preprocessedSource); + result.explicitOpaqueBindings = ExtractExplicitOpaqueBindings(result.preprocessedSource); + result.outcome = ShaderPreprocessOutcome::Preprocessed; + return result; + } + + // glslang has no "detach this thread" API in the vendored revision (there is no + // InitThread/DetachThread pair any more; thread attachment is implicit through + // thread_local state, and glslang::InitializeProcess() is process-wide, refcounted and + // mutex-guarded, so it needs no per-worker counterpart). + // + // What DOES need undoing is the thread pool allocator: TShader::parse sets the calling + // thread's TLS allocator to the shader's own pool and never restores it. Left pointing + // there, the next allocation this worker makes - in an unrelated job, or in glslang code + // reached from a different object - would come out of a pool the GL thread may already + // have deleted with the TShader. SetThreadPoolAllocator(nullptr) reverts the thread to + // its own thread_local default and is the documented idiom. A scope guard, so it also + // runs when a body throws. + struct GlslangThreadAllocatorGuard { + ~GlslangThreadAllocatorGuard() { glslang::SetThreadPoolAllocator(nullptr); } + }; +} // namespace + +namespace MobileGL::MG_State::GLState { + // Pure CPU work only. Everything this reads is either an input the node owns or a + // process-wide constant; everything it writes is `artifacts`. Do not add a GL/EGL call, + // a pActiveBackendObject read, or a pGLContext->RecordError() here - the first two are + // what CompileEnv exists to replace, and the third is why the design's section 6 + // deferral mechanism (and JobNode's debug assert on it) exists. + void ShaderCompileTask::RunBody() { + // Own the failure rather than letting JobNode's backstop settle the node as + // Cancelled: an abandoned node publishes nothing, so the shader would report + // COMPILE_STATUS false with an EMPTY info log. GL models a failed compile as + // status + log, so turn a throw into exactly that - a completed job whose result + // is "this shader did not compile", with a log the application can read. + // (JobNode still catches: it is the last resort for anything below.) + try { + RunCompilePipeline(); + } catch (const std::exception& e) { + artifacts = {}; + artifacts.env = env; + artifacts.compileStatus = false; + artifacts.infoLog = std::format("Error: shader compilation failed: {}", e.what()); + } catch (...) { + artifacts = {}; + artifacts.env = env; + artifacts.compileStatus = false; + artifacts.infoLog = "Error: shader compilation failed: unknown exception"; + } + } + + void ShaderCompileTask::RunCompilePipeline() { + using namespace MG_Util::ShaderTranspiler; + const GlslangThreadAllocatorGuard glslangGuard; + + const CompileEnv& compileEnv = *env; + artifacts.env = env; + + // P0b layer 2: another shader object in this context may already have run the + // source-only half over byte-identical text under the same environment. + ShaderPreprocessResultPtr cached = + cache ? cache->Find(stage, sourceHash, *source, compileEnv.fingerprint) : nullptr; + SharedPtr fresh; + if (!cached) fresh = MakeShared(RunSourceOnlyPipeline(stage, *source, compileEnv)); + const ShaderPreprocessResult& shared = cached ? *cached : *fresh; + const Bool shouldPopulateCache = !cached && cache != nullptr; + + if (!shared.Preprocessed()) { + // Rejected lexically, or a glslang failure this context has already seen for + // this exact source (ParseFailed) - either way the parse can be skipped. + artifacts.infoLog = shared.infoLog; + if (shouldPopulateCache) { + cache->Insert(stage, sourceHash, *source, compileEnv.fingerprint, Move(fresh)); + } + return; + } + + ShaderAttrib attrib{.shaderType = MG_Util::ConvertShaderStageToGLEnum(stage), + .sourceStr = shared.preprocessedSource, + .flags = 0, + .env = &compileEnv}; + + auto result = ShaderCompiler::CompileShader(attrib); + if (result) { + artifacts.compileStatus = true; + artifacts.shader = result.value(); + // Copy, not move: `shared` may alias a cache entry that has to outlive us, and + // `fresh` is about to be handed to the cache. + artifacts.preprocessedSource = shared.preprocessedSource; + artifacts.explicitUniformLocations = shared.explicitUniformLocations; + artifacts.explicitOpaqueBindings = shared.explicitOpaqueBindings; + artifacts.infoLog.clear(); + if (shouldPopulateCache) { + cache->Insert(stage, sourceHash, *source, compileEnv.fingerprint, Move(fresh)); + } + } else { + artifacts.infoLog = result.error().log; + // Deferred, not logged here, for two reasons. MGLOG from a pool thread interleaves + // mid-line with the GL thread's own output and lands out of order relative to the + // glCompileShader that caused it; diagnostics.logLines is replayed by the join, on + // the GL thread, exactly where a serial implementation would have printed it. + // And a one-line summary rather than the old full source dump: a shaderpack stage + // is ~100KB, so the dump was the single largest thing this driver ever wrote to + // the log, for every failing shader. The info log is what names the offending + // line; the source is recoverable from the application. + const SizeT firstLineEnd = artifacts.infoLog.find('\n'); + diagnostics.logLines.push_back(std::format( + "ShaderCompileTask: shader {} (stage {}) failed to compile; compileStatus = false. " + "Preprocessed source: {} bytes. First log line: {}", + externalIndex, static_cast(stage), shared.preprocessedSource.length(), + artifacts.infoLog.substr(0, firstLineEnd == String::npos ? artifacts.infoLog.length() + : firstLineEnd))); + if (shouldPopulateCache) { + fresh->outcome = ShaderPreprocessOutcome::ParseFailed; + fresh->infoLog = artifacts.infoLog; + fresh->explicitUniformLocations.clear(); + fresh->explicitOpaqueBindings.clear(); + cache->Insert(stage, sourceHash, *source, compileEnv.fingerprint, Move(fresh)); + } + } + } +} // namespace MobileGL::MG_State::GLState diff --git a/MobileGL/MG_State/GLState/ProgramState/ShaderCompileTask.h b/MobileGL/MG_State/GLState/ProgramState/ShaderCompileTask.h new file mode 100644 index 00000000..9fa78088 --- /dev/null +++ b/MobileGL/MG_State/GLState/ProgramState/ShaderCompileTask.h @@ -0,0 +1,83 @@ +// MobileGL - MobileGL/MG_State/GLState/ProgramState/ShaderCompileTask.h +// Copyright (c) 2025-2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#pragma once +#include +#include +#include +#include + +namespace MobileGL::MG_State::GLState { + // Everything one glCompileShader PRODUCES, in one block. + // + // This is exactly the set a single run of the compile pipeline writes, which is what + // makes "discard the artifacts" a complete invalidation and "move the artifacts" a + // complete publish. It lives on the job node rather than on ShaderObject: a worker fills + // it in, and the GL thread reads it through ShaderObject's join gate. + struct ShaderCompileArtifacts { + // The CompileEnv snapshot this compile ran against. Held so the consume-once + // re-parse in TakeShaderForLink() reproduces the original parse exactly, instead of + // re-reading whatever the backend says now. + SharedPtr env; + SharedPtr shader; + // The source the parse actually consumed (after PreprocessShaderSource), kept for + // TakeShaderForLink's re-parse so a later link never depends on the preprocessor + // being deterministic across backend-state changes. + String preprocessedSource; + UnorderedMap explicitUniformLocations; + UnorderedMap explicitOpaqueBindings; + // GL-thread-owned, and the one field here a worker never touches: TakeShaderForLink + // flips it after the join. Stage 4 replaces it with an atomic claim on this node, + // because two ProgramLinkTasks for two programs sharing this shader can then race + // for the parse on two workers. + Bool shaderConsumedByLink = false; + String infoLog; + Bool compileStatus = false; + }; + + // The unit of asynchronous shader compilation: one glCompileShader's worth of pure CPU + // work - preprocess, the two lexical rejections, the two lexical extractions, and the + // glslang parse - with every input it needs owned by the node itself. + // + // That ownership is the whole point. The node reads no GL-thread state (the source is a + // SharedPtr snapshot, the device limits come from the CompileEnv snapshot, + // the P0b cross-object memo is shared-owned and internally locked) and writes nothing + // but its own `artifacts`. So a node whose ShaderObject was re-sourced, deleted, or + // destroyed while it was still running is safe to simply abandon - no wait, no + // synchronization with the GL thread beyond the node's own terminal state. + class ShaderCompileTask final : public MG_Util::Async::JobNode { + public: + ShaderCompileTask(const ShaderStage stage, SharedPtr source, const Uint64 sourceHash, + SharedPtr env, + SharedPtr cache, const Uint externalIndex) + : stage(stage), source(Move(source)), sourceHash(sourceHash), env(Move(env)), cache(Move(cache)), + externalIndex(externalIndex) {} + + // ---- inputs: immutable after construction, all owned by the node ---- + const ShaderStage stage; + // The exact text at enqueue. ShaderObject compares this pointer against its own + // m_source to decide whether its layer-1 memo is armed, which is why glShaderSource + // only ever swaps the pointer when the text genuinely differs. + const SharedPtr source; + const Uint64 sourceHash; + const SharedPtr env; + // P0b layer 2, or null. Null is the "no context" case (the default fragment shader, + // the backends' internal blit/mipmap shaders) and doubles as the marker for + // "compile inline regardless of the async flag" - see ShaderObject::Compile(). + const SharedPtr cache; + const Uint externalIndex; // logs only + + // ---- output: valid iff IsComplete(), immutable afterwards ---- + ShaderCompileArtifacts artifacts; + + private: + void RunBody() override; + // The real body; RunBody wraps it so a throw becomes a GL-visible compile failure. + void RunCompilePipeline(); + }; +} // namespace MobileGL::MG_State::GLState diff --git a/MobileGL/MG_State/GLState/ProgramState/ShaderObject.cpp b/MobileGL/MG_State/GLState/ProgramState/ShaderObject.cpp index 0194cca5..9fe1e05e 100644 --- a/MobileGL/MG_State/GLState/ProgramState/ShaderObject.cpp +++ b/MobileGL/MG_State/GLState/ProgramState/ShaderObject.cpp @@ -8,313 +8,127 @@ #include "ShaderObject.h" #include "ShaderPreprocessCache.h" -#include -#include +#include #include -#include #include -#include - -#include - -namespace { - struct ComputeLocalSize { - MobileGL::Uint x = 1; - MobileGL::Uint y = 1; - MobileGL::Uint z = 1; - bool declared = false; - }; - - static MobileGL::String StripGlslComments(const MobileGL::String& source) { - MobileGL::String result; - result.reserve(source.length()); - - bool inLineComment = false; - bool inBlockComment = false; - for (MobileGL::SizeT i = 0; i < source.length(); ++i) { - if (inLineComment) { - if (source[i] == '\n') { - inLineComment = false; - result.push_back(source[i]); - } else { - result.push_back(' '); - } - continue; - } - - if (inBlockComment) { - if (source[i] == '*' && i + 1 < source.length() && source[i + 1] == '/') { - inBlockComment = false; - result.append(" "); - ++i; - } else { - result.push_back(source[i] == '\n' ? '\n' : ' '); - } - continue; - } - - if (source[i] == '/' && i + 1 < source.length()) { - if (source[i + 1] == '/') { - inLineComment = true; - result.append(" "); - ++i; - continue; - } - if (source[i + 1] == '*') { - inBlockComment = true; - result.append(" "); - ++i; - continue; - } - } - - result.push_back(source[i]); - } - - return result; - } - - // Hoisted out of ParseComputeLocalSize: constructing a std::regex costs far more than - // running it over a small source, and it was being rebuilt on every compute compile. A - // const regex carries no mutable state, so sharing one instance is safe. - static const std::regex kComputeLocalSizePattern(R"(local_size_([xyz])\s*=\s*([0-9]+))"); - - static ComputeLocalSize ParseComputeLocalSize(const MobileGL::String& source) { - ComputeLocalSize localSize; - const MobileGL::String uncommentedSource = StripGlslComments(source); - - for (std::sregex_iterator it(uncommentedSource.begin(), uncommentedSource.end(), kComputeLocalSizePattern), - end; - it != end; ++it) { - const char axis = (*it)[1].str()[0]; - // The [0-9]+ capture is unbounded, so `local_size_x = 99999999999999999999999` - // is a legal match. std::stoull would throw std::out_of_range on it and let the - // exception escape glCompileShader; std::from_chars reports the overflow instead. - // An overflowing literal saturates to UINT_MAX, which the device-limit check - // below rejects anyway - the same verdict a non-overflowing huge value gets. - const MobileGL::String digits = (*it)[2].str(); - unsigned long long value = 0; - const std::from_chars_result parsed = - std::from_chars(digits.data(), digits.data() + digits.size(), value); - const MobileGL::Uint clampedValue = (parsed.ec != std::errc() || value > UINT_MAX) - ? UINT_MAX - : static_cast(value); - - // TODO: Replace this literal layout scanner with parser/AST-backed validation so expressions and - // specialization-id layouts are handled consistently with glslang. - localSize.declared = true; - if (axis == 'x') { - localSize.x = clampedValue; - } else if (axis == 'y') { - localSize.y = clampedValue; - } else { - localSize.z = clampedValue; - } - } - - return localSize; - } - - // The device limits come from the CompileEnv snapshot, never from a live driver query. - // GL_MAX_COMPUTE_WORK_GROUP_SIZE is a real GLES call on the DirectGLES backend: issued - // off the context thread it would silently no-op and turn a legal local_size_z into - // COMPILE_STATUS=FALSE. CaptureCompileEnv() issues it once, on the GL thread. - static std::optional ValidateComputeLocalSizeLimits( - const MobileGL::String& source, const MobileGL::MG_Util::ShaderTranspiler::CompileEnv& env) { - const ComputeLocalSize localSize = ParseComputeLocalSize(source); - if (!localSize.declared) return std::nullopt; - - if (localSize.x > env.maxComputeWorkGroupSize[0] || localSize.y > env.maxComputeWorkGroupSize[1] || - localSize.z > env.maxComputeWorkGroupSize[2]) { - return "Compute shader local_size exceeds GL_MAX_COMPUTE_WORK_GROUP_SIZE."; - } - - const unsigned long long invocations = static_cast(localSize.x) * localSize.y * localSize.z; - if (invocations > env.maxComputeWorkGroupInvocations) { - return "Compute shader local_size product exceeds GL_MAX_COMPUTE_WORK_GROUP_INVOCATIONS."; - } - - return std::nullopt; - } - - // The half of ShaderObject::Compile() that depends on nothing but the source text and - // the stage: preprocessing, the two lexical rejections, and the two lexical - // side-channel extractions. Split out so P0b layer 2 can memoize exactly this and - // nothing else - the glslang parse stays per-object because its TShader is - // consume-once. Deliberately free of any per-object state so the memo is sound. - // - // The former caveat is gone: the compute local-size verdict reads `env` rather than the - // live backend, and env.fingerprint is part of the P0b cache key, so a memo can never be - // returned against limits other than the ones it was computed against. - static MobileGL::MG_State::GLState::ShaderPreprocessResult RunSourceOnlyPipeline( - const MobileGL::ShaderStage stage, const MobileGL::String& source, - const MobileGL::MG_Util::ShaderTranspiler::CompileEnv& env) { - using namespace MobileGL; - using namespace MobileGL::MG_Util::ShaderTranspiler; - using MobileGL::MG_State::GLState::ShaderPreprocessOutcome; - - MobileGL::MG_State::GLState::ShaderPreprocessResult result; - result.preprocessedSource = source; - PreprocessShaderSource(stage, result.preprocessedSource, env); - - if (stage == ShaderStage::Compute) { - if (const std::optional localSizeError = - ValidateComputeLocalSizeLimits(result.preprocessedSource, env)) { - result.outcome = ShaderPreprocessOutcome::ComputeLocalSizeRejected; - result.infoLog = *localSizeError; - return result; - } - } - - if (const std::optional reservedError = FindReservedIdentifierViolation(result.preprocessedSource)) { - result.outcome = ShaderPreprocessOutcome::ReservedIdentifierRejected; - result.infoLog = *reservedError; - return result; - } - - // The parse this feeds runs in the link-compatible configuration (Vulkan-client - // env with relaxed rules): the TShader it produces is what glLinkProgram links and - // what the backends' SPIR-V is generated from - there is no second, GL-client - // parse. The GL frontend semantics the relaxed parse cannot provide are restored - // on top: explicit default-block uniform locations through the lexical - // side-channels below, dead-uniform/global-UBO filtering in - // ProgramObject::DoReflection. - result.explicitUniformLocations = ExtractExplicitUniformLocations(result.preprocessedSource); - result.explicitOpaqueBindings = ExtractExplicitOpaqueBindings(result.preprocessedSource); - result.outcome = ShaderPreprocessOutcome::Preprocessed; - return result; - } -} +#include +#include namespace MobileGL::MG_State::GLState { void ShaderObject::SetShaderSource(const String& source) { // P0b layer 1. glShaderSource always REPLACES the source, but replacing it with a // byte-identical one cannot change what a compile would produce: the whole - // pipeline below (preprocess -> lexical checks -> glslang parse) is a pure - // function of (stage, source) plus context-lifetime backend limits. So keeping the - // compiled state is not an optimization that changes observable behaviour - the - // COMPILE_STATUS, the info log and the reflection a caller can query are exactly - // what a real recompile would have rebuilt, byte for byte. + // pipeline (preprocess -> lexical checks -> glslang parse) is a pure function of + // (stage, source, CompileEnv). So keeping the compiled state is not an optimization + // that changes observable behaviour - the COMPILE_STATUS, the info log and the + // reflection a caller can query are exactly what a real recompile would have + // rebuilt, byte for byte. A compile still IN FLIGHT is left running for the same + // reason: it is computing the right answer for text this object still holds. if (SourceMatchesCompiledState(source)) return; - m_source = source; + // The text genuinely changed, so whatever a running job is computing is now about + // an old source. Drop it where it stands - it owns its own copy of that old string, + // so swapping the pointer below cannot race its storage. + CancelCompile(); + m_source = MakeShared(source); InvalidateCompiledState(); } void ShaderObject::SetShaderSource(String&& source) { if (SourceMatchesCompiledState(source)) return; - m_source = Move(source); + CancelCompile(); + m_source = MakeShared(Move(source)); InvalidateCompiledState(); } Bool ShaderObject::SourceMatchesCompiledState(const String& candidate) const { - if (!m_hasCompiledState) return false; - if (candidate.length() != m_compiledSourceLength) return false; - if (ShaderPreprocessCache::HashSource(candidate) != m_compiledSourceHash) return false; - // The hash is a fast reject only; confirm against the actual stored text. While - // m_hasCompiledState holds, m_source IS the source that produced the state. - return candidate == m_source; + // The memo is armed exactly while a job exists that was built from the string this + // object still points at - pending or finished, success or failure. + if (!HasMemoizedCompile()) return false; + if (candidate.length() != m_source->length()) return false; + // Never let correctness ride on a hash: the answer is the full text comparison. + // (The stored hash on the node is a cache-lookup accelerator, not a substitute.) + return candidate == *m_source; } - void ShaderObject::RememberCompiledSource(const Uint64 sourceHash) { - m_hasCompiledState = true; - m_compiledSourceHash = sourceHash; - m_compiledSourceLength = m_source.length(); + void ShaderObject::JoinPendingCompile() const { + MOBILEGL_ASSERT(!MG_Util::Async::ShaderCompilePool::IsPoolThread(), + "ShaderObject::EnsureCompileJoined() reached from a pool thread; a job body must never read " + "GL-thread-owned objects"); + m_compiled->Wait(); + m_compileJoined = true; + // Errors and worker-side log lines are raised HERE, on the GL thread, at the first + // join of the job that produced them - which for a single shader is trivially the + // order a serial implementation would have produced them in. + MG_Util::Async::ApplyDeferredDiagnostics(*m_compiled); + // A node that settled as Cancelled published nothing. Dropping it here is what keeps + // the object's state machine to two reachable cases - "no job" and "a job that + // completed" - so every reader below can treat a live node as authoritative. + if (!m_compiled->IsComplete()) m_compiled.reset(); } - // EnsureCompileJoined() is defined inline in ShaderObject.h (see the comment there for - // why: no LTO, and it is called from every Compiled() read). - void ShaderObject::InvalidateCompiledState() { - // The compile artifacts are exactly what one Compile() writes, so discarding them - // wholesale IS the invalidation. (Kept as an explicit reset rather than a - // default-construct so the intent survives a future field addition.) - Compiled() = CompileArtifacts{}; - m_hasCompiledState = false; - m_compiledSourceHash = 0; - m_compiledSourceLength = 0; + // The job node holds exactly what one Compile() produces, so discarding it IS the + // invalidation - and it re-arms nothing, so the next Compile() genuinely recompiles. + m_compiled.reset(); + } + + void ShaderObject::CancelCompile() { + if (!m_compiled || m_compiled->IsTerminal()) return; + // Cooperative and non-blocking. A node that no worker has picked up settles + // immediately; one that is running is flagged and settles when its body returns, + // writing only into itself the whole time. + m_compiled->Cancel(); + m_compiled.reset(); } void ShaderObject::Compile() { - using namespace MG_Util::ShaderTranspiler; - - // P0b layer 1: the state this object holds was produced by a previous Compile() of - // the exact source it still holds, so a recompile is a no-op. This covers the - // failure case too - the info log stays queryable because nothing is cleared. + // P0b layer 1, as a tri-state: the memo is "the node in m_compiled was built from + // the string m_source still points at". SetShaderSource only swaps that pointer when + // the text actually differs, so this is a pointer compare, and it covers Pending as + // well as Complete - a second glCompileShader on an in-flight object is a no-op, not + // a duplicate job racing to write the same fields. // - // shaderConsumedByLink interaction: if the stored TShader already fed a link, - // the no-op leaves preprocessedSource and both side-channel maps intact, which - // is precisely what TakeShaderForLink's on-demand re-parse needs. A real recompile - // would have handed the next link a fresh parse; the no-op hands it a fresh - // re-parse of the identical source instead. Same result, one parse either way. - if (m_hasCompiledState) return; + // The failure case is covered too: the info log stays queryable because nothing is + // cleared. And if the stored TShader already fed a link, the no-op leaves + // preprocessedSource and both side-channel maps intact, which is precisely what + // TakeShaderForLink's on-demand re-parse needs - a real recompile would have handed + // the next link a fresh parse, the no-op hands it a fresh re-parse of the identical + // source instead. Same result, one parse either way. + if (HasMemoizedCompile()) return; - InvalidateCompiledState(); + // The compile-environment snapshot is taken HERE, on the GL thread, and handed to + // the job. Everything the pipeline needs to know about the device comes through it, + // never through pActiveBackendObject - that is what makes the body movable. + m_compiled = MakeShared(m_stage, m_source, ShaderPreprocessCache::HashSource(*m_source), + MG_Util::ShaderTranspiler::GetCurrentCompileEnv(), + m_preprocessCache, m_externalIndex); + m_compileJoined = false; - const Uint64 sourceHash = ShaderPreprocessCache::HashSource(m_source); - - // The compile-environment snapshot, taken here on the GL thread. Everything below - // reads the device through it and never through pActiveBackendObject, which is what - // makes the whole body movable onto a worker in stage 3. - CompileArtifacts& compiled = Compiled(); - compiled.env = MG_Util::ShaderTranspiler::GetCurrentCompileEnv(); - const MG_Util::ShaderTranspiler::CompileEnv& env = *compiled.env; - - // P0b layer 2: another shader object in this context may already have run the - // source-only half over byte-identical text under the same environment. - ShaderPreprocessResultPtr cached = - m_preprocessCache ? m_preprocessCache->Find(m_stage, sourceHash, m_source, env.fingerprint) : nullptr; - SharedPtr fresh; - if (!cached) fresh = MakeShared(RunSourceOnlyPipeline(m_stage, m_source, env)); - const ShaderPreprocessResult& shared = cached ? *cached : *fresh; - const Bool shouldPopulateCache = !cached && m_preprocessCache != nullptr; - - if (!shared.Preprocessed()) { - // Rejected lexically, or a glslang failure this context has already seen for - // this exact source (ParseFailed) - either way the parse can be skipped. - compiled.infoLog = shared.infoLog; - if (shouldPopulateCache) { - m_preprocessCache->Insert(m_stage, sourceHash, m_source, env.fingerprint, Move(fresh)); - } - RememberCompiledSource(sourceHash); + // Two reasons to stay on this thread, one rule. Without the async flag the whole + // path must be byte-identical to the synchronous implementation, and a cache-less + // object is an internal shader that compiles and reads its status in the same + // breath (see the constructor comment) - a job would only add a round trip. + if (!m_preprocessCache || !MG_Util::Async::AsyncShaderCompileEnabled()) { + m_compiled->RunInline(); + // Inline means the node is already terminal, so this join only replays + // diagnostics; it is here so the synchronous and asynchronous paths publish + // through the identical code. + EnsureCompileJoined(); return; } - - ShaderAttrib attrib{.shaderType = MG_Util::ConvertShaderStageToGLEnum(m_stage), - .sourceStr = shared.preprocessedSource, - .flags = 0, - .env = &env}; - - auto result = ShaderCompiler::CompileShader(attrib); - if (result) { - compiled.compileStatus = true; - compiled.shader = result.value(); - // Copy, not move: `shared` may alias a cache entry that has to outlive us, and - // `fresh` is about to be handed to the cache. - compiled.preprocessedSource = shared.preprocessedSource; - compiled.explicitUniformLocations = shared.explicitUniformLocations; - compiled.explicitOpaqueBindings = shared.explicitOpaqueBindings; - compiled.infoLog.clear(); - if (shouldPopulateCache) { - m_preprocessCache->Insert(m_stage, sourceHash, m_source, env.fingerprint, Move(fresh)); - } - } else { - compiled.infoLog = result.error().log; - MGLOG_D("ShaderObject::Compile: Shader %d compilation failed.\nSource:\n%s\nInfoLog:\n%s\nSetting " - "compileStatus = false as a result.", - m_externalIndex, shared.preprocessedSource.c_str(), compiled.infoLog.c_str()); - if (shouldPopulateCache) { - fresh->outcome = ShaderPreprocessOutcome::ParseFailed; - fresh->infoLog = compiled.infoLog; - fresh->explicitUniformLocations.clear(); - fresh->explicitOpaqueBindings.clear(); - m_preprocessCache->Insert(m_stage, sourceHash, m_source, env.fingerprint, Move(fresh)); - } - } - RememberCompiledSource(sourceHash); + MG_Util::Async::ShaderCompilePool::Get().Post(m_compiled); } SharedPtr ShaderObject::TakeShaderForLink(String& outReparseLog) { - CompileArtifacts& compiled = Compiled(); + EnsureCompileJoined(); + // Unreachable through the GL frontend: callers gate on GetCompileStatus(). + if (!m_compiled) return nullptr; + + // Mutating the node's artifacts from here is legal precisely because the node is + // terminal by now: no worker will ever touch it again, and this thread is the GL + // thread. Stage 4, where two link JOBS can reach the same node concurrently, + // replaces this flag with an atomic claim on the node. + ShaderCompileArtifacts& compiled = m_compiled->artifacts; if (compiled.shader && !compiled.shaderConsumedByLink) { compiled.shaderConsumedByLink = true; return compiled.shader; diff --git a/MobileGL/MG_State/GLState/ProgramState/ShaderObject.h b/MobileGL/MG_State/GLState/ProgramState/ShaderObject.h index 860bbf68..3c6d5f11 100644 --- a/MobileGL/MG_State/GLState/ProgramState/ShaderObject.h +++ b/MobileGL/MG_State/GLState/ProgramState/ShaderObject.h @@ -8,39 +8,47 @@ #pragma once #include -#include +#include +#include namespace MobileGL { - enum class ShaderStage { - Vertex, - TessControl, - TessEval, - Geometry, - Fragment, - Compute, - ShaderStageCount, - Unknown = -1 - }; - namespace MG_State::GLState { - // P0b layer 2. Declared, not included: the cache keys on ShaderStage, so including - // its header here would be circular. - class ShaderPreprocessCache; - + // The GL-visible shader name. It owns the source text and one compile job node; the + // job node owns everything a compile produces. + // + // Every member below is GL-thread-owned, and every read of worker-produced state + // goes through Compiled(), which joins first. That is invariant I5 of the P1 design: + // because Compiled() is the SOLE accessor of the node's artifacts, the compiler + // enumerates every reader for us and none can be forgotten. class ShaderObject { public: - // `preprocessCache` is the owning context's cross-object memo (P0b layer 2); - // null is fully supported and simply means "no sharing" - that is what the - // context-less internal shader objects (the default FS, the blit pipeline) use. - // Shared ownership rather than a raw pointer: once compiles run on a worker the - // job outlives neither the object nor the context deterministically, and the - // cache has to stay alive for whoever is still reading it. + // `preprocessCache` is the owning context's cross-object memo (P0b layer 2). + // Null is fully supported and means two things at once: "no sharing", and + // "compile inline, never on a worker". Those coincide exactly - the only + // cache-less shader objects are the internal ones (ProgramObject's default + // fragment shader, the DirectVulkan blit and depth-mipmap shaders) and every one + // of them compiles and reads its status in the same breath, so a job would only + // add a round trip. Shared ownership rather than a raw pointer: a compile job + // outlives neither the object nor the context deterministically, and the cache + // has to stay alive for whoever is still reading it. ShaderObject(const ShaderStage stage, Uint externalIndex, SharedPtr preprocessCache = nullptr) : m_stage(stage), m_externalIndex(externalIndex), m_preprocessCache(Move(preprocessCache)) {} + // Cancel-not-join: the node owns its inputs, so an in-flight compile whose + // object just went away is safe to abandon where it stands. Nothing can observe + // its result any more - this object was the only route to it. + ~ShaderObject() { CancelCompile(); } + + ShaderObject(const ShaderObject&) = delete; + ShaderObject& operator=(const ShaderObject&) = delete; + void SetShaderSource(const String& source); void SetShaderSource(String&& source); void Compile(); + // Drops a compile that is still in flight, without waiting for it. Called at the + // points where the object's compiled state stops being observable: a real source + // change, and the release of an orphaned shader name. + void CancelCompile(); void MarkAsDeleted(); // Hands out a link-consumable TShader. glslang's mapIO mutates the TShader's @@ -54,10 +62,14 @@ namespace MobileGL { Uint GetExternalIndex() const { return m_externalIndex; } ShaderStage GetShaderStage() const { return m_stage; } - const String& GetShaderSource() const { return m_source; } + // No join: the source is GL-thread-owned, and a worker only ever reads the + // immutable snapshot it was handed at enqueue. + const String& GetShaderSource() const { return *m_source; } + // The snapshot itself, for whoever needs to hand it to a job. + const SharedPtr& GetShaderSourcePtr() const { return m_source; } + const SharedPtr& GetCompiledShader() const { return Compiled().shader; } const String& GetInfoLog() const { return Compiled().infoLog; } - const UnorderedMap& GetUniformLocations() const { return Compiled().uniforms; } // Explicit layout(location = N) qualifiers on this shader's default-block // uniforms, captured lexically at Compile() because the relaxed parse drops // them from reflection (see ExtractExplicitUniformLocations). @@ -73,96 +85,98 @@ namespace MobileGL { Bool GetCompileStatus() const { return Compiled().compileStatus; } Bool GetDeleteStatus() const { return m_deleteStatus; } - // Blocks until a pending compile (P1 stage 3 onwards) has published its - // artifacts. Public for the few sites that must join without reading anything. - // A no-op today - nothing is ever pending. + // Blocks until a pending compile has published its artifacts. Public for the + // sites that must join without reading anything - ProgramObject::Link's + // prologue, which needs every attached shader settled before it runs. void JoinCompile() const { EnsureCompileJoined(); } - // True while this object holds the outcome (success OR failure) of a previous - // Compile() of exactly the source it currently holds - i.e. while the P0b - // layer-1 memo is armed and a glCompileShader would be a no-op. Diagnostics - // and tests only; nothing in the GL frontend branches on it. + // True while this object holds the outcome (success OR failure) of a Compile() + // of exactly the source it currently holds - i.e. while the P0b layer-1 memo is + // armed and a glCompileShader would be a no-op. Diagnostics and tests only; + // nothing in the GL frontend branches on it. // - // Deliberately does NOT join: the memo bookkeeping below is GL-thread-owned and - // says nothing about whether a worker has finished, which is exactly the - // property GL_COMPLETION_STATUS_KHR needs when stage 3 lands. - Bool HasMemoizedCompile() const { return m_hasCompiledState; } + // Tri-state, and deliberately NOT joining: an in-flight compile of the current + // source counts as memoized (a second glCompileShader must not enqueue a + // duplicate job), but asking that question must never block. + // A node that settled as Cancelled (the job body threw, or the enqueue failed) + // carries no result, so it must NOT satisfy the memo: otherwise a second + // glCompileShader on the same source enqueues nothing and the eventual join + // reports GL_FALSE forever. The synchronous path retries in exactly this case. + Bool HasMemoizedCompile() const { + return m_compiled != nullptr && m_compiled->source == m_source && !m_compiled->IsCancelled(); + } + + // MUST NOT JOIN - this is what GL_COMPLETION_STATUS_KHR will read when the + // extension surface lands. "No job at all" counts as complete: there is nothing + // outstanding to wait for. + Bool IsCompileComplete() const { return m_compiled == nullptr || m_compiled->IsTerminal(); } private: - // ---- P1: everything a compile PRODUCES, in one block ---- - // - // Same rule as ProgramObject::LinkArtifacts: this is exactly what - // InvalidateCompiledState() clears, i.e. exactly what one run of Compile() - // writes. Stage 3 lifts this struct wholesale into ShaderCompileTask, where a - // worker fills it in and the GL thread reads it through the same gate. - struct CompileArtifacts { - // The CompileEnv snapshot this compile ran against. Held so the - // consume-once re-parse in TakeShaderForLink() reproduces the original - // parse exactly, instead of re-reading whatever the backend says now. - SharedPtr env; - SharedPtr shader; - // The source Compile() actually parsed (after PreprocessShaderSource), kept - // for TakeShaderForLink's re-parse so a later link never depends on the - // preprocessor being deterministic across backend-state changes. - String preprocessedSource; - UnorderedMap uniforms; - UnorderedMap explicitUniformLocations; - UnorderedMap explicitOpaqueBindings; - Bool shaderConsumedByLink = false; - String infoLog; - Bool compileStatus = false; - }; - // ---- The one and only join gate for compile output (P1 invariant I5) ---- - // Blocks until a pending compile has published into m_compiled. Today nothing - // is ever pending - glCompileShader still runs the whole body inline - so this - // is an unconditional no-op. It exists NOW so that every reader of compile - // output is already routed through it when stage 3 makes it block. + // The fast path - no job, or a job whose result this object has already pulled - + // is two predictable branches and stays inline: it runs on every Compiled() read + // and the project never builds with LTO, so an out-of-line body would be a real + // cross-TU call at each of those sites. The blocking half is out of line. // - // Defined inline (not in ShaderObject.cpp): called from every Compiled() read, - // and the project never builds with LTO, so an out-of-line empty body would be - // a real cross-TU call at each of those call sites instead of folding away. - void EnsureCompileJoined() const {} - CompileArtifacts& Compiled() { - EnsureCompileJoined(); - return m_compiled; + // The gate keys on "has this object pulled the job's result yet", NOT on "is the + // job terminal". Those differ in the case that matters: a worker can finish a + // compile before the GL thread ever looks at it, and the pull is where deferred + // diagnostics get replayed and an abandoned node gets dropped. Keying on + // terminality would silently skip both. + void EnsureCompileJoined() const { + if (m_compiled && !m_compileJoined) JoinPendingCompile(); } - const CompileArtifacts& Compiled() const { + void JoinPendingCompile() const; + + // The artifacts of a compile that ran to completion. A node that was abandoned + // (cancelled at teardown, or whose body threw) never publishes: JoinPendingCompile + // drops it, so anything reachable here is either Complete or absent, and "absent" + // reads as the never-compiled defaults - COMPILE_STATUS false, empty info log, + // which is exactly what GL requires before the first glCompileShader. + static const ShaderCompileArtifacts& EmptyArtifacts() { + static const ShaderCompileArtifacts empty; + return empty; + } + const ShaderCompileArtifacts& Compiled() const { EnsureCompileJoined(); - return m_compiled; + return m_compiled ? m_compiled->artifacts : EmptyArtifacts(); } void InvalidateCompiledState(); // ---- P0b layer 1: per-object no-op recompile ---- - // True iff `candidate` is byte-identical to the source that produced the - // compiled state this object is currently holding. The stored hash and length - // are only a fast reject; the answer is always confirmed against the full - // stored text, so no behaviour rides on a 64-bit hash. + // True iff `candidate` is byte-identical to the source that produced (or is + // producing) the compiled state this object currently holds. Bool SourceMatchesCompiledState(const String& candidate) const; - // Arms the layer-1 memo for the source that Compile() just processed. - void RememberCompiledSource(Uint64 sourceHash); // ---- GL-thread-owned state: never produced by a compile, so it never joins ---- - const Uint m_externalIndex = 0; const ShaderStage m_stage; - // glShaderSource text. A worker only ever reads the snapshot handed to it, so - // GL_SHADER_SOURCE_LENGTH and glGetShaderSource never join. - String m_source; + const Uint m_externalIndex = 0; + // The pre-glShaderSource state, shared by every untouched object rather than + // allocated per glCreateShader. + static const SharedPtr& EmptySource() { + static const SharedPtr empty = MakeShared(); + return empty; + } + // glShaderSource text, as an immutable snapshot. Never null. A job holds its own + // SharedPtr to the exact string it was given, so replacing the source under a + // running compile cannot race its storage - and the layer-1 memo collapses to a + // pointer comparison against the job's snapshot, because the setter only swaps + // the pointer when the text genuinely differs. + SharedPtr m_source = EmptySource(); - // P0b layer 2: the owning context's cross-object memo, or null. + // P0b layer 2: the owning context's cross-object memo, or null. Internally + // locked, because several workers hit it at once. const SharedPtr m_preprocessCache; - // P0b layer 1. m_hasCompiledState is the invariant "m_source is byte-identical - // to the source that produced the compile artifacts"; it is armed at the end of - // every Compile() and disarmed by InvalidateCompiledState(). Stage 3 replaces - // all three with a pointer compare against the in-flight job's source snapshot. - Bool m_hasCompiledState = false; - Uint64 m_compiledSourceHash = 0; - SizeT m_compiledSourceLength = 0; Bool m_deleteStatus = false; - // ---- Compile OUTPUT ---- reachable only through Compiled(). - CompileArtifacts m_compiled; + // ---- Compile OUTPUT ---- pending OR completed; reachable only through Compiled(). + // Mutable because the join is a read-side operation: a const getter has to be + // able to settle an outstanding job before answering. + mutable SharedPtr m_compiled; + // Exactly-once latch for the pull above. Armed with every new job node, set by + // the one join that consumes it. + mutable Bool m_compileJoined = false; }; } // namespace MG_State::GLState } // namespace MobileGL diff --git a/MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.h b/MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.h index 0b13d1fd..14cce385 100644 --- a/MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.h +++ b/MobileGL/MG_State/GLState/ProgramState/ShaderPreprocessCache.h @@ -10,7 +10,9 @@ #include #include #include -#include +// Deliberately NOT ShaderObject.h: ShaderCompileTask.h needs this header, and ShaderObject.h +// needs ShaderCompileTask.h. Only ShaderStage was ever used from there. +#include namespace MobileGL::MG_State::GLState { // Where the shared, source-only half of ShaderObject::Compile() stopped. The two diff --git a/MobileGL/MG_State/GLState/ProgramState/ShaderStage.h b/MobileGL/MG_State/GLState/ProgramState/ShaderStage.h new file mode 100644 index 00000000..f9d17d2c --- /dev/null +++ b/MobileGL/MG_State/GLState/ProgramState/ShaderStage.h @@ -0,0 +1,25 @@ +// MobileGL - MobileGL/MG_State/GLState/ProgramState/ShaderStage.h +// Copyright (c) 2025-2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +#pragma once + +namespace MobileGL { + // Split out of ShaderObject.h so the compile pipeline's headers form a DAG: + // ShaderStage.h <- ShaderPreprocessCache.h <- ShaderCompileTask.h <- ShaderObject.h. + // Every existing includer of ShaderObject.h still sees this type unchanged. + enum class ShaderStage { + Vertex, + TessControl, + TessEval, + Geometry, + Fragment, + Compute, + ShaderStageCount, + Unknown = -1 + }; +} // namespace MobileGL diff --git a/MobileGL/MG_Test/Program/AsyncCompileTest.cpp b/MobileGL/MG_Test/Program/AsyncCompileTest.cpp new file mode 100644 index 00000000..b91a6fdc --- /dev/null +++ b/MobileGL/MG_Test/Program/AsyncCompileTest.cpp @@ -0,0 +1,571 @@ +// MobileGL - MobileGL/MG_Test/Program/AsyncCompileTest.cpp +// Copyright (c) 2025-2026 MobileGL-Dev +// Licensed under the GNU Lesser General Public License v3.0: +// https://www.gnu.org/licenses/gpl-3.0.txt +// https://www.gnu.org/licenses/lgpl-3.0.txt +// SPDX-License-Identifier: LGPL-3.0-only +// End of Source File Header + +// P1 stage 3: glCompileShader enqueues, and every observable read joins. +// +// Every test here drives the real GL entry points and flips +// MG_Config::Features.AsyncShaderCompile itself rather than reading the environment. That is +// what lets one binary assert the property that actually matters - the async path and the +// synchronous path are indistinguishable through the GL surface - and it makes the file +// behave identically whether or not the suite was launched with +// MOBILEGL_ASYNC_SHADER_COMPILE=1. + +#include + +#include +#include +#include + +#include "Config.h" +#include "Includes.h" +#include "Init.h" +#include "MG_Impl/GLImpl/Getter/GL_Getter.h" +#include "MG_Impl/GLImpl/Program/GL_Program.h" +#include "MG_State/GLState/Core.h" +#include "MG_Util/Async/ShaderCompilePool.h" + +using namespace MobileGL; +using namespace MobileGL::MG_Impl::GLImpl; + +namespace { + // Restores whatever the environment asked for when the test ends, so a case that forces + // one mode cannot leak into the next. + class AsyncModeScope { + public: + explicit AsyncModeScope(const Bool async) + : m_saved(MG_Config::Features.AsyncShaderCompile) { + MG_Config::Features.AsyncShaderCompile = + async ? MG_Config::QuirkOverride::ForceOn : MG_Config::QuirkOverride::ForceOff; + } + ~AsyncModeScope() { MG_Config::Features.AsyncShaderCompile = m_saved; } + AsyncModeScope(const AsyncModeScope&) = delete; + AsyncModeScope& operator=(const AsyncModeScope&) = delete; + + private: + const MG_Config::QuirkOverride m_saved; + }; + + const char* kVs = R"(#version 460 +layout(location = 0) in vec3 aPos; +uniform mat4 uModel; +uniform vec4 uColor; +out vec4 vColor; +void main() { + vColor = uColor; + gl_Position = uModel * vec4(aPos, 1.0); +} +)"; + + const char* kFs = R"(#version 460 +in vec4 vColor; +layout(location = 0) out vec4 fragColor; +uniform float uAlpha; +void main() { fragColor = vec4(vColor.rgb, vColor.a * uAlpha); } +)"; + + // Fails in glslang, not in the lexical pre-checks: that routes through the same + // ParseFailed path a real broken shaderpack source takes. + const char* kBrokenFs = R"(#version 460 +layout(location = 0) out vec4 fragColor; +void main() { fragColor = thisIdentifierWasNeverDeclared; } +)"; + + // Rejected by the lexical reserved-identifier scan, before glslang is ever reached - the + // other half of the "compile failed" surface, and the one that never allocates a parse. + const char* kReservedIdentifierFs = R"(#version 460 +layout(location = 0) out vec4 fragColor; +float gl_NotAllowedToDeclareThis = 1.0; +void main() { fragColor = vec4(gl_NotAllowedToDeclareThis); } +)"; + + // Big enough that a compile is not instantaneous, so the pool actually has a backlog to + // observe. Templated on an index so every instance is a distinct source (no P0b hit). + String MakeBulkySource(const int index) { + String source = "#version 460\nlayout(location = 0) out vec4 fragColor;\n"; + source += "uniform float uSeed" + std::to_string(index) + ";\n"; + source += "void main() {\n float acc = uSeed" + std::to_string(index) + ";\n"; + for (int i = 0; i < 220; ++i) { + source += " acc = acc * 1.0001 + sin(acc + " + std::to_string(i) + ".0) * cos(acc);\n"; + } + source += " fragColor = vec4(acc, acc, acc, 1.0);\n}\n"; + return source; + } + + GLuint MakeShader(const GLenum type, const char* source) { + const GLuint shader = CreateShader(type); + ShaderSource(shader, 1, &source, nullptr); + return shader; + } + + GLint QueryCompileStatus(const GLuint shader) { + GLint status = GL_FALSE; + GetShaderiv(shader, GL_COMPILE_STATUS, &status); + return status; + } + + String QueryShaderInfoLog(const GLuint shader) { + GLint length = 0; + GetShaderiv(shader, GL_INFO_LOG_LENGTH, &length); + if (length <= 0) return String(); + std::vector buffer(static_cast(length)); + GLsizei written = 0; + GetShaderInfoLog(shader, length, &written, buffer.data()); + return String(buffer.data(), static_cast(written)); + } + + GLint QueryLinkStatus(const GLuint program) { + GLint status = GL_FALSE; + GetProgramiv(program, GL_LINK_STATUS, &status); + return status; + } + + // The non-joining view of the object, i.e. what GL_COMPLETION_STATUS_KHR will report. + Bool CompileIsSettled(const GLuint shader) { + const auto& object = MG_State::pGLContext->GetShaderObject(shader); + return object == nullptr || object->IsCompileComplete(); + } + + Bool HasMemoizedCompile(const GLuint shader) { + const auto& object = MG_State::pGLContext->GetShaderObject(shader); + return object != nullptr && object->HasMemoizedCompile(); + } + + // Enqueues `count` distinct heavy compiles and returns their names WITHOUT reading + // anything back, so the pool is left with a real backlog for the caller to race against. + Vector SaturatePool(const int count, Vector& sourceStorage) { + Vector shaders; + shaders.reserve(static_cast(count)); + sourceStorage.reserve(sourceStorage.size() + static_cast(count)); + for (int i = 0; i < count; ++i) { + sourceStorage.push_back(MakeBulkySource(1000 + i)); + const char* text = sourceStorage.back().c_str(); + const GLuint shader = CreateShader(GL_FRAGMENT_SHADER); + ShaderSource(shader, 1, &text, nullptr); + CompileShader(shader); + shaders.push_back(shader); + } + return shaders; + } + + class AsyncCompileTest : public ::testing::Test { + protected: + void SetUp() override { MobileGL::Initialize(); } + }; +} // namespace + +// --------------------------------------------------------------------------------------- +// Correctness through the full GL surface +// --------------------------------------------------------------------------------------- + +// N shaders compiled with the flag on: every status, every info log and every link has to +// come out the same as the synchronous path produces. +TEST_F(AsyncCompileTest, ManyShadersCompileAndLinkCorrectlyWithAsyncOn) { + const AsyncModeScope async(true); + ASSERT_TRUE(MG_Util::Async::AsyncShaderCompileEnabled()); + + constexpr int kCount = 24; + Vector vertexShaders; + Vector fragmentShaders; + Vector sources; + sources.reserve(kCount); + + // Enqueue everything first, read nothing: this is the shape a shaderpack load has, and + // the only shape where the pool has more than one job in flight at a time. + for (int i = 0; i < kCount; ++i) { + vertexShaders.push_back(MakeShader(GL_VERTEX_SHADER, kVs)); + sources.push_back(MakeBulkySource(i)); + const char* text = sources.back().c_str(); + const GLuint fs = CreateShader(GL_FRAGMENT_SHADER); + ShaderSource(fs, 1, &text, nullptr); + CompileShader(fs); + fragmentShaders.push_back(fs); + CompileShader(vertexShaders.back()); + } + + for (int i = 0; i < kCount; ++i) { + EXPECT_EQ(QueryCompileStatus(vertexShaders[i]), GL_TRUE) << QueryShaderInfoLog(vertexShaders[i]); + EXPECT_EQ(QueryCompileStatus(fragmentShaders[i]), GL_TRUE) << QueryShaderInfoLog(fragmentShaders[i]); + EXPECT_TRUE(QueryShaderInfoLog(vertexShaders[i]).empty()); + EXPECT_TRUE(QueryShaderInfoLog(fragmentShaders[i]).empty()); + } + + // And the artifacts are actually usable: link, and reflect a uniform out of each stage. + for (int i = 0; i < kCount; ++i) { + const GLuint program = CreateProgram(); + AttachShader(program, vertexShaders[i]); + AttachShader(program, fragmentShaders[i]); + LinkProgram(program); + ASSERT_EQ(QueryLinkStatus(program), GL_TRUE) << "program " << i; + EXPECT_GE(GetUniformLocation(program, "uColor"), 0); + EXPECT_GE(GetUniformLocation(program, ("uSeed" + std::to_string(i)).c_str()), 0); + } + EXPECT_EQ(GetError(), GL_NO_ERROR); +} + +// glCompileShader must return before the work is done. Timing-based assertions flake, so +// this observes the state machine instead: with a saturated pool at least one of the just +// -enqueued shaders has to be unsettled at the moment we ask. Skipped rather than failed if +// the machine drained the whole batch first - it can then never be a false red. +TEST_F(AsyncCompileTest, CompileShaderReturnsBeforeTheWorkIsDone) { + const AsyncModeScope async(true); + Vector sources; + const Vector shaders = SaturatePool(64, sources); + + int unsettled = 0; + for (const GLuint shader : shaders) { + if (!CompileIsSettled(shader)) ++unsettled; + } + if (unsettled == 0) { + GTEST_SKIP() << "the pool drained 64 compiles before the first observation; nothing to prove here"; + } + + // Whatever was outstanding still has to produce the right answer once asked. + for (const GLuint shader : shaders) { + EXPECT_EQ(QueryCompileStatus(shader), GL_TRUE) << QueryShaderInfoLog(shader); + EXPECT_TRUE(CompileIsSettled(shader)) << "reading COMPILE_STATUS must have joined"; + } + EXPECT_EQ(GetError(), GL_NO_ERROR); +} + +// The synchronous path must stay synchronous: with the flag off, a compile is finished by +// the time glCompileShader returns. This is the guard that keeps the default shippable. +TEST_F(AsyncCompileTest, CompileIsFullySynchronousWithAsyncOff) { + const AsyncModeScope async(false); + ASSERT_FALSE(MG_Util::Async::AsyncShaderCompileEnabled()); + + Vector sources; + const Vector shaders = SaturatePool(8, sources); + for (const GLuint shader : shaders) { + EXPECT_TRUE(CompileIsSettled(shader)); + EXPECT_TRUE(HasMemoizedCompile(shader)); + EXPECT_EQ(QueryCompileStatus(shader), GL_TRUE) << QueryShaderInfoLog(shader); + } + EXPECT_EQ(GetError(), GL_NO_ERROR); +} + +// --------------------------------------------------------------------------------------- +// Diagnostics: the failing paths must read identically in both modes +// --------------------------------------------------------------------------------------- + +// A compile failure is reported through COMPILE_STATUS and the info log, never through +// glGetError - that is exactly why moving the work off-thread is legal. Both failure +// classes are covered: the glslang parse failure and the lexical reserved-identifier +// rejection (which never reaches glslang at all). +TEST_F(AsyncCompileTest, FailingCompileLogIsByteIdenticalAcrossModes) { + for (const char* source : {kBrokenFs, kReservedIdentifierFs}) { + String syncLog; + { + const AsyncModeScope async(false); + const GLuint fs = MakeShader(GL_FRAGMENT_SHADER, source); + CompileShader(fs); + ASSERT_EQ(QueryCompileStatus(fs), GL_FALSE); + syncLog = QueryShaderInfoLog(fs); + EXPECT_FALSE(syncLog.empty()); + // GL defines compile FAILURE as a status plus a log, not as a GL error. + EXPECT_EQ(GetError(), GL_NO_ERROR); + } + { + const AsyncModeScope async(true); + const GLuint fs = MakeShader(GL_FRAGMENT_SHADER, source); + CompileShader(fs); + EXPECT_EQ(QueryCompileStatus(fs), GL_FALSE); + EXPECT_EQ(QueryShaderInfoLog(fs), syncLog); + EXPECT_EQ(GetError(), GL_NO_ERROR); + } + } +} + +// A link whose vertex shader failed to compile has to reproduce that shader's log verbatim +// inside the program info log, whichever thread produced it. +TEST_F(AsyncCompileTest, LinkDiagnosticsQuoteTheAsyncCompileLog) { + const AsyncModeScope async(true); + const GLuint vs = MakeShader(GL_VERTEX_SHADER, kVs); + const GLuint fs = MakeShader(GL_FRAGMENT_SHADER, kBrokenFs); + CompileShader(vs); + CompileShader(fs); + + const GLuint program = CreateProgram(); + AttachShader(program, vs); + AttachShader(program, fs); + // No status read between the enqueue and the link: the link's own prologue is what has + // to join the two compiles. + LinkProgram(program); + EXPECT_EQ(QueryLinkStatus(program), GL_FALSE); + + GLint length = 0; + GetProgramiv(program, GL_INFO_LOG_LENGTH, &length); + ASSERT_GT(length, 1); + std::vector buffer(static_cast(length)); + GLsizei written = 0; + GetProgramInfoLog(program, length, &written, buffer.data()); + const String programLog(buffer.data(), static_cast(written)); + const String shaderLog = QueryShaderInfoLog(fs); + ASSERT_FALSE(shaderLog.empty()); + EXPECT_NE(programLog.find(shaderLog), String::npos) + << "program log:\n" << programLog << "\nshader log:\n" << shaderLog; + EXPECT_EQ(GetError(), GL_NO_ERROR); +} + +// --------------------------------------------------------------------------------------- +// Mutation over an in-flight compile +// --------------------------------------------------------------------------------------- + +// glShaderSource with DIFFERENT text over a pending compile: the running job is abandoned +// and the next compile reflects the new source. The re-source happens with the pool +// saturated, so the job it replaces is very likely still queued or running. +TEST_F(AsyncCompileTest, ShaderSourceOverAPendingCompileCancelsAndTheNewSourceWins) { + const AsyncModeScope async(true); + Vector backlog; + SaturatePool(48, backlog); + + const String firstSource = MakeBulkySource(7001); + const char* firstText = firstSource.c_str(); + const GLuint fs = CreateShader(GL_FRAGMENT_SHADER); + ShaderSource(fs, 1, &firstText, nullptr); + CompileShader(fs); + + // Replace the text while that compile is (very probably) still outstanding. This must + // not wait, must not corrupt the abandoned job's view of the old string, and must + // disarm the layer-1 memo. + const String secondSource = MakeBulkySource(7002); + const char* secondText = secondSource.c_str(); + ShaderSource(fs, 1, &secondText, nullptr); + EXPECT_FALSE(HasMemoizedCompile(fs)) << "a real source change must invalidate the compiled state"; + EXPECT_EQ(QueryCompileStatus(fs), GL_FALSE) << "the replaced compile must not publish"; + + CompileShader(fs); + ASSERT_EQ(QueryCompileStatus(fs), GL_TRUE) << QueryShaderInfoLog(fs); + + const GLuint vs = MakeShader(GL_VERTEX_SHADER, kVs); + CompileShader(vs); + const GLuint program = CreateProgram(); + AttachShader(program, vs); + AttachShader(program, fs); + LinkProgram(program); + ASSERT_EQ(QueryLinkStatus(program), GL_TRUE); + // The SECOND source's uniform is the one that exists. + EXPECT_GE(GetUniformLocation(program, "uSeed7002"), 0); + EXPECT_EQ(GetUniformLocation(program, "uSeed7001"), -1); + EXPECT_EQ(GetError(), GL_NO_ERROR); +} + +// glShaderSource with byte-identical text over a pending compile is a no-op: the job stays, +// the memo stays armed, and the result is still the right one. +TEST_F(AsyncCompileTest, IdenticalShaderSourceOverAPendingCompileKeepsTheJob) { + const AsyncModeScope async(true); + Vector backlog; + SaturatePool(48, backlog); + + const String source = MakeBulkySource(7100); + const char* text = source.c_str(); + const GLuint fs = CreateShader(GL_FRAGMENT_SHADER); + ShaderSource(fs, 1, &text, nullptr); + CompileShader(fs); + + ShaderSource(fs, 1, &text, nullptr); + EXPECT_TRUE(HasMemoizedCompile(fs)) << "identical re-source must not disturb an in-flight compile"; + EXPECT_EQ(QueryCompileStatus(fs), GL_TRUE) << QueryShaderInfoLog(fs); + EXPECT_EQ(GetError(), GL_NO_ERROR); +} + +// A second glCompileShader on a pending object must be a no-op, not a duplicate job racing +// the first one to write the same fields. Observed through the object identity of the node: +// HasMemoizedCompile stays true across the second call, and the result is still correct. +TEST_F(AsyncCompileTest, RepeatedCompileShaderOnAPendingObjectEnqueuesOneJob) { + const AsyncModeScope async(true); + Vector backlog; + SaturatePool(48, backlog); + + const String source = MakeBulkySource(7200); + const char* text = source.c_str(); + const GLuint fs = CreateShader(GL_FRAGMENT_SHADER); + ShaderSource(fs, 1, &text, nullptr); + + // A copy, not the slot reference: creating another shader can reallocate the table. + const SharedPtr object = MG_State::pGLContext->GetShaderObject(fs); + ASSERT_NE(object, nullptr); + EXPECT_FALSE(object->HasMemoizedCompile()); + CompileShader(fs); + EXPECT_TRUE(object->HasMemoizedCompile()); + for (int i = 0; i < 8; ++i) { + CompileShader(fs); + EXPECT_TRUE(object->HasMemoizedCompile()); + } + + EXPECT_EQ(QueryCompileStatus(fs), GL_TRUE) << QueryShaderInfoLog(fs); + EXPECT_EQ(GetError(), GL_NO_ERROR); +} + +// glDeleteShader on an unattached object with a compile still in flight. The name goes away +// immediately - no wait for a worker - and the abandoned job must neither crash nor keep the +// object alive in a way anything can observe. +TEST_F(AsyncCompileTest, DeleteShaderWhileACompileIsPending) { + const AsyncModeScope async(true); + Vector backlog; + SaturatePool(48, backlog); + + Vector doomed; + Vector sources; + for (int i = 0; i < 16; ++i) { + sources.push_back(MakeBulkySource(7300 + i)); + const char* text = sources.back().c_str(); + const GLuint fs = CreateShader(GL_FRAGMENT_SHADER); + ShaderSource(fs, 1, &text, nullptr); + CompileShader(fs); + doomed.push_back(fs); + } + for (const GLuint fs : doomed) { + DeleteShader(fs); + EXPECT_EQ(IsShader(fs), GL_FALSE) << "an unattached deleted shader's name goes immediately"; + } + EXPECT_EQ(GetError(), GL_NO_ERROR); + + // The context still works afterwards - the abandoned jobs did not take the pool, the + // preprocess cache or the glslang process state down with them. + const GLuint vs = MakeShader(GL_VERTEX_SHADER, kVs); + const GLuint fs = MakeShader(GL_FRAGMENT_SHADER, kFs); + CompileShader(vs); + CompileShader(fs); + const GLuint program = CreateProgram(); + AttachShader(program, vs); + AttachShader(program, fs); + LinkProgram(program); + EXPECT_EQ(QueryLinkStatus(program), GL_TRUE); + EXPECT_EQ(GetError(), GL_NO_ERROR); +} + +// glDeleteShader on a shader still ATTACHED to a program only flags it: the pending compile +// has to survive, because the link that follows still needs its artifacts. +TEST_F(AsyncCompileTest, DeleteShaderWhileAttachedKeepsThePendingCompileAlive) { + const AsyncModeScope async(true); + Vector backlog; + SaturatePool(48, backlog); + + const GLuint vs = MakeShader(GL_VERTEX_SHADER, kVs); + const GLuint fs = MakeShader(GL_FRAGMENT_SHADER, kFs); + const GLuint program = CreateProgram(); + AttachShader(program, vs); + AttachShader(program, fs); + CompileShader(vs); + CompileShader(fs); + DeleteShader(vs); + DeleteShader(fs); + + LinkProgram(program); + ASSERT_EQ(QueryLinkStatus(program), GL_TRUE); + EXPECT_GE(GetUniformLocation(program, "uColor"), 0); + EXPECT_GE(GetUniformLocation(program, "uAlpha"), 0); + EXPECT_EQ(GetError(), GL_NO_ERROR); +} + +// --------------------------------------------------------------------------------------- +// Stress +// --------------------------------------------------------------------------------------- + +// The adversarial interleaving: enqueue, query, re-source, re-enqueue, delete, all with the +// pool busy. Nothing here asserts timing - what it hunts for is a missed join or a use of an +// abandoned node, both of which surface as a wrong status, a wrong log, or a crash. +TEST_F(AsyncCompileTest, StressCompileQueryResourceDeleteInterleaved) { + const AsyncModeScope async(true); + constexpr int kRounds = 6; + constexpr int kPerRound = 12; + + for (int round = 0; round < kRounds; ++round) { + Vector sources; + Vector shaders; + sources.reserve(kPerRound * 2); + + for (int i = 0; i < kPerRound; ++i) { + sources.push_back(MakeBulkySource(round * 1000 + i)); + const char* text = sources.back().c_str(); + const GLuint fs = CreateShader(GL_FRAGMENT_SHADER); + ShaderSource(fs, 1, &text, nullptr); + CompileShader(fs); + shaders.push_back(fs); + + // Immediately query a PREVIOUS one while this one is still outstanding: the + // join has to settle exactly the object asked about and no other. + if (i > 0) { + const GLuint earlier = shaders[static_cast(i - 1)]; + EXPECT_EQ(QueryCompileStatus(earlier), GL_TRUE) << QueryShaderInfoLog(earlier); + } + } + + // Re-source half of them mid-flight, then recompile. + for (int i = 0; i < kPerRound; i += 2) { + sources.push_back(MakeBulkySource(round * 1000 + 500 + i)); + const char* text = sources.back().c_str(); + ShaderSource(shaders[static_cast(i)], 1, &text, nullptr); + CompileShader(shaders[static_cast(i)]); + } + + for (int i = 0; i < kPerRound; ++i) { + const GLuint shader = shaders[static_cast(i)]; + EXPECT_EQ(QueryCompileStatus(shader), GL_TRUE) << QueryShaderInfoLog(shader); + const String expectedUniform = + "uSeed" + std::to_string(round * 1000 + (i % 2 == 0 ? 500 + i : i)); + const GLuint vs = MakeShader(GL_VERTEX_SHADER, kVs); + CompileShader(vs); + const GLuint program = CreateProgram(); + AttachShader(program, vs); + AttachShader(program, shader); + LinkProgram(program); + ASSERT_EQ(QueryLinkStatus(program), GL_TRUE) << "round " << round << " shader " << i; + EXPECT_GE(GetUniformLocation(program, expectedUniform.c_str()), 0) + << "round " << round << " shader " << i << " expected " << expectedUniform; + DeleteProgram(program); + DeleteShader(vs); + } + for (const GLuint shader : shaders) { + DeleteShader(shader); + } + EXPECT_EQ(GetError(), GL_NO_ERROR); + } +} + +// The P0b cross-object memo is hit from several workers at once here: 8 objects share each +// of 6 distinct sources, all enqueued before anything is read. Every object must still end +// up with its own parse and its own correct reflection - a torn cache entry or an entry +// evicted from under a reader shows up as a link failure or a missing uniform. +TEST_F(AsyncCompileTest, ConcurrentCompilesShareThePreprocessCacheSafely) { + const AsyncModeScope async(true); + constexpr int kDistinct = 6; + constexpr int kDuplicates = 8; + + Vector sources; + sources.reserve(kDistinct); + for (int i = 0; i < kDistinct; ++i) { + sources.push_back(MakeBulkySource(8100 + i)); + } + + Vector shaders; + for (int duplicate = 0; duplicate < kDuplicates; ++duplicate) { + for (int i = 0; i < kDistinct; ++i) { + const char* text = sources[static_cast(i)].c_str(); + const GLuint fs = CreateShader(GL_FRAGMENT_SHADER); + ShaderSource(fs, 1, &text, nullptr); + CompileShader(fs); + shaders.push_back(fs); + } + } + + for (SizeT s = 0; s < shaders.size(); ++s) { + const GLuint fs = shaders[s]; + ASSERT_EQ(QueryCompileStatus(fs), GL_TRUE) << QueryShaderInfoLog(fs); + const GLuint vs = MakeShader(GL_VERTEX_SHADER, kVs); + CompileShader(vs); + const GLuint program = CreateProgram(); + AttachShader(program, vs); + AttachShader(program, fs); + LinkProgram(program); + ASSERT_EQ(QueryLinkStatus(program), GL_TRUE) << "shader index " << s; + const String uniform = "uSeed" + std::to_string(8100 + static_cast(s % kDistinct)); + EXPECT_GE(GetUniformLocation(program, uniform.c_str()), 0) << uniform; + } + EXPECT_EQ(GetError(), GL_NO_ERROR); +} diff --git a/MobileGL/MG_Test/Program/CMakeLists.txt b/MobileGL/MG_Test/Program/CMakeLists.txt index 95d23788..55da30a5 100644 --- a/MobileGL/MG_Test/Program/CMakeLists.txt +++ b/MobileGL/MG_Test/Program/CMakeLists.txt @@ -28,6 +28,22 @@ add_executable( ProgramTest.cpp ) +add_executable( + AsyncCompileTest + AsyncCompileTest.cpp +) + +target_include_directories(AsyncCompileTest PRIVATE + ${MGL_ROOT}/include + ${MGL_ROOT}/MobileGL +) + +target_link_libraries( + AsyncCompileTest PRIVATE + GTest::gtest_main + ${LINK_LIBRARIES} +) + target_include_directories(ProgramTest PRIVATE ${MGL_ROOT}/include ${MGL_ROOT}/MobileGL @@ -42,3 +58,6 @@ target_link_libraries( include(GoogleTest) gtest_discover_tests(ProgramUtilTest DISCOVERY_TIMEOUT 30 PROPERTIES LABELS unit) gtest_discover_tests(ProgramTest DISCOVERY_TIMEOUT 30 PROPERTIES LABELS unit) +# Heavier than the rest of the unit suite by design: several cases deliberately saturate the +# compile pool so there is something in flight to race against. +gtest_discover_tests(AsyncCompileTest DISCOVERY_TIMEOUT 60 PROPERTIES LABELS unit TIMEOUT 300) diff --git a/MobileGL/MG_Test/Util/JobNodeTest.cpp b/MobileGL/MG_Test/Util/JobNodeTest.cpp index 9760d9b4..e30702d9 100644 --- a/MobileGL/MG_Test/Util/JobNodeTest.cpp +++ b/MobileGL/MG_Test/Util/JobNodeTest.cpp @@ -135,12 +135,25 @@ TEST(ShaderCompilePoolLifecycle, DetectedThreadCountIsPositive) { EXPECT_GE(DetectShaderCompileThreadCount(), 1u); } -TEST(ShaderCompilePoolLifecycle, AsyncIsOffByDefaultInThisStage) { - // Stage 1 ships the machinery wired to nothing. If this ever fails without the default - // constant having been deliberately flipped, something enabled async by accident. - EXPECT_EQ(MG_Config::Features.AsyncShaderCompile, MG_Config::QuirkOverride::Auto); +TEST(ShaderCompilePoolLifecycle, AsyncIsOffByDefaultAndTheOverrideDecidesEitherWay) { + // The shipped default is still off, and an unset MOBILEGL_ASYNC_SHADER_COMPILE resolves + // to it. If the first expectation ever fails without the constant having been + // deliberately flipped, something enabled async by accident. + // + // Driven through Features rather than read from it: from stage 3 on, the whole suite is + // also run with MOBILEGL_ASYNC_SHADER_COMPILE=1 exported, so a test that simply asserted + // "the resolved answer is false" would either fail there or - worse - silently pass in a + // binary that never loaded the config and prove nothing at all. EXPECT_FALSE(kAsyncShaderCompileDefault); + + const MG_Config::QuirkOverride saved = MG_Config::Features.AsyncShaderCompile; + MG_Config::Features.AsyncShaderCompile = MG_Config::QuirkOverride::Auto; + EXPECT_EQ(AsyncShaderCompileEnabled(), kAsyncShaderCompileDefault); + MG_Config::Features.AsyncShaderCompile = MG_Config::QuirkOverride::ForceOn; + EXPECT_TRUE(AsyncShaderCompileEnabled()); + MG_Config::Features.AsyncShaderCompile = MG_Config::QuirkOverride::ForceOff; EXPECT_FALSE(AsyncShaderCompileEnabled()); + MG_Config::Features.AsyncShaderCompile = saved; } // --------------------------------------------------------------------------------------- diff --git a/MobileGL/MG_Util/Async/JobNode.cpp b/MobileGL/MG_Util/Async/JobNode.cpp index 35b8a7e2..011d201e 100644 --- a/MobileGL/MG_Util/Async/JobNode.cpp +++ b/MobileGL/MG_Util/Async/JobNode.cpp @@ -9,6 +9,8 @@ #include "JobNode.h" #include "ShaderCompilePool.h" +#include + namespace MobileGL::MG_Util::Async { namespace { Bool IsTerminalState(const JobState state) { @@ -122,4 +124,35 @@ namespace MobileGL::MG_Util::Async { } fn(); } + + void ApplyDeferredDiagnostics(JobNode& node) { + MOBILEGL_ASSERT(!ShaderCompilePool::IsPoolThread(), + "ApplyDeferredDiagnostics() called from a pool thread; deferred diagnostics exist precisely " + "so that a worker never touches the GL error state"); + MOBILEGL_ASSERT(node.IsTerminal(), + "ApplyDeferredDiagnostics() called on a job that has not settled; its diagnostics are still " + "being written"); + + if (!node.diagnostics.logLines.empty()) { + Vector lines; + lines.swap(node.diagnostics.logLines); + for (const String& line : lines) { + MGLOG_W("%s", line.c_str()); + } + } + + if (node.diagnostics.errors.empty()) return; + Vector errors; + errors.swap(node.diagnostics.errors); + // Ascending sequence == job-enqueue order == the order a serial implementation would + // have recorded them in, which is what decides WHICH payload the application sees: + // MobileGL implements GL's sticky-flag semantics, so a repeat of an already-pending + // code is discarded and only the first occurrence of each code survives. + std::sort(errors.begin(), errors.end(), + [](const DeferredError& a, const DeferredError& b) { return a.sequence < b.sequence; }); + if (!MG_State::pGLContext) return; + for (DeferredError& error : errors) { + MG_State::pGLContext->RecordError(error.code, Move(error.info)); + } + } } // namespace MobileGL::MG_Util::Async diff --git a/MobileGL/MG_Util/Async/JobNode.h b/MobileGL/MG_Util/Async/JobNode.h index 6f61e3d6..a7f4aef4 100644 --- a/MobileGL/MG_Util/Async/JobNode.h +++ b/MobileGL/MG_Util/Async/JobNode.h @@ -113,4 +113,15 @@ namespace MobileGL::MG_Util::Async { std::atomic m_cancelled{false}; Vector> m_continuations; }; + + // Replays a settled node's worker-side diagnostics on the calling thread: log lines + // first, in the order the body produced them, then any deferred GL error in ascending + // `sequence`. GL thread only - it is the join that calls this, which is exactly the + // point at which a deferred error becomes indistinguishable from one a serial + // implementation would have raised inside glCompileShader/glLinkProgram (an application + // cannot observe a pending job's effects by any other route). + // + // Drains what it replays, so calling it twice on one node is a no-op the second time. + // Must be called with the node terminal. + void ApplyDeferredDiagnostics(JobNode& node); } // namespace MobileGL::MG_Util::Async diff --git a/MobileGL/MG_Util/Async/ShaderCompilePool.cpp b/MobileGL/MG_Util/Async/ShaderCompilePool.cpp index bf58e609..1284100c 100644 --- a/MobileGL/MG_Util/Async/ShaderCompilePool.cpp +++ b/MobileGL/MG_Util/Async/ShaderCompilePool.cpp @@ -36,11 +36,29 @@ namespace MobileGL::MG_Util::Async { // first pool use, so it is guaranteed to run before any static destructor. Bool g_processTeardown = false; std::once_flag g_teardownSentinelOnce; + // The process-wide pool from Get(), for the atexit handler to stop. Never the + // stack-allocated pools a test builds - those join themselves in their destructor. + std::atomic g_processPool{nullptr}; Bool InProcessTeardown() { return g_processTeardown; } void EnsureProcessTeardownSentinel() { - std::call_once(g_teardownSentinelOnce, [] { std::atexit(+[] { g_processTeardown = true; }); }); + std::call_once(g_teardownSentinelOnce, [] { + std::atexit(+[] { + g_processTeardown = true; + // Latching the flag is not enough: a worker that is ALREADY inside + // glslang has to be out of it before static destruction reaches + // glslang's process globals, the SPIRV-Tools tables, or anything else a + // job body touches. This is the same wait Init.cpp's DestroyImpl does - + // it just also has to happen for a process that exits without ever + // calling eglTerminate, which is the norm for a test binary and legal + // for an application. Registered here, during main, so it runs before + // the destructors of statics constructed at load time. + if (ShaderCompilePool* pool = g_processPool.load(std::memory_order_acquire)) { + pool->StopAndDrain(); + } + }); + }); } Uint64 ReadCpuMaxFrequencyKHz(const Uint cpu) { @@ -120,10 +138,20 @@ namespace MobileGL::MG_Util::Async { // re-enter this mutex. void DispatchLocked() { while (!queue.empty() && inFlight < maxConcurrency && !stopped.load(std::memory_order_acquire)) { - SharedPtr node = Move(queue.front()); + // Copy rather than move into the handler: if asio::post throws (it allocates) + // the local SharedPtr is still valid, so the node can be settled instead of + // being stranded Pending in a queue nothing will dispatch from again - a + // joiner would block on it forever. Reclaiming the slot matters just as much: + // a leaked `inFlight` shrinks the pool's concurrency budget permanently. + SharedPtr node = queue.front(); queue.pop_front(); ++inFlight; - asio::post(*pool, [this, node = Move(node)]() mutable { RunOnWorker(Move(node)); }); + try { + asio::post(*pool, [this, node]() mutable { RunOnWorker(Move(node)); }); + } catch (...) { + --inFlight; + node->Cancel(); + } } } @@ -147,10 +175,17 @@ namespace MobileGL::MG_Util::Async { ShaderCompilePool::~ShaderCompilePool() { StopAndDrain(); } ShaderCompilePool& ShaderCompilePool::Get() { - // Leak-at-exit, like the other MobileGL singletons: a process that exits without - // eglTerminate hands the threads to the OS rather than joining them from a static - // destructor, where the rest of the library may already be gone. - static ShaderCompilePool* pool = new ShaderCompilePool(DetectShaderCompileThreadCount()); + // Leak-at-exit, like the other MobileGL singletons: the object itself is never + // destroyed, so no static destructor can race a late entry point for it. Its THREADS + // are a different matter and are stopped explicitly - by Init.cpp's DestroyImpl on + // the normal path, and by the atexit sentinel below for a process that exits without + // ever calling eglTerminate. + static ShaderCompilePool* pool = [] { + auto* created = new ShaderCompilePool(DetectShaderCompileThreadCount()); + g_processPool.store(created, std::memory_order_release); + EnsureProcessTeardownSentinel(); + return created; + }(); return *pool; } @@ -174,7 +209,18 @@ namespace MobileGL::MG_Util::Async { if (!node) return; EnsureProcessTeardownSentinel(); - { + // Enqueueing can throw: the thread_pool construction and asio::post both allocate, + // and under memory pressure a throw here would escape glCompileShader leaving the + // node Pending with nothing left to dispatch it - the first observable read would + // then block the GL thread forever. Settle the node instead: a cancelled node is a + // state every joiner already handles. + // + // `node` is still valid in the catch for every throw this try can produce. The + // thread_pool construction runs before the move; deque::push_back is strongly + // exception-safe and SharedPtr's move constructor is noexcept, so a throwing + // push_back never consumed it; and DispatchLocked contains its own asio::post + // failures rather than propagating them (see above). Keep it that way. + try { const std::lock_guard lock(m_impl->mutex); if (!m_impl->stopped.load(std::memory_order_acquire) && !InProcessTeardown()) { if (!m_impl->pool) m_impl->pool = MakeUnique(m_impl->threadCount); @@ -182,12 +228,27 @@ namespace MobileGL::MG_Util::Async { m_impl->DispatchLocked(); return; } + } catch (...) { + MGLOG_E("ShaderCompilePool::Post: enqueue failed; cancelling the job so its joiner " + "cannot block forever"); + if (node) node->Cancel(); + return; } // A stopped pool is a synchronous pool, not a black hole: the node still runs, just // on the caller's thread. Everything downstream already handles "terminal by the time // Post returns", because that is exactly what the inline path looks like. Run it // outside the lock - a body, or a continuation it releases, is free to Post again. + // + // Say so once. StopAndDrain is a one-way latch (see its tail), so from the first + // eglTerminate onwards EVERY compile in this process silently runs on the GL thread; + // without this line the only symptom is that asynchronous compilation stopped helping, + // with nothing in the log to point at. Once, not per node: a pack load posts hundreds. + static std::atomic warnedStopped{false}; + if (!warnedStopped.exchange(true, std::memory_order_relaxed)) { + MGLOG_W("ShaderCompilePool::Post: the pool is stopped (eglTerminate, or process exit); shader " + "compilation runs inline on the calling thread until MobileGL is re-initialized"); + } node->RunInline(); } diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp index f59e5e73..095fa235 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.cpp @@ -249,6 +249,56 @@ namespace MobileGL { return retryResult; } + // Namespace-level rather than a function-local static, because it has to be + // CLEARABLE: what PrewarmBuiltins latches is not a property of this process, it is + // a property of the built-in symbol tables glslang currently holds, and + // glslang::FinalizeProcess() deletes those. A function-local latch survived the + // teardown that invalidated it, so an Initialize -> Destroy -> Initialize cycle + // came back up with the tables gone and the prewarm skipped - which is exactly the + // serialized-first-parse stall this function exists to prevent, only now + // unfixable for the rest of the process. Reset it from DestroyImpl. + namespace { + Bool g_builtinsPrewarmed = false; + } // namespace + + void ShaderCompiler::ResetPrewarmLatch() { g_builtinsPrewarmed = false; } + + void ShaderCompiler::PrewarmBuiltins() { + if (g_builtinsPrewarmed) return; + g_builtinsPrewarmed = true; + + // One vertex and one fragment shader is enough: the built-in table is cached + // per (version, spvVersion, profile, source), not per stage language, and + // both configurations CompileShader can reach - the declared-460 path and + // the retargeted-legacy path - resolve to the same combination here because + // ParseShaderSource always passes 460/ECoreProfile as the default. Parsing + // both anyway costs microseconds and keeps this honest if that ever changes. + static constexpr const char* kPrewarmVertexSource = + "#version 460\nvoid main() { gl_Position = vec4(0.0); }\n"; + static constexpr const char* kPrewarmFragmentSource = + "#version 460\nlayout(location = 0) out vec4 c;\nvoid main() { c = vec4(0.0); }\n"; + static constexpr const char* kPrewarmLegacyVertexSource = + "#version 330 core\nvoid main() { gl_Position = vec4(0.0); }\n"; + + const CompileEnv& env = *GetDefaultCompileEnv(); + for (const auto& [type, source] : + {std::pair{GL_VERTEX_SHADER, kPrewarmVertexSource}, + std::pair{GL_FRAGMENT_SHADER, kPrewarmFragmentSource}, + std::pair{GL_VERTEX_SHADER, kPrewarmLegacyVertexSource}}) { + ShaderAttrib attrib{.shaderType = static_cast(type), + .sourceStr = source, + .flags = 0, + .env = &env}; + // The result is deliberately discarded: the value is the symbol table + // glslang cached as a side effect. A failure here is not fatal - it just + // means the first real compile pays for the table, exactly as before. + (void)CompileShader(attrib); + } + // The parses above left this thread's glslang allocator pointing at the last + // TShader's pool, and that TShader is about to be destroyed with it. + glslang::SetThreadPoolAllocator(nullptr); + } + Result> ShaderCompiler::LinkProgram(const ProgramAttrib& attrib) { SharedPtr program = MakeShared(); for (auto& s : attrib.shaders) { diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h index 726b4009..bd1a5964 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderCompiler.h @@ -77,6 +77,30 @@ namespace MobileGL { static bool UseUnformattedFloatStorageImagesForVulkan( const Vector& inputBinary, Vector& outputBinary); static Result DecompileShader(SpvcSession& session); + + // Parses one trivial shader in each configuration the production path can + // reach, on the calling thread, so the built-in symbol tables those + // configurations need are already cached before any worker asks for one. + // + // Why it matters: glslang builds a built-in TSymbolTable per distinct + // (version, spvVersion, profile, source) combination, and does it under a + // process-wide lock held for the whole build. Without this, the first + // parallel compiles of a shaderpack load all pile up behind that lock and + // show no speedup at all - which is easy to misread as asynchronous + // compilation not working. Call once, from the GL thread, right after + // glslang::InitializeProcess(). Idempotent and cheap on repeat. + // + // Only worth its cost when compiles can actually run in parallel, so the GL + // frontend calls it only when asynchronous compilation is enabled: a + // synchronous build would pay for three throwaway parses at every + // eglInitialize to prewarm tables the first real compile builds anyway. + static void PrewarmBuiltins(); + // Clears the "already prewarmed" latch. MUST be called wherever + // glslang::FinalizeProcess() is, and for the same reason: finalizing deletes + // the cached built-in tables the latch is asserting the existence of. Without + // it, the second eglInitialize of a process comes back up unwarmed and with + // no way left to warm it. + static void ResetPrewarmLatch(); }; } // namespace ShaderTranspiler } // namespace MG_Util