From 2eafce6e28f7cfa6d0bb60a3da75c1e927fb1ceb Mon Sep 17 00:00:00 2001 From: Swung0x48 Date: Thu, 16 Jul 2026 17:18:07 -0400 Subject: [PATCH] [Fix] (MG_Util/ShaderTranspiler): normalize legacy desktop shaders to GLSL 330 --- MobileGL/MG_Test/Program/ProgramUtilTest.cpp | 247 +++++++++++++++- .../ShaderSourceProcessor.cpp | 266 +++++++++++++++--- 2 files changed, 464 insertions(+), 49 deletions(-) diff --git a/MobileGL/MG_Test/Program/ProgramUtilTest.cpp b/MobileGL/MG_Test/Program/ProgramUtilTest.cpp index 652826f6..75874de4 100644 --- a/MobileGL/MG_Test/Program/ProgramUtilTest.cpp +++ b/MobileGL/MG_Test/Program/ProgramUtilTest.cpp @@ -107,7 +107,7 @@ void main() { PreprocessShaderSource(ShaderStage::Vertex, source); - EXPECT_EQ(source.find("#version 460 core\n"), 0); + EXPECT_EQ(source.find("#version 330 core\n"), 0); EXPECT_NE(source.find("in vec3 position;"), String::npos); EXPECT_NE(source.find("out vec2 uv;"), String::npos); EXPECT_EQ(source.find("attribute"), String::npos); @@ -136,7 +136,7 @@ void main() { PreprocessShaderSource(ShaderStage::Fragment, source); - EXPECT_EQ(source.find("#version 460 core\n"), 0); + EXPECT_EQ(source.find("#version 330 core\n"), 0); EXPECT_NE(source.find("out vec4 mg_FragColor;\n"), String::npos); EXPECT_NE(source.find("in vec2 uv;"), String::npos); EXPECT_NE(source.find("texture(texture0, uv)"), String::npos); @@ -153,6 +153,243 @@ void main() { } } +TEST_F(ProgramUtilTest, PreprocessMinecraft112BlurShaderKeepsLegacySampleIdentifier) { + using namespace MG_Util::ShaderTranspiler; + + // assets/minecraft/shaders/program/blur.fsh from the unmodified Minecraft 1.12 client jar. + String source = R"(#version 120 + +uniform sampler2D DiffuseSampler; + +varying vec2 texCoord; +varying vec2 oneTexel; + +uniform vec2 InSize; + +uniform vec2 BlurDir; +uniform float Radius; + +void main() { + vec4 blurred = vec4(0.0); + float totalStrength = 0.0; + float totalAlpha = 0.0; + float totalSamples = 0.0; + for(float r = -Radius; r <= Radius; r += 1.0) { + vec4 sample = texture2D(DiffuseSampler, texCoord + oneTexel * r * BlurDir); + + // Accumulate average alpha + totalAlpha = totalAlpha + sample.a; + totalSamples = totalSamples + 1.0; + + // Accumulate smoothed blur + float strength = 1.0 - abs(r / Radius); + totalStrength = totalStrength + strength; + blurred = blurred + sample; + } + gl_FragColor = vec4(blurred.rgb / (Radius * 2.0 + 1.0), totalAlpha); +} +)"; + + PreprocessShaderSource(ShaderStage::Fragment, source); + + EXPECT_EQ(source.find("#version 330 core\n"), 0); + EXPECT_NE(source.find("vec4 sample = texture(DiffuseSampler"), String::npos); + EXPECT_NE(source.find("totalAlpha = totalAlpha + sample.a;"), String::npos); + EXPECT_NE(source.find("float totalSamples = 0.0;"), String::npos); + EXPECT_NE(source.find("totalSamples = totalSamples + 1.0;"), String::npos); + EXPECT_NE(source.find("blurred = blurred + sample;"), String::npos); + + ShaderAttrib attrib{.shaderType = GL_FRAGMENT_SHADER, .sourceStr = source}; + auto res = ShaderCompiler::CompileShader(attrib); + if (!res) { + FAIL() << "errc: " << res.error().errc << "\nlog: " << res.error().log << "\nsource:\n" << source; + } +} + +TEST_F(ProgramUtilTest, PreprocessLegacySampleInterfaceIdentifiersKeepNames) { + using namespace MG_Util::ShaderTranspiler; + + String vertexSource = R"(#version 150 +attribute vec3 sample; + +void main() { + gl_Position = vec4(sample, 1.0); +} +)"; + PreprocessShaderSource(ShaderStage::Vertex, vertexSource); + + EXPECT_EQ(vertexSource.find("#version 330 core\n"), 0); + EXPECT_NE(vertexSource.find("in vec3 sample;"), String::npos); + + ShaderAttrib vertexAttrib{.shaderType = GL_VERTEX_SHADER, .sourceStr = vertexSource}; + auto vertexResult = ShaderCompiler::CompileShader(vertexAttrib); + if (!vertexResult) { + FAIL() << "errc: " << vertexResult.error().errc << "\nlog: " << vertexResult.error().log + << "\nsource:\n" << vertexSource; + } + + String fragmentSource = R"(#version 150 +uniform sampler2D sample; +varying vec2 texCoord; + +void main() { + gl_FragColor = texture2D(sample, texCoord); +} +)"; + PreprocessShaderSource(ShaderStage::Fragment, fragmentSource); + + EXPECT_EQ(fragmentSource.find("#version 330 core\n"), 0); + EXPECT_NE(fragmentSource.find("uniform sampler2D sample;"), String::npos); + EXPECT_NE(fragmentSource.find("texture(sample, texCoord)"), String::npos); + + ShaderAttrib fragmentAttrib{.shaderType = GL_FRAGMENT_SHADER, .sourceStr = fragmentSource}; + auto fragmentResult = ShaderCompiler::CompileShader(fragmentAttrib); + if (!fragmentResult) { + FAIL() << "errc: " << fragmentResult.error().errc << "\nlog: " << fragmentResult.error().log + << "\nsource:\n" << fragmentSource; + } +} + +TEST_F(ProgramUtilTest, PreprocessEsslVersionsRemainVulkanCompatible) { + using namespace MG_Util::ShaderTranspiler; + + const auto verifyVersion = [](const char* inputVersion, const char* expectedVersion) { + SCOPED_TRACE(inputVersion); + String source = inputVersion; + source += R"( +precision mediump float; +out vec4 fragColor; + +void main() { + fragColor = vec4(1.0); +} +)"; + PreprocessShaderSource(ShaderStage::Fragment, source); + + EXPECT_EQ(source.find(expectedVersion), 0); + + ShaderAttrib attrib{.shaderType = GL_FRAGMENT_SHADER, .sourceStr = source}; + auto res = ShaderCompiler::CompileShader(attrib); + if (!res) { + FAIL() << "errc: " << res.error().errc << "\nlog: " << res.error().log << "\nsource:\n" << source; + } + }; + + // Preserve the pre-existing desktop-core route: the current resource table cannot parse ESSL built-ins. + verifyVersion("#version 300 es", "#version 460 core\n"); + verifyVersion("#version 310 es", "#version 460 core\n"); +} + +TEST_F(ProgramUtilTest, PreprocessModernDesktopVersionsRecognizesUtf8Bom) { + using namespace MG_Util::ShaderTranspiler; + + const auto verifyVersion = [](const char* inputVersion) { + SCOPED_TRACE(inputVersion); + String source = "\xef\xbb\xbf"; + source += inputVersion; + source += R"( +out vec4 fragColor; + +void main() { + fragColor = vec4(1.0); +} +)"; + PreprocessShaderSource(ShaderStage::Fragment, source); + + EXPECT_EQ(source.find("#version 460 core\n"), 0); + EXPECT_EQ(source.find("\xef\xbb\xbf"), String::npos); + + ShaderAttrib attrib{.shaderType = GL_FRAGMENT_SHADER, .sourceStr = source}; + auto res = ShaderCompiler::CompileShader(attrib); + if (!res) { + FAIL() << "errc: " << res.error().errc << "\nlog: " << res.error().log << "\nsource:\n" << source; + } + }; + + verifyVersion("#version 400 core"); + verifyVersion("#version 460 core"); +} + +TEST_F(ProgramUtilTest, PreprocessUsesRealSpacedVersionDirectiveForInjectedOutput) { + using namespace MG_Util::ShaderTranspiler; + + String source = R"(// #version 460 core +/* "#version 400 core" */ +#line 7 "#version 460 core" +# version 120 +varying vec2 uv; + +void main() { + gl_FragColor = vec4(uv, 0.0, 1.0); +} +)"; + PreprocessShaderSource(ShaderStage::Fragment, source); + + const SizeT versionPos = source.find("#version 330 core\n"); + const SizeT outputPos = source.find("out vec4 mg_FragColor;\n"); + EXPECT_NE(versionPos, String::npos); + EXPECT_EQ(outputPos, versionPos + std::strlen("#version 330 core\n")); + EXPECT_NE(source.find("// #version 460 core"), String::npos); + EXPECT_EQ(source.find("#line"), String::npos); + + ShaderAttrib attrib{.shaderType = GL_FRAGMENT_SHADER, .sourceStr = source}; + auto res = ShaderCompiler::CompileShader(attrib); + if (!res) { + FAIL() << "errc: " << res.error().errc << "\nlog: " << res.error().log << "\nsource:\n" << source; + } +} + +TEST_F(ProgramUtilTest, PreprocessModernSampleQualifierStaysAtVersion460) { + using namespace MG_Util::ShaderTranspiler; + + String source = R"(#version 400 core +sample in vec4 interpolatedColor; +out vec4 fragColor; + +void main() { + fragColor = interpolatedColor; +} +)"; + PreprocessShaderSource(ShaderStage::Fragment, source); + + EXPECT_EQ(source.find("#version 460 core\n"), 0); + EXPECT_NE(source.find("sample in vec4 interpolatedColor;"), String::npos); + + ShaderAttrib attrib{.shaderType = GL_FRAGMENT_SHADER, .sourceStr = source}; + auto res = ShaderCompiler::CompileShader(attrib); + if (!res) { + FAIL() << "errc: " << res.error().errc << "\nlog: " << res.error().log << "\nsource:\n" << source; + } +} + +TEST_F(ProgramUtilTest, PreprocessGpuShader5SampleQualifierUsesVersion460) { + using namespace MG_Util::ShaderTranspiler; + + for (const char* extension : {"GL_ARB_gpu_shader5", "GL_NV_gpu_shader5"}) { + SCOPED_TRACE(extension); + String source = "#version 150\n#extension "; + source += extension; + source += R"( : enable +sample in vec4 interpolatedColor; +out vec4 fragColor; + +void main() { + fragColor = interpolatedColor; +} +)"; + PreprocessShaderSource(ShaderStage::Fragment, source); + + EXPECT_EQ(source.find("#version 460 core\n"), 0); + EXPECT_NE(source.find("sample in vec4 interpolatedColor;"), String::npos); + + ShaderAttrib attrib{.shaderType = GL_FRAGMENT_SHADER, .sourceStr = source}; + auto res = ShaderCompiler::CompileShader(attrib); + if (!res) { + FAIL() << "errc: " << res.error().errc << "\nlog: " << res.error().log << "\nsource:\n" << source; + } + } +} + TEST_F(ProgramUtilTest, PreprocessLegacyFragmentShaderModernizesFragData) { using namespace MG_Util::ShaderTranspiler; @@ -164,7 +401,7 @@ void main() { PreprocessShaderSource(ShaderStage::Fragment, source); - EXPECT_EQ(source.find("#version 460 core\n"), 0); + EXPECT_EQ(source.find("#version 330 core\n"), 0); EXPECT_NE(source.find("layout(location = 0) out vec4 mg_FragData[8];\n"), String::npos); EXPECT_NE(source.find("mg_FragData[0] = vec4(1.0);"), String::npos); EXPECT_NE(source.find("mg_FragData[1].a = 0.5;"), String::npos); @@ -182,7 +419,7 @@ TEST_F(ProgramUtilTest, PreprocessKeepsDefaultPrecisionStatements) { // Mirrors the GL CTS helper shaders (e.g. glcPixelStorageModesTests): the old qualifier strip // turned "precision highp float;" into invalid "precision float;". Precision qualifiers are - // legal (and ignored) in the forced 460 core profile, so they now pass through untouched. + // legal (and ignored) in the normalized desktop core profile, so they now pass through untouched. String source = R"(#version 330 precision highp float; precision mediump int; @@ -212,7 +449,7 @@ TEST_F(ProgramUtilTest, PreprocessKeepsPrecisionInLegacyShaderForGlslang) { using namespace MG_Util::ShaderTranspiler; // Legacy ES-style shader: precision statements and qualifier macros are left for glslang - // (its preprocessor expands the #define; the 460 core parse ignores the qualifiers). + // (its preprocessor expands the #define; the normalized 330 core parse ignores the qualifiers). String source = R"(#define HIGHP_OR_DEFAULT highp precision HIGHP_OR_DEFAULT float; precision mediump int; diff --git a/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.cpp b/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.cpp index 43c2d5aa..8c1115d5 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.cpp +++ b/MobileGL/MG_Util/ShaderTranspiler/ShaderSourceProcessor.cpp @@ -19,6 +19,220 @@ namespace { return (ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || ch == '_'; } + bool IsIdentifierStart(char ch) { + return (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || ch == '_'; + } + + MobileGL::String MaskCommentsAndQuotedText(const MobileGL::String& source) { + enum class Region { Code, SingleLineComment, MultiLineComment, QuotedText }; + + MobileGL::String masked = source; + Region region = Region::Code; + char quote = '\0'; + bool escaped = false; + + for (SizeT pos = 0; pos < source.size(); pos++) { + const char ch = source[pos]; + const char next = pos + 1 < source.size() ? source[pos + 1] : '\0'; + + if (region == Region::Code) { + if (ch == '/' && next == '/') { + masked[pos] = ' '; + masked[pos + 1] = ' '; + pos++; + region = Region::SingleLineComment; + } else if (ch == '/' && next == '*') { + masked[pos] = ' '; + masked[pos + 1] = ' '; + pos++; + region = Region::MultiLineComment; + } else if (ch == '"' || ch == '\'') { + masked[pos] = ' '; + quote = ch; + escaped = false; + region = Region::QuotedText; + } + continue; + } + + if (region == Region::SingleLineComment) { + if (ch == '\n' || ch == '\r') { + region = Region::Code; + } else { + masked[pos] = ' '; + } + continue; + } + + if (region == Region::MultiLineComment) { + if (ch == '*' && next == '/') { + masked[pos] = ' '; + masked[pos + 1] = ' '; + pos++; + region = Region::Code; + } else if (ch != '\n' && ch != '\r') { + masked[pos] = ' '; + } + continue; + } + + if (ch != '\n' && ch != '\r') { + masked[pos] = ' '; + } + if (escaped) { + escaped = false; + } else if (ch == '\\') { + escaped = true; + } else if (ch == quote) { + region = Region::Code; + } + } + + return masked; + } + + void SkipDirectiveWhitespace(const MobileGL::String& source, SizeT& pos, SizeT lineEnd) { + while (pos < lineEnd && std::isspace(static_cast(source[pos]))) { + pos++; + } + } + + MobileGL::String ReadDirectiveIdentifier(const MobileGL::String& source, SizeT& pos, SizeT lineEnd) { + if (pos >= lineEnd || !IsIdentifierStart(source[pos])) { + return {}; + } + + const SizeT start = pos++; + while (pos < lineEnd && IsIdentifierChar(source[pos])) { + pos++; + } + return source.substr(start, pos - start); + } + + bool HasUtf8Bom(const MobileGL::String& source) { + return source.size() >= 3 && static_cast(source[0]) == 0xef && + static_cast(source[1]) == 0xbb && static_cast(source[2]) == 0xbf; + } + + struct ShaderLanguageInfo { + unsigned version = 110; + MobileGL::ShaderProfile profile = MobileGL::ShaderProfile::Core; + SizeT versionDirectiveStart = MobileGL::String::npos; + SizeT versionDirectiveEnd = MobileGL::String::npos; + bool hasUtf8Bom = false; + bool enablesGpuShader5 = false; + + bool HasVersionDirective() const { return versionDirectiveStart != MobileGL::String::npos; } + }; + + ShaderLanguageInfo InspectShaderLanguage(const MobileGL::String& source) { + const MobileGL::String code = MaskCommentsAndQuotedText(source); + ShaderLanguageInfo info; + info.hasUtf8Bom = HasUtf8Bom(source); + + SizeT lineStart = 0; + while (lineStart < code.size()) { + SizeT lineEnd = code.find('\n', lineStart); + const bool hasLineBreak = lineEnd != MobileGL::String::npos; + if (!hasLineBreak) { + lineEnd = code.size(); + } + + SizeT probe = lineStart; + if (lineStart == 0 && info.hasUtf8Bom) { + probe = 3; + } + SkipDirectiveWhitespace(code, probe, lineEnd); + if (probe < lineEnd && code[probe] == '#') { + const SizeT directiveStart = probe; + probe++; + SkipDirectiveWhitespace(code, probe, lineEnd); + const MobileGL::String directive = ReadDirectiveIdentifier(code, probe, lineEnd); + + if (directive == "version" && !info.HasVersionDirective()) { + SkipDirectiveWhitespace(code, probe, lineEnd); + unsigned version = 0; + bool hasVersionDigits = false; + while (probe < lineEnd && code[probe] >= '0' && code[probe] <= '9') { + hasVersionDigits = true; + version = version * 10 + static_cast(code[probe] - '0'); + probe++; + } + if (hasVersionDigits) { + info.version = version; + info.versionDirectiveStart = directiveStart; + info.versionDirectiveEnd = lineEnd + (hasLineBreak ? 1 : 0); + SkipDirectiveWhitespace(code, probe, lineEnd); + const MobileGL::String profile = ReadDirectiveIdentifier(code, probe, lineEnd); + if (profile == "es" || profile == "ES") { + info.profile = MobileGL::ShaderProfile::ES; + } else if (profile == "compatibility") { + info.profile = MobileGL::ShaderProfile::Compatibility; + } else { + info.profile = MobileGL::ShaderProfile::Core; + } + } + } else if (directive == "extension") { + SkipDirectiveWhitespace(code, probe, lineEnd); + const MobileGL::String extension = ReadDirectiveIdentifier(code, probe, lineEnd); + SkipDirectiveWhitespace(code, probe, lineEnd); + if (probe < lineEnd && code[probe] == ':') { + probe++; + SkipDirectiveWhitespace(code, probe, lineEnd); + const MobileGL::String behavior = ReadDirectiveIdentifier(code, probe, lineEnd); + const bool isGpuShader5 = extension == "GL_ARB_gpu_shader5" || + extension == "GL_NV_gpu_shader5"; + const bool enablesExtension = behavior == "enable" || behavior == "require" || + behavior == "warn"; + // Gate the whole source if it ever opts into either extension. This is deliberately + // conservative around conditional directives and keeps legal sample qualifiers intact. + info.enablesGpuShader5 = info.enablesGpuShader5 || (isGpuShader5 && enablesExtension); + } + } + } + + lineStart = lineEnd + (hasLineBreak ? 1 : 0); + } + + return info; + } + + MobileGL::String GetNormalizedVersionDirective(const ShaderLanguageInfo& info) { + if (info.profile == MobileGL::ShaderProfile::ES) { + // Preserve the pre-existing behavior for standard lowercase "es" directives. MobileGL's Vulkan + // glslang resource table cannot parse its ESSL built-ins today, even at ESSL 310, whereas the same + // source is accepted through the normalized desktop core path. + return "#version 460 core\n"; + } + + // Keep compatibility-profile handling on its pre-existing 460 path. Vulkan glslang does not accept that + // profile today, and this legacy-sample fix must not broaden or otherwise alter that separate limitation. + if (info.profile == MobileGL::ShaderProfile::Compatibility) { + return "#version 460 compatibility\n"; + } + + const bool useLegacyDesktopVersion = + info.version < 400 && !info.enablesGpuShader5; + return useLegacyDesktopVersion ? "#version 330 core\n" : "#version 460 core\n"; + } + + void NormalizeVersionDirective(MobileGL::String& source, const ShaderLanguageInfo& info) { + const MobileGL::String replacement = GetNormalizedVersionDirective(info); + if (info.HasVersionDirective()) { + source.replace(info.versionDirectiveStart, info.versionDirectiveEnd - info.versionDirectiveStart, + replacement); + if (info.hasUtf8Bom) { + source.erase(0, 3); + } + return; + } + + if (info.hasUtf8Bom) { + source.erase(0, 3); + } + source.insert(0, replacement); + } + bool HasSingleLineFunctionDefinition(const MobileGL::String& source, const MobileGL::String& functionName) { SizeT lineStart = 0; while (lineStart < source.size()) { @@ -153,12 +367,8 @@ namespace { } SizeT FindAfterVersionDirective(const MobileGL::String& source) { - const SizeT versionPos = source.find("#version"); - if (versionPos == MobileGL::String::npos) { - return 0; - } - const SizeT lineEnd = source.find('\n', versionPos); - return lineEnd == MobileGL::String::npos ? source.size() : lineEnd + 1; + const ShaderLanguageInfo info = InspectShaderLanguage(source); + return info.HasVersionDirective() ? info.versionDirectiveEnd : 0; } bool IsExtensionAdvertised(MobileGL::GLExtension extension) { @@ -263,7 +473,7 @@ namespace { void ModernizeLegacyGLSL(MobileGL::ShaderStage stage, MobileGL::String& source) { // Precision qualifiers (highp/mediump/lowp and default-precision statements) are legal and - // ignored in the forced "#version 460 core" profile, so glslang handles them natively. + // ignored in the normalized desktop core profiles, so glslang handles them natively. ReplaceIdentifier(source, "texture2D", "texture"); ReplaceIdentifier(source, "texture2DProj", "textureProj"); @@ -308,6 +518,11 @@ namespace MobileGL { namespace MG_Util { namespace ShaderTranspiler { void PreprocessShaderSource(ShaderStage stage, String& source) { + // Normalize while the inspector's source span still refers to the untouched input. Later passes + // remove comments and directives, so any subsequent insertion re-inspects the current source. + const ShaderLanguageInfo originalLanguage = InspectShaderLanguage(source); + NormalizeVersionDirective(source, originalLanguage); + // remove multi-line comment size_t commentStartPos = source.find("/*"); while (commentStartPos != String::npos) { @@ -345,43 +560,6 @@ namespace MobileGL { noperspectivePos = source.find(str_np); } - // force #version - ShaderProfile profile = ShaderProfile::Core; - SizeT versionPos = source.find("#version"); - SizeT lineEnd = source.find('\n', versionPos); - - if (versionPos != String::npos) { - String versionLine = source.substr(versionPos, lineEnd - versionPos); - - if (versionLine.find("ES") != String::npos) - profile = ShaderProfile::ES; - else if (versionLine.find("compatibility") != String::npos) - profile = ShaderProfile::Compatibility; - else - profile = ShaderProfile::Core; - } else { - profile = ShaderProfile::Core; - source.insert(0, "#version 460 core\n"); - versionPos = 0; - lineEnd = source.find('\n', versionPos); - } - - SizeT firstLineEnd = lineEnd; - - if (profile != ShaderProfile::ES) { - constexpr const char* versionDirectiveCore = "#version 460 core\n"; - constexpr const char* versionDirectiveCompat = "#version 460 compatibility\n"; - - const char* replacement = - (profile == ShaderProfile::Compatibility) ? versionDirectiveCompat : versionDirectiveCore; - - if (firstLineEnd != String::npos) { - source.replace(versionPos, firstLineEnd - versionPos + 1, replacement); - } else { - source = replacement; - } - } - FilterUnsupportedGpuShaderInt64(source); // Some shader packs define helpers with built-in GLSL names such as round(), tanh(), or fma().