swung0x48/MobileGL
1
0
Fork 0
mirror of https://github.com/MobileGL-Dev/MobileGL synced 2026-09-12 06:08:30 +09:00
Code Issues Packages Projects Releases Wiki Activity
Files
5175f9ccd5a9311296b00dd9eb87c666d8bcac10
MobileGL/scripts/gen_pipe.py
T

1399 lines
67 KiB
Python
Raw Blame History

#!/usr/bin/env python3
# MobileGL - scripts/gen_pipe.py
# Copyright (c) 2025-2026 MobileGL-Dev
# Licensed under the GNU Lesser General Public License v3.0:
# https://www.gnu.org/licenses/gpl-3.0.txt
# https://www.gnu.org/licenses/lgpl-3.0.txt
# SPDX-License-Identifier: LGPL-3.0-only
# End of Source File Header
"""The seven MGPipe generators, G1..G7 (plan B section 4.1).
Reads the four hand-maintained sources of truth
MobileGL/MG_Pipe/PipeCalls.def the call catalogue
MobileGL/MG_Pipe/PipeFields.def per-payload field lists for the verify comparator
MobileGL/MG_Pipe/Coverage.def accessor -> call mapping for the read inventory,
and the sticky-field list
MobileGL/MG_Pipe/FillPoints.def verb -> class and class -> may-read field tables
(G5b), checked against MG_Backend::GLFunctionsTable
plus the vendored copy of the backend read inventory
scripts/data/backend_read_inventory.md
and writes MobileGL/MG_Pipe/generated/*.inc. The outputs are COMMITTED; CI regenerates
them and fails on a diff, which is what keeps the seven generators from drifting apart
from the catalogue (they all consume the same .def).
python3 scripts/gen_pipe.py # write the generated files, print the summary
python3 scripts/gen_pipe.py --check # fail if regenerating would change anything
python3 scripts/gen_pipe.py --self-test # the negative controls: each gate below must trip
Both modes refuse a catalogue whose call payload has no field list in PipeFields.def: a
payload the G4 comparator cannot see is a payload MOBILEGL_PIPE_VERIFY is blind to. Both
also refuse a field list that does not name every data member of its struct (or names one
that is not a member), and a backend accessor read through MGB_CTX-> / pGLContext-> that
has no Coverage.def row (P1 brief D8, D12).
"""
import argparse
import os
import re
import sys
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
PIPE_DIR = os.path.join(REPO_ROOT, "MobileGL", "MG_Pipe")
GENERATED_DIR = os.path.join(PIPE_DIR, "generated")
INVENTORY = os.path.join(REPO_ROOT, "scripts", "data", "backend_read_inventory.md")
FUNCTION_TABLE_HEADER = os.path.join(REPO_ROOT, "MobileGL", "MG_Backend", "BackendObject.h")
GENERATED_BANNER = """// MobileGL - MobileGL/MG_Pipe/generated/{name}
// Copyright (c) 2025-2026 MobileGL-Dev
// Licensed under the GNU Lesser General Public License v3.0:
// https://www.gnu.org/licenses/gpl-3.0.txt
// https://www.gnu.org/licenses/lgpl-3.0.txt
// SPDX-License-Identifier: LGPL-3.0-only
// End of Source File Header
// {title}
//
// GENERATED by scripts/gen_pipe.py from {sources} - DO NOT EDIT.
// Regenerate with `python3 scripts/gen_pipe.py`; CI runs it and diffs the result.
// This file is included from MG_Pipe/MGPipe.h inside namespace MobileGL::MG_Pipe.
"""
# G7. The pipeline subset of RenderStateParameters, BY MEMBER NAME, in DECLARATION order.
#
# P0 took this list from what VulkanRenderer::ComputePipelineStateHash hashed. P2 replaced
# that provenance with a RULE, and the rule is the only thing that decides membership:
#
# A member is in the pipeline subset if and only if some public RenderState setter that
# calls BumpVersions() writes it. Everything else is dynamic. There is no third set.
#
# That makes G7's invariant - the pipeline-subset hash moves IFF m_pipelineStateVersion
# moves - true by construction, and it makes the subset a strict SUPERSET of the 24 members
# the Vulkan hash read: it adds sample coverage, the front face, the provoking vertex, the
# scissor-test mask, the back polygon mode, the eleven capability bools the hash never read,
# and the three capabilities P2 gave storage to. The alternative - demoting those setters to
# ++m_version - would change MG_State semantics in the PULL build for the push path's sake.
#
# Names only: offsets are NOT computed here. The chunk table with real offsets is
# MGPipeRenderStateSpans.cpp, built in C++ with offsetof, because a python guess at the
# layout of a struct it cannot see is exactly the kind of drift the G7 setter-consistency
# test exists to catch (plan B section 4.5.2). StencilStates is named once and straddles:
# per face, Func and the three ops are pipeline, Ref/ValueMask/WriteMask are dynamic.
PIPELINE_STATE_MEMBERS = [
"PatchVertices",
"PatchDefaultOuterLevel",
"PatchDefaultInnerLevel",
"BlendStates",
"LogicOp",
"DepthTestEnabled",
"DepthFunc",
"DepthMask",
"ColorMasks",
"FramebufferSrgbEnabled",
"DepthClampEnabled",
"TextureCubeMapSeamlessEnabled",
"SampleCoverageValue",
"SampleCoverageInvert",
"SampleMaskValue",
"MinSampleShadingValue",
"StencilStates",
"CullFaceEnabled",
"CullFaceModeSetting",
"FrontFaceModeSetting",
"ProvokingVertexModeSetting",
"PolygonModeFront",
"PolygonModeBack",
"ColorLogicOpEnabled",
"DebugOutputEnabled",
"DebugOutputSynchronousEnabled",
"DitherEnabled",
"LineSmoothEnabled",
"MultisampleEnabled",
"PolygonOffsetFillEnabled",
"PolygonOffsetLineEnabled",
"PolygonOffsetPointEnabled",
"PolygonSmoothEnabled",
"PrimitiveRestartEnabled",
"PrimitiveRestartFixedIndexEnabled",
"RasterizerDiscardEnabled",
"SampleAlphaToCoverageEnabled",
"SampleAlphaToOneEnabled",
"SampleCoverageEnabled",
"SampleMaskEnabled",
"SampleShadingEnabled",
"StencilTestEnabled",
"ProgramPointSizeEnabled",
"ScissorTestEnabledMask",
]
def read(path):
with open(path, "r", encoding="utf-8") as handle:
return handle.read()
class Call(object):
def __init__(self, index, name, payload, cls, flags):
self.Index = index # 1-based; this is the wire opcode
self.Name = name
self.Payload = payload
self.Class = cls
self.Flags = flags
@property
def IsScreen(self):
return self.Class == "kScreen"
@property
def Signature(self):
"""(parameter declaration list, argument list) for this call."""
params = ["const %s* payload" % self.Payload]
args = ["payload"]
if "kVarTail" in self.Flags:
params.append("const void* varTail")
params.append("Uint32 varTailCount")
args.append("varTail")
args.append("varTailCount")
if "kReplySlot" in self.Flags:
params.append("MGPReplySlot* reply")
args.append("reply")
return ", ".join(params), ", ".join(args)
CALL_RE = re.compile(r"^\s*X\(\s*(\w+)\s*,\s*(\w+)\s*,\s*(\w+)\s*,\s*([\w|]+?)\s*\)\s*\\?\s*$")
def parse_calls():
text = read(os.path.join(PIPE_DIR, "PipeCalls.def"))
documented = re.search(r"#define MGP_CALL_LIST_DOCUMENTED_COUNT (\d+)", text)
if not documented:
sys.exit("PipeCalls.def: MGP_CALL_LIST_DOCUMENTED_COUNT is missing")
calls = []
inside = False
for line in text.splitlines():
if line.startswith("#define MGP_CALL_LIST(X)"):
inside = True
continue
if not inside:
continue
match = CALL_RE.match(line)
if match:
calls.append(Call(len(calls) + 1, match.group(1), match.group(2), match.group(3),
match.group(4).split("|")))
# The macro ends at the first line without a continuation backslash.
if not line.rstrip().endswith("\\"):
inside = False
count = int(documented.group(1))
if len(calls) != count:
sys.exit("PipeCalls.def: parsed %d calls but MGP_CALL_LIST_DOCUMENTED_COUNT says %d"
% (len(calls), count))
seen = set()
for call in calls:
if call.Name in seen:
sys.exit("PipeCalls.def: duplicate call %s" % call.Name)
seen.add(call.Name)
return calls
# Types the G4 comparator may fall back to memcmp for: NONE since P1. The value structs and
# MGHostSpan have field lists of their own, and the fallback branch of MGPipeFieldEqual is a
# static_assert, so a future struct without a field list is a compile error rather than a
# padding false positive. Kept as a (deliberately empty) set so the check below keeps its
# shape.
MEMCMP_FALLBACK_TYPES = set()
# Where the structs named in PipeFields.def are declared: the payload header, the value
# header, the host-span header and - for DynamicBackendParameters, the caps block - the
# backend object header.
FIELD_LIST_STRUCT_HEADERS = [
os.path.join(PIPE_DIR, "MGPipeTypes.h"),
os.path.join(PIPE_DIR, "MGPipeValueTypes.h"),
os.path.join(PIPE_DIR, "MGPipeHostSpan.h"),
FUNCTION_TABLE_HEADER,
]
BACKEND_DIR = os.path.join(REPO_ROOT, "MobileGL", "MG_Backend")
def parse_verify_payloads():
text = read(os.path.join(PIPE_DIR, "PipeFields.def"))
match = re.search(r"#define MGP_VERIFY_PAYLOAD_LIST\(P\)(.*?)\n\n", text, re.S)
if not match:
sys.exit("PipeFields.def: MGP_VERIFY_PAYLOAD_LIST is missing")
payloads = re.findall(r"P\((\w+)\)", match.group(1))
for payload in payloads:
if ("#define MGP_FIELDS_%s(F)" % payload) not in text:
sys.exit("PipeFields.def: %s is in the payload list with no field macro" % payload)
return payloads
FIELD_LIST_RE = re.compile(r"#define MGP_FIELDS_(\w+)\(F\)")
def parse_field_lists(text=None):
"""PipeFields.def -> {payload: [field, ...]} for every MGP_FIELDS_<Payload>(F) macro; the
macro ends at the first line without a continuation backslash."""
if text is None:
text = read(os.path.join(PIPE_DIR, "PipeFields.def"))
lists = {}
lines = text.splitlines()
i = 0
while i < len(lines):
match = FIELD_LIST_RE.match(lines[i])
if not match:
i += 1
continue
name = match.group(1)
body = []
while i < len(lines):
body.append(lines[i])
if not lines[i].rstrip().endswith("\\"):
break
i += 1
i += 1
fields = re.findall(r"\bF\((\w+)\)", "\n".join(body))
if name in lists:
sys.exit("PipeFields.def: MGP_FIELDS_%s is defined twice" % name)
lists[name] = fields
return lists
def mask_comments_and_strings(text):
"""Replace comment and string-literal bodies with spaces, keeping every offset and
newline, so the regexes below cannot match inside a comment or a literal
(gen_pipe_dirty_surface.py's shape)."""
out = list(text)
i = 0
n = len(text)
while i < n:
c = text[i]
if c == "/" and i + 1 < n and text[i + 1] == "/":
while i < n and text[i] != "\n":
out[i] = " "
i += 1
elif c == "/" and i + 1 < n and text[i + 1] == "*":
out[i] = out[i + 1] = " "
i += 2
while i < n and not (text[i] == "*" and i + 1 < n and text[i + 1] == "/"):
if text[i] != "\n":
out[i] = " "
i += 1
if i < n:
out[i] = " "
if i + 1 < n:
out[i + 1] = " "
i += 2
elif c in "\"'":
quote = c
i += 1
while i < n and text[i] != quote:
if text[i] == "\\":
out[i] = " "
i += 1
if i < n and text[i] != "\n":
out[i] = " "
i += 1
if i < n:
out[i] = " "
i += 1
else:
i += 1
return "".join(out)
PADDING_MEMBER_RE = re.compile(r"^Pad\d*$")
NESTED_TYPE_RE = re.compile(r"^\s*(?:struct|class|union|enum)\b")
FUNCTION_HEAD_RE = re.compile(r"\)\s*(?:const\s*)?(?:noexcept\s*)?(?:override\s*)?(?:=\s*(?:default|delete|0)\s*)?$")
NON_MEMBER_RE = re.compile(r"^\s*(?:static|using|typedef|friend|template|explicit|virtual|operator)\b")
def find_struct_body(masked, name):
"""The text between the braces of `struct <name> {` (an optional base list allowed), or
None. Comments and strings must already be masked."""
match = re.search(r"\bstruct\s+%s\s*(?::[^{;]*)?\{" % re.escape(name), masked)
if not match:
return None
i = match.end()
depth = 1
start = i
while i < len(masked) and depth:
if masked[i] == "{":
depth += 1
elif masked[i] == "}":
depth -= 1
i += 1
return masked[start:i - 1]
def matching_brace(text, open_index):
depth = 0
j = open_index
while j < len(text):
if text[j] == "{":
depth += 1
elif text[j] == "}":
depth -= 1
if depth == 0:
return j
j += 1
return len(text) - 1
def strip_balanced(text, open_char, close_char):
out = []
depth = 0
for c in text:
if c == open_char:
depth += 1
elif c == close_char:
depth -= 1
elif depth == 0:
out.append(c)
return "".join(out)
def member_names(statement):
"""The data-member names declared by one struct-body statement, or [] for anything
that is not a data member (a function, a static, a using, an access label...)."""
statement = re.sub(r"^\s*(?:public|private|protected)\s*:", "", statement).strip()
if not statement or NON_MEMBER_RE.match(statement):
return []
# The declarator part is what precedes the default initializer.
left = re.split(r"=|\{\.\.\.\}", statement, maxsplit=1)[0]
if "(" in left:
return [] # a function declaration
left = strip_balanced(left, "<", ">")
left = strip_balanced(left, "[", "]")
names = []
for k, chunk in enumerate(left.split(",")):
tokens = re.findall(r"[A-Za-z_]\w*", chunk)
if k == 0 and len(tokens) < 2:
return [] # no type: not a declaration
if not tokens:
return []
names.append(tokens[-1])
return [n for n in names if not PADDING_MEMBER_RE.match(n)]
def struct_data_members(body):
"""Direct data members of a struct body, in declaration order: statics, member
functions, nested types and Pad-named members excluded."""
members = []
statement = []
i = 0
n = len(body)
while i < n:
c = body[i]
if c == "{":
head = "".join(statement)
close = matching_brace(body, i)
if NESTED_TYPE_RE.match(head.strip()) or FUNCTION_HEAD_RE.search(head.rstrip()):
# A nested type or a member-function body: not a data member. Swallow the
# nested type's trailing semicolon too.
statement = []
i = close + 1
if NESTED_TYPE_RE.match(head.strip()):
while i < n and body[i] in " \t\n":
i += 1
if i < n and body[i] == ";":
i += 1
continue
statement.append("{...}") # a brace default initializer
i = close + 1
continue
if c == ";":
members.extend(member_names("".join(statement)))
statement = []
i += 1
continue
statement.append(c)
i += 1
return members
def check_field_lists_cover_struct_members(field_lists, payloads, header_texts=None):
"""Every payload in MGP_VERIFY_PAYLOAD_LIST: its MGP_FIELDS_ list must name every direct
data member of `struct <Payload> {` (Pad-named members are padding and excluded) and
nothing that is not a member. A member without an F(...) is a field MOBILEGL_PIPE_VERIFY
is blind to; an F(...) that is not a member is a list that stopped describing its
struct. Runs in both modes, --check included, so it is part of pipe-gates."""
if header_texts is None:
header_texts = [read(path) for path in FIELD_LIST_STRUCT_HEADERS]
masked = [mask_comments_and_strings(t) for t in header_texts]
problems = []
for payload in payloads:
body = None
for m in masked:
body = find_struct_body(m, payload)
if body is not None:
break
if body is None:
problems.append("%s: struct not found in %s" % (payload, ", ".join(os.path.basename(p) for p in FIELD_LIST_STRUCT_HEADERS)))
continue
members = struct_data_members(body)
listed = field_lists.get(payload, [])
missing = [m for m in members if m not in listed]
extra = [f for f in listed if f not in members]
if missing:
problems.append("%s: member(s) with no F(...) in PipeFields.def: %s" % (payload, ", ".join(missing)))
if extra:
problems.append("%s: F(...) name(s) that are not members: %s" % (payload, ", ".join(extra)))
if not members:
problems.append("%s: no data members parsed" % payload)
if problems:
sys.exit("PipeFields.def does not cover its structs:\n " + "\n ".join(problems))
ACCESSOR_READ_RE = re.compile(r"\b(?:MGB_CTX|pGLContext)\s*->\s*(\w+)")
def scan_live_accessors(accessors, backend_dir=None, verbose=True):
"""Every accessor a backend reads through MGB_CTX-> or pGLContext-> (comments and
strings masked) must have a Coverage.def row - a read without a row is a PipeInputs
field that does not exist. Rows no backend reads are printed, not refused (the dead
GetBoundTransformFeedbackName row is deliberate). Returns the set of names read."""
if backend_dir is None:
backend_dir = BACKEND_DIR
known = set(name for name, _ in accessors)
read_names = {}
for root, _, files in os.walk(backend_dir):
for name in sorted(files):
if not name.endswith((".cpp", ".h")):
continue
path = os.path.join(root, name)
masked = mask_comments_and_strings(read(path))
for match in ACCESSOR_READ_RE.finditer(masked):
read_names.setdefault(match.group(1), set()).add(os.path.relpath(path, REPO_ROOT))
unknown = sorted(n for n in read_names if n not in known)
if unknown:
sys.exit("Coverage.def: accessor(s) read by a backend with no row: %s"
% ", ".join("%s (%s)" % (n, ", ".join(sorted(read_names[n]))) for n in unknown))
unread = sorted(known - set(read_names))
if verbose and unread:
print("gen_pipe: %d accessor row(s) no backend reads: %s" % (len(unread), ", ".join(unread)))
return set(read_names)
def check_call_payloads_have_field_lists(calls, payloads):
"""Every payload PipeCalls.def names must have a G4 field list, or the verify comparator
is silently blind to that call. Runs in both modes, --check included."""
known = set(payloads)
missing = sorted({c.Payload for c in calls
if c.Payload not in known and c.Payload not in MEMCMP_FALLBACK_TYPES})
if missing:
sys.exit("PipeFields.def: call payload(s) with no field list, so MOBILEGL_PIPE_VERIFY "
"would be blind to them: %s" % ", ".join(missing))
# The MGPipeCallFlags enumerators, MGPipe.h:39-54. Kept here rather than parsed out of the
# header because this list is what the generated kMGPipeCallFlags[] table spells into C++:
# a flag token in PipeCalls.def that is not one of these would generate an expression that
# does not compile, and a build break several minutes later is a worse diagnosis than this
# one line. kNone is listed but is NOT a flag - it is the empty set, and it may not be
# combined with anything.
KNOWN_CALL_FLAGS = ("kNeedsAck", "kHasBlob", "kVarTail", "kHostSpan", "kReplySlot", "kOptional")
def check_call_flags_are_known(calls):
"""Every flag token in PipeCalls.def must be an MGPipeCallFlags enumerator, and kNone may
not be combined with one. Runs in both modes, --check included: P5 R-13.4 exports these
flags as a table six packages read, so a typo here is a wrong decode rather than a
compile error in the one consumer that used to hard-code its own copy."""
for call in calls:
for flag in call.Flags:
if flag != "kNone" and flag not in KNOWN_CALL_FLAGS:
sys.exit("PipeCalls.def: %s carries flag %s, which is not an MGPipeCallFlags "
"enumerator (%s)" % (call.Name, flag, ", ".join(KNOWN_CALL_FLAGS)))
if "kNone" in call.Flags and len(call.Flags) != 1:
sys.exit("PipeCalls.def: %s combines kNone with %s; kNone is the empty set"
% (call.Name, "|".join(f for f in call.Flags if f != "kNone")))
def parse_coverage():
text = read(os.path.join(PIPE_DIR, "Coverage.def"))
accessors = []
block = re.search(r"#define MGP_COVERAGE_ACCESSOR_LIST\(X\)(.*?)\n\n", text, re.S)
if not block:
sys.exit("Coverage.def: MGP_COVERAGE_ACCESSOR_LIST is missing")
for name, call in re.findall(r"X\((\w+)\s*,\s*(\w+)\)", block.group(1)):
accessors.append((name, call))
deltas = []
block = re.search(r"#define MGP_COVERAGE_DELTA_LIST\(X\)(.*?)\n\n", text, re.S)
if not block:
sys.exit("Coverage.def: MGP_COVERAGE_DELTA_LIST is missing")
for kind, call in re.findall(r"X\(([^,]+),\s*(\w+)\)", block.group(1)):
deltas.append((kind.strip(), call))
sticky = []
block = re.search(r"#define MGP_COVERAGE_STICKY_LIST\(X\)(.*?)\n\n", text, re.S)
if not block:
sys.exit("Coverage.def: MGP_COVERAGE_STICKY_LIST is missing")
accessor_names = set(name for name, _ in accessors)
for name, reason in re.findall(r"X\((\w+)\s*,\s*\"([^\"]*)\"\)", block.group(1)):
if name not in accessor_names:
sys.exit("Coverage.def: sticky field %s is not an accessor in MGP_COVERAGE_ACCESSOR_LIST" % name)
if name in dict(sticky):
sys.exit("Coverage.def: sticky field %s is listed twice" % name)
sticky.append((name, reason))
emitted = []
block = re.search(r"#define MGP_COVERAGE_EMITTED_LIST\(X\)(.*?)\n\n", text, re.S)
if not block:
sys.exit("Coverage.def: MGP_COVERAGE_EMITTED_LIST is missing")
seen = set()
for name, call in re.findall(r"X\((\w+)\s*,\s*(\w+)\)", block.group(1)):
if name not in accessor_names:
sys.exit("Coverage.def: emitted field %s is not an accessor in "
"MGP_COVERAGE_ACCESSOR_LIST" % name)
if name in seen:
sys.exit("Coverage.def: emitted field %s is listed twice" % name)
seen.add(name)
emitted.append((name, call))
return accessors, deltas, sticky, emitted
INVENTORY_ROW_RE = re.compile(r"^\|\s*(\d+)\s*\|([^|]*)\|([^|]*)\|([^|]*)\|")
def parse_inventory():
if not os.path.exists(INVENTORY):
sys.exit("missing %s - copy it from MobileGL-CS/docs/CS_Refactor/" % INVENTORY)
rows = []
current_file = None
for line in read(INVENTORY).splitlines():
heading = re.match(r"^### `([^`]+)`", line)
if heading:
current_file = heading.group(1)
continue
match = INVENTORY_ROW_RE.match(line)
if match and current_file:
rows.append({
"file": current_file,
"line": int(match.group(1)),
"kind": match.group(2).strip(),
"member": match.group(3).strip(),
"delta": match.group(4).strip(),
})
return rows
def banner(name, title, sources):
return GENERATED_BANNER.format(name=name, title=title, sources=sources)
def gen_tables(calls):
screen = [c for c in calls if c.IsScreen]
context = [c for c in calls if not c.IsScreen]
out = [banner("PipeTables.inc", "G1: the two MGPipe interface tables.", "PipeCalls.def")]
for struct_name, group, what in (("MGPipeScreen", screen, "share group"),
("MGPipeContext", context, "context")):
out.append("// %s: %d calls. A null entry means the backend does not implement this\n"
"// call and the frontend keeps its own path (plan B section 4.1).\n"
"struct %s {" % (what, len(group), struct_name))
for call in group:
params, _ = call.Signature
out.append(" void (*%s)(%s);" % (call.Name, params))
out.append("};\n")
out.append("inline constexpr SizeT kMGPipeScreenCallCount = %d;" % len(screen))
out.append("inline constexpr SizeT kMGPipeContextCallCount = %d;" % len(context))
out.append("inline constexpr SizeT kMGPipeCallCount = %d;" % len(calls))
out.append("")
out.append("// A table that is not exactly its call count of function pointers has grown a")
out.append("// member that no generator knows about.")
out.append("static_assert(sizeof(MGPipeScreen) == kMGPipeScreenCallCount * sizeof(void (*)()),")
out.append(" \"MGPipeScreen is not exactly its catalogue's function pointers\");")
out.append("static_assert(sizeof(MGPipeContext) == kMGPipeContextCallCount * sizeof(void (*)()),")
out.append(" \"MGPipeContext is not exactly its catalogue's function pointers\");")
out.append("static_assert(kMGPipeScreenCallCount + kMGPipeContextCallCount == kMGPipeCallCount);")
out.append("static_assert(kMGPipeCallCount == MGP_CALL_LIST_DOCUMENTED_COUNT,")
out.append(" \"the catalogue and its documented count disagree\");")
return "\n".join(out) + "\n"
def gen_thunks(calls):
out = [banner("PipeThunks.inc", "G2: monolith thunks over the two tables.", "PipeCalls.def")]
out.append("// One inline call through the installed table. These are the names MG_Impl call")
out.append("// sites move onto, replacing gBackendFunctionsTable.GL.* one at a time. An")
out.append("// unimplemented (null) entry is the caller's business to check, exactly as it is")
out.append("// with the table this replaces.\n")
for call in calls:
params, args = call.Signature
table = "gMGPipeScreen" if call.IsScreen else "gMGPipeContext"
out.append("inline void MGP_%s(%s) {" % (call.Name, params))
out.append(" %s.%s(%s);" % (table, call.Name, args))
out.append("}")
out.append("")
return "\n".join(out)
def gen_wire(calls, residual_fields=None):
out = [banner("PipeWire.inc", "G3: wire records, size assertions and the applier's bounds gate.",
"PipeCalls.def")]
out.append("""// Every record is a fixed header plus its payload, padded to the stream's 8-byte
// granularity. The size assertion is stated as a COMPOSITION so it fires on any padding
// the compiler inserts between the header and the payload while staying honest about the
// tail padding the alignment requires.
//
// The applier's precondition is checked BEFORE dispatch, on every record, in every build:
// a record that is shorter than its own type, longer than what is left in the buffer, or
// not a multiple of 8 is protocol corruption and is fatal. There is no recovery path -
// silently applying a truncated record is how a corrupt stream becomes a wrong picture.
//
// OVERSIZED PAYLOADS ARE CHUNKED, NEVER EMITTED WHOLE (plan section 8.2: G3 has to define
// the path for a record larger than the segment). The bound is the ring's,
// RingProducer::MaxRecordBytes() == Capacity()/2, and it is exact rather than
// conservative: a record has to be placeable at every head offset of an empty ring, the
// wrap pad in front of it costs up to total-8 bytes, and only a record of at most half the
// ring survives that at every offset. An emitter holding more than Capacity()/2 bytes of
// record (a large resource_subdata, a create_shader_state archive) splits it into several
// records of at most that size; the transport refuses a bigger one outright - nullptr plus
// an MGLOG_E - rather than let the producer wait on free bytes that can never suffice.
struct MGPWireRecHeader {
Uint16 Op; // MGPWireOp
Uint16 Flags; // MGPipeCallFlags of the call, for asserts and tracing
Uint32 Size; // bytes of this record including the header and the variable tail
};
static_assert(sizeof(MGPWireRecHeader) == 8, "the wire header is 8 bytes");
static_assert(std::is_trivially_copyable_v<MGPWireRecHeader>);
// The opcode is the call's position in PipeCalls.def. Reordering that file is a protocol
// break; appending to it is not.
enum class MGPWireOp : Uint16 {
kInvalid = 0,""")
for call in calls:
out.append(" %s = %d," % (call.Name, call.Index))
out.append(" kOpCount = %d," % (len(calls) + 1))
out.append("};\n")
out.append("""// THE FLAGS, EXPORTED ONCE, INDEXED BY OPCODE (P5 R-13.4). MGPWireRecHeader::Flags is
// documented as "MGPipeCallFlags of the call", and until this table existed nothing
// generated said what those were: every consumer that needed to know whether a record owns
// an MGPBlobRef, a variable tail or a reply slot had to hard-code its own copy of
// PipeCalls.def's fourth column, and six of them were about to. A hard-coded copy is how
// GetCaps and CreateSamplerState came to carry an MGPBlobRef member with no kHasBlob on
// their line at all - nothing compared the two, because nothing had both in one place.
//
// Index 0 is MGPWireOp::kInvalid and is kNone: the catalogue is 1-based, and an encoder
// that reads flags for an opcode it never got from the catalogue must see the empty set
// rather than another call's flags.
//
// kHasBlob here means EXACTLY "the payload owns an MGPBlobRef member". Three calls carry
// bytes without one - resource_respecify, resource_flush_range and map_persistent, whose
// companion pointers have no carrier - and they are deliberately NOT flagged; MG_Remote's
// CONTRACT-P5.md table 1 is where those live, because a decoder that trusts kHasBlob has
// to find a member to read.""")
out.append("inline constexpr Uint32 kMGPipeCallFlags[static_cast<SizeT>(MGPWireOp::kOpCount)] = {")
out.append(" /* 0 %-24s*/ static_cast<Uint32>(kNone)," % "kInvalid")
for call in calls:
out.append(" /* %2d %-24s*/ static_cast<Uint32>(%s),"
% (call.Index, call.Name, " | ".join(call.Flags)))
out.append("};")
out.append("static_assert(sizeof(kMGPipeCallFlags) / sizeof(kMGPipeCallFlags[0]) ==")
out.append(" static_cast<SizeT>(MGPWireOp::kOpCount),")
out.append(" \"the flags table and the opcode space disagree\");")
out.append("""
// The only supported read of the table. Out-of-range is kNone rather than undefined
// behaviour, because the one caller that can pass a bad opcode is a decoder holding bytes
// off a stream, and it must reach its own Fatal{ProtocolCorruption} rather than read past
// the array on the way there.
inline constexpr Uint32 MGPipeCallFlagsFor(MGPWireOp op) {
const SizeT index = static_cast<SizeT>(op);
return index < static_cast<SizeT>(MGPWireOp::kOpCount) ? kMGPipeCallFlags[index]
: static_cast<Uint32>(kNone);
}
// Spot checks the generator states about its own output, so that a catalogue edit that
// silently drops a flag is a build break here and not a wrong decode six packages away.
static_assert(MGPipeCallFlagsFor(MGPWireOp::kInvalid) == static_cast<Uint32>(kNone),
"opcode 0 is not a call and carries no flags");
static_assert((MGPipeCallFlagsFor(MGPWireOp::GetCaps) & static_cast<Uint32>(kHasBlob)) != 0,
"MGPCaps owns two MGPBlobRef members; R-13.1 gave the call its flag");
static_assert((MGPipeCallFlagsFor(MGPWireOp::CreateSamplerState) & static_cast<Uint32>(kHasBlob)) != 0,
"MGPSamplerDesc owns an MGPBlobRef member; R-13.1 gave the call its flag");
static_assert((MGPipeCallFlagsFor(MGPWireOp::ResourceFlushRange) & static_cast<Uint32>(kHasBlob)) == 0,
"R-13.2: resource_flush_range carries no bytes on the wire and owns no blobref");
static_assert((MGPipeCallFlagsFor(MGPWireOp::ResourceRespecify) & static_cast<Uint32>(kHasBlob)) == 0,
"R-13.3: initial bytes follow as resource_subdata; MGPResourceDesc owns no blobref");
static_assert((MGPipeCallFlagsFor(MGPWireOp::DrawVbo) &
static_cast<Uint32>(kHostSpan | kVarTail)) == static_cast<Uint32>(kHostSpan | kVarTail),
"draw_vbo is the conditional-tail plus host-span shape the codec is measured on");
""")
for call in calls:
out.append("struct alignas(8) MGPWireRec_%s {" % call.Name)
out.append(" MGPWireRecHeader Header;")
out.append(" %s Payload;" % call.Payload)
out.append("};")
out.append("static_assert(sizeof(MGPWireRec_%s) ==" % call.Name)
out.append(" ((sizeof(MGPWireRecHeader) + sizeof(%s) + 7u) & ~SizeT(7u))," % call.Payload)
out.append(" \"MGPWireRec_%s gained padding; the wire format moved\");" % call.Name)
out.append("")
out.append("""[[noreturn]] inline void MGPipeWireProtocolFatal(const char* call, Uint64 size, Uint64 remaining) {
MGLOG_F("MGPipe: protocol corruption applying %s: size=%llu remaining=%llu", call,
static_cast<unsigned long long>(size), static_cast<unsigned long long>(remaining));
std::abort();
}
#define MGP_WIRE_CHECK_BOUNDS(RecType, CallName) \\
do { \\
if (!(size >= sizeof(RecType) && size <= remaining && (size % 8) == 0)) { \\
MGPipeWireProtocolFatal(CallName, size, remaining); \\
} \\
} while (0)
// Returns whether the record was applied. P0 is a SKELETON: every case validates its
// bounds and then reports "not applied", because no applier exists until P5 wires
// MG_Remote/Server/PipeApplier.cpp to the real backend tables. The switch and the opcode
// enum come from the same list, so a call added to the catalogue cannot be forgotten here;
// the default arm is for the opcode that never came from this catalogue at all - a byte
// off a corrupt stream - and it is fatal for the same reason the bounds check is.
inline Bool MGPipeApplyWireRecord(MGPWireOp op, const void* record, Uint64 size, Uint64 remaining) {
(void)record;
switch (op) {""")
for call in calls:
out.append(" case MGPWireOp::%s:" % call.Name)
out.append(" MGP_WIRE_CHECK_BOUNDS(MGPWireRec_%s, \"%s\");" % (call.Name, call.Name))
out.append(" return false;")
out.append(""" case MGPWireOp::kInvalid:
case MGPWireOp::kOpCount:
default:
MGPipeWireProtocolFatal("<unknown opcode>", size, remaining);
}
}
#undef MGP_WIRE_CHECK_BOUNDS""")
# The migration carrier's layout, asserted MEMBER BY MEMBER and not only by sizeof
# (plan 6.3): a heterogeneous POD is where padding differs across ABIs, and the monolith
# verify harness is blind to it because both sides are the same translation unit. The
# first member is pinned at 0 and the rest are pinned to ascend, which is the strongest
# statement a generator that cannot see the layout can make; sizeof plus
# MGL_RESIDUAL_BLOCK_SIZE pins the rest, and the ratchet only ever goes DOWN.
if residual_fields:
out.append("")
out.append("// The ResidualValueBlock layout, from PipeFields.def's")
out.append("// MGP_FIELDS_ResidualValueBlock. Retiring a field without lowering")
out.append("// MGL_RESIDUAL_BLOCK_SIZE is a build break, which is the point.")
out.append("static_assert(offsetof(ResidualValueBlock, %s) == 0," % residual_fields[0])
out.append(" \"the residual block's first member must sit at offset 0\");")
for previous, member in zip(residual_fields, residual_fields[1:]):
out.append("static_assert(offsetof(ResidualValueBlock, %s) >" % member)
out.append(" offsetof(ResidualValueBlock, %s)," % previous)
out.append(" \"the residual block's members must stay in declaration order\");")
out.append("static_assert(sizeof(ResidualValueBlock) == MGL_RESIDUAL_BLOCK_SIZE,")
out.append(" \"the residual ratchet only ever goes down\");")
return "\n".join(out) + "\n"
def gen_verify(payloads):
out = [banner("PipeVerify.inc", "G4: the MOBILEGL_PIPE_VERIFY field-wise comparators.",
"PipeFields.def")]
out.append("""// Field by field, never memcmp over a whole payload: RenderStateParameters is documented
// in DirectGLES.cpp to false-DIFFER on padding under memcmp (harmlessly there, fatally
// here - a comparator with false positives is a comparator nobody reads). Each function
// reports the FIRST differing field by name, which with the draw serial is what the verify
// harness prints.
//
// Floating-point fields are compared by BITS, so a NaN patch level - which
// glPatchParameterfv accepts and ComputePipelineStateHash already hashes bitwise - equals
// itself instead of tripping every draw.
#include "../PipeFields.def"
""")
out.append("template <class T>")
out.append("struct MGPipeHasFieldVerifier : std::false_type {};\n")
out.append("""// A vector type (FloatVec4, IntVec4, BoolVec4...) is detected through its VecBase and
// compared BITWISE over its data: VecBase::operator== is IEEE ==, under which a NaN patch
// level would differ from itself. The probe rather than an overload because a
// derived-to-base conversion loses overload resolution to the exact-match generic template.
template <class Derived, class T, SizeT N>
std::true_type MGPipeVecBaseProbe(const VecBase<Derived, T, N>*);
std::false_type MGPipeVecBaseProbe(const void*);
template <class T>
inline constexpr Bool kMGPipeIsVecBase = decltype(MGPipeVecBaseProbe(static_cast<const T*>(nullptr)))::value;
template <class T>
inline Bool MGPipeFieldEqual(const T& a, const T& b);
template <class T, SizeT N>
inline Bool MGPipeFieldEqual(const Array<T, N>& a, const Array<T, N>& b);
template <class T, SizeT N>
inline Bool MGPipeFieldEqual(const T (&a)[N], const T (&b)[N]);
""")
for payload in payloads:
out.append("inline Bool MGPipeVerify(const %s& a, const %s& b, const char** outField);"
% (payload, payload))
out.append("")
for payload in payloads:
out.append("template <>")
out.append("struct MGPipeHasFieldVerifier<%s> : std::true_type {};" % payload)
out.append("")
out.append("""template <class T>
inline Bool MGPipeFieldEqual(const T& a, const T& b) {
if constexpr (MGPipeHasFieldVerifier<T>::value) {
const char* unusedField = nullptr;
return MGPipeVerify(a, b, &unusedField);
} else if constexpr (kMGPipeIsVecBase<T>) {
return std::memcmp(a.data.data(), b.data.data(), sizeof(a.data)) == 0;
} else if constexpr (std::is_floating_point_v<T>) {
return std::memcmp(&a, &b, sizeof(T)) == 0;
} else if constexpr (std::is_scalar_v<T> || std::is_enum_v<T>) {
return a == b;
} else if constexpr (requires(const T& x, const T& y) { x == y; }) {
return a == b;
} else {
// NO MEMCMP FALLBACK. Every value struct has a field list in PipeFields.def since P1
// (and gen_pipe.py asserts each list covers its struct's members); a type reaching
// this branch is one nobody gave a field list, and a memcmp would false-differ on
// its padding. A compile error is the honest answer.
static_assert(sizeof(T) == 0, "no field list in PipeFields.def for this type");
return false;
}
}
template <class T, SizeT N>
inline Bool MGPipeFieldEqual(const Array<T, N>& a, const Array<T, N>& b) {
for (SizeT i = 0; i < N; ++i) {
if (!MGPipeFieldEqual(a[i], b[i])) return false;
}
return true;
}
template <class T, SizeT N>
inline Bool MGPipeFieldEqual(const T (&a)[N], const T (&b)[N]) {
for (SizeT i = 0; i < N; ++i) {
if (!MGPipeFieldEqual(a[i], b[i])) return false;
}
return true;
}
#define MGP_VERIFY_FIELD(FieldName) \\
if (!MGPipeFieldEqual(a.FieldName, b.FieldName)) { \\
if (outField != nullptr) *outField = #FieldName; \\
return false; \\
}
""")
for payload in payloads:
out.append("inline Bool MGPipeVerify(const %s& a, const %s& b, const char** outField) {"
% (payload, payload))
out.append(" MGP_FIELDS_%s(MGP_VERIFY_FIELD)" % payload)
out.append(" return true;")
out.append("}")
out.append("")
out.append("#undef MGP_VERIFY_FIELD")
out.append("")
out.append("inline constexpr SizeT kMGPipeVerifiedPayloadCount = %d;" % len(payloads))
return "\n".join(out) + "\n"
def gen_emitted_by(accessors, calls, emitted):
"""P2 brief D5: which P2 call SUPPLIES each PipeInputs field, so the per-verb residual
fill loop can skip it. Emitters are the distinct calls named in MGP_COVERAGE_EMITTED_LIST,
sorted so the enum is stable against the order rows are written in."""
call_names = set(c.Name for c in calls)
emitted_map = dict(emitted)
for name, call in emitted:
if call not in call_names:
sys.exit("Coverage.def: MGP_COVERAGE_EMITTED_LIST names %s for %s, which is not a "
"call in PipeCalls.def" % (call, name))
emitters = sorted(set(call for _, call in emitted))
out = []
out.append("// P2 brief D5: the call that now SUPPLIES a field, so the residual fill loop no")
out.append("// longer pulls it out of GLContext. kNone means the field is still pulled - which")
out.append("// is what makes MOBILEGL_PIPE_PUSH a true per-subsystem A/B instead of a single")
out.append("// switch. Rows come from Coverage.def's MGP_COVERAGE_EMITTED_LIST.")
out.append("enum class MGPipeFieldEmitter : Uint8 {")
out.append(" kNone = 0,")
for emitter in emitters:
out.append(" %s," % emitter)
out.append("};")
out.append("")
out.append("inline constexpr const char* kMGPipeFieldEmitterNames[] = {")
out.append(" \"kNone\",")
for emitter in emitters:
out.append(" \"%s\"," % emitter)
out.append("};")
out.append("")
out.append("inline constexpr MGPipeFieldEmitter kMGPipeFieldEmittedBy[kMGPipeInputFieldCount] = {")
for name, _ in accessors:
emitter = emitted_map.get(name)
if emitter is None:
out.append(" MGPipeFieldEmitter::kNone, // %s" % name)
else:
out.append(" MGPipeFieldEmitter::%s, // %s" % (emitter, name))
out.append("};")
out.append("inline constexpr SizeT kMGPipeEmittedFieldCount = %d;" % len(emitted))
out.append("")
return "\n".join(out)
def gen_filled(accessors, calls, sticky, emitted):
call_names = set(c.Name for c in calls)
sticky_map = dict(sticky)
out = [banner("PipeFilled.inc", "G5: PipeInputs field ids and the per-verb poison generations.",
"Coverage.def and PipeCalls.def")]
out.append("""// One field id per GLContext accessor the backends actually read (plan B section 6.2:
// PipeInputs is organized by MEMO KEY, not by read point, which is why the field set is
// small and stable across the whole migration).
//
// The poison is a per-verb GENERATION, not a bit. A bitmap cannot see the dangerous case:
// a field filled by the previous DRAW and then read by the glTexSubImage that follows is
// stale, and its bit is already set. So every verb bumps CurrentVerbSerial, filling a
// field stamps it with that serial, and reading a non-sticky field whose stamp is older is
// Fatal{UnmigratedPipeInput} (section 6.2.2).
//
// PipeInputs itself is MG_Backend/MGPipe/PipeInputs.h (P1); the verb enum and the
// per-class fill masks are G5b, generated/PipeFillPoints.inc.
""")
out.append("enum class MGPipeInputField : Uint16 {")
for name, _ in accessors:
out.append(" %s," % name)
out.append(" kFieldCount,")
out.append("};")
out.append("")
out.append("inline constexpr SizeT kMGPipeInputFieldCount = static_cast<SizeT>(MGPipeInputField::kFieldCount);")
out.append("static_assert(kMGPipeInputFieldCount == %d, \"the PipeInputs field set moved\");" % len(accessors))
out.append("")
out.append("inline constexpr const char* kMGPipeInputFieldNames[kMGPipeInputFieldCount] = {")
for name, _ in accessors:
out.append(" \"%s\"," % name)
out.append("};")
out.append("")
out.append("// Fields whose value is valid ACROSS verbs: a sticky field is a field the poison")
out.append("// cannot protect, so every true is argued for in Coverage.def's")
out.append("// MGP_COVERAGE_STICKY_LIST (the seven forwarded, argument-keyed accessors).")
out.append("inline constexpr Bool kMGPipeInputFieldSticky[kMGPipeInputFieldCount] = {")
for name, _ in accessors:
if name in sticky_map:
out.append(" true, // %s: %s" % (name, sticky_map[name]))
else:
out.append(" false, // %s" % name)
out.append("};")
out.append("inline constexpr SizeT kMGPipeInputStickyFieldCount = %d;" % len(sticky))
out.append("")
out.append("// Which call is expected to have filled a field by the time a verb reads it. Names")
out.append("// come from Coverage.def, so this table and the coverage table cannot disagree.")
out.append("inline constexpr const char* kMGPipeInputFieldFilledBy[kMGPipeInputFieldCount] = {")
for name, call in accessors:
marker = "" if call in call_names else " // pseudo-call: not filled by a forward record"
out.append(" \"%s\",%s" % (call, marker))
out.append("};")
out.append("")
out.append(gen_emitted_by(accessors, calls, emitted))
out.append("""struct MGPipeFilledState {
Uint64 CurrentVerbSerial;
Uint64 FilledGen[kMGPipeInputFieldCount];
};
[[noreturn]] inline void MGPipeInputPoisonFatal(MGPipeInputField field, const char* verb) {
MGLOG_F("MGPipe: Fatal{UnmigratedPipeInput, \\"%s@%s\\"}",
kMGPipeInputFieldNames[static_cast<SizeT>(field)], verb);
std::abort();
}
// FilledGen == 0 is "never filled" on BOTH branches: before the first MGPipeFillForVerb the
// serial is 0 as well, and a read in that window is the poison's "<Field>@<none>" case
// (P1 brief D6), never a fresh read of default-constructed storage.
inline Bool MGPipeInputFieldIsFresh(const MGPipeFilledState& state, MGPipeInputField field) {
const SizeT index = static_cast<SizeT>(field);
const Uint64 gen = state.FilledGen[index];
if (gen == 0) return false;
return kMGPipeInputFieldSticky[index] || gen == state.CurrentVerbSerial;
}""")
return "\n".join(out) + "\n"
FUNCTION_POINTER_MEMBER_RE = re.compile(r"\(\s*\*\s*(\w+)\s*\)\s*\(")
def parse_function_table():
"""The function-pointer members of MG_Backend::GLFunctionsTable, in declaration order.
Data members (PrefersCpuXfbPrimitiveAccounting) are not verbs and are skipped; comments
are masked line by line."""
text = read(FUNCTION_TABLE_HEADER)
start = text.find("struct GLFunctionsTable {")
if start < 0:
sys.exit("%s: struct GLFunctionsTable is missing" % FUNCTION_TABLE_HEADER)
members = []
for line in text[start:].splitlines()[1:]:
if re.match(r"^\s*};", line):
break
code = line.split("//", 1)[0]
for name in FUNCTION_POINTER_MEMBER_RE.findall(code):
members.append(name)
if not members:
sys.exit("%s: GLFunctionsTable has no function-pointer members" % FUNCTION_TABLE_HEADER)
return members
def parse_fill_points(accessors, table_members=None, text=None):
"""FillPoints.def -> (verbs, classes, fields): verbs is [(verb, class)] in file order,
classes is [class], fields is {class: [field]}. Refuses a verb set that is not exactly
GLFunctionsTable's function-pointer members in declaration order, a verb in two
classes, a class with no verbs, a field that is not an accessor, a duplicate
(class, field) row, and a class row that names no class."""
if text is None:
text = read(os.path.join(PIPE_DIR, "FillPoints.def"))
if table_members is None:
table_members = parse_function_table()
def block(macro):
match = re.search(r"#define %s\(X\)(.*?)\n\n" % macro, text, re.S)
if not match:
sys.exit("FillPoints.def: %s is missing (or not followed by a blank line)" % macro)
return match.group(1)
classes = re.findall(r"X\((\w+)\)", block("MGP_FILL_CLASS_LIST"))
if len(classes) != len(set(classes)):
sys.exit("FillPoints.def: a class is listed twice in MGP_FILL_CLASS_LIST")
verbs = re.findall(r"X\((\w+)\s*,\s*(\w+)\)", block("MGP_FILL_VERB_LIST"))
seen = set()
for verb, cls in verbs:
if verb in seen:
sys.exit("FillPoints.def: verb %s is in two classes" % verb)
seen.add(verb)
if cls not in classes:
sys.exit("FillPoints.def: verb %s names unknown class %s" % (verb, cls))
verb_names = [verb for verb, _ in verbs]
missing = [m for m in table_members if m not in seen]
if missing:
sys.exit("FillPoints.def: GLFunctionsTable member(s) without a verb row: %s" % ", ".join(missing))
extra = [v for v in verb_names if v not in table_members]
if extra:
sys.exit("FillPoints.def: verb(s) that are not GLFunctionsTable members: %s" % ", ".join(extra))
if verb_names != table_members:
sys.exit("FillPoints.def: verb rows are not in GLFunctionsTable declaration order "
"(first difference at %s)" % next(a for a, b in zip(verb_names, table_members) if a != b))
for cls in classes:
if not any(c == cls for _, c in verbs):
sys.exit("FillPoints.def: class %s has no verbs" % cls)
accessor_names = set(name for name, _ in accessors)
fields = {cls: [] for cls in classes}
for cls, field in re.findall(r"X\((\w+)\s*,\s*(\w+)\)", block("MGP_FILL_FIELD_LIST")):
if cls not in fields:
sys.exit("FillPoints.def: field row names unknown class %s" % cls)
if field not in accessor_names:
sys.exit("FillPoints.def: %s is not an accessor in Coverage.def" % field)
if field in fields[cls]:
sys.exit("FillPoints.def: duplicate row (%s, %s)" % (cls, field))
fields[cls].append(field)
return verbs, classes, fields
def gen_fill_points(accessors, sticky, verbs, classes, fields):
field_index = {name: i for i, (name, _) in enumerate(accessors)}
# Two words minimum (the P1 contract shape, headroom for the 64th field); grows on demand.
words = max(2, (len(accessors) + 63) // 64)
sticky_names = [name for name, _ in sticky]
out = [banner("PipeFillPoints.inc", "G5b: the verb enum, the verb classes and their may-read field masks.",
"FillPoints.def, Coverage.def and MG_Backend/BackendObject.h")]
out.append("""// One verb per function-pointer member of MG_Backend::GLFunctionsTable, in declaration
// order, so the enum IS the table's member list. MG_Impl spells MGP_FILL(Verb) before every
// call through the table; MGPipeFillForVerb fills exactly the fields of the verb's class
// (plus the sticky fields, OR'ed into every mask) and stamps them with the new serial. A
// read of any other field is Fatal{UnmigratedPipeInput, \"Field@Verb\"} in a poison build.
""")
out.append("enum class MGPipeVerb : Uint8 {")
for verb, _ in verbs:
out.append(" %s," % verb)
out.append(" kVerbCount,")
out.append("};")
out.append("")
out.append("inline constexpr SizeT kMGPipeVerbCount = static_cast<SizeT>(MGPipeVerb::kVerbCount);")
out.append("static_assert(kMGPipeVerbCount == %d, \"the GLFunctionsTable verb set moved\");" % len(verbs))
out.append("")
out.append("inline constexpr const char* kMGPipeVerbNames[kMGPipeVerbCount] = {")
for verb, _ in verbs:
out.append(" \"%s\"," % verb)
out.append("};")
out.append("")
out.append("enum class MGPipeVerbClass : Uint8 {")
for cls in classes:
out.append(" %s," % cls)
out.append(" kClassCount,")
out.append("};")
out.append("")
out.append("inline constexpr SizeT kMGPipeVerbClassCount = static_cast<SizeT>(MGPipeVerbClass::kClassCount);")
out.append("static_assert(kMGPipeVerbClassCount == %d, \"the verb class set moved\");" % len(classes))
out.append("")
out.append("inline constexpr const char* kMGPipeVerbClassNames[kMGPipeVerbClassCount] = {")
for cls in classes:
out.append(" \"%s\"," % cls)
out.append("};")
out.append("")
out.append("inline constexpr MGPipeVerbClass kMGPipeVerbClass[kMGPipeVerbCount] = {")
for verb, cls in verbs:
out.append(" MGPipeVerbClass::%s, // %s" % (cls, verb))
out.append("};")
out.append("")
out.append("// One bit per MGPipeInputField. The %d sticky fields are OR'ed into every class." % len(sticky))
out.append("struct MGPipeFieldMask {")
out.append(" Uint64 Words[%d];" % words)
out.append("};")
out.append("")
out.append("inline constexpr Bool MGPipeFieldMaskHas(const MGPipeFieldMask& mask, MGPipeInputField field) {")
out.append(" const SizeT index = static_cast<SizeT>(field);")
out.append(" return (mask.Words[index / 64] >> (index % 64)) & 1u;")
out.append("}")
out.append("")
out.append("inline constexpr MGPipeFieldMask kMGPipeClassFieldMask[kMGPipeVerbClassCount] = {")
for cls in classes:
bits = [0] * words
names = fields[cls] + [n for n in sticky_names if n not in fields[cls]]
for name in names:
index = field_index[name]
bits[index // 64] |= 1 << (index % 64)
out.append(" // %s: %d fields (%d own + %d sticky)" % (cls, len(names), len(fields[cls]), len(names) - len(fields[cls])))
out.append(" {{%s}}," % ", ".join("0x%016xull" % b for b in bits))
out.append("};")
out.append("")
out.append("static_assert(kMGPipeInputFieldCount <= %d * 64, \"MGPipeFieldMask needs another word\");" % words)
return "\n".join(out) + "\n"
def gen_coverage(accessors, deltas, rows, calls):
call_names = set(c.Name for c in calls)
pseudo = {"kClientResolved", "kReverseChannel", "kStructuralHandle"}
accessor_map = dict(accessors)
delta_map = dict(deltas)
for _, call in accessors + deltas:
if call not in call_names and call not in pseudo:
sys.exit("Coverage.def: %s is not a call in PipeCalls.def and not a pseudo-call" % call)
mapped = 0
by_pseudo = {name: 0 for name in pseudo}
unmapped_rows = []
per_accessor = {}
for row in rows:
call = None
if row["member"] and row["member"] != "-":
call = accessor_map.get(row["member"])
if call is None:
call = delta_map.get(row["delta"])
if call is None:
unmapped_rows.append(row)
continue
if call in pseudo:
by_pseudo[call] += 1
else:
mapped += 1
key = row["member"] if row["member"] and row["member"] != "-" else row["delta"]
per_accessor.setdefault(key, [call, 0])[1] += 1
out = [banner("PipeCoverage.inc", "G6: backend read inventory -> MGPipe call coverage.",
"Coverage.def and scripts/data/backend_read_inventory.md")]
out.append("""// The acceptance rule (plan B section 10.3-5): regenerate, `git diff --exit-code`, and
// ZERO unmapped rows. P0 permits unmapped rows and only counts them; the count below is
// the number the later gate has to drive to zero.
//
// Three pseudo-calls stand for read points that never become a forward record:
// kClientResolved (the frontend answers it), kReverseChannel (it becomes one of the ten
// MGPipeCallbacks) and kStructuralHandle (the row is a signature carrying a
// SharedPtr<MG_State...> that becomes an MGPipeHandle parameter).
""")
out.append("struct MGPipeCoverageEntry {")
out.append(" const char* Accessor;")
out.append(" const char* Call;")
out.append(" Uint32 ReadPoints;")
out.append("};")
out.append("")
out.append("inline constexpr MGPipeCoverageEntry kMGPipeCoverage[] = {")
for key in sorted(per_accessor):
call, count = per_accessor[key]
out.append(" {\"%s\", \"%s\", %d}," % (key, call, count))
out.append("};")
out.append("")
out.append("inline constexpr SizeT kMGPipeCoverageEntryCount = %d;" % len(per_accessor))
out.append("inline constexpr Uint32 kMGPipeInventoryReadPoints = %d;" % len(rows))
out.append("inline constexpr Uint32 kMGPipeInventoryMappedToCall = %d;" % mapped)
out.append("inline constexpr Uint32 kMGPipeInventoryClientResolved = %d;" % by_pseudo["kClientResolved"])
out.append("inline constexpr Uint32 kMGPipeInventoryReverseChannel = %d;" % by_pseudo["kReverseChannel"])
out.append("inline constexpr Uint32 kMGPipeInventoryStructuralHandle = %d;"
% by_pseudo["kStructuralHandle"])
out.append("inline constexpr Uint32 kMGPipeInventoryUnmapped = %d;" % len(unmapped_rows))
out.append("static_assert(kMGPipeCoverageEntryCount == sizeof(kMGPipeCoverage) / sizeof(kMGPipeCoverage[0]));")
out.append("static_assert(kMGPipeInventoryMappedToCall + kMGPipeInventoryClientResolved +")
out.append(" kMGPipeInventoryReverseChannel + kMGPipeInventoryStructuralHandle +")
out.append(" kMGPipeInventoryUnmapped ==")
out.append(" kMGPipeInventoryReadPoints,")
out.append(" \"every inventory row must land in exactly one bucket\");")
return "\n".join(out) + "\n", unmapped_rows, mapped, by_pseudo
def gen_span_table():
out = [banner("PipeSpanTable.inc", "G7: the render-state pipeline subset, by member name.",
"the field list in scripts/gen_pipe.py")]
out.append("""// D-B1 rejected three CSOs and demanded this table instead, so the table needs its own
// completeness trip wire: MG_Test walks every public RenderState setter and asserts that
// the pipeline-subset hash moves IF AND ONLY IF m_pipelineStateVersion moves
// (MG_Test/Pipe/RenderStateSpansTest.cpp).
//
// P2 replaced P0's provenance with a RULE, and the rule is the only thing that decides
// membership: a member is pipeline state IF AND ONLY IF some public RenderState setter that
// calls BumpVersions() writes it. That is what makes the G7 invariant true by construction
// rather than by inspection, and it turns the subset into a strict SUPERSET of the 24
// members VulkanRenderer::ComputePipelineStateHash used to hash.
//
// The three questions P0 left open are ANSWERED here, and the answers are in this list:
// - FramebufferSrgb, DepthClamp and TextureCubeMapSeamless had NO STORAGE at all -
// SetCapability fell to "not supported currently" and IsCapabilityEnabled answered a
// compile-time false. P2 gave all three real storage in the three padding bytes between
// ColorMasks and ClearColor, and their setters call BumpVersions(), so: pipeline state.
// - ProvokingVertexModeSetting: SetProvokingVertexMode calls BumpVersions(), so pipeline.
// - FrontFaceModeSetting likewise. ClipOrigin and ClipDepthMode do NOT (SetClipControl is
// ++m_version only), so they are dynamic, in chunk D1.
//
// The complement of this list is the DYNAMIC subset - the half whose whole purpose is that
// glViewport must not mint a new CSO.
inline constexpr const char* const kMGPipePipelineStateMembers[] = {""")
for member in PIPELINE_STATE_MEMBERS:
out.append(" \"%s\"," % member)
out.append("};")
out.append("inline constexpr SizeT kMGPipePipelineStateMemberCount = %d;" % len(PIPELINE_STATE_MEMBERS))
out.append("static_assert(kMGPipePipelineStateMemberCount ==")
out.append(" sizeof(kMGPipePipelineStateMembers) / sizeof(kMGPipePipelineStateMembers[0]));")
out.append("")
out.append("// Defined by MG_Pipe/MGPipeRenderStateSpans.cpp (P2), which computes every")
out.append("// boundary in C++ with offsetof rather than guessing it in python. 7 pipeline")
out.append("// chunks / 396 bytes and 8 dynamic chunks / 772 bytes, and the two halves")
out.append("// partition [0, sizeof(RenderStateParameters)) exactly - asserted there.")
out.append("extern const MGPStateChunk kMGPipePipelineChunks[];")
out.append("extern const MGPStateChunk kMGPipeDynamicChunks[];")
return "\n".join(out) + "\n"
def write(path, text, check, changed):
existing = read(path) if os.path.exists(path) else None
if existing == text:
return
changed.append(os.path.relpath(path, REPO_ROOT))
if not check:
with open(path, "w", encoding="utf-8", newline="\n") as handle:
handle.write(text)
def expect_trip(name, fn):
"""Runs one negative control; a gate that lets it through is the failure."""
try:
fn()
except SystemExit as trip:
print("gen_pipe: self-test %s: tripped as expected (%s)" % (name, str(trip).splitlines()[0][:100]))
return 1
print("gen_pipe: self-test %s: DID NOT TRIP" % name, file=sys.stderr)
return 0
def self_test(accessors):
"""The negative controls (check_include_closure.py's shape): each gate must go red for
its reason, and zero trips is itself an error."""
canned_struct = "struct Canned {\n Uint32 A;\n Uint32 B, C;\n Uint8 Pad0[3];\n void F() { return; }\n};\n"
controls = [
("struct member without F(...)",
lambda: check_field_lists_cover_struct_members({"Canned": ["A", "B"]}, ["Canned"], [canned_struct])),
("F(...) that is not a member",
lambda: check_field_lists_cover_struct_members({"Canned": ["A", "B", "C", "D"]}, ["Canned"], [canned_struct])),
("payload with no struct",
lambda: check_field_lists_cover_struct_members({"Nowhere": ["A"]}, ["Nowhere"], [canned_struct])),
]
fill_text = read(os.path.join(PIPE_DIR, "FillPoints.def"))
verb_row = re.compile(r"X\(\s*DrawArrays\s*,\s*kDraw\s*\)")
field_row = re.compile(r"X\(\s*kDraw\s*,\s*GetBoundVertexArray\s*\)")
if not verb_row.search(fill_text) or not field_row.search(fill_text):
sys.exit("gen_pipe: self-test: FillPoints.def lost the rows the controls edit")
controls.append(("verb missing from FillPoints.def", lambda: parse_fill_points(
accessors, text=verb_row.sub("", fill_text, count=1))))
controls.append(("verb that is not a GLFunctionsTable member", lambda: parse_fill_points(
accessors, text=verb_row.sub("X(DrawArrays, kDraw) X(NotAVerb, kDraw)", fill_text, count=1))))
controls.append(("field row naming a non-accessor", lambda: parse_fill_points(
accessors, text=field_row.sub("X(kDraw, NotAnAccessor)", fill_text, count=1))))
# The EMITTED list's own gate: a row naming a call that is not in PipeCalls.def would
# generate an enumerator nothing can dispatch on.
calls_for_control = parse_calls()
controls.append(("emitted row naming a call that does not exist", lambda: gen_emitted_by(
[("GetViewport", "SetDynamicState")], calls_for_control, [("GetViewport", "NotACall")])))
# P5 R-13.4's gate. The flags are now a GENERATED TABLE six packages read instead of six
# hard-coded copies, so a token that is not an MGPipeCallFlags enumerator has to stop the
# generator rather than emit an expression that fails to compile minutes later - and
# kNone, the empty set, may not be OR'd with a real flag and quietly read as one.
flag_typo = Call(1, "Canned", "MGPHandleOnly", "kScreen", ["kHasBlobb"])
flag_kNone = Call(1, "Canned", "MGPHandleOnly", "kScreen", ["kNone", "kHasBlob"])
controls.append(("call flag that is not an MGPipeCallFlags enumerator",
lambda: check_call_flags_are_known([flag_typo])))
controls.append(("kNone combined with a real flag",
lambda: check_call_flags_are_known([flag_kNone])))
trips = 0
for name, fn in controls:
trips += expect_trip(name, fn)
# The positive control: the canned struct's exact list passes, and the parser sees the
# padding member as padding and the function as not a member.
check_field_lists_cover_struct_members({"Canned": ["A", "B", "C"]}, ["Canned"], [canned_struct])
# ... and the real catalogue's real flags pass the same gate.
check_call_flags_are_known(calls_for_control)
if trips == 0:
sys.exit("gen_pipe: self-test: no negative control tripped - the gates are not checking anything")
if trips != len(controls):
sys.exit("gen_pipe: self-test: %d of %d negative controls did not trip" % (len(controls) - trips, len(controls)))
print("gen_pipe: self-test: %d negative-control trip(s), positive control OK" % trips)
return 0
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--check", action="store_true",
help="do not write; exit 1 if regenerating would change anything")
parser.add_argument("--self-test", action="store_true",
help="run the negative controls (each gate must trip) and exit")
args = parser.parse_args()
calls = parse_calls()
payloads = parse_verify_payloads()
check_call_flags_are_known(calls)
check_call_payloads_have_field_lists(calls, payloads)
check_field_lists_cover_struct_members(parse_field_lists(), payloads)
accessors, deltas, sticky, emitted = parse_coverage()
if args.self_test:
return self_test(accessors)
scan_live_accessors(accessors)
verbs, classes, fields = parse_fill_points(accessors)
rows = parse_inventory()
if not os.path.isdir(GENERATED_DIR):
os.makedirs(GENERATED_DIR)
coverage_text, unmapped, mapped, pseudo = gen_coverage(accessors, deltas, rows, calls)
changed = []
write(os.path.join(GENERATED_DIR, "PipeTables.inc"), gen_tables(calls), args.check, changed)
write(os.path.join(GENERATED_DIR, "PipeThunks.inc"), gen_thunks(calls), args.check, changed)
write(os.path.join(GENERATED_DIR, "PipeWire.inc"),
gen_wire(calls, parse_field_lists().get("ResidualValueBlock")), args.check, changed)
write(os.path.join(GENERATED_DIR, "PipeVerify.inc"), gen_verify(payloads), args.check, changed)
write(os.path.join(GENERATED_DIR, "PipeFilled.inc"), gen_filled(accessors, calls, sticky, emitted),
args.check, changed)
write(os.path.join(GENERATED_DIR, "PipeFillPoints.inc"),
gen_fill_points(accessors, sticky, verbs, classes, fields), args.check, changed)
write(os.path.join(GENERATED_DIR, "PipeCoverage.inc"), coverage_text, args.check, changed)
write(os.path.join(GENERATED_DIR, "PipeSpanTable.inc"), gen_span_table(), args.check, changed)
screen = sum(1 for c in calls if c.IsScreen)
print("gen_pipe: %d calls (%d screen, %d context), %d verify payloads, %d PipeInputs fields "
"(%d sticky, %d emitted by a P2 call), %d verbs, %d classes"
% (len(calls), screen, len(calls) - screen, len(payloads), len(accessors), len(sticky),
len(emitted), len(verbs), len(classes)))
print("gen_pipe: inventory %d rows: %d -> call, %d client-resolved, %d reverse-channel, "
"%d structural handle, %d UNMAPPED"
% (len(rows), mapped, pseudo["kClientResolved"], pseudo["kReverseChannel"],
pseudo["kStructuralHandle"], len(unmapped)))
for row in unmapped:
print("gen_pipe: UNMAPPED %s:%d %s %s" % (row["file"], row["line"], row["kind"], row["member"]))
if changed:
if args.check:
print("gen_pipe: OUT OF DATE: %s" % ", ".join(changed), file=sys.stderr)
return 1
print("gen_pipe: wrote %s" % ", ".join(changed))
else:
print("gen_pipe: generated files are up to date")
return 0
if __name__ == "__main__":
sys.exit(main())
Reference in New Issue View Git Blame Copy Permalink