MobileGL/scripts/gen_pipe.py

#!/usr/bin/env python3
# MobileGL - scripts/gen_pipe.py
# Copyright (c) 2025-2026 MobileGL-Dev
# Licensed under the GNU Lesser General Public License v3.0:
#   https://www.gnu.org/licenses/gpl-3.0.txt
#   https://www.gnu.org/licenses/lgpl-3.0.txt
# SPDX-License-Identifier: LGPL-3.0-only
# End of Source File Header
"""The seven MGPipe generators, G1..G7 (plan B section 4.1).

Reads the four hand-maintained sources of truth

    MobileGL/MG_Pipe/PipeCalls.def      the call catalogue
    MobileGL/MG_Pipe/PipeFields.def     per-payload field lists for the verify comparator
    MobileGL/MG_Pipe/Coverage.def       accessor -> call mapping for the read inventory,
                                        and the sticky-field list
    MobileGL/MG_Pipe/FillPoints.def     verb -> class and class -> may-read field tables
                                        (G5b), checked against MG_Backend::GLFunctionsTable

plus the vendored copy of the backend read inventory

    scripts/data/backend_read_inventory.md

and writes MobileGL/MG_Pipe/generated/*.inc. The outputs are COMMITTED; CI regenerates
them and fails on a diff, which is what keeps the seven generators from drifting apart
from the catalogue (they all consume the same .def).

    python3 scripts/gen_pipe.py            # write the generated files, print the summary
    python3 scripts/gen_pipe.py --check    # fail if regenerating would change anything

Both modes refuse a catalogue whose call payload has no field list in PipeFields.def: a
payload the G4 comparator cannot see is a payload MOBILEGL_PIPE_VERIFY is blind to.
"""

import argparse
import os
import re
import sys

REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
PIPE_DIR = os.path.join(REPO_ROOT, "MobileGL", "MG_Pipe")
GENERATED_DIR = os.path.join(PIPE_DIR, "generated")
INVENTORY = os.path.join(REPO_ROOT, "scripts", "data", "backend_read_inventory.md")
FUNCTION_TABLE_HEADER = os.path.join(REPO_ROOT, "MobileGL", "MG_Backend", "BackendObject.h")

GENERATED_BANNER = """// MobileGL - MobileGL/MG_Pipe/generated/{name}
// Copyright (c) 2025-2026 MobileGL-Dev
// Licensed under the GNU Lesser General Public License v3.0:
//   https://www.gnu.org/licenses/gpl-3.0.txt
//   https://www.gnu.org/licenses/lgpl-3.0.txt
// SPDX-License-Identifier: LGPL-3.0-only
// End of Source File Header

// {title}
//
// GENERATED by scripts/gen_pipe.py from {sources} - DO NOT EDIT.
// Regenerate with `python3 scripts/gen_pipe.py`; CI runs it and diffs the result.
// This file is included from MG_Pipe/MGPipe.h inside namespace MobileGL::MG_Pipe.
"""

# G7. The pipeline subset of RenderStateParameters, BY MEMBER NAME, taken from the fields
# VulkanRenderer::ComputePipelineStateHash hashes today
# (MobileGL/MG_Backend/DirectVulkan/Renderer/VulkanRenderer.cpp:4805-4906 at dev@81b17c0b,
# including ResolveEffectiveSampleMask, which the hash folds in twice - once as the
# effective enable bit and once as the mask word).
#
# Names only: offsets are NOT computed here. The chunk table with real offsets is
# MGPipeRenderStateSpans.cpp, built in C++ with offsetof, because a python guess at the
# layout of a struct it cannot see is exactly the kind of drift the G7 setter-consistency
# test exists to catch (plan B section 4.5.2).
PIPELINE_STATE_MEMBERS = [
    "CullFaceEnabled",
    "DepthTestEnabled",
    "PolygonOffsetFillEnabled",
    "RasterizerDiscardEnabled",
    "ColorLogicOpEnabled",
    "StencilTestEnabled",
    "PrimitiveRestartEnabled",
    "PrimitiveRestartFixedIndexEnabled",
    "DepthMask",
    "SampleShadingEnabled",
    "MultisampleEnabled",
    "SampleMaskEnabled",
    "SampleMaskValue",
    "MinSampleShadingValue",
    "PatchVertices",
    "PatchDefaultOuterLevel",
    "PatchDefaultInnerLevel",
    "PolygonModeFront",
    "CullFaceModeSetting",
    "DepthFunc",
    "LogicOp",
    "StencilStates",
    "BlendStates",
    "ColorMasks",
]


def read(path):
    with open(path, "r", encoding="utf-8") as handle:
        return handle.read()


class Call(object):
    def __init__(self, index, name, payload, cls, flags):
        self.Index = index  # 1-based; this is the wire opcode
        self.Name = name
        self.Payload = payload
        self.Class = cls
        self.Flags = flags

    @property
    def IsScreen(self):
        return self.Class == "kScreen"

    @property
    def Signature(self):
        """(parameter declaration list, argument list) for this call."""
        params = ["const %s* payload" % self.Payload]
        args = ["payload"]
        if "kVarTail" in self.Flags:
            params.append("const void* varTail")
            params.append("Uint32 varTailCount")
            args.append("varTail")
            args.append("varTailCount")
        if "kReplySlot" in self.Flags:
            params.append("MGPReplySlot* reply")
            args.append("reply")
        return ", ".join(params), ", ".join(args)


CALL_RE = re.compile(r"^\s*X\(\s*(\w+)\s*,\s*(\w+)\s*,\s*(\w+)\s*,\s*([\w|]+?)\s*\)\s*\\?\s*$")


def parse_calls():
    text = read(os.path.join(PIPE_DIR, "PipeCalls.def"))
    documented = re.search(r"#define MGP_CALL_LIST_DOCUMENTED_COUNT (\d+)", text)
    if not documented:
        sys.exit("PipeCalls.def: MGP_CALL_LIST_DOCUMENTED_COUNT is missing")
    calls = []
    inside = False
    for line in text.splitlines():
        if line.startswith("#define MGP_CALL_LIST(X)"):
            inside = True
            continue
        if not inside:
            continue
        match = CALL_RE.match(line)
        if match:
            calls.append(Call(len(calls) + 1, match.group(1), match.group(2), match.group(3),
                              match.group(4).split("|")))
        # The macro ends at the first line without a continuation backslash.
        if not line.rstrip().endswith("\\"):
            inside = False
    count = int(documented.group(1))
    if len(calls) != count:
        sys.exit("PipeCalls.def: parsed %d calls but MGP_CALL_LIST_DOCUMENTED_COUNT says %d"
                 % (len(calls), count))
    seen = set()
    for call in calls:
        if call.Name in seen:
            sys.exit("PipeCalls.def: duplicate call %s" % call.Name)
        seen.add(call.Name)
    return calls


# The member types the G4 comparator falls back to memcmp for (see gen_verify): the
# MG_Pipe / MG_Backend value structs and MGHostSpan. They are not call payloads and get
# field lists of their own in P1 (P0.5 moved the types). Nothing else may be missing from
# PipeFields.def.
MEMCMP_FALLBACK_TYPES = {
    "RenderStateParameters",
    "PixelStoreParameters",
    "DynamicBackendParameters",
    "MGHostSpan",
}


def parse_verify_payloads():
    text = read(os.path.join(PIPE_DIR, "PipeFields.def"))
    match = re.search(r"#define MGP_VERIFY_PAYLOAD_LIST\(P\)(.*?)\n\n", text, re.S)
    if not match:
        sys.exit("PipeFields.def: MGP_VERIFY_PAYLOAD_LIST is missing")
    payloads = re.findall(r"P\((\w+)\)", match.group(1))
    for payload in payloads:
        if ("#define MGP_FIELDS_%s(F)" % payload) not in text:
            sys.exit("PipeFields.def: %s is in the payload list with no field macro" % payload)
    return payloads


def check_call_payloads_have_field_lists(calls, payloads):
    """Every payload PipeCalls.def names must have a G4 field list, or the verify comparator
    is silently blind to that call. Runs in both modes, --check included."""
    known = set(payloads)
    missing = sorted({c.Payload for c in calls
                      if c.Payload not in known and c.Payload not in MEMCMP_FALLBACK_TYPES})
    if missing:
        sys.exit("PipeFields.def: call payload(s) with no field list, so MOBILEGL_PIPE_VERIFY "
                 "would be blind to them: %s" % ", ".join(missing))


def parse_coverage():
    text = read(os.path.join(PIPE_DIR, "Coverage.def"))
    accessors = []
    block = re.search(r"#define MGP_COVERAGE_ACCESSOR_LIST\(X\)(.*?)\n\n", text, re.S)
    if not block:
        sys.exit("Coverage.def: MGP_COVERAGE_ACCESSOR_LIST is missing")
    for name, call in re.findall(r"X\((\w+)\s*,\s*(\w+)\)", block.group(1)):
        accessors.append((name, call))
    deltas = []
    block = re.search(r"#define MGP_COVERAGE_DELTA_LIST\(X\)(.*?)\n\n", text, re.S)
    if not block:
        sys.exit("Coverage.def: MGP_COVERAGE_DELTA_LIST is missing")
    for kind, call in re.findall(r"X\(([^,]+),\s*(\w+)\)", block.group(1)):
        deltas.append((kind.strip(), call))
    sticky = []
    block = re.search(r"#define MGP_COVERAGE_STICKY_LIST\(X\)(.*?)\n\n", text, re.S)
    if not block:
        sys.exit("Coverage.def: MGP_COVERAGE_STICKY_LIST is missing")
    accessor_names = set(name for name, _ in accessors)
    for name, reason in re.findall(r"X\((\w+)\s*,\s*\"([^\"]*)\"\)", block.group(1)):
        if name not in accessor_names:
            sys.exit("Coverage.def: sticky field %s is not an accessor in MGP_COVERAGE_ACCESSOR_LIST" % name)
        if name in dict(sticky):
            sys.exit("Coverage.def: sticky field %s is listed twice" % name)
        sticky.append((name, reason))
    return accessors, deltas, sticky


INVENTORY_ROW_RE = re.compile(r"^\|\s*(\d+)\s*\|([^|]*)\|([^|]*)\|([^|]*)\|")


def parse_inventory():
    if not os.path.exists(INVENTORY):
        sys.exit("missing %s - copy it from MobileGL-CS/docs/CS_Refactor/" % INVENTORY)
    rows = []
    current_file = None
    for line in read(INVENTORY).splitlines():
        heading = re.match(r"^### `([^`]+)`", line)
        if heading:
            current_file = heading.group(1)
            continue
        match = INVENTORY_ROW_RE.match(line)
        if match and current_file:
            rows.append({
                "file": current_file,
                "line": int(match.group(1)),
                "kind": match.group(2).strip(),
                "member": match.group(3).strip(),
                "delta": match.group(4).strip(),
            })
    return rows


def banner(name, title, sources):
    return GENERATED_BANNER.format(name=name, title=title, sources=sources)


def gen_tables(calls):
    screen = [c for c in calls if c.IsScreen]
    context = [c for c in calls if not c.IsScreen]
    out = [banner("PipeTables.inc", "G1: the two MGPipe interface tables.", "PipeCalls.def")]
    for struct_name, group, what in (("MGPipeScreen", screen, "share group"),
                                     ("MGPipeContext", context, "context")):
        out.append("// %s: %d calls. A null entry means the backend does not implement this\n"
                   "// call and the frontend keeps its own path (plan B section 4.1).\n"
                   "struct %s {" % (what, len(group), struct_name))
        for call in group:
            params, _ = call.Signature
            out.append("    void (*%s)(%s);" % (call.Name, params))
        out.append("};\n")
    out.append("inline constexpr SizeT kMGPipeScreenCallCount = %d;" % len(screen))
    out.append("inline constexpr SizeT kMGPipeContextCallCount = %d;" % len(context))
    out.append("inline constexpr SizeT kMGPipeCallCount = %d;" % len(calls))
    out.append("")
    out.append("// A table that is not exactly its call count of function pointers has grown a")
    out.append("// member that no generator knows about.")
    out.append("static_assert(sizeof(MGPipeScreen) == kMGPipeScreenCallCount * sizeof(void (*)()),")
    out.append("              \"MGPipeScreen is not exactly its catalogue's function pointers\");")
    out.append("static_assert(sizeof(MGPipeContext) == kMGPipeContextCallCount * sizeof(void (*)()),")
    out.append("              \"MGPipeContext is not exactly its catalogue's function pointers\");")
    out.append("static_assert(kMGPipeScreenCallCount + kMGPipeContextCallCount == kMGPipeCallCount);")
    out.append("static_assert(kMGPipeCallCount == MGP_CALL_LIST_DOCUMENTED_COUNT,")
    out.append("              \"the catalogue and its documented count disagree\");")
    return "\n".join(out) + "\n"


def gen_thunks(calls):
    out = [banner("PipeThunks.inc", "G2: monolith thunks over the two tables.", "PipeCalls.def")]
    out.append("// One inline call through the installed table. These are the names MG_Impl call")
    out.append("// sites move onto, replacing gBackendFunctionsTable.GL.* one at a time. An")
    out.append("// unimplemented (null) entry is the caller's business to check, exactly as it is")
    out.append("// with the table this replaces.\n")
    for call in calls:
        params, args = call.Signature
        table = "gMGPipeScreen" if call.IsScreen else "gMGPipeContext"
        out.append("inline void MGP_%s(%s) {" % (call.Name, params))
        out.append("    %s.%s(%s);" % (table, call.Name, args))
        out.append("}")
        out.append("")
    return "\n".join(out)


def gen_wire(calls):
    out = [banner("PipeWire.inc", "G3: wire records, size assertions and the applier's bounds gate.",
                  "PipeCalls.def")]
    out.append("""// Every record is a fixed header plus its payload, padded to the stream's 8-byte
// granularity. The size assertion is stated as a COMPOSITION so it fires on any padding
// the compiler inserts between the header and the payload while staying honest about the
// tail padding the alignment requires.
//
// The applier's precondition is checked BEFORE dispatch, on every record, in every build:
// a record that is shorter than its own type, longer than what is left in the buffer, or
// not a multiple of 8 is protocol corruption and is fatal. There is no recovery path -
// silently applying a truncated record is how a corrupt stream becomes a wrong picture.
//
// OVERSIZED PAYLOADS ARE CHUNKED, NEVER EMITTED WHOLE (plan section 8.2: G3 has to define
// the path for a record larger than the segment). The bound is the ring's,
// RingProducer::MaxRecordBytes() == Capacity()/2, and it is exact rather than
// conservative: a record has to be placeable at every head offset of an empty ring, the
// wrap pad in front of it costs up to total-8 bytes, and only a record of at most half the
// ring survives that at every offset. An emitter holding more than Capacity()/2 bytes of
// record (a large resource_subdata, a create_shader_state archive) splits it into several
// records of at most that size; the transport refuses a bigger one outright - nullptr plus
// an MGLOG_E - rather than let the producer wait on free bytes that can never suffice.

struct MGPWireRecHeader {
    Uint16 Op;    // MGPWireOp
    Uint16 Flags; // MGPipeCallFlags of the call, for asserts and tracing
    Uint32 Size;  // bytes of this record including the header and the variable tail
};
static_assert(sizeof(MGPWireRecHeader) == 8, "the wire header is 8 bytes");
static_assert(std::is_trivially_copyable_v<MGPWireRecHeader>);

// The opcode is the call's position in PipeCalls.def. Reordering that file is a protocol
// break; appending to it is not.
enum class MGPWireOp : Uint16 {
    kInvalid = 0,""")
    for call in calls:
        out.append("    %s = %d," % (call.Name, call.Index))
    out.append("    kOpCount = %d," % (len(calls) + 1))
    out.append("};\n")
    for call in calls:
        out.append("struct alignas(8) MGPWireRec_%s {" % call.Name)
        out.append("    MGPWireRecHeader Header;")
        out.append("    %s Payload;" % call.Payload)
        out.append("};")
        out.append("static_assert(sizeof(MGPWireRec_%s) ==" % call.Name)
        out.append("                  ((sizeof(MGPWireRecHeader) + sizeof(%s) + 7u) & ~SizeT(7u))," % call.Payload)
        out.append("              \"MGPWireRec_%s gained padding; the wire format moved\");" % call.Name)
        out.append("")
    out.append("""[[noreturn]] inline void MGPipeWireProtocolFatal(const char* call, Uint64 size, Uint64 remaining) {
    MGLOG_F("MGPipe: protocol corruption applying %s: size=%llu remaining=%llu", call,
            static_cast<unsigned long long>(size), static_cast<unsigned long long>(remaining));
    std::abort();
}

#define MGP_WIRE_CHECK_BOUNDS(RecType, CallName)                                                                       \\
    do {                                                                                                               \\
        if (!(size >= sizeof(RecType) && size <= remaining && (size % 8) == 0)) {                                       \\
            MGPipeWireProtocolFatal(CallName, size, remaining);                                                         \\
        }                                                                                                              \\
    } while (0)

// Returns whether the record was applied. P0 is a SKELETON: every case validates its
// bounds and then reports "not applied", because no applier exists until P5 wires
// MG_Remote/Server/PipeApplier.cpp to the real backend tables. The switch and the opcode
// enum come from the same list, so a call added to the catalogue cannot be forgotten here;
// the default arm is for the opcode that never came from this catalogue at all - a byte
// off a corrupt stream - and it is fatal for the same reason the bounds check is.
inline Bool MGPipeApplyWireRecord(MGPWireOp op, const void* record, Uint64 size, Uint64 remaining) {
    (void)record;
    switch (op) {""")
    for call in calls:
        out.append("    case MGPWireOp::%s:" % call.Name)
        out.append("        MGP_WIRE_CHECK_BOUNDS(MGPWireRec_%s, \"%s\");" % (call.Name, call.Name))
        out.append("        return false;")
    out.append("""    case MGPWireOp::kInvalid:
    case MGPWireOp::kOpCount:
    default:
        MGPipeWireProtocolFatal("<unknown opcode>", size, remaining);
    }
}

#undef MGP_WIRE_CHECK_BOUNDS""")
    return "\n".join(out) + "\n"


def gen_verify(payloads):
    out = [banner("PipeVerify.inc", "G4: the MOBILEGL_PIPE_VERIFY field-wise comparators.",
                  "PipeFields.def")]
    out.append("""// Field by field, never memcmp over a whole payload: RenderStateParameters is documented
// in DirectGLES.cpp to false-DIFFER on padding under memcmp (harmlessly there, fatally
// here - a comparator with false positives is a comparator nobody reads). Each function
// reports the FIRST differing field by name, which with the draw serial is what the verify
// harness prints.
//
// Floating-point fields are compared by BITS, so a NaN patch level - which
// glPatchParameterfv accepts and ComputePipelineStateHash already hashes bitwise - equals
// itself instead of tripping every draw.

#include "../PipeFields.def"
""")
    out.append("template <class T>")
    out.append("struct MGPipeHasFieldVerifier : std::false_type {};\n")
    for payload in payloads:
        out.append("inline Bool MGPipeVerify(const %s& a, const %s& b, const char** outField);"
                   % (payload, payload))
    out.append("")
    for payload in payloads:
        out.append("template <>")
        out.append("struct MGPipeHasFieldVerifier<%s> : std::true_type {};" % payload)
    out.append("")
    out.append("""template <class T>
inline Bool MGPipeFieldEqual(const T& a, const T& b) {
    if constexpr (MGPipeHasFieldVerifier<T>::value) {
        const char* unusedField = nullptr;
        return MGPipeVerify(a, b, &unusedField);
    } else if constexpr (std::is_floating_point_v<T>) {
        return std::memcmp(&a, &b, sizeof(T)) == 0;
    } else if constexpr (std::is_scalar_v<T> || std::is_enum_v<T>) {
        return a == b;
    } else if constexpr (requires(const T& x, const T& y) { x == y; }) {
        return a == b;
    } else {
        // MEMCMP FALLBACK. Only reached by the payload members that are still value structs
        // without a field list (RenderStateParameters, PixelStoreParameters,
        // DynamicBackendParameters) and by MGHostSpan. P0.5 moved the first two into
        // MGPipeValueTypes.h; P1 gives them field lists of their own, at which point this
        // branch stops being reachable from any payload.
        return std::memcmp(&a, &b, sizeof(T)) == 0;
    }
}

template <class T, SizeT N>
inline Bool MGPipeFieldEqual(const T (&a)[N], const T (&b)[N]) {
    for (SizeT i = 0; i < N; ++i) {
        if (!MGPipeFieldEqual(a[i], b[i])) return false;
    }
    return true;
}

#define MGP_VERIFY_FIELD(FieldName)                                                                                    \\
    if (!MGPipeFieldEqual(a.FieldName, b.FieldName)) {                                                                 \\
        if (outField != nullptr) *outField = #FieldName;                                                               \\
        return false;                                                                                                  \\
    }
""")
    for payload in payloads:
        out.append("inline Bool MGPipeVerify(const %s& a, const %s& b, const char** outField) {"
                   % (payload, payload))
        out.append("    MGP_FIELDS_%s(MGP_VERIFY_FIELD)" % payload)
        out.append("    return true;")
        out.append("}")
        out.append("")
    out.append("#undef MGP_VERIFY_FIELD")
    out.append("")
    out.append("inline constexpr SizeT kMGPipeVerifiedPayloadCount = %d;" % len(payloads))
    return "\n".join(out) + "\n"


def gen_filled(accessors, calls, sticky):
    call_names = set(c.Name for c in calls)
    sticky_map = dict(sticky)
    out = [banner("PipeFilled.inc", "G5: PipeInputs field ids and the per-verb poison generations.",
                  "Coverage.def and PipeCalls.def")]
    out.append("""// One field id per GLContext accessor the backends actually read (plan B section 6.2:
// PipeInputs is organized by MEMO KEY, not by read point, which is why the field set is
// small and stable across the whole migration).
//
// The poison is a per-verb GENERATION, not a bit. A bitmap cannot see the dangerous case:
// a field filled by the previous DRAW and then read by the glTexSubImage that follows is
// stale, and its bit is already set. So every verb bumps CurrentVerbSerial, filling a
// field stamps it with that serial, and reading a non-sticky field whose stamp is older is
// Fatal{UnmigratedPipeInput} (section 6.2.2).
//
// PipeInputs itself is MG_Backend/MGPipe/PipeInputs.h (P1); the verb enum and the
// per-class fill masks are G5b, generated/PipeFillPoints.inc.
""")
    out.append("enum class MGPipeInputField : Uint16 {")
    for name, _ in accessors:
        out.append("    %s," % name)
    out.append("    kFieldCount,")
    out.append("};")
    out.append("")
    out.append("inline constexpr SizeT kMGPipeInputFieldCount = static_cast<SizeT>(MGPipeInputField::kFieldCount);")
    out.append("static_assert(kMGPipeInputFieldCount == %d, \"the PipeInputs field set moved\");" % len(accessors))
    out.append("")
    out.append("inline constexpr const char* kMGPipeInputFieldNames[kMGPipeInputFieldCount] = {")
    for name, _ in accessors:
        out.append("    \"%s\"," % name)
    out.append("};")
    out.append("")
    out.append("// Fields whose value is valid ACROSS verbs: a sticky field is a field the poison")
    out.append("// cannot protect, so every true is argued for in Coverage.def's")
    out.append("// MGP_COVERAGE_STICKY_LIST (the seven forwarded, argument-keyed accessors).")
    out.append("inline constexpr Bool kMGPipeInputFieldSticky[kMGPipeInputFieldCount] = {")
    for name, _ in accessors:
        if name in sticky_map:
            out.append("    true,  // %s: %s" % (name, sticky_map[name]))
        else:
            out.append("    false, // %s" % name)
    out.append("};")
    out.append("inline constexpr SizeT kMGPipeInputStickyFieldCount = %d;" % len(sticky))
    out.append("")
    out.append("// Which call is expected to have filled a field by the time a verb reads it. Names")
    out.append("// come from Coverage.def, so this table and the coverage table cannot disagree.")
    out.append("inline constexpr const char* kMGPipeInputFieldFilledBy[kMGPipeInputFieldCount] = {")
    for name, call in accessors:
        marker = "" if call in call_names else " // pseudo-call: not filled by a forward record"
        out.append("    \"%s\",%s" % (call, marker))
    out.append("};")
    out.append("")
    out.append("""struct MGPipeFilledState {
    Uint64 CurrentVerbSerial;
    Uint64 FilledGen[kMGPipeInputFieldCount];
};

[[noreturn]] inline void MGPipeInputPoisonFatal(MGPipeInputField field, const char* verb) {
    MGLOG_F("MGPipe: Fatal{UnmigratedPipeInput, \\"%s@%s\\"}",
            kMGPipeInputFieldNames[static_cast<SizeT>(field)], verb);
    std::abort();
}

inline Bool MGPipeInputFieldIsFresh(const MGPipeFilledState& state, MGPipeInputField field) {
    const SizeT index = static_cast<SizeT>(field);
    return kMGPipeInputFieldSticky[index] ? state.FilledGen[index] != 0
                                          : state.FilledGen[index] == state.CurrentVerbSerial;
}""")
    return "\n".join(out) + "\n"


FUNCTION_POINTER_MEMBER_RE = re.compile(r"\(\s*\*\s*(\w+)\s*\)\s*\(")


def parse_function_table():
    """The function-pointer members of MG_Backend::GLFunctionsTable, in declaration order.
    Data members (PrefersCpuXfbPrimitiveAccounting) are not verbs and are skipped; comments
    are masked line by line."""
    text = read(FUNCTION_TABLE_HEADER)
    start = text.find("struct GLFunctionsTable {")
    if start < 0:
        sys.exit("%s: struct GLFunctionsTable is missing" % FUNCTION_TABLE_HEADER)
    members = []
    for line in text[start:].splitlines()[1:]:
        if re.match(r"^\s*};", line):
            break
        code = line.split("//", 1)[0]
        for name in FUNCTION_POINTER_MEMBER_RE.findall(code):
            members.append(name)
    if not members:
        sys.exit("%s: GLFunctionsTable has no function-pointer members" % FUNCTION_TABLE_HEADER)
    return members


def parse_fill_points(accessors, table_members=None):
    """FillPoints.def -> (verbs, classes, fields): verbs is [(verb, class)] in file order,
    classes is [class], fields is {class: [field]}. Refuses a verb set that is not exactly
    GLFunctionsTable's function-pointer members in declaration order, a verb in two
    classes, a class with no verbs, a field that is not an accessor, a duplicate
    (class, field) row, and a class row that names no class."""
    text = read(os.path.join(PIPE_DIR, "FillPoints.def"))
    if table_members is None:
        table_members = parse_function_table()

    def block(macro):
        match = re.search(r"#define %s\(X\)(.*?)\n\n" % macro, text, re.S)
        if not match:
            sys.exit("FillPoints.def: %s is missing (or not followed by a blank line)" % macro)
        return match.group(1)

    classes = re.findall(r"X\((\w+)\)", block("MGP_FILL_CLASS_LIST"))
    if len(classes) != len(set(classes)):
        sys.exit("FillPoints.def: a class is listed twice in MGP_FILL_CLASS_LIST")
    verbs = re.findall(r"X\((\w+)\s*,\s*(\w+)\)", block("MGP_FILL_VERB_LIST"))
    seen = set()
    for verb, cls in verbs:
        if verb in seen:
            sys.exit("FillPoints.def: verb %s is in two classes" % verb)
        seen.add(verb)
        if cls not in classes:
            sys.exit("FillPoints.def: verb %s names unknown class %s" % (verb, cls))
    verb_names = [verb for verb, _ in verbs]
    missing = [m for m in table_members if m not in seen]
    if missing:
        sys.exit("FillPoints.def: GLFunctionsTable member(s) without a verb row: %s" % ", ".join(missing))
    extra = [v for v in verb_names if v not in table_members]
    if extra:
        sys.exit("FillPoints.def: verb(s) that are not GLFunctionsTable members: %s" % ", ".join(extra))
    if verb_names != table_members:
        sys.exit("FillPoints.def: verb rows are not in GLFunctionsTable declaration order "
                 "(first difference at %s)" % next(a for a, b in zip(verb_names, table_members) if a != b))
    for cls in classes:
        if not any(c == cls for _, c in verbs):
            sys.exit("FillPoints.def: class %s has no verbs" % cls)
    accessor_names = set(name for name, _ in accessors)
    fields = {cls: [] for cls in classes}
    for cls, field in re.findall(r"X\((\w+)\s*,\s*(\w+)\)", block("MGP_FILL_FIELD_LIST")):
        if cls not in fields:
            sys.exit("FillPoints.def: field row names unknown class %s" % cls)
        if field not in accessor_names:
            sys.exit("FillPoints.def: %s is not an accessor in Coverage.def" % field)
        if field in fields[cls]:
            sys.exit("FillPoints.def: duplicate row (%s, %s)" % (cls, field))
        fields[cls].append(field)
    return verbs, classes, fields


def gen_fill_points(accessors, sticky, verbs, classes, fields):
    field_index = {name: i for i, (name, _) in enumerate(accessors)}
    # Two words minimum (the P1 contract shape, headroom for the 64th field); grows on demand.
    words = max(2, (len(accessors) + 63) // 64)
    sticky_names = [name for name, _ in sticky]
    out = [banner("PipeFillPoints.inc", "G5b: the verb enum, the verb classes and their may-read field masks.",
                  "FillPoints.def, Coverage.def and MG_Backend/BackendObject.h")]
    out.append("""// One verb per function-pointer member of MG_Backend::GLFunctionsTable, in declaration
// order, so the enum IS the table's member list. MG_Impl spells MGP_FILL(Verb) before every
// call through the table; MGPipeFillForVerb fills exactly the fields of the verb's class
// (plus the sticky fields, OR'ed into every mask) and stamps them with the new serial. A
// read of any other field is Fatal{UnmigratedPipeInput, \"Field@Verb\"} in a poison build.
""")
    out.append("enum class MGPipeVerb : Uint8 {")
    for verb, _ in verbs:
        out.append("    %s," % verb)
    out.append("    kVerbCount,")
    out.append("};")
    out.append("")
    out.append("inline constexpr SizeT kMGPipeVerbCount = static_cast<SizeT>(MGPipeVerb::kVerbCount);")
    out.append("static_assert(kMGPipeVerbCount == %d, \"the GLFunctionsTable verb set moved\");" % len(verbs))
    out.append("")
    out.append("inline constexpr const char* kMGPipeVerbNames[kMGPipeVerbCount] = {")
    for verb, _ in verbs:
        out.append("    \"%s\"," % verb)
    out.append("};")
    out.append("")
    out.append("enum class MGPipeVerbClass : Uint8 {")
    for cls in classes:
        out.append("    %s," % cls)
    out.append("    kClassCount,")
    out.append("};")
    out.append("")
    out.append("inline constexpr SizeT kMGPipeVerbClassCount = static_cast<SizeT>(MGPipeVerbClass::kClassCount);")
    out.append("static_assert(kMGPipeVerbClassCount == %d, \"the verb class set moved\");" % len(classes))
    out.append("")
    out.append("inline constexpr const char* kMGPipeVerbClassNames[kMGPipeVerbClassCount] = {")
    for cls in classes:
        out.append("    \"%s\"," % cls)
    out.append("};")
    out.append("")
    out.append("inline constexpr MGPipeVerbClass kMGPipeVerbClass[kMGPipeVerbCount] = {")
    for verb, cls in verbs:
        out.append("    MGPipeVerbClass::%s, // %s" % (cls, verb))
    out.append("};")
    out.append("")
    out.append("// One bit per MGPipeInputField. The %d sticky fields are OR'ed into every class." % len(sticky))
    out.append("struct MGPipeFieldMask {")
    out.append("    Uint64 Words[%d];" % words)
    out.append("};")
    out.append("")
    out.append("inline constexpr Bool MGPipeFieldMaskHas(const MGPipeFieldMask& mask, MGPipeInputField field) {")
    out.append("    const SizeT index = static_cast<SizeT>(field);")
    out.append("    return (mask.Words[index / 64] >> (index % 64)) & 1u;")
    out.append("}")
    out.append("")
    out.append("inline constexpr MGPipeFieldMask kMGPipeClassFieldMask[kMGPipeVerbClassCount] = {")
    for cls in classes:
        bits = [0] * words
        names = fields[cls] + [n for n in sticky_names if n not in fields[cls]]
        for name in names:
            index = field_index[name]
            bits[index // 64] |= 1 << (index % 64)
        out.append("    // %s: %d fields (%d own + %d sticky)" % (cls, len(names), len(fields[cls]), len(names) - len(fields[cls])))
        out.append("    {{%s}}," % ", ".join("0x%016xull" % b for b in bits))
    out.append("};")
    out.append("")
    out.append("static_assert(kMGPipeInputFieldCount <= %d * 64, \"MGPipeFieldMask needs another word\");" % words)
    return "\n".join(out) + "\n"


def gen_coverage(accessors, deltas, rows, calls):
    call_names = set(c.Name for c in calls)
    pseudo = {"kClientResolved", "kReverseChannel", "kStructuralHandle"}
    accessor_map = dict(accessors)
    delta_map = dict(deltas)
    for _, call in accessors + deltas:
        if call not in call_names and call not in pseudo:
            sys.exit("Coverage.def: %s is not a call in PipeCalls.def and not a pseudo-call" % call)

    mapped = 0
    by_pseudo = {name: 0 for name in pseudo}
    unmapped_rows = []
    per_accessor = {}
    for row in rows:
        call = None
        if row["member"] and row["member"] != "-":
            call = accessor_map.get(row["member"])
        if call is None:
            call = delta_map.get(row["delta"])
        if call is None:
            unmapped_rows.append(row)
            continue
        if call in pseudo:
            by_pseudo[call] += 1
        else:
            mapped += 1
        key = row["member"] if row["member"] and row["member"] != "-" else row["delta"]
        per_accessor.setdefault(key, [call, 0])[1] += 1

    out = [banner("PipeCoverage.inc", "G6: backend read inventory -> MGPipe call coverage.",
                  "Coverage.def and scripts/data/backend_read_inventory.md")]
    out.append("""// The acceptance rule (plan B section 10.3-5): regenerate, `git diff --exit-code`, and
// ZERO unmapped rows. P0 permits unmapped rows and only counts them; the count below is
// the number the later gate has to drive to zero.
//
// Three pseudo-calls stand for read points that never become a forward record:
// kClientResolved (the frontend answers it), kReverseChannel (it becomes one of the ten
// MGPipeCallbacks) and kStructuralHandle (the row is a signature carrying a
// SharedPtr<MG_State...> that becomes an MGPipeHandle parameter).
""")
    out.append("struct MGPipeCoverageEntry {")
    out.append("    const char* Accessor;")
    out.append("    const char* Call;")
    out.append("    Uint32 ReadPoints;")
    out.append("};")
    out.append("")
    out.append("inline constexpr MGPipeCoverageEntry kMGPipeCoverage[] = {")
    for key in sorted(per_accessor):
        call, count = per_accessor[key]
        out.append("    {\"%s\", \"%s\", %d}," % (key, call, count))
    out.append("};")
    out.append("")
    out.append("inline constexpr SizeT kMGPipeCoverageEntryCount = %d;" % len(per_accessor))
    out.append("inline constexpr Uint32 kMGPipeInventoryReadPoints = %d;" % len(rows))
    out.append("inline constexpr Uint32 kMGPipeInventoryMappedToCall = %d;" % mapped)
    out.append("inline constexpr Uint32 kMGPipeInventoryClientResolved = %d;" % by_pseudo["kClientResolved"])
    out.append("inline constexpr Uint32 kMGPipeInventoryReverseChannel = %d;" % by_pseudo["kReverseChannel"])
    out.append("inline constexpr Uint32 kMGPipeInventoryStructuralHandle = %d;"
               % by_pseudo["kStructuralHandle"])
    out.append("inline constexpr Uint32 kMGPipeInventoryUnmapped = %d;" % len(unmapped_rows))
    out.append("static_assert(kMGPipeCoverageEntryCount == sizeof(kMGPipeCoverage) / sizeof(kMGPipeCoverage[0]));")
    out.append("static_assert(kMGPipeInventoryMappedToCall + kMGPipeInventoryClientResolved +")
    out.append("                  kMGPipeInventoryReverseChannel + kMGPipeInventoryStructuralHandle +")
    out.append("                  kMGPipeInventoryUnmapped ==")
    out.append("              kMGPipeInventoryReadPoints,")
    out.append("              \"every inventory row must land in exactly one bucket\");")
    return "\n".join(out) + "\n", unmapped_rows, mapped, by_pseudo


def gen_span_table():
    out = [banner("PipeSpanTable.inc", "G7: the render-state pipeline subset, by member name.",
                  "the field list in scripts/gen_pipe.py")]
    out.append("""// D-B1 rejected three CSOs and demanded this table instead, so the table needs its own
// completeness trip wire: MG_Test walks every public RenderState setter and asserts that
// the pipeline-subset hash moves IF AND ONLY IF m_pipelineStateVersion moves. That test
// and MGPipeRenderStateSpans.cpp land with P2; what P0 pins is the MEMBER LIST, taken from
// what VulkanRenderer::ComputePipelineStateHash hashes today, so the later offsets are
// derived from a list that was reviewed rather than invented.
//
// Deliberately absent, and each absence is a question P2 has to answer before the chunk
// table freezes:
//   - FramebufferSrgb and DepthClamp have NO STORAGE at all (RenderState.cpp's SetCapability
//     falls to "not supported currently" and IsCapabilityEnabled returns false), so six
//     backend read points are constant false today. Pipeline state or dead capability?
//   - ProvokingVertexModeSetting is Vulkan pipeline state but is not hashed today.
//   - FrontFaceModeSetting, ClipOrigin and ClipDepthMode are pipeline state on Vulkan and
//     are handled elsewhere in the payload path rather than in the memo word.
//
// The complement of this list is the DYNAMIC subset - the half whose whole purpose is that
// glViewport must not mint a new CSO.

inline constexpr const char* const kMGPipePipelineStateMembers[] = {""")
    for member in PIPELINE_STATE_MEMBERS:
        out.append("    \"%s\"," % member)
    out.append("};")
    out.append("inline constexpr SizeT kMGPipePipelineStateMemberCount = %d;" % len(PIPELINE_STATE_MEMBERS))
    out.append("static_assert(kMGPipePipelineStateMemberCount ==")
    out.append("                  sizeof(kMGPipePipelineStateMembers) / sizeof(kMGPipePipelineStateMembers[0]));")
    out.append("")
    out.append("// Filled in by MG_Pipe/MGPipeRenderStateSpans.cpp (P2), which computes the offsets")
    out.append("// in C++ with offsetof rather than guessing them in python.")
    out.append("extern const MGPStateChunk kMGPipePipelineChunks[];")
    out.append("extern const MGPStateChunk kMGPipeDynamicChunks[];")
    return "\n".join(out) + "\n"


def write(path, text, check, changed):
    existing = read(path) if os.path.exists(path) else None
    if existing == text:
        return
    changed.append(os.path.relpath(path, REPO_ROOT))
    if not check:
        with open(path, "w", encoding="utf-8", newline="\n") as handle:
            handle.write(text)


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--check", action="store_true",
                        help="do not write; exit 1 if regenerating would change anything")
    args = parser.parse_args()

    calls = parse_calls()
    payloads = parse_verify_payloads()
    check_call_payloads_have_field_lists(calls, payloads)
    accessors, deltas, sticky = parse_coverage()
    verbs, classes, fields = parse_fill_points(accessors)
    rows = parse_inventory()

    if not os.path.isdir(GENERATED_DIR):
        os.makedirs(GENERATED_DIR)

    coverage_text, unmapped, mapped, pseudo = gen_coverage(accessors, deltas, rows, calls)
    changed = []
    write(os.path.join(GENERATED_DIR, "PipeTables.inc"), gen_tables(calls), args.check, changed)
    write(os.path.join(GENERATED_DIR, "PipeThunks.inc"), gen_thunks(calls), args.check, changed)
    write(os.path.join(GENERATED_DIR, "PipeWire.inc"), gen_wire(calls), args.check, changed)
    write(os.path.join(GENERATED_DIR, "PipeVerify.inc"), gen_verify(payloads), args.check, changed)
    write(os.path.join(GENERATED_DIR, "PipeFilled.inc"), gen_filled(accessors, calls, sticky), args.check, changed)
    write(os.path.join(GENERATED_DIR, "PipeFillPoints.inc"),
          gen_fill_points(accessors, sticky, verbs, classes, fields), args.check, changed)
    write(os.path.join(GENERATED_DIR, "PipeCoverage.inc"), coverage_text, args.check, changed)
    write(os.path.join(GENERATED_DIR, "PipeSpanTable.inc"), gen_span_table(), args.check, changed)

    screen = sum(1 for c in calls if c.IsScreen)
    print("gen_pipe: %d calls (%d screen, %d context), %d verify payloads, %d PipeInputs fields "
          "(%d sticky), %d verbs, %d classes"
          % (len(calls), screen, len(calls) - screen, len(payloads), len(accessors), len(sticky),
             len(verbs), len(classes)))
    print("gen_pipe: inventory %d rows: %d -> call, %d client-resolved, %d reverse-channel, "
          "%d structural handle, %d UNMAPPED"
          % (len(rows), mapped, pseudo["kClientResolved"], pseudo["kReverseChannel"],
             pseudo["kStructuralHandle"], len(unmapped)))
    for row in unmapped:
        print("gen_pipe:   UNMAPPED %s:%d %s %s" % (row["file"], row["line"], row["kind"], row["member"]))

    if changed:
        if args.check:
            print("gen_pipe: OUT OF DATE: %s" % ", ".join(changed), file=sys.stderr)
            return 1
        print("gen_pipe: wrote %s" % ", ".join(changed))
    else:
        print("gen_pipe: generated files are up to date")
    return 0


if __name__ == "__main__":
    sys.exit(main())