firestorm/MW4COMPARE/tools/decompile/verify_roundtrip.py

#!/usr/bin/env python3
"""Round-trip verifier for the mech .data decompiler.

Regenerates a whole `.data` from the compiled records for every chassis we hold
both forms of, and compares it key by key against the authored source. This is
the standard `.armature` and `.subsystems` were held to; nothing should be
emitted for the new chassis until this passes.

Comparison is semantic, not byte-exact: NotationFile is order-independent, and
Windows path lookup is case-insensitive (our own tree writes both
`mechs\\atlas_destroyed\\...` and `Mechs\\Atlas_Destroyed\\...` for the same
key), so keys are matched case-insensitively and numbers within tolerance.

    python3 verify_roundtrip.py [--show CHASSIS]
"""
import argparse, collections, os, re, sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import datamap, data, subsystems

REC = "/home/rich/Repositories/FS_Ours_extracted/core/mechs"
NUM = re.compile(r'^-?\d+\.?\d*(?:[eE][-+]?\d+)?$')


def canon(v):
    """Normalise one value for comparison."""
    v = str(v).strip()
    v = re.sub(r'^content[\\/]', '', v.replace("/", "\\"), flags=re.I)
    v = re.sub(r'\s+', " ", v).lower()
    # normalise every number in place so "20.0 20.0 20.0" == "20 20 20"
    return re.sub(r'-?\d+\.\d+(?:[eE][-+]?\d+)?|-?\d+',
                  lambda m: f"{float(m.group(0)):.4g}", v)


def canon_set(val):
    """Sorted, de-duplicated: some sources repeat a key with the same value."""
    return sorted({canon(v) for v in (val if isinstance(val, list) else [val])})


def source_kv(path):
    """Authored [GameData] page, keeping repeated keys as lists."""
    txt = open(path, "rb").read().decode("latin-1")
    # protect braced blocks first: Shadow={...} contains a line reading
    # "[shadow]", which otherwise looks like the start of the next page.
    # Both CR and LF must go -- splitlines() splits on a bare CR too.
    txt = re.sub(r'\{.*?\}',
                 lambda x: x.group(0).replace("\r", "").replace("\n", "\x01"),
                 txt, flags=re.S)
    m = re.search(r'^\[GameData\]\r?\n(.*?)(?=^\[[A-Za-z]|\Z)', txt, re.M | re.S)
    body = m.group(1) if m else ""
    kv = collections.OrderedDict()
    for line in body.splitlines():
        if "=" not in line or line.lstrip().startswith("//"):
            continue
        k, v = line.split("=", 1)
        kv.setdefault(k.strip(), []).append(v.replace("\x01", "\n").strip())
    return kv


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--show")
    args = ap.parse_args()

    manifest = subsystems.load_manifest(data.MANIFEST)
    totals = collections.Counter()
    wrong = collections.Counter()
    missing = collections.Counter()
    extra = collections.Counter()
    examples = []
    chassis = 0

    for ch, kv_ignored, _blob in datamap.corpus():
        src_path = [p for p in
                    __import__("glob").glob(datamap.SRC + "/*/*.data")
                    if os.path.basename(p).lower() == ch.lower() + ".data"]
        if not src_path:
            continue
        chassis += 1
        want = source_kv(src_path[0])
        got = data.decompile(os.path.join(REC, ch), manifest)
        got_l = {k.lower(): v for k, v in got.items()}

        for key, wvals in want.items():
            if key in data.UNRECOVERABLE:
                totals["omitted"] += 1
                continue
            totals["keys"] += 1
            if key.lower() not in got_l:
                missing[key] += 1
                continue
            if canon_set(got_l[key.lower()]) == canon_set(wvals):
                totals["ok"] += 1
            else:
                wrong[key] += 1
                if len(examples) < 12:
                    examples.append((ch, key, wvals, got_l[key.lower()]))
        for key in got:
            if key.lower() not in {k.lower() for k in want}:
                extra[key] += 1

        if args.show and args.show.lower() == ch.lower():
            sys.stdout.write(data.emit(got))
            return

    print(f"chassis            : {chassis}")
    print(f"keys compared      : {totals['keys']}   exact: {totals['ok']}   "
          f"wrong: {sum(wrong.values())}   missing: {sum(missing.values())}")
    print(f"deliberately omitted: {totals['omitted']} (unreadable by the engine)")
    if missing:
        print("\nmissing from output:")
        for k, n in missing.most_common(15):
            print(f"   {k:30s} x{n}")
    if wrong:
        print("\nvalue mismatches:")
        for k, n in wrong.most_common(15):
            print(f"   {k:30s} x{n}")
        print("\nexamples:")
        for ch, k, w, g in examples:
            print(f"   {ch:12s} {k:26s} want={w!r:44s} got={g!r}")
    if extra:
        print("\nemitted but not in source:")
        for k, n in extra.most_common(10):
            print(f"   {k:30s} x{n}")


if __name__ == "__main__":
    main()