firestorm/MW4COMPARE/tools/decompile/verify_contents.py

#!/usr/bin/env python3
"""Round-trip verifier for the .contents decompiler.

Regenerates each `.contents` and compares page names and both key values against
the authored source. Page order is not compared -- NotationFile is
order-independent, and the packer groups pages by parent joint rather than
preserving the authored sequence.

    python3 verify_contents.py [--show CHASSIS]
"""
import argparse, collections, glob, os, re, sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import contents, subsystems

REC = "/home/rich/Repositories/FS_Ours_extracted/core/mechs"
SRC = "/home/rich/Repositories/firestorm/Gameleap/mw4/Content/Mechs"


def source_pages(path):
    txt = open(path, "rb").read().decode("latin-1")
    txt = re.sub(r'//[^\n]*', '', txt)
    out = {}
    for m in re.finditer(r'^\[([^\]]+)\]\r?\n(.*?)(?=^\[|\Z)', txt, re.M | re.S):
        name, body = m.group(1), m.group(2)
        if name.lower() == "includes":
            continue
        kv = dict(re.findall(r'^([A-Za-z_]\w*)=([^\r\n]*)', body, re.M))
        out[name.lower()] = {k.lower(): v.strip().lower().replace("/", "\\")
                             for k, v in kv.items()}
    return out


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--show")
    args = ap.parse_args()

    manifest = subsystems.load_manifest(contents.MANIFEST)
    t = collections.Counter()
    bad = collections.Counter()
    examples = []
    missing_pages = collections.Counter()
    extra_pages = collections.Counter()

    for d in sorted(glob.glob(REC + "/*")):
        ch = os.path.basename(d)
        src = [p for p in glob.glob(SRC + "/*/*.contents")
               if os.path.basename(p).lower() == ch.lower() + ".contents"]
        if not src:
            continue
        t["chassis"] += 1
        want = source_pages(src[0])
        chassis, pages = contents.decompile(d, manifest)
        got = {n.lower(): {k.lower(): (v or "").lower().replace("/", "\\")
                           for k, v in kv} for n, kv in pages}

        if args.show and args.show.lower() == ch.lower():
            sys.stdout.write(contents.emit(chassis, pages))
            return

        for name in want:
            if name not in got:
                missing_pages[name] += 1
        for name in got:
            if name not in want:
                extra_pages[name] += 1
        for name in set(want) & set(got):
            t["pages"] += 1
            for key in ("model", "executionstate"):
                t["keys"] += 1
                if want[name].get(key) == got[name].get(key):
                    t["ok"] += 1
                else:
                    bad[key] += 1
                    if len(examples) < 10:
                        examples.append((ch, name, key,
                                         want[name].get(key), got[name].get(key)))

    print(f"chassis        : {t['chassis']}")
    print(f"pages compared : {t['pages']}")
    print(f"keys compared  : {t['keys']}   exact: {t['ok']}   wrong: {t['keys'] - t['ok']}")
    if missing_pages:
        print(f"\npages in source but not decoded: {sum(missing_pages.values())}")
        for n, c in missing_pages.most_common(10):
            print(f"   {n:28s} x{c}")
    if extra_pages:
        print(f"\npages decoded but not in source: {sum(extra_pages.values())}")
        for n, c in extra_pages.most_common(10):
            print(f"   {n:28s} x{c}")
    if bad:
        print("\nvalue mismatches:")
        for k, c in bad.most_common():
            print(f"   {k:20s} x{c}")
        for e in examples:
            print(f"   {e[0]:14s} {e[1]:24s} {e[2]:16s} want={e[3]!r} got={e[4]!r}")


if __name__ == "__main__":
    main()