firestorm/MW4COMPARE/tools/decompile/verify_subsystems.py

#!/usr/bin/env python3
"""Verification harness for the .subsystems decompiler.

Rebuilds every chassis from our own packed records and compares the resulting
key/value pairs, page by page, against the known source .subsystems.

Page names are ignored - the packer does not store them (see subsystems.py), so
only order and content can be verified.

Chassis whose source has moved on since the package was built are reported
separately rather than counted as failures; core.mw4 has not been repacked since
the initial mirror, so battlemaster and battlemaster2c legitimately differ.

    python3 verify_subsystems.py
"""
import sys, os, glob, re, collections

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import subsystems


def source_pages(path):
    """Parse a source .subsystems leniently.

    The runtime NotationFile accepts a page header with no closing bracket -
    hellspawn had '[HeatSink10' and sunder '[HeatSink16' - so match on a leading
    '[' rather than a full bracketed pattern.

    GroupIndex may appear several times in one page, so values are collected as
    lists rather than overwritten.
    """
    txt = open(path, "rb").read().decode("latin-1")
    out = []
    for m in re.finditer(r'^\[([^\r\n\]]*)\]?[ \t]*\r?\n(.*?)(?=^\[|\Z)', txt, re.M | re.S):
        kv = collections.OrderedDict()
        for k, v in re.findall(r'^([A-Za-z0-9_]+)=([^\r\n]*)', m.group(2), re.M):
            if k in kv:
                kv[k] = (kv[k] if isinstance(kv[k], list) else [kv[k]]) + [v]
            else:
                kv[k] = v
        out.append((m.group(1), kv))
    return out


def norm(key, value):
    """Compare as a sorted list so a single value and a one-element list match."""
    if not isinstance(value, list):
        value = [value]
    return sorted(_norm1(key, v) for v in value)


def _norm1(key, value):
    v = str(value).strip()
    if key in ("Model", "Site", "EjectSite", "InternalLocation", "ExecutionState",
               "ArmorType", "InternalType"):
        return v.lower().replace("\\", "/")
    try:
        return f"{float(v):.4g}"
    except ValueError:
        return v.lower()


def main():
    manifest = subsystems.load_manifest(subsystems.OUR_MANIFEST)
    ppt = subsystems.armor_points_per_ton()
    srcs = {os.path.basename(p)[:-len(".subsystems")].lower(): p
            for p in glob.glob(subsystems.OUR_MECH_SOURCE + "/*/*.subsystems")}

    t = collections.Counter()
    mismatched_keys = collections.Counter()
    stale, examples = [], []

    for rec in sorted(glob.glob(subsystems.OUR_RECORDS + "/*/*.subsystems")):
        chassis = os.path.basename(rec)[:-len(".subsystems")].lower()
        if chassis not in srcs:
            continue
        src = source_pages(srcs[chassis])
        got = subsystems.decompile(rec, manifest, ppt)
        if len(src) != len(got):
            stale.append((chassis, len(got), len(src)))
            continue
        t["chassis"] += 1
        for (_sname, skv), (_gname, gkv) in zip(src, got):
            t["pages"] += 1
            page_ok = True
            for key, sval in skv.items():
                if key in ("SubsystemIndex",):          # not emitted; defaults to 0
                    continue
                # An explicit "=0" and an omitted key are identical to the packer.
                if gkv.get(key) is None and str(sval).strip() in ("0", "0.0"):
                    continue
                t["keys"] += 1
                gval = gkv.get(key)
                if gval is not None and norm(key, gval) == norm(key, sval):
                    t["keys_ok"] += 1
                else:
                    page_ok = False
                    mismatched_keys[key] += 1
                    if len(examples) < 12:
                        examples.append((chassis, key, sval, gval))
            extra = [k for k in gkv if k not in skv]
            if extra:
                page_ok = False
                for k in extra:
                    mismatched_keys["EXTRA:" + k] += 1
            t["pages_ok"] += page_ok

    print(f"chassis verified : {t['chassis']}")
    print(f"pages compared   : {t['pages']}   fully exact: {t['pages_ok']}")
    print(f"keys compared    : {t['keys']}   exact: {t['keys_ok']}"
          f"   wrong: {t['keys'] - t['keys_ok']}")
    if mismatched_keys:
        print("\nmismatches by key:")
        for k, v in mismatched_keys.most_common(12):
            print(f"   {k:22s} x{v}")
    if examples:
        print("\nexamples (chassis, key, source, decompiled):")
        for e in examples:
            print(f"   {e[0]:14s} {e[1]:18s} src={e[2]!r:28s} got={e[3]!r}")
    if stale:
        print("\nskipped - source has moved on since the package was built:")
        for ch, g, s in stale:
            print(f"   {ch:16s} messages={g} sourcePages={s}")


if __name__ == "__main__":
    main()