firestorm/MW4COMPARE/tools/classify-survivors.py

#!/usr/bin/env python3
"""
classify-survivors.py - split a pruned extracted tree into NEW vs DIFFERS.

    python3 classify-survivors.py <prunedExtractedDir>

Run after prune-identical.py. Everything still present is something we do not
have byte-for-byte; this reports which of two reasons applies:

    NEW      no file at that entry path exists on our side at all
    DIFFERS  we have that entry path, but the bytes differ

"Our side" means our extracted packages OR our Content/ source tree, matched
case-insensitively - the same notion of "have it" the pruner uses.

Writes _classified.tsv next to the tree and prints a summary.
"""
import sys, os, collections

OURS_EXTRACTED = "/home/rich/Repositories/FS_Ours_extracted"
OUR_SOURCE = "/home/rich/Repositories/firestorm/Gameleap/mw4/Content"


def index_paths(root, skip_top=()):
    out = {}
    for dp, dirs, fs in os.walk(root):
        if dp == root:
            dirs[:] = [d for d in dirs if d.lower() not in skip_top]
        for f in fs:
            p = os.path.join(dp, f)
            out[os.path.relpath(p, root).replace("\\", "/").lower()] = p
    return out


def package_of(rel):
    parts = rel.split("/")
    head = parts[0].lower()
    if head in ("core", "props", "textures"):
        return parts[0], "/".join(parts[1:])
    if head in ("maps", "missions", "variants") and len(parts) > 2:
        return "/".join(parts[:2]), "/".join(parts[2:])
    if head == "pilots" and len(parts) > 3:
        return "/".join(parts[:3]), "/".join(parts[3:])
    return parts[0], "/".join(parts[1:])


def main():
    if len(sys.argv) < 2:
        sys.exit(__doc__)
    root = sys.argv[1]
    merged = os.path.join(root, "_merged")

    ours = index_paths(OURS_EXTRACTED, skip_top={"_merged"})
    ours_entries = set()
    for rel in ours:
        ours_entries.add(package_of(rel)[1])
    src_entries = set(index_paths(OUR_SOURCE))

    rows, summary = [], collections.Counter()
    for dp, _dirs, fs in os.walk(root):
        if dp == merged or dp.startswith(merged + os.sep):
            continue
        for f in fs:
            rel = os.path.relpath(os.path.join(dp, f), root).replace("\\", "/")
            if rel.startswith("_"):
                continue
            pkg, entry = package_of(rel)
            e = entry.lower()
            verdict = "DIFFERS" if (rel.lower() in ours or e in ours_entries
                                    or e in src_entries) else "NEW"
            rows.append((verdict, rel, pkg, entry))
            summary[(verdict, pkg.split("/")[0])] += 1

    with open(os.path.join(root, "_classified.tsv"), "w", encoding="utf-8") as fh:
        fh.write("verdict\tpath\tpackage\tentry\n")
        for r in sorted(rows):
            fh.write("\t".join(r) + "\n")

    tot = collections.Counter(v for v, _, _, _ in rows)
    print(f"survivors: {len(rows)}   NEW={tot['NEW']}   DIFFERS={tot['DIFFERS']}\n")
    print(f"{'package group':16s} {'NEW':>7s} {'DIFFERS':>8s}")
    groups = sorted({g for _v, g in summary})
    for g in groups:
        print(f"{g:16s} {summary[('NEW', g)]:7d} {summary[('DIFFERS', g)]:8d}")
    print(f"\n-> {os.path.join(root, '_classified.tsv')}")


if __name__ == "__main__":
    main()