firestorm/MW4COMPARE/tools/split-source.py

#!/usr/bin/env python3
"""
split-source.py - separate repackable source files from compiled records, and
rename the ones that need it, so the tree can be dropped into Content/.

    python3 split-source.py <extractedDir> [--apply]

Dry-run by default.

The problem
-----------
A .mw4 does not store the source tree. Some records are the source file byte for
byte; others are what the packer produced from it. Classification was derived
empirically by extracting OUR OWN packages and hash-matching every record against
Gameleap/mw4/Content (22,138 matched, 28,728 did not):

  verbatim source   .tga .erf .mw4anim .bid .wav .bsp .material .abl .obb .ebf
                    .bounds .animscript .fgd .tcf .mlr .d3f .gaf .script .h .abi
  compiled          .data .instance .subsystems .damage .torso .engine .audio
                    .video .lights .mw4, and every {hint} {handle} {gamemodel}
                    {element} {footsteps} {zones} {nametable} [shadow]
                    [joint_*]{armature} [joint_*]{sites} record

Worth being concrete: our source Content/Mechs/Atlas/atlas.data is 4,891 bytes of
text; the record packed under that name is a 12-byte binary stub. The real
payload went into the {gamemodel}/{element} records. So a mech's .data .damage
.subsystems .instance CANNOT be recovered from a package - they have to be
re-authored.

.armature likewise does not exist in a package. The packer splits it into
per-joint X.contents[joint_*]{armature} records, which are compiled.

What this does
--------------
  * X.data{solidobb}        -> X_skeleton_SOLID.obb   (mechs)
    X.data{hierarchicalobb} -> X_skeleton.obb         (mechs)
    ...or _SOLID.obb / .obb outside Content/Mechs. These really are .obb files
    ('#BBO' magic, headers byte-identical to ours). The exact source spelling is
    declared inside the .data text, which we do not have, so a convention is
    applied and recorded in _source-vs-compiled.tsv - whatever .data gets
    authored later must reference the same name.
  * everything classified as compiled moves to _compiled/, keeping its path
  * everything left under Content/ is genuine, repackable source

Resource/ (variants, pilot options) is left alone: those are whole .mw4
packages, best taken from FS_Build_V4H/resource/Variants/*.mw4 directly rather
than from their unpacked records.
"""
import sys, os, re, collections

SOURCE_EXT = {".tga", ".erf", ".mw4anim", ".bid", ".wav", ".bsp", ".material",
              ".abl", ".obb", ".ebf", ".bounds", ".animscript", ".fgd", ".tcf",
              ".mlr", ".d3f", ".gaf", ".h", ".abi", ".hpp", ".include", ".tpl"}
COMPILED_EXT = {".data", ".instance", ".subsystems", ".damage", ".torso",
                ".engine", ".audio", ".video", ".lights", ".mw4"}
OBB_QUALS = {"{solidobb}": ("_skeleton_SOLID.obb", "_SOLID.obb"),
             "{hierarchicalobb}": ("_skeleton.obb", ".obb")}

QUAL = re.compile(r'(\[[^\]]*\]|\{[^}]*\})')


def is_text(path, limit=8192):
    with open(path, "rb") as fh:
        chunk = fh.read(limit)
    if not chunk:
        return False
    printable = sum(1 for b in chunk if 9 <= b <= 13 or 32 <= b < 127)
    return printable / len(chunk) > 0.90


def classify(path, name, rel):
    quals = "".join(QUAL.findall(name)).lower()
    stem = QUAL.sub("", name)
    ext = os.path.splitext(stem)[1].lower()

    if quals in OBB_QUALS:
        mech = "/mechs/" in rel.lower()
        suffix = OBB_QUALS[quals][0 if mech else 1]
        return "source", stem.rsplit(".", 1)[0] + suffix
    if quals:
        return "compiled", name
    if ext in SOURCE_EXT:
        return "source", name
    if ext in COMPILED_EXT:
        return "compiled", name
    # .script and .contents are genuinely mixed - decide on content
    return ("source" if is_text(path) else "compiled"), name


def main():
    if len(sys.argv) < 2:
        sys.exit(__doc__)
    root = sys.argv[1]
    apply_ = "--apply" in sys.argv
    content = os.path.join(root, "Content")
    if not os.path.isdir(content):
        sys.exit(f"no Content/ in {root} - run restructure.py first")

    plan, stats = [], collections.Counter()
    renames = collections.Counter()
    for dp, _dirs, fs in os.walk(content):
        for f in fs:
            p = os.path.join(dp, f)
            rel = os.path.relpath(p, root).replace("\\", "/")
            verdict, newname = classify(p, f, rel)
            new = ("Content/" if verdict == "source" else "_compiled/Content/") + \
                  os.path.relpath(os.path.join(dp, newname), content).replace("\\", "/")
            plan.append((rel, new, verdict))
            stats[verdict] += 1
            if verdict == "source" and newname != f:
                renames[os.path.splitext(f)[1] or f] += 1

    print(f"Content/: {sum(stats.values())} files")
    print(f"  repackable source : {stats['source']}")
    print(f"  compiled records  : {stats['compiled']}  (-> _compiled/)")
    if renames:
        print("\nrenamed:")
        for k, v in renames.most_common():
            print(f"  {v:5d}  {k}")
    print("\nsample renames:")
    shown = 0
    for old, new, v in plan:
        if v == "source" and os.path.basename(old) != os.path.basename(new) and shown < 6:
            print(f"  {os.path.basename(old)}  ->  {os.path.basename(new)}")
            shown += 1

    if not apply_:
        print("\nDRY RUN - nothing changed. Re-run with --apply.")
        return

    for old, new, _v in plan:
        src, dest = os.path.join(root, old), os.path.join(root, new)
        os.makedirs(os.path.dirname(dest), exist_ok=True)
        os.replace(src, dest)

    with open(os.path.join(root, "_source-vs-compiled.tsv"), "w", encoding="utf-8") as fh:
        fh.write("verdict\toriginalPath\tnewPath\n")
        for old, new, v in sorted(plan):
            fh.write(f"{v}\t{old}\t{new}\n")

    removed = 0
    for dp, _dirs, _fs in os.walk(root, topdown=False):
        if dp == root or not os.path.isdir(dp):
            continue
        if not os.listdir(dp):
            os.rmdir(dp)
            removed += 1
    print(f"\nmoved {len(plan)} files, removed {removed} empty directories")
    print(f"log: {os.path.join(root, '_source-vs-compiled.tsv')}")


if __name__ == "__main__":
    main()