firestorm/MW4COMPARE/tools/restructure.py

#!/usr/bin/env python3
"""
restructure.py - reshape an extracted tree to match the repo's directory layout.

    python3 restructure.py <extractedDir> [--apply]

Dry-run by default.

The extractor writes one directory per package, which is lossless but does not
look like the repo. This flattens it to the shape of Gameleap/mw4/Content:

    <root>/Content/Mechs/Champion/champion.subsystems
    <root>/Content/Maps/alpine02/...
    <root>/Resource/Variants/<name>/...        (variant records have no path)
    <root>/Resource/Pilots/Tesla/options/...
    <root>/_pkgroots/...                       (4-byte package-root pseudo-entries)

Path components are re-cased to match the repo wherever a counterpart exists, so
'mechs/longbow/longbow.data{element}' becomes
'Content/Mechs/Longbow/longbow.data{Element}'. Components with no counterpart
(the new chassis, for instance) keep the name the packer used.

REFUSES TO RUN if two packages would land on the same path with different bytes.
Flatten only trees where that has been checked - the pruned V4H tree has zero
collisions; our own full extraction has 60 (mission-local copies of shared
Culturals props) and must stay per-package.

Provenance is preserved in _layout.tsv: newPath, package, original entry path.
"""
import sys, os, hashlib, collections

REPO_CONTENT = "/home/rich/Repositories/firestorm/Gameleap/mw4/Content"


def build_case_map(root):
    """lowercased relative path -> the repo's actual spelling, for dirs and files."""
    out = {}
    for dp, dirs, files in os.walk(root):
        rel = os.path.relpath(dp, root).replace("\\", "/")
        base = "" if rel == "." else rel
        for name in dirs + files:
            r = f"{base}/{name}" if base else name
            out[r.lower()] = r
    return out


def recase(path, case_map):
    """Re-case each component against the repo, keeping unknown ones as-is."""
    done = []
    for part in path.split("/"):
        probe = "/".join(done + [part]).lower()
        canonical = case_map.get(probe)
        done.append(canonical.rsplit("/", 1)[-1] if canonical else part)
    return "/".join(done)


def package_of(rel):
    parts = rel.split("/")
    head = parts[0].lower()
    if head in ("core", "props", "textures"):
        return parts[0], "/".join(parts[1:])
    if head in ("maps", "missions", "variants") and len(parts) > 2:
        return "/".join(parts[:2]), "/".join(parts[2:])
    if head == "pilots" and len(parts) > 3:
        return "/".join(parts[:3]), "/".join(parts[3:])
    return parts[0], "/".join(parts[1:])


def main():
    if len(sys.argv) < 2:
        sys.exit(__doc__)
    root = sys.argv[1]
    apply_ = "--apply" in sys.argv

    print("indexing repo Content/ for canonical casing ...", flush=True)
    case_map = build_case_map(REPO_CONTENT)
    print(f"  {len(case_map)} path components")

    plan = []                                   # (oldRel, newRel, package, entry)
    claims = collections.defaultdict(dict)      # newRel -> md5 -> [package]

    for dp, dirs, fs in os.walk(root):
        if dp == root:
            dirs[:] = [d for d in dirs if d not in ("_merged", "Content", "Resource", "_pkgroots")]
        for f in fs:
            old = os.path.relpath(os.path.join(dp, f), root).replace("\\", "/")
            if old.startswith("_"):
                continue
            pkg, entry = package_of(old)
            low = pkg.lower()
            if low.startswith("variants/"):
                new = f"Resource/{pkg}/{entry}"
            elif low.startswith("pilots/"):
                new = f"Resource/{pkg}/{entry}"
            elif entry.lower().startswith("resource/"):
                new = f"_pkgroots/{pkg}/{entry}"
            else:
                new = "Content/" + recase(entry, case_map)
            plan.append((old, new, pkg, entry))
            if new.startswith("Content/"):
                with open(os.path.join(root, old), "rb") as fh:
                    h = hashlib.md5(fh.read()).hexdigest()
                claims[new].setdefault(h, []).append(pkg)

    clashes = {k: v for k, v in claims.items() if len(v) > 1}
    dupes = {k: v for k, v in claims.items()
             if len(v) == 1 and len(next(iter(v.values()))) > 1}

    print(f"\n{len(plan)} files -> {len({n for _o, n, _p, _e in plan})} destinations")
    print(f"  identical copies from several packages : {len(dupes)}")
    print(f"  CONFLICTING copies (different bytes)   : {len(clashes)}")
    if clashes:
        for k, v in list(clashes.items())[:10]:
            print(f"    {k}  " + "; ".join(f"{h[:6]}={','.join(p)}" for h, p in v.items()))
        sys.exit("\nrefusing to flatten: resolve the conflicts first")

    sample = [(o, n) for o, n, _p, _e in plan if o != n][:8]
    print("\nsample:")
    for o, n in sample:
        print(f"  {o}\n    -> {n}")

    if not apply_:
        print("\nDRY RUN - nothing changed. Re-run with --apply.")
        return

    import shutil
    shutil.rmtree(os.path.join(root, "_merged"), ignore_errors=True)

    moved, deduped = 0, 0
    with open(os.path.join(root, "_layout.tsv"), "w", encoding="utf-8") as fh:
        fh.write("newPath\tpackage\toriginalEntry\n")
        for old, new, pkg, entry in sorted(plan):
            src, dest = os.path.join(root, old), os.path.join(root, new)
            os.makedirs(os.path.dirname(dest), exist_ok=True)
            if os.path.exists(dest):            # identical copy from another package
                os.remove(src)
                deduped += 1
            else:
                os.replace(src, dest)
                moved += 1
            fh.write(f"{new}\t{pkg}\t{entry}\n")

    removed = 0
    for dp, _dirs, _fs in os.walk(root, topdown=False):
        if dp == root or not os.path.isdir(dp):
            continue
        if not os.listdir(dp):
            os.rmdir(dp)
            removed += 1

    print(f"\nmoved {moved}, dropped {deduped} identical duplicates, "
          f"removed {removed} empty directories")
    print(f"provenance: {os.path.join(root, '_layout.tsv')}")


if __name__ == "__main__":
    main()