firestorm/MW4COMPARE/tools/extract-all.py

#!/usr/bin/env python3
"""
extract-all.py - unpack every *.mw4 under a resource root into a real directory tree.

    python3 extract-all.py <resourceRoot> <outDir> [--no-merged]

Layout
------
    <outDir>/<packagePathWithoutExtension>/<entryPath>

e.g.  core.mw4          entry  mechs\\atlas\\atlas.subsystems
      ->  <outDir>/core/mechs/atlas/atlas.subsystems

      Missions/freezer.mw4  entry  missions\\freezer\\freezer.contents
      ->  <outDir>/Missions/freezer/missions/freezer/freezer.contents

This is lossless: 842 entry paths are claimed by more than one package (281 of
them with different content - typically a mission packing its own copy of a
global props.mw4 asset), so a single flat tree cannot represent the data.

<outDir>/_merged/ is then built as a flattened view of the whole set using
HARDLINKS (no extra disk). It mirrors the layout of Gameleap/mw4/Content, which
makes it directly diffable against our source tree. Where two packages disagree,
precedence is props > core > textures > maps > missions and every conflict is
listed in _conflicts.tsv.

Entry-name qualifiers are preserved verbatim in the filename:
    foo.data{gamemodel}   foo.contents[joint_hip]{armature}   bar.tga{hint}
All characters used by these packages are legal on Linux ('!' '#' '$' '%' '^'
appear in lobby skin names; no ':' or NUL).
"""
import sys, os, hashlib

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from mw4db import read_records, walk_packages

# Higher wins when the same entry path comes from several packages.
PRECEDENCE = {"props": 50, "core": 40, "textures": 30, "maps": 20, "missions": 10}

# Saved mechlab loadouts and pilot options name their records '{Mech}',
# '<chassis>{Subsystem}' etc. with no directory part, so they have no place in a
# Content-shaped tree - 797 variants would all collide on the same few names.
MERGE_EXCLUDE = ("variants/", "pilots/")


def in_merged(pkgrel):
    return not pkgrel.lower().startswith(MERGE_EXCLUDE)


def precedence(pkgrel):
    head = pkgrel.split("/")[0].lower()
    if head.endswith(".mw4"):
        head = head[:-4]
    return PRECEDENCE.get(head, 0)


def entry_path(name):
    """Entry name -> relative filesystem path. Qualifiers kept as-is."""
    p = name.replace("\\", "/").strip("/")
    # Defensive: never let an entry escape the output directory.
    parts = [seg for seg in p.split("/") if seg not in ("", ".", "..")]
    return "/".join(parts) if parts else "_unnamed"


def main():
    args = [a for a in sys.argv[1:] if not a.startswith("--")]
    if len(args) != 2:
        sys.exit(__doc__)
    root, out = args
    want_merged = "--no-merged" not in sys.argv

    os.makedirs(out, exist_ok=True)
    manifest = open(os.path.join(out, "_manifest.tsv"), "w", encoding="utf-8")
    manifest.write("package\trecordId\tentryName\tdataLen\trecLen\tmd5\toutPath\n")
    notes = open(os.path.join(out, "_notes.txt"), "w", encoding="utf-8")

    # merged bookkeeping: mergedRelPath -> (precedence, pkgrel, md5, absSourceFile)
    best = {}
    seen_paths = {}          # mergedRelPath -> {md5: [pkgrel, ...]}
    total_files = total_bytes = 0
    packages = 0

    for path, pkgrel in walk_packages(root):
        res = read_records(path)
        if res is None:
            notes.write(f"SKIP not-a-#VBD-package: {pkgrel}\n")
            continue
        _content, recs = res
        packages += 1
        pkgdir = os.path.join(out, pkgrel[:-4] if pkgrel.lower().endswith(".mw4") else pkgrel)
        written = {}                                  # relPath -> md5, for intra-package dupes

        for rid, name, dlen, rlen, blob in recs:
            rel = entry_path(name)
            h = hashlib.md5(blob).hexdigest()

            if rel in written:                        # duplicate entry name in one package
                if written[rel] == h:
                    notes.write(f"DUP-IDENTICAL {pkgrel}\t{name}\trec{rid}\n")
                    continue
                rel = f"{rel}#rec{rid}"
                notes.write(f"DUP-DIFFERENT {pkgrel}\t{name}\trec{rid} -> {rel}\n")
            written[rel] = h

            dest = os.path.join(pkgdir, rel)
            os.makedirs(os.path.dirname(dest), exist_ok=True)
            with open(dest, "wb") as fh:
                fh.write(blob)
            total_files += 1
            total_bytes += len(blob)
            manifest.write(f"{pkgrel}\t{rid}\t{name}\t{dlen}\t{rlen}\t{h}\t"
                           f"{os.path.relpath(dest, out)}\n")

            if want_merged and in_merged(pkgrel):
                seen_paths.setdefault(rel, {}).setdefault(h, []).append(pkgrel)
                pr = precedence(pkgrel)
                cur = best.get(rel)
                if cur is None or pr > cur[0]:
                    best[rel] = (pr, pkgrel, h, dest)

        print(f"  {pkgrel}: {len(recs)} records", flush=True)

    manifest.close()

    conflicts = 0
    if want_merged:
        mroot = os.path.join(out, "_merged")
        with open(os.path.join(out, "_conflicts.tsv"), "w", encoding="utf-8") as cf:
            cf.write("mergedPath\tchosenPackage\tallVersions\n")
            for rel, (_pr, pkgrel, _h, src) in best.items():
                dest = os.path.join(mroot, rel)
                os.makedirs(os.path.dirname(dest), exist_ok=True)
                if os.path.lexists(dest):
                    os.unlink(dest)
                os.link(src, dest)
                versions = seen_paths[rel]
                if len(versions) > 1:
                    conflicts += 1
                    detail = "; ".join(f"{h[:8]}={','.join(ps)}" for h, ps in versions.items())
                    cf.write(f"{rel}\t{pkgrel}\t{detail}\n")

    notes.write(f"\npackages={packages} files={total_files} bytes={total_bytes} "
                f"mergedPaths={len(best)} conflicts={conflicts}\n")
    notes.close()
    print(f"\npackages   : {packages}")
    print(f"files      : {total_files}")
    print(f"bytes      : {total_bytes/1e9:.2f} GB")
    if want_merged:
        print(f"merged     : {len(best)} paths, {conflicts} with conflicting versions")
    print(f"out        : {out}")


if __name__ == "__main__":
    main()