TeslaRel410/dpl3-revive/viewer/archive.py

#!/usr/bin/env python3
"""
archive.py -- batch-build EVERY resolvable DPL3 scene into the historical archive.

Writes one JSON per unique scene to viewer/data/ plus a manifest, for the lazy-
loading archive viewer (archive.html + serve.py). Reuses bundle.build_scene.

    python viewer/archive.py [--limit N]
"""
import glob
import json
import os
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
sys.path.insert(0, os.path.join(os.path.dirname(HERE), "parser"))
import bundle
import scn

DATA = os.path.join(HERE, "data")


def unique_scenes():
    """One canonical .SCN per basename, >=60% of its models resolvable."""
    idx = bundle.archive_index()
    seen = {}
    for s in sorted(glob.glob(bundle.ARCHIVE + "/**/*.SCN", recursive=True)):
        name = os.path.basename(s)[:-4].lower()
        if name in seen:
            continue
        try:
            sc = scn.load(s)
        except Exception:
            continue
        if not sc.instances:
            continue
        uniq = set(i.name.lower() for i in sc.instances)
        res = sum(1 for n in uniq if n in idx["geo"])
        if uniq and res / len(uniq) >= 0.6:
            seen[name] = s
    return seen


def build_scenes(limit=None):
    scenes = unique_scenes()
    items = sorted(scenes.items())
    if limit:
        items = items[:limit]
    print("building %d scenes -> data/" % len(items))
    manifest = []
    for i, (name, path) in enumerate(items):
        rel = os.path.relpath(path, bundle.ARCHIVE).replace("\\", "/")
        try:
            md = bundle.build_scene({"name": name.upper(), "scene": rel, "up": "y"})
        except Exception as e:
            print("  SKIP scene %-14s %s" % (name, e)); continue
        out = os.path.join(DATA, name + ".json")
        with open(out, "w", encoding="utf-8") as fp:
            json.dump(md, fp, separators=(",", ":"))
        manifest.append({"id": name, "tris": md["tris"], "groups": len(md["groups"]),
                         "textures": len(md["textures"]), "kb": round(os.path.getsize(out) / 1024)})
    manifest.sort(key=lambda m: -m["tris"])
    return manifest


def unique_maps():
    """One canonical .MAP per basename from the game MAPS dirs (skip EDITBACK)."""
    seen = {}
    for f in sorted(glob.glob(bundle.ARCHIVE + "/**/MAPS/*.MAP", recursive=True)):
        if "EDITBACK" in f.upper():
            continue
        name = os.path.basename(f)[:-4].lower()
        seen.setdefault(name, f)
    return seen


def build_maps(limit=None):
    maps = unique_maps()
    mdir = os.path.join(DATA, "maps")
    os.makedirs(mdir, exist_ok=True)
    items = sorted(maps.items())
    if limit:
        items = items[:limit]
    print("building %d maps -> data/maps/" % len(items))
    manifest = []
    for name, path in items:
        rel = os.path.relpath(path, bundle.ARCHIVE).replace("\\", "/")
        try:
            md = bundle.build_map({"name": name.upper(), "map": rel})
            if not md["groups"]:
                continue
        except Exception as e:
            print("  SKIP map %-14s %s" % (name, e)); continue
        out = os.path.join(mdir, name + ".json")
        with open(out, "w", encoding="utf-8") as fp:
            json.dump(md, fp, separators=(",", ":"))
        manifest.append({"id": name, "tris": md["tris"], "groups": len(md["groups"]),
                         "textures": len(md["textures"]), "kb": round(os.path.getsize(out) / 1024)})
    manifest.sort(key=lambda m: -m["tris"])
    return manifest


def build_models(limit=None):
    idx = bundle.archive_index()
    mdir = os.path.join(DATA, "models")
    os.makedirs(mdir, exist_ok=True)
    names = sorted(n for n, p in idx["geo"].items()
                   if p.lower().endswith((".b2z", ".biz", ".bgf")))
    if limit:
        names = names[:limit]
    print("building %d models -> data/models/" % len(names))
    manifest = []
    for i, name in enumerate(names):
        try:
            md = bundle.build_model_archive(idx["geo"][name], name)
            if not md["groups"]:
                continue
        except Exception as e:
            print("  SKIP model %-14s %s" % (name, e)); continue
        out = os.path.join(mdir, name + ".json")
        with open(out, "w", encoding="utf-8") as fp:
            json.dump(md, fp, separators=(",", ":"))
        manifest.append({"id": name, "tris": md["tris"], "groups": len(md["groups"]),
                         "textures": len(md["textures"]), "kb": round(os.path.getsize(out) / 1024)})
        if (i + 1) % 50 == 0:
            print("  ...%d/%d models" % (i + 1, len(names)))
    manifest.sort(key=lambda m: -m["tris"])
    return manifest


def classify(id, path):
    """Map a scene/model to its VWE product line. Directory is the strong signal
    for the themed CONVERT/ and project folders; object name fills the gaps for the
    generic geometry/example dirs."""
    rel = os.path.relpath(path, bundle.ARCHIVE).replace(os.sep, "/").lower()
    n = id.lower()
    if "/btech" in rel or rel[:2] == "bt":
        return "BattleTech"
    if "hpdave" in rel:
        return "Hull Pressure"
    if "stdave" in rel:
        return "Star Trek"
    if any(d in rel for d in ("/canals", "/canyons", "/raptor", "/rocky", "/thor", "/grg")) \
            or rel[:2] == "rp":
        return "Red Planet"
    if "/mileston" in rel:
        return "Milestone"
    if "spheres" in rel:
        return "Engine / Demo"
    # name-driven for generic geometry / example dirs
    if any(k in n for k in ("shark", "grouper", "angel", "butter", "golden", "fish",
                            "ocean", "plant", "coral", "kelp", "atlant", "sea", "reef")):
        return "Hull Pressure"
    if any(k in n for k in ("trek", "klngn", "klingon", "vulcan", "divcal", "field")):
        return "Star Trek"
    if any(k in n for k in ("madcat", "vulture", "mech", "loki", "warhawk", "thor", "gauss")):
        return "BattleTech"
    if any(k in n for k in ("canal", "canyon", "cliff", "rock", "strut", "tcliff",
                            "redrock", "mars", "raptor", "rapids", "river", "mine")):
        return "Red Planet"
    return "Engine / Demo"


def reclassify():
    """Add/refresh the `product` tag on every manifest entry WITHOUT rebuilding the
    (heavy) per-item JSON. Uses the source paths to classify."""
    mpath = os.path.join(DATA, "manifest.json")
    man = json.load(open(mpath, encoding="utf-8"))
    idx = bundle.archive_index()
    scene_paths = unique_scenes()
    for s in man.get("scenes", []):
        p = scene_paths.get(s["id"])
        s["product"] = classify(s["id"], p) if p else "Engine / Demo"
    for m in man.get("models", []):
        p = idx["geo"].get(m["id"])
        m["product"] = classify(m["id"], p) if p else "Engine / Demo"
    map_paths = unique_maps()
    for m in man.get("maps", []):
        p = map_paths.get(m["id"])
        m["product"] = classify(m["id"], p) if p else "Engine / Demo"
    json.dump(man, open(mpath, "w", encoding="utf-8"), separators=(",", ":"))
    from collections import Counter
    for key in ("scenes", "models", "maps"):
        c = Counter(x["product"] for x in man.get(key, []))
        print("%-8s by product: %s" % (key, dict(c)))


def main():
    if "--reclassify" in sys.argv:
        reclassify()
        return
    limit = None
    if "--limit" in sys.argv:
        limit = int(sys.argv[sys.argv.index("--limit") + 1])
    os.makedirs(DATA, exist_ok=True)
    only = [a for a in sys.argv if a.endswith("-only")]
    def want(kind):
        return not only or ("--%s-only" % kind) in only
    scenes = build_scenes(limit) if want("scenes") else _load("scenes")
    models = build_models(limit) if want("models") else _load("models")
    maps = build_maps(limit) if want("maps") else _load("maps")
    with open(os.path.join(DATA, "manifest.json"), "w", encoding="utf-8") as fp:
        json.dump({"scenes": scenes, "models": models, "maps": maps},
                  fp, separators=(",", ":"))
    sm = sum(m["kb"] for m in scenes) / 1024
    mm = sum(m["kb"] for m in models) / 1024
    pm = sum(m["kb"] for m in maps) / 1024
    print("done: %d scenes (%.1f MB), %d models (%.1f MB), %d maps (%.1f MB)"
          % (len(scenes), sm, len(models), mm, len(maps), pm))


def _load(key):
    try:
        return json.load(open(os.path.join(DATA, "manifest.json"), encoding="utf-8")).get(key, [])
    except Exception:
        return []


if __name__ == "__main__":
    main()