#!/usr/bin/env python3 """ archive.py -- batch-build EVERY resolvable DPL3 scene into the historical archive. Writes one JSON per unique scene to viewer/data/ plus a manifest, for the lazy- loading archive viewer (archive.html + serve.py). Reuses bundle.build_scene. python viewer/archive.py [--limit N] """ import glob import json import os import sys HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) sys.path.insert(0, os.path.join(os.path.dirname(HERE), "parser")) import bundle import scn DATA = os.path.join(HERE, "data") def unique_scenes(): """One canonical .SCN per basename, >=60% of its models resolvable.""" idx = bundle.archive_index() seen = {} for s in sorted(glob.glob(bundle.ARCHIVE + "/**/*.SCN", recursive=True)): name = os.path.basename(s)[:-4].lower() if name in seen: continue try: sc = scn.load(s) except Exception: continue if not sc.instances: continue uniq = set(i.name.lower() for i in sc.instances) res = sum(1 for n in uniq if n in idx["geo"]) if uniq and res / len(uniq) >= 0.6: seen[name] = s return seen def build_scenes(limit=None): scenes = unique_scenes() items = sorted(scenes.items()) if limit: items = items[:limit] print("building %d scenes -> data/" % len(items)) manifest = [] for i, (name, path) in enumerate(items): rel = os.path.relpath(path, bundle.ARCHIVE).replace("\\", "/") try: md = bundle.build_scene({"name": name.upper(), "scene": rel, "up": "y"}) except Exception as e: print(" SKIP scene %-14s %s" % (name, e)); continue out = os.path.join(DATA, name + ".json") with open(out, "w", encoding="utf-8") as fp: json.dump(md, fp, separators=(",", ":")) manifest.append({"id": name, "tris": md["tris"], "groups": len(md["groups"]), "textures": len(md["textures"]), "kb": round(os.path.getsize(out) / 1024)}) manifest.sort(key=lambda m: -m["tris"]) return manifest def unique_maps(): """One canonical .MAP per basename from the game MAPS dirs (skip EDITBACK).""" seen = {} for f in sorted(glob.glob(bundle.ARCHIVE + "/**/MAPS/*.MAP", recursive=True)): if "EDITBACK" in f.upper(): continue name = os.path.basename(f)[:-4].lower() seen.setdefault(name, f) return seen def build_maps(limit=None): maps = unique_maps() mdir = os.path.join(DATA, "maps") os.makedirs(mdir, exist_ok=True) items = sorted(maps.items()) if limit: items = items[:limit] print("building %d maps -> data/maps/" % len(items)) manifest = [] for name, path in items: rel = os.path.relpath(path, bundle.ARCHIVE).replace("\\", "/") try: md = bundle.build_map({"name": name.upper(), "map": rel}) if not md["groups"]: continue except Exception as e: print(" SKIP map %-14s %s" % (name, e)); continue out = os.path.join(mdir, name + ".json") with open(out, "w", encoding="utf-8") as fp: json.dump(md, fp, separators=(",", ":")) manifest.append({"id": name, "tris": md["tris"], "groups": len(md["groups"]), "textures": len(md["textures"]), "kb": round(os.path.getsize(out) / 1024)}) manifest.sort(key=lambda m: -m["tris"]) return manifest def build_models(limit=None): idx = bundle.archive_index() mdir = os.path.join(DATA, "models") os.makedirs(mdir, exist_ok=True) names = sorted(n for n, p in idx["geo"].items() if p.lower().endswith((".b2z", ".biz", ".bgf"))) if limit: names = names[:limit] print("building %d models -> data/models/" % len(names)) manifest = [] for i, name in enumerate(names): try: md = bundle.build_model_archive(idx["geo"][name], name) if not md["groups"]: continue except Exception as e: print(" SKIP model %-14s %s" % (name, e)); continue out = os.path.join(mdir, name + ".json") with open(out, "w", encoding="utf-8") as fp: json.dump(md, fp, separators=(",", ":")) manifest.append({"id": name, "tris": md["tris"], "groups": len(md["groups"]), "textures": len(md["textures"]), "kb": round(os.path.getsize(out) / 1024)}) if (i + 1) % 50 == 0: print(" ...%d/%d models" % (i + 1, len(names))) manifest.sort(key=lambda m: -m["tris"]) return manifest def classify(id, path): """Map a scene/model to its VWE product line. Directory is the strong signal for the themed CONVERT/ and project folders; object name fills the gaps for the generic geometry/example dirs.""" rel = os.path.relpath(path, bundle.ARCHIVE).replace(os.sep, "/").lower() n = id.lower() if "/btech" in rel or rel[:2] == "bt": return "BattleTech" if "hpdave" in rel: return "Hull Pressure" if "stdave" in rel: return "Star Trek" if any(d in rel for d in ("/canals", "/canyons", "/raptor", "/rocky", "/thor", "/grg")) \ or rel[:2] == "rp": return "Red Planet" if "/mileston" in rel: return "Milestone" if "spheres" in rel: return "Engine / Demo" # name-driven for generic geometry / example dirs if any(k in n for k in ("shark", "grouper", "angel", "butter", "golden", "fish", "ocean", "plant", "coral", "kelp", "atlant", "sea", "reef")): return "Hull Pressure" if any(k in n for k in ("trek", "klngn", "klingon", "vulcan", "divcal", "field")): return "Star Trek" if any(k in n for k in ("madcat", "vulture", "mech", "loki", "warhawk", "thor", "gauss")): return "BattleTech" if any(k in n for k in ("canal", "canyon", "cliff", "rock", "strut", "tcliff", "redrock", "mars", "raptor", "rapids", "river", "mine")): return "Red Planet" return "Engine / Demo" def reclassify(): """Add/refresh the `product` tag on every manifest entry WITHOUT rebuilding the (heavy) per-item JSON. Uses the source paths to classify.""" mpath = os.path.join(DATA, "manifest.json") man = json.load(open(mpath, encoding="utf-8")) idx = bundle.archive_index() scene_paths = unique_scenes() for s in man.get("scenes", []): p = scene_paths.get(s["id"]) s["product"] = classify(s["id"], p) if p else "Engine / Demo" for m in man.get("models", []): p = idx["geo"].get(m["id"]) m["product"] = classify(m["id"], p) if p else "Engine / Demo" map_paths = unique_maps() for m in man.get("maps", []): p = map_paths.get(m["id"]) m["product"] = classify(m["id"], p) if p else "Engine / Demo" json.dump(man, open(mpath, "w", encoding="utf-8"), separators=(",", ":")) from collections import Counter for key in ("scenes", "models", "maps"): c = Counter(x["product"] for x in man.get(key, [])) print("%-8s by product: %s" % (key, dict(c))) def main(): if "--reclassify" in sys.argv: reclassify() return limit = None if "--limit" in sys.argv: limit = int(sys.argv[sys.argv.index("--limit") + 1]) os.makedirs(DATA, exist_ok=True) only = [a for a in sys.argv if a.endswith("-only")] def want(kind): return not only or ("--%s-only" % kind) in only scenes = build_scenes(limit) if want("scenes") else _load("scenes") models = build_models(limit) if want("models") else _load("models") maps = build_maps(limit) if want("maps") else _load("maps") with open(os.path.join(DATA, "manifest.json"), "w", encoding="utf-8") as fp: json.dump({"scenes": scenes, "models": models, "maps": maps}, fp, separators=(",", ":")) sm = sum(m["kb"] for m in scenes) / 1024 mm = sum(m["kb"] for m in models) / 1024 pm = sum(m["kb"] for m in maps) / 1024 print("done: %d scenes (%.1f MB), %d models (%.1f MB), %d maps (%.1f MB)" % (len(scenes), sm, len(models), mm, len(maps), pm)) def _load(key): try: return json.load(open(os.path.join(DATA, "manifest.json"), encoding="utf-8")).get(key, []) except Exception: return [] if __name__ == "__main__": main()