diff --git a/restoration/README.md b/restoration/README.md index c323881..4d0536f 100644 --- a/restoration/README.md +++ b/restoration/README.md @@ -1,42 +1,57 @@ -# TREK.SCN Restoration +# Scene Restorations -A modern re-rendering of the unreleased **Star Trek** simulator-pod scenes found -in the `STDAVE` directory of the Glaze developer drive (see -[HISTORY.md](../HISTORY.md)). The original scenes ran on Division Ltd.'s dVS -pixel-pipeline hardware in 1996; this restoration parses the original data -files and re-renders them in WebGL, in the browser. +Modern re-renderings of the Division dVS scenes recovered from the Glaze +developer drive (see [HISTORY.md](../HISTORY.md)) and this repo's CONTENT +tree. The originals ran on Division Ltd.'s pixel-pipeline hardware in +1994–1997; these tools parse the original data files and re-render them in +WebGL, in the browser. -## Files +## Viewers (open in any browser, fully self-contained) -- **`trek-scn.html`** — the finished, self-contained viewer (open in any - browser; no server needed). Renders `TREK.SCN` (the Enterprise-D among - three parallax layers of drifting stars) and `KLNGVID.SCN` (a Klingon - cruiser at rest while two more drift past on spline paths). Fly camera: - drag to look, WASD to fly. -- **`convert.py`** — the converter. Parses the DIV-VIZ2 text formats from - the drive dump (expects it at `..\sda4\STDAVE`) and emits `trekdata.json`: - - `.VGF` geometry (vertex pools + polygon connection lists, converted from - Wavefront OBJ in 1996 by Ken Olsen's `obj2vgf`) - - `.VMF` materials (ambient/diffuse/emissive, shading ramps, texture - bit-slice references) - - `.TGA` textures (decoded and re-encoded as PNG data URIs) - - `.SPL` splines (starfield drift paths, ship flyby paths) - - `.SCN` scene scripts (camera, lights, fog, object placements) -- **`viewer_template.html`** — the viewer source with a `%%DATA%%` - placeholder; `convert.py`'s JSON output is injected to produce - `trek-scn.html`. +- **`vwe-archive.html`** — the Scene Archive: 26 curated scenes across all + projects — Star Trek (Enterprise-D, Klingon flyby), Hull Pressure (sea + demos, fish schools, sunken temple), BattleTech (polar map with mech + lineup, Ravine and desert maps, stadium), Red Planet (Blade arena, the + Mars canal milestone demo), and the Canyon/Maya-temple Division demos. + Fly camera: drag to look, WASD to fly, R resets. +- **`trek-scn.html`** — the original standalone Star Trek restoration + (TREK.SCN and KLNGVID.SCN). + +## Toolkit + +- **`divformats.py`** — parsers for every Division format found on the + drive: text `VGF`/`VMF` (DIV-VIZ2 geometry/materials, both the 1995 + `VERSION 2:07` and 1996 `02:05` dialects, with header SCALE support), + binary **`BGF`/`BMF` (DIV-BIZ2)** — the binary block-stream layout was + recovered from Division's own reader source (`DPL3/BIZREAD.C`, 1994), + including all 14 vertex layouts, pmesh/tristrip/polygon connectivity, + LOD blocks (newer writers nest patches inside LODs; the largest LOD is + taken), and embedded materials/textures/ramps — plus `TGA` and raw `SVT` + textures, `SPL` splines, and `SCN` scene scripts. +- **`audit.py`** — scans every `.SCN` on the drive and reports what + fraction of its geometry survives, and in which format. +- **`extract_all.py`** — extracts every scene with ≥60 % of its geometry + recoverable (108 of 241 on the drive; the rest reference lost models) + into `allscenes.json`, resolving models/materials/textures across the + drive and this repo's CONTENT dirs. +- **`gallery_build.py`** — curates scenes from `allscenes.json` into the + self-contained `vwe-archive.html` (via `gallery_template.html`). +- **`convert.py` / `viewer_template.html`** — the original single-scene + Star Trek pipeline. ## Rebuild ``` -python convert.py -python -c "open('trek-scn.html','w',encoding='utf-8').write(open('viewer_template.html',encoding='utf-8').read().replace('%%DATA%%',open('trekdata.json',encoding='utf-8').read()))" +python extract_all.py # requires the sda4/ drive dump beside this repo +python gallery_build.py ``` ## Fidelity notes -Faithful: vertices, triangles, material colors, textures, fog and light -parameters, star paths and layer speeds, camera start positions — all from -the original files. Approximated: Division's shading ramps and rasterizer -behavior, point-sprite star sizes, ship motion timing (original simulation -ran at 30 Hz). Not simulated: the event-driven SPECIALFX particle effects. +Faithful: vertices, triangles, material colors and shading ramps, textures, +fog and light parameters, spline motion, camera start positions. Approximated: +Division's rasterizer behavior, point sizes, timing (originals ran at 30 Hz). +Not simulated: event-driven SPECIALFX particles. A few dev scenes had camera +start positions inside geometry; those get overridden or auto-framed cameras +(see `START_OVERRIDE` in `gallery_build.py`). Scenes missing one or more +models from the drive say so in their header. diff --git a/restoration/audit.py b/restoration/audit.py new file mode 100644 index 0000000..fd26bd0 --- /dev/null +++ b/restoration/audit.py @@ -0,0 +1,66 @@ +"""Audit every .SCN on the drive: which geometry does it need, and in what form +(VGF text / BGF binary / missing) does that geometry survive?""" +import os, re, sys +from collections import defaultdict + +SDA4 = r"c:\VWE\TeslaRel410\sda4" + +# index every VGF/BGF/VMF/BMF/TGA on the drive by lowercase basename +index = defaultdict(list) +for root, dirs, files in os.walk(SDA4): + dirs[:] = [d for d in dirs if d.upper() not in ("EDITBACK", "RECYCLED")] + for f in files: + base, ext = os.path.splitext(f.lower()) + if ext in (".vgf", ".bgf", ".vmf", ".bmf", ".tga", ".spl", ".bsl"): + index[(base, ext)].append(os.path.join(root, f)) + +def parse_scene_refs(path): + geos, mats, spls = [], set(), [] + geodir = matdir = None + for raw in open(path, errors="replace"): + line = raw.split("//")[0].split("#")[0].strip() + if not line: continue + t = line.split() + k = t[0].upper() + try: + if k == "GEOMETRY": geodir = t[1] + elif k == "MATERIAL": matdir = t[1] + elif k in ("STATIC", "SCHILD", "DCHILD"): + geos.append(t[1].lower()) + elif k == "DYNAMIC": + geos.append(t[1].lower()) + spls.append(os.path.basename(t[3].replace("\\", "/")).lower()) + elif k == "MORPH": + geos.append(t[1].lower()) + except IndexError: + pass + return geos, spls, geodir, matdir + +scenes = [] +for root, dirs, files in os.walk(SDA4): + dirs[:] = [d for d in dirs if d.upper() not in ("EDITBACK", "RECYCLED")] + for f in files: + if f.lower().endswith(".scn"): + scenes.append(os.path.join(root, f)) + +report = [] +for s in sorted(scenes): + geos, spls, geodir, matdir = parse_scene_refs(s) + if not geos: continue + uniq = sorted(set(g.split(":")[0] for g in geos)) + n_vgf = sum(1 for g in uniq if (g, ".vgf") in index) + n_bgf = sum(1 for g in uniq if (g, ".bgf") in index and (g, ".vgf") not in index) + n_missing = len(uniq) - n_vgf - n_bgf + spl_missing = sum(1 for x in set(spls) if (os.path.splitext(x)[0], ".spl") not in index) + rel = os.path.relpath(s, SDA4) + report.append((rel, len(uniq), n_vgf, n_bgf, n_missing, spl_missing)) + +print(f"{'scene':<44} {'geo':>3} {'vgf':>3} {'bgf':>3} {'mis':>3} {'spl-':>4}") +full, partial, locked = 0, 0, 0 +for rel, n, v, b, m, sm in report: + tag = "FULL-VGF" if v == n else ("HAS-BGF" if m == 0 else "MISSING") + if v == n: full += 1 + elif m == 0: locked += 1 + else: partial += 1 + print(f"{rel:<44} {n:>3} {v:>3} {b:>3} {m:>3} {sm:>4} {tag}") +print(f"\n{len(report)} scenes with geometry: {full} fully VGF, {locked} need BGF, {partial} have missing geometry") diff --git a/restoration/divformats.py b/restoration/divformats.py new file mode 100644 index 0000000..702e8d5 --- /dev/null +++ b/restoration/divformats.py @@ -0,0 +1,411 @@ +"""Parsers for Division Ltd. dVS/DPL file formats (1994-1996), reverse-engineered +from the VWE 'Glaze' drive. Binary BIZ2 layout comes from Division's own reader +source (DPL3/BIZREAD.C, PJA 1994); text formats from surviving examples. + +Formats: + .VGF/.VMF DIV-VIZ2 text geometry / materials + .BGF/.BMF DIV-BIZ2 binary geometry / materials (b2z block stream) + .SVT raw square texture, 4 B/texel, size implies 64/128/256 edge + .TGA type-2 uncompressed 24/32-bit + .SPL text spline paths + .SCN text scene scripts +""" +import os, re, struct, zlib, base64 + +# ================================================================ helpers +def strip_comments(text): + text = re.sub(r"/\*.*?\*/", " ", text, flags=re.S) + text = re.sub(r"//[^\n]*", " ", text) + return text + +def png_datauri(w, h, rgb_rows): + raw = b"".join(b"\x00" + r for r in rgb_rows) + def chunk(tag, data): + c = struct.pack(">I", len(data)) + tag + data + return c + struct.pack(">I", zlib.crc32(tag + data) & 0xffffffff) + png = (b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 9)) + + chunk(b"IEND", b"")) + return "data:image/png;base64," + base64.b64encode(png).decode() + +def r3(x): return round(x, 3) +def r4(x): return round(x, 4) + +# ================================================================ VGF (text geometry) +def parse_vgf(path): + text = strip_comments(open(path, "r", errors="replace").read()) + scale = 1.0 + hm = re.search(r"HEADER\s*\((.*?)\)", text, re.S) + if hm: + sm = re.search(r"SCALE\s*=\s*([\d.eE+-]+)", hm.group(1)) + if sm: scale = float(sm.group(1)) + groups = [] + for gm in re.finditer(r"GEOGROUP\s*\(([^)]*)\)\s*\{", text): + params = gm.group(1) + mmat = re.search(r'F_MATERIAL\s*=\s*"([^"]+)"', params) + material = mmat.group(1) if mmat else None + vflags = re.search(r"VERTEX\s*=\s*([^;]+);", params) + flags = vflags.group(1).upper() if vflags else "" + has_n = "NORMALS" in flags + has_uv = "2D_TEXTURE" in flags or "3D_TEXTURE" in flags + depth, i = 1, gm.end() + while depth > 0 and i < len(text): + if text[i] == "{": depth += 1 + elif text[i] == "}": depth -= 1 + i += 1 + body = text[gm.end():i - 1] + g = {"material": material} + sph = re.search(r"SPHERELIST\s*\{(.*?)\n\s*\}", body, re.S) + if sph: + pts = [] + for row in re.finditer(r"\{([^}]*)\}", sph.group(1)): + nums = [float(x) for x in re.findall(r"[-\d.eE+]+", row.group(1))] + pts.append([r3(v * scale) for v in nums[:3]]) + g["mode"] = "points"; g["points"] = pts + groups.append(g); continue + vp = re.search(r"VERTEX_POOL\s*\{(.*?)\n\s*\}", body, re.S) + if not vp: continue + verts, norms, uvs = [], [], [] + for row in re.finditer(r"\{([^}]*)\}", vp.group(1)): + nums = [float(x) for x in re.findall(r"[-\d.eE+]+", row.group(1))] + k = 0 + verts.append([r3(v * scale) for v in nums[k:k+3]]); k += 3 + if has_n: norms.append([r3(v) for v in nums[k:k+3]]); k += 3 + if has_uv: uvs.append([r4(v) for v in nums[k:k+2]]); k += 2 + indices = [] + for cm in re.finditer(r"CONNECTION_LIST\s*(?:\(\s*PCOUNT\s*=\s*\d+\s*\))?\s*\{(.*?)(?:\n\s*\}|(?=CONNECTION_LIST))", body, re.S): + for row in re.finditer(r"\{([^}]*)\}", cm.group(1)): + idx = [int(x) for x in re.findall(r"\d+", row.group(1))] + for t in range(1, len(idx) - 1): + indices += [idx[0], idx[t], idx[t + 1]] + g["mode"] = "mesh"; g["v"] = verts + g["n"] = norms if has_n else None + g["uv"] = uvs if has_uv else None + g["i"] = indices + groups.append(g) + return groups + +# ================================================================ VMF (text materials) +def parse_vmf(path, prefix): + text = strip_comments(open(path, "r", errors="replace").read()) + textures, materials = {}, {} + for tm in re.finditer(r"TEXTURE\s*\(\s*NAME\s*=\s*\"?([\w-]+)\"?\s*;?([^)]*)\)\s*\{(.*?)\n\}", text, re.S): + name, params, body = tm.group(1), tm.group(2), tm.group(3) + mmap = re.search(r'MAP\s*\{\s*"([\w-]+)"\s*\}', body) + mslice = re.search(r"BITSLICE\s*\{\s*(\d+)\s*\}", body) + textures[name] = {"map": mmap.group(1).lower() if mmap else None, + "slice": int(mslice.group(1)) if mslice else None} + ramps = {} + for rm in re.finditer(r"RAMP\s*\(\s*NAME\s*=\s*\"?([\w-]+)\"?\s*\)\s*\{(.*?)\n\}", text, re.S): + nums = [float(x) for x in re.findall(r"[-\d.eE+]+", rm.group(2))] + if len(nums) >= 6: ramps[rm.group(1)] = [nums[0:3], nums[3:6]] + for mm in re.finditer(r"MATERIAL\s*\(\s*NAME\s*=\s*\"?([\w-]+)\"?\s*;?([^)]*)\)\s*\{(.*?)\n\}", text, re.S): + name, params, body = mm.group(1), mm.group(2), mm.group(3) + def col(tag, n=3): + m = re.search(tag + r"\s*\{([^}]*)\}", body) + return [float(x) for x in re.findall(r"[-\d.eE+]+", m.group(1))][:n] if m else None + mtex = re.search(r'TEXTURE\s*\{\s*"([\w-]+)"\s*\}', body) + mramp = re.search(r"RAMP\s*\{\s*\"?([\w-]+)\"?\s*\}", body) + materials[prefix + ":" + name] = { + "ambient": col("AMBIENT"), "diffuse": col("DIFFUSE"), + "emissive": col("EMISSIVE"), + "texture": mtex.group(1) if mtex else None, + "ramp": mramp.group(1) if mramp else None, + "immune": "IMMUNE" in params} + return textures, materials, ramps + +# ================================================================ BIZ2 (binary container) +class B2Z: + def __init__(self, path): + self.d = open(path, "rb").read() + self.p = 0 + def u8(self): v = self.d[self.p]; self.p += 1; return v + def u16(self): v = struct.unpack_from("> 12) & 0xC + if mode == 0x8: ln = self.i32() + elif mode == 0x4: ln = self.u16() + elif mode == 0x0: ln = self.u8() + else: raise ValueError("bad block header 0x%x @%d" % (hdr, self.p)) + return hdr & 0x3fff, ln # strip length-size bits, keep 0x2000 flag + +VTX_LAYOUT = { # tag: (floats_per_vertex, n_off, uv_off, uv3_off, col_off, lum_off) + 0x80: (3, -1, -1, -1, -1, -1), + 0x81: (6, 3, -1, -1, -1, -1), + 0x82: (7, -1, -1, -1, 3, -1), + 0x83: (10, 3, -1, -1, 6, -1), + 0x84: (5, -1, -1, -1, -1, 3), + 0x85: (8, 3, -1, -1, -1, 6), + 0x88: (5, -1, 3, -1, -1, -1), + 0x89: (8, 3, 6, -1, -1, -1), + 0x8a: (9, -1, 7, -1, 3, -1), + 0x8c: (7, -1, 5, -1, -1, 3), + 0x90: (6, -1, -1, 3, -1, -1), + 0x91: (9, 3, -1, 6, -1, -1), + 0x92: (10, -1, -1, 7, 3, -1), + 0x94: (8, -1, -1, 5, 3, -1), # lum@3, 3dtex@5 per BIZREAD.C +} + +def _tri_from_strip(nv): + out = [] + for t in range(nv - 2): + if t % 2 == 0: out += [t, t + 1, t + 2] + else: out += [t + 1, t, t + 2] + return out + +def _read_vertex_block(bz, tag, ln): + fpv, n_off, uv_off, uv3_off, col_off, lum_off = VTX_LAYOUT[tag] + if tag == 0x94: fpv, lum_off, uv3_off = 8, 3, 5 + nv = (ln // 4) // fpv + verts, norms, uvs, cols = [], [], [], [] + for _ in range(nv): + f = bz.floats(fpv) + verts.append([r3(f[0]), r3(f[1]), r3(f[2])]) + if n_off >= 0: norms.append([r3(f[n_off]), r3(f[n_off+1]), r3(f[n_off+2])]) + if uv_off >= 0: uvs.append([r4(f[uv_off]), r4(f[uv_off+1])]) + if uv3_off >= 0: uvs.append([r4(f[uv3_off]), r4(f[uv3_off+1])]) + if col_off >= 0: cols.append([r3(f[col_off]), r3(f[col_off+1]), r3(f[col_off+2])]) + if lum_off >= 0: cols.append([r3(f[lum_off])] * 3) + return verts, norms or None, uvs or None, cols or None + +def _parse_bgf_vertices(bz, length, geotype, out_groups, material): + end = bz.p + length + pend_v = pend_n = pend_uv = pend_c = None + conns = [] + while bz.p < end: + tag, ln = bz.block() + nxt = bz.p + ln + if tag == 0x47: # triangle connectivity + for _ in range(ln // 12): + conns.append(list(bz.ints(3))) + elif tag == 0x4d: # polygon connectivity + vpp = bz.u8() + n = (ln - 1) // (vpp * 4) + for _ in range(n): + conns.append(list(bz.ints(vpp))) + elif tag in VTX_LAYOUT: + v, n, uv, c = _read_vertex_block(bz, tag, ln) + if geotype == "pmesh": + if pend_v is None: + pend_v, pend_n, pend_uv, pend_c = v, n, uv, c + else: + pend_v += v + if pend_n and n: pend_n += n + if pend_uv and uv: pend_uv += uv + if pend_c and c: pend_c += c + else: # strip / polygon: emit immediately + idx = (_tri_from_strip(len(v)) if geotype == "tristrip" + else [x for t in range(1, len(v)-1) for x in (0, t, t+1)]) + out_groups.append({"material": material, "mode": "mesh", + "v": v, "n": n, "uv": uv, "c": c, "i": idx}) + bz.p = nxt + if geotype == "pmesh" and pend_v is not None: + idx = [] + for poly in conns: + for t in range(1, len(poly) - 1): + idx += [poly[0], poly[t], poly[t + 1]] + out_groups.append({"material": material, "mode": "mesh", + "v": pend_v, "n": pend_n, "uv": pend_uv, + "c": pend_c, "i": idx}) + +def _parse_bgf_patch(bz, length, out_groups, obj_mtl): + end = bz.p + length + f_mtl = obj_mtl + pend = [] # geometry blocks seen before the material tag + while bz.p < end: + tag, ln = bz.block() + nxt = bz.p + ln + if tag == 0x2008: bz.cstr() + elif tag in (0x2030, 0x2031): + t = bz.u8() + name = bz.cstr() if t == 1 else (None if t != 2 else "DEFAULT") + if tag == 0x2030: f_mtl = name or f_mtl + elif tag == 0x43: _parse_bgf_vertices(bz, ln, "polygon", pend, None) + elif tag == 0x44: _parse_bgf_vertices(bz, ln, "tristrip", pend, None) + elif tag == 0x45: _parse_bgf_vertices(bz, ln, "tristrip", pend, None) + elif tag == 0x46: _parse_bgf_vertices(bz, ln, "pmesh", pend, None) + bz.p = nxt + for g in pend: + g["material"] = f_mtl + out_groups.append(g) + +def parse_bgf(path): + """Parse DIV-BIZ2 geometry -> groups like parse_vgf. Also returns any + embedded materials/textures/ramps.""" + bz = B2Z(path) + if bz.d[:8] != b"DIV-BIZ2": + raise ValueError("not a DIV-BIZ2 file: " + path) + bz.p = 8 + groups, materials, textures, ramps = [], {}, {}, {} + while bz.p < len(bz.d) - 2: + tag, ln = bz.block() + nxt = bz.p + ln + top = tag & 0xfff + if top == 0x005: break # trailer + elif top == 0x003 or top == 0x004: pass + elif top == 0x010: _parse_biz_texture(bz, ln, textures) + elif top == 0x020: _parse_biz_material(bz, ln, materials) + elif top == 0x030: _parse_biz_ramp(bz, ln, ramps) + elif top == 0x040: _parse_bgf_object(bz, ln, groups) + bz.p = nxt + return groups, materials, textures, ramps + +def _parse_bgf_lod(bz, length, groups, obj_mtl): + end = bz.p + length + while bz.p < end: + tag, ln = bz.block() + nxt = bz.p + ln + if tag == 0x42: _parse_bgf_patch(bz, ln, groups, obj_mtl) + bz.p = nxt + +def _parse_bgf_object(bz, length, groups): + end = bz.p + length + obj_mtl = None + lods = [] # (size, offset) of nested LOD blocks + while bz.p < end: + tag, ln = bz.block() + nxt = bz.p + ln + if tag == 0x2008: bz.cstr() + elif tag == 0x2030: + t = bz.u8() + if t == 1: obj_mtl = bz.cstr() + elif tag == 0x2031: + t = bz.u8() + if t == 1: bz.cstr() + elif tag == 0x42: _parse_bgf_patch(bz, ln, groups, obj_mtl) + elif tag == 0x41: lods.append((ln, bz.p)) + bz.p = nxt + if lods: + # newer writers nest patches inside LODs (Division's 1994 reader + # predates this); take the largest LOD = highest detail + ln, off = max(lods) + bz.p = off + _parse_bgf_lod(bz, ln, groups, obj_mtl) + bz.p = end + +def _parse_biz_material(bz, length, materials): + end = bz.p + length + m = {"ambient": None, "diffuse": None, "emissive": None, + "texture": None, "ramp": None, "immune": False} + name = "unnamed" + while bz.p < end: + tag, ln = bz.block() + nxt = bz.p + ln + if tag == 0x2008: name = bz.cstr() + elif tag == 0x0021: + mode = bz.u8() + if mode == 2: m["texture"] = bz.cstr() + elif tag == 0x0023: m["ambient"] = [r3(x) for x in bz.floats(3)] + elif tag == 0x0024: m["diffuse"] = [r3(x) for x in bz.floats(3)] + elif tag == 0x0026: m["emissive"] = [r3(x) for x in bz.floats(3)] + elif tag == 0x0028: m["ramp"] = bz.cstr() + bz.p = nxt + materials[name] = m + +def _parse_biz_texture(bz, length, textures): + end = bz.p + length + name, mapname = None, None + while bz.p < end: + tag, ln = bz.block() + nxt = bz.p + ln + if tag == 0x2008: name = bz.cstr() + elif tag == 0x0011: mapname = bz.cstr() + bz.p = nxt + if name: textures[name] = {"map": (mapname or "").lower() or None, "slice": None} + +def _parse_biz_ramp(bz, length, ramps): + end = bz.p + length + name, data = None, None + while bz.p < end: + tag, ln = bz.block() + nxt = bz.p + ln + if tag == 0x2008: name = bz.cstr() + elif tag == 0x0031: + f = bz.floats(6); data = [list(f[0:3]), list(f[3:6])] + bz.p = nxt + if name and data: ramps[name] = data + +# ================================================================ textures +def tga_datauri(path): + d = open(path, "rb").read() + idlen, imgtype = d[0], d[2] + w, h = struct.unpack("= 3: pts.append([r3(v) for v in nums[:3]]) + return pts + +def parse_scn(path): + scene = {"lights": [], "placements": []} + last = None + for raw in open(path, errors="replace"): + line = raw.split("//")[0].split("#")[0].strip() + if not line: continue + t = line.split() + k = t[0].upper() + try: + if k == "VIEWANGLE": scene["fov"] = float(t[1]) + elif k == "CLIP": scene["clip"] = [float(t[1]), float(t[2])] + elif k == "FOG": scene["fog"] = [float(x) for x in t[1:6]] + elif k == "BACKGND": scene["bg"] = [float(x) for x in t[1:4]] + elif k == "AMBIENT": scene["ambient"] = [float(x) for x in t[1:4]] + elif k == "LIGHT": scene["lights"].append([float(x) for x in t[1:7]]) + elif k == "START": scene["start"] = [float(x) for x in t[1:7]] + elif k in ("GEOMETRY", "MATERIAL", "TEXTURE"): scene[k.lower()] = t[1] + elif k == "STATIC": + last = {"kind": "static", "geo": t[1].lower(), "scale": float(t[2]), + "pos": [float(x) for x in t[3:6]], "rot": [float(x) for x in t[6:9]], + "children": []} + scene["placements"].append(last) + elif k == "DYNAMIC": + last = {"kind": "dynamic", "geo": t[1].lower(), "scale": float(t[2]), + "spl": os.path.basename(t[3].replace("\\", "/")).lower(), + "phase": float(t[4]), "speed": float(t[5]), "children": []} + scene["placements"].append(last) + elif k in ("SCHILD", "DCHILD") and last is not None: + last["children"].append({"geo": t[1].lower(), "scale": float(t[2])}) + except (ValueError, IndexError): + pass + return scene diff --git a/restoration/extract_all.py b/restoration/extract_all.py new file mode 100644 index 0000000..7f0f4c0 --- /dev/null +++ b/restoration/extract_all.py @@ -0,0 +1,231 @@ +"""Extract every recoverable scene on the Glaze drive to one master JSON.""" +import os, re, json, hashlib, traceback +from collections import defaultdict +import divformats as df + +SDA4 = r"c:\VWE\TeslaRel410\sda4" +MIN_COVERAGE = 0.6 + +# ---------------------------------------------------------------- file index +# sda4 dev drive + the release repo's CONTENT (same-era Division files) +ROOTS = [SDA4, r"c:\VWE\TeslaRel410\CONTENT"] +index = defaultdict(list) +for r in ROOTS: + for root, dirs, files in os.walk(r): + dirs[:] = [d for d in dirs if d.upper() not in ("EDITBACK", "RECYCLED")] + for f in files: + base, ext = os.path.splitext(f.lower()) + index[(base, ext)].append(os.path.join(root, f)) + +def nearest(cands, refdir): + """Pick candidate path sharing the longest prefix with refdir.""" + def score(p): + a, b = p.lower(), refdir.lower() + n = 0 + while n < min(len(a), len(b)) and a[n] == b[n]: n += 1 + return n + return max(cands, key=score) + +def find(base, exts, refdir): + for ext in exts: + c = index.get((base.lower(), ext)) + if c: return nearest(c, refdir) + return None + +# ---------------------------------------------------------------- caches +model_cache = {} # abspath -> model id +models = {} # id -> groups +image_cache = {} # abspath -> image id +images = {} # id -> datauri +mat_file_cache = {} # abspath -> (textures, materials, ramps) + +def load_model(path): + if path in model_cache: return model_cache[path] + mid = "m%d" % len(models) + if path.lower().endswith(".vgf"): + groups = df.parse_vgf(path) + extra = ({}, {}, {}) + else: + groups, m, t, r = df.parse_bgf(path) + extra = (m, t, r) + models[mid] = groups + model_cache[path] = (mid, extra) + return model_cache[path] + +def load_image(path): + if path in image_cache: return image_cache[path] + try: + uri = df.tga_datauri(path) if path.lower().endswith(".tga") else df.svt_datauri(path) + except Exception: + return None + iid = "i%d" % len(images) + images[iid] = uri + image_cache[path] = iid + return iid + +def load_mat_file(path): + if path in mat_file_cache: return mat_file_cache[path] + if path.lower().endswith(".vmf"): + t, m, r = df.parse_vmf(path, "x") + m = {k.split(":", 1)[1]: v for k, v in m.items()} + else: + g, m, t, r = df.parse_bgf(path) + mat_file_cache[path] = (t, m, r) + return mat_file_cache[path] + +def resolve_texture_image(tex, refdir): + """tex: {map, slice} -> image id or None""" + if not tex or not tex.get("map"): return None + m, sl = tex["map"], tex.get("slice") + tries = [] + if sl is not None: tries.append(("%s%d" % (m, sl + 1), (".tga",))) + dm = re.match(r"(.+?)(\d)$", m) + tries.append((m, (".tga", ".svt"))) + if dm: tries.append((dm.group(1) + dm.group(2), (".tga", ".svt"))) + for base, exts in tries: + p = find(base, exts, refdir) + if p: return load_image(p) + return None + +def resolve_material(ref, refdir, embedded): + """ref like 'file:name' or 'name'. Returns resolved material dict or None.""" + if not ref: return None + if ":" in ref: fbase, name = ref.split(":", 1) + else: fbase, name = None, ref + # 1) material file .vmf/.bmf + if fbase: + p = find(fbase, (".vmf", ".bmf"), refdir) + if p: + t, m, r = load_mat_file(p) + if name in m: + return finish_material(m[name], t, r, refdir) + # 2) embedded in the BGF itself + em, et, er = embedded + if name in em: + return finish_material(em[name], et, er, refdir) + if fbase: # sometimes embedded keyed with prefix + if ref in em: return finish_material(em[ref], et, er, refdir) + return None + +def finish_material(m, texdict, rampdict, refdir): + out = {"ambient": m.get("ambient"), "diffuse": m.get("diffuse"), + "emissive": m.get("emissive"), "immune": m.get("immune", False)} + tex = m.get("texture") + if tex: + tinfo = texdict.get(tex) + if tinfo is None and re.match(r".*_tex$", tex): + tinfo = {"map": tex[:-4], "slice": None} + # heuristic: trailing digit in texture name selects TGA slice + if tinfo and tinfo.get("slice") is None: + dm = re.match(r"(.+?)(\d)_?tex$", tex) + if dm and tinfo.get("map") and not re.search(r"\d$", tinfo["map"]): + tinfo = {"map": tinfo["map"], "slice": int(dm.group(2)) - 1} + out["img"] = resolve_texture_image(tinfo, refdir) + rname = m.get("ramp") + if rname and rname in rampdict: + out["ramp"] = rampdict[rname] + return out + +# ---------------------------------------------------------------- scenes +def dos_to_real(dospath, scndir): + """Map a scene's GEOMETRY/MATERIAL dir (DOS path) onto the dump.""" + if not dospath: return scndir + p = dospath.replace("\\", "/").strip() + p = re.sub(r"^[A-Za-z]:", "", p).lstrip("/") + cand = os.path.join(SDA4, *p.split("/")) + if os.path.isdir(cand): return cand + return scndir + +seen_hashes = set() +scenes_out = {} +skipped = [] + +PRIORITY = ["STDAVE", "HPDAVE", "CANYON", "BTDAVE", "BTRAVINE", "RPDAVE", "ERIC", + "FX", "RD", "CONVERT", "DPL3RLS", "DPL3", "PROBLEMS", "SPHERES"] +def prio(rel): + top = rel.split(os.sep)[0].upper() + return PRIORITY.index(top) if top in PRIORITY else 99 + +all_scn = [] +for root, dirs, files in os.walk(SDA4): + dirs[:] = [d for d in dirs if d.upper() not in ("EDITBACK", "RECYCLED")] + for f in files: + if f.lower().endswith(".scn"): + all_scn.append(os.path.join(root, f)) +all_scn.sort(key=lambda p: (prio(os.path.relpath(p, SDA4)), p)) + +for scn_path in all_scn: + rel = os.path.relpath(scn_path, SDA4) + try: + content = open(scn_path, "rb").read() + h = hashlib.md5(content).hexdigest() + if h in seen_hashes: + skipped.append((rel, "duplicate")); continue + scene = df.parse_scn(scn_path) + pls = scene["placements"] + if not pls: + skipped.append((rel, "no placements")); continue + scndir = os.path.dirname(scn_path) + geodir = dos_to_real(scene.get("geometry"), scndir) + + needed = [] + for p in pls: + needed.append(p["geo"].split(":")[0]) + for c in p["children"]: needed.append(c["geo"].split(":")[0]) + uniq = sorted(set(needed)) + found_models, missing = {}, [] + for g in uniq: + fp = find(g, (".vgf",), geodir) or find(g, (".bgf",), geodir) + if fp: found_models[g] = fp + else: missing.append(g) + if len(found_models) == 0 or len(found_models) / len(uniq) < MIN_COVERAGE: + skipped.append((rel, "coverage %d/%d" % (len(found_models), len(uniq)))) + continue + + # load models + materials + used_models, used_mats = {}, {} + for g, fp in found_models.items(): + mid, embedded = load_model(fp) + used_models[g] = mid + for grp in models[mid]: + mref = grp.get("material") + if mref and mref not in used_mats: + r = resolve_material(mref, os.path.dirname(fp), embedded) + if r is None: + r = resolve_material(mref, dos_to_real(scene.get("material"), scndir), embedded) + used_mats[mref] = r or {} + # splines + spls = {} + for p in pls: + if p["kind"] == "dynamic": + sp = find(os.path.splitext(p["spl"])[0], (".spl",), scndir) + if sp: spls[p["spl"]] = df.parse_spl(sp) + placements = [p for p in pls if p["geo"].split(":")[0] in used_models + and (p["kind"] != "dynamic" or p["spl"] in spls)] + for p in placements: + p["children"] = [c for c in p["children"] if c["geo"].split(":")[0] in used_models] + + scenes_out[rel.replace(os.sep, "/")] = { + "fov": scene.get("fov", 60), "clip": scene.get("clip", [1, 5000]), + "fog": scene.get("fog"), "bg": scene.get("bg", [0, 0, 0]), + "ambient": scene.get("ambient", [0.2, 0.2, 0.2]), + "lights": scene.get("lights", []), "start": scene.get("start"), + "placements": placements, "modelmap": used_models, + "materials": used_mats, "splines": spls, "missing": missing, + } + seen_hashes.add(h) + nv = sum(len(g.get("v", g.get("points", []))) for m in used_models.values() for g in models[m]) + print("OK %-46s models=%d/%d verts=%d" % (rel, len(found_models), len(uniq), nv)) + except Exception as e: + skipped.append((rel, "ERROR " + str(e))) + print("ERR %-46s %s" % (rel, e)) + traceback.print_exc() + +out = {"models": models, "images": images, "scenes": scenes_out} +dst = os.path.join(os.path.dirname(os.path.abspath(__file__)), "allscenes.json") +with open(dst, "w") as fh: + json.dump(out, fh, separators=(",", ":")) +print("\n%d scenes extracted, %d skipped" % (len(scenes_out), len(skipped))) +print("models=%d images=%d size=%.1f MB" % (len(models), len(images), os.path.getsize(dst) / 1e6)) +with open("extract_report.txt", "w") as fh: + for rel, why in skipped: fh.write("%s\t%s\n" % (rel, why)) diff --git a/restoration/gallery_build.py b/restoration/gallery_build.py new file mode 100644 index 0000000..8437948 --- /dev/null +++ b/restoration/gallery_build.py @@ -0,0 +1,111 @@ +"""Build the gallery HTML: curated scenes from allscenes.json injected into +gallery_template.html.""" +import json, os + +CURATED = [ + # (scene key, project, display name, blurb) + ("STDAVE/TREK.SCN", "Star Trek", "Enterprise-D", + "The Enterprise-D at rest among three parallax layers of drifting stars."), + ("STDAVE/KLNGVID.SCN", "Star Trek", "Klingon flyby", + "A Klingon cruiser at rest while two more drift past on spline paths."), + ("STDAVE/TREKVID.SCN", "Star Trek", "Enterprise flyby", + "Camera-pass variant of the Enterprise scene, built for video capture."), + ("HPDAVE/SDEMO.SCN", "Hull Pressure", "Sea demo", + "The big Hull Pressure demo: seafloor, wrecks, and marine life on spline paths."), + ("HPDAVE/FISHSPLS.SCN", "Hull Pressure", "Fish schools", + "Schools of fish and sharks swimming spline circuits over the seabed."), + ("HPDAVE/BDDEMO.SCN", "Hull Pressure", "Big demo", + "Full underwater environment demo for the unreleased submarine game."), + ("HPDAVE/TMPRIGS.SCN", "Hull Pressure", "Temple rigs", + "Sunken temple environment with rig structures."), + ("HPDAVE/DUANE.SCN", "Hull Pressure", "Duane's scene", + "An underwater scene named for one of the team."), + ("HPDAVE/SANDTEST.SCN", "Hull Pressure", "Sand test", + "Seafloor sand and lighting test."), + ("BTDAVE/MAPS/POLAR4.SCN", "BattleTech", "Polar map", + "The polar arena environment, assembled from 99 models."), + ("BTRAVINE/RAVTEST.SCN", "BattleTech", "Ravine map", + "The Ravine arena in development — this environment shipped as RAV/RAV1."), + ("BTRAVINE/DESTEST.SCN", "BattleTech", "Desert test", + "Desert environment test with full map furniture."), + ("BTDAVE/MAPS/DTEST.SCN", "BattleTech", "Desert map test", + "A smaller desert map assembly test."), + ("CONVERT/BTECH/STAD/STAD.SCN", "BattleTech", "Stadium", + "Arena stadium model in the art-conversion workspace."), + ("DPL3RLS/SCENES/MISSILE.SCN", "BattleTech", "Missile test", + "Weapon-effects staging scene with arena props."), + ("RPDAVE/SCENES/BLADE.SCN", "Red Planet", "Blade arena", + "The Blade racing arena — late Red Planet development work."), + ("CONVERT/MILESTON/MILESTON.SCN", "Red Planet", "Milestone", + "The Red Planet milestone demo: Mars mining-canal raceway."), + ("CONVERT/MILESTON/WELLENT.SCN", "Red Planet", "Well entrance", + "Raceway well-entrance sequence from the milestone build."), + ("CONVERT/RD/CHOICE2.SCN", "Red Planet", "Choice point", + "A raceway fork (“choice”) section of the Mars canals."), + ("RD/VTER.SCN", "Red Planet", "Terrain", + "Generated Mars terrain heightfield test."), + ("CANYON/CANYON.SCN", "Canyon demo", "Canyon", + "River-canyon flythrough environment — Division demo work."), + ("CANYON/CANABOVE.SCN", "Canyon demo", "Canyon from above", + "The canyon seen from altitude."), + ("CANYON/TEMPLE.SCN", "Canyon demo", "Maya temple", + "Mayan temple set piece at the canyon rim."), + ("ERIC/VANSMAK.SCN", "Misc", "Van's MAK", + "A 31-model scene from developer Eric's directory."), + ("HPDAVE/BATEST.SCN", "Hull Pressure", "Bathysphere test", + "Single-model test of the submersible."), + ("CONVERT/CALIB/DIVCAL.SCN", "Misc", "Division calibration", + "Display-calibration model used to align the pod projectors."), +] + +# camera overrides for scenes whose START sits inside geometry (dev scenes); +# None = use the viewer's auto-framing instead +START_OVERRIDE = { + "BTRAVINE/RAVTEST.SCN": [280, 30, 160, -8, 25, 0], + "CANYON/CANYON.SCN": None, + "CANYON/CANABOVE.SCN": None, + "CANYON/TEMPLE.SCN": None, + "BTDAVE/MAPS/DTEST.SCN": None, + "CONVERT/CALIB/DIVCAL.SCN": None, +} + +here = os.path.dirname(os.path.abspath(__file__)) +data = json.load(open(os.path.join(here, "allscenes.json"))) + +out = {"models": {}, "images": {}, "scenes": {}, "catalog": []} +for key, project, name, blurb in CURATED: + sc = data["scenes"].get(key) + if not sc: + print("MISSING SCENE:", key) + continue + if key in START_OVERRIDE: + sc = dict(sc); sc["start"] = START_OVERRIDE[key] + if sc["start"] is None: del sc["start"] + out["scenes"][key] = sc + out["catalog"].append({"key": key, "project": project, "name": name, "blurb": blurb}) + for mid in sc["modelmap"].values(): + out["models"][mid] = data["models"][mid] + for mat in sc["materials"].values(): + if mat and mat.get("img"): + out["images"][mat["img"]] = data["images"][mat["img"]] + +# per-model bounding radius for auto-framing +for mid, groups in out["models"].items(): + lo = [1e9] * 3; hi = [-1e9] * 3 + for g in groups: + for v in g.get("v", g.get("points", [])): + for i in range(3): + lo[i] = min(lo[i], v[i]); hi[i] = max(hi[i], v[i]) + if lo[0] > hi[0]: lo = hi = [0, 0, 0] + for g in groups: pass + c = [(lo[i] + hi[i]) / 2 for i in range(3)] + r = max(hi[i] - lo[i] for i in range(3)) / 2 or 1 + out.setdefault("bounds", {})[mid] = {"c": [round(x, 2) for x in c], "r": round(r, 2)} + +blob = json.dumps(out, separators=(",", ":")) +print("gallery data: %.1f MB, %d scenes, %d models, %d images" + % (len(blob) / 1e6, len(out["scenes"]), len(out["models"]), len(out["images"]))) +tpl = open(os.path.join(here, "gallery_template.html"), encoding="utf-8").read() +open(os.path.join(here, "vwe-archive.html"), "w", encoding="utf-8").write( + tpl.replace("%%DATA%%", blob)) +print("wrote vwe-archive.html", os.path.getsize(os.path.join(here, "vwe-archive.html")) / 1e6, "MB") diff --git a/restoration/gallery_template.html b/restoration/gallery_template.html new file mode 100644 index 0000000..ab86117 --- /dev/null +++ b/restoration/gallery_template.html @@ -0,0 +1,648 @@ +VWE Scene Archive — 1994–1997 + + + +
RESTORING SCENES…
+ + + + +
+

+

+

+
+ +
+ drag look   W A S D fly   Q E rise/sink   shift fast   R reset +
+ + + + + + + diff --git a/restoration/vwe-archive.html b/restoration/vwe-archive.html new file mode 100644 index 0000000..3675560 --- /dev/null +++ b/restoration/vwe-archive.html @@ -0,0 +1,648 @@ +VWE Scene Archive — 1994–1997 + + + +
RESTORING SCENES…
+ + + + +
+

+

+

+
+ +
+ drag look   W A S D fly   Q E rise/sink   shift fast   R reset +
+ + + + + + +