#!/usr/bin/env python3 """ manifest410.py -- per-TU manifest of BTL4OPT.EXE for the 4.10 literal-source reconstruction (the "size of the mountain" measurement). Inputs (all already in the repo): reference/decomp/all/part_*.c -- Ghidra export; every function is headed by /* @00ADDR file=TAG name=NAME */ reference/decomp/section_dump.txt -- hex dump of the data sections (VA -> bytes); used to resolve s_*_00ADDR string symbols to their REAL contents (assert file paths). game/original/BT/BT.MAK, BT_L4/BTL4.MAK -- the authoritative TU census + link order for bt.lib (36 TUs) / btl4.lib (13 TUs). game/reconstructed/*.cpp|*.hpp -- the port; @00ADDR annotations are harvested to measure which binary ranges are already semantically decoded. Attribution signals, strongest first: 1. an assert string inside the body naming the source file (+ line number); 2. the exporter's file= tag; 3. link-order fill: a function between two anchors of the SAME TU inherits it (confident); between anchors of DIFFERENT TUs it lands in a boundary zone (uncertain -- counted separately, never claimed). Outputs: reference/BT410_SOURCE_MANIFEST.md -- human summary tables reference/BT410_SOURCE_MANIFEST.json -- machine form (per-function attribution) Usage: py -3 tools/manifest410.py (from the BT411 repo root) """ import json import os import re import sys from collections import defaultdict HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(HERE) DECOMP_DIR = os.path.join(ROOT, "reference", "decomp", "all") SECTION_DUMP = os.path.join(ROOT, "reference", "decomp", "section_dump.txt") RECON_DIR = os.path.join(ROOT, "game", "reconstructed") ORIG_BT = os.path.join(ROOT, "game", "original", "BT") ORIG_BTL4 = os.path.join(ROOT, "game", "original", "BT_L4") OUT_MD = os.path.join(ROOT, "reference", "BT410_SOURCE_MANIFEST.md") OUT_JSON = os.path.join(ROOT, "reference", "BT410_SOURCE_MANIFEST.json") # ---------------------------------------------------------------- census ---- # The authoritative TU lists, in LINK ORDER, from the surviving makefiles. BT_LIB_TUS = [ # game/original/BT/BT.MAK BT_OBJS "btmssn", "messmgr", "mechdmg", "dmgtable", "mech", "mech2", "mech3", "mech4", "mechsub", "mechtech", "heat", "mechmppr", "powersub", "sensor", "gnrator", "gyro", "torso", "hud", "myomers", "mechweap", "emitter", "ppc", "projweap", "mislanch", "ammobin", "gauss", "projtile", "misthrst", "seeker", "missile", "btplayer", "btteam", "btdirect", "btreg", "btcnsl", "bttool", ] BTL4_LIB_TUS = [ # game/original/BT_L4/BTL4.MAK BTL4_OBJS "btl4mode", "btl4rdr", "btl4gaug", "btl4gau2", "btl4gau3", "btl4grnd", "btl4galm", "btl4vid", "btl4mppr", "btl4arnd", "btl4mssn", "btl4app", "btl4pb", ] MAIN_TUS = ["btl4"] # btl4.obj, linked directly def tu_key(prefix, name): return "%s/%s.cpp" % (prefix, name) CENSUS = ( [tu_key("bt", n) for n in BT_LIB_TUS] + [tu_key("bt_l4", n) for n in BTL4_LIB_TUS] + [tu_key("bt_l4", n) for n in MAIN_TUS] ) CENSUS_SET = set(CENSUS) # ------------------------------------------------------- section dump ------- def load_string_table(): """Parse section_dump.txt (objdump -s style: ' VA GGGGGGGG x4 ascii') into (VA->byte map, [(section_name, lo, hi)]); strings are read lazily via read_cstr().""" mem = {} sections = [] # (name, lo, hi) cur_name, cur_lo, cur_hi = None, None, None pat = re.compile(r"^ ([0-9a-f]{6,8}) ((?:[0-9a-f]{2,8} ){1,4})", re.I) sec = re.compile(r"^Contents of section (\S+):") with open(SECTION_DUMP, "r", errors="replace") as f: for line in f: sm = sec.match(line) if sm: if cur_name is not None: sections.append((cur_name, cur_lo, cur_hi)) cur_name, cur_lo, cur_hi = sm.group(1), None, None continue m = pat.match(line) if not m: continue va = int(m.group(1), 16) off = 0 for group in m.group(2).split(): if len(group) % 2: continue for i in range(0, len(group), 2): mem[va + off] = int(group[i:i + 2], 16) off += 1 if cur_lo is None: cur_lo = va cur_hi = va + off if cur_name is not None: sections.append((cur_name, cur_lo, cur_hi)) return mem, sections def read_cstr(mem, va, maxlen=260): out = [] for i in range(maxlen): b = mem.get(va + i) if b is None or b == 0: break out.append(chr(b) if 32 <= b < 127 else "?") return "".join(out) PATH_RE = re.compile(r"^[a-z]:[\\/].*\.(cpp|hpp|tcp|thp|c|h|asm)$", re.I) def normalize_path(p): r"""d:\tesla_bt\bt_l4\BTL4MSSN.CPP -> bt_l4/btl4mssn.cpp (last two comps).""" parts = re.split(r"[\\/]+", p.lower()) if len(parts) >= 2: return "%s/%s" % (parts[-2], parts[-1]) return parts[-1] # ------------------------------------------------------------ decomp -------- FUNC_HDR = re.compile(r"^/\* @([0-9a-f]{8}) file=(\S+) name=(\S+) \*/") SYM_LINE = re.compile(r"s_\w+?_(00[0-9a-f]{6})\s*,\s*(0x[0-9a-f]+|\d+)\s*[),]") # every symbolic reference into the image (code or data) ANY_REF = re.compile(r"\b(?:s_\w+?|DAT|PTR_FUN|PTR_DAT|PTR_LAB|UNK)_(00[0-9a-f]{6})\b") def parse_decomp(mem): """Return sorted list of dicts: addr, name, tag, part, paths{norm:[lines]}, refs[set of VAs].""" funcs = [] for fn in sorted(os.listdir(DECOMP_DIR)): if not fn.endswith(".c"): continue part = fn cur = None with open(os.path.join(DECOMP_DIR, fn), "r", errors="replace") as f: for line in f: m = FUNC_HDR.match(line) if m: cur = { "addr": int(m.group(1), 16), "tag": m.group(2), "name": m.group(3), "part": part, "paths": defaultdict(list), # norm path -> [line numbers] "refs": set(), # referenced VAs } funcs.append(cur) continue if cur is None: continue for sm in SYM_LINE.finditer(line): va = int(sm.group(1), 16) s = read_cstr(mem, va) if PATH_RE.match(s): cur["paths"][normalize_path(s)].append(int(sm.group(2), 0)) for sm in ANY_REF.finditer(line): va = int(sm.group(1), 16) cur["refs"].add(va) s = read_cstr(mem, va) if PATH_RE.match(s): cur["paths"][normalize_path(s)] # touch (no line info) funcs.sort(key=lambda d: d["addr"]) return funcs # ------------------------------------------------------- attribution -------- # # Anchors are sparse (release build: only hard Fail() sites carry file # strings), so attribution leans on a second linker invariant: each OBJ's # DATA (vtables, literals, statics) is laid out contiguously per section in # the SAME link order as its code. Every data VA referenced by an anchored # function becomes a known (VA -> TU) point; an unanchored function's data # refs then VOTE: a ref whose nearest known neighbors below AND above in the # same section agree contributes one vote for that TU. Assignments feed # back as new known points/anchors and the pass iterates to a fixed point, # then plain link-order fill closes the gaps between agreeing code anchors. # def _data_sections(sections): out = [(n, lo, hi) for (n, lo, hi) in sections if n.upper() != "CODE" and lo is not None] # .bss is not in the dump (no contents) but its statics are laid out in # the same per-OBJ link order -- treat the VA range above the last dumped # data section as a virtual section so bss refs can vote too. data_hi = max(hi for (n, lo, hi) in out if n.upper() == "DATA") out.append((".bss~", data_hi, 0x540000)) return out # Verified TU anchor addresses from game/reconstructed/CLASSMAP.md and the # context files ([T1] each: ctors / named functions whose owning TU is # established). These seed the data-cluster voting where Fail() strings are # absent, and let the link-order sanity check run. SEED_ANCHORS = { 0x49B54C: "bt/btmssn.cpp", # BTMission ctor 0x49BCA4: "bt/messmgr.cpp", # SubsystemMessageManager ctor 0x49DE14: "bt/dmgtable.cpp", # DamageZonePercentTable SelectZone 0x49EA48: "bt/dmgtable.cpp", # DamageLookupTable ctor 0x49F624: "bt/mech.cpp", # mech<->player bind 0x4A2D48: "bt/mech.cpp", # Mech::Make 0x4ADDA0: "bt/heat.cpp", # heat subsystem ctor (file= tagged) 0x4AFBE0: "bt/mechmppr.cpp", # CycleControlMode 0x4B0EFC: "bt/powersub.cpp", # PoweredSubsystem::HandleMessage 0x4B3778: "bt/gyro.cpp", # Gyroscope ctor 0x4B6918: "bt/torso.cpp", # Torso::Recenter 0x4B8D18: "bt/myomers.cpp", # MyomersSimulation 0x4B9728: "bt/mechweap.cpp", # MechWeapon::SendDamageMessage 0x4BBFCC: "bt/projweap.cpp", # CheckForJam 0x4BEF78: "bt/missile.cpp", # Missile MoveAndCollide 0x4C0BC8: "bt/btplayer.cpp", # BTPlayer ctor 0x4C1C24: "bt_l4/btl4rdr.cpp", # MapDisplay ctor 0x4C2F94: "bt_l4/btl4gaug.cpp", # gauge TU start (CLASSMAP extent) 0x4C6798: "bt_l4/btl4gau2.cpp", # gau2 TU start (CLASSMAP extent) 0x4CC40C: "bt_l4/btl4vid.cpp", # BTReticleRenderable ctor 0x4D1AE4: "bt_l4/btl4mppr.cpp", # L4MechControlsMapper::SetControlMode 0x4D2C30: "bt_l4/btl4mssn.cpp", # BTL4Mission::SetPlayerData 0x4D34C4: "bt_l4/btl4app.cpp", # BTL4Application ctor 0x4D3BE4: "bt_l4/btl4pb.cpp", # BTL4PlaybackApplication ctor } def _section_of(va, dsecs): for i, (_, lo, hi) in enumerate(dsecs): if lo <= va < hi: return i return None def attribute(funcs, sections, recon_affinity=None): """Set f['file'] + f['how'] in {assert, seed, tag, dp, rtl, none}. recon_affinity: {addr: tu_key} soft votes from name-matched game/reconstructed files' @address annotations.""" dsecs = _data_sections(sections) recon_affinity = recon_affinity or {} # pass 1: direct evidence for f in funcs: if f["paths"]: best = max( f["paths"].items(), key=lambda kv: (len(kv[1]) > 0, len(kv[1]), kv[0]), )[0] f["file"], f["how"] = best, "assert" elif f["addr"] in SEED_ANCHORS: f["file"], f["how"] = SEED_ANCHORS[f["addr"]], "seed" elif f["tag"] != "?": f["file"], f["how"] = f["tag"].lower(), "tag" else: f["file"], f["how"] = None, "none" # ---- pass 2: monotone DP labeling of the BT region ------------------- # # bt.lib + btl4.lib are the last game libs before the Borland RTL, so # inside [first BT anchor .. RTL sentinel) the TU label sequence along # the address axis must be NON-DECREASING in makefile link order. A DP # picks, among all monotone labelings that respect the anchors, the one # maximizing data-cluster affinity. The REGION TOTALS are exact under # the sentinels even where an individual TU boundary is soft. # import bisect # anchor-known data points (assert/seed/tag only -- no cascade); VAs # referenced from >1 file are shared globals: non-discriminative. claims = defaultdict(set) for f in funcs: if f["file"] is not None: for va in f["refs"]: claims[va].add(f["file"]) shared = {va for va, owners in claims.items() if len(owners) > 1} known = [[] for _ in dsecs] for f in funcs: if f["file"] is None: continue for va in f["refs"]: if va not in shared: si = _section_of(va, dsecs) if si is not None: known[si].append((va, f["file"])) for k in known: k.sort() def affinity(f): votes = defaultdict(float) for va in f["refs"]: if va in shared: continue si = _section_of(va, dsecs) if si is None or not known[si]: continue k = known[si] i = bisect.bisect_left(k, (va, "")) if i < len(k) and k[i][0] == va: votes[k[i][1]] += 2.0 # unique exact VA match continue lo = k[i - 1][1] if i > 0 else None hi = k[i][1] if i < len(k) else None if lo is not None and lo == hi: votes[lo] += 1.0 # inside one TU's data cluster return votes RTL_SENTINEL = 0x4D4B58 # first known Borland RTL fn # btl4.obj is linked right after the startup OBJ (see BTL4.MAK tlink32 # response), i.e. at the BOTTOM of the image -- it is NOT part of the # bt.lib/btl4.lib region and is excluded from the ordered labeling. labels = [t for t in CENSUS if t != "bt_l4/btl4.cpp"] + [""] label_ix = {t: i for i, t in enumerate(labels)} lo_i = next(i for i, f in enumerate(funcs) if f["file"] in CENSUS_SET) hi_i = next(i for i, f in enumerate(funcs) if f["addr"] >= RTL_SENTINEL) region = funcs[lo_i:hi_i + 1] NEG = -1e18 n, m = len(region), len(labels) score = [[0.0] * m for _ in range(n)] for r, f in enumerate(region): if f["file"] in label_ix: # anchored: pin it for c in range(m): score[r][c] = 0.0 if c == label_ix[f["file"]] else NEG elif f["addr"] >= RTL_SENTINEL: for c in range(m): score[r][c] = 0.0 if labels[c] == "" else NEG else: votes = affinity(f) t = recon_affinity.get(f["addr"]) if t is not None: votes[t] += 0.75 # port citation, soft for t, v in votes.items(): if t in label_ix: score[r][label_ix[t]] = v # DP: best[r][c] = max over c' <= c of best[r-1][c'] + score[r][c] best = [row[:] for row in score] back = [[0] * m for _ in range(n)] for c in range(1, m): if best[0][c] < best[0][c - 1]: best[0][c] = best[0][c - 1] for r in range(1, n): run_best, run_ix = NEG, 0 for c in range(m): if best[r - 1][c] > run_best: run_best, run_ix = best[r - 1][c], c back[r][c] = run_ix best[r][c] = (run_best + score[r][c]) if run_best > NEG / 2 \ and score[r][c] > NEG / 2 else NEG # trace back from the best final cell c = max(range(m), key=lambda cc: best[n - 1][cc]) for r in range(n - 1, -1, -1): f = region[r] if f["file"] is None: f["file"] = labels[c] f["how"] = "dp" c = back[r][c] for f in funcs: if f["file"] == "": f["file"], f["how"] = None, "rtl" return funcs # ------------------------------------------------- reconstructed map -------- ADDR_ANNOT = re.compile(r"@\s?(00[0-9a-f]{6})\b|@([0-9a-f]{6})\b", re.I) def recon_addresses(): """Harvest every @00XXXXXX annotation from game/reconstructed sources.""" hits = defaultdict(set) # file -> set of addrs for fn in sorted(os.listdir(RECON_DIR)): if not fn.endswith((".cpp", ".hpp", ".md")): continue with open(os.path.join(RECON_DIR, fn), "r", errors="replace") as f: text = f.read() for m in ADDR_ANNOT.finditer(text): a = int(m.group(1) or m.group(2), 16) if 0x400000 <= a <= 0x540000: hits[fn].add(a) return hits # ------------------------------------------------------ survivors ----------- def survivors(): """Map census key -> 'full' | 'partial(TCP)' from game/original.""" out = {} for d, prefix in ((ORIG_BT, "bt"), (ORIG_BTL4, "bt_l4")): if not os.path.isdir(d): continue for fn in os.listdir(d): base, ext = os.path.splitext(fn.lower()) key = tu_key(prefix, base) if ext == ".cpp" and key in CENSUS_SET: out[key] = "full" elif ext == ".tcp": k2 = tu_key(prefix, base) if k2 in CENSUS_SET and out.get(k2) != "full": out[k2] = "partial(TCP)" return out # ------------------------------------------------------------- main --------- def main(): print("loading section dump ...") mem, sections = load_string_table() print(" %d bytes mapped, sections: %s" % ( len(mem), ", ".join("%s@%x-%x" % s for s in sections))) print("parsing decomp ...") funcs = parse_decomp(mem) print(" %d functions" % len(funcs)) # soft affinity: a recon file named like a census TU votes for that TU # at every address it cites (cross-citations are outvoted by the DP's # monotone constraint) recon_pre = recon_addresses() recon_affinity = {} for fn, addrs in recon_pre.items(): base = os.path.splitext(fn)[0].lower() for key in (tu_key("bt", base), tu_key("bt_l4", base)): if key in CENSUS_SET: for a in addrs: recon_affinity[a] = key attribute(funcs, sections, recon_affinity) # sizes: delta to next function for f, g in zip(funcs, funcs[1:]): f["size"] = g["addr"] - f["addr"] funcs[-1]["size"] = 0 recon = recon_addresses() recon_all = set().union(*recon.values()) if recon else set() surv = survivors() # per-TU aggregation (census TUs + whatever engine files showed up) tus = defaultdict(lambda: { "funcs": 0, "bytes": 0, "assert": 0, "data": 0, "fill": 0, "tag": 0, "lo": None, "hi": None, "max_line": 0, "recon_hits": 0, }) boundary = 0 unattributed = 0 for f in funcs: if f["how"] == "boundary": boundary += 1 continue if f["file"] is None: unattributed += 1 continue t = tus[f["file"]] t["funcs"] += 1 t["bytes"] += f["size"] bucket = {"assert": "assert", "dp": "fill", "data": "data"}.get(f["how"], "tag") t[bucket] += 1 t["lo"] = f["addr"] if t["lo"] is None else min(t["lo"], f["addr"]) t["hi"] = max(t["hi"] or 0, f["addr"] + f["size"]) for lines in f["paths"].values(): for n in lines: t["max_line"] = max(t["max_line"], n) if f["addr"] in recon_all: t["recon_hits"] += 1 # ------------------------------------------------------------- write ---- def tbl_row(key): t = tus.get(key) s = surv.get(key, "") if not t: return "| %s | %s | — | — | — | — | — | — | 0 |" % (key, s or "—") return "| %s | %s | %d | %.1f | %06x-%06x | %d | %d | %d | %d |" % ( key, s or "—", t["funcs"], t["bytes"] / 1024.0, t["lo"], t["hi"], t["assert"], t["data"] + t["fill"], t["max_line"], t["recon_hits"], ) bt_keys = [tu_key("bt", n) for n in BT_LIB_TUS] btl4_keys = [tu_key("bt_l4", n) for n in BTL4_LIB_TUS + MAIN_TUS] engine_keys = sorted(k for k in tus if k not in CENSUS_SET) # link-order sanity: census TU code ranges must ascend in makefile order order_violations = [] prev = None for k in bt_keys + btl4_keys: t = tus.get(k) if not t or t["lo"] is None: continue if prev and prev[1] > t["lo"]: order_violations.append("%s (ends %06x) overlaps %s (starts %06x)" % (prev[0], prev[1], k, t["lo"])) prev = (k, t["hi"]) bt_total = sum(tus[k]["bytes"] for k in bt_keys if k in tus) btl4_total = sum(tus[k]["bytes"] for k in btl4_keys if k in tus) bt_funcs = sum(tus[k]["funcs"] for k in bt_keys if k in tus) btl4_funcs = sum(tus[k]["funcs"] for k in btl4_keys if k in tus) hdr = ("| TU | survives | funcs | KB | addr range | assert-anchored | " "DP-inferred | max assert line | recon @addr hits |\n" "|---|---|---|---|---|---|---|---|---|") with open(OUT_MD, "w", encoding="utf-8") as f: f.write("# BT 4.10 source manifest — measured from BTL4OPT.EXE\n\n") f.write("Generated by `tools/manifest410.py`. Census = the surviving " "makefiles (BT.MAK 36 TUs, BTL4.MAK 13 TUs + btl4.obj, in " "LINK ORDER). Attribution = Fail() assert strings resolved " "via section_dump [T1 anchors] + CLASSMAP seed addresses " "[T1] + a monotone-in-link-order DP over data-section " "cluster affinity and recon @addr citations [T3 for " "unanchored boundaries]. A `—` row = TU NOT LOCALIZED: its " "code is contained inside an adjacent row's range (no " "anchor evidence yet); 8 such TUs survive as full .cpp " "anyway. `bt_l4/btl4.cpp` links at the image BOTTOM with " "the startup OBJ and is not part of this region.\n\n") f.write("Totals: **%d functions** in the image; BT-side: **%d funcs / " "%.0f KB** (bt.lib) + **%d funcs / %.0f KB** (btl4.lib+main); " "boundary-uncertain: %d; unattributed (outside any anchor " "span): %d.\n\n" % ( len(funcs), bt_funcs, bt_total / 1024.0, btl4_funcs, btl4_total / 1024.0, boundary, unattributed)) f.write("`recon @addr hits` = binary function addresses cited in " "`game/reconstructed/` — a proxy for how much of the TU is " "already semantically decoded (not a completeness claim).\n\n") f.write("## bt.lib (link order)\n\n%s\n" % hdr) for k in bt_keys: f.write(tbl_row(k) + "\n") f.write("\n## btl4.lib + main (link order)\n\n%s\n" % hdr) for k in btl4_keys: f.write(tbl_row(k) + "\n") if order_violations: f.write("\n## ⚠ Link-order violations (attribution suspects)\n\n") for v in order_violations: f.write("- %s\n" % v) f.write("\n## Engine/other TUs seen in the image (context only — " "their source survives)\n\n") f.write("| TU | funcs | KB |\n|---|---|---|\n") for k in engine_keys: t = tus[k] f.write("| %s | %d | %.1f |\n" % (k, t["funcs"], t["bytes"] / 1024.0)) with open(OUT_JSON, "w", encoding="utf-8") as f: json.dump({ "census": CENSUS, "functions": [ {"addr": "%08x" % fu["addr"], "name": fu["name"], "file": fu["file"], "how": fu["how"], "size": fu["size"], "lines": {k: v for k, v in fu["paths"].items() if v}} for fu in funcs ], }, f, indent=1) print("wrote %s\nwrote %s" % (OUT_MD, OUT_JSON)) print("BT-side: %d funcs / %.0f KB btl4-side: %d funcs / %.0f KB " "boundary %d unattributed %d" % ( bt_funcs, bt_total / 1024.0, btl4_funcs, btl4_total / 1024.0, boundary, unattributed)) if __name__ == "__main__": sys.exit(main())