vwemirror/BT411
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

4.10 literal-source campaign: manifest DP upgrade, Route A closed, toolchain facts

Browse Source 
- manifest410.py: monotone-DP TU labeling (data-cluster affinity + CLASSMAP
  seeds + recon @addr soft votes) replaces naive fill; BT region = 917 fn /
  ~230 KB, extents match every independently-known TU boundary.
- source-completeness: ROUTE A CLOSED (Nick holds no further assets);
  toolchain = BC++ 4.52 PROVEN by CW32.LIB byte-match, archived at
  TeslaRel410/BORLAND; OPT.MAK = the shipped binary's exact recipe; extender
  corrected to Borland PowerPack DPMI32 (was "Phar Lap TNT"); "all BT headers
  survive" corrected (17/36 bt-side; mech.hpp etc. reconstruction-only);
  engine gaps enumerated (vdata.hpp first, back-dated from BT412).
- phases/phase-03: rounds 1-3 of the source410 campaign - 6/10 surviving
  originals compile clean under BC4.52; console wire IDs recovered from the
  binary ctors (Killed=9 Damaged=10 ScoreUpdate=13 DWH=15 [T1], TeamScore
  guessed 12 [T4] - answers the TeslaSuite console-port spec's open item);
  round-3 finding: MECH.HPP is the capstone grown with the mech TU
  reconstructions, BTREG.CPP green = the header-family milestone.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Cyd
2026-07-19 07:33:59 -05:00
co-authored by Claude Fable 5
parent 4e01e83563
commit ec43e365fa
7 changed files with 7540 additions and 7159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/manifest410.py
+249 -43
View File
@@ -75,11 +75,21 @@ CENSUS_SET = set(CENSUS)
# ------------------------------------------------------- section dump -------
def load_string_table():
"""Parse section_dump.txt (objdump -s style: ' VA GGGGGGGG x4 ascii')
into a VA->byte map; strings are read lazily via read_cstr()."""
into (VA->byte map, [(section_name, lo, hi)]); strings are read lazily
via read_cstr()."""
mem = {}
sections = [] # (name, lo, hi)
cur_name, cur_lo, cur_hi = None, None, None
pat = re.compile(r"^ ([0-9a-f]{6,8}) ((?:[0-9a-f]{2,8} ){1,4})", re.I)
sec = re.compile(r"^Contents of section (\S+):")
with open(SECTION_DUMP, "r", errors="replace") as f:
for line in f:
sm = sec.match(line)
if sm:
if cur_name is not None:
sections.append((cur_name, cur_lo, cur_hi))
cur_name, cur_lo, cur_hi = sm.group(1), None, None
continue
m = pat.match(line)
if not m:
continue
@@ -91,7 +101,12 @@ def load_string_table():
for i in range(0, len(group), 2):
mem[va + off] = int(group[i:i + 2], 16)
off += 1
return mem
if cur_lo is None:
cur_lo = va
cur_hi = va + off
if cur_name is not None:
sections.append((cur_name, cur_lo, cur_hi))
return mem, sections
def read_cstr(mem, va, maxlen=260):
out = []
@@ -113,11 +128,13 @@ def normalize_path(p):
# ------------------------------------------------------------ decomp --------
FUNC_HDR = re.compile(r"^/\* @([0-9a-f]{8}) file=(\S+) name=(\S+) \*/")
SYM_REF = re.compile(r"s_\w+?_(00[0-9a-f]{6})\b")
SYM_LINE = re.compile(r"s_\w+?_(00[0-9a-f]{6})\s*,\s*(0x[0-9a-f]+|\d+)\s*[),]")
# every symbolic reference into the image (code or data)
ANY_REF = re.compile(r"\b(?:s_\w+?|DAT|PTR_FUN|PTR_DAT|PTR_LAB|UNK)_(00[0-9a-f]{6})\b")
def parse_decomp(mem):
"""Return sorted list of dicts: addr, name, tag, part, paths{norm: [lines]}."""
"""Return sorted list of dicts:
addr, name, tag, part, paths{norm:[lines]}, refs[set of VAs]."""
funcs = []
for fn in sorted(os.listdir(DECOMP_DIR)):
if not fn.endswith(".c"):
@@ -134,60 +151,211 @@ def parse_decomp(mem):
"name": m.group(3),
"part": part,
"paths": defaultdict(list), # norm path -> [line numbers]
"refs": set(), # referenced VAs
}
funcs.append(cur)
continue
if cur is None:
continue
# line-number pairs first (subset of SYM_REF hits)
consumed = set()
for sm in SYM_LINE.finditer(line):
va = int(sm.group(1), 16)
s = read_cstr(mem, va)
if PATH_RE.match(s):
num = int(sm.group(2), 0)
cur["paths"][normalize_path(s)].append(num)
consumed.add(va)
for sm in SYM_REF.finditer(line):
cur["paths"][normalize_path(s)].append(int(sm.group(2), 0))
for sm in ANY_REF.finditer(line):
va = int(sm.group(1), 16)
if va in consumed:
continue
cur["refs"].add(va)
s = read_cstr(mem, va)
if PATH_RE.match(s):
cur["paths"][normalize_path(s)] # touch, no line
cur["paths"][normalize_path(s)] # touch (no line info)
funcs.sort(key=lambda d: d["addr"])
return funcs
# ------------------------------------------------------- attribution --------
def attribute(funcs):
"""Set f['file'] (best attribution) + f['how'] in
{assert, tag, fill, boundary, none}."""
#
# Anchors are sparse (release build: only hard Fail() sites carry file
# strings), so attribution leans on a second linker invariant: each OBJ's
# DATA (vtables, literals, statics) is laid out contiguously per section in
# the SAME link order as its code. Every data VA referenced by an anchored
# function becomes a known (VA -> TU) point; an unanchored function's data
# refs then VOTE: a ref whose nearest known neighbors below AND above in the
# same section agree contributes one vote for that TU. Assignments feed
# back as new known points/anchors and the pass iterates to a fixed point,
# then plain link-order fill closes the gaps between agreeing code anchors.
#
def _data_sections(sections):
out = [(n, lo, hi) for (n, lo, hi) in sections
if n.upper() != "CODE" and lo is not None]
# .bss is not in the dump (no contents) but its statics are laid out in
# the same per-OBJ link order -- treat the VA range above the last dumped
# data section as a virtual section so bss refs can vote too.
data_hi = max(hi for (n, lo, hi) in out if n.upper() == "DATA")
out.append((".bss~", data_hi, 0x540000))
return out
# Verified TU anchor addresses from game/reconstructed/CLASSMAP.md and the
# context files ([T1] each: ctors / named functions whose owning TU is
# established). These seed the data-cluster voting where Fail() strings are
# absent, and let the link-order sanity check run.
SEED_ANCHORS = {
0x49B54C: "bt/btmssn.cpp", # BTMission ctor
0x49BCA4: "bt/messmgr.cpp", # SubsystemMessageManager ctor
0x49DE14: "bt/dmgtable.cpp", # DamageZonePercentTable SelectZone
0x49EA48: "bt/dmgtable.cpp", # DamageLookupTable ctor
0x49F624: "bt/mech.cpp", # mech<->player bind
0x4A2D48: "bt/mech.cpp", # Mech::Make
0x4ADDA0: "bt/heat.cpp", # heat subsystem ctor (file= tagged)
0x4AFBE0: "bt/mechmppr.cpp", # CycleControlMode
0x4B0EFC: "bt/powersub.cpp", # PoweredSubsystem::HandleMessage
0x4B3778: "bt/gyro.cpp", # Gyroscope ctor
0x4B6918: "bt/torso.cpp", # Torso::Recenter
0x4B8D18: "bt/myomers.cpp", # MyomersSimulation
0x4B9728: "bt/mechweap.cpp", # MechWeapon::SendDamageMessage
0x4BBFCC: "bt/projweap.cpp", # CheckForJam
0x4BEF78: "bt/missile.cpp", # Missile MoveAndCollide
0x4C0BC8: "bt/btplayer.cpp", # BTPlayer ctor
0x4C1C24: "bt_l4/btl4rdr.cpp", # MapDisplay ctor
0x4C2F94: "bt_l4/btl4gaug.cpp", # gauge TU start (CLASSMAP extent)
0x4C6798: "bt_l4/btl4gau2.cpp", # gau2 TU start (CLASSMAP extent)
0x4CC40C: "bt_l4/btl4vid.cpp", # BTReticleRenderable ctor
0x4D1AE4: "bt_l4/btl4mppr.cpp", # L4MechControlsMapper::SetControlMode
0x4D2C30: "bt_l4/btl4mssn.cpp", # BTL4Mission::SetPlayerData
0x4D34C4: "bt_l4/btl4app.cpp", # BTL4Application ctor
0x4D3BE4: "bt_l4/btl4pb.cpp", # BTL4PlaybackApplication ctor
}
def _section_of(va, dsecs):
for i, (_, lo, hi) in enumerate(dsecs):
if lo <= va < hi:
return i
return None
def attribute(funcs, sections, recon_affinity=None):
"""Set f['file'] + f['how'] in {assert, seed, tag, dp, rtl, none}.
recon_affinity: {addr: tu_key} soft votes from name-matched
game/reconstructed files' @address annotations."""
dsecs = _data_sections(sections)
recon_affinity = recon_affinity or {}
# pass 1: direct evidence
for f in funcs:
best = None
if f["paths"]:
# the file MOST referenced in-body (asserts overwhelmingly name
# the owning TU); ties broken by presence of line numbers
best = max(
f["paths"].items(),
key=lambda kv: (len(kv[1]) > 0, len(kv[1]), kv[0]),
)[0]
f["file"], f["how"] = best, "assert"
elif f["addr"] in SEED_ANCHORS:
f["file"], f["how"] = SEED_ANCHORS[f["addr"]], "seed"
elif f["tag"] != "?":
f["file"], f["how"] = f["tag"].lower(), "tag"
else:
f["file"], f["how"] = None, "none"
# pass 2: link-order fill between agreeing anchors
anchors = [(i, f["file"]) for i, f in enumerate(funcs) if f["file"]]
for (i0, f0), (i1, f1) in zip(anchors, anchors[1:]):
for j in range(i0 + 1, i1):
if funcs[j]["file"] is None:
if f0 == f1:
funcs[j]["file"], funcs[j]["how"] = f0, "fill"
else:
funcs[j]["file"], funcs[j]["how"] = None, "boundary"
funcs[j]["between"] = (f0, f1)
# ---- pass 2: monotone DP labeling of the BT region -------------------
#
# bt.lib + btl4.lib are the last game libs before the Borland RTL, so
# inside [first BT anchor .. RTL sentinel) the TU label sequence along
# the address axis must be NON-DECREASING in makefile link order. A DP
# picks, among all monotone labelings that respect the anchors, the one
# maximizing data-cluster affinity. The REGION TOTALS are exact under
# the sentinels even where an individual TU boundary is soft.
#
import bisect
# anchor-known data points (assert/seed/tag only -- no cascade); VAs
# referenced from >1 file are shared globals: non-discriminative.
claims = defaultdict(set)
for f in funcs:
if f["file"] is not None:
for va in f["refs"]:
claims[va].add(f["file"])
shared = {va for va, owners in claims.items() if len(owners) > 1}
known = [[] for _ in dsecs]
for f in funcs:
if f["file"] is None:
continue
for va in f["refs"]:
if va not in shared:
si = _section_of(va, dsecs)
if si is not None:
known[si].append((va, f["file"]))
for k in known:
k.sort()
def affinity(f):
votes = defaultdict(float)
for va in f["refs"]:
if va in shared:
continue
si = _section_of(va, dsecs)
if si is None or not known[si]:
continue
k = known[si]
i = bisect.bisect_left(k, (va, ""))
if i < len(k) and k[i][0] == va:
votes[k[i][1]] += 2.0 # unique exact VA match
continue
lo = k[i - 1][1] if i > 0 else None
hi = k[i][1] if i < len(k) else None
if lo is not None and lo == hi:
votes[lo] += 1.0 # inside one TU's data cluster
return votes
RTL_SENTINEL = 0x4D4B58 # first known Borland RTL fn
# btl4.obj is linked right after the startup OBJ (see BTL4.MAK tlink32
# response), i.e. at the BOTTOM of the image -- it is NOT part of the
# bt.lib/btl4.lib region and is excluded from the ordered labeling.
labels = [t for t in CENSUS if t != "bt_l4/btl4.cpp"] + ["<rtl>"]
label_ix = {t: i for i, t in enumerate(labels)}
lo_i = next(i for i, f in enumerate(funcs)
if f["file"] in CENSUS_SET)
hi_i = next(i for i, f in enumerate(funcs) if f["addr"] >= RTL_SENTINEL)
region = funcs[lo_i:hi_i + 1]
NEG = -1e18
n, m = len(region), len(labels)
score = [[0.0] * m for _ in range(n)]
for r, f in enumerate(region):
if f["file"] in label_ix: # anchored: pin it
for c in range(m):
score[r][c] = 0.0 if c == label_ix[f["file"]] else NEG
elif f["addr"] >= RTL_SENTINEL:
for c in range(m):
score[r][c] = 0.0 if labels[c] == "<rtl>" else NEG
else:
votes = affinity(f)
t = recon_affinity.get(f["addr"])
if t is not None:
votes[t] += 0.75 # port citation, soft
for t, v in votes.items():
if t in label_ix:
score[r][label_ix[t]] = v
# DP: best[r][c] = max over c' <= c of best[r-1][c'] + score[r][c]
best = [row[:] for row in score]
back = [[0] * m for _ in range(n)]
for c in range(1, m):
if best[0][c] < best[0][c - 1]:
best[0][c] = best[0][c - 1]
for r in range(1, n):
run_best, run_ix = NEG, 0
for c in range(m):
if best[r - 1][c] > run_best:
run_best, run_ix = best[r - 1][c], c
back[r][c] = run_ix
best[r][c] = (run_best + score[r][c]) if run_best > NEG / 2 \
and score[r][c] > NEG / 2 else NEG
# trace back from the best final cell
c = max(range(m), key=lambda cc: best[n - 1][cc])
for r in range(n - 1, -1, -1):
f = region[r]
if f["file"] is None:
f["file"] = labels[c]
f["how"] = "dp"
c = back[r][c]
for f in funcs:
if f["file"] == "<rtl>":
f["file"], f["how"] = None, "rtl"
return funcs
# ------------------------------------------------- reconstructed map --------
@@ -228,12 +396,24 @@ def survivors():
# ------------------------------------------------------------- main ---------
def main():
print("loading section dump ...")
mem = load_string_table()
print(" %d bytes mapped" % len(mem))
mem, sections = load_string_table()
print(" %d bytes mapped, sections: %s" % (
len(mem), ", ".join("%s@%x-%x" % s for s in sections)))
print("parsing decomp ...")
funcs = parse_decomp(mem)
print(" %d functions" % len(funcs))
attribute(funcs)
# soft affinity: a recon file named like a census TU votes for that TU
# at every address it cites (cross-citations are outvoted by the DP's
# monotone constraint)
recon_pre = recon_addresses()
recon_affinity = {}
for fn, addrs in recon_pre.items():
base = os.path.splitext(fn)[0].lower()
for key in (tu_key("bt", base), tu_key("bt_l4", base)):
if key in CENSUS_SET:
for a in addrs:
recon_affinity[a] = key
attribute(funcs, sections, recon_affinity)
# sizes: delta to next function
for f, g in zip(funcs, funcs[1:]):
@@ -246,7 +426,7 @@ def main():
# per-TU aggregation (census TUs + whatever engine files showed up)
tus = defaultdict(lambda: {
"funcs": 0, "bytes": 0, "assert": 0, "fill": 0, "tag": 0,
"funcs": 0, "bytes": 0, "assert": 0, "data": 0, "fill": 0, "tag": 0,
"lo": None, "hi": None, "max_line": 0, "recon_hits": 0,
})
boundary = 0
@@ -261,7 +441,8 @@ def main():
t = tus[f["file"]]
t["funcs"] += 1
t["bytes"] += f["size"]
t[f["how"] if f["how"] in ("assert", "fill", "tag") else "tag"] += 1
bucket = {"assert": "assert", "dp": "fill", "data": "data"}.get(f["how"], "tag")
t[bucket] += 1
t["lo"] = f["addr"] if t["lo"] is None else min(t["lo"], f["addr"])
t["hi"] = max(t["hi"] or 0, f["addr"] + f["size"])
for lines in f["paths"].values():
@@ -275,30 +456,51 @@ def main():
t = tus.get(key)
s = surv.get(key, "")
if not t:
return "| %s | %s | — | — | — | — | — | 0 |" % (key, s or "—")
return "| %s | %s | %d | %.1f | %06x-%06x | %d | %d | %d |" % (
return "| %s | %s | — | — | — | — | — | — | 0 |" % (key, s or "—")
return "| %s | %s | %d | %.1f | %06x-%06x | %d | %d | %d | %d |" % (
key, s or "—", t["funcs"], t["bytes"] / 1024.0,
t["lo"], t["hi"], t["assert"], t["max_line"], t["recon_hits"],
t["lo"], t["hi"], t["assert"], t["data"] + t["fill"],
t["max_line"], t["recon_hits"],
)
bt_keys = [tu_key("bt", n) for n in BT_LIB_TUS]
btl4_keys = [tu_key("bt_l4", n) for n in BTL4_LIB_TUS + MAIN_TUS]
engine_keys = sorted(k for k in tus if k not in CENSUS_SET)
# link-order sanity: census TU code ranges must ascend in makefile order
order_violations = []
prev = None
for k in bt_keys + btl4_keys:
t = tus.get(k)
if not t or t["lo"] is None:
continue
if prev and prev[1] > t["lo"]:
order_violations.append("%s (ends %06x) overlaps %s (starts %06x)"
% (prev[0], prev[1], k, t["lo"]))
prev = (k, t["hi"])
bt_total = sum(tus[k]["bytes"] for k in bt_keys if k in tus)
btl4_total = sum(tus[k]["bytes"] for k in btl4_keys if k in tus)
bt_funcs = sum(tus[k]["funcs"] for k in bt_keys if k in tus)
btl4_funcs = sum(tus[k]["funcs"] for k in btl4_keys if k in tus)
hdr = ("| TU | survives | funcs | KB | addr range | assert-anchored | "
"max assert line | recon @addr hits |\n|---|---|---|---|---|---|---|---|")
"DP-inferred | max assert line | recon @addr hits |\n"
"|---|---|---|---|---|---|---|---|---|")
with open(OUT_MD, "w") as f:
with open(OUT_MD, "w", encoding="utf-8") as f:
f.write("# BT 4.10 source manifest — measured from BTL4OPT.EXE\n\n")
f.write("Generated by `tools/manifest410.py`. Census = the surviving "
"makefiles (BT.MAK 36 TUs, BTL4.MAK 13 TUs + btl4.obj); "
"attribution = in-body assert strings (resolved via "
"section_dump), exporter file= tags, link-order fill.\n\n")
"makefiles (BT.MAK 36 TUs, BTL4.MAK 13 TUs + btl4.obj, in "
"LINK ORDER). Attribution = Fail() assert strings resolved "
"via section_dump [T1 anchors] + CLASSMAP seed addresses "
"[T1] + a monotone-in-link-order DP over data-section "
"cluster affinity and recon @addr citations [T3 for "
"unanchored boundaries]. A `—` row = TU NOT LOCALIZED: its "
"code is contained inside an adjacent row's range (no "
"anchor evidence yet); 8 such TUs survive as full .cpp "
"anyway. `bt_l4/btl4.cpp` links at the image BOTTOM with "
"the startup OBJ and is not part of this region.\n\n")
f.write("Totals: **%d functions** in the image; BT-side: **%d funcs / "
"%.0f KB** (bt.lib) + **%d funcs / %.0f KB** (btl4.lib+main); "
"boundary-uncertain: %d; unattributed (outside any anchor "
@@ -314,6 +516,10 @@ def main():
f.write("\n## btl4.lib + main (link order)\n\n%s\n" % hdr)
for k in btl4_keys:
f.write(tbl_row(k) + "\n")
if order_violations:
f.write("\n## ⚠ Link-order violations (attribution suspects)\n\n")
for v in order_violations:
f.write("- %s\n" % v)
f.write("\n## Engine/other TUs seen in the image (context only — "
"their source survives)\n\n")
f.write("| TU | funcs | KB |\n|---|---|---|\n")
@@ -321,7 +527,7 @@ def main():
t = tus[k]
f.write("| %s | %d | %.1f |\n" % (k, t["funcs"], t["bytes"] / 1024.0))
with open(OUT_JSON, "w") as f:
with open(OUT_JSON, "w", encoding="utf-8") as f:
json.dump({
"census": CENSUS,
"functions": [