#60 PART 2: the RE-EXPORT -- dark code 90KB -> 41KB, coverage 87.3% -> 93.5%
Installed JDK 21 + Ghidra 12.1.2 (no admin, %LOCALAPPDATA%\bt411-tools beside DXSDK/cmake; runner uses 8.3 SHORT paths because Ghidra's .bat expands %JAVA_HOME% unquoted and the profile has a space). New tooling: reference/ghidra_scripts/ExportGaps.java -- ExportAll's exact output contract PLUS a gap-fill pass (force disassembly + createFunction at E8 call targets outside functions, data->code pointers at a plausible prologue, and the census's discovered starts; iterated to a fixpoint, logged to gapfill_report.tsv). tools/ghidra_reexport.sh (headless runner, 'reprocess' mode) and tools/gapdiff.py (score two censused exports); gapcensus.py now censuses any export dir. Results: 6267 -> 6472 functions (+205 created in 2 rounds: 195 census starts, 6 call targets, 4 data pointers; 56.1KB newly covered), ZERO decompile failures. Dark real code 90.4 -> 40.8 KB (54.8% recovered); game-side dark 53.1 -> 21.1 KB; regions 428 -> 321. EVERY historically dark function now has pseudocode -- including @0x4c05c4 VehicleDead, the absence that opened this issue. VALIDATION: the new pseudocode confirms this week's hand reconstruction of the crouch field-for-field (mapPosture/duckState/squatCapable/myomerEff/ novice gate/SetLegAnimation/ForceUpdate/stability alarm) -- and exposed one branch the raw pass missed: AIRBORNE AUTO-RISE (mode 3|4 && legState 1 -> forced squ), now implemented in mech4.cpp and re-benched un-regressed. PROMOTION: the re-export is canonical reference/decomp/; the previous export is preserved at reference/decomp/archive_2025export/ so old `part_0NN.c:LINE` citations still resolve (addresses are stable across both; line/shard membership is NOT -- cite @ADDR). New lead recorded: @0x4c0904 is the MASTER BTPlayer Performance (team resolution, EndMission console post, score heartbeat) -- our @0x4c083c PlayerSimulation attribution needs a re-check. KB: source-completeness, gotcha #20 (the rule is cheap now -- look it up), CLAUDE.md router/layout. Log: phases/phase-04-gap-census.md. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Joe DiPrima
co-authored by
Claude Fable 5
Claude Fable 5
parent
e69c0d7aa8
commit
f44be87ab2
+13
-4
@@ -30,10 +30,19 @@ import sys
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
EXE = os.path.join(ROOT, 'content', 'BTL4OPT.EXE')
|
||||
IDX = os.path.join(ROOT, 'reference', 'decomp', 'functions_index.tsv')
|
||||
PARTS = os.path.join(ROOT, 'reference', 'decomp', 'all', 'part_*.c')
|
||||
OUT_MD = os.path.join(ROOT, 'reference', 'decomp', 'GAP_CENSUS.md')
|
||||
OUT_TSV = os.path.join(ROOT, 'reference', 'decomp', 'gap_census.tsv')
|
||||
|
||||
# Census any export tree: gapcensus.py [<decompDir>]
|
||||
# default = reference/decomp (the canonical export)
|
||||
# e.g. reference/decomp/reexport (a candidate re-export -- writes its
|
||||
# own GAP_CENSUS.md/gap_census.tsv inside that dir, so the two are
|
||||
# directly diffable)
|
||||
DECOMP = os.path.join(ROOT, 'reference', 'decomp')
|
||||
if len(sys.argv) > 1:
|
||||
DECOMP = os.path.abspath(sys.argv[1])
|
||||
IDX = os.path.join(DECOMP, 'functions_index.tsv')
|
||||
PARTS = os.path.join(DECOMP, 'all', 'part_*.c')
|
||||
OUT_MD = os.path.join(DECOMP, 'GAP_CENSUS.md')
|
||||
OUT_TSV = os.path.join(DECOMP, 'gap_census.tsv')
|
||||
|
||||
MIN_REGION = 16 # ignore alignment slivers below this many bytes
|
||||
TOP_REGIONS = 40 # detail table size
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env python3
|
||||
"""gapdiff.py -- score a candidate re-export against the canonical one.
|
||||
|
||||
python tools/gapdiff.py [oldDecompDir] [newDecompDir]
|
||||
defaults: reference/decomp reference/decomp/reexport
|
||||
|
||||
Both dirs must already have been censused (tools/gapcensus.py <dir>).
|
||||
Reports: index/coverage deltas, dark regions closed, and -- the part that
|
||||
matters for reconstruction -- which previously-dark addresses the KB already
|
||||
cites (past raw-disasm digs) now have real pseudocode, plus how much of the
|
||||
never-touched game-side dark came into the light.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import glob
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
OLD = os.path.abspath(sys.argv[1]) if len(sys.argv) > 1 else os.path.join(ROOT, 'reference', 'decomp')
|
||||
NEW = os.path.abspath(sys.argv[2]) if len(sys.argv) > 2 else os.path.join(ROOT, 'reference', 'decomp', 'reexport')
|
||||
|
||||
# addresses the KB/port cite (a past dig reached them) -- the practical audience
|
||||
CITE = set()
|
||||
_pat = re.compile(r'(?:@|FUN_|0x)0{0,2}(4[0-9a-f]{5})\b', re.I)
|
||||
for base in ('game', 'context', 'docs'):
|
||||
for dp, _d, fs in os.walk(os.path.join(ROOT, base)):
|
||||
for fn in fs:
|
||||
if fn.lower().endswith(('.cpp', '.hpp', '.h', '.md')):
|
||||
try:
|
||||
for m in _pat.finditer(open(os.path.join(dp, fn), encoding='utf-8',
|
||||
errors='replace').read()):
|
||||
CITE.add(int(m.group(1), 16))
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def load_index(d):
|
||||
idx = {}
|
||||
p = os.path.join(d, 'functions_index.tsv')
|
||||
for line in open(p, encoding='utf-8', errors='replace'):
|
||||
f = line.rstrip('\n').split('\t')
|
||||
if len(f) < 4:
|
||||
continue
|
||||
try:
|
||||
idx[int(f[0], 16)] = (int(f[1]), f[2], f[3])
|
||||
except ValueError:
|
||||
pass
|
||||
return idx
|
||||
|
||||
|
||||
def load_regions(d):
|
||||
p = os.path.join(d, 'gap_census.tsv')
|
||||
out = []
|
||||
for line in open(p, encoding='utf-8', errors='replace').read().splitlines()[1:]:
|
||||
f = line.split('\t')
|
||||
if len(f) < 8:
|
||||
f += [''] * (8 - len(f))
|
||||
out.append({'lo': int(f[0], 16), 'hi': int(f[1], 16), 'size': int(f[2]),
|
||||
'code': int(f[3]), 'visited': f[6], 'tu': f[7]})
|
||||
return out
|
||||
|
||||
|
||||
def exported_addrs(d):
|
||||
got = set()
|
||||
hdr = re.compile(r'/\* @(00[0-9a-f]{6}) ')
|
||||
for p in sorted(glob.glob(os.path.join(d, 'all', 'part_*.c'))):
|
||||
for m in hdr.finditer(open(p, encoding='utf-8', errors='replace').read()):
|
||||
got.add(int(m.group(1), 16))
|
||||
return got
|
||||
|
||||
|
||||
def main():
|
||||
oi, ni = load_index(OLD), load_index(NEW)
|
||||
orr, nr = load_regions(OLD), load_regions(NEW)
|
||||
oe, ne = exported_addrs(OLD), exported_addrs(NEW)
|
||||
|
||||
o_dark = sum(r['code'] for r in orr)
|
||||
n_dark = sum(r['code'] for r in nr)
|
||||
o_game = sum(r['code'] for r in orr if 'bt/' in r['tu'] or 'bt_l4/' in r['tu'])
|
||||
n_game = sum(r['code'] for r in nr if 'bt/' in r['tu'] or 'bt_l4/' in r['tu'])
|
||||
|
||||
new_fns = sorted(set(ni) - set(oi))
|
||||
# which new functions land in what USED to be dark
|
||||
def in_old_dark(a):
|
||||
for r in orr:
|
||||
if r['lo'] <= a < r['hi']:
|
||||
return True
|
||||
return False
|
||||
new_in_dark = [a for a in new_fns if in_old_dark(a)]
|
||||
cited_new = [a for a in new_in_dark if a in CITE]
|
||||
|
||||
print('=== GAP DIFF: %s -> %s' % (os.path.basename(OLD), os.path.basename(NEW)))
|
||||
print('indexed functions : %d -> %d (+%d)' % (len(oi), len(ni), len(ni) - len(oi)))
|
||||
print('exported pseudocode: %d -> %d (+%d)' % (len(oe), len(ne), len(ne) - len(oe)))
|
||||
print('dark regions : %d -> %d (%+d)' % (len(orr), len(nr), len(nr) - len(orr)))
|
||||
print('dark REAL code : %.1f KB -> %.1f KB (%+.1f KB, %.1f%% recovered)'
|
||||
% (o_dark / 1024, n_dark / 1024, (n_dark - o_dark) / 1024,
|
||||
100.0 * (o_dark - n_dark) / o_dark if o_dark else 0))
|
||||
print(' game-side dark : %.1f KB -> %.1f KB (%+.1f KB)'
|
||||
% (o_game / 1024, n_game / 1024, (n_game - o_game) / 1024))
|
||||
print('new functions in previously-DARK bytes: %d' % len(new_in_dark))
|
||||
print(' ...of which the KB/port already cites (past raw-disasm digs): %d' % len(cited_new))
|
||||
if cited_new:
|
||||
print(' ' + ' '.join('%x' % a for a in cited_new[:24])
|
||||
+ (' ...' if len(cited_new) > 24 else ''))
|
||||
|
||||
# celebrity check -- the addresses that historically burned us
|
||||
fam = [(0x4c05c4, 'VehicleDead handler'), (0x4b838c, 'Searchlight ToggleLamp'),
|
||||
(0x4a07b8, 'TakeDamage death tail'), (0x4a9b5c, 'master perf'),
|
||||
(0x4b8be3, 'myomer integrator'), (0x4aa011, 'duck consumer'),
|
||||
(0x486467, 'btmssn dispatch cluster'), (0x4c090a, 'btplayer hole')]
|
||||
print('--- celebrities (function created? exported?)')
|
||||
for a, what in fam:
|
||||
# nearest containing function in the new index
|
||||
owner = max((x for x in ni if x <= a and x + ni[x][0] > a), default=None)
|
||||
print(' %-24s %s' % (what, ('IN %06x (%s)%s' % (owner, ni[owner][2],
|
||||
' +pseudocode' if owner in ne else ''))
|
||||
if owner else 'still dark'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
# #60 re-export -- headless Ghidra: analyze BTL4OPT.EXE, fill the dark regions
|
||||
# (ExportGaps.java), and write a full pseudocode export we can diff against the
|
||||
# canonical one.
|
||||
#
|
||||
# tools/ghidra_reexport.sh # first run: import + full auto-analysis
|
||||
# tools/ghidra_reexport.sh reprocess # re-run the script on the saved project
|
||||
#
|
||||
# Output: reference/decomp/reexport/{all/part_*.c, functions_index.tsv,
|
||||
# vtables.tsv, gapfill_report.tsv, headless.log}
|
||||
# Score it with: python tools/gapcensus.py reference/decomp/reexport
|
||||
set -e
|
||||
|
||||
# NB: Ghidra's launcher .bat uses %JAVA_HOME% UNQUOTED, so a space anywhere in
|
||||
# these paths kills it ("'C:\Users\Joe' is not recognized"). The tools live under
|
||||
# the user profile (space in "Joe Laptop"), so use the 8.3 SHORT forms.
|
||||
TOOLS="/c/Users/JOELAP~1/AppData/Local/BT411-~1"
|
||||
GHIDRA="$TOOLS/GHIDRA~1.2_P"
|
||||
export JAVA_HOME='C:\Users\JOELAP~1\AppData\Local\BT411-~1\JDK-21~1.12_'
|
||||
PROJDIR="$TOOLS/ghidra_projects"
|
||||
PROJ="BT411"
|
||||
|
||||
OUT_W='C:\git\bt411\reference\decomp\reexport'
|
||||
SCRIPTS_W='C:\git\bt411\reference\ghidra_scripts'
|
||||
EXE_W='C:\git\bt411\content\BTL4OPT.EXE'
|
||||
PROJDIR_W="$(cygpath -w "$PROJDIR")"
|
||||
LOG_W='C:\git\bt411\reference\decomp\reexport\headless.log'
|
||||
|
||||
mkdir -p "$PROJDIR" /c/git/bt411/reference/decomp/reexport
|
||||
|
||||
if [ ! -x "$GHIDRA/support/analyzeHeadless.bat" ] && [ ! -f "$GHIDRA/support/analyzeHeadless.bat" ]; then
|
||||
echo "ghidra not found at $GHIDRA" >&2; exit 1
|
||||
fi
|
||||
"$TOOLS/JDK-21~1.12_/bin/java.exe" -version 2>&1 | head -1
|
||||
|
||||
if [ "$1" = "reprocess" ]; then
|
||||
MODE=(-process BTL4OPT.EXE -noanalysis)
|
||||
else
|
||||
MODE=(-import "$EXE_W")
|
||||
fi
|
||||
|
||||
echo "== analyzeHeadless: ${MODE[0]} =="
|
||||
"$GHIDRA/support/analyzeHeadless.bat" "$PROJDIR_W" "$PROJ" \
|
||||
"${MODE[@]}" \
|
||||
-scriptPath "$SCRIPTS_W" \
|
||||
-postScript ExportGaps.java "$OUT_W" \
|
||||
-log "$LOG_W"
|
||||
|
||||
echo "== done; scoring =="
|
||||
python /c/git/bt411/tools/gapcensus.py /c/git/bt411/reference/decomp/reexport | tail -3
|
||||
Reference in New Issue
Block a user