Add MW4COMPARE: .mw4 decompiler toolchain and V4H comparison harness
Tooling built to recover editable source for six 'Mech chassis that exist
in the parallel FS_Build_V4H build but not in this repo. Reverse-engineers
every compiled record type in the .mw4 package format back to the .data /
.instance / .subsystems / .damage / .contents / .torso / .engine /
.armature sources the content pipeline consumes.
Nothing here is wired into the game build. It is a standalone analysis
harness run from Linux.
Package format
--------------
"#VBD" container. Directory records are [len][name][FILETIME][origSize]
[storedSize][offset], payload base at dword 0x0C. A record is stored raw
when storedSize == origSize, otherwise LZW (9->12-bit LSB-first codes,
256=clear, 257=EOF, dict from 258), per Database.cpp:451.
GameModel records are flat /Zp4 structs following the C++ inheritance
chain Entity(0) -> Mover(28) -> MWObject(80) -> Vehicle(664) -> Mech(756),
1636 bytes total. CreateMessage records follow Replicator -> Entity ->
Mover -> MWMover -> MWObject -> Vehicle -> Mech from start=16 (the
undeclared Connection__Message header), ending at 341 and padded to 344.
tools/decompile/
----------------
datamap.py header-driven layout engine; CHAIN + ANCHORS
{Vehicle:664, Mech:756} assert the struct offsets
mw4msg.py CreateMessage reader/walker
data.py .data constants.py define/table symbol resolution
damage.py .damage contents.py .contents
smallmodel.py .torso + .engine instance.py .instance
armature.py / armature_parts.py .armature + armaturedata/armaturevideo
assembly.py joint hierarchy renderer
make_generic_doll.py builds generic MFD/Radar damage dolls
verify_*.py per-type round-trip verifiers
Verified round-trip across all 64 shared chassis:
.armature 2938/2976 pages .subsystems 7579/7585 keys
.data map 6071/6071 values .data trip 8291/8306 keys
.damage 6605/6605 keys .contents 7480/7480 keys
.torso+.engine 1280/1280 keys .instance 896/896 keys, 64/64 pages
armature_parts 1202/1202 .data, 1149/1202 .video
Layout-discovery lessons (documented in DECOMPILING.md)
-------------------------------------------------------
- Never let a field map be discovered by the values that verify it. A
value-matching pass reported 4288/4288 while mis-assigning 34 keys. The
map was rebuilt from header declaration order, anchored on uniquely
resolved fields.
- Read the factory, not the data. 12 .data fields and 5 Torso fields are
declared plain Stuff::Scalar but multiplied by Radians_Per_Degree in
Mech_Tool.cpp:889 / Torso_Tool.cpp.
- Strip typedefs before walking a header. A stray `typedef int AttributeID;`
masked a missing ClassID - two 4-byte errors cancelling out, caught only
by the ANCHORS assertion.
- A verifier that silently narrows its own input reports success. Braced
blocks must be hidden before splitting pages, replacing both CR and LF,
because a `Shadow={...}` block contains a line reading `[shadow]` and
splitlines() also splits on bare CR.
- NSWIZZLE is undefined, so the #else branch is live and orders members
differently. bool is 1 byte; char x[MaxStringLength] is 256.
- V4H carries stale Mech IDs (their Atlas is 5, ours 6), so 64 of 65 shared
chassis are off by one; --retarget-ids emits $(M_<Chassis>)/$(IDS_<Chassis>).
reports/ holds generated diffs. The two ~5 MB manifest-*.tsv intermediates
are gitignored; regenerate everything with run-comparison.sh.
Co-authored-by: Claude Opus 5 (Anthropic) <noreply@anthropic.com>
Co-authored-by: GitHub Copilot <copilot@github.com>
This commit is contained in:
co-authored by
Claude Opus 5
GitHub Copilot
Claude Opus 5
GitHub Copilot
parent
e088555b96
commit
83478b7666
@@ -0,0 +1 @@
|
||||
__pycache__/
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
classify-survivors.py - split a pruned extracted tree into NEW vs DIFFERS.
|
||||
|
||||
python3 classify-survivors.py <prunedExtractedDir>
|
||||
|
||||
Run after prune-identical.py. Everything still present is something we do not
|
||||
have byte-for-byte; this reports which of two reasons applies:
|
||||
|
||||
NEW no file at that entry path exists on our side at all
|
||||
DIFFERS we have that entry path, but the bytes differ
|
||||
|
||||
"Our side" means our extracted packages OR our Content/ source tree, matched
|
||||
case-insensitively - the same notion of "have it" the pruner uses.
|
||||
|
||||
Writes _classified.tsv next to the tree and prints a summary.
|
||||
"""
|
||||
import sys, os, collections
|
||||
|
||||
OURS_EXTRACTED = "/home/rich/Repositories/FS_Ours_extracted"
|
||||
OUR_SOURCE = "/home/rich/Repositories/firestorm/Gameleap/mw4/Content"
|
||||
|
||||
|
||||
def index_paths(root, skip_top=()):
|
||||
out = {}
|
||||
for dp, dirs, fs in os.walk(root):
|
||||
if dp == root:
|
||||
dirs[:] = [d for d in dirs if d.lower() not in skip_top]
|
||||
for f in fs:
|
||||
p = os.path.join(dp, f)
|
||||
out[os.path.relpath(p, root).replace("\\", "/").lower()] = p
|
||||
return out
|
||||
|
||||
|
||||
def package_of(rel):
|
||||
parts = rel.split("/")
|
||||
head = parts[0].lower()
|
||||
if head in ("core", "props", "textures"):
|
||||
return parts[0], "/".join(parts[1:])
|
||||
if head in ("maps", "missions", "variants") and len(parts) > 2:
|
||||
return "/".join(parts[:2]), "/".join(parts[2:])
|
||||
if head == "pilots" and len(parts) > 3:
|
||||
return "/".join(parts[:3]), "/".join(parts[3:])
|
||||
return parts[0], "/".join(parts[1:])
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
sys.exit(__doc__)
|
||||
root = sys.argv[1]
|
||||
merged = os.path.join(root, "_merged")
|
||||
|
||||
ours = index_paths(OURS_EXTRACTED, skip_top={"_merged"})
|
||||
ours_entries = set()
|
||||
for rel in ours:
|
||||
ours_entries.add(package_of(rel)[1])
|
||||
src_entries = set(index_paths(OUR_SOURCE))
|
||||
|
||||
rows, summary = [], collections.Counter()
|
||||
for dp, _dirs, fs in os.walk(root):
|
||||
if dp == merged or dp.startswith(merged + os.sep):
|
||||
continue
|
||||
for f in fs:
|
||||
rel = os.path.relpath(os.path.join(dp, f), root).replace("\\", "/")
|
||||
if rel.startswith("_"):
|
||||
continue
|
||||
pkg, entry = package_of(rel)
|
||||
e = entry.lower()
|
||||
verdict = "DIFFERS" if (rel.lower() in ours or e in ours_entries
|
||||
or e in src_entries) else "NEW"
|
||||
rows.append((verdict, rel, pkg, entry))
|
||||
summary[(verdict, pkg.split("/")[0])] += 1
|
||||
|
||||
with open(os.path.join(root, "_classified.tsv"), "w", encoding="utf-8") as fh:
|
||||
fh.write("verdict\tpath\tpackage\tentry\n")
|
||||
for r in sorted(rows):
|
||||
fh.write("\t".join(r) + "\n")
|
||||
|
||||
tot = collections.Counter(v for v, _, _, _ in rows)
|
||||
print(f"survivors: {len(rows)} NEW={tot['NEW']} DIFFERS={tot['DIFFERS']}\n")
|
||||
print(f"{'package group':16s} {'NEW':>7s} {'DIFFERS':>8s}")
|
||||
groups = sorted({g for _v, g in summary})
|
||||
for g in groups:
|
||||
print(f"{g:16s} {summary[('NEW', g)]:7d} {summary[('DIFFERS', g)]:8d}")
|
||||
print(f"\n-> {os.path.join(root, '_classified.tsv')}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
armature.py - rebuild a mech's .armature source from its packed records.
|
||||
|
||||
python3 armature.py <mechRecordDir> [-o out.armature]
|
||||
python3 armature.py --verify # check against all 65 known chassis
|
||||
|
||||
The packer merges <mech>.armature into <mech>.contents (via `!include=`) and
|
||||
then, for every contents page that has Child= entries, emits two records
|
||||
(MWMover_Tool.cpp ~78):
|
||||
|
||||
<mech>.contents[<joint>]{sites} children whose name starts 'site_'
|
||||
but is not 'site_eye*' - stored as
|
||||
YawPitchRoll + Point3D + name
|
||||
<mech>.contents[<joint>]{armature} every other child, as a CreateMessage
|
||||
carrying jointName + localToParent
|
||||
|
||||
Between them those two records hold every page of the original .armature.
|
||||
Verified on Atlas: 61 of 61 page names recovered, values exact.
|
||||
|
||||
Rotation notes
|
||||
--------------
|
||||
* {sites} stores YawPitchRoll directly, so those angles are exact.
|
||||
* Joint messages only carry a matrix. Across all 65 chassis, 1348 of 1453 joint
|
||||
matrices are the identity, i.e. `Rotation=0 0 0`.
|
||||
* site_lfoot / site_rfoot appear in BOTH streams - a deliberate hack in
|
||||
MWMover_Tool.cpp ("Jerry this will get deleted when you fix your foot
|
||||
problem") - and their matrix is the identity while the real angle is in the
|
||||
{sites} record. The {sites} value always wins.
|
||||
* site_eye* goes only to the armature stream, so its angle comes from the
|
||||
matrix.
|
||||
"""
|
||||
import sys, os, glob, math, struct, re, argparse, collections
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import mw4msg
|
||||
|
||||
OUR_MECH_SOURCE = "/home/rich/Repositories/firestorm/Gameleap/mw4/Content/Mechs"
|
||||
OUR_RECORDS = "/home/rich/Repositories/FS_Ours_extracted/core/mechs"
|
||||
|
||||
REC_RE = re.compile(r'^(?P<chassis>.+?)\.contents\[(?P<parent>[^\]]+)\]\{(?P<kind>armature|sites)\}$',
|
||||
re.I)
|
||||
|
||||
|
||||
def ypr_from_matrix(r):
|
||||
"""3x3 -> the (a, b, c) triple as written in Rotation=, in degrees.
|
||||
|
||||
Stuff's YawPitchRoll writes the X-axis angle in the middle slot; confirmed
|
||||
against kodiak/site_eyepoint (matrix gives 0.0688 deg, source says
|
||||
0.069104).
|
||||
"""
|
||||
m00, m01, m02, m10, m11, m12, m20, m21, m22 = r
|
||||
pitch = math.asin(max(-1.0, min(1.0, m21)))
|
||||
if abs(m21) < 0.999999:
|
||||
yaw = math.atan2(-m20, m22)
|
||||
roll = math.atan2(-m01, m11)
|
||||
else:
|
||||
yaw = math.atan2(m02, m00)
|
||||
roll = 0.0
|
||||
return (math.degrees(yaw), math.degrees(pitch), math.degrees(roll))
|
||||
|
||||
|
||||
def is_identity(r, eps=1e-5):
|
||||
ident = (1, 0, 0, 0, 1, 0, 0, 0, 1)
|
||||
return all(abs(a - b) < eps for a, b in zip(r, ident))
|
||||
|
||||
|
||||
def rebuild(mech_dir):
|
||||
"""-> (chassisName, entries, children)
|
||||
|
||||
entries list of {'name','parent','rot','trans'} - one per armature page.
|
||||
Keyed on (parent, name), not name alone: Victor legitimately has
|
||||
two different [site_lshellport] pages hanging off different joints.
|
||||
children {parentName: [childName, ...]} preserving order and multiplicity.
|
||||
"""
|
||||
entries = [] # one dict per armature page occurrence
|
||||
children = collections.defaultdict(list)
|
||||
site_children = collections.defaultdict(set)
|
||||
chassis = None
|
||||
# {sites} first: site_lfoot/site_rfoot are written to BOTH streams and only
|
||||
# the {sites} copy carries their real angle.
|
||||
records = (sorted(glob.glob(os.path.join(mech_dir, "*{sites}"))) +
|
||||
sorted(glob.glob(os.path.join(mech_dir, "*{armature}"))))
|
||||
for path in records:
|
||||
m = REC_RE.match(os.path.basename(path))
|
||||
if not m:
|
||||
continue
|
||||
chassis = chassis or m.group("chassis")
|
||||
parent, kind = m.group("parent"), m.group("kind").lower()
|
||||
with open(path, "rb") as fh:
|
||||
data = fh.read()
|
||||
children[parent] # ensure the parent exists
|
||||
|
||||
if kind == "sites":
|
||||
for name, rot, trans in mw4msg.read_sites(data):
|
||||
entries.append({
|
||||
"name": name, "parent": parent,
|
||||
"rot": tuple(math.degrees(a) for a in rot), "trans": trans})
|
||||
children[parent].append(name)
|
||||
site_children[parent].add(name)
|
||||
else:
|
||||
for _off, msg in mw4msg.walk(data):
|
||||
name = mw4msg.joint_name(msg)
|
||||
if not name:
|
||||
continue
|
||||
# site_lfoot/site_rfoot are deliberately written to both streams;
|
||||
# counting the armature copy too would double them.
|
||||
if name in site_children[parent]:
|
||||
continue
|
||||
children[parent].append(name)
|
||||
r = mw4msg.rotation3x3(msg)
|
||||
entries.append({
|
||||
"name": name, "parent": parent,
|
||||
"rot": (0.0, 0.0, 0.0) if is_identity(r) else ypr_from_matrix(r),
|
||||
"trans": mw4msg.translation(msg)})
|
||||
return chassis, entries, children
|
||||
|
||||
|
||||
def emit(entries, children):
|
||||
"""Render as .armature text, children before the joint that owns them."""
|
||||
by_name = {}
|
||||
for e in entries:
|
||||
by_name.setdefault(e["name"], []).append(e)
|
||||
out, done = [], set()
|
||||
|
||||
def visit(name):
|
||||
if name in done:
|
||||
return
|
||||
done.add(name)
|
||||
for c in children.get(name, ()):
|
||||
visit(c)
|
||||
for e in by_name.get(name, [{"rot": (0.0, 0.0, 0.0), "trans": (0.0, 0.0, 0.0)}]):
|
||||
out.append(f"[{name}]")
|
||||
out.append("Rotation=%.6f %.6f %.6f" % (e["rot"] or (0.0, 0.0, 0.0)))
|
||||
out.append("Translation=%.6f %.6f %.6f" % (e["trans"] or (0.0, 0.0, 0.0)))
|
||||
for c in children.get(name, ()):
|
||||
out.append(f"Child={c}")
|
||||
out.append("")
|
||||
|
||||
all_children = {c for lst in children.values() for c in lst}
|
||||
for name in list(children) + list(by_name):
|
||||
if name not in all_children:
|
||||
visit(name)
|
||||
for name in list(children) + list(by_name):
|
||||
visit(name)
|
||||
return "\r\n".join(out) + "\r\n"
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("mechdir", nargs="?", help="directory of packed records for one mech")
|
||||
ap.add_argument("-o", "--out", help="write .armature here instead of stdout")
|
||||
ap.add_argument("--verify", action="store_true",
|
||||
help="rebuild all 65 known chassis and diff against their source")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.verify:
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
os.execvp("python3", ["python3", os.path.join(here, "verify_armature.py")])
|
||||
if not args.mechdir:
|
||||
ap.error("give a mech record directory, or --verify")
|
||||
|
||||
chassis, entries, children = rebuild(args.mechdir)
|
||||
text = emit(entries, children)
|
||||
if args.out:
|
||||
os.makedirs(os.path.dirname(args.out), exist_ok=True)
|
||||
open(args.out, "wb").write(text.encode("latin-1"))
|
||||
print(f"{chassis}: {len(entries)} pages -> {args.out}")
|
||||
else:
|
||||
sys.stdout.write(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,225 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
armature_parts.py - generate a mech's `armaturedata/` and `armaturevideo/` files.
|
||||
|
||||
Every joint in a mech's `.contents` points at `armaturedata\\<part>.data`, which
|
||||
in turn points at `armaturevideo\\<part>.video`, which names the geometry. Both
|
||||
are packed as compiled records -- the `.data` becomes a 12-byte stub and the
|
||||
`.video` becomes a binary renderer graph -- so neither survives extraction as
|
||||
text and both have to be regenerated.
|
||||
|
||||
They are formulaic. Measured over the 89 shipped mechs:
|
||||
|
||||
* `armaturedata/<part>.data` -- **one single shape** across all 1336 files,
|
||||
parameterised only by the part name.
|
||||
* `armaturevideo/<part>.video` -- three shapes cover 91% of 1354 files:
|
||||
with a damaged LOD, without one, and with a running-lights block.
|
||||
|
||||
Rules, and their measured accuracy:
|
||||
|
||||
* A `DAMLOD` block iff `<part>_dam.erf` exists next to the mech. This holds for
|
||||
1327 of 1354 shipped files. **All 27 exceptions run the same way**: the
|
||||
`_dam.erf` exists but the author's `.video` ignores it. No shipped `.video`
|
||||
ever references a `_dam.erf` that is absent, so generating by this rule is
|
||||
safe by construction -- it can never produce a dangling reference, only
|
||||
occasionally more damage-state switching than the original author chose.
|
||||
The exceptions are mostly `_gun` and `_toe` parts and follow no clean rule.
|
||||
* A running-lights block iff the part is the torso and a running-lights `.erf`
|
||||
is present. 68 of the 71 torsos carry it; almost nothing else does.
|
||||
|
||||
python3 armature_parts.py <mech-source-dir>
|
||||
python3 armature_parts.py --verify
|
||||
"""
|
||||
import argparse, os, re, sys, collections
|
||||
|
||||
DATA_TEMPLATE = """[gamedata]
|
||||
Class=MechWarrior4::MWMover
|
||||
|
||||
[renderers]
|
||||
VideoRenderer=armaturevideo\\{part}.video
|
||||
"""
|
||||
|
||||
LIGHT_BLOCK = """[lightlod]
|
||||
Type=ShapeComponent
|
||||
Geometry={lights}
|
||||
Unique=true
|
||||
|
||||
[lightwatcher]
|
||||
Type=AttributeWatcherOfInt
|
||||
Attribute=IsDark
|
||||
SimulationShouldExecute=1
|
||||
|
||||
[lightswitch]
|
||||
Type=SwitchComponent
|
||||
Input=LightWatcher
|
||||
Child=LightLOD
|
||||
|
||||
"""
|
||||
|
||||
LOD_BLOCK = """[lod]
|
||||
Type=ShapeComponent
|
||||
Geometry={part}.erf
|
||||
Unique=true
|
||||
|
||||
"""
|
||||
|
||||
DAM_BLOCK = """[damlod]
|
||||
Type=ShapeComponent
|
||||
Geometry={part}_DAM.erf
|
||||
Unique=true
|
||||
|
||||
"""
|
||||
|
||||
# joint_cage has its own shape: two empty group components, then the cage
|
||||
# geometry twice (intact and destroyed), shadow-disabled.
|
||||
CAGE_TEMPLATE = """[fake1]
|
||||
Type=GroupComponent
|
||||
|
||||
[fake2]
|
||||
Type=GroupComponent
|
||||
|
||||
[cage]
|
||||
Type=ShapeComponent
|
||||
Geometry={cage}
|
||||
Unique=true
|
||||
DisableShadow=true
|
||||
|
||||
[destroyedcage]
|
||||
Type=ShapeComponent
|
||||
Geometry={cage}
|
||||
Unique=true
|
||||
DisableShadow=true
|
||||
|
||||
[watcher]
|
||||
Type=AttributeWatcherOfInt
|
||||
Attribute=VisualRepresentation
|
||||
SimulationShouldExecute=1
|
||||
|
||||
[damageappearance]
|
||||
Type=SwitchComponent
|
||||
Input=Watcher
|
||||
Child=Fake1
|
||||
Child=Fake2
|
||||
Child=Cage
|
||||
Child=DestroyedCage
|
||||
|
||||
[locator]
|
||||
Child=DamageAppearance
|
||||
"""
|
||||
|
||||
|
||||
def parts_of(mech_dir):
|
||||
"""Part names referenced as armaturedata\\<part>.data by the .contents."""
|
||||
out = []
|
||||
for fn in os.listdir(mech_dir):
|
||||
if not fn.lower().endswith(".contents"):
|
||||
continue
|
||||
t = open(os.path.join(mech_dir, fn), "rb").read().decode("latin-1")
|
||||
for m in re.finditer(r'armaturedata[\\/]([A-Za-z0-9_\-]+)\.data', t, re.I):
|
||||
if m.group(1) not in out:
|
||||
out.append(m.group(1))
|
||||
return out
|
||||
|
||||
|
||||
def video_text(mech_dir, part):
|
||||
files = {f.lower(): f for f in os.listdir(mech_dir)}
|
||||
cage = next((files[f] for f in files if f.endswith("_cage.erf")), None)
|
||||
if part.lower() == "joint_cage" and cage:
|
||||
return CAGE_TEMPLATE.format(cage=cage)
|
||||
|
||||
has_dam = f"{part.lower()}_dam.erf" in files
|
||||
lights = next((files[f] for f in files
|
||||
if f.startswith("runninglights") and f.endswith(".erf")), None)
|
||||
is_torso = part.lower().endswith("torso")
|
||||
|
||||
text = ""
|
||||
if lights and is_torso:
|
||||
text += LIGHT_BLOCK.format(lights=lights)
|
||||
text += LOD_BLOCK.format(part=part)
|
||||
if has_dam:
|
||||
text += DAM_BLOCK.format(part=part)
|
||||
text += """[watcher]
|
||||
Type=AttributeWatcherOfInt
|
||||
Attribute=VisualRepresentation
|
||||
SimulationShouldExecute=1
|
||||
|
||||
[damageappearance]
|
||||
Type=SwitchComponent
|
||||
Input=Watcher
|
||||
Child=LOD
|
||||
"""
|
||||
if has_dam:
|
||||
text += "Child=DAMLOD\n"
|
||||
text += "\n[locator]\n"
|
||||
if lights and is_torso:
|
||||
text += "Child=LightSwitch\n"
|
||||
text += "Child=DamageAppearance\n"
|
||||
return text
|
||||
|
||||
|
||||
def generate(mech_dir, write=True):
|
||||
"""-> {relativePath: text} for every part."""
|
||||
out = {}
|
||||
for part in parts_of(mech_dir):
|
||||
out[f"armaturedata/{part}.data"] = DATA_TEMPLATE.format(part=part)
|
||||
out[f"armaturevideo/{part}.video"] = video_text(mech_dir, part)
|
||||
if write:
|
||||
for rel, text in out.items():
|
||||
p = os.path.join(mech_dir, rel)
|
||||
os.makedirs(os.path.dirname(p), exist_ok=True)
|
||||
with open(p, "wb") as fh:
|
||||
fh.write(text.replace("\n", "\r\n").encode("latin-1"))
|
||||
return out
|
||||
|
||||
|
||||
def norm(text, part):
|
||||
t = re.sub(r'//[^\n]*', '', text).replace("\r", "")
|
||||
t = re.sub(re.escape(part), "PART", t, flags=re.I)
|
||||
return re.sub(r'\n+', "\n", t).strip().lower()
|
||||
|
||||
|
||||
def verify():
|
||||
import glob
|
||||
MECHS = "/home/rich/Repositories/firestorm/Gameleap/mw4/Content/Mechs"
|
||||
t = collections.Counter()
|
||||
diffs = collections.Counter()
|
||||
for d in sorted(glob.glob(MECHS + "/*")):
|
||||
if not os.path.isdir(d) or not glob.glob(d + "/armaturedata/*.data"):
|
||||
continue
|
||||
gen = generate(d, write=False)
|
||||
for rel, text in gen.items():
|
||||
p = os.path.join(d, rel)
|
||||
part = os.path.basename(rel).rsplit(".", 1)[0]
|
||||
kind = rel.split("/")[0]
|
||||
if not os.path.exists(p):
|
||||
t[kind + " absent"] += 1
|
||||
continue
|
||||
t[kind] += 1
|
||||
want = open(p, "rb").read().decode("latin-1")
|
||||
if norm(want, part) == norm(text, part):
|
||||
t[kind + " ok"] += 1
|
||||
else:
|
||||
diffs[kind] += 1
|
||||
print(f"armaturedata : {t['armaturedata ok']}/{t['armaturedata']} reproduced")
|
||||
print(f"armaturevideo: {t['armaturevideo ok']}/{t['armaturevideo']} reproduced")
|
||||
if t["armaturedata absent"] or t["armaturevideo absent"]:
|
||||
print(f" referenced but absent on disk: "
|
||||
f"{t['armaturedata absent']} data, {t['armaturevideo absent']} video")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("mech_dir", nargs="?")
|
||||
ap.add_argument("--verify", action="store_true")
|
||||
args = ap.parse_args()
|
||||
if args.verify:
|
||||
verify()
|
||||
return
|
||||
if not args.mech_dir:
|
||||
ap.error("mech_dir required")
|
||||
out = generate(args.mech_dir)
|
||||
print(f"wrote {len(out)} files into {args.mech_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
assembly.py - show how a mech's parts bolt together.
|
||||
|
||||
There is no `.erf` viewer on Linux, but the assembly itself is not hidden: the
|
||||
`.armature` gives every joint's parent, offset and rotation, and the `.contents`
|
||||
plus `armaturevideo/` say which geometry hangs off each joint. Printing that as
|
||||
a tree answers "what are the parts and how do they fit" without a renderer.
|
||||
|
||||
For actual 3D you need MW4Ed2 on the Windows box (Gameleap/mw4/run-editor.bat) --
|
||||
its Game View renders a loaded mech through DDrawCompat.
|
||||
|
||||
python3 assembly.py <mech-source-dir> [--geometry-only]
|
||||
"""
|
||||
import argparse, collections, os, re, sys
|
||||
|
||||
|
||||
def pages(path):
|
||||
"""-> OrderedDict pageName -> [(key, value)]"""
|
||||
txt = re.sub(r'//[^\n]*', '', open(path, "rb").read().decode("latin-1"))
|
||||
out, cur = collections.OrderedDict(), None
|
||||
for line in txt.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("!"):
|
||||
continue
|
||||
m = re.match(r'^\[([^\]]+)\]$', line)
|
||||
if m:
|
||||
cur = m.group(1)
|
||||
out.setdefault(cur, [])
|
||||
elif "=" in line and cur is not None:
|
||||
k, v = line.split("=", 1)
|
||||
out[cur].append((k.strip(), v.strip()))
|
||||
return out
|
||||
|
||||
|
||||
def find(mech_dir, ext):
|
||||
for f in sorted(os.listdir(mech_dir)):
|
||||
if f.lower().endswith(ext) and "}" not in f:
|
||||
return os.path.join(mech_dir, f)
|
||||
return None
|
||||
|
||||
|
||||
def geometry_for(mech_dir, part):
|
||||
"""The .erf a joint draws, via armaturevideo/<part>.video.
|
||||
|
||||
Takes the [lod] block specifically -- a torso's file starts with the
|
||||
running-lights block, whose geometry is not the part.
|
||||
"""
|
||||
vid = os.path.join(mech_dir, "armaturevideo", part + ".video")
|
||||
if not os.path.exists(vid):
|
||||
return None
|
||||
t = open(vid, "rb").read().decode("latin-1")
|
||||
m = re.search(r'^\[lod\][^\[]*?Geometry=([^\r\n]+)', t, re.M | re.I | re.S)
|
||||
if not m:
|
||||
m = re.search(r'Geometry=([^\r\n]+)', t)
|
||||
return m.group(1).strip() if m else None
|
||||
|
||||
|
||||
def build(mech_dir):
|
||||
arm = find(mech_dir, ".armature")
|
||||
con = find(mech_dir, ".contents")
|
||||
if not arm:
|
||||
raise SystemExit(f"no .armature in {mech_dir}")
|
||||
|
||||
apages = pages(arm)
|
||||
children = collections.defaultdict(list)
|
||||
info = {}
|
||||
for name, kv in apages.items():
|
||||
d = dict(kv)
|
||||
info[name] = d
|
||||
for k, v in kv:
|
||||
if k.lower() == "child":
|
||||
children[name].append(v.strip())
|
||||
|
||||
models = {}
|
||||
if con:
|
||||
for name, kv in pages(con).items():
|
||||
for k, v in kv:
|
||||
if k.lower() == "model":
|
||||
models[name.lower()] = v.strip()
|
||||
|
||||
parented = {c for cs in children.values() for c in cs}
|
||||
roots = [n for n in apages if n not in parented]
|
||||
return apages, children, info, models, roots
|
||||
|
||||
|
||||
def render(mech_dir, geometry_only=False):
|
||||
apages, children, info, models, roots = build(mech_dir)
|
||||
sizes = {f.lower(): os.path.getsize(os.path.join(mech_dir, f))
|
||||
for f in os.listdir(mech_dir) if f.lower().endswith(".erf")}
|
||||
lines = []
|
||||
|
||||
def walk(name, depth, last, prefix):
|
||||
d = info.get(name, {})
|
||||
model = models.get(name.lower(), "")
|
||||
part = None
|
||||
m = re.match(r'armaturedata[\\/](.+)\.data', model, re.I)
|
||||
if m:
|
||||
part = m.group(1)
|
||||
geo = geometry_for(mech_dir, part) if part else None
|
||||
size = sizes.get((geo or "").lower())
|
||||
tag = ""
|
||||
if geo:
|
||||
tag = f" <- {geo}" + (f" ({size:,} B)" if size else "")
|
||||
elif model and model.lower() != "basic.data":
|
||||
tag = f" <- {model}"
|
||||
trans = d.get("Translation", d.get("translation", ""))
|
||||
rot = d.get("Rotation", d.get("rotation", ""))
|
||||
pos = f" @[{trans}]" if trans and trans.strip() not in ("0.0 0.0 0.0", "0 0 0") else ""
|
||||
if geometry_only and not geo:
|
||||
pass
|
||||
else:
|
||||
branch = "" if depth == 0 else ("`-- " if last else "|-- ")
|
||||
lines.append(f"{prefix}{branch}{name}{tag}{pos}")
|
||||
kids = children.get(name, [])
|
||||
newprefix = prefix + ("" if depth == 0 else (" " if last else "| "))
|
||||
for i, k in enumerate(kids):
|
||||
walk(k, depth + 1, i == len(kids) - 1, newprefix)
|
||||
|
||||
for r in roots:
|
||||
walk(r, 0, True, "")
|
||||
return lines, sizes
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("mech_dir")
|
||||
ap.add_argument("--geometry-only", action="store_true",
|
||||
help="hide joints and sites that draw nothing")
|
||||
args = ap.parse_args()
|
||||
lines, sizes = render(args.mech_dir, args.geometry_only)
|
||||
name = os.path.basename(args.mech_dir.rstrip("/"))
|
||||
print(f"=== {name}: {len(lines)} nodes, {len(sizes)} .erf, "
|
||||
f"{sum(sizes.values()):,} B of geometry ===")
|
||||
print("\n".join(lines))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
constants.py - reverse tables for the `.data` keys authored as symbolic names.
|
||||
|
||||
Four `[GameData]` keys are written in the source as a symbol rather than a
|
||||
literal, and the record stores only the resolved integer. Emitting a faithful
|
||||
`.data` therefore needs the int -> symbol direction:
|
||||
|
||||
MechID $(M_Annihilator) MechLabHeaders.h #define M_Annihilator 0
|
||||
TechType $(Tech_IS) MechLabHeaders.h #define Tech_IS 0
|
||||
NameIndex $(IDS_Annihilator) MissionLang.defines #define IDS_ANNIHILATOR 576
|
||||
MoveTypeFlag LEGJUMPMOVETYPE MWObject.hpp enum + MWObject_Tool.cpp:20
|
||||
|
||||
MoveTypeFlag is the odd one out: a bare token, not `$(...)`, matched by the
|
||||
`stricmp` chain in `MWObject__GameModel::ConvertStringToMoveType`. Its values
|
||||
come from the anonymous enum at MWObject.hpp:136, where LEG is 0 and the order
|
||||
is NOT the same as the stricmp chain, so the enum is the authority.
|
||||
|
||||
The `.defines` and `.h` files spell symbols in mixed case while sources
|
||||
reference them in any case, so lookups here are case-insensitive.
|
||||
"""
|
||||
import re
|
||||
|
||||
REPO = "/home/rich/Repositories/firestorm/Gameleap"
|
||||
MECHLAB_HEADERS = REPO + "/mw4/Content/ShellScripts/MechLabHeaders.h"
|
||||
MISSIONLANG_DEFINES = REPO + "/mw4/Content/Defines/MissionLang.defines"
|
||||
|
||||
# MWObject.hpp:136. Declaration order is the value order.
|
||||
MOVE_TYPE = [
|
||||
"LEGMOVETYPE", "LEGJUMPMOVETYPE", "TRACKMOVETYPE", "WHEELMOVETYPE",
|
||||
"FLYERMOVETYPE", "HOVERMOVETYPE", "HELIMOVETYPE", "NONEMOVETYPE",
|
||||
"DROPSHIPMOVETYPE", "WATERMOVETYPE",
|
||||
]
|
||||
|
||||
_DEFINE = re.compile(r'^\s*#define\s+(\w+)\s+(-?\d+)\s*$', re.M)
|
||||
|
||||
|
||||
def defines(path, prefix):
|
||||
"""-> {value: [symbols]} for every `#define <prefix>NAME <int>` in a file.
|
||||
|
||||
Values are not unique: IDS_FIRSTSKIN and IDS_WOLFHOUND are both 501, so a
|
||||
single winner cannot be picked here without knowing which mech is being
|
||||
emitted.
|
||||
"""
|
||||
txt = open(path, encoding="latin-1", errors="replace").read()
|
||||
out = {}
|
||||
for name, val in _DEFINE.findall(txt):
|
||||
if name.lower().startswith(prefix.lower()):
|
||||
out.setdefault(int(val), []).append(name)
|
||||
return out
|
||||
|
||||
|
||||
def tables():
|
||||
"""-> {sourceKey: {intValue: [sourceToken]}} for the four symbolic keys."""
|
||||
wrap = lambda d: {v: [f"$({s})" for s in syms] for v, syms in d.items()}
|
||||
return {
|
||||
"MechID": wrap(defines(MECHLAB_HEADERS, "M_")),
|
||||
"TechType": wrap(defines(MECHLAB_HEADERS, "Tech_")),
|
||||
"NameIndex": wrap(defines(MISSIONLANG_DEFINES, "IDS_")),
|
||||
"MoveTypeFlag": {i: [s] for i, s in enumerate(MOVE_TYPE)},
|
||||
}
|
||||
|
||||
|
||||
def symbol(tbl, key, value, hint=None):
|
||||
"""Pick the source token for a value, preferring one naming the chassis."""
|
||||
syms = tbl.get(key, {}).get(value)
|
||||
if not syms:
|
||||
return None
|
||||
if hint and len(syms) > 1:
|
||||
h = norm(hint)
|
||||
for s in syms:
|
||||
if h and h in norm(s):
|
||||
return s
|
||||
return syms[0]
|
||||
|
||||
|
||||
def norm(tok):
|
||||
"""Compare symbols ignoring case and $() wrapping."""
|
||||
return re.sub(r'[^a-z0-9_]', '', tok.lower())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for key, tbl in tables().items():
|
||||
print(f"{key:14s} {len(tbl):5d} values e.g. {list(tbl.items())[:2]}")
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
contents.py - decompiler for a mech `<chassis>.contents` file.
|
||||
|
||||
`.contents` is the thin half of the pair `armature.py` already handles. It
|
||||
`!include`s `<chassis>.armature` and then gives every joint and site exactly two
|
||||
entries:
|
||||
|
||||
[joint_torso]
|
||||
Model=basic.data
|
||||
ExecutionState=AlwaysExecuteState
|
||||
|
||||
Both values ride in the same CreateMessages `armature.py` walks -- the `.armature`
|
||||
source contributes a page's geometry, the `.contents` source contributes its
|
||||
Model and ExecutionState, and the packer merges them into one message per page.
|
||||
So nothing new has to be located: `Model` is `dataListID` (record id in the HIGH
|
||||
word) and `ExecutionState` is the enum at offset 76.
|
||||
|
||||
Site pages are the exception. `{sites}` records store only name, rotation and
|
||||
translation, so a site's Model and ExecutionState are not in the package at all.
|
||||
They do not need to be: all **2926** site pages across the 89 mech `.contents`
|
||||
files carry the identical pair `basic.data` / `AlwaysExecuteState`, so they are
|
||||
emitted as constants rather than guessed per mech.
|
||||
|
||||
python3 contents.py <chassis-record-dir> [-o out.contents]
|
||||
python3 contents.py --verify
|
||||
"""
|
||||
import argparse, collections, glob, os, re, struct, subprocess, sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import mw4msg, subsystems
|
||||
|
||||
MANIFEST = "/home/rich/Repositories/FS_Ours_extracted/_manifest.tsv"
|
||||
REC_RE = re.compile(r'^(?P<chassis>.+?)\.contents\[(?P<parent>[^\]]+)\]\{(?P<kind>armature|sites)\}$',
|
||||
re.I)
|
||||
EXEC_STATE = {1: "NeverExecuteState", 2: "AlwaysExecuteState", 6: "ActiveState"}
|
||||
SITE_DEFAULTS = [("Model", "basic.data"), ("ExecutionState", "AlwaysExecuteState")]
|
||||
|
||||
EXEC_OFF = 76
|
||||
DATALIST_OFF = 84
|
||||
|
||||
|
||||
def relative(path, chassis, folder=None):
|
||||
"""Model= is written relative to the mech folder, not from the package root.
|
||||
|
||||
The package path uses the FOLDER name, which need not match the record stem
|
||||
(V4H's jenner2c/ holds jenner_2c.* records), so both are tried.
|
||||
"""
|
||||
if not path:
|
||||
return path
|
||||
for name in (chassis, folder):
|
||||
if name:
|
||||
path = re.sub(rf'^mechs[\\/]{re.escape(name)}[\\/]', '', path, flags=re.I)
|
||||
return path
|
||||
|
||||
|
||||
def decompile(mech_dir, manifest=None):
|
||||
"""-> (chassis, [(pageName, [(key, value)])])"""
|
||||
manifest = manifest if manifest is not None else subsystems.load_manifest(MANIFEST)
|
||||
chassis = None
|
||||
folder = os.path.basename(mech_dir.rstrip("/"))
|
||||
pages = collections.OrderedDict()
|
||||
|
||||
root = [p for p in glob.glob(os.path.join(mech_dir, "*.contents"))
|
||||
if not re.search(r'[\[\{]', os.path.basename(p))]
|
||||
for path in sorted(glob.glob(os.path.join(mech_dir, "*{armature}"))) + root:
|
||||
base = os.path.basename(path)
|
||||
m = REC_RE.match(base)
|
||||
if m:
|
||||
chassis = chassis or m.group("chassis")
|
||||
else:
|
||||
chassis = chassis or base[:-len(".contents")]
|
||||
with open(path, "rb") as fh:
|
||||
data = fh.read()
|
||||
for _off, msg in mw4msg.walk(data):
|
||||
name = mw4msg.joint_name(msg)
|
||||
if not name:
|
||||
continue
|
||||
model = manifest.get(mw4msg.record_id(msg, DATALIST_OFF))
|
||||
state = EXEC_STATE.get(struct.unpack_from("<i", msg, EXEC_OFF)[0])
|
||||
pages.setdefault(name, [("Model", relative(model, chassis, folder)),
|
||||
("ExecutionState", state)])
|
||||
|
||||
for path in sorted(glob.glob(os.path.join(mech_dir, "*{sites}"))):
|
||||
with open(path, "rb") as fh:
|
||||
for name, _rot, _trans in mw4msg.read_sites(fh.read()):
|
||||
pages.setdefault(name, list(SITE_DEFAULTS))
|
||||
|
||||
return chassis, list(pages.items())
|
||||
|
||||
|
||||
def emit(chassis, pages):
|
||||
lines = ["[includes]", f"!include={chassis}.armature", ""]
|
||||
for name, kv in pages:
|
||||
lines.append(f"[{name}]")
|
||||
lines.extend(f"{k}={v}" for k, v in kv)
|
||||
lines.append("")
|
||||
return "\r\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("mech_dir", nargs="?")
|
||||
ap.add_argument("-o", "--output")
|
||||
ap.add_argument("-m", "--manifest", default=MANIFEST)
|
||||
ap.add_argument("--verify", action="store_true")
|
||||
args = ap.parse_args()
|
||||
if args.verify:
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
raise SystemExit(subprocess.call([sys.executable,
|
||||
os.path.join(here, "verify_contents.py")]))
|
||||
if not args.mech_dir:
|
||||
ap.error("mech_dir required")
|
||||
chassis, pages = decompile(args.mech_dir, subsystems.load_manifest(args.manifest))
|
||||
text = emit(chassis, pages)
|
||||
if args.output:
|
||||
with open(args.output, "wb") as fh:
|
||||
fh.write(text.encode("latin-1"))
|
||||
print(f"wrote {args.output}")
|
||||
else:
|
||||
sys.stdout.write(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,237 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
damage.py - decompiler for a mech `<chassis>.damage` file.
|
||||
|
||||
The record is a bare concatenation of variable-length objects with no index and
|
||||
no length prefixes: parsing means walking forward, reading a classID, and using
|
||||
it to decide what follows. Written by `MWObject::CreateDamageStream`
|
||||
(MWObject_Tool.cpp:1149), which iterates the source pages in order and dispatches
|
||||
on whether a page carries a `DamageZone` entry.
|
||||
|
||||
Armour page -- DamageObject::ConstructDamageObjectStream (DamageObject.cpp:157):
|
||||
|
||||
classID, baseArmorValue, currentArmorValue, scaleSplashDamage,
|
||||
damageObjectName (MString), internalDamageZoneID, armorZone, damageLevel,
|
||||
armorType, maxArmorValue, attachedToZone
|
||||
|
||||
Internal page -- InternalDamageObject::ConstructInternalDamageObjectStream
|
||||
(DamageObject.cpp:931) plus the MW4 subclass (MWDamageObject.cpp:88):
|
||||
|
||||
classID, baseInternalDamage, currentInternalDamage,
|
||||
parentEntityName (MString), damageMode, damageZone, damagePropagationZone,
|
||||
internalType, attachedTo, damageEffects[], missileSlots, projectileSlots,
|
||||
beamSlots, omniSlots
|
||||
|
||||
MString is an int length NOT counting the terminator, the characters, then a NUL
|
||||
-- the same encoding as the {FootSteps} stream.
|
||||
|
||||
Two traps:
|
||||
|
||||
* The armour page stores its own name, but the internal page does NOT. Internal
|
||||
page names are reconstructed as `<Zone>Internal`, which every one of the 89
|
||||
mech .damage files follows, using the damageZone that IS stored.
|
||||
* ArmorZone and InternalZone are DIFFERENT enums. ArmorZone has
|
||||
CenterRearTorso at 7 and Head at 8; InternalZone has Head at 7 and no rear
|
||||
torso entry. Conflating them silently mislabels head and torso zones.
|
||||
|
||||
python3 damage.py <chassis-record-dir> [-o out.damage]
|
||||
python3 damage.py --verify
|
||||
"""
|
||||
import argparse, collections, os, re, struct, subprocess, sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import subsystems
|
||||
|
||||
MANIFEST = "/home/rich/Repositories/FS_Ours_extracted/_manifest.tsv"
|
||||
|
||||
ARMOR_CLASS = 468 # Adept::DamageObject
|
||||
INTERNAL_CLASS = 1162 # MechWarrior4::MWInternalDamageObject
|
||||
|
||||
# DamageObject.hpp:363 -- note CenterRearTorso, absent from the internal enum.
|
||||
ARMOR_ZONE = {
|
||||
-1: "NullZone", 0: "LeftLeg", 1: "RightLeg", 2: "LeftArm", 3: "RightArm",
|
||||
4: "RightTorso", 5: "LeftTorso", 6: "CenterTorso", 7: "CenterRearTorso",
|
||||
8: "Head", 9: "Special1", 10: "Special2", 11: "DefaultZone",
|
||||
}
|
||||
# DamageObject.hpp:204
|
||||
INTERNAL_ZONE = {
|
||||
-1: "NullZone", 0: "LeftLeg", 1: "RightLeg", 2: "LeftArm", 3: "RightArm",
|
||||
4: "RightTorso", 5: "LeftTorso", 6: "CenterTorso", 7: "Head",
|
||||
8: "Special1", 9: "Special2", 10: "VehicleHull", 11: "VehicleWeapon",
|
||||
12: "VehicleSpecial", 13: "DefaultZone",
|
||||
}
|
||||
# DamageObject.hpp:185
|
||||
DAMAGE_MODE = {
|
||||
0: "GeneralDamageMode", 1: "GimpLeftDamageMode", 2: "GimpRightDamageMode",
|
||||
3: "DestructionDamageMode", 4: "DetachableDamageMode", 5: "EngineDamageMode",
|
||||
6: "NextDamageMode", 7: "HeadShotDamageMode", 8: "GyroHitDamageMode",
|
||||
9: "TorsoLeftDamageMode", 10: "TorsoRightDamageMode",
|
||||
11: "ArmLeftDamageMode", 12: "ArmRightDamageMode",
|
||||
}
|
||||
|
||||
|
||||
class Reader:
|
||||
def __init__(self, blob):
|
||||
self.b, self.o = blob, 0
|
||||
|
||||
def i32(self):
|
||||
v = struct.unpack_from("<i", self.b, self.o)[0]
|
||||
self.o += 4
|
||||
return v
|
||||
|
||||
def u32(self):
|
||||
v = struct.unpack_from("<I", self.b, self.o)[0]
|
||||
self.o += 4
|
||||
return v
|
||||
|
||||
def f32(self):
|
||||
v = struct.unpack_from("<f", self.b, self.o)[0]
|
||||
self.o += 4
|
||||
return v
|
||||
|
||||
def mstring(self):
|
||||
n = self.u32()
|
||||
s = self.b[self.o:self.o + n].decode("latin-1")
|
||||
self.o += n + 1 # length excludes the terminator
|
||||
return s
|
||||
|
||||
def done(self):
|
||||
return self.o >= len(self.b)
|
||||
|
||||
|
||||
def parse(blob):
|
||||
"""-> [(kind, {field: value})] in stream order."""
|
||||
r = Reader(blob)
|
||||
out = []
|
||||
while not r.done():
|
||||
class_id = r.i32()
|
||||
if class_id == ARMOR_CLASS:
|
||||
rec = {
|
||||
"baseArmorValue": r.f32(),
|
||||
"currentArmorValue": r.f32(),
|
||||
"scaleSplashDamage": r.f32(),
|
||||
"name": r.mstring(),
|
||||
"internalDamageZone": r.i32(),
|
||||
"armorZone": r.i32(),
|
||||
"damageLevel": r.i32(),
|
||||
"armorType": r.i32(),
|
||||
"maxArmorValue": r.f32(),
|
||||
"attachedToZone": r.i32(),
|
||||
}
|
||||
out.append(("armor", rec))
|
||||
elif class_id == INTERNAL_CLASS:
|
||||
rec = {
|
||||
"baseInternalDamage": r.f32(),
|
||||
"currentInternalDamage": r.f32(),
|
||||
"parentEntityName": r.mstring(),
|
||||
"damageMode": r.i32(),
|
||||
"damageZone": r.i32(),
|
||||
"damagePropagationZone": r.i32(),
|
||||
"internalType": r.i32(),
|
||||
"attachedTo": r.i32(),
|
||||
}
|
||||
count = r.i32()
|
||||
rec["effects"] = [(r.u32(), r.f32()) for _ in range(count)]
|
||||
rec["missileSlots"] = r.i32()
|
||||
rec["projectileSlots"] = r.i32()
|
||||
rec["beamSlots"] = r.i32()
|
||||
rec["omniSlots"] = r.i32()
|
||||
out.append(("internal", rec))
|
||||
else:
|
||||
raise ValueError(f"unknown classID {class_id} at offset {r.o - 4}")
|
||||
return out
|
||||
|
||||
|
||||
def fmt(x):
|
||||
"""Match the authored style: 1.0, 80.0, 0.14, .99"""
|
||||
if x == int(x):
|
||||
return f"{x:.1f}"
|
||||
return f"{x:g}"
|
||||
|
||||
|
||||
def decompile(mech_dir, manifest=None):
|
||||
"""-> [(pageName, [(key, value)])] in stream order."""
|
||||
manifest = manifest if manifest is not None else subsystems.load_manifest(MANIFEST)
|
||||
blob = record(mech_dir)
|
||||
pages = []
|
||||
for kind, r in parse(blob):
|
||||
if kind == "armor":
|
||||
kv = [
|
||||
("BaseArmorValue", fmt(r["baseArmorValue"])),
|
||||
("MaxArmorValue", fmt(r["maxArmorValue"])),
|
||||
("ScaleSplashDamage", f"{r['scaleSplashDamage']:g}"),
|
||||
("InternalDamageZone", INTERNAL_ZONE.get(r["internalDamageZone"])),
|
||||
("ArmorZone", ARMOR_ZONE.get(r["armorZone"])),
|
||||
]
|
||||
if r["attachedToZone"] != -1:
|
||||
kv.append(("SpecialAttachedToZone", ARMOR_ZONE.get(r["attachedToZone"])))
|
||||
pages.append((r["name"], kv))
|
||||
else:
|
||||
zone = INTERNAL_ZONE.get(r["damageZone"], "Null")
|
||||
kv = [
|
||||
("DamageZone", zone),
|
||||
("BaseInternalDamage", fmt(r["baseInternalDamage"])),
|
||||
]
|
||||
for rid, pct in r["effects"]:
|
||||
path = manifest.get(rid >> 16, f"<unresolved:{rid}>")
|
||||
kv.append(("DamageEffect", f"{path},{pct:g}"))
|
||||
# no source writes GeneralDamageMode explicitly, so 0 means "omitted"
|
||||
if r["damageMode"]:
|
||||
kv.append(("DamageMode", DAMAGE_MODE.get(r["damageMode"])))
|
||||
kv.append(("ParentEntityName", r["parentEntityName"]))
|
||||
if r["damagePropagationZone"] != -1:
|
||||
kv.append(("DamagePropagationZone",
|
||||
INTERNAL_ZONE.get(r["damagePropagationZone"])))
|
||||
if r["attachedTo"] != -1:
|
||||
kv.append(("SpecialAttachedToZone", ARMOR_ZONE.get(r["attachedTo"])))
|
||||
for key, field in (("MissileSlots", "missileSlots"),
|
||||
("ProjectileSlots", "projectileSlots"),
|
||||
("BeamSlots", "beamSlots"),
|
||||
("OmniSlots", "omniSlots")):
|
||||
if r[field]:
|
||||
kv.append((key, str(r[field])))
|
||||
pages.append((zone + "Internal", kv))
|
||||
return pages
|
||||
|
||||
|
||||
def record(mech_dir):
|
||||
ch = os.path.basename(mech_dir.rstrip("/")).lower()
|
||||
for fn in os.listdir(mech_dir):
|
||||
if fn.lower().endswith(".damage"):
|
||||
return open(os.path.join(mech_dir, fn), "rb").read()
|
||||
raise SystemExit(f"no .damage record in {mech_dir}")
|
||||
|
||||
|
||||
def emit(pages):
|
||||
lines = []
|
||||
for name, kv in pages:
|
||||
lines.append(f"[{name}]")
|
||||
lines.extend(f"{k}={v}" for k, v in kv)
|
||||
lines.append("")
|
||||
return "\r\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("mech_dir", nargs="?")
|
||||
ap.add_argument("-o", "--output")
|
||||
ap.add_argument("-m", "--manifest", default=MANIFEST)
|
||||
ap.add_argument("--verify", action="store_true")
|
||||
args = ap.parse_args()
|
||||
if args.verify:
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
raise SystemExit(subprocess.call([sys.executable,
|
||||
os.path.join(here, "verify_damage.py")]))
|
||||
if not args.mech_dir:
|
||||
ap.error("mech_dir required")
|
||||
text = emit(decompile(args.mech_dir, subsystems.load_manifest(args.manifest)))
|
||||
if args.output:
|
||||
with open(args.output, "wb") as fh:
|
||||
fh.write(text.encode("latin-1"))
|
||||
print(f"wrote {args.output}")
|
||||
else:
|
||||
sys.stdout.write(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,278 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
data.py - decompiler for a mech `<chassis>.data` file.
|
||||
|
||||
Rebuilds the `[GameData]` page from the compiled records:
|
||||
|
||||
<chassis>.data{GameModel} 1636-byte flat struct (datamap.py)
|
||||
<chassis>.data{FootSteps} foot-step texture list
|
||||
<chassis>.data[shadow] inline Shadow notation block
|
||||
|
||||
Value sources, in order of preference:
|
||||
|
||||
* a struct member - typed read through datamap.chain_layout()
|
||||
* a ResourceID member - record id (HIGH word) resolved via the manifest
|
||||
* a symbolic constant - int reversed through constants.py
|
||||
* an explicit factory key - handled below, because SaveGameModel writes these
|
||||
outside the attribute table
|
||||
|
||||
Four keys cannot be recovered and are deliberately omitted: BattleDamageRatio,
|
||||
BattleKillBonus, DragoonValue and VehicleTradeValue. They appear in every source
|
||||
.data but are read by NOTHING in the engine - no factory, no attribute
|
||||
registration, no runtime reference - so they never enter the package. They are
|
||||
authoring metadata; omitting them changes no behaviour.
|
||||
|
||||
python3 data.py <chassis-record-dir> [-o out.data]
|
||||
python3 data.py --verify
|
||||
"""
|
||||
import argparse, collections, os, glob, re, struct, subprocess, sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import datamap, constants, subsystems
|
||||
|
||||
MANIFEST = "/home/rich/Repositories/FS_Ours_extracted/_manifest.tsv"
|
||||
|
||||
# Adept.hpp:212. NoMaterial is 0.
|
||||
MATERIALS = [
|
||||
"NoMaterial", "Grass", "Water", "Concrete", "GreyDirt", "BrownDirt", "Rock",
|
||||
"DarkConcrete", "DarkGreyDirt", "DarkBrownDirt", "DarkRock", "Blacktop",
|
||||
"Snow", "Wood", "Lava", "Glass", "Steel", "Us", "Them", "LightMineral",
|
||||
"DarkMineral", "Ash", "CrackedLava", "OpenLava",
|
||||
]
|
||||
|
||||
# Identical in all 64 chassis; SaveGameModel writes them outside the attribute table.
|
||||
CONSTANTS = {
|
||||
"Class": "MechWarrior4::Mech",
|
||||
"FaceLighting": "yes",
|
||||
"LookupLighting": "yes",
|
||||
"VertexLighting": "yes",
|
||||
"LightMapLighting": "no",
|
||||
# byte-identical in all 89 mech .data files, destroyed variants included
|
||||
"Shadow": "{\n[shadow]\nLightType=Shadow\nInnerRadius=4.0\nOuterRadius=10.0\n"
|
||||
"BlobDistance=200.0\nShadowMap=ShadowMask\nIntensity=0.4\n}",
|
||||
}
|
||||
|
||||
# Keys whose source name differs from the struct member name.
|
||||
ALIASES = {
|
||||
"AnimationScript": "animScriptName",
|
||||
"HeatManager": "heatManagerResource",
|
||||
"FootEffectsFile": "footFallEffectsTable",
|
||||
}
|
||||
|
||||
# CraterName is stored as MString::GetHashValue (DeathEntity_Tool.cpp:34), which
|
||||
# is one-way. All 64 chassis hold the same hash, so the single authored value is
|
||||
# recoverable by constancy rather than by inversion.
|
||||
CRATER_HASH = {217688981: "crater01"}
|
||||
|
||||
# Read by nothing in the engine - see module docstring.
|
||||
UNRECOVERABLE = ["BattleDamageRatio", "BattleKillBonus", "DragoonValue", "VehicleTradeValue"]
|
||||
|
||||
SYMBOLIC = ("MechID", "TechType", "NameIndex", "MoveTypeFlag")
|
||||
|
||||
|
||||
def stem(mech_dir):
|
||||
"""File stem used inside a mech folder; it need not match the folder name.
|
||||
|
||||
jenner2c/ holds jenner_2c.*, the same trap Black Hawk/nova sprang on
|
||||
.subsystems -- never assume the folder name.
|
||||
"""
|
||||
names = [f for f in os.listdir(mech_dir) if f.lower().endswith(".data")]
|
||||
if names:
|
||||
return names[0][:-len(".data")]
|
||||
for f in os.listdir(mech_dir):
|
||||
m = re.match(r'(.+)\.data[\{\[]', f, re.I)
|
||||
if m:
|
||||
return m.group(1)
|
||||
return os.path.basename(mech_dir.rstrip("/"))
|
||||
|
||||
|
||||
def record(mech_dir, suffix):
|
||||
"""Read one qualified record. Not glob -- '[shadow]' is a character class."""
|
||||
want = (stem(mech_dir) + suffix).lower()
|
||||
for fn in os.listdir(mech_dir):
|
||||
if fn.lower() == want:
|
||||
return open(os.path.join(mech_dir, fn), "rb").read()
|
||||
return None
|
||||
|
||||
|
||||
def bool_words():
|
||||
"""-> {key: (trueWord, falseWord)}; sources spell these inconsistently.
|
||||
|
||||
Collider and CanBeShot are written true/false, the CanLoad* flags Yes/No.
|
||||
"""
|
||||
seen = collections.defaultdict(collections.Counter)
|
||||
for _ch, kv, _b in datamap.corpus():
|
||||
for k, v in kv.items():
|
||||
w = v.strip().lower()
|
||||
if w in ("true", "false", "yes", "no"):
|
||||
seen[k][w] += 1
|
||||
out = {}
|
||||
for k, c in seen.items():
|
||||
plain = c["true"] + c["false"] >= c["yes"] + c["no"]
|
||||
out[k] = ("true", "false") if plain else ("Yes", "No")
|
||||
return out
|
||||
|
||||
|
||||
def foot_steps(blob):
|
||||
"""-> (defaultTexture, [(texture, materialName)]).
|
||||
|
||||
Stream written by Mech_Tool.cpp:196: int material (-1 = default), int length
|
||||
NOT counting the terminator, the characters, a NUL, then a one-byte
|
||||
isDefault flag.
|
||||
"""
|
||||
default, rows, off = None, [], 0
|
||||
while off + 8 <= len(blob):
|
||||
material, length = struct.unpack_from("<iI", blob, off)
|
||||
off += 8
|
||||
text = blob[off:off + length].decode("latin-1")
|
||||
off += length + 1 # skip the terminator
|
||||
is_default = blob[off] if off < len(blob) else 0
|
||||
off += 1
|
||||
if is_default or material < 0:
|
||||
default = text
|
||||
else:
|
||||
name = MATERIALS[material] if 0 <= material < len(MATERIALS) else str(material)
|
||||
rows.append((text, name))
|
||||
return default, rows
|
||||
|
||||
|
||||
def decompile(mech_dir, manifest=None, retarget_ids=False, obb_dir=None):
|
||||
"""-> OrderedDict key -> value or [values]."""
|
||||
manifest = manifest if manifest is not None else subsystems.load_manifest(MANIFEST)
|
||||
ch = os.path.basename(mech_dir.rstrip("/"))
|
||||
gm = record(mech_dir, ".data{GameModel}")
|
||||
if gm is None:
|
||||
raise SystemExit(f"no GameModel record in {mech_dir}")
|
||||
|
||||
layout = datamap.chain_layout()[0]
|
||||
angles = datamap.angle_fields()
|
||||
tables = constants.tables()
|
||||
words = bool_words()
|
||||
by_member = {datamap.norm(n): n for n in layout}
|
||||
out = collections.OrderedDict()
|
||||
|
||||
def member(key):
|
||||
n = ALIASES.get(key) or by_member.get(datamap.norm(key))
|
||||
return (n, *layout[n]) if n in layout else None
|
||||
|
||||
# every key the corpus knows about, so output matches the authored shape.
|
||||
# Keys only a handful of mechs author (VehicleBattleValue: 1 of 64) are left
|
||||
# out rather than emitted at their default.
|
||||
corpus = datamap.corpus()
|
||||
common = collections.Counter(k for _c, kv, _b in corpus for k in kv)
|
||||
for key in sorted(common):
|
||||
if key in UNRECOVERABLE or common[key] * 2 < len(corpus):
|
||||
continue
|
||||
if key in CONSTANTS:
|
||||
out[key] = CONSTANTS[key]
|
||||
continue
|
||||
if key == "CraterName":
|
||||
name = CRATER_HASH.get(datamap.read(gm, *layout["m_craterID"]))
|
||||
if name:
|
||||
out[key] = name
|
||||
continue
|
||||
m = member(key)
|
||||
if key in SYMBOLIC and m:
|
||||
_n, off, typ, size = m
|
||||
raw = datamap.read(gm, off, typ, size)
|
||||
sym = constants.symbol(tables, key, raw, hint=ch)
|
||||
# MechID/NameIndex are authored as $(M_Chassis)/$(IDS_Chassis). A
|
||||
# foreign package may store an int from a different roster -- V4H's
|
||||
# are shifted by one against ours -- so --retarget-ids emits the
|
||||
# chassis's own symbol and lets the build resolve it.
|
||||
if retarget_ids and key in ("MechID", "NameIndex"):
|
||||
prefix = "M_" if key == "MechID" else "IDS_"
|
||||
if not sym or constants.norm(ch) not in constants.norm(sym):
|
||||
out[key] = f"$({prefix}{ch[:1].upper() + ch[1:]})"
|
||||
continue
|
||||
if sym:
|
||||
out[key] = sym
|
||||
continue
|
||||
if m:
|
||||
name, off, typ, size = m
|
||||
val = datamap.read(gm, off, typ, size, datamap.norm(name) in angles
|
||||
or typ == "Radian")
|
||||
if typ == "ResourceID":
|
||||
path = manifest.get(val >> 16)
|
||||
if path:
|
||||
out[key] = path
|
||||
elif typ in datamap.VECTORS:
|
||||
out[key] = " ".join(f"{v:g}" for v in val)
|
||||
elif typ in ("bool", "BYTE"):
|
||||
yes, no = words.get(key, ("true", "false"))
|
||||
out[key] = yes if val else no
|
||||
elif typ in datamap.FLOATS:
|
||||
out[key] = f"{val:g}"
|
||||
elif typ == "char":
|
||||
if val:
|
||||
out[key] = val
|
||||
else:
|
||||
out[key] = str(val)
|
||||
|
||||
# The OBB filenames are source-side names the package never stores. Prefer the
|
||||
# .obb files actually shipped next to the output so the keys cannot disagree
|
||||
# with them; fall back to the usual convention when none are present.
|
||||
solid = hier = None
|
||||
if obb_dir and os.path.isdir(obb_dir):
|
||||
for fn in sorted(os.listdir(obb_dir)):
|
||||
if not fn.lower().endswith(".obb"):
|
||||
continue
|
||||
if fn.lower().endswith("_solid.obb"):
|
||||
solid = fn
|
||||
else:
|
||||
hier = fn
|
||||
out["SolidOBB"] = solid or f"{ch}_Skeleton_SOLID.obb"
|
||||
out["HierarchicalOBB"] = hier or f"{ch}_Skeleton.obb"
|
||||
|
||||
fs = record(mech_dir, ".data{FootSteps}")
|
||||
if fs:
|
||||
default, rows = foot_steps(fs)
|
||||
if default:
|
||||
out["DefaultFootStepTexture"] = default
|
||||
if rows:
|
||||
out["FootStepTexture"] = [f"{t},{m}" for t, m in rows]
|
||||
|
||||
sh = record(mech_dir, ".data[shadow]")
|
||||
if sh is None:
|
||||
out.pop("Shadow", None)
|
||||
return out
|
||||
|
||||
|
||||
def emit(kv):
|
||||
lines = ["[GameData]"]
|
||||
for key, val in kv.items():
|
||||
for v in (val if isinstance(val, list) else [val]):
|
||||
lines.append(f"{key}={v}")
|
||||
return "\r\n".join(lines) + "\r\n"
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("mech_dir", nargs="?")
|
||||
ap.add_argument("-o", "--output")
|
||||
ap.add_argument("-m", "--manifest", default=MANIFEST,
|
||||
help="package manifest for resolving ResourceIDs")
|
||||
ap.add_argument("--retarget-ids", action="store_true",
|
||||
help="emit the chassis's own MechID/NameIndex symbol when the "
|
||||
"stored int belongs to a different roster")
|
||||
ap.add_argument("--verify", action="store_true")
|
||||
args = ap.parse_args()
|
||||
if args.verify:
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
raise SystemExit(subprocess.call([sys.executable,
|
||||
os.path.join(here, "verify_roundtrip.py")]))
|
||||
if not args.mech_dir:
|
||||
ap.error("mech_dir required")
|
||||
text = emit(decompile(args.mech_dir, subsystems.load_manifest(args.manifest),
|
||||
retarget_ids=args.retarget_ids,
|
||||
obb_dir=os.path.dirname(args.output) if args.output else None))
|
||||
if args.output:
|
||||
with open(args.output, "wb") as fh:
|
||||
fh.write(text.encode("latin-1"))
|
||||
print(f"wrote {args.output}")
|
||||
else:
|
||||
sys.stdout.write(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,267 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
datamap.py - field map for the mech `<chassis>.data{GameModel}` record (1636 B).
|
||||
|
||||
Built from the engine headers, not from value matching.
|
||||
|
||||
Why: value matching alone cannot separate keys that hold the same value in every
|
||||
mech. `dampenWorldJoint` and `fallAdjustmentSeconds` are both 0.5 everywhere, so
|
||||
34 of the 80 numeric keys came out ambiguous, and ordering them by their
|
||||
appearance in the source file assigned several of them wrongly - it put
|
||||
`tiltSpeed` at 672, which is really `slopeDecel2`, and swapped
|
||||
`percentageOfTurnToStartTilt` with `percentageOfSpeedToStartTilt`.
|
||||
|
||||
So the layout is taken from the declaration order in
|
||||
|
||||
Vehicle__GameModel mw4/Code/MW4/Vehicle.hpp
|
||||
Mech__GameModel mw4/Code/MW4/Mech.hpp
|
||||
|
||||
and each block is anchored using the fields that value matching *did* resolve
|
||||
unambiguously. Every anchor agrees:
|
||||
|
||||
Mech block base 756: footReturnSeconds 764, dampenTorsoJoint 800,
|
||||
undampenRootJoint 816, undampenHipJoint 824,
|
||||
scaleInternalTiltDegree 832
|
||||
Vehicle block base 664: minSpeed 692, maxSpeed 696, acceleration 720,
|
||||
decceleration 724, reverseAccelerationMultiplier 728
|
||||
|
||||
All fields in both blocks are 4-byte Scalar/Radian, so field i sits at
|
||||
base + 4*i.
|
||||
"""
|
||||
import glob, math, os, re, struct, collections
|
||||
|
||||
REPO = "/home/rich/Repositories/firestorm/Gameleap"
|
||||
SRC = REPO + "/mw4/Content/Mechs"
|
||||
CODE = REPO + "/code"
|
||||
REC = "/home/rich/Repositories/FS_Ours_extracted/core/mechs"
|
||||
GAMEMODEL_SIZE = 1636
|
||||
|
||||
# The GameModel inheritance chain for a mech, base class first. Bases are NOT
|
||||
# hardcoded: each block starts where the previous one ended. MWMover__GameModel
|
||||
# is just `typedef Adept::Mover__GameModel` (MWMover.hpp:173), so it adds
|
||||
# nothing. `Entity__GameModel` declares no base class and wraps its members in
|
||||
# `#if NSWIZZLE`, which is never defined anywhere in the tree - the #else branch
|
||||
# is live, and the two branches order their members DIFFERENTLY.
|
||||
CHAIN = [
|
||||
("Entity__GameModel", CODE + "/mw4/Libraries/Adept/Entity.hpp"),
|
||||
("Mover__GameModel", CODE + "/mw4/Libraries/Adept/Mover.hpp"),
|
||||
("MWObject__GameModel", CODE + "/mw4/Code/MW4/MWObject.hpp"),
|
||||
("Vehicle__GameModel", CODE + "/mw4/Code/MW4/Vehicle.hpp"),
|
||||
("Mech__GameModel", CODE + "/mw4/Code/MW4/Mech.hpp"),
|
||||
]
|
||||
# Independently measured block starts, used to check the computed chain.
|
||||
ANCHORS = {"Vehicle__GameModel": 664, "Mech__GameModel": 756}
|
||||
# The factories that convert authored degrees into stored radians.
|
||||
FACTORIES = [CODE + "/mw4/Code/MW4/Mech_Tool.cpp", CODE + "/mw4/Code/MW4/Vehicle_Tool.cpp"]
|
||||
SCALAR_TYPES = r'(?:Stuff::Scalar|Stuff::Radian|Stuff::Angle|float)'
|
||||
DEG2RAD = math.pi / 180.0
|
||||
MAX_STRING_LENGTH = 256 # Entity.hpp:199
|
||||
|
||||
# Member sizes. bool is ONE byte, not four -- getting this wrong shifts every
|
||||
# field after the six m_canLoad* flags by exactly 16 bytes.
|
||||
SIZES = {
|
||||
"Scalar": 4, "Radian": 4, "Angle": 4, "float": 4, "int": 4, "unsigned": 4,
|
||||
"DWORD": 4, "WORD": 2, "BYTE": 1, "bool": 1, "ResourceID": 4,
|
||||
"Point3D": 12, "Vector3D": 12, "UnitQuaternion": 16, "RGBAColor": 16,
|
||||
"Motion3D": 24, "LinearMatrix4D": 48, "char": 1,
|
||||
"ClassID": 4, "ReplicatorID": 4, "FactoryRequest": 4, "ObjectID": 4,
|
||||
}
|
||||
MEMBER_RE = re.compile(
|
||||
r'\b((?:Stuff::|Adept::)?(?:Scalar|Radian|Angle|Point3D|Vector3D|UnitQuaternion'
|
||||
r'|RGBAColor|Motion3D|LinearMatrix4D|ResourceID)|(?:Stuff::)?(?:RegisteredClass::)?ClassID'
|
||||
r'|ReplicatorID|ObjectID|(?:\w+::)?FactoryRequest|float|int|bool|BYTE|WORD|DWORD'
|
||||
r'|unsigned|char)\s+([A-Za-z_][\w\s,\[\]]*?);', re.S)
|
||||
|
||||
|
||||
def members(cls, path):
|
||||
"""-> [(typeName, memberName, sizeInBytes)] in declaration order."""
|
||||
txt = open(path, encoding="latin-1").read()
|
||||
m = re.search(rf'class\s+{cls}\s*(?::[^{{;]*)?\{{', txt, re.S)
|
||||
if not m:
|
||||
return []
|
||||
depth, i = 1, m.end() # brace-match; indentation is inconsistent between headers
|
||||
while i < len(txt) and depth:
|
||||
depth += (txt[i] == "{") - (txt[i] == "}")
|
||||
i += 1
|
||||
body = txt[m.end():i - 1]
|
||||
ctor = body.find(f"{cls}(")
|
||||
if ctor > 0:
|
||||
body = body[:ctor]
|
||||
body = re.sub(r'//[^\n]*', '', body)
|
||||
# `typedef int AttributeID;` is not a member. This masked a missing ClassID
|
||||
# for a while: two 4-byte errors that happened to cancel.
|
||||
body = re.sub(r'\btypedef\b[^;]*;', '', body, flags=re.S)
|
||||
body = re.sub(r'#\s*if\s+NSWIZZLE\b.*?#\s*else', '', body, flags=re.S)
|
||||
body = re.sub(r'#\s*(endif|else|if\w*|ifdef|ifndef)[^\n]*', '', body)
|
||||
out = []
|
||||
for d in MEMBER_RE.finditer(body):
|
||||
base = (d.group(1).replace("Stuff::", "").replace("Adept::", "")
|
||||
.replace("RegisteredClass::", ""))
|
||||
base = base.rsplit("::", 1)[-1]
|
||||
for name in d.group(2).split(","):
|
||||
name = name.strip()
|
||||
arr = re.fullmatch(r'([A-Za-z_]\w*)\s*\[\s*([A-Za-z_]\w*|\d+)\s*\]', name)
|
||||
if arr:
|
||||
n = arr.group(2)
|
||||
count = MAX_STRING_LENGTH if n == "MaxStringLength" else int(n)
|
||||
out.append((base, arr.group(1), SIZES[base] * count))
|
||||
elif re.fullmatch(r'[A-Za-z_]\w*', name):
|
||||
out.append((base, name, SIZES[base]))
|
||||
return out
|
||||
|
||||
|
||||
def chain_layout(chain=None, anchors=None, start=0):
|
||||
"""-> ({memberName: (offset, typeName, size)}, {className: baseOffset}).
|
||||
|
||||
Each block starts where the previous ended; /Zp4 means align = min(4, size).
|
||||
Raises if a computed base contradicts an independently measured anchor.
|
||||
Defaults to the mech chain; pass another for Torso/Engine and friends.
|
||||
"""
|
||||
chain = CHAIN if chain is None else chain
|
||||
anchors = ANCHORS if anchors is None else anchors
|
||||
fields, bases, off = {}, {}, start
|
||||
for cls, path in chain:
|
||||
off += (-off) % 4
|
||||
bases[cls] = off
|
||||
want = anchors.get(cls)
|
||||
if want is not None and off != want:
|
||||
raise AssertionError(f"{cls} computed at {off}, measured {want}")
|
||||
for typ, name, size in members(cls, path):
|
||||
off += (-off) % min(4, size)
|
||||
fields.setdefault(name, (off, typ, size))
|
||||
off += size
|
||||
return fields, bases
|
||||
|
||||
|
||||
def angle_fields():
|
||||
"""Fields the tool factory scales by Radians_Per_Degree.
|
||||
|
||||
This cannot be inferred from the header: torsoHitSpringMotionLimit and its
|
||||
siblings are declared plain Stuff::Scalar, yet Mech_Tool.cpp:889 stores
|
||||
`model->torsoHitSpringMotionLimit * Radians_Per_Degree`. Only the writer
|
||||
knows. Stuff::Radian fields (tiltSpeed, tiltDegree, topSpeedTurnRate,
|
||||
fullStopTurnRate) are handled by their declared type as well.
|
||||
"""
|
||||
out = set()
|
||||
for path in FACTORIES:
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
txt = open(path, encoding="latin-1").read()
|
||||
for m in re.finditer(r'model->(\w+)\s*=\s*model->\w+\s*\*\s*Radians_Per_Degree', txt):
|
||||
out.add(norm(m.group(1)))
|
||||
return out
|
||||
|
||||
|
||||
def declared_fields(cls, path):
|
||||
"""-> [(fieldName, isAngle)] in declaration order, scalars only.
|
||||
|
||||
isAngle marks Stuff::Radian / Stuff::Angle. Those are authored in DEGREES in
|
||||
the .data but stored in RADIANS in the record, so they need a pi/180 factor
|
||||
on the way in and 180/pi on the way back out.
|
||||
"""
|
||||
txt = open(path, encoding="latin-1").read()
|
||||
m = re.search(rf'class\s+{cls}\s*:(.*?)^\t\t\}};', txt, re.S | re.M)
|
||||
if not m:
|
||||
return []
|
||||
body = m.group(1)
|
||||
ctor = body.find(f"{cls}(")
|
||||
if ctor > 0:
|
||||
body = body[:ctor]
|
||||
body = re.sub(r'//[^\n]*', '', body)
|
||||
out = []
|
||||
for decl in re.finditer(rf'\b({SCALAR_TYPES})\s+([A-Za-z_][\w\s,]*?);', body, re.S):
|
||||
angle = decl.group(1) in ("Stuff::Radian", "Stuff::Angle")
|
||||
for name in decl.group(2).split(","):
|
||||
n = name.strip()
|
||||
if re.fullmatch(r'[A-Za-z_]\w*', n):
|
||||
out.append((n, angle))
|
||||
return out
|
||||
|
||||
|
||||
def norm(name):
|
||||
return re.sub(r'^m_', '', name).replace("_", "").lower()
|
||||
|
||||
|
||||
def header_field_map():
|
||||
"""normalised member name -> (offset, typeName, size, isAngle)."""
|
||||
angles = angle_fields()
|
||||
out = {}
|
||||
for name, (off, typ, size) in chain_layout()[0].items():
|
||||
key = norm(name)
|
||||
out.setdefault(key, (off, typ, size, typ == "Radian" or key in angles))
|
||||
return out
|
||||
|
||||
|
||||
def gamedata(path):
|
||||
txt = open(path, "rb").read().decode("latin-1")
|
||||
# Shadow={...} contains a line reading "[shadow]"; without hiding braced
|
||||
# blocks the page scan stops there and every later key is lost silently.
|
||||
txt = re.sub(r'\{.*?\}',
|
||||
lambda x: x.group(0).replace("\r", "").replace("\n", "\x01"),
|
||||
txt, flags=re.S)
|
||||
m = re.search(r'^\[GameData\]\r?\n(.*?)(?=^\[[A-Za-z]|\Z)', txt, re.M | re.S)
|
||||
if not m:
|
||||
return collections.OrderedDict()
|
||||
kv = collections.OrderedDict()
|
||||
for k, v in re.findall(r'^([A-Za-z0-9_]+)=([^\r\n]*)', m.group(1), re.M):
|
||||
kv.setdefault(k, v.replace("\x01", "\n"))
|
||||
return kv
|
||||
|
||||
|
||||
def corpus():
|
||||
"""-> [(chassis, {key: value}, gameModelBytes)] for our 64 mech chassis."""
|
||||
out = []
|
||||
for d in sorted(glob.glob(REC + "/*")):
|
||||
ch = os.path.basename(d)
|
||||
gm = [g for g in glob.glob(d + "/*.data{GameModel}")
|
||||
if os.path.basename(g).lower().startswith(ch.lower() + ".data")]
|
||||
if not gm:
|
||||
continue
|
||||
blob = open(gm[0], "rb").read()
|
||||
if len(blob) != GAMEMODEL_SIZE:
|
||||
continue
|
||||
s = [p for p in glob.glob(SRC + "/*/*.data")
|
||||
if os.path.basename(p).lower() == ch.lower() + ".data"]
|
||||
if s:
|
||||
out.append((ch, gamedata(s[0]), blob))
|
||||
return out
|
||||
|
||||
|
||||
def build():
|
||||
"""-> (corpus, {sourceKey: (offset, typeName, size, isAngle)})"""
|
||||
pairs = corpus()
|
||||
hmap = header_field_map()
|
||||
keys = collections.Counter()
|
||||
for _c, kv, _b in pairs:
|
||||
keys.update(kv.keys())
|
||||
return pairs, {k: hmap[norm(k)] for k in keys if norm(k) in hmap}
|
||||
|
||||
|
||||
FLOATS = {"Scalar", "Radian", "Angle", "float"}
|
||||
VECTORS = {"Point3D": 3, "Vector3D": 3, "RGBAColor": 4, "UnitQuaternion": 4}
|
||||
|
||||
|
||||
def read(blob, off, typ="Scalar", size=4, angle=False):
|
||||
"""Decoded value in the units and form the .data source uses."""
|
||||
if typ in FLOATS:
|
||||
v = struct.unpack_from("<f", blob, off)[0]
|
||||
return v / DEG2RAD if angle else v
|
||||
if typ in VECTORS:
|
||||
return list(struct.unpack_from(f"<{VECTORS[typ]}f", blob, off))
|
||||
if typ == "char":
|
||||
return blob[off:off + size].split(b"\0")[0].decode("latin-1")
|
||||
if typ in ("bool", "BYTE"):
|
||||
return blob[off]
|
||||
if typ == "WORD":
|
||||
return struct.unpack_from("<H", blob, off)[0]
|
||||
if typ == "ResourceID":
|
||||
return struct.unpack_from("<I", blob, off)[0]
|
||||
return struct.unpack_from("<i", blob, off)[0]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pairs, field_map = build()
|
||||
print(f"chassis: {len(pairs)} mapped keys: {len(field_map)}")
|
||||
for k, (off, typ, size, angle) in sorted(field_map.items(), key=lambda kv: kv[1][0]):
|
||||
note = " [degrees]" if angle else ""
|
||||
print(f" {off:5d} {typ:14s} {k}{note}")
|
||||
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
destroyed.py - regenerate the .data and .video of a *_destroyed mech variant.
|
||||
|
||||
python3 destroyed.py <mechDir> [--erf X.erf] [--obb X.obb] [-o outDir]
|
||||
python3 destroyed.py --verify # regenerate all 89 of ours and diff
|
||||
|
||||
A destroyed variant is four files:
|
||||
|
||||
<stem>.data text, DeathEntity boilerplate
|
||||
<stem>.video text, four-page render graph
|
||||
<stem>.erf geometry, verbatim in the package
|
||||
<stem>_SOLID.obb collision, verbatim in the package
|
||||
|
||||
The .erf and .obb come straight out of the package. The .data and .video are
|
||||
compiled away (a 12-byte stub plus {Element}/{GameModel}, and a binary element
|
||||
tree with embedded #FRE/#RLM blobs), so they are regenerated from a template.
|
||||
|
||||
That is safe here because the template is invariant. Across all 89 destroyed
|
||||
variants in our own Content tree, every one of Class, OBBCollides, Collider,
|
||||
CanBeShot, CanBeWalkedOn, VertexLighting, FaceLighting, LookupLighting,
|
||||
LightMapLighting and CraterName is identical. The only things that vary are the
|
||||
three filename references, and those are read from the files actually present
|
||||
rather than guessed - which also preserves V4H's own `champion_stroyed` typo,
|
||||
since that is the name their build really uses.
|
||||
|
||||
The .data and .video each appear in two forms in our tree, differing only by a
|
||||
trailing blank line; the majority form is emitted.
|
||||
"""
|
||||
import sys, os, re, glob, argparse, collections
|
||||
|
||||
DATA_TEMPLATE = (
|
||||
"[GameData]\r\n"
|
||||
"Class=MechWarrior4::DeathEntity\r\n"
|
||||
"SolidOBB={obb}\r\n"
|
||||
"OBBCollides=false\r\n"
|
||||
"Collider=false\r\n"
|
||||
"CanBeShot=false\r\n"
|
||||
"CanBeWalkedOn=false\r\n"
|
||||
"VertexLighting=yes\r\n"
|
||||
"FaceLighting=yes\r\n"
|
||||
"LookupLighting=no\r\n"
|
||||
"LightMapLighting=no\r\n"
|
||||
"CraterName=crater1\r\n"
|
||||
"\r\n"
|
||||
"[Renderers]\r\n"
|
||||
"VideoRenderer={video}\r\n"
|
||||
"\r\n"
|
||||
)
|
||||
|
||||
VIDEO_TEMPLATE = (
|
||||
"[lod]\r\n"
|
||||
"Type=ShapeComponent\r\n"
|
||||
"Geometry={erf}\r\n"
|
||||
"\r\n"
|
||||
"[watcher]\r\n"
|
||||
"Type=AttributeWatcherOfInt\r\n"
|
||||
"Attribute=VisualRepresentation\r\n"
|
||||
"SimulationShouldExecute=1\r\n"
|
||||
"\r\n"
|
||||
"[damageappearance]\r\n"
|
||||
"Type=SwitchComponent\r\n"
|
||||
"Input=Watcher\r\n"
|
||||
"Child=LOD\r\n"
|
||||
"\r\n"
|
||||
"[locator]\r\n"
|
||||
"Child=DamageAppearance\r\n"
|
||||
"\r\n"
|
||||
)
|
||||
|
||||
OUR_MECH_SOURCE = "/home/rich/Repositories/firestorm/Gameleap/mw4/Content/Mechs"
|
||||
|
||||
|
||||
def find_parts(mech_dir, compiled_dir=None):
|
||||
"""-> (stem, erfName, obbName). Looks in mech_dir, then the compiled tree."""
|
||||
erfs = [f for f in os.listdir(mech_dir) if f.lower().endswith(".erf")]
|
||||
obbs = [f for f in os.listdir(mech_dir) if f.lower().endswith(".obb")]
|
||||
stem = None
|
||||
if compiled_dir and os.path.isdir(compiled_dir):
|
||||
datas = [f for f in os.listdir(compiled_dir)
|
||||
if f.lower().endswith(".data")]
|
||||
if datas:
|
||||
stem = datas[0][:-len(".data")]
|
||||
if stem is None and obbs:
|
||||
stem = re.sub(r"_solid\.obb$", "", obbs[0], flags=re.I)
|
||||
stem = re.sub(r"\.obb$", "", stem, flags=re.I)
|
||||
return stem, (erfs[0] if erfs else None), (obbs[0] if obbs else None)
|
||||
|
||||
|
||||
def generate(stem, erf, obb):
|
||||
return (DATA_TEMPLATE.format(obb=obb, video=f"{stem}.video"),
|
||||
VIDEO_TEMPLATE.format(erf=erf))
|
||||
|
||||
|
||||
def verify():
|
||||
ok = collections.Counter()
|
||||
problems = []
|
||||
for src_dir in sorted(glob.glob(OUR_MECH_SOURCE + "/*_[Dd]estroyed") +
|
||||
glob.glob(OUR_MECH_SOURCE + "/*_DESTROYED")):
|
||||
datas = glob.glob(src_dir + "/*.data")
|
||||
videos = glob.glob(src_dir + "/*.video")
|
||||
if not datas or not videos:
|
||||
continue
|
||||
stem = os.path.basename(datas[0])[:-len(".data")]
|
||||
want_data = open(datas[0], "rb").read().decode("latin-1")
|
||||
want_video = open(videos[0], "rb").read().decode("latin-1")
|
||||
kv = dict(re.findall(r'^([A-Za-z0-9_]+)=([^\r\n]*)', want_data, re.M))
|
||||
erf = re.search(r'Geometry=([^\r\n]*)', want_video).group(1).strip()
|
||||
got_data, got_video = generate(stem, erf, kv["SolidOBB"].strip())
|
||||
|
||||
ok["files"] += 1
|
||||
for what, want, got, exact_key, soft_key in (
|
||||
(".data", want_data, got_data, "data_exact", "data_soft"),
|
||||
(".video", want_video, got_video, "video_exact", "video_soft")):
|
||||
if got == want:
|
||||
ok[exact_key] += 1
|
||||
elif got.rstrip("\r\n").lower() == want.rstrip("\r\n").lower():
|
||||
# NotationFile compares with _stricmp throughout, and the sources
|
||||
# themselves are inconsistent (one uses [renderers], others
|
||||
# [Renderers]), so case and a trailing blank line are cosmetic.
|
||||
ok[soft_key] += 1
|
||||
else:
|
||||
problems.append((os.path.basename(src_dir), what, want, got))
|
||||
|
||||
print(f"destroyed variants checked : {ok['files']}")
|
||||
print(f" .data equivalent : {ok['data_exact'] + ok['data_soft']}"
|
||||
f" (byte-exact {ok['data_exact']}, +{ok['data_soft']} case / trailing-blank-line only)")
|
||||
print(f" .video equivalent : {ok['video_exact'] + ok['video_soft']}"
|
||||
f" (byte-exact {ok['video_exact']}, +{ok['video_soft']} case / trailing-blank-line only)")
|
||||
if problems:
|
||||
print(f"\n{len(problems)} real differences:")
|
||||
for name, what, want, got in problems[:6]:
|
||||
print(f" {name} {what}")
|
||||
for w, g in zip(want.splitlines(), got.splitlines()):
|
||||
if w != g:
|
||||
print(f" source={w!r}\n got ={g!r}")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("mechdir", nargs="?", help="the *_destroyed folder holding .erf/.obb")
|
||||
ap.add_argument("--compiled", help="matching folder in the _compiled tree, for the stem")
|
||||
ap.add_argument("-o", "--out", help="output folder (defaults to mechdir)")
|
||||
ap.add_argument("--verify", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.verify:
|
||||
verify()
|
||||
return
|
||||
if not args.mechdir:
|
||||
ap.error("give a *_destroyed folder, or --verify")
|
||||
|
||||
stem, erf, obb = find_parts(args.mechdir, args.compiled)
|
||||
if not (stem and erf and obb):
|
||||
sys.exit(f"{args.mechdir}: need a stem, an .erf and an .obb "
|
||||
f"(got stem={stem!r} erf={erf!r} obb={obb!r})")
|
||||
data, video = generate(stem, erf, obb)
|
||||
out = args.out or args.mechdir
|
||||
os.makedirs(out, exist_ok=True)
|
||||
open(os.path.join(out, f"{stem}.data"), "wb").write(data.encode("latin-1"))
|
||||
open(os.path.join(out, f"{stem}.video"), "wb").write(video.encode("latin-1"))
|
||||
print(f"{os.path.basename(args.mechdir)}: {stem}.data + {stem}.video "
|
||||
f"(SolidOBB={obb}, Geometry={erf})")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
instance.py - decompiler for a mech `<chassis>.instance` file.
|
||||
|
||||
One page named after the chassis, holding the model/armature/subsystem/damage
|
||||
references and the mechlab bar ratings. The record is a single
|
||||
`Mech__CreateMessage`, so the layout comes from the CreateMessage chain rather
|
||||
than the GameModel one:
|
||||
|
||||
Replicator -> Entity -> Mover -> MWMover -> MWObject -> Vehicle -> Mech
|
||||
|
||||
`datamap.chain_layout(..., start=16)` computes it -- 16 because the
|
||||
`Connection__Message` header (messageLength, priority, flags) sits in front and
|
||||
is not declared in any of these classes. The result ends at 341 and pads to
|
||||
exactly the 344-byte record, and every offset established independently while
|
||||
decoding `.armature` lands on the nose: classID 16, replicatorID 24,
|
||||
localToParent 28, dataListID 84, alignment 88, jointName 152.
|
||||
|
||||
Two parser gaps had to be closed to get there, both fields typed with names the
|
||||
member regex did not know: `Stuff::RegisteredClass::ClassID` / `ReplicatorID` in
|
||||
the Replicator base, and `Entity__ExecutionStateEngine::FactoryRequest` /
|
||||
`ObjectID` in Entity. Missing them silently shifted everything after offset 76
|
||||
by 8 bytes while still producing a plausible-looking table.
|
||||
|
||||
python3 instance.py <chassis-record-dir> [-o out.instance]
|
||||
python3 instance.py --verify
|
||||
"""
|
||||
import argparse, os, re, struct, subprocess, sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import datamap, mw4msg, subsystems
|
||||
|
||||
MANIFEST = "/home/rich/Repositories/FS_Ours_extracted/_manifest.tsv"
|
||||
C = datamap.CODE
|
||||
CHAIN = [
|
||||
("Replicator__CreateMessage", C + "/mw4/Libraries/Adept/Replicator.hpp"),
|
||||
("Entity__CreateMessage", C + "/mw4/Libraries/Adept/Entity.hpp"),
|
||||
("Mover__CreateMessage", C + "/mw4/Libraries/Adept/Mover.hpp"),
|
||||
("MWMover__CreateMessage", C + "/mw4/Code/MW4/MWMover.hpp"),
|
||||
("MWObject__CreateMessage", C + "/mw4/Code/MW4/MWObject.hpp"),
|
||||
("Vehicle__CreateMessage", C + "/mw4/Code/MW4/Vehicle.hpp"),
|
||||
("Mech__CreateMessage", C + "/mw4/Code/MW4/Mech.hpp"),
|
||||
]
|
||||
CONNECTION_HEADER = 16
|
||||
|
||||
EXEC_STATE = {1: "NeverExecuteState", 2: "AlwaysExecuteState", 6: "ActiveState"}
|
||||
# Entity.hpp:894
|
||||
ALIGNMENT = {0: "DefaultAlignment", 1: "Player", 2: "Enemy",
|
||||
3: "Team1", 4: "Team2", 5: "Team3", 6: "Team4"}
|
||||
|
||||
# source key -> message member, in the order 61 of 89 sources use
|
||||
KEYS = [
|
||||
("Model", "dataListID"),
|
||||
("ExecutionState", "executionState"),
|
||||
("Armature", "armatureStreamResourceID"),
|
||||
("Subsystems", "subsystemStreamResourceID"),
|
||||
("DamageObjects", "damageStreamResourceID"),
|
||||
("Alignment", "alignment"),
|
||||
("CurrentHeat", "currentHeat"),
|
||||
("CurrentCoolant", "currentCoolant"),
|
||||
("MaxCoolant", "maxCoolant"),
|
||||
("DoesHaveInstanceName", "doesHaveInstanceName"),
|
||||
("PowerRating", "m_powerBar"),
|
||||
("ArmorRating", "m_armorBar"),
|
||||
("SpeedRating", "m_speedBar"),
|
||||
("HeatRating", "m_heatBar"),
|
||||
]
|
||||
|
||||
|
||||
def record(mech_dir):
|
||||
for fn in os.listdir(mech_dir):
|
||||
if fn.lower().endswith(".instance"):
|
||||
return open(os.path.join(mech_dir, fn), "rb").read()
|
||||
raise SystemExit(f"no .instance record in {mech_dir}")
|
||||
|
||||
|
||||
def relative(path, chassis):
|
||||
"""References are written relative to the mech folder."""
|
||||
if not path:
|
||||
return path
|
||||
return re.sub(rf'^mechs[\\/]{re.escape(chassis or "")}[\\/]', '', path, flags=re.I)
|
||||
|
||||
|
||||
def num(x):
|
||||
return f"{int(x)}" if float(x) == int(x) else f"{x:g}"
|
||||
|
||||
|
||||
def decompile(mech_dir, manifest=None):
|
||||
"""-> (pageName, [(key, value)])"""
|
||||
manifest = manifest if manifest is not None else subsystems.load_manifest(MANIFEST)
|
||||
blob = record(mech_dir)
|
||||
layout, _ = datamap.chain_layout(CHAIN, {}, start=CONNECTION_HEADER)
|
||||
|
||||
msgs = list(mw4msg.walk(blob, start=0))
|
||||
if len(msgs) != 1:
|
||||
raise SystemExit(f"expected one message in {mech_dir}, got {len(msgs)}")
|
||||
msg = msgs[0][1]
|
||||
|
||||
name = datamap.read(msg, *layout["jointName"])
|
||||
folder = os.path.basename(mech_dir.rstrip("/"))
|
||||
kv = []
|
||||
for key, member in KEYS:
|
||||
off, typ, size = layout[member]
|
||||
val = datamap.read(msg, off, typ, size)
|
||||
if member == "executionState":
|
||||
kv.append((key, EXEC_STATE.get(val, str(val))))
|
||||
elif member == "alignment":
|
||||
kv.append((key, ALIGNMENT.get(val, str(val))))
|
||||
elif typ == "ResourceID":
|
||||
path = manifest.get(val >> 16)
|
||||
kv.append((key, relative(path, folder) if path else ""))
|
||||
elif typ == "bool":
|
||||
kv.append((key, "yes" if val else "no"))
|
||||
elif typ in datamap.FLOATS:
|
||||
kv.append((key, num(val)))
|
||||
else:
|
||||
kv.append((key, str(val)))
|
||||
return name, kv
|
||||
|
||||
|
||||
def emit(name, kv):
|
||||
lines = [f"[{name}]"] + [f"{k}={v}" for k, v in kv]
|
||||
return "\r\n".join(lines) + "\r\n"
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("mech_dir", nargs="?")
|
||||
ap.add_argument("-o", "--output")
|
||||
ap.add_argument("-m", "--manifest", default=MANIFEST)
|
||||
ap.add_argument("--verify", action="store_true")
|
||||
args = ap.parse_args()
|
||||
if args.verify:
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
raise SystemExit(subprocess.call([sys.executable,
|
||||
os.path.join(here, "verify_instance.py")]))
|
||||
if not args.mech_dir:
|
||||
ap.error("mech_dir required")
|
||||
name, kv = decompile(args.mech_dir, subsystems.load_manifest(args.manifest))
|
||||
text = emit(name, kv)
|
||||
if args.output:
|
||||
with open(args.output, "wb") as fh:
|
||||
fh.write(text.encode("latin-1"))
|
||||
print(f"wrote {args.output}")
|
||||
else:
|
||||
sys.stdout.write(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
make_generic_doll.py - build a generic MFD/Radar damage paper doll.
|
||||
|
||||
The six chassis imported from V4H have no damage art. V4H shipped Mad Cat copies
|
||||
under six names, which renders an aligned doll that lies about the mech. Instead
|
||||
this builds one honest generic doll from art the game already owns.
|
||||
|
||||
Source: `hsh/mfd_texture.bmp`, the MFD sprite sheet, which contains a generic
|
||||
mech already separated into components, together with a validated 11-zone
|
||||
mapping in `huddamage.cpp` (`sm2_texture` / `sm2_offset`, lines 472 and 487) --
|
||||
the same zone order `coord.cpp` uses. `RenderAux4SmallMech` draws it, so the
|
||||
geometry below is proven in game rather than measured by us.
|
||||
|
||||
Output is the exploded runtime BMP: every zone appears exactly once, and no two
|
||||
source rectangles touch, so each `texuv` isolates one component. The game
|
||||
reassembles the figure from the `offset` values.
|
||||
|
||||
python3 make_generic_doll.py [-o OUTDIR]
|
||||
"""
|
||||
import argparse, os, sys
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
HSH = "/home/rich/Repositories/firestorm/Gameleap/mw4/hsh"
|
||||
|
||||
# huddamage.cpp:472 -- source rects into mfd_texture.bmp, in coord.cpp zone order
|
||||
SM2_TEXTURE = [
|
||||
(172, 0, 231, 140, "LL", "left leg"),
|
||||
(109, 0, 168, 140, "RL", "right leg"),
|
||||
(30, 144, 67, 246, "LA", "left arm"),
|
||||
(0, 36, 36, 128, "RA", "right arm"),
|
||||
(37, 24, 66, 101, "RT", "right torso"),
|
||||
(0, 132, 28, 209, "LT", "left torso"),
|
||||
(67, 33, 99, 125, "CT", "center torso"),
|
||||
(None, None, None, None, "CTR", "center torso rear"),
|
||||
(67, 0, 99, 31, "HD", "head"),
|
||||
(None, None, None, None, "S1", "special 1"),
|
||||
(None, None, None, None, "S2", "special 2"),
|
||||
]
|
||||
# huddamage.cpp:487 -- assembled positions (the *3 is already applied here)
|
||||
SM2_OFFSET = [
|
||||
(29 * 3, 35 * 3), (6 * 3, 35 * 3), (43 * 3, 11 * 3), (0 * 3, 11 * 3),
|
||||
(12 * 3, 7 * 3), (33 * 3, 7 * 3), (22 * 3, 10 * 3), None,
|
||||
(22 * 3, 0 * 3), None, None,
|
||||
]
|
||||
|
||||
# The native figure is 166x245. Real dolls assemble to roughly 330x335 (Atlas,
|
||||
# Assassin II), so 1.5 lands in the right range and keeps the arithmetic exact:
|
||||
# the generic centre torso becomes 48x138 against Atlas's 48x174.
|
||||
#
|
||||
# Radar is authored on a 410 working canvas against the MFD's 340, and real rows
|
||||
# follow that -- Assassin II's radar rects run about 1.27x its MFD ones -- so the
|
||||
# Radar doll is scaled up to match. The runtime halves Radar values at draw time.
|
||||
MFD_SCALE = 1.5
|
||||
RADAR_SCALE = 1.8
|
||||
CANVAS = 512
|
||||
|
||||
|
||||
def even(v):
|
||||
"""Coordinates are authored even; runtime Radar halves them with integer /2."""
|
||||
return int(round(v / 2.0)) * 2
|
||||
|
||||
|
||||
def components(scale):
|
||||
sheet = Image.open(os.path.join(HSH, "mfd_texture.bmp")).convert("L")
|
||||
out = []
|
||||
for (rect, off) in zip(SM2_TEXTURE, SM2_OFFSET):
|
||||
x0, y0, x1, y1, zone, label = rect
|
||||
if x0 is None:
|
||||
out.append((zone, label, None, None, None))
|
||||
continue
|
||||
piece = sheet.crop((x0, y0, x1, y1))
|
||||
w = even((x1 - x0) * scale)
|
||||
h = even((y1 - y0) * scale)
|
||||
piece = piece.resize((w, h), Image.LANCZOS)
|
||||
out.append((zone, label, piece, (even(off[0] * scale), even(off[1] * scale)), (w, h)))
|
||||
return out
|
||||
|
||||
|
||||
def layout(comps, outline=False):
|
||||
"""Place every piece in the 512 canvas without touching. -> (image, {zone: rect})"""
|
||||
img = Image.new("L", (CANVAS, CANVAS), 0)
|
||||
draw = ImageDraw.Draw(img)
|
||||
rects = {}
|
||||
x, y, row_h, gap = 4, 4, 0, 8
|
||||
for zone, _label, piece, _off, size in comps:
|
||||
if piece is None:
|
||||
rects[zone] = (0, 0, 0, 0)
|
||||
continue
|
||||
w, h = size
|
||||
if x + w + gap > CANVAS:
|
||||
x = 4
|
||||
y += row_h + gap
|
||||
row_h = 0
|
||||
img.paste(piece, (x, y))
|
||||
if outline:
|
||||
draw.rectangle([x, y, x + w - 1, y + h - 1], outline=255, width=2)
|
||||
rects[zone] = (x, y, x + w, y + h)
|
||||
x += w + gap
|
||||
row_h = max(row_h, h)
|
||||
return img, rects
|
||||
|
||||
|
||||
def row(values, width=3):
|
||||
return "{" + ",".join("{" + ",".join(f"{v:{width}d}" for v in t) + "}" for t in values) + "}"
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("-o", "--outdir", default="/home/rich/Repositories/FS_Build_V4H_extracted")
|
||||
args = ap.parse_args()
|
||||
|
||||
comps = components(MFD_SCALE)
|
||||
order = [c[0] for c in comps]
|
||||
|
||||
written = []
|
||||
tables = {}
|
||||
for kind, sub, outlined, scale in (("MFD", "hsh/hud", False, MFD_SCALE),
|
||||
("Radar", "hsh/radar/hud", True, RADAR_SCALE)):
|
||||
comps = components(scale)
|
||||
img, rects = layout(comps, outline=outlined)
|
||||
d = os.path.join(args.outdir, sub)
|
||||
os.makedirs(d, exist_ok=True)
|
||||
p = os.path.join(d, "generic.bmp")
|
||||
img.save(p)
|
||||
written.append(p)
|
||||
texuv = [rects[z] for z in order]
|
||||
offset = [(c[3] if c[3] else (0, 0)) for c in comps]
|
||||
offset = [o if rects[z] != (0, 0, 0, 0) else (0, 0) for z, o in zip(order, offset)]
|
||||
tables[kind] = (texuv, offset)
|
||||
|
||||
doc = os.path.join(args.outdir, "hsh", "GENERIC-DOLL-COORDS.txt")
|
||||
with open(doc, "w", newline="\r\n") as fh:
|
||||
fh.write(TEXT.format(
|
||||
scale=MFD_SCALE,
|
||||
rscale=RADAR_SCALE,
|
||||
zones=" ".join(f"{i}:{z}" for i, z in enumerate(order)),
|
||||
mfd_texuv=row(tables["MFD"][0]),
|
||||
mfd_offset=row(tables["MFD"][1]),
|
||||
rad_texuv=row(tables["Radar"][0]),
|
||||
rad_offset=row(tables["Radar"][1]),
|
||||
))
|
||||
written.append(doc)
|
||||
for p in written:
|
||||
print("wrote", p)
|
||||
|
||||
|
||||
TEXT = """GENERIC MECH DAMAGE PAPER DOLL - coordinates for coord.cpp
|
||||
=========================================================
|
||||
|
||||
WHAT THIS IS
|
||||
A generic 'Mech damage doll for chassis that have no bespoke MFD/Radar art.
|
||||
Six imported V4H chassis (champion, dasher, griffin, jenner2c, marauder,
|
||||
thunderbolt) ship with none. V4H filled the gap with pixel-identical copies
|
||||
of the Mad Cat doll under six names, which renders an aligned display that
|
||||
misrepresents the mech. This is the honest placeholder instead.
|
||||
|
||||
WHERE THE ART CAME FROM
|
||||
Not drawn from scratch. The game already owns a generic 'Mech, separated
|
||||
into components, inside hsh/mfd_texture.bmp (the MFD sprite sheet), with a
|
||||
validated eleven-zone mapping in huddamage.cpp:
|
||||
|
||||
sm2_texture[][4] line 472 source rectangles
|
||||
sm2_offset[][2] line 487 assembled positions
|
||||
|
||||
drawn by RenderAux4SmallMech(). Those tables use the same zone order as
|
||||
coord.cpp, so the geometry below is proven in game, not measured by us.
|
||||
Components were scaled {scale}x for the MFD and {rscale}x for the Radar. The
|
||||
native figure is 166x245 and real dolls assemble to roughly 330x335, so this
|
||||
lands in the right range; the generic centre torso ends up 48x138 against the
|
||||
Atlas's 48x174. The Radar is larger because it is authored on a 410 working
|
||||
canvas against the MFD's 340, and the runtime halves Radar values at draw.
|
||||
|
||||
FILES
|
||||
hsh/hud/generic.bmp 512x512 external MFD doll
|
||||
hsh/radar/hud/generic.bmp 512x512 Radar doll, components outlined 2px
|
||||
white per the Radar pipeline
|
||||
|
||||
Both are the EXPLODED runtime view: each zone appears exactly once and no
|
||||
two source rectangles touch, so a texuv isolates one component. The game
|
||||
reassembles the figure from the offsets.
|
||||
|
||||
ZONE ORDER (index: zone)
|
||||
{zones}
|
||||
|
||||
CTR, S1 and S2 are all-zero, and that is normal rather than a shortcut. Across
|
||||
the 65 shipped rows CTR is zero in ALL 65, S1 in 31 and S2 in 49. Neither
|
||||
draw loop guards zero rectangles - a degenerate quad simply renders nothing -
|
||||
so absent zones are safe. The generic figure has no rear silhouette and no
|
||||
special hardpoints.
|
||||
|
||||
ONE FILE SERVES ALL SIX
|
||||
The BMP name comes only from `texturename[]` in huddamage.cpp; both displays
|
||||
use it (`LoadDamageTexture` and `LoadRadarDamageTexture` are both called with
|
||||
`texturename[m_MechID]`). Nothing in the mech data files names it. So six
|
||||
entries pointing at "hud\\\\generic" share one pair of images - do not make
|
||||
per-chassis copies. (`m_HudMap` is unrelated: that is the mission map.)
|
||||
|
||||
--------------------------------------------------------------------------
|
||||
HOW TO INSTALL
|
||||
--------------------------------------------------------------------------
|
||||
|
||||
1. Copy the art
|
||||
hsh/hud/generic.bmp -> Gameleap/mw4/hsh/hud/generic.bmp
|
||||
hsh/radar/hud/generic.bmp -> Gameleap/mw4/hsh/radar/hud/generic.bmp
|
||||
Loose hsh art is not packed into a .mw4, so no resource rebuild is needed.
|
||||
|
||||
2. huddamage.cpp - add one texturename[] entry per new chassis, all pointing at
|
||||
the same generic art. The array is sized [LastMechID+1], so LastMechID in
|
||||
MechLabHeaders.h must also rise from 64 to 70.
|
||||
|
||||
"hud\\\\generic", // M_Champion 65
|
||||
"hud\\\\generic", // M_Jenner2c 66
|
||||
"hud\\\\generic", // M_Dasher 67
|
||||
"hud\\\\generic", // M_Marauder 68
|
||||
"hud\\\\generic", // M_Thunderbolt 69
|
||||
"hud\\\\generic", // M_Griffin 70
|
||||
|
||||
Keep V4H's append order (65-70). Inserting alphabetically would renumber
|
||||
every existing chassis, and Mech IDs are positional across code, tables and
|
||||
shell scripts.
|
||||
|
||||
3. coord.cpp - widen all four arrays from [65] to [71] and append the SAME row
|
||||
six times to each, once per new chassis. All six share one doll, so all six
|
||||
rows are identical.
|
||||
|
||||
texuv2 / offset2 are the external MFD. texuv3 / offset3 are the Radar, which
|
||||
the runtime divides by two at draw time - store the full-size values here,
|
||||
do not pre-divide.
|
||||
|
||||
texuv2 (MFD source rectangles)
|
||||
{mfd_texuv}
|
||||
|
||||
offset2 (MFD exploded positions)
|
||||
{mfd_offset}
|
||||
|
||||
texuv3 (Radar source rectangles)
|
||||
{rad_texuv}
|
||||
|
||||
offset3 (Radar exploded positions)
|
||||
{rad_offset}
|
||||
|
||||
4. Rebuild. DXRasterizer.cpp includes coord.cpp directly, so a coordinate change
|
||||
needs a Release/Profile rebuild and redeploy of MW4.exe. The art alone does
|
||||
not.
|
||||
|
||||
--------------------------------------------------------------------------
|
||||
NOTES AND LIMITS
|
||||
--------------------------------------------------------------------------
|
||||
|
||||
* Dasher's four commented-out rows were REMOVED from coord.cpp on 2026-08-08.
|
||||
They were unique authored Dasher geometry rather than a copy, but the art they
|
||||
mapped has never existed, and leaving them invited someone to enable them
|
||||
against wrong art. Recoverable from git (last touched in deafc2b0) if real
|
||||
Dasher art is ever produced. The matching commented `"hud\\\\dasher"` entry at
|
||||
huddamage.cpp:81 is still there and should go with them.
|
||||
|
||||
* The generic cannot represent chassis-specific zones. Marauder's cage and
|
||||
Dasher's S1/S2 will not appear.
|
||||
|
||||
* This is a placeholder. The point of a generic silhouette over a borrowed one
|
||||
is that it reads as "no art yet" rather than as the wrong mech. Replace it
|
||||
per chassis when real 1024x1024 renders exist; the authoring pipelines are
|
||||
in MFD-RADAR-MAPPINGS.md.
|
||||
|
||||
* PRE-EXISTING BUGS, unrelated to this work but found while checking it. Three
|
||||
chassis share another mech's rows while having their own art. Overlaying each
|
||||
row on its own art shows:
|
||||
Rifleman (id 49, uses Mad Cat's rows) - BROKEN. CT lands in empty
|
||||
space beside the mech, S1/S2 float in empty corners, and the
|
||||
leg boxes run well past the feet.
|
||||
Battlemaster (id 9, uses Atlas's rows) - BROKEN. CT sits over a
|
||||
left-side arm piece and the leg boxes are narrow centre
|
||||
strips while the real legs fall outside them.
|
||||
Templar (id 54, uses Archer's rows) - fine. Its art was evidently
|
||||
authored to that layout; only the leg boxes clip slightly.
|
||||
Rifleman and Battlemaster need their own measured rows, or art authored to
|
||||
the borrowed layout.
|
||||
"""
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
mw4msg.py - reader for the GameOS "CreateMessage" streams found inside .mw4
|
||||
records (.subsystems, [joint_*]{armature}, .contents, .instance, ...).
|
||||
|
||||
Layout, derived from the engine source rather than guessed:
|
||||
|
||||
MWObject::CreateSubsystemStream / CreateArmatureStream (MWObject_Tool.cpp)
|
||||
WORD span number of replicator IDs consumed
|
||||
N x CreateMessage one per [Page] that was serialised
|
||||
|
||||
Each message is a plain C struct, /Zp4, built by a per-class factory in one of
|
||||
the 51 mw4/Code/MW4/*_Tool.cpp files. The common prefix is:
|
||||
|
||||
off 0 u32 messageLength Connection__Message
|
||||
off 4 u32 messageID
|
||||
off 8 u32 priority
|
||||
off 12 u32 messageFlags
|
||||
off 16 u32 classID Replicator__CreateMessage
|
||||
off 20 u32 replicatorFlags
|
||||
off 24 u32 replicatorID
|
||||
off 28 12f localToParent Entity__CreateMessage (LinearMatrix4D)
|
||||
off 76 u32 executionState
|
||||
off 80 f32 initialAge
|
||||
off 84 u32 dataListID (ResourceID; record id is the HIGH word)
|
||||
off 88 u32 alignment
|
||||
off 92 u32 nameID
|
||||
|
||||
Mover__CreateMessage then adds two 24-byte Motion3D fields (96, 120), and
|
||||
MWMover__CreateMessage adds:
|
||||
|
||||
off 144 u32 siteStreamResourceID
|
||||
off 148 u32 armatureStreamResourceID
|
||||
off 152 char jointName[128]
|
||||
|
||||
Verified: a joint message is exactly 280 bytes, which is 152 + 128.
|
||||
|
||||
localToParent is 3 rows of 4 floats; the rotation is columns 0..2 and the
|
||||
translation is column 3 of each row. Checked against every joint of all 65
|
||||
chassis: 1453/1453 translations matched the source .armature exactly.
|
||||
"""
|
||||
import struct
|
||||
|
||||
HDR_LEN = 0
|
||||
HDR_CLASSID = 16
|
||||
HDR_REPLICATORID = 24
|
||||
ENT_MATRIX = 28
|
||||
ENT_EXECSTATE = 76
|
||||
ENT_INITIALAGE = 80
|
||||
ENT_DATALISTID = 84
|
||||
ENT_ALIGNMENT = 88
|
||||
ENT_NAMEID = 92
|
||||
MWMOVER_SITEID = 144
|
||||
MWMOVER_ARMID = 148
|
||||
MWMOVER_JOINTNAME = 152
|
||||
|
||||
|
||||
def walk(data, start=2):
|
||||
"""Yield (offset, messageBytes) for each message in a span-prefixed stream."""
|
||||
off = start
|
||||
while off + 4 <= len(data):
|
||||
length, = struct.unpack_from("<I", data, off)
|
||||
if length < 16 or off + length > len(data):
|
||||
raise ValueError(f"bad messageLength {length} at offset {off}")
|
||||
yield off, data[off:off + length]
|
||||
off += length
|
||||
if off != len(data):
|
||||
raise ValueError(f"trailing {len(data) - off} bytes")
|
||||
|
||||
|
||||
def span(data):
|
||||
return struct.unpack_from("<H", data, 0)[0]
|
||||
|
||||
|
||||
def u32(msg, off):
|
||||
return struct.unpack_from("<I", msg, off)[0]
|
||||
|
||||
|
||||
def class_id(msg):
|
||||
return u32(msg, HDR_CLASSID)
|
||||
|
||||
|
||||
def record_id(msg, off=ENT_DATALISTID):
|
||||
"""ResourceID packs the package record id in its high word."""
|
||||
return u32(msg, off) >> 16
|
||||
|
||||
|
||||
def matrix(msg):
|
||||
return struct.unpack_from("<12f", msg, ENT_MATRIX)
|
||||
|
||||
|
||||
def translation(msg):
|
||||
m = matrix(msg)
|
||||
return (m[3], m[7], m[11])
|
||||
|
||||
|
||||
def rotation3x3(msg):
|
||||
m = matrix(msg)
|
||||
return (m[0], m[1], m[2], m[4], m[5], m[6], m[8], m[9], m[10])
|
||||
|
||||
|
||||
def joint_name(msg):
|
||||
if len(msg) <= MWMOVER_JOINTNAME:
|
||||
return None
|
||||
return msg[MWMOVER_JOINTNAME:].split(b"\0")[0].decode("latin-1")
|
||||
|
||||
|
||||
def read_sites(data):
|
||||
"""{sites} record: repeated [YawPitchRoll 3f][Point3D 3f][u32 len][name][NUL].
|
||||
|
||||
Written by MWMover__CreateMessage::ConstructCreateMessage (MWMover_Tool.cpp
|
||||
~145): `site_stream << rotation; << translation; << site_name;`
|
||||
Angles are radians.
|
||||
"""
|
||||
out, off = [], 0
|
||||
while off + 28 <= len(data):
|
||||
rx, ry, rz, tx, ty, tz = struct.unpack_from("<6f", data, off)
|
||||
off += 24
|
||||
n, = struct.unpack_from("<I", data, off)
|
||||
off += 4
|
||||
name = data[off:off + n].decode("latin-1")
|
||||
off += n + 1 # names are length-prefixed AND NUL-terminated
|
||||
out.append((name, (rx, ry, rz), (tx, ty, tz)))
|
||||
return out
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
smallmodel.py - decompilers for a mech's `.torso` and `.engine` files.
|
||||
|
||||
Both are single-page `[GameData]` subsystem models stored as flat structs, so
|
||||
they reuse `datamap.chain_layout()` with their own class chains:
|
||||
|
||||
Entity__GameModel -> Subsystem__GameModel -> Torso__GameModel 1608 bytes
|
||||
Entity__GameModel -> Subsystem__GameModel -> Engine__GameModel 64 bytes
|
||||
|
||||
Both chains compute to exactly the record size with no anchoring, which is the
|
||||
check that the member list and alignment are right.
|
||||
|
||||
Sources author the numbers as `$(SYMBOL)` macros from a `!include`d defines file
|
||||
(`!NAME=value` syntax, not `#define`), and the record keeps only the resolved
|
||||
float, so the symbol is restored by reverse lookup where one matches.
|
||||
|
||||
Two quirks worth keeping:
|
||||
|
||||
* The five Torso angles are declared plain `Stuff::Scalar` but `Torso_Tool.cpp`
|
||||
multiplies them by `Radians_Per_Degree`, exactly like the Mech spring fields.
|
||||
* `TotalCritLocations` is read by **nothing** in the engine -- only the 3DS Max
|
||||
export plugin writes it, and the factory reads `TotalSlotsTaken`, which no
|
||||
source sets (the record holds the default 1 while every source says 2). It
|
||||
is emitted as the constant it always is rather than derived.
|
||||
|
||||
python3 smallmodel.py torso|engine <chassis-record-dir> [-o out]
|
||||
python3 smallmodel.py --verify
|
||||
"""
|
||||
import argparse, collections, os, re, subprocess, sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import datamap
|
||||
|
||||
C = datamap.CODE
|
||||
CONTENT = "/home/rich/Repositories/firestorm/Gameleap/mw4/Content"
|
||||
|
||||
SPECS = {
|
||||
"torso": {
|
||||
"chain": [("Entity__GameModel", C + "/mw4/Libraries/Adept/Entity.hpp"),
|
||||
("Subsystem__GameModel", C + "/mw4/Code/MW4/Subsystem.hpp"),
|
||||
("Torso__GameModel", C + "/mw4/Code/MW4/Torso.hpp")],
|
||||
"size": 1608,
|
||||
"factory": C + "/mw4/Code/MW4/Torso_Tool.cpp",
|
||||
"defines": CONTENT + "/Defines/MechTorso.defines",
|
||||
"include": r"Content\Defines\MechTorso.defines",
|
||||
"class": "MechWarrior4::Torso",
|
||||
"keys": [("TwistJointName", "twistJointName"),
|
||||
("PitchJointName", "pitchJointName"),
|
||||
("LeftArmJointName", "leftArmJointName"),
|
||||
("RightArmJointName", "rightArmJointName"),
|
||||
("EyeJointName", "eyeJointName"),
|
||||
("ArmRatioAngle", "armRatioAngle"),
|
||||
("TwistSpeed", "twistSpeed"),
|
||||
("PitchSpeed", "pitchSpeed"),
|
||||
("TwistRadius", "twistRadius"),
|
||||
("PitchRadius", "pitchRadius"),
|
||||
("CageJointName", "cageJointName"),
|
||||
("CageRatioAngle", "cageRatioAngle")],
|
||||
},
|
||||
"engine": {
|
||||
"chain": [("Entity__GameModel", C + "/mw4/Libraries/Adept/Entity.hpp"),
|
||||
("Subsystem__GameModel", C + "/mw4/Code/MW4/Subsystem.hpp"),
|
||||
("Engine__GameModel", C + "/mw4/Code/MW4/Engine.hpp")],
|
||||
"size": 64,
|
||||
"factory": C + "/mw4/Code/MW4/Engine_Tool.cpp",
|
||||
"defines": CONTENT + "/Subsystems/HeatSink.Defines",
|
||||
"include": r"Content\Subsystems\HeatSink.defines",
|
||||
"class": "Mechwarrior4::Engine", # lowercase 'w' in every source
|
||||
"keys": [("NumHeatSinks", "m_numHeatSinks"),
|
||||
("TonsPerUpgrade", "m_tonsPerUpgrade"),
|
||||
("MPSPerUpgrade", "m_mpsPerUpgrade"),
|
||||
("HeatSinkEfficiency", "m_heatSinkEfficiency")],
|
||||
},
|
||||
}
|
||||
|
||||
# Written only by the 3DS Max exporter; the engine never reads it. Uniformly 2.
|
||||
TOTAL_CRIT_LOCATIONS = "2"
|
||||
|
||||
|
||||
def defines(path):
|
||||
"""-> {value: symbol} from a `!NAME=value` defines file."""
|
||||
out = {}
|
||||
if not os.path.exists(path):
|
||||
return out
|
||||
txt = open(path, encoding="latin-1", errors="replace").read()
|
||||
txt = re.sub(r'//[^\n]*', '', txt)
|
||||
for name, val in re.findall(r'^\s*!(\w+)\s*=\s*([-\d.]+)\s*$', txt, re.M):
|
||||
out.setdefault(round(float(val), 6), name)
|
||||
return out
|
||||
|
||||
|
||||
def angle_members(factory):
|
||||
out = set()
|
||||
if not os.path.exists(factory):
|
||||
return out
|
||||
txt = open(factory, encoding="latin-1").read()
|
||||
for m in re.finditer(r'model->(\w+)\s*=\s*model->\w+\s*\*\s*Radians_Per_Degree', txt):
|
||||
out.add(m.group(1))
|
||||
return out
|
||||
|
||||
|
||||
def record(mech_dir, kind):
|
||||
for fn in os.listdir(mech_dir):
|
||||
if fn.lower().endswith(f".{kind}{{gamemodel}}"):
|
||||
return open(os.path.join(mech_dir, fn), "rb").read()
|
||||
return None
|
||||
|
||||
|
||||
def num(x):
|
||||
return f"{int(x)}" if float(x) == int(x) else f"{x:g}"
|
||||
|
||||
|
||||
def decompile(mech_dir, kind):
|
||||
"""-> [(key, value)] for the [GameData] page."""
|
||||
spec = SPECS[kind]
|
||||
blob = record(mech_dir, kind)
|
||||
if blob is None:
|
||||
raise SystemExit(f"no .{kind}{{GameModel}} record in {mech_dir}")
|
||||
layout, _ = datamap.chain_layout(spec["chain"], {})
|
||||
angles = angle_members(spec["factory"])
|
||||
symbols = defines(spec["defines"])
|
||||
|
||||
kv = [("Class", spec["class"]), ("TotalCritLocations", TOTAL_CRIT_LOCATIONS)]
|
||||
for key, member in spec["keys"]:
|
||||
off, typ, size = layout[member]
|
||||
val = datamap.read(blob, off, typ, size, member in angles)
|
||||
if typ == "char":
|
||||
kv.append((key, val))
|
||||
elif typ == "int":
|
||||
kv.append((key, str(val)))
|
||||
else:
|
||||
sym = symbols.get(round(val, 6))
|
||||
kv.append((key, f"$({sym})" if sym else num(val)))
|
||||
return kv
|
||||
|
||||
|
||||
def emit(kind, kv):
|
||||
lines = [f"!include = {SPECS[kind]['include']}", "", "[GameData]"]
|
||||
lines += [f"{k}={v}" for k, v in kv]
|
||||
return "\r\n".join(lines) + "\r\n"
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("kind", nargs="?", choices=sorted(SPECS))
|
||||
ap.add_argument("mech_dir", nargs="?")
|
||||
ap.add_argument("-o", "--output")
|
||||
ap.add_argument("--verify", action="store_true")
|
||||
args = ap.parse_args()
|
||||
if args.verify:
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
raise SystemExit(subprocess.call([sys.executable,
|
||||
os.path.join(here, "verify_smallmodel.py")]))
|
||||
if not args.kind or not args.mech_dir:
|
||||
ap.error("kind and mech_dir required")
|
||||
text = emit(args.kind, decompile(args.mech_dir, args.kind))
|
||||
if args.output:
|
||||
with open(args.output, "wb") as fh:
|
||||
fh.write(text.encode("latin-1"))
|
||||
print(f"wrote {args.output}")
|
||||
else:
|
||||
sys.stdout.write(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
subsystems.py - rebuild a mech's .subsystems source from its packed record.
|
||||
|
||||
python3 subsystems.py <record.subsystems> <manifest.tsv> [-o out.subsystems]
|
||||
python3 subsystems.py --verify # check against all known chassis
|
||||
|
||||
The packed record is `WORD span` followed by one CreateMessage per [Page]
|
||||
(MWObject::CreateSubsystemStream, MWObject_Tool.cpp:1196). Message layout and
|
||||
the derivation of every offset below is documented in ../../DECOMPILING.md.
|
||||
|
||||
Layout beyond the Entity header (Subsystem.hpp, Weapon.hpp, Armor.hpp):
|
||||
|
||||
off 96 i32 subsystemIndex
|
||||
off 100 u8 locationID -> InternalLocation
|
||||
off 104 i32 criticalHitsTaken -> CriticalHitsTaken
|
||||
Armor (152):
|
||||
off 108 9xf32 armour points -> tons, see ARMOR_POINTS_PER_TON
|
||||
off 144 i32 m_armorType -> ArmorType
|
||||
off 148 i32 m_internalType -> InternalType
|
||||
Engine (112):
|
||||
off 108 i32 m_engineUpgrades -> EngineUpgrades
|
||||
SearchLight (236):
|
||||
off 108 char[128] siteName -> Site
|
||||
Weapon (380, or 376 without the trailing field):
|
||||
off 108 char[128] siteName -> Site
|
||||
off 236 char[128] ejectSiteName -> EjectSite
|
||||
off 364 i32 groupIndex -> GroupIndex
|
||||
off 368 i32 ammoCount -> AmmoCount
|
||||
off 372 i32 initialAmmoCount
|
||||
off 376 i32 m_weaponFacing -> WeaponFacing (absent when len == 376)
|
||||
|
||||
`m_weaponFacing` is the "MSL 5.04 Rear Firing Weapons" field. V4H packed
|
||||
champion/griffin/marauder without it (376 bytes) - matching their own release
|
||||
note "Any new mech will not have rear facing weapons" - while dasher, jenner2c
|
||||
and thunderbolt have it. Both forms are read; re-packing with our own exe always
|
||||
emits the 380-byte form.
|
||||
|
||||
PAGE NAMES ARE NOT STORED. The packer only serialises page order, so names are
|
||||
regenerated from the Model= reference with a per-kind counter. They are labels;
|
||||
the runtime keys on order.
|
||||
"""
|
||||
import sys, os, re, glob, struct, argparse, collections
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import mw4msg
|
||||
|
||||
OUR_MECH_SOURCE = "/home/rich/Repositories/firestorm/Gameleap/mw4/Content/Mechs"
|
||||
OUR_RECORDS = "/home/rich/Repositories/FS_Ours_extracted/core/mechs"
|
||||
OUR_MANIFEST = "/home/rich/Repositories/FS_Ours_extracted/_manifest.tsv"
|
||||
ARMOR_DATA = "/home/rich/Repositories/firestorm/Gameleap/mw4/Content/Subsystems/Armor.data"
|
||||
|
||||
# Derived empirically from 63 aligned chassis and cross-checked against the
|
||||
# engine's own text tables.
|
||||
EXEC_STATE = {1: "NeverExecuteState", 2: "AlwaysExecuteState", 6: "ActiveState"}
|
||||
# InternalDamageObject enum, DamageObject.hpp:205
|
||||
ZONE = {255: "NullZone", 0: "LeftLeg", 1: "RightLeg", 2: "LeftArm", 3: "RightArm",
|
||||
4: "RightTorso", 5: "LeftTorso", 6: "CenterTorso", 7: "Head",
|
||||
8: "Special1", 9: "Special2", 10: "VehicleHull", 11: "VehicleWeapon",
|
||||
12: "VehicleSpecial", 13: "DefaultZone"}
|
||||
ARMOR_TYPE = {0: "Standard", 1: "FerroFiberus", 2: "Reactive", 3: "Reflective",
|
||||
4: "Solarian"}
|
||||
# Armor.hpp:275 - a separate 2-value enum, not the armour table. (The engine's own
|
||||
# InternalTypeAsciiToText returns the typo "Statndard"; TextToAscii wants "Standard".)
|
||||
INTERNAL_TYPE = {0: "Standard", 1: "EndoSteel"}
|
||||
ARMOR_KEYS = ["LeftLeg", "RightLeg", "LeftArm", "RightArm", "LeftFrontTorso",
|
||||
"RightFrontTorso", "CenterFrontTorso", "CenterRearTorso", "Head"]
|
||||
|
||||
CLASS_ENGINE, CLASS_LAMS = 1073, 1183
|
||||
|
||||
# Model= basename -> page-name stem. Anything unmatched falls back to the
|
||||
# weapon-category table below.
|
||||
PAGE_STEM = {
|
||||
"heatsinksubsystem.data": "HeatSink", "armor.data": "Armor",
|
||||
"advancedgyrosubsystem.data": "AdvancedGyro", "sensorsubsystem.data": "Sensor",
|
||||
"searchlightsubsystem.data": "SearchLight", "jumpjetsubsystem.data": "JumpJet",
|
||||
"ecmsubsystem.data": "ECM", "beaglesubsystem.data": "Beagle",
|
||||
"lams.data": "LAMS", "narcbeacon.data": "Narc",
|
||||
}
|
||||
WEAPON_CATEGORY = [
|
||||
("laserweaponsubsystem", "Beam"), ("pulselaserweaponsubsystem", "Beam"),
|
||||
("ppcweaponsubsystem", "Beam"), ("flamerweaponsubsystem", "Beam"),
|
||||
("lrmweaponsubsystem", "Missile"), ("srmweaponsubsystem", "Missile"),
|
||||
("ssrmweaponsubsystem", "Missile"), ("smrmweaponsubsystem", "Missile"),
|
||||
("missileweaponsubsystem", "Missile"), ("narcbeaconweaponsubsystem", "Narc"),
|
||||
("machinegunweaponsubsystem", "Ballistic"), ("ultraacweaponsubsystem", "Ballistic"),
|
||||
("acweaponsubsystem", "Ballistic"), ("gaussweaponsubsystem", "Ballistic"),
|
||||
("lbxweaponsubsystem", "Ballistic"), ("rtxweaponsubsystem", "Ballistic"),
|
||||
]
|
||||
|
||||
|
||||
def armor_points_per_ton(path=ARMOR_DATA):
|
||||
txt = open(path, "rb").read().decode("latin-1")
|
||||
|
||||
def get(key, default):
|
||||
m = re.search(rf"^{key}=(\d+)", txt, re.M | re.I)
|
||||
return int(m.group(1)) if m else default
|
||||
return {0: get("PointsPerStandardTon", 32), 1: get("PointsPerFerroTon", 38),
|
||||
2: get("PointsPerReactiveTon", 30), 3: get("PointsPerReflectiveTon", 30),
|
||||
4: get("PointsPerSolarianTon", 60)}
|
||||
|
||||
|
||||
def load_manifest(path, package="core.mw4"):
|
||||
"""record id -> entry name, for resolving dataListID back to a Model= path."""
|
||||
out = {}
|
||||
with open(path, encoding="latin-1") as fh:
|
||||
for line in fh:
|
||||
parts = line.rstrip("\n").split("\t")
|
||||
if len(parts) > 2 and parts[0].lower() == package.lower():
|
||||
out[int(parts[1])] = parts[2]
|
||||
return out
|
||||
|
||||
|
||||
def cstr(msg, off, size=128):
|
||||
return msg[off:off + size].split(b"\0")[0].decode("latin-1").strip()
|
||||
|
||||
|
||||
def group_flags_to_list(flags):
|
||||
"""groupIndex is a BITMASK, not an index (Weapon_Tool.cpp ~76).
|
||||
|
||||
A page may carry several `GroupIndex=` lines; the factory ORs
|
||||
`1 << (n-1)` for each. Returns the group numbers, so the caller can emit one
|
||||
line per group.
|
||||
"""
|
||||
return [n for n in range(1, 7) if flags & (1 << (n - 1))]
|
||||
|
||||
|
||||
def page_name(model, counters):
|
||||
base = os.path.basename(model.replace("\\", "/")).lower()
|
||||
stem = PAGE_STEM.get(base)
|
||||
if stem is None:
|
||||
low = model.lower().replace("\\", "/")
|
||||
stem = next((s for key, s in WEAPON_CATEGORY if key in low), None)
|
||||
if stem is None:
|
||||
stem = "Torso" if base.endswith(".torso") else \
|
||||
"Engine" if base.endswith(".engine") else "Subsystem"
|
||||
counters[stem] += 1
|
||||
# Singletons keep a bare name, matching how the sources are written.
|
||||
if stem in ("Armor", "AdvancedGyro", "Sensor", "SearchLight", "Torso",
|
||||
"Engine", "ECM", "Beagle", "LAMS"):
|
||||
return stem if counters[stem] == 1 else f"{stem}{counters[stem]}"
|
||||
return f"{stem}{counters[stem]}"
|
||||
|
||||
|
||||
def decompile(record_path, manifest, ppt=None):
|
||||
ppt = ppt or armor_points_per_ton()
|
||||
data = open(record_path, "rb").read()
|
||||
# Model= is written relative to the mech's own directory in the source, but the
|
||||
# package stores the full entry path. Use the parent folder, not the file name:
|
||||
# Black Hawk's chassis files are nova.* inside mechs/blackhawk/.
|
||||
own_dir = "mechs\\" + os.path.basename(os.path.dirname(record_path)).lower() + "\\"
|
||||
counters = collections.Counter()
|
||||
pages = []
|
||||
for _off, msg in mw4msg.walk(data):
|
||||
cid, n = mw4msg.class_id(msg), len(msg)
|
||||
model = manifest.get(mw4msg.record_id(msg), "?")
|
||||
if model.lower().startswith(own_dir):
|
||||
model = model[len(own_dir):]
|
||||
kv = collections.OrderedDict()
|
||||
kv["Model"] = model
|
||||
kv["ExecutionState"] = EXEC_STATE.get(struct.unpack_from("<i", msg, 76)[0], "?")
|
||||
kv["InternalLocation"] = ZONE.get(msg[100], f"?{msg[100]}")
|
||||
crit = struct.unpack_from("<i", msg, 104)[0]
|
||||
|
||||
if n == 152: # Armor
|
||||
vals = struct.unpack_from("<9f", msg, 108)
|
||||
atype, itype = struct.unpack_from("<2i", msg, 144)
|
||||
kv["ArmorType"] = ARMOR_TYPE.get(atype, str(atype))
|
||||
kv["InternalType"] = INTERNAL_TYPE.get(itype, str(itype))
|
||||
mult = ppt.get(atype, 32)
|
||||
for key, v in zip(ARMOR_KEYS, vals):
|
||||
kv[key] = f"{v / mult:g}"
|
||||
elif n == 112 and cid == CLASS_ENGINE:
|
||||
kv["EngineUpgrades"] = str(struct.unpack_from("<i", msg, 108)[0])
|
||||
elif n == 112 and cid == CLASS_LAMS:
|
||||
ammo = struct.unpack_from("<i", msg, 108)[0]
|
||||
if ammo >= 0:
|
||||
kv["AmmoCount"] = str(ammo)
|
||||
elif n == 236: # SearchLight
|
||||
kv["Site"] = cstr(msg, 108)
|
||||
elif n in (376, 380): # Weapon
|
||||
kv["Site"] = cstr(msg, 108)
|
||||
eject = cstr(msg, 236)
|
||||
grp, ammo, _init = struct.unpack_from("<3i", msg, 364)
|
||||
kv["GroupIndex"] = group_flags_to_list(grp)
|
||||
if ammo >= 0:
|
||||
kv["AmmoCount"] = str(ammo)
|
||||
if eject:
|
||||
kv["EjectSite"] = eject
|
||||
if n == 380:
|
||||
facing = struct.unpack_from("<i", msg, 376)[0]
|
||||
if facing:
|
||||
kv["WeaponFacing"] = str(facing)
|
||||
if crit:
|
||||
kv["CriticalHitsTaken"] = str(crit)
|
||||
pages.append((page_name(model, counters), kv))
|
||||
return pages
|
||||
|
||||
|
||||
def emit(pages):
|
||||
out = []
|
||||
for name, kv in pages:
|
||||
out.append(f"[{name}]")
|
||||
for k, v in kv.items():
|
||||
if isinstance(v, list): # GroupIndex: one line per group
|
||||
out.extend(f"{k}={n}" for n in v)
|
||||
else:
|
||||
out.append(f"{k}={v}")
|
||||
out.append("")
|
||||
return "\r\n".join(out) + "\r\n"
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("record", nargs="?")
|
||||
ap.add_argument("manifest", nargs="?", default=OUR_MANIFEST)
|
||||
ap.add_argument("-o", "--out")
|
||||
ap.add_argument("--verify", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.verify:
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
os.execvp("python3", ["python3", os.path.join(here, "verify_subsystems.py")])
|
||||
if not args.record:
|
||||
ap.error("give a packed .subsystems record, or --verify")
|
||||
|
||||
pages = decompile(args.record, load_manifest(args.manifest))
|
||||
text = emit(pages)
|
||||
if args.out:
|
||||
os.makedirs(os.path.dirname(args.out), exist_ok=True)
|
||||
open(args.out, "wb").write(text.encode("latin-1"))
|
||||
print(f"{os.path.basename(args.record)}: {len(pages)} pages -> {args.out}")
|
||||
else:
|
||||
sys.stdout.write(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verification harness for the .armature decompiler.
|
||||
|
||||
Rebuilds every chassis from our own packed records and compares against the
|
||||
known source .armature. Compares page transforms as a multiset keyed on
|
||||
(name, transform) so duplicate page names - Victor has two [site_lshellport]
|
||||
pages under different joints - are handled.
|
||||
|
||||
python3 verify_armature.py
|
||||
"""
|
||||
import sys, os, glob, re, collections
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import armature
|
||||
|
||||
|
||||
def source_entries(path):
|
||||
"""-> (list of (name, rot, trans), {parent: [child, ...]})"""
|
||||
txt = open(path, "rb").read().decode("latin-1")
|
||||
entries, children = [], {}
|
||||
for m in re.finditer(r'^\[([^\]]+)\]\s*\r?\n(.*?)(?=^\[|\Z)', txt, re.M | re.S):
|
||||
name, body = m.group(1).lower(), m.group(2)
|
||||
r = re.search(r'Rotation=([-\d.eE ]+)', body)
|
||||
t = re.search(r'Translation=([-\d.eE ]+)', body)
|
||||
kids = [c.lower() for c in re.findall(r'Child=(\S+)', body)]
|
||||
if kids:
|
||||
children[name] = kids
|
||||
if r and t:
|
||||
entries.append((name,
|
||||
tuple(float(x) for x in r.group(1).split()),
|
||||
tuple(float(x) for x in t.group(1).split())))
|
||||
return entries, children
|
||||
|
||||
|
||||
def close(a, b, tol):
|
||||
return a and b and max(abs(x - y) for x, y in zip(a, b)) <= tol
|
||||
|
||||
|
||||
def angles_close(a, b, tol=0.01):
|
||||
return a and b and max(abs(((x - y + 180) % 360) - 180) for x, y in zip(a, b)) <= tol
|
||||
|
||||
|
||||
def main():
|
||||
src_paths = {os.path.basename(p)[:-len(".armature")].lower(): p
|
||||
for p in glob.glob(armature.OUR_MECH_SOURCE + "/*/*.armature")}
|
||||
t = collections.Counter()
|
||||
residual = collections.Counter()
|
||||
for mech_dir in sorted(glob.glob(armature.OUR_RECORDS + "/*")):
|
||||
if not glob.glob(mech_dir + "/*{armature}"):
|
||||
continue
|
||||
chassis, entries, children = armature.rebuild(mech_dir)
|
||||
if not chassis or chassis.lower() not in src_paths:
|
||||
continue
|
||||
t["chassis"] += 1
|
||||
src_list, src_children = source_entries(src_paths[chassis.lower()])
|
||||
|
||||
got = [(e["name"].lower(), e["rot"], e["trans"]) for e in entries]
|
||||
pool = list(got)
|
||||
for name, rot, tr in src_list:
|
||||
t["pages"] += 1
|
||||
hit = next((g for g in pool if g[0] == name
|
||||
and close(g[2], tr, 2e-3) and angles_close(g[1], rot)), None)
|
||||
if hit:
|
||||
pool.remove(hit)
|
||||
t["exact"] += 1
|
||||
continue
|
||||
near = next((g for g in pool if g[0] == name and close(g[2], tr, 2e-3)), None)
|
||||
if near:
|
||||
pool.remove(near)
|
||||
t["rot_only"] += 1
|
||||
residual[name] += 1
|
||||
else:
|
||||
t["missing"] += 1
|
||||
residual["MISSING " + name] += 1
|
||||
|
||||
for parent, kids in src_children.items():
|
||||
t["childlists"] += 1
|
||||
if sorted(kids) == sorted(c.lower() for c in children.get(parent, ())):
|
||||
t["childlists_ok"] += 1
|
||||
else:
|
||||
residual["CHILDREN " + parent] += 1
|
||||
|
||||
print(f"chassis : {t['chassis']}")
|
||||
print(f"pages compared : {t['pages']}")
|
||||
print(f" fully exact : {t['exact']}")
|
||||
print(f" translation ok, rotation lost by the packer : {t['rot_only']}")
|
||||
print(f" not recovered : {t['missing']}")
|
||||
print(f"child lists : {t['childlists_ok']}/{t['childlists']} exact")
|
||||
if residual:
|
||||
print("\nresidual, by page:")
|
||||
for k, v in residual.most_common(10):
|
||||
print(f" {k:22s} x{v}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Round-trip verifier for the .contents decompiler.
|
||||
|
||||
Regenerates each `.contents` and compares page names and both key values against
|
||||
the authored source. Page order is not compared -- NotationFile is
|
||||
order-independent, and the packer groups pages by parent joint rather than
|
||||
preserving the authored sequence.
|
||||
|
||||
python3 verify_contents.py [--show CHASSIS]
|
||||
"""
|
||||
import argparse, collections, glob, os, re, sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import contents, subsystems
|
||||
|
||||
REC = "/home/rich/Repositories/FS_Ours_extracted/core/mechs"
|
||||
SRC = "/home/rich/Repositories/firestorm/Gameleap/mw4/Content/Mechs"
|
||||
|
||||
|
||||
def source_pages(path):
|
||||
txt = open(path, "rb").read().decode("latin-1")
|
||||
txt = re.sub(r'//[^\n]*', '', txt)
|
||||
out = {}
|
||||
for m in re.finditer(r'^\[([^\]]+)\]\r?\n(.*?)(?=^\[|\Z)', txt, re.M | re.S):
|
||||
name, body = m.group(1), m.group(2)
|
||||
if name.lower() == "includes":
|
||||
continue
|
||||
kv = dict(re.findall(r'^([A-Za-z_]\w*)=([^\r\n]*)', body, re.M))
|
||||
out[name.lower()] = {k.lower(): v.strip().lower().replace("/", "\\")
|
||||
for k, v in kv.items()}
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--show")
|
||||
args = ap.parse_args()
|
||||
|
||||
manifest = subsystems.load_manifest(contents.MANIFEST)
|
||||
t = collections.Counter()
|
||||
bad = collections.Counter()
|
||||
examples = []
|
||||
missing_pages = collections.Counter()
|
||||
extra_pages = collections.Counter()
|
||||
|
||||
for d in sorted(glob.glob(REC + "/*")):
|
||||
ch = os.path.basename(d)
|
||||
src = [p for p in glob.glob(SRC + "/*/*.contents")
|
||||
if os.path.basename(p).lower() == ch.lower() + ".contents"]
|
||||
if not src:
|
||||
continue
|
||||
t["chassis"] += 1
|
||||
want = source_pages(src[0])
|
||||
chassis, pages = contents.decompile(d, manifest)
|
||||
got = {n.lower(): {k.lower(): (v or "").lower().replace("/", "\\")
|
||||
for k, v in kv} for n, kv in pages}
|
||||
|
||||
if args.show and args.show.lower() == ch.lower():
|
||||
sys.stdout.write(contents.emit(chassis, pages))
|
||||
return
|
||||
|
||||
for name in want:
|
||||
if name not in got:
|
||||
missing_pages[name] += 1
|
||||
for name in got:
|
||||
if name not in want:
|
||||
extra_pages[name] += 1
|
||||
for name in set(want) & set(got):
|
||||
t["pages"] += 1
|
||||
for key in ("model", "executionstate"):
|
||||
t["keys"] += 1
|
||||
if want[name].get(key) == got[name].get(key):
|
||||
t["ok"] += 1
|
||||
else:
|
||||
bad[key] += 1
|
||||
if len(examples) < 10:
|
||||
examples.append((ch, name, key,
|
||||
want[name].get(key), got[name].get(key)))
|
||||
|
||||
print(f"chassis : {t['chassis']}")
|
||||
print(f"pages compared : {t['pages']}")
|
||||
print(f"keys compared : {t['keys']} exact: {t['ok']} wrong: {t['keys'] - t['ok']}")
|
||||
if missing_pages:
|
||||
print(f"\npages in source but not decoded: {sum(missing_pages.values())}")
|
||||
for n, c in missing_pages.most_common(10):
|
||||
print(f" {n:28s} x{c}")
|
||||
if extra_pages:
|
||||
print(f"\npages decoded but not in source: {sum(extra_pages.values())}")
|
||||
for n, c in extra_pages.most_common(10):
|
||||
print(f" {n:28s} x{c}")
|
||||
if bad:
|
||||
print("\nvalue mismatches:")
|
||||
for k, c in bad.most_common():
|
||||
print(f" {k:20s} x{c}")
|
||||
for e in examples:
|
||||
print(f" {e[0]:14s} {e[1]:24s} {e[2]:16s} want={e[3]!r} got={e[4]!r}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Round-trip verifier for the .damage decompiler.
|
||||
|
||||
Regenerates a whole `.damage` from the compiled record for every chassis and
|
||||
compares it page by page against the authored source: page names, page order,
|
||||
key sets and values.
|
||||
|
||||
Comparison is semantic. Sources carry an `!include` line and comments, spell
|
||||
numbers freely (`.99` vs `0.99`, `1.0` vs `1`), and Windows path lookup is
|
||||
case-insensitive, so none of those count as differences.
|
||||
|
||||
python3 verify_damage.py [--show CHASSIS]
|
||||
"""
|
||||
import argparse, collections, glob, os, re, sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import damage, subsystems
|
||||
|
||||
REC = "/home/rich/Repositories/FS_Ours_extracted/core/mechs"
|
||||
SRC = "/home/rich/Repositories/firestorm/Gameleap/mw4/Content/Mechs"
|
||||
NUM = re.compile(r'^-?(?:\d+\.?\d*|\.\d+)$')
|
||||
|
||||
|
||||
def canon(v):
|
||||
v = str(v).strip()
|
||||
if NUM.match(v):
|
||||
return f"{float(v):.4g}"
|
||||
if "," in v: # DamageEffect: path,percent
|
||||
path, _, pct = v.rpartition(",")
|
||||
return canon(path) + "," + (f"{float(pct):.4g}" if NUM.match(pct.strip()) else pct)
|
||||
v = re.sub(r'^content[\\/]', '', v.replace("/", "\\"), flags=re.I)
|
||||
return v.lower()
|
||||
|
||||
|
||||
def source_pages(path):
|
||||
"""-> [(pageName, [(key, value)])] preserving order and repeats."""
|
||||
txt = open(path, "rb").read().decode("latin-1")
|
||||
txt = re.sub(r'//[^\n]*', '', txt)
|
||||
pages, cur = [], None
|
||||
for line in txt.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("!"):
|
||||
continue
|
||||
m = re.match(r'^\[([^\]]+)\]$', line)
|
||||
if m:
|
||||
cur = (m.group(1), [])
|
||||
pages.append(cur)
|
||||
elif "=" in line and cur is not None:
|
||||
k, v = line.split("=", 1)
|
||||
cur[1].append((k.strip(), v.strip()))
|
||||
return pages
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--show")
|
||||
args = ap.parse_args()
|
||||
|
||||
manifest = subsystems.load_manifest(damage.MANIFEST)
|
||||
t = collections.Counter()
|
||||
bad = collections.Counter()
|
||||
examples = []
|
||||
order_problems = []
|
||||
|
||||
for d in sorted(glob.glob(REC + "/*")):
|
||||
ch = os.path.basename(d)
|
||||
src = [p for p in glob.glob(SRC + "/*/*.damage")
|
||||
if os.path.basename(p).lower() == ch.lower() + ".damage"]
|
||||
if not src or not glob.glob(os.path.join(d, "*.damage")):
|
||||
continue
|
||||
t["chassis"] += 1
|
||||
want = source_pages(src[0])
|
||||
got = damage.decompile(d, manifest)
|
||||
|
||||
if args.show and args.show.lower() == ch.lower():
|
||||
sys.stdout.write(damage.emit(got))
|
||||
return
|
||||
|
||||
if [n.lower() for n, _ in want] != [n.lower() for n, _ in got]:
|
||||
order_problems.append((ch, [n for n, _ in want], [n for n, _ in got]))
|
||||
continue
|
||||
t["pages"] += len(want)
|
||||
for (wname, wkv), (_gname, gkv) in zip(want, got):
|
||||
wmap = collections.defaultdict(list)
|
||||
for k, v in wkv:
|
||||
wmap[k.lower()].append(canon(v))
|
||||
gmap = collections.defaultdict(list)
|
||||
for k, v in gkv:
|
||||
gmap[k.lower()].append(canon(v))
|
||||
for key in set(wmap) | set(gmap):
|
||||
t["keys"] += 1
|
||||
w, g = sorted(wmap.get(key, [])), sorted(gmap.get(key, []))
|
||||
# The writer defaults MaxArmorValue to BaseArmorValue, so a source
|
||||
# that omits it and one that states it equal produce identical
|
||||
# bytes -- 51 pages do state it. The distinction is unrecoverable
|
||||
# and harmless, so emitting it explicitly counts as a match.
|
||||
if not w and key == "maxarmorvalue" and g == sorted(gmap.get("basearmorvalue", [])):
|
||||
t["ok"] += 1
|
||||
t["implicit_max"] += 1
|
||||
elif w == g:
|
||||
t["ok"] += 1
|
||||
else:
|
||||
bad[key] += 1
|
||||
if len(examples) < 12:
|
||||
examples.append((ch, wname, key, wmap.get(key), gmap.get(key)))
|
||||
|
||||
print(f"chassis : {t['chassis']}")
|
||||
print(f"pages compared : {t['pages']}")
|
||||
print(f"keys compared : {t['keys']} exact: {t['ok']} wrong: {t['keys'] - t['ok']}")
|
||||
print(f" MaxArmorValue omitted by source, implied by BaseArmorValue: {t['implicit_max']}")
|
||||
if order_problems:
|
||||
print(f"\npage name/order mismatches: {len(order_problems)}")
|
||||
for ch, w, g in order_problems[:3]:
|
||||
print(f" {ch}\n want {w}\n got {g}")
|
||||
if bad:
|
||||
print("\nkeys not matching:")
|
||||
for k, n in bad.most_common(15):
|
||||
print(f" {k:28s} x{n}")
|
||||
print("\nexamples (chassis, page, key, source, decoded):")
|
||||
for e in examples:
|
||||
print(f" {e[0]:14s} {e[1]:22s} {e[2]:22s} {e[3]} != {e[4]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verification harness for the mech .data{GameModel} field map.
|
||||
|
||||
Reads every mapped value through its header-derived offset and compares against
|
||||
the source .data, across all 64 chassis we hold both forms for.
|
||||
|
||||
Unlike a value-discovered map this one can genuinely fail: the offsets come from
|
||||
declaration order, so a mis-parsed header shows up at once as a whole column of
|
||||
wrong values. Keys that miss consistently by a constant factor are reported with
|
||||
their decoded/source ratio, which is how unit conversions (deg -> rad, kph ->
|
||||
m/s) announce themselves.
|
||||
|
||||
python3 verify_data.py
|
||||
"""
|
||||
import sys, os, re, collections
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import datamap
|
||||
|
||||
|
||||
def close(a, b):
|
||||
return abs(a - b) <= max(1e-4, abs(b) * 1e-5)
|
||||
|
||||
|
||||
def matches(got, want_text, typ):
|
||||
"""Compare a decoded value against the raw source text."""
|
||||
if typ in datamap.VECTORS:
|
||||
nums = [float(x) for x in re.findall(r'-?\d+\.?\d*(?:[eE][-+]?\d+)?', want_text)]
|
||||
return len(nums) == len(got) and all(close(a, b) for a, b in zip(got, nums))
|
||||
if typ == "char":
|
||||
return got.lower() == want_text.strip().lower()
|
||||
if typ in ("bool", "BYTE"):
|
||||
w = want_text.strip().lower()
|
||||
if w in ("true", "yes", "1"):
|
||||
return got == 1
|
||||
if w in ("false", "no", "0"):
|
||||
return got == 0
|
||||
return None
|
||||
try:
|
||||
return close(float(got), float(want_text))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
pairs, field_map = datamap.build()
|
||||
t = collections.Counter()
|
||||
bad = collections.Counter()
|
||||
examples = []
|
||||
|
||||
for ch, kv, blob in pairs:
|
||||
for key, (off, typ, size, angle) in field_map.items():
|
||||
if key not in kv:
|
||||
continue
|
||||
got = datamap.read(blob, off, typ, size, angle)
|
||||
ok = matches(got, kv[key], typ)
|
||||
if ok is None:
|
||||
t["skipped"] += 1
|
||||
continue
|
||||
t["values"] += 1
|
||||
if ok:
|
||||
t["ok"] += 1
|
||||
else:
|
||||
bad[key] += 1
|
||||
if len(examples) < 12:
|
||||
examples.append((ch, key, typ, kv[key], got, off))
|
||||
|
||||
print(f"chassis : {len(pairs)}")
|
||||
print(f"mapped keys : {len(field_map)}")
|
||||
print(f"values compared : {t['values']} exact: {t['ok']} wrong: {t['values'] - t['ok']}")
|
||||
print(f"not comparable : {t['skipped']} (enum/resource text, decoded separately)")
|
||||
if bad:
|
||||
print("\nkeys not matching:")
|
||||
for k, v in bad.most_common(25):
|
||||
print(f" {k:34s} x{v}")
|
||||
print("\nexamples (chassis, key, type, source, decoded, offset):")
|
||||
for ch, key, typ, src, got, off in examples:
|
||||
print(f" {ch:14s} {key:28s} {typ:10s} src={src!r:24s} got={got!r} @{off}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Round-trip verifier for the .instance decompiler.
|
||||
|
||||
Regenerates each `.instance` and compares the page name and every key against
|
||||
the authored source. Keys the source carries but the decompiler does not emit
|
||||
are reported separately, so an unhandled key can never be mistaken for a pass.
|
||||
|
||||
python3 verify_instance.py [--show CHASSIS]
|
||||
"""
|
||||
import argparse, collections, glob, os, re, sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import instance, subsystems
|
||||
|
||||
REC = "/home/rich/Repositories/FS_Ours_extracted/core/mechs"
|
||||
SRC = "/home/rich/Repositories/firestorm/Gameleap/mw4/Content/Mechs"
|
||||
NUM = re.compile(r'^-?(?:\d+\.?\d*|\.\d+)$')
|
||||
|
||||
|
||||
def canon(v):
|
||||
v = str(v).strip()
|
||||
if NUM.match(v):
|
||||
return f"{float(v):.5g}"
|
||||
return re.sub(r'^content[\\/]', '', v.replace("/", "\\"), flags=re.I).lower()
|
||||
|
||||
|
||||
def source_page(path):
|
||||
txt = re.sub(r'//[^\n]*', '', open(path, "rb").read().decode("latin-1"))
|
||||
m = re.search(r'^\[([^\]]+)\]', txt, re.M)
|
||||
kv = [(k, v.strip()) for k, v in re.findall(r'^([A-Za-z]\w*)=([^\r\n]*)', txt, re.M)]
|
||||
return (m.group(1) if m else None), kv
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--show")
|
||||
args = ap.parse_args()
|
||||
|
||||
manifest = subsystems.load_manifest(instance.MANIFEST)
|
||||
t = collections.Counter()
|
||||
bad = collections.Counter()
|
||||
unhandled = collections.Counter()
|
||||
examples = []
|
||||
name_bad = []
|
||||
|
||||
for d in sorted(glob.glob(REC + "/*")):
|
||||
ch = os.path.basename(d)
|
||||
src = [p for p in glob.glob(SRC + "/*/*.instance")
|
||||
if os.path.basename(p).lower() == ch.lower() + ".instance"]
|
||||
if not src or not glob.glob(os.path.join(d, "*.instance")):
|
||||
continue
|
||||
t["chassis"] += 1
|
||||
want_name, want = source_page(src[0])
|
||||
got_name, got = instance.decompile(d, manifest)
|
||||
|
||||
if args.show and args.show.lower() == ch.lower():
|
||||
sys.stdout.write(instance.emit(got_name, got))
|
||||
return
|
||||
|
||||
if (want_name or "").lower() != (got_name or "").lower():
|
||||
name_bad.append((ch, want_name, got_name))
|
||||
gmap = {k.lower(): v for k, v in got}
|
||||
for key, wv in want:
|
||||
if key.lower() not in gmap:
|
||||
unhandled[key] += 1
|
||||
continue
|
||||
t["keys"] += 1
|
||||
if canon(wv) == canon(gmap[key.lower()]):
|
||||
t["ok"] += 1
|
||||
else:
|
||||
bad[key] += 1
|
||||
if len(examples) < 12:
|
||||
examples.append((ch, key, wv, gmap[key.lower()]))
|
||||
|
||||
print(f"chassis : {t['chassis']}")
|
||||
print(f"page names : {t['chassis'] - len(name_bad)}/{t['chassis']} match")
|
||||
print(f"keys compared : {t['keys']} exact: {t['ok']} wrong: {t['keys'] - t['ok']}")
|
||||
if unhandled:
|
||||
print("\nkeys present in source but NOT emitted:")
|
||||
for k, n in unhandled.most_common():
|
||||
print(f" {k:26s} x{n}")
|
||||
if name_bad:
|
||||
print("\npage name mismatches:")
|
||||
for e in name_bad[:5]:
|
||||
print(f" {e[0]:14s} want={e[1]!r} got={e[2]!r}")
|
||||
if bad:
|
||||
print("\nvalue mismatches:")
|
||||
for k, n in bad.most_common(15):
|
||||
print(f" {k:26s} x{n}")
|
||||
print("\nexamples (chassis, key, source, decoded):")
|
||||
for e in examples:
|
||||
print(f" {e[0]:14s} {e[1]:22s} want={e[2]!r:32s} got={e[3]!r}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Round-trip verifier for the mech .data decompiler.
|
||||
|
||||
Regenerates a whole `.data` from the compiled records for every chassis we hold
|
||||
both forms of, and compares it key by key against the authored source. This is
|
||||
the standard `.armature` and `.subsystems` were held to; nothing should be
|
||||
emitted for the new chassis until this passes.
|
||||
|
||||
Comparison is semantic, not byte-exact: NotationFile is order-independent, and
|
||||
Windows path lookup is case-insensitive (our own tree writes both
|
||||
`mechs\\atlas_destroyed\\...` and `Mechs\\Atlas_Destroyed\\...` for the same
|
||||
key), so keys are matched case-insensitively and numbers within tolerance.
|
||||
|
||||
python3 verify_roundtrip.py [--show CHASSIS]
|
||||
"""
|
||||
import argparse, collections, os, re, sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import datamap, data, subsystems
|
||||
|
||||
REC = "/home/rich/Repositories/FS_Ours_extracted/core/mechs"
|
||||
NUM = re.compile(r'^-?\d+\.?\d*(?:[eE][-+]?\d+)?$')
|
||||
|
||||
|
||||
def canon(v):
|
||||
"""Normalise one value for comparison."""
|
||||
v = str(v).strip()
|
||||
v = re.sub(r'^content[\\/]', '', v.replace("/", "\\"), flags=re.I)
|
||||
v = re.sub(r'\s+', " ", v).lower()
|
||||
# normalise every number in place so "20.0 20.0 20.0" == "20 20 20"
|
||||
return re.sub(r'-?\d+\.\d+(?:[eE][-+]?\d+)?|-?\d+',
|
||||
lambda m: f"{float(m.group(0)):.4g}", v)
|
||||
|
||||
|
||||
def canon_set(val):
|
||||
"""Sorted, de-duplicated: some sources repeat a key with the same value."""
|
||||
return sorted({canon(v) for v in (val if isinstance(val, list) else [val])})
|
||||
|
||||
|
||||
def source_kv(path):
|
||||
"""Authored [GameData] page, keeping repeated keys as lists."""
|
||||
txt = open(path, "rb").read().decode("latin-1")
|
||||
# protect braced blocks first: Shadow={...} contains a line reading
|
||||
# "[shadow]", which otherwise looks like the start of the next page.
|
||||
# Both CR and LF must go -- splitlines() splits on a bare CR too.
|
||||
txt = re.sub(r'\{.*?\}',
|
||||
lambda x: x.group(0).replace("\r", "").replace("\n", "\x01"),
|
||||
txt, flags=re.S)
|
||||
m = re.search(r'^\[GameData\]\r?\n(.*?)(?=^\[[A-Za-z]|\Z)', txt, re.M | re.S)
|
||||
body = m.group(1) if m else ""
|
||||
kv = collections.OrderedDict()
|
||||
for line in body.splitlines():
|
||||
if "=" not in line or line.lstrip().startswith("//"):
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
kv.setdefault(k.strip(), []).append(v.replace("\x01", "\n").strip())
|
||||
return kv
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--show")
|
||||
args = ap.parse_args()
|
||||
|
||||
manifest = subsystems.load_manifest(data.MANIFEST)
|
||||
totals = collections.Counter()
|
||||
wrong = collections.Counter()
|
||||
missing = collections.Counter()
|
||||
extra = collections.Counter()
|
||||
examples = []
|
||||
chassis = 0
|
||||
|
||||
for ch, kv_ignored, _blob in datamap.corpus():
|
||||
src_path = [p for p in
|
||||
__import__("glob").glob(datamap.SRC + "/*/*.data")
|
||||
if os.path.basename(p).lower() == ch.lower() + ".data"]
|
||||
if not src_path:
|
||||
continue
|
||||
chassis += 1
|
||||
want = source_kv(src_path[0])
|
||||
got = data.decompile(os.path.join(REC, ch), manifest)
|
||||
got_l = {k.lower(): v for k, v in got.items()}
|
||||
|
||||
for key, wvals in want.items():
|
||||
if key in data.UNRECOVERABLE:
|
||||
totals["omitted"] += 1
|
||||
continue
|
||||
totals["keys"] += 1
|
||||
if key.lower() not in got_l:
|
||||
missing[key] += 1
|
||||
continue
|
||||
if canon_set(got_l[key.lower()]) == canon_set(wvals):
|
||||
totals["ok"] += 1
|
||||
else:
|
||||
wrong[key] += 1
|
||||
if len(examples) < 12:
|
||||
examples.append((ch, key, wvals, got_l[key.lower()]))
|
||||
for key in got:
|
||||
if key.lower() not in {k.lower() for k in want}:
|
||||
extra[key] += 1
|
||||
|
||||
if args.show and args.show.lower() == ch.lower():
|
||||
sys.stdout.write(data.emit(got))
|
||||
return
|
||||
|
||||
print(f"chassis : {chassis}")
|
||||
print(f"keys compared : {totals['keys']} exact: {totals['ok']} "
|
||||
f"wrong: {sum(wrong.values())} missing: {sum(missing.values())}")
|
||||
print(f"deliberately omitted: {totals['omitted']} (unreadable by the engine)")
|
||||
if missing:
|
||||
print("\nmissing from output:")
|
||||
for k, n in missing.most_common(15):
|
||||
print(f" {k:30s} x{n}")
|
||||
if wrong:
|
||||
print("\nvalue mismatches:")
|
||||
for k, n in wrong.most_common(15):
|
||||
print(f" {k:30s} x{n}")
|
||||
print("\nexamples:")
|
||||
for ch, k, w, g in examples:
|
||||
print(f" {ch:12s} {k:26s} want={w!r:44s} got={g!r}")
|
||||
if extra:
|
||||
print("\nemitted but not in source:")
|
||||
for k, n in extra.most_common(10):
|
||||
print(f" {k:30s} x{n}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Round-trip verifier for the .torso and .engine decompilers.
|
||||
|
||||
Regenerates both files for every chassis and compares key order and values
|
||||
against the authored source. `$(SYMBOL)` and its numeric expansion are treated
|
||||
as equal, since the record stores only the resolved float.
|
||||
|
||||
python3 verify_smallmodel.py [--show KIND CHASSIS]
|
||||
"""
|
||||
import argparse, collections, glob, os, re, sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import smallmodel
|
||||
|
||||
REC = "/home/rich/Repositories/FS_Ours_extracted/core/mechs"
|
||||
SRC = "/home/rich/Repositories/firestorm/Gameleap/mw4/Content/Mechs"
|
||||
NUM = re.compile(r'^-?(?:\d+\.?\d*|\.\d+)$')
|
||||
|
||||
|
||||
def resolve(value, symbols):
|
||||
"""$(NAME) -> its numeric value, so symbol and literal compare equal."""
|
||||
m = re.fullmatch(r'\$\((\w+)\)', value.strip())
|
||||
if m:
|
||||
return symbols.get(m.group(1).lower(), value.strip().lower())
|
||||
return value.strip().lower()
|
||||
|
||||
|
||||
def canon(value, symbols):
|
||||
v = resolve(value, symbols)
|
||||
return f"{float(v):.5g}" if NUM.match(str(v)) else str(v)
|
||||
|
||||
|
||||
def symbol_values(path):
|
||||
"""-> {symbolNameLower: numericString}"""
|
||||
out = {}
|
||||
if os.path.exists(path):
|
||||
txt = re.sub(r'//[^\n]*', '', open(path, encoding="latin-1", errors="replace").read())
|
||||
for name, val in re.findall(r'^\s*!(\w+)\s*=\s*([-\d.]+)\s*$', txt, re.M):
|
||||
out[name.lower()] = val
|
||||
return out
|
||||
|
||||
|
||||
def source_kv(path):
|
||||
txt = re.sub(r'//[^\n]*', '', open(path, "rb").read().decode("latin-1"))
|
||||
return [(m.group(1), m.group(2).strip())
|
||||
for m in re.finditer(r'^([A-Za-z]\w*)=([^\r\n]*)', txt, re.M)]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--show", nargs=2, metavar=("KIND", "CHASSIS"))
|
||||
args = ap.parse_args()
|
||||
|
||||
t = collections.Counter()
|
||||
bad = collections.Counter()
|
||||
examples = []
|
||||
order_bad = []
|
||||
|
||||
for kind, spec in sorted(smallmodel.SPECS.items()):
|
||||
symbols = symbol_values(spec["defines"])
|
||||
for d in sorted(glob.glob(REC + "/*")):
|
||||
ch = os.path.basename(d)
|
||||
src = [p for p in glob.glob(f"{SRC}/*/*.{kind}")
|
||||
if os.path.basename(p).lower() == f"{ch.lower()}.{kind}"]
|
||||
if not src or smallmodel.record(d, kind) is None:
|
||||
continue
|
||||
t[f"{kind} files"] += 1
|
||||
want = source_kv(src[0])
|
||||
got = smallmodel.decompile(d, kind)
|
||||
|
||||
if args.show and args.show[0] == kind and args.show[1].lower() == ch.lower():
|
||||
sys.stdout.write(smallmodel.emit(kind, got))
|
||||
return
|
||||
|
||||
if [k.lower() for k, _ in want] != [k.lower() for k, _ in got]:
|
||||
order_bad.append((kind, ch, [k for k, _ in want], [k for k, _ in got]))
|
||||
continue
|
||||
for (wk, wv), (_gk, gv) in zip(want, got):
|
||||
t["keys"] += 1
|
||||
if canon(wv, symbols) == canon(gv, symbols):
|
||||
t["ok"] += 1
|
||||
else:
|
||||
bad[f"{kind}.{wk}"] += 1
|
||||
if len(examples) < 12:
|
||||
examples.append((kind, ch, wk, wv, gv))
|
||||
|
||||
for kind in sorted(smallmodel.SPECS):
|
||||
print(f"{kind:8s} files : {t[kind + ' files']}")
|
||||
print(f"keys compared : {t['keys']} exact: {t['ok']} wrong: {t['keys'] - t['ok']}")
|
||||
if order_bad:
|
||||
print(f"\nkey order mismatches: {len(order_bad)}")
|
||||
for kind, ch, w, g in order_bad[:3]:
|
||||
print(f" {kind} {ch}\n want {w}\n got {g}")
|
||||
if bad:
|
||||
print("\nvalue mismatches:")
|
||||
for k, n in bad.most_common(15):
|
||||
print(f" {k:34s} x{n}")
|
||||
print("\nexamples (kind, chassis, key, source, decoded):")
|
||||
for e in examples:
|
||||
print(f" {e[0]:7s} {e[1]:14s} {e[2]:22s} want={e[3]!r:22s} got={e[4]!r}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verification harness for the .subsystems decompiler.
|
||||
|
||||
Rebuilds every chassis from our own packed records and compares the resulting
|
||||
key/value pairs, page by page, against the known source .subsystems.
|
||||
|
||||
Page names are ignored - the packer does not store them (see subsystems.py), so
|
||||
only order and content can be verified.
|
||||
|
||||
Chassis whose source has moved on since the package was built are reported
|
||||
separately rather than counted as failures; core.mw4 has not been repacked since
|
||||
the initial mirror, so battlemaster and battlemaster2c legitimately differ.
|
||||
|
||||
python3 verify_subsystems.py
|
||||
"""
|
||||
import sys, os, glob, re, collections
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import subsystems
|
||||
|
||||
|
||||
def source_pages(path):
|
||||
"""Parse a source .subsystems leniently.
|
||||
|
||||
The runtime NotationFile accepts a page header with no closing bracket -
|
||||
hellspawn had '[HeatSink10' and sunder '[HeatSink16' - so match on a leading
|
||||
'[' rather than a full bracketed pattern.
|
||||
|
||||
GroupIndex may appear several times in one page, so values are collected as
|
||||
lists rather than overwritten.
|
||||
"""
|
||||
txt = open(path, "rb").read().decode("latin-1")
|
||||
out = []
|
||||
for m in re.finditer(r'^\[([^\r\n\]]*)\]?[ \t]*\r?\n(.*?)(?=^\[|\Z)', txt, re.M | re.S):
|
||||
kv = collections.OrderedDict()
|
||||
for k, v in re.findall(r'^([A-Za-z0-9_]+)=([^\r\n]*)', m.group(2), re.M):
|
||||
if k in kv:
|
||||
kv[k] = (kv[k] if isinstance(kv[k], list) else [kv[k]]) + [v]
|
||||
else:
|
||||
kv[k] = v
|
||||
out.append((m.group(1), kv))
|
||||
return out
|
||||
|
||||
|
||||
def norm(key, value):
|
||||
"""Compare as a sorted list so a single value and a one-element list match."""
|
||||
if not isinstance(value, list):
|
||||
value = [value]
|
||||
return sorted(_norm1(key, v) for v in value)
|
||||
|
||||
|
||||
def _norm1(key, value):
|
||||
v = str(value).strip()
|
||||
if key in ("Model", "Site", "EjectSite", "InternalLocation", "ExecutionState",
|
||||
"ArmorType", "InternalType"):
|
||||
return v.lower().replace("\\", "/")
|
||||
try:
|
||||
return f"{float(v):.4g}"
|
||||
except ValueError:
|
||||
return v.lower()
|
||||
|
||||
|
||||
def main():
|
||||
manifest = subsystems.load_manifest(subsystems.OUR_MANIFEST)
|
||||
ppt = subsystems.armor_points_per_ton()
|
||||
srcs = {os.path.basename(p)[:-len(".subsystems")].lower(): p
|
||||
for p in glob.glob(subsystems.OUR_MECH_SOURCE + "/*/*.subsystems")}
|
||||
|
||||
t = collections.Counter()
|
||||
mismatched_keys = collections.Counter()
|
||||
stale, examples = [], []
|
||||
|
||||
for rec in sorted(glob.glob(subsystems.OUR_RECORDS + "/*/*.subsystems")):
|
||||
chassis = os.path.basename(rec)[:-len(".subsystems")].lower()
|
||||
if chassis not in srcs:
|
||||
continue
|
||||
src = source_pages(srcs[chassis])
|
||||
got = subsystems.decompile(rec, manifest, ppt)
|
||||
if len(src) != len(got):
|
||||
stale.append((chassis, len(got), len(src)))
|
||||
continue
|
||||
t["chassis"] += 1
|
||||
for (_sname, skv), (_gname, gkv) in zip(src, got):
|
||||
t["pages"] += 1
|
||||
page_ok = True
|
||||
for key, sval in skv.items():
|
||||
if key in ("SubsystemIndex",): # not emitted; defaults to 0
|
||||
continue
|
||||
# An explicit "=0" and an omitted key are identical to the packer.
|
||||
if gkv.get(key) is None and str(sval).strip() in ("0", "0.0"):
|
||||
continue
|
||||
t["keys"] += 1
|
||||
gval = gkv.get(key)
|
||||
if gval is not None and norm(key, gval) == norm(key, sval):
|
||||
t["keys_ok"] += 1
|
||||
else:
|
||||
page_ok = False
|
||||
mismatched_keys[key] += 1
|
||||
if len(examples) < 12:
|
||||
examples.append((chassis, key, sval, gval))
|
||||
extra = [k for k in gkv if k not in skv]
|
||||
if extra:
|
||||
page_ok = False
|
||||
for k in extra:
|
||||
mismatched_keys["EXTRA:" + k] += 1
|
||||
t["pages_ok"] += page_ok
|
||||
|
||||
print(f"chassis verified : {t['chassis']}")
|
||||
print(f"pages compared : {t['pages']} fully exact: {t['pages_ok']}")
|
||||
print(f"keys compared : {t['keys']} exact: {t['keys_ok']}"
|
||||
f" wrong: {t['keys'] - t['keys_ok']}")
|
||||
if mismatched_keys:
|
||||
print("\nmismatches by key:")
|
||||
for k, v in mismatched_keys.most_common(12):
|
||||
print(f" {k:22s} x{v}")
|
||||
if examples:
|
||||
print("\nexamples (chassis, key, source, decompiled):")
|
||||
for e in examples:
|
||||
print(f" {e[0]:14s} {e[1]:18s} src={e[2]!r:28s} got={e[3]!r}")
|
||||
if stale:
|
||||
print("\nskipped - source has moved on since the package was built:")
|
||||
for ch, g, s in stale:
|
||||
print(f" {ch:16s} messages={g} sourcePages={s}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+64
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
diffindex.py - case-insensitive package/entry diff between two mw4index manifests.
|
||||
|
||||
python3 diffindex.py <A.tsv> <B.tsv> [labelA] [labelB]
|
||||
|
||||
Per package it prints:
|
||||
* name-set diff - which source assets exist on each side (the reliable signal)
|
||||
* blob multiset - stored-byte md5 counts, names ignored (a rough upper bound
|
||||
on how much payload differs; see the caveat in mw4index.py)
|
||||
|
||||
Packages present on only one side are reported as A-ONLY / B-ONLY.
|
||||
"""
|
||||
import sys, collections
|
||||
|
||||
|
||||
def load(path):
|
||||
names = collections.defaultdict(set)
|
||||
blobs = collections.defaultdict(collections.Counter)
|
||||
sizes = collections.defaultdict(dict)
|
||||
with open(path, encoding="latin-1") as fh:
|
||||
for line in fh:
|
||||
line = line.rstrip("\n")
|
||||
if not line:
|
||||
continue
|
||||
pkg, _rid, name, dlen, _rlen, h = line.split("\t")
|
||||
pkg = pkg.lower()
|
||||
key = name.lower().replace("\\", "/")
|
||||
names[pkg].add(key)
|
||||
blobs[pkg][h] += 1
|
||||
sizes[pkg][key] = int(dlen)
|
||||
return names, blobs, sizes
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 3:
|
||||
sys.exit(__doc__)
|
||||
la = sys.argv[3] if len(sys.argv) > 3 else "A"
|
||||
lb = sys.argv[4] if len(sys.argv) > 4 else "B"
|
||||
an, ab, asz = load(sys.argv[1])
|
||||
bn, bb, bsz = load(sys.argv[2])
|
||||
|
||||
for pkg in sorted(set(an) | set(bn)):
|
||||
if pkg not in an:
|
||||
print(f"== {pkg}: {lb}-ONLY PACKAGE ({len(bn[pkg])} entries)")
|
||||
continue
|
||||
if pkg not in bn:
|
||||
print(f"== {pkg}: {la}-ONLY PACKAGE ({len(an[pkg])} entries)")
|
||||
continue
|
||||
aonly = sorted(an[pkg] - bn[pkg])
|
||||
bonly = sorted(bn[pkg] - an[pkg])
|
||||
shared = sum((ab[pkg] & bb[pkg]).values())
|
||||
print(f"== {pkg}: {la}={len(an[pkg])} {lb}={len(bn[pkg])} | "
|
||||
f"names: common={len(an[pkg] & bn[pkg])} {la}_only={len(aonly)} {lb}_only={len(bonly)} | "
|
||||
f"blobs: identical={shared} {la}_uniq={sum(ab[pkg].values()) - shared} "
|
||||
f"{lb}_uniq={sum(bb[pkg].values()) - shared}")
|
||||
for n in aonly:
|
||||
print(f" +{la} {n} ({asz[pkg][n]}B)")
|
||||
for n in bonly:
|
||||
print(f" -{lb} {n} ({bsz[pkg][n]}B)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
dumprec.py - extract one record from a *.mw4, decoded.
|
||||
|
||||
python3 dumprec.py <pkg.mw4> "<entry name>" [outfile]
|
||||
|
||||
The entry name is matched case-insensitively with backslashes normalised to
|
||||
'/', e.g. "mechs/atlas/atlas.subsystems". With no outfile the bytes go to
|
||||
stdout. Pass "--list" as the entry name to print every entry name instead.
|
||||
"""
|
||||
import sys, os
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from mw4db import read_records, norm
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 3:
|
||||
sys.exit(__doc__)
|
||||
path, want = sys.argv[1], norm(sys.argv[2])
|
||||
_content, recs = read_records(path)
|
||||
if want == "--list":
|
||||
for rid, name, dlen, rlen, _blob in recs:
|
||||
print(f"{rid}\t{name}\t{dlen}\t{rlen}")
|
||||
return
|
||||
for _rid, name, _dlen, _rlen, blob in recs:
|
||||
if norm(name) == want:
|
||||
if len(sys.argv) > 3:
|
||||
open(sys.argv[3], "wb").write(blob)
|
||||
else:
|
||||
sys.stdout.buffer.write(blob)
|
||||
return
|
||||
sys.exit(f"not found: {want}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
extract-all.py - unpack every *.mw4 under a resource root into a real directory tree.
|
||||
|
||||
python3 extract-all.py <resourceRoot> <outDir> [--no-merged]
|
||||
|
||||
Layout
|
||||
------
|
||||
<outDir>/<packagePathWithoutExtension>/<entryPath>
|
||||
|
||||
e.g. core.mw4 entry mechs\\atlas\\atlas.subsystems
|
||||
-> <outDir>/core/mechs/atlas/atlas.subsystems
|
||||
|
||||
Missions/freezer.mw4 entry missions\\freezer\\freezer.contents
|
||||
-> <outDir>/Missions/freezer/missions/freezer/freezer.contents
|
||||
|
||||
This is lossless: 842 entry paths are claimed by more than one package (281 of
|
||||
them with different content - typically a mission packing its own copy of a
|
||||
global props.mw4 asset), so a single flat tree cannot represent the data.
|
||||
|
||||
<outDir>/_merged/ is then built as a flattened view of the whole set using
|
||||
HARDLINKS (no extra disk). It mirrors the layout of Gameleap/mw4/Content, which
|
||||
makes it directly diffable against our source tree. Where two packages disagree,
|
||||
precedence is props > core > textures > maps > missions and every conflict is
|
||||
listed in _conflicts.tsv.
|
||||
|
||||
Entry-name qualifiers are preserved verbatim in the filename:
|
||||
foo.data{gamemodel} foo.contents[joint_hip]{armature} bar.tga{hint}
|
||||
All characters used by these packages are legal on Linux ('!' '#' '$' '%' '^'
|
||||
appear in lobby skin names; no ':' or NUL).
|
||||
"""
|
||||
import sys, os, hashlib
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from mw4db import read_records, walk_packages
|
||||
|
||||
# Higher wins when the same entry path comes from several packages.
|
||||
PRECEDENCE = {"props": 50, "core": 40, "textures": 30, "maps": 20, "missions": 10}
|
||||
|
||||
# Saved mechlab loadouts and pilot options name their records '{Mech}',
|
||||
# '<chassis>{Subsystem}' etc. with no directory part, so they have no place in a
|
||||
# Content-shaped tree - 797 variants would all collide on the same few names.
|
||||
MERGE_EXCLUDE = ("variants/", "pilots/")
|
||||
|
||||
|
||||
def in_merged(pkgrel):
|
||||
return not pkgrel.lower().startswith(MERGE_EXCLUDE)
|
||||
|
||||
|
||||
def precedence(pkgrel):
|
||||
head = pkgrel.split("/")[0].lower()
|
||||
if head.endswith(".mw4"):
|
||||
head = head[:-4]
|
||||
return PRECEDENCE.get(head, 0)
|
||||
|
||||
|
||||
def entry_path(name):
|
||||
"""Entry name -> relative filesystem path. Qualifiers kept as-is."""
|
||||
p = name.replace("\\", "/").strip("/")
|
||||
# Defensive: never let an entry escape the output directory.
|
||||
parts = [seg for seg in p.split("/") if seg not in ("", ".", "..")]
|
||||
return "/".join(parts) if parts else "_unnamed"
|
||||
|
||||
|
||||
def main():
|
||||
args = [a for a in sys.argv[1:] if not a.startswith("--")]
|
||||
if len(args) != 2:
|
||||
sys.exit(__doc__)
|
||||
root, out = args
|
||||
want_merged = "--no-merged" not in sys.argv
|
||||
|
||||
os.makedirs(out, exist_ok=True)
|
||||
manifest = open(os.path.join(out, "_manifest.tsv"), "w", encoding="utf-8")
|
||||
manifest.write("package\trecordId\tentryName\tdataLen\trecLen\tmd5\toutPath\n")
|
||||
notes = open(os.path.join(out, "_notes.txt"), "w", encoding="utf-8")
|
||||
|
||||
# merged bookkeeping: mergedRelPath -> (precedence, pkgrel, md5, absSourceFile)
|
||||
best = {}
|
||||
seen_paths = {} # mergedRelPath -> {md5: [pkgrel, ...]}
|
||||
total_files = total_bytes = 0
|
||||
packages = 0
|
||||
|
||||
for path, pkgrel in walk_packages(root):
|
||||
res = read_records(path)
|
||||
if res is None:
|
||||
notes.write(f"SKIP not-a-#VBD-package: {pkgrel}\n")
|
||||
continue
|
||||
_content, recs = res
|
||||
packages += 1
|
||||
pkgdir = os.path.join(out, pkgrel[:-4] if pkgrel.lower().endswith(".mw4") else pkgrel)
|
||||
written = {} # relPath -> md5, for intra-package dupes
|
||||
|
||||
for rid, name, dlen, rlen, blob in recs:
|
||||
rel = entry_path(name)
|
||||
h = hashlib.md5(blob).hexdigest()
|
||||
|
||||
if rel in written: # duplicate entry name in one package
|
||||
if written[rel] == h:
|
||||
notes.write(f"DUP-IDENTICAL {pkgrel}\t{name}\trec{rid}\n")
|
||||
continue
|
||||
rel = f"{rel}#rec{rid}"
|
||||
notes.write(f"DUP-DIFFERENT {pkgrel}\t{name}\trec{rid} -> {rel}\n")
|
||||
written[rel] = h
|
||||
|
||||
dest = os.path.join(pkgdir, rel)
|
||||
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
||||
with open(dest, "wb") as fh:
|
||||
fh.write(blob)
|
||||
total_files += 1
|
||||
total_bytes += len(blob)
|
||||
manifest.write(f"{pkgrel}\t{rid}\t{name}\t{dlen}\t{rlen}\t{h}\t"
|
||||
f"{os.path.relpath(dest, out)}\n")
|
||||
|
||||
if want_merged and in_merged(pkgrel):
|
||||
seen_paths.setdefault(rel, {}).setdefault(h, []).append(pkgrel)
|
||||
pr = precedence(pkgrel)
|
||||
cur = best.get(rel)
|
||||
if cur is None or pr > cur[0]:
|
||||
best[rel] = (pr, pkgrel, h, dest)
|
||||
|
||||
print(f" {pkgrel}: {len(recs)} records", flush=True)
|
||||
|
||||
manifest.close()
|
||||
|
||||
conflicts = 0
|
||||
if want_merged:
|
||||
mroot = os.path.join(out, "_merged")
|
||||
with open(os.path.join(out, "_conflicts.tsv"), "w", encoding="utf-8") as cf:
|
||||
cf.write("mergedPath\tchosenPackage\tallVersions\n")
|
||||
for rel, (_pr, pkgrel, _h, src) in best.items():
|
||||
dest = os.path.join(mroot, rel)
|
||||
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
||||
if os.path.lexists(dest):
|
||||
os.unlink(dest)
|
||||
os.link(src, dest)
|
||||
versions = seen_paths[rel]
|
||||
if len(versions) > 1:
|
||||
conflicts += 1
|
||||
detail = "; ".join(f"{h[:8]}={','.join(ps)}" for h, ps in versions.items())
|
||||
cf.write(f"{rel}\t{pkgrel}\t{detail}\n")
|
||||
|
||||
notes.write(f"\npackages={packages} files={total_files} bytes={total_bytes} "
|
||||
f"mergedPaths={len(best)} conflicts={conflicts}\n")
|
||||
notes.close()
|
||||
print(f"\npackages : {packages}")
|
||||
print(f"files : {total_files}")
|
||||
print(f"bytes : {total_bytes/1e9:.2f} GB")
|
||||
if want_merged:
|
||||
print(f"merged : {len(best)} paths, {conflicts} with conflicting versions")
|
||||
print(f"out : {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+149
@@ -0,0 +1,149 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
mw4db.py - reader for GameOS "#VBD" resource packages (*.mw4).
|
||||
|
||||
Format (from mw4\\Libraries\\stuff\\Database.cpp):
|
||||
|
||||
header (20 bytes, little-endian)
|
||||
char[4] tag "#VBD"
|
||||
u32 format
|
||||
u32 contentVersion (63 for this codebase, VER_CONTENTVERSION)
|
||||
u32 indexSize byte offset of the payload area
|
||||
u16 recordCount
|
||||
u16 nextId
|
||||
|
||||
then `recordCount` directory records, packed, no alignment:
|
||||
s64 FILETIME when this entry was last (re)packed
|
||||
u32 dataLength decompressed size
|
||||
u32 recLength stored size
|
||||
u32 dataOffset offset from `indexSize`
|
||||
u16 recordId
|
||||
u8 nameLength
|
||||
char[] name (latin-1, backslash-separated, may carry
|
||||
{qualifier} / [page] suffixes)
|
||||
|
||||
payload at indexSize + dataOffset, `recLength` bytes.
|
||||
Storage rule (Database.cpp:451): recLength == dataLength -> raw,
|
||||
otherwise LZW-compressed with gos_LZCompress.
|
||||
|
||||
The decompressor is a faithful port of gos_LZDecompress from
|
||||
CoreTech\\Libraries\\GameOS\\FileIO.cpp - variable-width LZW, LSB-first,
|
||||
9 -> 12 bits, CLEAR = 256, EOF = 257, dictionary starts at 258.
|
||||
"""
|
||||
import struct, hashlib, os
|
||||
|
||||
|
||||
def lz_decompress(src: bytes) -> bytes:
|
||||
src = bytes(src) + b"\x00\x00\x00\x00" # u32-read padding margin
|
||||
bitpos = 0
|
||||
|
||||
def getcode(width, mask):
|
||||
nonlocal bitpos
|
||||
i = bitpos >> 3
|
||||
v = src[i] | (src[i + 1] << 8) | (src[i + 2] << 16) | (src[i + 3] << 24)
|
||||
v = (v >> (bitpos & 7)) & mask
|
||||
bitpos += width
|
||||
return v
|
||||
|
||||
CLEAR, EOF, FREE = 256, 257, 258
|
||||
chain = [0] * 8192
|
||||
suffix = [0] * 8192
|
||||
width, mask, maxindex, free = 9, 511, 512, FREE
|
||||
oldchain = oldsuffix = 0
|
||||
out = bytearray()
|
||||
|
||||
while True:
|
||||
code = getcode(width, mask)
|
||||
if code == EOF:
|
||||
break
|
||||
if code == CLEAR:
|
||||
width, mask, maxindex, free = 9, 511, 512, FREE
|
||||
code = getcode(width, mask) # ClearHash emits one literal
|
||||
out.append(code & 0xFF)
|
||||
oldchain, oldsuffix = code, code & 0xFF
|
||||
continue
|
||||
|
||||
stack = []
|
||||
if code >= free: # KwKwK special case
|
||||
stack.append(oldsuffix)
|
||||
suffix[code] = oldsuffix
|
||||
chain[code] = oldchain
|
||||
walk = oldchain
|
||||
else:
|
||||
walk = code
|
||||
while walk >= 256:
|
||||
stack.append(suffix[walk])
|
||||
walk = chain[walk]
|
||||
stack.append(walk)
|
||||
oldsuffix = walk & 0xFF
|
||||
out += bytes(reversed(stack))
|
||||
|
||||
suffix[free] = oldsuffix
|
||||
chain[free] = oldchain
|
||||
free += 1
|
||||
oldchain = code
|
||||
if free == maxindex and width != 12:
|
||||
width += 1
|
||||
maxindex <<= 1
|
||||
mask = (mask << 1) | 1
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def read_index(path):
|
||||
"""Parse directory only. Fast - no decompression.
|
||||
|
||||
Returns (contentVersion, [(recordId, name, dataLen, recLen, storedBytes), ...])
|
||||
or None when the file is not a #VBD package.
|
||||
"""
|
||||
with open(path, "rb") as fh:
|
||||
data = fh.read()
|
||||
if data[:4] != b"#VBD":
|
||||
return None
|
||||
_tag, _fmt, content, indexsize, nrec, _nextid = struct.unpack_from("<4sIIIHH", data, 0)
|
||||
out, off = [], 20
|
||||
for _ in range(nrec):
|
||||
off += 8 # FILETIME
|
||||
dlen, rlen, doff = struct.unpack_from("<III", data, off); off += 12
|
||||
rid, nlen = struct.unpack_from("<HB", data, off); off += 3
|
||||
name = data[off:off + nlen].decode("latin-1"); off += nlen
|
||||
raw = data[indexsize + doff: indexsize + doff + rlen] if dlen else b""
|
||||
out.append((rid, name, dlen, rlen, raw))
|
||||
return content, out
|
||||
|
||||
|
||||
def read_records(path):
|
||||
"""Parse and decode every record.
|
||||
|
||||
Returns (contentVersion, [(recordId, name, dataLen, recLen, decodedBytes), ...])
|
||||
"""
|
||||
r = read_index(path)
|
||||
if r is None:
|
||||
return None
|
||||
content, recs = r
|
||||
out = []
|
||||
for rid, name, dlen, rlen, raw in recs:
|
||||
blob = b"" if not dlen else (raw if rlen == dlen else lz_decompress(raw))
|
||||
out.append((rid, name, dlen, rlen, blob))
|
||||
return content, out
|
||||
|
||||
|
||||
def norm(name: str) -> str:
|
||||
"""Canonical comparison key for an entry name.
|
||||
|
||||
Package entry names are case-inconsistent between builds (V4H's packer
|
||||
lowercased everything) and use backslashes. Always compare through this.
|
||||
"""
|
||||
return name.lower().replace("\\", "/")
|
||||
|
||||
|
||||
def md5(b: bytes) -> str:
|
||||
return hashlib.md5(b).hexdigest()
|
||||
|
||||
|
||||
def walk_packages(root):
|
||||
"""Yield (absolutePath, relativePathWithForwardSlashes) for every *.mw4 under root."""
|
||||
for dirpath, _dirs, files in os.walk(root):
|
||||
for f in sorted(files):
|
||||
if f.lower().endswith(".mw4"):
|
||||
p = os.path.join(dirpath, f)
|
||||
yield p, os.path.relpath(p, root).replace("\\", "/")
|
||||
Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
mw4index.py - fast TSV manifest of every *.mw4 package under a directory.
|
||||
|
||||
python3 mw4index.py <resource-root> > manifest.tsv
|
||||
|
||||
Columns: pkgRelPath \t recordId \t entryName \t dataLen \t recLen \t md5(storedBytes)
|
||||
|
||||
Directory-only, so a 780 MB resource tree indexes in about a second.
|
||||
|
||||
CAUTION: md5 of the *stored* bytes is NOT a reliable equality test. Two builds
|
||||
that packed identical source can still produce different stored bytes (raw vs
|
||||
LZW selection, dictionary state). Use it only as a cheap "definitely identical"
|
||||
signal; confirm real differences with pkgcmp.py, which decodes.
|
||||
"""
|
||||
import sys, os
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from mw4db import read_index, walk_packages, md5
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
sys.exit(__doc__)
|
||||
root = sys.argv[1]
|
||||
for path, rel in walk_packages(root):
|
||||
try:
|
||||
r = read_index(path)
|
||||
except Exception as e: # truncated / unreadable package
|
||||
print(f"!ERR\t{rel}\t{e}", file=sys.stderr)
|
||||
continue
|
||||
if r is None:
|
||||
print(f"!NOTVBD\t{rel}", file=sys.stderr)
|
||||
continue
|
||||
_content, recs = r
|
||||
for rid, name, dlen, rlen, raw in recs:
|
||||
print(f"{rel}\t{rid}\t{name}\t{dlen}\t{rlen}\t{md5(raw)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
pkgcmp.py - authoritative content diff of two versions of the same *.mw4.
|
||||
|
||||
python3 pkgcmp.py <A.mw4> <B.mw4>
|
||||
|
||||
Decodes every record on both sides and compares the *decompressed* bytes, so
|
||||
the result is free of packer noise. This is the only trustworthy equality test;
|
||||
stored-byte comparison badly over-reports (peaks.mw4: 9 stored-byte diffs, 1
|
||||
real one).
|
||||
|
||||
Throughput is roughly 2 MB/s of package (props.mw4, 114 MB, takes ~50 s).
|
||||
"""
|
||||
import sys, os
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from mw4db import read_records, norm, md5
|
||||
|
||||
|
||||
def load(path):
|
||||
_content, recs = read_records(path)
|
||||
return {norm(name): (dlen, md5(blob), blob) for _rid, name, dlen, _rlen, blob in recs}
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 3:
|
||||
sys.exit(__doc__)
|
||||
a, b = load(sys.argv[1]), load(sys.argv[2])
|
||||
aonly = sorted(set(a) - set(b))
|
||||
bonly = sorted(set(b) - set(a))
|
||||
diff = sorted(k for k in set(a) & set(b) if a[k][1] != b[k][1])
|
||||
print(f"# entries: A={len(a)} B={len(b)} "
|
||||
f"A_only={len(aonly)} B_only={len(bonly)} decoded-differ={len(diff)}")
|
||||
for k in aonly:
|
||||
print(f"+A {k} {a[k][0]}B")
|
||||
for k in bonly:
|
||||
print(f"-B {k} {b[k][0]}B")
|
||||
for k in diff:
|
||||
print(f"~ {k} A={a[k][0]}B B={b[k][0]}B")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
prune-identical.py - delete everything from an extracted tree that we already have,
|
||||
leaving only genuine differences and additions.
|
||||
|
||||
python3 prune-identical.py <extractedDir> [--apply]
|
||||
|
||||
Dry-run by default; nothing is deleted without --apply.
|
||||
|
||||
Why two baselines
|
||||
-----------------
|
||||
A .mw4 is not a zip of the source tree. Our own packer rewrites ~7,400 files on
|
||||
the way in (.data .video .instance .contents .audio .damage .torso .subsystems
|
||||
.engine .lights) and generates a further ~17,700 qualified records that have no
|
||||
source file at all ('foo.data{gamemodel}', 'foo.contents[joint_hip]{armature}',
|
||||
'bar.tga{hint}'). Comparing only against Content/ would therefore "keep" about
|
||||
24,000 files we in fact already have.
|
||||
|
||||
So each candidate is tested, in order:
|
||||
|
||||
1. same package + same entry path in OUR extracted packages, identical bytes
|
||||
2. same entry path anywhere in OUR extracted packages, identical bytes
|
||||
3. unqualified name + same path in our Content/ source tree, identical bytes
|
||||
|
||||
Any hit means we have that exact asset -> delete. Rule 3 is what recognises
|
||||
raw pass-through assets (.tga, .wav, .erf, scripts); rules 1-2 are what
|
||||
recognise the packer-generated forms.
|
||||
|
||||
Afterwards _merged/ is rebuilt from the survivors and empty directories removed.
|
||||
"""
|
||||
import sys, os, hashlib, collections, shutil
|
||||
|
||||
OURS_EXTRACTED = "/home/rich/Repositories/FS_Ours_extracted"
|
||||
OUR_SOURCE = "/home/rich/Repositories/firestorm/Gameleap/mw4/Content"
|
||||
|
||||
PRECEDENCE = {"props": 50, "core": 40, "textures": 30, "maps": 20, "missions": 10}
|
||||
MERGE_EXCLUDE = ("variants/", "pilots/")
|
||||
|
||||
|
||||
def digest(path, cache={}):
|
||||
key = (path, os.path.getsize(path))
|
||||
if key not in cache:
|
||||
with open(path, "rb") as fh:
|
||||
cache[key] = hashlib.md5(fh.read()).hexdigest()
|
||||
return cache[key]
|
||||
|
||||
|
||||
def index_paths(root, skip_top=()):
|
||||
"""relative-lowercased-path -> absolute path. No hashing (lazy)."""
|
||||
out = {}
|
||||
for dp, dirs, fs in os.walk(root):
|
||||
if dp == root:
|
||||
dirs[:] = [d for d in dirs if d.lower() not in skip_top]
|
||||
for f in fs:
|
||||
p = os.path.join(dp, f)
|
||||
out[os.path.relpath(p, root).replace("\\", "/").lower()] = p
|
||||
return out
|
||||
|
||||
|
||||
def package_of(rel):
|
||||
"""Split '<pkgdir>/<entryPath>' for an extracted file.
|
||||
|
||||
Package directories are: core, props, textures, maps/<x>, Missions/<x>,
|
||||
Variants/<x>, Pilots/Tesla/options
|
||||
"""
|
||||
parts = rel.split("/")
|
||||
head = parts[0].lower()
|
||||
if head in ("core", "props", "textures"):
|
||||
return parts[0], "/".join(parts[1:])
|
||||
if head in ("maps", "missions", "variants") and len(parts) > 2:
|
||||
return "/".join(parts[:2]), "/".join(parts[2:])
|
||||
if head == "pilots" and len(parts) > 3:
|
||||
return "/".join(parts[:3]), "/".join(parts[3:])
|
||||
return parts[0], "/".join(parts[1:])
|
||||
|
||||
|
||||
def merged_precedence(pkg):
|
||||
return PRECEDENCE.get(pkg.split("/")[0].lower(), 0)
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
sys.exit(__doc__)
|
||||
root = sys.argv[1]
|
||||
apply_ = "--apply" in sys.argv
|
||||
merged_root = os.path.join(root, "_merged")
|
||||
|
||||
print("indexing our extracted packages ...", flush=True)
|
||||
ours_pkg = index_paths(OURS_EXTRACTED, skip_top={"_merged"})
|
||||
ours_by_entry = {}
|
||||
for rel, p in ours_pkg.items():
|
||||
if rel.startswith("_"):
|
||||
continue
|
||||
_pkg, entry = package_of(rel)
|
||||
ours_by_entry.setdefault(entry, []).append(p)
|
||||
print(f" {len(ours_pkg)} files, {len(ours_by_entry)} distinct entry paths")
|
||||
|
||||
print("indexing our Content source tree ...", flush=True)
|
||||
src = index_paths(OUR_SOURCE)
|
||||
print(f" {len(src)} files")
|
||||
|
||||
stats = collections.Counter()
|
||||
doomed, kept = [], []
|
||||
|
||||
for dp, _dirs, fs in os.walk(root):
|
||||
if dp == merged_root or dp.startswith(merged_root + os.sep):
|
||||
continue
|
||||
for f in fs:
|
||||
p = os.path.join(dp, f)
|
||||
rel = os.path.relpath(p, root).replace("\\", "/")
|
||||
if rel.startswith("_"):
|
||||
continue
|
||||
pkg, entry = package_of(rel)
|
||||
h = digest(p)
|
||||
reason = None
|
||||
|
||||
cand = ours_pkg.get(rel.lower())
|
||||
if cand and digest(cand) == h:
|
||||
reason = "same-package"
|
||||
if reason is None:
|
||||
for c in ours_by_entry.get(entry.lower(), ()):
|
||||
if digest(c) == h:
|
||||
reason = "other-package"
|
||||
break
|
||||
if reason is None and "{" not in f and "[" not in f:
|
||||
c = src.get(entry.lower())
|
||||
if c and digest(c) == h:
|
||||
reason = "our-source"
|
||||
|
||||
if reason:
|
||||
stats["deleted:" + reason] += 1
|
||||
doomed.append((rel, reason))
|
||||
else:
|
||||
stats["kept"] += 1
|
||||
kept.append((rel, pkg, entry, h))
|
||||
|
||||
total = len(doomed) + len(kept)
|
||||
print(f"\nexamined {total} files")
|
||||
for k, v in sorted(stats.items()):
|
||||
print(f" {v:7d} {k}")
|
||||
print(f"\n -> would delete {len(doomed)}, keep {len(kept)}")
|
||||
|
||||
with open(os.path.join(root, "_survivors.tsv"), "w", encoding="utf-8") as fh:
|
||||
fh.write("path\tpackage\tentry\tmd5\n")
|
||||
for rel, pkg, entry, h in sorted(kept):
|
||||
fh.write(f"{rel}\t{pkg}\t{entry}\t{h}\n")
|
||||
|
||||
if not apply_:
|
||||
print(f"\nDRY RUN - nothing changed. Survivor list written to "
|
||||
f"{os.path.join(root, '_survivors.tsv')}. Re-run with --apply.")
|
||||
return
|
||||
|
||||
for rel, _reason in doomed:
|
||||
os.remove(os.path.join(root, rel))
|
||||
|
||||
with open(os.path.join(root, "_pruned.tsv"), "w", encoding="utf-8") as fh:
|
||||
fh.write("deletedPath\tmatchedVia\n")
|
||||
for rel, reason in sorted(doomed):
|
||||
fh.write(f"{rel}\t{reason}\n")
|
||||
|
||||
# rebuild _merged from survivors
|
||||
shutil.rmtree(merged_root, ignore_errors=True)
|
||||
best = {}
|
||||
for rel, pkg, entry, h in kept:
|
||||
if pkg.lower().startswith(MERGE_EXCLUDE):
|
||||
continue
|
||||
pr = merged_precedence(pkg)
|
||||
cur = best.get(entry.lower())
|
||||
if cur is None or pr > cur[0]:
|
||||
best[entry.lower()] = (pr, entry, os.path.join(root, rel))
|
||||
for _pr, entry, srcfile in best.values():
|
||||
dest = os.path.join(merged_root, entry)
|
||||
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
||||
os.link(srcfile, dest)
|
||||
|
||||
removed = 0
|
||||
for dp, dirs, fs in os.walk(root, topdown=False):
|
||||
if dp == root or not os.path.isdir(dp):
|
||||
continue
|
||||
if not os.listdir(dp):
|
||||
os.rmdir(dp)
|
||||
removed += 1
|
||||
|
||||
print(f"\ndeleted {len(doomed)} files, removed {removed} empty directories")
|
||||
print(f"_merged rebuilt with {len(best)} paths")
|
||||
print(f"log: {os.path.join(root, '_pruned.tsv')}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
restructure.py - reshape an extracted tree to match the repo's directory layout.
|
||||
|
||||
python3 restructure.py <extractedDir> [--apply]
|
||||
|
||||
Dry-run by default.
|
||||
|
||||
The extractor writes one directory per package, which is lossless but does not
|
||||
look like the repo. This flattens it to the shape of Gameleap/mw4/Content:
|
||||
|
||||
<root>/Content/Mechs/Champion/champion.subsystems
|
||||
<root>/Content/Maps/alpine02/...
|
||||
<root>/Resource/Variants/<name>/... (variant records have no path)
|
||||
<root>/Resource/Pilots/Tesla/options/...
|
||||
<root>/_pkgroots/... (4-byte package-root pseudo-entries)
|
||||
|
||||
Path components are re-cased to match the repo wherever a counterpart exists, so
|
||||
'mechs/longbow/longbow.data{element}' becomes
|
||||
'Content/Mechs/Longbow/longbow.data{Element}'. Components with no counterpart
|
||||
(the new chassis, for instance) keep the name the packer used.
|
||||
|
||||
REFUSES TO RUN if two packages would land on the same path with different bytes.
|
||||
Flatten only trees where that has been checked - the pruned V4H tree has zero
|
||||
collisions; our own full extraction has 60 (mission-local copies of shared
|
||||
Culturals props) and must stay per-package.
|
||||
|
||||
Provenance is preserved in _layout.tsv: newPath, package, original entry path.
|
||||
"""
|
||||
import sys, os, hashlib, collections
|
||||
|
||||
REPO_CONTENT = "/home/rich/Repositories/firestorm/Gameleap/mw4/Content"
|
||||
|
||||
|
||||
def build_case_map(root):
|
||||
"""lowercased relative path -> the repo's actual spelling, for dirs and files."""
|
||||
out = {}
|
||||
for dp, dirs, files in os.walk(root):
|
||||
rel = os.path.relpath(dp, root).replace("\\", "/")
|
||||
base = "" if rel == "." else rel
|
||||
for name in dirs + files:
|
||||
r = f"{base}/{name}" if base else name
|
||||
out[r.lower()] = r
|
||||
return out
|
||||
|
||||
|
||||
def recase(path, case_map):
|
||||
"""Re-case each component against the repo, keeping unknown ones as-is."""
|
||||
done = []
|
||||
for part in path.split("/"):
|
||||
probe = "/".join(done + [part]).lower()
|
||||
canonical = case_map.get(probe)
|
||||
done.append(canonical.rsplit("/", 1)[-1] if canonical else part)
|
||||
return "/".join(done)
|
||||
|
||||
|
||||
def package_of(rel):
|
||||
parts = rel.split("/")
|
||||
head = parts[0].lower()
|
||||
if head in ("core", "props", "textures"):
|
||||
return parts[0], "/".join(parts[1:])
|
||||
if head in ("maps", "missions", "variants") and len(parts) > 2:
|
||||
return "/".join(parts[:2]), "/".join(parts[2:])
|
||||
if head == "pilots" and len(parts) > 3:
|
||||
return "/".join(parts[:3]), "/".join(parts[3:])
|
||||
return parts[0], "/".join(parts[1:])
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
sys.exit(__doc__)
|
||||
root = sys.argv[1]
|
||||
apply_ = "--apply" in sys.argv
|
||||
|
||||
print("indexing repo Content/ for canonical casing ...", flush=True)
|
||||
case_map = build_case_map(REPO_CONTENT)
|
||||
print(f" {len(case_map)} path components")
|
||||
|
||||
plan = [] # (oldRel, newRel, package, entry)
|
||||
claims = collections.defaultdict(dict) # newRel -> md5 -> [package]
|
||||
|
||||
for dp, dirs, fs in os.walk(root):
|
||||
if dp == root:
|
||||
dirs[:] = [d for d in dirs if d not in ("_merged", "Content", "Resource", "_pkgroots")]
|
||||
for f in fs:
|
||||
old = os.path.relpath(os.path.join(dp, f), root).replace("\\", "/")
|
||||
if old.startswith("_"):
|
||||
continue
|
||||
pkg, entry = package_of(old)
|
||||
low = pkg.lower()
|
||||
if low.startswith("variants/"):
|
||||
new = f"Resource/{pkg}/{entry}"
|
||||
elif low.startswith("pilots/"):
|
||||
new = f"Resource/{pkg}/{entry}"
|
||||
elif entry.lower().startswith("resource/"):
|
||||
new = f"_pkgroots/{pkg}/{entry}"
|
||||
else:
|
||||
new = "Content/" + recase(entry, case_map)
|
||||
plan.append((old, new, pkg, entry))
|
||||
if new.startswith("Content/"):
|
||||
with open(os.path.join(root, old), "rb") as fh:
|
||||
h = hashlib.md5(fh.read()).hexdigest()
|
||||
claims[new].setdefault(h, []).append(pkg)
|
||||
|
||||
clashes = {k: v for k, v in claims.items() if len(v) > 1}
|
||||
dupes = {k: v for k, v in claims.items()
|
||||
if len(v) == 1 and len(next(iter(v.values()))) > 1}
|
||||
|
||||
print(f"\n{len(plan)} files -> {len({n for _o, n, _p, _e in plan})} destinations")
|
||||
print(f" identical copies from several packages : {len(dupes)}")
|
||||
print(f" CONFLICTING copies (different bytes) : {len(clashes)}")
|
||||
if clashes:
|
||||
for k, v in list(clashes.items())[:10]:
|
||||
print(f" {k} " + "; ".join(f"{h[:6]}={','.join(p)}" for h, p in v.items()))
|
||||
sys.exit("\nrefusing to flatten: resolve the conflicts first")
|
||||
|
||||
sample = [(o, n) for o, n, _p, _e in plan if o != n][:8]
|
||||
print("\nsample:")
|
||||
for o, n in sample:
|
||||
print(f" {o}\n -> {n}")
|
||||
|
||||
if not apply_:
|
||||
print("\nDRY RUN - nothing changed. Re-run with --apply.")
|
||||
return
|
||||
|
||||
import shutil
|
||||
shutil.rmtree(os.path.join(root, "_merged"), ignore_errors=True)
|
||||
|
||||
moved, deduped = 0, 0
|
||||
with open(os.path.join(root, "_layout.tsv"), "w", encoding="utf-8") as fh:
|
||||
fh.write("newPath\tpackage\toriginalEntry\n")
|
||||
for old, new, pkg, entry in sorted(plan):
|
||||
src, dest = os.path.join(root, old), os.path.join(root, new)
|
||||
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
||||
if os.path.exists(dest): # identical copy from another package
|
||||
os.remove(src)
|
||||
deduped += 1
|
||||
else:
|
||||
os.replace(src, dest)
|
||||
moved += 1
|
||||
fh.write(f"{new}\t{pkg}\t{entry}\n")
|
||||
|
||||
removed = 0
|
||||
for dp, _dirs, _fs in os.walk(root, topdown=False):
|
||||
if dp == root or not os.path.isdir(dp):
|
||||
continue
|
||||
if not os.listdir(dp):
|
||||
os.rmdir(dp)
|
||||
removed += 1
|
||||
|
||||
print(f"\nmoved {moved}, dropped {deduped} identical duplicates, "
|
||||
f"removed {removed} empty directories")
|
||||
print(f"provenance: {os.path.join(root, '_layout.tsv')}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
split-source.py - separate repackable source files from compiled records, and
|
||||
rename the ones that need it, so the tree can be dropped into Content/.
|
||||
|
||||
python3 split-source.py <extractedDir> [--apply]
|
||||
|
||||
Dry-run by default.
|
||||
|
||||
The problem
|
||||
-----------
|
||||
A .mw4 does not store the source tree. Some records are the source file byte for
|
||||
byte; others are what the packer produced from it. Classification was derived
|
||||
empirically by extracting OUR OWN packages and hash-matching every record against
|
||||
Gameleap/mw4/Content (22,138 matched, 28,728 did not):
|
||||
|
||||
verbatim source .tga .erf .mw4anim .bid .wav .bsp .material .abl .obb .ebf
|
||||
.bounds .animscript .fgd .tcf .mlr .d3f .gaf .script .h .abi
|
||||
compiled .data .instance .subsystems .damage .torso .engine .audio
|
||||
.video .lights .mw4, and every {hint} {handle} {gamemodel}
|
||||
{element} {footsteps} {zones} {nametable} [shadow]
|
||||
[joint_*]{armature} [joint_*]{sites} record
|
||||
|
||||
Worth being concrete: our source Content/Mechs/Atlas/atlas.data is 4,891 bytes of
|
||||
text; the record packed under that name is a 12-byte binary stub. The real
|
||||
payload went into the {gamemodel}/{element} records. So a mech's .data .damage
|
||||
.subsystems .instance CANNOT be recovered from a package - they have to be
|
||||
re-authored.
|
||||
|
||||
.armature likewise does not exist in a package. The packer splits it into
|
||||
per-joint X.contents[joint_*]{armature} records, which are compiled.
|
||||
|
||||
What this does
|
||||
--------------
|
||||
* X.data{solidobb} -> X_skeleton_SOLID.obb (mechs)
|
||||
X.data{hierarchicalobb} -> X_skeleton.obb (mechs)
|
||||
...or _SOLID.obb / .obb outside Content/Mechs. These really are .obb files
|
||||
('#BBO' magic, headers byte-identical to ours). The exact source spelling is
|
||||
declared inside the .data text, which we do not have, so a convention is
|
||||
applied and recorded in _source-vs-compiled.tsv - whatever .data gets
|
||||
authored later must reference the same name.
|
||||
* everything classified as compiled moves to _compiled/, keeping its path
|
||||
* everything left under Content/ is genuine, repackable source
|
||||
|
||||
Resource/ (variants, pilot options) is left alone: those are whole .mw4
|
||||
packages, best taken from FS_Build_V4H/resource/Variants/*.mw4 directly rather
|
||||
than from their unpacked records.
|
||||
"""
|
||||
import sys, os, re, collections
|
||||
|
||||
SOURCE_EXT = {".tga", ".erf", ".mw4anim", ".bid", ".wav", ".bsp", ".material",
|
||||
".abl", ".obb", ".ebf", ".bounds", ".animscript", ".fgd", ".tcf",
|
||||
".mlr", ".d3f", ".gaf", ".h", ".abi", ".hpp", ".include", ".tpl"}
|
||||
COMPILED_EXT = {".data", ".instance", ".subsystems", ".damage", ".torso",
|
||||
".engine", ".audio", ".video", ".lights", ".mw4"}
|
||||
OBB_QUALS = {"{solidobb}": ("_skeleton_SOLID.obb", "_SOLID.obb"),
|
||||
"{hierarchicalobb}": ("_skeleton.obb", ".obb")}
|
||||
|
||||
QUAL = re.compile(r'(\[[^\]]*\]|\{[^}]*\})')
|
||||
|
||||
|
||||
def is_text(path, limit=8192):
|
||||
with open(path, "rb") as fh:
|
||||
chunk = fh.read(limit)
|
||||
if not chunk:
|
||||
return False
|
||||
printable = sum(1 for b in chunk if 9 <= b <= 13 or 32 <= b < 127)
|
||||
return printable / len(chunk) > 0.90
|
||||
|
||||
|
||||
def classify(path, name, rel):
|
||||
quals = "".join(QUAL.findall(name)).lower()
|
||||
stem = QUAL.sub("", name)
|
||||
ext = os.path.splitext(stem)[1].lower()
|
||||
|
||||
if quals in OBB_QUALS:
|
||||
mech = "/mechs/" in rel.lower()
|
||||
suffix = OBB_QUALS[quals][0 if mech else 1]
|
||||
return "source", stem.rsplit(".", 1)[0] + suffix
|
||||
if quals:
|
||||
return "compiled", name
|
||||
if ext in SOURCE_EXT:
|
||||
return "source", name
|
||||
if ext in COMPILED_EXT:
|
||||
return "compiled", name
|
||||
# .script and .contents are genuinely mixed - decide on content
|
||||
return ("source" if is_text(path) else "compiled"), name
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
sys.exit(__doc__)
|
||||
root = sys.argv[1]
|
||||
apply_ = "--apply" in sys.argv
|
||||
content = os.path.join(root, "Content")
|
||||
if not os.path.isdir(content):
|
||||
sys.exit(f"no Content/ in {root} - run restructure.py first")
|
||||
|
||||
plan, stats = [], collections.Counter()
|
||||
renames = collections.Counter()
|
||||
for dp, _dirs, fs in os.walk(content):
|
||||
for f in fs:
|
||||
p = os.path.join(dp, f)
|
||||
rel = os.path.relpath(p, root).replace("\\", "/")
|
||||
verdict, newname = classify(p, f, rel)
|
||||
new = ("Content/" if verdict == "source" else "_compiled/Content/") + \
|
||||
os.path.relpath(os.path.join(dp, newname), content).replace("\\", "/")
|
||||
plan.append((rel, new, verdict))
|
||||
stats[verdict] += 1
|
||||
if verdict == "source" and newname != f:
|
||||
renames[os.path.splitext(f)[1] or f] += 1
|
||||
|
||||
print(f"Content/: {sum(stats.values())} files")
|
||||
print(f" repackable source : {stats['source']}")
|
||||
print(f" compiled records : {stats['compiled']} (-> _compiled/)")
|
||||
if renames:
|
||||
print("\nrenamed:")
|
||||
for k, v in renames.most_common():
|
||||
print(f" {v:5d} {k}")
|
||||
print("\nsample renames:")
|
||||
shown = 0
|
||||
for old, new, v in plan:
|
||||
if v == "source" and os.path.basename(old) != os.path.basename(new) and shown < 6:
|
||||
print(f" {os.path.basename(old)} -> {os.path.basename(new)}")
|
||||
shown += 1
|
||||
|
||||
if not apply_:
|
||||
print("\nDRY RUN - nothing changed. Re-run with --apply.")
|
||||
return
|
||||
|
||||
for old, new, _v in plan:
|
||||
src, dest = os.path.join(root, old), os.path.join(root, new)
|
||||
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
||||
os.replace(src, dest)
|
||||
|
||||
with open(os.path.join(root, "_source-vs-compiled.tsv"), "w", encoding="utf-8") as fh:
|
||||
fh.write("verdict\toriginalPath\tnewPath\n")
|
||||
for old, new, v in sorted(plan):
|
||||
fh.write(f"{v}\t{old}\t{new}\n")
|
||||
|
||||
removed = 0
|
||||
for dp, _dirs, _fs in os.walk(root, topdown=False):
|
||||
if dp == root or not os.path.isdir(dp):
|
||||
continue
|
||||
if not os.listdir(dp):
|
||||
os.rmdir(dp)
|
||||
removed += 1
|
||||
print(f"\nmoved {len(plan)} files, removed {removed} empty directories")
|
||||
print(f"log: {os.path.join(root, '_source-vs-compiled.tsv')}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+47
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
treediff.py - case-insensitive loose-file diff of two directory trees.
|
||||
|
||||
python3 treediff.py <dirA> <dirB> [labelA] [labelB]
|
||||
|
||||
Compares by relative path (lowercased, '/'-normalised) and md5. Written for the
|
||||
hsh/, content/ and root config comparisons, where the two builds disagree on
|
||||
filename case but not on identity.
|
||||
"""
|
||||
import sys, os, hashlib
|
||||
|
||||
|
||||
def scan(root):
|
||||
out = {}
|
||||
for dirpath, _dirs, files in os.walk(root):
|
||||
for f in files:
|
||||
p = os.path.join(dirpath, f)
|
||||
rel = os.path.relpath(p, root).replace("\\", "/")
|
||||
with open(p, "rb") as fh:
|
||||
h = hashlib.md5(fh.read()).hexdigest()
|
||||
out[rel.lower()] = (os.path.getsize(p), h, rel)
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 3:
|
||||
sys.exit(__doc__)
|
||||
la = sys.argv[3] if len(sys.argv) > 3 else "A"
|
||||
lb = sys.argv[4] if len(sys.argv) > 4 else "B"
|
||||
a, b = scan(sys.argv[1]), scan(sys.argv[2])
|
||||
aonly = sorted(set(a) - set(b))
|
||||
bonly = sorted(set(b) - set(a))
|
||||
diff = sorted(k for k in set(a) & set(b) if a[k][1] != b[k][1])
|
||||
print(f"# {la}={len(a)} files, {lb}={len(b)} files | "
|
||||
f"identical={len(set(a) & set(b)) - len(diff)} differ={len(diff)} "
|
||||
f"{la}_only={len(aonly)} {lb}_only={len(bonly)}")
|
||||
for k in aonly:
|
||||
print(f"+{la}\t{a[k][2]}\t{a[k][0]}")
|
||||
for k in bonly:
|
||||
print(f"-{lb}\t{b[k][2]}\t{b[k][0]}")
|
||||
for k in diff:
|
||||
print(f"~DIF\t{a[k][2]}\t{la}={a[k][0]}B {lb}={b[k][0]}B")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+93
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
variants-report.py - inventory of resource/Variants/*.mw4 (saved mechlab loadouts).
|
||||
|
||||
python3 variants-report.py <variantsDir> <v4hManifest.tsv> <oursManifest.tsv>
|
||||
|
||||
A variant package holds 4 records:
|
||||
[0] 'Resource\\Variants\\<name>' 4 B (raw id)
|
||||
[1] '{Mech}' ~350 B
|
||||
[2] '<chassis>{Subsystem}' ~4 KB <- names the chassis it needs
|
||||
[3] '<8-char token>' 16 B
|
||||
|
||||
So the chassis a variant depends on is readable straight from the manifest,
|
||||
with no decompression: the record whose name ends in '{Subsystem}'.
|
||||
|
||||
The report groups variants by chassis and flags any whose chassis is absent
|
||||
from the target build's core.mw4 (those will not load).
|
||||
"""
|
||||
import sys, os, collections
|
||||
|
||||
|
||||
def chassis_in_core(manifest):
|
||||
"""Set of chassis names that core.mw4 defines (one *.subsystems per chassis)."""
|
||||
out = set()
|
||||
with open(manifest, encoding="latin-1") as fh:
|
||||
for line in fh:
|
||||
pkg, _rid, name, *_ = line.rstrip("\n").split("\t")
|
||||
if pkg.lower() != "core.mw4":
|
||||
continue
|
||||
n = name.lower().replace("\\", "/")
|
||||
if n.endswith(".subsystems"):
|
||||
out.add(n.rsplit("/", 1)[-1][:-len(".subsystems")])
|
||||
return out
|
||||
|
||||
|
||||
def variant_chassis(manifest):
|
||||
"""variantPkgRelPath -> chassis name it requires."""
|
||||
out = {}
|
||||
with open(manifest, encoding="latin-1") as fh:
|
||||
for line in fh:
|
||||
pkg, _rid, name, *_ = line.rstrip("\n").split("\t")
|
||||
if not pkg.lower().startswith("variants/"):
|
||||
continue
|
||||
if name.endswith("{Subsystem}"):
|
||||
out[pkg] = name[:-len("{Subsystem}")].lower()
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 4:
|
||||
sys.exit(__doc__)
|
||||
vdir, v4h_manifest, ours_manifest = sys.argv[1:4]
|
||||
have_v4h = chassis_in_core(v4h_manifest)
|
||||
have_ours = chassis_in_core(ours_manifest)
|
||||
vc = variant_chassis(v4h_manifest)
|
||||
|
||||
print(f"# variant packages: {len(vc)} (files on disk: "
|
||||
f"{len([f for f in os.listdir(vdir) if f.lower().endswith('.mw4')])})")
|
||||
print(f"# chassis defined in V4H core.mw4 : {len(have_v4h)}")
|
||||
print(f"# chassis defined in OUR core.mw4 : {len(have_ours)}")
|
||||
print(f"# chassis only V4H defines : {sorted(have_v4h - have_ours)}")
|
||||
print()
|
||||
|
||||
by = collections.defaultdict(list)
|
||||
for pkg, ch in vc.items():
|
||||
by[ch].append(pkg)
|
||||
|
||||
loadable_now = broken = needs_new_mech = 0
|
||||
print(f"{'chassis':22s} {'count':>5s} status")
|
||||
for ch in sorted(by, key=lambda c: (-len(by[c]), c)):
|
||||
n = len(by[ch])
|
||||
if ch in have_ours:
|
||||
status = "loadable in OUR build today"
|
||||
loadable_now += n
|
||||
elif ch in have_v4h:
|
||||
status = "needs the new chassis ported"
|
||||
needs_new_mech += n
|
||||
else:
|
||||
status = "BROKEN - chassis missing from V4H too"
|
||||
broken += n
|
||||
print(f"{ch:22s} {n:5d} {status}")
|
||||
|
||||
print()
|
||||
print(f"loadable in our build today : {loadable_now}")
|
||||
print(f"blocked on new chassis : {needs_new_mech}")
|
||||
print(f"broken / orphaned : {broken}")
|
||||
for ch in sorted(by):
|
||||
if ch not in have_v4h:
|
||||
print(f" orphan chassis '{ch}': " + ", ".join(sorted(by[ch])))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user