diff --git a/MUNGA/JMOVER.cpp b/MUNGA/JMOVER.cpp index ae2c912..272ba9d 100644 --- a/MUNGA/JMOVER.cpp +++ b/MUNGA/JMOVER.cpp @@ -7,10 +7,315 @@ #include "notation.h" #include "namelist.h" +// EXCEPTION_EXECUTE_HANDLER, for the guarded read in RPReadDword below. +// excpt.h rather than windows.h: this layer does not otherwise want it. +#include + //############################################################################# //############################ JointedMover ############################# //############################################################################# +//############################################################################# +// RP412SEGCHECK - segment validity at teardown +//############################################################################# +// +// The podium crash (three identical dumps, 2026-08-11) dies inside +// SocketIterator::DeletePlugs, called from ~JointedMover, calling through a +// segment whose FIRST DWORD - the vtable pointer - has been replaced by a +// small float (9.75 / 12.80 / 7.52 across the three machines). The table +// itself was untouched: numItems still 15, every TableEntry hooked up, which +// means nothing ever ran delete against that segment. Something wrote over +// it, or freed it behind the collection's back. +// +// Six configurations of the local repro rig - parked pods, driven pods, +// light page heap, full page heap, two, four and six pods - reached the +// podium and tore down clean. So this exists to make the next REAL playtest +// the instrument: look at each segment before calling through it, and if one +// is wrong, write what it looks like to rpl4-fail.log, which is fclose'd and +// survives the abort. Text written only to rpl4.log does not. +// +// This deliberately does NOT skip the delete, repair the pointer, or +// otherwise make the teardown survivable. The ownership bug is still +// unfixed, and a guard that hid it would cost exactly the evidence this is +// here to collect. It stops on the same object the crash would have died on, +// one step earlier, holding the forensics. +// +// RP412SEGCHECK=0 turns it off. +// + +// +// The vtable every EntitySegment should carry, taken from the first one ever +// built - i.e. from fresh memory, before any race has run. Comparing against +// a captured value rather than a computed one keeps this independent of how +// the linker laid the image out. +// +static void + *gGoodSegmentVtable = NULL; + +static int + RPSegCheckEnabled() +{ + static int + enabled = -1; + + if (enabled < 0) + { + const char + *setting = getenv("RP412SEGCHECK"); + + enabled = (setting != NULL && atoi(setting) == 0) ? 0 : 1; + } + return enabled; +} + +// +// Read a dword that may not be there any more. If the block has been freed +// under a page-heap build the pages are decommitted, and the plain read +// would fault inside the checker - losing the report we came for. +// +static int + RPReadDword(const void *address, unsigned long *out) +{ + __try + { + *out = *(const unsigned long *) address; + return 1; + } + __except (EXCEPTION_EXECUTE_HANDLER) + { + return 0; + } +} + +// +// WHERE the corruption was first seen. Recorded in a static rather than +// simply logged, because rpl4.log does not reliably survive the abort that +// follows at teardown - so the answer has to travel inside the Fail report. +// +// This is the question the dumps cannot answer: the crash proves a segment +// is wrong by teardown, not whether it was already wrong when the race +// ended. Bracketing the podium separates "something during the race broke +// it" from "the podium's second MakeEntityRenderables broke it". +// +static char + gFirstBadWhen[64] = ""; + +// +// Public entry point for the bracket call sites (see RPL4APP.cpp). Counts +// corrupted segments on one entity and remembers the first moment any were +// seen. Reports; never aborts - the teardown check does that, with this +// answer folded in. +// +int + RPCheckJointedMoverSegments(Entity *entity, const char *when) +{ + if (!RPSegCheckEnabled() || gGoodSegmentVtable == NULL || entity == NULL) + { + return 0; + } + if (!entity->IsDerivedFrom(*JointedMover::GetClassDerivations())) + { + return 0; + } + + JointedMover + *mover = (JointedMover *) entity; + EntitySegment::SegmentTableIterator + iterator(mover->segmentTable); + EntitySegment + *segment; + int + bad = 0; + + iterator.First(); + while ((segment = iterator.ReadAndNext()) != NULL) + { + unsigned long + vtable = 0; + + if (!RPReadDword(segment, &vtable) + || (void *) vtable != gGoodSegmentVtable) + { + ++bad; + } + } + + if (bad > 0 && gFirstBadWhen[0] == '\0') + { + strncpy(gFirstBadWhen, when, sizeof(gFirstBadWhen) - 1); + gFirstBadWhen[sizeof(gFirstBadWhen) - 1] = '\0'; + DEBUG_STREAM << "SegCheck: " << bad + << " segment(s) already corrupt at " << when << "\n" << std::flush; + } + return bad; +} + +static void + RPCheckSegmentsBeforeTeardown( + Entity *mover, + EntitySegment::SegmentTable &segment_table + ) +{ + if (!RPSegCheckEnabled() || gGoodSegmentVtable == NULL) + { + return; + } + + // + // Walk the table WITHOUT dereferencing anything: the iterator reads the + // TableEntry array, and GetPlug hands back the pointer without touching + // the object. The only read of segment memory is the guarded one below. + // + EntitySegment::SegmentTableIterator + iterator(segment_table); + EntitySegment + *segment, + *bad_segment = NULL, + *before_bad = NULL, + *after_bad = NULL; + unsigned long + bad_value = 0; + int + index = 0, + total = 0, + bad_count = 0, + bad_index = -1, + bad_readable = 0; + + iterator.First(); + while ((segment = iterator.ReadAndNext()) != NULL) + { + unsigned long + vtable = 0; + int + readable = RPReadDword(segment, &vtable); + + if (!readable || (void *) vtable != gGoodSegmentVtable) + { + ++bad_count; + if (bad_segment == NULL) + { + bad_segment = segment; + bad_index = index; + bad_value = vtable; + bad_readable = readable; + } + } + else if (bad_segment == NULL) + { + before_bad = segment; // last good one before the first bad + } + else if (after_bad == NULL) + { + after_bad = segment; // first one after it + } + ++index; + ++total; + } + + if (bad_count == 0) + { + return; + } + + // + // One buffer, one message: Fail_With_Message fprintf's the string it is + // given into rpl4-fail.log and closes the file, so everything worth + // having has to be inside it. Newlines are fine. + // + static char + report[1400]; + char + *out = report; + Entity + *viewpoint = (application != NULL) + ? application->GetViewpointEntity() + : NULL; + + out += sprintf(out, + "segment plug corrupted BEFORE DeletePlugs\n" + " first seen: %s\n" + " entity %d:%d class%d, %s pod, owner %d\n" + " segments %d, corrupted %d, first bad index %d at 0x%08lX\n", + (gFirstBadWhen[0] != '\0') ? gFirstBadWhen + : "teardown (clean at every earlier check)", + (int) mover->entityID.GetHostID(), (int) mover->entityID, + (int) mover->GetClassID(), + (mover == viewpoint) ? "VIEWPOINT (local)" : "replicant", + (int) mover->GetOwnerID(), + total, bad_count, bad_index, (unsigned long) bad_segment); + + if (!bad_readable) + { + out += sprintf(out, + " segment memory is NOT READABLE - freed and decommitted\n"); + } + else + { + float + as_float; + + memcpy(&as_float, &bad_value, sizeof(as_float)); + out += sprintf(out, + " vtable expected 0x%08lX found 0x%08lX (as float %.6g)\n", + (unsigned long) gGoodSegmentVtable, bad_value, (double) as_float); + + // + // The first 8 dwords, hex and float. The dumps showed a float where + // the vtable belongs; how FAR the float data runs says whether this + // was a stray single write or something walking through the object. + // + for (int row = 0; row < 2; ++row) + { + char + hex[80], + flt[80]; + int + hex_used = 0, + flt_used = 0; + + for (int col = 0; col < 4; ++col) + { + const unsigned char + *at = (const unsigned char *) bad_segment + + (row * 16) + (col * 4); + unsigned long + word = 0; + + if (!RPReadDword(at, &word)) + { + hex_used += sprintf(hex + hex_used, " ????????"); + flt_used += sprintf(flt + flt_used, " ????????"); + continue; + } + + float + word_float; + + memcpy(&word_float, &word, sizeof(word_float)); + hex_used += sprintf(hex + hex_used, " %08lX", word); + flt_used += sprintf(flt + flt_used, " %9.4g", (double) word_float); + } + out += sprintf(out, " +%02X:%s |%s\n", row * 16, hex, flt); + } + } + + // + // Heap adjacency: if the corruption came from a neighbour running past + // its end, the deltas say which side it came from and how far it reached. + // + out += sprintf(out, + " neighbours: prev good 0x%08lX (delta %ld), next 0x%08lX (delta %ld)\n", + (unsigned long) before_bad, + (before_bad != NULL) + ? (long) ((char *) bad_segment - (char *) before_bad) : 0L, + (unsigned long) after_bad, + (after_bad != NULL) + ? (long) ((char *) after_bad - (char *) bad_segment) : 0L); + + Fail(report); +} + //############################################################################# // Shared Data Support // @@ -307,6 +612,17 @@ JointedMover::JointedMover( primary_dzone_index ); Register_Object(current_segment); + + // + // The reference vtable for RPCheckSegmentsBeforeTeardown, taken once + // from the very first segment ever built - fresh memory, before any + // race has run. See the RP412SEGCHECK block at the top of this file. + // + if (gGoodSegmentVtable == NULL) + { + gGoodSegmentVtable = *(void **) current_segment; + } + segmentTable.AddValue(current_segment, ii); // //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -429,6 +745,13 @@ void // JointedMover::~JointedMover() { + // + // Look at the segments before calling through them. This is where the + // podium crash lands; see the RP412SEGCHECK block at the top of this + // file for what it records and why it does not try to survive. + // + RPCheckSegmentsBeforeTeardown((Entity *) this, segmentTable); + EntitySegment::SegmentTableIterator iterator(segmentTable); iterator.DeletePlugs(); // diff --git a/MUNGA/JMOVER.h b/MUNGA/JMOVER.h index 4c73cd9..a114022 100644 --- a/MUNGA/JMOVER.h +++ b/MUNGA/JMOVER.h @@ -5,6 +5,16 @@ #include "slot.h" #include "segment.h" +// +// RP412SEGCHECK - count corrupted segment plugs on one entity and remember +// the first moment any were seen, so the teardown report can say WHERE it +// went wrong rather than only that it did. Safe on any entity: returns 0 +// for anything that is not a JointedMover. Reports; never aborts. +// See the block comment at the top of JMOVER.cpp. +// +extern int + RPCheckJointedMoverSegments(Entity *entity, const char *when); + //############################################################################## //######################## JointedMover ################################## //############################################################################## diff --git a/RP_L4/RPL4APP.cpp b/RP_L4/RPL4APP.cpp index 0c76afa..b45bba8 100644 --- a/RP_L4/RPL4APP.cpp +++ b/RP_L4/RPL4APP.cpp @@ -253,8 +253,21 @@ void // Time podiumOutsideStart = Now(); + // + // Bracket the exterior build. The podium crash kills the pod at teardown + // with a segment plug whose vtable has been overwritten; these two calls + // say whether it was ALREADY wrong when the podium began, or whether this + // second MakeEntityRenderables on the own pod is what did it. The answer + // rides in the teardown's rpl4-fail.log report - see RP412SEGCHECK in + // MUNGA\JMOVER.cpp. Costs a handful of pointer compares, twice, once a + // race. + // + RPCheckJointedMoverSegments(GetViewpointEntity(), "podium: before exterior"); + dpl_renderer->ShowViewpointFromOutside(); + RPCheckJointedMoverSegments(GetViewpointEntity(), "podium: after exterior"); + // //--------------------------------------------------------------------- // The name plates are drawn per rank slot, so they have to be re-sorted @@ -500,6 +513,13 @@ void Check(this); DEBUG_STREAM << "WinnersCircle: standing the finishers up\n" << std::flush; + + // + // The earliest of the three checks: anything corrupt HERE was broken by + // the race itself, before the podium touched anything. + // + RPCheckJointedMoverSegments(GetViewpointEntity(), "podium: entry (race just ended)"); + ShowWinnersCircle(); Check_Fpu(); } diff --git a/RP_L4/RPL4ENVIRON.cpp b/RP_L4/RPL4ENVIRON.cpp index 256d682..3726ce1 100644 --- a/RP_L4/RPL4ENVIRON.cpp +++ b/RP_L4/RPL4ENVIRON.cpp @@ -416,6 +416,14 @@ namespace "# same race at every frame rate, bit for bit.\n" "#RP412PHYSTRACE=1\n" "\n" +"# Segment validity check at pod teardown, on by default. The winners'\n" +"# circle crash of 2026-08-11 died deleting a pod whose skeleton segment\n" +"# had been overwritten; this looks at the segments before deleting them\n" +"# and, if one is wrong, writes what it looks like - and where it first\n" +"# went wrong - to rpl4-fail.log, which survives the abort. It costs a\n" +"# handful of pointer compares per pod per race. 0 turns it off.\n" +"#RP412SEGCHECK=0\n" +"\n" "# Try this drop zone first at spawn instead of a random pick. The pick\n" "# is seeded by RANDOM=, but a seed only repeats a run if the same\n" "# NUMBER of draws comes before the pick, and that count rides on load\n" diff --git a/tools/podium-repro/README.md b/tools/podium-repro/README.md new file mode 100644 index 0000000..0d3fd7c --- /dev/null +++ b/tools/podium-repro/README.md @@ -0,0 +1,313 @@ +# Podium-teardown crash repro rig + +For the intermittent double-delete in `JointedMover::~JointedMover` after the +winners' circle (build 4.12.217, 2026-08-11 six-player playtest). This rig +exists to catch the FIRST free in a debugger — the diagnosis must come from +the trap, not from reading code. + +## Evidence already banked (2026-08-11 debug session) + +The three podium dumps are `playtestlogs\20260811\rpl4crash (N)1.dmp` — note +the `1` suffix; the plain `rpl4crash (N).dmp` files are the OLD, already-fixed +`L4NetworkManager::Send` NULL-host crash. Symbols: `playtestlogs\symbols-4.12.217`, +load with `.reload /f /i rpl4opt.exe`. + +All three podium dumps are byte-for-byte the same failure: + +- Crash at `SocketIterator::DeletePlugs` inlined in `~JointedMover+0x68`: + `call [edx]` where edx = the vtable dword of the plug being deleted, + overwritten by a small float (9.75 / 12.80 / 7.52 in the three dumps) — + freed memory reused by something that writes floats. +- The iterator lives on the stack, which IS in the minidumps: + `numItems=15` (a full skeleton), `currentPosition=1`, so the dead plug is + `array[0]` — segment index 0, the root segment. +- `segmentTable.socketsNode` read back == the entity pointer, and the table + header was readable/writable — the ENTITY IS INTACT. The freed thing is + the segment plug (or its TableEntry), not the pod. +- The frame above returns to `Application::Shutdown+0x71`, which is + `delete viewpointEntity` (MUNGA\APP.cpp ~1038) — the FIRST entity deleted + in Shutdown. So the dying pod is the LOCAL pod: the same entity that got + the podium's second `MakeEntityRenderables(outsideEntity)` pass + (`DPLRenderer::ShowViewpointFromOutside`). `videoRenderer->Shutdown()` + runs immediately before it. +- No entries were removed from the table (15 of 15 still present), so the + first free BYPASSED the Link/TableEntry machinery: a clean + `delete segment` would have unhooked its TableEntry and decremented + numItems. Whatever freed it did not run ~Plug against that table. + +Nothing above says WHO freed it first. That is what this rig answers. + +## What the rig is + +- `setup.ps1` — builds N sandboxed installs under `%TEMP%\rp412-podium-repro` + (junctions to `dist\{AUDIO,GAUGE,VIDEO}`, INIs/RES/DLLs copied from dist, + `rpl4opt.exe` copied from `Release\` — build Release first). +- `runpod.cmd` — one pod under cdb. Clears `NoDefaultCurrentDirectoryInExePath` + (Git-Bash exports it; CreateProcess then refuses CWD-relative exe names) and + keeps the game's CWD = the pod dir (a wrong CWD produces a phantom audio-init + crash — see project discipline notes). +- `cdbrun.txt` — cdb stdin script. At the create-process event it writes + `ed $peb+68 02001000` (FLG_HEAP_PAGE_ALLOCS | FLG_USER_STACK_TRACE_DB), which + turns on PAGE HEAP WITHOUT gflags/elevation — verified working on this + machine: cdb log shows `verifier.dll` loading and + `Page heap: pid X: page heap enabled with flags 0x2` (light page heap + + stack traces: fills freed blocks, validates on every heap op, records + alloc/free stacks). A double free traps at the SECOND free; the freed-fill + pattern makes the teardown's vtable call fault deterministically instead of + only when the heap happened to reuse the block. On any break it prints + registers, `.exr -1`, `kb 30`, `!heap -p -a` on eax/ecx/edx, writes a FULL + `podbreak*.dmp` in the pod dir, then quits. +- `feeder.ps1` — drives `-Races` (default 5) full races through ONE set of + processes over the console protocol: egg -> mesh -> RunMission -> 45 s -> + StopMission(0) (the buzzer) -> podium -> ending fade -> + `Application::Shutdown` teardown -> back to WaitingForEgg -> next race. + `-PodCount` (2..8) sets the size of the field. It watches the cdb logs for + the break marker every tick, so a trap is caught in seconds rather than at + the race timeout, and leaves everything up for post-mortem. Derived from + `tools\two-pod-test.ps1` (proven harness). + +**No human pilots are needed at any point.** Bots fill the field: pods sit on +their pads unless `RP412INPUTSCRIPT` drives them, and the race still runs to +the buzzer, still ranks players, and still places them on the stand — which is +the path under test. "More players" means more scripted instances on this one +machine, not more people. + +## Run it + + powershell -NoProfile -ExecutionPolicy Bypass -File tools\podium-repro\setup.ps1 + powershell -NoProfile -ExecutionPolicy Bypass -File tools\podium-repro\feeder.ps1 + +800x600 game windows appear, one per pod; ~2 min per race. Watch for +`FEEDER DONE`. To reproduce the population that actually died (six placed), +run the six-pod field — pass the same count to both scripts: + + ... -File tools\podium-repro\setup.ps1 -PodCount 6 + ... -File tools\podium-repro\feeder.ps1 -PodCount 6 -Races 5 + +Six 32-bit instances with light page heap fit comfortably on this machine; +they will contend for CPU, which perturbs teardown timing — no bad thing for +an order-dependent bug. + +Status when this rig was assembled: single-pod boot under page heap to +WaitingForEgg is VERIFIED (verifier.dll loads, "page heap enabled with flags +0x2" in the cdb log, game reaches the front end). The race loop itself has +NOT been exercised — the session that built this lost the ability to run +anything before it could. The feeder is a close copy of the proven +two-pod-test, but treat its first run as a shakedown: if the pods never reach +`WaitingForLaunch`, the mesh/egg path is the thing to debug, not the crash. + +## Gotchas this rig has already paid for + +- **`dist\environ.ini` ships `RP412PODIUM=0`.** The podium is OFF by default, + so a sandbox that copies environ.ini verbatim tests NOTHING - every pod + logs `WinnersCircle: disabled` and goes straight to the results. setup.ps1 + now appends `RP412PODIUM=1` (appended, not edited: last duplicate key wins; + written ASCII/no-BOM, because a BOM here has invalidated runs before). + **Check for `WinnersCircle: N placed on 8 spots` in a pod's rpl4.log - if + it says "disabled", the run proved nothing.** +- **One mission per process is ALL you get, by design.** RESOLVED, not a + rig fault: in APPMGR.cpp RunMissions, a finished application has + `Application::Shutdown` called on it; Shutdown returns False, so the app is + removed from runningApplications, and with none left RunMissions returns + and the process exits. The teardown IS the exit. So `-Races 1` is the only + honest setting for a console-driven pod, and repeated teardowns means + repeated LAUNCHES, not repeated races. (This is why "race 2" never + started: nothing was there to race.) +- **A clean exit is not a trap.** cdb's `g` returns on process exit as well + as on a fault, so the "=== POD BREAK ===" banner prints for a normal quit. + Detection now keys on `ExceptionAddress:` from `.exr -1`, which only + appears for a real exception. Six "traps" were once six clean shutdowns. + +- **Pilot colours must be in `colorLookUp`** (RPL4GAUG.cpp, matched on the + first three chars): Aqua Black Blue Green Pink Purple Red White Yellow. + Anything else → `determineEntityColor` returns its 255 fallback → the GPS + gauge's own-pod blip adds 0xC0 → `translationTable[447]` on a 256-entry + table → out-of-bounds read, which page heap turns into a hard AV at mission + start. It only bites the pod that OWNS the odd colour (own blip flashes, + others don't), so it looks like one machine at random. A six-pod run with + "Orange" died exactly this way. +- **`!heap` is `!ext.heap`** in current debuggers — exts.dll forwards and + prints a notice INSTEAD of running, silently costing the heap forensics. +- **Retire the survivors on a trap.** When one pod breaks, the feeder stops + driving; without an explicit StopMission the other pods race on forever + with the clock counting up. The feeder now buzzes then kills them, keeping + only the trapped pod frozen. + +## Reading the result + +- Crash repro'd: `%TEMP%\rp412-podium-repro\pod?-cdb.log` has everything after + `=== POD BREAK ===`. The money shot is `!heap -p -a ` — with + page heap + stack DB the output includes the block's ALLOC stack and its + FREE stack. The free stack that is NOT `~JointedMover` IS the first free — + that is the bug. If the break is a verifier stop inside RtlFreeHeap + (double free), the block address is in the verifier message/args; if it is + an AV at `call [edx]` in `~JointedMover`, use eax/ecx (the plug) — the + script already runs `!heap -p -a` on both. The full `podbreak*.dmp` in the + pod dir supports any follow-up (`cdb -y C:\VWE\RP412\Release -z `). +## Results so far + +**2026-08-13, six pods, podium ON, light page heap, pods PARKED: no crash.** +All six placed on the stand (`WinnersCircle: 6 placed on 8 spots`, the fatal +night's own line), own vehicle given an exterior, full `Application::Shutdown` +teardown, six clean exits, no exception on any pod. The target path ran and +survived. What that run did NOT have: any driving - scores came out 999/1000 +across the field, so nothing collided, took damage, died or respawned. + +**2026-08-13, SIX pods, FULL page heap: INFRASTRUCTURE FAILURE, not a +result.** All six confirmed `flags 0x3`, meshed and ACKed, then every pod +died during LoadingMission (~7 s in) with no exception, no `Fail`, no +`rpl4-fail.log` - an allocation failure taking the process out silently. +Six 32-bit processes each carrying full page heap AND a 2400x1350 cockpit +canvas do not fit. **Do not record this as a negative.** The ceiling is +somewhere between 2 (works) and 6 (dies); 4 is the next thing to try. + +Note on why a size filter does NOT rescue this: filtering page heap to +segment-sized blocks would isolate the SEGMENT on its own page, so a +neighbour could no longer reach it - the corruption would silently stop +happening instead of trapping. To trap an overrun you must guard the +CULPRIT, whose size is unknown. Unfiltered full page heap is the instrument +precisely because it guards everything: it traps an overrunning write at the +guard page AND a write through a stale pointer into a decommitted block. +Hence: reduce the field, do not filter the heap. + +**2026-08-13, TWO pods, FULL page heap (gflags +hpa, `flags 0x3`), driven: +no crash.** The important half of this result is that a 32-bit rpl4opt +SURVIVES full page heap - it booted, meshed, raced 45 s, placed 2 on the +stand and tore down clean, no address-space failure. So full PH is a usable +instrument here and the six-pod run is affordable. As a crash test it is +weak on its own: two pods, and the fatal night was six. + +**2026-08-13, same but DRIVEN (crashlap.txt): no crash.** Scores spread +827..1259, so the pods really did drive and score. Podium, exterior and +teardown all ran on all six; every pod exited clean. NOTE: deaths and +respawns remain UNVERIFIED - the game logs no collision, damage or death +line, so "they drove hard" is all the evidence supports. An input script +cannot press buttons (only the four analog channels), so it cannot fire a +weapon or pop a chute; if deaths turn out to matter, they need a hazardous +map or a different mechanism, not a fiercer script. + +### What the two clean runs change + +Light page heap traps a **double free** at the second free. It does NOT trap +a **buffer overrun** at the moment of the write - it only notices at the +overrun block's own free, via the fill-pattern check. The dump evidence fits +BOTH stories, and the second is arguably the better fit: + +- the segment's vtable dword held a small float (9.75 / 12.80 / 7.52), which + is what you get either from a freed block reused by float data OR from a + neighbour writing floats past its end; +- but `numItems` was still 15 with every TableEntry intact, so nothing was + ever unhooked - no `delete` ever ran against that segment. A proper free + would have unhooked it. An overrun explains that with no free at all. + +If it IS an overrun, these runs could not have caught it, and running the +same configuration again will not either. **Escalating to FULL page heap +(guard page immediately after every block, so the overrunning WRITE faults +with the culprit's stack) is the discriminating test.** That needs real +gflags and an elevated shell - see below. + +Escalate in this order, cheapest first, and LOG which ones ran - a silent cap +reads as "covered everything": + +1. **Driving, damage, deaths and respawns** (`setup.ps1 -Drive 1`, now the + default; installs crashlap.txt as RP412INPUTSCRIPT). This is the biggest + difference between the sterile run and the fatal night, and it is the one + the original diagnosis flagged. Damage is also topically close to the bug: + a destroyed segment swaps its video object (DestroyedGraphicState), and + segments are exactly what gets double-freed. +2. **Repeated launches** - one teardown per process (see above), so loop the + whole rig N times rather than raising -Races. The fatal night crashed on + pods that had done one race, so a single teardown CAN do it; repetition + just buys more samples of an intermittent event. +3. **Real network** - loopback gives ~0 ms and near-simultaneous teardown on + every pod. The fatal night was three machines on a LAN, where the podium + and its teardown land at genuinely different times per pod. If 1 and 2 + come up dry, this is the remaining structural difference and needs a + second machine. + +## Full page heap (the overrun trap) + +The PEB trick gives LIGHT page heap, which cannot catch an overrun at the +write. Real full page heap needs gflags and an ELEVATED shell. Run the rig +with the no-PEB cdb script so only ONE mechanism is in play: + +gflags is NOT on PATH - it ships beside cdb, and you want the **x86** copy +(same bitness as the game and as the x86 cdb): + + $gflags = 'C:\Program Files (x86)\Windows Kits\10\Debuggers\x86\gflags.exe' + + & $gflags -i rpl4opt.exe +hpa # ELEVATED. full page heap + powershell ... setup.ps1 -PodCount 2 + powershell ... feeder.ps1 -PodCount 2 -Races 1 -CdbScript cdbrun-gflags.txt + & $gflags -i rpl4opt.exe -hpa # ALWAYS turn it back off + +`-i` takes the image NAME, never a path. The setting lives in HKLM +(Image File Execution Options), so an unelevated shell fails with an access +error rather than doing nothing quietly. `& $gflags -i rpl4opt.exe` with no +flag prints the current setting - a quick way to confirm it stuck. + +If `+hpa` does not give full page heap, the explicit form is: + + & $gflags /p /enable rpl4opt.exe /full + +Start at `-PodCount 2` for the FIRST full-PH attempt - not because two pods +are likelier to crash (they are not; the fatal night was six) but to find +out whether the process survives full page heap at all before spending a +six-pod run on it. Scale to 6 once it boots and races. + +**Confirm the mode actually changed.** The feeder logs each pod's page-heap +line verbatim, e.g. `Page heap: pid 0x97B4: page heap enabled with flags +0x2`. The PEB trick produces `0x2`; gflags full page heap should produce a +DIFFERENT value. If it still says `0x2`, the gflags setting did not take +(wrong image name, not elevated, or 32/64-bit gflags mismatch - use the x86 +gflags from the same Debuggers folder as the x86 cdb) and the run is just +the light-page-heap test again. + +Full page heap gives every allocation its own page with a guard page right +after it, so a write one byte past the end faults immediately, in the +culprit's own stack - which is exactly the evidence the dumps cannot give. + +**Address-space warning:** this is a 32-bit process. Full page heap costs at +least two pages per live allocation, and a game with many small objects can +exhaust the 2 GB user address space and die in allocation rather than in the +bug. Two mitigations, in order of preference: + +1. Filter to the block size of interest, so only candidate blocks are + guarded and the rest use the normal heap. EntitySegment is a few hundred + bytes; a window either side of that is a good first cut. Check the exact + flag spelling with `gflags /p /?` before relying on it - the size-range + options live under `gflags /p /enable /full /size `. +2. Drop to `-PodCount 2` while using unfiltered full page heap, and accept + the smaller field. + +If the game dies during startup or loading with an out-of-memory or +allocation failure under +hpa, that is the address space, not the bug. +Narrow the size filter or reduce the pod count and try again - and note it +in the run log so a memory failure is never mistaken for a negative result. + +## When the first free is identified + +- Fix OWNERSHIP; do not ship a guard that skips DeletePlugs — it would leak + and mask the real first free. +- Re-run this rig: all races must survive WITH page heap still on. +- Then: commit (narrative style + Co-Authored-By), `stamp-version.ps1`, + rebuild, `pack-dist.ps1 -Zip`. + +## Leads from static reading (context, NOT conclusions) + +- The dead plug is the ROOT segment (index 0) of the LOCAL pod, and the local + pod is exactly the one that gets a SECOND renderable build at the podium + (`ShowViewpointFromOutside` -> `MakeEntityRenderables(..., outsideEntity)`), + and `videoRenderer->Shutdown()` runs just before the fatal delete. +- The renderable build path (RPL4VID.cpp `MakeEntityRenderables`) only READS + segments; segments are built once in the JointedMover ctor. No second + segment build happens at the podium (confirmed earlier). +- `EntitySegment` is a Plug in two sockets: the mover's `segmentTable` and the + parent segment's `childPointerTable` — but both teardown paths looked + self-consistent on paper. The float scribble suggests the reuser allocates + scale/quat/matrix-sized float data (renderables are full of those). +- Whatever freed the segment did NOT unhook its links (numItems stayed 15). + Look for frees that bypass `delete` on the plug: pool teardown + (`TableEntryOf::operator delete` deletes the SHARED per-V MemoryBlock + pool when its global allocationCount hits zero), or a wholesale `delete` of + something that owns segment-adjacent memory. diff --git a/tools/podium-repro/cdbrun-gflags.txt b/tools/podium-repro/cdbrun-gflags.txt new file mode 100644 index 0000000..6846f5a --- /dev/null +++ b/tools/podium-repro/cdbrun-gflags.txt @@ -0,0 +1,29 @@ +$$ Variant of cdbrun.txt for use WITH `gflags -i rpl4opt.exe +hpa`. +$$ +$$ No PEB write here: gflags has already put page heap settings in the +$$ registry (IFEO), and ntdll applies them during process init. Writing +$$ NtGlobalFlag on top would only muddy which mode is actually in force - +$$ and the whole point of this variant is to read that off the log +$$ cleanly. Watch the "page heap enabled with flags 0x..." line: it should +$$ differ from the 0x2 the PEB trick produces. +g +.echo === HEAP CHECK: compare the "page heap enabled with flags" value above === +g +.echo === POD BREAK === +.symfix+ C:\Users\cyd\AppData\Local\Temp\claude\sym +.reload +r +.exr -1 +kb 30 +.echo === HEAP FORENSICS eax === +!ext.heap -p -a @eax +.echo === HEAP FORENSICS ecx === +!ext.heap -p -a @ecx +.echo === HEAP FORENSICS edx === +!ext.heap -p -a @edx +.echo === HEAP FORENSICS edi (overrun culprit is often the base) === +!ext.heap -p -a @edi +.dump /ma /u podbreak.dmp +.echo === DUMP WRITTEN === +~*kb 8 +q diff --git a/tools/podium-repro/cdbrun.txt b/tools/podium-repro/cdbrun.txt new file mode 100644 index 0000000..2afe5e8 --- /dev/null +++ b/tools/podium-repro/cdbrun.txt @@ -0,0 +1,20 @@ +ed $peb+68 02001000 +g +.echo === HEAP CHECK: look for "page heap enabled" ModLoad verifier.dll above === +g +.echo === POD BREAK === +.symfix+ C:\Users\cyd\AppData\Local\Temp\claude\sym +.reload +r +.exr -1 +kb 30 +.echo === HEAP FORENSICS eax === +!ext.heap -p -a @eax +.echo === HEAP FORENSICS ecx === +!ext.heap -p -a @ecx +.echo === HEAP FORENSICS edx === +!ext.heap -p -a @edx +.dump /ma /u podbreak.dmp +.echo === DUMP WRITTEN === +~*kb 8 +q diff --git a/tools/podium-repro/crashlap.txt b/tools/podium-repro/crashlap.txt new file mode 100644 index 0000000..90456ea --- /dev/null +++ b/tools/podium-repro/crashlap.txt @@ -0,0 +1,29 @@ +# RP412INPUTSCRIPT - "crashlap": drive hard and hit things. +# +# The sterile six-pod run (pods parked on their pads for 45 s) reached the +# podium with 6 placed and tore down clean under page heap - no crash. The +# fatal playtest differed in that its pods were DRIVEN: they collided, took +# damage, died and respawned. Damage matters to this bug's neighbourhood - +# a destroyed segment swaps its video object (DestroyedGraphicState), which +# is the segment machinery the crash lives in - and death/respawn runs +# VTV::Reset mid-race. +# +# So: full throttle into a hard turn, held, so the pod leaves the track and +# keeps ramming terrain for the whole race. Pedals yaw it further off line. +# Blunt on purpose - the goal is contact and damage, not a clean lap. +# +# Times are SIMULATION seconds from the green light; each row holds until +# the next. +# +# t throttle stickX stickY pedals +0.0 0.0 0 0 0 +1.0 1.0 0 0 0 +4.0 1.0 0.85 0 0.6 +9.0 1.0 -0.85 0 -0.6 +14.0 1.0 0.9 0.4 0.8 +19.0 1.0 -0.9 -0.4 -0.8 +24.0 1.0 0.85 0 0.6 +29.0 1.0 -0.85 0 -0.6 +34.0 1.0 0.9 0.4 0.8 +39.0 1.0 -0.9 -0.4 -0.8 +44.0 1.0 0.85 0 0.6 diff --git a/tools/podium-repro/feeder.ps1 b/tools/podium-repro/feeder.ps1 new file mode 100644 index 0000000..d7b615d --- /dev/null +++ b/tools/podium-repro/feeder.ps1 @@ -0,0 +1,361 @@ +# Podium-teardown crash repro: N pods under cdb+page heap, R races in one +# process pair-or-field, each race driven race->buzzer->podium->teardown by +# the console protocol. Derived from tools\two-pod-test.ps1. +# +# Everything here is bot-driven: no human pilots. Pods sit on their spawn +# pads unless RP412INPUTSCRIPT drives them; the race still runs to the +# buzzer, players are still ranked, and the podium still places them - which +# is the path under test. -PodCount raises the field toward the six that +# died on 2026-08-11 (podium has eight spots; eight is the ceiling). +# +# Run setup.ps1 with the SAME -PodCount first. +param( + [int]$Races = 5, + [ValidateRange(2, 8)] + [int]$PodCount = 2, + [string]$Scratch = "$env:TEMP\rp412-podium-repro", + [string]$MungaNetDll = 'C:\VWE\TeslaSuite\Console\lib\Munga Net.dll', + [string]$ReferenceEgg = 'C:\VWE\RP412\assets\RP411\TEST.EGG', + # seconds of racing before the buzzer + [int]$RaceSeconds = 45, + # cdbrun.txt enables LIGHT page heap itself via the PEB. + # cdbrun-gflags.txt does not - use it when `gflags -i rpl4opt.exe +hpa` + # has already been set (full page heap), so only one mechanism is in play. + [string]$CdbScript = 'cdbrun.txt' +) +$ErrorActionPreference = 'Stop' +Add-Type -Path $MungaNetDll + +# Keep the feeder's own narrative on disk. Without this the state machine's +# view of the run lives only in the console that launched it, which is where +# a race-2 hang went unexplained the first time. +try { Start-Transcript -Path "$Scratch\feeder.log" -Force | Out-Null } catch {} + +function Log([string]$m) { + $line = "{0:HH:mm:ss.f} $m" -f (Get-Date) + Write-Output $line +} + +#----------------------------------------------------------------- +# The field. Pod i: dir podA+i, console port 1501+100i, game port +# 1502+100i - the numbering two-pod-test established. +#----------------------------------------------------------------- +$vehicles = @('speck','roach','flea','bug','puck','vole','wasp','grunt') +# ONLY names in RPL4GAUG.cpp's colorLookUp (matched on the first 3 chars): +# Aqua Black Blue Green Pink Purple Red White Yellow. An unlisted name makes +# determineEntityColor fall through to 255, and the GPS gauge's own-pod blip +# adds 0xC0 to it -> translationTable[447] on a 256-entry table -> an +# out-of-bounds read that page heap turns into a hard AV. ("Orange" cost one +# six-pod run learning this.) +$colors = @('Red','Blue','Green','Yellow','Purple','Pink','White','Black') +$keys = @() +$podInfo = @{} +for ($i = 0; $i -lt $PodCount; $i++) { + $k = [string][char](65 + $i) # A, B, C... + $keys += $k + $podInfo[$k] = @{ + dir = "$Scratch\pod$k" + consolePort = 1501 + (100 * $i) + gamePort = 1502 + (100 * $i) + name = "POD$k" + vehicle = $vehicles[$i] + color = $colors[$i] + cdbLog = "$Scratch\pod$k-cdb.log" + } +} +Log "field: $PodCount pods ($($keys -join ',')), $Races races, $RaceSeconds s each" + +#----------------------------------------------------------------- +# The egg (RPMission.ToEggString layout; blank plasma name bitmaps, +# ordinals lifted verbatim from TEST.EGG) +#----------------------------------------------------------------- +function Add-BlankBitmap([Text.StringBuilder]$sb, [int]$w, [int]$h) { + $row = 'bitmap=' + ('0' * ($w / 4)) + for ($i = 0; $i -lt $h; $i++) { [void]$sb.AppendLine($row) } + [void]$sb.AppendLine("x=$w"); [void]$sb.AppendLine("y=$h") +} +$test = [IO.File]::ReadAllText($ReferenceEgg) +$ordinals = $test.Substring($test.IndexOf('[ordinals]')) + +$sb = New-Object Text.StringBuilder +[void]$sb.AppendLine('[mission]') +[void]$sb.AppendLine('adventure=Red Planet') +[void]$sb.AppendLine('map=wise') +[void]$sb.AppendLine('scenario=race') +[void]$sb.AppendLine('time=day') +[void]$sb.AppendLine('weather=clear') +[void]$sb.AppendLine('temperature=0') +[void]$sb.AppendLine('compression=0') +[void]$sb.AppendLine("length=$RaceSeconds") +[void]$sb.AppendLine('[pilots]') +foreach ($k in $keys) { [void]$sb.AppendLine("pilot=127.0.0.1:$($podInfo[$k].gamePort)") } +$index = 1 +foreach ($k in $keys) { + $p = $podInfo[$k] + [void]$sb.AppendLine("[127.0.0.1:$($p.gamePort)]") + [void]$sb.AppendLine('hostType=0') + # All on 'one': DropZone falls back to a random free pad when the named + # one is taken, so a whole field can share a nominal zone without wedging. + [void]$sb.AppendLine('dropzone=one') + [void]$sb.AppendLine("name=$($p.name)") + [void]$sb.AppendLine("bitmapindex=$index") + [void]$sb.AppendLine('loadzones=1') + [void]$sb.AppendLine("vehicle=$($p.vehicle)") + [void]$sb.AppendLine("color=$($p.color)") + [void]$sb.AppendLine('badge=None') + $index++ +} +[void]$sb.AppendLine('[largebitmap]') +foreach ($k in $keys) { [void]$sb.AppendLine("bitmap=BitMap::Large::$($podInfo[$k].name)") } +[void]$sb.AppendLine('[smallbitmap]') +foreach ($k in $keys) { [void]$sb.AppendLine("bitmap=BitMap::Small::$($podInfo[$k].name)") } +foreach ($k in $keys) { + $n = $podInfo[$k].name + [void]$sb.AppendLine("[BitMap::Large::$n]") + Add-BlankBitmap $sb 128 32 + [void]$sb.AppendLine('width=8') + [void]$sb.AppendLine("[BitMap::Small::$n]") + Add-BlankBitmap $sb 64 16 + [void]$sb.AppendLine('width=4') +} +[void]$sb.Append($ordinals) +$eggText = $sb.ToString() +[IO.File]::WriteAllText("$Scratch\field.egg", $eggText) + +# wire form: newlines -> NUL, chunked into EggFileMessages +$wire = $eggText.Replace("`r`n", "`0").Replace("`n", "`0") +$bytes = [Text.Encoding]::ASCII.GetBytes($wire) +$chunks = @() +for ($off = 0; $off -lt $bytes.Length; $off += 1000) { + $n = [Math]::Min(1000, $bytes.Length - $off) + $buf = New-Object byte[] 1000 + [Buffer]::BlockCopy($bytes, $off, $buf, 0, $n) + $chunks += New-Object Munga.Net.EggFileMessage([int]($off / 1000), $bytes.Length, $n, $buf) +} +Log "egg built: $($eggText.Length) chars, $($chunks.Count) chunks" + +#----------------------------------------------------------------- +# Launch every pod under cdb (page heap via the PEB trick in cdbrun.txt) +#----------------------------------------------------------------- +Get-Process rpl4opt -ErrorAction SilentlyContinue | Stop-Process -Force -Confirm:$false -ErrorAction SilentlyContinue +Get-Process cdb -ErrorAction SilentlyContinue | Stop-Process -Force -Confirm:$false -ErrorAction SilentlyContinue +Start-Sleep -Seconds 1 +foreach ($k in $keys) { + Remove-Item "$($podInfo[$k].dir)\rpl4.log", "$($podInfo[$k].dir)\podbreak*.dmp", $podInfo[$k].cdbLog -Force -Confirm:$false -ErrorAction SilentlyContinue +} + +foreach ($k in $keys) { + $p = $podInfo[$k] + if (-not (Test-Path "$($p.dir)\rpl4opt.exe")) { + Log "pod ${k}: $($p.dir) not set up - run setup.ps1 -PodCount $PodCount first" + exit 1 + } + Start-Process -FilePath cmd.exe -ArgumentList '/c', "$Scratch\runpod.cmd", $p.dir, $p.cdbLog, "$Scratch\$CdbScript", "$($p.consolePort)" -WindowStyle Hidden | Out-Null + Start-Sleep -Seconds 3 +} +Log "$PodCount pods launched under cdb" +Start-Sleep -Seconds 12 + +# The game PID sits behind cmd -> cdb, but page heap announces it in the cdb +# log ("Page heap: pid 0x97B4: ..."), which is the only place the pod and its +# process are tied together. Needed to retire survivors selectively later. +foreach ($k in $keys) { + $podInfo[$k].pid = $null + $hit = Select-String -Path $podInfo[$k].cdbLog -Pattern 'Page heap: pid 0x([0-9a-fA-F]+)' -ErrorAction SilentlyContinue | + Select-Object -First 1 + if ($null -ne $hit) { + $podInfo[$k].pid = [Convert]::ToInt32($hit.Matches[0].Groups[1].Value, 16) + # Log the line VERBATIM - its "flags 0x..." value is how you tell light + # page heap (the PEB trick) from full (gflags +hpa). Do not paraphrase it. + Log "pod ${k}: game pid $($podInfo[$k].pid) | $($hit.Line.Trim())" + } else { + Log "pod ${k}: WARNING - no page-heap line in cdb log; is page heap on?" + } +} + +#----------------------------------------------------------------- +# Break detection: cdb writes the marker the instant it traps, well +# before a race would time out. +#----------------------------------------------------------------- +function Find-Break { + # Match on ExceptionAddress, NOT on the "=== POD BREAK ===" banner. The + # cdb script's `g` returns on process EXIT as well as on a fault, so the + # banner prints for a perfectly clean shutdown too - which had this rig + # reporting six "traps" that were all just the pods quitting normally. + # `.exr -1` prints ExceptionAddress only when there was a real exception. + foreach ($k in $keys) { + $log = $podInfo[$k].cdbLog + if (Test-Path $log) { + if (Select-String -Path $log -Pattern 'ExceptionAddress:' -Quiet -ErrorAction SilentlyContinue) { + return $k + } + } + } + return $null +} +function PodsAlive { + return @(Get-Process rpl4opt -ErrorAction SilentlyContinue).Count -ge $PodCount +} + +#----------------------------------------------------------------- +# Console feeder +#----------------------------------------------------------------- +$pods = @{} +foreach ($k in $keys) { + $sock = New-Object Munga.Net.MungaSocket + $connected = $false + for ($try = 0; $try -lt 20 -and -not $connected; $try++) { + try { $sock.Connect([Net.IPAddress]::Loopback, [uint16]$podInfo[$k].consolePort); $connected = $true } + catch { Start-Sleep -Seconds 2 } + } + if (-not $connected) { Log "pod ${k}: console connect FAILED"; exit 1 } + $pods[$k] = @{ sock = $sock; state = $null } + Log "console connected to pod $k (port $($podInfo[$k].consolePort))" +} + +$stateQuery = New-Object Munga.Net.StateQueryMessage(1) +$raceResults = @() +$brokePod = $null + +for ($race = 1; $race -le $Races -and $brokePod -eq $null; $race++) { + Log "=== RACE $race of $Races ===" + foreach ($k in $keys) { + $pods[$k].ack = $false; $pods[$k].eggSent = [DateTime]::MinValue + $pods[$k].lastQuery = [DateTime]::MinValue + $pods[$k].score = $null; $pods[$k].runSent = $false; $pods[$k].stopSent = $false + } + $runStart = $null + $raceDeadline = (Get-Date).AddMinutes(6) + $raceOK = $false + + while ((Get-Date) -lt $raceDeadline) { + $brokePod = Find-Break + if ($brokePod -ne $null) { + Log "*** POD $brokePod TRAPPED IN THE DEBUGGER ***" + # Retire the survivors. They are of no forensic value once one pod has + # trapped, and leaving them racing with no buzzer ever sent strands + # them mid-mission with the race clock running up. + foreach ($k in $keys) { + if ($k -ne $brokePod) { + try { $pods[$k].sock.Send(0, 1, (New-Object Munga.Net.StopMissionMessage(0))) } catch {} + } + } + Start-Sleep -Seconds 5 + foreach ($k in $keys) { + if ($k -ne $brokePod) { + Get-Process -Id $podInfo[$k].pid -ErrorAction SilentlyContinue | + Stop-Process -Force -Confirm:$false -ErrorAction SilentlyContinue + } + } + Log "survivors retired; pod $brokePod left frozen for post-mortem" + break + } + if (-not (PodsAlive)) { Log 'A POD PROCESS IS GONE (died without trapping)'; break } + + foreach ($k in $keys) { + $pod = $pods[$k] + $now = Get-Date + if (($now - $pod.lastQuery).TotalSeconds -ge 1) { + try { $pod.sock.Send(0, 1, $stateQuery) } catch { Log "pod ${k}: send failed ($_)" } + $pod.lastQuery = $now + } + for ($m = $pod.sock.Receive(); $m -ne $null; $m = $pod.sock.Receive()) { + $msg = $m.Message + if ($msg -eq $null) { continue } + switch ($msg.GetType().Name) { + 'StateResponseMessage' { + if ("$($pod.state)" -ne "$($msg.ApplicationState)") { + Log "pod ${k}: state -> $($msg.ApplicationState)" + } + $pod.state = $msg.ApplicationState + } + 'AcknowledgeEggFileMessage' { + if (-not $pod.ack) { Log "pod ${k}: EGG ACK (mesh complete)" } + $pod.ack = $true + } + 'EndMissionMessage' { + $pod.score = $msg.FinalScore + Log "pod ${k}: FINAL SCORE host=$($msg.PlayerHostID) score=$($msg.FinalScore)" + } + default {} + } + } + if ("$($pod.state)" -eq 'WaitingForEgg' -and -not $pod.ack -and + ($now - $pod.eggSent).TotalSeconds -ge 6) { + Log "pod ${k}: sending egg" + foreach ($chunk in $chunks) { $pod.sock.Send(0, 1, $chunk) } + $pod.eggSent = $now + } + } + + $allWaiting = $true + foreach ($k in $keys) { if ("$($pods[$k].state)" -ne 'WaitingForLaunch') { $allWaiting = $false } } + if ($allWaiting -and -not $pods[$keys[0]].runSent) { + Log 'all pods WaitingForLaunch: RunMission' + foreach ($k in $keys) { + $pods[$k].sock.Send(0, 1, (New-Object Munga.Net.RunMissionMessage)) + $pods[$k].runSent = $true + } + } + + $allRunning = $true + foreach ($k in $keys) { if ("$($pods[$k].state)" -ne 'RunningMission') { $allRunning = $false } } + if ($allRunning -and $runStart -eq $null) { + $runStart = Get-Date + Log "all pods RUNNING - $RaceSeconds s to the buzzer" + } + + if ($runStart -ne $null -and -not $pods[$keys[0]].stopSent -and + ((Get-Date) - $runStart).TotalSeconds -ge $RaceSeconds) { + Log 'buzzer: StopMission(0) -> all pods' + foreach ($k in $keys) { + $pods[$k].sock.Send(0, 1, (New-Object Munga.Net.StopMissionMessage(0))) + $pods[$k].stopSent = $true + } + } + + # Race complete = every pod came through podium+teardown and is asking + # for the next egg, with every process still alive. + if ($pods[$keys[0]].stopSent) { + $allBack = $true + foreach ($k in $keys) { if ("$($pods[$k].state)" -ne 'WaitingForEgg') { $allBack = $false } } + if ($allBack -and (PodsAlive)) { $raceOK = $true; break } + } + Start-Sleep -Milliseconds 250 + } + + $scores = @() + foreach ($k in $keys) { $scores += "$k=$($pods[$k].score)" } + $raceResults += [pscustomobject]@{ race = $race; ok = $raceOK; scores = ($scores -join ' ') } + if (-not $raceOK) { + Log "RACE $race DID NOT COMPLETE - stopping loop (see cdb logs)" + break + } + Log "race $race survived podium teardown ($($scores -join ' '))" +} + +Log '=== SUMMARY ===' +foreach ($r in $raceResults) { Log "race $($r.race): ok=$($r.ok) $($r.scores)" } +$dumpPaths = @() +foreach ($k in $keys) { $dumpPaths += "$($podInfo[$k].dir)\podbreak*.dmp" } +$dumps = Get-ChildItem $dumpPaths -ErrorAction SilentlyContinue +foreach ($d in $dumps) { Log "BREAK DUMP: $($d.FullName) ($([Math]::Round($d.Length/1MB)) MB)" } +if ($dumps -eq $null -or @($dumps).Count -eq 0) { Log 'no break dumps written' } +$anyFailed = $false +foreach ($r in $raceResults) { if (-not $r.ok) { $anyFailed = $true } } +if ($null -ne $brokePod) { + Log "FIRST FREE EVIDENCE: see $($podInfo[$brokePod].cdbLog) after '=== POD BREAK ==='" + Log 'processes left up for post-mortem' +} elseif ($anyFailed) { + # A race that never came back without anyone trapping is a HARNESS problem + # (mesh, egg, state machine), not a crash. Leave it standing so it can be + # looked at - killing it here is what hid the race-2 hang the first time. + Log 'a race did not complete and nothing trapped - pods LEFT UP for inspection' + Log "check each pod's rpl4.log tail and the feeder states above" +} else { + foreach ($k in $keys) { try { $pods[$k].sock.Shutdown() } catch {} } + Get-Process rpl4opt -ErrorAction SilentlyContinue | Stop-Process -Force -Confirm:$false -ErrorAction SilentlyContinue + Log 'clean shutdown' +} +Log 'FEEDER DONE' +try { Stop-Transcript | Out-Null } catch {} diff --git a/tools/podium-repro/runpod.cmd b/tools/podium-repro/runpod.cmd new file mode 100644 index 0000000..18f0946 --- /dev/null +++ b/tools/podium-repro/runpod.cmd @@ -0,0 +1,9 @@ +@echo off +rem runpod.cmd +rem Runs one pod under cdb with page heap enabled via the PEB trick in the +rem cdb script (no gflags/elevation needed). Clears the Git-Bash-inherited +rem NoDefaultCurrentDirectoryInExePath, which otherwise makes CreateProcess +rem refuse to find rpl4opt.exe in the working directory. +cd /d "%~1" +set NoDefaultCurrentDirectoryInExePath= +"C:\Program Files (x86)\Windows Kits\10\Debuggers\x86\cdb.exe" -y C:\VWE\RP412\Release -logo "%~2" -xe cpr "%~1\rpl4opt.exe" -windowed -res 800 600 -net %~4 < "%~3" diff --git a/tools/podium-repro/setup.ps1 b/tools/podium-repro/setup.ps1 new file mode 100644 index 0000000..88f5b5e --- /dev/null +++ b/tools/podium-repro/setup.ps1 @@ -0,0 +1,70 @@ +# Builds the sandboxed pod installs for the podium-teardown repro. +# Assets are junctioned from dist, the exe comes from Release (build first). +# Use the SAME -PodCount you intend to pass to feeder.ps1. +param( + [ValidateRange(2, 8)] + [int]$PodCount = 2, + [string]$Scratch = "$env:TEMP\rp412-podium-repro", + [string]$Dist = 'C:\VWE\RP412\dist', + [string]$ReleaseExe = 'C:\VWE\RP412\Release\rpl4opt.exe', + # Drive the pods into things so the race contains damage, deaths and + # respawns. Parked pods reach the podium and tear down clean; the fatal + # playtest's pods did not. -Drive 0 restores the sterile run. + [int]$Drive = 1 +) +$ErrorActionPreference = 'Stop' +if (-not (Test-Path $ReleaseExe)) { throw "build Release first: $ReleaseExe missing" } +New-Item -ItemType Directory -Force $Scratch | Out-Null +for ($i = 0; $i -lt $PodCount; $i++) { + $dir = "$Scratch\pod$([char](65 + $i))" + New-Item -ItemType Directory -Force $dir | Out-Null + foreach ($assets in 'AUDIO','GAUGE','VIDEO') { + if (-not (Test-Path "$dir\$assets")) { + New-Item -ItemType Junction -Path "$dir\$assets" -Target "$Dist\$assets" | Out-Null + } + } + foreach ($file in 'environ.ini','JOYSTICK.INI','RPDPL.INI','RPL4.RES','libsndfile-1.dll','OpenAL32.dll') { + Copy-Item "$Dist\$file" $dir -Force + } + Copy-Item $ReleaseExe $dir -Force + + # + # dist ships RP412PODIUM=0 - the podium is OFF by default, and a rig that + # copies environ.ini verbatim tests nothing (every pod just logs + # "WinnersCircle: disabled" and goes straight to the results). Force it on. + # + # Appended, not edited: environ.ini takes the LAST value for a duplicate + # key. Written as ASCII with no BOM - a BOM in this file has invalidated + # test runs before. + # + $override = "`r`n# --- podium-repro overrides (setup.ps1) ---`r`nRP412PODIUM=1`r`n" + if ($Drive -ne 0) { + Copy-Item "$PSScriptRoot\crashlap.txt" $dir -Force + $override += "RP412INPUTSCRIPT=crashlap.txt`r`n" + } + [IO.File]::AppendAllText("$dir\environ.ini", $override, [Text.Encoding]::ASCII) +} +# +# Retire evidence from pods this run will not use. A -PodCount 2 run after a +# -PodCount 6 run leaves podC..podF holding the SIX-pod logs and dumps, and +# a grep across the tree then reports results that belong to a different +# experiment - which has already caused one wrong reading. +# +# Files only, never the directories: they contain junctions to the dist +# asset folders, and a recursive delete through a junction can take the +# TARGET's contents with it. +# +for ($i = $PodCount; $i -lt 8; $i++) { + $k = [char](65 + $i) + $stale = @( + "$Scratch\pod$k\rpl4.log" + "$Scratch\pod$k\podbreak*.dmp" + "$Scratch\pod$k-cdb.log" + ) + Remove-Item $stale -Force -Confirm:$false -ErrorAction SilentlyContinue +} + +foreach ($file in 'runpod.cmd','cdbrun.txt','cdbrun-gflags.txt','feeder.ps1') { + Copy-Item "$PSScriptRoot\$file" $Scratch -Force +} +Write-Output "$PodCount pod sandboxes ready under $Scratch"