From 6467a5f9305077125842d7004f8e8cb9b0954e50 Mon Sep 17 00:00:00 2001 From: Cyd Date: Thu, 13 Aug 2026 16:33:31 -0500 Subject: [PATCH] The teardown looks at the segments before it calls through them The podium crash dies in SocketIterator::DeletePlugs, calling through a segment whose vtable dword has been replaced by a small float. The three dumps prove the segment is wrong BY teardown; nothing in them says when it went wrong, and six configurations of the local repro rig - parked pods, driven pods, light and full page heap, two, four and six pods - reached the podium and tore down clean. So the next real playtest becomes the instrument. RP412SEGCHECK walks the segment table in ~JointedMover before the delete, guarded-reads each segment's first dword, and if one does not match the vtable captured from the very first segment ever built it writes the forensics into rpl4-fail.log, which is closed on the way down and survives the abort - rpl4.log does not. The report carries the entity and whether it was the local pod, which index went bad and what is in it, the first two rows of the object as hex and float, and the heap deltas to its neighbours on either side. Three bracket calls in the winners' circle answer the question the dumps cannot: at podium entry, and either side of the second MakeEntityRenderables on the own pod. Whichever fires first is recorded and travels inside the teardown report, so the log says whether the race broke the segment or the podium did. It deliberately does not skip the delete or repair the pointer. The ownership bug is unfixed and a guard would cost exactly the evidence this is here to collect - it stops on the same object, one step earlier, holding the forensics. On by default, a handful of pointer compares per pod per race; RP412SEGCHECK=0 turns it off, and the environ.ini template says so. tools/podium-repro is the rig itself, banked with what the dumps already established: page heap turned on through the PEB without gflags or elevation, N sandboxed installs, and a feeder that drives full races through them. Co-Authored-By: Claude Opus 5 (1M context) --- MUNGA/JMOVER.cpp | 323 ++++++++++++++++++++++++ MUNGA/JMOVER.h | 10 + RP_L4/RPL4APP.cpp | 20 ++ RP_L4/RPL4ENVIRON.cpp | 8 + tools/podium-repro/README.md | 313 +++++++++++++++++++++++ tools/podium-repro/cdbrun-gflags.txt | 29 +++ tools/podium-repro/cdbrun.txt | 20 ++ tools/podium-repro/crashlap.txt | 29 +++ tools/podium-repro/feeder.ps1 | 361 +++++++++++++++++++++++++++ tools/podium-repro/runpod.cmd | 9 + tools/podium-repro/setup.ps1 | 70 ++++++ 11 files changed, 1192 insertions(+) create mode 100644 tools/podium-repro/README.md create mode 100644 tools/podium-repro/cdbrun-gflags.txt create mode 100644 tools/podium-repro/cdbrun.txt create mode 100644 tools/podium-repro/crashlap.txt create mode 100644 tools/podium-repro/feeder.ps1 create mode 100644 tools/podium-repro/runpod.cmd create mode 100644 tools/podium-repro/setup.ps1 diff --git a/MUNGA/JMOVER.cpp b/MUNGA/JMOVER.cpp index ae2c912..272ba9d 100644 --- a/MUNGA/JMOVER.cpp +++ b/MUNGA/JMOVER.cpp @@ -7,10 +7,315 @@ #include "notation.h" #include "namelist.h" +// EXCEPTION_EXECUTE_HANDLER, for the guarded read in RPReadDword below. +// excpt.h rather than windows.h: this layer does not otherwise want it. +#include + //############################################################################# //############################ JointedMover ############################# //############################################################################# +//############################################################################# +// RP412SEGCHECK - segment validity at teardown +//############################################################################# +// +// The podium crash (three identical dumps, 2026-08-11) dies inside +// SocketIterator::DeletePlugs, called from ~JointedMover, calling through a +// segment whose FIRST DWORD - the vtable pointer - has been replaced by a +// small float (9.75 / 12.80 / 7.52 across the three machines). The table +// itself was untouched: numItems still 15, every TableEntry hooked up, which +// means nothing ever ran delete against that segment. Something wrote over +// it, or freed it behind the collection's back. +// +// Six configurations of the local repro rig - parked pods, driven pods, +// light page heap, full page heap, two, four and six pods - reached the +// podium and tore down clean. So this exists to make the next REAL playtest +// the instrument: look at each segment before calling through it, and if one +// is wrong, write what it looks like to rpl4-fail.log, which is fclose'd and +// survives the abort. Text written only to rpl4.log does not. +// +// This deliberately does NOT skip the delete, repair the pointer, or +// otherwise make the teardown survivable. The ownership bug is still +// unfixed, and a guard that hid it would cost exactly the evidence this is +// here to collect. It stops on the same object the crash would have died on, +// one step earlier, holding the forensics. +// +// RP412SEGCHECK=0 turns it off. +// + +// +// The vtable every EntitySegment should carry, taken from the first one ever +// built - i.e. from fresh memory, before any race has run. Comparing against +// a captured value rather than a computed one keeps this independent of how +// the linker laid the image out. +// +static void + *gGoodSegmentVtable = NULL; + +static int + RPSegCheckEnabled() +{ + static int + enabled = -1; + + if (enabled < 0) + { + const char + *setting = getenv("RP412SEGCHECK"); + + enabled = (setting != NULL && atoi(setting) == 0) ? 0 : 1; + } + return enabled; +} + +// +// Read a dword that may not be there any more. If the block has been freed +// under a page-heap build the pages are decommitted, and the plain read +// would fault inside the checker - losing the report we came for. +// +static int + RPReadDword(const void *address, unsigned long *out) +{ + __try + { + *out = *(const unsigned long *) address; + return 1; + } + __except (EXCEPTION_EXECUTE_HANDLER) + { + return 0; + } +} + +// +// WHERE the corruption was first seen. Recorded in a static rather than +// simply logged, because rpl4.log does not reliably survive the abort that +// follows at teardown - so the answer has to travel inside the Fail report. +// +// This is the question the dumps cannot answer: the crash proves a segment +// is wrong by teardown, not whether it was already wrong when the race +// ended. Bracketing the podium separates "something during the race broke +// it" from "the podium's second MakeEntityRenderables broke it". +// +static char + gFirstBadWhen[64] = ""; + +// +// Public entry point for the bracket call sites (see RPL4APP.cpp). Counts +// corrupted segments on one entity and remembers the first moment any were +// seen. Reports; never aborts - the teardown check does that, with this +// answer folded in. +// +int + RPCheckJointedMoverSegments(Entity *entity, const char *when) +{ + if (!RPSegCheckEnabled() || gGoodSegmentVtable == NULL || entity == NULL) + { + return 0; + } + if (!entity->IsDerivedFrom(*JointedMover::GetClassDerivations())) + { + return 0; + } + + JointedMover + *mover = (JointedMover *) entity; + EntitySegment::SegmentTableIterator + iterator(mover->segmentTable); + EntitySegment + *segment; + int + bad = 0; + + iterator.First(); + while ((segment = iterator.ReadAndNext()) != NULL) + { + unsigned long + vtable = 0; + + if (!RPReadDword(segment, &vtable) + || (void *) vtable != gGoodSegmentVtable) + { + ++bad; + } + } + + if (bad > 0 && gFirstBadWhen[0] == '\0') + { + strncpy(gFirstBadWhen, when, sizeof(gFirstBadWhen) - 1); + gFirstBadWhen[sizeof(gFirstBadWhen) - 1] = '\0'; + DEBUG_STREAM << "SegCheck: " << bad + << " segment(s) already corrupt at " << when << "\n" << std::flush; + } + return bad; +} + +static void + RPCheckSegmentsBeforeTeardown( + Entity *mover, + EntitySegment::SegmentTable &segment_table + ) +{ + if (!RPSegCheckEnabled() || gGoodSegmentVtable == NULL) + { + return; + } + + // + // Walk the table WITHOUT dereferencing anything: the iterator reads the + // TableEntry array, and GetPlug hands back the pointer without touching + // the object. The only read of segment memory is the guarded one below. + // + EntitySegment::SegmentTableIterator + iterator(segment_table); + EntitySegment + *segment, + *bad_segment = NULL, + *before_bad = NULL, + *after_bad = NULL; + unsigned long + bad_value = 0; + int + index = 0, + total = 0, + bad_count = 0, + bad_index = -1, + bad_readable = 0; + + iterator.First(); + while ((segment = iterator.ReadAndNext()) != NULL) + { + unsigned long + vtable = 0; + int + readable = RPReadDword(segment, &vtable); + + if (!readable || (void *) vtable != gGoodSegmentVtable) + { + ++bad_count; + if (bad_segment == NULL) + { + bad_segment = segment; + bad_index = index; + bad_value = vtable; + bad_readable = readable; + } + } + else if (bad_segment == NULL) + { + before_bad = segment; // last good one before the first bad + } + else if (after_bad == NULL) + { + after_bad = segment; // first one after it + } + ++index; + ++total; + } + + if (bad_count == 0) + { + return; + } + + // + // One buffer, one message: Fail_With_Message fprintf's the string it is + // given into rpl4-fail.log and closes the file, so everything worth + // having has to be inside it. Newlines are fine. + // + static char + report[1400]; + char + *out = report; + Entity + *viewpoint = (application != NULL) + ? application->GetViewpointEntity() + : NULL; + + out += sprintf(out, + "segment plug corrupted BEFORE DeletePlugs\n" + " first seen: %s\n" + " entity %d:%d class%d, %s pod, owner %d\n" + " segments %d, corrupted %d, first bad index %d at 0x%08lX\n", + (gFirstBadWhen[0] != '\0') ? gFirstBadWhen + : "teardown (clean at every earlier check)", + (int) mover->entityID.GetHostID(), (int) mover->entityID, + (int) mover->GetClassID(), + (mover == viewpoint) ? "VIEWPOINT (local)" : "replicant", + (int) mover->GetOwnerID(), + total, bad_count, bad_index, (unsigned long) bad_segment); + + if (!bad_readable) + { + out += sprintf(out, + " segment memory is NOT READABLE - freed and decommitted\n"); + } + else + { + float + as_float; + + memcpy(&as_float, &bad_value, sizeof(as_float)); + out += sprintf(out, + " vtable expected 0x%08lX found 0x%08lX (as float %.6g)\n", + (unsigned long) gGoodSegmentVtable, bad_value, (double) as_float); + + // + // The first 8 dwords, hex and float. The dumps showed a float where + // the vtable belongs; how FAR the float data runs says whether this + // was a stray single write or something walking through the object. + // + for (int row = 0; row < 2; ++row) + { + char + hex[80], + flt[80]; + int + hex_used = 0, + flt_used = 0; + + for (int col = 0; col < 4; ++col) + { + const unsigned char + *at = (const unsigned char *) bad_segment + + (row * 16) + (col * 4); + unsigned long + word = 0; + + if (!RPReadDword(at, &word)) + { + hex_used += sprintf(hex + hex_used, " ????????"); + flt_used += sprintf(flt + flt_used, " ????????"); + continue; + } + + float + word_float; + + memcpy(&word_float, &word, sizeof(word_float)); + hex_used += sprintf(hex + hex_used, " %08lX", word); + flt_used += sprintf(flt + flt_used, " %9.4g", (double) word_float); + } + out += sprintf(out, " +%02X:%s |%s\n", row * 16, hex, flt); + } + } + + // + // Heap adjacency: if the corruption came from a neighbour running past + // its end, the deltas say which side it came from and how far it reached. + // + out += sprintf(out, + " neighbours: prev good 0x%08lX (delta %ld), next 0x%08lX (delta %ld)\n", + (unsigned long) before_bad, + (before_bad != NULL) + ? (long) ((char *) bad_segment - (char *) before_bad) : 0L, + (unsigned long) after_bad, + (after_bad != NULL) + ? (long) ((char *) after_bad - (char *) bad_segment) : 0L); + + Fail(report); +} + //############################################################################# // Shared Data Support // @@ -307,6 +612,17 @@ JointedMover::JointedMover( primary_dzone_index ); Register_Object(current_segment); + + // + // The reference vtable for RPCheckSegmentsBeforeTeardown, taken once + // from the very first segment ever built - fresh memory, before any + // race has run. See the RP412SEGCHECK block at the top of this file. + // + if (gGoodSegmentVtable == NULL) + { + gGoodSegmentVtable = *(void **) current_segment; + } + segmentTable.AddValue(current_segment, ii); // //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -429,6 +745,13 @@ void // JointedMover::~JointedMover() { + // + // Look at the segments before calling through them. This is where the + // podium crash lands; see the RP412SEGCHECK block at the top of this + // file for what it records and why it does not try to survive. + // + RPCheckSegmentsBeforeTeardown((Entity *) this, segmentTable); + EntitySegment::SegmentTableIterator iterator(segmentTable); iterator.DeletePlugs(); // diff --git a/MUNGA/JMOVER.h b/MUNGA/JMOVER.h index 4c73cd9..a114022 100644 --- a/MUNGA/JMOVER.h +++ b/MUNGA/JMOVER.h @@ -5,6 +5,16 @@ #include "slot.h" #include "segment.h" +// +// RP412SEGCHECK - count corrupted segment plugs on one entity and remember +// the first moment any were seen, so the teardown report can say WHERE it +// went wrong rather than only that it did. Safe on any entity: returns 0 +// for anything that is not a JointedMover. Reports; never aborts. +// See the block comment at the top of JMOVER.cpp. +// +extern int + RPCheckJointedMoverSegments(Entity *entity, const char *when); + //############################################################################## //######################## JointedMover ################################## //############################################################################## diff --git a/RP_L4/RPL4APP.cpp b/RP_L4/RPL4APP.cpp index 0c76afa..b45bba8 100644 --- a/RP_L4/RPL4APP.cpp +++ b/RP_L4/RPL4APP.cpp @@ -253,8 +253,21 @@ void // Time podiumOutsideStart = Now(); + // + // Bracket the exterior build. The podium crash kills the pod at teardown + // with a segment plug whose vtable has been overwritten; these two calls + // say whether it was ALREADY wrong when the podium began, or whether this + // second MakeEntityRenderables on the own pod is what did it. The answer + // rides in the teardown's rpl4-fail.log report - see RP412SEGCHECK in + // MUNGA\JMOVER.cpp. Costs a handful of pointer compares, twice, once a + // race. + // + RPCheckJointedMoverSegments(GetViewpointEntity(), "podium: before exterior"); + dpl_renderer->ShowViewpointFromOutside(); + RPCheckJointedMoverSegments(GetViewpointEntity(), "podium: after exterior"); + // //--------------------------------------------------------------------- // The name plates are drawn per rank slot, so they have to be re-sorted @@ -500,6 +513,13 @@ void Check(this); DEBUG_STREAM << "WinnersCircle: standing the finishers up\n" << std::flush; + + // + // The earliest of the three checks: anything corrupt HERE was broken by + // the race itself, before the podium touched anything. + // + RPCheckJointedMoverSegments(GetViewpointEntity(), "podium: entry (race just ended)"); + ShowWinnersCircle(); Check_Fpu(); } diff --git a/RP_L4/RPL4ENVIRON.cpp b/RP_L4/RPL4ENVIRON.cpp index 256d682..3726ce1 100644 --- a/RP_L4/RPL4ENVIRON.cpp +++ b/RP_L4/RPL4ENVIRON.cpp @@ -416,6 +416,14 @@ namespace "# same race at every frame rate, bit for bit.\n" "#RP412PHYSTRACE=1\n" "\n" +"# Segment validity check at pod teardown, on by default. The winners'\n" +"# circle crash of 2026-08-11 died deleting a pod whose skeleton segment\n" +"# had been overwritten; this looks at the segments before deleting them\n" +"# and, if one is wrong, writes what it looks like - and where it first\n" +"# went wrong - to rpl4-fail.log, which survives the abort. It costs a\n" +"# handful of pointer compares per pod per race. 0 turns it off.\n" +"#RP412SEGCHECK=0\n" +"\n" "# Try this drop zone first at spawn instead of a random pick. The pick\n" "# is seeded by RANDOM=, but a seed only repeats a run if the same\n" "# NUMBER of draws comes before the pick, and that count rides on load\n" diff --git a/tools/podium-repro/README.md b/tools/podium-repro/README.md new file mode 100644 index 0000000..0d3fd7c --- /dev/null +++ b/tools/podium-repro/README.md @@ -0,0 +1,313 @@ +# Podium-teardown crash repro rig + +For the intermittent double-delete in `JointedMover::~JointedMover` after the +winners' circle (build 4.12.217, 2026-08-11 six-player playtest). This rig +exists to catch the FIRST free in a debugger — the diagnosis must come from +the trap, not from reading code. + +## Evidence already banked (2026-08-11 debug session) + +The three podium dumps are `playtestlogs\20260811\rpl4crash (N)1.dmp` — note +the `1` suffix; the plain `rpl4crash (N).dmp` files are the OLD, already-fixed +`L4NetworkManager::Send` NULL-host crash. Symbols: `playtestlogs\symbols-4.12.217`, +load with `.reload /f /i rpl4opt.exe`. + +All three podium dumps are byte-for-byte the same failure: + +- Crash at `SocketIterator::DeletePlugs` inlined in `~JointedMover+0x68`: + `call [edx]` where edx = the vtable dword of the plug being deleted, + overwritten by a small float (9.75 / 12.80 / 7.52 in the three dumps) — + freed memory reused by something that writes floats. +- The iterator lives on the stack, which IS in the minidumps: + `numItems=15` (a full skeleton), `currentPosition=1`, so the dead plug is + `array[0]` — segment index 0, the root segment. +- `segmentTable.socketsNode` read back == the entity pointer, and the table + header was readable/writable — the ENTITY IS INTACT. The freed thing is + the segment plug (or its TableEntry), not the pod. +- The frame above returns to `Application::Shutdown+0x71`, which is + `delete viewpointEntity` (MUNGA\APP.cpp ~1038) — the FIRST entity deleted + in Shutdown. So the dying pod is the LOCAL pod: the same entity that got + the podium's second `MakeEntityRenderables(outsideEntity)` pass + (`DPLRenderer::ShowViewpointFromOutside`). `videoRenderer->Shutdown()` + runs immediately before it. +- No entries were removed from the table (15 of 15 still present), so the + first free BYPASSED the Link/TableEntry machinery: a clean + `delete segment` would have unhooked its TableEntry and decremented + numItems. Whatever freed it did not run ~Plug against that table. + +Nothing above says WHO freed it first. That is what this rig answers. + +## What the rig is + +- `setup.ps1` — builds N sandboxed installs under `%TEMP%\rp412-podium-repro` + (junctions to `dist\{AUDIO,GAUGE,VIDEO}`, INIs/RES/DLLs copied from dist, + `rpl4opt.exe` copied from `Release\` — build Release first). +- `runpod.cmd` — one pod under cdb. Clears `NoDefaultCurrentDirectoryInExePath` + (Git-Bash exports it; CreateProcess then refuses CWD-relative exe names) and + keeps the game's CWD = the pod dir (a wrong CWD produces a phantom audio-init + crash — see project discipline notes). +- `cdbrun.txt` — cdb stdin script. At the create-process event it writes + `ed $peb+68 02001000` (FLG_HEAP_PAGE_ALLOCS | FLG_USER_STACK_TRACE_DB), which + turns on PAGE HEAP WITHOUT gflags/elevation — verified working on this + machine: cdb log shows `verifier.dll` loading and + `Page heap: pid X: page heap enabled with flags 0x2` (light page heap + + stack traces: fills freed blocks, validates on every heap op, records + alloc/free stacks). A double free traps at the SECOND free; the freed-fill + pattern makes the teardown's vtable call fault deterministically instead of + only when the heap happened to reuse the block. On any break it prints + registers, `.exr -1`, `kb 30`, `!heap -p -a` on eax/ecx/edx, writes a FULL + `podbreak*.dmp` in the pod dir, then quits. +- `feeder.ps1` — drives `-Races` (default 5) full races through ONE set of + processes over the console protocol: egg -> mesh -> RunMission -> 45 s -> + StopMission(0) (the buzzer) -> podium -> ending fade -> + `Application::Shutdown` teardown -> back to WaitingForEgg -> next race. + `-PodCount` (2..8) sets the size of the field. It watches the cdb logs for + the break marker every tick, so a trap is caught in seconds rather than at + the race timeout, and leaves everything up for post-mortem. Derived from + `tools\two-pod-test.ps1` (proven harness). + +**No human pilots are needed at any point.** Bots fill the field: pods sit on +their pads unless `RP412INPUTSCRIPT` drives them, and the race still runs to +the buzzer, still ranks players, and still places them on the stand — which is +the path under test. "More players" means more scripted instances on this one +machine, not more people. + +## Run it + + powershell -NoProfile -ExecutionPolicy Bypass -File tools\podium-repro\setup.ps1 + powershell -NoProfile -ExecutionPolicy Bypass -File tools\podium-repro\feeder.ps1 + +800x600 game windows appear, one per pod; ~2 min per race. Watch for +`FEEDER DONE`. To reproduce the population that actually died (six placed), +run the six-pod field — pass the same count to both scripts: + + ... -File tools\podium-repro\setup.ps1 -PodCount 6 + ... -File tools\podium-repro\feeder.ps1 -PodCount 6 -Races 5 + +Six 32-bit instances with light page heap fit comfortably on this machine; +they will contend for CPU, which perturbs teardown timing — no bad thing for +an order-dependent bug. + +Status when this rig was assembled: single-pod boot under page heap to +WaitingForEgg is VERIFIED (verifier.dll loads, "page heap enabled with flags +0x2" in the cdb log, game reaches the front end). The race loop itself has +NOT been exercised — the session that built this lost the ability to run +anything before it could. The feeder is a close copy of the proven +two-pod-test, but treat its first run as a shakedown: if the pods never reach +`WaitingForLaunch`, the mesh/egg path is the thing to debug, not the crash. + +## Gotchas this rig has already paid for + +- **`dist\environ.ini` ships `RP412PODIUM=0`.** The podium is OFF by default, + so a sandbox that copies environ.ini verbatim tests NOTHING - every pod + logs `WinnersCircle: disabled` and goes straight to the results. setup.ps1 + now appends `RP412PODIUM=1` (appended, not edited: last duplicate key wins; + written ASCII/no-BOM, because a BOM here has invalidated runs before). + **Check for `WinnersCircle: N placed on 8 spots` in a pod's rpl4.log - if + it says "disabled", the run proved nothing.** +- **One mission per process is ALL you get, by design.** RESOLVED, not a + rig fault: in APPMGR.cpp RunMissions, a finished application has + `Application::Shutdown` called on it; Shutdown returns False, so the app is + removed from runningApplications, and with none left RunMissions returns + and the process exits. The teardown IS the exit. So `-Races 1` is the only + honest setting for a console-driven pod, and repeated teardowns means + repeated LAUNCHES, not repeated races. (This is why "race 2" never + started: nothing was there to race.) +- **A clean exit is not a trap.** cdb's `g` returns on process exit as well + as on a fault, so the "=== POD BREAK ===" banner prints for a normal quit. + Detection now keys on `ExceptionAddress:` from `.exr -1`, which only + appears for a real exception. Six "traps" were once six clean shutdowns. + +- **Pilot colours must be in `colorLookUp`** (RPL4GAUG.cpp, matched on the + first three chars): Aqua Black Blue Green Pink Purple Red White Yellow. + Anything else → `determineEntityColor` returns its 255 fallback → the GPS + gauge's own-pod blip adds 0xC0 → `translationTable[447]` on a 256-entry + table → out-of-bounds read, which page heap turns into a hard AV at mission + start. It only bites the pod that OWNS the odd colour (own blip flashes, + others don't), so it looks like one machine at random. A six-pod run with + "Orange" died exactly this way. +- **`!heap` is `!ext.heap`** in current debuggers — exts.dll forwards and + prints a notice INSTEAD of running, silently costing the heap forensics. +- **Retire the survivors on a trap.** When one pod breaks, the feeder stops + driving; without an explicit StopMission the other pods race on forever + with the clock counting up. The feeder now buzzes then kills them, keeping + only the trapped pod frozen. + +## Reading the result + +- Crash repro'd: `%TEMP%\rp412-podium-repro\pod?-cdb.log` has everything after + `=== POD BREAK ===`. The money shot is `!heap -p -a ` — with + page heap + stack DB the output includes the block's ALLOC stack and its + FREE stack. The free stack that is NOT `~JointedMover` IS the first free — + that is the bug. If the break is a verifier stop inside RtlFreeHeap + (double free), the block address is in the verifier message/args; if it is + an AV at `call [edx]` in `~JointedMover`, use eax/ecx (the plug) — the + script already runs `!heap -p -a` on both. The full `podbreak*.dmp` in the + pod dir supports any follow-up (`cdb -y C:\VWE\RP412\Release -z `). +## Results so far + +**2026-08-13, six pods, podium ON, light page heap, pods PARKED: no crash.** +All six placed on the stand (`WinnersCircle: 6 placed on 8 spots`, the fatal +night's own line), own vehicle given an exterior, full `Application::Shutdown` +teardown, six clean exits, no exception on any pod. The target path ran and +survived. What that run did NOT have: any driving - scores came out 999/1000 +across the field, so nothing collided, took damage, died or respawned. + +**2026-08-13, SIX pods, FULL page heap: INFRASTRUCTURE FAILURE, not a +result.** All six confirmed `flags 0x3`, meshed and ACKed, then every pod +died during LoadingMission (~7 s in) with no exception, no `Fail`, no +`rpl4-fail.log` - an allocation failure taking the process out silently. +Six 32-bit processes each carrying full page heap AND a 2400x1350 cockpit +canvas do not fit. **Do not record this as a negative.** The ceiling is +somewhere between 2 (works) and 6 (dies); 4 is the next thing to try. + +Note on why a size filter does NOT rescue this: filtering page heap to +segment-sized blocks would isolate the SEGMENT on its own page, so a +neighbour could no longer reach it - the corruption would silently stop +happening instead of trapping. To trap an overrun you must guard the +CULPRIT, whose size is unknown. Unfiltered full page heap is the instrument +precisely because it guards everything: it traps an overrunning write at the +guard page AND a write through a stale pointer into a decommitted block. +Hence: reduce the field, do not filter the heap. + +**2026-08-13, TWO pods, FULL page heap (gflags +hpa, `flags 0x3`), driven: +no crash.** The important half of this result is that a 32-bit rpl4opt +SURVIVES full page heap - it booted, meshed, raced 45 s, placed 2 on the +stand and tore down clean, no address-space failure. So full PH is a usable +instrument here and the six-pod run is affordable. As a crash test it is +weak on its own: two pods, and the fatal night was six. + +**2026-08-13, same but DRIVEN (crashlap.txt): no crash.** Scores spread +827..1259, so the pods really did drive and score. Podium, exterior and +teardown all ran on all six; every pod exited clean. NOTE: deaths and +respawns remain UNVERIFIED - the game logs no collision, damage or death +line, so "they drove hard" is all the evidence supports. An input script +cannot press buttons (only the four analog channels), so it cannot fire a +weapon or pop a chute; if deaths turn out to matter, they need a hazardous +map or a different mechanism, not a fiercer script. + +### What the two clean runs change + +Light page heap traps a **double free** at the second free. It does NOT trap +a **buffer overrun** at the moment of the write - it only notices at the +overrun block's own free, via the fill-pattern check. The dump evidence fits +BOTH stories, and the second is arguably the better fit: + +- the segment's vtable dword held a small float (9.75 / 12.80 / 7.52), which + is what you get either from a freed block reused by float data OR from a + neighbour writing floats past its end; +- but `numItems` was still 15 with every TableEntry intact, so nothing was + ever unhooked - no `delete` ever ran against that segment. A proper free + would have unhooked it. An overrun explains that with no free at all. + +If it IS an overrun, these runs could not have caught it, and running the +same configuration again will not either. **Escalating to FULL page heap +(guard page immediately after every block, so the overrunning WRITE faults +with the culprit's stack) is the discriminating test.** That needs real +gflags and an elevated shell - see below. + +Escalate in this order, cheapest first, and LOG which ones ran - a silent cap +reads as "covered everything": + +1. **Driving, damage, deaths and respawns** (`setup.ps1 -Drive 1`, now the + default; installs crashlap.txt as RP412INPUTSCRIPT). This is the biggest + difference between the sterile run and the fatal night, and it is the one + the original diagnosis flagged. Damage is also topically close to the bug: + a destroyed segment swaps its video object (DestroyedGraphicState), and + segments are exactly what gets double-freed. +2. **Repeated launches** - one teardown per process (see above), so loop the + whole rig N times rather than raising -Races. The fatal night crashed on + pods that had done one race, so a single teardown CAN do it; repetition + just buys more samples of an intermittent event. +3. **Real network** - loopback gives ~0 ms and near-simultaneous teardown on + every pod. The fatal night was three machines on a LAN, where the podium + and its teardown land at genuinely different times per pod. If 1 and 2 + come up dry, this is the remaining structural difference and needs a + second machine. + +## Full page heap (the overrun trap) + +The PEB trick gives LIGHT page heap, which cannot catch an overrun at the +write. Real full page heap needs gflags and an ELEVATED shell. Run the rig +with the no-PEB cdb script so only ONE mechanism is in play: + +gflags is NOT on PATH - it ships beside cdb, and you want the **x86** copy +(same bitness as the game and as the x86 cdb): + + $gflags = 'C:\Program Files (x86)\Windows Kits\10\Debuggers\x86\gflags.exe' + + & $gflags -i rpl4opt.exe +hpa # ELEVATED. full page heap + powershell ... setup.ps1 -PodCount 2 + powershell ... feeder.ps1 -PodCount 2 -Races 1 -CdbScript cdbrun-gflags.txt + & $gflags -i rpl4opt.exe -hpa # ALWAYS turn it back off + +`-i` takes the image NAME, never a path. The setting lives in HKLM +(Image File Execution Options), so an unelevated shell fails with an access +error rather than doing nothing quietly. `& $gflags -i rpl4opt.exe` with no +flag prints the current setting - a quick way to confirm it stuck. + +If `+hpa` does not give full page heap, the explicit form is: + + & $gflags /p /enable rpl4opt.exe /full + +Start at `-PodCount 2` for the FIRST full-PH attempt - not because two pods +are likelier to crash (they are not; the fatal night was six) but to find +out whether the process survives full page heap at all before spending a +six-pod run on it. Scale to 6 once it boots and races. + +**Confirm the mode actually changed.** The feeder logs each pod's page-heap +line verbatim, e.g. `Page heap: pid 0x97B4: page heap enabled with flags +0x2`. The PEB trick produces `0x2`; gflags full page heap should produce a +DIFFERENT value. If it still says `0x2`, the gflags setting did not take +(wrong image name, not elevated, or 32/64-bit gflags mismatch - use the x86 +gflags from the same Debuggers folder as the x86 cdb) and the run is just +the light-page-heap test again. + +Full page heap gives every allocation its own page with a guard page right +after it, so a write one byte past the end faults immediately, in the +culprit's own stack - which is exactly the evidence the dumps cannot give. + +**Address-space warning:** this is a 32-bit process. Full page heap costs at +least two pages per live allocation, and a game with many small objects can +exhaust the 2 GB user address space and die in allocation rather than in the +bug. Two mitigations, in order of preference: + +1. Filter to the block size of interest, so only candidate blocks are + guarded and the rest use the normal heap. EntitySegment is a few hundred + bytes; a window either side of that is a good first cut. Check the exact + flag spelling with `gflags /p /?` before relying on it - the size-range + options live under `gflags /p /enable /full /size `. +2. Drop to `-PodCount 2` while using unfiltered full page heap, and accept + the smaller field. + +If the game dies during startup or loading with an out-of-memory or +allocation failure under +hpa, that is the address space, not the bug. +Narrow the size filter or reduce the pod count and try again - and note it +in the run log so a memory failure is never mistaken for a negative result. + +## When the first free is identified + +- Fix OWNERSHIP; do not ship a guard that skips DeletePlugs — it would leak + and mask the real first free. +- Re-run this rig: all races must survive WITH page heap still on. +- Then: commit (narrative style + Co-Authored-By), `stamp-version.ps1`, + rebuild, `pack-dist.ps1 -Zip`. + +## Leads from static reading (context, NOT conclusions) + +- The dead plug is the ROOT segment (index 0) of the LOCAL pod, and the local + pod is exactly the one that gets a SECOND renderable build at the podium + (`ShowViewpointFromOutside` -> `MakeEntityRenderables(..., outsideEntity)`), + and `videoRenderer->Shutdown()` runs just before the fatal delete. +- The renderable build path (RPL4VID.cpp `MakeEntityRenderables`) only READS + segments; segments are built once in the JointedMover ctor. No second + segment build happens at the podium (confirmed earlier). +- `EntitySegment` is a Plug in two sockets: the mover's `segmentTable` and the + parent segment's `childPointerTable` — but both teardown paths looked + self-consistent on paper. The float scribble suggests the reuser allocates + scale/quat/matrix-sized float data (renderables are full of those). +- Whatever freed the segment did NOT unhook its links (numItems stayed 15). + Look for frees that bypass `delete` on the plug: pool teardown + (`TableEntryOf::operator delete` deletes the SHARED per-V MemoryBlock + pool when its global allocationCount hits zero), or a wholesale `delete` of + something that owns segment-adjacent memory. diff --git a/tools/podium-repro/cdbrun-gflags.txt b/tools/podium-repro/cdbrun-gflags.txt new file mode 100644 index 0000000..6846f5a --- /dev/null +++ b/tools/podium-repro/cdbrun-gflags.txt @@ -0,0 +1,29 @@ +$$ Variant of cdbrun.txt for use WITH `gflags -i rpl4opt.exe +hpa`. +$$ +$$ No PEB write here: gflags has already put page heap settings in the +$$ registry (IFEO), and ntdll applies them during process init. Writing +$$ NtGlobalFlag on top would only muddy which mode is actually in force - +$$ and the whole point of this variant is to read that off the log +$$ cleanly. Watch the "page heap enabled with flags 0x..." line: it should +$$ differ from the 0x2 the PEB trick produces. +g +.echo === HEAP CHECK: compare the "page heap enabled with flags" value above === +g +.echo === POD BREAK === +.symfix+ C:\Users\cyd\AppData\Local\Temp\claude\sym +.reload +r +.exr -1 +kb 30 +.echo === HEAP FORENSICS eax === +!ext.heap -p -a @eax +.echo === HEAP FORENSICS ecx === +!ext.heap -p -a @ecx +.echo === HEAP FORENSICS edx === +!ext.heap -p -a @edx +.echo === HEAP FORENSICS edi (overrun culprit is often the base) === +!ext.heap -p -a @edi +.dump /ma /u podbreak.dmp +.echo === DUMP WRITTEN === +~*kb 8 +q diff --git a/tools/podium-repro/cdbrun.txt b/tools/podium-repro/cdbrun.txt new file mode 100644 index 0000000..2afe5e8 --- /dev/null +++ b/tools/podium-repro/cdbrun.txt @@ -0,0 +1,20 @@ +ed $peb+68 02001000 +g +.echo === HEAP CHECK: look for "page heap enabled" ModLoad verifier.dll above === +g +.echo === POD BREAK === +.symfix+ C:\Users\cyd\AppData\Local\Temp\claude\sym +.reload +r +.exr -1 +kb 30 +.echo === HEAP FORENSICS eax === +!ext.heap -p -a @eax +.echo === HEAP FORENSICS ecx === +!ext.heap -p -a @ecx +.echo === HEAP FORENSICS edx === +!ext.heap -p -a @edx +.dump /ma /u podbreak.dmp +.echo === DUMP WRITTEN === +~*kb 8 +q diff --git a/tools/podium-repro/crashlap.txt b/tools/podium-repro/crashlap.txt new file mode 100644 index 0000000..90456ea --- /dev/null +++ b/tools/podium-repro/crashlap.txt @@ -0,0 +1,29 @@ +# RP412INPUTSCRIPT - "crashlap": drive hard and hit things. +# +# The sterile six-pod run (pods parked on their pads for 45 s) reached the +# podium with 6 placed and tore down clean under page heap - no crash. The +# fatal playtest differed in that its pods were DRIVEN: they collided, took +# damage, died and respawned. Damage matters to this bug's neighbourhood - +# a destroyed segment swaps its video object (DestroyedGraphicState), which +# is the segment machinery the crash lives in - and death/respawn runs +# VTV::Reset mid-race. +# +# So: full throttle into a hard turn, held, so the pod leaves the track and +# keeps ramming terrain for the whole race. Pedals yaw it further off line. +# Blunt on purpose - the goal is contact and damage, not a clean lap. +# +# Times are SIMULATION seconds from the green light; each row holds until +# the next. +# +# t throttle stickX stickY pedals +0.0 0.0 0 0 0 +1.0 1.0 0 0 0 +4.0 1.0 0.85 0 0.6 +9.0 1.0 -0.85 0 -0.6 +14.0 1.0 0.9 0.4 0.8 +19.0 1.0 -0.9 -0.4 -0.8 +24.0 1.0 0.85 0 0.6 +29.0 1.0 -0.85 0 -0.6 +34.0 1.0 0.9 0.4 0.8 +39.0 1.0 -0.9 -0.4 -0.8 +44.0 1.0 0.85 0 0.6 diff --git a/tools/podium-repro/feeder.ps1 b/tools/podium-repro/feeder.ps1 new file mode 100644 index 0000000..d7b615d --- /dev/null +++ b/tools/podium-repro/feeder.ps1 @@ -0,0 +1,361 @@ +# Podium-teardown crash repro: N pods under cdb+page heap, R races in one +# process pair-or-field, each race driven race->buzzer->podium->teardown by +# the console protocol. Derived from tools\two-pod-test.ps1. +# +# Everything here is bot-driven: no human pilots. Pods sit on their spawn +# pads unless RP412INPUTSCRIPT drives them; the race still runs to the +# buzzer, players are still ranked, and the podium still places them - which +# is the path under test. -PodCount raises the field toward the six that +# died on 2026-08-11 (podium has eight spots; eight is the ceiling). +# +# Run setup.ps1 with the SAME -PodCount first. +param( + [int]$Races = 5, + [ValidateRange(2, 8)] + [int]$PodCount = 2, + [string]$Scratch = "$env:TEMP\rp412-podium-repro", + [string]$MungaNetDll = 'C:\VWE\TeslaSuite\Console\lib\Munga Net.dll', + [string]$ReferenceEgg = 'C:\VWE\RP412\assets\RP411\TEST.EGG', + # seconds of racing before the buzzer + [int]$RaceSeconds = 45, + # cdbrun.txt enables LIGHT page heap itself via the PEB. + # cdbrun-gflags.txt does not - use it when `gflags -i rpl4opt.exe +hpa` + # has already been set (full page heap), so only one mechanism is in play. + [string]$CdbScript = 'cdbrun.txt' +) +$ErrorActionPreference = 'Stop' +Add-Type -Path $MungaNetDll + +# Keep the feeder's own narrative on disk. Without this the state machine's +# view of the run lives only in the console that launched it, which is where +# a race-2 hang went unexplained the first time. +try { Start-Transcript -Path "$Scratch\feeder.log" -Force | Out-Null } catch {} + +function Log([string]$m) { + $line = "{0:HH:mm:ss.f} $m" -f (Get-Date) + Write-Output $line +} + +#----------------------------------------------------------------- +# The field. Pod i: dir podA+i, console port 1501+100i, game port +# 1502+100i - the numbering two-pod-test established. +#----------------------------------------------------------------- +$vehicles = @('speck','roach','flea','bug','puck','vole','wasp','grunt') +# ONLY names in RPL4GAUG.cpp's colorLookUp (matched on the first 3 chars): +# Aqua Black Blue Green Pink Purple Red White Yellow. An unlisted name makes +# determineEntityColor fall through to 255, and the GPS gauge's own-pod blip +# adds 0xC0 to it -> translationTable[447] on a 256-entry table -> an +# out-of-bounds read that page heap turns into a hard AV. ("Orange" cost one +# six-pod run learning this.) +$colors = @('Red','Blue','Green','Yellow','Purple','Pink','White','Black') +$keys = @() +$podInfo = @{} +for ($i = 0; $i -lt $PodCount; $i++) { + $k = [string][char](65 + $i) # A, B, C... + $keys += $k + $podInfo[$k] = @{ + dir = "$Scratch\pod$k" + consolePort = 1501 + (100 * $i) + gamePort = 1502 + (100 * $i) + name = "POD$k" + vehicle = $vehicles[$i] + color = $colors[$i] + cdbLog = "$Scratch\pod$k-cdb.log" + } +} +Log "field: $PodCount pods ($($keys -join ',')), $Races races, $RaceSeconds s each" + +#----------------------------------------------------------------- +# The egg (RPMission.ToEggString layout; blank plasma name bitmaps, +# ordinals lifted verbatim from TEST.EGG) +#----------------------------------------------------------------- +function Add-BlankBitmap([Text.StringBuilder]$sb, [int]$w, [int]$h) { + $row = 'bitmap=' + ('0' * ($w / 4)) + for ($i = 0; $i -lt $h; $i++) { [void]$sb.AppendLine($row) } + [void]$sb.AppendLine("x=$w"); [void]$sb.AppendLine("y=$h") +} +$test = [IO.File]::ReadAllText($ReferenceEgg) +$ordinals = $test.Substring($test.IndexOf('[ordinals]')) + +$sb = New-Object Text.StringBuilder +[void]$sb.AppendLine('[mission]') +[void]$sb.AppendLine('adventure=Red Planet') +[void]$sb.AppendLine('map=wise') +[void]$sb.AppendLine('scenario=race') +[void]$sb.AppendLine('time=day') +[void]$sb.AppendLine('weather=clear') +[void]$sb.AppendLine('temperature=0') +[void]$sb.AppendLine('compression=0') +[void]$sb.AppendLine("length=$RaceSeconds") +[void]$sb.AppendLine('[pilots]') +foreach ($k in $keys) { [void]$sb.AppendLine("pilot=127.0.0.1:$($podInfo[$k].gamePort)") } +$index = 1 +foreach ($k in $keys) { + $p = $podInfo[$k] + [void]$sb.AppendLine("[127.0.0.1:$($p.gamePort)]") + [void]$sb.AppendLine('hostType=0') + # All on 'one': DropZone falls back to a random free pad when the named + # one is taken, so a whole field can share a nominal zone without wedging. + [void]$sb.AppendLine('dropzone=one') + [void]$sb.AppendLine("name=$($p.name)") + [void]$sb.AppendLine("bitmapindex=$index") + [void]$sb.AppendLine('loadzones=1') + [void]$sb.AppendLine("vehicle=$($p.vehicle)") + [void]$sb.AppendLine("color=$($p.color)") + [void]$sb.AppendLine('badge=None') + $index++ +} +[void]$sb.AppendLine('[largebitmap]') +foreach ($k in $keys) { [void]$sb.AppendLine("bitmap=BitMap::Large::$($podInfo[$k].name)") } +[void]$sb.AppendLine('[smallbitmap]') +foreach ($k in $keys) { [void]$sb.AppendLine("bitmap=BitMap::Small::$($podInfo[$k].name)") } +foreach ($k in $keys) { + $n = $podInfo[$k].name + [void]$sb.AppendLine("[BitMap::Large::$n]") + Add-BlankBitmap $sb 128 32 + [void]$sb.AppendLine('width=8') + [void]$sb.AppendLine("[BitMap::Small::$n]") + Add-BlankBitmap $sb 64 16 + [void]$sb.AppendLine('width=4') +} +[void]$sb.Append($ordinals) +$eggText = $sb.ToString() +[IO.File]::WriteAllText("$Scratch\field.egg", $eggText) + +# wire form: newlines -> NUL, chunked into EggFileMessages +$wire = $eggText.Replace("`r`n", "`0").Replace("`n", "`0") +$bytes = [Text.Encoding]::ASCII.GetBytes($wire) +$chunks = @() +for ($off = 0; $off -lt $bytes.Length; $off += 1000) { + $n = [Math]::Min(1000, $bytes.Length - $off) + $buf = New-Object byte[] 1000 + [Buffer]::BlockCopy($bytes, $off, $buf, 0, $n) + $chunks += New-Object Munga.Net.EggFileMessage([int]($off / 1000), $bytes.Length, $n, $buf) +} +Log "egg built: $($eggText.Length) chars, $($chunks.Count) chunks" + +#----------------------------------------------------------------- +# Launch every pod under cdb (page heap via the PEB trick in cdbrun.txt) +#----------------------------------------------------------------- +Get-Process rpl4opt -ErrorAction SilentlyContinue | Stop-Process -Force -Confirm:$false -ErrorAction SilentlyContinue +Get-Process cdb -ErrorAction SilentlyContinue | Stop-Process -Force -Confirm:$false -ErrorAction SilentlyContinue +Start-Sleep -Seconds 1 +foreach ($k in $keys) { + Remove-Item "$($podInfo[$k].dir)\rpl4.log", "$($podInfo[$k].dir)\podbreak*.dmp", $podInfo[$k].cdbLog -Force -Confirm:$false -ErrorAction SilentlyContinue +} + +foreach ($k in $keys) { + $p = $podInfo[$k] + if (-not (Test-Path "$($p.dir)\rpl4opt.exe")) { + Log "pod ${k}: $($p.dir) not set up - run setup.ps1 -PodCount $PodCount first" + exit 1 + } + Start-Process -FilePath cmd.exe -ArgumentList '/c', "$Scratch\runpod.cmd", $p.dir, $p.cdbLog, "$Scratch\$CdbScript", "$($p.consolePort)" -WindowStyle Hidden | Out-Null + Start-Sleep -Seconds 3 +} +Log "$PodCount pods launched under cdb" +Start-Sleep -Seconds 12 + +# The game PID sits behind cmd -> cdb, but page heap announces it in the cdb +# log ("Page heap: pid 0x97B4: ..."), which is the only place the pod and its +# process are tied together. Needed to retire survivors selectively later. +foreach ($k in $keys) { + $podInfo[$k].pid = $null + $hit = Select-String -Path $podInfo[$k].cdbLog -Pattern 'Page heap: pid 0x([0-9a-fA-F]+)' -ErrorAction SilentlyContinue | + Select-Object -First 1 + if ($null -ne $hit) { + $podInfo[$k].pid = [Convert]::ToInt32($hit.Matches[0].Groups[1].Value, 16) + # Log the line VERBATIM - its "flags 0x..." value is how you tell light + # page heap (the PEB trick) from full (gflags +hpa). Do not paraphrase it. + Log "pod ${k}: game pid $($podInfo[$k].pid) | $($hit.Line.Trim())" + } else { + Log "pod ${k}: WARNING - no page-heap line in cdb log; is page heap on?" + } +} + +#----------------------------------------------------------------- +# Break detection: cdb writes the marker the instant it traps, well +# before a race would time out. +#----------------------------------------------------------------- +function Find-Break { + # Match on ExceptionAddress, NOT on the "=== POD BREAK ===" banner. The + # cdb script's `g` returns on process EXIT as well as on a fault, so the + # banner prints for a perfectly clean shutdown too - which had this rig + # reporting six "traps" that were all just the pods quitting normally. + # `.exr -1` prints ExceptionAddress only when there was a real exception. + foreach ($k in $keys) { + $log = $podInfo[$k].cdbLog + if (Test-Path $log) { + if (Select-String -Path $log -Pattern 'ExceptionAddress:' -Quiet -ErrorAction SilentlyContinue) { + return $k + } + } + } + return $null +} +function PodsAlive { + return @(Get-Process rpl4opt -ErrorAction SilentlyContinue).Count -ge $PodCount +} + +#----------------------------------------------------------------- +# Console feeder +#----------------------------------------------------------------- +$pods = @{} +foreach ($k in $keys) { + $sock = New-Object Munga.Net.MungaSocket + $connected = $false + for ($try = 0; $try -lt 20 -and -not $connected; $try++) { + try { $sock.Connect([Net.IPAddress]::Loopback, [uint16]$podInfo[$k].consolePort); $connected = $true } + catch { Start-Sleep -Seconds 2 } + } + if (-not $connected) { Log "pod ${k}: console connect FAILED"; exit 1 } + $pods[$k] = @{ sock = $sock; state = $null } + Log "console connected to pod $k (port $($podInfo[$k].consolePort))" +} + +$stateQuery = New-Object Munga.Net.StateQueryMessage(1) +$raceResults = @() +$brokePod = $null + +for ($race = 1; $race -le $Races -and $brokePod -eq $null; $race++) { + Log "=== RACE $race of $Races ===" + foreach ($k in $keys) { + $pods[$k].ack = $false; $pods[$k].eggSent = [DateTime]::MinValue + $pods[$k].lastQuery = [DateTime]::MinValue + $pods[$k].score = $null; $pods[$k].runSent = $false; $pods[$k].stopSent = $false + } + $runStart = $null + $raceDeadline = (Get-Date).AddMinutes(6) + $raceOK = $false + + while ((Get-Date) -lt $raceDeadline) { + $brokePod = Find-Break + if ($brokePod -ne $null) { + Log "*** POD $brokePod TRAPPED IN THE DEBUGGER ***" + # Retire the survivors. They are of no forensic value once one pod has + # trapped, and leaving them racing with no buzzer ever sent strands + # them mid-mission with the race clock running up. + foreach ($k in $keys) { + if ($k -ne $brokePod) { + try { $pods[$k].sock.Send(0, 1, (New-Object Munga.Net.StopMissionMessage(0))) } catch {} + } + } + Start-Sleep -Seconds 5 + foreach ($k in $keys) { + if ($k -ne $brokePod) { + Get-Process -Id $podInfo[$k].pid -ErrorAction SilentlyContinue | + Stop-Process -Force -Confirm:$false -ErrorAction SilentlyContinue + } + } + Log "survivors retired; pod $brokePod left frozen for post-mortem" + break + } + if (-not (PodsAlive)) { Log 'A POD PROCESS IS GONE (died without trapping)'; break } + + foreach ($k in $keys) { + $pod = $pods[$k] + $now = Get-Date + if (($now - $pod.lastQuery).TotalSeconds -ge 1) { + try { $pod.sock.Send(0, 1, $stateQuery) } catch { Log "pod ${k}: send failed ($_)" } + $pod.lastQuery = $now + } + for ($m = $pod.sock.Receive(); $m -ne $null; $m = $pod.sock.Receive()) { + $msg = $m.Message + if ($msg -eq $null) { continue } + switch ($msg.GetType().Name) { + 'StateResponseMessage' { + if ("$($pod.state)" -ne "$($msg.ApplicationState)") { + Log "pod ${k}: state -> $($msg.ApplicationState)" + } + $pod.state = $msg.ApplicationState + } + 'AcknowledgeEggFileMessage' { + if (-not $pod.ack) { Log "pod ${k}: EGG ACK (mesh complete)" } + $pod.ack = $true + } + 'EndMissionMessage' { + $pod.score = $msg.FinalScore + Log "pod ${k}: FINAL SCORE host=$($msg.PlayerHostID) score=$($msg.FinalScore)" + } + default {} + } + } + if ("$($pod.state)" -eq 'WaitingForEgg' -and -not $pod.ack -and + ($now - $pod.eggSent).TotalSeconds -ge 6) { + Log "pod ${k}: sending egg" + foreach ($chunk in $chunks) { $pod.sock.Send(0, 1, $chunk) } + $pod.eggSent = $now + } + } + + $allWaiting = $true + foreach ($k in $keys) { if ("$($pods[$k].state)" -ne 'WaitingForLaunch') { $allWaiting = $false } } + if ($allWaiting -and -not $pods[$keys[0]].runSent) { + Log 'all pods WaitingForLaunch: RunMission' + foreach ($k in $keys) { + $pods[$k].sock.Send(0, 1, (New-Object Munga.Net.RunMissionMessage)) + $pods[$k].runSent = $true + } + } + + $allRunning = $true + foreach ($k in $keys) { if ("$($pods[$k].state)" -ne 'RunningMission') { $allRunning = $false } } + if ($allRunning -and $runStart -eq $null) { + $runStart = Get-Date + Log "all pods RUNNING - $RaceSeconds s to the buzzer" + } + + if ($runStart -ne $null -and -not $pods[$keys[0]].stopSent -and + ((Get-Date) - $runStart).TotalSeconds -ge $RaceSeconds) { + Log 'buzzer: StopMission(0) -> all pods' + foreach ($k in $keys) { + $pods[$k].sock.Send(0, 1, (New-Object Munga.Net.StopMissionMessage(0))) + $pods[$k].stopSent = $true + } + } + + # Race complete = every pod came through podium+teardown and is asking + # for the next egg, with every process still alive. + if ($pods[$keys[0]].stopSent) { + $allBack = $true + foreach ($k in $keys) { if ("$($pods[$k].state)" -ne 'WaitingForEgg') { $allBack = $false } } + if ($allBack -and (PodsAlive)) { $raceOK = $true; break } + } + Start-Sleep -Milliseconds 250 + } + + $scores = @() + foreach ($k in $keys) { $scores += "$k=$($pods[$k].score)" } + $raceResults += [pscustomobject]@{ race = $race; ok = $raceOK; scores = ($scores -join ' ') } + if (-not $raceOK) { + Log "RACE $race DID NOT COMPLETE - stopping loop (see cdb logs)" + break + } + Log "race $race survived podium teardown ($($scores -join ' '))" +} + +Log '=== SUMMARY ===' +foreach ($r in $raceResults) { Log "race $($r.race): ok=$($r.ok) $($r.scores)" } +$dumpPaths = @() +foreach ($k in $keys) { $dumpPaths += "$($podInfo[$k].dir)\podbreak*.dmp" } +$dumps = Get-ChildItem $dumpPaths -ErrorAction SilentlyContinue +foreach ($d in $dumps) { Log "BREAK DUMP: $($d.FullName) ($([Math]::Round($d.Length/1MB)) MB)" } +if ($dumps -eq $null -or @($dumps).Count -eq 0) { Log 'no break dumps written' } +$anyFailed = $false +foreach ($r in $raceResults) { if (-not $r.ok) { $anyFailed = $true } } +if ($null -ne $brokePod) { + Log "FIRST FREE EVIDENCE: see $($podInfo[$brokePod].cdbLog) after '=== POD BREAK ==='" + Log 'processes left up for post-mortem' +} elseif ($anyFailed) { + # A race that never came back without anyone trapping is a HARNESS problem + # (mesh, egg, state machine), not a crash. Leave it standing so it can be + # looked at - killing it here is what hid the race-2 hang the first time. + Log 'a race did not complete and nothing trapped - pods LEFT UP for inspection' + Log "check each pod's rpl4.log tail and the feeder states above" +} else { + foreach ($k in $keys) { try { $pods[$k].sock.Shutdown() } catch {} } + Get-Process rpl4opt -ErrorAction SilentlyContinue | Stop-Process -Force -Confirm:$false -ErrorAction SilentlyContinue + Log 'clean shutdown' +} +Log 'FEEDER DONE' +try { Stop-Transcript | Out-Null } catch {} diff --git a/tools/podium-repro/runpod.cmd b/tools/podium-repro/runpod.cmd new file mode 100644 index 0000000..18f0946 --- /dev/null +++ b/tools/podium-repro/runpod.cmd @@ -0,0 +1,9 @@ +@echo off +rem runpod.cmd +rem Runs one pod under cdb with page heap enabled via the PEB trick in the +rem cdb script (no gflags/elevation needed). Clears the Git-Bash-inherited +rem NoDefaultCurrentDirectoryInExePath, which otherwise makes CreateProcess +rem refuse to find rpl4opt.exe in the working directory. +cd /d "%~1" +set NoDefaultCurrentDirectoryInExePath= +"C:\Program Files (x86)\Windows Kits\10\Debuggers\x86\cdb.exe" -y C:\VWE\RP412\Release -logo "%~2" -xe cpr "%~1\rpl4opt.exe" -windowed -res 800 600 -net %~4 < "%~3" diff --git a/tools/podium-repro/setup.ps1 b/tools/podium-repro/setup.ps1 new file mode 100644 index 0000000..88f5b5e --- /dev/null +++ b/tools/podium-repro/setup.ps1 @@ -0,0 +1,70 @@ +# Builds the sandboxed pod installs for the podium-teardown repro. +# Assets are junctioned from dist, the exe comes from Release (build first). +# Use the SAME -PodCount you intend to pass to feeder.ps1. +param( + [ValidateRange(2, 8)] + [int]$PodCount = 2, + [string]$Scratch = "$env:TEMP\rp412-podium-repro", + [string]$Dist = 'C:\VWE\RP412\dist', + [string]$ReleaseExe = 'C:\VWE\RP412\Release\rpl4opt.exe', + # Drive the pods into things so the race contains damage, deaths and + # respawns. Parked pods reach the podium and tear down clean; the fatal + # playtest's pods did not. -Drive 0 restores the sterile run. + [int]$Drive = 1 +) +$ErrorActionPreference = 'Stop' +if (-not (Test-Path $ReleaseExe)) { throw "build Release first: $ReleaseExe missing" } +New-Item -ItemType Directory -Force $Scratch | Out-Null +for ($i = 0; $i -lt $PodCount; $i++) { + $dir = "$Scratch\pod$([char](65 + $i))" + New-Item -ItemType Directory -Force $dir | Out-Null + foreach ($assets in 'AUDIO','GAUGE','VIDEO') { + if (-not (Test-Path "$dir\$assets")) { + New-Item -ItemType Junction -Path "$dir\$assets" -Target "$Dist\$assets" | Out-Null + } + } + foreach ($file in 'environ.ini','JOYSTICK.INI','RPDPL.INI','RPL4.RES','libsndfile-1.dll','OpenAL32.dll') { + Copy-Item "$Dist\$file" $dir -Force + } + Copy-Item $ReleaseExe $dir -Force + + # + # dist ships RP412PODIUM=0 - the podium is OFF by default, and a rig that + # copies environ.ini verbatim tests nothing (every pod just logs + # "WinnersCircle: disabled" and goes straight to the results). Force it on. + # + # Appended, not edited: environ.ini takes the LAST value for a duplicate + # key. Written as ASCII with no BOM - a BOM in this file has invalidated + # test runs before. + # + $override = "`r`n# --- podium-repro overrides (setup.ps1) ---`r`nRP412PODIUM=1`r`n" + if ($Drive -ne 0) { + Copy-Item "$PSScriptRoot\crashlap.txt" $dir -Force + $override += "RP412INPUTSCRIPT=crashlap.txt`r`n" + } + [IO.File]::AppendAllText("$dir\environ.ini", $override, [Text.Encoding]::ASCII) +} +# +# Retire evidence from pods this run will not use. A -PodCount 2 run after a +# -PodCount 6 run leaves podC..podF holding the SIX-pod logs and dumps, and +# a grep across the tree then reports results that belong to a different +# experiment - which has already caused one wrong reading. +# +# Files only, never the directories: they contain junctions to the dist +# asset folders, and a recursive delete through a junction can take the +# TARGET's contents with it. +# +for ($i = $PodCount; $i -lt 8; $i++) { + $k = [char](65 + $i) + $stale = @( + "$Scratch\pod$k\rpl4.log" + "$Scratch\pod$k\podbreak*.dmp" + "$Scratch\pod$k-cdb.log" + ) + Remove-Item $stale -Force -Confirm:$false -ErrorAction SilentlyContinue +} + +foreach ($file in 'runpod.cmd','cdbrun.txt','cdbrun-gflags.txt','feeder.ps1') { + Copy-Item "$PSScriptRoot\$file" $Scratch -Force +} +Write-Output "$PodCount pod sandboxes ready under $Scratch"