RP412/tools/podium-repro/feeder.ps1

# Podium-teardown crash repro: N pods under cdb+page heap, R races in one
# process pair-or-field, each race driven race->buzzer->podium->teardown by
# the console protocol. Derived from tools\two-pod-test.ps1.
#
# Everything here is bot-driven: no human pilots. Pods sit on their spawn
# pads unless RP412INPUTSCRIPT drives them; the race still runs to the
# buzzer, players are still ranked, and the podium still places them - which
# is the path under test. -PodCount raises the field toward the six that
# died on 2026-08-11 (podium has eight spots; eight is the ceiling).
#
# Run setup.ps1 with the SAME -PodCount first.
param(
  [int]$Races = 5,
  [ValidateRange(2, 8)]
  [int]$PodCount = 2,
  [string]$Scratch = "$env:TEMP\rp412-podium-repro",
  [string]$MungaNetDll = 'C:\VWE\TeslaSuite\Console\lib\Munga Net.dll',
  [string]$ReferenceEgg = 'C:\VWE\RP412\assets\RP411\TEST.EGG',
  # seconds of racing before the buzzer
  [int]$RaceSeconds = 45,
  # cdbrun.txt enables LIGHT page heap itself via the PEB.
  # cdbrun-gflags.txt does not - use it when `gflags -i rpl4opt.exe +hpa`
  # has already been set (full page heap), so only one mechanism is in play.
  [string]$CdbScript = 'cdbrun.txt',
  # Run even when page heap could not be confirmed. Off by default: an
  # uninstrumented clean run looks exactly like an instrumented one in the
  # summary, and banking it as a negative is how a rig starts lying.
  [switch]$AllowNoPageHeap
)
$ErrorActionPreference = 'Stop'
Add-Type -Path $MungaNetDll

# Keep the feeder's own narrative on disk. Without this the state machine's
# view of the run lives only in the console that launched it, which is where
# a race-2 hang went unexplained the first time.
try { Start-Transcript -Path "$Scratch\feeder.log" -Force | Out-Null } catch {}

function Log([string]$m) {
  $line = "{0:HH:mm:ss.f} $m" -f (Get-Date)
  Write-Output $line
}

#-----------------------------------------------------------------
# The field. Pod i: dir podA+i, console port 1501+100i, game port
# 1502+100i - the numbering two-pod-test established.
#-----------------------------------------------------------------
$vehicles = @('speck','roach','flea','bug','puck','vole','wasp','grunt')
# ONLY names in RPL4GAUG.cpp's colorLookUp (matched on the first 3 chars):
# Aqua Black Blue Green Pink Purple Red White Yellow. An unlisted name makes
# determineEntityColor fall through to 255, and the GPS gauge's own-pod blip
# adds 0xC0 to it -> translationTable[447] on a 256-entry table -> an
# out-of-bounds read that page heap turns into a hard AV. ("Orange" cost one
# six-pod run learning this.)
$colors   = @('Red','Blue','Green','Yellow','Purple','Pink','White','Black')
$keys = @()
$podInfo = @{}
for ($i = 0; $i -lt $PodCount; $i++) {
  $k = [string][char](65 + $i)          # A, B, C...
  $keys += $k
  $podInfo[$k] = @{
    dir         = "$Scratch\pod$k"
    consolePort = 1501 + (100 * $i)
    gamePort    = 1502 + (100 * $i)
    name        = "POD$k"
    vehicle     = $vehicles[$i]
    color       = $colors[$i]
    cdbLog      = "$Scratch\pod$k-cdb.log"
  }
}
Log "field: $PodCount pods ($($keys -join ',')), $Races races, $RaceSeconds s each"

#-----------------------------------------------------------------
# The egg (RPMission.ToEggString layout; blank plasma name bitmaps,
# ordinals lifted verbatim from TEST.EGG)
#-----------------------------------------------------------------
function Add-BlankBitmap([Text.StringBuilder]$sb, [int]$w, [int]$h) {
  $row = 'bitmap=' + ('0' * ($w / 4))
  for ($i = 0; $i -lt $h; $i++) { [void]$sb.AppendLine($row) }
  [void]$sb.AppendLine("x=$w"); [void]$sb.AppendLine("y=$h")
}
$test = [IO.File]::ReadAllText($ReferenceEgg)
$ordinals = $test.Substring($test.IndexOf('[ordinals]'))

$sb = New-Object Text.StringBuilder
[void]$sb.AppendLine('[mission]')
[void]$sb.AppendLine('adventure=Red Planet')
[void]$sb.AppendLine('map=wise')
[void]$sb.AppendLine('scenario=race')
[void]$sb.AppendLine('time=day')
[void]$sb.AppendLine('weather=clear')
[void]$sb.AppendLine('temperature=0')
[void]$sb.AppendLine('compression=0')
[void]$sb.AppendLine("length=$RaceSeconds")
[void]$sb.AppendLine('[pilots]')
foreach ($k in $keys) { [void]$sb.AppendLine("pilot=127.0.0.1:$($podInfo[$k].gamePort)") }
$index = 1
foreach ($k in $keys) {
  $p = $podInfo[$k]
  [void]$sb.AppendLine("[127.0.0.1:$($p.gamePort)]")
  [void]$sb.AppendLine('hostType=0')
  # All on 'one': DropZone falls back to a random free pad when the named
  # one is taken, so a whole field can share a nominal zone without wedging.
  [void]$sb.AppendLine('dropzone=one')
  [void]$sb.AppendLine("name=$($p.name)")
  [void]$sb.AppendLine("bitmapindex=$index")
  [void]$sb.AppendLine('loadzones=1')
  [void]$sb.AppendLine("vehicle=$($p.vehicle)")
  [void]$sb.AppendLine("color=$($p.color)")
  [void]$sb.AppendLine('badge=None')
  $index++
}
[void]$sb.AppendLine('[largebitmap]')
foreach ($k in $keys) { [void]$sb.AppendLine("bitmap=BitMap::Large::$($podInfo[$k].name)") }
[void]$sb.AppendLine('[smallbitmap]')
foreach ($k in $keys) { [void]$sb.AppendLine("bitmap=BitMap::Small::$($podInfo[$k].name)") }
foreach ($k in $keys) {
  $n = $podInfo[$k].name
  [void]$sb.AppendLine("[BitMap::Large::$n]")
  Add-BlankBitmap $sb 128 32
  [void]$sb.AppendLine('width=8')
  [void]$sb.AppendLine("[BitMap::Small::$n]")
  Add-BlankBitmap $sb 64 16
  [void]$sb.AppendLine('width=4')
}
[void]$sb.Append($ordinals)
$eggText = $sb.ToString()
[IO.File]::WriteAllText("$Scratch\field.egg", $eggText)

# wire form: newlines -> NUL, chunked into EggFileMessages
$wire = $eggText.Replace("`r`n", "`0").Replace("`n", "`0")
$bytes = [Text.Encoding]::ASCII.GetBytes($wire)
$chunks = @()
for ($off = 0; $off -lt $bytes.Length; $off += 1000) {
  $n = [Math]::Min(1000, $bytes.Length - $off)
  $buf = New-Object byte[] 1000
  [Buffer]::BlockCopy($bytes, $off, $buf, 0, $n)
  $chunks += New-Object Munga.Net.EggFileMessage([int]($off / 1000), $bytes.Length, $n, $buf)
}
Log "egg built: $($eggText.Length) chars, $($chunks.Count) chunks"

#-----------------------------------------------------------------
# Launch every pod under cdb (page heap via the PEB trick in cdbrun.txt)
#-----------------------------------------------------------------
Get-Process rpl4opt -ErrorAction SilentlyContinue | Stop-Process -Force -Confirm:$false -ErrorAction SilentlyContinue
Get-Process cdb -ErrorAction SilentlyContinue | Stop-Process -Force -Confirm:$false -ErrorAction SilentlyContinue
Start-Sleep -Seconds 1
foreach ($k in $keys) {
  Remove-Item "$($podInfo[$k].dir)\rpl4.log", "$($podInfo[$k].dir)\podbreak*.dmp", $podInfo[$k].cdbLog -Force -Confirm:$false -ErrorAction SilentlyContinue
}

foreach ($k in $keys) {
  $p = $podInfo[$k]
  if (-not (Test-Path "$($p.dir)\rpl4opt.exe")) {
    Log "pod ${k}: $($p.dir) not set up - run setup.ps1 -PodCount $PodCount first"
    exit 1
  }
  Start-Process -FilePath cmd.exe -ArgumentList '/c', "$Scratch\runpod.cmd", $p.dir, $p.cdbLog, "$Scratch\$CdbScript", "$($p.consolePort)" -WindowStyle Hidden | Out-Null
  Start-Sleep -Seconds 3
}
Log "$PodCount pods launched under cdb"
Start-Sleep -Seconds 12

# The game PID sits behind cmd -> cdb, but page heap announces it in the cdb
# log ("Page heap: pid 0x97B4: ..."), which is the only place the pod and its
# process are tied together. Needed to retire survivors selectively later.
foreach ($k in $keys) {
  $podInfo[$k].pid = $null
  $hit = Select-String -Path $podInfo[$k].cdbLog -Pattern 'Page heap: pid 0x([0-9a-fA-F]+)' -ErrorAction SilentlyContinue |
    Select-Object -First 1
  if ($null -ne $hit) {
    $podInfo[$k].pid = [Convert]::ToInt32($hit.Matches[0].Groups[1].Value, 16)
    # Log the line VERBATIM - its "flags 0x..." value is how you tell light
    # page heap (the PEB trick) from full (gflags +hpa). Do not paraphrase it.
    Log "pod ${k}: game pid $($podInfo[$k].pid) | $($hit.Line.Trim())"
  } else {
    Log "pod ${k}: WARNING - no page-heap line in cdb log; is page heap on?"
  }
}

#
# Stop rather than bank a negative that proves nothing.
#
# The PEB trick this rig was built on FAILS SILENTLY on Windows 26200: the
# write lands at the create-process event - read it back there and it is
# 0x02001000 - and ntdll zeroes NtGlobalFlag again during its own init, so
# every pod runs on an ordinary heap. It worked on 26100, which is what the
# README verified against. Nothing in the run's output would tell you: the
# races look identical and the pods exit clean either way.
#
$instrumented = @($keys | Where-Object { $podInfo[$_].pid }).Count
if ($instrumented -lt $PodCount) {
  Log "PAGE HEAP NOT CONFIRMED on $($PodCount - $instrumented) of $PodCount pods"
  if (-not $AllowNoPageHeap) {
    Log 'ABORTING - a clean result without the heap instrument proves nothing.'
    Log 'Full page heap needs an ELEVATED shell:'
    Log "  & 'C:\Program Files (x86)\Windows Kits\10\Debuggers\x86\gflags.exe' -i rpl4opt.exe +hpa"
    Log "  ...feeder.ps1 -PodCount 4 -CdbScript cdbrun-gflags.txt"
    Log "  & '...\gflags.exe' -i rpl4opt.exe -hpa      # always put it back"
    Log 'Or pass -AllowNoPageHeap to run uninstrumented on purpose.'
    Get-Process rpl4opt -ErrorAction SilentlyContinue | Stop-Process -Force -Confirm:$false -ErrorAction SilentlyContinue
    Get-Process cdb -ErrorAction SilentlyContinue | Stop-Process -Force -Confirm:$false -ErrorAction SilentlyContinue
    exit 1
  }
  Log 'CONTINUING UNINSTRUMENTED (-AllowNoPageHeap) - this run can only catch'
  Log 'a crash that faults on its own, as the fatal night did. A clean result'
  Log 'is NOT evidence of absence and must not be recorded as one.'
}

#-----------------------------------------------------------------
# Break detection: cdb writes the marker the instant it traps, well
# before a race would time out.
#-----------------------------------------------------------------
function Find-Break {
  # Match on ExceptionAddress, NOT on the "=== POD BREAK ===" banner. The
  # cdb script's `g` returns on process EXIT as well as on a fault, so the
  # banner prints for a perfectly clean shutdown too - which had this rig
  # reporting six "traps" that were all just the pods quitting normally.
  # `.exr -1` prints ExceptionAddress only when there was a real exception.
  foreach ($k in $keys) {
    $log = $podInfo[$k].cdbLog
    if (Test-Path $log) {
      if (Select-String -Path $log -Pattern 'ExceptionAddress:' -Quiet -ErrorAction SilentlyContinue) {
        return $k
      }
    }
  }
  return $null
}
function PodsAlive {
  return @(Get-Process rpl4opt -ErrorAction SilentlyContinue).Count -ge $PodCount
}

#-----------------------------------------------------------------
# Console feeder
#-----------------------------------------------------------------
$pods = @{}
foreach ($k in $keys) {
  $sock = New-Object Munga.Net.MungaSocket
  $connected = $false
  for ($try = 0; $try -lt 20 -and -not $connected; $try++) {
    try { $sock.Connect([Net.IPAddress]::Loopback, [uint16]$podInfo[$k].consolePort); $connected = $true }
    catch { Start-Sleep -Seconds 2 }
  }
  if (-not $connected) { Log "pod ${k}: console connect FAILED"; exit 1 }
  $pods[$k] = @{ sock = $sock; state = $null }
  Log "console connected to pod $k (port $($podInfo[$k].consolePort))"
}

$stateQuery = New-Object Munga.Net.StateQueryMessage(1)
$raceResults = @()
$brokePod = $null

for ($race = 1; $race -le $Races -and $brokePod -eq $null; $race++) {
  Log "=== RACE $race of $Races ==="
  foreach ($k in $keys) {
    $pods[$k].ack = $false; $pods[$k].eggSent = [DateTime]::MinValue
    $pods[$k].lastQuery = [DateTime]::MinValue
    $pods[$k].score = $null; $pods[$k].runSent = $false; $pods[$k].stopSent = $false
  }
  $runStart = $null
  $raceDeadline = (Get-Date).AddMinutes(6)
  $raceOK = $false

  while ((Get-Date) -lt $raceDeadline) {
    $brokePod = Find-Break
    if ($brokePod -ne $null) {
      Log "*** POD $brokePod TRAPPED IN THE DEBUGGER ***"
      # Retire the survivors. They are of no forensic value once one pod has
      # trapped, and leaving them racing with no buzzer ever sent strands
      # them mid-mission with the race clock running up.
      foreach ($k in $keys) {
        if ($k -ne $brokePod) {
          try { $pods[$k].sock.Send(0, 1, (New-Object Munga.Net.StopMissionMessage(0))) } catch {}
        }
      }
      Start-Sleep -Seconds 5
      foreach ($k in $keys) {
        if ($k -ne $brokePod) {
          Get-Process -Id $podInfo[$k].pid -ErrorAction SilentlyContinue |
            Stop-Process -Force -Confirm:$false -ErrorAction SilentlyContinue
        }
      }
      Log "survivors retired; pod $brokePod left frozen for post-mortem"
      break
    }
    if (-not (PodsAlive)) { Log 'A POD PROCESS IS GONE (died without trapping)'; break }

    foreach ($k in $keys) {
      $pod = $pods[$k]
      $now = Get-Date
      if (($now - $pod.lastQuery).TotalSeconds -ge 1) {
        try { $pod.sock.Send(0, 1, $stateQuery) } catch { Log "pod ${k}: send failed ($_)" }
        $pod.lastQuery = $now
      }
      for ($m = $pod.sock.Receive(); $m -ne $null; $m = $pod.sock.Receive()) {
        $msg = $m.Message
        if ($msg -eq $null) { continue }
        switch ($msg.GetType().Name) {
          'StateResponseMessage' {
            if ("$($pod.state)" -ne "$($msg.ApplicationState)") {
              Log "pod ${k}: state -> $($msg.ApplicationState)"
            }
            $pod.state = $msg.ApplicationState
          }
          'AcknowledgeEggFileMessage' {
            if (-not $pod.ack) { Log "pod ${k}: EGG ACK (mesh complete)" }
            $pod.ack = $true
          }
          'EndMissionMessage' {
            $pod.score = $msg.FinalScore
            Log "pod ${k}: FINAL SCORE host=$($msg.PlayerHostID) score=$($msg.FinalScore)"
          }
          default {}
        }
      }
      if ("$($pod.state)" -eq 'WaitingForEgg' -and -not $pod.ack -and
          ($now - $pod.eggSent).TotalSeconds -ge 6) {
        Log "pod ${k}: sending egg"
        foreach ($chunk in $chunks) { $pod.sock.Send(0, 1, $chunk) }
        $pod.eggSent = $now
      }
    }

    $allWaiting = $true
    foreach ($k in $keys) { if ("$($pods[$k].state)" -ne 'WaitingForLaunch') { $allWaiting = $false } }
    if ($allWaiting -and -not $pods[$keys[0]].runSent) {
      Log 'all pods WaitingForLaunch: RunMission'
      foreach ($k in $keys) {
        $pods[$k].sock.Send(0, 1, (New-Object Munga.Net.RunMissionMessage))
        $pods[$k].runSent = $true
      }
    }

    $allRunning = $true
    foreach ($k in $keys) { if ("$($pods[$k].state)" -ne 'RunningMission') { $allRunning = $false } }
    if ($allRunning -and $runStart -eq $null) {
      $runStart = Get-Date
      Log "all pods RUNNING - $RaceSeconds s to the buzzer"
    }

    if ($runStart -ne $null -and -not $pods[$keys[0]].stopSent -and
        ((Get-Date) - $runStart).TotalSeconds -ge $RaceSeconds) {
      Log 'buzzer: StopMission(0) -> all pods'
      foreach ($k in $keys) {
        $pods[$k].sock.Send(0, 1, (New-Object Munga.Net.StopMissionMessage(0)))
        $pods[$k].stopSent = $true
      }
    }

    # Race complete = every pod came through podium+teardown and is asking
    # for the next egg, with every process still alive.
    if ($pods[$keys[0]].stopSent) {
      $allBack = $true
      foreach ($k in $keys) { if ("$($pods[$k].state)" -ne 'WaitingForEgg') { $allBack = $false } }
      if ($allBack -and (PodsAlive)) { $raceOK = $true; break }
    }
    Start-Sleep -Milliseconds 250
  }

  $scores = @()
  foreach ($k in $keys) { $scores += "$k=$($pods[$k].score)" }
  $raceResults += [pscustomobject]@{ race = $race; ok = $raceOK; scores = ($scores -join ' ') }
  if (-not $raceOK) {
    Log "RACE $race DID NOT COMPLETE - stopping loop (see cdb logs)"
    break
  }
  Log "race $race survived podium teardown ($($scores -join ' '))"
}

Log '=== SUMMARY ==='
foreach ($r in $raceResults) { Log "race $($r.race): ok=$($r.ok) $($r.scores)" }
$dumpPaths = @()
foreach ($k in $keys) { $dumpPaths += "$($podInfo[$k].dir)\podbreak*.dmp" }
$dumps = Get-ChildItem $dumpPaths -ErrorAction SilentlyContinue
foreach ($d in $dumps) { Log "BREAK DUMP: $($d.FullName) ($([Math]::Round($d.Length/1MB)) MB)" }
if ($dumps -eq $null -or @($dumps).Count -eq 0) { Log 'no break dumps written' }
$anyFailed = $false
foreach ($r in $raceResults) { if (-not $r.ok) { $anyFailed = $true } }
if ($null -ne $brokePod) {
  Log "FIRST FREE EVIDENCE: see $($podInfo[$brokePod].cdbLog) after '=== POD BREAK ==='"
  Log 'processes left up for post-mortem'
} elseif ($anyFailed) {
  # A race that never came back without anyone trapping is a HARNESS problem
  # (mesh, egg, state machine), not a crash. Leave it standing so it can be
  # looked at - killing it here is what hid the race-2 hang the first time.
  Log 'a race did not complete and nothing trapped - pods LEFT UP for inspection'
  Log "check each pod's rpl4.log tail and the feeder states above"
} else {
  foreach ($k in $keys) { try { $pods[$k].sock.Shutdown() } catch {} }
  Get-Process rpl4opt -ErrorAction SilentlyContinue | Stop-Process -Force -Confirm:$false -ErrorAction SilentlyContinue
  Log 'clean shutdown'
}
Log 'FEEDER DONE'
try { Stop-Transcript | Out-Null } catch {}