{ "meta": { "updatedAt": "2026-07-19T05:30:00Z", "map": "Hatchet Axe and Saw (SoD 72x72, symmetric, full dwellings)", "faction": "Inferno (red)", "opponent": "VCMI Nullkiller AI (NK2), unthrottled", "dayCap": 40, "budget": { "rounds": 25, "evalRuns": 5, "fitness": "median peak_army over evalRuns" }, "verification": { "runs": 15, "note": "champion re-run on fresh samples; median + IQR + range reported" }, "gameSeconds": 7, "liveGameMinutes": 27, "protocolNotes": [ "The game reward is heavy-tailed and nondeterministic (~4-10x peak-army spread for the same policy and seed), so in-loop numbers are upward-biased by selection (winner's curse). Only fresh-sample verified medians are comparable.", "One autoresearch loop per writer unless marked as a rerun; loop-level variance is measured by the _b reruns.", "Writers are isolated: they see only the interface SPEC, a faction cheat-sheet, and factual per-game stats. They never see the engine, the harness, or each other's policies.", "Champion policies are hand-reviewed: pure-stdlib strategy code, no engine introspection." ] }, "liveplay": { "note": "LLM-in-the-loop reference (one decision call per action, same harness/map/opponent). Single runs - directional only.", "models": { "glm-5.2": { "peakArmy": 141565, "n": 1, "detail": "cap-120, lost the decisive battle vs NK2 on day 99" }, "gpt-oss-120b": { "peakArmy": 39683, "n": 1, "detail": "cap-40, late conquest spike" } } }, "writers": [ { "id": "gpt-oss-120b", "displayName": "GPT-OSS 120B", "family": "OpenAI (open-weights)", "channel": "openrouter:openai/gpt-oss-120b", "run": "pilot_gptoss120b", "status": "verified", "verified": { "median": 44340, "mean": 43155, "iqr": [ 43666, 45995 ], "range": [ 31511, 48066 ], "n": 15, "wins": 0 }, "inLoopBest": 45674, "keptTrajectory": [ [ 0, 4071 ], [ 1, 44059 ], [ 8, 45064 ], [ 20, 45674 ] ], "rounds": 25, "invalidRounds": 0, "note": "Jumped 4k -> 44k in a single revision at round 1 (tightest IQR, only -3% verification shrink). CAVEAT: a second independent loop verified only 2,424 - its 25 candidates never solved hero movement. gpt-oss's loop outcome is bimodal (cracks map-conquest or never leaves town), so this rank carries much more loop-level uncertainty than the Anthropic entries." }, { "id": "claude-sonnet-5", "displayName": "Claude Sonnet 5", "family": "Anthropic", "channel": "claude:claude-sonnet-5", "run": "sonnet5_median (2026-07-01)", "status": "verified", "verified": { "median": 38263, "mean": 41079, "iqr": [ 29929, 53817 ], "range": [ 25447, 69759 ], "n": 15, "wins": 0 }, "inLoopBest": 62283, "keptTrajectory": [ [ 1, 6547 ], [ 4, 7917 ], [ 5, 8323 ], [ 6, 15444 ], [ 7, 21832 ], [ 14, 62283 ] ], "rounds": 25, "invalidRounds": 0, "note": "Steady climber, but high-variance policy: in-loop best 62k shrank -39% under fresh-sample verification. A second independent loop verified 42,252 (~10% from this run) — loop-level variance is ~4k, so the gap to GPT-OSS 120B (44,340) is a tie, not a deficit." }, { "id": "glm-5.2", "displayName": "GLM-5.2", "family": "Z.AI", "channel": "openrouter:z-ai/glm-5.2", "run": "pilot_glm52", "status": "verified", "verified": { "median": 13895, "mean": 12723, "iqr": [ 8453, 19147 ], "range": [ 1210, 22267 ], "n": 15, "wins": 0 }, "inLoopBest": 17462, "keptTrajectory": [ [ 0, 6541 ], [ 7, 10587 ], [ 13, 15818 ], [ 14, 17462 ] ], "rounds": 25, "invalidRounds": 1, "note": "The best live player we have measured (141k army) writes the weakest bot - playing and coding dissociate." }, { "id": "claude-opus-4-8", "displayName": "Claude Opus 4.8", "family": "Anthropic", "channel": "claude:claude-opus-4-8", "run": "ladder_opus48", "status": "verified", "verified": { "median": 62837, "mean": 60293, "iqr": [ 52890, 70570 ], "range": [ 29994, 83889 ], "n": 15, "wins": 0 }, "inLoopBest": 76619, "keptTrajectory": [ [ 0, 23907 ], [ 1, 27611 ], [ 4, 36024 ], [ 5, 76619 ] ], "rounds": 25, "invalidRounds": 4, "note": "New champion. Strongest opening draft of any writer (23.9k at round 0) and the biggest verified score by a wide margin." }, { "id": "gpt-5.5", "displayName": "GPT-5.5", "family": "OpenAI", "channel": "codex:gpt-5.5", "run": "ladder_gpt55", "status": "verified", "verified": { "median": 20036, "mean": 26042, "iqr": [ 13847, 32815 ], "range": [ 12375, 57310 ], "n": 15, "wins": 0 }, "inLoopBest": 36022, "keptTrajectory": [ [ 0, 2374 ], [ 1, 6517 ], [ 2, 14536 ], [ 4, 36022 ] ], "rounds": 25, "invalidRounds": 0, "note": "Fast early climb, then 20 rounds without a kept improvement; high-variance policy (in-loop 36k shrank -44% under verification)." }, { "id": "gpt-5.4-mini", "displayName": "GPT-5.4 Mini", "family": "OpenAI", "channel": "codex:gpt-5.4-mini (low reasoning)", "run": "ladder_gpt54mini_cx", "status": "verified", "verified": { "median": 32659, "mean": 30400, "iqr": [ 31056, 36212 ], "range": [ 2475, 36501 ], "n": 15, "wins": 0 }, "inLoopBest": 35111, "keptTrajectory": [ [ 0, 10751 ], [ 7, 23760 ], [ 11, 24740 ], [ 12, 33832 ], [ 15, 35111 ] ], "rounds": 25, "invalidRounds": 0, "note": "Ran at low reasoning effort (higher efforts time out on the policy prompt). Beats GPT-5.5 — echoing Haiku > Sonnet: the small models punch above their weight at bot-writing." }, { "id": "claude-haiku-4-5", "displayName": "Claude Haiku 4.5", "family": "Anthropic", "channel": "claude:claude-haiku-4-5-20251001", "run": "ladder_haiku45", "status": "verified", "verified": { "median": 55557, "mean": 56357, "iqr": [ 53730, 59730 ], "range": [ 44642, 76436 ], "n": 15, "wins": 0 }, "inLoopBest": 59156, "keptTrajectory": [ [ 0, 5674 ], [ 1, 31363 ], [ 11, 58127 ], [ 13, 59156 ] ], "rounds": 25, "invalidRounds": 0, "note": "The smallest, cheapest model in the field writes the #2 solo bot — only -6% verification shrink, tight IQR. BRACKET CAVEAT: the bot hardcodes its own color ('owner' != 'red' in its mine logic) — a red-seat specialist (170.9k median as attacker, 2.5k as defender), so it drops to #5 in the head-to-head bracket." } ], "reruns": [ { "of": "claude-sonnet-5", "run": "ladder_sonnet5_b", "status": "verified", "verified": { "median": 42252, "mean": 37796, "iqr": [ 28782, 44118 ], "range": [ 20258, 45921 ], "n": 15, "wins": 0 }, "inLoopBest": 40944, "keptTrajectory": [ [ 0, 21807 ], [ 1, 39591 ], [ 13, 40944 ] ], "note": "Loop-variance check: two independent Sonnet-5 loops verified 38,263 and 42,252 (~10% apart) — rankings within ~4k should be read as ties." }, { "of": "gpt-oss-120b", "run": "ladder_gptoss120b_b", "status": "verified", "verified": { "median": 2424, "mean": 2677, "iqr": [ 2049, 3626 ], "range": [ 1265, 4296 ], "n": 15, "wins": 0 }, "inLoopBest": 3123, "keptTrajectory": [ [ 0, 2314 ], [ 1, 2553 ], [ 11, 2835 ], [ 12, 3064 ], [ 13, 3123 ] ], "note": "Loop-variance check FAILED to reproduce run A: 25/25 valid candidates, zero invalid rounds, but every policy scored 1.3-3.1k - none ever moved the hero (0 MOVE_HERO in 85 offers on the champion). gpt-oss either discovers the movement/coordinate join (run A, round 1) or never does. Loop outcome is bimodal; single-loop ranks understate uncertainty for this writer." } ], "bracket": { "tag": "bracket1", "date": "2026-07-05", "games": 420, "gamesPerPairing": 20, "dayCap": 120, "eliminations": 0, "decidedBy": "Peak army in the shared world (paired same-game comparison). Elimination wins would count first; none occurred - these bots farm, they don't hunt.", "standings": [ { "id": "claude-opus-4-8", "wins": 106, "games": 120 }, { "id": "gpt-oss-120b", "wins": 82, "games": 120 }, { "id": "claude-sonnet-5", "wins": 79, "games": 120 }, { "id": "gpt-5.4-mini", "wins": 52, "games": 120 }, { "id": "claude-haiku-4-5", "wins": 46, "games": 120 }, { "id": "gpt-5.5", "wins": 38, "games": 120 }, { "id": "glm-5.2", "wins": 17, "games": 120 } ], "matrix": { "claude-opus-4-8": { "gpt-oss-120b": 17, "claude-sonnet-5": 13, "gpt-5.4-mini": 19, "claude-haiku-4-5": 18, "gpt-5.5": 19, "glm-5.2": 20 }, "gpt-oss-120b": { "claude-opus-4-8": 3, "claude-sonnet-5": 12, "gpt-5.4-mini": 14, "claude-haiku-4-5": 16, "gpt-5.5": 18, "glm-5.2": 19 }, "claude-sonnet-5": { "claude-opus-4-8": 7, "gpt-oss-120b": 8, "gpt-5.4-mini": 17, "claude-haiku-4-5": 11, "gpt-5.5": 16, "glm-5.2": 20 }, "gpt-5.4-mini": { "claude-opus-4-8": 1, "gpt-oss-120b": 6, "claude-sonnet-5": 3, "claude-haiku-4-5": 10, "gpt-5.5": 13, "glm-5.2": 19 }, "claude-haiku-4-5": { "claude-opus-4-8": 2, "gpt-oss-120b": 4, "claude-sonnet-5": 9, "gpt-5.4-mini": 10, "gpt-5.5": 10, "glm-5.2": 11 }, "gpt-5.5": { "claude-opus-4-8": 1, "gpt-oss-120b": 2, "claude-sonnet-5": 4, "gpt-5.4-mini": 7, "claude-haiku-4-5": 10, "glm-5.2": 14 }, "glm-5.2": { "claude-opus-4-8": 0, "gpt-oss-120b": 1, "claude-sonnet-5": 0, "gpt-5.4-mini": 1, "claude-haiku-4-5": 9, "gpt-5.5": 6 } }, "sideSplit": { "note": "Median peak army as attacker vs as defender across all bracket games. Sides alternate every game; a side-locked bot forfeits half its games.", "claude-opus-4-8": [ 237269, 174340 ], "gpt-oss-120b": [ 132908, 175118 ], "claude-sonnet-5": [ 148682, 98986 ], "gpt-5.4-mini": [ 110672, 99525 ], "claude-haiku-4-5": [ 170929, 2460 ], "gpt-5.5": [ 23700, 8878 ], "glm-5.2": [ 7539, 6057 ] }, "notes": [ "Zero eliminations in 420 games: peak-army fitness bred farmers, not fighters - the case for a Season 2 fitness that rewards aggression.", "Haiku 4.5's bot hardcodes its own color ('owner' != 'red' inside its mine logic): 170.9k median as red/attacker (second only to Opus) but 2.5k as defender. It memorized the seat it was evolved in. Side-robustness is part of the duel.", "GPT-5.5's bot is also side-sensitive (23.7k attacker vs 8.9k defender), cause not yet isolated.", "Side-robust bots scale ~3.1x from the solo cap-40 to the bracket's cap-120 (Opus 3.21x, Sonnet 3.14x, mini 3.17x, gpt-oss 3.06x)." ] }, "colorSlots": { "gpt-oss-120b": "#3987e5", "claude-sonnet-5": "#199e70", "glm-5.2": "#c98500", "gpt-5.5": "#008300", "claude-opus-4-8": "#9085e9", "gpt-5.4-mini": "#e66767", "claude-haiku-4-5": "#d55181", "gpt-5.6-terra": "#a8642a", "gpt-5.6-luna": "#7f9bd1", "gpt-5.6-sol": "#d4a017" }, "season2": { "meta": { "name": "Season 2 — The War Season", "date": "2026-07-08", "spec": "plans/2026-07-05-PLAN-season2-war-season.md", "goal": "Bots bred to WIN games, not grow armies: in-loop fitness = 6-game sparring vs three frozen Season-1 champions (one game as each side, fresh random seed per game, day cap 120), scored elimination=3 / decision=1 / loss=0, tuple (league_pts, elim_wins, median_peak) kept lexicographically.", "protocol": { "rounds": 25, "drafts": 3, "stuckRestart": 8, "loopsPerWriter": 2, "sparring": [ "opus-4-8 (S1)", "gpt-oss-120b (S1)", "glm-5.2 (S1)" ], "sides": "attacker=red/Inferno, defender=blue/Rampart — sides and seeds randomized every eval; champions must play both", "championRule": "better loop by tuple; final ranking = league round-robin vs the other S2 champions (unseen during evolution)" }, "league": { "tag": "s2_league", "games": 900, "gamesPerPairing": 20, "dayCap": 120, "eliminations": 0, "decidedBy": "All 420 games went to the day-120 cap; every point is a decision (bigger engine-scored peak army at the cap, garrisons included). The 3-point elimination bounty was never claimed.", "pairings": 45, "amended": "2026-07-10 — post-season codex entrants gpt-5.6-luna/-terra/-sol added; full 10-champion round-robin (45 pairings x 20). THE CROWN CHANGES HANDS: gpt-5.6-terra leads on points (129 v 124) although sonnet-5 won all 9 of its pairings incl. terra 14-6 — terra wins by margin against the tail (18-2, 18-2, 17-3), sonnet by narrower scores. Season rank is league points per the published protocol." }, "notes": [ "Season-1 results below are frozen and unchanged; Season 2 used new SPEC_s2/wiki docs (either-side rule + S1 pitfalls) and day-capped engine scoring.", "Both subscription channels burst-failed mid-season (codex quota in 4a, claude 5h-window in 4b); affected rounds were topped up and the harness now retries channel outages in-round.", "Systematic seat asymmetry: ALL champions develop bigger as defender (Rampart) than attacker (Inferno). League rank tracks seat-robustness — the podium plays both sides; the tail only farms as defender.", "2026-07-10 amendment: gpt-5.6-luna (Stage 6) and gpt-5.6-terra (Stage 7) evolved post-season on the frozen protocol and merged into the league (was 7 champions/420 games; now 9/720). All league/robustness/efficiency columns recomputed over 160 games per champion; development/boss unchanged for the original seven. NK2 now 9-for-9 (0/135 boss games); still 0 eliminations league-wide.", "2026-07-10 late amendment: gpt-5.6-sol (Stage 8, codex HIGH) merged — 10 champions/900 games. Sol profile: season-worst drafts but most improve-loop keeps (10); reached 68/180, 8th. NK2 now 10-for-10 (0/150 boss); still 0 eliminations in 900 league games." ] }, "standings": [ { "id": "gpt-5.6-terra", "displayName": "GPT-5.6-Terra", "channel": "codex:gpt-5.6-terra", "championLoop": "s2_gpt56terra_a", "loops": [ { "tag": "s2_gpt56terra_a", "bestFitness": [ 6, 0, 314268 ], "bestRound": 7, "keptTrajectory": [ [ 0, [ 5, 0, 272231 ] ], [ 3, [ 5, 0, 274562 ] ], [ 6, [ 6, 0, 279920 ] ], [ 7, [ 6, 0, 314268 ] ] ], "rounds": 31 }, { "tag": "s2_gpt56terra_b", "bestFitness": [ 6, 0, 277018 ], "bestRound": 7, "keptTrajectory": [ [ 0, [ 2, 0, 15090 ] ], [ 3, [ 5, 0, 209429 ] ], [ 6, [ 5, 0, 221700 ] ], [ 7, [ 6, 0, 277018 ] ] ], "rounds": 31 } ], "development": { "median": 45032, "mean": 48703, "iqr": [ 40245, 57074 ], "range": [ 31992, 77853 ], "n": 15, "wins": 0 }, "boss": { "wins": 0, "runs": 15, "dayCap": 120 }, "robustness": { "median_attacker": 251773, "median_defender": 278252, "att_def_ratio": 0.9, "iqr": [ 230475, 300579 ], "n_games": 180 }, "efficiency": { "writer_calls": 27, "pts_per_call": 4.78 }, "league": { "name": "gpt-5.6-terra", "points": 129, "elim_wins": 0, "decision_wins": 129, "games": 180 } }, { "id": "sonnet-5", "displayName": "Claude Sonnet 5", "channel": "claude:claude-sonnet-5", "championLoop": "s2_sonnet5_a", "loops": [ { "tag": "s2_sonnet5_a", "bestFitness": [ 6, 0, 325991 ], "bestRound": 2, "keptTrajectory": [ [ 0, [ 6, 0, 302088 ] ], [ 2, [ 6, 0, 325991 ] ] ], "rounds": 25 }, { "tag": "s2_sonnet5_b", "bestFitness": [ 5, 0, 192775 ], "bestRound": 23, "keptTrajectory": [ [ 0, [ 4, 0, 149699 ] ], [ 3, [ 4, 0, 176301 ] ], [ 23, [ 5, 0, 192775 ] ] ], "rounds": 25 } ], "development": { "median": 56287, "mean": 55241, "iqr": [ 50848, 61678 ], "range": [ 22424, 76203 ], "n": 15, "wins": 0 }, "boss": { "wins": 0, "runs": 15, "dayCap": 120 }, "robustness": { "median_attacker": 310896, "median_defender": 305034, "att_def_ratio": 1.02, "iqr": [ 288735, 326672 ], "n_games": 180 }, "efficiency": { "writer_calls": 27, "pts_per_call": 4.59 }, "league": { "name": "sonnet-5", "points": 124, "elim_wins": 0, "decision_wins": 124, "games": 180 } }, { "id": "opus-4-8", "displayName": "Claude Opus 4.8", "channel": "claude:claude-opus-4-8", "championLoop": "s2_opus48_b", "loops": [ { "tag": "s2_opus48_a", "bestFitness": [ 6, 0, 290105 ], "bestRound": 31, "keptTrajectory": [ [ 0, [ 5, 0, 296213 ] ], [ 1, [ 5, 0, 296865 ] ], [ 2, [ 6, 0, 290087 ] ], [ 31, [ 6, 0, 290105 ] ] ], "rounds": 32 }, { "tag": "s2_opus48_b", "bestFitness": [ 6, 0, 292198 ], "bestRound": 6, "keptTrajectory": [ [ 0, [ 5, 0, 271418 ] ], [ 1, [ 6, 0, 268159 ] ], [ 6, [ 6, 0, 292198 ] ] ], "rounds": 25 } ], "development": { "median": 43495, "mean": 43139, "iqr": [ 30893, 49563 ], "range": [ 2816, 77890 ], "n": 15, "wins": 0 }, "boss": { "wins": 0, "runs": 15, "dayCap": 120 }, "robustness": { "median_attacker": 261992, "median_defender": 282574, "att_def_ratio": 0.93, "iqr": [ 242615, 300898 ], "n_games": 180 }, "efficiency": { "writer_calls": 27, "pts_per_call": 3.93 }, "league": { "name": "opus-4-8", "points": 106, "elim_wins": 0, "decision_wins": 106, "games": 180 } }, { "id": "haiku-4-5", "displayName": "Claude Haiku 4.5", "channel": "claude:claude-haiku-4-5-20251001", "championLoop": "s2_haiku45_a", "loops": [ { "tag": "s2_haiku45_a", "bestFitness": [ 5, 0, 303560 ], "bestRound": 24, "keptTrajectory": [ [ 0, [ 4, 0, 224824 ] ], [ 3, [ 4, 0, 276968 ] ], [ 6, [ 5, 0, 225132 ] ], [ 8, [ 5, 0, 258934 ] ], [ 19, [ 5, 0, 275358 ] ], [ 20, [ 5, 0, 279876 ] ], [ 24, [ 5, 0, 303560 ] ] ], "rounds": 25 }, { "tag": "s2_haiku45_b", "bestFitness": [ 4, 0, 200820 ], "bestRound": 16, "keptTrajectory": [ [ 0, [ 1, 0, 6355 ] ], [ 2, [ 3, 0, 171838 ] ], [ 5, [ 4, 0, 162383 ] ], [ 15, [ 4, 0, 191565 ] ], [ 16, [ 4, 0, 200820 ] ] ], "rounds": 25 } ], "development": { "median": 30227, "mean": 35179, "iqr": [ 23543, 36621 ], "range": [ 8409, 66800 ], "n": 15, "wins": 0 }, "boss": { "wins": 0, "runs": 15, "dayCap": 120 }, "robustness": { "median_attacker": 160007, "median_defender": 298644, "att_def_ratio": 0.54, "iqr": [ 157346, 300023 ], "n_games": 180 }, "efficiency": { "writer_calls": 27, "pts_per_call": 3.3 }, "league": { "name": "haiku-4-5", "points": 89, "elim_wins": 0, "decision_wins": 89, "games": 180 } }, { "id": "gpt-5.4-mini", "displayName": "GPT-5.4 Mini", "channel": "codex:gpt-5.4-mini (low)", "championLoop": "s2_gpt54mini_a", "loops": [ { "tag": "s2_gpt54mini_a", "bestFitness": [ 6, 0, 279068 ], "bestRound": 1, "keptTrajectory": [ [ 0, [ 3, 0, 145601 ] ], [ 1, [ 6, 0, 279068 ] ] ], "rounds": 25 }, { "tag": "s2_gpt54mini_b", "bestFitness": [ 3, 0, 78354 ], "bestRound": 8, "keptTrajectory": [ [ 0, [ 0, 0, 3309 ] ], [ 1, [ 1, 0, 4467 ] ], [ 3, [ 2, 0, 68145 ] ], [ 4, [ 2, 0, 78723 ] ], [ 7, [ 3, 0, 76855 ] ], [ 8, [ 3, 0, 78354 ] ] ], "rounds": 25 } ], "development": { "median": 13873, "mean": 16435, "iqr": [ 9831, 18508 ], "range": [ 7236, 53923 ], "n": 15, "wins": 0 }, "boss": { "wins": 0, "runs": 15, "dayCap": 120 }, "robustness": { "median_attacker": 161953, "median_defender": 289205, "att_def_ratio": 0.56, "iqr": [ 156547, 290889 ], "n_games": 180 }, "efficiency": { "writer_calls": 27, "pts_per_call": 2.96 }, "league": { "name": "gpt-5.4-mini", "points": 80, "elim_wins": 0, "decision_wins": 80, "games": 180 } }, { "id": "gpt-5.6-luna", "displayName": "GPT-5.6-Luna", "channel": "codex:gpt-5.6-luna", "championLoop": "s2_gpt56luna_b", "loops": [ { "tag": "s2_gpt56luna_a", "bestFitness": [ 5, 0, 205482 ], "bestRound": 6, "keptTrajectory": [ [ 0, [ 4, 0, 268579 ] ], [ 1, [ 5, 0, 191624 ] ], [ 6, [ 5, 0, 205482 ] ] ], "rounds": 26 }, { "tag": "s2_gpt56luna_b", "bestFitness": [ 5, 0, 252034 ], "bestRound": 5, "keptTrajectory": [ [ 0, [ 4, 0, 213249 ] ], [ 5, [ 5, 0, 252034 ] ] ], "rounds": 25 } ], "development": { "median": 35274, "mean": 33432, "iqr": [ 30260, 36737 ], "range": [ 19795, 39209 ], "n": 15, "wins": 0 }, "boss": { "wins": 0, "runs": 15, "dayCap": 120 }, "robustness": { "median_attacker": 201288, "median_defender": 291022, "att_def_ratio": 0.69, "iqr": [ 188907, 291234 ], "n_games": 180 }, "efficiency": { "writer_calls": 27, "pts_per_call": 2.89 }, "league": { "name": "gpt-5.6-luna", "points": 78, "elim_wins": 0, "decision_wins": 78, "games": 180 } }, { "id": "gpt-oss-120b", "displayName": "GPT-OSS 120B", "channel": "openrouter:openai/gpt-oss-120b", "championLoop": "s2_gptoss120b_b", "loops": [ { "tag": "s2_gptoss120b_a", "bestFitness": [ 5, 0, 79150 ], "bestRound": 5, "keptTrajectory": [ [ 0, [ 2, 0, 87887 ] ], [ 2, [ 2, 0, 116427 ] ], [ 4, [ 4, 0, 87762 ] ], [ 5, [ 5, 0, 79150 ] ] ], "rounds": 25 }, { "tag": "s2_gptoss120b_b", "bestFitness": [ 6, 0, 255867 ], "bestRound": 11, "keptTrajectory": [ [ 0, [ 1, 0, 23678 ] ], [ 1, [ 2, 0, 16498 ] ], [ 3, [ 5, 0, 37643 ] ], [ 4, [ 5, 0, 192188 ] ], [ 5, [ 5, 0, 252672 ] ], [ 11, [ 6, 0, 255867 ] ] ], "rounds": 25 } ], "development": { "median": 35613, "mean": 29898, "iqr": [ 19774, 39134 ], "range": [ 9436, 44528 ], "n": 15, "wins": 0 }, "boss": { "wins": 0, "runs": 15, "dayCap": 120 }, "robustness": { "median_attacker": 163924, "median_defender": 298657, "att_def_ratio": 0.55, "iqr": [ 157679, 298681 ], "n_games": 180 }, "efficiency": { "writer_calls": 26, "pts_per_call": 2.96 }, "league": { "name": "gpt-oss-120b", "points": 77, "elim_wins": 0, "decision_wins": 77, "games": 180 } }, { "id": "gpt-5.5", "displayName": "GPT-5.5", "channel": "codex:gpt-5.5 (xhigh)", "championLoop": "s2_gpt55_b", "loops": [ { "tag": "s2_gpt55_a", "bestFitness": [ 3, 0, 186147 ], "bestRound": 27, "keptTrajectory": [ [ 0, [ 1, 0, 4211 ] ], [ 1, [ 2, 0, 77977 ] ], [ 6, [ 2, 0, 80115 ] ], [ 11, [ 3, 0, 69204 ] ], [ 14, [ 3, 0, 136259 ] ], [ 18, [ 3, 0, 137844 ] ], [ 20, [ 3, 0, 178795 ] ], [ 27, [ 3, 0, 186147 ] ] ], "rounds": 29 }, { "tag": "s2_gpt55_b", "bestFitness": [ 6, 0, 177289 ], "bestRound": 11, "keptTrajectory": [ [ 0, [ 2, 0, 54719 ] ], [ 2, [ 4, 0, 221666 ] ], [ 5, [ 4, 0, 276569 ] ], [ 11, [ 6, 0, 177289 ] ] ], "rounds": 35 } ], "development": { "median": 5515, "mean": 6155, "iqr": [ 4754, 6039 ], "range": [ 3525, 14925 ], "n": 15, "wins": 0 }, "boss": { "wins": 0, "runs": 15, "dayCap": 120 }, "robustness": { "median_attacker": 113888, "median_defender": 459680, "att_def_ratio": 0.25, "iqr": [ 113561, 459988 ], "n_games": 180 }, "efficiency": { "writer_calls": 27, "pts_per_call": 2.85 }, "league": { "name": "gpt-5.5", "points": 77, "elim_wins": 0, "decision_wins": 77, "games": 180 } }, { "id": "glm-5.2", "displayName": "GLM-5.2", "channel": "openrouter:z-ai/glm-5.2", "championLoop": "s2_glm52_b", "loops": [ { "tag": "s2_glm52_a", "bestFitness": [ 4, 0, 272863 ], "bestRound": 3, "keptTrajectory": [ [ 0, [ 2, 0, 5090 ] ], [ 3, [ 4, 0, 272863 ] ] ], "rounds": 25 }, { "tag": "s2_glm52_b", "bestFitness": [ 5, 0, 235317 ], "bestRound": 19, "keptTrajectory": [ [ 0, [ 3, 0, 166132 ] ], [ 1, [ 4, 0, 103804 ] ], [ 14, [ 4, 0, 185537 ] ], [ 18, [ 5, 0, 38170 ] ], [ 19, [ 5, 0, 235317 ] ] ], "rounds": 25 } ], "development": { "median": 20112, "mean": 22208, "iqr": [ 18676, 22703 ], "range": [ 13626, 37976 ], "n": 15, "wins": 0 }, "boss": { "wins": 0, "runs": 15, "dayCap": 120 }, "robustness": { "median_attacker": 23102, "median_defender": 347042, "att_def_ratio": 0.07, "iqr": [ 22878, 350457 ], "n_games": 180 }, "efficiency": { "writer_calls": 27, "pts_per_call": 2.67 }, "league": { "name": "glm-5.2", "points": 72, "elim_wins": 0, "decision_wins": 72, "games": 180 } }, { "id": "gpt-5.6-sol", "displayName": "GPT-5.6-Sol", "channel": "codex:gpt-5.6-sol", "championLoop": "s2_gpt56sol_a", "loops": [ { "tag": "s2_gpt56sol_a", "bestFitness": [ 5, 0, 52003 ], "bestRound": 16, "keptTrajectory": [ [ 0, [ 2, 0, 8834 ] ], [ 4, [ 3, 0, 16622 ] ], [ 5, [ 4, 0, 18368 ] ], [ 9, [ 5, 0, 44401 ] ], [ 16, [ 5, 0, 52003 ] ] ], "rounds": 25 }, { "tag": "s2_gpt56sol_b", "bestFitness": [ 3, 0, 78805 ], "bestRound": 19, "keptTrajectory": [ [ 0, [ 1, 0, 2562 ] ], [ 9, [ 2, 0, 78482 ] ], [ 15, [ 3, 0, 70125 ] ], [ 16, [ 3, 0, 75489 ] ], [ 19, [ 3, 0, 78805 ] ] ], "rounds": 25 } ], "development": { "median": 13149, "mean": 17642, "iqr": [ 10688, 30563 ], "range": [ 7233, 32333 ], "n": 15, "wins": 0 }, "boss": { "wins": 0, "runs": 15, "dayCap": 120 }, "robustness": { "median_attacker": 27878, "median_defender": 254503, "att_def_ratio": 0.11, "iqr": [ 25324, 256009 ], "n_games": 180 }, "efficiency": { "writer_calls": 27, "pts_per_call": 2.52 }, "league": { "name": "gpt-5.6-sol", "points": 68, "elim_wins": 0, "decision_wins": 68, "games": 180 } } ] }, "season3": { "meta": { "name": "Season 3 — The Film Season", "date": "2026-07-19", "goal": "One shot, one session: each model gets a single agentic CLI session (its own native tooling), a 300-game eval budget, and — new this season — grep-able game film of every game its bot plays. It reads the film, finds its own bugs, and submits one bot. Twice.", "format": { "runsPerModel": 2, "gamesBudget": 300, "wallClockHours": 4, "channels": { "claude": "claude -p (Claude Code CLI)", "codex": "codex exec (Codex CLI)" }, "maps": "12 randomly generated 72x72 mirror-symmetric 2-player maps (training) + 8 held-out maps from the same generator (league only, sealed until league day; SHA-256 seal published in-repo)", "factions": "both players same faction per map; all 8 factions across the pool", "film": "after every eval, per-game digests (one line per decision + fight/capture/sighting events) land in the workspace; writers grep them to debug their own bots; mirror self-play (--opponent self) isolates seat bugs" }, "league": { "tag": "s3r_final_league", "games": 3640, "pairings": 91, "gamesPerPairing": 40, "seedsPerPairing": 20, "dayCap": 120, "failed": 0, "eliminations": 46, "metric": "balanced overall median margin: every pairing plays 20 seeds from both seats; per game margin = log((peak_self+1)/(peak_opp+1)), day-capped. Seats cancel by design; worst-seat and seat-gap published as diagnostics, never the rank." }, "champion": { "run": "s3r_opus48_b", "model": "claude-opus-4-8", "overall": 0.777, "ci": [ 0.654, 0.838 ], "pairingsWon": 12 }, "effortNote": "codex sessions ran at xhigh reasoning effort; Claude Code sessions at its default high effort." }, "standings": [ { "id": "gpt-5.6-sol", "displayName": "GPT-5.6-Sol", "channel": "codex:gpt-5.6-sol", "combined": 0.678, "gap": 0.076, "runs": [ { "tag": "s3r_gpt56sol_a", "run": "A", "overall": 0.6395, "ci": [ 0.5497, 0.7233 ], "worstSeat": "defender", "worstSeatMedian": 0.4068, "seatGap": 0.3896, "pairingsWon": 13, "elimsFor": 4, "elimsAgainst": 2, "games": 520, "trajectory": [ [ 16, -1.425 ], [ 32, -3.226 ], [ 40, 0.477 ], [ 48, 0.589 ], [ 56, 1.049 ], [ 128, 1.444 ], [ 192, 1.007 ], [ 272, 1.497 ] ], "gamesUsed": 292 }, { "tag": "s3r_gpt56sol_b", "run": "B", "overall": 0.7157, "ci": [ 0.5976, 0.8195 ], "worstSeat": "defender", "worstSeatMedian": 0.5356, "seatGap": 0.3752, "pairingsWon": 11, "elimsFor": 2, "elimsAgainst": 1, "games": 520, "trajectory": [ [ 24, -2.023 ], [ 40, 1.127 ], [ 64, 1.537 ], [ 144, 1.789 ], [ 228, 1.641 ] ], "gamesUsed": 300 } ], "effort": "xhigh" }, { "id": "gpt-5.5", "displayName": "GPT-5.5", "channel": "codex:gpt-5.5", "combined": 0.594, "gap": 0.068, "runs": [ { "tag": "s3r_gpt55_a", "run": "A", "overall": 0.6282, "ci": [ 0.5797, 0.732 ], "worstSeat": "defender", "worstSeatMedian": 0.5834, "seatGap": 0.1112, "pairingsWon": 10, "elimsFor": 1, "elimsAgainst": 1, "games": 520, "trajectory": [ [ 10, -1.658 ], [ 20, 0.886 ], [ 24, 0.895 ], [ 48, 0.191 ], [ 52, -2.708 ], [ 56, 1.011 ], [ 76, 0.937 ], [ 80, -0.081 ], [ 84, 1.731 ], [ 124, 1.436 ], [ 136, 1.218 ], [ 148, 0.783 ], [ 208, 1.428 ], [ 268, 1.475 ] ], "gamesUsed": 288 }, { "tag": "s3r_gpt55_b", "run": "B", "overall": 0.5602, "ci": [ 0.4538, 0.6448 ], "worstSeat": "defender", "worstSeatMedian": 0.3503, "seatGap": 0.301, "pairingsWon": 9, "elimsFor": 20, "elimsAgainst": 0, "games": 520, "trajectory": [ [ 12, -0.785 ], [ 24, -3.174 ], [ 32, 0.183 ], [ 64, 0.877 ], [ 112, 1.305 ], [ 176, 1.442 ] ], "gamesUsed": 192 } ], "effort": "xhigh" }, { "id": "claude-opus-4-8", "displayName": "Claude Opus 4.8", "channel": "claude:claude-opus-4-8", "combined": 0.185, "gap": 1.183, "runs": [ { "tag": "s3r_opus48_a", "run": "A", "overall": -0.4062, "ci": [ -0.5242, -0.2399 ], "worstSeat": "defender", "worstSeatMedian": -0.5632, "seatGap": 0.4071, "pairingsWon": 4, "elimsFor": 0, "elimsAgainst": 2, "games": 520, "trajectory": [ [ 12, -2.691 ], [ 24, -3.431 ], [ 30, -3.63 ], [ 32, -3.805 ], [ 40, -0.935 ], [ 52, 0.053 ], [ 64, 0.063 ], [ 76, 0.214 ], [ 96, 0.49 ] ], "gamesUsed": 96 }, { "tag": "s3r_opus48_b", "run": "B", "overall": 0.7771, "ci": [ 0.6538, 0.838 ], "worstSeat": "defender", "worstSeatMedian": 0.6681, "seatGap": 0.1639, "pairingsWon": 12, "elimsFor": 7, "elimsAgainst": 1, "games": 520, "trajectory": [ [ 10, 0.195 ], [ 22, 0.308 ], [ 102, 1.458 ], [ 152, 1.351 ] ], "gamesUsed": 152 } ], "note": "Why the 1.18 spread (from the session transcripts + league film): run A burned a third of its budget on four catastrophic early redesigns (−2.7→−3.8), then film-debugged to +0.49 vs the practice set, declared itself done and stopped with 204 of 300 games unused — final verification only n=20. Run B's first draft was already positive; it A/B-tested four drafts against its previous version, verified its final bot on 54 games (+1.4), tried one more draft, measured it worse, and reverted. In the league, A's bot is uniformly mediocre on all 8 maps (no crash, no broken seat — just under-cooked); B's strength transferred. Same model, same harness, same information — different research discipline between two sessions. Not a harness artifact: zero errors or failed games in either run.", "effort": "high" }, { "id": "gpt-5.6-terra", "displayName": "GPT-5.6-Terra", "channel": "codex:gpt-5.6-terra", "combined": 0.175, "gap": 0.519, "runs": [ { "tag": "s3r_gpt56terra_a", "run": "A", "overall": -0.0847, "ci": [ -0.1705, -0.0426 ], "worstSeat": "defender", "worstSeatMedian": -0.2143, "seatGap": 0.1924, "pairingsWon": 6, "elimsFor": 0, "elimsAgainst": 3, "games": 520, "trajectory": [ [ 12, -2.518 ], [ 52, 0.232 ], [ 120, 0.536 ], [ 152, 0.182 ], [ 192, 0.582 ] ], "gamesUsed": 212 }, { "tag": "s3r_gpt56terra_b", "run": "B", "overall": 0.434, "ci": [ 0.3561, 0.5703 ], "worstSeat": "defender", "worstSeatMedian": 0.0575, "seatGap": 0.5947, "pairingsWon": 8, "elimsFor": 0, "elimsAgainst": 0, "games": 520, "trajectory": [ [ 10, -1.886 ], [ 20, 0.09 ], [ 30, 0.87 ], [ 110, 1.298 ], [ 190, 1.426 ] ], "gamesUsed": 210 } ], "effort": "xhigh" }, { "id": "gpt-5.6-luna", "displayName": "GPT-5.6-Luna", "channel": "codex:gpt-5.6-luna", "combined": -0.173, "gap": 0.563, "runs": [ { "tag": "s3r_gpt56luna_a", "run": "A", "overall": -0.4545, "ci": [ -0.6036, -0.3103 ], "worstSeat": "defender", "worstSeatMedian": -0.6743, "seatGap": 0.5084, "pairingsWon": 2, "elimsFor": 0, "elimsAgainst": 1, "games": 520, "trajectory": [ [ 20, -2.417 ], [ 40, 0.595 ], [ 60, -0.603 ], [ 80, 0.319 ], [ 100, -1.724 ], [ 120, -2.93 ], [ 140, -1.337 ], [ 160, -2.637 ], [ 180, -0.233 ], [ 200, -1.008 ], [ 220, 0 ], [ 240, -0.454 ], [ 260, -0.062 ], [ 300, 0.217 ] ], "gamesUsed": 300 }, { "tag": "s3r_gpt56luna_b", "run": "B", "overall": 0.1087, "ci": [ 0.0278, 0.2541 ], "worstSeat": "defender", "worstSeatMedian": -0.0305, "seatGap": 0.321, "pairingsWon": 7, "elimsFor": 4, "elimsAgainst": 3, "games": 520, "trajectory": [ [ 8, -2.428 ], [ 28, 0.755 ], [ 32, 0.945 ], [ 52, 0.772 ], [ 100, 0.995 ], [ 120, 1.021 ], [ 128, 0.133 ], [ 136, 0.245 ], [ 156, 0.77 ], [ 164, 0.573 ], [ 204, 0.901 ], [ 212, 0.036 ], [ 284, 0.9 ] ], "gamesUsed": 292 } ], "effort": "xhigh" }, { "id": "claude-sonnet-5", "displayName": "Claude Sonnet 5", "channel": "claude:claude-sonnet-5", "combined": -0.363, "gap": 0.103, "runs": [ { "tag": "s3r_sonnet5_a", "run": "A", "overall": -0.312, "ci": [ -0.439, -0.1683 ], "worstSeat": "defender", "worstSeatMedian": -0.5351, "seatGap": 0.5063, "pairingsWon": 5, "elimsFor": 0, "elimsAgainst": 1, "games": 520, "trajectory": [ [ 20, 0.068 ], [ 40, 0.325 ], [ 120, 0.559 ], [ 200, 0.77 ], [ 260, 0.646 ] ], "gamesUsed": 260 }, { "tag": "s3r_sonnet5_b", "run": "B", "overall": -0.4146, "ci": [ -0.5109, -0.2646 ], "worstSeat": "defender", "worstSeatMedian": -0.597, "seatGap": 0.449, "pairingsWon": 3, "elimsFor": 1, "elimsAgainst": 3, "games": 520, "trajectory": [ [ 10, 0.201 ], [ 20, -3.153 ], [ 40, 0.152 ], [ 80, 0.493 ], [ 150, 0.188 ] ], "gamesUsed": 150 } ], "effort": "high" }, { "id": "claude-haiku-4-5", "displayName": "Claude Haiku 4.5", "channel": "claude:claude-haiku-4-5", "combined": -4.033, "gap": 0.079, "runs": [ { "tag": "s3r_haiku45_a", "run": "A", "overall": -3.9932, "ci": [ -4.0796, -3.8897 ], "worstSeat": "defender", "worstSeatMedian": -4.0621, "seatGap": 0.1885, "pairingsWon": 0, "elimsFor": 3, "elimsAgainst": 14, "games": 520, "trajectory": [ [ 4, -3.116 ], [ 8, -2.874 ], [ 12, -3.37 ], [ 18, -2.804 ], [ 38, -2.818 ], [ 58, -2.694 ], [ 78, -3.273 ], [ 98, -2.736 ], [ 114, -3.014 ], [ 128, -3.183 ], [ 138, -3.56 ], [ 156, -3.347 ], [ 186, -3.016 ], [ 206, -3.042 ], [ 232, -3.15 ] ], "gamesUsed": 232 }, { "tag": "s3r_haiku45_b", "run": "B", "overall": -4.0722, "ci": [ -4.1723, -4.0129 ], "worstSeat": "defender", "worstSeatMedian": -4.2486, "seatGap": 0.284, "pairingsWon": 0, "elimsFor": 4, "elimsAgainst": 14, "games": 520, "trajectory": [ [ 10, -3.055 ], [ 20, -3.271 ], [ 30, -3.063 ], [ 50, -2.993 ], [ 70, -3.188 ], [ 100, -3.188 ], [ 140, -3.173 ], [ 160, -3.11 ] ], "gamesUsed": 160 } ], "effort": "high" } ] } }