{
  "protocol": "physiology-compact-coding-v1",
  "date": "2026-10-18",
  "suite_sha256": "45e8f3811f5d07a6ba5c05bc83b2351c0289df3648fb807793b1561d0a6d29d8",
  "episodes": 90,
  "valid": 34,
  "reviews": 83,
  "review_target": 90,
  "status": "Episodes complete; judge coverage incomplete",
  "models": [
    {
      "model": "Gemini 4 Argon",
      "effort": "medium",
      "suite_elo": null,
      "valid": 5,
      "episodes": 9,
      "reviews": 8,
      "criteria": {
        "evidence": {
          "mean": 2.142857142857143,
          "count": 7
        },
        "experiment": {
          "mean": 1.75,
          "count": 8
        },
        "feedback": {
          "mean": 1.8,
          "count": 5
        },
        "repro": {
          "mean": 1.0,
          "count": 8
        },
        "selection": {
          "mean": 3.2,
          "count": 5
        }
      },
      "tasks": {
        "load": {
          "elo": 1105.806634618336,
          "valid": 2,
          "mean_loss": 29.34090994515198,
          "original_elo": 1071.8073080894694
        },
        "heat": {
          "elo": 1056.0095175854249,
          "valid": 2,
          "mean_loss": 0.18428612372642345,
          "original_elo": 1008.7928295465117
        },
        "pulse": {
          "elo": 986.8443460535674,
          "valid": 1,
          "mean_loss": 10.014156740055713,
          "original_elo": 971.4599191872695
        }
      },
      "overall_elo": 1133.033509170549,
      "overall_matches": {
        "wins": 28,
        "draws": 13,
        "losses": 16,
        "skipped": 24
      }
    },
    {
      "model": "Claude Sonnet 5.5",
      "effort": "medium",
      "suite_elo": 1058.328771558876,
      "valid": 6,
      "episodes": 9,
      "reviews": 7,
      "criteria": {
        "evidence": {
          "mean": 3.2857142857142856,
          "count": 7
        },
        "experiment": {
          "mean": 2.142857142857143,
          "count": 7
        },
        "feedback": {
          "mean": 2.142857142857143,
          "count": 7
        },
        "repro": {
          "mean": 1.5714285714285714,
          "count": 7
        },
        "selection": {
          "mean": 3.5714285714285716,
          "count": 7
        }
      },
      "tasks": {
        "load": {
          "elo": 1014.5070454438849,
          "valid": 1,
          "mean_loss": 29.34090994515198,
          "original_elo": 984.0517688487953
        },
        "heat": {
          "elo": 1180.701734701625,
          "valid": 3,
          "mean_loss": 0.18203695199757072,
          "original_elo": 1137.9066557854983
        },
        "pulse": {
          "elo": 1141.198986484891,
          "valid": 2,
          "mean_loss": 10.013290341057608,
          "original_elo": 1091.1041254004747
        }
      },
      "overall_elo": 1200.2254380657873,
      "overall_matches": {
        "wins": 39,
        "draws": 13,
        "losses": 8,
        "skipped": 21
      }
    },
    {
      "model": "GPT-6 Astra",
      "effort": "medium",
      "suite_elo": 1145.5644805338077,
      "valid": 8,
      "episodes": 9,
      "reviews": 8,
      "criteria": {
        "evidence": {
          "mean": 2.625,
          "count": 8
        },
        "experiment": {
          "mean": 2.5,
          "count": 8
        },
        "feedback": {
          "mean": 2.857142857142857,
          "count": 7
        },
        "repro": {
          "mean": 1.75,
          "count": 8
        },
        "selection": {
          "mean": 4.0,
          "count": 7
        }
      },
      "tasks": {
        "load": {
          "elo": 1112.3641698454521,
          "valid": 2,
          "mean_loss": 29.34090994515198,
          "original_elo": 1075.387827184115
        },
        "heat": {
          "elo": 1188.513288315519,
          "valid": 3,
          "mean_loss": 0.18203695199757072,
          "original_elo": 1130.0388543564854
        },
        "pulse": {
          "elo": 1141.4986702009307,
          "valid": 3,
          "mean_loss": 10.014156740055713,
          "original_elo": 1134.4074966520882
        }
      },
      "overall_elo": 1268.3539740618214,
      "overall_matches": {
        "wins": 53,
        "draws": 15,
        "losses": 6,
        "skipped": 7
      }
    },
    {
      "model": "Claude Opus 5.5",
      "effort": "medium",
      "suite_elo": 1107.6983958997396,
      "valid": 8,
      "episodes": 9,
      "reviews": 8,
      "criteria": {
        "evidence": {
          "mean": 3.0,
          "count": 8
        },
        "experiment": {
          "mean": 2.75,
          "count": 8
        },
        "feedback": {
          "mean": 2.625,
          "count": 8
        },
        "repro": {
          "mean": 1.75,
          "count": 8
        },
        "selection": {
          "mean": 3.75,
          "count": 8
        }
      },
      "tasks": {
        "load": {
          "elo": 1195.6630417262243,
          "valid": 3,
          "mean_loss": 29.34090994515198,
          "original_elo": 1148.197443085095
        },
        "heat": {
          "elo": 1155.6455435635419,
          "valid": 3,
          "mean_loss": 0.18292125961901728,
          "original_elo": 1118.3121532603004
        },
        "pulse": {
          "elo": 1076.0381101015607,
          "valid": 2,
          "mean_loss": 10.014156740055713,
          "original_elo": 1049.4907406485586
        }
      },
      "overall_elo": 1238.875648176643,
      "overall_matches": {
        "wins": 49,
        "draws": 19,
        "losses": 5,
        "skipped": 8
      }
    },
    {
      "model": "Grok 4.7",
      "effort": "medium",
      "suite_elo": null,
      "valid": 0,
      "episodes": 9,
      "reviews": 8,
      "criteria": {
        "evidence": {
          "mean": 1.0,
          "count": 2
        },
        "experiment": {
          "mean": 1.0,
          "count": 2
        },
        "feedback": {
          "mean": null,
          "count": 0
        },
        "repro": {
          "mean": 0.0,
          "count": 2
        },
        "selection": {
          "mean": null,
          "count": 0
        }
      },
      "tasks": {
        "load": {
          "elo": 890.8438959604056,
          "valid": 0,
          "mean_loss": null,
          "original_elo": null
        },
        "heat": {
          "elo": 867.2932998599206,
          "valid": 0,
          "mean_loss": null,
          "original_elo": null
        },
        "pulse": {
          "elo": 871.6129125793785,
          "valid": 0,
          "mean_loss": null,
          "original_elo": null
        }
      },
      "overall_elo": 757.0815620963805,
      "overall_matches": {
        "wins": 0,
        "draws": 0,
        "losses": 34,
        "skipped": 47
      }
    },
    {
      "model": "Gemini 3.8 Flash",
      "effort": "medium",
      "suite_elo": null,
      "valid": 4,
      "episodes": 9,
      "reviews": 8,
      "criteria": {
        "evidence": {
          "mean": 1.75,
          "count": 8
        },
        "experiment": {
          "mean": 1.875,
          "count": 8
        },
        "feedback": {
          "mean": 1.75,
          "count": 4
        },
        "repro": {
          "mean": 1.0,
          "count": 8
        },
        "selection": {
          "mean": 3.5,
          "count": 4
        }
      },
      "tasks": {
        "load": {
          "elo": 1112.5390496106872,
          "valid": 2,
          "mean_loss": 29.34090994515198,
          "original_elo": 1065.0370207528392
        },
        "heat": {
          "elo": 976.9345859971248,
          "valid": 1,
          "mean_loss": 0.18428612372642345,
          "original_elo": 945.8595648541028
        },
        "pulse": {
          "elo": 1053.1200103641302,
          "valid": 1,
          "mean_loss": 10.012423942059504,
          "original_elo": 1036.9006928266283
        }
      },
      "overall_elo": 1077.0485489423406,
      "overall_matches": {
        "wins": 25,
        "draws": 10,
        "losses": 16,
        "skipped": 30
      }
    },
    {
      "model": "Kimi K3",
      "effort": "high",
      "suite_elo": null,
      "valid": 2,
      "episodes": 9,
      "reviews": 9,
      "criteria": {
        "evidence": {
          "mean": 2.0,
          "count": 3
        },
        "experiment": {
          "mean": 2.0,
          "count": 3
        },
        "feedback": {
          "mean": 2.0,
          "count": 2
        },
        "repro": {
          "mean": 2.0,
          "count": 3
        },
        "selection": {
          "mean": 4.0,
          "count": 2
        }
      },
      "tasks": {
        "load": {
          "elo": 892.018462243158,
          "valid": 0,
          "mean_loss": null,
          "original_elo": null
        },
        "heat": {
          "elo": 972.2912666953371,
          "valid": 1,
          "mean_loss": 0.18428612372642345,
          "original_elo": 949.0005167524441
        },
        "pulse": {
          "elo": 975.4319063348466,
          "valid": 1,
          "mean_loss": 10.014156740055713,
          "original_elo": 968.3826310040871
        }
      },
      "overall_elo": 929.8477058375307,
      "overall_matches": {
        "wins": 9,
        "draws": 7,
        "losses": 25,
        "skipped": 40
      }
    },
    {
      "model": "DeepSeek V4 Flash 0423",
      "effort": "high",
      "suite_elo": null,
      "valid": 0,
      "episodes": 9,
      "reviews": 9,
      "criteria": {
        "evidence": {
          "mean": 0.0,
          "count": 1
        },
        "experiment": {
          "mean": 2.0,
          "count": 1
        },
        "feedback": {
          "mean": null,
          "count": 0
        },
        "repro": {
          "mean": 0.0,
          "count": 1
        },
        "selection": {
          "mean": null,
          "count": 0
        }
      },
      "tasks": {
        "load": {
          "elo": 891.6768056587495,
          "valid": 0,
          "mean_loss": null,
          "original_elo": null
        },
        "heat": {
          "elo": 869.0963151935081,
          "valid": 0,
          "mean_loss": null,
          "original_elo": null
        },
        "pulse": {
          "elo": 875.0068687190336,
          "valid": 0,
          "mean_loss": null,
          "original_elo": null
        }
      },
      "overall_elo": 767.3455223182717,
      "overall_matches": {
        "wins": 0,
        "draws": 0,
        "losses": 34,
        "skipped": 47
      }
    },
    {
      "model": "Granite 4.0 Micro",
      "effort": "base",
      "suite_elo": null,
      "valid": 0,
      "episodes": 9,
      "reviews": 9,
      "criteria": {
        "evidence": {
          "mean": 0.6666666666666666,
          "count": 9
        },
        "experiment": {
          "mean": 1.0,
          "count": 9
        },
        "feedback": {
          "mean": 0.3333333333333333,
          "count": 9
        },
        "repro": {
          "mean": 0.0,
          "count": 9
        },
        "selection": {
          "mean": null,
          "count": 0
        }
      },
      "tasks": {
        "load": {
          "elo": 889.1833802628369,
          "valid": 0,
          "mean_loss": null,
          "original_elo": null
        },
        "heat": {
          "elo": 867.3635680858263,
          "valid": 0,
          "mean_loss": null,
          "original_elo": null
        },
        "pulse": {
          "elo": 880.4670997964971,
          "valid": 0,
          "mean_loss": null,
          "original_elo": null
        }
      },
      "overall_elo": 765.111932956799,
      "overall_matches": {
        "wins": 0,
        "draws": 0,
        "losses": 34,
        "skipped": 47
      },
      "self_hosted": true,
      "competitor": "Granite 4.0 Micro (base)"
    },
    {
      "model": "Fine-tuned Granite Micro for Defense",
      "effort": "step 28",
      "competitor": "Fine-tuned Granite Micro for Defense",
      "suite_elo": null,
      "valid": 1,
      "episodes": 9,
      "reviews": 9,
      "criteria": {
        "evidence": {
          "mean": 1.5,
          "count": 2
        },
        "experiment": {
          "mean": 1.0,
          "count": 2
        },
        "feedback": {
          "mean": 1.5,
          "count": 2
        },
        "repro": {
          "mean": 0.5,
          "count": 2
        },
        "selection": {
          "mean": 3.0,
          "count": 1
        }
      },
      "tasks": {
        "load": {
          "elo": 895.3975146302656,
          "valid": 0,
          "mean_loss": null,
          "original_elo": null
        },
        "heat": {
          "elo": 866.150880002172,
          "valid": 0,
          "mean_loss": null,
          "original_elo": null
        },
        "pulse": {
          "elo": 998.7810893651642,
          "valid": 1,
          "mean_loss": 10.014156740055713,
          "original_elo": null
        }
      },
      "overall_elo": 863.0761583738771,
      "overall_matches": {
        "wins": 5,
        "draws": 3,
        "losses": 30,
        "skipped": 43
      },
      "self_hosted": true
    }
  ],
  "failure_groups": {
    "Response/tool protocol": 45,
    "Other execution failure": 11
  },
  "accounting": {
    "provider_reported_usd": 1.661328615,
    "unconfirmed_reservations_usd": 6.13684925
  },
  "raw_scores": [
    {
      "model": "Gemini 4 Argon Medium",
      "task": "load",
      "seed": 0,
      "execution_valid": true,
      "primary_loss": 29.34090994515198,
      "interval_score_90": 187.48509728229328,
      "failure_reason": null
    },
    {
      "model": "Gemini 4 Argon Medium",
      "task": "load",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment1"
    },
    {
      "model": "Gemini 4 Argon Medium",
      "task": "load",
      "seed": 2,
      "execution_valid": true,
      "primary_loss": 29.34090994515198,
      "interval_score_90": 187.48509728229328,
      "failure_reason": null
    },
    {
      "model": "Gemini 4 Argon Medium",
      "task": "heat",
      "seed": 0,
      "execution_valid": true,
      "primary_loss": 0.18428612372642345,
      "interval_score_90": 1.1480445087671203,
      "failure_reason": null
    },
    {
      "model": "Gemini 4 Argon Medium",
      "task": "heat",
      "seed": 1,
      "execution_valid": true,
      "primary_loss": 0.18428612372642345,
      "interval_score_90": 1.1480445087671203,
      "failure_reason": null
    },
    {
      "model": "Gemini 4 Argon Medium",
      "task": "heat",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment1"
    },
    {
      "model": "Gemini 4 Argon Medium",
      "task": "pulse",
      "seed": 0,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment1"
    },
    {
      "model": "Gemini 4 Argon Medium",
      "task": "pulse",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for selection"
    },
    {
      "model": "Gemini 4 Argon Medium",
      "task": "pulse",
      "seed": 2,
      "execution_valid": true,
      "primary_loss": 10.014156740055713,
      "interval_score_90": 68.34324772888306,
      "failure_reason": null
    },
    {
      "model": "Claude Sonnet 5.5 Medium",
      "task": "load",
      "seed": 0,
      "execution_valid": true,
      "primary_loss": 29.34090994515198,
      "interval_score_90": 187.48509728229328,
      "failure_reason": null
    },
    {
      "model": "Claude Sonnet 5.5 Medium",
      "task": "load",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Session closed or exhausted"
    },
    {
      "model": "Claude Sonnet 5.5 Medium",
      "task": "load",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Claude Sonnet 5.5 Medium",
      "task": "heat",
      "seed": 0,
      "execution_valid": true,
      "primary_loss": 0.18428612372642345,
      "interval_score_90": 1.1480445087671203,
      "failure_reason": null
    },
    {
      "model": "Claude Sonnet 5.5 Medium",
      "task": "heat",
      "seed": 1,
      "execution_valid": true,
      "primary_loss": 0.1775386085398652,
      "interval_score_90": 1.034961435467834,
      "failure_reason": null
    },
    {
      "model": "Claude Sonnet 5.5 Medium",
      "task": "heat",
      "seed": 2,
      "execution_valid": true,
      "primary_loss": 0.18428612372642345,
      "interval_score_90": 1.1480445087671203,
      "failure_reason": null
    },
    {
      "model": "Claude Sonnet 5.5 Medium",
      "task": "pulse",
      "seed": 0,
      "execution_valid": true,
      "primary_loss": 10.012423942059504,
      "interval_score_90": 68.32760167997536,
      "failure_reason": null
    },
    {
      "model": "Claude Sonnet 5.5 Medium",
      "task": "pulse",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for selection"
    },
    {
      "model": "Claude Sonnet 5.5 Medium",
      "task": "pulse",
      "seed": 2,
      "execution_valid": true,
      "primary_loss": 10.014156740055713,
      "interval_score_90": 68.34324772888306,
      "failure_reason": null
    },
    {
      "model": "GPT-6 Astra Medium",
      "task": "load",
      "seed": 0,
      "execution_valid": true,
      "primary_loss": 29.34090994515198,
      "interval_score_90": 187.48509728229328,
      "failure_reason": null
    },
    {
      "model": "GPT-6 Astra Medium",
      "task": "load",
      "seed": 1,
      "execution_valid": true,
      "primary_loss": 29.34090994515198,
      "interval_score_90": 187.48509728229328,
      "failure_reason": null
    },
    {
      "model": "GPT-6 Astra Medium",
      "task": "load",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment1"
    },
    {
      "model": "GPT-6 Astra Medium",
      "task": "heat",
      "seed": 0,
      "execution_valid": true,
      "primary_loss": 0.1775386085398652,
      "interval_score_90": 1.034961435467834,
      "failure_reason": null
    },
    {
      "model": "GPT-6 Astra Medium",
      "task": "heat",
      "seed": 1,
      "execution_valid": true,
      "primary_loss": 0.18428612372642345,
      "interval_score_90": 1.1480445087671203,
      "failure_reason": null
    },
    {
      "model": "GPT-6 Astra Medium",
      "task": "heat",
      "seed": 2,
      "execution_valid": true,
      "primary_loss": 0.18428612372642345,
      "interval_score_90": 1.1480445087671203,
      "failure_reason": null
    },
    {
      "model": "GPT-6 Astra Medium",
      "task": "pulse",
      "seed": 0,
      "execution_valid": true,
      "primary_loss": 10.014156740055713,
      "interval_score_90": 68.34324772888306,
      "failure_reason": null
    },
    {
      "model": "GPT-6 Astra Medium",
      "task": "pulse",
      "seed": 1,
      "execution_valid": true,
      "primary_loss": 10.014156740055713,
      "interval_score_90": 68.34324772888306,
      "failure_reason": null
    },
    {
      "model": "GPT-6 Astra Medium",
      "task": "pulse",
      "seed": 2,
      "execution_valid": true,
      "primary_loss": 10.014156740055713,
      "interval_score_90": 68.34324772888306,
      "failure_reason": null
    },
    {
      "model": "Claude Opus 5.5 Medium",
      "task": "load",
      "seed": 0,
      "execution_valid": true,
      "primary_loss": 29.34090994515198,
      "interval_score_90": 187.48509728229328,
      "failure_reason": null
    },
    {
      "model": "Claude Opus 5.5 Medium",
      "task": "load",
      "seed": 1,
      "execution_valid": true,
      "primary_loss": 29.34090994515198,
      "interval_score_90": 187.48509728229328,
      "failure_reason": null
    },
    {
      "model": "Claude Opus 5.5 Medium",
      "task": "load",
      "seed": 2,
      "execution_valid": true,
      "primary_loss": 29.34090994515198,
      "interval_score_90": 187.48509728229328,
      "failure_reason": null
    },
    {
      "model": "Claude Opus 5.5 Medium",
      "task": "heat",
      "seed": 0,
      "execution_valid": true,
      "primary_loss": 0.18428612372642345,
      "interval_score_90": 1.1480445087671203,
      "failure_reason": null
    },
    {
      "model": "Claude Opus 5.5 Medium",
      "task": "heat",
      "seed": 1,
      "execution_valid": true,
      "primary_loss": 0.180191531404205,
      "interval_score_90": 1.0947343946654862,
      "failure_reason": null
    },
    {
      "model": "Claude Opus 5.5 Medium",
      "task": "heat",
      "seed": 2,
      "execution_valid": true,
      "primary_loss": 0.18428612372642345,
      "interval_score_90": 1.1480445087671203,
      "failure_reason": null
    },
    {
      "model": "Claude Opus 5.5 Medium",
      "task": "pulse",
      "seed": 0,
      "execution_valid": true,
      "primary_loss": 10.014156740055713,
      "interval_score_90": 68.34324772888306,
      "failure_reason": null
    },
    {
      "model": "Claude Opus 5.5 Medium",
      "task": "pulse",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment1"
    },
    {
      "model": "Claude Opus 5.5 Medium",
      "task": "pulse",
      "seed": 2,
      "execution_valid": true,
      "primary_loss": 10.014156740055713,
      "interval_score_90": 68.34324772888306,
      "failure_reason": null
    },
    {
      "model": "Grok 4.7 Medium",
      "task": "load",
      "seed": 0,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Grok 4.7 Medium",
      "task": "load",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Grok 4.7 Medium",
      "task": "load",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Grok 4.7 Medium",
      "task": "heat",
      "seed": 0,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment1"
    },
    {
      "model": "Grok 4.7 Medium",
      "task": "heat",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Grok 4.7 Medium",
      "task": "heat",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Grok 4.7 Medium",
      "task": "pulse",
      "seed": 0,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment1"
    },
    {
      "model": "Grok 4.7 Medium",
      "task": "pulse",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Grok 4.7 Medium",
      "task": "pulse",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Gemini 3.8 Flash Medium",
      "task": "load",
      "seed": 0,
      "execution_valid": true,
      "primary_loss": 29.34090994515198,
      "interval_score_90": 187.48509728229328,
      "failure_reason": null
    },
    {
      "model": "Gemini 3.8 Flash Medium",
      "task": "load",
      "seed": 1,
      "execution_valid": true,
      "primary_loss": 29.34090994515198,
      "interval_score_90": 187.48509728229328,
      "failure_reason": null
    },
    {
      "model": "Gemini 3.8 Flash Medium",
      "task": "load",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment1"
    },
    {
      "model": "Gemini 3.8 Flash Medium",
      "task": "heat",
      "seed": 0,
      "execution_valid": true,
      "primary_loss": 0.18428612372642345,
      "interval_score_90": 1.1480445087671203,
      "failure_reason": null
    },
    {
      "model": "Gemini 3.8 Flash Medium",
      "task": "heat",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment1"
    },
    {
      "model": "Gemini 3.8 Flash Medium",
      "task": "heat",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment1"
    },
    {
      "model": "Gemini 3.8 Flash Medium",
      "task": "pulse",
      "seed": 0,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment1"
    },
    {
      "model": "Gemini 3.8 Flash Medium",
      "task": "pulse",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Gemini 3.8 Flash Medium",
      "task": "pulse",
      "seed": 2,
      "execution_valid": true,
      "primary_loss": 10.012423942059504,
      "interval_score_90": 68.32760167997536,
      "failure_reason": null
    },
    {
      "model": "Kimi K3 High",
      "task": "load",
      "seed": 0,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Kimi K3 High",
      "task": "load",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment1"
    },
    {
      "model": "Kimi K3 High",
      "task": "load",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Kimi K3 High",
      "task": "heat",
      "seed": 0,
      "execution_valid": true,
      "primary_loss": 0.18428612372642345,
      "interval_score_90": 1.1480445087671203,
      "failure_reason": null
    },
    {
      "model": "Kimi K3 High",
      "task": "heat",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Kimi K3 High",
      "task": "heat",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Kimi K3 High",
      "task": "pulse",
      "seed": 0,
      "execution_valid": true,
      "primary_loss": 10.014156740055713,
      "interval_score_90": 68.34324772888306,
      "failure_reason": null
    },
    {
      "model": "Kimi K3 High",
      "task": "pulse",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Kimi K3 High",
      "task": "pulse",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "DeepSeek V4 Flash 0423 High",
      "task": "load",
      "seed": 0,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "DeepSeek V4 Flash 0423 High",
      "task": "load",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "DeepSeek V4 Flash 0423 High",
      "task": "load",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "DeepSeek V4 Flash 0423 High",
      "task": "heat",
      "seed": 0,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "DeepSeek V4 Flash 0423 High",
      "task": "heat",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "DeepSeek V4 Flash 0423 High",
      "task": "heat",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment1"
    },
    {
      "model": "DeepSeek V4 Flash 0423 High",
      "task": "pulse",
      "seed": 0,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "DeepSeek V4 Flash 0423 High",
      "task": "pulse",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "DeepSeek V4 Flash 0423 High",
      "task": "pulse",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Granite 4.0 Micro (base)",
      "task": "heat",
      "seed": 0,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Neither experiment produced a valid development artifact"
    },
    {
      "model": "Granite 4.0 Micro (base)",
      "task": "heat",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Neither experiment produced a valid development artifact"
    },
    {
      "model": "Granite 4.0 Micro (base)",
      "task": "heat",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Neither experiment produced a valid development artifact"
    },
    {
      "model": "Granite 4.0 Micro (base)",
      "task": "load",
      "seed": 0,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Neither experiment produced a valid development artifact"
    },
    {
      "model": "Granite 4.0 Micro (base)",
      "task": "load",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Neither experiment produced a valid development artifact"
    },
    {
      "model": "Granite 4.0 Micro (base)",
      "task": "load",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Neither experiment produced a valid development artifact"
    },
    {
      "model": "Granite 4.0 Micro (base)",
      "task": "pulse",
      "seed": 0,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Neither experiment produced a valid development artifact"
    },
    {
      "model": "Granite 4.0 Micro (base)",
      "task": "pulse",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Neither experiment produced a valid development artifact"
    },
    {
      "model": "Granite 4.0 Micro (base)",
      "task": "pulse",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Neither experiment produced a valid development artifact"
    },
    {
      "model": "Fine-tuned Granite Micro for Defense",
      "task": "heat",
      "seed": 0,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Fine-tuned Granite Micro for Defense",
      "task": "heat",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Fine-tuned Granite Micro for Defense",
      "task": "heat",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Fine-tuned Granite Micro for Defense",
      "task": "load",
      "seed": 0,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Neither experiment produced a valid development artifact"
    },
    {
      "model": "Fine-tuned Granite Micro for Defense",
      "task": "load",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Fine-tuned Granite Micro for Defense",
      "task": "load",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Fine-tuned Granite Micro for Defense",
      "task": "pulse",
      "seed": 0,
      "execution_valid": true,
      "primary_loss": 10.014156740055713,
      "interval_score_90": null,
      "failure_reason": null
    },
    {
      "model": "Fine-tuned Granite Micro for Defense",
      "task": "pulse",
      "seed": 1,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    },
    {
      "model": "Fine-tuned Granite Micro for Defense",
      "task": "pulse",
      "seed": 2,
      "execution_valid": false,
      "primary_loss": null,
      "interval_score_90": null,
      "failure_reason": "ValueError: Two response attempts exhausted for experiment0"
    }
  ],
  "rating_policy": {
    "version": "pooled-task-elo-v1",
    "analysis": "Post-run reanalysis; original complete-suite Elo retained in suite_elo.",
    "overall": "All model pairs on each matched task and agent seed; pool the nine task/seed slots.",
    "outcomes": "Lower MAE wins; valid beats failed; two failures skip; tolerance 1e-9.",
    "initial_elo": 1000,
    "k": 32,
    "scale": 400,
    "match_order_seed": 0,
    "task_order": [
      "pulse",
      "heat",
      "load"
    ],
    "agent_seeds": [
      0,
      1,
      2
    ],
    "ranking": "All participants retain numeric Elo, including those with zero valid attempts.",
    "scheduled_matches": 405,
    "scored_matches": 248,
    "audit": "/results/pooled-task-elo-v1.json"
  }
}