{
  "schema_version": 1,
  "kind": "published-results-report",
  "dataset": "linux-results-2026-10-24",
  "exported": "2026-10-24T08:12:53.079Z",
  "source_bindings": {
    "source.json": "sha256:df6381aaa6b6b53ef0d9e0cd3f344cbe101a142a48721d1691e2aaad7c654dee",
    "analysis.json": "sha256:c66eff9528e366dd0e8af2c050086e98770c1424c0a969298a93d071415e4fba",
    "native-evidence.json": "sha256:25020676c958116a567e08640b7612709f62cdf6fb22778e12f0cdb3757d0eeb",
    "cline-workflow-audit.json": "sha256:1383d088c1359c8e7f2c661ec85528448092665fec246773b6377f0c925a721c"
  },
  "panels": [
    {
      "model": "deepseek/deepseek-v4.1-flash-1017",
      "rates": {
        "input": 1.5e-7,
        "cached_input": 3e-9,
        "output": 6e-7
      },
      "metrics": [
        {
          "key": "pass",
          "label": "Pass rate",
          "direction": "Higher is better",
          "kind": "percent"
        },
        {
          "key": "cost",
          "label": "Avg cost / task",
          "direction": "Lower is better",
          "kind": "cost"
        },
        {
          "key": "benchmark",
          "label": "Whole benchmark",
          "direction": "Lower is better",
          "kind": "cost"
        },
        {
          "key": "cache",
          "label": "Cache rate",
          "direction": "Higher is better",
          "kind": "percent"
        },
        {
          "key": "input",
          "label": "Tokens in",
          "direction": "Lower is better",
          "kind": "tokens"
        },
        {
          "key": "output",
          "label": "Tokens out",
          "direction": "Lower is better",
          "kind": "tokens"
        }
      ],
      "views": [
        {
          "id": "all",
          "task": null,
          "confidence": [
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"codex\",\"codex-cli 0.149.1\"]",
              "difference_pp": -81.25,
              "tasks": 8,
              "attempts_per_harness": 16,
              "interval": [
                -93.75,
                -62.5
              ],
              "interpretation": "directional"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": -56.25,
              "tasks": 8,
              "attempts_per_harness": 16,
              "interval": [
                -81.25,
                -31.25
              ],
              "interpretation": "directional"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": -81.25,
              "tasks": 8,
              "attempts_per_harness": 16,
              "interval": [
                -100,
                -56.25
              ],
              "interpretation": "directional"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": -56.25,
              "tasks": 8,
              "attempts_per_harness": 16,
              "interval": [
                -87.5,
                -25
              ],
              "interpretation": "directional"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": 25,
              "tasks": 8,
              "attempts_per_harness": 16,
              "interval": [
                0,
                62.5
              ],
              "interpretation": "inconclusive"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": 0,
              "tasks": 8,
              "attempts_per_harness": 16,
              "interval": [
                -25,
                31.25
              ],
              "interpretation": "inconclusive"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 25,
              "tasks": 8,
              "attempts_per_harness": 16,
              "interval": [
                -12.5,
                62.5
              ],
              "interpretation": "inconclusive"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": -25,
              "tasks": 8,
              "attempts_per_harness": 16,
              "interval": [
                -50,
                -6.25
              ],
              "interpretation": "directional"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 0,
              "tasks": 8,
              "attempts_per_harness": 16,
              "interval": [
                -25,
                25
              ],
              "interpretation": "inconclusive"
            },
            {
              "left": "[\"pi\",\"0.73.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 25,
              "tasks": 8,
              "attempts_per_harness": 16,
              "interval": [
                6.25,
                50
              ],
              "interpretation": "directional"
            }
          ],
          "confidenceByBaseline": {
            "[\"cline\",\"3.0.61\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 81.25,
                "interval": [
                  62.5,
                  93.75
                ],
                "difference_text": "+81.3 pp",
                "interval_text": "+62.5 pp to +93.8 pp",
                "note": "Directional in this sample",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 81.25,
                "interval": [
                  56.25,
                  100
                ],
                "difference_text": "+81.3 pp",
                "interval_text": "+56.3 pp to +100.0 pp",
                "note": "Directional in this sample",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 56.25,
                "interval": [
                  31.25,
                  81.25
                ],
                "difference_text": "+56.3 pp",
                "interval_text": "+31.3 pp to +81.3 pp",
                "note": "Directional in this sample",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 56.25,
                "interval": [
                  25,
                  87.5
                ],
                "difference_text": "+56.3 pp",
                "interval_text": "+25.0 pp to +87.5 pp",
                "note": "Directional in this sample",
                "label": "qwen @ 0.22.2"
              }
            ],
            "[\"codex\",\"codex-cli 0.149.1\"]": [
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": [
                  -31.25,
                  25
                ],
                "difference_text": "0.0 pp",
                "interval_text": "-31.3 pp to +25.0 pp",
                "note": "No clear separation",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": -25,
                "interval": [
                  -62.5,
                  0
                ],
                "difference_text": "-25.0 pp",
                "interval_text": "-62.5 pp to 0.0 pp",
                "note": "No clear separation",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": -25,
                "interval": [
                  -62.5,
                  12.5
                ],
                "difference_text": "-25.0 pp",
                "interval_text": "-62.5 pp to +12.5 pp",
                "note": "No clear separation",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -81.25,
                "interval": [
                  -93.75,
                  -62.5
                ],
                "difference_text": "-81.3 pp",
                "interval_text": "-93.8 pp to -62.5 pp",
                "note": "Directional in this sample",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 25,
                "interval": [
                  0,
                  62.5
                ],
                "difference_text": "+25.0 pp",
                "interval_text": "0.0 pp to +62.5 pp",
                "note": "No clear separation",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 25,
                "interval": [
                  6.25,
                  50
                ],
                "difference_text": "+25.0 pp",
                "interval_text": "+6.3 pp to +50.0 pp",
                "note": "Directional in this sample",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 0,
                "interval": [
                  -25,
                  25
                ],
                "difference_text": "0.0 pp",
                "interval_text": "-25.0 pp to +25.0 pp",
                "note": "No clear separation",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -56.25,
                "interval": [
                  -81.25,
                  -31.25
                ],
                "difference_text": "-56.3 pp",
                "interval_text": "-81.3 pp to -31.3 pp",
                "note": "Directional in this sample",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"pi\",\"0.73.1\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 0,
                "interval": [
                  -25,
                  31.25
                ],
                "difference_text": "0.0 pp",
                "interval_text": "-25.0 pp to +31.3 pp",
                "note": "No clear separation",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": -25,
                "interval": [
                  -50,
                  -6.25
                ],
                "difference_text": "-25.0 pp",
                "interval_text": "-50.0 pp to -6.3 pp",
                "note": "Directional in this sample",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": -25,
                "interval": [
                  -50,
                  -6.25
                ],
                "difference_text": "-25.0 pp",
                "interval_text": "-50.0 pp to -6.3 pp",
                "note": "Directional in this sample",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -81.25,
                "interval": [
                  -100,
                  -56.25
                ],
                "difference_text": "-81.3 pp",
                "interval_text": "-100.0 pp to -56.3 pp",
                "note": "Directional in this sample",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"qwen\",\"0.22.2\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 25,
                "interval": [
                  -12.5,
                  62.5
                ],
                "difference_text": "+25.0 pp",
                "interval_text": "-12.5 pp to +62.5 pp",
                "note": "No clear separation",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 25,
                "interval": [
                  6.25,
                  50
                ],
                "difference_text": "+25.0 pp",
                "interval_text": "+6.3 pp to +50.0 pp",
                "note": "Directional in this sample",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": [
                  -25,
                  25
                ],
                "difference_text": "0.0 pp",
                "interval_text": "-25.0 pp to +25.0 pp",
                "note": "No clear separation",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -56.25,
                "interval": [
                  -87.5,
                  -25
                ],
                "difference_text": "-56.3 pp",
                "interval_text": "-87.5 pp to -25.0 pp",
                "note": "Directional in this sample",
                "label": "cline @ 3.0.61"
              }
            ]
          },
          "total": "All harnesses combined: $10.088 for 80 selected runs at the shared reference prices.",
          "rows": [
            {
              "harness": "cline",
              "version": "3.0.61",
              "selected": 16,
              "passes": 6,
              "pass_rate": 37.5,
              "tasks": 8,
              "cost": {
                "value": 0.30820311562499975,
                "n": 16
              },
              "benchmark": {
                "value": 4.931249850000005,
                "n": 16
              },
              "cache": {
                "value": 96.44368797122053,
                "n": 16
              },
              "input": {
                "value": 12672609.5,
                "n": 16
              },
              "output": {
                "value": 163416,
                "n": 16
              },
              "estimated": true,
              "id": "[\"cline\",\"3.0.61\"]",
              "label": "cline @ 3.0.61",
              "recorded": {
                "passes": 0,
                "pass_rate": 0,
                "pass_score": 0
              },
              "pass_estimate": {
                "passed": 6,
                "total": 16,
                "rate": 37.5
              },
              "pass_display": {
                "passed": 6,
                "rate": 37.5,
                "estimated": true,
                "score": 46.15384615384615
              },
              "scores": {
                "pass": 46.15384615384615,
                "cost": 20.222160919305296,
                "benchmark": 20.222160919305253,
                "cache": 97.3718660349374,
                "input": 35.05667084589011,
                "output": 35.64950800411221
              },
              "display": {
                "pass": "37.5%",
                "cost": "$0.308",
                "benchmark": "$4.931",
                "cache": "96.4%",
                "input": "12.67M",
                "output": "163.4K"
              }
            },
            {
              "harness": "codex",
              "version": "codex-cli 0.149.1",
              "selected": 16,
              "passes": 13,
              "pass_rate": 81.25,
              "tasks": 8,
              "cost": {
                "value": 0.07716122587499999,
                "n": 16
              },
              "benchmark": {
                "value": 1.2345796139999983,
                "n": 16
              },
              "cache": {
                "value": 98.5733397753614,
                "n": 16
              },
              "input": {
                "value": 6830421,
                "n": 16
              },
              "output": {
                "value": 73591,
                "n": 16
              },
              "estimated": false,
              "id": "[\"codex\",\"codex-cli 0.149.1\"]",
              "label": "codex @ codex-cli 0.149.1",
              "recorded": {
                "passes": 13,
                "pass_rate": 81.25,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 13,
                "rate": 81.25,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 80.77286136040152,
                "benchmark": 80.77286136040159,
                "cache": 99.52201369660447,
                "input": 65.04130565304833,
                "output": 79.16321289288092
              },
              "display": {
                "pass": "81.3%",
                "cost": "$0.077",
                "benchmark": "$1.235",
                "cache": "98.6%",
                "input": "6.83M",
                "output": "73.6K"
              }
            },
            {
              "harness": "hermes",
              "version": "Hermes Agent v0.20.5 (2026.8.19)",
              "selected": 16,
              "passes": 9,
              "pass_rate": 56.25,
              "tasks": 8,
              "cost": {
                "value": 0.102815898375,
                "n": 16
              },
              "benchmark": {
                "value": 1.645054374000001,
                "n": 16
              },
              "cache": {
                "value": 98.94361733149506,
                "n": 16
              },
              "input": {
                "value": 9497098.5,
                "n": 16
              },
              "output": {
                "value": 74948.5,
                "n": 16
              },
              "estimated": false,
              "id": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)",
              "recorded": {
                "passes": 9,
                "pass_rate": 56.25,
                "pass_score": 69.23076923076923
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 9,
                "rate": 56.25,
                "estimated": false,
                "score": 69.23076923076923
              },
              "scores": {
                "pass": 69.23076923076923,
                "cost": 60.618378076784474,
                "benchmark": 60.61837807678442,
                "cache": 99.89585481933662,
                "input": 46.778445016654295,
                "output": 77.72937417026358
              },
              "display": {
                "pass": "56.3%",
                "cost": "$0.103",
                "benchmark": "$1.645",
                "cache": "98.9%",
                "input": "9.50M",
                "output": "74.9K"
              }
            },
            {
              "harness": "pi",
              "version": "0.73.1",
              "selected": 16,
              "passes": 13,
              "pass_rate": 81.25,
              "tasks": 8,
              "cost": {
                "value": 0.06232533000000001,
                "n": 16
              },
              "benchmark": {
                "value": 0.9972052799999997,
                "n": 16
              },
              "cache": {
                "value": 98.3556967307641,
                "n": 16
              },
              "input": {
                "value": 4442595,
                "n": 16
              },
              "output": {
                "value": 58257,
                "n": 16
              },
              "estimated": false,
              "id": "[\"pi\",\"0.73.1\"]",
              "label": "pi @ 0.73.1",
              "recorded": {
                "passes": 13,
                "pass_rate": 81.25,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 13,
                "rate": 81.25,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 100,
                "benchmark": 100,
                "cache": 99.3022760462951,
                "input": 100,
                "output": 100
              },
              "display": {
                "pass": "81.3%",
                "cost": "$0.062",
                "benchmark": "$0.997",
                "cache": "98.4%",
                "input": "4.44M",
                "output": "58.3K"
              }
            },
            {
              "harness": "qwen",
              "version": "0.22.2",
              "selected": 16,
              "passes": 9,
              "pass_rate": 56.25,
              "tasks": 8,
              "cost": {
                "value": 0.08001148462500002,
                "n": 16
              },
              "benchmark": {
                "value": 1.2801837540000038,
                "n": 16
              },
              "cache": {
                "value": 99.04676976881203,
                "n": 16
              },
              "input": {
                "value": 8035598,
                "n": 16
              },
              "output": {
                "value": 70098,
                "n": 16
              },
              "estimated": false,
              "id": "[\"qwen\",\"0.22.2\"]",
              "label": "qwen @ 0.22.2",
              "recorded": {
                "passes": 9,
                "pass_rate": 56.25,
                "pass_score": 69.23076923076923
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 9,
                "rate": 56.25,
                "estimated": false,
                "score": 69.23076923076923
              },
              "scores": {
                "pass": 69.23076923076923,
                "cost": 77.89547999528824,
                "benchmark": 77.89547999528799,
                "cache": 100,
                "input": 55.286426722690706,
                "output": 83.10793460583754
              },
              "display": {
                "pass": "56.3%",
                "cost": "$0.080",
                "benchmark": "$1.280",
                "cache": "99.0%",
                "input": "8.04M",
                "output": "70.1K"
              }
            }
          ],
          "decision": {
            "html": "<h3>All 8 selected tasks</h3><p>8 distinct tasks; 2 repetitions per task and harness version. Pass differences and cost means use equal task weights; pass counts include every selected attempt. Recorded verifier image/environment identities match within each task. Recorded configuration details are unavailable. Observed differences are not causal and do not establish future superiority. Exploratory 95% task-cluster intervals use 10,000 deterministic resamples; repetitions are not independent tasks or paired random seeds. No correction for multiple comparisons or winner selection. Secondary cost/token intervals describe means, not headline token medians. Original grades remain unchanged; corrected verification is a separate audit with limited replay coverage.</p><p>Time is agent end-to-end elapsed time, excluding setup and verification. All-outcome values include fast failures; successful timing uses only passes. Reference cost includes all selected outcomes and weights tasks equally. Unmeasured workflows: not measured.</p><div class=\"decision-grid\"><article class=\"decision-row\" data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h3>cline @ 3.0.61</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>0 / 16 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.3082</strong><small>n=16; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>20.31 min</strong><small>n=16; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small></dd></div></dl><p class=\"outcome-counts\">0 completed · 16 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.3082; exploratory 95% interval: $0.1730 to $0.4683<small>n=16; missing=0; 8 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h3>codex @ codex-cli 0.149.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>13 / 16 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.07716</strong><small>n=16; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>8.131 min</strong><small>n=16; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>8.238 min</strong><small>n=13; missing=0</small></dd></div></dl><p class=\"outcome-counts\">13 completed · 3 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.07716; exploratory 95% interval: $0.06797 to $0.08644<small>n=16; missing=0; 8 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h3>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>9 / 16 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.1028</strong><small>n=16; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>9.395 min</strong><small>n=16; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>9.425 min</strong><small>n=9; missing=0</small></dd></div></dl><p class=\"outcome-counts\">9 completed · 7 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.1028; exploratory 95% interval: $0.07257 to $0.1396<small>n=16; missing=0; 8 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h3>pi @ 0.73.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>13 / 16 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.06233</strong><small>n=16; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>7.175 min</strong><small>n=16; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>7.682 min</strong><small>n=13; missing=0</small></dd></div></dl><p class=\"outcome-counts\">13 completed · 3 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.06233; exploratory 95% interval: $0.05310 to $0.07186<small>n=16; missing=0; 8 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h3>qwen @ 0.22.2</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>9 / 16 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.08001</strong><small>n=16; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>8.759 min</strong><small>n=16; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>6.743 min</strong><small>n=9; missing=0</small></dd></div></dl><p class=\"outcome-counts\">9 completed · 7 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.08001; exploratory 95% interval: $0.06430 to $0.09732<small>n=16; missing=0; 8 task clusters</small></p></article></div><div class=\"decision-table-scroll\" tabindex=\"0\" role=\"region\" aria-label=\"Observed differences\"><table class=\"decision-differences\"><caption>Observed differences from <span id=\"decision-reference\">codex @ codex-cli 0.149.1</span>. Reference stays fixed when hidden. Positive means more passes, higher cost or longer time.</caption><thead><tr><th scope=\"col\">Harness version</th><th scope=\"col\">Pass difference</th><th scope=\"col\">Mean cost difference</th><th scope=\"col\">All-outcome median time difference</th><th scope=\"col\">Coverage</th></tr></thead><tbody id=\"decision-differences\"><tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−81.25 pp<small>Exploratory 95%: −93.75 pp to −62.5 pp</small></td><td>+$0.2310<small>+299.4%</small></td><td>+12.18 min<small>+149.8%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>−25 pp<small>Exploratory 95%: −62.5 pp to 0 pp</small></td><td>+$0.02565<small>+33.25%</small></td><td>+1.264 min<small>+15.55%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: −31.25 pp to +25 pp</small></td><td>−$0.01484<small>−19.23%</small></td><td>−57.32 s<small>−11.75%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>−25 pp<small>Exploratory 95%: −62.5 pp to +12.5 pp</small></td><td>+$0.002850<small>+3.694%</small></td><td>+37.7 s<small>+7.727%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr></tbody></table></div><details id=\"decision-explanation\" class=\"joint-explanation\"><summary>Explain time, tokens and failure evidence together</summary><p>API-active union (<code>model_time</code>) is proxy-observed API-active time including transport, not provider compute. Non-model residual (<code>non_model_time</code>) includes unobserved tools and harness work. Timing correlation is a hypothesis, not a diagnosed cause; these observations do not establish a full tool/harness split.</p><p>These are observed median, quartile and min–max distributions, not confidence intervals. Secondary token mean intervals do not replace the headline token medians. No winner-selection or multiple-comparison correction is applied.</p><section data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h4>cline @ 3.0.61</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>20.31 min</strong><small>n=16; missing=0</small><small>Q1–Q3: 12.92 min to 44.01 min; range: 7.655 min to 101.8 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small><small>Q1–Q3: Unavailable; range: Unavailable</small></dd></div><div><dt>API-active union · median</dt><dd><strong>17.46 min</strong><small>n=16; missing=0</small><small>Q1–Q3: 10.66 min to 33.53 min; range: 6.64 min to 72.14 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>3.776 min</strong><small>n=16; missing=0</small><small>Q1–Q3: 58.2 s to 6.473 min; range: 16.03 s to 32.09 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>252.5</strong><small>n=16; missing=0</small><small>Q1–Q3: 122.8 to 574.3; range: 77 to 1,267</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>96.44%</strong><small>n=16; missing=0</small><small>Q1–Q3: 95.82% to 97.35%; range: 94.77% to 97.8%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>12,670,000</strong><small>n=16; missing=0</small><small>Q1–Q3: 6,387,000 to 29,210,000; range: 3,874,000 to 57,170,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>163,400</strong><small>n=16; missing=0</small><small>Q1–Q3: 98,900 to 288,100; range: 69,660 to 604,500</small></dd></div></dl><p>Secondary input Mean 19,820,000; exploratory 95% interval: 11,380,000 to 29,380,000<small>n=16; missing=0; 8 task clusters</small></p><p>Secondary output Mean 227,500; exploratory 95% interval: 138,900 to 329,100<small>n=16; missing=0; 8 task clusters</small></p><p>Visibility: 0 none, 16 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 16 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:cline:cattrs-partial-structuring-recovery:0</code></li><li><code>pinned:cline:cattrs-partial-structuring-recovery:1</code></li><li><code>pinned:cline:happy-dom-deterministic-intersectionobserver:0</code></li><li><code>pinned:cline:happy-dom-deterministic-intersectionobserver:1</code></li><li><code>pinned:cline:ink-grid-box-layout:0</code></li><li><code>pinned:cline:ink-grid-box-layout:1</code></li><li><code>pinned:cline:psd-tools-blend-range-api:0</code></li><li><code>pinned:cline:psd-tools-blend-range-api:1</code></li><li><code>pinned:cline:superjson-error-stack-serialization:0</code></li><li><code>pinned:cline:superjson-error-stack-serialization:1</code></li><li><code>pinned:cline:textual-richlog-follow-state:0</code></li><li><code>pinned:cline:textual-richlog-follow-state:1</code></li><li><code>pinned:cline:tomlkit-toml-table-converters:0</code></li><li><code>pinned:cline:tomlkit-toml-table-converters:1</code></li><li><code>pinned:cline:true-myth-iterable-collection-combinators:0</code></li><li><code>pinned:cline:true-myth-iterable-collection-combinators:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h4>codex @ codex-cli 0.149.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>8.131 min</strong><small>n=16; missing=0</small><small>Q1–Q3: 6.785 min to 10.25 min; range: 4.179 min to 15.63 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>8.238 min</strong><small>n=13; missing=0</small><small>Q1–Q3: 7.273 min to 10.09 min; range: 4.179 min to 15.63 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>6.46 min</strong><small>n=16; missing=0</small><small>Q1–Q3: 5.912 min to 8.047 min; range: 3.698 min to 10.52 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>1.46 min</strong><small>n=16; missing=0</small><small>Q1–Q3: 25.71 s to 2.351 min; range: 14.64 s to 5.715 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>64</strong><small>n=16; missing=0</small><small>Q1–Q3: 50.5 to 77.5; range: 30 to 95</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.57%</strong><small>n=16; missing=0</small><small>Q1–Q3: 98.47% to 98.93%; range: 96.8% to 99.1%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>6,830,000</strong><small>n=16; missing=0</small><small>Q1–Q3: 4,251,000 to 8,473,000; range: 2,412,000 to 11,510,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>73,590</strong><small>n=16; missing=0</small><small>Q1–Q3: 65,060 to 83,600; range: 43,700 to 102,100</small></dd></div></dl><p>Secondary input Mean 6,668,000; exploratory 95% interval: 5,344,000 to 7,972,000<small>n=16; missing=0; 8 task clusters</small></p><p>Secondary output Mean 73,880; exploratory 95% interval: 65,160 to 83,250<small>n=16; missing=0; 8 task clusters</small></p><p>Visibility: 0 none, 16 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 3 verification unknown; 13 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:codex:cattrs-partial-structuring-recovery:0</code></li><li><code>pinned:codex:cattrs-partial-structuring-recovery:1</code></li><li><code>pinned:codex:happy-dom-deterministic-intersectionobserver:0</code></li><li><code>pinned:codex:happy-dom-deterministic-intersectionobserver:1</code></li><li><code>pinned:codex:ink-grid-box-layout:0</code></li><li><code>pinned:codex:ink-grid-box-layout:1</code></li><li><code>pinned:codex:psd-tools-blend-range-api:0</code></li><li><code>pinned:codex:psd-tools-blend-range-api:1</code></li><li><code>pinned:codex:superjson-error-stack-serialization:0</code></li><li><code>pinned:codex:superjson-error-stack-serialization:1</code></li><li><code>pinned:codex:textual-richlog-follow-state:0</code></li><li><code>pinned:codex:textual-richlog-follow-state:1</code></li><li><code>pinned:codex:tomlkit-toml-table-converters:0</code></li><li><code>pinned:codex:tomlkit-toml-table-converters:1</code></li><li><code>pinned:codex:true-myth-iterable-collection-combinators:0</code></li><li><code>pinned:codex:true-myth-iterable-collection-combinators:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h4>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>9.395 min</strong><small>n=16; missing=0</small><small>Q1–Q3: 7.938 min to 14.17 min; range: 4.381 min to 24.29 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>9.425 min</strong><small>n=9; missing=0</small><small>Q1–Q3: 8.785 min to 15.54 min; range: 7.461 min to 21.32 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>7.353 min</strong><small>n=16; missing=0</small><small>Q1–Q3: 6.502 min to 9.697 min; range: 3.857 min to 20.46 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>1.642 min</strong><small>n=16; missing=0</small><small>Q1–Q3: 1.091 min to 3.912 min; range: 24.37 s to 7.605 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>98.5</strong><small>n=16; missing=0</small><small>Q1–Q3: 68.75 to 111.5; range: 33 to 195</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.94%</strong><small>n=16; missing=0</small><small>Q1–Q3: 98.66% to 99.24%; range: 96.98% to 99.56%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>9,497,000</strong><small>n=16; missing=0</small><small>Q1–Q3: 5,911,000 to 13,610,000; range: 2,305,000 to 44,120,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>74,950</strong><small>n=16; missing=0</small><small>Q1–Q3: 64,880 to 93,700; range: 43,960 to 192,200</small></dd></div></dl><p>Secondary input Mean 12,450,000; exploratory 95% interval: 7,016,000 to 18,980,000<small>n=16; missing=0; 8 task clusters</small></p><p>Secondary output Mean 84,660; exploratory 95% interval: 65,720 to 108,500<small>n=16; missing=0; 8 task clusters</small></p><p>Visibility: 0 none, 16 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 9 pass; 7 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:hermes:cattrs-partial-structuring-recovery:0</code></li><li><code>pinned:hermes:cattrs-partial-structuring-recovery:1</code></li><li><code>pinned:hermes:happy-dom-deterministic-intersectionobserver:0</code></li><li><code>pinned:hermes:happy-dom-deterministic-intersectionobserver:1</code></li><li><code>pinned:hermes:ink-grid-box-layout:0</code></li><li><code>pinned:hermes:ink-grid-box-layout:1</code></li><li><code>pinned:hermes:psd-tools-blend-range-api:0</code></li><li><code>pinned:hermes:psd-tools-blend-range-api:1</code></li><li><code>pinned:hermes:superjson-error-stack-serialization:0</code></li><li><code>pinned:hermes:superjson-error-stack-serialization:1</code></li><li><code>pinned:hermes:textual-richlog-follow-state:0</code></li><li><code>pinned:hermes:textual-richlog-follow-state:1</code></li><li><code>pinned:hermes:tomlkit-toml-table-converters:0</code></li><li><code>pinned:hermes:tomlkit-toml-table-converters:1</code></li><li><code>pinned:hermes:true-myth-iterable-collection-combinators:0</code></li><li><code>pinned:hermes:true-myth-iterable-collection-combinators:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h4>pi @ 0.73.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>7.175 min</strong><small>n=16; missing=0</small><small>Q1–Q3: 5.978 min to 9.584 min; range: 4.027 min to 12.59 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>7.682 min</strong><small>n=13; missing=0</small><small>Q1–Q3: 5.901 min to 9.66 min; range: 4.027 min to 12.59 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>5.784 min</strong><small>n=16; missing=0</small><small>Q1–Q3: 4.873 min to 6.268 min; range: 3.655 min to 9.585 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>1.273 min</strong><small>n=16; missing=0</small><small>Q1–Q3: 22.09 s to 2.701 min; range: 13.89 s to 4.251 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>57</strong><small>n=16; missing=0</small><small>Q1–Q3: 49 to 66.25; range: 33 to 109</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.36%</strong><small>n=16; missing=0</small><small>Q1–Q3: 98.17% to 98.53%; range: 97.59% to 99.17%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>4,443,000</strong><small>n=16; missing=0</small><small>Q1–Q3: 2,989,000 to 5,413,000; range: 2,082,000 to 10,930,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>58,260</strong><small>n=16; missing=0</small><small>Q1–Q3: 52,520 to 71,780; range: 40,470 to 100,100</small></dd></div></dl><p>Secondary input Mean 4,687,000; exploratory 95% interval: 3,501,000 to 5,868,000<small>n=16; missing=0; 8 task clusters</small></p><p>Secondary output Mean 63,140; exploratory 95% interval: 53,430 to 74,160<small>n=16; missing=0; 8 task clusters</small></p><p>Visibility: 0 none, 16 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 13 pass; 3 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:pi:cattrs-partial-structuring-recovery:0</code></li><li><code>pinned:pi:cattrs-partial-structuring-recovery:1</code></li><li><code>pinned:pi:happy-dom-deterministic-intersectionobserver:0</code></li><li><code>pinned:pi:happy-dom-deterministic-intersectionobserver:1</code></li><li><code>pinned:pi:ink-grid-box-layout:0</code></li><li><code>pinned:pi:ink-grid-box-layout:1</code></li><li><code>pinned:pi:psd-tools-blend-range-api:0</code></li><li><code>pinned:pi:psd-tools-blend-range-api:1</code></li><li><code>pinned:pi:superjson-error-stack-serialization:0</code></li><li><code>pinned:pi:superjson-error-stack-serialization:1</code></li><li><code>pinned:pi:textual-richlog-follow-state:0</code></li><li><code>pinned:pi:textual-richlog-follow-state:1</code></li><li><code>pinned:pi:tomlkit-toml-table-converters:0</code></li><li><code>pinned:pi:tomlkit-toml-table-converters:1</code></li><li><code>pinned:pi:true-myth-iterable-collection-combinators:0</code></li><li><code>pinned:pi:true-myth-iterable-collection-combinators:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h4>qwen @ 0.22.2</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>8.759 min</strong><small>n=16; missing=0</small><small>Q1–Q3: 5.397 min to 11.42 min; range: 3.773 min to 21.33 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>6.743 min</strong><small>n=9; missing=0</small><small>Q1–Q3: 5.466 min to 11.29 min; range: 3.773 min to 21.33 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>6.608 min</strong><small>n=16; missing=0</small><small>Q1–Q3: 4.897 min to 8.495 min; range: 3.424 min to 13.71 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>1.22 min</strong><small>n=16; missing=0</small><small>Q1–Q3: 33.94 s to 2.551 min; range: 10.98 s to 11.71 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>83</strong><small>n=16; missing=0</small><small>Q1–Q3: 65.5 to 101.5; range: 37 to 142</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>99.05%</strong><small>n=16; missing=0</small><small>Q1–Q3: 98.74% to 99.28%; range: 97.92% to 99.56%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>8,036,000</strong><small>n=16; missing=0</small><small>Q1–Q3: 5,445,000 to 10,880,000; range: 2,738,000 to 20,860,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>70,100</strong><small>n=16; missing=0</small><small>Q1–Q3: 54,600 to 83,540; range: 36,980 to 128,700</small></dd></div></dl><p>Secondary input Mean 8,597,000; exploratory 95% interval: 5,996,000 to 11,440,000<small>n=16; missing=0; 8 task clusters</small></p><p>Secondary output Mean 71,310; exploratory 95% interval: 58,890 to 85,080<small>n=16; missing=0; 8 task clusters</small></p><p>Visibility: 0 none, 16 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 9 pass; 7 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:qwen:cattrs-partial-structuring-recovery:0</code></li><li><code>pinned:qwen:cattrs-partial-structuring-recovery:1</code></li><li><code>pinned:qwen:happy-dom-deterministic-intersectionobserver:0</code></li><li><code>pinned:qwen:happy-dom-deterministic-intersectionobserver:1</code></li><li><code>pinned:qwen:ink-grid-box-layout:0</code></li><li><code>pinned:qwen:ink-grid-box-layout:1</code></li><li><code>pinned:qwen:psd-tools-blend-range-api:0</code></li><li><code>pinned:qwen:psd-tools-blend-range-api:1</code></li><li><code>pinned:qwen:superjson-error-stack-serialization:0</code></li><li><code>pinned:qwen:superjson-error-stack-serialization:1</code></li><li><code>pinned:qwen:textual-richlog-follow-state:0</code></li><li><code>pinned:qwen:textual-richlog-follow-state:1</code></li><li><code>pinned:qwen:tomlkit-toml-table-converters:0</code></li><li><code>pinned:qwen:tomlkit-toml-table-converters:1</code></li><li><code>pinned:qwen:true-myth-iterable-collection-combinators:0</code></li><li><code>pinned:qwen:true-myth-iterable-collection-combinators:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><p><a href=\"report.html\">Detailed timing report and request timelines</a> · <a href=\"data/diagnostics.json\">Download diagnostics</a></p></details>",
            "differences": {
              "[\"cline\",\"3.0.61\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+81.25 pp<small>Exploratory 95%: +62.5 pp to +93.75 pp</small></td><td>−$0.2310<small>−74.96%</small></td><td>−12.18 min<small>−59.96%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>+56.25 pp<small>Exploratory 95%: +31.25 pp to +81.25 pp</small></td><td>−$0.2054<small>−66.64%</small></td><td>−10.91 min<small>−53.74%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+81.25 pp<small>Exploratory 95%: +56.25 pp to +100 pp</small></td><td>−$0.2459<small>−79.78%</small></td><td>−13.13 min<small>−64.67%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>+56.25 pp<small>Exploratory 95%: +25 pp to +87.5 pp</small></td><td>−$0.2282<small>−74.04%</small></td><td>−11.55 min<small>−56.87%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr>",
              "[\"codex\",\"codex-cli 0.149.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−81.25 pp<small>Exploratory 95%: −93.75 pp to −62.5 pp</small></td><td>+$0.2310<small>+299.4%</small></td><td>+12.18 min<small>+149.8%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>−25 pp<small>Exploratory 95%: −62.5 pp to 0 pp</small></td><td>+$0.02565<small>+33.25%</small></td><td>+1.264 min<small>+15.55%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: −31.25 pp to +25 pp</small></td><td>−$0.01484<small>−19.23%</small></td><td>−57.32 s<small>−11.75%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>−25 pp<small>Exploratory 95%: −62.5 pp to +12.5 pp</small></td><td>+$0.002850<small>+3.694%</small></td><td>+37.7 s<small>+7.727%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr>",
              "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−56.25 pp<small>Exploratory 95%: −81.25 pp to −31.25 pp</small></td><td>+$0.2054<small>+199.8%</small></td><td>+10.91 min<small>+116.2%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+25 pp<small>Exploratory 95%: 0 pp to +62.5 pp</small></td><td>−$0.02565<small>−24.95%</small></td><td>−1.264 min<small>−13.46%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19) (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+25 pp<small>Exploratory 95%: +6.25 pp to +50 pp</small></td><td>−$0.04049<small>−39.38%</small></td><td>−2.22 min<small>−23.62%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: −25 pp to +25 pp</small></td><td>−$0.02280<small>−22.18%</small></td><td>−38.16 s<small>−6.769%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr>",
              "[\"pi\",\"0.73.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−81.25 pp<small>Exploratory 95%: −100 pp to −56.25 pp</small></td><td>+$0.2459<small>+394.5%</small></td><td>+13.13 min<small>+183%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>0 pp<small>Exploratory 95%: −25 pp to +31.25 pp</small></td><td>+$0.01484<small>+23.8%</small></td><td>+57.32 s<small>+13.31%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>−25 pp<small>Exploratory 95%: −50 pp to −6.25 pp</small></td><td>+$0.04049<small>+64.97%</small></td><td>+2.22 min<small>+30.93%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>−25 pp<small>Exploratory 95%: −50 pp to −6.25 pp</small></td><td>+$0.01769<small>+28.38%</small></td><td>+1.584 min<small>+22.07%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr>",
              "[\"qwen\",\"0.22.2\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−56.25 pp<small>Exploratory 95%: −87.5 pp to −25 pp</small></td><td>+$0.2282<small>+285.2%</small></td><td>+11.55 min<small>+131.9%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+25 pp<small>Exploratory 95%: −12.5 pp to +62.5 pp</small></td><td>−$0.002850<small>−3.562%</small></td><td>−37.7 s<small>−7.173%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: −25 pp to +25 pp</small></td><td>+$0.02280<small>+28.5%</small></td><td>+38.16 s<small>+7.261%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+25 pp<small>Exploratory 95%: +6.25 pp to +50 pp</small></td><td>−$0.01769<small>−22.1%</small></td><td>−1.584 min<small>−18.08%</small></td><td>Exploratory task-cluster pass difference; equal task weights.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr>"
            }
          }
        },
        {
          "id": "[\"cattrs-partial-structuring-recovery\",\"91df27b2a20f26c35777a92f4555438b62c06345\"]",
          "task": "cattrs-partial-structuring-recovery",
          "confidence": [
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"codex\",\"codex-cli 0.149.1\"]",
              "difference_pp": -50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": -50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": -50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": -50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": -50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": -50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"pi\",\"0.73.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            }
          ],
          "confidenceByBaseline": {
            "[\"cline\",\"3.0.61\"]": [
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              }
            ],
            "[\"codex\",\"codex-cli 0.149.1\"]": [
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": [
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"pi\",\"0.73.1\"]": [
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"qwen\",\"0.22.2\"]": [
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ]
          },
          "total": "All harnesses combined: $1.256 for 10 selected runs at the shared reference prices.",
          "rows": [
            {
              "harness": "cline",
              "version": "3.0.61",
              "selected": 2,
              "passes": 0,
              "pass_rate": 0,
              "tasks": 1,
              "cost": {
                "value": 0.3519759030000002,
                "n": 2
              },
              "benchmark": {
                "value": 0.7039518060000004,
                "n": 2
              },
              "cache": {
                "value": 97.60649478350717,
                "n": 2
              },
              "input": {
                "value": 28596106.5,
                "n": 2
              },
              "output": {
                "value": 267557,
                "n": 2
              },
              "estimated": true,
              "id": "[\"cline\",\"3.0.61\"]",
              "label": "cline @ 3.0.61",
              "recorded": {
                "passes": 0,
                "pass_rate": 0,
                "pass_score": 0
              },
              "pass_estimate": {
                "passed": 0,
                "total": 2,
                "rate": 0
              },
              "pass_display": {
                "passed": 0,
                "rate": 0,
                "estimated": true,
                "score": 0
              },
              "scores": {
                "pass": 0,
                "cost": 12.653076423814156,
                "benchmark": 12.653076423814156,
                "cache": 98.66837203054529,
                "input": 10.183770996936243,
                "output": 17.53831893764693
              },
              "display": {
                "pass": "0.0%",
                "cost": "$0.352",
                "benchmark": "$0.704",
                "cache": "97.6%",
                "input": "28.60M",
                "output": "267.6K"
              }
            },
            {
              "harness": "codex",
              "version": "codex-cli 0.149.1",
              "selected": 2,
              "passes": 1,
              "pass_rate": 50,
              "tasks": 1,
              "cost": {
                "value": 0.06852040199999998,
                "n": 2
              },
              "benchmark": {
                "value": 0.13704080399999996,
                "n": 2
              },
              "cache": {
                "value": 98.4147510509751,
                "n": 2
              },
              "input": {
                "value": 6515773,
                "n": 2
              },
              "output": {
                "value": 60015.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"codex\",\"codex-cli 0.149.1\"]",
              "label": "codex @ codex-cli 0.149.1",
              "recorded": {
                "passes": 1,
                "pass_rate": 50,
                "pass_score": 50
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 1,
                "rate": 50,
                "estimated": false,
                "score": 50
              },
              "scores": {
                "pass": 50,
                "cost": 64.99637874278673,
                "benchmark": 64.99637874278673,
                "cache": 99.48542145201493,
                "input": 44.694037069738314,
                "output": 78.18813473186094
              },
              "display": {
                "pass": "50.0%",
                "cost": "$0.069",
                "benchmark": "$0.137",
                "cache": "98.4%",
                "input": "6.52M",
                "output": "60.0K"
              }
            },
            {
              "harness": "hermes",
              "version": "Hermes Agent v0.20.5 (2026.8.19)",
              "selected": 2,
              "passes": 1,
              "pass_rate": 50,
              "tasks": 1,
              "cost": {
                "value": 0.088562802,
                "n": 2
              },
              "benchmark": {
                "value": 0.177125604,
                "n": 2
              },
              "cache": {
                "value": 98.92379166171975,
                "n": 2
              },
              "input": {
                "value": 9331933,
                "n": 2
              },
              "output": {
                "value": 78515.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)",
              "recorded": {
                "passes": 1,
                "pass_rate": 50,
                "pass_score": 50
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 1,
                "rate": 50,
                "estimated": false,
                "score": 50
              },
              "scores": {
                "pass": 50,
                "cost": 50.287230072056666,
                "benchmark": 50.287230072056666,
                "cache": 100,
                "input": 31.206417791469356,
                "output": 59.76526927804065
              },
              "display": {
                "pass": "50.0%",
                "cost": "$0.089",
                "benchmark": "$0.177",
                "cache": "98.9%",
                "input": "9.33M",
                "output": "78.5K"
              }
            },
            {
              "harness": "pi",
              "version": "0.73.1",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.044535780000000004,
                "n": 2
              },
              "benchmark": {
                "value": 0.08907156000000001,
                "n": 2
              },
              "cache": {
                "value": 98.08129545057017,
                "n": 2
              },
              "input": {
                "value": 2912162,
                "n": 2
              },
              "output": {
                "value": 46925,
                "n": 2
              },
              "estimated": false,
              "id": "[\"pi\",\"0.73.1\"]",
              "label": "pi @ 0.73.1",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 100,
                "benchmark": 100,
                "cache": 99.1483381328219,
                "input": 100,
                "output": 100
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.045",
                "benchmark": "$0.089",
                "cache": "98.1%",
                "input": "2.91M",
                "output": "46.9K"
              }
            },
            {
              "harness": "qwen",
              "version": "0.22.2",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.07440574200000005,
                "n": 2
              },
              "benchmark": {
                "value": 0.1488114840000001,
                "n": 2
              },
              "cache": {
                "value": 98.62538798115148,
                "n": 2
              },
              "input": {
                "value": 6611401,
                "n": 2
              },
              "output": {
                "value": 68183,
                "n": 2
              },
              "estimated": false,
              "id": "[\"qwen\",\"0.22.2\"]",
              "label": "qwen @ 0.22.2",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 59.85529987725944,
                "benchmark": 59.85529987725944,
                "cache": 99.6983499362937,
                "input": 44.04757781293254,
                "output": 68.82214041623278
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.074",
                "benchmark": "$0.149",
                "cache": "98.6%",
                "input": "6.61M",
                "output": "68.2K"
              }
            }
          ],
          "decision": {
            "html": "<h3>cattrs-partial-structuring-recovery</h3><p>1 distinct task; 2 repetitions per task and harness version. Pass differences and cost means use equal task weights; pass counts include every selected attempt. Recorded verifier image/environment identities match within each task. Recorded configuration details are unavailable. Observed differences are not causal and do not establish future superiority. Exploratory 95% task-cluster intervals use 10,000 deterministic resamples; repetitions are not independent tasks or paired random seeds. No correction for multiple comparisons or winner selection. Secondary cost/token intervals describe means, not headline token medians. Original grades remain unchanged; corrected verification is a separate audit with limited replay coverage.</p><p>Time is agent end-to-end elapsed time, excluding setup and verification. All-outcome values include fast failures; successful timing uses only passes. Reference cost includes all selected outcomes and weights tasks equally. Unmeasured workflows: not measured.</p><div class=\"decision-grid\"><article class=\"decision-row\" data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h3>cline @ 3.0.61</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>0 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.3520</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>42.85 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small></dd></div></dl><p class=\"outcome-counts\">0 completed · 2 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.3520; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h3>codex @ codex-cli 0.149.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>1 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.06852</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>7.605 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>8.416 min</strong><small>n=1; missing=0</small></dd></div></dl><p class=\"outcome-counts\">1 completed · 1 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.06852; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h3>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>1 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.08856</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>11.4 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>13.43 min</strong><small>n=1; missing=0</small></dd></div></dl><p class=\"outcome-counts\">1 completed · 1 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.08856; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h3>pi @ 0.73.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.04454</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>6.791 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>6.791 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.04454; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h3>qwen @ 0.22.2</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.07441</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>8.392 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>8.392 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.07441; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article></div><div class=\"decision-table-scroll\" tabindex=\"0\" role=\"region\" aria-label=\"Observed differences\"><table class=\"decision-differences\"><caption>Observed differences from <span id=\"decision-reference\">codex @ codex-cli 0.149.1</span>. Reference stays fixed when hidden. Positive means more passes, higher cost or longer time.</caption><thead><tr><th scope=\"col\">Harness version</th><th scope=\"col\">Pass difference</th><th scope=\"col\">Mean cost difference</th><th scope=\"col\">All-outcome median time difference</th><th scope=\"col\">Coverage</th></tr></thead><tbody id=\"decision-differences\"><tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.2835<small>+413.7%</small></td><td>+35.25 min<small>+463.5%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.02004<small>+29.25%</small></td><td>+3.791 min<small>+49.84%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.02398<small>−35%</small></td><td>−48.82 s<small>−10.7%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.005885<small>+8.589%</small></td><td>+47.23 s<small>+10.35%</small></td><td>one task; interval unavailable.</td></tr></tbody></table></div><details id=\"decision-explanation\" class=\"joint-explanation\"><summary>Explain time, tokens and failure evidence together</summary><p>API-active union (<code>model_time</code>) is proxy-observed API-active time including transport, not provider compute. Non-model residual (<code>non_model_time</code>) includes unobserved tools and harness work. Timing correlation is a hypothesis, not a diagnosed cause; these observations do not establish a full tool/harness split.</p><p>These are observed median, quartile and min–max distributions, not confidence intervals. Secondary token mean intervals do not replace the headline token medians. No winner-selection or multiple-comparison correction is applied.</p><section data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h4>cline @ 3.0.61</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>42.85 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 29.9 min to 55.8 min; range: 16.96 min to 68.75 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small><small>Q1–Q3: Unavailable; range: Unavailable</small></dd></div><div><dt>API-active union · median</dt><dd><strong>31 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 21.76 min to 40.23 min; range: 12.53 min to 49.46 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>11.84 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 8.119 min to 15.55 min; range: 4.401 min to 19.27 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>576</strong><small>n=2; missing=0</small><small>Q1–Q3: 398 to 754; range: 220 to 932</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>97.61%</strong><small>n=2; missing=0</small><small>Q1–Q3: 97.51% to 97.7%; range: 97.41% to 97.8%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>28,600,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 19,730,000 to 37,460,000; range: 10,870,000 to 46,330,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>267,600</strong><small>n=2; missing=0</small><small>Q1–Q3: 186,400 to 348,800; range: 105,200 to 430,000</small></dd></div></dl><p>Secondary input Mean 28,600,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 267,600; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:cline:cattrs-partial-structuring-recovery:0</code></li><li><code>pinned:cline:cattrs-partial-structuring-recovery:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h4>codex @ codex-cli 0.149.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>7.605 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 7.2 min to 8.01 min; range: 6.795 min to 8.416 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>8.416 min</strong><small>n=1; missing=0</small><small>Q1–Q3: Unavailable; range: 8.416 min to 8.416 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>5.544 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 5.077 min to 6.011 min; range: 4.61 min to 6.478 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>2.051 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 1.989 min to 2.113 min; range: 1.927 min to 2.175 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>62</strong><small>n=2; missing=0</small><small>Q1–Q3: 51.5 to 72.5; range: 41 to 83</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.41%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.12% to 98.7%; range: 97.84% to 98.99%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>6,516,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 5,213,000 to 7,818,000; range: 3,911,000 to 9,121,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>60,020</strong><small>n=2; missing=0</small><small>Q1–Q3: 57,800 to 62,230; range: 55,590 to 64,440</small></dd></div></dl><p>Secondary input Mean 6,516,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 60,020; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 1 verification unknown; 1 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:codex:cattrs-partial-structuring-recovery:0</code></li><li><code>pinned:codex:cattrs-partial-structuring-recovery:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h4>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>11.4 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 10.38 min to 12.41 min; range: 9.365 min to 13.43 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>13.43 min</strong><small>n=1; missing=0</small><small>Q1–Q3: Unavailable; range: 13.43 min to 13.43 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>7.803 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 7.206 min to 8.401 min; range: 6.608 min to 8.998 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>3.523 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 3.105 min to 3.941 min; range: 2.687 min to 4.358 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>87.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 76.25 to 98.75; range: 65 to 110</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.92%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.78% to 99.06%; range: 98.64% to 99.2%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>9,332,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 7,746,000 to 10,920,000; range: 6,159,000 to 12,500,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>78,520</strong><small>n=2; missing=0</small><small>Q1–Q3: 72,530 to 84,500; range: 66,540 to 90,490</small></dd></div></dl><p>Secondary input Mean 9,332,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 78,520; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 1 pass; 1 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:hermes:cattrs-partial-structuring-recovery:0</code></li><li><code>pinned:hermes:cattrs-partial-structuring-recovery:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h4>pi @ 0.73.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>6.791 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 6.346 min to 7.237 min; range: 5.901 min to 7.682 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>6.791 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 6.346 min to 7.237 min; range: 5.901 min to 7.682 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>4.312 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 3.983 min to 4.64 min; range: 3.655 min to 4.968 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>2.467 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 2.35 min to 2.584 min; range: 2.233 min to 2.701 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>48</strong><small>n=2; missing=0</small><small>Q1–Q3: 42.5 to 53.5; range: 37 to 59</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.08%</strong><small>n=2; missing=0</small><small>Q1–Q3: 97.83% to 98.33%; range: 97.59% to 98.58%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>2,912,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 2,520,000 to 3,304,000; range: 2,128,000 to 3,697,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>46,930</strong><small>n=2; missing=0</small><small>Q1–Q3: 43,700 to 50,150; range: 40,470 to 53,380</small></dd></div></dl><p>Secondary input Mean 2,912,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 46,930; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:pi:cattrs-partial-structuring-recovery:0</code></li><li><code>pinned:pi:cattrs-partial-structuring-recovery:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h4>qwen @ 0.22.2</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>8.392 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 7.568 min to 9.217 min; range: 6.743 min to 10.04 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>8.392 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 7.568 min to 9.217 min; range: 6.743 min to 10.04 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>6.373 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 5.695 min to 7.051 min; range: 5.017 min to 7.729 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>1.996 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 1.849 min to 2.142 min; range: 1.702 min to 2.289 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>77.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 72.25 to 82.75; range: 67 to 88</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.63%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.44% to 98.81%; range: 98.26% to 98.99%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>6,611,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 6,327,000 to 6,896,000; range: 6,043,000 to 7,180,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>68,180</strong><small>n=2; missing=0</small><small>Q1–Q3: 59,590 to 76,770; range: 51,000 to 85,370</small></dd></div></dl><p>Secondary input Mean 6,611,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 68,180; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:qwen:cattrs-partial-structuring-recovery:0</code></li><li><code>pinned:qwen:cattrs-partial-structuring-recovery:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><p><a href=\"report.html\">Detailed timing report and request timelines</a> · <a href=\"data/diagnostics.json\">Download diagnostics</a></p></details>",
            "differences": {
              "[\"cline\",\"3.0.61\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.2835<small>−80.53%</small></td><td>−35.25 min<small>−82.25%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.2634<small>−74.84%</small></td><td>−31.46 min<small>−73.41%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.3074<small>−87.35%</small></td><td>−36.06 min<small>−84.15%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.2776<small>−78.86%</small></td><td>−34.46 min<small>−80.42%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"codex\",\"codex-cli 0.149.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.2835<small>+413.7%</small></td><td>+35.25 min<small>+463.5%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.02004<small>+29.25%</small></td><td>+3.791 min<small>+49.84%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.02398<small>−35%</small></td><td>−48.82 s<small>−10.7%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.005885<small>+8.589%</small></td><td>+47.23 s<small>+10.35%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.2634<small>+297.4%</small></td><td>+31.46 min<small>+276%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.02004<small>−22.63%</small></td><td>−3.791 min<small>−33.26%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19) (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.04403<small>−49.71%</small></td><td>−4.604 min<small>−40.4%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.01416<small>−15.99%</small></td><td>−3.004 min<small>−26.36%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"pi\",\"0.73.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.3074<small>+690.3%</small></td><td>+36.06 min<small>+531%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.02398<small>+53.85%</small></td><td>+48.82 s<small>+11.98%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.04403<small>+98.86%</small></td><td>+4.604 min<small>+67.8%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.02987<small>+67.07%</small></td><td>+1.601 min<small>+23.57%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"qwen\",\"0.22.2\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.2776<small>+373%</small></td><td>+34.46 min<small>+410.6%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.005885<small>−7.91%</small></td><td>−47.23 s<small>−9.379%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.01416<small>+19.03%</small></td><td>+3.004 min<small>+35.79%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.02987<small>−40.14%</small></td><td>−1.601 min<small>−19.08%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr>"
            }
          }
        },
        {
          "id": "[\"happy-dom-deterministic-intersectionobserver\",\"32b2a8e60eab482c023af377160bf294d6da0318\"]",
          "task": "happy-dom-deterministic-intersectionobserver",
          "confidence": [
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"codex\",\"codex-cli 0.149.1\"]",
              "difference_pp": -50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": -50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": -50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"pi\",\"0.73.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            }
          ],
          "confidenceByBaseline": {
            "[\"cline\",\"3.0.61\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              }
            ],
            "[\"codex\",\"codex-cli 0.149.1\"]": [
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              }
            ],
            "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              }
            ],
            "[\"pi\",\"0.73.1\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              }
            ],
            "[\"qwen\",\"0.22.2\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ]
          },
          "total": "All harnesses combined: $0.919 for 10 selected runs at the shared reference prices.",
          "rows": [
            {
              "harness": "cline",
              "version": "3.0.61",
              "selected": 2,
              "passes": 0,
              "pass_rate": 0,
              "tasks": 1,
              "cost": {
                "value": 0.1814994869999998,
                "n": 2
              },
              "benchmark": {
                "value": 0.3629989739999996,
                "n": 2
              },
              "cache": {
                "value": 96.16520065420457,
                "n": 2
              },
              "input": {
                "value": 10706940.5,
                "n": 2
              },
              "output": {
                "value": 139832.5,
                "n": 2
              },
              "estimated": true,
              "id": "[\"cline\",\"3.0.61\"]",
              "label": "cline @ 3.0.61",
              "recorded": {
                "passes": 0,
                "pass_rate": 0,
                "pass_score": 0
              },
              "pass_estimate": {
                "passed": 0,
                "total": 2,
                "rate": 0
              },
              "pass_display": {
                "passed": 0,
                "rate": 0,
                "estimated": true,
                "score": 0
              },
              "scores": {
                "pass": 0,
                "cost": 32.44325368258482,
                "benchmark": 32.44325368258482,
                "cache": 97.13146172569802,
                "input": 38.65446436355932,
                "output": 41.92551803050078
              },
              "display": {
                "pass": "0.0%",
                "cost": "$0.181",
                "benchmark": "$0.363",
                "cache": "96.2%",
                "input": "10.71M",
                "output": "139.8K"
              }
            },
            {
              "harness": "codex",
              "version": "codex-cli 0.149.1",
              "selected": 2,
              "passes": 1,
              "pass_rate": 50,
              "tasks": 1,
              "cost": {
                "value": 0.06599297699999998,
                "n": 2
              },
              "benchmark": {
                "value": 0.13198595399999996,
                "n": 2
              },
              "cache": {
                "value": 98.20931830903106,
                "n": 2
              },
              "input": {
                "value": 5165473.5,
                "n": 2
              },
              "output": {
                "value": 61746,
                "n": 2
              },
              "estimated": false,
              "id": "[\"codex\",\"codex-cli 0.149.1\"]",
              "label": "codex @ codex-cli 0.149.1",
              "recorded": {
                "passes": 1,
                "pass_rate": 50,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 1,
                "rate": 50,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 89.22819014514228,
                "benchmark": 89.22819014514228,
                "cache": 99.19611852880243,
                "input": 80.12257733971533,
                "output": 94.94623133482331
              },
              "display": {
                "pass": "50.0%",
                "cost": "$0.066",
                "benchmark": "$0.132",
                "cache": "98.2%",
                "input": "5.17M",
                "output": "61.7K"
              }
            },
            {
              "harness": "hermes",
              "version": "Hermes Agent v0.20.5 (2026.8.19)",
              "selected": 2,
              "passes": 1,
              "pass_rate": 50,
              "tasks": 1,
              "cost": {
                "value": 0.07690026899999997,
                "n": 2
              },
              "benchmark": {
                "value": 0.15380053799999993,
                "n": 2
              },
              "cache": {
                "value": 98.81560350125174,
                "n": 2
              },
              "input": {
                "value": 6800481.5,
                "n": 2
              },
              "output": {
                "value": 74948.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)",
              "recorded": {
                "passes": 1,
                "pass_rate": 50,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 1,
                "rate": 50,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 76.57234462989982,
                "benchmark": 76.57234462989982,
                "cache": 99.80849563135533,
                "input": 60.85908034600197,
                "output": 78.22104511764745
              },
              "display": {
                "pass": "50.0%",
                "cost": "$0.077",
                "benchmark": "$0.154",
                "cache": "98.8%",
                "input": "6.80M",
                "output": "74.9K"
              }
            },
            {
              "harness": "pi",
              "version": "0.73.1",
              "selected": 2,
              "passes": 1,
              "pass_rate": 50,
              "tasks": 1,
              "cost": {
                "value": 0.05888433899999999,
                "n": 2
              },
              "benchmark": {
                "value": 0.11776867799999997,
                "n": 2
              },
              "cache": {
                "value": 98.11470055931659,
                "n": 2
              },
              "input": {
                "value": 4138710.5,
                "n": 2
              },
              "output": {
                "value": 58625.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"pi\",\"0.73.1\"]",
              "label": "pi @ 0.73.1",
              "recorded": {
                "passes": 1,
                "pass_rate": 50,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 1,
                "rate": 50,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 100,
                "benchmark": 100,
                "cache": 99.10055006669303,
                "input": 100,
                "output": 100
              },
              "display": {
                "pass": "50.0%",
                "cost": "$0.059",
                "benchmark": "$0.118",
                "cache": "98.1%",
                "input": "4.14M",
                "output": "58.6K"
              }
            },
            {
              "harness": "qwen",
              "version": "0.22.2",
              "selected": 2,
              "passes": 0,
              "pass_rate": 0,
              "tasks": 1,
              "cost": {
                "value": 0.076048986,
                "n": 2
              },
              "benchmark": {
                "value": 0.152097972,
                "n": 2
              },
              "cache": {
                "value": 99.00520278977969,
                "n": 2
              },
              "input": {
                "value": 7621791,
                "n": 2
              },
              "output": {
                "value": 70098,
                "n": 2
              },
              "estimated": false,
              "id": "[\"qwen\",\"0.22.2\"]",
              "label": "qwen @ 0.22.2",
              "recorded": {
                "passes": 0,
                "pass_rate": 0,
                "pass_score": 0
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 0,
                "rate": 0,
                "estimated": false,
                "score": 0
              },
              "scores": {
                "pass": 0,
                "cost": 77.42948604206241,
                "benchmark": 77.42948604206241,
                "cache": 100,
                "input": 54.30102321094872,
                "output": 83.63362720762362
              },
              "display": {
                "pass": "0.0%",
                "cost": "$0.076",
                "benchmark": "$0.152",
                "cache": "99.0%",
                "input": "7.62M",
                "output": "70.1K"
              }
            }
          ],
          "decision": {
            "html": "<h3>happy-dom-deterministic-intersectionobserver</h3><p>1 distinct task; 2 repetitions per task and harness version. Pass differences and cost means use equal task weights; pass counts include every selected attempt. Recorded verifier image/environment identities match within each task. Recorded configuration details are unavailable. Observed differences are not causal and do not establish future superiority. Exploratory 95% task-cluster intervals use 10,000 deterministic resamples; repetitions are not independent tasks or paired random seeds. No correction for multiple comparisons or winner selection. Secondary cost/token intervals describe means, not headline token medians. Original grades remain unchanged; corrected verification is a separate audit with limited replay coverage.</p><p>Time is agent end-to-end elapsed time, excluding setup and verification. All-outcome values include fast failures; successful timing uses only passes. Reference cost includes all selected outcomes and weights tasks equally. Unmeasured workflows: not measured.</p><div class=\"decision-grid\"><article class=\"decision-row\" data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h3>cline @ 3.0.61</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>0 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.1815</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>18.11 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small></dd></div></dl><p class=\"outcome-counts\">0 completed · 2 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.1815; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h3>codex @ codex-cli 0.149.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>1 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.06599</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>7.058 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>8.238 min</strong><small>n=1; missing=0</small></dd></div></dl><p class=\"outcome-counts\">1 completed · 1 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.06599; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h3>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>1 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.07690</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>10.81 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>9.425 min</strong><small>n=1; missing=0</small></dd></div></dl><p class=\"outcome-counts\">1 completed · 1 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.07690; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h3>pi @ 0.73.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>1 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.05888</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>7.624 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>6.616 min</strong><small>n=1; missing=0</small></dd></div></dl><p class=\"outcome-counts\">1 completed · 1 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.05888; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h3>qwen @ 0.22.2</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>0 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.07605</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>8.759 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small></dd></div></dl><p class=\"outcome-counts\">0 completed · 2 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.07605; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article></div><div class=\"decision-table-scroll\" tabindex=\"0\" role=\"region\" aria-label=\"Observed differences\"><table class=\"decision-differences\"><caption>Observed differences from <span id=\"decision-reference\">codex @ codex-cli 0.149.1</span>. Reference stays fixed when hidden. Positive means more passes, higher cost or longer time.</caption><thead><tr><th scope=\"col\">Harness version</th><th scope=\"col\">Pass difference</th><th scope=\"col\">Mean cost difference</th><th scope=\"col\">All-outcome median time difference</th><th scope=\"col\">Coverage</th></tr></thead><tbody id=\"decision-differences\"><tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.1155<small>+175%</small></td><td>+11.05 min<small>+156.6%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.01091<small>+16.53%</small></td><td>+3.756 min<small>+53.21%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.007109<small>−10.77%</small></td><td>+33.98 s<small>+8.024%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.01006<small>+15.24%</small></td><td>+1.701 min<small>+24.1%</small></td><td>one task; interval unavailable.</td></tr></tbody></table></div><details id=\"decision-explanation\" class=\"joint-explanation\"><summary>Explain time, tokens and failure evidence together</summary><p>API-active union (<code>model_time</code>) is proxy-observed API-active time including transport, not provider compute. Non-model residual (<code>non_model_time</code>) includes unobserved tools and harness work. Timing correlation is a hypothesis, not a diagnosed cause; these observations do not establish a full tool/harness split.</p><p>These are observed median, quartile and min–max distributions, not confidence intervals. Secondary token mean intervals do not replace the headline token medians. No winner-selection or multiple-comparison correction is applied.</p><section data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h4>cline @ 3.0.61</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>18.11 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 15.76 min to 20.46 min; range: 13.41 min to 22.81 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small><small>Q1–Q3: Unavailable; range: Unavailable</small></dd></div><div><dt>API-active union · median</dt><dd><strong>14.96 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 12.11 min to 17.81 min; range: 9.259 min to 20.66 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>3.12 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 2.618 min to 3.623 min; range: 2.115 min to 4.126 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>226.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 174.8 to 278.3; range: 123 to 330</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>96.17%</strong><small>n=2; missing=0</small><small>Q1–Q3: 95.8% to 96.53%; range: 95.44% to 96.89%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>10,710,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 8,288,000 to 13,130,000; range: 5,870,000 to 15,540,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>139,800</strong><small>n=2; missing=0</small><small>Q1–Q3: 111,800 to 167,900; range: 83,710 to 196,000</small></dd></div></dl><p>Secondary input Mean 10,710,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 139,800; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:cline:happy-dom-deterministic-intersectionobserver:0</code></li><li><code>pinned:cline:happy-dom-deterministic-intersectionobserver:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h4>codex @ codex-cli 0.149.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>7.058 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 6.468 min to 7.648 min; range: 5.878 min to 8.238 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>8.238 min</strong><small>n=1; missing=0</small><small>Q1–Q3: Unavailable; range: 8.238 min to 8.238 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>5.417 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 5.102 min to 5.733 min; range: 4.786 min to 6.049 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>1.614 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 1.336 min to 1.892 min; range: 1.058 min to 2.169 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>56.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 52.75 to 60.25; range: 49 to 64</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.21%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.15% to 98.27%; range: 98.08% to 98.34%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>5,165,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 4,765,000 to 5,566,000; range: 4,365,000 to 5,966,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>61,750</strong><small>n=2; missing=0</small><small>Q1–Q3: 59,980 to 63,510; range: 58,220 to 65,270</small></dd></div></dl><p>Secondary input Mean 5,165,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 61,750; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 1 pass; 1 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:codex:happy-dom-deterministic-intersectionobserver:0</code></li><li><code>pinned:codex:happy-dom-deterministic-intersectionobserver:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h4>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>10.81 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 10.12 min to 11.51 min; range: 9.425 min to 12.2 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>9.425 min</strong><small>n=1; missing=0</small><small>Q1–Q3: Unavailable; range: 9.425 min to 9.425 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>7.531 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 7.425 min to 7.637 min; range: 7.319 min to 7.743 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>3.18 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 2.392 min to 3.969 min; range: 1.604 min to 4.757 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>83</strong><small>n=2; missing=0</small><small>Q1–Q3: 76.5 to 89.5; range: 70 to 96</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.82%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.75% to 98.88%; range: 98.69% to 98.94%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>6,800,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 5,983,000 to 7,617,000; range: 5,166,000 to 8,434,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>74,950</strong><small>n=2; missing=0</small><small>Q1–Q3: 73,540 to 76,350; range: 72,140 to 77,760</small></dd></div></dl><p>Secondary input Mean 6,800,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 74,950; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 1 verification unknown; 1 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:hermes:happy-dom-deterministic-intersectionobserver:0</code></li><li><code>pinned:hermes:happy-dom-deterministic-intersectionobserver:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h4>pi @ 0.73.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>7.624 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 7.12 min to 8.128 min; range: 6.616 min to 8.632 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>6.616 min</strong><small>n=1; missing=0</small><small>Q1–Q3: Unavailable; range: 6.616 min to 6.616 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>5.491 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 5.284 min to 5.698 min; range: 5.076 min to 5.906 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>2.111 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 1.814 min to 2.407 min; range: 1.517 min to 2.704 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>57.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 55.25 to 59.75; range: 53 to 62</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.11%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.05% to 98.17%; range: 97.99% to 98.24%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>4,139,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 3,639,000 to 4,639,000; range: 3,139,000 to 5,139,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>58,630</strong><small>n=2; missing=0</small><small>Q1–Q3: 56,070 to 61,180; range: 53,520 to 63,730</small></dd></div></dl><p>Secondary input Mean 4,139,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 58,630; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 1 verification unknown; 1 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:pi:happy-dom-deterministic-intersectionobserver:0</code></li><li><code>pinned:pi:happy-dom-deterministic-intersectionobserver:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h4>qwen @ 0.22.2</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>8.759 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 8.661 min to 8.857 min; range: 8.562 min to 8.956 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small><small>Q1–Q3: Unavailable; range: Unavailable</small></dd></div><div><dt>API-active union · median</dt><dd><strong>6.608 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 6.543 min to 6.673 min; range: 6.478 min to 6.739 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>2.109 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 2.071 min to 2.146 min; range: 2.034 min to 2.183 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>84</strong><small>n=2; missing=0</small><small>Q1–Q3: 75.5 to 92.5; range: 67 to 101</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>99.01%</strong><small>n=2; missing=0</small><small>Q1–Q3: 99% to 99.01%; range: 98.99% to 99.02%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>7,622,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 6,987,000 to 8,257,000; range: 6,352,000 to 8,892,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>70,100</strong><small>n=2; missing=0</small><small>Q1–Q3: 68,980 to 71,220; range: 67,860 to 72,340</small></dd></div></dl><p>Secondary input Mean 7,622,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 70,100; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:qwen:happy-dom-deterministic-intersectionobserver:0</code></li><li><code>pinned:qwen:happy-dom-deterministic-intersectionobserver:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><p><a href=\"report.html\">Detailed timing report and request timelines</a> · <a href=\"data/diagnostics.json\">Download diagnostics</a></p></details>",
            "differences": {
              "[\"cline\",\"3.0.61\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.1155<small>−63.64%</small></td><td>−11.05 min<small>−61.03%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.1046<small>−57.63%</small></td><td>−7.299 min<small>−40.3%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.1226<small>−67.56%</small></td><td>−10.49 min<small>−57.91%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.1055<small>−58.1%</small></td><td>−9.353 min<small>−51.64%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"codex\",\"codex-cli 0.149.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.1155<small>+175%</small></td><td>+11.05 min<small>+156.6%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.01091<small>+16.53%</small></td><td>+3.756 min<small>+53.21%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.007109<small>−10.77%</small></td><td>+33.98 s<small>+8.024%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.01006<small>+15.24%</small></td><td>+1.701 min<small>+24.1%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.1046<small>+136%</small></td><td>+7.299 min<small>+67.5%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.01091<small>−14.18%</small></td><td>−3.756 min<small>−34.73%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19) (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.01802<small>−23.43%</small></td><td>−3.189 min<small>−29.49%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.0008513<small>−1.107%</small></td><td>−2.054 min<small>−19%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"pi\",\"0.73.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.1226<small>+208.2%</small></td><td>+10.49 min<small>+137.6%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.007109<small>+12.07%</small></td><td>−33.98 s<small>−7.428%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.01802<small>+30.6%</small></td><td>+3.189 min<small>+41.83%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.01716<small>+29.15%</small></td><td>+1.135 min<small>+14.88%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"qwen\",\"0.22.2\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.1055<small>+138.7%</small></td><td>+9.353 min<small>+106.8%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.01006<small>−13.22%</small></td><td>−1.701 min<small>−19.42%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.0008513<small>+1.119%</small></td><td>+2.054 min<small>+23.45%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.01716<small>−22.57%</small></td><td>−1.135 min<small>−12.96%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr>"
            }
          }
        },
        {
          "id": "[\"ink-grid-box-layout\",\"477fda7df7781406d2ba046eb48dd5391e964b72\"]",
          "task": "ink-grid-box-layout",
          "confidence": [
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"codex\",\"codex-cli 0.149.1\"]",
              "difference_pp": -50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": -50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": -50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": -50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": -50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": -50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"pi\",\"0.73.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            }
          ],
          "confidenceByBaseline": {
            "[\"cline\",\"3.0.61\"]": [
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              }
            ],
            "[\"codex\",\"codex-cli 0.149.1\"]": [
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": [
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"pi\",\"0.73.1\"]": [
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"qwen\",\"0.22.2\"]": [
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ]
          },
          "total": "All harnesses combined: $1.107 for 10 selected runs at the shared reference prices.",
          "rows": [
            {
              "harness": "cline",
              "version": "3.0.61",
              "selected": 2,
              "passes": 0,
              "pass_rate": 0,
              "tasks": 1,
              "cost": {
                "value": 0.20331233099999987,
                "n": 2
              },
              "benchmark": {
                "value": 0.40662466199999975,
                "n": 2
              },
              "cache": {
                "value": 96.99300871127828,
                "n": 2
              },
              "input": {
                "value": 14481916.5,
                "n": 2
              },
              "output": {
                "value": 165905,
                "n": 2
              },
              "estimated": true,
              "id": "[\"cline\",\"3.0.61\"]",
              "label": "cline @ 3.0.61",
              "recorded": {
                "passes": 0,
                "pass_rate": 0,
                "pass_score": 0
              },
              "pass_estimate": {
                "passed": 0,
                "total": 2,
                "rate": 0
              },
              "pass_display": {
                "passed": 0,
                "rate": 0,
                "estimated": true,
                "score": 0
              },
              "scores": {
                "pass": 0,
                "cost": 36.617799143722365,
                "benchmark": 36.617799143722365,
                "cache": 97.6871623980868,
                "input": 50.26288820267677,
                "output": 39.75317199602182
              },
              "display": {
                "pass": "0.0%",
                "cost": "$0.203",
                "benchmark": "$0.407",
                "cache": "97.0%",
                "input": "14.48M",
                "output": "165.9K"
              }
            },
            {
              "harness": "codex",
              "version": "codex-cli 0.149.1",
              "selected": 2,
              "passes": 1,
              "pass_rate": 50,
              "tasks": 1,
              "cost": {
                "value": 0.09039451499999998,
                "n": 2
              },
              "benchmark": {
                "value": 0.18078902999999996,
                "n": 2
              },
              "cache": {
                "value": 98.89859515404457,
                "n": 2
              },
              "input": {
                "value": 8594838.5,
                "n": 2
              },
              "output": {
                "value": 84565.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"codex\",\"codex-cli 0.149.1\"]",
              "label": "codex @ codex-cli 0.149.1",
              "recorded": {
                "passes": 1,
                "pass_rate": 50,
                "pass_score": 50
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 1,
                "rate": 50,
                "estimated": false,
                "score": 50
              },
              "scores": {
                "pass": 50,
                "cost": 82.35953365090786,
                "benchmark": 82.35953365090786,
                "cache": 99.606386626425,
                "input": 84.69070710287343,
                "output": 77.98984219332942
              },
              "display": {
                "pass": "50.0%",
                "cost": "$0.090",
                "benchmark": "$0.181",
                "cache": "98.9%",
                "input": "8.59M",
                "output": "84.6K"
              }
            },
            {
              "harness": "hermes",
              "version": "Hermes Agent v0.20.5 (2026.8.19)",
              "selected": 2,
              "passes": 1,
              "pass_rate": 50,
              "tasks": 1,
              "cost": {
                "value": 0.07444850099999994,
                "n": 2
              },
              "benchmark": {
                "value": 0.1488970019999999,
                "n": 2
              },
              "cache": {
                "value": 98.09293526290048,
                "n": 2
              },
              "input": {
                "value": 7469021.5,
                "n": 2
              },
              "output": {
                "value": 65952.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)",
              "recorded": {
                "passes": 1,
                "pass_rate": 50,
                "pass_score": 50
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 1,
                "rate": 50,
                "estimated": false,
                "score": 50
              },
              "scores": {
                "pass": 50,
                "cost": 100,
                "benchmark": 100,
                "cache": 98.79496083739629,
                "input": 97.45626652701428,
                "output": 100
              },
              "display": {
                "pass": "50.0%",
                "cost": "$0.074",
                "benchmark": "$0.149",
                "cache": "98.1%",
                "input": "7.47M",
                "output": "66.0K"
              }
            },
            {
              "harness": "pi",
              "version": "0.73.1",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.07643693700000004,
                "n": 2
              },
              "benchmark": {
                "value": 0.15287387400000008,
                "n": 2
              },
              "cache": {
                "value": 98.81025884074671,
                "n": 2
              },
              "input": {
                "value": 7279029.5,
                "n": 2
              },
              "output": {
                "value": 72986,
                "n": 2
              },
              "estimated": false,
              "id": "[\"pi\",\"0.73.1\"]",
              "label": "pi @ 0.73.1",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 97.39859277720653,
                "benchmark": 97.39859277720653,
                "cache": 99.51741811315345,
                "input": 100,
                "output": 90.36322034362755
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.076",
                "benchmark": "$0.153",
                "cache": "98.8%",
                "input": "7.28M",
                "output": "73.0K"
              }
            },
            {
              "harness": "qwen",
              "version": "0.22.2",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.10897672500000004,
                "n": 2
              },
              "benchmark": {
                "value": 0.21795345000000008,
                "n": 2
              },
              "cache": {
                "value": 99.28941155647478,
                "n": 2
              },
              "input": {
                "value": 12909489.5,
                "n": 2
              },
              "output": {
                "value": 94567.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"qwen\",\"0.22.2\"]",
              "label": "qwen @ 0.22.2",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 68.31596471631893,
                "benchmark": 68.31596471631893,
                "cache": 100,
                "input": 56.38510724998073,
                "output": 69.74119015518016
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.109",
                "benchmark": "$0.218",
                "cache": "99.3%",
                "input": "12.91M",
                "output": "94.6K"
              }
            }
          ],
          "decision": {
            "html": "<h3>ink-grid-box-layout</h3><p>1 distinct task; 2 repetitions per task and harness version. Pass differences and cost means use equal task weights; pass counts include every selected attempt. Recorded verifier image/environment identities match within each task. Recorded configuration details are unavailable. Observed differences are not causal and do not establish future superiority. Exploratory 95% task-cluster intervals use 10,000 deterministic resamples; repetitions are not independent tasks or paired random seeds. No correction for multiple comparisons or winner selection. Secondary cost/token intervals describe means, not headline token medians. Original grades remain unchanged; corrected verification is a separate audit with limited replay coverage.</p><p>Time is agent end-to-end elapsed time, excluding setup and verification. All-outcome values include fast failures; successful timing uses only passes. Reference cost includes all selected outcomes and weights tasks equally. Unmeasured workflows: not measured.</p><div class=\"decision-grid\"><article class=\"decision-row\" data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h3>cline @ 3.0.61</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>0 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.2033</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>24.61 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small></dd></div></dl><p class=\"outcome-counts\">0 completed · 2 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.2033; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h3>codex @ codex-cli 0.149.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>1 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.09039</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>12.6 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>10.73 min</strong><small>n=1; missing=0</small></dd></div></dl><p class=\"outcome-counts\">1 completed · 1 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.09039; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h3>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>1 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.07445</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>10.99 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>16.74 min</strong><small>n=1; missing=0</small></dd></div></dl><p class=\"outcome-counts\">1 completed · 1 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.07445; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h3>pi @ 0.73.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.07644</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>11.07 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>11.07 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.07644; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h3>qwen @ 0.22.2</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.1090</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>18.52 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>18.52 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.1090; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article></div><div class=\"decision-table-scroll\" tabindex=\"0\" role=\"region\" aria-label=\"Observed differences\"><table class=\"decision-differences\"><caption>Observed differences from <span id=\"decision-reference\">codex @ codex-cli 0.149.1</span>. Reference stays fixed when hidden. Positive means more passes, higher cost or longer time.</caption><thead><tr><th scope=\"col\">Harness version</th><th scope=\"col\">Pass difference</th><th scope=\"col\">Mean cost difference</th><th scope=\"col\">All-outcome median time difference</th><th scope=\"col\">Coverage</th></tr></thead><tbody id=\"decision-differences\"><tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.1129<small>+124.9%</small></td><td>+12.01 min<small>+95.35%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.01595<small>−17.64%</small></td><td>−1.602 min<small>−12.72%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.01396<small>−15.44%</small></td><td>−1.523 min<small>−12.09%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.01858<small>+20.56%</small></td><td>+5.923 min<small>+47.03%</small></td><td>one task; interval unavailable.</td></tr></tbody></table></div><details id=\"decision-explanation\" class=\"joint-explanation\"><summary>Explain time, tokens and failure evidence together</summary><p>API-active union (<code>model_time</code>) is proxy-observed API-active time including transport, not provider compute. Non-model residual (<code>non_model_time</code>) includes unobserved tools and harness work. Timing correlation is a hypothesis, not a diagnosed cause; these observations do not establish a full tool/harness split.</p><p>These are observed median, quartile and min–max distributions, not confidence intervals. Secondary token mean intervals do not replace the headline token medians. No winner-selection or multiple-comparison correction is applied.</p><section data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h4>cline @ 3.0.61</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>24.61 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 21.21 min to 28 min; range: 17.81 min to 31.4 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small><small>Q1–Q3: Unavailable; range: Unavailable</small></dd></div><div><dt>API-active union · median</dt><dd><strong>19.75 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 17 min to 22.49 min; range: 14.26 min to 25.23 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>4.82 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 4.168 min to 5.472 min; range: 3.517 min to 6.124 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>269</strong><small>n=2; missing=0</small><small>Q1–Q3: 222 to 316; range: 175 to 363</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>96.99%</strong><small>n=2; missing=0</small><small>Q1–Q3: 96.76% to 97.22%; range: 96.53% to 97.45%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>14,480,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 11,750,000 to 17,210,000; range: 9,024,000 to 19,940,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>165,900</strong><small>n=2; missing=0</small><small>Q1–Q3: 151,200 to 180,600; range: 136,600 to 195,300</small></dd></div></dl><p>Secondary input Mean 14,480,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 165,900; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:cline:ink-grid-box-layout:0</code></li><li><code>pinned:cline:ink-grid-box-layout:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h4>codex @ codex-cli 0.149.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>12.6 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 11.66 min to 13.53 min; range: 10.73 min to 14.46 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>10.73 min</strong><small>n=1; missing=0</small><small>Q1–Q3: Unavailable; range: 10.73 min to 10.73 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>8.352 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 8.042 min to 8.662 min; range: 7.733 min to 8.971 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>4.13 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 3.504 min to 4.756 min; range: 2.879 min to 5.381 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>80.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 79.75 to 81.25; range: 79 to 82</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.9%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.84% to 98.96%; range: 98.77% to 99.02%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>8,595,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 8,473,000 to 8,717,000; range: 8,351,000 to 8,839,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>84,570</strong><small>n=2; missing=0</small><small>Q1–Q3: 83,600 to 85,530; range: 82,640 to 86,490</small></dd></div></dl><p>Secondary input Mean 8,595,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 84,570; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 1 pass; 1 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:codex:ink-grid-box-layout:0</code></li><li><code>pinned:codex:ink-grid-box-layout:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h4>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>10.99 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 8.121 min to 13.87 min; range: 5.248 min to 16.74 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>16.74 min</strong><small>n=1; missing=0</small><small>Q1–Q3: Unavailable; range: 16.74 min to 16.74 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>6.739 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 5.613 min to 7.864 min; range: 4.488 min to 8.989 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>4.146 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 2.416 min to 5.875 min; range: 41.17 s to 7.605 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>69.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 51.25 to 87.75; range: 33 to 106</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.09%</strong><small>n=2; missing=0</small><small>Q1–Q3: 97.54% to 98.65%; range: 96.98% to 99.21%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>7,469,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 4,887,000 to 10,050,000; range: 2,305,000 to 12,630,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>65,950</strong><small>n=2; missing=0</small><small>Q1–Q3: 54,960 to 76,950; range: 43,960 to 87,940</small></dd></div></dl><p>Secondary input Mean 7,469,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 65,950; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 1 pass; 1 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:hermes:ink-grid-box-layout:0</code></li><li><code>pinned:hermes:ink-grid-box-layout:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h4>pi @ 0.73.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>11.07 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 10.32 min to 11.83 min; range: 9.559 min to 12.59 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>11.07 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 10.32 min to 11.83 min; range: 9.559 min to 12.59 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>7.303 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 6.295 min to 8.311 min; range: 5.286 min to 9.319 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>3.748 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 3.496 min to 3.999 min; range: 3.245 min to 4.251 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>81.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 67.75 to 95.25; range: 54 to 109</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.81%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.63% to 98.99%; range: 98.45% to 99.17%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>7,279,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 5,455,000 to 9,103,000; range: 3,631,000 to 10,930,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>72,990</strong><small>n=2; missing=0</small><small>Q1–Q3: 64,570 to 81,400; range: 56,160 to 89,820</small></dd></div></dl><p>Secondary input Mean 7,279,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 72,990; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:pi:ink-grid-box-layout:0</code></li><li><code>pinned:pi:ink-grid-box-layout:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h4>qwen @ 0.22.2</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>18.52 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 17.11 min to 19.93 min; range: 15.7 min to 21.33 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>18.52 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 17.11 min to 19.93 min; range: 15.7 min to 21.33 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>9.625 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 9.607 min to 9.642 min; range: 9.59 min to 9.659 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>8.861 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 7.436 min to 10.28 min; range: 6.012 min to 11.71 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>107</strong><small>n=2; missing=0</small><small>Q1–Q3: 104 to 110; range: 101 to 113</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>99.29%</strong><small>n=2; missing=0</small><small>Q1–Q3: 99.26% to 99.32%; range: 99.24% to 99.34%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>12,910,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 12,760,000 to 13,050,000; range: 12,620,000 to 13,200,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>94,570</strong><small>n=2; missing=0</small><small>Q1–Q3: 91,850 to 97,280; range: 89,140 to 100,000</small></dd></div></dl><p>Secondary input Mean 12,910,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 94,570; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:qwen:ink-grid-box-layout:0</code></li><li><code>pinned:qwen:ink-grid-box-layout:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><p><a href=\"report.html\">Detailed timing report and request timelines</a> · <a href=\"data/diagnostics.json\">Download diagnostics</a></p></details>",
            "differences": {
              "[\"cline\",\"3.0.61\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.1129<small>−55.54%</small></td><td>−12.01 min<small>−48.81%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.1289<small>−63.38%</small></td><td>−13.61 min<small>−55.32%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.1269<small>−62.4%</small></td><td>−13.53 min<small>−55%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.09434<small>−46.4%</small></td><td>−6.087 min<small>−24.74%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"codex\",\"codex-cli 0.149.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.1129<small>+124.9%</small></td><td>+12.01 min<small>+95.35%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.01595<small>−17.64%</small></td><td>−1.602 min<small>−12.72%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.01396<small>−15.44%</small></td><td>−1.523 min<small>−12.09%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.01858<small>+20.56%</small></td><td>+5.923 min<small>+47.03%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.1289<small>+173.1%</small></td><td>+13.61 min<small>+123.8%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.01595<small>+21.42%</small></td><td>+1.602 min<small>+14.57%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19) (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.001988<small>+2.671%</small></td><td>+4.708 s<small>+0.7137%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.03453<small>+46.38%</small></td><td>+7.525 min<small>+68.45%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"pi\",\"0.73.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.1269<small>+166%</small></td><td>+13.53 min<small>+122.2%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.01396<small>+18.26%</small></td><td>+1.523 min<small>+13.76%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.001988<small>−2.601%</small></td><td>−4.708 s<small>−0.7086%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.03254<small>+42.57%</small></td><td>+7.446 min<small>+67.25%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"qwen\",\"0.22.2\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.09434<small>+86.56%</small></td><td>+6.087 min<small>+32.87%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.01858<small>−17.05%</small></td><td>−5.923 min<small>−31.98%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.03453<small>−31.68%</small></td><td>−7.525 min<small>−40.63%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.03254<small>−29.86%</small></td><td>−7.446 min<small>−40.21%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr>"
            }
          }
        },
        {
          "id": "[\"psd-tools-blend-range-api\",\"01595992c5f4d4695251fdf58b3edb5302d3ba46\"]",
          "task": "psd-tools-blend-range-api",
          "confidence": [
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"codex\",\"codex-cli 0.149.1\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"pi\",\"0.73.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            }
          ],
          "confidenceByBaseline": {
            "[\"cline\",\"3.0.61\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              }
            ],
            "[\"codex\",\"codex-cli 0.149.1\"]": [
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"pi\",\"0.73.1\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"qwen\",\"0.22.2\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ]
          },
          "total": "All harnesses combined: $1.027 for 10 selected runs at the shared reference prices.",
          "rows": [
            {
              "harness": "cline",
              "version": "3.0.61",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.25396847999999994,
                "n": 2
              },
              "benchmark": {
                "value": 0.5079369599999999,
                "n": 2
              },
              "cache": {
                "value": 96.79369019053024,
                "n": 2
              },
              "input": {
                "value": 19044360,
                "n": 2
              },
              "output": {
                "value": 189242,
                "n": 2
              },
              "estimated": true,
              "id": "[\"cline\",\"3.0.61\"]",
              "label": "cline @ 3.0.61",
              "recorded": {
                "passes": 0,
                "pass_rate": 0,
                "pass_score": 0
              },
              "pass_estimate": {
                "passed": 2,
                "total": 2,
                "rate": 100
              },
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": true,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 18.665532431426136,
                "benchmark": 18.665532431426136,
                "cache": 97.80824425615138,
                "input": 18.185150354225605,
                "output": 24.05491381405819
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.254",
                "benchmark": "$0.508",
                "cache": "96.8%",
                "input": "19.04M",
                "output": "189.2K"
              }
            },
            {
              "harness": "codex",
              "version": "codex-cli 0.149.1",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.08043954900000001,
                "n": 2
              },
              "benchmark": {
                "value": 0.16087909800000003,
                "n": 2
              },
              "cache": {
                "value": 98.54146733574282,
                "n": 2
              },
              "input": {
                "value": 6989765.5,
                "n": 2
              },
              "output": {
                "value": 74137.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"codex\",\"codex-cli 0.149.1\"]",
              "label": "codex @ codex-cli 0.149.1",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 58.93191793007192,
                "benchmark": 58.93191793007192,
                "cache": 99.5743409261696,
                "input": 49.54737751931735,
                "output": 61.40212443095599
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.080",
                "benchmark": "$0.161",
                "cache": "98.5%",
                "input": "6.99M",
                "output": "74.1K"
              }
            },
            {
              "harness": "hermes",
              "version": "Hermes Agent v0.20.5 (2026.8.19)",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.07856929800000002,
                "n": 2
              },
              "benchmark": {
                "value": 0.15713859600000005,
                "n": 2
              },
              "cache": {
                "value": 98.96271109522823,
                "n": 2
              },
              "input": {
                "value": 8957983,
                "n": 2
              },
              "output": {
                "value": 63431,
                "n": 2
              },
              "estimated": false,
              "id": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 60.33472387649432,
                "benchmark": 60.33472387649432,
                "cache": 100,
                "input": 38.6609965658564,
                "output": 71.76617111506992
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.079",
                "benchmark": "$0.157",
                "cache": "99.0%",
                "input": "8.96M",
                "output": "63.4K"
              }
            },
            {
              "harness": "pi",
              "version": "0.73.1",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.047404568999999994,
                "n": 2
              },
              "benchmark": {
                "value": 0.09480913799999999,
                "n": 2
              },
              "cache": {
                "value": 98.02848959452236,
                "n": 2
              },
              "input": {
                "value": 3463245.5,
                "n": 2
              },
              "output": {
                "value": 45522,
                "n": 2
              },
              "estimated": false,
              "id": "[\"pi\",\"0.73.1\"]",
              "label": "pi @ 0.73.1",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 100,
                "benchmark": 100,
                "cache": 99.0559863504478,
                "input": 100,
                "output": 100
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.047",
                "benchmark": "$0.095",
                "cache": "98.0%",
                "input": "3.46M",
                "output": "45.5K"
              }
            },
            {
              "harness": "qwen",
              "version": "0.22.2",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.053281250999999995,
                "n": 2
              },
              "benchmark": {
                "value": 0.10656250199999999,
                "n": 2
              },
              "cache": {
                "value": 98.8302468690217,
                "n": 2
              },
              "input": {
                "value": 5210320.5,
                "n": 2
              },
              "output": {
                "value": 48117,
                "n": 2
              },
              "estimated": false,
              "id": "[\"qwen\",\"0.22.2\"]",
              "label": "qwen @ 0.22.2",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 88.97045041228479,
                "benchmark": 88.97045041228479,
                "cache": 99.86614733495017,
                "input": 66.46895330143319,
                "output": 94.60689569175136
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.053",
                "benchmark": "$0.107",
                "cache": "98.8%",
                "input": "5.21M",
                "output": "48.1K"
              }
            }
          ],
          "decision": {
            "html": "<h3>psd-tools-blend-range-api</h3><p>1 distinct task; 2 repetitions per task and harness version. Pass differences and cost means use equal task weights; pass counts include every selected attempt. Recorded verifier image/environment identities match within each task. Recorded configuration details are unavailable. Observed differences are not causal and do not establish future superiority. Exploratory 95% task-cluster intervals use 10,000 deterministic resamples; repetitions are not independent tasks or paired random seeds. No correction for multiple comparisons or winner selection. Secondary cost/token intervals describe means, not headline token medians. Original grades remain unchanged; corrected verification is a separate audit with limited replay coverage.</p><p>Time is agent end-to-end elapsed time, excluding setup and verification. All-outcome values include fast failures; successful timing uses only passes. Reference cost includes all selected outcomes and weights tasks equally. Unmeasured workflows: not measured.</p><div class=\"decision-grid\"><article class=\"decision-row\" data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h3>cline @ 3.0.61</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>0 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.2540</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>25.74 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small></dd></div></dl><p class=\"outcome-counts\">0 completed · 2 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.2540; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h3>codex @ codex-cli 0.149.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.08044</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>7.015 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>7.015 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.08044; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h3>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.07857</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>8.123 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>8.123 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.07857; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h3>pi @ 0.73.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.04740</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>4.499 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>4.499 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.04740; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h3>qwen @ 0.22.2</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.05328</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>5.214 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>5.214 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.05328; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article></div><div class=\"decision-table-scroll\" tabindex=\"0\" role=\"region\" aria-label=\"Observed differences\"><table class=\"decision-differences\"><caption>Observed differences from <span id=\"decision-reference\">codex @ codex-cli 0.149.1</span>. Reference stays fixed when hidden. Positive means more passes, higher cost or longer time.</caption><thead><tr><th scope=\"col\">Harness version</th><th scope=\"col\">Pass difference</th><th scope=\"col\">Mean cost difference</th><th scope=\"col\">All-outcome median time difference</th><th scope=\"col\">Coverage</th></tr></thead><tbody id=\"decision-differences\"><tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.1735<small>+215.7%</small></td><td>+18.72 min<small>+266.9%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.001870<small>−2.325%</small></td><td>+1.108 min<small>+15.79%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.03303<small>−41.07%</small></td><td>−2.516 min<small>−35.86%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.02716<small>−33.76%</small></td><td>−1.801 min<small>−25.68%</small></td><td>one task; interval unavailable.</td></tr></tbody></table></div><details id=\"decision-explanation\" class=\"joint-explanation\"><summary>Explain time, tokens and failure evidence together</summary><p>API-active union (<code>model_time</code>) is proxy-observed API-active time including transport, not provider compute. Non-model residual (<code>non_model_time</code>) includes unobserved tools and harness work. Timing correlation is a hypothesis, not a diagnosed cause; these observations do not establish a full tool/harness split.</p><p>These are observed median, quartile and min–max distributions, not confidence intervals. Secondary token mean intervals do not replace the headline token medians. No winner-selection or multiple-comparison correction is applied.</p><section data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h4>cline @ 3.0.61</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>25.74 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 20.73 min to 30.75 min; range: 15.71 min to 35.77 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small><small>Q1–Q3: Unavailable; range: Unavailable</small></dd></div><div><dt>API-active union · median</dt><dd><strong>20.91 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 17.26 min to 24.57 min; range: 13.61 min to 28.22 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>4.803 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 3.444 min to 6.162 min; range: 2.084 min to 7.522 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>365.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 294.8 to 436.3; range: 224 to 507</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>96.79%</strong><small>n=2; missing=0</small><small>Q1–Q3: 96.52% to 97.06%; range: 96.25% to 97.34%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>19,040,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 14,990,000 to 23,100,000; range: 10,930,000 to 27,160,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>189,200</strong><small>n=2; missing=0</small><small>Q1–Q3: 160,300 to 218,200; range: 131,400 to 247,100</small></dd></div></dl><p>Secondary input Mean 19,040,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 189,200; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:cline:psd-tools-blend-range-api:0</code></li><li><code>pinned:cline:psd-tools-blend-range-api:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h4>codex @ codex-cli 0.149.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>7.015 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 6.886 min to 7.144 min; range: 6.757 min to 7.273 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>7.015 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 6.886 min to 7.144 min; range: 6.757 min to 7.273 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>6.239 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 6.137 min to 6.341 min; range: 6.035 min to 6.443 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>45.96 s</strong><small>n=2; missing=0</small><small>Q1–Q3: 44.33 s to 47.58 s; range: 42.7 s to 49.21 s</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>62</strong><small>n=2; missing=0</small><small>Q1–Q3: 61 to 63; range: 60 to 64</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.54%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.54% to 98.54%; range: 98.54% to 98.54%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>6,990,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 6,772,000 to 7,207,000; range: 6,555,000 to 7,425,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>74,140</strong><small>n=2; missing=0</small><small>Q1–Q3: 73,470 to 74,810; range: 72,800 to 75,480</small></dd></div></dl><p>Secondary input Mean 6,990,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 74,140; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:codex:psd-tools-blend-range-api:0</code></li><li><code>pinned:codex:psd-tools-blend-range-api:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h4>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>8.123 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 7.792 min to 8.454 min; range: 7.461 min to 8.785 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>8.123 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 7.792 min to 8.454 min; range: 7.461 min to 8.785 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>6.609 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 6.396 min to 6.821 min; range: 6.184 min to 7.033 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>1.444 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 1.326 min to 1.563 min; range: 1.207 min to 1.681 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>101.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 101.3 to 101.8; range: 101 to 102</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.96%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.95% to 98.97%; range: 98.94% to 98.98%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>8,958,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 8,580,000 to 9,336,000; range: 8,203,000 to 9,713,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>63,430</strong><small>n=2; missing=0</small><small>Q1–Q3: 61,990 to 64,880; range: 60,540 to 66,320</small></dd></div></dl><p>Secondary input Mean 8,958,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 63,430; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:hermes:psd-tools-blend-range-api:0</code></li><li><code>pinned:hermes:psd-tools-blend-range-api:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h4>pi @ 0.73.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>4.499 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 4.263 min to 4.736 min; range: 4.027 min to 4.972 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>4.499 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 4.263 min to 4.736 min; range: 4.027 min to 4.972 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>4.123 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 3.891 min to 4.355 min; range: 3.659 min to 4.587 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>21.84 s</strong><small>n=2; missing=0</small><small>Q1–Q3: 21.58 s to 22.09 s; range: 21.33 s to 22.34 s</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>50.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 48.25 to 52.75; range: 46 to 55</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.03%</strong><small>n=2; missing=0</small><small>Q1–Q3: 97.9% to 98.15%; range: 97.78% to 98.28%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>3,463,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 3,002,000 to 3,924,000; range: 2,542,000 to 4,385,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>45,520</strong><small>n=2; missing=0</small><small>Q1–Q3: 43,300 to 47,740; range: 41,080 to 49,960</small></dd></div></dl><p>Secondary input Mean 3,463,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 45,520; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:pi:psd-tools-blend-range-api:0</code></li><li><code>pinned:pi:psd-tools-blend-range-api:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h4>qwen @ 0.22.2</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>5.214 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 5.088 min to 5.34 min; range: 4.962 min to 5.466 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>5.214 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 5.088 min to 5.34 min; range: 4.962 min to 5.466 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>4.472 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 4.353 min to 4.591 min; range: 4.233 min to 4.71 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>42.99 s</strong><small>n=2; missing=0</small><small>Q1–Q3: 42.57 s to 43.4 s; range: 42.16 s to 43.81 s</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>65.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 63.25 to 67.75; range: 61 to 70</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.83%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.7% to 98.96%; range: 98.57% to 99.09%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>5,210,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 4,975,000 to 5,445,000; range: 4,740,000 to 5,680,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>48,120</strong><small>n=2; missing=0</small><small>Q1–Q3: 46,770 to 49,470; range: 45,410 to 50,820</small></dd></div></dl><p>Secondary input Mean 5,210,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 48,120; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:qwen:psd-tools-blend-range-api:0</code></li><li><code>pinned:qwen:psd-tools-blend-range-api:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><p><a href=\"report.html\">Detailed timing report and request timelines</a> · <a href=\"data/diagnostics.json\">Download diagnostics</a></p></details>",
            "differences": {
              "[\"cline\",\"3.0.61\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.1735<small>−68.33%</small></td><td>−18.72 min<small>−72.74%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.1754<small>−69.06%</small></td><td>−17.62 min<small>−68.44%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.2066<small>−81.33%</small></td><td>−21.24 min<small>−82.52%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.2007<small>−79.02%</small></td><td>−20.53 min<small>−79.74%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"codex\",\"codex-cli 0.149.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.1735<small>+215.7%</small></td><td>+18.72 min<small>+266.9%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.001870<small>−2.325%</small></td><td>+1.108 min<small>+15.79%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.03303<small>−41.07%</small></td><td>−2.516 min<small>−35.86%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.02716<small>−33.76%</small></td><td>−1.801 min<small>−25.68%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.1754<small>+223.2%</small></td><td>+17.62 min<small>+216.9%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.001870<small>+2.38%</small></td><td>−1.108 min<small>−13.64%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19) (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.03116<small>−39.67%</small></td><td>−3.624 min<small>−44.61%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.02529<small>−32.19%</small></td><td>−2.909 min<small>−35.82%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"pi\",\"0.73.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.2066<small>+435.7%</small></td><td>+21.24 min<small>+472.1%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.03303<small>+69.69%</small></td><td>+2.516 min<small>+55.92%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.03116<small>+65.74%</small></td><td>+3.624 min<small>+80.54%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.005877<small>+12.4%</small></td><td>+42.87 s<small>+15.88%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"qwen\",\"0.22.2\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.2007<small>+376.7%</small></td><td>+20.53 min<small>+393.7%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.02716<small>+50.97%</small></td><td>+1.801 min<small>+34.55%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.02529<small>+47.46%</small></td><td>+2.909 min<small>+55.8%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.005877<small>−11.03%</small></td><td>−42.87 s<small>−13.7%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr>"
            }
          }
        },
        {
          "id": "[\"superjson-error-stack-serialization\",\"f143004734ea2dd4090e45013a07f091f5bbaee7\"]",
          "task": "superjson-error-stack-serialization",
          "confidence": [
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"codex\",\"codex-cli 0.149.1\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": 100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": 100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"pi\",\"0.73.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            }
          ],
          "confidenceByBaseline": {
            "[\"cline\",\"3.0.61\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              }
            ],
            "[\"codex\",\"codex-cli 0.149.1\"]": [
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              }
            ],
            "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              }
            ],
            "[\"pi\",\"0.73.1\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              }
            ],
            "[\"qwen\",\"0.22.2\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              }
            ]
          },
          "total": "All harnesses combined: $0.612 for 10 selected runs at the shared reference prices.",
          "rows": [
            {
              "harness": "cline",
              "version": "3.0.61",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.09823409399999998,
                "n": 2
              },
              "benchmark": {
                "value": 0.19646818799999996,
                "n": 2
              },
              "cache": {
                "value": 96.10446827103199,
                "n": 2
              },
              "input": {
                "value": 4718935,
                "n": 2
              },
              "output": {
                "value": 95419.5,
                "n": 2
              },
              "estimated": true,
              "id": "[\"cline\",\"3.0.61\"]",
              "label": "cline @ 3.0.61",
              "recorded": {
                "passes": 0,
                "pass_rate": 0,
                "pass_score": 0
              },
              "pass_estimate": {
                "passed": 2,
                "total": 2,
                "rate": 100
              },
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": true,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 45.15338635891528,
                "benchmark": 45.15338635891528,
                "cache": 97.30724562196599,
                "input": 46.53724834099219,
                "output": 52.34150252306918
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.098",
                "benchmark": "$0.196",
                "cache": "96.1%",
                "input": "4.72M",
                "output": "95.4K"
              }
            },
            {
              "harness": "codex",
              "version": "codex-cli 0.149.1",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.06120286799999999,
                "n": 2
              },
              "benchmark": {
                "value": 0.12240573599999997,
                "n": 2
              },
              "cache": {
                "value": 98.51792628158546,
                "n": 2
              },
              "input": {
                "value": 3682024,
                "n": 2
              },
              "output": {
                "value": 70225.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"codex\",\"codex-cli 0.149.1\"]",
              "label": "codex @ codex-cli 0.149.1",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 72.47376054337848,
                "benchmark": 72.47376054337848,
                "cache": 99.75090881116253,
                "input": 59.642807868715686,
                "output": 71.11946515154752
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.061",
                "benchmark": "$0.122",
                "cache": "98.5%",
                "input": "3.68M",
                "output": "70.2K"
              }
            },
            {
              "harness": "hermes",
              "version": "Hermes Agent v0.20.5 (2026.8.19)",
              "selected": 2,
              "passes": 0,
              "pass_rate": 0,
              "tasks": 1,
              "cost": {
                "value": 0.04435602,
                "n": 2
              },
              "benchmark": {
                "value": 0.08871204,
                "n": 2
              },
              "cache": {
                "value": 98.26595332038364,
                "n": 2
              },
              "input": {
                "value": 2592814,
                "n": 2
              },
              "output": {
                "value": 49944,
                "n": 2
              },
              "estimated": false,
              "id": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)",
              "recorded": {
                "passes": 0,
                "pass_rate": 0,
                "pass_score": 0
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 0,
                "rate": 0,
                "estimated": false,
                "score": 0
              },
              "scores": {
                "pass": 0,
                "cost": 100,
                "benchmark": 100,
                "cache": 99.49578232987749,
                "input": 84.69803464498418,
                "output": 100
              },
              "display": {
                "pass": "0.0%",
                "cost": "$0.044",
                "benchmark": "$0.089",
                "cache": "98.3%",
                "input": "2.59M",
                "output": "49.9K"
              }
            },
            {
              "harness": "pi",
              "version": "0.73.1",
              "selected": 2,
              "passes": 0,
              "pass_rate": 0,
              "tasks": 1,
              "cost": {
                "value": 0.053274243,
                "n": 2
              },
              "benchmark": {
                "value": 0.106548486,
                "n": 2
              },
              "cache": {
                "value": 98.36392086416069,
                "n": 2
              },
              "input": {
                "value": 2196062.5,
                "n": 2
              },
              "output": {
                "value": 69037.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"pi\",\"0.73.1\"]",
              "label": "pi @ 0.73.1",
              "recorded": {
                "passes": 0,
                "pass_rate": 0,
                "pass_score": 0
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 0,
                "rate": 0,
                "estimated": false,
                "score": 0
              },
              "scores": {
                "pass": 0,
                "cost": 83.25978465803823,
                "benchmark": 83.25978465803823,
                "cache": 99.59497596797564,
                "input": 100,
                "output": 72.34329168929929
              },
              "display": {
                "pass": "0.0%",
                "cost": "$0.053",
                "benchmark": "$0.107",
                "cache": "98.4%",
                "input": "2.20M",
                "output": "69.0K"
              }
            },
            {
              "harness": "qwen",
              "version": "0.22.2",
              "selected": 2,
              "passes": 0,
              "pass_rate": 0,
              "tasks": 1,
              "cost": {
                "value": 0.048787722000000026,
                "n": 2
              },
              "benchmark": {
                "value": 0.09757544400000005,
                "n": 2
              },
              "cache": {
                "value": 98.76393855026302,
                "n": 2
              },
              "input": {
                "value": 2879117,
                "n": 2
              },
              "output": {
                "value": 58368.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"qwen\",\"0.22.2\"]",
              "label": "qwen @ 0.22.2",
              "recorded": {
                "passes": 0,
                "pass_rate": 0,
                "pass_score": 0
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 0,
                "rate": 0,
                "estimated": false,
                "score": 0
              },
              "scores": {
                "pass": 0,
                "cost": 90.91635801319025,
                "benchmark": 90.91635801319025,
                "cache": 100,
                "input": 76.27555601248577,
                "output": 85.56670121726616
              },
              "display": {
                "pass": "0.0%",
                "cost": "$0.049",
                "benchmark": "$0.098",
                "cache": "98.8%",
                "input": "2.88M",
                "output": "58.4K"
              }
            }
          ],
          "decision": {
            "html": "<h3>superjson-error-stack-serialization</h3><p>1 distinct task; 2 repetitions per task and harness version. Pass differences and cost means use equal task weights; pass counts include every selected attempt. Recorded verifier image/environment identities match within each task. Recorded configuration details are unavailable. Observed differences are not causal and do not establish future superiority. Exploratory 95% task-cluster intervals use 10,000 deterministic resamples; repetitions are not independent tasks or paired random seeds. No correction for multiple comparisons or winner selection. Secondary cost/token intervals describe means, not headline token medians. Original grades remain unchanged; corrected verification is a separate audit with limited replay coverage.</p><p>Time is agent end-to-end elapsed time, excluding setup and verification. All-outcome values include fast failures; successful timing uses only passes. Reference cost includes all selected outcomes and weights tasks equally. Unmeasured workflows: not measured.</p><div class=\"decision-grid\"><article class=\"decision-row\" data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h3>cline @ 3.0.61</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>0 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.09823</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>9.726 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small></dd></div></dl><p class=\"outcome-counts\">0 completed · 2 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.09823; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h3>codex @ codex-cli 0.149.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.06120</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>6.919 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>6.919 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.06120; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h3>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>0 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.04436</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>5.094 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small></dd></div></dl><p class=\"outcome-counts\">0 completed · 2 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.04436; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h3>pi @ 0.73.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>0 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.05327</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>6.205 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small></dd></div></dl><p class=\"outcome-counts\">0 completed · 2 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.05327; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h3>qwen @ 0.22.2</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>0 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.04879</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>4.969 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small></dd></div></dl><p class=\"outcome-counts\">0 completed · 2 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.04879; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article></div><div class=\"decision-table-scroll\" tabindex=\"0\" role=\"region\" aria-label=\"Observed differences\"><table class=\"decision-differences\"><caption>Observed differences from <span id=\"decision-reference\">codex @ codex-cli 0.149.1</span>. Reference stays fixed when hidden. Positive means more passes, higher cost or longer time.</caption><thead><tr><th scope=\"col\">Harness version</th><th scope=\"col\">Pass difference</th><th scope=\"col\">Mean cost difference</th><th scope=\"col\">All-outcome median time difference</th><th scope=\"col\">Coverage</th></tr></thead><tbody id=\"decision-differences\"><tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.03703<small>+60.51%</small></td><td>+2.808 min<small>+40.58%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.01685<small>−27.53%</small></td><td>−1.825 min<small>−26.38%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.007929<small>−12.95%</small></td><td>−42.8 s<small>−10.31%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.01242<small>−20.29%</small></td><td>−1.95 min<small>−28.18%</small></td><td>one task; interval unavailable.</td></tr></tbody></table></div><details id=\"decision-explanation\" class=\"joint-explanation\"><summary>Explain time, tokens and failure evidence together</summary><p>API-active union (<code>model_time</code>) is proxy-observed API-active time including transport, not provider compute. Non-model residual (<code>non_model_time</code>) includes unobserved tools and harness work. Timing correlation is a hypothesis, not a diagnosed cause; these observations do not establish a full tool/harness split.</p><p>These are observed median, quartile and min–max distributions, not confidence intervals. Secondary token mean intervals do not replace the headline token medians. No winner-selection or multiple-comparison correction is applied.</p><section data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h4>cline @ 3.0.61</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>9.726 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 8.871 min to 10.58 min; range: 8.016 min to 11.44 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small><small>Q1–Q3: Unavailable; range: Unavailable</small></dd></div><div><dt>API-active union · median</dt><dd><strong>9.424 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 8.571 min to 10.28 min; range: 7.718 min to 11.13 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>16.27 s</strong><small>n=2; missing=0</small><small>Q1–Q3: 16.15 s to 16.4 s; range: 16.03 s to 16.52 s</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>92</strong><small>n=2; missing=0</small><small>Q1–Q3: 84.5 to 99.5; range: 77 to 107</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>96.1%</strong><small>n=2; missing=0</small><small>Q1–Q3: 96.03% to 96.18%; range: 95.95% to 96.26%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>4,719,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 4,297,000 to 5,141,000; range: 3,874,000 to 5,564,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>95,420</strong><small>n=2; missing=0</small><small>Q1–Q3: 91,940 to 98,900; range: 88,460 to 102,400</small></dd></div></dl><p>Secondary input Mean 4,719,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 95,420; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:cline:superjson-error-stack-serialization:0</code></li><li><code>pinned:cline:superjson-error-stack-serialization:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h4>codex @ codex-cli 0.149.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>6.919 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 6.366 min to 7.471 min; range: 5.814 min to 8.024 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>6.919 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 6.366 min to 7.471 min; range: 5.814 min to 8.024 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>6.64 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 6.09 min to 7.189 min; range: 5.541 min to 7.739 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>14.98 s</strong><small>n=2; missing=0</small><small>Q1–Q3: 14.81 s to 15.15 s; range: 14.64 s to 15.32 s</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>50</strong><small>n=2; missing=0</small><small>Q1–Q3: 49.5 to 50.5; range: 49 to 51</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.52%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.52% to 98.52%; range: 98.51% to 98.52%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>3,682,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 3,659,000 to 3,705,000; range: 3,635,000 to 3,729,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>70,230</strong><small>n=2; missing=0</small><small>Q1–Q3: 68,150 to 72,300; range: 66,070 to 74,380</small></dd></div></dl><p>Secondary input Mean 3,682,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 70,230; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:codex:superjson-error-stack-serialization:0</code></li><li><code>pinned:codex:superjson-error-stack-serialization:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h4>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>5.094 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 4.737 min to 5.45 min; range: 4.381 min to 5.807 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small><small>Q1–Q3: Unavailable; range: Unavailable</small></dd></div><div><dt>API-active union · median</dt><dd><strong>4.59 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 4.223 min to 4.956 min; range: 3.857 min to 5.323 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>25.47 s</strong><small>n=2; missing=0</small><small>Q1–Q3: 24.92 s to 26.02 s; range: 24.37 s to 26.56 s</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>40.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 40.25 to 40.75; range: 40 to 41</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.27%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.26% to 98.27%; range: 98.25% to 98.28%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>2,593,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 2,542,000 to 2,643,000; range: 2,492,000 to 2,694,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>49,940</strong><small>n=2; missing=0</small><small>Q1–Q3: 47,740 to 52,150; range: 45,530 to 54,360</small></dd></div></dl><p>Secondary input Mean 2,593,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 49,940; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:hermes:superjson-error-stack-serialization:0</code></li><li><code>pinned:hermes:superjson-error-stack-serialization:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h4>pi @ 0.73.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>6.205 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 6.104 min to 6.307 min; range: 6.003 min to 6.408 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small><small>Q1–Q3: Unavailable; range: Unavailable</small></dd></div><div><dt>API-active union · median</dt><dd><strong>5.86 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 5.761 min to 5.959 min; range: 5.662 min to 6.058 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>19.19 s</strong><small>n=2; missing=0</small><small>Q1–Q3: 19 s to 19.38 s; range: 18.8 s to 19.58 s</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>36</strong><small>n=2; missing=0</small><small>Q1–Q3: 34.5 to 37.5; range: 33 to 39</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.36%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.31% to 98.42%; range: 98.26% to 98.47%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>2,196,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 2,139,000 to 2,253,000; range: 2,082,000 to 2,310,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>69,040</strong><small>n=2; missing=0</small><small>Q1–Q3: 67,650 to 70,430; range: 66,260 to 71,820</small></dd></div></dl><p>Secondary input Mean 2,196,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 69,040; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:pi:superjson-error-stack-serialization:0</code></li><li><code>pinned:pi:superjson-error-stack-serialization:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h4>qwen @ 0.22.2</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>4.969 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 4.858 min to 5.081 min; range: 4.746 min to 5.192 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small><small>Q1–Q3: Unavailable; range: Unavailable</small></dd></div><div><dt>API-active union · median</dt><dd><strong>4.744 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 4.637 min to 4.852 min; range: 4.529 min to 4.96 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>11.45 s</strong><small>n=2; missing=0</small><small>Q1–Q3: 11.21 s to 11.68 s; range: 10.98 s to 11.91 s</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>39</strong><small>n=2; missing=0</small><small>Q1–Q3: 38 to 40; range: 37 to 41</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.76%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.52% to 99.01%; range: 98.27% to 99.26%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>2,879,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 2,808,000 to 2,950,000; range: 2,738,000 to 3,021,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>58,370</strong><small>n=2; missing=0</small><small>Q1–Q3: 57,080 to 59,660; range: 55,800 to 60,940</small></dd></div></dl><p>Secondary input Mean 2,879,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 58,370; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:qwen:superjson-error-stack-serialization:0</code></li><li><code>pinned:qwen:superjson-error-stack-serialization:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><p><a href=\"report.html\">Detailed timing report and request timelines</a> · <a href=\"data/diagnostics.json\">Download diagnostics</a></p></details>",
            "differences": {
              "[\"cline\",\"3.0.61\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.03703<small>−37.7%</small></td><td>−2.808 min<small>−28.87%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.05388<small>−54.85%</small></td><td>−4.633 min<small>−47.63%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.04496<small>−45.77%</small></td><td>−3.521 min<small>−36.2%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.04945<small>−50.34%</small></td><td>−4.757 min<small>−48.91%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"codex\",\"codex-cli 0.149.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.03703<small>+60.51%</small></td><td>+2.808 min<small>+40.58%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.01685<small>−27.53%</small></td><td>−1.825 min<small>−26.38%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.007929<small>−12.95%</small></td><td>−42.8 s<small>−10.31%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.01242<small>−20.29%</small></td><td>−1.95 min<small>−28.18%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.05388<small>+121.5%</small></td><td>+4.633 min<small>+90.95%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.01685<small>+37.98%</small></td><td>+1.825 min<small>+35.83%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19) (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.008918<small>+20.11%</small></td><td>+1.112 min<small>+21.82%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.004432<small>+9.991%</small></td><td>−7.469 s<small>−2.444%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"pi\",\"0.73.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.04496<small>+84.39%</small></td><td>+3.521 min<small>+56.74%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.007929<small>+14.88%</small></td><td>+42.8 s<small>+11.5%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.008918<small>−16.74%</small></td><td>−1.112 min<small>−17.91%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.004487<small>−8.422%</small></td><td>−1.236 min<small>−19.92%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"qwen\",\"0.22.2\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.04945<small>+101.4%</small></td><td>+4.757 min<small>+95.73%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.01242<small>+25.45%</small></td><td>+1.95 min<small>+39.23%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.004432<small>−9.084%</small></td><td>+7.469 s<small>+2.505%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.004487<small>+9.196%</small></td><td>+1.236 min<small>+24.88%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr>"
            }
          }
        },
        {
          "id": "[\"textual-richlog-follow-state\",\"a12215b791587a6d94227bd30a0e9291de932106\"]",
          "task": "textual-richlog-follow-state",
          "confidence": [
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"codex\",\"codex-cli 0.149.1\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": -50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"pi\",\"0.73.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 50,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            }
          ],
          "confidenceByBaseline": {
            "[\"cline\",\"3.0.61\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              }
            ],
            "[\"codex\",\"codex-cli 0.149.1\"]": [
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"pi\",\"0.73.1\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"qwen\",\"0.22.2\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 50,
                "interval": null,
                "difference_text": "+50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -50,
                "interval": null,
                "difference_text": "-50.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ]
          },
          "total": "All harnesses combined: $2.370 for 10 selected runs at the shared reference prices.",
          "rows": [
            {
              "harness": "cline",
              "version": "3.0.61",
              "selected": 2,
              "passes": 1,
              "pass_rate": 50,
              "tasks": 1,
              "cost": {
                "value": 0.7801968959999992,
                "n": 2
              },
              "benchmark": {
                "value": 1.5603937919999984,
                "n": 2
              },
              "cache": {
                "value": 94.9839150602455,
                "n": 2
              },
              "input": {
                "value": 46257864,
                "n": 2
              },
              "output": {
                "value": 494898,
                "n": 2
              },
              "estimated": true,
              "id": "[\"cline\",\"3.0.61\"]",
              "label": "cline @ 3.0.61",
              "recorded": {
                "passes": 0,
                "pass_rate": 0,
                "pass_score": 0
              },
              "pass_estimate": {
                "passed": 1,
                "total": 2,
                "rate": 50
              },
              "pass_display": {
                "passed": 1,
                "rate": 50,
                "estimated": true,
                "score": 50
              },
              "scores": {
                "pass": 50,
                "cost": 8.195184744749367,
                "benchmark": 8.195184744749367,
                "cache": 95.46407650180866,
                "input": 10.590453333513194,
                "output": 13.21019684864356
              },
              "display": {
                "pass": "50.0%",
                "cost": "$0.780",
                "benchmark": "$1.560",
                "cache": "95.0%",
                "input": "46.26M",
                "output": "494.9K"
              }
            },
            {
              "harness": "codex",
              "version": "codex-cli 0.149.1",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.09593764199999998,
                "n": 2
              },
              "benchmark": {
                "value": 0.19187528399999995,
                "n": 2
              },
              "cache": {
                "value": 98.9310490669978,
                "n": 2
              },
              "input": {
                "value": 9764633,
                "n": 2
              },
              "output": {
                "value": 86041.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"codex\",\"codex-cli 0.149.1\"]",
              "label": "codex @ codex-cli 0.149.1",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 66.64597510120171,
                "benchmark": 66.64597510120171,
                "cache": 99.4311640086196,
                "input": 50.17001150990519,
                "output": 75.98310117791995
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.096",
                "benchmark": "$0.192",
                "cache": "98.9%",
                "input": "9.76M",
                "output": "86.0K"
              }
            },
            {
              "harness": "hermes",
              "version": "Hermes Agent v0.20.5 (2026.8.19)",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.1574306910000001,
                "n": 2
              },
              "benchmark": {
                "value": 0.3148613820000002,
                "n": 2
              },
              "cache": {
                "value": 99.49702394957535,
                "n": 2
              },
              "input": {
                "value": 23509946.5,
                "n": 2
              },
              "output": {
                "value": 115906.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 40.61379429503996,
                "benchmark": 40.61379429503996,
                "cache": 100,
                "input": 20.83763780576872,
                "output": 56.40494709097419
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.157",
                "benchmark": "$0.315",
                "cache": "99.5%",
                "input": "23.51M",
                "output": "115.9K"
              }
            },
            {
              "harness": "pi",
              "version": "0.73.1",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.06393857700000002,
                "n": 2
              },
              "benchmark": {
                "value": 0.12787715400000005,
                "n": 2
              },
              "cache": {
                "value": 98.60173155441342,
                "n": 2
              },
              "input": {
                "value": 4898917.5,
                "n": 2
              },
              "output": {
                "value": 65377,
                "n": 2
              },
              "estimated": false,
              "id": "[\"pi\",\"0.73.1\"]",
              "label": "pi @ 0.73.1",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 100,
                "benchmark": 100,
                "cache": 99.10018173446508,
                "input": 100,
                "output": 100
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.064",
                "benchmark": "$0.128",
                "cache": "98.6%",
                "input": "4.90M",
                "output": "65.4K"
              }
            },
            {
              "harness": "qwen",
              "version": "0.22.2",
              "selected": 2,
              "passes": 1,
              "pass_rate": 50,
              "tasks": 1,
              "cost": {
                "value": 0.08748910200000001,
                "n": 2
              },
              "benchmark": {
                "value": 0.17497820400000003,
                "n": 2
              },
              "cache": {
                "value": 99.22935144881808,
                "n": 2
              },
              "input": {
                "value": 10324021,
                "n": 2
              },
              "output": {
                "value": 74880,
                "n": 2
              },
              "estimated": false,
              "id": "[\"qwen\",\"0.22.2\"]",
              "label": "qwen @ 0.22.2",
              "recorded": {
                "passes": 1,
                "pass_rate": 50,
                "pass_score": 50
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 1,
                "rate": 50,
                "estimated": false,
                "score": 50
              },
              "scores": {
                "pass": 50,
                "cost": 73.08176165758337,
                "benchmark": 73.08176165758337,
                "cache": 99.73097436472781,
                "input": 47.45164214602043,
                "output": 87.30902777777779
              },
              "display": {
                "pass": "50.0%",
                "cost": "$0.087",
                "benchmark": "$0.175",
                "cache": "99.2%",
                "input": "10.32M",
                "output": "74.9K"
              }
            }
          ],
          "decision": {
            "html": "<h3>textual-richlog-follow-state</h3><p>1 distinct task; 2 repetitions per task and harness version. Pass differences and cost means use equal task weights; pass counts include every selected attempt. Recorded verifier image/environment identities match within each task. Recorded configuration details are unavailable. Observed differences are not causal and do not establish future superiority. Exploratory 95% task-cluster intervals use 10,000 deterministic resamples; repetitions are not independent tasks or paired random seeds. No correction for multiple comparisons or winner selection. Secondary cost/token intervals describe means, not headline token medians. Original grades remain unchanged; corrected verification is a separate audit with limited replay coverage.</p><p>Time is agent end-to-end elapsed time, excluding setup and verification. All-outcome values include fast failures; successful timing uses only passes. Reference cost includes all selected outcomes and weights tasks equally. Unmeasured workflows: not measured.</p><div class=\"decision-grid\"><article class=\"decision-row\" data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h3>cline @ 3.0.61</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>0 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.7802</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>85.3 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small></dd></div></dl><p class=\"outcome-counts\">0 completed · 2 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.7802; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h3>codex @ codex-cli 0.149.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.09594</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>12.86 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>12.86 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.09594; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h3>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.1574</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>18.43 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>18.43 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.1574; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h3>pi @ 0.73.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.06394</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>8.655 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>8.655 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.06394; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h3>qwen @ 0.22.2</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>1 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.08749</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>11.54 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>11.29 min</strong><small>n=1; missing=0</small></dd></div></dl><p class=\"outcome-counts\">1 completed · 1 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.08749; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article></div><div class=\"decision-table-scroll\" tabindex=\"0\" role=\"region\" aria-label=\"Observed differences\"><table class=\"decision-differences\"><caption>Observed differences from <span id=\"decision-reference\">codex @ codex-cli 0.149.1</span>. Reference stays fixed when hidden. Positive means more passes, higher cost or longer time.</caption><thead><tr><th scope=\"col\">Harness version</th><th scope=\"col\">Pass difference</th><th scope=\"col\">Mean cost difference</th><th scope=\"col\">All-outcome median time difference</th><th scope=\"col\">Coverage</th></tr></thead><tbody id=\"decision-differences\"><tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.6843<small>+713.2%</small></td><td>+72.43 min<small>+563.2%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.06149<small>+64.1%</small></td><td>+5.567 min<small>+43.28%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.03200<small>−33.35%</small></td><td>−4.207 min<small>−32.71%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.008449<small>−8.806%</small></td><td>−1.322 min<small>−10.28%</small></td><td>one task; interval unavailable.</td></tr></tbody></table></div><details id=\"decision-explanation\" class=\"joint-explanation\"><summary>Explain time, tokens and failure evidence together</summary><p>API-active union (<code>model_time</code>) is proxy-observed API-active time including transport, not provider compute. Non-model residual (<code>non_model_time</code>) includes unobserved tools and harness work. Timing correlation is a hypothesis, not a diagnosed cause; these observations do not establish a full tool/harness split.</p><p>These are observed median, quartile and min–max distributions, not confidence intervals. Secondary token mean intervals do not replace the headline token medians. No winner-selection or multiple-comparison correction is applied.</p><section data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h4>cline @ 3.0.61</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>85.3 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 77.03 min to 93.57 min; range: 68.76 min to 101.8 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small><small>Q1–Q3: Unavailable; range: Unavailable</small></dd></div><div><dt>API-active union · median</dt><dd><strong>60.92 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 56.52 min to 65.32 min; range: 52.11 min to 69.72 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>24.36 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 20.49 min to 28.22 min; range: 16.62 min to 32.09 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>1,022</strong><small>n=2; missing=0</small><small>Q1–Q3: 898.8 to 1,144; range: 776 to 1,267</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>94.98%</strong><small>n=2; missing=0</small><small>Q1–Q3: 94.88% to 95.09%; range: 94.77% to 95.2%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>46,260,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 40,800,000 to 51,710,000; range: 35,350,000 to 57,170,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>494,900</strong><small>n=2; missing=0</small><small>Q1–Q3: 453,000 to 536,800; range: 411,100 to 578,700</small></dd></div></dl><p>Secondary input Mean 46,260,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 494,900; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:cline:textual-richlog-follow-state:0</code></li><li><code>pinned:cline:textual-richlog-follow-state:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h4>codex @ codex-cli 0.149.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>12.86 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 11.48 min to 14.25 min; range: 10.09 min to 15.63 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>12.86 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 11.48 min to 14.25 min; range: 10.09 min to 15.63 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>8.452 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 7.723 min to 9.18 min; range: 6.994 min to 9.909 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>4.4 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 3.742 min to 5.057 min; range: 3.085 min to 5.715 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>83.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 77.75 to 89.25; range: 72 to 95</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.93%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.87% to 98.99%; range: 98.8% to 99.06%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>9,765,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 8,890,000 to 10,640,000; range: 8,014,000 to 11,510,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>86,040</strong><small>n=2; missing=0</small><small>Q1–Q3: 81,420 to 90,660; range: 76,800 to 95,280</small></dd></div></dl><p>Secondary input Mean 9,765,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 86,040; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:codex:textual-richlog-follow-state:0</code></li><li><code>pinned:codex:textual-richlog-follow-state:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h4>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>18.43 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 16.98 min to 19.87 min; range: 15.54 min to 21.32 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>18.43 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 16.98 min to 19.87 min; range: 15.54 min to 21.32 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>13.57 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 12.68 min to 14.46 min; range: 11.79 min to 15.35 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>4.776 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 4.216 min to 5.336 min; range: 3.655 min to 5.897 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>186</strong><small>n=2; missing=0</small><small>Q1–Q3: 184 to 188; range: 182 to 190</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>99.5%</strong><small>n=2; missing=0</small><small>Q1–Q3: 99.49% to 99.5%; range: 99.49% to 99.51%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>23,510,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 22,430,000 to 24,590,000; range: 21,350,000 to 25,670,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>115,900</strong><small>n=2; missing=0</small><small>Q1–Q3: 109,600 to 122,200; range: 103,300 to 128,500</small></dd></div></dl><p>Secondary input Mean 23,510,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 115,900; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:hermes:textual-richlog-follow-state:0</code></li><li><code>pinned:hermes:textual-richlog-follow-state:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h4>pi @ 0.73.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>8.655 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 7.662 min to 9.648 min; range: 6.669 min to 10.64 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>8.655 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 7.662 min to 9.648 min; range: 6.669 min to 10.64 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>6.46 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 6.268 min to 6.652 min; range: 6.076 min to 6.844 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>2.182 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 1.381 min to 2.983 min; range: 34.81 s to 3.783 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>67</strong><small>n=2; missing=0</small><small>Q1–Q3: 66.5 to 67.5; range: 66 to 68</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.6%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.56% to 98.65%; range: 98.51% to 98.69%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>4,899,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 4,700,000 to 5,098,000; range: 4,500,000 to 5,298,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>65,380</strong><small>n=2; missing=0</small><small>Q1–Q3: 62,180 to 68,570; range: 58,990 to 71,770</small></dd></div></dl><p>Secondary input Mean 4,899,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 65,380; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:pi:textual-richlog-follow-state:0</code></li><li><code>pinned:pi:textual-richlog-follow-state:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h4>qwen @ 0.22.2</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>11.54 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 11.42 min to 11.67 min; range: 11.29 min to 11.79 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>11.29 min</strong><small>n=1; missing=0</small><small>Q1–Q3: Unavailable; range: 11.29 min to 11.29 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>8.161 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 8.027 min to 8.295 min; range: 7.893 min to 8.429 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>3.355 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 3.346 min to 3.363 min; range: 3.337 min to 3.372 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>107.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 105.3 to 109.8; range: 103 to 112</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>99.23%</strong><small>n=2; missing=0</small><small>Q1–Q3: 99.15% to 99.31%; range: 99.08% to 99.38%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>10,320,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 10,090,000 to 10,560,000; range: 9,848,000 to 10,800,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>74,880</strong><small>n=2; missing=0</small><small>Q1–Q3: 74,750 to 75,010; range: 74,630 to 75,130</small></dd></div></dl><p>Secondary input Mean 10,320,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 74,880; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 1 pass; 1 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:qwen:textual-richlog-follow-state:0</code></li><li><code>pinned:qwen:textual-richlog-follow-state:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><p><a href=\"report.html\">Detailed timing report and request timelines</a> · <a href=\"data/diagnostics.json\">Download diagnostics</a></p></details>",
            "differences": {
              "[\"cline\",\"3.0.61\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.6843<small>−87.7%</small></td><td>−72.43 min<small>−84.92%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.6228<small>−79.82%</small></td><td>−66.87 min<small>−78.39%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.7163<small>−91.8%</small></td><td>−76.64 min<small>−89.85%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.6927<small>−88.79%</small></td><td>−73.76 min<small>−86.47%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"codex\",\"codex-cli 0.149.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.6843<small>+713.2%</small></td><td>+72.43 min<small>+563.2%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.06149<small>+64.1%</small></td><td>+5.567 min<small>+43.28%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.03200<small>−33.35%</small></td><td>−4.207 min<small>−32.71%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.008449<small>−8.806%</small></td><td>−1.322 min<small>−10.28%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.6228<small>+395.6%</small></td><td>+66.87 min<small>+362.8%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.06149<small>−39.06%</small></td><td>−5.567 min<small>−30.21%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19) (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.09349<small>−59.39%</small></td><td>−9.774 min<small>−53.04%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.06994<small>−44.43%</small></td><td>−6.888 min<small>−37.38%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"pi\",\"0.73.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.7163<small>+1,120%</small></td><td>+76.64 min<small>+885.5%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.03200<small>+50.05%</small></td><td>+4.207 min<small>+48.61%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.09349<small>+146.2%</small></td><td>+9.774 min<small>+112.9%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.02355<small>+36.83%</small></td><td>+2.886 min<small>+33.34%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"qwen\",\"0.22.2\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.6927<small>+791.8%</small></td><td>+73.76 min<small>+639.1%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.008449<small>+9.657%</small></td><td>+1.322 min<small>+11.45%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.06994<small>+79.94%</small></td><td>+6.888 min<small>+59.69%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+50 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.02355<small>−26.92%</small></td><td>−2.886 min<small>−25.01%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr>"
            }
          }
        },
        {
          "id": "[\"tomlkit-toml-table-converters\",\"962de91aaf103135f400e5108a798a59a10dc17e\"]",
          "task": "tomlkit-toml-table-converters",
          "confidence": [
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"codex\",\"codex-cli 0.149.1\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": 100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"pi\",\"0.73.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            }
          ],
          "confidenceByBaseline": {
            "[\"cline\",\"3.0.61\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              }
            ],
            "[\"codex\",\"codex-cli 0.149.1\"]": [
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              }
            ],
            "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              }
            ],
            "[\"pi\",\"0.73.1\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              }
            ],
            "[\"qwen\",\"0.22.2\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              }
            ]
          },
          "total": "All harnesses combined: $2.022 for 10 selected runs at the shared reference prices.",
          "rows": [
            {
              "harness": "cline",
              "version": "3.0.61",
              "selected": 2,
              "passes": 0,
              "pass_rate": 0,
              "tasks": 1,
              "cost": {
                "value": 0.5021648579999992,
                "n": 2
              },
              "benchmark": {
                "value": 1.0043297159999984,
                "n": 2
              },
              "cache": {
                "value": 96.27201494849965,
                "n": 2
              },
              "input": {
                "value": 28379725,
                "n": 2
              },
              "output": {
                "value": 397370.5,
                "n": 2
              },
              "estimated": true,
              "id": "[\"cline\",\"3.0.61\"]",
              "label": "cline @ 3.0.61",
              "recorded": {
                "passes": 0,
                "pass_rate": 0,
                "pass_score": 0
              },
              "pass_estimate": {
                "passed": 0,
                "total": 2,
                "rate": 0
              },
              "pass_display": {
                "passed": 0,
                "rate": 0,
                "estimated": true,
                "score": 0
              },
              "scores": {
                "pass": 0,
                "cost": 17.337376881916367,
                "benchmark": 17.337376881916367,
                "cache": 96.78764209162286,
                "input": 22.643246190722426,
                "output": 23.731882462336788
              },
              "display": {
                "pass": "0.0%",
                "cost": "$0.502",
                "benchmark": "$1.004",
                "cache": "96.3%",
                "input": "28.38M",
                "output": "397.4K"
              }
            },
            {
              "harness": "codex",
              "version": "codex-cli 0.149.1",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.09316274400000002,
                "n": 2
              },
              "benchmark": {
                "value": 0.18632548800000004,
                "n": 2
              },
              "cache": {
                "value": 99.00279670327109,
                "n": 2
              },
              "input": {
                "value": 7875966,
                "n": 2
              },
              "output": {
                "value": 96997.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"codex\",\"codex-cli 0.149.1\"]",
              "label": "codex @ codex-cli 0.149.1",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 93.45174933877001,
                "benchmark": 93.45174933877001,
                "cache": 99.53304974983529,
                "input": 81.59114704151847,
                "output": 97.22260883012449
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.093",
                "benchmark": "$0.186",
                "cache": "99.0%",
                "input": "7.88M",
                "output": "97.0K"
              }
            },
            {
              "harness": "hermes",
              "version": "Hermes Agent v0.20.5 (2026.8.19)",
              "selected": 2,
              "passes": 0,
              "pass_rate": 0,
              "tasks": 1,
              "cost": {
                "value": 0.20549859299999998,
                "n": 2
              },
              "benchmark": {
                "value": 0.41099718599999996,
                "n": 2
              },
              "cache": {
                "value": 99.442127551483,
                "n": 2
              },
              "input": {
                "value": 30328771.5,
                "n": 2
              },
              "output": {
                "value": 153487.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)",
              "recorded": {
                "passes": 0,
                "pass_rate": 0,
                "pass_score": 0
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 0,
                "rate": 0,
                "estimated": false,
                "score": 0
              },
              "scores": {
                "pass": 0,
                "cost": 42.366330946119916,
                "benchmark": 42.366330946119916,
                "cache": 99.97473362774404,
                "input": 21.188101865583313,
                "output": 61.4405081847056
              },
              "display": {
                "pass": "0.0%",
                "cost": "$0.205",
                "benchmark": "$0.411",
                "cache": "99.4%",
                "input": "30.33M",
                "output": "153.5K"
              }
            },
            {
              "harness": "pi",
              "version": "0.73.1",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.08706221400000001,
                "n": 2
              },
              "benchmark": {
                "value": 0.17412442800000003,
                "n": 2
              },
              "cache": {
                "value": 98.7536171686117,
                "n": 2
              },
              "input": {
                "value": 6426091,
                "n": 2
              },
              "output": {
                "value": 94303.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"pi\",\"0.73.1\"]",
              "label": "pi @ 0.73.1",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 100,
                "benchmark": 100,
                "cache": 99.28253562451992,
                "input": 100,
                "output": 100
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.087",
                "benchmark": "$0.174",
                "cache": "98.8%",
                "input": "6.43M",
                "output": "94.3K"
              }
            },
            {
              "harness": "qwen",
              "version": "0.22.2",
              "selected": 2,
              "passes": 0,
              "pass_rate": 0,
              "tasks": 1,
              "cost": {
                "value": 0.12330579299999998,
                "n": 2
              },
              "benchmark": {
                "value": 0.24661158599999997,
                "n": 2
              },
              "cache": {
                "value": 99.46725931949545,
                "n": 2
              },
              "input": {
                "value": 15991167.5,
                "n": 2
              },
              "output": {
                "value": 105828.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"qwen\",\"0.22.2\"]",
              "label": "qwen @ 0.22.2",
              "recorded": {
                "passes": 0,
                "pass_rate": 0,
                "pass_score": 0
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 0,
                "rate": 0,
                "estimated": false,
                "score": 0
              },
              "scores": {
                "pass": 0,
                "cost": 70.60675081178061,
                "benchmark": 70.60675081178061,
                "cache": 100,
                "input": 40.18525226503943,
                "output": 89.10973886996413
              },
              "display": {
                "pass": "0.0%",
                "cost": "$0.123",
                "benchmark": "$0.247",
                "cache": "99.5%",
                "input": "15.99M",
                "output": "105.8K"
              }
            }
          ],
          "decision": {
            "html": "<h3>tomlkit-toml-table-converters</h3><p>1 distinct task; 2 repetitions per task and harness version. Pass differences and cost means use equal task weights; pass counts include every selected attempt. Recorded verifier image/environment identities match within each task. Recorded configuration details are unavailable. Observed differences are not causal and do not establish future superiority. Exploratory 95% task-cluster intervals use 10,000 deterministic resamples; repetitions are not independent tasks or paired random seeds. No correction for multiple comparisons or winner selection. Secondary cost/token intervals describe means, not headline token medians. Original grades remain unchanged; corrected verification is a separate audit with limited replay coverage.</p><p>Time is agent end-to-end elapsed time, excluding setup and verification. All-outcome values include fast failures; successful timing uses only passes. Reference cost includes all selected outcomes and weights tasks equally. Unmeasured workflows: not measured.</p><div class=\"decision-grid\"><article class=\"decision-row\" data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h3>cline @ 3.0.61</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>0 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.5022</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>49.56 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small></dd></div></dl><p class=\"outcome-counts\">0 completed · 2 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.5022; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h3>codex @ codex-cli 0.149.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.09316</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>10.21 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>10.21 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.09316; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h3>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>0 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.2055</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>19 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small></dd></div></dl><p class=\"outcome-counts\">0 completed · 2 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.2055; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h3>pi @ 0.73.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.08706</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>9.744 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>9.744 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.08706; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h3>qwen @ 0.22.2</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>0 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.1233</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>11.77 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small></dd></div></dl><p class=\"outcome-counts\">0 completed · 2 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.1233; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article></div><div class=\"decision-table-scroll\" tabindex=\"0\" role=\"region\" aria-label=\"Observed differences\"><table class=\"decision-differences\"><caption>Observed differences from <span id=\"decision-reference\">codex @ codex-cli 0.149.1</span>. Reference stays fixed when hidden. Positive means more passes, higher cost or longer time.</caption><thead><tr><th scope=\"col\">Harness version</th><th scope=\"col\">Pass difference</th><th scope=\"col\">Mean cost difference</th><th scope=\"col\">All-outcome median time difference</th><th scope=\"col\">Coverage</th></tr></thead><tbody id=\"decision-differences\"><tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.4090<small>+439%</small></td><td>+39.35 min<small>+385.3%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.1123<small>+120.6%</small></td><td>+8.788 min<small>+86.07%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.006101<small>−6.548%</small></td><td>−27.97 s<small>−4.566%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.03014<small>+32.36%</small></td><td>+1.555 min<small>+15.23%</small></td><td>one task; interval unavailable.</td></tr></tbody></table></div><details id=\"decision-explanation\" class=\"joint-explanation\"><summary>Explain time, tokens and failure evidence together</summary><p>API-active union (<code>model_time</code>) is proxy-observed API-active time including transport, not provider compute. Non-model residual (<code>non_model_time</code>) includes unobserved tools and harness work. Timing correlation is a hypothesis, not a diagnosed cause; these observations do not establish a full tool/harness split.</p><p>These are observed median, quartile and min–max distributions, not confidence intervals. Secondary token mean intervals do not replace the headline token medians. No winner-selection or multiple-comparison correction is applied.</p><section data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h4>cline @ 3.0.61</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>49.56 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 36.24 min to 62.87 min; range: 22.92 min to 76.19 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small><small>Q1–Q3: Unavailable; range: Unavailable</small></dd></div><div><dt>API-active union · median</dt><dd><strong>47.07 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 34.53 min to 59.6 min; range: 22 min to 72.14 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>2.466 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 1.681 min to 3.251 min; range: 53.79 s to 4.036 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>598.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 439.8 to 757.3; range: 281 to 916</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>96.27%</strong><small>n=2; missing=0</small><small>Q1–Q3: 95.71% to 96.83%; range: 95.15% to 97.4%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>28,380,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 21,400,000 to 35,360,000; range: 14,420,000 to 42,340,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>397,400</strong><small>n=2; missing=0</small><small>Q1–Q3: 293,800 to 500,900; range: 190,300 to 604,500</small></dd></div></dl><p>Secondary input Mean 28,380,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 397,400; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:cline:tomlkit-toml-table-converters:0</code></li><li><code>pinned:cline:tomlkit-toml-table-converters:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h4>codex @ codex-cli 0.149.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>10.21 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 9.899 min to 10.52 min; range: 9.587 min to 10.83 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>10.21 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 9.899 min to 10.52 min; range: 9.587 min to 10.83 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>9.89 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 9.577 min to 10.2 min; range: 9.264 min to 10.52 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>18.15 s</strong><small>n=2; missing=0</small><small>Q1–Q3: 18.06 s to 18.24 s; range: 17.96 s to 18.34 s</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>68</strong><small>n=2; missing=0</small><small>Q1–Q3: 63.5 to 72.5; range: 59 to 77</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>99%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.96% to 99.05%; range: 98.91% to 99.1%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>7,876,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 7,132,000 to 8,620,000; range: 6,387,000 to 9,365,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>97,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 94,440 to 99,560; range: 91,880 to 102,100</small></dd></div></dl><p>Secondary input Mean 7,876,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 97,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:codex:tomlkit-toml-table-converters:0</code></li><li><code>pinned:codex:tomlkit-toml-table-converters:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h4>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>19 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 16.35 min to 21.64 min; range: 13.71 min to 24.29 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small><small>Q1–Q3: Unavailable; range: Unavailable</small></dd></div><div><dt>API-active union · median</dt><dd><strong>16.67 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 14.78 min to 18.57 min; range: 12.89 min to 20.46 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>2.253 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 1.498 min to 3.008 min; range: 44.62 s to 3.763 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>155.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 135.8 to 175.3; range: 116 to 195</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>99.44%</strong><small>n=2; missing=0</small><small>Q1–Q3: 99.38% to 99.5%; range: 99.32% to 99.56%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>30,330,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 23,440,000 to 37,220,000; range: 16,540,000 to 44,120,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>153,500</strong><small>n=2; missing=0</small><small>Q1–Q3: 134,100 to 172,800; range: 114,800 to 192,200</small></dd></div></dl><p>Secondary input Mean 30,330,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 153,500; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:hermes:tomlkit-toml-table-converters:0</code></li><li><code>pinned:hermes:tomlkit-toml-table-converters:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h4>pi @ 0.73.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>9.744 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 9.702 min to 9.786 min; range: 9.66 min to 9.829 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>9.744 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 9.702 min to 9.786 min; range: 9.66 min to 9.829 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>9.222 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 9.041 min to 9.403 min; range: 8.859 min to 9.585 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>30.58 s</strong><small>n=2; missing=0</small><small>Q1–Q3: 22.24 s to 38.93 s; range: 13.89 s to 47.28 s</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>71</strong><small>n=2; missing=0</small><small>Q1–Q3: 65.5 to 76.5; range: 60 to 82</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.75%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.59% to 98.91%; range: 98.44% to 99.07%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>6,426,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 5,815,000 to 7,038,000; range: 5,203,000 to 7,649,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>94,300</strong><small>n=2; missing=0</small><small>Q1–Q3: 91,410 to 97,190; range: 88,530 to 100,100</small></dd></div></dl><p>Secondary input Mean 6,426,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 94,300; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:pi:tomlkit-toml-table-converters:0</code></li><li><code>pinned:pi:tomlkit-toml-table-converters:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h4>qwen @ 0.22.2</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>11.77 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 10.49 min to 13.04 min; range: 9.209 min to 14.32 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small><small>Q1–Q3: Unavailable; range: Unavailable</small></dd></div><div><dt>API-active union · median</dt><dd><strong>11.2 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 9.945 min to 12.45 min; range: 8.691 min to 13.71 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>32.51 s</strong><small>n=2; missing=0</small><small>Q1–Q3: 31.08 s to 33.94 s; range: 29.65 s to 35.37 s</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>118</strong><small>n=2; missing=0</small><small>Q1–Q3: 106 to 130; range: 94 to 142</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>99.47%</strong><small>n=2; missing=0</small><small>Q1–Q3: 99.42% to 99.52%; range: 99.37% to 99.56%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>15,990,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 13,560,000 to 18,430,000; range: 11,120,000 to 20,860,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>105,800</strong><small>n=2; missing=0</small><small>Q1–Q3: 94,380 to 117,300; range: 82,930 to 128,700</small></dd></div></dl><p>Secondary input Mean 15,990,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 105,800; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:qwen:tomlkit-toml-table-converters:0</code></li><li><code>pinned:qwen:tomlkit-toml-table-converters:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><p><a href=\"report.html\">Detailed timing report and request timelines</a> · <a href=\"data/diagnostics.json\">Download diagnostics</a></p></details>",
            "differences": {
              "[\"cline\",\"3.0.61\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.4090<small>−81.45%</small></td><td>−39.35 min<small>−79.4%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.2967<small>−59.08%</small></td><td>−30.56 min<small>−61.66%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.4151<small>−82.66%</small></td><td>−39.81 min<small>−80.34%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.3789<small>−75.45%</small></td><td>−37.79 min<small>−76.26%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"codex\",\"codex-cli 0.149.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.4090<small>+439%</small></td><td>+39.35 min<small>+385.3%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.1123<small>+120.6%</small></td><td>+8.788 min<small>+86.07%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.006101<small>−6.548%</small></td><td>−27.97 s<small>−4.566%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.03014<small>+32.36%</small></td><td>+1.555 min<small>+15.23%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.2967<small>+144.4%</small></td><td>+30.56 min<small>+160.8%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.1123<small>−54.67%</small></td><td>−8.788 min<small>−46.26%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19) (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.1184<small>−57.63%</small></td><td>−9.255 min<small>−48.71%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.08219<small>−40%</small></td><td>−7.234 min<small>−38.07%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"pi\",\"0.73.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.4151<small>+476.8%</small></td><td>+39.81 min<small>+408.6%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.006101<small>+7.007%</small></td><td>+27.97 s<small>+4.784%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.1184<small>+136%</small></td><td>+9.255 min<small>+94.97%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.03624<small>+41.63%</small></td><td>+2.021 min<small>+20.74%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"qwen\",\"0.22.2\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.3789<small>+307.3%</small></td><td>+37.79 min<small>+321.2%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.03014<small>−24.45%</small></td><td>−1.555 min<small>−13.21%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.08219<small>+66.66%</small></td><td>+7.234 min<small>+61.48%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.03624<small>−29.39%</small></td><td>−2.021 min<small>−17.18%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr>"
            }
          }
        },
        {
          "id": "[\"true-myth-iterable-collection-combinators\",\"5945ad1b96e228ff716275d8a273282101152508\"]",
          "task": "true-myth-iterable-collection-combinators",
          "confidence": [
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"codex\",\"codex-cli 0.149.1\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"cline\",\"3.0.61\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": -100,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"codex\",\"codex-cli 0.149.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"pi\",\"0.73.1\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            },
            {
              "left": "[\"pi\",\"0.73.1\"]",
              "right": "[\"qwen\",\"0.22.2\"]",
              "difference_pp": 0,
              "tasks": 1,
              "attempts_per_harness": 2,
              "interval": null,
              "interpretation": "insufficient-tasks"
            }
          ],
          "confidenceByBaseline": {
            "[\"cline\",\"3.0.61\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 100,
                "interval": null,
                "difference_text": "+100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              }
            ],
            "[\"codex\",\"codex-cli 0.149.1\"]": [
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"pi\",\"0.73.1\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"qwen\",\"0.22.2\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "qwen @ 0.22.2"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ],
            "[\"qwen\",\"0.22.2\"]": [
              {
                "harness": "[\"codex\",\"codex-cli 0.149.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "codex @ codex-cli 0.149.1"
              },
              {
                "harness": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)"
              },
              {
                "harness": "[\"pi\",\"0.73.1\"]",
                "difference": 0,
                "interval": null,
                "difference_text": "0.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "pi @ 0.73.1"
              },
              {
                "harness": "[\"cline\",\"3.0.61\"]",
                "difference": -100,
                "interval": null,
                "difference_text": "-100.0 pp",
                "interval_text": "Unavailable: only one task",
                "note": "More distinct tasks needed",
                "label": "cline @ 3.0.61"
              }
            ]
          },
          "total": "All harnesses combined: $0.775 for 10 selected runs at the shared reference prices.",
          "rows": [
            {
              "harness": "cline",
              "version": "3.0.61",
              "selected": 2,
              "passes": 1,
              "pass_rate": 50,
              "tasks": 1,
              "cost": {
                "value": 0.09427287599999996,
                "n": 2
              },
              "benchmark": {
                "value": 0.18854575199999993,
                "n": 2
              },
              "cache": {
                "value": 96.44368797122053,
                "n": 2
              },
              "input": {
                "value": 6349182,
                "n": 2
              },
              "output": {
                "value": 70057,
                "n": 2
              },
              "estimated": true,
              "id": "[\"cline\",\"3.0.61\"]",
              "label": "cline @ 3.0.61",
              "recorded": {
                "passes": 0,
                "pass_rate": 0,
                "pass_score": 0
              },
              "pass_estimate": {
                "passed": 1,
                "total": 2,
                "rate": 50
              },
              "pass_display": {
                "passed": 1,
                "rate": 50,
                "estimated": true,
                "score": 50
              },
              "scores": {
                "pass": 50,
                "cost": 65.38371652096414,
                "benchmark": 65.38371652096414,
                "cache": 97.70508103300617,
                "input": 74.95288369430897,
                "output": 71.9792454715446
              },
              "display": {
                "pass": "50.0%",
                "cost": "$0.094",
                "benchmark": "$0.189",
                "cache": "96.4%",
                "input": "6.35M",
                "output": "70.1K"
              }
            },
            {
              "harness": "codex",
              "version": "codex-cli 0.149.1",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.06163911000000002,
                "n": 2
              },
              "benchmark": {
                "value": 0.12327822000000004,
                "n": 2
              },
              "cache": {
                "value": 97.70340996626325,
                "n": 2
              },
              "input": {
                "value": 4758895,
                "n": 2
              },
              "output": {
                "value": 57334.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"codex\",\"codex-cli 0.149.1\"]",
              "label": "codex @ codex-cli 0.149.1",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 100,
                "benchmark": 100,
                "cache": 98.98127901126513,
                "input": 100,
                "output": 87.9514079655356
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.062",
                "benchmark": "$0.123",
                "cache": "97.7%",
                "input": "4.76M",
                "output": "57.3K"
              }
            },
            {
              "harness": "hermes",
              "version": "Hermes Agent v0.20.5 (2026.8.19)",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.09676101300000006,
                "n": 2
              },
              "benchmark": {
                "value": 0.19352202600000012,
                "n": 2
              },
              "cache": {
                "value": 98.70897905364868,
                "n": 2
              },
              "input": {
                "value": 10573609.5,
                "n": 2
              },
              "output": {
                "value": 75101.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]",
              "label": "hermes @ Hermes Agent v0.20.5 (2026.8.19)",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 63.70242320633825,
                "benchmark": 63.70242320633825,
                "cache": 100,
                "input": 45.00728913811315,
                "output": 67.14446449138832
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.097",
                "benchmark": "$0.194",
                "cache": "98.7%",
                "input": "10.57M",
                "output": "75.1K"
              }
            },
            {
              "harness": "pi",
              "version": "0.73.1",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.06706598100000002,
                "n": 2
              },
              "benchmark": {
                "value": 0.13413196200000005,
                "n": 2
              },
              "cache": {
                "value": 98.10791248453525,
                "n": 2
              },
              "input": {
                "value": 6181473.5,
                "n": 2
              },
              "output": {
                "value": 52370.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"pi\",\"0.73.1\"]",
              "label": "pi @ 0.73.1",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 91.90816130759349,
                "benchmark": 91.90816130759349,
                "cache": 99.39107204341893,
                "input": 76.9864175588555,
                "output": 96.28798655731757
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.067",
                "benchmark": "$0.134",
                "cache": "98.1%",
                "input": "6.18M",
                "output": "52.4K"
              }
            },
            {
              "harness": "qwen",
              "version": "0.22.2",
              "selected": 2,
              "passes": 2,
              "pass_rate": 100,
              "tasks": 1,
              "cost": {
                "value": 0.06779655599999995,
                "n": 2
              },
              "benchmark": {
                "value": 0.1355931119999999,
                "n": 2
              },
              "cache": {
                "value": 98.35654697732707,
                "n": 2
              },
              "input": {
                "value": 7229000,
                "n": 2
              },
              "output": {
                "value": 50426.5,
                "n": 2
              },
              "estimated": false,
              "id": "[\"qwen\",\"0.22.2\"]",
              "label": "qwen @ 0.22.2",
              "recorded": {
                "passes": 2,
                "pass_rate": 100,
                "pass_score": 100
              },
              "pass_estimate": null,
              "pass_display": {
                "passed": 2,
                "rate": 100,
                "estimated": false,
                "score": 100
              },
              "scores": {
                "pass": 100,
                "cost": 90.91775989329024,
                "benchmark": 90.91775989329024,
                "cache": 99.64295844238237,
                "input": 65.83061280951722,
                "output": 100
              },
              "display": {
                "pass": "100.0%",
                "cost": "$0.068",
                "benchmark": "$0.136",
                "cache": "98.4%",
                "input": "7.23M",
                "output": "50.4K"
              }
            }
          ],
          "decision": {
            "html": "<h3>true-myth-iterable-collection-combinators</h3><p>1 distinct task; 2 repetitions per task and harness version. Pass differences and cost means use equal task weights; pass counts include every selected attempt. Recorded verifier image/environment identities match within each task. Recorded configuration details are unavailable. Observed differences are not causal and do not establish future superiority. Exploratory 95% task-cluster intervals use 10,000 deterministic resamples; repetitions are not independent tasks or paired random seeds. No correction for multiple comparisons or winner selection. Secondary cost/token intervals describe means, not headline token medians. Original grades remain unchanged; corrected verification is a separate audit with limited replay coverage.</p><p>Time is agent end-to-end elapsed time, excluding setup and verification. All-outcome values include fast failures; successful timing uses only passes. Reference cost includes all selected outcomes and weights tasks equally. Unmeasured workflows: not measured.</p><div class=\"decision-grid\"><article class=\"decision-row\" data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h3>cline @ 3.0.61</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>0 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.09427</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>7.915 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small></dd></div></dl><p class=\"outcome-counts\">0 completed · 2 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.09427; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h3>codex @ codex-cli 0.149.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.06164</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>6.092 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>6.092 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.06164; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h3>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.09676</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>8.512 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>8.512 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.09676; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h3>pi @ 0.73.1</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.06707</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>6.514 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>6.514 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.06707; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article><article class=\"decision-row\" data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h3>qwen @ 0.22.2</h3><dl class=\"decision-measures\"><div><dt>Original passes</dt><dd><strong>2 / 2 passes</strong></dd></div><div><dt>All-attempt reference cost / task</dt><dd><strong>$0.06780</strong><small>n=2; missing=0</small></dd></div><div><dt>All-outcome agent time · median</dt><dd><strong>5.13 min</strong><small>n=2; missing=0</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>5.13 min</strong><small>n=2; missing=0</small></dd></div></dl><p class=\"outcome-counts\">2 completed · 0 verification failures · 0 timeouts · 0 adapter errors</p><p class=\"decision-interval\">Mean $0.06780; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p></article></div><div class=\"decision-table-scroll\" tabindex=\"0\" role=\"region\" aria-label=\"Observed differences\"><table class=\"decision-differences\"><caption>Observed differences from <span id=\"decision-reference\">codex @ codex-cli 0.149.1</span>. Reference stays fixed when hidden. Positive means more passes, higher cost or longer time.</caption><thead><tr><th scope=\"col\">Harness version</th><th scope=\"col\">Pass difference</th><th scope=\"col\">Mean cost difference</th><th scope=\"col\">All-outcome median time difference</th><th scope=\"col\">Coverage</th></tr></thead><tbody id=\"decision-differences\"><tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.03263<small>+52.94%</small></td><td>+1.823 min<small>+29.93%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.03512<small>+56.98%</small></td><td>+2.421 min<small>+39.74%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.005427<small>+8.804%</small></td><td>+25.36 s<small>+6.939%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.006157<small>+9.99%</small></td><td>−57.71 s<small>−15.79%</small></td><td>one task; interval unavailable.</td></tr></tbody></table></div><details id=\"decision-explanation\" class=\"joint-explanation\"><summary>Explain time, tokens and failure evidence together</summary><p>API-active union (<code>model_time</code>) is proxy-observed API-active time including transport, not provider compute. Non-model residual (<code>non_model_time</code>) includes unobserved tools and harness work. Timing correlation is a hypothesis, not a diagnosed cause; these observations do not establish a full tool/harness split.</p><p>These are observed median, quartile and min–max distributions, not confidence intervals. Secondary token mean intervals do not replace the headline token medians. No winner-selection or multiple-comparison correction is applied.</p><section data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><h4>cline @ 3.0.61</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>7.915 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 7.785 min to 8.045 min; range: 7.655 min to 8.174 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>Unavailable</strong><small>n=0; missing=0</small><small>Q1–Q3: Unavailable; range: Unavailable</small></dd></div><div><dt>API-active union · median</dt><dd><strong>7.082 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 6.861 min to 7.303 min; range: 6.64 min to 7.524 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>48.75 s</strong><small>n=2; missing=0</small><small>Q1–Q3: 43.29 s to 54.21 s; range: 37.82 s to 59.67 s</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>117.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 115.3 to 119.8; range: 113 to 122</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>96.44%</strong><small>n=2; missing=0</small><small>Q1–Q3: 96.44% to 96.45%; range: 96.43% to 96.45%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>6,349,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 6,311,000 to 6,387,000; range: 6,273,000 to 6,425,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>70,060</strong><small>n=2; missing=0</small><small>Q1–Q3: 69,860 to 70,260; range: 69,660 to 70,460</small></dd></div></dl><p>Secondary input Mean 6,349,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 70,060; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 verification unknown. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:cline:true-myth-iterable-collection-combinators:0</code></li><li><code>pinned:cline:true-myth-iterable-collection-combinators:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><h4>codex @ codex-cli 0.149.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>6.092 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 5.135 min to 7.048 min; range: 4.179 min to 8.005 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>6.092 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 5.135 min to 7.048 min; range: 4.179 min to 8.005 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>4.915 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 4.306 min to 5.523 min; range: 3.698 min to 6.132 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>1.166 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 49.05 s to 1.514 min; range: 28.17 s to 1.862 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>49</strong><small>n=2; missing=0</small><small>Q1–Q3: 39.5 to 58.5; range: 30 to 68</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>97.7%</strong><small>n=2; missing=0</small><small>Q1–Q3: 97.25% to 98.15%; range: 96.8% to 98.6%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>4,759,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 3,585,000 to 5,933,000; range: 2,412,000 to 7,106,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>57,330</strong><small>n=2; missing=0</small><small>Q1–Q3: 50,520 to 64,150; range: 43,700 to 70,970</small></dd></div></dl><p>Secondary input Mean 4,759,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 57,330; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:codex:true-myth-iterable-collection-combinators:0</code></li><li><code>pinned:codex:true-myth-iterable-collection-combinators:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><h4>hermes @ Hermes Agent v0.20.5 (2026.8.19)</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>8.512 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 8.305 min to 8.72 min; range: 8.097 min to 8.927 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>8.512 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 8.305 min to 8.72 min; range: 8.097 min to 8.927 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>7.094 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 6.947 min to 7.241 min; range: 6.8 min to 7.388 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>1.348 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 1.289 min to 1.408 min; range: 1.229 min to 1.467 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>76</strong><small>n=2; missing=0</small><small>Q1–Q3: 73.5 to 78.5; range: 71 to 81</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.71%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.69% to 98.73%; range: 98.66% to 98.75%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>10,570,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 9,927,000 to 11,220,000; range: 9,281,000 to 11,870,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>75,100</strong><small>n=2; missing=0</small><small>Q1–Q3: 72,090 to 78,110; range: 69,080 to 81,130</small></dd></div></dl><p>Secondary input Mean 10,570,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 75,100; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:hermes:true-myth-iterable-collection-combinators:0</code></li><li><code>pinned:hermes:true-myth-iterable-collection-combinators:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><h4>pi @ 0.73.1</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>6.514 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 5.909 min to 7.12 min; range: 5.303 min to 7.726 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>6.514 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 5.909 min to 7.12 min; range: 5.303 min to 7.726 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>5.164 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 4.713 min to 5.615 min; range: 4.262 min to 6.066 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>1.338 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 1.183 min to 1.492 min; range: 1.028 min to 1.647 min</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>58.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 54.25 to 62.75; range: 50 to 67</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.11%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.03% to 98.18%; range: 97.96% to 98.26%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>6,181,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 5,970,000 to 6,393,000; range: 5,758,000 to 6,604,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>52,370</strong><small>n=2; missing=0</small><small>Q1–Q3: 49,790 to 54,950; range: 47,220 to 57,530</small></dd></div></dl><p>Secondary input Mean 6,181,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 52,370; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:pi:true-myth-iterable-collection-combinators:0</code></li><li><code>pinned:pi:true-myth-iterable-collection-combinators:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><section data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><h4>qwen @ 0.22.2</h4><dl class=\"decision-measures\"><div><dt>All-outcome agent time · median</dt><dd><strong>5.13 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 4.452 min to 5.808 min; range: 3.773 min to 6.486 min</small></dd></div><div><dt>Successful agent time · median</dt><dd><strong>5.13 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 4.452 min to 5.808 min; range: 3.773 min to 6.486 min</small></dd></div><div><dt>API-active union · median</dt><dd><strong>4.574 min</strong><small>n=2; missing=0</small><small>Q1–Q3: 3.999 min to 5.149 min; range: 3.424 min to 5.724 min</small></dd></div><div><dt>Non-model residual · median</dt><dd><strong>31.47 s</strong><small>n=2; missing=0</small><small>Q1–Q3: 25.07 s to 37.88 s; range: 18.67 s to 44.28 s</small></dd></div><div><dt>Requests per attempt · median</dt><dd><strong>62.5</strong><small>n=2; missing=0</small><small>Q1–Q3: 54.75 to 70.25; range: 47 to 78</small></dd></div><div><dt>Cached input percent · median</dt><dd><strong>98.36%</strong><small>n=2; missing=0</small><small>Q1–Q3: 98.14% to 98.58%; range: 97.92% to 98.8%</small></dd></div><div><dt>Input tokens · median</dt><dd><strong>7,229,000</strong><small>n=2; missing=0</small><small>Q1–Q3: 5,980,000 to 8,478,000; range: 4,730,000 to 9,728,000</small></dd></div><div><dt>Output tokens · median</dt><dd><strong>50,430</strong><small>n=2; missing=0</small><small>Q1–Q3: 43,700 to 57,150; range: 36,980 to 63,870</small></dd></div></dl><p>Secondary input Mean 7,229,000; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Secondary output Mean 50,430; exploratory 95% interval: Unavailable: one task<small>n=2; missing=0; 1 task clusters</small></p><p>Visibility: 0 none, 2 partial, 0 full. A visibility flag alone is not native timing qualification.</p><p>Evidence labels: 2 pass. Verification-unknown does not diagnose an agent or verifier cause.</p><p>Selected run IDs (image/environment identities and request timelines in the linked evidence):</p><ul class=\"run-evidence\"><li><code>pinned:qwen:true-myth-iterable-collection-combinators:0</code></li><li><code>pinned:qwen:true-myth-iterable-collection-combinators:1</code></li></ul><p><a href=\"data/analysis.json\">Inspect run identities and measurements</a> · <a href=\"data/diagnostics.json\">Inspect bound failure evidence</a></p></section><p><a href=\"report.html\">Detailed timing report and request timelines</a> · <a href=\"data/diagnostics.json\">Download diagnostics</a></p></details>",
            "differences": {
              "[\"cline\",\"3.0.61\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.03263<small>−34.62%</small></td><td>−1.823 min<small>−23.04%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.002488<small>+2.639%</small></td><td>+35.84 s<small>+7.548%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.02721<small>−28.86%</small></td><td>−1.401 min<small>−17.7%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>+100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.02648<small>−28.08%</small></td><td>−2.785 min<small>−35.19%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"codex\",\"codex-cli 0.149.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.03263<small>+52.94%</small></td><td>+1.823 min<small>+29.93%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.03512<small>+56.98%</small></td><td>+2.421 min<small>+39.74%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.005427<small>+8.804%</small></td><td>+25.36 s<small>+6.939%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.006157<small>+9.99%</small></td><td>−57.71 s<small>−15.79%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"hermes\",\"Hermes Agent v0.20.5 (2026.8.19)\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.002488<small>−2.571%</small></td><td>−35.84 s<small>−7.018%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.03512<small>−36.3%</small></td><td>−2.421 min<small>−28.44%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19) (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.02970<small>−30.69%</small></td><td>−1.998 min<small>−23.47%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.02896<small>−29.93%</small></td><td>−3.382 min<small>−39.74%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"pi\",\"0.73.1\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.02721<small>+40.57%</small></td><td>+1.401 min<small>+21.5%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.005427<small>−8.092%</small></td><td>−25.36 s<small>−6.489%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.02970<small>+44.28%</small></td><td>+1.998 min<small>+30.67%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.0007306<small>+1.089%</small></td><td>−1.385 min<small>−21.25%</small></td><td>one task; interval unavailable.</td></tr>",
              "[\"qwen\",\"0.22.2\"]": "<tr data-decision-harness=\"[&quot;cline&quot;,&quot;3.0.61&quot;]\"><th scope=\"row\">cline @ 3.0.61</th><td>−100 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.02648<small>+39.05%</small></td><td>+2.785 min<small>+54.29%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;codex&quot;,&quot;codex-cli 0.149.1&quot;]\"><th scope=\"row\">codex @ codex-cli 0.149.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.006157<small>−9.082%</small></td><td>+57.71 s<small>+18.75%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;hermes&quot;,&quot;Hermes Agent v0.20.5 (2026.8.19)&quot;]\"><th scope=\"row\">hermes @ Hermes Agent v0.20.5 (2026.8.19)</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>+$0.02896<small>+42.72%</small></td><td>+3.382 min<small>+65.94%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;pi&quot;,&quot;0.73.1&quot;]\"><th scope=\"row\">pi @ 0.73.1</th><td>0 pp<small>Exploratory 95%: one task; interval unavailable</small></td><td>−$0.0007306<small>−1.078%</small></td><td>+1.385 min<small>+26.99%</small></td><td>one task; interval unavailable.</td></tr><tr data-decision-harness=\"[&quot;qwen&quot;,&quot;0.22.2&quot;]\"><th scope=\"row\">qwen @ 0.22.2 (reference)</th><td>0 pp<small>Exploratory 95%: Reference</small></td><td>$0<small>0%</small></td><td>0 s<small>0%</small></td><td>Reference row; no comparison interval.</td></tr>"
            }
          }
        }
      ],
      "id": "[\"linux-results-2026-10-24\",\"[\\\"linux\\\",\\\"x64\\\",30.168060302734375,\\\"pinned\\\",\\\"deepseek/deepseek-v4.1-flash-1017\\\",\\\"deepseek-v41-1017-reference-2026-10-18\\\",\\\"https://github.com/datacurve-ai/deep-swe.git\\\",\\\"public-task-pack\\\",\\\"0b9fabbb63b9104d678fe965e1632f2dd9eaa2ea\\\",\\\"extended\\\",[[\\\"relace\\\"],\\\"deepseek\\\",false],null,null]\"]",
      "dataset": "linux-results-2026-10-24",
      "population_key": "[\"linux\",\"x64\",30.168060302734375,\"pinned\",\"deepseek/deepseek-v4.1-flash-1017\",\"deepseek-v41-1017-reference-2026-10-18\",\"https://github.com/datacurve-ai/deep-swe.git\",\"public-task-pack\",\"0b9fabbb63b9104d678fe965e1632f2dd9eaa2ea\",\"extended\",[[\"relace\"],\"deepseek\",false],null,null]",
      "label": "deepseek/deepseek-v4.1-flash-1017 · configuration not recorded · linux-results-2026-10-24",
      "context": "linux · pinned · extended",
      "baseline": "[\"codex\",\"codex-cli 0.149.1\"]",
      "links": {
        "comparison": "data/comparison.json",
        "summary": "data/summary.json",
        "provenance": "data/provenance.json",
        "readme": "data/README.md",
        "analysis": "data/analysis.json",
        "diagnostics": "data/diagnostics.json",
        "report": "report.html",
        "source": "data/source.json",
        "audit": null,
        "published": "data/report.json"
      },
      "audit": "",
      "evidence": "\n",
      "pricing": "$0.15 per million uncached input tokens, $0.003 per million cached input tokens, and $0.6 per million output tokens. Reference estimates, not billing. Reference book: deepseek-v41-1017-reference-2026-10-18. Original books: deepseek-v41-1017-reference-2026-10-18.",
      "provenance": "Dataset: linux-results-2026-10-24. Analysis: sha256:c66eff9528e366dd0e8af2c050086e98770c1424c0a969298a93d071415e4fba."
    }
  ],
  "populations": [
    {
      "key": "[\"linux\",\"x64\",30.168060302734375,\"pinned\",\"deepseek/deepseek-v4.1-flash-1017\",\"deepseek-v41-1017-reference-2026-10-18\",\"https://github.com/datacurve-ai/deep-swe.git\",\"public-task-pack\",\"0b9fabbb63b9104d678fe965e1632f2dd9eaa2ea\",\"extended\",[[\"relace\"],\"deepseek\",false],null,null]",
      "dataset": "linux-results-2026-10-24",
      "run_ids": [
        "pinned:cline:cattrs-partial-structuring-recovery:0",
        "pinned:cline:cattrs-partial-structuring-recovery:1",
        "pinned:cline:happy-dom-deterministic-intersectionobserver:0",
        "pinned:cline:happy-dom-deterministic-intersectionobserver:1",
        "pinned:cline:ink-grid-box-layout:0",
        "pinned:cline:ink-grid-box-layout:1",
        "pinned:cline:psd-tools-blend-range-api:0",
        "pinned:cline:psd-tools-blend-range-api:1",
        "pinned:cline:superjson-error-stack-serialization:0",
        "pinned:cline:superjson-error-stack-serialization:1",
        "pinned:cline:textual-richlog-follow-state:0",
        "pinned:cline:textual-richlog-follow-state:1",
        "pinned:cline:tomlkit-toml-table-converters:0",
        "pinned:cline:tomlkit-toml-table-converters:1",
        "pinned:cline:true-myth-iterable-collection-combinators:0",
        "pinned:cline:true-myth-iterable-collection-combinators:1",
        "pinned:codex:cattrs-partial-structuring-recovery:0",
        "pinned:codex:cattrs-partial-structuring-recovery:1",
        "pinned:codex:happy-dom-deterministic-intersectionobserver:0",
        "pinned:codex:happy-dom-deterministic-intersectionobserver:1",
        "pinned:codex:ink-grid-box-layout:0",
        "pinned:codex:ink-grid-box-layout:1",
        "pinned:codex:psd-tools-blend-range-api:0",
        "pinned:codex:psd-tools-blend-range-api:1",
        "pinned:codex:superjson-error-stack-serialization:0",
        "pinned:codex:superjson-error-stack-serialization:1",
        "pinned:codex:textual-richlog-follow-state:0",
        "pinned:codex:textual-richlog-follow-state:1",
        "pinned:codex:tomlkit-toml-table-converters:0",
        "pinned:codex:tomlkit-toml-table-converters:1",
        "pinned:codex:true-myth-iterable-collection-combinators:0",
        "pinned:codex:true-myth-iterable-collection-combinators:1",
        "pinned:hermes:cattrs-partial-structuring-recovery:0",
        "pinned:hermes:cattrs-partial-structuring-recovery:1",
        "pinned:hermes:happy-dom-deterministic-intersectionobserver:0",
        "pinned:hermes:happy-dom-deterministic-intersectionobserver:1",
        "pinned:hermes:ink-grid-box-layout:0",
        "pinned:hermes:ink-grid-box-layout:1",
        "pinned:hermes:psd-tools-blend-range-api:0",
        "pinned:hermes:psd-tools-blend-range-api:1",
        "pinned:hermes:superjson-error-stack-serialization:0",
        "pinned:hermes:superjson-error-stack-serialization:1",
        "pinned:hermes:textual-richlog-follow-state:0",
        "pinned:hermes:textual-richlog-follow-state:1",
        "pinned:hermes:tomlkit-toml-table-converters:0",
        "pinned:hermes:tomlkit-toml-table-converters:1",
        "pinned:hermes:true-myth-iterable-collection-combinators:0",
        "pinned:hermes:true-myth-iterable-collection-combinators:1",
        "pinned:pi:cattrs-partial-structuring-recovery:0",
        "pinned:pi:cattrs-partial-structuring-recovery:1",
        "pinned:pi:happy-dom-deterministic-intersectionobserver:0",
        "pinned:pi:happy-dom-deterministic-intersectionobserver:1",
        "pinned:pi:ink-grid-box-layout:0",
        "pinned:pi:ink-grid-box-layout:1",
        "pinned:pi:psd-tools-blend-range-api:0",
        "pinned:pi:psd-tools-blend-range-api:1",
        "pinned:pi:superjson-error-stack-serialization:0",
        "pinned:pi:superjson-error-stack-serialization:1",
        "pinned:pi:textual-richlog-follow-state:0",
        "pinned:pi:textual-richlog-follow-state:1",
        "pinned:pi:tomlkit-toml-table-converters:0",
        "pinned:pi:tomlkit-toml-table-converters:1",
        "pinned:pi:true-myth-iterable-collection-combinators:0",
        "pinned:pi:true-myth-iterable-collection-combinators:1",
        "pinned:qwen:cattrs-partial-structuring-recovery:0",
        "pinned:qwen:cattrs-partial-structuring-recovery:1",
        "pinned:qwen:happy-dom-deterministic-intersectionobserver:0",
        "pinned:qwen:happy-dom-deterministic-intersectionobserver:1",
        "pinned:qwen:ink-grid-box-layout:0",
        "pinned:qwen:ink-grid-box-layout:1",
        "pinned:qwen:psd-tools-blend-range-api:0",
        "pinned:qwen:psd-tools-blend-range-api:1",
        "pinned:qwen:superjson-error-stack-serialization:0",
        "pinned:qwen:superjson-error-stack-serialization:1",
        "pinned:qwen:textual-richlog-follow-state:0",
        "pinned:qwen:textual-richlog-follow-state:1",
        "pinned:qwen:tomlkit-toml-table-converters:0",
        "pinned:qwen:tomlkit-toml-table-converters:1",
        "pinned:qwen:true-myth-iterable-collection-combinators:0",
        "pinned:qwen:true-myth-iterable-collection-combinators:1"
      ]
    }
  ],
  "context": {
    "description": "Explore coding-agent measurements from two tool cohorts. Descriptive snapshot; full experiment cost is unknown.",
    "noticeHtml": "<p>80 completion-selected outcomes across two tool cohorts; 83 observed paid primary attempts and 52 prerequisites. This is a descriptive snapshot, not one randomized five-CLI experiment. Full experiment cost is unknown. Three partial native captures retain valid required grade artifacts. Native tool timing remains partial. <a href=\"https://github.com/Tom910/agent-overhead-bench/blob/main/docs/snapshots/2026-10-24-five-cli.md\">Read result interpretation and native workflow findings</a>.</p>",
    "reportData": "data/report.json",
    "source": "data/source.json",
    "readme": "data/README.md",
    "summaryLabel": "Selection summary",
    "limitsHtml": "This is a completion-selected descriptive snapshot across original tool cohorts. It is not the frozen v1 dataset.",
    "auditHtml": "",
    "limitsAuditHtml": "Pass-rate estimates and recorded results are identified in the report.",
    "tableNoteHtml": "<p>Estimated pass rates use retained traces. <a href=\"#cline-result-notes\">Method notes</a>.</p>",
    "methodologyHtml": "<details id=\"cline-result-notes\"><summary>Pass-rate method notes</summary><p>Estimated pass rates use the retained per-run audit. Other pass rates use recorded grades. Original results remain in the source data.</p><p><a href=\"data/cline-workflow-audit.json\">Calculation and per-run evidence</a></p></details>"
  },
  "estimate_evidence": {
    "file": "cline-workflow-audit.json",
    "sha256": "sha256:1383d088c1359c8e7f2c661ec85528448092665fec246773b6377f0c925a721c"
  },
  "appendix_html": "<section id=\"extra-attempts\" style=\"width:100%;max-width:1280px;margin:auto;padding:24px;box-sizing:border-box\"><h2>Original incomplete attempts: time, reliability and cost</h2><p>Three extra attempts are outside the selected 80. Prefix estimates do not give complete charges.</p><div style=\"max-width:100%;overflow:auto\"><table><tr><th>Attempt</th><th>Outcome</th><th>Adapter time</th><th>Known prefix reference estimate</th><th>Full cost</th></tr><tr><td>pinned:cline:textual-richlog-follow-state:0</td><td>timeout</td><td>10,800.372 s</td><td>$1.836203754</td><td>unknown</td></tr><tr><td>pinned:cline:tomlkit-toml-table-converters:1</td><td>budget-interrupted</td><td>unknown</td><td>$0.444080070</td><td>unknown</td></tr><tr><td>pinned:pi:textual-richlog-follow-state:0</td><td>interrupted</td><td>unknown</td><td>$0.003784620</td><td>unknown</td></tr></table></div><p><a href=\"data/extra-attempts.json\">All extra attempt facts and captured request series</a> · <a href=\"data/cohorts.json\">Original cohorts and collection windows</a> · <a href=\"data/collection-ledger.json\">Collection ledger</a> · <a href=\"https://github.com/Tom910/agent-overhead-bench/tree/main/evidence/linux-results-2026-09-19-r1\">Historical snapshot</a></p></section>"
}