{
  "protocol": "cold-start-two-runs-descriptive-1",
  "recorded_at": "2026-09-30T18:54:30.662403+00:00",
  "unique_tasks": 4,
  "repetitions_per_arm_per_task": 2,
  "assignment_denominator": 16,
  "totals": {
    "agent": {
      "agent_tool_calls": 296,
      "assignments": 8,
      "observed_model_requests": 304,
      "quality_passed": 6,
      "reported_cost_usd": 1.9429724000000004
    },
    "instincts": {
      "agent_tool_calls": 264,
      "assignments": 8,
      "observed_model_requests": 272,
      "quality_passed": 8,
      "reported_cost_usd": 1.7720458000000001
    }
  },
  "cost_regression_pairs": 3,
  "paired_comparisons": 8,
  "cost_reduction_percent": 8.797170767840045,
  "request_reduction_percent": 10.526315789473683,
  "cpu_action_elimination_tested": false,
  "gpu_utilization_measured": false,
  "limitations": [
    "Same four authored tasks on one pinned Marshmallow repository, two runs per arm; not eight distinct tasks or broad efficacy.",
    "Second run reverses each first-run pair order; same model, product binary, prompts and grading; no outcome retries.",
    "All sixteen outcomes, failures and cost regressions retained.",
    "Intervention is shipped advisory hints, not learned automatic CPU execution.",
    "Replication registered after observing first pilot. This is descriptive evidence, not an independently powered confirmatory trial."
  ],
  "source": {
    "commit": "0584c9e",
    "file": "evals/reflex-proof/harness/results/20260930-cold-start-replication/report.json",
    "sha256": "4744cace0ac4f31424bb500940bbeb272a9c19a224cac72a227e8dcb3e59ab47"
  },
  "rounds": [
    {
      "round": 1,
      "derived_from": "outcomes",
      "agent_cost_usd": 1.0104017,
      "instincts_cost_usd": 0.8452748,
      "cost_reduction_percent": 16.34269815658465,
      "agent_requests": 156,
      "instincts_requests": 124
    },
    {
      "round": 2,
      "derived_from": "outcomes",
      "agent_cost_usd": 0.9325707,
      "instincts_cost_usd": 0.926771,
      "cost_reduction_percent": 0.6219045912551385,
      "agent_requests": 148,
      "instincts_requests": 148
    }
  ],
  "outcomes": [
    {
      "round": 1,
      "task": "mm-curve--odd",
      "arm": "agent",
      "quality_passed": true,
      "cost_usd": 0.2500275,
      "observed_model_requests": 40,
      "agent_tool_calls": 39,
      "collection_errors": []
    },
    {
      "round": 1,
      "task": "mm-curve--odd",
      "arm": "instincts",
      "quality_passed": true,
      "cost_usd": 0.18224880000000002,
      "observed_model_requests": 22,
      "agent_tool_calls": 21,
      "collection_errors": []
    },
    {
      "round": 1,
      "task": "mm-curve--finite",
      "arm": "instincts",
      "quality_passed": true,
      "cost_usd": 0.17719150000000003,
      "observed_model_requests": 26,
      "agent_tool_calls": 25,
      "collection_errors": []
    },
    {
      "round": 1,
      "task": "mm-curve--finite",
      "arm": "agent",
      "quality_passed": false,
      "cost_usd": 0.3325417,
      "observed_model_requests": 51,
      "agent_tool_calls": 50,
      "collection_errors": []
    },
    {
      "round": 1,
      "task": "mm-curve--no-adjacent-duplicates",
      "arm": "agent",
      "quality_passed": true,
      "cost_usd": 0.21653869999999997,
      "observed_model_requests": 33,
      "agent_tool_calls": 32,
      "collection_errors": []
    },
    {
      "round": 1,
      "task": "mm-curve--no-adjacent-duplicates",
      "arm": "instincts",
      "quality_passed": true,
      "cost_usd": 0.24739560000000005,
      "observed_model_requests": 38,
      "agent_tool_calls": 37,
      "collection_errors": []
    },
    {
      "round": 1,
      "task": "mm-curve--max-utf8-bytes",
      "arm": "instincts",
      "quality_passed": true,
      "cost_usd": 0.2384389,
      "observed_model_requests": 38,
      "agent_tool_calls": 37,
      "collection_errors": []
    },
    {
      "round": 1,
      "task": "mm-curve--max-utf8-bytes",
      "arm": "agent",
      "quality_passed": false,
      "cost_usd": 0.21129379999999998,
      "observed_model_requests": 32,
      "agent_tool_calls": 31,
      "collection_errors": []
    },
    {
      "round": 2,
      "task": "mm-curve--odd",
      "arm": "instincts",
      "quality_passed": true,
      "cost_usd": 0.2230678,
      "observed_model_requests": 36,
      "agent_tool_calls": 35,
      "collection_errors": []
    },
    {
      "round": 2,
      "task": "mm-curve--odd",
      "arm": "agent",
      "quality_passed": true,
      "cost_usd": 0.22857440000000007,
      "observed_model_requests": 34,
      "agent_tool_calls": 33,
      "collection_errors": []
    },
    {
      "round": 2,
      "task": "mm-curve--finite",
      "arm": "agent",
      "quality_passed": true,
      "cost_usd": 0.22850790000000007,
      "observed_model_requests": 40,
      "agent_tool_calls": 39,
      "collection_errors": []
    },
    {
      "round": 2,
      "task": "mm-curve--finite",
      "arm": "instincts",
      "quality_passed": true,
      "cost_usd": 0.22051389999999998,
      "observed_model_requests": 38,
      "agent_tool_calls": 37,
      "collection_errors": []
    },
    {
      "round": 2,
      "task": "mm-curve--no-adjacent-duplicates",
      "arm": "instincts",
      "quality_passed": true,
      "cost_usd": 0.21233859999999996,
      "observed_model_requests": 33,
      "agent_tool_calls": 32,
      "collection_errors": []
    },
    {
      "round": 2,
      "task": "mm-curve--no-adjacent-duplicates",
      "arm": "agent",
      "quality_passed": true,
      "cost_usd": 0.23650029999999997,
      "observed_model_requests": 37,
      "agent_tool_calls": 36,
      "collection_errors": []
    },
    {
      "round": 2,
      "task": "mm-curve--max-utf8-bytes",
      "arm": "agent",
      "quality_passed": true,
      "cost_usd": 0.23898810000000006,
      "observed_model_requests": 37,
      "agent_tool_calls": 36,
      "collection_errors": []
    },
    {
      "round": 2,
      "task": "mm-curve--max-utf8-bytes",
      "arm": "instincts",
      "quality_passed": true,
      "cost_usd": 0.27085070000000006,
      "observed_model_requests": 41,
      "agent_tool_calls": 40,
      "collection_errors": []
    }
  ]
}