{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.12.14"
    },
    "execution": {
      "method": "Python subprocess, exact code cell source",
      "deterministic_replay_passed": true
    },
    "runtime": {
      "packages": [],
      "local_assets": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "id": "question",
      "metadata": {},
      "source": [
        "# When benchmark averages reverse an agent ranking\n",
        "An 83% overall score can lose to 80% under equal family weighting. The crossover can be calculated rather than argued over.\n",
        "\nAggregation and bootstrap demonstration on authored labels; not AgentBench model evaluation.\n"
      ]
    },
    {
      "cell_type": "code",
      "id": "experiment",
      "metadata": {},
      "execution_count": 1,
      "source": [
        "\"\"\"When benchmark averages reverse an agent ranking\n",
        "Deterministic, standard-library experiment. RESULTS contains the full data.\n",
        "\"\"\"\n",
        "from random import Random\n",
        "counts = [('lookup', 80, 76, 64), ('editing', 10, 6, 8),\n",
        "          ('planning', 5, 1, 4), ('recovery', 5, 0, 4)]\n",
        "rows = {name: [(int(i < a), int(i < b)) for i in range(n)]\n",
        "        for name, n, a, b in counts}\n",
        "\n",
        "def macro(data, policy):\n",
        "    return sum(sum(row[policy] for row in group) / len(group)\n",
        "               for group in data.values()) / len(data)\n",
        "\n",
        "for policy, name in enumerate(('A', 'B')):\n",
        "    micro = sum(row[policy] for group in rows.values() for row in group) / 100\n",
        "    print(name, f'micro={micro:.4f}', f'macro={macro(rows,policy):.4f}')\n",
        "for name, n, a, b in counts:\n",
        "    print(name, f'A={a}/{n}', f'B={b}/{n}')\n",
        "for weight in (0, 0.25, 0.5, 0.75, 1):\n",
        "    a = 0.95 * weight + ((0.6 + 0.2 + 0) / 3) * (1-weight)\n",
        "    print(f'lookup_weight={weight:.2f}', f'A={a:.3f}', 'B=0.800')\n",
        "\n",
        "rng = Random(35)\n",
        "differences = []\n",
        "for _ in range(1000):\n",
        "    sample = {name: [rng.choice(group) for _ in group] for name, group in rows.items()}\n",
        "    differences.append(macro(sample, 0) - macro(sample, 1))\n",
        "differences.sort()\n",
        "print('macro_A_minus_B_bootstrap_95pct',\n",
        "      f'[{differences[24]:.3f},{differences[974]:.3f}]')\n",
        "crossover = (0.8 - 0.8/3) / (0.95 - 0.8/3)\n",
        "print('lookup_weight_crossover', f'{crossover:.4f}')\n",
        "assert abs(macro(rows, 0) - 0.4375) < 1e-12\n",
        "\n",
        "\n",
        "RESULTS=dict(families=[dict(name=n,tasks=t,A=a,B=b,A_rate=a/t,B_rate=b/t) for n,t,a,b in counts],paired_rows=rows,bootstrap_differences=differences,crossover=crossover,micro={'A':.83,'B':.80},macro={'A':macro(rows,0),'B':macro(rows,1)},sensitivity=[dict(lookup_weight=i/100,A=.95*i/100+(.8/3)*(1-i/100),B=.8) for i in range(101)])\n",
        "assert abs(crossover-32/41)<1e-12\n",
        "print('Analytic crossover: 32/41')\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "A micro=0.8300 macro=0.4375\n",
            "B micro=0.8000 macro=0.8000\n",
            "lookup A=76/80 B=64/80\n",
            "editing A=6/10 B=8/10\n",
            "planning A=1/5 B=4/5\n",
            "recovery A=0/5 B=4/5\n",
            "lookup_weight=0.00 A=0.267 B=0.800\n",
            "lookup_weight=0.25 A=0.438 B=0.800\n",
            "lookup_weight=0.50 A=0.608 B=0.800\n",
            "lookup_weight=0.75 A=0.779 B=0.800\n",
            "lookup_weight=1.00 A=0.950 B=0.800\n",
            "macro_A_minus_B_bootstrap_95pct [-0.506,-0.203]\n",
            "lookup_weight_crossover 0.7805\n",
            "Analytic crossover: 32/41\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from pathlib import Path\n",
        "import base64\n",
        "print('__NOTEBOOK_IMAGE__svg+xml:' + base64.b64encode(Path('figure.svg').read_bytes()).decode())\n"
      ]
    }
  ]
}