{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.12.14"
    },
    "execution": {
      "method": "Python subprocess, exact code cell source",
      "deterministic_replay_passed": true
    },
    "runtime": {
      "packages": [],
      "local_assets": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "id": "question",
      "metadata": {},
      "source": [
        "# WebArena’s evaluation lesson: check the final state\n",
        "A cart message, a URL, and a persisted-state predicate disagree on the same 20 terminal states. The disagreement is the test.\n",
        "\nOffline evaluator unit test on hand-labeled synthetic states, not a WebArena run.\n"
      ]
    },
    {
      "cell_type": "code",
      "id": "experiment",
      "metadata": {},
      "execution_count": 1,
      "source": [
        "\"\"\"WebArena’s evaluation lesson: check the final state\n",
        "Deterministic, standard-library experiment. RESULTS contains the full data.\n",
        "\"\"\"\n",
        "TEMPLATES = [\n",
        "    ('valid', {'blue-mug': 2}, 'Added successfully', '/cart', True),\n",
        "    ('quiet', {'blue-mug': 2}, '', '/catalog', True),\n",
        "    ('color', {'red-mug': 2}, 'Added successfully', '/cart', False),\n",
        "    ('count', {'blue-mug': 1}, 'Added successfully', '/cart', False),\n",
        "    ('extra', {'blue-mug': 2, 'red-mug': 1}, 'Added successfully', '/cart', False),\n",
        "]\n",
        "cases = [dict(name=f'{name}-{i}', cart=cart.copy(), message=msg,\n",
        "              url=url, label=label)\n",
        "         for i in range(4) for name, cart, msg, url, label in TEMPLATES]\n",
        "\n",
        "def confusion(check):\n",
        "    result = dict(TP=0, FP=0, FN=0, TN=0)\n",
        "    for case in cases:\n",
        "        predicted, actual = check(case), case['label']\n",
        "        key = ('TP' if actual else 'FP') if predicted else ('FN' if actual else 'TN')\n",
        "        result[key] += 1\n",
        "    return result\n",
        "\n",
        "checks = {\n",
        "    'message': lambda c: 'successfully' in c['message'],\n",
        "    'url': lambda c: c['url'] == '/cart',\n",
        "    'state': lambda c: c['cart'] == {'blue-mug': 2},\n",
        "}\n",
        "for name, check in checks.items():\n",
        "    print(name, confusion(check))\n",
        "assert len(cases) == 20\n",
        "assert confusion(checks['state']) == dict(TP=8, FP=0, FN=0, TN=12)\n",
        "\n",
        "\n",
        "rows = []\n",
        "for case in cases:\n",
        "    rows.append(dict(case, predictions={name:check(case) for name,check in checks.items()}))\n",
        "weak = lambda c: c['cart'].get('blue-mug') == 2\n",
        "assert confusion(weak) == dict(TP=8,FP=4,FN=0,TN=8)\n",
        "RESULTS=dict(cases=rows, confusion={name:confusion(check) for name,check in checks.items()}, positive_only=confusion(weak))\n",
        "print('Positive-only predicate',confusion(weak))\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "message {'TP': 4, 'FP': 12, 'FN': 4, 'TN': 0}\n",
            "url {'TP': 4, 'FP': 12, 'FN': 4, 'TN': 0}\n",
            "state {'TP': 8, 'FP': 0, 'FN': 0, 'TN': 12}\n",
            "Positive-only predicate {'TP': 8, 'FP': 4, 'FN': 0, 'TN': 8}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from pathlib import Path\n",
        "import base64\n",
        "print('__NOTEBOOK_IMAGE__svg+xml:' + base64.b64encode(Path('figure.svg').read_bytes()).decode())\n"
      ]
    }
  ]
}