{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.12.14"
    },
    "execution": {
      "method": "Python subprocess, exact code cell source",
      "deterministic_replay_passed": true
    },
    "runtime": {
      "packages": [],
      "local_assets": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "id": "question",
      "metadata": {},
      "source": [
        "# Selective tool calls: finding the cost threshold\n",
        "A carry-detection fixture makes the error–latency trade-off explicit, including the point at which a perfect calculator stops being worthwhile.\n",
        "\nSynthetic routing utility calculation; no learned Toolformer, model confidence, or measured tool latency.\n"
      ]
    },
    {
      "cell_type": "code",
      "id": "experiment",
      "metadata": {},
      "execution_count": 1,
      "source": [
        "\"\"\"Selective tool calls: finding the cost threshold\n",
        "Deterministic, standard-library experiment. RESULTS contains the full data.\n",
        "\"\"\"\n",
        "TASKS = [(a, b) for a in range(10, 20) for b in range(20, 25)]\n",
        "\n",
        "def limited_solver(a, b):\n",
        "    tens = 10 * (a // 10 + b // 10)\n",
        "    units = (a % 10 + b % 10) % 10\n",
        "    return tens + units\n",
        "\n",
        "def evaluate(threshold):\n",
        "    calls = errors = 0\n",
        "    for a, b in TASKS:\n",
        "        feature = a % 10 + b % 10\n",
        "        use_tool = feature >= threshold\n",
        "        answer = a + b if use_tool else limited_solver(a, b)\n",
        "        calls += use_tool\n",
        "        errors += answer != a + b\n",
        "    return calls, errors\n",
        "\n",
        "print('n_tasks', len(TASKS), 'latency is simulated, not measured')\n",
        "for latency_ms in (0, 250, 5000):\n",
        "    cost_per_call = 0.02 + 0.2 * latency_ms / 1000\n",
        "    for threshold in (0, 8, 10, 12, 15):\n",
        "        calls, errors = evaluate(threshold)\n",
        "        objective = errors / 50 + calls / 50 * cost_per_call\n",
        "        print(f'ms={latency_ms} threshold={threshold} calls={calls}',\n",
        "              f'errors={errors} objective={objective:.3f}')\n",
        "assert evaluate(10) == (10, 0)\n",
        "assert evaluate(12) == (3, 7)\n",
        "assert limited_solver(19, 24) == 33\n",
        "\n",
        "\n",
        "thresholds=[]\n",
        "for threshold in range(16):\n",
        "    calls,errors=evaluate(threshold)\n",
        "    thresholds.append(dict(threshold=threshold,calls=calls,errors=errors,call_rate=calls/50,error_rate=errors/50))\n",
        "cases=[dict(a=a,b=b,units_sum=a%10+b%10,limited=limited_solver(a,b),truth=a+b,wrong=limited_solver(a,b)!=a+b) for a,b in TASKS]\n",
        "crossover_ms=round((1-0.02)/0.2*1000, 9)\n",
        "assert crossover_ms==4900\n",
        "assert abs((10/50)*(0.02+0.2*crossover_ms/1000)-10/50)<1e-12\n",
        "RESULTS=dict(tasks=cases,thresholds=thresholds,call_cost_fixed=0.02,call_cost_per_second=0.2,crossover_ms=crossover_ms)\n",
        "print('Threshold 10 / never-call indifference:',int(crossover_ms),'simulated ms')\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "n_tasks 50 latency is simulated, not measured\n",
            "ms=0 threshold=0 calls=50 errors=0 objective=0.020\n",
            "ms=0 threshold=8 calls=20 errors=0 objective=0.008\n",
            "ms=0 threshold=10 calls=10 errors=0 objective=0.004\n",
            "ms=0 threshold=12 calls=3 errors=7 objective=0.141\n",
            "ms=0 threshold=15 calls=0 errors=10 objective=0.200\n",
            "ms=250 threshold=0 calls=50 errors=0 objective=0.070\n",
            "ms=250 threshold=8 calls=20 errors=0 objective=0.028\n",
            "ms=250 threshold=10 calls=10 errors=0 objective=0.014\n",
            "ms=250 threshold=12 calls=3 errors=7 objective=0.144\n",
            "ms=250 threshold=15 calls=0 errors=10 objective=0.200\n",
            "ms=5000 threshold=0 calls=50 errors=0 objective=1.020\n",
            "ms=5000 threshold=8 calls=20 errors=0 objective=0.408\n",
            "ms=5000 threshold=10 calls=10 errors=0 objective=0.204\n",
            "ms=5000 threshold=12 calls=3 errors=7 objective=0.201\n",
            "ms=5000 threshold=15 calls=0 errors=10 objective=0.200\n",
            "Threshold 10 / never-call indifference: 4900 simulated ms\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from pathlib import Path\n",
        "import base64\n",
        "print('__NOTEBOOK_IMAGE__svg+xml:' + base64.b64encode(Path('figure.svg').read_bytes()).decode())\n"
      ]
    }
  ]
}