{ "nbformat": 4, "nbformat_minor": 5, "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "name": "python", "version": "3.12.14" }, "execution_note": "Code cells executed with Python via an AST-based offline notebook runner; outputs are actual stdout and rich MIME return values. No Jupyter package dependency." }, "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# The cost of asking a learned index for a key\n", "\n", "This notebook reruns the attached offline analysis. Place all bundle files in one directory and run from that directory. The default path uses Python’s standard library only. Published measurements, metadata and illustrative numerical probes are identified in the article and provenance. No model or API calls occur.\n" ], "id": "cell-1" }, { "cell_type": "code", "metadata": {}, "source": [ "import sys, json, runpy\n", "from pathlib import Path\n", "if str(Path.cwd()) not in sys.path:\n", " sys.path.insert(0, str(Path.cwd()))\n", "analysis = runpy.run_path('analyze.py', run_name='offline_analysis')\n", "result = analysis['analyze']()\n", "analysis['draw'](result)\n", "print('All analysis assertions passed; figures and results regenerated.')\n", "print(json.dumps(result, indent=2))\n" ], "execution_count": 1, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "All analysis assertions passed; figures and results regenerated.\n", "{\n", " \"evidence\": \"published batch timing means, not new model runs\",\n", " \"rows\": [\n", " {\n", " \"dataset\": \"Quora\",\n", " \"keys_approx\": \"513000\",\n", " \"parameter_fraction\": 0.05,\n", " \"size\": \"S\",\n", " \"support_score_s\": 0.00134,\n", " \"support_grad_s\": 0.0025,\n", " \"key_score_s\": 0.00128,\n", " \"key_grad_s\": 0.00129,\n", " \"support_grad_over_score\": 1.8656716417910448,\n", " \"key_grad_over_score\": 1.0078124999999998,\n", " \"support_over_key_grad\": 1.9379844961240311,\n", " \"key_grad_us_per_query_equivalent\": 0.31494140625,\n", " \"support_grad_us_per_query_equivalent\": 0.6103515625,\n", " \"grad_batch_saving_ms\": 1.2100000000000002,\n", " \"combined_ratios\": {\n", " \"0\": 1.9379844961240311,\n", " \"5\": 1.1923688394276628,\n", " \"20\": 1.0568341944574917,\n", " \"100\": 1.0119458979168723\n", " }\n", " },\n", " {\n", " \"dataset\": \"Quora\",\n", " \"keys_approx\": \"513000\",\n", " \"parameter_fraction\": 0.1,\n", " \"size\": \"M\",\n", " \"support_score_s\": 0.00218,\n", " \"support_grad_s\": 0.00408,\n", " \"key_score_s\": 0.00213,\n", " \"key_grad_s\": 0.00212,\n", " \"support_grad_over_score\": 1.871559633027523,\n", " \"key_grad_over_score\": 0.9953051643192488,\n", " \"support_over_key_grad\": 1.9245283018867927,\n", " \"key_grad_us_per_query_equivalent\": 0.517578125,\n", " \"support_grad_us_per_query_equivalent\": 0.9960937500000001,\n", " \"grad_batch_saving_ms\": 1.9600000000000004,\n", " \"combined_ratios\": {\n", " \"0\": 1.9245283018867927,\n", " \"5\": 1.2752808988764048,\n", " \"20\": 1.0886075949367089,\n", " \"100\": 1.019193106149628\n", " }\n", " },\n", " {\n", " \"dataset\": \"Quora\",\n", " \"keys_approx\": \"513000\",\n", " \"parameter_fraction\": 0.2,\n", " \"size\": \"L\",\n", " \"support_score_s\": 0.00199,\n", " \"support_grad_s\": 0.00312,\n", " \"key_score_s\": 0.0019,\n", " \"key_grad_s\": 0.0019,\n", " \"support_grad_over_score\": 1.5678391959798994,\n", " \"key_grad_over_score\": 1.0,\n", " \"support_over_key_grad\": 1.6421052631578947,\n", " \"key_grad_us_per_query_equivalent\": 0.4638671875,\n", " \"support_grad_us_per_query_equivalent\": 0.76171875,\n", " \"grad_batch_saving_ms\": 1.22,\n", " \"combined_ratios\": {\n", " \"0\": 1.6421052631578947,\n", " \"5\": 1.1768115942028987,\n", " \"20\": 1.0557077625570777,\n", " \"100\": 1.011972522080471\n", " }\n", " },\n", " {\n", " \"dataset\": \"Natural Questions\",\n", " \"keys_approx\": \"2600000\",\n", " \"parameter_fraction\": 0.05,\n", " \"size\": \"S\",\n", " \"support_score_s\": 0.00456,\n", " \"support_grad_s\": 0.00864,\n", " \"key_score_s\": 0.00446,\n", " \"key_grad_s\": 0.00446,\n", " \"support_grad_over_score\": 1.8947368421052633,\n", " \"key_grad_over_score\": 1.0,\n", " \"support_over_key_grad\": 1.9372197309417039,\n", " \"key_grad_us_per_query_equivalent\": 1.0888671875,\n", " \"support_grad_us_per_query_equivalent\": 2.109375,\n", " \"grad_batch_saving_ms\": 4.18,\n", " \"combined_ratios\": {\n", " \"0\": 1.9372197309417039,\n", " \"5\": 1.441860465116279,\n", " \"20\": 1.1708912510220766,\n", " \"100\": 1.0400153168677004\n", " }\n", " },\n", " {\n", " \"dataset\": \"Natural Questions\",\n", " \"keys_approx\": \"2600000\",\n", " \"parameter_fraction\": 0.1,\n", " \"size\": \"M\",\n", " \"support_score_s\": 0.00855,\n", " \"support_grad_s\": 0.0162,\n", " \"key_score_s\": 0.00768,\n", " \"key_grad_s\": 0.00769,\n", " \"support_grad_over_score\": 1.894736842105263,\n", " \"key_grad_over_score\": 1.0013020833333333,\n", " \"support_over_key_grad\": 2.106631989596879,\n", " \"key_grad_us_per_query_equivalent\": 1.87744140625,\n", " \"support_grad_us_per_query_equivalent\": 3.955078125,\n", " \"grad_batch_saving_ms\": 8.51,\n", " \"combined_ratios\": {\n", " \"0\": 2.106631989596879,\n", " \"5\": 1.6706067769897557,\n", " \"20\": 1.307331166486096,\n", " \"100\": 1.0790231219240412\n", " }\n", " },\n", " {\n", " \"dataset\": \"Natural Questions\",\n", " \"keys_approx\": \"2600000\",\n", " \"parameter_fraction\": 0.2,\n", " \"size\": \"L\",\n", " \"support_score_s\": 0.00708,\n", " \"support_grad_s\": 0.0123,\n", " \"key_score_s\": 0.006,\n", " \"key_grad_s\": 0.00599,\n", " \"support_grad_over_score\": 1.7372881355932204,\n", " \"key_grad_over_score\": 0.9983333333333333,\n", " \"support_over_key_grad\": 2.0534223706176964,\n", " \"key_grad_us_per_query_equivalent\": 1.46240234375,\n", " \"support_grad_us_per_query_equivalent\": 3.0029296875,\n", " \"grad_batch_saving_ms\": 6.3100000000000005,\n", " \"combined_ratios\": {\n", " \"0\": 2.0534223706176964,\n", " \"5\": 1.5741583257506824,\n", " \"20\": 1.2427856868026166,\n", " \"100\": 1.0595339182941788\n", " }\n", " },\n", " {\n", " \"dataset\": \"HotpotQA\",\n", " \"keys_approx\": \"5200000\",\n", " \"parameter_fraction\": 0.05,\n", " \"size\": \"S\",\n", " \"support_score_s\": 0.00869,\n", " \"support_grad_s\": 0.0168,\n", " \"key_score_s\": 0.00815,\n", " \"key_grad_s\": 0.00815,\n", " \"support_grad_over_score\": 1.9332566168009204,\n", " \"key_grad_over_score\": 1.0,\n", " \"support_over_key_grad\": 2.061349693251534,\n", " \"key_grad_us_per_query_equivalent\": 1.9897460937499998,\n", " \"support_grad_us_per_query_equivalent\": 4.1015625,\n", " \"grad_batch_saving_ms\": 8.65,\n", " \"combined_ratios\": {\n", " \"0\": 2.061349693251534,\n", " \"5\": 1.6577946768060838,\n", " \"20\": 1.3072824156305505,\n", " \"100\": 1.079981507165973\n", " }\n", " },\n", " {\n", " \"dataset\": \"HotpotQA\",\n", " \"keys_approx\": \"5200000\",\n", " \"parameter_fraction\": 0.1,\n", " \"size\": \"M\",\n", " \"support_score_s\": 0.0159,\n", " \"support_grad_s\": 0.0305,\n", " \"key_score_s\": 0.0149,\n", " \"key_grad_s\": 0.0149,\n", " \"support_grad_over_score\": 1.9182389937106916,\n", " \"key_grad_over_score\": 1.0,\n", " \"support_over_key_grad\": 2.046979865771812,\n", " \"key_grad_us_per_query_equivalent\": 3.6376953125,\n", " \"support_grad_us_per_query_equivalent\": 7.4462890625,\n", " \"grad_batch_saving_ms\": 15.6,\n", " \"combined_ratios\": {\n", " \"0\": 2.046979865771812,\n", " \"5\": 1.7839195979899496,\n", " \"20\": 1.4469914040114613,\n", " \"100\": 1.1357702349869452\n", " }\n", " },\n", " {\n", " \"dataset\": \"HotpotQA\",\n", " \"keys_approx\": \"5200000\",\n", " \"parameter_fraction\": 0.2,\n", " \"size\": \"L\",\n", " \"support_score_s\": 0.0314,\n", " \"support_grad_s\": 0.0596,\n", " \"key_score_s\": 0.0285,\n", " \"key_grad_s\": 0.0286,\n", " \"support_grad_over_score\": 1.8980891719745225,\n", " \"key_grad_over_score\": 1.0035087719298246,\n", " \"support_over_key_grad\": 2.0839160839160837,\n", " \"key_grad_us_per_query_equivalent\": 6.982421875,\n", " \"support_grad_us_per_query_equivalent\": 14.55078125,\n", " \"grad_batch_saving_ms\": 31.0,\n", " \"combined_ratios\": {\n", " \"0\": 2.0839160839160837,\n", " \"5\": 1.922619047619048,\n", " \"20\": 1.6378600823045266,\n", " \"100\": 1.241057542768274\n", " }\n", " }\n", " ],\n", " \"support_grad_over_score_range\": [\n", " 1.5678391959798994,\n", " 1.9332566168009204\n", " ],\n", " \"key_grad_over_score_range\": [\n", " 0.9953051643192488,\n", " 1.0078124999999998\n", " ],\n", " \"support_over_key_grad_range\": [\n", " 1.6421052631578947,\n", " 2.106631989596879\n", " ],\n", " \"median_support_over_key_grad\": 2.046979865771812,\n", " \"nonmonotonic_pairs\": [\n", " {\n", " \"dataset\": \"Quora\",\n", " \"column\": \"support_score_s\",\n", " \"M_to_L_ratio\": 0.9128440366972477\n", " },\n", " {\n", " \"dataset\": \"Quora\",\n", " \"column\": \"support_grad_s\",\n", " \"M_to_L_ratio\": 0.7647058823529411\n", " },\n", " {\n", " \"dataset\": \"Quora\",\n", " \"column\": \"key_score_s\",\n", " \"M_to_L_ratio\": 0.892018779342723\n", " },\n", " {\n", " \"dataset\": \"Quora\",\n", " \"column\": \"key_grad_s\",\n", " \"M_to_L_ratio\": 0.8962264150943396\n", " },\n", " {\n", " \"dataset\": \"Natural Questions\",\n", " \"column\": \"support_score_s\",\n", " \"M_to_L_ratio\": 0.8280701754385965\n", " },\n", " {\n", " \"dataset\": \"Natural Questions\",\n", " \"column\": \"support_grad_s\",\n", " \"M_to_L_ratio\": 0.7592592592592593\n", " },\n", " {\n", " \"dataset\": \"Natural Questions\",\n", " \"column\": \"key_score_s\",\n", " \"M_to_L_ratio\": 0.78125\n", " },\n", " {\n", " \"dataset\": \"Natural Questions\",\n", " \"column\": \"key_grad_s\",\n", " \"M_to_L_ratio\": 0.7789336801040312\n", " }\n", " ]\n", "}\n" ] } ], "id": "cell-2" }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Regenerated figures\n", "\n", "The following rich SVG outputs come from files regenerated by the preceding code, not screenshots of an external result.\n" ], "id": "cell-3" }, { "cell_type": "code", "metadata": {}, "source": [ "class LocalSVG:\n", " def __init__(self, filename):\n", " self.filename = filename\n", " def _repr_mimebundle_(self, include=None, exclude=None):\n", " return {'image/svg+xml': Path(self.filename).read_text(), 'text/plain': self.filename}\n" ], "execution_count": 2, "outputs": [], "id": "cell-4" }, { "cell_type": "code", "metadata": {}, "source": [ "LocalSVG('downstream-sensitivity.svg')\n" ], "execution_count": 3, "outputs": [ { "output_type": "execute_result", "data": { "image/svg+xml": "A twofold component ratio is not a twofold pipeline gainCalculated sensitivity using M-size published means; downstream time is an assumption, not a measurement.A twofold component ratio is not a twofold pipeline gainCalculated sensitivity using M-size published means; downstream time is an assumption, not a measurement.1×1.25×1.5×1.75×2×2.25×Quora · MNatural Questions · MHotpotQA · M0 ms20 ms40 ms60 ms80 ms100 msAssumed equal downstream time per batch", "text/plain": "downstream-sensitivity.svg" }, "metadata": {}, "execution_count": 3 } ], "id": "cell-5" }, { "cell_type": "code", "metadata": {}, "source": [ "LocalSVG('gradient-overhead.svg')\n", "\n", "import base64\n", "from pathlib import Path\n", "print('__NOTEBOOK_IMAGE__svg+xml:' + base64.b64encode(Path('downstream-sensitivity.svg').read_bytes()).decode())\n", "print('__NOTEBOOK_IMAGE__svg+xml:' + base64.b64encode(Path('gradient-overhead.svg').read_bytes()).decode())\n" ], "execution_count": 4, "outputs": [ { "output_type": "execute_result", "data": { "image/svg+xml": "A predicted key changes the cost of the interfaceGradient time / score time within each published configuration. Batch = 4096; 100-run means.A predicted key changes the cost of the interfaceGradient time / score time within each published configuration. Batch = 4096; 100-run means.1×1.25×1.5×1.75×2×2.25×Quora · S1.87×Quora · M1.87×Quora · L1.57×Natural Questions · S1.89×Natural Questions · M1.89×Natural Questions · L1.74×HotpotQA · S1.93×HotpotQA · M1.92×HotpotQA · L1.90×KeyNetSupportNet", "text/plain": "gradient-overhead.svg" }, "metadata": {}, "execution_count": 4 } ], "id": "cell-6" } ] }