Python starts when you choose Run.
The cost of asking a learned index for a key
This notebook reruns the attached offline analysis. Place all bundle files in one directory and run from that directory. The default path uses Python’s standard library only. Published measurements, metadata and illustrative numerical probes are identified in the article and provenance. No model or API calls occur.
All analysis assertions passed; figures and results regenerated.
{
"evidence": "published batch timing means, not new model runs",
"rows": [
{
"dataset": "Quora",
"keys_approx": "513000",
"parameter_fraction": 0.05,
"size": "S",
"support_score_s": 0.00134,
"support_grad_s": 0.0025,
"key_score_s": 0.00128,
"key_grad_s": 0.00129,
"support_grad_over_score": 1.8656716417910448,
"key_grad_over_score": 1.0078124999999998,
"support_over_key_grad": 1.9379844961240311,
"key_grad_us_per_query_equivalent": 0.31494140625,
"support_grad_us_per_query_equivalent": 0.6103515625,
"grad_batch_saving_ms": 1.2100000000000002,
"combined_ratios": {
"0": 1.9379844961240311,
"5": 1.1923688394276628,
"20": 1.0568341944574917,
"100": 1.0119458979168723
}
},
{
"dataset": "Quora",
"keys_approx": "513000",
"parameter_fraction": 0.1,
"size": "M",
"support_score_s": 0.00218,
"support_grad_s": 0.00408,
"key_score_s": 0.00213,
"key_grad_s": 0.00212,
"support_grad_over_score": 1.871559633027523,
"key_grad_over_score": 0.9953051643192488,
"support_over_key_grad": 1.9245283018867927,
"key_grad_us_per_query_equivalent": 0.517578125,
"support_grad_us_per_query_equivalent": 0.9960937500000001,
"grad_batch_saving_ms": 1.9600000000000004,
"combined_ratios": {
"0": 1.9245283018867927,
"5": 1.2752808988764048,
"20": 1.0886075949367089,
"100": 1.019193106149628
}
},
{
"dataset": "Quora",
"keys_approx": "513000",
"parameter_fraction": 0.2,
"size": "L",
"support_score_s": 0.00199,
"support_grad_s": 0.00312,
"key_score_s": 0.0019,
"key_grad_s": 0.0019,
"support_grad_over_score": 1.5678391959798994,
"key_grad_over_score": 1.0,
"support_over_key_grad": 1.6421052631578947,
"key_grad_us_per_query_equivalent": 0.4638671875,
"support_grad_us_per_query_equivalent": 0.76171875,
"grad_batch_saving_ms": 1.22,
"combined_ratios": {
"0": 1.6421052631578947,
"5": 1.1768115942028987,
"20": 1.0557077625570777,
"100": 1.011972522080471
}
},
{
"dataset": "Natural Questions",
"keys_approx": "2600000",
"parameter_fraction": 0.05,
"size": "S",
"support_score_s": 0.00456,
"support_grad_s": 0.00864,
"key_score_s": 0.00446,
"key_grad_s": 0.00446,
"support_grad_over_score": 1.8947368421052633,
"key_grad_over_score": 1.0,
"support_over_key_grad": 1.9372197309417039,
"key_grad_us_per_query_equivalent": 1.0888671875,
"support_grad_us_per_query_equivalent": 2.109375,
"grad_batch_saving_ms": 4.18,
"combined_ratios": {
"0": 1.9372197309417039,
"5": 1.441860465116279,
"20": 1.1708912510220766,
"100": 1.0400153168677004
}
},
{
"dataset": "Natural Questions",
"keys_approx": "2600000",
"parameter_fraction": 0.1,
"size": "M",
"support_score_s": 0.00855,
"support_grad_s": 0.0162,
"key_score_s": 0.00768,
"key_grad_s": 0.00769,
"support_grad_over_score": 1.894736842105263,
"key_grad_over_score": 1.0013020833333333,
"support_over_key_grad": 2.106631989596879,
"key_grad_us_per_query_equivalent": 1.87744140625,
"support_grad_us_per_query_equivalent": 3.955078125,
"grad_batch_saving_ms": 8.51,
"combined_ratios": {
"0": 2.106631989596879,
"5": 1.6706067769897557,
"20": 1.307331166486096,
"100": 1.0790231219240412
}
},
{
"dataset": "Natural Questions",
"keys_approx": "2600000",
"parameter_fraction": 0.2,
"size": "L",
"support_score_s": 0.00708,
"support_grad_s": 0.0123,
"key_score_s": 0.006,
"key_grad_s": 0.00599,
"support_grad_over_score": 1.7372881355932204,
"key_grad_over_score": 0.9983333333333333,
"support_over_key_grad": 2.0534223706176964,
"key_grad_us_per_query_equivalent": 1.46240234375,
"support_grad_us_per_query_equivalent": 3.0029296875,
"grad_batch_saving_ms": 6.3100000000000005,
"combined_ratios": {
"0": 2.0534223706176964,
"5": 1.5741583257506824,
"20": 1.2427856868026166,
"100": 1.0595339182941788
}
},
{
"dataset": "HotpotQA",
"keys_approx": "5200000",
"parameter_fraction": 0.05,
"size": "S",
"support_score_s": 0.00869,
"support_grad_s": 0.0168,
"key_score_s": 0.00815,
"key_grad_s": 0.00815,
"support_grad_over_score": 1.9332566168009204,
"key_grad_over_score": 1.0,
"support_over_key_grad": 2.061349693251534,
"key_grad_us_per_query_equivalent": 1.9897460937499998,
"support_grad_us_per_query_equivalent": 4.1015625,
"grad_batch_saving_ms": 8.65,
"combined_ratios": {
"0": 2.061349693251534,
"5": 1.6577946768060838,
"20": 1.3072824156305505,
"100": 1.079981507165973
}
},
{
"dataset": "HotpotQA",
"keys_approx": "5200000",
"parameter_fraction": 0.1,
"size": "M",
"support_score_s": 0.0159,
"support_grad_s": 0.0305,
"key_score_s": 0.0149,
"key_grad_s": 0.0149,
"support_grad_over_score": 1.9182389937106916,
"key_grad_over_score": 1.0,
"support_over_key_grad": 2.046979865771812,
"key_grad_us_per_query_equivalent": 3.6376953125,
"support_grad_us_per_query_equivalent": 7.4462890625,
"grad_batch_saving_ms": 15.6,
"combined_ratios": {
"0": 2.046979865771812,
"5": 1.7839195979899496,
"20": 1.4469914040114613,
"100": 1.1357702349869452
}
},
{
"dataset": "HotpotQA",
"keys_approx": "5200000",
"parameter_fraction": 0.2,
"size": "L",
"support_score_s": 0.0314,
"support_grad_s": 0.0596,
"key_score_s": 0.0285,
"key_grad_s": 0.0286,
"support_grad_over_score": 1.8980891719745225,
"key_grad_over_score": 1.0035087719298246,
"support_over_key_grad": 2.0839160839160837,
"key_grad_us_per_query_equivalent": 6.982421875,
"support_grad_us_per_query_equivalent": 14.55078125,
"grad_batch_saving_ms": 31.0,
"combined_ratios": {
"0": 2.0839160839160837,
"5": 1.922619047619048,
"20": 1.6378600823045266,
"100": 1.241057542768274
}
}
],
"support_grad_over_score_range": [
1.5678391959798994,
1.9332566168009204
],
"key_grad_over_score_range": [
0.9953051643192488,
1.0078124999999998
],
"support_over_key_grad_range": [
1.6421052631578947,
2.106631989596879
],
"median_support_over_key_grad": 2.046979865771812,
"nonmonotonic_pairs": [
{
"dataset": "Quora",
"column": "support_score_s",
"M_to_L_ratio": 0.9128440366972477
},
{
"dataset": "Quora",
"column": "support_grad_s",
"M_to_L_ratio": 0.7647058823529411
},
{
"dataset": "Quora",
"column": "key_score_s",
"M_to_L_ratio": 0.892018779342723
},
{
"dataset": "Quora",
"column": "key_grad_s",
"M_to_L_ratio": 0.8962264150943396
},
{
"dataset": "Natural Questions",
"column": "support_score_s",
"M_to_L_ratio": 0.8280701754385965
},
{
"dataset": "Natural Questions",
"column": "support_grad_s",
"M_to_L_ratio": 0.7592592592592593
},
{
"dataset": "Natural Questions",
"column": "key_score_s",
"M_to_L_ratio": 0.78125
},
{
"dataset": "Natural Questions",
"column": "key_grad_s",
"M_to_L_ratio": 0.7789336801040312
}
]
}
Regenerated figures
The following rich SVG outputs come from files regenerated by the preceding code, not screenshots of an external result.
Python runs locally in this page. Saved results and figures are visible before execution.