{
  "evidence": "published batch timing means, not new model runs",
  "rows": [
    {
      "dataset": "Quora",
      "keys_approx": "513000",
      "parameter_fraction": 0.05,
      "size": "S",
      "support_score_s": 0.00134,
      "support_grad_s": 0.0025,
      "key_score_s": 0.00128,
      "key_grad_s": 0.00129,
      "support_grad_over_score": 1.8656716417910448,
      "key_grad_over_score": 1.0078124999999998,
      "support_over_key_grad": 1.9379844961240311,
      "key_grad_us_per_query_equivalent": 0.31494140625,
      "support_grad_us_per_query_equivalent": 0.6103515625,
      "grad_batch_saving_ms": 1.2100000000000002,
      "combined_ratios": {
        "0": 1.9379844961240311,
        "5": 1.1923688394276628,
        "20": 1.0568341944574917,
        "100": 1.0119458979168723
      }
    },
    {
      "dataset": "Quora",
      "keys_approx": "513000",
      "parameter_fraction": 0.1,
      "size": "M",
      "support_score_s": 0.00218,
      "support_grad_s": 0.00408,
      "key_score_s": 0.00213,
      "key_grad_s": 0.00212,
      "support_grad_over_score": 1.871559633027523,
      "key_grad_over_score": 0.9953051643192488,
      "support_over_key_grad": 1.9245283018867927,
      "key_grad_us_per_query_equivalent": 0.517578125,
      "support_grad_us_per_query_equivalent": 0.9960937500000001,
      "grad_batch_saving_ms": 1.9600000000000004,
      "combined_ratios": {
        "0": 1.9245283018867927,
        "5": 1.2752808988764048,
        "20": 1.0886075949367089,
        "100": 1.019193106149628
      }
    },
    {
      "dataset": "Quora",
      "keys_approx": "513000",
      "parameter_fraction": 0.2,
      "size": "L",
      "support_score_s": 0.00199,
      "support_grad_s": 0.00312,
      "key_score_s": 0.0019,
      "key_grad_s": 0.0019,
      "support_grad_over_score": 1.5678391959798994,
      "key_grad_over_score": 1.0,
      "support_over_key_grad": 1.6421052631578947,
      "key_grad_us_per_query_equivalent": 0.4638671875,
      "support_grad_us_per_query_equivalent": 0.76171875,
      "grad_batch_saving_ms": 1.22,
      "combined_ratios": {
        "0": 1.6421052631578947,
        "5": 1.1768115942028987,
        "20": 1.0557077625570777,
        "100": 1.011972522080471
      }
    },
    {
      "dataset": "Natural Questions",
      "keys_approx": "2600000",
      "parameter_fraction": 0.05,
      "size": "S",
      "support_score_s": 0.00456,
      "support_grad_s": 0.00864,
      "key_score_s": 0.00446,
      "key_grad_s": 0.00446,
      "support_grad_over_score": 1.8947368421052633,
      "key_grad_over_score": 1.0,
      "support_over_key_grad": 1.9372197309417039,
      "key_grad_us_per_query_equivalent": 1.0888671875,
      "support_grad_us_per_query_equivalent": 2.109375,
      "grad_batch_saving_ms": 4.18,
      "combined_ratios": {
        "0": 1.9372197309417039,
        "5": 1.441860465116279,
        "20": 1.1708912510220766,
        "100": 1.0400153168677004
      }
    },
    {
      "dataset": "Natural Questions",
      "keys_approx": "2600000",
      "parameter_fraction": 0.1,
      "size": "M",
      "support_score_s": 0.00855,
      "support_grad_s": 0.0162,
      "key_score_s": 0.00768,
      "key_grad_s": 0.00769,
      "support_grad_over_score": 1.894736842105263,
      "key_grad_over_score": 1.0013020833333333,
      "support_over_key_grad": 2.106631989596879,
      "key_grad_us_per_query_equivalent": 1.87744140625,
      "support_grad_us_per_query_equivalent": 3.955078125,
      "grad_batch_saving_ms": 8.51,
      "combined_ratios": {
        "0": 2.106631989596879,
        "5": 1.6706067769897557,
        "20": 1.307331166486096,
        "100": 1.0790231219240412
      }
    },
    {
      "dataset": "Natural Questions",
      "keys_approx": "2600000",
      "parameter_fraction": 0.2,
      "size": "L",
      "support_score_s": 0.00708,
      "support_grad_s": 0.0123,
      "key_score_s": 0.006,
      "key_grad_s": 0.00599,
      "support_grad_over_score": 1.7372881355932204,
      "key_grad_over_score": 0.9983333333333333,
      "support_over_key_grad": 2.0534223706176964,
      "key_grad_us_per_query_equivalent": 1.46240234375,
      "support_grad_us_per_query_equivalent": 3.0029296875,
      "grad_batch_saving_ms": 6.3100000000000005,
      "combined_ratios": {
        "0": 2.0534223706176964,
        "5": 1.5741583257506824,
        "20": 1.2427856868026166,
        "100": 1.0595339182941788
      }
    },
    {
      "dataset": "HotpotQA",
      "keys_approx": "5200000",
      "parameter_fraction": 0.05,
      "size": "S",
      "support_score_s": 0.00869,
      "support_grad_s": 0.0168,
      "key_score_s": 0.00815,
      "key_grad_s": 0.00815,
      "support_grad_over_score": 1.9332566168009204,
      "key_grad_over_score": 1.0,
      "support_over_key_grad": 2.061349693251534,
      "key_grad_us_per_query_equivalent": 1.9897460937499998,
      "support_grad_us_per_query_equivalent": 4.1015625,
      "grad_batch_saving_ms": 8.65,
      "combined_ratios": {
        "0": 2.061349693251534,
        "5": 1.6577946768060838,
        "20": 1.3072824156305505,
        "100": 1.079981507165973
      }
    },
    {
      "dataset": "HotpotQA",
      "keys_approx": "5200000",
      "parameter_fraction": 0.1,
      "size": "M",
      "support_score_s": 0.0159,
      "support_grad_s": 0.0305,
      "key_score_s": 0.0149,
      "key_grad_s": 0.0149,
      "support_grad_over_score": 1.9182389937106916,
      "key_grad_over_score": 1.0,
      "support_over_key_grad": 2.046979865771812,
      "key_grad_us_per_query_equivalent": 3.6376953125,
      "support_grad_us_per_query_equivalent": 7.4462890625,
      "grad_batch_saving_ms": 15.6,
      "combined_ratios": {
        "0": 2.046979865771812,
        "5": 1.7839195979899496,
        "20": 1.4469914040114613,
        "100": 1.1357702349869452
      }
    },
    {
      "dataset": "HotpotQA",
      "keys_approx": "5200000",
      "parameter_fraction": 0.2,
      "size": "L",
      "support_score_s": 0.0314,
      "support_grad_s": 0.0596,
      "key_score_s": 0.0285,
      "key_grad_s": 0.0286,
      "support_grad_over_score": 1.8980891719745225,
      "key_grad_over_score": 1.0035087719298246,
      "support_over_key_grad": 2.0839160839160837,
      "key_grad_us_per_query_equivalent": 6.982421875,
      "support_grad_us_per_query_equivalent": 14.55078125,
      "grad_batch_saving_ms": 31.0,
      "combined_ratios": {
        "0": 2.0839160839160837,
        "5": 1.922619047619048,
        "20": 1.6378600823045266,
        "100": 1.241057542768274
      }
    }
  ],
  "support_grad_over_score_range": [
    1.5678391959798994,
    1.9332566168009204
  ],
  "key_grad_over_score_range": [
    0.9953051643192488,
    1.0078124999999998
  ],
  "support_over_key_grad_range": [
    1.6421052631578947,
    2.106631989596879
  ],
  "median_support_over_key_grad": 2.046979865771812,
  "nonmonotonic_pairs": [
    {
      "dataset": "Quora",
      "column": "support_score_s",
      "M_to_L_ratio": 0.9128440366972477
    },
    {
      "dataset": "Quora",
      "column": "support_grad_s",
      "M_to_L_ratio": 0.7647058823529411
    },
    {
      "dataset": "Quora",
      "column": "key_score_s",
      "M_to_L_ratio": 0.892018779342723
    },
    {
      "dataset": "Quora",
      "column": "key_grad_s",
      "M_to_L_ratio": 0.8962264150943396
    },
    {
      "dataset": "Natural Questions",
      "column": "support_score_s",
      "M_to_L_ratio": 0.8280701754385965
    },
    {
      "dataset": "Natural Questions",
      "column": "support_grad_s",
      "M_to_L_ratio": 0.7592592592592593
    },
    {
      "dataset": "Natural Questions",
      "column": "key_score_s",
      "M_to_L_ratio": 0.78125
    },
    {
      "dataset": "Natural Questions",
      "column": "key_grad_s",
      "M_to_L_ratio": 0.7789336801040312
    }
  ]
}