{ "release": "MLPerf Inference v6.1", "results_revision": "4bb63cd28eb136be221bc801a9e38c66c8a7553f", "method": "Within-submission metadata-matched Server/Offline ratios; no system ranking.", "source_rows": 492, "closed_rows": 476, "eligible_rows": 305, "complete_pairs": 152, "unmatched_row_ids": [ "264" ], "pairs_with_server_greater_than_offline": 11, "rows_with_measured_power": 0, "match_fields": [ "Organization", "Availability", "Division", "SystemType", "SystemName", "Platform", "Model", "MlperfModel", "number_of_nodes", "host_processor_model_name", "host_processors_per_node", "host_processor_core_count", "accelerator_model_name", "accelerators_per_node", "total_accelerators", "framework", "operating_system", "notes", "compliance", "errors", "version", "inferred", "has_power", "Units", "weight_data_types", "design_power_watts", "PrivateID" ], "excluded_match_fields": [ "Accuracy", "Location", "Result", "Scenario" ], "focus_pairs": [ { "offline_row_id": "285", "server_row_id": "286", "metadata": { "Organization": "NVIDIA", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)", "Platform": "B300-SXM-270GBx8_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8570", "host_processors_per_node": 2, "host_processor_core_count": "56", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 70132.3, "server_tokens_per_second": 59867.8, "server_over_offline": 0.8536409044049603, "server_over_offline_percent": 85.36409044049603, "difference_percent_of_offline": -14.635909559503968, "offline_accuracy": "exact_match: 80.78851412944394 TOKENS_PER_SAMPLE: 3793.9359617137648", "server_accuracy": "exact_match: 81.17593436645396 TOKENS_PER_SAMPLE: 3818.7203737465816", "offline_location": "./closed/NVIDIA/results/B300-SXM-270GBx8_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/NVIDIA/results/B300-SXM-270GBx8_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "287", "server_row_id": "288", "metadata": { "Organization": "NVIDIA", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)", "Platform": "B300-SXM-270GBx8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8570", "host_processors_per_node": 2, "host_processor_core_count": "56", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 106637.0, "server_tokens_per_second": 100630.0, "server_over_offline": 0.9436687078593734, "server_over_offline_percent": 94.36687078593734, "difference_percent_of_offline": -5.633129214062659, "offline_accuracy": "exact_match: 83.430", "server_accuracy": "exact_match: 83.702", "offline_location": "./closed/NVIDIA/results/B300-SXM-270GBx8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/NVIDIA/results/B300-SXM-270GBx8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "289", "server_row_id": "290", "metadata": { "Organization": "NVIDIA", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)", "Platform": "B300-SXM-270GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8570", "host_processors_per_node": 2, "host_processor_core_count": "56", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 109277.0, "server_tokens_per_second": 106949.0, "server_over_offline": 0.9786963404925099, "server_over_offline_percent": 97.86963404925099, "difference_percent_of_offline": -2.1303659507490136, "offline_accuracy": "ROUGE1: 44.7951 ROUGE2: 22.3759 ROUGEL: 29.1534 TOKENS_PER_SAMPLE: 272.8", "server_accuracy": "ROUGE1: 44.8452 ROUGE2: 22.4177 ROUGEL: 29.2149 TOKENS_PER_SAMPLE: 272.7", "offline_location": "./closed/NVIDIA/results/B300-SXM-270GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/NVIDIA/results/B300-SXM-270GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "300", "server_row_id": "301", "metadata": { "Organization": "NVIDIA", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)", "Platform": "GB200-NVL72_GB200-186GB_aarch64x72_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB200", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB200 NVL72, 1200W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 883761.0, "server_tokens_per_second": 899005.0, "server_over_offline": 1.0172490073673766, "server_over_offline_percent": 101.72490073673765, "difference_percent_of_offline": 1.7249007367376556, "offline_accuracy": "exact_match: 84.038", "server_accuracy": "exact_match: 83.914", "offline_location": "./closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/gpt-oss-120b/Server/performance/run_1" } ], "b300_ratio_span_percentage_points": 12.505543608754962, "accuracy_and_log_checks": [ { "row_id": "285", "model": "deepseek-r1", "scenario": "Offline", "thresholds": { "exact_match": 80.544618, "TOKENS_PER_SAMPLE": 3497.6046600000004 }, "reported_accuracy": { "exact_match": 80.78851412944394, "TOKENS_PER_SAMPLE": 3793.9359617137648 }, "log_tokens_per_second": 70132.3, "summary_tokens_per_second": 70132.3, "log_result": "VALID" }, { "row_id": "286", "model": "deepseek-r1", "scenario": "Server", "thresholds": { "exact_match": 80.544618, "TOKENS_PER_SAMPLE": 3497.6046600000004 }, "reported_accuracy": { "exact_match": 81.17593436645396, "TOKENS_PER_SAMPLE": 3818.7203737465816 }, "log_tokens_per_second": 59867.76, "summary_tokens_per_second": 59867.8, "log_result": "VALID", "limits_ns": { "ttft": 2000000000, "tpot": 80000000 } }, { "row_id": "287", "model": "gpt-oss-120b", "scenario": "Offline", "thresholds": { "exact_match": 82.2987 }, "reported_accuracy": { "exact_match": 83.43 }, "log_tokens_per_second": 106637.0, "summary_tokens_per_second": 106637.0, "log_result": "VALID" }, { "row_id": "288", "model": "gpt-oss-120b", "scenario": "Server", "thresholds": { "exact_match": 82.2987 }, "reported_accuracy": { "exact_match": 83.702 }, "log_tokens_per_second": 100629.85, "summary_tokens_per_second": 100630.0, "log_result": "VALID", "limits_ns": { "ttft": 3000000000, "tpot": 80000000 } }, { "row_id": "289", "model": "llama2-70b-99.9", "scenario": "Offline", "thresholds": { "ROUGE1": 44.3867688, "ROUGE2": 22.0131648, "ROUGEL": 28.5875838, "TOKENS_PER_SAMPLE": 265.005 }, "reported_accuracy": { "ROUGE1": 44.7951, "ROUGE2": 22.3759, "ROUGEL": 29.1534, "TOKENS_PER_SAMPLE": 272.8 }, "log_tokens_per_second": 109277.0, "summary_tokens_per_second": 109277.0, "log_result": "VALID" }, { "row_id": "290", "model": "llama2-70b-99.9", "scenario": "Server", "thresholds": { "ROUGE1": 44.3867688, "ROUGE2": 22.0131648, "ROUGEL": 28.5875838, "TOKENS_PER_SAMPLE": 265.005 }, "reported_accuracy": { "ROUGE1": 44.8452, "ROUGE2": 22.4177, "ROUGEL": 29.2149, "TOKENS_PER_SAMPLE": 272.7 }, "log_tokens_per_second": 106949.36, "summary_tokens_per_second": 106949.0, "log_result": "VALID", "limits_ns": { "ttft": 2000000000, "tpot": 200000000 } }, { "row_id": "300", "model": "gpt-oss-120b", "scenario": "Offline", "thresholds": { "exact_match": 82.2987 }, "reported_accuracy": { "exact_match": 84.038 }, "log_tokens_per_second": 883761.0, "summary_tokens_per_second": 883761.0, "log_result": "VALID" }, { "row_id": "301", "model": "gpt-oss-120b", "scenario": "Server", "thresholds": { "exact_match": 82.2987 }, "reported_accuracy": { "exact_match": 83.914 }, "log_tokens_per_second": 899005.11, "summary_tokens_per_second": 899005.0, "log_result": "VALID", "limits_ns": { "ttft": 3000000000, "tpot": 80000000 } } ], "negative_and_positive_test_cases": 13, "verified_source_files": 15, "all_pairs": [ { "offline_row_id": "6", "server_row_id": "7", "metadata": { "Organization": "AMD", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "smci355-ccs-aus-m13-13 (AS -4126GS-NMR-LCC)", "Platform": "8xMI355X_2xEPYC_9575F", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 8, "host_processor_core_count": "64", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "PyTorch 2.10.0+git8514f05, ROCm 7.2.2", "operating_system": "Ubuntu 22.04.5 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 121818.0, "server_tokens_per_second": 113241.0, "server_over_offline": 0.9295916859577402, "server_over_offline_percent": 92.95916859577402, "difference_percent_of_offline": -7.040831404225978, "offline_accuracy": "exact_match: 83.706", "server_accuracy": "exact_match: 83.451", "offline_location": "./closed/AMD/results/8xMI355X_2xEPYC_9575F/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/AMD/results/8xMI355X_2xEPYC_9575F/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "9", "server_row_id": "10", "metadata": { "Organization": "AMD", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "smci355-ccs-aus-m13-13 (AS -4126GS-NMR-LCC)", "Platform": "8xMI355X_2xEPYC_9575F", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 8, "host_processor_core_count": "64", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "PyTorch 2.10.0+git8514f05, ROCm 7.2.2", "operating_system": "Ubuntu 22.04.5 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 106517.0, "server_tokens_per_second": 103275.0, "server_over_offline": 0.969563543847461, "server_over_offline_percent": 96.9563543847461, "difference_percent_of_offline": -3.043645615253898, "offline_accuracy": "ROUGE1: 44.5481 ROUGE2: 22.1846 ROUGEL: 28.9482 TOKENS_PER_SAMPLE: 283.2", "server_accuracy": "ROUGE1: 44.5266 ROUGE2: 22.1733 ROUGEL: 28.9289 TOKENS_PER_SAMPLE: 284.2", "offline_location": "./closed/AMD/results/8xMI355X_2xEPYC_9575F/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/AMD/results/8xMI355X_2xEPYC_9575F/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "13", "server_row_id": "14", "metadata": { "Organization": "AMD", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "9x8xMI355X_2xEPYC_9575F", "Platform": "9x8xMI355X_2xEPYC_9575F", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 9, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 72, "framework": "vLLM 0.22.1.dev0, PyTorch 2.10.0, ROCm 7.2.2", "operating_system": "Ubuntu 22.04.2 LTS (kernel 5.15.0-70-generic)", "notes": "9 nodes x 8 AMD Instinct MI355X (gfx950) = 72 GPUs, 288GB HBM3E each; distributed asynchronous ZMQ backend. MXFP4 (W4A4) weights with FP8 KV cache, Quark quantization; 72-GPU distributed ZMQ (async server / sync offline)", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "mxfp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 1039900.0, "server_tokens_per_second": 964468.0, "server_over_offline": 0.9274622559861525, "server_over_offline_percent": 92.74622559861525, "difference_percent_of_offline": -7.253774401384749, "offline_accuracy": "exact_match: 83.230", "server_accuracy": "exact_match: 83.446", "offline_location": "./closed/AMD/results/9x8xMI355X_2xEPYC_9575F/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/AMD/results/9x8xMI355X_2xEPYC_9575F/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "15", "server_row_id": "16", "metadata": { "Organization": "ASUSTeK", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "XA NB3I-E12", "Platform": "XA_NB3I-E12_B300x8_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6767P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1", "operating_system": "Ubuntu 22.04", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 69846.9, "server_tokens_per_second": 54568.6, "server_over_offline": 0.7812601561415038, "server_over_offline_percent": 78.12601561415038, "difference_percent_of_offline": -21.87398438584962, "offline_accuracy": "exact_match: 80.76572470373746 TOKENS_PER_SAMPLE: 3759.735870556062", "server_accuracy": "exact_match: 80.9024612579763 TOKENS_PER_SAMPLE: 3802.827256153145", "offline_location": "./closed/ASUSTeK/results/XA_NB3I-E12_B300x8_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/ASUSTeK/results/XA_NB3I-E12_B300x8_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "17", "server_row_id": "18", "metadata": { "Organization": "ASUSTeK", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "XA NB3I-E12", "Platform": "XA_NB3I-E12_B300x8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6767P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1", "operating_system": "Ubuntu 22.04", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 109368.0, "server_tokens_per_second": 109949.0, "server_over_offline": 1.0053123399897594, "server_over_offline_percent": 100.53123399897595, "difference_percent_of_offline": 0.531233998975944, "offline_accuracy": "exact_match: 83.572", "server_accuracy": "exact_match: 83.403", "offline_location": "./closed/ASUSTeK/results/XA_NB3I-E12_B300x8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/ASUSTeK/results/XA_NB3I-E12_B300x8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "19", "server_row_id": "20", "metadata": { "Organization": "ASUSTeK", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "XA NB3I-E12", "Platform": "XA_NB3I-E12_B300x8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6767P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1", "operating_system": "Ubuntu 22.04", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 113007.0, "server_tokens_per_second": 113455.0, "server_over_offline": 1.0039643561903244, "server_over_offline_percent": 100.39643561903245, "difference_percent_of_offline": 0.3964356190324425, "offline_accuracy": "ROUGE1: 44.7974 ROUGE2: 22.3739 ROUGEL: 29.1541 TOKENS_PER_SAMPLE: 272.7", "server_accuracy": "ROUGE1: 44.8853 ROUGE2: 22.409 ROUGEL: 29.2169 TOKENS_PER_SAMPLE: 272.2", "offline_location": "./closed/ASUSTeK/results/XA_NB3I-E12_B300x8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/ASUSTeK/results/XA_NB3I-E12_B300x8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "23", "server_row_id": "24", "metadata": { "Organization": "Azure", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Azure GB200 NVL72 (288x GB200-186GB_aarch64, TensorRT)", "Platform": "Azure-GB200-NVL72_186GBx288", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 72, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB200", "accelerators_per_node": 4, "total_accelerators": 288, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB200 NVL72, 1200W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 2091190.0, "server_tokens_per_second": 1598510.0, "server_over_offline": 0.764402086850071, "server_over_offline_percent": 76.44020868500711, "difference_percent_of_offline": -23.559791314992896, "offline_accuracy": "exact_match: 81.56335460346399 TOKENS_PER_SAMPLE: 3758.743163172288", "server_accuracy": "exact_match: 80.85688240656336 TOKENS_PER_SAMPLE: 3746.623518687329", "offline_location": "./closed/Azure/results/Azure-GB200-NVL72_186GBx288/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Azure/results/Azure-GB200-NVL72_186GBx288/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "25", "server_row_id": "26", "metadata": { "Organization": "Azure", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Azure GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)", "Platform": "Azure-GB200-NVL72_186GBx72", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB200", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB200 NVL72, 1200W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 542075.0, "server_tokens_per_second": 426796.0, "server_over_offline": 0.7873375455425909, "server_over_offline_percent": 78.7337545542591, "difference_percent_of_offline": -21.266245445740907, "offline_accuracy": "exact_match: 81.35824977210574 TOKENS_PER_SAMPLE: 3739.669553327256", "server_accuracy": "exact_match: 81.51777575205105 TOKENS_PER_SAMPLE: 3732.908158614403", "offline_location": "./closed/Azure/results/Azure-GB200-NVL72_186GBx72/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Azure/results/Azure-GB200-NVL72_186GBx72/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "28", "server_row_id": "29", "metadata": { "Organization": "Azure", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Azure GB300 NVL72 (72x GB300-288GB_aarch64 TensorRT)", "Platform": "Azure-GB300-NVL72_288GBx72", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72, 1400W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 661143.0, "server_tokens_per_second": 589391.0, "server_over_offline": 0.8914727978667247, "server_over_offline_percent": 89.14727978667247, "difference_percent_of_offline": -10.852720213327526, "offline_accuracy": "exact_match: 81.47219690063811 TOKENS_PER_SAMPLE: 3779.8644029170464", "server_accuracy": "exact_match: 81.13035551504102 TOKENS_PER_SAMPLE: 3771.7791704649044", "offline_location": "./closed/Azure/results/Azure-GB300-NVL72_288GBx72/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Azure/results/Azure-GB300-NVL72_288GBx72/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "31", "server_row_id": "32", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco UCS C240 M8 - (5x NVIDIA RTX PRO 4500 Blackwell-32GB, TensorRT)", "Platform": "C240M8_RTXPro4500_32GBx5_TRT", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6787P", "host_processors_per_node": 2, "host_processor_core_count": "86", "accelerator_model_name": "NVIDIA RTX PRO 4500 Blackwell", "accelerators_per_node": 5, "total_accelerators": 5, "framework": "TensorRT 10.14, CUDA 13.1", "operating_system": "Ubuntu 24.04", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 12017.7, "server_tokens_per_second": 11426.1, "server_over_offline": 0.9507726103996604, "server_over_offline_percent": 95.07726103996605, "difference_percent_of_offline": -4.922738960033957, "offline_accuracy": "ROUGE1: 38.7698 ROUGE2: 16.0132 ROUGEL: 24.5304 ROUGELSUM: 35.6924 GEN_LEN: 8178850", "server_accuracy": "ROUGE1: 38.7615 ROUGE2: 16.0056 ROUGEL: 24.5235 ROUGELSUM: 35.6866 GEN_LEN: 8178120", "offline_location": "./closed/Cisco/results/C240M8_RTXPro4500_32GBx5_TRT/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/Cisco/results/C240M8_RTXPro4500_32GBx5_TRT/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "35", "server_row_id": "36", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco UCS C845A M8 (8x NVIDIA RTX PRO 4500 Blackwell-32GB, TensorRT)", "Platform": "C845A_M8_RTXPro4500_32GBx8_TRT", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9655", "host_processors_per_node": 2, "host_processor_core_count": "96", "accelerator_model_name": "NVIDIA RTX PRO 4500 Blackwell", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1", "operating_system": "Ubuntu 24.04", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 18800.9, "server_tokens_per_second": 18186.2, "server_over_offline": 0.9673047566871799, "server_over_offline_percent": 96.73047566871799, "difference_percent_of_offline": -3.2695243312820144, "offline_accuracy": "ROUGE1: 38.7819 ROUGE2: 16.0194 ROUGEL: 24.5377 ROUGELSUM: 35.7086 GEN_LEN: 8178827", "server_accuracy": "ROUGE1: 38.7633 ROUGE2: 16.0118 ROUGEL: 24.5232 ROUGELSUM: 35.6834 GEN_LEN: 8178277", "offline_location": "./closed/Cisco/results/C845A_M8_RTXPro4500_32GBx8_TRT/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/Cisco/results/C845A_M8_RTXPro4500_32GBx8_TRT/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "38", "server_row_id": "39", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco UCS C845A M8 (8x RTXPro6000-PCIe-96GB, TensorRT)", "Platform": "C845A_M8_RTXPro6000_96GBx8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 15508.4, "server_tokens_per_second": 14817.4, "server_over_offline": 0.9554435015862371, "server_over_offline_percent": 95.54435015862371, "difference_percent_of_offline": -4.455649841376285, "offline_accuracy": "exact_match: 83.570", "server_accuracy": "exact_match: 84.117", "offline_location": "./closed/Cisco/results/C845A_M8_RTXPro6000_96GBx8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Cisco/results/C845A_M8_RTXPro6000_96GBx8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "40", "server_row_id": "41", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco UCS C845A M8 (8x RTXPro6000-PCIe-96GB, TensorRT)", "Platform": "C845A_M8_RTXPro6000_96GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 29904.0, "server_tokens_per_second": 29316.4, "server_over_offline": 0.9803504547886571, "server_over_offline_percent": 98.03504547886571, "difference_percent_of_offline": -1.9649545211342878, "offline_accuracy": "ROUGE1: 44.7661 ROUGE2: 22.3346 ROUGEL: 29.1319 TOKENS_PER_SAMPLE: 274.4", "server_accuracy": "ROUGE1: 44.7625 ROUGE2: 22.3336 ROUGEL: 29.1052 TOKENS_PER_SAMPLE: 273.5", "offline_location": "./closed/Cisco/results/C845A_M8_RTXPro6000_96GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Cisco/results/C845A_M8_RTXPro6000_96GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "43", "server_row_id": "44", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco UCS C845A M8 (8x RTXPro6000-PCIe-96GB, TensorRT)", "Platform": "C845A_M8_RTXPro6000_96GBx8_TRT", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 50453.7, "server_tokens_per_second": 49174.7, "server_over_offline": 0.9746500256670968, "server_over_offline_percent": 97.46500256670969, "difference_percent_of_offline": -2.5349974332903202, "offline_accuracy": "ROUGE1: 38.769 ROUGE2: 16.0147 ROUGEL: 24.5312 ROUGELSUM: 35.6942 GEN_LEN: 8178808", "server_accuracy": "ROUGE1: 38.7598 ROUGE2: 16.0015 ROUGEL: 24.5351 ROUGELSUM: 35.6872 GEN_LEN: 8179087", "offline_location": "./closed/Cisco/results/C845A_M8_RTXPro6000_96GBx8_TRT/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/Cisco/results/C845A_M8_RTXPro6000_96GBx8_TRT/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "46", "server_row_id": "47", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco UCS C880A M8 (8x NVIDIA B300-SXM-270GB, TensorRT)", "Platform": "C880A_M8_B300_270GBx8_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6767P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 70325.4, "server_tokens_per_second": 66171.6, "server_over_offline": 0.9409345698709145, "server_over_offline_percent": 94.09345698709146, "difference_percent_of_offline": -5.90654301290855, "offline_accuracy": "exact_match: 80.92525068368278 TOKENS_PER_SAMPLE: 3782.986326344576", "server_accuracy": "exact_match: 80.76572470373746 TOKENS_PER_SAMPLE: 3811.0783956244304", "offline_location": "./closed/Cisco/results/C880A_M8_B300_270GBx8_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Cisco/results/C880A_M8_B300_270GBx8_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "49", "server_row_id": "50", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco UCS C880A M8 (8x NVIDIA B300-SXM-270GB, TensorRT)", "Platform": "C880A_M8_B300_270GBx8_TRT", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6767P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 171114.0, "server_tokens_per_second": 173950.0, "server_over_offline": 1.0165737461575324, "server_over_offline_percent": 101.65737461575324, "difference_percent_of_offline": 1.6573746157532376, "offline_accuracy": "ROUGE1: 38.4898 ROUGE2: 15.9329 ROUGEL: 24.3413 ROUGELSUM: 35.6705 GEN_LEN: 8186559", "server_accuracy": "ROUGE1: 38.4912 ROUGE2: 15.9156 ROUGEL: 24.3266 ROUGELSUM: 35.6593 GEN_LEN: 8186471", "offline_location": "./closed/Cisco/results/C880A_M8_B300_270GBx8_TRT/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/Cisco/results/C880A_M8_B300_270GBx8_TRT/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "54", "server_row_id": "55", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco_UCS_B300x16_G200_4x2_RO", "Platform": "Cisco_UCS_B300x16_G200_4x2_RO", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 2, "host_processor_model_name": "Intel(R) Xeon(R) 6776P", "host_processors_per_node": 2, "host_processor_core_count": "128", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 16, "framework": "TensorRT 10.14.1.48; TensorRT-LLM 1.3.0rc0", "operating_system": "Ubuntu 24.04.4 LTS, Linux 6.17.0-35-generic", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 137067.0, "server_tokens_per_second": 88915.5, "server_over_offline": 0.6487010002407582, "server_over_offline_percent": 64.87010002407581, "difference_percent_of_offline": -35.12989997592418, "offline_accuracy": "exact_match: 81.33546034639927 TOKENS_PER_SAMPLE: 3825.4619416590704", "server_accuracy": "exact_match: 81.28988149498633 TOKENS_PER_SAMPLE: 3794.9457611668186", "offline_location": "./closed/Cisco/results/Cisco_UCS_B300x16_G200_4x2_RO/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Cisco/results/Cisco_UCS_B300x16_G200_4x2_RO/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "56", "server_row_id": "57", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco_UCS_B300x16_G200_4x2_RO", "Platform": "Cisco_UCS_B300x16_G200_4x2_RO", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 2, "host_processor_model_name": "Intel(R) Xeon(R) 6776P", "host_processors_per_node": 2, "host_processor_core_count": "128", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 16, "framework": "TensorRT 10.14.1.48; TensorRT-LLM 1.3.0rc0", "operating_system": "Ubuntu 24.04.4 LTS, Linux 6.17.0-35-generic", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 220527.0, "server_tokens_per_second": 214774.0, "server_over_offline": 0.9739124914409574, "server_over_offline_percent": 97.39124914409574, "difference_percent_of_offline": -2.6087508559042627, "offline_accuracy": "exact_match: 83.296", "server_accuracy": "exact_match: 83.572", "offline_location": "./closed/Cisco/results/Cisco_UCS_B300x16_G200_4x2_RO/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Cisco/results/Cisco_UCS_B300x16_G200_4x2_RO/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "58", "server_row_id": "59", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco_UCS_B300x16_G200_4x2_RO", "Platform": "Cisco_UCS_B300x16_G200_4x2_RO", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 2, "host_processor_model_name": "Intel(R) Xeon(R) 6776P", "host_processors_per_node": 2, "host_processor_core_count": "128", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 16, "framework": "TensorRT 10.14.1.48; TensorRT-LLM 1.3.0rc0", "operating_system": "Ubuntu 24.04.4 LTS, Linux 6.17.0-35-generic", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 227570.0, "server_tokens_per_second": 179597.0, "server_over_offline": 0.7891945335501165, "server_over_offline_percent": 78.91945335501165, "difference_percent_of_offline": -21.080546644988353, "offline_accuracy": "ROUGE1: 44.8082 ROUGE2: 22.3989 ROUGEL: 29.1755 TOKENS_PER_SAMPLE: 272.1", "server_accuracy": "ROUGE1: 44.7969 ROUGE2: 22.3691 ROUGEL: 29.184 TOKENS_PER_SAMPLE: 272.7", "offline_location": "./closed/Cisco/results/Cisco_UCS_B300x16_G200_4x2_RO/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Cisco/results/Cisco_UCS_B300x16_G200_4x2_RO/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "60", "server_row_id": "61", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco UCS H200 x8 + MI350X x8 G200 2x2 TOR", "Platform": "Cisco_UCS_H200x8_MI350Xx8_G200_2x2_TOR", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 2, "host_processor_model_name": "AMD EPYC", "host_processors_per_node": 2, "host_processor_core_count": "128", "accelerator_model_name": "NVIDIA H200-SXM-141GB + AMD Instinct MI350X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 16, "framework": "vLLM", "operating_system": "Ubuntu 22.04", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "mxfp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 94249.3, "server_tokens_per_second": 73357.6, "server_over_offline": 0.7783357542178032, "server_over_offline_percent": 77.83357542178032, "difference_percent_of_offline": -22.16642457821968, "offline_accuracy": "exact_match: 83.57105741751104", "server_accuracy": "exact_match: 83.43465835281891", "offline_location": "./closed/Cisco/results/Cisco_UCS_H200x8_MI350Xx8_G200_2x2_TOR/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Cisco/results/Cisco_UCS_H200x8_MI350Xx8_G200_2x2_TOR/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "63", "server_row_id": "64", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco UCS H200 x8 + MI350X x8 G200 2x2 TOR", "Platform": "Cisco_UCS_H200x8_MI350Xx8_G200_2x2_TOR", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 2, "host_processor_model_name": "AMD EPYC", "host_processors_per_node": 2, "host_processor_core_count": "128", "accelerator_model_name": "NVIDIA H200-SXM-141GB + AMD Instinct MI350X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 16, "framework": "vLLM", "operating_system": "Ubuntu 22.04", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp8,fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 99683.8, "server_tokens_per_second": 95104.6, "server_over_offline": 0.9540627464041299, "server_over_offline_percent": 95.40627464041299, "difference_percent_of_offline": -4.593725359587008, "offline_accuracy": "ROUGE1: 44.4169 ROUGE2: 22.068 ROUGEL: 28.7957 TOKENS_PER_SAMPLE: 286.6", "server_accuracy": "ROUGE1: 44.6044 ROUGE2: 22.1926 ROUGEL: 28.9466 TOKENS_PER_SAMPLE: 284.7", "offline_location": "./closed/Cisco/results/Cisco_UCS_H200x8_MI350Xx8_G200_2x2_TOR/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Cisco/results/Cisco_UCS_H200x8_MI350Xx8_G200_2x2_TOR/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "66", "server_row_id": "67", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco UCS H200 x8 + MI350X x8 G200 2x2 TOR", "Platform": "Cisco_UCS_H200x8_MI350Xx8_G200_2x2_TOR", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 2, "host_processor_model_name": "AMD EPYC", "host_processors_per_node": 2, "host_processor_core_count": "128", "accelerator_model_name": "NVIDIA H200-SXM-141GB + AMD Instinct MI350X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 16, "framework": "vLLM", "operating_system": "Ubuntu 22.04", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp8,fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 163292.0, "server_tokens_per_second": 161110.0, "server_over_offline": 0.9866374347794136, "server_over_offline_percent": 98.66374347794135, "difference_percent_of_offline": -1.336256522058643, "offline_accuracy": "ROUGE1: 38.5675 ROUGE2: 16.0757 ROUGEL: 24.5063 ROUGELSUM: 35.5154 GEN_LEN: 8141814", "server_accuracy": "ROUGE1: 38.6437 ROUGE2: 16.1045 ROUGEL: 24.5278 ROUGELSUM: 35.5601 GEN_LEN: 8147699", "offline_location": "./closed/Cisco/results/Cisco_UCS_H200x8_MI350Xx8_G200_2x2_TOR/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/Cisco/results/Cisco_UCS_H200x8_MI350Xx8_G200_2x2_TOR/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "69", "server_row_id": "70", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco UCS X210 M8 - X580P PCIe Node (4x NVIDIA RTX PRO 4500 Blackwell-32GB, TensorRT)", "Platform": "X210M8_X580P_RTXPro4500_32GBx4_TRT", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6747P", "host_processors_per_node": 2, "host_processor_core_count": "48", "accelerator_model_name": "NVIDIA RTX PRO 4500 Blackwell", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "TensorRT 10.14, CUDA 13.1", "operating_system": "Ubuntu 24.04", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 9518.66, "server_tokens_per_second": 9277.73, "server_over_offline": 0.9746886641607116, "server_over_offline_percent": 97.46886641607117, "difference_percent_of_offline": -2.531133583928835, "offline_accuracy": "ROUGE1: 38.7664 ROUGE2: 16.0107 ROUGEL: 24.5281 ROUGELSUM: 35.6917 GEN_LEN: 8178365", "server_accuracy": "ROUGE1: 38.7836 ROUGE2: 16.0298 ROUGEL: 24.5367 ROUGELSUM: 35.7048 GEN_LEN: 8178628", "offline_location": "./closed/Cisco/results/X210M8_X580P_RTXPro4500_32GBx4_TRT/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/Cisco/results/X210M8_X580P_RTXPro4500_32GBx4_TRT/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "72", "server_row_id": "73", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco UCS X210c M8 - X580P PCIe Node (4x RTXPro6000-PCIe-96GB, TensorRT)", "Platform": "X210M8_X580P_RTXPro6000_96GBx4_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6747P", "host_processors_per_node": 2, "host_processor_core_count": "48", "accelerator_model_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "TensorRT 10.14, CUDA 13.1", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 7801.36, "server_tokens_per_second": 6950.76, "server_over_offline": 0.8909677287037132, "server_over_offline_percent": 89.09677287037132, "difference_percent_of_offline": -10.903227129628679, "offline_accuracy": "exact_match: 83.969", "server_accuracy": "exact_match: 83.637", "offline_location": "./closed/Cisco/results/X210M8_X580P_RTXPro6000_96GBx4_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Cisco/results/X210M8_X580P_RTXPro6000_96GBx4_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "74", "server_row_id": "75", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco UCS X210c M8 - X580P PCIe Node (4x RTXPro6000-PCIe-96GB, TensorRT)", "Platform": "X210M8_X580P_RTXPro6000_96GBx4_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6747P", "host_processors_per_node": 2, "host_processor_core_count": "48", "accelerator_model_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "TensorRT 10.14, CUDA 13.1", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 14860.7, "server_tokens_per_second": 14333.3, "server_over_offline": 0.964510420101341, "server_over_offline_percent": 96.4510420101341, "difference_percent_of_offline": -3.5489579898658974, "offline_accuracy": "ROUGE1: 44.7766 ROUGE2: 22.3407 ROUGEL: 29.1379 TOKENS_PER_SAMPLE: 275.2", "server_accuracy": "ROUGE1: 44.7151 ROUGE2: 22.3389 ROUGEL: 29.0874 TOKENS_PER_SAMPLE: 274.6", "offline_location": "./closed/Cisco/results/X210M8_X580P_RTXPro6000_96GBx4_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Cisco/results/X210M8_X580P_RTXPro6000_96GBx4_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "77", "server_row_id": "78", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Cisco UCS X215c M8 - X580P PCIe Node (4x RTXPro6000-PCIe-96GB, TensorRT)", "Platform": "X215M8_X580P_RTXPro6000_96GBx4_TRT", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9845", "host_processors_per_node": 2, "host_processor_core_count": "160", "accelerator_model_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "TensorRT 10.14, CUDA 13.1", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 24913.5, "server_tokens_per_second": 24498.6, "server_over_offline": 0.9833463784695045, "server_over_offline_percent": 98.33463784695044, "difference_percent_of_offline": -1.6653621530495544, "offline_accuracy": "ROUGE1: 38.7701 ROUGE2: 16.0155 ROUGEL: 24.534 ROUGELSUM: 35.6954 GEN_LEN: 8178898", "server_accuracy": "ROUGE1: 38.766 ROUGE2: 16.0082 ROUGEL: 24.5262 ROUGELSUM: 35.6898 GEN_LEN: 8179590", "offline_location": "./closed/Cisco/results/X215M8_X580P_RTXPro6000_96GBx4_TRT/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/Cisco/results/X215M8_X580P_RTXPro6000_96GBx4_TRT/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "80", "server_row_id": "81", "metadata": { "Organization": "Cisco", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "X410M8_1_node_4S_GNR_86C", "Platform": "X410M8_1_node_4S_GNR_86C", "Model": "llama3_1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6788P", "host_processors_per_node": 4, "host_processor_core_count": "86", "accelerator_model_name": null, "accelerators_per_node": 0, "total_accelerators": 0, "framework": "PyTorch", "operating_system": "Ubuntu 24.04.3", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "INT8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 2068.63, "server_tokens_per_second": 1030.45, "server_over_offline": 0.49813161367668457, "server_over_offline_percent": 49.81316136766846, "difference_percent_of_offline": -50.18683863233154, "offline_accuracy": "ROUGE1: 38.6284 ROUGE2: 15.804 ROUGEL: 24.4033 ROUGELSUM: 35.6616 GEN_LEN: 8196513", "server_accuracy": "ROUGE1: 38.8382 ROUGE2: 16.0493 ROUGEL: 24.6331 ROUGELSUM: 35.8769 GEN_LEN: 8171982", "offline_location": "./closed/Cisco/results/X410M8_1_node_4S_GNR_86C/llama3_1-8b/Offline/performance/run_1", "server_location": "./closed/Cisco/results/X410M8_1_node_4S_GNR_86C/llama3_1-8b/Server/performance/run_1" }, { "offline_row_id": "83", "server_row_id": "84", "metadata": { "Organization": "CoreWeave", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "CoreWeave B200 SXM (8x B200-SXM-180GB)", "Platform": "B200-SXM-180GBx8_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8562Y+", "host_processors_per_node": 2, "host_processor_core_count": "32", "accelerator_model_name": "NVIDIA B200-SXM-180GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0", "operating_system": "Ubuntu 24.04", "notes": "B200 TGP 1000W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 58913.2, "server_tokens_per_second": 56121.3, "server_over_offline": 0.9526099414053218, "server_over_offline_percent": 95.26099414053219, "difference_percent_of_offline": -4.739005859467815, "offline_accuracy": "exact_match: 80.9024612579763 TOKENS_PER_SAMPLE: 3888.048541476755", "server_accuracy": "exact_match: 81.60893345487693 TOKENS_PER_SAMPLE: 3816.3475387420235", "offline_location": "./closed/CoreWeave/results/B200-SXM-180GBx8_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/CoreWeave/results/B200-SXM-180GBx8_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "85", "server_row_id": "86", "metadata": { "Organization": "CoreWeave", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "CoreWeave B200 SXM (8x B200-SXM-180GB)", "Platform": "B200-SXM-180GBx8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8562Y+", "host_processors_per_node": 2, "host_processor_core_count": "32", "accelerator_model_name": "NVIDIA B200-SXM-180GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0", "operating_system": "Ubuntu 24.04", "notes": "B200 TGP 1000W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 91487.4, "server_tokens_per_second": 90204.7, "server_over_offline": 0.9859794900718569, "server_over_offline_percent": 98.59794900718569, "difference_percent_of_offline": -1.4020509928143077, "offline_accuracy": "exact_match: 83.901", "server_accuracy": "exact_match: 83.728", "offline_location": "./closed/CoreWeave/results/B200-SXM-180GBx8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/CoreWeave/results/B200-SXM-180GBx8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "87", "server_row_id": "88", "metadata": { "Organization": "CoreWeave", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "CoreWeave B200 SXM (8x B200-SXM-180GB)", "Platform": "B200-SXM-180GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8562Y+", "host_processors_per_node": 2, "host_processor_core_count": "32", "accelerator_model_name": "NVIDIA B200-SXM-180GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0", "operating_system": "Ubuntu 24.04", "notes": "B200 TGP 1000W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 102703.0, "server_tokens_per_second": 102398.0, "server_over_offline": 0.9970302717544764, "server_over_offline_percent": 99.70302717544764, "difference_percent_of_offline": -0.2969728245523551, "offline_accuracy": "ROUGE1: 44.833 ROUGE2: 22.4096 ROUGEL: 29.2097 TOKENS_PER_SAMPLE: 271.6", "server_accuracy": "ROUGE1: 44.8305 ROUGE2: 22.4096 ROUGEL: 29.21 TOKENS_PER_SAMPLE: 271.5", "offline_location": "./closed/CoreWeave/results/B200-SXM-180GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/CoreWeave/results/B200-SXM-180GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "89", "server_row_id": "90", "metadata": { "Organization": "CoreWeave", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "CoreWeave B300 SXM (8x B300-SXM-270GB)", "Platform": "B300-SXM-270GBx8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6747P", "host_processors_per_node": 2, "host_processor_core_count": "48", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 112840.0, "server_tokens_per_second": 111247.0, "server_over_offline": 0.9858826657213754, "server_over_offline_percent": 98.58826657213754, "difference_percent_of_offline": -1.4117334278624605, "offline_accuracy": "exact_match: 83.290", "server_accuracy": "exact_match: 83.467", "offline_location": "./closed/CoreWeave/results/B300-SXM-270GBx8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/CoreWeave/results/B300-SXM-270GBx8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "91", "server_row_id": "92", "metadata": { "Organization": "CoreWeave", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "CoreWeave B300 SXM (8x B300-SXM-270GB)", "Platform": "B300-SXM-270GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6747P", "host_processors_per_node": 2, "host_processor_core_count": "48", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 115530.0, "server_tokens_per_second": 114576.0, "server_over_offline": 0.9917424045702415, "server_over_offline_percent": 99.17424045702415, "difference_percent_of_offline": -0.8257595429758458, "offline_accuracy": "ROUGE1: 44.7888 ROUGE2: 22.3737 ROUGEL: 29.1521 TOKENS_PER_SAMPLE: 272.9", "server_accuracy": "ROUGE1: 44.8422 ROUGE2: 22.4119 ROUGEL: 29.1975 TOKENS_PER_SAMPLE: 272.8", "offline_location": "./closed/CoreWeave/results/B300-SXM-270GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/CoreWeave/results/B300-SXM-270GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "93", "server_row_id": "94", "metadata": { "Organization": "CoreWeave", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "CoreWeave GB200 NVL72 (72x GB200-186GB_aarch64)", "Platform": "GB200-NVL72_GB200-186GB_aarch64x72_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace (Neoverse-V2)", "host_processors_per_node": 2, "host_processor_core_count": "72", "accelerator_model_name": "NVIDIA GB200", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0", "operating_system": "Ubuntu 24.04", "notes": "GB200 TGP 1200W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 529991.0, "server_tokens_per_second": 419778.0, "server_over_offline": 0.7920474121258663, "server_over_offline_percent": 79.20474121258663, "difference_percent_of_offline": -20.795258787413374, "offline_accuracy": "exact_match: 80.76572470373746 TOKENS_PER_SAMPLE: 3815.436645396536", "server_accuracy": "exact_match: 81.03919781221514 TOKENS_PER_SAMPLE: 3802.557201458523", "offline_location": "./closed/CoreWeave/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/CoreWeave/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "95", "server_row_id": "96", "metadata": { "Organization": "CoreWeave", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "CoreWeave GB200 NVL72 (72x GB200-186GB_aarch64)", "Platform": "GB200-NVL72_GB200-186GB_aarch64x72_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace (Neoverse-V2)", "host_processors_per_node": 2, "host_processor_core_count": "72", "accelerator_model_name": "NVIDIA GB200", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0", "operating_system": "Ubuntu 24.04", "notes": "GB200 TGP 1200W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 911566.0, "server_tokens_per_second": 901058.0, "server_over_offline": 0.9884725845413278, "server_over_offline_percent": 98.84725845413278, "difference_percent_of_offline": -1.1527415458672197, "offline_accuracy": "exact_match: 83.394", "server_accuracy": "exact_match: 83.424", "offline_location": "./closed/CoreWeave/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/CoreWeave/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "97", "server_row_id": "98", "metadata": { "Organization": "CoreWeave", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "CoreWeave GB200 NVL72 (72x GB200-186GB_aarch64)", "Platform": "GB200-NVL72_GB200-186GB_aarch64x72_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace (Neoverse-V2)", "host_processors_per_node": 2, "host_processor_core_count": "72", "accelerator_model_name": "NVIDIA GB200", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0", "operating_system": "Ubuntu 24.04", "notes": "GB200 TGP 1200W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 894002.0, "server_tokens_per_second": 910621.0, "server_over_offline": 1.0185894438714902, "server_over_offline_percent": 101.85894438714902, "difference_percent_of_offline": 1.8589443871490197, "offline_accuracy": "ROUGE1: 44.8296 ROUGE2: 22.4099 ROUGEL: 29.2105 TOKENS_PER_SAMPLE: 271.5", "server_accuracy": "ROUGE1: 44.8247 ROUGE2: 22.3971 ROUGEL: 29.2015 TOKENS_PER_SAMPLE: 271.7", "offline_location": "./closed/CoreWeave/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/CoreWeave/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "101", "server_row_id": "102", "metadata": { "Organization": "CoreWeave", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "CoreWeave GB300 NVL72 (72x GB300-288GB_aarch64)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x72_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace (Neoverse-V2)", "host_processors_per_node": 2, "host_processor_core_count": "72", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "GB300 TGP 1400W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 1197710.0, "server_tokens_per_second": 1160490.0, "server_over_offline": 0.9689240300239624, "server_over_offline_percent": 96.89240300239625, "difference_percent_of_offline": -3.1075969976037565, "offline_accuracy": "exact_match: 83.098", "server_accuracy": "exact_match: 82.875", "offline_location": "./closed/CoreWeave/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/CoreWeave/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "103", "server_row_id": "104", "metadata": { "Organization": "CoreWeave", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "CoreWeave GB300 NVL72 (72x GB300-288GB_aarch64)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x72_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace (Neoverse-V2)", "host_processors_per_node": 2, "host_processor_core_count": "72", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "GB300 TGP 1400W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 1136100.0, "server_tokens_per_second": 944902.0, "server_over_offline": 0.8317067159581023, "server_over_offline_percent": 83.17067159581023, "difference_percent_of_offline": -16.82932840418977, "offline_accuracy": "ROUGE1: 44.8147 ROUGE2: 22.3736 ROUGEL: 29.189 TOKENS_PER_SAMPLE: 272.4", "server_accuracy": "ROUGE1: 44.8524 ROUGE2: 22.4096 ROUGEL: 29.218 TOKENS_PER_SAMPLE: 272.3", "offline_location": "./closed/CoreWeave/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/CoreWeave/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "107", "server_row_id": "108", "metadata": { "Organization": "Crusoe", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Crusoe Cloud 64x8 MI355X (512 GPU) \u2014 DeepSeek-R1 / SGLang", "Platform": "512xMI355X_Crusoe_DS", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 64, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 512, "framework": "SGLang 0.5.15.post1 (ROCm 7.2.0), PyTorch 2.x", "operating_system": "Ubuntu 24.04 (node amdgpu driver 6.14.x; container ROCm 7.2.0 userspace)", "notes": "Crusoe Cloud mi355x-288gb-roce.8x, 64 nodes = 512 MI355X GPUs. First-ever 512-GPU DeepSeek-R1 (Offline + Server).. AMD MLPerf Inference v6.1 DeepSeek-R1 harness (SGLang backend). Model amd/Deepseek-S3_sq_a05_v2_mlperf6_1 \u2014 MXFP4 experts / FP8 KV cache / BF16 MLA-absorb. Per node TP8 / EP8 / DP-attention8 / PP1 (one 8-GPU replica); device_count=512. Server uses engine warmup (dsr_code6_1 WarmUp.ENCODED_SAMPLES) + prefill-delayer for TTFT/TPOT.", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 2901950.0, "server_tokens_per_second": 2405310.0, "server_over_offline": 0.8288599045469426, "server_over_offline_percent": 82.88599045469425, "difference_percent_of_offline": -17.11400954530574, "offline_accuracy": "exact_match: 80.69735642661804 TOKENS_PER_SAMPLE: 3898.373746581586", "server_accuracy": "exact_match: 80.69735642661804 TOKENS_PER_SAMPLE: 3898.373746581586", "offline_location": "./closed/Crusoe/results/512xMI355X_Crusoe_DS/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Crusoe/results/512xMI355X_Crusoe_DS/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "109", "server_row_id": "110", "metadata": { "Organization": "Crusoe", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Crusoe Cloud 64x8 MI355X (512 GPU) \u2014 gpt-oss-120b / vLLM", "Platform": "512xMI355X_Crusoe_GPT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 64, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 512, "framework": "vLLM 0.22.1 (ROCm 7.2.2), PyTorch 2.10", "operating_system": "Ubuntu 24.04 (node amdgpu driver 6.14.x; container ROCm 7.2.2 userspace)", "notes": "Crusoe Cloud mi355x-288gb-roce.8x, 64 nodes = 512 MI355X GPUs. First-ever 512-GPU gpt-oss-120b (Offline + Server).. AMD MLPerf Inference v6.1 gpt-oss-120b harness (vLLM backend). Native-MXFP4 model openai/gpt-oss-120b@b5c939d (NOT the v6.0 Quark repack). tensor_parallel_size=1 (one GPU per replica, 8 replicas/node); device_count=512. Dedicated ZMQ head dispatches Offline sample-ranges / Server per-query and relays token streams.", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "mxfp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 5749440.0, "server_tokens_per_second": 5392930.0, "server_over_offline": 0.9379922218511716, "server_over_offline_percent": 93.79922218511716, "difference_percent_of_offline": -6.200777814882841, "offline_accuracy": "exact_match: 83.706", "server_accuracy": "exact_match: 83.706", "offline_location": "./closed/Crusoe/results/512xMI355X_Crusoe_GPT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Crusoe/results/512xMI355X_Crusoe_GPT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "117", "server_row_id": "118", "metadata": { "Organization": "Crusoe", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB200 NVL72 (4x GB200-186GB_aarch64, TensorRT)", "Platform": "GB200-NVL72_GB200-186GB_aarch64x4_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB200", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB200 NVL72, 1200W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 50912.9, "server_tokens_per_second": 50613.5, "server_over_offline": 0.9941193685686731, "server_over_offline_percent": 99.41193685686731, "difference_percent_of_offline": -0.5880631431326888, "offline_accuracy": "exact_match: 83.691", "server_accuracy": "exact_match: 83.642", "offline_location": "./closed/Crusoe/results/GB200-NVL72_GB200-186GB_aarch64x4_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Crusoe/results/GB200-NVL72_GB200-186GB_aarch64x4_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "121", "server_row_id": "122", "metadata": { "Organization": "Dell", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "PowerEdge R770_86C 16x64GB (6400 MT/s)", "Platform": "1-node-2S-GNR_86C", "Model": "llama3_1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6787P", "host_processors_per_node": 2, "host_processor_core_count": "86", "accelerator_model_name": null, "accelerators_per_node": 0, "total_accelerators": 0, "framework": "PyTorch", "operating_system": "Ubuntu 24.04 LTS", "notes": "Dell PowerEdge R770. vLLM, FP8 KV Cache", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "INT8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 1227.99, "server_tokens_per_second": 516.012, "server_over_offline": 0.4202086336207949, "server_over_offline_percent": 42.02086336207949, "difference_percent_of_offline": -57.97913663792051, "offline_accuracy": "ROUGE1: 38.6229 ROUGE2: 15.8107 ROUGEL: 24.4082 ROUGELSUM: 35.6517 GEN_LEN: 8188478", "server_accuracy": "ROUGE1: 38.6453 ROUGE2: 15.8841 ROUGEL: 24.4777 ROUGELSUM: 35.7016 GEN_LEN: 8176464", "offline_location": "./closed/Dell/results/1-node-2S-GNR_86C/llama3_1-8b/Offline/performance/run_1", "server_location": "./closed/Dell/results/1-node-2S-GNR_86C/llama3_1-8b/Server/performance/run_1" }, { "offline_row_id": "131", "server_row_id": "132", "metadata": { "Organization": "Dell", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "PowerEdge XE7740 (8x RTXPro6000-PCIe-96GB, TensorRT)", "Platform": "XE7740_RTXPro6000_PCIe_96GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6767P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 29796.7, "server_tokens_per_second": 28973.3, "server_over_offline": 0.9723660673833008, "server_over_offline_percent": 97.23660673833008, "difference_percent_of_offline": -2.763393261669922, "offline_accuracy": "ROUGE1: 44.7408 ROUGE2: 22.3357 ROUGEL: 29.1354 TOKENS_PER_SAMPLE: 274.4", "server_accuracy": "ROUGE1: 44.6929 ROUGE2: 22.2906 ROUGEL: 29.0805 TOKENS_PER_SAMPLE: 267.3", "offline_location": "./closed/Dell/results/XE7740_RTXPro6000_PCIe_96GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Dell/results/XE7740_RTXPro6000_PCIe_96GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "133", "server_row_id": "134", "metadata": { "Organization": "Dell", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "PowerEdge XE7745 (8x NVIDIA RTX PRO 4500 Blackwell, TensorRT)", "Platform": "XE7745_RTXPRO4500_32GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9655", "host_processors_per_node": 1, "host_processor_core_count": "96", "accelerator_model_name": "NVIDIA RTX PRO 4500 Blackwell", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.3", "operating_system": "Ubuntu 24.04.3", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 4374.06, "server_tokens_per_second": 3012.33, "server_over_offline": 0.6886805393615999, "server_over_offline_percent": 68.86805393616, "difference_percent_of_offline": -31.13194606384001, "offline_accuracy": "ROUGE1: 44.7919 ROUGE2: 22.3659 ROUGEL: 29.1627 TOKENS_PER_SAMPLE: 273.2", "server_accuracy": "ROUGE1: 44.7861 ROUGE2: 22.3547 ROUGEL: 29.1687 TOKENS_PER_SAMPLE: 271.3", "offline_location": "./closed/Dell/results/XE7745_RTXPRO4500_32GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Dell/results/XE7745_RTXPRO4500_32GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "137", "server_row_id": "138", "metadata": { "Organization": "Dell", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Dell XE9780 (8x B300-SXM-270GB, TensorRT)", "Platform": "XE9780-B300-SXM-270GBx8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6747P", "host_processors_per_node": 2, "host_processor_core_count": "96", "accelerator_model_name": "Nvidia B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, Dell Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 109257.0, "server_tokens_per_second": 103332.0, "server_over_offline": 0.9457700650759219, "server_over_offline_percent": 94.57700650759219, "difference_percent_of_offline": -5.422993492407812, "offline_accuracy": "exact_match: 83.610", "server_accuracy": "exact_match: 83.460", "offline_location": "./closed/Dell/results/XE9780-B300-SXM-270GBx8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Dell/results/XE9780-B300-SXM-270GBx8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "139", "server_row_id": "140", "metadata": { "Organization": "Dell", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Dell XE9780 (8x B300-SXM-270GB, TensorRT)", "Platform": "XE9780-B300-SXM-270GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6747P", "host_processors_per_node": 2, "host_processor_core_count": "96", "accelerator_model_name": "Nvidia B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, Dell Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 104006.0, "server_tokens_per_second": 96180.6, "server_over_offline": 0.9247601099936543, "server_over_offline_percent": 92.47601099936543, "difference_percent_of_offline": -7.523989000634568, "offline_accuracy": "ROUGE1: 44.9129 ROUGE2: 22.4979 ROUGEL: 29.3269 TOKENS_PER_SAMPLE: 271.0", "server_accuracy": "ROUGE1: 44.8204 ROUGE2: 22.399 ROUGEL: 29.2127 TOKENS_PER_SAMPLE: 272.0", "offline_location": "./closed/Dell/results/XE9780-B300-SXM-270GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Dell/results/XE9780-B300-SXM-270GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "141", "server_row_id": "142", "metadata": { "Organization": "Dell", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "XE9785L_8xMI355X", "Platform": "XE9785L_8xMI355X", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9655", "host_processors_per_node": 2, "host_processor_core_count": "96", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0, ROCm 7.2.2", "operating_system": "Ubuntu 24.04.4 LTS", "notes": "8x AMD Instinct MI355X (gfx950), 288GB HBM3E each", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 117804.0, "server_tokens_per_second": 110188.0, "server_over_offline": 0.9353502427761367, "server_over_offline_percent": 93.53502427761367, "difference_percent_of_offline": -6.464975722386335, "offline_accuracy": "exact_match: 84.217", "server_accuracy": "exact_match: 83.843", "offline_location": "./closed/Dell/results/XE9785L_8xMI355X/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Dell/results/XE9785L_8xMI355X/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "144", "server_row_id": "145", "metadata": { "Organization": "Dell", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "XE9785L_8xMI355X", "Platform": "XE9785L_8xMI355X", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9655", "host_processors_per_node": 2, "host_processor_core_count": "96", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0, ROCm 7.2.2", "operating_system": "Ubuntu 24.04.4 LTS", "notes": "8x AMD Instinct MI355X (gfx950), 288GB HBM3E each", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 103792.0, "server_tokens_per_second": 103932.0, "server_over_offline": 1.0013488515492524, "server_over_offline_percent": 100.13488515492523, "difference_percent_of_offline": 0.134885154925235, "offline_accuracy": "ROUGE1: 44.5322 ROUGE2: 22.1855 ROUGEL: 28.9228 TOKENS_PER_SAMPLE: 284.4", "server_accuracy": "ROUGE1: 44.5341 ROUGE2: 22.1716 ROUGEL: 28.9333 TOKENS_PER_SAMPLE: 284.0", "offline_location": "./closed/Dell/results/XE9785L_8xMI355X/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Dell/results/XE9785L_8xMI355X/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "147", "server_row_id": "148", "metadata": { "Organization": "Dell", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "XE9785L_8xMI355X", "Platform": "XE9785L_8xMI355X", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9655", "host_processors_per_node": 2, "host_processor_core_count": "96", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0, ROCm 7.2.2", "operating_system": "Ubuntu 24.04.4 LTS", "notes": "8x AMD Instinct MI355X (gfx950), 288GB HBM3E each", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 158458.0, "server_tokens_per_second": 148563.0, "server_over_offline": 0.9375544308270961, "server_over_offline_percent": 93.7554430827096, "difference_percent_of_offline": -6.244556917290389, "offline_accuracy": "ROUGE1: 38.5617 ROUGE2: 16.0263 ROUGEL: 24.4381 ROUGELSUM: 35.6994 GEN_LEN: 8151245", "server_accuracy": "ROUGE1: 38.5211 ROUGE2: 16.0095 ROUGEL: 24.4208 ROUGELSUM: 35.662 GEN_LEN: 8168331", "offline_location": "./closed/Dell/results/XE9785L_8xMI355X/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/Dell/results/XE9785L_8xMI355X/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "149", "server_row_id": "150", "metadata": { "Organization": "Dell", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Dell XE9785L (8x B300-SXM-270GB, TensorRT)", "Platform": "XE9785L_x86_B300-SXM-270GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9965", "host_processors_per_node": 2, "host_processor_core_count": "192", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 113101.0, "server_tokens_per_second": 106898.0, "server_over_offline": 0.9451552152500862, "server_over_offline_percent": 94.51552152500862, "difference_percent_of_offline": -5.484478474991383, "offline_accuracy": "ROUGE1: 44.8017 ROUGE2: 22.3823 ROUGEL: 29.1656 TOKENS_PER_SAMPLE: 272.7", "server_accuracy": "ROUGE1: 44.8186 ROUGE2: 22.3916 ROUGEL: 29.1862 TOKENS_PER_SAMPLE: 272.3", "offline_location": "./closed/Dell/results/XE9785L_x86_B300-SXM-270GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Dell/results/XE9785L_x86_B300-SXM-270GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "153", "server_row_id": "154", "metadata": { "Organization": "Dell_AMD", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "XE7745_8xMI350P_2xEPYC_9455", "Platform": "XE7745_8xMI350P_2xEPYC-9455", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9455", "host_processors_per_node": 2, "host_processor_core_count": "48", "accelerator_model_name": "AMD Instinct MI350P", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0, ROCm 7.2.2", "operating_system": "Ubuntu 24.04.4 LTS (kernel 6.8.0-136-generic)", "notes": "8x AMD Instinct MI350P (gfx950), 144GB HBM3E each. MXFP4 (W4A4) weights with FP8 KV cache, Quark quantization", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 48971.1, "server_tokens_per_second": 40867.9, "server_over_offline": 0.8345309784750599, "server_over_offline_percent": 83.45309784750599, "difference_percent_of_offline": -16.546902152494013, "offline_accuracy": "exact_match: 83.287", "server_accuracy": "exact_match: 83.631", "offline_location": "./closed/Dell_AMD/results/XE7745_8xMI350P_2xEPYC-9455/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Dell_AMD/results/XE7745_8xMI350P_2xEPYC-9455/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "156", "server_row_id": "157", "metadata": { "Organization": "Dell_AMD", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "XE7745_8xMI350P_2xEPYC_9455", "Platform": "XE7745_8xMI350P_2xEPYC-9455", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9455", "host_processors_per_node": 2, "host_processor_core_count": "48", "accelerator_model_name": "AMD Instinct MI350P", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0, ROCm 7.2.2", "operating_system": "Ubuntu 24.04.4 LTS (kernel 6.8.0-136-generic)", "notes": "8x AMD Instinct MI350P (gfx950), 144GB HBM3E each. MXFP4 (W4A4) weights with FP8 KV cache, Quark quantization", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 42286.2, "server_tokens_per_second": 41830.0, "server_over_offline": 0.9892116104071779, "server_over_offline_percent": 98.92116104071779, "difference_percent_of_offline": -1.0788389592822134, "offline_accuracy": "ROUGE1: 44.5012 ROUGE2: 22.1742 ROUGEL: 28.9465 TOKENS_PER_SAMPLE: 284.2", "server_accuracy": "ROUGE1: 44.4477 ROUGE2: 22.0889 ROUGEL: 28.8674 TOKENS_PER_SAMPLE: 287.4", "offline_location": "./closed/Dell_AMD/results/XE7745_8xMI350P_2xEPYC-9455/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Dell_AMD/results/XE7745_8xMI350P_2xEPYC-9455/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "159", "server_row_id": "160", "metadata": { "Organization": "Dell_AMD", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "XE7745_8xMI350P_2xEPYC_9455", "Platform": "XE7745_8xMI350P_2xEPYC-9455", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9455", "host_processors_per_node": 2, "host_processor_core_count": "48", "accelerator_model_name": "AMD Instinct MI350P", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0, ROCm 7.2.2", "operating_system": "Ubuntu 24.04.4 LTS (kernel 6.8.0-136-generic)", "notes": "8x AMD Instinct MI350P (gfx950), 144GB HBM3E each. MXFP4 (W4A4) weights with FP8 KV cache, Quark quantization", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 73643.0, "server_tokens_per_second": 72305.1, "server_over_offline": 0.9818326249609604, "server_over_offline_percent": 98.18326249609603, "difference_percent_of_offline": -1.8167375039039646, "offline_accuracy": "ROUGE1: 38.5604 ROUGE2: 16.0125 ROUGEL: 24.4415 ROUGELSUM: 35.6907 GEN_LEN: 8149596", "server_accuracy": "ROUGE1: 38.5633 ROUGE2: 16.0282 ROUGEL: 24.4369 ROUGELSUM: 35.7015 GEN_LEN: 8151565", "offline_location": "./closed/Dell_AMD/results/XE7745_8xMI350P_2xEPYC-9455/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/Dell_AMD/results/XE7745_8xMI350P_2xEPYC-9455/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "163", "server_row_id": "164", "metadata": { "Organization": "Dell_MangoBoost", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Heterogeneous Four-Cluster (2x8x MI300X + 2x8x MI355X, LLMBoost)", "Platform": "2x8xMI355X_2x8xMI300X", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 4, "host_processor_model_name": "AMD EPYC 9534 (on MI300X node) and AMD EPYC 9965 (on MI355X node)", "host_processors_per_node": 2, "host_processor_core_count": "64(AMD EPYC 9534) and 192(AMD EPYC 9965)", "accelerator_model_name": "AMD Instinct MI300X 192GB HBM3 (x2x8) and AMD Instinct MI355X 288GB HBM3e (x2x8)", "accelerators_per_node": 8, "total_accelerators": 32, "framework": "ROCm", "operating_system": "Ubuntu 22.04.5 LTS (jammy)", "notes": "Heterogeneous Four-Cluster with LLMBoost, vLLM 0.22.1 on MI355X, and vLLM 0.24.0 on MI300X", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 285454.0, "server_tokens_per_second": 253501.0, "server_over_offline": 0.8880625249602387, "server_over_offline_percent": 88.80625249602387, "difference_percent_of_offline": -11.193747503976127, "offline_accuracy": "exact_match: 84.250", "server_accuracy": "exact_match: 83.574", "offline_location": "./closed/Dell_MangoBoost/results/2x8xMI355X_2x8xMI300X/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Dell_MangoBoost/results/2x8xMI355X_2x8xMI300X/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "166", "server_row_id": "167", "metadata": { "Organization": "Dell_MangoBoost", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "PowerEdge XE9785", "Platform": "XE9785", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9965", "host_processors_per_node": 2, "host_processor_core_count": "192", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "ROCm", "operating_system": "Ubuntu 22.04.5 LTS (jammy)", "notes": "Intra-Node P/D Disaggregation for Interactive Scenario, and vLLM 0.22.1", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 115294.0, "server_tokens_per_second": 109065.0, "server_over_offline": 0.9459729040539837, "server_over_offline_percent": 94.59729040539837, "difference_percent_of_offline": -5.402709594601629, "offline_accuracy": "exact_match: 83.720", "server_accuracy": "exact_match: 83.806", "offline_location": "./closed/Dell_MangoBoost/results/XE9785/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Dell_MangoBoost/results/XE9785/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "169", "server_row_id": "170", "metadata": { "Organization": "Dell_MangoBoost", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "PowerEdge XE9785L", "Platform": "XE9785L", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9965", "host_processors_per_node": 2, "host_processor_core_count": "192", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "ROCm", "operating_system": "Ubuntu 22.04.5 LTS (jammy)", "notes": "Intra-Node P/D Disaggregation for Interactive Scenario, and vLLM 0.22.1", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 116953.0, "server_tokens_per_second": 110153.0, "server_over_offline": 0.941856985284687, "server_over_offline_percent": 94.18569852846869, "difference_percent_of_offline": -5.814301471531302, "offline_accuracy": "exact_match: 83.643", "server_accuracy": "exact_match: 83.598", "offline_location": "./closed/Dell_MangoBoost/results/XE9785L/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Dell_MangoBoost/results/XE9785L/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "171", "server_row_id": "172", "metadata": { "Organization": "Fujitsu", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "PRIMERGY_CDI_RTX_PRO_6000_PCIE_96GBx8, TensorRT)", "Platform": "PRIMERGY_CDI_RTX_PRO_6000_PCIE_96GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8592+", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA RTX PRO 6000 Blackwell", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.16, CUDA 13.2, cuDNN 9.21", "operating_system": "Ubuntu 24.04", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 26626.5, "server_tokens_per_second": 27340.9, "server_over_offline": 1.0268304133100483, "server_over_offline_percent": 102.68304133100483, "difference_percent_of_offline": 2.6830413310048318, "offline_accuracy": "ROUGE1: 44.7182 ROUGE2: 22.3154 ROUGEL: 29.104 TOKENS_PER_SAMPLE: 273.6", "server_accuracy": "ROUGE1: 44.7475 ROUGE2: 22.3697 ROUGEL: 29.1476 TOKENS_PER_SAMPLE: 274.5", "offline_location": "./closed/Fujitsu/results/PRIMERGY_CDI_RTX_PRO_6000_PCIE_96GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Fujitsu/results/PRIMERGY_CDI_RTX_PRO_6000_PCIE_96GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "173", "server_row_id": "174", "metadata": { "Organization": "GigaComputing", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "G894-AD3-AAX7", "Platform": "G894-AD3-AAX7_B300-SXM-270GBx8_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6979P", "host_processors_per_node": 2, "host_processor_core_count": "56", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 72445.1, "server_tokens_per_second": 70495.7, "server_over_offline": 0.9730913477930183, "server_over_offline_percent": 97.30913477930183, "difference_percent_of_offline": -2.6908652206981687, "offline_accuracy": "exact_match: 81.24430264357338 TOKENS_PER_SAMPLE: 3793.010711030082", "server_accuracy": "exact_match: 81.26709206927985 TOKENS_PER_SAMPLE: 3788.4774384685506", "offline_location": "./closed/GigaComputing/results/G894-AD3-AAX7_B300-SXM-270GBx8_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/GigaComputing/results/G894-AD3-AAX7_B300-SXM-270GBx8_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "175", "server_row_id": "176", "metadata": { "Organization": "GigaComputing", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "G894-AD3-AAX7", "Platform": "G894-AD3-AAX7_B300-SXM-270GBx8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6979P", "host_processors_per_node": 2, "host_processor_core_count": "56", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 111625.0, "server_tokens_per_second": 111218.0, "server_over_offline": 0.9963538633818589, "server_over_offline_percent": 99.6353863381859, "difference_percent_of_offline": -0.36461366181410604, "offline_accuracy": "exact_match: 83.316", "server_accuracy": "exact_match: 83.071", "offline_location": "./closed/GigaComputing/results/G894-AD3-AAX7_B300-SXM-270GBx8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/GigaComputing/results/G894-AD3-AAX7_B300-SXM-270GBx8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "177", "server_row_id": "178", "metadata": { "Organization": "GigaComputing", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "G894-AD3-AAX7", "Platform": "G894-AD3-AAX7_B300-SXM-270GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6979P", "host_processors_per_node": 2, "host_processor_core_count": "56", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 115034.0, "server_tokens_per_second": 114832.0, "server_over_offline": 0.9982439974268477, "server_over_offline_percent": 99.82439974268476, "difference_percent_of_offline": -0.17560025731523377, "offline_accuracy": "ROUGE1: 44.803 ROUGE2: 22.3802 ROUGEL: 29.1563 TOKENS_PER_SAMPLE: 272.6", "server_accuracy": "ROUGE1: 44.8381 ROUGE2: 22.394 ROUGEL: 29.1852 TOKENS_PER_SAMPLE: 272.5", "offline_location": "./closed/GigaComputing/results/G894-AD3-AAX7_B300-SXM-270GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/GigaComputing/results/G894-AD3-AAX7_B300-SXM-270GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "184", "server_row_id": "185", "metadata": { "Organization": "GigaComputing", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "G894-ZD3-AAX7", "Platform": "G894-ZD3-AAX7_B300-SXM-270GBx8_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 22.04.4", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 72327.1, "server_tokens_per_second": 70771.7, "server_over_offline": 0.9784949209908871, "server_over_offline_percent": 97.8494920990887, "difference_percent_of_offline": -2.1505079009112915, "offline_accuracy": "exact_match: 80.9936189608022 TOKENS_PER_SAMPLE: 3803.4314038286234", "server_accuracy": "exact_match: 80.76572470373746 TOKENS_PER_SAMPLE: 3851.8625797629898", "offline_location": "./closed/GigaComputing/results/G894-ZD3-AAX7_B300-SXM-270GBx8_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/GigaComputing/results/G894-ZD3-AAX7_B300-SXM-270GBx8_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "186", "server_row_id": "187", "metadata": { "Organization": "GigaComputing", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "G894-ZD3-AAX7", "Platform": "G894-ZD3-AAX7_B300-SXM-270GBx8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 22.04.4", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 111541.0, "server_tokens_per_second": 110592.0, "server_over_offline": 0.9914919177701473, "server_over_offline_percent": 99.14919177701474, "difference_percent_of_offline": -0.8508082229852665, "offline_accuracy": "exact_match: 82.928", "server_accuracy": "exact_match: 83.965", "offline_location": "./closed/GigaComputing/results/G894-ZD3-AAX7_B300-SXM-270GBx8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/GigaComputing/results/G894-ZD3-AAX7_B300-SXM-270GBx8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "188", "server_row_id": "189", "metadata": { "Organization": "GigaComputing", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "G894-ZD3-AAX7", "Platform": "G894-ZD3-AAX7_B300-SXM-270GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 22.04.4", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 114826.0, "server_tokens_per_second": 114578.0, "server_over_offline": 0.9978402104053089, "server_over_offline_percent": 99.78402104053089, "difference_percent_of_offline": -0.21597895946910972, "offline_accuracy": "ROUGE1: 44.8002 ROUGE2: 22.3763 ROUGEL: 29.1593 TOKENS_PER_SAMPLE: 272.5", "server_accuracy": "ROUGE1: 44.8817 ROUGE2: 22.4296 ROUGEL: 29.2316 TOKENS_PER_SAMPLE: 271.9", "offline_location": "./closed/GigaComputing/results/G894-ZD3-AAX7_B300-SXM-270GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/GigaComputing/results/G894-ZD3-AAX7_B300-SXM-270GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "196", "server_row_id": "197", "metadata": { "Organization": "Google", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)", "Platform": "GB200-NVL72_GB200-186GB_aarch64x72_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB200", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "PyTorch 2.10.0a0+b4e4ee81d3.nv25.12, TensorRT 10.14.1.48, CUDA Toolkit 13.1, cuDNN 9.17.0, TensorRT-LLM 1.3.0rc0", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB200 NVL72, 1200W TGP Per GPU. DeepSeek-R1 used the pinned aarch64 MLPerf TensorRT-LLM container. Runtime logs show MNNVL/NVLinkOneSided selection for Offline, Server, and Interactive; colocated Offline/Server also initialized the NIXL UCX backend, while disaggregated Interactive used a UCX KV-cache transceiver between context and generation services.", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 510781.0, "server_tokens_per_second": 388590.0, "server_over_offline": 0.7607761447665438, "server_over_offline_percent": 76.07761447665438, "difference_percent_of_offline": -23.922385523345625, "offline_accuracy": "exact_match: 81.19872379216044 TOKENS_PER_SAMPLE: 3852.355970829535", "server_accuracy": "exact_match: 81.01640838650866 TOKENS_PER_SAMPLE: 3795.0934366453967", "offline_location": "./closed/Google/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Google/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "198", "server_row_id": "199", "metadata": { "Organization": "HPE", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB200 NVL72 by HPE (8x GB200-186GB_aarch64, TensorRT)", "Platform": "HPE_GB200-NVL72_GB200-186GB_aarch64x8", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 2, "host_processor_model_name": "NVIDIA Grace CPU (Arm Neoverse-V2)", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB200", "accelerators_per_node": 4, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, TensorRT-LLM 1.3.0rc0", "operating_system": "Ubuntu 24.04.4 LTS, Linux 6.14.0-37-generic-64k", "notes": "NVIDIA GB200 NVL72 platform by HPE; 1200 W GPU power limit. Two dep8 replicas across two nodes; CentML DeepSeek-R1 NVFP4 v2 checkpoint", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 101453.0, "server_tokens_per_second": 100714.0, "server_over_offline": 0.9927158388613446, "server_over_offline_percent": 99.27158388613447, "difference_percent_of_offline": -0.7284161138655376, "offline_accuracy": "exact_match: 83.727", "server_accuracy": "exact_match: 83.485", "offline_location": "./closed/HPE/results/HPE_GB200-NVL72_GB200-186GB_aarch64x8/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/HPE/results/HPE_GB200-NVL72_GB200-186GB_aarch64x8/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "206", "server_row_id": "207", "metadata": { "Organization": "HPE", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "HPE ProLiant Compute DL380a Gen12 (8x RTX Pro 6000 Blackwell Server Edition 96GB, TensorRT)", "Platform": "HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6787P", "host_processors_per_node": 2, "host_processor_core_count": "172", "accelerator_model_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.16.1.11, CUDA 13.2", "operating_system": "Ubuntu 24.04.4 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 15256.0, "server_tokens_per_second": 14581.7, "server_over_offline": 0.9558009963293131, "server_over_offline_percent": 95.58009963293131, "difference_percent_of_offline": -4.419900367068685, "offline_accuracy": "exact_match: 82.714", "server_accuracy": "exact_match: 82.341", "offline_location": "./closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "208", "server_row_id": "209", "metadata": { "Organization": "HPE", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "HPE ProLiant Compute DL380a Gen12 (8x RTX Pro 6000 Blackwell Server Edition 96GB, TensorRT)", "Platform": "HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6787P", "host_processors_per_node": 2, "host_processor_core_count": "172", "accelerator_model_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.16.1.11, CUDA 13.2", "operating_system": "Ubuntu 24.04.4 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 29284.8, "server_tokens_per_second": 28643.3, "server_over_offline": 0.9780944380702618, "server_over_offline_percent": 97.80944380702617, "difference_percent_of_offline": -2.190556192973825, "offline_accuracy": "ROUGE1: 44.7216 ROUGE2: 22.3081 ROUGEL: 29.0908 TOKENS_PER_SAMPLE: 274.8", "server_accuracy": "ROUGE1: 44.7706 ROUGE2: 22.352 ROUGEL: 29.1377 TOKENS_PER_SAMPLE: 272.0", "offline_location": "./closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "211", "server_row_id": "212", "metadata": { "Organization": "HPE", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "HPE ProLiant Compute DL380a Gen12 (8x RTX Pro 6000 Blackwell Server Edition 96GB, TensorRT)", "Platform": "HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6787P", "host_processors_per_node": 2, "host_processor_core_count": "172", "accelerator_model_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.16.1.11, CUDA 13.2", "operating_system": "Ubuntu 24.04.4 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 50965.3, "server_tokens_per_second": 49523.3, "server_over_offline": 0.9717062393432394, "server_over_offline_percent": 97.17062393432394, "difference_percent_of_offline": -2.829376065676059, "offline_accuracy": "ROUGE1: 38.7974 ROUGE2: 16.0868 ROUGEL: 24.5879 ROUGELSUM: 35.7199 GEN_LEN: 8172782", "server_accuracy": "ROUGE1: 38.7821 ROUGE2: 16.0772 ROUGEL: 24.5857 ROUGELSUM: 35.7074 GEN_LEN: 8172428", "offline_location": "./closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "218", "server_row_id": "219", "metadata": { "Organization": "HPE", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "HPE ProLiant Compute XD685 (8x AMD MI355X-288GB, vLLM)", "Platform": "HPE_XD685_MI355X_288GBx8_vLLM", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0, ROCm 7.2.2", "operating_system": "Ubuntu 22.04.5 LTS (kernel 5.15.0-186-generic)", "notes": "HPE ProLiant Compute XD685; 8x AMD Instinct MI355X (gfx950), 288GB HBM3E each. MXFP4 (W4A4) weights with FP8 KV cache, Quark quantization", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 117355.0, "server_tokens_per_second": 108836.0, "server_over_offline": 0.9274082910826126, "server_over_offline_percent": 92.74082910826125, "difference_percent_of_offline": -7.2591708917387425, "offline_accuracy": "exact_match: 83.687", "server_accuracy": "exact_match: 83.864", "offline_location": "./closed/HPE/results/HPE_XD685_MI355X_288GBx8_vLLM/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/HPE/results/HPE_XD685_MI355X_288GBx8_vLLM/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "221", "server_row_id": "222", "metadata": { "Organization": "HPE", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "HPE ProLiant Compute XD685 (8x AMD MI355X-288GB, vLLM)", "Platform": "HPE_XD685_MI355X_288GBx8_vLLM", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0, ROCm 7.2.2", "operating_system": "Ubuntu 22.04.5 LTS (kernel 5.15.0-186-generic)", "notes": "HPE ProLiant Compute XD685; 8x AMD Instinct MI355X (gfx950), 288GB HBM3E each. MXFP4 (W4A4) weights with FP8 KV cache, Quark quantization", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 102991.0, "server_tokens_per_second": 101979.0, "server_over_offline": 0.9901738986901768, "server_over_offline_percent": 99.01738986901768, "difference_percent_of_offline": -0.9826101309823199, "offline_accuracy": "ROUGE1: 44.5406 ROUGE2: 22.1922 ROUGEL: 28.9322 TOKENS_PER_SAMPLE: 284.2", "server_accuracy": "ROUGE1: 44.5325 ROUGE2: 22.1714 ROUGEL: 28.9308 TOKENS_PER_SAMPLE: 284.1", "offline_location": "./closed/HPE/results/HPE_XD685_MI355X_288GBx8_vLLM/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/HPE/results/HPE_XD685_MI355X_288GBx8_vLLM/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "224", "server_row_id": "225", "metadata": { "Organization": "HPE", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "HPE ProLiant Compute XD685 (8x AMD MI355X-288GB, vLLM)", "Platform": "HPE_XD685_MI355X_288GBx8_vLLM", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0, ROCm 7.2.2", "operating_system": "Ubuntu 22.04.5 LTS (kernel 5.15.0-186-generic)", "notes": "HPE ProLiant Compute XD685; 8x AMD Instinct MI355X (gfx950), 288GB HBM3E each. MXFP4 (W4A4) weights with FP8 KV cache, Quark quantization", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 157694.0, "server_tokens_per_second": 148118.0, "server_over_offline": 0.9392747980265578, "server_over_offline_percent": 93.92747980265578, "difference_percent_of_offline": -6.072520197344222, "offline_accuracy": "ROUGE1: 38.5626 ROUGE2: 16.0276 ROUGEL: 24.4431 ROUGELSUM: 35.699 GEN_LEN: 8151086", "server_accuracy": "ROUGE1: 38.5316 ROUGE2: 16.0159 ROUGEL: 24.4259 ROUGELSUM: 35.6716 GEN_LEN: 8162750", "offline_location": "./closed/HPE/results/HPE_XD685_MI355X_288GBx8_vLLM/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/HPE/results/HPE_XD685_MI355X_288GBx8_vLLM/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "228", "server_row_id": "229", "metadata": { "Organization": "HPE", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "HPE Compute XD690 (8x NVIDIA B300-SXM-270GB, TensorRT)", "Platform": "HPE_XD690_B300_SXM_270GBx16_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 2, "host_processor_model_name": "Intel(R) Xeon(R) 6776P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 16, "framework": "TensorRT 10.14, CUDA 13.2", "operating_system": "RHEL 9.4", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 136854.0, "server_tokens_per_second": 80536.9, "server_over_offline": 0.5884877314510354, "server_over_offline_percent": 58.848773145103536, "difference_percent_of_offline": -41.151226854896464, "offline_accuracy": "exact_match: 81.24430264357338 TOKENS_PER_SAMPLE: 3826.166362807657", "server_accuracy": "exact_match: 80.92525068368278 TOKENS_PER_SAMPLE: 3807.0186873290795", "offline_location": "./closed/HPE/results/HPE_XD690_B300_SXM_270GBx16_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/HPE/results/HPE_XD690_B300_SXM_270GBx16_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "230", "server_row_id": "231", "metadata": { "Organization": "HPE", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "HPE Compute XD690 (8x NVIDIA B300-SXM-270GB, TensorRT)", "Platform": "HPE_XD690_B300_SXM_270GBx8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6776P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.2", "operating_system": "RHEL 9.4", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 114569.0, "server_tokens_per_second": 113685.0, "server_over_offline": 0.9922841257233632, "server_over_offline_percent": 99.22841257233632, "difference_percent_of_offline": -0.771587427663678, "offline_accuracy": "exact_match: 83.235", "server_accuracy": "exact_match: 83.448", "offline_location": "./closed/HPE/results/HPE_XD690_B300_SXM_270GBx8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/HPE/results/HPE_XD690_B300_SXM_270GBx8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "232", "server_row_id": "233", "metadata": { "Organization": "Intel", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "1-node-2S-GNR_128C", "Platform": "1-node-2S-GNR_128C", "Model": "llama3_1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6980P", "host_processors_per_node": 2, "host_processor_core_count": "128", "accelerator_model_name": null, "accelerators_per_node": 0, "total_accelerators": 0, "framework": "PyTorch", "operating_system": "CentOS Stream 9", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "INT8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 1916.74, "server_tokens_per_second": 1139.51, "server_over_offline": 0.5945042102736938, "server_over_offline_percent": 59.45042102736938, "difference_percent_of_offline": -40.54957897263062, "offline_accuracy": "ROUGE1: 38.6418 ROUGE2: 15.8693 ROUGEL: 24.4809 ROUGELSUM: 35.6703 GEN_LEN: 8194131", "server_accuracy": "ROUGE1: 38.6681 ROUGE2: 15.9242 ROUGEL: 24.5144 ROUGELSUM: 35.7046 GEN_LEN: 8206137", "offline_location": "./closed/Intel/results/1-node-2S-GNR_128C/llama3_1-8b/Offline/performance/run_1", "server_location": "./closed/Intel/results/1-node-2S-GNR_128C/llama3_1-8b/Server/performance/run_1" }, { "offline_row_id": "238", "server_row_id": "239", "metadata": { "Organization": "Intel", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "1-node-4x-BMG-B70", "Platform": "1-node-4x-BMG-B70", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 698X", "host_processors_per_node": 1, "host_processor_core_count": "86", "accelerator_model_name": "Intel(R) Arc(TM) Pro B70", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "PyTorch", "operating_system": "Ubuntu 24.04.4 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "MXFP4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 1956.4, "server_tokens_per_second": 1296.87, "server_over_offline": 0.6628859129012471, "server_over_offline_percent": 66.28859129012471, "difference_percent_of_offline": -33.71140870987529, "offline_accuracy": "exact_match: 83.674", "server_accuracy": "exact_match: 83.489", "offline_location": "./closed/Intel/results/1-node-4x-BMG-B70/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Intel/results/1-node-4x-BMG-B70/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "240", "server_row_id": "241", "metadata": { "Organization": "Intel", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "1-node-4x-BMG-B70", "Platform": "1-node-4x-BMG-B70", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 698X", "host_processors_per_node": 1, "host_processor_core_count": "86", "accelerator_model_name": "Intel(R) Arc(TM) Pro B70", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "PyTorch", "operating_system": "Ubuntu 24.04.4 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "INT8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 2464.98, "server_tokens_per_second": 1745.48, "server_over_offline": 0.7081112219977445, "server_over_offline_percent": 70.81112219977445, "difference_percent_of_offline": -29.188877800225555, "offline_accuracy": "ROUGE1: 44.5947 ROUGE2: 22.2074 ROUGEL: 28.8264 TOKENS_PER_SAMPLE: 290.8", "server_accuracy": "ROUGE1: 44.5937 ROUGE2: 22.2052 ROUGEL: 28.8276 TOKENS_PER_SAMPLE: 290.8", "offline_location": "./closed/Intel/results/1-node-4x-BMG-B70/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Intel/results/1-node-4x-BMG-B70/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "242", "server_row_id": "243", "metadata": { "Organization": "Intel", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "1-node-4x-BMG-B70", "Platform": "1-node-4x-BMG-B70", "Model": "llama3_1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 698X", "host_processors_per_node": 1, "host_processor_core_count": "86", "accelerator_model_name": "Intel(R) Arc(TM) Pro B70", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "PyTorch", "operating_system": "Ubuntu 24.04.4 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "INT8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 4613.28, "server_tokens_per_second": 4173.25, "server_over_offline": 0.904616671869039, "server_over_offline_percent": 90.46166718690391, "difference_percent_of_offline": -9.538332813096096, "offline_accuracy": "ROUGE1: 38.843 ROUGE2: 16.0878 ROUGEL: 24.6429 ROUGELSUM: 35.9449 GEN_LEN: 8127480", "server_accuracy": "ROUGE1: 38.8131 ROUGE2: 16.0827 ROUGEL: 24.6388 ROUGELSUM: 35.9153 GEN_LEN: 8126165", "offline_location": "./closed/Intel/results/1-node-4x-BMG-B70/llama3_1-8b/Offline/performance/run_1", "server_location": "./closed/Intel/results/1-node-4x-BMG-B70/llama3_1-8b/Server/performance/run_1" }, { "offline_row_id": "245", "server_row_id": "246", "metadata": { "Organization": "Inventec", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Inventec P5800G7(LC) (8x B300-SXM-270GB, TensorRT)", "Platform": "P5800G7-B300-SXM-270GBx8_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6787P", "host_processors_per_node": 2, "host_processor_core_count": "86", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 70269.6, "server_tokens_per_second": 57432.0, "server_over_offline": 0.8173093343351889, "server_over_offline_percent": 81.7309334335189, "difference_percent_of_offline": -18.269066566481108, "offline_accuracy": "exact_match: 81.22151321786691 TOKENS_PER_SAMPLE: 3748.471741112124", "server_accuracy": "exact_match: 81.10756608933455 TOKENS_PER_SAMPLE: 3769.7967183226983", "offline_location": "./closed/Inventec/results/P5800G7-B300-SXM-270GBx8_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Inventec/results/P5800G7-B300-SXM-270GBx8_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "247", "server_row_id": "248", "metadata": { "Organization": "Inventec", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Inventec P5800G7(LC) (8x B300-SXM-270GB, TensorRT)", "Platform": "P5800G7-B300-SXM-270GBx8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6787P", "host_processors_per_node": 2, "host_processor_core_count": "86", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 106280.0, "server_tokens_per_second": 100299.0, "server_over_offline": 0.9437241249529544, "server_over_offline_percent": 94.37241249529545, "difference_percent_of_offline": -5.6275875047045565, "offline_accuracy": "exact_match: 83.856", "server_accuracy": "exact_match: 83.423", "offline_location": "./closed/Inventec/results/P5800G7-B300-SXM-270GBx8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Inventec/results/P5800G7-B300-SXM-270GBx8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "249", "server_row_id": "250", "metadata": { "Organization": "Inventec", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Inventec P5800G7(LC) (8x B300-SXM-270GB, TensorRT)", "Platform": "P5800G7-B300-SXM-270GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6787P", "host_processors_per_node": 2, "host_processor_core_count": "86", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 109933.0, "server_tokens_per_second": 106617.0, "server_over_offline": 0.9698361729417009, "server_over_offline_percent": 96.98361729417009, "difference_percent_of_offline": -3.016382705829912, "offline_accuracy": "ROUGE1: 44.7871 ROUGE2: 22.3677 ROUGEL: 29.1429 TOKENS_PER_SAMPLE: 272.9", "server_accuracy": "ROUGE1: 44.8264 ROUGE2: 22.3815 ROUGEL: 29.1549 TOKENS_PER_SAMPLE: 272.1", "offline_location": "./closed/Inventec/results/P5800G7-B300-SXM-270GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Inventec/results/P5800G7-B300-SXM-270GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "251", "server_row_id": "252", "metadata": { "Organization": "Inventec", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Inventec P9000G7(AC) (8x B300-SXM-270GB, TensorRT)", "Platform": "P9000G7-B300-SXM-270GBx8_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6767P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 69062.2, "server_tokens_per_second": 57517.4, "server_over_offline": 0.8328347489654254, "server_over_offline_percent": 83.28347489654254, "difference_percent_of_offline": -16.716525103457457, "offline_accuracy": "exact_match: 81.13035551504102 TOKENS_PER_SAMPLE: 3743.316089334549", "server_accuracy": "exact_match: 80.62898814949864 TOKENS_PER_SAMPLE: 3774.6118960802187", "offline_location": "./closed/Inventec/results/P9000G7-B300-SXM-270GBx8_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Inventec/results/P9000G7-B300-SXM-270GBx8_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "253", "server_row_id": "254", "metadata": { "Organization": "Inventec", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Inventec P9000G7(AC) (8x B300-SXM-270GB, TensorRT)", "Platform": "P9000G7-B300-SXM-270GBx8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6767P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 105617.0, "server_tokens_per_second": 100300.0, "server_over_offline": 0.9496577255555451, "server_over_offline_percent": 94.96577255555451, "difference_percent_of_offline": -5.034227444445493, "offline_accuracy": "exact_match: 83.550", "server_accuracy": "exact_match: 83.799", "offline_location": "./closed/Inventec/results/P9000G7-B300-SXM-270GBx8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Inventec/results/P9000G7-B300-SXM-270GBx8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "255", "server_row_id": "256", "metadata": { "Organization": "Inventec", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Inventec P9000G7(AC) (8x B300-SXM-270GB, TensorRT)", "Platform": "P9000G7-B300-SXM-270GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6767P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 109755.0, "server_tokens_per_second": 106645.0, "server_over_offline": 0.9716641610860554, "server_over_offline_percent": 97.16641610860553, "difference_percent_of_offline": -2.8335838913944644, "offline_accuracy": "ROUGE1: 44.7774 ROUGE2: 22.365 ROUGEL: 29.1457 TOKENS_PER_SAMPLE: 272.9", "server_accuracy": "ROUGE1: 44.8297 ROUGE2: 22.3857 ROUGEL: 29.1786 TOKENS_PER_SAMPLE: 272.2", "offline_location": "./closed/Inventec/results/P9000G7-B300-SXM-270GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Inventec/results/P9000G7-B300-SXM-270GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "259", "server_row_id": "260", "metadata": { "Organization": "Lambda", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Lambda 4x GB300-288GB (GB300 NVL4, aarch64)", "Platform": "GB300-NVL4_GB300-288GB_aarch64x4", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "NVIDIA Grace CPU (Arm Neoverse-V2)", "host_processors_per_node": 2, "host_processor_core_count": "72", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "TensorRT-LLM 1.3.0rc14 (GPT-OSS-120B); vLLM (CentML) 0.20.2rc1.dev2091+g2e1040b7e + NVIDIA Dynamo (Qwen3-VL-235B); CUDA 13.0", "operating_system": "Ubuntu 24.04.4 LTS (aarch64)", "notes": "Single GB300 NVL4 node (Supermicro PIO-FRU-121GL-NB3-LCC-R1): 2x NVIDIA Grace CPU (72 cores each) + 4x NVIDIA GB300 288GB HBM3e, 1400W TGP per GPU, direct all-to-all 5th-gen NVLink (no NVLink Switch). GPU HBM is exposed as coherent NUMA memory alongside the 960GB of Grace LPDDR5X.. GPT-OSS-120B served with TensorRT-LLM (trtllm-serve, in-flight batching) as 4 data-parallel replicas (one GPU per replica) behind the MLPerf LoadGen harness. Qwen3-VL-235B-A22B served with vLLM + NVIDIA Dynamo as 4 data-parallel replicas (TP=1, one GPU per replica) and benchmarked with the NVIDIA/MLCommons inference-endpoints client; endpoint result format. Both workflows orchestrated by nv-sflow on Slurm with Pyxis/Enroot containers.", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "mxfp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 65511.9, "server_tokens_per_second": 58195.8, "server_over_offline": 0.8883241060021156, "server_over_offline_percent": 88.83241060021156, "difference_percent_of_offline": -11.167589399788437, "offline_accuracy": "exact_match: 82.738", "server_accuracy": "exact_match: 83.079", "offline_location": "./closed/Lambda/results/GB300-NVL4_GB300-288GB_aarch64x4/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Lambda/results/GB300-NVL4_GB300-288GB_aarch64x4/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "265", "server_row_id": "266", "metadata": { "Organization": "MangoBoost", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "8xMI300X_2xEPYC_9534", "Platform": "8xMI300X_2xEPYC_9534", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9534", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "AMD Instinct MI300X 192GB HBM3", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.24.0 (stock vllm/vllm-openai-rocm:v0.24.0), ROCm 7.2.1", "operating_system": "Ubuntu 22.04.5 LTS", "notes": "8xMI300X_2xEPYC_9534. Self-quantized w-fp8-a-fp8 gpt-oss-120b (AMD Quark 0.12) on stock vLLM 0.24.0 (vllm/vllm-openai-rocm:v0.24.0).", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 30198.4, "server_tokens_per_second": 24863.0, "server_over_offline": 0.8233217653915439, "server_over_offline_percent": 82.33217653915439, "difference_percent_of_offline": -17.66782346084561, "offline_accuracy": "exact_match: 83.713", "server_accuracy": "exact_match: 82.842", "offline_location": "./closed/MangoBoost/results/8xMI300X_2xEPYC_9534/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/MangoBoost/results/8xMI300X_2xEPYC_9534/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "267", "server_row_id": "268", "metadata": { "Organization": "MiTAC", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "G8825Z5", "Platform": "8xMI350X_2xEPYC-9755", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9755", "host_processors_per_node": 2, "host_processor_core_count": "128", "accelerator_model_name": "AMD Instinct MI350X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "Pytorch 2.10, ROCm 7.2.2", "operating_system": "Ubuntu 24.04.4 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 97019.7, "server_tokens_per_second": 87549.5, "server_over_offline": 0.9023888962757048, "server_over_offline_percent": 90.23888962757049, "difference_percent_of_offline": -9.761110372429515, "offline_accuracy": "exact_match: 83.639", "server_accuracy": "exact_match: 83.316", "offline_location": "./closed/MiTAC/results/8xMI350X_2xEPYC-9755/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/MiTAC/results/8xMI350X_2xEPYC-9755/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "270", "server_row_id": "271", "metadata": { "Organization": "MiTAC", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "G8825Z5", "Platform": "8xMI350X_2xEPYC-9755", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9755", "host_processors_per_node": 2, "host_processor_core_count": "128", "accelerator_model_name": "AMD Instinct MI350X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "Pytorch 2.10, ROCm 7.2.2", "operating_system": "Ubuntu 24.04.4 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 84016.4, "server_tokens_per_second": 85004.8, "server_over_offline": 1.0117643698135128, "server_over_offline_percent": 101.17643698135129, "difference_percent_of_offline": 1.176436981351281, "offline_accuracy": "ROUGE1: 44.4551 ROUGE2: 22.0787 ROUGEL: 28.8207 TOKENS_PER_SAMPLE: 283.0", "server_accuracy": "ROUGE1: 44.5423 ROUGE2: 22.1706 ROUGEL: 28.9236 TOKENS_PER_SAMPLE: 284.5", "offline_location": "./closed/MiTAC/results/8xMI350X_2xEPYC-9755/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/MiTAC/results/8xMI350X_2xEPYC-9755/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "273", "server_row_id": "274", "metadata": { "Organization": "MiTAC", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "G8825Z5", "Platform": "8xMI350X_2xEPYC-9755", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9755", "host_processors_per_node": 2, "host_processor_core_count": "128", "accelerator_model_name": "AMD Instinct MI350X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "Pytorch 2.10, ROCm 7.2.2", "operating_system": "Ubuntu 24.04.4 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 129943.0, "server_tokens_per_second": 121706.0, "server_over_offline": 0.936610667754323, "server_over_offline_percent": 93.6610667754323, "difference_percent_of_offline": -6.3389332245677, "offline_accuracy": "ROUGE1: 38.5694 ROUGE2: 16.0367 ROUGEL: 24.4456 ROUGELSUM: 35.7078 GEN_LEN: 8150329", "server_accuracy": "ROUGE1: 38.5208 ROUGE2: 16.0104 ROUGEL: 24.424 ROUGELSUM: 35.666 GEN_LEN: 8168845", "offline_location": "./closed/MiTAC/results/8xMI350X_2xEPYC-9755/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/MiTAC/results/8xMI350X_2xEPYC-9755/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "275", "server_row_id": "276", "metadata": { "Organization": "MiTAC", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "G4826", "Platform": "8xMI355X_2xEPYC-9755", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9755", "host_processors_per_node": 2, "host_processor_core_count": "128", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0+git8514f05, ROCm 7.2.53211", "operating_system": "Ubuntu 24.04.4 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 118042.0, "server_tokens_per_second": 108861.0, "server_over_offline": 0.9222225987360431, "server_over_offline_percent": 92.2222598736043, "difference_percent_of_offline": -7.777740126395694, "offline_accuracy": "exact_match: 83.861", "server_accuracy": "exact_match: 84.039", "offline_location": "./closed/MiTAC/results/8xMI355X_2xEPYC-9755/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/MiTAC/results/8xMI355X_2xEPYC-9755/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "278", "server_row_id": "279", "metadata": { "Organization": "MiTAC", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "G4826", "Platform": "8xMI355X_2xEPYC-9755", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9755", "host_processors_per_node": 2, "host_processor_core_count": "128", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0+git8514f05, ROCm 7.2.53211", "operating_system": "Ubuntu 24.04.4 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 104904.0, "server_tokens_per_second": 104541.0, "server_over_offline": 0.9965396934339968, "server_over_offline_percent": 99.65396934339968, "difference_percent_of_offline": -0.34603065660031573, "offline_accuracy": "ROUGE1: 44.5341 ROUGE2: 22.1837 ROUGEL: 28.9253 TOKENS_PER_SAMPLE: 284.3", "server_accuracy": "ROUGE1: 44.5342 ROUGE2: 22.1734 ROUGEL: 28.9337 TOKENS_PER_SAMPLE: 284.0", "offline_location": "./closed/MiTAC/results/8xMI355X_2xEPYC-9755/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/MiTAC/results/8xMI355X_2xEPYC-9755/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "281", "server_row_id": "282", "metadata": { "Organization": "MiTAC", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "G4826", "Platform": "8xMI355X_2xEPYC-9755", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9755", "host_processors_per_node": 2, "host_processor_core_count": "128", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0+git8514f05, ROCm 7.2.53211", "operating_system": "Ubuntu 24.04.4 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 159278.0, "server_tokens_per_second": 151116.0, "server_over_offline": 0.9487562626351411, "server_over_offline_percent": 94.8756262635141, "difference_percent_of_offline": -5.124373736485888, "offline_accuracy": "ROUGE1: 38.5554 ROUGE2: 16.0227 ROUGEL: 24.4331 ROUGELSUM: 35.6963 GEN_LEN: 8150276", "server_accuracy": "ROUGE1: 38.5121 ROUGE2: 15.9993 ROUGEL: 24.4128 ROUGELSUM: 35.6518 GEN_LEN: 8170944", "offline_location": "./closed/MiTAC/results/8xMI355X_2xEPYC-9755/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/MiTAC/results/8xMI355X_2xEPYC-9755/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "283", "server_row_id": "284", "metadata": { "Organization": "MiTAC", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "MiTAC_G4520G6_RTX_PRO_6000_PCIE_96GBx8", "Platform": "MiTAC_G4520G6_RTX_PRO_6000_PCIE_96GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6787P", "host_processors_per_node": 2, "host_processor_core_count": "172", "accelerator_model_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.2", "operating_system": "Ubuntu 24.04.4", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 30034.2, "server_tokens_per_second": 29203.3, "server_over_offline": 0.9723348715797324, "server_over_offline_percent": 97.23348715797324, "difference_percent_of_offline": -2.766512842026758, "offline_accuracy": "ROUGE1: 44.7186 ROUGE2: 22.3366 ROUGEL: 29.1057 TOKENS_PER_SAMPLE: 275.5", "server_accuracy": "ROUGE1: 44.8047 ROUGE2: 22.3347 ROUGEL: 29.1315 TOKENS_PER_SAMPLE: 267.8", "offline_location": "./closed/MiTAC/results/MiTAC_G4520G6_RTX_PRO_6000_PCIE_96GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/MiTAC/results/MiTAC_G4520G6_RTX_PRO_6000_PCIE_96GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "285", "server_row_id": "286", "metadata": { "Organization": "NVIDIA", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)", "Platform": "B300-SXM-270GBx8_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8570", "host_processors_per_node": 2, "host_processor_core_count": "56", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 70132.3, "server_tokens_per_second": 59867.8, "server_over_offline": 0.8536409044049603, "server_over_offline_percent": 85.36409044049603, "difference_percent_of_offline": -14.635909559503968, "offline_accuracy": "exact_match: 80.78851412944394 TOKENS_PER_SAMPLE: 3793.9359617137648", "server_accuracy": "exact_match: 81.17593436645396 TOKENS_PER_SAMPLE: 3818.7203737465816", "offline_location": "./closed/NVIDIA/results/B300-SXM-270GBx8_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/NVIDIA/results/B300-SXM-270GBx8_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "287", "server_row_id": "288", "metadata": { "Organization": "NVIDIA", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)", "Platform": "B300-SXM-270GBx8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8570", "host_processors_per_node": 2, "host_processor_core_count": "56", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 106637.0, "server_tokens_per_second": 100630.0, "server_over_offline": 0.9436687078593734, "server_over_offline_percent": 94.36687078593734, "difference_percent_of_offline": -5.633129214062659, "offline_accuracy": "exact_match: 83.430", "server_accuracy": "exact_match: 83.702", "offline_location": "./closed/NVIDIA/results/B300-SXM-270GBx8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/NVIDIA/results/B300-SXM-270GBx8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "289", "server_row_id": "290", "metadata": { "Organization": "NVIDIA", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)", "Platform": "B300-SXM-270GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8570", "host_processors_per_node": 2, "host_processor_core_count": "56", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 109277.0, "server_tokens_per_second": 106949.0, "server_over_offline": 0.9786963404925099, "server_over_offline_percent": 97.86963404925099, "difference_percent_of_offline": -2.1303659507490136, "offline_accuracy": "ROUGE1: 44.7951 ROUGE2: 22.3759 ROUGEL: 29.1534 TOKENS_PER_SAMPLE: 272.8", "server_accuracy": "ROUGE1: 44.8452 ROUGE2: 22.4177 ROUGEL: 29.2149 TOKENS_PER_SAMPLE: 272.7", "offline_location": "./closed/NVIDIA/results/B300-SXM-270GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/NVIDIA/results/B300-SXM-270GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "297", "server_row_id": "298", "metadata": { "Organization": "NVIDIA", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)", "Platform": "GB200-NVL72_GB200-186GB_aarch64x72_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB200", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB200 NVL72, 1200W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 523790.0, "server_tokens_per_second": 395940.0, "server_over_offline": 0.7559136295080089, "server_over_offline_percent": 75.59136295080089, "difference_percent_of_offline": -24.40863704919911, "offline_accuracy": "exact_match: 81.0619872379216 TOKENS_PER_SAMPLE: 3795.436417502279", "server_accuracy": "exact_match: 80.78851412944394 TOKENS_PER_SAMPLE: 3823.4881494986325", "offline_location": "./closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "300", "server_row_id": "301", "metadata": { "Organization": "NVIDIA", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)", "Platform": "GB200-NVL72_GB200-186GB_aarch64x72_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB200", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB200 NVL72, 1200W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 883761.0, "server_tokens_per_second": 899005.0, "server_over_offline": 1.0172490073673766, "server_over_offline_percent": 101.72490073673765, "difference_percent_of_offline": 1.7249007367376556, "offline_accuracy": "exact_match: 84.038", "server_accuracy": "exact_match: 83.914", "offline_location": "./closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "304", "server_row_id": "305", "metadata": { "Organization": "NVIDIA", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x288_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 72, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 288, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 2705130.0, "server_tokens_per_second": 2028030.0, "server_over_offline": 0.749697796409045, "server_over_offline_percent": 74.9697796409045, "difference_percent_of_offline": -25.0302203590955, "offline_accuracy": "exact_match: 81.19872379216044 TOKENS_PER_SAMPLE: 3809.0271194165907", "server_accuracy": "exact_match: 81.13035551504102 TOKENS_PER_SAMPLE: 3861.4316317228804", "offline_location": "./closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x288_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x288_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "307", "server_row_id": "308", "metadata": { "Organization": "NVIDIA", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x72_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72, 1400W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 679740.0, "server_tokens_per_second": 596944.0, "server_over_offline": 0.8781946038191073, "server_over_offline_percent": 87.81946038191073, "difference_percent_of_offline": -12.18053961808927, "offline_accuracy": "exact_match: 80.9936189608022 TOKENS_PER_SAMPLE: 3800.5963992707384", "server_accuracy": "exact_match: 81.42661804922516 TOKENS_PER_SAMPLE: 3855.550136736554", "offline_location": "./closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "310", "server_row_id": "311", "metadata": { "Organization": "NVIDIA", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x72_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72, 1400W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 1124500.0, "server_tokens_per_second": 1120710.0, "server_over_offline": 0.9966296131614051, "server_over_offline_percent": 99.66296131614051, "difference_percent_of_offline": -0.33703868385949187, "offline_accuracy": "exact_match: 82.861", "server_accuracy": "exact_match: 83.069", "offline_location": "./closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "317", "server_row_id": "318", "metadata": { "Organization": "NVIDIA", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB300 NVL72 (8x GB300-288GB_aarch64, TensorRT)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x8_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 2, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72, 1400W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 75918.2, "server_tokens_per_second": 67578.3, "server_over_offline": 0.8901462363438544, "server_over_offline_percent": 89.01462363438543, "difference_percent_of_offline": -10.985376365614563, "offline_accuracy": "exact_match: 81.13035551504102 TOKENS_PER_SAMPLE: 3843.4847310847767", "server_accuracy": "exact_match: 80.78851412944394 TOKENS_PER_SAMPLE: 3829.014357338195", "offline_location": "./closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x8_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x8_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "319", "server_row_id": "320", "metadata": { "Organization": "NVIDIA", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB300 NVL72 (8x GB300-288GB_aarch64, TensorRT)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 2, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72, 1400W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 131619.0, "server_tokens_per_second": 128173.0, "server_over_offline": 0.9738183696882668, "server_over_offline_percent": 97.38183696882669, "difference_percent_of_offline": -2.618163031173315, "offline_accuracy": "exact_match: 82.963", "server_accuracy": "exact_match: 83.020", "offline_location": "./closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "321", "server_row_id": "322", "metadata": { "Organization": "NVIDIA", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB300 NVL72 (8x GB300-288GB_aarch64, TensorRT)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 2, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.1", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72, 1400W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 131464.0, "server_tokens_per_second": 127911.0, "server_over_offline": 0.9729735897279864, "server_over_offline_percent": 97.29735897279865, "difference_percent_of_offline": -2.702641027201358, "offline_accuracy": "ROUGE1: 44.8017 ROUGE2: 22.3794 ROUGEL: 29.1616 TOKENS_PER_SAMPLE: 272.6", "server_accuracy": "ROUGE1: 44.8324 ROUGE2: 22.3953 ROUGEL: 29.1865 TOKENS_PER_SAMPLE: 272.1", "offline_location": "./closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "332", "server_row_id": "333", "metadata": { "Organization": "Nebius", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Nebius B200 n1 (8x B200-SXM-180GB, TensorRT)", "Platform": "B200-SXM-180GBx8_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8580", "host_processors_per_node": 2, "host_processor_core_count": "60", "accelerator_model_name": "NVIDIA B200-SXM-180GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04.3 LTS (Linux kernel: 6.11.0-1016-nvidia #16-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 21 17:06:33 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux)", "notes": "B200 TGP 1000W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 56929.9, "server_tokens_per_second": 55869.0, "server_over_offline": 0.9813648012731446, "server_over_offline_percent": 98.13648012731446, "difference_percent_of_offline": -1.8635198726855373, "offline_accuracy": "exact_match: 81.08477666362808 TOKENS_PER_SAMPLE: 3808.5396536007293", "server_accuracy": "exact_match: 80.62898814949864 TOKENS_PER_SAMPLE: 3840.4710574293526", "offline_location": "./closed/Nebius/results/B200-SXM-180GBx8_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Nebius/results/B200-SXM-180GBx8_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "334", "server_row_id": "335", "metadata": { "Organization": "Nebius", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Nebius B200 n1 (8x B200-SXM-180GB, TensorRT)", "Platform": "B200-SXM-180GBx8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8580", "host_processors_per_node": 2, "host_processor_core_count": "60", "accelerator_model_name": "NVIDIA B200-SXM-180GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04.3 LTS (Linux kernel: 6.11.0-1016-nvidia #16-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 21 17:06:33 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux)", "notes": "B200 TGP 1000W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 88491.0, "server_tokens_per_second": 89856.3, "server_over_offline": 1.0154286876631522, "server_over_offline_percent": 101.54286876631522, "difference_percent_of_offline": 1.5428687663152196, "offline_accuracy": "exact_match: 83.040", "server_accuracy": "exact_match: 82.998", "offline_location": "./closed/Nebius/results/B200-SXM-180GBx8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Nebius/results/B200-SXM-180GBx8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "336", "server_row_id": "337", "metadata": { "Organization": "Nebius", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Nebius B300 n1 (8x B300-SXM-270GB, TensorRT)", "Platform": "B300-SXM-270GBx8_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8570", "host_processors_per_node": 2, "host_processor_core_count": "48", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 69655.1, "server_tokens_per_second": 63909.3, "server_over_offline": 0.9175107063230115, "server_over_offline_percent": 91.75107063230115, "difference_percent_of_offline": -8.248929367698853, "offline_accuracy": "exact_match: 80.81130355515042 TOKENS_PER_SAMPLE: 3831.6855059252507", "server_accuracy": "exact_match: 81.67730173199635 TOKENS_PER_SAMPLE: 3826.9986326344574", "offline_location": "./closed/Nebius/results/B300-SXM-270GBx8_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Nebius/results/B300-SXM-270GBx8_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "338", "server_row_id": "339", "metadata": { "Organization": "Nebius", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Nebius B300 n1 (8x B300-SXM-270GB, TensorRT)", "Platform": "B300-SXM-270GBx8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8570", "host_processors_per_node": 2, "host_processor_core_count": "48", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 108529.0, "server_tokens_per_second": 108124.0, "server_over_offline": 0.996268278524634, "server_over_offline_percent": 99.62682785246339, "difference_percent_of_offline": -0.37317214753660455, "offline_accuracy": "exact_match: 83.782", "server_accuracy": "exact_match: 83.224", "offline_location": "./closed/Nebius/results/B300-SXM-270GBx8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Nebius/results/B300-SXM-270GBx8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "344", "server_row_id": "345", "metadata": { "Organization": "Nebius", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Nebius GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x72_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72, 1400W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 689961.0, "server_tokens_per_second": 603023.0, "server_over_offline": 0.8739957765728787, "server_over_offline_percent": 87.39957765728788, "difference_percent_of_offline": -12.600422342712125, "offline_accuracy": "exact_match: 80.94804010938924 TOKENS_PER_SAMPLE: 3831.7297174111213", "server_accuracy": "exact_match: 80.67456700091158 TOKENS_PER_SAMPLE: 3850.7272105742936", "offline_location": "./closed/Nebius/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Nebius/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "346", "server_row_id": "347", "metadata": { "Organization": "Nebius", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Nebius GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x72_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72, 1400W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 1186750.0, "server_tokens_per_second": 1122490.0, "server_over_offline": 0.9458521171266063, "server_over_offline_percent": 94.58521171266064, "difference_percent_of_offline": -5.414788287339367, "offline_accuracy": "exact_match: 82.905", "server_accuracy": "exact_match: 83.174", "offline_location": "./closed/Nebius/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Nebius/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "348", "server_row_id": "349", "metadata": { "Organization": "Nebius", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Nebius GB300 NVL72 (8x GB300-288GB_aarch64, TensorRT)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 2, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72, 1400W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 132236.0, "server_tokens_per_second": 127921.0, "server_over_offline": 0.9673689464291116, "server_over_offline_percent": 96.73689464291117, "difference_percent_of_offline": -3.2631053570888358, "offline_accuracy": "exact_match: 82.975", "server_accuracy": "exact_match: 83.209", "offline_location": "./closed/Nebius/results/GB300-NVL72_GB300-288GB_aarch64x8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Nebius/results/GB300-NVL72_GB300-288GB_aarch64x8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "350", "server_row_id": "351", "metadata": { "Organization": "Nebius", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "Nebius RTX PRO 6000 n1 (8x RTX-PRO-6000-PCIE-96GB, TensorRT)", "Platform": "RTX_PRO_6000_PCIE_96GBx8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6776P", "host_processors_per_node": 2, "host_processor_core_count": "96", "accelerator_model_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04.4 LTS, Linux 6.11.0-1016-nvidia", "notes": "KVM virtual machine with 192 vCPUs presented as 2 sockets x 48 cores x 2 threads; 2 virtual NUMA nodes; 8 attached GPUs with 600 W power limit each. nvidia-smi reports CUDA 13.0 driver compatibility; CUDA toolkit and runtime versions are container-specific", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 15738.5, "server_tokens_per_second": 14784.3, "server_over_offline": 0.9393716046637227, "server_over_offline_percent": 93.93716046637228, "difference_percent_of_offline": -6.062839533627729, "offline_accuracy": "exact_match: 83.832", "server_accuracy": "exact_match: 83.717", "offline_location": "./closed/Nebius/results/RTX_PRO_6000_PCIE_96GBx8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Nebius/results/RTX_PRO_6000_PCIE_96GBx8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "354", "server_row_id": "355", "metadata": { "Organization": "Oracle", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "BM.Standard4.Ax.120", "Platform": "1-node-1S-GNR_120C", "Model": "llama3_1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6987P-C", "host_processors_per_node": 1, "host_processor_core_count": "120", "accelerator_model_name": null, "accelerators_per_node": 0, "total_accelerators": 0, "framework": "PyTorch", "operating_system": "Oracle-Linux-9.7-2026.06.15-1", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "INT8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 853.14, "server_tokens_per_second": 324.995, "server_over_offline": 0.38093982230349066, "server_over_offline_percent": 38.09398223034906, "difference_percent_of_offline": -61.90601776965094, "offline_accuracy": "ROUGE1: 38.7451 ROUGE2: 16.0022 ROUGEL: 24.5948 ROUGELSUM: 35.807 GEN_LEN: 8190646", "server_accuracy": "ROUGE1: 38.7167 ROUGE2: 16.0067 ROUGEL: 24.5623 ROUGELSUM: 35.7763 GEN_LEN: 8199230", "offline_location": "./closed/Oracle/results/1-node-1S-GNR_120C/llama3_1-8b/Offline/performance/run_1", "server_location": "./closed/Oracle/results/1-node-1S-GNR_120C/llama3_1-8b/Server/performance/run_1" }, { "offline_row_id": "357", "server_row_id": "358", "metadata": { "Organization": "Oracle", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "8xMI355X_2xEPYC_9575F", "Platform": "8xMI355X_2xEPYC-9575F", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0, ROCm 7.2.2", "operating_system": "Ubuntu 22.04.2 LTS (kernel 5.15.0-70-generic)", "notes": "8x AMD Instinct MI355X (gfx950), 288GB HBM3E each. MXFP4 (W4A4) weights with FP8 KV cache, Quark quantization", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 115567.0, "server_tokens_per_second": 109083.0, "server_over_offline": 0.9438940181885832, "server_over_offline_percent": 94.38940181885832, "difference_percent_of_offline": -5.610598181141679, "offline_accuracy": "exact_match: 83.895", "server_accuracy": "exact_match: 83.679", "offline_location": "./closed/Oracle/results/8xMI355X_2xEPYC-9575F/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Oracle/results/8xMI355X_2xEPYC-9575F/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "360", "server_row_id": "361", "metadata": { "Organization": "Oracle", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "8xMI355X_2xEPYC_9575F", "Platform": "8xMI355X_2xEPYC-9575F", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0, ROCm 7.2.2", "operating_system": "Ubuntu 22.04.2 LTS (kernel 5.15.0-70-generic)", "notes": "8x AMD Instinct MI355X (gfx950), 288GB HBM3E each. MXFP4 (W4A4) weights with FP8 KV cache, Quark quantization", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 101553.0, "server_tokens_per_second": 100392.0, "server_over_offline": 0.9885675460104576, "server_over_offline_percent": 98.85675460104576, "difference_percent_of_offline": -1.1432453989542402, "offline_accuracy": "ROUGE1: 44.5168 ROUGE2: 22.1764 ROUGEL: 28.9174 TOKENS_PER_SAMPLE: 284.5", "server_accuracy": "ROUGE1: 44.5335 ROUGE2: 22.1719 ROUGEL: 28.9333 TOKENS_PER_SAMPLE: 284.1", "offline_location": "./closed/Oracle/results/8xMI355X_2xEPYC-9575F/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Oracle/results/8xMI355X_2xEPYC-9575F/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "363", "server_row_id": "364", "metadata": { "Organization": "Oracle", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "8xMI355X_2xEPYC_9575F", "Platform": "8xMI355X_2xEPYC-9575F", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0, ROCm 7.2.2", "operating_system": "Ubuntu 22.04.2 LTS (kernel 5.15.0-70-generic)", "notes": "8x AMD Instinct MI355X (gfx950), 288GB HBM3E each. MXFP4 (W4A4) weights with FP8 KV cache, Quark quantization", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 144321.0, "server_tokens_per_second": 148594.0, "server_over_offline": 1.0296076108120094, "server_over_offline_percent": 102.96076108120093, "difference_percent_of_offline": 2.9607610812009355, "offline_accuracy": "ROUGE1: 38.5632 ROUGE2: 16.0256 ROUGEL: 24.4374 ROUGELSUM: 35.7022 GEN_LEN: 8150941", "server_accuracy": "ROUGE1: 38.5373 ROUGE2: 16.018 ROUGEL: 24.4335 ROUGELSUM: 35.6767 GEN_LEN: 8166290", "offline_location": "./closed/Oracle/results/8xMI355X_2xEPYC-9575F/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/Oracle/results/8xMI355X_2xEPYC-9575F/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "367", "server_row_id": "368", "metadata": { "Organization": "Oracle", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)", "Platform": "B300-SXM-270GBx8_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8570", "host_processors_per_node": 2, "host_processor_core_count": "56", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 70354.0, "server_tokens_per_second": 60745.5, "server_over_offline": 0.8634263865593996, "server_over_offline_percent": 86.34263865593996, "difference_percent_of_offline": -13.657361344060037, "offline_accuracy": "exact_match: 80.9936189608022 TOKENS_PER_SAMPLE: 3810.1563354603463", "server_accuracy": "exact_match: 81.35824977210574 TOKENS_PER_SAMPLE: 3858.553099361896", "offline_location": "./closed/Oracle/results/B300-SXM-270GBx8_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Oracle/results/B300-SXM-270GBx8_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "369", "server_row_id": "370", "metadata": { "Organization": "Oracle", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)", "Platform": "B300-SXM-270GBx8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8570", "host_processors_per_node": 2, "host_processor_core_count": "56", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 109931.0, "server_tokens_per_second": 100608.0, "server_over_offline": 0.9151922569611848, "server_over_offline_percent": 91.51922569611848, "difference_percent_of_offline": -8.480774303881521, "offline_accuracy": "exact_match: 83.311", "server_accuracy": "exact_match: 83.635", "offline_location": "./closed/Oracle/results/B300-SXM-270GBx8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Oracle/results/B300-SXM-270GBx8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "371", "server_row_id": "372", "metadata": { "Organization": "Oracle", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)", "Platform": "B300-SXM-270GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8570", "host_processors_per_node": 2, "host_processor_core_count": "56", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 108026.0, "server_tokens_per_second": 106696.0, "server_over_offline": 0.9876881491492788, "server_over_offline_percent": 98.76881491492789, "difference_percent_of_offline": -1.2311850850721173, "offline_accuracy": "ROUGE1: 44.7932 ROUGE2: 22.3753 ROUGEL: 29.152 TOKENS_PER_SAMPLE: 272.9", "server_accuracy": "ROUGE1: 44.8216 ROUGE2: 22.385 ROUGEL: 29.1799 TOKENS_PER_SAMPLE: 272.5", "offline_location": "./closed/Oracle/results/B300-SXM-270GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Oracle/results/B300-SXM-270GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "375", "server_row_id": "376", "metadata": { "Organization": "Oracle", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB300 NVL72 (4x GB300-288GB_aarch64, TensorRT)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x4_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72, 1400W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 60745.8, "server_tokens_per_second": 59996.0, "server_over_offline": 0.9876567598089085, "server_over_offline_percent": 98.76567598089085, "difference_percent_of_offline": -1.2343240191091454, "offline_accuracy": "exact_match: 83.395", "server_accuracy": "exact_match: 83.331", "offline_location": "./closed/Oracle/results/GB300-NVL72_GB300-288GB_aarch64x4_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Oracle/results/GB300-NVL72_GB300-288GB_aarch64x4_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "377", "server_row_id": "378", "metadata": { "Organization": "Oracle", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB300 NVL72 (4x GB300-288GB_aarch64, TensorRT)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x4_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72, 1400W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 66001.9, "server_tokens_per_second": 58991.9, "server_over_offline": 0.8937909363215303, "server_over_offline_percent": 89.37909363215303, "difference_percent_of_offline": -10.620906367846972, "offline_accuracy": "ROUGE1: 44.7161 ROUGE2: 22.308 ROUGEL: 29.1704 TOKENS_PER_SAMPLE: 267.0", "server_accuracy": "ROUGE1: 44.7161 ROUGE2: 22.308 ROUGEL: 29.1704 TOKENS_PER_SAMPLE: 267.0", "offline_location": "./closed/Oracle/results/GB300-NVL72_GB300-288GB_aarch64x4_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Oracle/results/GB300-NVL72_GB300-288GB_aarch64x4_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "385", "server_row_id": "386", "metadata": { "Organization": "Oracle", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x72_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72, 1400W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 666613.0, "server_tokens_per_second": 586852.0, "server_over_offline": 0.8803488680838808, "server_over_offline_percent": 88.03488680838808, "difference_percent_of_offline": -11.96511319161192, "offline_accuracy": "exact_match: 81.42661804922516 TOKENS_PER_SAMPLE: 3825.77484047402", "server_accuracy": "exact_match: 81.38103919781221 TOKENS_PER_SAMPLE: 3780.2771194165907", "offline_location": "./closed/Oracle/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Oracle/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "387", "server_row_id": "388", "metadata": { "Organization": "Oracle", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x72_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 18, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 72, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72, 1400W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 1097620.0, "server_tokens_per_second": 1092610.0, "server_over_offline": 0.9954355787977625, "server_over_offline_percent": 99.54355787977624, "difference_percent_of_offline": -0.4564421202237545, "offline_accuracy": "exact_match: 83.395", "server_accuracy": "exact_match: 83.331", "offline_location": "./closed/Oracle/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Oracle/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "393", "server_row_id": "394", "metadata": { "Organization": "Oracle", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB300 NVL72 (8x GB300-288GB_aarch64, TensorRT)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x8_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 2, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72, 1400W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 77031.7, "server_tokens_per_second": 68138.0, "server_over_offline": 0.8845449340985595, "server_over_offline_percent": 88.45449340985596, "difference_percent_of_offline": -11.54550659014405, "offline_accuracy": "exact_match: 81.42661804922516 TOKENS_PER_SAMPLE: 3803.590474020055", "server_accuracy": "exact_match: 80.9024612579763 TOKENS_PER_SAMPLE: 3861.967183226983", "offline_location": "./closed/Oracle/results/GB300-NVL72_GB300-288GB_aarch64x8_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Oracle/results/GB300-NVL72_GB300-288GB_aarch64x8_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "395", "server_row_id": "396", "metadata": { "Organization": "Oracle", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA GB300 NVL72 (8x GB300-288GB_aarch64, TensorRT)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 2, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72, 1400W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 132253.0, "server_tokens_per_second": 127984.0, "server_over_offline": 0.9677209590708717, "server_over_offline_percent": 96.77209590708718, "difference_percent_of_offline": -3.2279040929128255, "offline_accuracy": "ROUGE1: 44.8005 ROUGE2: 22.3791 ROUGEL: 29.1527 TOKENS_PER_SAMPLE: 272.6", "server_accuracy": "ROUGE1: 44.8018 ROUGE2: 22.3668 ROUGEL: 29.141 TOKENS_PER_SAMPLE: 272.9", "offline_location": "./closed/Oracle/results/GB300-NVL72_GB300-288GB_aarch64x8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Oracle/results/GB300-NVL72_GB300-288GB_aarch64x8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "397", "server_row_id": "398", "metadata": { "Organization": "Quanta_Cloud_Technology", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "1-node-2S-GNR_86C", "Platform": "1-node-2S-GNR_86C", "Model": "llama3_1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6788P", "host_processors_per_node": 2, "host_processor_core_count": "86", "accelerator_model_name": null, "accelerators_per_node": 0, "total_accelerators": 0, "framework": "PyTorch", "operating_system": "Ubuntu 25.10", "notes": "QuantaGrid D75E-4U. N/A", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "INT8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 1170.51, "server_tokens_per_second": 516.012, "server_over_offline": 0.44084373478227434, "server_over_offline_percent": 44.084373478227434, "difference_percent_of_offline": -55.915626521772566, "offline_accuracy": "ROUGE1: 38.703 ROUGE2: 15.9823 ROUGEL: 24.5421 ROUGELSUM: 35.7696 GEN_LEN: 8188918", "server_accuracy": "ROUGE1: 38.6189 ROUGE2: 15.8198 ROUGEL: 24.4192 ROUGELSUM: 35.6324 GEN_LEN: 8192507", "offline_location": "./closed/Quanta_Cloud_Technology/results/1-node-2S-GNR_86C/llama3_1-8b/Offline/performance/run_1", "server_location": "./closed/Quanta_Cloud_Technology/results/1-node-2S-GNR_86C/llama3_1-8b/Server/performance/run_1" }, { "offline_row_id": "401", "server_row_id": "402", "metadata": { "Organization": "Quanta_Cloud_Technology", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "1-node-4x-BMG-B70", "Platform": "1-node-4x-BMG-B70", "Model": "llama3_1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6788P", "host_processors_per_node": 2, "host_processor_core_count": "86", "accelerator_model_name": "Intel(R) Arc(TM) Pro B70", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "PyTorch", "operating_system": "Ubuntu 25.10", "notes": "QuantaGrid D75E-4U. N/A", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "INT8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 4977.67, "server_tokens_per_second": 4122.73, "server_over_offline": 0.8282449419105725, "server_over_offline_percent": 82.82449419105725, "difference_percent_of_offline": -17.17550580894275, "offline_accuracy": "ROUGE1: 38.8427 ROUGE2: 16.0895 ROUGEL: 24.6432 ROUGELSUM: 35.9449 GEN_LEN: 8127665", "server_accuracy": "ROUGE1: 38.8436 ROUGE2: 16.0905 ROUGEL: 24.6447 ROUGELSUM: 35.9462 GEN_LEN: 8127418", "offline_location": "./closed/Quanta_Cloud_Technology/results/1-node-4x-BMG-B70/llama3_1-8b/Offline/performance/run_1", "server_location": "./closed/Quanta_Cloud_Technology/results/1-node-4x-BMG-B70/llama3_1-8b/Server/performance/run_1" }, { "offline_row_id": "404", "server_row_id": "405", "metadata": { "Organization": "Quanta_Cloud_Technology", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "QuantaGrid D75H-10U (8x B300-SXM-270GB, TensorRT)", "Platform": "D75H-10U_B300-270GBx8_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6776P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Rocky Linux 10.1", "notes": "QuantaGrid D75H-10U", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 69841.6, "server_tokens_per_second": 59622.2, "server_over_offline": 0.8536774644338044, "server_over_offline_percent": 85.36774644338044, "difference_percent_of_offline": -14.632253556619556, "offline_accuracy": "exact_match: 81.58614402917047 TOKENS_PER_SAMPLE: 3773.390154968095", "server_accuracy": "exact_match: 81.17593436645396 TOKENS_PER_SAMPLE: 3821.9033728350046", "offline_location": "./closed/Quanta_Cloud_Technology/results/D75H-10U_B300-270GBx8_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Quanta_Cloud_Technology/results/D75H-10U_B300-270GBx8_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "406", "server_row_id": "407", "metadata": { "Organization": "Quanta_Cloud_Technology", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "QuantaGrid D75H-10U (8x B300-SXM-270GB, TensorRT)", "Platform": "D75H-10U_B300-270GBx8_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6776P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Rocky Linux 10.1", "notes": "QuantaGrid D75H-10U", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 108293.0, "server_tokens_per_second": 100183.0, "server_over_offline": 0.9251105796311858, "server_over_offline_percent": 92.51105796311859, "difference_percent_of_offline": -7.488942036881419, "offline_accuracy": "exact_match: 83.431", "server_accuracy": "exact_match: 83.451", "offline_location": "./closed/Quanta_Cloud_Technology/results/D75H-10U_B300-270GBx8_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Quanta_Cloud_Technology/results/D75H-10U_B300-270GBx8_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "408", "server_row_id": "409", "metadata": { "Organization": "Quanta_Cloud_Technology", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "QuantaGrid D75H-10U (8x B300-SXM-270GB, TensorRT)", "Platform": "D75H-10U_B300-270GBx8_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6776P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Rocky Linux 10.1", "notes": "QuantaGrid D75H-10U", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 113985.0, "server_tokens_per_second": 106285.0, "server_over_offline": 0.9324472518313813, "server_over_offline_percent": 93.24472518313813, "difference_percent_of_offline": -6.755274816861867, "offline_accuracy": "ROUGE1: 44.7969 ROUGE2: 22.3741 ROUGEL: 29.152 TOKENS_PER_SAMPLE: 272.6", "server_accuracy": "ROUGE1: 44.8968 ROUGE2: 22.4583 ROUGEL: 29.2397 TOKENS_PER_SAMPLE: 272.0", "offline_location": "./closed/Quanta_Cloud_Technology/results/D75H-10U_B300-270GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Quanta_Cloud_Technology/results/D75H-10U_B300-270GBx8_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "413", "server_row_id": "414", "metadata": { "Organization": "RedHat_Intel", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "1-node-2S-GNR_96C", "Platform": "1-node-2S-GNR_96C", "Model": "llama3_1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6972P", "host_processors_per_node": 2, "host_processor_core_count": "96", "accelerator_model_name": null, "accelerators_per_node": 0, "total_accelerators": 0, "framework": "PyTorch", "operating_system": "RHEL 9.9", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "INT8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 1507.21, "server_tokens_per_second": 644.788, "server_over_offline": 0.4278023633070375, "server_over_offline_percent": 42.78023633070375, "difference_percent_of_offline": -57.21976366929624, "offline_accuracy": "ROUGE1: 38.7872 ROUGE2: 16.0024 ROUGEL: 24.5918 ROUGELSUM: 35.7668 GEN_LEN: 8190420", "server_accuracy": "ROUGE1: 38.606 ROUGE2: 15.7593 ROUGEL: 24.3946 ROUGELSUM: 35.6342 GEN_LEN: 8185295", "offline_location": "./closed/RedHat_Intel/results/1-node-2S-GNR_96C/llama3_1-8b/Offline/performance/run_1", "server_location": "./closed/RedHat_Intel/results/1-node-2S-GNR_96C/llama3_1-8b/Server/performance/run_1" }, { "offline_row_id": "418", "server_row_id": "419", "metadata": { "Organization": "RedHat_Supermicro", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "SMC Lab OpenShift GB200-NVL4 4xGB200-186GB", "Platform": "GB200-NVL4_186GB_Openshift_VLLM", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "72", "accelerator_model_name": "NVIDIA GB200", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "vLLM 0.24.0, Red Hat OpenShift 4.21", "operating_system": "Red Hat Enterprise Linux CoreOS 9.6", "notes": "NVIDIA GB200 NVL4 with 2x Grace CPUs (72 ARM cores each), 4x GB200 GPUs (189GB HBM3e each). SMC Lab, Red Hat OpenShift 4.21, RHEL CoreOS 9.6, vLLM 0.24.0", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 61205.6, "server_tokens_per_second": 51095.2, "server_over_offline": 0.8348125008169187, "server_over_offline_percent": 83.48125008169187, "difference_percent_of_offline": -16.518749918308128, "offline_accuracy": "exact_match: 83.695", "server_accuracy": "exact_match: 83.948", "offline_location": "./closed/RedHat_Supermicro/results/GB200-NVL4_186GB_Openshift_VLLM/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/RedHat_Supermicro/results/GB200-NVL4_186GB_Openshift_VLLM/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "420", "server_row_id": "421", "metadata": { "Organization": "Supermicro", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "1-node-2S-GNR_128C", "Platform": "1-node-2S-GNR_128C", "Model": "llama3_1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6980P", "host_processors_per_node": 2, "host_processor_core_count": "128", "accelerator_model_name": null, "accelerators_per_node": 0, "total_accelerators": 0, "framework": "PyTorch", "operating_system": "Ubuntu 24.04.3", "notes": "SYS-422GA-NRT. N/A", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "INT8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 1833.97, "server_tokens_per_second": 1101.19, "server_over_offline": 0.6004405742732979, "server_over_offline_percent": 60.04405742732979, "difference_percent_of_offline": -39.95594257267021, "offline_accuracy": "ROUGE1: 38.6493 ROUGE2: 15.8388 ROUGEL: 24.4665 ROUGELSUM: 35.6409 GEN_LEN: 8186666", "server_accuracy": "ROUGE1: 38.7236 ROUGE2: 15.8895 ROUGEL: 24.4948 ROUGELSUM: 35.7134 GEN_LEN: 8199179", "offline_location": "./closed/Supermicro/results/1-node-2S-GNR_128C/llama3_1-8b/Offline/performance/run_1", "server_location": "./closed/Supermicro/results/1-node-2S-GNR_128C/llama3_1-8b/Server/performance/run_1" }, { "offline_row_id": "423", "server_row_id": "424", "metadata": { "Organization": "Supermicro", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "1-node-2S-GNR_86C", "Platform": "1-node-2S-GNR_86C", "Model": "llama3_1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6787P", "host_processors_per_node": 2, "host_processor_core_count": "86", "accelerator_model_name": null, "accelerators_per_node": 0, "total_accelerators": 0, "framework": "PyTorch", "operating_system": "Ubuntu 24.04.3", "notes": "SYS-422GB-NRT. N/A", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "INT8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 1189.89, "server_tokens_per_second": 516.059, "server_over_offline": 0.4337031154140298, "server_over_offline_percent": 43.37031154140298, "difference_percent_of_offline": -56.629688458597016, "offline_accuracy": "ROUGE1: 38.7008 ROUGE2: 15.8479 ROUGEL: 24.4355 ROUGELSUM: 35.7034 GEN_LEN: 8193707", "server_accuracy": "ROUGE1: 38.7059 ROUGE2: 15.8614 ROUGEL: 24.4785 ROUGELSUM: 35.726 GEN_LEN: 8175493", "offline_location": "./closed/Supermicro/results/1-node-2S-GNR_86C/llama3_1-8b/Offline/performance/run_1", "server_location": "./closed/Supermicro/results/1-node-2S-GNR_86C/llama3_1-8b/Server/performance/run_1" }, { "offline_row_id": "427", "server_row_id": "428", "metadata": { "Organization": "Supermicro", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "1-node-4S-GNR_86C", "Platform": "1-node-4S-GNR_86C", "Model": "llama3_1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6788P", "host_processors_per_node": 4, "host_processor_core_count": "86", "accelerator_model_name": null, "accelerators_per_node": 0, "total_accelerators": 0, "framework": "PyTorch", "operating_system": "Ubuntu 24.04.3 LTS", "notes": "SYS-442B-NR. N/A", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "INT8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 2063.24, "server_tokens_per_second": 966.241, "server_over_offline": 0.4683124600143464, "server_over_offline_percent": 46.83124600143464, "difference_percent_of_offline": -53.16875399856536, "offline_accuracy": "ROUGE1: 38.5949 ROUGE2: 15.7867 ROUGEL: 24.3919 ROUGELSUM: 35.6229 GEN_LEN: 8190845", "server_accuracy": "ROUGE1: 38.7724 ROUGE2: 16.0093 ROUGEL: 24.5982 ROUGELSUM: 35.8002 GEN_LEN: 8168484", "offline_location": "./closed/Supermicro/results/1-node-4S-GNR_86C/llama3_1-8b/Offline/performance/run_1", "server_location": "./closed/Supermicro/results/1-node-4S-GNR_86C/llama3_1-8b/Server/performance/run_1" }, { "offline_row_id": "430", "server_row_id": "431", "metadata": { "Organization": "Supermicro", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "1-node-4x-BMG-B70", "Platform": "1-node-4x-BMG-B70", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6731P", "host_processors_per_node": 1, "host_processor_core_count": "32", "accelerator_model_name": "Intel(R) Arc(TM) Pro B70", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "PyTorch", "operating_system": "Ubuntu 24.04.4 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "MXFP4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 2063.93, "server_tokens_per_second": 1351.13, "server_over_offline": 0.6546394499813464, "server_over_offline_percent": 65.46394499813464, "difference_percent_of_offline": -34.536055001865364, "offline_accuracy": "exact_match: 83.780", "server_accuracy": "exact_match: 84.186", "offline_location": "./closed/Supermicro/results/1-node-4x-BMG-B70/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Supermicro/results/1-node-4x-BMG-B70/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "432", "server_row_id": "433", "metadata": { "Organization": "Supermicro", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "1-node-4x-BMG-B70", "Platform": "1-node-4x-BMG-B70", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6731P", "host_processors_per_node": 1, "host_processor_core_count": "32", "accelerator_model_name": "Intel(R) Arc(TM) Pro B70", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "PyTorch", "operating_system": "Ubuntu 24.04.4 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "INT8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 2484.14, "server_tokens_per_second": 1759.95, "server_over_offline": 0.708474562625295, "server_over_offline_percent": 70.84745626252949, "difference_percent_of_offline": -29.152543737470506, "offline_accuracy": "ROUGE1: 44.585 ROUGE2: 22.2008 ROUGEL: 28.8241 TOKENS_PER_SAMPLE: 290.8", "server_accuracy": "ROUGE1: 44.591 ROUGE2: 22.2044 ROUGEL: 28.8277 TOKENS_PER_SAMPLE: 290.8", "offline_location": "./closed/Supermicro/results/1-node-4x-BMG-B70/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Supermicro/results/1-node-4x-BMG-B70/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "434", "server_row_id": "435", "metadata": { "Organization": "Supermicro", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "1-node-4x-BMG-B70", "Platform": "1-node-4x-BMG-B70", "Model": "llama3_1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6731P", "host_processors_per_node": 1, "host_processor_core_count": "32", "accelerator_model_name": "Intel(R) Arc(TM) Pro B70", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "PyTorch", "operating_system": "Ubuntu 24.04.4 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "INT8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 4596.62, "server_tokens_per_second": 4208.73, "server_over_offline": 0.9156140816513002, "server_over_offline_percent": 91.56140816513002, "difference_percent_of_offline": -8.43859183486998, "offline_accuracy": "ROUGE1: 38.8447 ROUGE2: 16.0909 ROUGEL: 24.6442 ROUGELSUM: 35.9477 GEN_LEN: 8127563", "server_accuracy": "ROUGE1: 38.8128 ROUGE2: 16.082 ROUGEL: 24.6384 ROUGELSUM: 35.9155 GEN_LEN: 8126189", "offline_location": "./closed/Supermicro/results/1-node-4x-BMG-B70/llama3_1-8b/Offline/performance/run_1", "server_location": "./closed/Supermicro/results/1-node-4x-BMG-B70/llama3_1-8b/Server/performance/run_1" }, { "offline_row_id": "437", "server_row_id": "438", "metadata": { "Organization": "Supermicro", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "AS-4126GS-NMR-LCC", "Platform": "8xMI355X_2xEPYC-9575F", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0, ROCm 7.2.4", "operating_system": "Ubuntu 26.04 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 117005.0, "server_tokens_per_second": 109184.0, "server_over_offline": 0.9331567027050126, "server_over_offline_percent": 93.31567027050126, "difference_percent_of_offline": -6.684329729498739, "offline_accuracy": "exact_match: 83.113", "server_accuracy": "exact_match: 83.861", "offline_location": "./closed/Supermicro/results/8xMI355X_2xEPYC-9575F/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Supermicro/results/8xMI355X_2xEPYC-9575F/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "440", "server_row_id": "441", "metadata": { "Organization": "Supermicro", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "AS-4126GS-NMR-LCC", "Platform": "8xMI355X_2xEPYC-9575F", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0, ROCm 7.2.4", "operating_system": "Ubuntu 26.04 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 102243.0, "server_tokens_per_second": 103032.0, "server_over_offline": 1.0077169097150904, "server_over_offline_percent": 100.77169097150905, "difference_percent_of_offline": 0.7716909715090425, "offline_accuracy": "ROUGE1: 44.5269 ROUGE2: 22.1836 ROUGEL: 28.9207 TOKENS_PER_SAMPLE: 284.3", "server_accuracy": "ROUGE1: 44.5345 ROUGE2: 22.1731 ROUGEL: 28.933 TOKENS_PER_SAMPLE: 284.0", "offline_location": "./closed/Supermicro/results/8xMI355X_2xEPYC-9575F/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Supermicro/results/8xMI355X_2xEPYC-9575F/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "442", "server_row_id": "443", "metadata": { "Organization": "Supermicro", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "AS-4126GS-NMR-LCC", "Platform": "8xMI355X_2xEPYC-9575F", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "AMD Instinct MI355X 288GB HBM3e", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "vLLM 0.22.0, PyTorch 2.10.0, ROCm 7.2.4", "operating_system": "Ubuntu 26.04 LTS", "notes": null, "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp8", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 159060.0, "server_tokens_per_second": 148710.0, "server_over_offline": 0.9349302150132026, "server_over_offline_percent": 93.49302150132026, "difference_percent_of_offline": -6.5069784986797385, "offline_accuracy": "ROUGE1: 38.5437 ROUGE2: 16.0099 ROUGEL: 24.4209 ROUGELSUM: 35.6859 GEN_LEN: 8150015", "server_accuracy": "ROUGE1: 38.5048 ROUGE2: 15.9985 ROUGEL: 24.4065 ROUGELSUM: 35.652 GEN_LEN: 8176781", "offline_location": "./closed/Supermicro/results/8xMI355X_2xEPYC-9575F/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/Supermicro/results/8xMI355X_2xEPYC-9575F/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "444", "server_row_id": "445", "metadata": { "Organization": "Supermicro", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "AS-8126GS-NB3RT", "Platform": "AS-8126GS-NB3RT_B300-SXM-288GB_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 69830.0, "server_tokens_per_second": 61164.9, "server_over_offline": 0.8759114993555779, "server_over_offline_percent": 87.5911499355578, "difference_percent_of_offline": -12.408850064442213, "offline_accuracy": "exact_match: 80.78851412944394 TOKENS_PER_SAMPLE: 3839.5469462169553", "server_accuracy": "exact_match: 81.22151321786691 TOKENS_PER_SAMPLE: 3850.996809480401", "offline_location": "./closed/Supermicro/results/AS-8126GS-NB3RT_B300-SXM-288GB_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Supermicro/results/AS-8126GS-NB3RT_B300-SXM-288GB_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "446", "server_row_id": "447", "metadata": { "Organization": "Supermicro", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "AS-8126GS-NB3RT", "Platform": "AS-8126GS-NB3RT_B300-SXM-288GB_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 108727.0, "server_tokens_per_second": 107288.0, "server_over_offline": 0.9867650169691061, "server_over_offline_percent": 98.6765016969106, "difference_percent_of_offline": -1.3234983030893877, "offline_accuracy": "exact_match: 83.743", "server_accuracy": "exact_match: 83.354", "offline_location": "./closed/Supermicro/results/AS-8126GS-NB3RT_B300-SXM-288GB_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Supermicro/results/AS-8126GS-NB3RT_B300-SXM-288GB_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "448", "server_row_id": "449", "metadata": { "Organization": "Supermicro", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "AS-8126GS-NB3RT", "Platform": "AS-8126GS-NB3RT_B300-SXM-288GB_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9575F", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 110802.0, "server_tokens_per_second": 106872.0, "server_over_offline": 0.9645313261493474, "server_over_offline_percent": 96.45313261493474, "difference_percent_of_offline": -3.5468673850652555, "offline_accuracy": "ROUGE1: 44.8018 ROUGE2: 22.3841 ROUGEL: 29.1588 TOKENS_PER_SAMPLE: 272.7", "server_accuracy": "ROUGE1: 44.828 ROUGE2: 22.4004 ROUGEL: 29.1931 TOKENS_PER_SAMPLE: 272.0", "offline_location": "./closed/Supermicro/results/AS-8126GS-NB3RT_B300-SXM-288GB_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Supermicro/results/AS-8126GS-NB3RT_B300-SXM-288GB_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "450", "server_row_id": "451", "metadata": { "Organization": "Supermicro", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "SYS-822GS-NB3RT", "Platform": "SYS-822GS-NB3RT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6767P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 69104.0, "server_tokens_per_second": 66754.3, "server_over_offline": 0.965997626765455, "server_over_offline_percent": 96.5997626765455, "difference_percent_of_offline": -3.4002373234545025, "offline_accuracy": "exact_match: 81.58614402917047 TOKENS_PER_SAMPLE: 3787.6494986326343", "server_accuracy": "exact_match: 80.92525068368278 TOKENS_PER_SAMPLE: 3729.1365086599817", "offline_location": "./closed/Supermicro/results/SYS-822GS-NB3RT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/Supermicro/results/SYS-822GS-NB3RT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "452", "server_row_id": "453", "metadata": { "Organization": "Supermicro", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "SYS-822GS-NB3RT", "Platform": "SYS-822GS-NB3RT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6767P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 107851.0, "server_tokens_per_second": 107205.0, "server_over_offline": 0.994010254888689, "server_over_offline_percent": 99.4010254888689, "difference_percent_of_offline": -0.5989745111311007, "offline_accuracy": "exact_match: 83.592", "server_accuracy": "exact_match: 83.457", "offline_location": "./closed/Supermicro/results/SYS-822GS-NB3RT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Supermicro/results/SYS-822GS-NB3RT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "454", "server_row_id": "455", "metadata": { "Organization": "Supermicro", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "SYS-822GS-NB3RT", "Platform": "SYS-822GS-NB3RT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6767P", "host_processors_per_node": 2, "host_processor_core_count": "64", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 111247.0, "server_tokens_per_second": 109848.0, "server_over_offline": 0.9874243799832805, "server_over_offline_percent": 98.74243799832804, "difference_percent_of_offline": -1.2575620016719524, "offline_accuracy": "ROUGE1: 44.7966 ROUGE2: 22.3806 ROUGEL: 29.1557 TOKENS_PER_SAMPLE: 272.7", "server_accuracy": "ROUGE1: 44.799 ROUGE2: 22.3988 ROUGEL: 29.1884 TOKENS_PER_SAMPLE: 272.3", "offline_location": "./closed/Supermicro/results/SYS-822GS-NB3RT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/Supermicro/results/SYS-822GS-NB3RT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "458", "server_row_id": "459", "metadata": { "Organization": "TTA", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "ManyCoreSoft DeepGadget dg5W (4x NVIDIA RTX PRO 6000 Blackwell 96GB, TensorRT)", "Platform": "DeepGadget_dg5W_RTXPro6000_96GBx4_TRT", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "AMD EPYC 9124", "host_processors_per_node": 1, "host_processor_core_count": "16", "accelerator_model_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "TensorRT 10.14.1.48, CUDA 13.1", "operating_system": "Rocky Linux 9.7", "notes": "DeepGadget dg5W; ManyCoreSoft dg5W (SKU2-T1) chassis, GENOAD8X-2T/BCM mainboard; 4x NVIDIA RTX PRO 6000 Blackwell Server Edition (TGP 600W each); direct liquid cooling (DLC)", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 24511.8, "server_tokens_per_second": 24510.5, "server_over_offline": 0.9999469643192259, "server_over_offline_percent": 99.99469643192259, "difference_percent_of_offline": -0.005303568077408283, "offline_accuracy": "ROUGE1: 38.5238 ROUGE2: 15.9869 ROUGEL: 24.3988 ROUGELSUM: 35.6579 GEN_LEN: 8178576", "server_accuracy": "ROUGE1: 38.5242 ROUGE2: 15.9887 ROUGEL: 24.3995 ROUGELSUM: 35.6582 GEN_LEN: 8178682", "offline_location": "./closed/TTA/results/DeepGadget_dg5W_RTXPro6000_96GBx4_TRT/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/TTA/results/DeepGadget_dg5W_RTXPro6000_96GBx4_TRT/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "460", "server_row_id": "461", "metadata": { "Organization": "VibeHPC", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "B300-SXM-270GBx1_TRT", "Platform": "B300-SXM-270GBx1_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8570", "host_processors_per_node": 2, "host_processor_core_count": "56", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 1, "total_accelerators": 1, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 14130.6, "server_tokens_per_second": 13854.0, "server_over_offline": 0.9804254596407795, "server_over_offline_percent": 98.04254596407796, "difference_percent_of_offline": -1.957454035922046, "offline_accuracy": "exact_match: 83.235", "server_accuracy": "exact_match: 83.289", "offline_location": "./closed/VibeHPC/results/B300-SXM-270GBx1_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/VibeHPC/results/B300-SXM-270GBx1_TRT/gpt-oss-120b/Server/performance/run_1" }, { "offline_row_id": "462", "server_row_id": "463", "metadata": { "Organization": "VibeHPC", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "B300-SXM-270GBx1_TRT", "Platform": "B300-SXM-270GBx1_TRT", "Model": "llama2-70b-99.9", "MlperfModel": "llama2-70b-99.9", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8570", "host_processors_per_node": 2, "host_processor_core_count": "56", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 1, "total_accelerators": 1, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 14636.8, "server_tokens_per_second": 14495.6, "server_over_offline": 0.9903530826410145, "server_over_offline_percent": 99.03530826410145, "difference_percent_of_offline": -0.9646917358985485, "offline_accuracy": "ROUGE1: 44.781 ROUGE2: 22.3679 ROUGEL: 29.1539 TOKENS_PER_SAMPLE: 273.1", "server_accuracy": "ROUGE1: 44.7255 ROUGE2: 22.327 ROUGEL: 29.1172 TOKENS_PER_SAMPLE: 274.2", "offline_location": "./closed/VibeHPC/results/B300-SXM-270GBx1_TRT/llama2-70b-99.9/Offline/performance/run_1", "server_location": "./closed/VibeHPC/results/B300-SXM-270GBx1_TRT/llama2-70b-99.9/Server/performance/run_1" }, { "offline_row_id": "464", "server_row_id": "465", "metadata": { "Organization": "VibeHPC", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "B300-SXM-270GBx1_TRT", "Platform": "B300-SXM-270GBx1_TRT", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) Platinum 8570", "host_processors_per_node": 2, "host_processor_core_count": "56", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 1, "total_accelerators": 1, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 21266.1, "server_tokens_per_second": 20420.4, "server_over_offline": 0.960232482683708, "server_over_offline_percent": 96.02324826837079, "difference_percent_of_offline": -3.9767517316292023, "offline_accuracy": "ROUGE1: 38.5346 ROUGE2: 15.7631 ROUGEL: 24.3218 ROUGELSUM: 35.5543 GEN_LEN: 8191123", "server_accuracy": "ROUGE1: 38.5378 ROUGE2: 15.764 ROUGEL: 24.3237 ROUGELSUM: 35.5571 GEN_LEN: 8191033", "offline_location": "./closed/VibeHPC/results/B300-SXM-270GBx1_TRT/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/VibeHPC/results/B300-SXM-270GBx1_TRT/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "467", "server_row_id": "468", "metadata": { "Organization": "VibeHPC", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)", "Platform": "B300-SXM-270GBx8_TRT", "Model": "deepseek-r1", "MlperfModel": "deepseek-r1", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6776P", "host_processors_per_node": 2, "host_processor_core_count": "48", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 36639.6, "server_tokens_per_second": 33415.2, "server_over_offline": 0.9119968558608718, "server_over_offline_percent": 91.19968558608717, "difference_percent_of_offline": -8.800314413912824, "offline_accuracy": "exact_match: 81.22151321786691 TOKENS_PER_SAMPLE: 3770.8865086599817", "server_accuracy": "exact_match: 81.83682771194167 TOKENS_PER_SAMPLE: 3750.4530537830447", "offline_location": "./closed/VibeHPC/results/B300-SXM-270GBx8_TRT/deepseek-r1/Offline/performance/run_1", "server_location": "./closed/VibeHPC/results/B300-SXM-270GBx8_TRT/deepseek-r1/Server/performance/run_1" }, { "offline_row_id": "469", "server_row_id": "470", "metadata": { "Organization": "VibeHPC", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)", "Platform": "B300-SXM-270GBx8_TRT", "Model": "llama3.1-8b", "MlperfModel": "llama3.1-8b", "number_of_nodes": 1, "host_processor_model_name": "Intel(R) Xeon(R) 6776P", "host_processors_per_node": 2, "host_processor_core_count": "48", "accelerator_model_name": "NVIDIA B300-SXM-270GB", "accelerators_per_node": 8, "total_accelerators": 8, "framework": "TensorRT 10.14, CUDA 13.1", "operating_system": "Ubuntu 24.04", "notes": "B300 TGP 1100W", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 161767.0, "server_tokens_per_second": 156867.0, "server_over_offline": 0.9697095204831641, "server_over_offline_percent": 96.9709520483164, "difference_percent_of_offline": -3.0290479516835944, "offline_accuracy": "ROUGE1: 38.5395 ROUGE2: 15.7648 ROUGEL: 24.3262 ROUGELSUM: 35.5597 GEN_LEN: 8190944", "server_accuracy": "ROUGE1: 38.5355 ROUGE2: 15.7593 ROUGEL: 24.3212 ROUGELSUM: 35.5549 GEN_LEN: 8191008", "offline_location": "./closed/VibeHPC/results/B300-SXM-270GBx8_TRT/llama3.1-8b/Offline/performance/run_1", "server_location": "./closed/VibeHPC/results/B300-SXM-270GBx8_TRT/llama3.1-8b/Server/performance/run_1" }, { "offline_row_id": "474", "server_row_id": "475", "metadata": { "Organization": "Wiwynn", "Availability": "available", "Division": "closed", "SystemType": "datacenter", "SystemName": "OneW GB300 NVL72 (4x GB300-288GB_aarch64, TensorRT)", "Platform": "GB300-NVL72_GB300-288GB_aarch64x4_TRT", "Model": "gpt-oss-120b", "MlperfModel": "gpt-oss-120b", "number_of_nodes": 1, "host_processor_model_name": "NVIDIA Grace CPU", "host_processors_per_node": 2, "host_processor_core_count": "144", "accelerator_model_name": "NVIDIA GB300", "accelerators_per_node": 4, "total_accelerators": 4, "framework": "TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf", "operating_system": "Ubuntu 24.04", "notes": "NVIDIA GB300 NVL72, 1400W TGP Per GPU", "compliance": "closed", "errors": 0, "version": "v6.1", "inferred": 0, "has_power": false, "Units": "Tokens/s", "weight_data_types": "fp4", "design_power_watts": null, "PrivateID": null }, "offline_tokens_per_second": 63990.5, "server_tokens_per_second": 60466.2, "server_over_offline": 0.9449246372508419, "server_over_offline_percent": 94.49246372508419, "difference_percent_of_offline": -5.5075362749158145, "offline_accuracy": "exact_match: 83.440", "server_accuracy": "exact_match: 83.187", "offline_location": "./closed/Wiwynn/results/GB300-NVL72_GB300-288GB_aarch64x4_TRT/gpt-oss-120b/Offline/performance/run_1", "server_location": "./closed/Wiwynn/results/GB300-NVL72_GB300-288GB_aarch64x4_TRT/gpt-oss-120b/Server/performance/run_1" } ] }