|
{ |
|
"base_current_gpu_type": "NVIDIA A100-PCIE-40GB", |
|
"base_current_gpu_total_memory": 40339.3125, |
|
"base_perplexity": Infinity, |
|
"base_token_generation_latency_sync": 32.41663513183594, |
|
"base_token_generation_latency_async": 31.96390364319086, |
|
"base_token_generation_throughput_sync": 0.03084835905802924, |
|
"base_token_generation_throughput_async": 0.03128529015613604, |
|
"base_token_generation_CO2_emissions": null, |
|
"base_token_generation_energy_consumption": null, |
|
"base_inference_latency_sync": 124.98411560058594, |
|
"base_inference_latency_async": 30.86845874786377, |
|
"base_inference_throughput_sync": 0.00800101673076376, |
|
"base_inference_throughput_async": 0.03239552736235023, |
|
"base_inference_CO2_emissions": null, |
|
"base_inference_energy_consumption": null, |
|
"smashed_current_gpu_type": "NVIDIA A100-PCIE-40GB", |
|
"smashed_current_gpu_total_memory": 40339.3125, |
|
"smashed_perplexity": Infinity, |
|
"smashed_token_generation_latency_sync": 54.6178581237793, |
|
"smashed_token_generation_latency_async": 52.420770190656185, |
|
"smashed_token_generation_throughput_sync": 0.01830902994646405, |
|
"smashed_token_generation_throughput_async": 0.01907640800322019, |
|
"smashed_token_generation_CO2_emissions": null, |
|
"smashed_token_generation_energy_consumption": null, |
|
"smashed_inference_latency_sync": 211.23184356689453, |
|
"smashed_inference_latency_async": 107.47067928314209, |
|
"smashed_inference_throughput_sync": 0.004734134698224666, |
|
"smashed_inference_throughput_async": 0.009304863490863414, |
|
"smashed_inference_CO2_emissions": null, |
|
"smashed_inference_energy_consumption": null |
|
} |