| { | |
| "current_gpu_type": "NVIDIA A100-PCIE-40GB", | |
| "current_gpu_total_memory": 40339.3125, | |
| "perplexity": 7.984891414642334, | |
| "token_generation_latency_sync": 22.21034393310547, | |
| "token_generation_latency_async": 22.44959268718958, | |
| "token_generation_throughput_sync": 0.045024066399505736, | |
| "token_generation_throughput_async": 0.04454423801509016, | |
| "token_generation_CO2_emissions": null, | |
| "token_generation_energy_consumption": null, | |
| "inference_latency_sync": 336.52049865722654, | |
| "inference_latency_async": 146.97182178497314, | |
| "inference_throughput_sync": 0.0029715871811380537, | |
| "inference_throughput_async": 0.00680402534210298, | |
| "inference_CO2_emissions": null, | |
| "inference_energy_consumption": null | |
| } |