han.zhao/test_gpu_scripts
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity
test_gpu_scripts/reports_single_gpu_aikubeworker0016.json

292 lines
10 KiB
JSON
Raw Blame History

{
"timestamp": "2026-05-22T15:26:29.511252",
"gpu_info": {
"driver_version": "580.159.03",
"cuda_version": "13.0",
"gpu_count": 8,
"gpus": [
{
"index": 0,
"name": "NVIDIA H100 80GB HBM3",
"uuid": "GPU-dfbc9513-255d-4fe7-2b77-7b1ec3972e75",
"pci_bus_id": "00000000:18:00.0",
"pcie_link_gen": 5,
"pcie_link_width": 16,
"vram_total_mb": 81559,
"vram_used_mb": 4,
"vram_free_mb": 81076,
"power_draw": 69.81,
"power_limit": 700.0,
"clock_sm": 345,
"clock_mem": 2619,
"temperature": 20,
"fan_speed": 0,
"persistence_mode": false,
"compute_mode": "Default",
"serial_number": "1651924016120",
"ecc_errors_single": 0,
"ecc_errors_double": 0
},
{
"index": 1,
"name": "NVIDIA H100 80GB HBM3",
"uuid": "GPU-bb845ef7-d7b5-f011-9395-ea74274e2282",
"pci_bus_id": "00000000:2A:00.0",
"pcie_link_gen": 5,
"pcie_link_width": 16,
"vram_total_mb": 81559,
"vram_used_mb": 0,
"vram_free_mb": 81079,
"power_draw": 67.45,
"power_limit": 700.0,
"clock_sm": 345,
"clock_mem": 2619,
"temperature": 20,
"fan_speed": 0,
"persistence_mode": false,
"compute_mode": "Default",
"serial_number": "1651924015483",
"ecc_errors_single": 0,
"ecc_errors_double": 0
},
{
"index": 2,
"name": "NVIDIA H100 80GB HBM3",
"uuid": "GPU-3720cf13-2a34-be38-27be-0a7adc4addc4",
"pci_bus_id": "00000000:3A:00.0",
"pcie_link_gen": 5,
"pcie_link_width": 16,
"vram_total_mb": 81559,
"vram_used_mb": 0,
"vram_free_mb": 81079,
"power_draw": 66.69,
"power_limit": 700.0,
"clock_sm": 345,
"clock_mem": 2619,
"temperature": 21,
"fan_speed": 0,
"persistence_mode": false,
"compute_mode": "Default",
"serial_number": "1651924025595",
"ecc_errors_single": 0,
"ecc_errors_double": 0
},
{
"index": 3,
"name": "NVIDIA H100 80GB HBM3",
"uuid": "GPU-87080b2d-ac43-be0d-d574-c193078850ae",
"pci_bus_id": "00000000:5D:00.0",
"pcie_link_gen": 5,
"pcie_link_width": 16,
"vram_total_mb": 81559,
"vram_used_mb": 0,
"vram_free_mb": 81079,
"power_draw": 66.86,
"power_limit": 700.0,
"clock_sm": 345,
"clock_mem": 2619,
"temperature": 20,
"fan_speed": 0,
"persistence_mode": false,
"compute_mode": "Default",
"serial_number": "1651924016862",
"ecc_errors_single": 0,
"ecc_errors_double": 0
},
{
"index": 4,
"name": "NVIDIA H100 80GB HBM3",
"uuid": "GPU-599bd883-cc5c-a5dd-6c33-c15f7049da48",
"pci_bus_id": "00000000:9A:00.0",
"pcie_link_gen": 5,
"pcie_link_width": 16,
"vram_total_mb": 81559,
"vram_used_mb": 0,
"vram_free_mb": 81079,
"power_draw": 67.07,
"power_limit": 700.0,
"clock_sm": 345,
"clock_mem": 2619,
"temperature": 20,
"fan_speed": 0,
"persistence_mode": false,
"compute_mode": "Default",
"serial_number": "1651924025670",
"ecc_errors_single": 0,
"ecc_errors_double": 0
},
{
"index": 5,
"name": "NVIDIA H100 80GB HBM3",
"uuid": "GPU-a1c6bba4-61b0-e623-06c9-9c88635e26fe",
"pci_bus_id": "00000000:AB:00.0",
"pcie_link_gen": 5,
"pcie_link_width": 16,
"vram_total_mb": 81559,
"vram_used_mb": 0,
"vram_free_mb": 81079,
"power_draw": 69.12,
"power_limit": 700.0,
"clock_sm": 345,
"clock_mem": 2619,
"temperature": 22,
"fan_speed": 0,
"persistence_mode": false,
"compute_mode": "Default",
"serial_number": "1651924027166",
"ecc_errors_single": 0,
"ecc_errors_double": 0
},
{
"index": 6,
"name": "NVIDIA H100 80GB HBM3",
"uuid": "GPU-98745a0c-39bd-3e56-d6ca-54ba3647ab6d",
"pci_bus_id": "00000000:BA:00.0",
"pcie_link_gen": 5,
"pcie_link_width": 16,
"vram_total_mb": 81559,
"vram_used_mb": 0,
"vram_free_mb": 81079,
"power_draw": 67.61,
"power_limit": 700.0,
"clock_sm": 345,
"clock_mem": 2619,
"temperature": 20,
"fan_speed": 0,
"persistence_mode": false,
"compute_mode": "Default",
"serial_number": "1651924026234",
"ecc_errors_single": 0,
"ecc_errors_double": 0
},
{
"index": 7,
"name": "NVIDIA H100 80GB HBM3",
"uuid": "GPU-8c73bd8b-666b-357e-ac5d-c75ac7a759db",
"pci_bus_id": "00000000:DB:00.0",
"pcie_link_gen": 5,
"pcie_link_width": 16,
"vram_total_mb": 81559,
"vram_used_mb": 0,
"vram_free_mb": 81079,
"power_draw": 66.19,
"power_limit": 700.0,
"clock_sm": 345,
"clock_mem": 2619,
"temperature": 20,
"fan_speed": 0,
"persistence_mode": false,
"compute_mode": "Default",
"serial_number": "1651924027255",
"ecc_errors_single": 0,
"ecc_errors_double": 0
}
],
"topology": "\t\u001b[4mGPU0\tGPU1\tGPU2\tGPU3\tGPU4\tGPU5\tGPU6\tGPU7\tNIC0\tNIC1\tNIC2\tNIC3\tNIC4\tNIC5\tNIC6\tNIC7\tNIC8\tNIC9\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\u001b[0m\nGPU0\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tPIX\tNODE\tNODE\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\t0-55,112-167\t0\t\tN/A\nGPU1\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tNODE\tPIX\tNODE\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\t0-55,112-167\t0\t\tN/A\nGPU2\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tNODE\tNODE\tPIX\tPIX\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\t0-55,112-167\t0\t\tN/A\nGPU3\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tNODE\tNODE\tNODE\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\t0-55,112-167\t0\t\tN/A\nGPU4\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tPIX\tNODE\tNODE\tNODE\t56-111,168-223\t1\t\tN/A\nGPU5\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tNODE\tPIX\tNODE\tNODE\t56-111,168-223\t1\t\tN/A\nGPU6\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tPIX\tPIX\t56-111,168-223\t1\t\tN/A\nGPU7\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tNODE\t56-111,168-223\t1\t\tN/A\nNIC0\tPIX\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\t X \tNODE\tNODE\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\t\t\t\t\nNIC1\tNODE\tPIX\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tNODE\t X \tNODE\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\t\t\t\t\nNIC2\tNODE\tNODE\tPIX\tNODE\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\t X \tPIX\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\t\t\t\t\nNIC3\tNODE\tNODE\tPIX\tNODE\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tPIX\t X \tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\t\t\t\t\nNIC4\tNODE\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tNODE\t X \tPIX\tSYS\tSYS\tSYS\tSYS\t\t\t\t\nNIC5\tNODE\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tNODE\tNODE\tPIX\t X \tSYS\tSYS\tSYS\tSYS\t\t\t\t\nNIC6\tSYS\tSYS\tSYS\tSYS\tPIX\tNODE\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\t X \tNODE\tNODE\tNODE\t\t\t\t\nNIC7\tSYS\tSYS\tSYS\tSYS\tNODE\tPIX\tNODE\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tNODE\t X \tNODE\tNODE\t\t\t\t\nNIC8\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tPIX\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\t X \tPIX\t\t\t\t\nNIC9\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tPIX\tNODE\tSYS\tSYS\tSYS\tSYS\tSYS\tSYS\tNODE\tNODE\tPIX\t X \t\t\t\t\n\nLegend:\n\n X = Self\n SYS = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)\n NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node\n PHB = Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU)\n PXB = Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge)\n PIX = Connection traversing at most a single PCIe bridge\n NV# = Connection traversing a bonded set of # NVLinks\n\nNIC Legend:\n\n NIC0: mlx5_0\n NIC1: mlx5_1\n NIC2: mlx5_2\n NIC3: mlx5_3\n NIC4: mlx5_4\n NIC5: mlx5_5\n NIC6: mlx5_6\n NIC7: mlx5_7\n NIC8: mlx5_8\n NIC9: mlx5_9\n\n",
"timestamp": "2026-05-22T15:26:36.627805",
"detected_gpu_type": "h100",
"gpu_label": "H100 SXM5"
},
"memory_bench": {
"memory": {
"source": "pytorch",
"h2d_bandwidth_gbps": 11.8,
"d2h_bandwidth_gbps": 10.1,
"d2d_bandwidth_gbps": 829.0,
"peak_bandwidth_gbps": 3400,
"efficiency_pct": 24.4,
"test_sizes_mb": [
1,
4,
16,
64,
256,
1024,
4096
],
"bandwidth_by_size": {
"1": {
"h2d_gbps": 3.6,
"d2h_gbps": 1.4,
"d2d_gbps": 40.3
},
"4": {
"h2d_gbps": 7.7,
"d2h_gbps": 10.1,
"d2d_gbps": 159.5
},
"16": {
"h2d_gbps": 10.9,
"d2h_gbps": 1.9,
"d2d_gbps": 439.5
},
"64": {
"h2d_gbps": 11.8,
"d2h_gbps": 1.4,
"d2d_gbps": 740.5
},
"256": {
"h2d_gbps": 9.0,
"d2h_gbps": 1.4,
"d2d_gbps": 792.1
},
"1024": {
"h2d_gbps": 8.4,
"d2h_gbps": 1.4,
"d2d_gbps": 818.9
},
"4096": {
"h2d_gbps": 6.1,
"d2h_gbps": 1.4,
"d2d_gbps": 829.0
}
},
"per_gpu": []
}
},
"compute_bench": {
"compute": {
"per_dtype_tflops": {
"fp32": 51.9,
"tf32": 357.8,
"fp16": 667.2,
"bf16": 699.1,
"fp8": 1146.2
},
"peak_tflops": {
"fp32": 67,
"tf32": 495,
"fp16": 990,
"bf16": 990,
"fp8": 1979
},
"efficiency_pct": {
"fp32": 77.5,
"tf32": 72.3,
"fp16": 67.4,
"bf16": 70.6,
"fp8": 57.9
},
"pass_thresholds_tflops": {
"fp32": 54,
"tf32": 444,
"fp16": 734,
"bf16": 745,
"fp8": 1400
},
"per_gpu": [
{
"index": 0,
"fp32": 51.9,
"tf32": 357.8,
"fp16": 667.2,
"bf16": 699.1,
"fp8": 1146.2
}
],
"matrix_size": 8192,
"warmup": 50,
"iterations": 500
}
}
}
Reference in New Issue View Git Blame Copy Permalink