{
  "tool": "VoltageGPU hwcheck acceptance test",
  "version": "1.1",
  "started_utc": "2026-10-24 15:46:57 UTC",
  "host": "bm-187f9222",
  "options": {"quick": false, "burn_seconds": 300, "gpus_listed": 8, "bare_metal": true, "install": true},
  "versions": {"driver": "595.112.07", "cuda": "13.2", "torch": "2.14.1+cu130", "torch_cuda": "13.0", "torch_source": "installed for this run, cu130", "nccl": "2.30.7", "system_libnccl2": "", "os": "Ubuntu 24.04.5 LTS", "kernel": "6.8.0-139-generic"},
  "virtualization": "none",
  "gpus": [
    {"index": 0, "name": "NVIDIA H100 80GB HBM3", "uuid": "GPU-7782710d-f9fd-498f-0210-9c0914e1a7d2", "serial": "1654923000571", "pci_bus_id": "00000000:18:00.0", "vbios": "96.00.DA.00.0C", "memory_mib": 81559},
    {"index": 1, "name": "NVIDIA H100 80GB HBM3", "uuid": "GPU-96ac5618-25d6-5696-5d79-d766211e11a2", "serial": "1655023028920", "pci_bus_id": "00000000:2A:00.0", "vbios": "96.00.DA.00.0C", "memory_mib": 81559},
    {"index": 2, "name": "NVIDIA H100 80GB HBM3", "uuid": "GPU-7ed715d4-dfdc-1ecb-46ff-0acd0c56e5af", "serial": "1654923002085", "pci_bus_id": "00000000:3A:00.0", "vbios": "96.00.DA.00.0C", "memory_mib": 81559},
    {"index": 3, "name": "NVIDIA H100 80GB HBM3", "uuid": "GPU-cafa2ecc-dbb7-910f-6736-caea4000194b", "serial": "1654923002164", "pci_bus_id": "00000000:5D:00.0", "vbios": "96.00.DA.00.0C", "memory_mib": 81559},
    {"index": 4, "name": "NVIDIA H100 80GB HBM3", "uuid": "GPU-acf63434-feb5-5e16-bb25-4c0285d948aa", "serial": "1654823034886", "pci_bus_id": "00000000:9A:00.0", "vbios": "96.00.DA.00.0C", "memory_mib": 81559},
    {"index": 5, "name": "NVIDIA H100 80GB HBM3", "uuid": "GPU-4dcc0b58-5fd6-1664-027a-074834a5ab22", "serial": "1654823017574", "pci_bus_id": "00000000:AB:00.0", "vbios": "96.00.DA.00.0C", "memory_mib": 81559},
    {"index": 6, "name": "NVIDIA H100 80GB HBM3", "uuid": "GPU-176c6e35-f73a-c5f5-87a5-4ff97382c589", "serial": "1654723004653", "pci_bus_id": "00000000:BA:00.0", "vbios": "96.00.DA.00.0C", "memory_mib": 81559},
    {"index": 7, "name": "NVIDIA H100 80GB HBM3", "uuid": "GPU-e53bf323-ab7e-fea4-70f9-43b6a05a5b46", "serial": "1654623009511", "pci_bus_id": "00000000:DB:00.0", "vbios": "96.00.DA.00.0C", "memory_mib": 81559}
  ],
  "checks": [
    {"section": "1 Topology", "name": "GPUs", "status": "PASS", "summary": "8x NVIDIA H100 80GB HBM3", "data": null},
    {"section": "1 Topology", "name": "NVLink mesh", "status": "PASS", "summary": "56/56 GPU pairs on NV18 (NV18 expected on H100 SXM)", "data": null},
    {"section": "1 Topology", "name": "NVSwitch", "status": "PASS", "summary": "4 NVSwitch chips visible (4 on an 8-GPU H100 SXM board)", "data": null},
    {"section": "1 Topology", "name": "Fabric state", "status": "PASS", "summary": "Completed / Success on 8/8 GPUs", "data": null},
    {"section": "1 Topology", "name": "PCIe links", "status": "PASS", "summary": "Gen5 x16 on 8/8 GPUs under load, host-to-GPU copies 53.1 to 55.5 GB/s", "data": null},
    {"section": "2 P2P", "name": "CUDA init", "status": "PASS", "summary": "8/8 GPUs in PyTorch 2.14.1+cu130 (installed for this run, cu130)", "data": null},
    {"section": "2 P2P", "name": "P2P copies", "status": "PASS", "summary": "slowest pair 387 GB/s (GPU 0>2), median 389 GB/s (FAIL under 112, WARN under 270: H100 SXM line rate 450 GB/s)", "data": {"gb_per_s": [[null, 388.7, 386.8, 389.9, 387.9, 388.6, 387.9, 388.4], [390.1, null, 388.8, 389.6, 388.3, 388.6, 388.1, 388.3], [389.3, 389.4, null, 386.9, 387.8, 388.6, 388.7, 389.2], [387.9, 388.8, 388.5, null, 388.4, 387.9, 387.7, 387.5], [389.5, 387.3, 390.5, 388.6, null, 388.3, 389.7, 389.8], [389.1, 388.6, 390.1, 389.2, 387.4, null, 390.6, 387.0], [390.2, 387.4, 390.0, 389.1, 391.3, 391.4, null, 390.2], [389.0, 390.7, 388.5, 389.3, 389.8, 391.3, 388.8, null]], "peer_access": [[true, true, true, true, true, true, true, true], [true, true, true, true, true, true, true, true], [true, true, true, true, true, true, true, true], [true, true, true, true, true, true, true, true], [true, true, true, true, true, true, true, true], [true, true, true, true, true, true, true, true], [true, true, true, true, true, true, true, true], [true, true, true, true, true, true, true, true]], "host_to_gpu_gb_per_s": [55.5, 53.2, 55.5, 53.1, 53.9, 54.3, 53.9, 55.3], "bytes": 268435456, "iterations": 20, "corrupted_pairs": []}},
    {"section": "3 NCCL", "name": "NCCL all-reduce", "status": "PASS", "summary": "bus bandwidth 478.5 GB/s on 8 GPUs (4 GiB, NCCL 2.30.7); H100 SXM: about 370 GB/s in ring, about 480 with NVLink SHARP, PASS from 340", "data": {"bytes":4294967296,"ranks":8,"algbw_gb_per_s":273.4,"busbw_gb_per_s":478.5,"time_us":15709,"wrong_values":0}},
    {"section": "4 Load", "name": "Throughput", "status": "PASS", "summary": "8/8 GPUs ran it, median 684 TFLOPS (bfloat16), slowest GPU 2 at 669", "data": [{"gpu":0,"samples":60,"busy_median":100,"temp_max_c":58,"sm_mhz_median":1365,"power_max_w":702,"tflops":687.0,"changed_results":0,"power_cap_samples":57,"thermal_samples":0,"hw_slowdown_samples":0,"power_brake_samples":0},{"gpu":1,"samples":60,"busy_median":100,"temp_max_c":65,"sm_mhz_median":1350,"power_max_w":701,"tflops":676.3,"changed_results":0,"power_cap_samples":57,"thermal_samples":0,"hw_slowdown_samples":0,"power_brake_samples":0},{"gpu":2,"samples":60,"busy_median":100,"temp_max_c":69,"sm_mhz_median":1335,"power_max_w":700,"tflops":669.2,"changed_results":0,"power_cap_samples":57,"thermal_samples":0,"hw_slowdown_samples":0,"power_brake_samples":0},{"gpu":3,"samples":60,"busy_median":100,"temp_max_c":58,"sm_mhz_median":1372,"power_max_w":701,"tflops":684.5,"changed_results":0,"power_cap_samples":57,"thermal_samples":0,"hw_slowdown_samples":0,"power_brake_samples":0},{"gpu":4,"samples":60,"busy_median":100,"temp_max_c":62,"sm_mhz_median":1380,"power_max_w":701,"tflops":684.6,"changed_results":0,"power_cap_samples":57,"thermal_samples":0,"hw_slowdown_samples":0,"power_brake_samples":0},{"gpu":5,"samples":60,"busy_median":100,"temp_max_c":65,"sm_mhz_median":1380,"power_max_w":701,"tflops":688.7,"changed_results":0,"power_cap_samples":57,"thermal_samples":0,"hw_slowdown_samples":0,"power_brake_samples":0},{"gpu":6,"samples":60,"busy_median":100,"temp_max_c":65,"sm_mhz_median":1365,"power_max_w":702,"tflops":683.3,"changed_results":0,"power_cap_samples":57,"thermal_samples":0,"hw_slowdown_samples":0,"power_brake_samples":0},{"gpu":7,"samples":60,"busy_median":100,"temp_max_c":60,"sm_mhz_median":1380,"power_max_w":701,"tflops":682.9,"changed_results":0,"power_cap_samples":58,"thermal_samples":0,"hw_slowdown_samples":0,"power_brake_samples":0}]},
    {"section": "4 Load", "name": "Thermal / power", "status": "PASS", "summary": "max 69 C (GPU 2), no thermal or hardware slowdown; at the power cap in 95% of the samples (normal under full load)", "data": null},
    {"section": "4 Load", "name": "Clocks", "status": "PASS", "summary": "median SM clock 1368 MHz, every GPU between 1335 and 1380 MHz, at least 90% busy", "data": null},
    {"section": "5 Errors", "name": "ECC / memory", "status": "PASS", "summary": "ECC on 8/8 GPUs, no uncorrected error, nothing pending, 0 corrected since the driver loaded", "data": [{"gpu":0,"ecc_mode":"Enabled","corrected_volatile":"0","uncorrected_volatile":"0","corrected_aggregate":"0","uncorrected_aggregate":"0","retired_sbe":"[N/A]","retired_dbe":"[N/A]","retired_pending":"[N/A]","remapped_correctable":"0","remapped_uncorrectable":"0","remap_pending":"No","remap_failure":"No"},{"gpu":1,"ecc_mode":"Enabled","corrected_volatile":"0","uncorrected_volatile":"0","corrected_aggregate":"0","uncorrected_aggregate":"0","retired_sbe":"[N/A]","retired_dbe":"[N/A]","retired_pending":"[N/A]","remapped_correctable":"0","remapped_uncorrectable":"0","remap_pending":"No","remap_failure":"No"},{"gpu":2,"ecc_mode":"Enabled","corrected_volatile":"0","uncorrected_volatile":"0","corrected_aggregate":"0","uncorrected_aggregate":"0","retired_sbe":"[N/A]","retired_dbe":"[N/A]","retired_pending":"[N/A]","remapped_correctable":"0","remapped_uncorrectable":"0","remap_pending":"No","remap_failure":"No"},{"gpu":3,"ecc_mode":"Enabled","corrected_volatile":"0","uncorrected_volatile":"0","corrected_aggregate":"0","uncorrected_aggregate":"0","retired_sbe":"[N/A]","retired_dbe":"[N/A]","retired_pending":"[N/A]","remapped_correctable":"0","remapped_uncorrectable":"0","remap_pending":"No","remap_failure":"No"},{"gpu":4,"ecc_mode":"Enabled","corrected_volatile":"0","uncorrected_volatile":"0","corrected_aggregate":"0","uncorrected_aggregate":"0","retired_sbe":"[N/A]","retired_dbe":"[N/A]","retired_pending":"[N/A]","remapped_correctable":"0","remapped_uncorrectable":"0","remap_pending":"No","remap_failure":"No"},{"gpu":5,"ecc_mode":"Enabled","corrected_volatile":"0","uncorrected_volatile":"0","corrected_aggregate":"0","uncorrected_aggregate":"0","retired_sbe":"[N/A]","retired_dbe":"[N/A]","retired_pending":"[N/A]","remapped_correctable":"0","remapped_uncorrectable":"0","remap_pending":"No","remap_failure":"No"},{"gpu":6,"ecc_mode":"Enabled","corrected_volatile":"0","uncorrected_volatile":"0","corrected_aggregate":"0","uncorrected_aggregate":"0","retired_sbe":"[N/A]","retired_dbe":"[N/A]","retired_pending":"[N/A]","remapped_correctable":"0","remapped_uncorrectable":"0","remap_pending":"No","remap_failure":"No"},{"gpu":7,"ecc_mode":"Enabled","corrected_volatile":"0","uncorrected_volatile":"0","corrected_aggregate":"0","uncorrected_aggregate":"0","retired_sbe":"[N/A]","retired_dbe":"[N/A]","retired_pending":"[N/A]","remapped_correctable":"0","remapped_uncorrectable":"0","remap_pending":"No","remap_failure":"No"}]},
    {"section": "5 Errors", "name": "Kernel log (Xid)", "status": "PASS", "summary": "no Xid during the test, read with journalctl", "data": []},
    {"section": "6 Hygiene", "name": "Previous tenant", "status": "PASS", "summary": "nothing found in the home directories, /tmp, /var/tmp and /workspace; not readable: /root (run as root to include it)", "data": []},
    {"section": "6 Hygiene", "name": "SSH keys", "status": "PASS", "summary": "1 key can log in (ubuntu: 1)", "data": null},
    {"section": "6 Hygiene", "name": "GPUs idle at start", "status": "PASS", "summary": "no compute process, no GPU memory in use", "data": null},
    {"section": "6 Hygiene", "name": "Virtualization", "status": "PASS", "summary": "none detected (systemd-detect-virt): bare metal, as listed", "data": null}
  ],
  "raw": {"nvidia_smi_topo": "\tGPU0\tGPU1\tGPU2\tGPU3\tGPU4\tGPU5\tGPU6\tGPU7\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\nGPU0\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t0-23,96-119\t0\t\tN/A\nGPU1\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t0-23,96-119\t0\t\tN/A\nGPU2\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tNV18\t0-23,96-119\t0\t\tN/A\nGPU3\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tNV18\t24-47,120-143\t1\t\tN/A\nGPU4\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tNV18\t48-71,144-167\t2\t\tN/A\nGPU5\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNV18\t48-71,144-167\t2\t\tN/A\nGPU6\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNV18\t48-71,144-167\t2\t\tN/A\nGPU7\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t X \t72-95,168-191\t3\t\tN/A\n\nLegend:\n\n  X    = Self\n  SYS  = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)\n  NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node\n  PHB  = Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU)\n  PXB  = Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge)\n  PIX  = Connection traversing at most a single PCIe bridge\n  NV#  = Connection traversing a bonded set of # NVLinks", "lspci_nvidia": "05:00.0 Bridge [0680]: NVIDIA Corporation GH100 [H100 NVSwitch] [10de:22a3] (rev a1)\n06:00.0 Bridge [0680]: NVIDIA Corporation GH100 [H100 NVSwitch] [10de:22a3] (rev a1)\n07:00.0 Bridge [0680]: NVIDIA Corporation GH100 [H100 NVSwitch] [10de:22a3] (rev a1)\n08:00.0 Bridge [0680]: NVIDIA Corporation GH100 [H100 NVSwitch] [10de:22a3] (rev a1)\n18:00.0 3D controller [0302]: NVIDIA Corporation GH100 [H100 SXM5 80GB] [10de:2330] (rev a1)\n2a:00.0 3D controller [0302]: NVIDIA Corporation GH100 [H100 SXM5 80GB] [10de:2330] (rev a1)\n3a:00.0 3D controller [0302]: NVIDIA Corporation GH100 [H100 SXM5 80GB] [10de:2330] (rev a1)\n5d:00.0 3D controller [0302]: NVIDIA Corporation GH100 [H100 SXM5 80GB] [10de:2330] (rev a1)\n9a:00.0 3D controller [0302]: NVIDIA Corporation GH100 [H100 SXM5 80GB] [10de:2330] (rev a1)\nab:00.0 3D controller [0302]: NVIDIA Corporation GH100 [H100 SXM5 80GB] [10de:2330] (rev a1)\nba:00.0 3D controller [0302]: NVIDIA Corporation GH100 [H100 SXM5 80GB] [10de:2330] (rev a1)\ndb:00.0 3D controller [0302]: NVIDIA Corporation GH100 [H100 SXM5 80GB] [10de:2330] (rev a1)"},
  "result": "PASS",
  "exit_code": 0
}