Files
infrastructure/llm-server/monitoring/scripts/amdgpu-textfile-collector.py
spektr b96be7ae1c GNMON Kosmetik: GPU-Label via PCI-ID-Fallback + Geist-Favicon (Dashboard v0.3.3)
- amdgpu-textfile-collector.py: product_name existiert im sysfs der RX 6900 XT
  nicht (gpu="unknown"); Fallback auf PCI-Device-ID (0x73bf -> AMD Radeon RX
  6900 XT (Navi 21)). Live verifiziert: gn_gpu_up{card="0|1",gpu="AMD Radeon
  RX 6900 XT (Navi 21)"} 1 in VictoriaMetrics.
- Dashboard v0.3.3: favicon.png (32x32) + apple-touch-icon.png (180x180) aus
  dem Betreiber-Maskottchen (Dokumente/ghost.png) gerendert; via NPM als
  image/png 200 verifiziert (2729 B favicon). App.razor referenziert beide.
- Server-Image-Store: v0.3.3 + Rollback v0.2.0/v0.1.0; monitor.yml gesamt
  ok=84 changed=0.

Generated by Mistral Vibe.
Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
2026-11-09 20:19:25 +01:00

253 lines
8.3 KiB
Python
Executable File

#!/usr/bin/env python3
"""AMD GPU textfile collector for Prometheus node_exporter (GNMON-001 v2.0).
Reads amdgpu metrics directly from sysfs (no rocm-smi, no ROCm runtime
required) and writes them as a Prometheus textfile exposition for the
node_exporter textfile collector. Same data source as the thermal load
profile from 2026-11-01 (GNAI-010 section 10.8).
Per AMD GPU (vendor 0x1002) the following metrics are emitted when the
underlying sysfs file exists (APUs expose a subset, discrete cards all):
gn_gpu_up{card,gpu}
gn_gpu_utilization_percent{card,gpu}
gn_gpu_vram_used_bytes{card,gpu}
gn_gpu_vram_total_bytes{card,gpu}
gn_gpu_edge_temp_celsius{card,gpu}
gn_gpu_junction_temp_celsius{card,gpu}
gn_gpu_memory_temp_celsius{card,gpu}
gn_gpu_fan_rpm{card,gpu}
gn_gpu_fan_pwm_percent{card,gpu}
gn_gpu_power_watts{card,gpu}
The output file is written atomically (tmp + rename) with mode 0644 so the
node_exporter textfile directory never exposes a partial file.
Usage: amdgpu-textfile-collector.py --textfile-dir DIR [--drm-root /sys/class/drm]
"""
from __future__ import annotations
import argparse
import os
import sys
import tempfile
from pathlib import Path
AMD_VENDOR_ID = "0x1002"
# PCI device-ID fallback for the gpu label: on the discrete RX 6900 XT hosts
# sysfs has no product_name file (Befund 09.11.2026: gpu="unknown"), so map
# the well-known Navi-21 IDs explicitly. Unknown IDs fall back to the raw hex.
AMD_PCI_DEVICE_NAMES = {
"0x73bf": "AMD Radeon RX 6900 XT (Navi 21)",
"0x73df": "AMD Radeon RX 6950 XT (Navi 21)",
"0x73ff": "AMD Radeon RX 6900 XTX (Navi 21)",
}
def gpu_label(device_dir: Path) -> str:
"""Product name from sysfs, falling back to the PCI device ID."""
name = _read_text(device_dir / "product_name")
if name:
return name
device_id = _read_text(device_dir / "device")
if device_id:
return AMD_PCI_DEVICE_NAMES.get(
device_id.lower(), f"AMD PCI GPU {device_id}"
)
return "unknown"
def _read_int(path: Path) -> int | None:
try:
return int(path.read_text().strip())
except (OSError, ValueError):
return None
def _read_text(path: Path) -> str | None:
try:
return path.read_text().strip()
except OSError:
return None
def _read_hwmon_temps(device_dir: Path) -> dict[str, float]:
"""Map hwmon temperature labels (edge/junction/mem/mcd) to celsius."""
temps: dict[str, float] = {}
for hwmon_dir in sorted(device_dir.glob("hwmon/hwmon*")):
for input_path in sorted(hwmon_dir.glob("temp*_input")):
value = _read_int(input_path)
if value is None:
continue
label_path = input_path.with_name(
input_path.name.replace("_input", "_label")
)
label = _read_text(label_path) or ""
label = label.lower()
if "junction" in label:
temps["junction"] = value / 1000.0
elif "edge" in label:
temps["edge"] = value / 1000.0
elif "mem" in label or "mcd" in label:
temps.setdefault("memory", value / 1000.0)
return temps
def collect(drm_root: Path) -> list[str]:
lines: list[str] = []
declared: set[str] = set()
def declare(metric: str, help_text: str, metric_type: str) -> None:
# HELP/TYPE must appear once per metric family; repeating them per
# card makes node_exporter reject the whole textfile.
if metric in declared:
return
lines.append(f"# HELP {metric} {help_text}")
lines.append(f"# TYPE {metric} {metric_type}")
declared.add(metric)
for device_link in sorted(drm_root.glob("card*")):
if "-" in device_link.name:
continue # connector entries like card0-eDP-1
device_dir = device_link / "device"
if not device_dir.is_dir():
continue
vendor = _read_text(device_dir / "vendor")
if vendor != AMD_VENDOR_ID:
continue
card = device_link.name.removeprefix("card")
gpu = gpu_label(device_dir)
def emit(metric: str, help_text: str, metric_type: str, value, **labels) -> None:
declare(metric, help_text, metric_type)
if value is None:
return
label_pairs = {"card": card, "gpu": gpu, **labels}
label_str = ",".join(f'{key}="{val}"' for key, val in label_pairs.items())
lines.append(f"{metric}{{{label_str}}} {value}")
emit("gn_gpu_up", "AMD GPU is visible in sysfs", "gauge", 1)
emit(
"gn_gpu_utilization_percent",
"GPU utilization in percent (amdgpu sysfs)",
"gauge",
_read_int(device_dir / "gpu_busy_percent"),
)
emit(
"gn_gpu_vram_used_bytes",
"Used VRAM in bytes",
"gauge",
_read_int(device_dir / "mem_info_vram_used"),
)
emit(
"gn_gpu_vram_total_bytes",
"Total VRAM in bytes",
"gauge",
_read_int(device_dir / "mem_info_vram_total"),
)
temps = _read_hwmon_temps(device_dir)
emit(
"gn_gpu_edge_temp_celsius",
"GPU edge temperature in celsius",
"gauge",
temps.get("edge"),
)
emit(
"gn_gpu_junction_temp_celsius",
"GPU junction (hotspot) temperature in celsius",
"gauge",
temps.get("junction"),
)
emit(
"gn_gpu_memory_temp_celsius",
"GPU memory temperature in celsius",
"gauge",
temps.get("memory"),
)
for hwmon_dir in sorted(device_dir.glob("hwmon/hwmon*")):
for fan_path in sorted(hwmon_dir.glob("fan*_input")):
fan_index = fan_path.name.removeprefix("fan").removesuffix("_input")
rpm = _read_int(fan_path)
if rpm is None:
continue
emit(
"gn_gpu_fan_rpm",
"GPU fan speed in rpm",
"gauge",
rpm,
fan=fan_index,
)
pwm = _read_int(hwmon_dir / "pwm1")
if pwm is not None:
emit(
"gn_gpu_fan_pwm_percent",
"Requested fan PWM in percent",
"gauge",
round(pwm / 255.0 * 100, 2),
)
# AMD power sensors report microwatts (power1_average preferred).
for power_name in ("power1_average", "power1_input"):
microwatts = _read_int(hwmon_dir / power_name)
if microwatts is not None:
emit(
"gn_gpu_power_watts",
"GPU board power in watts",
"gauge",
round(microwatts / 1e6, 3),
)
break
return lines
def write_textfile(lines: list[str], textfile_dir: Path) -> Path:
textfile_dir.mkdir(parents=True, exist_ok=True)
target = textfile_dir / "amdgpu.prom"
fd, tmp_name = tempfile.mkstemp(
dir=textfile_dir, prefix=".amdgpu.prom.", suffix=".tmp"
)
try:
with os.fdopen(fd, "w", encoding="ascii") as handle:
handle.write("\n".join(lines) + "\n")
os.chmod(tmp_name, 0o644)
os.replace(tmp_name, target)
except BaseException:
try:
os.unlink(tmp_name)
except OSError:
pass
raise
return target
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--textfile-dir",
type=Path,
default=Path("/var/lib/node_exporter/textfile"),
help="node_exporter textfile directory (default: %(default)s)",
)
parser.add_argument(
"--drm-root",
type=Path,
default=Path("/sys/class/drm"),
help="sysfs DRM root (default: %(default)s)",
)
parser.add_argument(
"--stdout", action="store_true", help="print exposition to stdout instead"
)
args = parser.parse_args()
lines = collect(args.drm_root)
if args.stdout:
print("\n".join(lines))
return 0
write_textfile(lines, args.textfile_dir)
return 0
if __name__ == "__main__":
sys.exit(main())