GNMON-001 M1: Metrikpfad auf der Workstation implementiert
llm-server/monitoring/ neu: amdgpu-textfile-collector.py (sysfs-only, atomar, APU-tauglich), Rollen gnmon_collector (node_exporter v1.12.1, SHA256SUMS-verifiziert, 10s-Timer) und gnmon_victoria_metrics (rootless Quadlet, v1.154.1 digest-gepinnt, Scrape 10s, Retention 60d), Workstation- Inventory. Verifikation: PromQL liefert gn_gpu_edge_temp_celsius von der Laptop-APU; Idempotenz changed=0. GNMON-001 v2.0 in die Plattform-Struktur übernommen, GNMON-000-Stand inkl. Alerting-Planlücke aktualisiert. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
+213
@@ -0,0 +1,213 @@
|
||||
#!/usr/bin/env python3
|
||||
"""AMD GPU textfile collector for Prometheus node_exporter (GNMON-001 v2.0).
|
||||
|
||||
Reads amdgpu metrics directly from sysfs (no rocm-smi, no ROCm runtime
|
||||
required) and writes them as a Prometheus textfile exposition for the
|
||||
node_exporter textfile collector. Same data source as the thermal load
|
||||
profile from 2026-11-01 (GNAI-010 section 10.8).
|
||||
|
||||
Per AMD GPU (vendor 0x1002) the following metrics are emitted when the
|
||||
underlying sysfs file exists (APUs expose a subset, discrete cards all):
|
||||
|
||||
gn_gpu_up{card,gpu}
|
||||
gn_gpu_utilization_percent{card,gpu}
|
||||
gn_gpu_vram_used_bytes{card,gpu}
|
||||
gn_gpu_vram_total_bytes{card,gpu}
|
||||
gn_gpu_edge_temp_celsius{card,gpu}
|
||||
gn_gpu_junction_temp_celsius{card,gpu}
|
||||
gn_gpu_memory_temp_celsius{card,gpu}
|
||||
gn_gpu_fan_rpm{card,gpu}
|
||||
gn_gpu_fan_pwm_percent{card,gpu}
|
||||
gn_gpu_power_watts{card,gpu}
|
||||
|
||||
The output file is written atomically (tmp + rename) with mode 0644 so the
|
||||
node_exporter textfile directory never exposes a partial file.
|
||||
|
||||
Usage: amdgpu-textfile-collector.py --textfile-dir DIR [--drm-root /sys/class/drm]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
AMD_VENDOR_ID = "0x1002"
|
||||
|
||||
|
||||
def _read_int(path: Path) -> int | None:
|
||||
try:
|
||||
return int(path.read_text().strip())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _read_text(path: Path) -> str | None:
|
||||
try:
|
||||
return path.read_text().strip()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _read_hwmon_temps(device_dir: Path) -> dict[str, float]:
|
||||
"""Map hwmon temperature labels (edge/junction/mem/mcd) to celsius."""
|
||||
temps: dict[str, float] = {}
|
||||
for hwmon_dir in sorted(device_dir.glob("hwmon/hwmon*")):
|
||||
for input_path in sorted(hwmon_dir.glob("temp*_input")):
|
||||
value = _read_int(input_path)
|
||||
if value is None:
|
||||
continue
|
||||
label_path = input_path.with_name(
|
||||
input_path.name.replace("_input", "_label")
|
||||
)
|
||||
label = _read_text(label_path) or ""
|
||||
label = label.lower()
|
||||
if "junction" in label:
|
||||
temps["junction"] = value / 1000.0
|
||||
elif "edge" in label:
|
||||
temps["edge"] = value / 1000.0
|
||||
elif "mem" in label or "mcd" in label:
|
||||
temps.setdefault("memory", value / 1000.0)
|
||||
return temps
|
||||
|
||||
|
||||
def collect(drm_root: Path) -> list[str]:
|
||||
lines: list[str] = []
|
||||
for device_link in sorted(drm_root.glob("card*")):
|
||||
if "-" in device_link.name:
|
||||
continue # connector entries like card0-eDP-1
|
||||
device_dir = device_link / "device"
|
||||
if not device_dir.is_dir():
|
||||
continue
|
||||
vendor = _read_text(device_dir / "vendor")
|
||||
if vendor != AMD_VENDOR_ID:
|
||||
continue
|
||||
card = device_link.name.removeprefix("card")
|
||||
gpu = _read_text(device_dir / "product_name") or "unknown"
|
||||
|
||||
def emit(metric: str, value, help_text: str, metric_type: str) -> None:
|
||||
lines.append(f"# HELP {metric} {help_text}")
|
||||
lines.append(f"# TYPE {metric} {metric_type}")
|
||||
if value is not None:
|
||||
lines.append(f'{metric}{{card="{card}",gpu="{gpu}"}} {value}')
|
||||
|
||||
emit("gn_gpu_up", 1, "AMD GPU is visible in sysfs", "gauge")
|
||||
emit(
|
||||
"gn_gpu_utilization_percent",
|
||||
_read_int(device_dir / "gpu_busy_percent"),
|
||||
"GPU utilization in percent (amdgpu sysfs)",
|
||||
"gauge",
|
||||
)
|
||||
emit(
|
||||
"gn_gpu_vram_used_bytes",
|
||||
_read_int(device_dir / "mem_info_vram_used"),
|
||||
"Used VRAM in bytes",
|
||||
"gauge",
|
||||
)
|
||||
emit(
|
||||
"gn_gpu_vram_total_bytes",
|
||||
_read_int(device_dir / "mem_info_vram_total"),
|
||||
"Total VRAM in bytes",
|
||||
"gauge",
|
||||
)
|
||||
temps = _read_hwmon_temps(device_dir)
|
||||
emit(
|
||||
"gn_gpu_edge_temp_celsius",
|
||||
temps.get("edge"),
|
||||
"GPU edge temperature in celsius",
|
||||
"gauge",
|
||||
)
|
||||
emit(
|
||||
"gn_gpu_junction_temp_celsius",
|
||||
temps.get("junction"),
|
||||
"GPU junction (hotspot) temperature in celsius",
|
||||
"gauge",
|
||||
)
|
||||
emit(
|
||||
"gn_gpu_memory_temp_celsius",
|
||||
temps.get("memory"),
|
||||
"GPU memory temperature in celsius",
|
||||
"gauge",
|
||||
)
|
||||
for hwmon_dir in sorted(device_dir.glob("hwmon/hwmon*")):
|
||||
for fan_path in sorted(hwmon_dir.glob("fan*_input")):
|
||||
fan_index = fan_path.name.removeprefix("fan").removesuffix("_input")
|
||||
rpm = _read_int(fan_path)
|
||||
if rpm is None:
|
||||
continue
|
||||
lines.append(f"# HELP gn_gpu_fan_rpm GPU fan speed in rpm")
|
||||
lines.append(f"# TYPE gn_gpu_fan_rpm gauge")
|
||||
lines.append(
|
||||
f'gn_gpu_fan_rpm{{card="{card}",gpu="{gpu}",fan="{fan_index}"}} {rpm}'
|
||||
)
|
||||
pwm = _read_int(hwmon_dir / "pwm1")
|
||||
if pwm is not None:
|
||||
lines.append(f"# HELP gn_gpu_fan_pwm_percent Requested fan PWM in percent")
|
||||
lines.append(f"# TYPE gn_gpu_fan_pwm_percent gauge")
|
||||
lines.append(
|
||||
f'gn_gpu_fan_pwm_percent{{card="{card}",gpu="{gpu}"}} {pwm / 255.0 * 100:.2f}'
|
||||
)
|
||||
# AMD power sensors report microwatts (power1_average preferred).
|
||||
for power_name in ("power1_average", "power1_input"):
|
||||
microwatts = _read_int(hwmon_dir / power_name)
|
||||
if microwatts is not None:
|
||||
lines.append(f"# HELP gn_gpu_power_watts GPU board power in watts")
|
||||
lines.append(f"# TYPE gn_gpu_power_watts gauge")
|
||||
lines.append(
|
||||
f'gn_gpu_power_watts{{card="{card}",gpu="{gpu}"}} {microwatts / 1e6:.3f}'
|
||||
)
|
||||
break
|
||||
return lines
|
||||
|
||||
|
||||
def write_textfile(lines: list[str], textfile_dir: Path) -> Path:
|
||||
textfile_dir.mkdir(parents=True, exist_ok=True)
|
||||
target = textfile_dir / "amdgpu.prom"
|
||||
fd, tmp_name = tempfile.mkstemp(
|
||||
dir=textfile_dir, prefix=".amdgpu.prom.", suffix=".tmp"
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="ascii") as handle:
|
||||
handle.write("\n".join(lines) + "\n")
|
||||
os.chmod(tmp_name, 0o644)
|
||||
os.replace(tmp_name, target)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(tmp_name)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
return target
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--textfile-dir",
|
||||
type=Path,
|
||||
default=Path("/var/lib/node_exporter/textfile"),
|
||||
help="node_exporter textfile directory (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--drm-root",
|
||||
type=Path,
|
||||
default=Path("/sys/class/drm"),
|
||||
help="sysfs DRM root (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stdout", action="store_true", help="print exposition to stdout instead"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
lines = collect(args.drm_root)
|
||||
if args.stdout:
|
||||
print("\n".join(lines))
|
||||
return 0
|
||||
write_textfile(lines, args.textfile_dir)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user