GNMON-001 M5: Monitoring-Suite auf dem KI-Server ausgerollt (System-Scope)
Rollen auf scope user|system generalisiert, gnmon_accounts-Rolle (gn-monitor 986), Testing-Inventory ghostnet-ki, Image-Transport per Tag-Ref. Fixes: Textfile-Collector HELP/TYPE je Familie (Multi-GPU), Restart-Handler je Scope, Loopback-Bindings VM/Loki (§7), scope- agnostische Playbook-Verifikationen. Server + Workstation idempotent. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
@@ -75,6 +75,17 @@ def _read_hwmon_temps(device_dir: Path) -> dict[str, float]:
|
||||
|
||||
def collect(drm_root: Path) -> list[str]:
|
||||
lines: list[str] = []
|
||||
declared: set[str] = set()
|
||||
|
||||
def declare(metric: str, help_text: str, metric_type: str) -> None:
|
||||
# HELP/TYPE must appear once per metric family; repeating them per
|
||||
# card makes node_exporter reject the whole textfile.
|
||||
if metric in declared:
|
||||
return
|
||||
lines.append(f"# HELP {metric} {help_text}")
|
||||
lines.append(f"# TYPE {metric} {metric_type}")
|
||||
declared.add(metric)
|
||||
|
||||
for device_link in sorted(drm_root.glob("card*")):
|
||||
if "-" in device_link.name:
|
||||
continue # connector entries like card0-eDP-1
|
||||
@@ -87,49 +98,51 @@ def collect(drm_root: Path) -> list[str]:
|
||||
card = device_link.name.removeprefix("card")
|
||||
gpu = _read_text(device_dir / "product_name") or "unknown"
|
||||
|
||||
def emit(metric: str, value, help_text: str, metric_type: str) -> None:
|
||||
lines.append(f"# HELP {metric} {help_text}")
|
||||
lines.append(f"# TYPE {metric} {metric_type}")
|
||||
if value is not None:
|
||||
lines.append(f'{metric}{{card="{card}",gpu="{gpu}"}} {value}')
|
||||
def emit(metric: str, help_text: str, metric_type: str, value, **labels) -> None:
|
||||
declare(metric, help_text, metric_type)
|
||||
if value is None:
|
||||
return
|
||||
label_pairs = {"card": card, "gpu": gpu, **labels}
|
||||
label_str = ",".join(f'{key}="{val}"' for key, val in label_pairs.items())
|
||||
lines.append(f"{metric}{{{label_str}}} {value}")
|
||||
|
||||
emit("gn_gpu_up", 1, "AMD GPU is visible in sysfs", "gauge")
|
||||
emit("gn_gpu_up", "AMD GPU is visible in sysfs", "gauge", 1)
|
||||
emit(
|
||||
"gn_gpu_utilization_percent",
|
||||
_read_int(device_dir / "gpu_busy_percent"),
|
||||
"GPU utilization in percent (amdgpu sysfs)",
|
||||
"gauge",
|
||||
_read_int(device_dir / "gpu_busy_percent"),
|
||||
)
|
||||
emit(
|
||||
"gn_gpu_vram_used_bytes",
|
||||
_read_int(device_dir / "mem_info_vram_used"),
|
||||
"Used VRAM in bytes",
|
||||
"gauge",
|
||||
_read_int(device_dir / "mem_info_vram_used"),
|
||||
)
|
||||
emit(
|
||||
"gn_gpu_vram_total_bytes",
|
||||
_read_int(device_dir / "mem_info_vram_total"),
|
||||
"Total VRAM in bytes",
|
||||
"gauge",
|
||||
_read_int(device_dir / "mem_info_vram_total"),
|
||||
)
|
||||
temps = _read_hwmon_temps(device_dir)
|
||||
emit(
|
||||
"gn_gpu_edge_temp_celsius",
|
||||
temps.get("edge"),
|
||||
"GPU edge temperature in celsius",
|
||||
"gauge",
|
||||
temps.get("edge"),
|
||||
)
|
||||
emit(
|
||||
"gn_gpu_junction_temp_celsius",
|
||||
temps.get("junction"),
|
||||
"GPU junction (hotspot) temperature in celsius",
|
||||
"gauge",
|
||||
temps.get("junction"),
|
||||
)
|
||||
emit(
|
||||
"gn_gpu_memory_temp_celsius",
|
||||
temps.get("memory"),
|
||||
"GPU memory temperature in celsius",
|
||||
"gauge",
|
||||
temps.get("memory"),
|
||||
)
|
||||
for hwmon_dir in sorted(device_dir.glob("hwmon/hwmon*")):
|
||||
for fan_path in sorted(hwmon_dir.glob("fan*_input")):
|
||||
@@ -137,26 +150,30 @@ def collect(drm_root: Path) -> list[str]:
|
||||
rpm = _read_int(fan_path)
|
||||
if rpm is None:
|
||||
continue
|
||||
lines.append(f"# HELP gn_gpu_fan_rpm GPU fan speed in rpm")
|
||||
lines.append(f"# TYPE gn_gpu_fan_rpm gauge")
|
||||
lines.append(
|
||||
f'gn_gpu_fan_rpm{{card="{card}",gpu="{gpu}",fan="{fan_index}"}} {rpm}'
|
||||
emit(
|
||||
"gn_gpu_fan_rpm",
|
||||
"GPU fan speed in rpm",
|
||||
"gauge",
|
||||
rpm,
|
||||
fan=fan_index,
|
||||
)
|
||||
pwm = _read_int(hwmon_dir / "pwm1")
|
||||
if pwm is not None:
|
||||
lines.append(f"# HELP gn_gpu_fan_pwm_percent Requested fan PWM in percent")
|
||||
lines.append(f"# TYPE gn_gpu_fan_pwm_percent gauge")
|
||||
lines.append(
|
||||
f'gn_gpu_fan_pwm_percent{{card="{card}",gpu="{gpu}"}} {pwm / 255.0 * 100:.2f}'
|
||||
emit(
|
||||
"gn_gpu_fan_pwm_percent",
|
||||
"Requested fan PWM in percent",
|
||||
"gauge",
|
||||
round(pwm / 255.0 * 100, 2),
|
||||
)
|
||||
# AMD power sensors report microwatts (power1_average preferred).
|
||||
for power_name in ("power1_average", "power1_input"):
|
||||
microwatts = _read_int(hwmon_dir / power_name)
|
||||
if microwatts is not None:
|
||||
lines.append(f"# HELP gn_gpu_power_watts GPU board power in watts")
|
||||
lines.append(f"# TYPE gn_gpu_power_watts gauge")
|
||||
lines.append(
|
||||
f'gn_gpu_power_watts{{card="{card}",gpu="{gpu}"}} {microwatts / 1e6:.3f}'
|
||||
emit(
|
||||
"gn_gpu_power_watts",
|
||||
"GPU board power in watts",
|
||||
"gauge",
|
||||
round(microwatts / 1e6, 3),
|
||||
)
|
||||
break
|
||||
return lines
|
||||
|
||||
Reference in New Issue
Block a user