GNMON-001 M1: Metrikpfad auf der Workstation implementiert
llm-server/monitoring/ neu: amdgpu-textfile-collector.py (sysfs-only, atomar, APU-tauglich), Rollen gnmon_collector (node_exporter v1.12.1, SHA256SUMS-verifiziert, 10s-Timer) und gnmon_victoria_metrics (rootless Quadlet, v1.154.1 digest-gepinnt, Scrape 10s, Retention 60d), Workstation- Inventory. Verifikation: PromQL liefert gn_gpu_edge_temp_celsius von der Laptop-APU; Idempotenz changed=0. GNMON-001 v2.0 in die Plattform-Struktur übernommen, GNMON-000-Stand inkl. Alerting-Planlücke aktualisiert. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
@@ -0,0 +1,12 @@
|
||||
[defaults]
|
||||
home = .ansible
|
||||
host_key_checking = True
|
||||
interpreter_python = auto_silent
|
||||
local_tmp = .ansible/tmp
|
||||
retry_files_enabled = False
|
||||
roles_path = roles:../../../shared/ansible/roles
|
||||
timeout = 15
|
||||
|
||||
[ssh_connection]
|
||||
pipelining = True
|
||||
ssh_args = -o ControlMaster=auto -o ControlPersist=60s
|
||||
@@ -0,0 +1,10 @@
|
||||
# GNMON-001 v2.0 workstation (development) inventory group vars.
|
||||
# Role defaults carry the M1 development values; this file only pins the
|
||||
# deployment-specific bits.
|
||||
|
||||
# VictoriaMetrics image: by tag during development. The digest of the
|
||||
# verified pull is recorded by the role and pinned here before M5.
|
||||
# Tag choice: v1.154.1 is the newest patch release of the second-newest
|
||||
# line (the v1.136 LTS line has no bare OSS tags in the Docker registry).
|
||||
gnmon_victoria_metrics_image_tag: v1.154.1
|
||||
gnmon_victoria_metrics_image_digest: null
|
||||
@@ -0,0 +1,10 @@
|
||||
# GNMON-001 section 2: workstation inventory for the development phase.
|
||||
# The monitoring suite is developed and first tested on the operator's
|
||||
# CachyOS laptop (rootless, user-scope systemd) before the server rollout
|
||||
# (ghostnet-ki) in milestone M5.
|
||||
all:
|
||||
hosts:
|
||||
gnmon-workstation:
|
||||
ansible_connection: local
|
||||
ansible_user: ghost
|
||||
# No become: everything in user scope during development.
|
||||
@@ -0,0 +1,28 @@
|
||||
---
|
||||
# GNMON-001 v2.0: host monitoring deployment.
|
||||
# M1 (workstation): metrics path (collector + node_exporter + VictoriaMetrics).
|
||||
# Tags: `metrics` for the M1 path; log path (M2) and dashboard (M3) follow.
|
||||
- name: GNMON metrics path
|
||||
hosts: all
|
||||
connection: local
|
||||
roles:
|
||||
- role: gnmon_collector
|
||||
tags: [collector]
|
||||
- role: gnmon_victoria_metrics
|
||||
tags: [metrics]
|
||||
post_tasks:
|
||||
- name: Verify the GPU metrics are ingested
|
||||
ansible.builtin.uri:
|
||||
url: "http://127.0.0.1:{{ gnmon_victoria_metrics_port }}/api/v1/query?query=gn_gpu_edge_temp_celsius"
|
||||
status_code: 200
|
||||
register: gnmon_query_result
|
||||
retries: 15
|
||||
delay: 4
|
||||
until: >-
|
||||
gnmon_query_result.json is defined
|
||||
and gnmon_query_result.json.data.result | length > 0
|
||||
tags: [metrics]
|
||||
- name: Show the ingested GPU temperature sample
|
||||
ansible.builtin.debug:
|
||||
msg: "gn_gpu_edge_temp_celsius = {{ gnmon_query_result.json.data.result[0].value[1] }} °C"
|
||||
tags: [metrics]
|
||||
@@ -0,0 +1,22 @@
|
||||
# GNMON-001 v2.0 section 3: node_exporter + AMD-GPU textfile collector.
|
||||
#
|
||||
# M1 scope: user-scope deployment for the workstation development phase
|
||||
# (CachyOS laptop). The system scope for ghostnet-ki (metrics account,
|
||||
# host-systemd, journald groups) is added with milestone M5.
|
||||
|
||||
gnmon_collector_scope: user
|
||||
gnmon_collector_user_home: "{{ lookup('ansible.builtin.env', 'HOME') }}"
|
||||
gnmon_collector_textfile_dir: "{{ gnmon_collector_user_home }}/.local/state/gnmon/textfile"
|
||||
gnmon_collector_binary_dir: "{{ gnmon_collector_user_home }}/.local/bin"
|
||||
gnmon_collector_script_dir: "{{ gnmon_collector_user_home }}/.local/share/gnmon"
|
||||
gnmon_collector_cache_dir: "{{ gnmon_collector_user_home }}/.cache/gnmon"
|
||||
|
||||
# Collection cadence (GNMON-001 section 3: host-systemd-Timer, 10 s).
|
||||
gnmon_collector_interval_seconds: 10
|
||||
|
||||
# node_exporter: pinned upstream release, verified against the release
|
||||
# SHA256SUMS file at download time.
|
||||
gnmon_collector_node_exporter_version: "1.12.1"
|
||||
gnmon_collector_node_exporter_url: "https://github.com/prometheus/node_exporter/releases/download"
|
||||
gnmon_collector_node_exporter_port: 9100
|
||||
gnmon_collector_node_exporter_listen: "127.0.0.1:{{ gnmon_collector_node_exporter_port }}"
|
||||
@@ -0,0 +1,14 @@
|
||||
# GNMON-001: collector role handlers (user scope, M1).
|
||||
- name: Restart collector timer
|
||||
ansible.builtin.systemd:
|
||||
name: gnmon-amdgpu-collector.timer
|
||||
scope: user
|
||||
state: restarted
|
||||
daemon_reload: true
|
||||
|
||||
- name: Restart node_exporter
|
||||
ansible.builtin.systemd:
|
||||
name: gnmon-node-exporter.service
|
||||
scope: user
|
||||
state: restarted
|
||||
daemon_reload: true
|
||||
@@ -0,0 +1,106 @@
|
||||
# GNMON-001 v2.0 section 3: AMD-GPU textfile collector + node_exporter.
|
||||
# M1: user-scope deployment on the workstation inventory.
|
||||
- name: Validate collector declarations
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- gnmon_collector_scope == 'user'
|
||||
- gnmon_collector_user_home is match('^/')
|
||||
- gnmon_collector_interval_seconds | int >= 5
|
||||
- gnmon_collector_node_exporter_version is match('^[0-9]+\\.[0-9]+\\.[0-9]+$')
|
||||
fail_msg: >-
|
||||
gnmon_collector only implements scope 'user' in milestone M1 and
|
||||
requires absolute paths and a pinned node_exporter release.
|
||||
|
||||
- name: Create collector directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
loop:
|
||||
- "{{ gnmon_collector_textfile_dir }}"
|
||||
- "{{ gnmon_collector_binary_dir }}"
|
||||
- "{{ gnmon_collector_script_dir }}"
|
||||
- "{{ gnmon_collector_cache_dir }}"
|
||||
|
||||
- name: Install the AMD GPU textfile collector
|
||||
ansible.builtin.copy:
|
||||
src: "{{ playbook_dir }}/../../scripts/amdgpu-textfile-collector.py"
|
||||
dest: "{{ gnmon_collector_script_dir }}/amdgpu-textfile-collector.py"
|
||||
mode: "0755"
|
||||
notify: Restart collector timer
|
||||
|
||||
- name: Fetch the node_exporter release checksums
|
||||
ansible.builtin.get_url:
|
||||
url: "{{ gnmon_collector_node_exporter_url }}/v{{ gnmon_collector_node_exporter_version }}/sha256sums.txt"
|
||||
dest: "{{ gnmon_collector_cache_dir }}/node_exporter-{{ gnmon_collector_node_exporter_version }}.SHA256SUMS"
|
||||
mode: "0644"
|
||||
|
||||
- name: Record the pinned node_exporter tarball checksum
|
||||
ansible.builtin.set_fact:
|
||||
gnmon_collector_tarball_name: "node_exporter-{{ gnmon_collector_node_exporter_version }}.linux-amd64.tar.gz"
|
||||
gnmon_collector_tarball_checksum: "{{ lookup('ansible.builtin.file',
|
||||
gnmon_collector_cache_dir ~ '/node_exporter-' ~ gnmon_collector_node_exporter_version ~ '.SHA256SUMS')
|
||||
| regex_findall('^([0-9a-f]{64})\\s+node_exporter-' ~ gnmon_collector_node_exporter_version ~ '\\.linux-amd64\\.tar\\.gz$', multiline=True) | first }}"
|
||||
|
||||
- name: Download node_exporter {{ gnmon_collector_node_exporter_version }}
|
||||
ansible.builtin.get_url:
|
||||
url: "{{ gnmon_collector_node_exporter_url }}/v{{ gnmon_collector_node_exporter_version }}/{{ gnmon_collector_tarball_name }}"
|
||||
dest: "{{ gnmon_collector_cache_dir }}/{{ gnmon_collector_tarball_name }}"
|
||||
checksum: "sha256:{{ gnmon_collector_tarball_checksum }}"
|
||||
mode: "0644"
|
||||
|
||||
- name: Extract node_exporter
|
||||
ansible.builtin.unarchive:
|
||||
src: "{{ gnmon_collector_cache_dir }}/{{ gnmon_collector_tarball_name }}"
|
||||
dest: "{{ gnmon_collector_cache_dir }}"
|
||||
remote_src: true
|
||||
creates: "{{ gnmon_collector_cache_dir }}/node_exporter-{{ gnmon_collector_node_exporter_version }}.linux-amd64"
|
||||
|
||||
- name: Install the node_exporter binary
|
||||
ansible.builtin.copy:
|
||||
src: "{{ gnmon_collector_cache_dir }}/node_exporter-{{ gnmon_collector_node_exporter_version }}.linux-amd64/node_exporter"
|
||||
dest: "{{ gnmon_collector_binary_dir }}/node_exporter"
|
||||
mode: "0755"
|
||||
notify: Restart node_exporter
|
||||
|
||||
- name: Install the collector service unit
|
||||
ansible.builtin.template:
|
||||
src: gnmon-amdgpu-collector.service.j2
|
||||
dest: "{{ gnmon_collector_user_home }}/.config/systemd/user/gnmon-amdgpu-collector.service"
|
||||
mode: "0644"
|
||||
notify: Restart collector timer
|
||||
|
||||
- name: Install the collector timer unit
|
||||
ansible.builtin.template:
|
||||
src: gnmon-amdgpu-collector.timer.j2
|
||||
dest: "{{ gnmon_collector_user_home }}/.config/systemd/user/gnmon-amdgpu-collector.timer"
|
||||
mode: "0644"
|
||||
notify: Restart collector timer
|
||||
|
||||
- name: Install the node_exporter service unit
|
||||
ansible.builtin.template:
|
||||
src: gnmon-node-exporter.service.j2
|
||||
dest: "{{ gnmon_collector_user_home }}/.config/systemd/user/gnmon-node-exporter.service"
|
||||
mode: "0644"
|
||||
notify: Restart node_exporter
|
||||
|
||||
- name: Run the collector once to seed the textfile
|
||||
ansible.builtin.command:
|
||||
cmd: "/usr/bin/python3 {{ gnmon_collector_script_dir }}/amdgpu-textfile-collector.py --textfile-dir {{ gnmon_collector_textfile_dir }}"
|
||||
creates: "{{ gnmon_collector_textfile_dir }}/amdgpu.prom"
|
||||
|
||||
- name: Enable and start the collector timer
|
||||
ansible.builtin.systemd:
|
||||
name: gnmon-amdgpu-collector.timer
|
||||
scope: user
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: true
|
||||
|
||||
- name: Enable and start node_exporter
|
||||
ansible.builtin.systemd:
|
||||
name: gnmon-node-exporter.service
|
||||
scope: user
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: true
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=GNMON AMD GPU textfile collector (GNMON-001)
|
||||
Documentation=https://github.com/ghostnet/infrastructure
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 {{ gnmon_collector_script_dir }}/amdgpu-textfile-collector.py --textfile-dir {{ gnmon_collector_textfile_dir }}
|
||||
# A failing sample must surface in the journal, not silently skip cycles.
|
||||
Nice=5
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=GNMON AMD GPU textfile collector timer (GNMON-001)
|
||||
|
||||
[Timer]
|
||||
OnBootSec=15s
|
||||
OnUnitActiveSec={{ gnmon_collector_interval_seconds }}s
|
||||
AccuracySec=1s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=GNMON node_exporter (GNMON-001)
|
||||
Documentation=https://github.com/prometheus/node_exporter
|
||||
|
||||
[Service]
|
||||
ExecStart={{ gnmon_collector_binary_dir }}/node_exporter --collector.textfile.directory={{ gnmon_collector_textfile_dir }} --web.listen-address={{ gnmon_collector_node_exporter_listen }}
|
||||
Restart=on-failure
|
||||
RestartSec=2s
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -0,0 +1,27 @@
|
||||
# GNMON-001 v2.0 sections 3-4: VictoriaMetrics (single) as rootless quadlet.
|
||||
#
|
||||
# M1 scope: user-scope quadlet under the current user on the workstation.
|
||||
# The image runs by tag during development; M5 pins the digest of the
|
||||
# verified pull in the server inventory (repo convention: digest pin).
|
||||
|
||||
gnmon_victoria_metrics_image: docker.io/victoriametrics/victoria-metrics
|
||||
gnmon_victoria_metrics_image_tag: v1.154.1
|
||||
# Digest of the verified pull on the workstation (2026-11-04, M1).
|
||||
gnmon_victoria_metrics_image_digest: c7884f32920dd45a9d86542a717227e63af03e060e53bbf8031e6e6fd0e1e05d
|
||||
|
||||
gnmon_victoria_metrics_user_home: "{{ lookup('ansible.builtin.env', 'HOME') }}"
|
||||
gnmon_victoria_metrics_quadlet_dir: "{{ gnmon_victoria_metrics_user_home }}/.config/containers/systemd"
|
||||
gnmon_victoria_metrics_config_dir: "{{ gnmon_victoria_metrics_user_home }}/.config/gnmon"
|
||||
gnmon_victoria_metrics_data_dir: "{{ gnmon_victoria_metrics_user_home }}/.local/share/gnmon/victoria-metrics-data"
|
||||
|
||||
gnmon_victoria_metrics_port: 8428
|
||||
gnmon_victoria_metrics_retention_days: 60
|
||||
gnmon_victoria_metrics_scrape_interval_seconds: 10
|
||||
|
||||
# node_exporter target (same host; NetworkMode=host in the quadlet).
|
||||
gnmon_victoria_metrics_node_target: "127.0.0.1:9100"
|
||||
|
||||
gnmon_victoria_metrics_image_ref: >-
|
||||
{{ (gnmon_victoria_metrics_image_digest is not none)
|
||||
| ternary(gnmon_victoria_metrics_image ~ '@sha256:' ~ gnmon_victoria_metrics_image_digest,
|
||||
gnmon_victoria_metrics_image ~ ':' ~ gnmon_victoria_metrics_image_tag) }}
|
||||
@@ -0,0 +1,7 @@
|
||||
# GNMON-001: VictoriaMetrics role handlers (user scope, M1).
|
||||
- name: Restart VictoriaMetrics
|
||||
ansible.builtin.systemd:
|
||||
name: gnmon-victoria-metrics.service
|
||||
scope: user
|
||||
state: restarted
|
||||
daemon_reload: true
|
||||
@@ -0,0 +1,62 @@
|
||||
# GNMON-001 v2.0 sections 3-4: VictoriaMetrics rootless quadlet (M1: user scope).
|
||||
- name: Validate VictoriaMetrics declarations
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- gnmon_victoria_metrics_user_home is match('^/')
|
||||
- gnmon_victoria_metrics_retention_days | int >= 1
|
||||
- gnmon_victoria_metrics_node_target | length > 0
|
||||
fail_msg: VictoriaMetrics declarations are incomplete or ambiguous.
|
||||
|
||||
- name: Create VictoriaMetrics directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
loop:
|
||||
- "{{ gnmon_victoria_metrics_quadlet_dir }}"
|
||||
- "{{ gnmon_victoria_metrics_config_dir }}"
|
||||
- "{{ gnmon_victoria_metrics_data_dir }}"
|
||||
|
||||
- name: Install the promscrape configuration
|
||||
ansible.builtin.template:
|
||||
src: scrape.yml.j2
|
||||
dest: "{{ gnmon_victoria_metrics_config_dir }}/scrape.yml"
|
||||
mode: "0644"
|
||||
notify: Restart VictoriaMetrics
|
||||
|
||||
- name: Install the VictoriaMetrics quadlet
|
||||
ansible.builtin.template:
|
||||
src: gnmon-victoria-metrics.container.j2
|
||||
dest: "{{ gnmon_victoria_metrics_quadlet_dir }}/gnmon-victoria-metrics.container"
|
||||
mode: "0644"
|
||||
notify: Restart VictoriaMetrics
|
||||
|
||||
- name: Enable and start VictoriaMetrics
|
||||
ansible.builtin.systemd:
|
||||
name: gnmon-victoria-metrics.service
|
||||
scope: user
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: true
|
||||
|
||||
- name: Record the running image digest
|
||||
ansible.builtin.command:
|
||||
cmd: "podman inspect --format '{{ '{{' }}.ImageDigest{{ '}}' }}' gnmon-victoria-metrics"
|
||||
changed_when: false
|
||||
register: gnmon_victoria_metrics_inspect
|
||||
failed_when: false
|
||||
|
||||
- name: Show the running image digest (digest pin for M5)
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
gnmon-victoria-metrics runs {{ gnmon_victoria_metrics_image }}:{{ gnmon_victoria_metrics_image_tag }}
|
||||
(digest {{ gnmon_victoria_metrics_inspect.stdout | default('n/a') }})
|
||||
|
||||
- name: Wait for the VictoriaMetrics health endpoint
|
||||
ansible.builtin.uri:
|
||||
url: "http://127.0.0.1:{{ gnmon_victoria_metrics_port }}/health"
|
||||
status_code: 200
|
||||
register: gnmon_victoria_metrics_health
|
||||
retries: 30
|
||||
delay: 2
|
||||
until: gnmon_victoria_metrics_health.status == 200
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
[Unit]
|
||||
Description=GNMON VictoriaMetrics (GNMON-001)
|
||||
Documentation=https://docs.victoriametrics.com/
|
||||
|
||||
[Container]
|
||||
ContainerName=gnmon-victoria-metrics
|
||||
Image={{ gnmon_victoria_metrics_image_ref }}
|
||||
# Rootless quadlet reaches node_exporter on the host loopback interface.
|
||||
Network=host
|
||||
Exec=-promscrape.config=/etc/gnmon/scrape.yml -promscrape.suppressDuplicateScrapeTargetErrors -retentionPeriod={{ gnmon_victoria_metrics_retention_days }}d
|
||||
Volume={{ gnmon_victoria_metrics_config_dir }}/scrape.yml:/etc/gnmon/scrape.yml:ro,Z
|
||||
Volume={{ gnmon_victoria_metrics_data_dir }}:/victoria-metrics-data:Z
|
||||
|
||||
[Service]
|
||||
# Quadlet services are user units; pods are transient managed containers.
|
||||
Restart=on-failure
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -0,0 +1,9 @@
|
||||
# GNMON-001 section 4: VictoriaMetrics promscrape configuration.
|
||||
global:
|
||||
scrape_interval: {{ gnmon_victoria_metrics_scrape_interval_seconds }}s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: node
|
||||
static_configs:
|
||||
- targets:
|
||||
- "{{ gnmon_victoria_metrics_node_target }}"
|
||||
+213
@@ -0,0 +1,213 @@
|
||||
#!/usr/bin/env python3
|
||||
"""AMD GPU textfile collector for Prometheus node_exporter (GNMON-001 v2.0).
|
||||
|
||||
Reads amdgpu metrics directly from sysfs (no rocm-smi, no ROCm runtime
|
||||
required) and writes them as a Prometheus textfile exposition for the
|
||||
node_exporter textfile collector. Same data source as the thermal load
|
||||
profile from 2026-11-01 (GNAI-010 section 10.8).
|
||||
|
||||
Per AMD GPU (vendor 0x1002) the following metrics are emitted when the
|
||||
underlying sysfs file exists (APUs expose a subset, discrete cards all):
|
||||
|
||||
gn_gpu_up{card,gpu}
|
||||
gn_gpu_utilization_percent{card,gpu}
|
||||
gn_gpu_vram_used_bytes{card,gpu}
|
||||
gn_gpu_vram_total_bytes{card,gpu}
|
||||
gn_gpu_edge_temp_celsius{card,gpu}
|
||||
gn_gpu_junction_temp_celsius{card,gpu}
|
||||
gn_gpu_memory_temp_celsius{card,gpu}
|
||||
gn_gpu_fan_rpm{card,gpu}
|
||||
gn_gpu_fan_pwm_percent{card,gpu}
|
||||
gn_gpu_power_watts{card,gpu}
|
||||
|
||||
The output file is written atomically (tmp + rename) with mode 0644 so the
|
||||
node_exporter textfile directory never exposes a partial file.
|
||||
|
||||
Usage: amdgpu-textfile-collector.py --textfile-dir DIR [--drm-root /sys/class/drm]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
AMD_VENDOR_ID = "0x1002"
|
||||
|
||||
|
||||
def _read_int(path: Path) -> int | None:
|
||||
try:
|
||||
return int(path.read_text().strip())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _read_text(path: Path) -> str | None:
|
||||
try:
|
||||
return path.read_text().strip()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _read_hwmon_temps(device_dir: Path) -> dict[str, float]:
|
||||
"""Map hwmon temperature labels (edge/junction/mem/mcd) to celsius."""
|
||||
temps: dict[str, float] = {}
|
||||
for hwmon_dir in sorted(device_dir.glob("hwmon/hwmon*")):
|
||||
for input_path in sorted(hwmon_dir.glob("temp*_input")):
|
||||
value = _read_int(input_path)
|
||||
if value is None:
|
||||
continue
|
||||
label_path = input_path.with_name(
|
||||
input_path.name.replace("_input", "_label")
|
||||
)
|
||||
label = _read_text(label_path) or ""
|
||||
label = label.lower()
|
||||
if "junction" in label:
|
||||
temps["junction"] = value / 1000.0
|
||||
elif "edge" in label:
|
||||
temps["edge"] = value / 1000.0
|
||||
elif "mem" in label or "mcd" in label:
|
||||
temps.setdefault("memory", value / 1000.0)
|
||||
return temps
|
||||
|
||||
|
||||
def collect(drm_root: Path) -> list[str]:
|
||||
lines: list[str] = []
|
||||
for device_link in sorted(drm_root.glob("card*")):
|
||||
if "-" in device_link.name:
|
||||
continue # connector entries like card0-eDP-1
|
||||
device_dir = device_link / "device"
|
||||
if not device_dir.is_dir():
|
||||
continue
|
||||
vendor = _read_text(device_dir / "vendor")
|
||||
if vendor != AMD_VENDOR_ID:
|
||||
continue
|
||||
card = device_link.name.removeprefix("card")
|
||||
gpu = _read_text(device_dir / "product_name") or "unknown"
|
||||
|
||||
def emit(metric: str, value, help_text: str, metric_type: str) -> None:
|
||||
lines.append(f"# HELP {metric} {help_text}")
|
||||
lines.append(f"# TYPE {metric} {metric_type}")
|
||||
if value is not None:
|
||||
lines.append(f'{metric}{{card="{card}",gpu="{gpu}"}} {value}')
|
||||
|
||||
emit("gn_gpu_up", 1, "AMD GPU is visible in sysfs", "gauge")
|
||||
emit(
|
||||
"gn_gpu_utilization_percent",
|
||||
_read_int(device_dir / "gpu_busy_percent"),
|
||||
"GPU utilization in percent (amdgpu sysfs)",
|
||||
"gauge",
|
||||
)
|
||||
emit(
|
||||
"gn_gpu_vram_used_bytes",
|
||||
_read_int(device_dir / "mem_info_vram_used"),
|
||||
"Used VRAM in bytes",
|
||||
"gauge",
|
||||
)
|
||||
emit(
|
||||
"gn_gpu_vram_total_bytes",
|
||||
_read_int(device_dir / "mem_info_vram_total"),
|
||||
"Total VRAM in bytes",
|
||||
"gauge",
|
||||
)
|
||||
temps = _read_hwmon_temps(device_dir)
|
||||
emit(
|
||||
"gn_gpu_edge_temp_celsius",
|
||||
temps.get("edge"),
|
||||
"GPU edge temperature in celsius",
|
||||
"gauge",
|
||||
)
|
||||
emit(
|
||||
"gn_gpu_junction_temp_celsius",
|
||||
temps.get("junction"),
|
||||
"GPU junction (hotspot) temperature in celsius",
|
||||
"gauge",
|
||||
)
|
||||
emit(
|
||||
"gn_gpu_memory_temp_celsius",
|
||||
temps.get("memory"),
|
||||
"GPU memory temperature in celsius",
|
||||
"gauge",
|
||||
)
|
||||
for hwmon_dir in sorted(device_dir.glob("hwmon/hwmon*")):
|
||||
for fan_path in sorted(hwmon_dir.glob("fan*_input")):
|
||||
fan_index = fan_path.name.removeprefix("fan").removesuffix("_input")
|
||||
rpm = _read_int(fan_path)
|
||||
if rpm is None:
|
||||
continue
|
||||
lines.append(f"# HELP gn_gpu_fan_rpm GPU fan speed in rpm")
|
||||
lines.append(f"# TYPE gn_gpu_fan_rpm gauge")
|
||||
lines.append(
|
||||
f'gn_gpu_fan_rpm{{card="{card}",gpu="{gpu}",fan="{fan_index}"}} {rpm}'
|
||||
)
|
||||
pwm = _read_int(hwmon_dir / "pwm1")
|
||||
if pwm is not None:
|
||||
lines.append(f"# HELP gn_gpu_fan_pwm_percent Requested fan PWM in percent")
|
||||
lines.append(f"# TYPE gn_gpu_fan_pwm_percent gauge")
|
||||
lines.append(
|
||||
f'gn_gpu_fan_pwm_percent{{card="{card}",gpu="{gpu}"}} {pwm / 255.0 * 100:.2f}'
|
||||
)
|
||||
# AMD power sensors report microwatts (power1_average preferred).
|
||||
for power_name in ("power1_average", "power1_input"):
|
||||
microwatts = _read_int(hwmon_dir / power_name)
|
||||
if microwatts is not None:
|
||||
lines.append(f"# HELP gn_gpu_power_watts GPU board power in watts")
|
||||
lines.append(f"# TYPE gn_gpu_power_watts gauge")
|
||||
lines.append(
|
||||
f'gn_gpu_power_watts{{card="{card}",gpu="{gpu}"}} {microwatts / 1e6:.3f}'
|
||||
)
|
||||
break
|
||||
return lines
|
||||
|
||||
|
||||
def write_textfile(lines: list[str], textfile_dir: Path) -> Path:
|
||||
textfile_dir.mkdir(parents=True, exist_ok=True)
|
||||
target = textfile_dir / "amdgpu.prom"
|
||||
fd, tmp_name = tempfile.mkstemp(
|
||||
dir=textfile_dir, prefix=".amdgpu.prom.", suffix=".tmp"
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="ascii") as handle:
|
||||
handle.write("\n".join(lines) + "\n")
|
||||
os.chmod(tmp_name, 0o644)
|
||||
os.replace(tmp_name, target)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(tmp_name)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
return target
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--textfile-dir",
|
||||
type=Path,
|
||||
default=Path("/var/lib/node_exporter/textfile"),
|
||||
help="node_exporter textfile directory (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--drm-root",
|
||||
type=Path,
|
||||
default=Path("/sys/class/drm"),
|
||||
help="sysfs DRM root (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stdout", action="store_true", help="print exposition to stdout instead"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
lines = collect(args.drm_root)
|
||||
if args.stdout:
|
||||
print("\n".join(lines))
|
||||
return 0
|
||||
write_textfile(lines, args.textfile_dir)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user