GNMON-001 M5: Monitoring-Suite auf dem KI-Server ausgerollt (System-Scope)

Rollen auf scope user|system generalisiert, gnmon_accounts-Rolle
(gn-monitor 986), Testing-Inventory ghostnet-ki, Image-Transport per
Tag-Ref. Fixes: Textfile-Collector HELP/TYPE je Familie (Multi-GPU),
Restart-Handler je Scope, Loopback-Bindings VM/Loki (§7), scope-
agnostische Playbook-Verifikationen. Server + Workstation idempotent.

Generated by Mistral Vibe.
Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
2026-11-05 06:53:56 +01:00
parent 218f57cbaa
commit 671e41c02f
32 changed files with 801 additions and 159 deletions
@@ -0,0 +1,74 @@
# GNMON-001 M5 server (ghostnet-ki) deployment.
# System scope: metrics/shipper services as gn-monitor (986),
# dashboard quadlet as gn-dashboard (984, pre-existing).
# --- Konten ---------------------------------------------------------------
gnmon_monitor_user: gn-monitor
gnmon_monitor_uid: 986
gnmon_monitor_group: gn-monitor
gnmon_monitor_home: /var/lib/gn-monitor
gnmon_dashboard_user: gn-dashboard
gnmon_dashboard_uid: 984
# --- Collector (scope system) ---------------------------------------------
gnmon_collector_scope: system
gnmon_collector_systemd_scope: system
gnmon_collector_user: gn-monitor
gnmon_collector_textfile_dir: /var/lib/gn-monitor/textfile
gnmon_collector_binary_dir: /usr/local/bin
gnmon_collector_script_dir: /usr/local/share/gnmon
gnmon_collector_cache_dir: /var/cache/gnmon
gnmon_collector_unit_dir: /etc/systemd/system
# --- VictoriaMetrics -------------------------------------------------------
gnmon_victoria_metrics_scope: system
gnmon_victoria_metrics_systemd_scope: system
gnmon_victoria_metrics_runuser_cli: true
gnmon_victoria_metrics_quadlet_dir: /etc/containers/systemd/users/986
gnmon_victoria_metrics_config_dir: /etc/gnmon
gnmon_victoria_metrics_data_dir: /var/lib/gn-monitor/victoria-metrics-data
# --- Loki ------------------------------------------------------------------
gnmon_loki_scope: system
gnmon_loki_systemd_scope: system
gnmon_loki_quadlet_dir: /etc/containers/systemd/users/986
gnmon_loki_config_dir: /etc/gnmon
gnmon_loki_data_dir: /var/lib/gn-monitor/loki-data
gnmon_loki_log_level: warn
# --- Alertmanager ----------------------------------------------------------
gnmon_alertmanager_scope: system
gnmon_alertmanager_systemd_scope: system
gnmon_alertmanager_quadlet_dir: /etc/containers/systemd/users/986
gnmon_alertmanager_config_dir: /etc/gnmon
gnmon_alertmanager_data_dir: /var/lib/gn-monitor/alertmanager-data
# --- vmalert ---------------------------------------------------------------
gnmon_vmalert_scope: system
gnmon_vmalert_systemd_scope: system
gnmon_vmalert_quadlet_dir: /etc/containers/systemd/users/986
gnmon_vmalert_config_dir: /etc/gnmon
# Server: no self-test rule (workstation only).
gnmon_vmalert_selftest_rule: false
# --- Alloy -----------------------------------------------------------------
gnmon_alloy_scope: system
gnmon_alloy_systemd_scope: system
gnmon_alloy_user: gn-monitor
gnmon_alloy_binary_dir: /usr/local/bin
gnmon_alloy_config_dir: /etc/gnmon
gnmon_alloy_data_dir: /var/lib/gn-monitor/alloy-data
gnmon_alloy_cache_dir: /var/cache/gnmon/alloy
gnmon_alloy_unit_dir: /etc/systemd/system
gnmon_alloy_journal_path: ""
gnmon_alloy_file_targets: []
# --- Image-Referenzen (Transport von der Workstation) -------------------
# Die Images wurden auf der Workstation per Digest-Pull verifiziert und
# als tar transportiert (podman load verliert die Digest-Namen); die
# getaggten Referenzen zeigen auf dieselben Image-IDs. Der Server pulled
# darum nicht (docker.io Rate-Limit vermeiden).
gnmon_victoria_metrics_image_ref: docker.io/victoriametrics/victoria-metrics:v1.154.1
gnmon_loki_image_ref: docker.io/grafana/loki:3.6.19
gnmon_alertmanager_image_ref: docker.io/prom/alertmanager:v0.34.2
gnmon_vmalert_image_ref: docker.io/victoriametrics/vmalert:v1.154.1
@@ -0,0 +1,10 @@
# GNMON-001 M5: server inventory (ghostnet-ki). All services run in
# system scope; the quadlets live under the workload accounts
# (established convention: /etc/containers/systemd/users/<uid>).
all:
hosts:
ghostnet-ki:
ansible_host: 10.0.0.90
ansible_user: ansible-deploy
ansible_private_key_file: ~/.ssh/id_ed25519_ghostnet_ai_ansible_deploy
ansible_become: true
@@ -4,8 +4,9 @@
# M2 (workstation): log path (Loki + Alloy). Dashboard (M3) follows.
- name: GNMON metrics and log paths
hosts: all
connection: local
roles:
- role: gnmon_accounts
tags: [accounts, never]
- role: gnmon_collector
tags: [collector]
- role: gnmon_victoria_metrics
@@ -43,11 +44,24 @@
args:
executable: /bin/bash
changed_when: false
when: gnmon_alloy_file_targets | length > 0
tags: [logs]
- name: Seed the journal with a verification line
# Journal-based rebuild of the log-path verify: in the server (system)
# scope Alloy has no file targets, only the journal. Both scopes label
# the message via SYSLOG_IDENTIFIER (JSON pipeline).
ansible.builtin.command:
argv:
- /usr/bin/logger
- -t
- gnmon-verify
- "gnmon-m2-verification {{ ansible_date_time['iso8601'] }}"
changed_when: false
tags: [logs]
- name: Verify the log line arrives in Loki
ansible.builtin.uri:
url: >-
http://127.0.0.1:{{ gnmon_loki_port }}/loki/api/v1/query_range?query={source="dev-test"}&limit=5&start={{
http://127.0.0.1:{{ gnmon_loki_port }}/loki/api/v1/query_range?query={syslog_identifier="gnmon-verify"}&limit=5&start={{
(now_epoch | int - 300) * 1000000000 }}&end={{ (now_epoch | int + 60) * 1000000000 }}
status_code: 200
vars:
@@ -63,7 +77,17 @@
ansible.builtin.debug:
msg: "Loki-Ergebnisstreams: {{ gnmon_loki_query.json.data.result | length }}"
tags: [logs]
- name: Verify the self-test alert reaches Alertmanager
- name: Wait until vmalert has evaluated its first cycle
ansible.builtin.uri:
url: "http://{{ gnmon_vmalert_http_listen }}/-/healthy"
status_code: 200
register: gnmon_vmalert_health
retries: 12
delay: 5
until: gnmon_vmalert_health.status == 200
tags: [alerting]
- name: Verify the self-test alert reaches Alertmanager (workstation)
ansible.builtin.uri:
url: "http://{{ gnmon_alertmanager_http_listen }}/api/v2/alerts"
status_code: 200
@@ -75,11 +99,21 @@
and gnmon_am_alerts.json | selectattr('labels.alertname', 'defined')
| selectattr('labels.alertname', 'equalto', 'GnmonSelfTest')
| list | length > 0
when: gnmon_vmalert_selftest_rule | bool
tags: [alerting]
- name: Verify Alertmanager answers (server, no self-test rule)
ansible.builtin.uri:
url: "http://{{ gnmon_alertmanager_http_listen }}/api/v2/alerts"
status_code: 200
when: not (gnmon_vmalert_selftest_rule | bool)
tags: [alerting]
- name: Show the alerting path verdict
ansible.builtin.debug:
msg: >-
vmalert -> Alertmanager funktioniert:
{{ gnmon_am_alerts.json | selectattr('labels.alertname', 'defined')
| map(attribute='labels.alertname') | list | join(', ') }}
when: gnmon_vmalert_selftest_rule | bool
tags: [alerting]
@@ -0,0 +1,14 @@
# GNMON-001 M5 (plan section 9.5): workload accounts for the monitoring
# suite. gn-monitor (983) carries metrics, shippers and the data-store
# quadlets; gn-dashboard (984, pre-existing) only the web app.
gnmon_monitor_user: gn-monitor
gnmon_monitor_uid: 986
gnmon_monitor_group: gn-monitor
gnmon_monitor_home: /var/lib/gn-monitor
gnmon_monitor_shell: /usr/sbin/nologin
# System journal access for the log shipper (GNMON-001 section 5).
gnmon_monitor_groups:
- systemd-journal
gnmon_dashboard_user: gn-dashboard
gnmon_dashboard_uid: 984
@@ -0,0 +1,68 @@
# GNMON-001 M5: monitoring workload accounts and linger.
- name: Validate account declarations
ansible.builtin.assert:
that:
- gnmon_monitor_uid | int > 0
- gnmon_monitor_home is match('^/')
- gnmon_dashboard_uid | int > 0
fail_msg: Monitoring account declarations are incomplete.
- name: Create the gn-monitor system group
ansible.builtin.group:
name: "{{ gnmon_monitor_group }}"
gid: "{{ gnmon_monitor_uid }}"
system: true
- name: Create the gn-monitor system account
ansible.builtin.user:
name: "{{ gnmon_monitor_user }}"
uid: "{{ gnmon_monitor_uid }}"
group: "{{ gnmon_monitor_group }}"
groups: "{{ gnmon_monitor_groups }}"
home: "{{ gnmon_monitor_home }}"
shell: "{{ gnmon_monitor_shell }}"
system: true
comment: Ghostnet monitoring metrics and data services
- name: Ensure the gn-monitor home directory exists
ansible.builtin.file:
path: "{{ gnmon_monitor_home }}"
state: directory
owner: "{{ gnmon_monitor_user }}"
group: "{{ gnmon_monitor_group }}"
mode: "0755"
- name: Require the pre-existing gn-dashboard account
ansible.builtin.getent:
database: passwd
key: "{{ gnmon_dashboard_user }}"
fail_key: false
- name: Assert the gn-dashboard account exists with the plan UID (984)
ansible.builtin.assert:
that:
- gnmon_dashboard_user in getent_passwd
fail_msg: >-
gn-dashboard (984) must pre-exist on the server (created with the
GNMON-001 v1 plan; re-create it before deploying the dashboard).
- name: Enable linger so the user managers run without sessions
ansible.builtin.command:
argv:
- /usr/bin/loginctl
- enable-linger
- "{{ item }}"
register: gnmon_linger_result
changed_when: false
loop:
- "{{ gnmon_monitor_user }}"
- "{{ gnmon_dashboard_user }}"
- name: Verify linger is active for both accounts
ansible.builtin.stat:
path: "/var/lib/systemd/linger/{{ item }}"
loop:
- "{{ gnmon_monitor_user }}"
- "{{ gnmon_dashboard_user }}"
register: gnmon_linger_state
failed_when: not gnmon_linger_state.stat.exists
@@ -25,3 +25,8 @@ gnmon_alertmanager_image_ref: >-
{{ (gnmon_alertmanager_image_digest is not none)
| ternary(gnmon_alertmanager_image ~ '@sha256:' ~ gnmon_alertmanager_image_digest,
gnmon_alertmanager_image ~ ':' ~ gnmon_alertmanager_image_tag) }}
gnmon_alertmanager_scope: user
gnmon_alertmanager_systemd_scope: user
gnmon_alertmanager_run_user: gn-monitor
gnmon_alertmanager_run_uid: 986
@@ -1,7 +1,15 @@
# GNMON-001: Alertmanager role handlers (user scope, M4).
- name: Restart Alertmanager
# GNMON-001: Alertmanager role handlers.
- name: Restart Alertmanager (user scope)
ansible.builtin.systemd:
name: gnmon-alertmanager.service
scope: user
state: restarted
daemon_reload: true
when: gnmon_alertmanager_scope == 'user'
- name: Restart Alertmanager (system scope)
ansible.builtin.command:
argv: [/usr/sbin/runuser, --user, "{{ gnmon_alertmanager_run_user }}", --,
/usr/bin/env, "XDG_RUNTIME_DIR=/run/user/{{ gnmon_alertmanager_run_uid }}",
/usr/bin/systemctl, --user, restart, gnmon-alertmanager.service]
when: gnmon_alertmanager_scope == 'system'
@@ -1,21 +1,24 @@
# GNMON-001 M4: Alertmanager rootless quadlet (user scope on the workstation).
# GNMON-001 M4: Alertmanager rootless quadlet.
- name: Validate Alertmanager declarations
ansible.builtin.assert:
that:
- gnmon_alertmanager_user_home is match('^/')
- gnmon_alertmanager_http_listen | length > 0
fail_msg: Alertmanager declarations are incomplete or ambiguous.
- name: Check for the host-local SMTP secrets file
- name: Check for the host-local SMTP secrets file when present
ansible.builtin.stat:
path: "{{ gnmon_alertmanager_secrets_file }}"
register: gnmon_alertmanager_secrets_stat
become: "{{ gnmon_alertmanager_scope == 'system' }}"
become_user: root
- name: Load the host-local SMTP secrets when present
ansible.builtin.include_vars:
file: "{{ gnmon_alertmanager_secrets_file }}"
name: gnmon_alertmanager_secrets
when: gnmon_alertmanager_secrets_stat.stat.exists
become: "{{ gnmon_alertmanager_scope == 'system' }}"
become_user: root
- name: Set stub SMTP values when no secrets file exists
ansible.builtin.set_fact:
@@ -29,38 +32,67 @@
- name: Create Alertmanager directories
ansible.builtin.file:
path: "{{ item }}"
path: "{{ item.path }}"
state: directory
mode: "0755"
owner: "{{ item.owner | default(omit) }}"
group: "{{ item.group | default(omit) }}"
mode: "{{ item.mode | default('0755') }}"
loop:
- "{{ gnmon_alertmanager_quadlet_dir }}"
- "{{ gnmon_alertmanager_config_dir }}"
- "{{ gnmon_alertmanager_data_dir }}"
- path: "{{ gnmon_alertmanager_quadlet_dir }}"
- path: "{{ gnmon_alertmanager_config_dir }}"
- path: "{{ gnmon_alertmanager_data_dir }}"
owner: "{{ gnmon_alertmanager_run_user if gnmon_alertmanager_scope == 'system' else omit }}"
group: "{{ gnmon_alertmanager_run_user if gnmon_alertmanager_scope == 'system' else omit }}"
- name: Install the Alertmanager configuration
ansible.builtin.template:
src: alertmanager.yml.j2
dest: "{{ gnmon_alertmanager_config_dir }}/alertmanager.yml"
owner: "{{ gnmon_alertmanager_run_user if gnmon_alertmanager_scope == 'system' else omit }}"
group: "{{ gnmon_alertmanager_run_user if gnmon_alertmanager_scope == 'system' else omit }}"
mode: "0640"
notify: Restart Alertmanager
notify: ["Restart Alertmanager (user scope)", "Restart Alertmanager (system scope)"]
- name: Install the Alertmanager quadlet
ansible.builtin.template:
src: gnmon-alertmanager.container.j2
dest: "{{ gnmon_alertmanager_quadlet_dir }}/gnmon-alertmanager.container"
mode: "0644"
notify: Restart Alertmanager
notify: ["Restart Alertmanager (user scope)", "Restart Alertmanager (system scope)"]
- name: Enable and start Alertmanager
- name: Reload the workload user manager (system scope)
ansible.builtin.command:
argv: [/usr/sbin/runuser, --user, "{{ gnmon_alertmanager_run_user }}", --,
/usr/bin/env, "XDG_RUNTIME_DIR=/run/user/{{ gnmon_alertmanager_run_uid }}",
/usr/bin/systemctl, --user, daemon-reload]
changed_when: false
when: gnmon_alertmanager_scope == 'system'
- name: Enable and start the workload unit (system scope)
ansible.builtin.command:
argv:
- /usr/sbin/runuser
- --user
- "{{ gnmon_alertmanager_run_user }}"
- --
- /usr/bin/env
- "HOME=/var/lib/gn-monitor"
- "XDG_RUNTIME_DIR=/run/user/{{ gnmon_alertmanager_run_uid }}"
- /usr/bin/systemctl
- --user
- start
- gnmon-alertmanager.service
changed_when: false
when: gnmon_alertmanager_scope == 'system'
- name: Enable and start Alertmanager (user scope)
ansible.builtin.systemd:
name: gnmon-alertmanager.service
scope: user
enabled: true
state: started
daemon_reload: true
- name: Flush handlers so config changes are applied before the readiness check
ansible.builtin.meta: flush_handlers
when: gnmon_alertmanager_scope == 'user'
- name: Wait for the Alertmanager readiness endpoint
ansible.builtin.uri:
@@ -70,17 +102,3 @@
retries: 30
delay: 2
until: gnmon_alertmanager_ready.status == 200
- name: Record the running image digest
ansible.builtin.command:
cmd: "podman inspect --format '{{ '{{' }}.ImageDigest{{ '}}' }}' gnmon-alertmanager"
changed_when: false
register: gnmon_alertmanager_inspect
failed_when: false
- name: Show the running image digest (digest pin)
ansible.builtin.debug:
msg: >-
gnmon-alertmanager runs {{ gnmon_alertmanager_image }}:{{ gnmon_alertmanager_image_tag }}
(digest {{ gnmon_alertmanager_inspect.stdout | default('n/a') }});
SMTP {{ 'konfiguriert (secrets-Datei gefunden)' if gnmon_alertmanager_secrets_stat.stat.exists else 'als Stub (keine secrets-Datei)' }}
@@ -6,6 +6,10 @@
# source requires membership in the `systemd-journal` group (operator
# step on the workstation; documented in GNMON-000).
gnmon_alloy_scope: user
gnmon_alloy_systemd_scope: user
gnmon_alloy_user: gn-monitor
gnmon_alloy_unit_dir: "{{ lookup('ansible.builtin.env', 'HOME') }}/.config/systemd/user"
gnmon_alloy_version: "1.21.3"
gnmon_alloy_url: "https://github.com/grafana/alloy/releases/download"
@@ -2,6 +2,6 @@
- name: Restart Alloy
ansible.builtin.systemd:
name: gnmon-alloy.service
scope: user
scope: "{{ gnmon_alloy_systemd_scope }}"
state: restarted
daemon_reload: true
@@ -2,14 +2,22 @@
- name: Validate Alloy declarations
ansible.builtin.assert:
that:
- gnmon_alloy_user_home is match('^/')
- gnmon_alloy_scope in ['user', 'system']
- gnmon_alloy_version is match('^[0-9]+\\.[0-9]+\\.[0-9]+$')
fail_msg: Alloy declarations are incomplete or ambiguous.
- name: Require the unzip binary for the Alloy release archive
ansible.builtin.package:
name: unzip
state: present
when: gnmon_alloy_scope == 'system'
- name: Create Alloy directories
ansible.builtin.file:
path: "{{ item }}"
state: directory
owner: "{{ gnmon_alloy_user if gnmon_alloy_scope == 'system' else omit }}"
group: "{{ gnmon_alloy_user if gnmon_alloy_scope == 'system' else omit }}"
mode: "0755"
loop:
- "{{ gnmon_alloy_config_dir }}"
@@ -22,12 +30,21 @@
dest: "{{ gnmon_alloy_cache_dir }}/alloy-{{ gnmon_alloy_version }}.SHA256SUMS"
mode: "0644"
- name: Read the pinned Alloy zip checksum
# Remote-side read (controller lookups would read the workstation file;
# found 05.11.2025 on the server rollout).
ansible.builtin.command:
argv:
- /usr/bin/awk
- '$0 ~ /alloy-linux-amd64.zip$/ { print $1; exit }'
- "{{ gnmon_alloy_cache_dir }}/alloy-{{ gnmon_alloy_version }}.SHA256SUMS"
changed_when: false
register: gnmon_alloy_checksum_read
- name: Record the pinned Alloy zip checksum
ansible.builtin.set_fact:
gnmon_alloy_zip_name: "alloy-linux-amd64.zip"
gnmon_alloy_zip_checksum: "{{ lookup('ansible.builtin.file',
gnmon_alloy_cache_dir ~ '/alloy-' ~ gnmon_alloy_version ~ '.SHA256SUMS')
| regex_findall('^([0-9a-f]{64})\\s+alloy-linux-amd64\\.zip$', multiline=True) | first }}"
gnmon_alloy_zip_checksum: "{{ gnmon_alloy_checksum_read.stdout }}"
- name: Download Alloy v{{ gnmon_alloy_version }}
ansible.builtin.get_url:
@@ -51,6 +68,7 @@
- name: Install the Alloy binary
ansible.builtin.copy:
remote_src: true
src: "{{ gnmon_alloy_cache_dir }}/alloy-{{ gnmon_alloy_version }}/alloy-linux-amd64"
dest: "{{ gnmon_alloy_binary_dir }}/alloy"
mode: "0755"
@@ -66,14 +84,14 @@
- name: Install the Alloy service unit
ansible.builtin.template:
src: gnmon-alloy.service.j2
dest: "{{ gnmon_alloy_user_home }}/.config/systemd/user/gnmon-alloy.service"
dest: "{{ gnmon_alloy_unit_dir }}/gnmon-alloy.service"
mode: "0644"
notify: Restart Alloy
- name: Enable and start Alloy
ansible.builtin.systemd:
name: gnmon-alloy.service
scope: user
scope: "{{ gnmon_alloy_systemd_scope }}"
enabled: true
state: started
daemon_reload: true
@@ -3,6 +3,11 @@ Description=GNMON Grafana Alloy log shipper (GNMON-001)
Documentation=https://grafana.com/docs/alloy/
[Service]
{% if gnmon_alloy_scope == 'system' %}
User={{ gnmon_alloy_user }}
Group={{ gnmon_alloy_user }}
SupplementaryGroups=systemd-journal
{% endif %}
ExecStart={{ gnmon_alloy_binary_dir }}/alloy run --server.http.listen-addr={{ gnmon_alloy_http_listen }} --storage.path={{ gnmon_alloy_data_dir }} {{ gnmon_alloy_config_dir }}/alloy-config.alloy
Restart=on-failure
RestartSec=2s
@@ -5,7 +5,12 @@
# host-systemd, journald groups) is added with milestone M5.
gnmon_collector_scope: user
gnmon_collector_systemd_scope: user
# System-scope overrides come from the inventory: unit dir, service user
# and paths move to /etc/systemd/system, /usr/local/... and /var/lib.
gnmon_collector_user: gn-monitor
gnmon_collector_user_home: "{{ lookup('ansible.builtin.env', 'HOME') }}"
gnmon_collector_unit_dir: "{{ gnmon_collector_user_home }}/.config/systemd/user"
gnmon_collector_textfile_dir: "{{ gnmon_collector_user_home }}/.local/state/gnmon/textfile"
gnmon_collector_binary_dir: "{{ gnmon_collector_user_home }}/.local/bin"
gnmon_collector_script_dir: "{{ gnmon_collector_user_home }}/.local/share/gnmon"
@@ -2,13 +2,13 @@
- name: Restart collector timer
ansible.builtin.systemd:
name: gnmon-amdgpu-collector.timer
scope: user
scope: "{{ gnmon_collector_systemd_scope }}"
state: restarted
daemon_reload: true
- name: Restart node_exporter
ansible.builtin.systemd:
name: gnmon-node-exporter.service
scope: user
scope: "{{ gnmon_collector_systemd_scope }}"
state: restarted
daemon_reload: true
@@ -3,18 +3,20 @@
- name: Validate collector declarations
ansible.builtin.assert:
that:
- gnmon_collector_scope == 'user'
- gnmon_collector_scope in ['user', 'system']
- gnmon_collector_user_home is match('^/')
- gnmon_collector_interval_seconds | int >= 5
- gnmon_collector_node_exporter_version is match('^[0-9]+\\.[0-9]+\\.[0-9]+$')
fail_msg: >-
gnmon_collector only implements scope 'user' in milestone M1 and
requires absolute paths and a pinned node_exporter release.
gnmon_collector requires scope user|system, absolute paths and a
pinned node_exporter release.
- name: Create collector directories
ansible.builtin.file:
path: "{{ item }}"
state: directory
owner: "{{ gnmon_collector_user if gnmon_collector_scope == 'system' else omit }}"
group: "{{ gnmon_collector_user if gnmon_collector_scope == 'system' else omit }}"
mode: "0755"
loop:
- "{{ gnmon_collector_textfile_dir }}"
@@ -35,12 +37,22 @@
dest: "{{ gnmon_collector_cache_dir }}/node_exporter-{{ gnmon_collector_node_exporter_version }}.SHA256SUMS"
mode: "0644"
- name: Read the pinned node_exporter tarball checksum
# The SHA256SUMS file lives on the managed host; controller-side
# lookups would read the wrong machine (only worked by accident on
# the workstation where both paths coincided).
ansible.builtin.command:
argv:
- /usr/bin/awk
- '$0 ~ /linux-amd64.tar.gz/ { print $1; exit }'
- "{{ gnmon_collector_cache_dir }}/node_exporter-{{ gnmon_collector_node_exporter_version }}.SHA256SUMS"
changed_when: false
register: gnmon_collector_checksum_read
- name: Record the pinned node_exporter tarball checksum
ansible.builtin.set_fact:
gnmon_collector_tarball_name: "node_exporter-{{ gnmon_collector_node_exporter_version }}.linux-amd64.tar.gz"
gnmon_collector_tarball_checksum: "{{ lookup('ansible.builtin.file',
gnmon_collector_cache_dir ~ '/node_exporter-' ~ gnmon_collector_node_exporter_version ~ '.SHA256SUMS')
| regex_findall('^([0-9a-f]{64})\\s+node_exporter-' ~ gnmon_collector_node_exporter_version ~ '\\.linux-amd64\\.tar\\.gz$', multiline=True) | first }}"
gnmon_collector_tarball_checksum: "{{ gnmon_collector_checksum_read.stdout }}"
- name: Download node_exporter {{ gnmon_collector_node_exporter_version }}
ansible.builtin.get_url:
@@ -58,6 +70,9 @@
- name: Install the node_exporter binary
ansible.builtin.copy:
# Both paths live on the managed host (controller default would look
# for the file on the workstation; only worked there by accident).
remote_src: true
src: "{{ gnmon_collector_cache_dir }}/node_exporter-{{ gnmon_collector_node_exporter_version }}.linux-amd64/node_exporter"
dest: "{{ gnmon_collector_binary_dir }}/node_exporter"
mode: "0755"
@@ -66,21 +81,21 @@
- name: Install the collector service unit
ansible.builtin.template:
src: gnmon-amdgpu-collector.service.j2
dest: "{{ gnmon_collector_user_home }}/.config/systemd/user/gnmon-amdgpu-collector.service"
dest: "{{ gnmon_collector_unit_dir }}/gnmon-amdgpu-collector.service"
mode: "0644"
notify: Restart collector timer
- name: Install the collector timer unit
ansible.builtin.template:
src: gnmon-amdgpu-collector.timer.j2
dest: "{{ gnmon_collector_user_home }}/.config/systemd/user/gnmon-amdgpu-collector.timer"
dest: "{{ gnmon_collector_unit_dir }}/gnmon-amdgpu-collector.timer"
mode: "0644"
notify: Restart collector timer
- name: Install the node_exporter service unit
ansible.builtin.template:
src: gnmon-node-exporter.service.j2
dest: "{{ gnmon_collector_user_home }}/.config/systemd/user/gnmon-node-exporter.service"
dest: "{{ gnmon_collector_unit_dir }}/gnmon-node-exporter.service"
mode: "0644"
notify: Restart node_exporter
@@ -92,7 +107,7 @@
- name: Enable and start the collector timer
ansible.builtin.systemd:
name: gnmon-amdgpu-collector.timer
scope: user
scope: "{{ gnmon_collector_systemd_scope }}"
enabled: true
state: started
daemon_reload: true
@@ -100,7 +115,7 @@
- name: Enable and start node_exporter
ansible.builtin.systemd:
name: gnmon-node-exporter.service
scope: user
scope: "{{ gnmon_collector_systemd_scope }}"
enabled: true
state: started
daemon_reload: true
@@ -3,6 +3,9 @@ Description=GNMON AMD GPU textfile collector (GNMON-001)
Documentation=https://github.com/ghostnet/infrastructure
[Service]
{% if gnmon_collector_scope == 'system' %}
User={{ gnmon_collector_user }}
{% endif %}
Type=oneshot
ExecStart=/usr/bin/python3 {{ gnmon_collector_script_dir }}/amdgpu-textfile-collector.py --textfile-dir {{ gnmon_collector_textfile_dir }}
# A failing sample must surface in the journal, not silently skip cycles.
@@ -3,6 +3,9 @@ Description=GNMON node_exporter (GNMON-001)
Documentation=https://github.com/prometheus/node_exporter
[Service]
{% if gnmon_collector_scope == 'system' %}
User={{ gnmon_collector_user }}
{% endif %}
ExecStart={{ gnmon_collector_binary_dir }}/node_exporter --collector.textfile.directory={{ gnmon_collector_textfile_dir }} --web.listen-address={{ gnmon_collector_node_exporter_listen }}
Restart=on-failure
RestartSec=2s
@@ -1,11 +1,15 @@
# GNMON-001 v2.0 section 5: Loki as rootless quadlet (log aggregation,
# filesystem store, no S3). M2 scope: user-scope on the workstation.
# GNMON-001 v2.0 section 5: Loki as rootless quadlet (filesystem store).
gnmon_loki_image: docker.io/grafana/loki
gnmon_loki_image_tag: "3.6.19"
# Digest of the verified pull on the workstation (2026-11-04, M2).
gnmon_loki_image_digest: 0f9cbc340cbe33ee498e654075034336962057db072f57b37f253e14d476443a
gnmon_loki_scope: user
gnmon_loki_systemd_scope: user
gnmon_loki_run_user: gn-monitor
gnmon_loki_run_uid: 986
gnmon_loki_user_home: "{{ lookup('ansible.builtin.env', 'HOME') }}"
gnmon_loki_quadlet_dir: "{{ gnmon_loki_user_home }}/.config/containers/systemd"
gnmon_loki_config_dir: "{{ gnmon_loki_user_home }}/.config/gnmon"
@@ -13,6 +17,7 @@ gnmon_loki_data_dir: "{{ gnmon_loki_user_home }}/.local/share/gnmon/loki-data"
gnmon_loki_port: 3100
gnmon_loki_retention_days: 30
gnmon_loki_log_level: warn
gnmon_loki_image_ref: >-
{{ (gnmon_loki_image_digest is not none)
@@ -1,7 +1,15 @@
# GNMON-001: Loki role handlers (user scope, M2).
- name: Restart Loki
# GNMON-001: Loki role handlers.
- name: Restart Loki (user scope)
ansible.builtin.systemd:
name: gnmon-loki.service
scope: user
state: restarted
daemon_reload: true
when: gnmon_loki_scope == 'user'
- name: Restart Loki (system scope)
ansible.builtin.command:
argv: [/usr/sbin/runuser, --user, "{{ gnmon_loki_run_user }}", --,
/usr/bin/env, "XDG_RUNTIME_DIR=/run/user/{{ gnmon_loki_run_uid }}",
/usr/bin/systemctl, --user, restart, gnmon-loki.service]
when: gnmon_loki_scope == 'system'
@@ -1,57 +1,77 @@
# GNMON-001 v2.0 section 5: Loki rootless quadlet (M2: user scope).
# GNMON-001 v2.0 section 5: Loki rootless quadlet.
- name: Validate Loki declarations
ansible.builtin.assert:
that:
- gnmon_loki_user_home is match('^/')
- gnmon_loki_retention_days | int >= 1
fail_msg: Loki declarations are incomplete or ambiguous.
- name: Create Loki directories
ansible.builtin.file:
path: "{{ item }}"
path: "{{ item.path }}"
state: directory
mode: "0755"
owner: "{{ item.owner | default(omit) }}"
group: "{{ item.group | default(omit) }}"
mode: "{{ item.mode | default('0755') }}"
loop:
- "{{ gnmon_loki_quadlet_dir }}"
- "{{ gnmon_loki_config_dir }}"
- "{{ gnmon_loki_data_dir }}"
- "{{ gnmon_loki_data_dir }}/chunks"
- "{{ gnmon_loki_data_dir }}/rules"
- path: "{{ gnmon_loki_quadlet_dir }}"
- path: "{{ gnmon_loki_config_dir }}"
- path: "{{ gnmon_loki_data_dir }}"
owner: "{{ gnmon_loki_run_user if gnmon_loki_scope == 'system' else omit }}"
group: "{{ gnmon_loki_run_user if gnmon_loki_scope == 'system' else omit }}"
- path: "{{ gnmon_loki_data_dir }}/chunks"
owner: "{{ gnmon_loki_run_user if gnmon_loki_scope == 'system' else omit }}"
group: "{{ gnmon_loki_run_user if gnmon_loki_scope == 'system' else omit }}"
- path: "{{ gnmon_loki_data_dir }}/rules"
owner: "{{ gnmon_loki_run_user if gnmon_loki_scope == 'system' else omit }}"
group: "{{ gnmon_loki_run_user if gnmon_loki_scope == 'system' else omit }}"
- name: Install the Loki configuration
ansible.builtin.template:
src: loki-config.yaml.j2
dest: "{{ gnmon_loki_config_dir }}/loki-config.yaml"
mode: "0644"
notify: Restart Loki
notify: ["Restart Loki (user scope)", "Restart Loki (system scope)"]
- name: Install the Loki quadlet
ansible.builtin.template:
src: gnmon-loki.container.j2
dest: "{{ gnmon_loki_quadlet_dir }}/gnmon-loki.container"
mode: "0644"
notify: Restart Loki
notify: ["Restart Loki (user scope)", "Restart Loki (system scope)"]
- name: Enable and start Loki
- name: Reload the workload user manager (system scope)
ansible.builtin.command:
argv: [/usr/sbin/runuser, --user, "{{ gnmon_loki_run_user }}", --,
/usr/bin/env, "XDG_RUNTIME_DIR=/run/user/{{ gnmon_loki_run_uid }}",
/usr/bin/systemctl, --user, daemon-reload]
changed_when: false
when: gnmon_loki_scope == 'system'
- name: Enable and start the workload unit (system scope)
ansible.builtin.command:
argv:
- /usr/sbin/runuser
- --user
- "{{ gnmon_loki_run_user }}"
- --
- /usr/bin/env
- "HOME=/var/lib/gn-monitor"
- "XDG_RUNTIME_DIR=/run/user/{{ gnmon_loki_run_uid }}"
- /usr/bin/systemctl
- --user
- start
- gnmon-loki.service
changed_when: false
when: gnmon_loki_scope == 'system'
- name: Enable and start Loki (user scope)
ansible.builtin.systemd:
name: gnmon-loki.service
scope: user
enabled: true
state: started
daemon_reload: true
- name: Record the running image digest
ansible.builtin.command:
cmd: "podman inspect --format '{{ '{{' }}.ImageDigest{{ '}}' }}' gnmon-loki"
changed_when: false
register: gnmon_loki_inspect
failed_when: false
- name: Show the running image digest (digest pin)
ansible.builtin.debug:
msg: >-
gnmon-loki runs {{ gnmon_loki_image }}:{{ gnmon_loki_image_tag }}
(digest {{ gnmon_loki_inspect.stdout | default('n/a') }})
when: gnmon_loki_scope == 'user'
- name: Wait for the Loki readiness endpoint
ansible.builtin.uri:
@@ -11,7 +11,7 @@ Network=host
# rootless container. Run as the mapped root instead (= host user); the
# same pattern the VictoriaMetrics quadlet uses.
User=0
Exec=-config.file=/etc/loki/loki-config.yaml -log.level=warn
Exec=-config.file=/etc/loki/loki-config.yaml -log.level={{ gnmon_loki_log_level }} -log.level=warn
Volume={{ gnmon_loki_config_dir }}/loki-config.yaml:/etc/loki/loki-config.yaml:ro,Z
Volume={{ gnmon_loki_data_dir }}:/loki:Z
@@ -3,7 +3,11 @@
auth_enabled: false
server:
# GNMON-001 section 7: Loki stays host-local (loopback); only the
# dashboard proxy is exposed later.
http_listen_address: 127.0.0.1
http_listen_port: {{ gnmon_loki_port }}
grpc_listen_address: 127.0.0.1
grpc_listen_port: 9096
common:
@@ -1,24 +1,32 @@
# GNMON-001 v2.0 sections 3-4: VictoriaMetrics (single) as rootless quadlet.
#
# M1 scope: user-scope quadlet under the current user on the workstation.
# The image runs by tag during development; M5 pins the digest of the
# verified pull in the server inventory (repo convention: digest pin).
# GNMON-001 v2.0 sections 3-4: VictoriaMetrics (single) rootless quadlet.
# M1 workstation: user scope. M5 server: system scope under the
# gn-monitor account (quadlet under /etc/containers/systemd/users/986).
gnmon_victoria_metrics_image: docker.io/victoriametrics/victoria-metrics
gnmon_victoria_metrics_image_tag: v1.154.1
# Digest of the verified pull on the workstation (2026-11-04, M1).
gnmon_victoria_metrics_image_digest: c7884f32920dd45a9d86542a717227e63af03e060e53bbf8031e6e6fd0e1e05d
# scope: user (workstation) or system (server, quadlet under
# /etc/containers/systemd/users/<uid>, controlled via runuser).
gnmon_victoria_metrics_scope: user
gnmon_victoria_metrics_systemd_scope: user
gnmon_victoria_metrics_user_home: "{{ lookup('ansible.builtin.env', 'HOME') }}"
gnmon_victoria_metrics_quadlet_dir: "{{ gnmon_victoria_metrics_user_home }}/.config/containers/systemd"
gnmon_victoria_metrics_config_dir: "{{ gnmon_victoria_metrics_user_home }}/.config/gnmon"
gnmon_victoria_metrics_data_dir: "{{ gnmon_victoria_metrics_user_home }}/.local/share/gnmon/victoria-metrics-data"
gnmon_victoria_metrics_run_user: gn-monitor
gnmon_victoria_metrics_run_uid: 986
gnmon_victoria_metrics_port: 8428
# GNMON-001 section 7: metric stores stay host-local; only the reverse
# proxy (dashboard) is exposed later. Bind loopback on both scopes.
gnmon_victoria_metrics_listen_address: "127.0.0.1"
gnmon_victoria_metrics_retention_days: 60
gnmon_victoria_metrics_scrape_interval_seconds: 5
# node_exporter target (same host; NetworkMode=host in the quadlet).
# node_exporter target (same host; Network=host in the quadlet).
gnmon_victoria_metrics_node_target: "127.0.0.1:9100"
gnmon_victoria_metrics_image_ref: >-
@@ -1,7 +1,25 @@
# GNMON-001: VictoriaMetrics role handlers (user scope, M1).
- name: Restart VictoriaMetrics
# GNMON-001: VictoriaMetrics role handlers. User scope restarts via the
# session manager; system scope via runuser into the workload account.
- name: Restart VictoriaMetrics (system scope)
ansible.builtin.command:
argv:
- /usr/sbin/runuser
- --user
- "{{ gnmon_victoria_metrics_run_user }}"
- --
- /usr/bin/env
- "HOME={{ gnmon_victoria_metrics_config_dir | dirname }}"
- "XDG_RUNTIME_DIR=/run/user/{{ gnmon_victoria_metrics_run_uid }}"
- /usr/bin/systemctl
- --user
- restart
- gnmon-victoria-metrics.service
when: gnmon_victoria_metrics_scope == 'system'
- name: Restart VictoriaMetrics (user scope)
ansible.builtin.systemd:
name: gnmon-victoria-metrics.service
scope: user
state: restarted
daemon_reload: true
when: gnmon_victoria_metrics_scope == 'user'
@@ -1,43 +1,80 @@
# GNMON-001 v2.0 sections 3-4: VictoriaMetrics rootless quadlet (M1: user scope).
# GNMON-001 v2.0 sections 3-4: VictoriaMetrics rootless quadlet.
- name: Validate VictoriaMetrics declarations
ansible.builtin.assert:
that:
- gnmon_victoria_metrics_user_home is match('^/')
- gnmon_victoria_metrics_retention_days | int >= 1
- gnmon_victoria_metrics_node_target | length > 0
fail_msg: VictoriaMetrics declarations are incomplete or ambiguous.
- name: Create VictoriaMetrics directories
ansible.builtin.file:
path: "{{ item }}"
path: "{{ item.path }}"
state: directory
mode: "0755"
owner: "{{ item.owner | default(omit) }}"
group: "{{ item.group | default(omit) }}"
mode: "{{ item.mode | default('0755') }}"
loop:
- "{{ gnmon_victoria_metrics_quadlet_dir }}"
- "{{ gnmon_victoria_metrics_config_dir }}"
- "{{ gnmon_victoria_metrics_data_dir }}"
- path: "{{ gnmon_victoria_metrics_quadlet_dir }}"
- path: "{{ gnmon_victoria_metrics_config_dir }}"
- path: "{{ gnmon_victoria_metrics_data_dir }}"
owner: "{{ gnmon_victoria_metrics_run_user if gnmon_victoria_metrics_scope == 'system' else omit }}"
group: "{{ gnmon_victoria_metrics_run_user if gnmon_victoria_metrics_scope == 'system' else omit }}"
- name: Install the promscrape configuration
ansible.builtin.template:
src: scrape.yml.j2
dest: "{{ gnmon_victoria_metrics_config_dir }}/scrape.yml"
mode: "0644"
notify: Restart VictoriaMetrics
notify: ["Restart VictoriaMetrics (system scope)", "Restart VictoriaMetrics (user scope)"]
- name: Install the VictoriaMetrics quadlet
ansible.builtin.template:
src: gnmon-victoria-metrics.container.j2
dest: "{{ gnmon_victoria_metrics_quadlet_dir }}/gnmon-victoria-metrics.container"
mode: "0644"
notify: Restart VictoriaMetrics
notify: ["Restart VictoriaMetrics (system scope)", "Restart VictoriaMetrics (user scope)"]
- name: Enable and start VictoriaMetrics
- name: Reload the workload user manager (system scope)
ansible.builtin.command:
argv:
- /usr/sbin/runuser
- --user
- "{{ gnmon_victoria_metrics_run_user }}"
- --
- /usr/bin/env
- "HOME=/var/lib/gn-monitor"
- "XDG_RUNTIME_DIR=/run/user/{{ gnmon_victoria_metrics_run_uid }}"
- /usr/bin/systemctl
- --user
- daemon-reload
changed_when: false
when: gnmon_victoria_metrics_scope == 'system'
- name: Run the quadlet generator and start VictoriaMetrics (user scope)
ansible.builtin.systemd:
name: gnmon-victoria-metrics.service
scope: user
scope: "{{ gnmon_victoria_metrics_systemd_scope }}"
enabled: true
state: started
daemon_reload: true
when: gnmon_victoria_metrics_scope == 'user'
- name: Enable and start the workload unit (system scope)
ansible.builtin.command:
argv:
- /usr/sbin/runuser
- --user
- "{{ gnmon_victoria_metrics_run_user }}"
- --
- /usr/bin/env
- "HOME=/var/lib/gn-monitor"
- "XDG_RUNTIME_DIR=/run/user/{{ gnmon_victoria_metrics_run_uid }}"
- /usr/bin/systemctl
- --user
- start
- gnmon-victoria-metrics.service
changed_when: false
when: gnmon_victoria_metrics_scope == 'system'
- name: Record the running image digest
ansible.builtin.command:
@@ -45,8 +82,10 @@
changed_when: false
register: gnmon_victoria_metrics_inspect
failed_when: false
become: "{{ gnmon_victoria_metrics_scope == 'system' }}"
become_user: "{{ gnmon_victoria_metrics_run_user if gnmon_victoria_metrics_scope == 'system' else omit }}"
- name: Show the running image digest (digest pin for M5)
- name: Show the running image digest (digest pin)
ansible.builtin.debug:
msg: >-
gnmon-victoria-metrics runs {{ gnmon_victoria_metrics_image }}:{{ gnmon_victoria_metrics_image_tag }}
@@ -7,7 +7,7 @@ ContainerName=gnmon-victoria-metrics
Image={{ gnmon_victoria_metrics_image_ref }}
# Rootless quadlet reaches node_exporter on the host loopback interface.
Network=host
Exec=-promscrape.config=/etc/gnmon/scrape.yml -promscrape.suppressDuplicateScrapeTargetErrors -retentionPeriod={{ gnmon_victoria_metrics_retention_days }}d
Exec=-promscrape.config=/etc/gnmon/scrape.yml -promscrape.suppressDuplicateScrapeTargetErrors -retentionPeriod={{ gnmon_victoria_metrics_retention_days }}d -httpListenAddr={{ gnmon_victoria_metrics_listen_address }}:{{ gnmon_victoria_metrics_port }}
Volume={{ gnmon_victoria_metrics_config_dir }}/scrape.yml:/etc/gnmon/scrape.yml:ro,Z
Volume={{ gnmon_victoria_metrics_data_dir }}:/victoria-metrics-data:Z
@@ -25,3 +25,8 @@ gnmon_vmalert_image_ref: >-
{{ (gnmon_vmalert_image_digest is not none)
| ternary(gnmon_vmalert_image ~ '@sha256:' ~ gnmon_vmalert_image_digest,
gnmon_vmalert_image ~ ':' ~ gnmon_vmalert_image_tag) }}
gnmon_vmalert_scope: user
gnmon_vmalert_systemd_scope: user
gnmon_vmalert_run_user: gn-monitor
gnmon_vmalert_run_uid: 986
@@ -1,7 +1,15 @@
# GNMON-001: vmalert role handlers (user scope, M4).
- name: Restart vmalert
# GNMON-001: vmalert role handlers.
- name: Restart vmalert (user scope)
ansible.builtin.systemd:
name: gnmon-vmalert.service
scope: user
state: restarted
daemon_reload: true
when: gnmon_vmalert_scope == 'user'
- name: Restart vmalert (system scope)
ansible.builtin.command:
argv: [/usr/sbin/runuser, --user, "{{ gnmon_vmalert_run_user }}", --,
/usr/bin/env, "XDG_RUNTIME_DIR=/run/user/{{ gnmon_vmalert_run_uid }}",
/usr/bin/systemctl, --user, restart, gnmon-vmalert.service]
when: gnmon_vmalert_scope == 'system'
@@ -1,8 +1,7 @@
# GNMON-001 M4: vmalert rootless quadlet (user scope on the workstation).
# GNMON-001 M4: vmalert rootless quadlet.
- name: Validate vmalert declarations
ansible.builtin.assert:
that:
- gnmon_vmalert_user_home is match('^/')
- gnmon_vmalert_datasource_url | length > 0
- gnmon_vmalert_notifier_url | length > 0
fail_msg: vmalert declarations are incomplete or ambiguous.
@@ -21,25 +20,48 @@
src: alerts.yml.j2
dest: "{{ gnmon_vmalert_config_dir }}/alerts.yml"
mode: "0644"
notify: Restart vmalert
notify: ["Restart vmalert (user scope)", "Restart vmalert (system scope)"]
- name: Install the vmalert quadlet
ansible.builtin.template:
src: gnmon-vmalert.container.j2
dest: "{{ gnmon_vmalert_quadlet_dir }}/gnmon-vmalert.container"
mode: "0644"
notify: Restart vmalert
notify: ["Restart vmalert (user scope)", "Restart vmalert (system scope)"]
- name: Enable and start vmalert
- name: Reload the workload user manager (system scope)
ansible.builtin.command:
argv: [/usr/sbin/runuser, --user, "{{ gnmon_vmalert_run_user }}", --,
/usr/bin/env, "XDG_RUNTIME_DIR=/run/user/{{ gnmon_vmalert_run_uid }}",
/usr/bin/systemctl, --user, daemon-reload]
changed_when: false
when: gnmon_vmalert_scope == 'system'
- name: Enable and start the workload unit (system scope)
ansible.builtin.command:
argv:
- /usr/sbin/runuser
- --user
- "{{ gnmon_vmalert_run_user }}"
- --
- /usr/bin/env
- "HOME=/var/lib/gn-monitor"
- "XDG_RUNTIME_DIR=/run/user/{{ gnmon_vmalert_run_uid }}"
- /usr/bin/systemctl
- --user
- start
- gnmon-vmalert.service
changed_when: false
when: gnmon_vmalert_scope == 'system'
- name: Enable and start vmalert (user scope)
ansible.builtin.systemd:
name: gnmon-vmalert.service
scope: user
enabled: true
state: started
daemon_reload: true
- name: Flush handlers so config changes are applied before the readiness check
ansible.builtin.meta: flush_handlers
when: gnmon_vmalert_scope == 'user'
- name: Wait for the vmalert readiness endpoint
ansible.builtin.uri:
@@ -49,16 +71,3 @@
retries: 30
delay: 2
until: gnmon_vmalert_ready.status == 200
- name: Record the running image digest
ansible.builtin.command:
cmd: "podman inspect --format '{{ '{{' }}.ImageDigest{{ '}}' }}' gnmon-vmalert"
changed_when: false
register: gnmon_vmalert_inspect
failed_when: false
- name: Show the running image digest (digest pin)
ansible.builtin.debug:
msg: >-
gnmon-vmalert runs {{ gnmon_vmalert_image }}:{{ gnmon_vmalert_image_tag }}
(digest {{ gnmon_vmalert_inspect.stdout | default('n/a') }})
@@ -75,6 +75,17 @@ def _read_hwmon_temps(device_dir: Path) -> dict[str, float]:
def collect(drm_root: Path) -> list[str]:
lines: list[str] = []
declared: set[str] = set()
def declare(metric: str, help_text: str, metric_type: str) -> None:
# HELP/TYPE must appear once per metric family; repeating them per
# card makes node_exporter reject the whole textfile.
if metric in declared:
return
lines.append(f"# HELP {metric} {help_text}")
lines.append(f"# TYPE {metric} {metric_type}")
declared.add(metric)
for device_link in sorted(drm_root.glob("card*")):
if "-" in device_link.name:
continue # connector entries like card0-eDP-1
@@ -87,49 +98,51 @@ def collect(drm_root: Path) -> list[str]:
card = device_link.name.removeprefix("card")
gpu = _read_text(device_dir / "product_name") or "unknown"
def emit(metric: str, value, help_text: str, metric_type: str) -> None:
lines.append(f"# HELP {metric} {help_text}")
lines.append(f"# TYPE {metric} {metric_type}")
if value is not None:
lines.append(f'{metric}{{card="{card}",gpu="{gpu}"}} {value}')
def emit(metric: str, help_text: str, metric_type: str, value, **labels) -> None:
declare(metric, help_text, metric_type)
if value is None:
return
label_pairs = {"card": card, "gpu": gpu, **labels}
label_str = ",".join(f'{key}="{val}"' for key, val in label_pairs.items())
lines.append(f"{metric}{{{label_str}}} {value}")
emit("gn_gpu_up", 1, "AMD GPU is visible in sysfs", "gauge")
emit("gn_gpu_up", "AMD GPU is visible in sysfs", "gauge", 1)
emit(
"gn_gpu_utilization_percent",
_read_int(device_dir / "gpu_busy_percent"),
"GPU utilization in percent (amdgpu sysfs)",
"gauge",
_read_int(device_dir / "gpu_busy_percent"),
)
emit(
"gn_gpu_vram_used_bytes",
_read_int(device_dir / "mem_info_vram_used"),
"Used VRAM in bytes",
"gauge",
_read_int(device_dir / "mem_info_vram_used"),
)
emit(
"gn_gpu_vram_total_bytes",
_read_int(device_dir / "mem_info_vram_total"),
"Total VRAM in bytes",
"gauge",
_read_int(device_dir / "mem_info_vram_total"),
)
temps = _read_hwmon_temps(device_dir)
emit(
"gn_gpu_edge_temp_celsius",
temps.get("edge"),
"GPU edge temperature in celsius",
"gauge",
temps.get("edge"),
)
emit(
"gn_gpu_junction_temp_celsius",
temps.get("junction"),
"GPU junction (hotspot) temperature in celsius",
"gauge",
temps.get("junction"),
)
emit(
"gn_gpu_memory_temp_celsius",
temps.get("memory"),
"GPU memory temperature in celsius",
"gauge",
temps.get("memory"),
)
for hwmon_dir in sorted(device_dir.glob("hwmon/hwmon*")):
for fan_path in sorted(hwmon_dir.glob("fan*_input")):
@@ -137,26 +150,30 @@ def collect(drm_root: Path) -> list[str]:
rpm = _read_int(fan_path)
if rpm is None:
continue
lines.append(f"# HELP gn_gpu_fan_rpm GPU fan speed in rpm")
lines.append(f"# TYPE gn_gpu_fan_rpm gauge")
lines.append(
f'gn_gpu_fan_rpm{{card="{card}",gpu="{gpu}",fan="{fan_index}"}} {rpm}'
emit(
"gn_gpu_fan_rpm",
"GPU fan speed in rpm",
"gauge",
rpm,
fan=fan_index,
)
pwm = _read_int(hwmon_dir / "pwm1")
if pwm is not None:
lines.append(f"# HELP gn_gpu_fan_pwm_percent Requested fan PWM in percent")
lines.append(f"# TYPE gn_gpu_fan_pwm_percent gauge")
lines.append(
f'gn_gpu_fan_pwm_percent{{card="{card}",gpu="{gpu}"}} {pwm / 255.0 * 100:.2f}'
emit(
"gn_gpu_fan_pwm_percent",
"Requested fan PWM in percent",
"gauge",
round(pwm / 255.0 * 100, 2),
)
# AMD power sensors report microwatts (power1_average preferred).
for power_name in ("power1_average", "power1_input"):
microwatts = _read_int(hwmon_dir / power_name)
if microwatts is not None:
lines.append(f"# HELP gn_gpu_power_watts GPU board power in watts")
lines.append(f"# TYPE gn_gpu_power_watts gauge")
lines.append(
f'gn_gpu_power_watts{{card="{card}",gpu="{gpu}"}} {microwatts / 1e6:.3f}'
emit(
"gn_gpu_power_watts",
"GPU board power in watts",
"gauge",
round(microwatts / 1e6, 3),
)
break
return lines