From 22f7fb034467f84639814119f5664b72d1f802e5 Mon Sep 17 00:00:00 2001 From: spektr Date: Wed, 4 Nov 2026 06:30:42 +0100 Subject: [PATCH] GNMON-001 M4: Alerting-Pfad vmalert + Alertmanager implementiert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plan-Ergänzung (Alerting-Lücke in v2.0): vmalert v1.154.1 wertet Regeln gegen VictoriaMetrics aus, Alertmanager v0.34.2 versendet E-Mail (SMTP-Credentials host-lokal außerhalb des Repos, ohne secrets-Datei Stub-Betrieb). Regeln inklusive der GPU0-Vorfall-Signaturen (Junction >95C, PWM-Anforderung bei 0 RPM). Verifikation: Selbsttest-Alert in Alertmanager nachweisbar, Idempotenz changed=0. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../monitoring/GNMON-000-DOC_Index.md | 33 ++++++- .../monitoring/ansible/playbooks/monitor.yml | 24 ++++++ .../gnmon_alertmanager/defaults/main.yml | 27 ++++++ .../gnmon_alertmanager/handlers/main.yml | 7 ++ .../roles/gnmon_alertmanager/tasks/main.yml | 86 +++++++++++++++++++ .../templates/alertmanager.yml.j2 | 24 ++++++ .../templates/gnmon-alertmanager.container.j2 | 19 ++++ .../roles/gnmon_vmalert/defaults/main.yml | 27 ++++++ .../roles/gnmon_vmalert/handlers/main.yml | 7 ++ .../roles/gnmon_vmalert/tasks/main.yml | 64 ++++++++++++++ .../gnmon_vmalert/templates/alerts.yml.j2 | 75 ++++++++++++++++ .../templates/gnmon-vmalert.container.j2 | 18 ++++ 12 files changed, 408 insertions(+), 3 deletions(-) create mode 100644 llm-server/monitoring/ansible/roles/gnmon_alertmanager/defaults/main.yml create mode 100644 llm-server/monitoring/ansible/roles/gnmon_alertmanager/handlers/main.yml create mode 100644 llm-server/monitoring/ansible/roles/gnmon_alertmanager/tasks/main.yml create mode 100644 llm-server/monitoring/ansible/roles/gnmon_alertmanager/templates/alertmanager.yml.j2 create mode 100644 llm-server/monitoring/ansible/roles/gnmon_alertmanager/templates/gnmon-alertmanager.container.j2 create mode 100644 llm-server/monitoring/ansible/roles/gnmon_vmalert/defaults/main.yml create mode 100644 llm-server/monitoring/ansible/roles/gnmon_vmalert/handlers/main.yml create mode 100644 llm-server/monitoring/ansible/roles/gnmon_vmalert/tasks/main.yml create mode 100644 llm-server/monitoring/ansible/roles/gnmon_vmalert/templates/alerts.yml.j2 create mode 100644 llm-server/monitoring/ansible/roles/gnmon_vmalert/templates/gnmon-vmalert.container.j2 diff --git a/docs/platform/llm-server/monitoring/GNMON-000-DOC_Index.md b/docs/platform/llm-server/monitoring/GNMON-000-DOC_Index.md index d7e46ca..2690ad9 100644 --- a/docs/platform/llm-server/monitoring/GNMON-000-DOC_Index.md +++ b/docs/platform/llm-server/monitoring/GNMON-000-DOC_Index.md @@ -67,7 +67,34 @@ User-Scope-systemd) im Verzeichnis `llm-server/monitoring/`: adm,systemd-journal ghost` — die Journald-Quelle läuft ohne Fehler, sammelt aber ohne Gruppenmitgliedschaft keine Einträge (dokumentiertes Alloy-Verhalten). Danach Journald-Ingestion in einer neuen Session - verifizieren. + verifizieren. (usermod durchgeführt 04.11.2026; aktivierungspflichtig + über Neustart des User-Managers, z. B. Laptop-Reboot.) + +### M4: Alerting (implementiert 04.11.2026, Workstation-Entwicklung) + +Plan-Ergänzung (Lücke in v2.0 festgestellt): Sammeln und Anzeigen ohne +Schwellwert-Alarm hätte den Vorfall vom 03.11. nicht verhindert. +Betreiberentscheidung 04.11.2026: Benachrichtigung per E-Mail. + +- `gnmon_vmalert`-Rolle: rootless Quadlet, vmalert **v1.154.1** + (Digest `sha256:aafb962b5a5713c71f0d697d629416f9cccf23aecb74f0ec5e829ddbf3f221f2`), + Evaluationsintervall 30 s, Datasource/Remote-Write/Notifier host-lokal. +- `gnmon_alertmanager`-Rolle: rootless Quadlet, Alertmanager **v0.34.2** + (Digest `sha256:39d8bb3966c694a124428a3a88fbcffd40bd069129ecd47c059f8a3f17c55f3b`), + E-Mail-Receiver (group_wait 30 s, repeat 4 h, send_resolved). + SMTP-Credentials liegen in einer host-lokalen secrets-Datei außerhalb + des Repos (`~/.config/gnmon/alertmanager-secrets.yml`); ohne sie wird + eine Stub-Konfiguration deployt, die den Pfad ohne Mail testbar hält. +- Alert-Regeln (`alerts.yml`): `GpuJunctionTempCritical` (> 95 °C, 2 min, + critical), `GpuFanStalledUnderLoad` (PWM > 5 % bei 0 RPM, 3 min, + critical — exakte Signatur des GPU0-Ausfalls), `GpuEdgeTempHigh`, + `GpuVramNearFull`, `NodeExporterDown`, `NodeExporterAbsent`; plus + Dev-only-Selbsttestregel (`GnmonSelfTest`, in M5 serverseitig zu + deaktivieren). +- Verifikation: Selbsttest-Alert erscheint in Alertmanager + (`/api/v2/alerts`), Idempotenzlauf `changed=0, failed=0`. +- Offen bis M5: echter SMTP-Versand gegen den Betreiber-Mailserver + (Operator-Schritt: secrets-Datei + Testmail). ## Noch offen @@ -78,8 +105,8 @@ User-Scope-systemd) im Verzeichnis `llm-server/monitoring/`: keinen Schwellwert-Alarm (Motivation des Vorfalls). Vorschlag: `vmalert`-Regeln gegen VictoriaMetrics mit Notification-Kanal; Kanal- Auswahl (Mail/ntfy/Matrix) ist eine Betreiberentscheidung. -3. M3 (Blazor-Dashboard) und M4 (Alerting: vmalert + Alertmanager, E-Mail - an SMTP — Betreiberentscheidung 04.11.2026) auf der Workstation. +3. M3 (Blazor-Dashboard) auf der Workstation; Journald-Ingestion nach + Laptop-Neustart verifizieren (M2-Operator-Schritt). 4. M5 Server-Rollout auf `ghostnet-ki` nach Behebung des GPU0-Kühldefekts (GNAI-010 10.10): System-Scope-Rollen, Konten 983/984, nftables, Abnahmekriterien aus GNMON-001 Abschnitt 10. diff --git a/llm-server/monitoring/ansible/playbooks/monitor.yml b/llm-server/monitoring/ansible/playbooks/monitor.yml index ec288c9..7fa72f1 100644 --- a/llm-server/monitoring/ansible/playbooks/monitor.yml +++ b/llm-server/monitoring/ansible/playbooks/monitor.yml @@ -14,6 +14,10 @@ tags: [logs] - role: gnmon_alloy tags: [logs] + - role: gnmon_alertmanager + tags: [alerting] + - role: gnmon_vmalert + tags: [alerting] post_tasks: - name: Verify the GPU metrics are ingested ansible.builtin.uri: @@ -59,3 +63,23 @@ ansible.builtin.debug: msg: "Loki-Ergebnisstreams: {{ gnmon_loki_query.json.data.result | length }}" tags: [logs] + - name: Verify the self-test alert reaches Alertmanager + ansible.builtin.uri: + url: "http://{{ gnmon_alertmanager_http_listen }}/api/v2/alerts" + status_code: 200 + register: gnmon_am_alerts + retries: 15 + delay: 8 + until: >- + gnmon_am_alerts.json is defined + and gnmon_am_alerts.json | selectattr('labels.alertname', 'defined') + | selectattr('labels.alertname', 'equalto', 'GnmonSelfTest') + | list | length > 0 + tags: [alerting] + - name: Show the alerting path verdict + ansible.builtin.debug: + msg: >- + vmalert -> Alertmanager funktioniert: + {{ gnmon_am_alerts.json | selectattr('labels.alertname', 'defined') + | map(attribute='labels.alertname') | list | join(', ') }} + tags: [alerting] diff --git a/llm-server/monitoring/ansible/roles/gnmon_alertmanager/defaults/main.yml b/llm-server/monitoring/ansible/roles/gnmon_alertmanager/defaults/main.yml new file mode 100644 index 0000000..641c6a8 --- /dev/null +++ b/llm-server/monitoring/ansible/roles/gnmon_alertmanager/defaults/main.yml @@ -0,0 +1,27 @@ +# GNMON-001 M4: Alertmanager (email receiver, operator decision 04.11.2026). +# +# SMTP credentials never enter the repository: they live in a host-local +# secrets file (YAML) outside the repo, read at deploy time. Without the +# file, a stub configuration is deployed (pipeline verifiable, email off). + +gnmon_alertmanager_image: docker.io/prom/alertmanager +gnmon_alertmanager_image_tag: v0.34.2 +# Digest of the verified pull on the workstation (2026-11-04, M4). +gnmon_alertmanager_image_digest: 39d8bb3966c694a124428a3a88fbcffd40bd069129ecd47c059f8a3f17c55f3b + +gnmon_alertmanager_user_home: "{{ lookup('ansible.builtin.env', 'HOME') }}" +gnmon_alertmanager_quadlet_dir: "{{ gnmon_alertmanager_user_home }}/.config/containers/systemd" +gnmon_alertmanager_config_dir: "{{ gnmon_alertmanager_user_home }}/.config/gnmon" +gnmon_alertmanager_data_dir: "{{ gnmon_alertmanager_user_home }}/.local/share/gnmon/alertmanager-data" + +# Host-local secrets file (operator-managed, never in the repo). Expected +# keys: smtp_smarthost, smtp_from, smtp_auth_username, smtp_auth_password, +# email_to. +gnmon_alertmanager_secrets_file: "{{ gnmon_alertmanager_config_dir }}/alertmanager-secrets.yml" + +gnmon_alertmanager_http_listen: "127.0.0.1:9093" + +gnmon_alertmanager_image_ref: >- + {{ (gnmon_alertmanager_image_digest is not none) + | ternary(gnmon_alertmanager_image ~ '@sha256:' ~ gnmon_alertmanager_image_digest, + gnmon_alertmanager_image ~ ':' ~ gnmon_alertmanager_image_tag) }} diff --git a/llm-server/monitoring/ansible/roles/gnmon_alertmanager/handlers/main.yml b/llm-server/monitoring/ansible/roles/gnmon_alertmanager/handlers/main.yml new file mode 100644 index 0000000..7678cf3 --- /dev/null +++ b/llm-server/monitoring/ansible/roles/gnmon_alertmanager/handlers/main.yml @@ -0,0 +1,7 @@ +# GNMON-001: Alertmanager role handlers (user scope, M4). +- name: Restart Alertmanager + ansible.builtin.systemd: + name: gnmon-alertmanager.service + scope: user + state: restarted + daemon_reload: true diff --git a/llm-server/monitoring/ansible/roles/gnmon_alertmanager/tasks/main.yml b/llm-server/monitoring/ansible/roles/gnmon_alertmanager/tasks/main.yml new file mode 100644 index 0000000..afef2ec --- /dev/null +++ b/llm-server/monitoring/ansible/roles/gnmon_alertmanager/tasks/main.yml @@ -0,0 +1,86 @@ +# GNMON-001 M4: Alertmanager rootless quadlet (user scope on the workstation). +- name: Validate Alertmanager declarations + ansible.builtin.assert: + that: + - gnmon_alertmanager_user_home is match('^/') + - gnmon_alertmanager_http_listen | length > 0 + fail_msg: Alertmanager declarations are incomplete or ambiguous. + +- name: Check for the host-local SMTP secrets file + ansible.builtin.stat: + path: "{{ gnmon_alertmanager_secrets_file }}" + register: gnmon_alertmanager_secrets_stat + +- name: Load the host-local SMTP secrets when present + ansible.builtin.include_vars: + file: "{{ gnmon_alertmanager_secrets_file }}" + name: gnmon_alertmanager_secrets + when: gnmon_alertmanager_secrets_stat.stat.exists + +- name: Set stub SMTP values when no secrets file exists + ansible.builtin.set_fact: + gnmon_alertmanager_secrets: + smtp_smarthost: "stub.example.org:587" + smtp_from: "gnmon@stub.example.org" + smtp_auth_username: "stub" + smtp_auth_password: "stub" + email_to: "operator@stub.example.org" + when: not gnmon_alertmanager_secrets_stat.stat.exists + +- name: Create Alertmanager directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0755" + loop: + - "{{ gnmon_alertmanager_quadlet_dir }}" + - "{{ gnmon_alertmanager_config_dir }}" + - "{{ gnmon_alertmanager_data_dir }}" + +- name: Install the Alertmanager configuration + ansible.builtin.template: + src: alertmanager.yml.j2 + dest: "{{ gnmon_alertmanager_config_dir }}/alertmanager.yml" + mode: "0640" + notify: Restart Alertmanager + +- name: Install the Alertmanager quadlet + ansible.builtin.template: + src: gnmon-alertmanager.container.j2 + dest: "{{ gnmon_alertmanager_quadlet_dir }}/gnmon-alertmanager.container" + mode: "0644" + notify: Restart Alertmanager + +- name: Enable and start Alertmanager + ansible.builtin.systemd: + name: gnmon-alertmanager.service + scope: user + enabled: true + state: started + daemon_reload: true + +- name: Flush handlers so config changes are applied before the readiness check + ansible.builtin.meta: flush_handlers + +- name: Wait for the Alertmanager readiness endpoint + ansible.builtin.uri: + url: "http://{{ gnmon_alertmanager_http_listen }}/-/ready" + status_code: 200 + register: gnmon_alertmanager_ready + retries: 30 + delay: 2 + until: gnmon_alertmanager_ready.status == 200 + +- name: Record the running image digest + ansible.builtin.command: + cmd: "podman inspect --format '{{ '{{' }}.ImageDigest{{ '}}' }}' gnmon-alertmanager" + changed_when: false + register: gnmon_alertmanager_inspect + failed_when: false + +- name: Show the running image digest (digest pin) + ansible.builtin.debug: + msg: >- + gnmon-alertmanager runs {{ gnmon_alertmanager_image }}:{{ gnmon_alertmanager_image_tag }} + (digest {{ gnmon_alertmanager_inspect.stdout | default('n/a') }}); + SMTP {{ 'konfiguriert (secrets-Datei gefunden)' if gnmon_alertmanager_secrets_stat.stat.exists else 'als Stub (keine secrets-Datei)' }} diff --git a/llm-server/monitoring/ansible/roles/gnmon_alertmanager/templates/alertmanager.yml.j2 b/llm-server/monitoring/ansible/roles/gnmon_alertmanager/templates/alertmanager.yml.j2 new file mode 100644 index 0000000..89536d3 --- /dev/null +++ b/llm-server/monitoring/ansible/roles/gnmon_alertmanager/templates/alertmanager.yml.j2 @@ -0,0 +1,24 @@ +# GNMON-001 M4: Alertmanager configuration (email receiver). +# SMTP values are injected from the host-local secrets file (see role +# defaults); the stub configuration keeps the pipeline verifiable. +global: + resolve_timeout: 5m + smtp_smarthost: "{{ gnmon_alertmanager_secrets.smtp_smarthost }}" + smtp_from: "{{ gnmon_alertmanager_secrets.smtp_from }}" + smtp_auth_username: "{{ gnmon_alertmanager_secrets.smtp_auth_username }}" + smtp_auth_password: "{{ gnmon_alertmanager_secrets.smtp_auth_password }}" + +route: + receiver: gnmon-email + group_by: + - alertname + - card + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + +receivers: + - name: gnmon-email + email_configs: + - to: "{{ gnmon_alertmanager_secrets.email_to }}" + send_resolved: true diff --git a/llm-server/monitoring/ansible/roles/gnmon_alertmanager/templates/gnmon-alertmanager.container.j2 b/llm-server/monitoring/ansible/roles/gnmon_alertmanager/templates/gnmon-alertmanager.container.j2 new file mode 100644 index 0000000..af37987 --- /dev/null +++ b/llm-server/monitoring/ansible/roles/gnmon_alertmanager/templates/gnmon-alertmanager.container.j2 @@ -0,0 +1,19 @@ +[Unit] +Description=GNMON Alertmanager (GNMON-001) +Documentation=https://prometheus.io/docs/alerting/latest/alertmanager/ + +[Container] +ContainerName=gnmon-alertmanager +Image={{ gnmon_alertmanager_image_ref }} +# Rootless quadlet; the UI stays on the host loopback interface. +Network=host +User=0 +Exec=--config.file=/etc/alertmanager/alertmanager.yml --web.listen-address={{ gnmon_alertmanager_http_listen }} --storage.path=/alertmanager +Volume={{ gnmon_alertmanager_config_dir }}/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro,Z +Volume={{ gnmon_alertmanager_data_dir }}:/alertmanager:Z + +[Service] +Restart=on-failure + +[Install] +WantedBy=default.target diff --git a/llm-server/monitoring/ansible/roles/gnmon_vmalert/defaults/main.yml b/llm-server/monitoring/ansible/roles/gnmon_vmalert/defaults/main.yml new file mode 100644 index 0000000..1a45370 --- /dev/null +++ b/llm-server/monitoring/ansible/roles/gnmon_vmalert/defaults/main.yml @@ -0,0 +1,27 @@ +# GNMON-001 M4 (Plan addition 04.11.2026): alerting path. +# vmalert evaluates alert rules against VictoriaMetrics and forwards +# firing alerts to Alertmanager (email receiver). + +gnmon_vmalert_image: docker.io/victoriametrics/vmalert +gnmon_vmalert_image_tag: v1.154.1 +# Digest of the verified pull on the workstation (2026-11-04, M4). +gnmon_vmalert_image_digest: aafb962b5a5713c71f0d697d629416f9cccf23aecb74f0ec5e829ddbf3f221f2 + +gnmon_vmalert_user_home: "{{ lookup('ansible.builtin.env', 'HOME') }}" +gnmon_vmalert_quadlet_dir: "{{ gnmon_vmalert_user_home }}/.config/containers/systemd" +gnmon_vmalert_config_dir: "{{ gnmon_vmalert_user_home }}/.config/gnmon" + +gnmon_vmalert_datasource_url: "http://127.0.0.1:8428" +gnmon_vmalert_remote_write_url: "http://127.0.0.1:8428" +gnmon_vmalert_remote_read_url: "http://127.0.0.1:8428" +gnmon_vmalert_notifier_url: "http://127.0.0.1:9093" +gnmon_vmalert_http_listen: "127.0.0.1:8880" + +# Dev-only self-test rule (fires permanently) to verify the alert path +# end to end without SMTP. Must be false on the server (M5). +gnmon_vmalert_selftest_rule: true + +gnmon_vmalert_image_ref: >- + {{ (gnmon_vmalert_image_digest is not none) + | ternary(gnmon_vmalert_image ~ '@sha256:' ~ gnmon_vmalert_image_digest, + gnmon_vmalert_image ~ ':' ~ gnmon_vmalert_image_tag) }} diff --git a/llm-server/monitoring/ansible/roles/gnmon_vmalert/handlers/main.yml b/llm-server/monitoring/ansible/roles/gnmon_vmalert/handlers/main.yml new file mode 100644 index 0000000..2a78cd4 --- /dev/null +++ b/llm-server/monitoring/ansible/roles/gnmon_vmalert/handlers/main.yml @@ -0,0 +1,7 @@ +# GNMON-001: vmalert role handlers (user scope, M4). +- name: Restart vmalert + ansible.builtin.systemd: + name: gnmon-vmalert.service + scope: user + state: restarted + daemon_reload: true diff --git a/llm-server/monitoring/ansible/roles/gnmon_vmalert/tasks/main.yml b/llm-server/monitoring/ansible/roles/gnmon_vmalert/tasks/main.yml new file mode 100644 index 0000000..47a79f1 --- /dev/null +++ b/llm-server/monitoring/ansible/roles/gnmon_vmalert/tasks/main.yml @@ -0,0 +1,64 @@ +# GNMON-001 M4: vmalert rootless quadlet (user scope on the workstation). +- name: Validate vmalert declarations + ansible.builtin.assert: + that: + - gnmon_vmalert_user_home is match('^/') + - gnmon_vmalert_datasource_url | length > 0 + - gnmon_vmalert_notifier_url | length > 0 + fail_msg: vmalert declarations are incomplete or ambiguous. + +- name: Create vmalert directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0755" + loop: + - "{{ gnmon_vmalert_quadlet_dir }}" + - "{{ gnmon_vmalert_config_dir }}" + +- name: Install the alert rules + ansible.builtin.template: + src: alerts.yml.j2 + dest: "{{ gnmon_vmalert_config_dir }}/alerts.yml" + mode: "0644" + notify: Restart vmalert + +- name: Install the vmalert quadlet + ansible.builtin.template: + src: gnmon-vmalert.container.j2 + dest: "{{ gnmon_vmalert_quadlet_dir }}/gnmon-vmalert.container" + mode: "0644" + notify: Restart vmalert + +- name: Enable and start vmalert + ansible.builtin.systemd: + name: gnmon-vmalert.service + scope: user + enabled: true + state: started + daemon_reload: true + +- name: Flush handlers so config changes are applied before the readiness check + ansible.builtin.meta: flush_handlers + +- name: Wait for the vmalert readiness endpoint + ansible.builtin.uri: + url: "http://{{ gnmon_vmalert_http_listen }}/-/healthy" + status_code: 200 + register: gnmon_vmalert_ready + retries: 30 + delay: 2 + until: gnmon_vmalert_ready.status == 200 + +- name: Record the running image digest + ansible.builtin.command: + cmd: "podman inspect --format '{{ '{{' }}.ImageDigest{{ '}}' }}' gnmon-vmalert" + changed_when: false + register: gnmon_vmalert_inspect + failed_when: false + +- name: Show the running image digest (digest pin) + ansible.builtin.debug: + msg: >- + gnmon-vmalert runs {{ gnmon_vmalert_image }}:{{ gnmon_vmalert_image_tag }} + (digest {{ gnmon_vmalert_inspect.stdout | default('n/a') }}) diff --git a/llm-server/monitoring/ansible/roles/gnmon_vmalert/templates/alerts.yml.j2 b/llm-server/monitoring/ansible/roles/gnmon_vmalert/templates/alerts.yml.j2 new file mode 100644 index 0000000..d59f352 --- /dev/null +++ b/llm-server/monitoring/ansible/roles/gnmon_vmalert/templates/alerts.yml.j2 @@ -0,0 +1,75 @@ +# GNMON-001 M4: alert rules against VictoriaMetrics. +# The GPU rules encode the failure signature of the 2026-11-03 incident +# (GNAI-010 section 10.10): junction temperature climbing toward the +# SW CTF (~110 C) and a stalled fan while PWM is requested. +groups: + - name: gnmon-gpu + interval: 30s + rules: + - alert: GpuJunctionTempCritical + expr: gn_gpu_junction_temp_celsius > 95 + for: 2m + labels: + severity: critical + annotations: + summary: "GPU {{ '{{' }} $labels.card {{ '}}' }} junction temperature {{ '{{' }} $value {{ '}}' }} °C (SW CTF at ~110 °C)" + description: >- + GPU junction above 95 °C for 2 minutes on card {{ '{{' }} $labels.card {{ '}}' }} + ({{ '{{' }} $labels.gpu {{ '}}' }}). Stop the load before the + amdgpu driver triggers an emergency shutdown. + - alert: GpuFanStalledUnderLoad + expr: (gn_gpu_fan_pwm_percent > 5) and on(card) (gn_gpu_fan_rpm == 0) + for: 3m + labels: + severity: critical + annotations: + summary: "GPU {{ '{{' }} $labels.card {{ '}}' }} fan stalled: PWM requested but 0 RPM" + description: >- + The driver requests {{ '{{' }} $value {{ '}}' }}% fan PWM but the fan + reports 0 RPM. This is the exact signature of the GPU0 failure + from 2026-11-03 (GNAI-010 10.10). + - alert: GpuEdgeTempHigh + expr: gn_gpu_edge_temp_celsius > 90 + for: 5m + labels: + severity: warning + annotations: + summary: "GPU {{ '{{' }} $labels.card {{ '}}' }} edge temperature {{ '{{' }} $value {{ '}}' }} °C" + - alert: GpuVramNearFull + expr: gn_gpu_vram_used_bytes / gn_gpu_vram_total_bytes > 0.95 + for: 10m + labels: + severity: warning + annotations: + summary: "GPU {{ '{{' }} $labels.card {{ '}}' }} VRAM above 95 %" + + - name: gnmon-pipeline + interval: 30s + rules: + - alert: NodeExporterDown + expr: up{job="node"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: node_exporter scrape failing + - alert: NodeExporterAbsent + expr: absent(up{job="node"}) + for: 5m + labels: + severity: critical + annotations: + summary: node_exporter target disappeared from VictoriaMetrics +{% if gnmon_vmalert_selftest_rule %} + - name: gnmon-selftest + interval: 30s + rules: + # Dev/workstation only (M5 must disable): fires permanently so the + # vmalert -> Alertmanager path is verifiable without SMTP. + - alert: GnmonSelfTest + expr: gn_gpu_up == 1 + labels: + severity: info + annotations: + summary: "Self-test alert (expected on the workstation inventory)" +{% endif %} diff --git a/llm-server/monitoring/ansible/roles/gnmon_vmalert/templates/gnmon-vmalert.container.j2 b/llm-server/monitoring/ansible/roles/gnmon_vmalert/templates/gnmon-vmalert.container.j2 new file mode 100644 index 0000000..8ef47db --- /dev/null +++ b/llm-server/monitoring/ansible/roles/gnmon_vmalert/templates/gnmon-vmalert.container.j2 @@ -0,0 +1,18 @@ +[Unit] +Description=GNMON vmalert (GNMON-001) +Documentation=https://docs.victoriametrics.com/vmalert/ + +[Container] +ContainerName=gnmon-vmalert +Image={{ gnmon_vmalert_image_ref }} +# Rootless quadlet; all endpoints stay on the host loopback interface. +Network=host +User=0 +Exec=-rule=/etc/vmalert/alerts.yml -datasource.url={{ gnmon_vmalert_datasource_url }} -remoteWrite.url={{ gnmon_vmalert_remote_write_url }} -remoteRead.url={{ gnmon_vmalert_remote_read_url }} -notifier.url={{ gnmon_vmalert_notifier_url }} -httpListenAddr={{ gnmon_vmalert_http_listen }} -evaluationInterval=30s +Volume={{ gnmon_vmalert_config_dir }}/alerts.yml:/etc/vmalert/alerts.yml:ro,Z + +[Service] +Restart=on-failure + +[Install] +WantedBy=default.target