Files
infrastructure/llm-server/monitoring/ansible/playbooks/monitor.yml
T
spektr 76feaa9549 GNMON-001 M5: Dashboard-Rollout + AI-Verfuegbarkeitsregel auf dem KI-Server
gnmon_dashboard-Rolle (Rootless-Quadlet unter gn-dashboard 984,
ASPNETCORE_URLS=127.0.0.1:8080), Containerfile aus MCR-Basisimages,
Image-Transport per podman save/load (v0.1.0, ID 1b71e6d35069).
keep-id verworfen (crun-Overlay-Rechtebefund), Default-Mapping wie alle
GNMON-Dienste. GnAiEnvironmentDown-Regel (gn_ai_active_env == 0, 2m,
critical) aus GNAI-001 M4 integriert; Suite gruen und idempotent.

Generated by Mistral Vibe.
Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
2026-11-06 14:45:41 +01:00

148 lines
5.3 KiB
YAML

---
# GNMON-001 v2.0: host monitoring deployment.
# M1 (workstation): metrics path (collector + node_exporter + VictoriaMetrics).
# M2 (workstation): log path (Loki + Alloy). Dashboard (M3) follows.
- name: GNMON metrics and log paths
hosts: all
roles:
- role: gnmon_accounts
tags: [accounts, never]
- role: gnmon_collector
tags: [collector]
- role: gnmon_victoria_metrics
tags: [metrics]
- role: gnmon_loki
tags: [logs]
- role: gnmon_alloy
tags: [logs]
- role: gnmon_alertmanager
tags: [alerting]
- role: gnmon_vmalert
tags: [alerting]
- role: gnmon_dashboard
tags: [dashboard]
post_tasks:
- name: Verify the GPU metrics are ingested
ansible.builtin.uri:
url: "http://127.0.0.1:{{ gnmon_victoria_metrics_port }}/api/v1/query?query=gn_gpu_edge_temp_celsius"
status_code: 200
register: gnmon_query_result
retries: 15
delay: 4
until: >-
gnmon_query_result.json is defined
and gnmon_query_result.json.data.result | length > 0
tags: [metrics]
- name: Show the ingested GPU temperature sample
ansible.builtin.debug:
msg: "gn_gpu_edge_temp_celsius = {{ gnmon_query_result.json.data.result[0].value[1] }} °C"
tags: [metrics]
- name: Seed the development log source with a fresh line
ansible.builtin.shell: |
set -e
dev_log="{{ lookup('ansible.builtin.env', 'HOME') }}/.local/state/gnmon/dev-test.log"
mkdir -p "$(dirname "$dev_log")"
echo "gnmon-m2-verification $(date -u +%%Y-%%m-%%dT%%H:%%M:%%SZ)" >> "$dev_log"
args:
executable: /bin/bash
changed_when: false
when: gnmon_alloy_file_targets | length > 0
tags: [logs]
- name: Seed the journal with a verification line
# Journal-based rebuild of the log-path verify: in the server (system)
# scope Alloy has no file targets, only the journal. Both scopes label
# the message via SYSLOG_IDENTIFIER (JSON pipeline).
ansible.builtin.command:
argv:
- /usr/bin/logger
- -t
- gnmon-verify
- "gnmon-m2-verification {{ ansible_date_time['iso8601'] }}"
changed_when: false
tags: [logs]
- name: Verify the log line arrives in Loki
ansible.builtin.uri:
url: >-
http://127.0.0.1:{{ gnmon_loki_port }}/loki/api/v1/query_range?query={syslog_identifier="gnmon-verify"}&limit=5&start={{
(now_epoch | int - 300) * 1000000000 }}&end={{ (now_epoch | int + 60) * 1000000000 }}
status_code: 200
vars:
now_epoch: "{{ ansible_facts['date_time']['epoch'] }}"
register: gnmon_loki_query
retries: 10
delay: 3
until: >-
gnmon_loki_query.json is defined
and gnmon_loki_query.json.data.result | length > 0
tags: [logs]
- name: Show the ingested log sample
ansible.builtin.debug:
msg: "Loki-Ergebnisstreams: {{ gnmon_loki_query.json.data.result | length }}"
tags: [logs]
- name: Wait until vmalert has evaluated its first cycle
ansible.builtin.uri:
url: "http://{{ gnmon_vmalert_http_listen }}/-/healthy"
status_code: 200
register: gnmon_vmalert_health
retries: 12
delay: 5
until: gnmon_vmalert_health.status == 200
tags: [alerting]
- name: Verify the self-test alert reaches Alertmanager (workstation)
ansible.builtin.uri:
url: "http://{{ gnmon_alertmanager_http_listen }}/api/v2/alerts"
status_code: 200
register: gnmon_am_alerts
retries: 15
delay: 8
until: >-
gnmon_am_alerts.json is defined
and gnmon_am_alerts.json | selectattr('labels.alertname', 'defined')
| selectattr('labels.alertname', 'equalto', 'GnmonSelfTest')
| list | length > 0
when: gnmon_vmalert_selftest_rule | bool
tags: [alerting]
- name: Verify Alertmanager answers (server, no self-test rule)
ansible.builtin.uri:
url: "http://{{ gnmon_alertmanager_http_listen }}/api/v2/alerts"
status_code: 200
when: not (gnmon_vmalert_selftest_rule | bool)
tags: [alerting]
- name: Show the alerting path verdict
ansible.builtin.debug:
msg: >-
vmalert -> Alertmanager funktioniert:
{{ gnmon_am_alerts.json | selectattr('labels.alertname', 'defined')
| map(attribute='labels.alertname') | list | join(', ') }}
when: gnmon_vmalert_selftest_rule | bool
tags: [alerting]
- name: Wait for the dashboard to answer on the host loopback
ansible.builtin.uri:
url: "http://{{ gnmon_dashboard_http_listen }}/"
status_code: 200
register: gnmon_dashboard_ready
retries: 20
delay: 3
until: gnmon_dashboard_ready.status == 200
tags: [dashboard]
- name: Verify the dashboard serves the Blazor shell
ansible.builtin.uri:
url: "http://{{ gnmon_dashboard_http_listen }}/"
status_code: 200
return_content: false
register: gnmon_dashboard_shell
until: gnmon_dashboard_shell.status == 200
retries: 3
delay: 2
tags: [dashboard]
- name: Show the dashboard verdict
ansible.builtin.debug:
msg: "Dashboard antwortet auf http://{{ gnmon_dashboard_http_listen }}/"
tags: [dashboard]