diff --git a/docs/platform/llm-server/monitoring/GNMON-000-DOC_Index.md b/docs/platform/llm-server/monitoring/GNMON-000-DOC_Index.md index 9b639be..3aab278 100644 --- a/docs/platform/llm-server/monitoring/GNMON-000-DOC_Index.md +++ b/docs/platform/llm-server/monitoring/GNMON-000-DOC_Index.md @@ -96,6 +96,32 @@ Betreiberentscheidung 04.11.2026: Benachrichtigung per E-Mail. - Offen bis M5: echter SMTP-Versand gegen den Betreiber-Mailserver (Operator-Schritt: secrets-Datei + Testmail). +### M3.5 + M3.6: Log-Browser und Alerts-Ansicht (04.11.2026) + +- `LokiClient` (query_range + Label-Values) und `AlertmanagerClient` + (v2 API) als weitere typed HTTP-Clients; Logs-Seite mit + Quellen-Filter (`syslog_identifier`), Zeilen-Suche (LogQL `|=`), + Live-Tail-Toggle (3-s-Polling, pausierbar); Alerts-Seite listet + aktive Alerts mit Severity-Farben. +- **Alloy-Journal-Pipeline überarbeitet** (Journald-Ingestion war + nach dem Reboot aktiv, aber ohne stabile Labels): + - Befund 1: In User-Scope öffnet die sd-journal-Auto-Discovery nur + das User-Journal; die Workstation setzt darum einen expliziten + Journal-Pfad (`gnmon_alloy_journal_path`, Inventory pro Host). + - Befund 2: Das in der Dokumentation propagierte + `relabel_rules = loki.relabel.*.rules` am `loki.source.journal` + überträgt die `__journal_*`-Felder in v1.21.3 (User-Scope) nicht + in stabile Labels (systematische Verifikation über statisches + Label und Test-Logger: Labels kamen nie an). Ersatz: + `format_as_json = true` am Source + `loki.process` mit + `stage.json`/`stage.labels`/`stage.output` extrahiert + `unit`, `user_unit`, `syslog_identifier`, `priority` deterministisch + und reduziert die Zeile auf MESSAGE. +- Verifikation: Logger-Probes landen mit `syslog_identifier=ghost`, + `priority=5` in Loki; Kernel-Einträge mit `syslog_identifier=kernel`; + user-unit-Einträge mit `unit=user@1000.service`. Ansible-Rolle + idempotent (`changed=0`). + ### M3: Blazor-Dashboard (implementiert 04.11.2026, Workstation) M3.1–M3.4 in kleinen visuell abnehmbaren Schritten umgesetzt diff --git a/llm-server/monitoring/ansible/inventories/workstation/group_vars/all.yml b/llm-server/monitoring/ansible/inventories/workstation/group_vars/all.yml index df1a3b3..615fab5 100644 --- a/llm-server/monitoring/ansible/inventories/workstation/group_vars/all.yml +++ b/llm-server/monitoring/ansible/inventories/workstation/group_vars/all.yml @@ -8,3 +8,7 @@ # line (the v1.136 LTS line has no bare OSS tags in the Docker registry). gnmon_victoria_metrics_image_tag: v1.154.1 gnmon_victoria_metrics_image_digest: null + +# In user scope the sd-journal auto-discovery only opens the user +# journal; point alloy at the system journal directory explicitly. +gnmon_alloy_journal_path: "/var/log/journal/1a18b069dd7a4191b5f060e72aedae87" diff --git a/llm-server/monitoring/ansible/roles/gnmon_alloy/defaults/main.yml b/llm-server/monitoring/ansible/roles/gnmon_alloy/defaults/main.yml index fb24daa..b3d23b9 100644 --- a/llm-server/monitoring/ansible/roles/gnmon_alloy/defaults/main.yml +++ b/llm-server/monitoring/ansible/roles/gnmon_alloy/defaults/main.yml @@ -18,6 +18,12 @@ gnmon_alloy_cache_dir: "{{ gnmon_alloy_user_home }}/.cache/gnmon" gnmon_alloy_http_listen: "127.0.0.1:12345" gnmon_alloy_loki_push_url: "http://127.0.0.1:3100/loki/api/v1/push" +# journald source. In user-scope operation the sd-journal auto-discovery +# only opens the user journal; the workstation inventory therefore sets +# an explicit journal path. Empty string = auto-discovery. +gnmon_alloy_journal_path: "" +gnmon_alloy_journal_max_age: "12h" + # journald source (needs systemd-journal group membership). gnmon_alloy_journal_enabled: true gnmon_alloy_journal_max_age: "12h" diff --git a/llm-server/monitoring/ansible/roles/gnmon_alloy/templates/alloy-config.alloy.j2 b/llm-server/monitoring/ansible/roles/gnmon_alloy/templates/alloy-config.alloy.j2 index 55a3bbe..9963de6 100644 --- a/llm-server/monitoring/ansible/roles/gnmon_alloy/templates/alloy-config.alloy.j2 +++ b/llm-server/monitoring/ansible/roles/gnmon_alloy/templates/alloy-config.alloy.j2 @@ -8,38 +8,50 @@ loki.write "gnmon" { } } -{% if gnmon_alloy_journal_enabled %} -// Relabel journal fields into Loki labels before writing -// (documented pattern: pass the loki.relabel rules export to the -// journal source; internal __journal_* labels are dropped otherwise). -loki.relabel "gnmon_journal" { +// Journal pipeline: the source passes entries as JSON (all journal +// fields); the process stage extracts unit/syslog_identifier/priority +// labels and reduces the line back to MESSAGE. The upstream +// relabel_rules handoff (__journal_* -> stable labels) does not work +// in alloy v1.21.3 in user scope (labels never arrived, found +// 04.11.2026); the JSON pipeline achieves the same deterministically. +loki.process "gnmon_journal" { forward_to = [loki.write.gnmon.receiver] - rule { - action = "replace" - source_labels = ["__journal_unit"] - target_label = "unit" + stage.json { + expressions = { + unit = "_SYSTEMD_UNIT", + user_unit = "_SYSTEMD_USER_UNIT", + syslog_identifier = "SYSLOG_IDENTIFIER", + priority = "PRIORITY", + message = "MESSAGE", + } } - rule { - action = "replace" - source_labels = ["__journal_syslog_identifier"] - target_label = "syslog_identifier" + + stage.labels { + values = { + unit = "", + user_unit = "", + syslog_identifier = "", + priority = "", + } } - rule { - action = "replace" - source_labels = ["__journal_priority_keyword"] - target_label = "priority" + + stage.output { + source = "message" } } loki.source.journal "gnmon" { - max_age = "{{ gnmon_alloy_journal_max_age }}" - forward_to = [loki.relabel.gnmon_journal.receiver] - relabel_rules = loki.relabel.gnmon_journal.rules -} + max_age = "{{ gnmon_alloy_journal_max_age }}" + format_as_json = true + forward_to = [loki.process.gnmon_journal.receiver] +{% if gnmon_alloy_journal_path | length > 0 %} + // Explicit journal path (see role defaults; auto-discovery in user + // scope only opened the user journal). + path = "{{ gnmon_alloy_journal_path }}" {% endif %} +} -{% if gnmon_alloy_file_targets | length > 0 %} local.file_match "gnmon_files" { path_targets = [ {% for target in gnmon_alloy_file_targets %} @@ -52,4 +64,3 @@ loki.source.file "gnmon_files" { targets = local.file_match.gnmon_files.targets forward_to = [loki.write.gnmon.receiver] } -{% endif %} diff --git a/llm-server/monitoring/src/Ghostnet.Dashboard/Components/Pages/Alerts.razor b/llm-server/monitoring/src/Ghostnet.Dashboard/Components/Pages/Alerts.razor index 26a983a..277c80b 100644 --- a/llm-server/monitoring/src/Ghostnet.Dashboard/Components/Pages/Alerts.razor +++ b/llm-server/monitoring/src/Ghostnet.Dashboard/Components/Pages/Alerts.razor @@ -1,17 +1,93 @@ @page "/alerts" @rendermode InteractiveServer +@using Ghostnet.Dashboard.Services +@inject AlertmanagerClient Alertmanager +@implements IDisposable Alerts - Ghostnet.AI Monitoring

Alerts

-

Aktive Alarme aus vmalert / Alertmanager — Anbindung folgt in M3.6.

-
- Keine aktiven Alerts (Platzhalter) - -- -
+@if (_error is not null) +{ +
Alertmanager nicht erreichbar: @_error
+} +else if (_alerts.Count == 0) +{ +
+ Keine aktiven Alerts + OK +
+} +else +{ + @foreach (var alert in _alerts) + { +
+ + @alert.AlertName + @if (!string.IsNullOrEmpty(alert.Summary)) + { + — @alert.Summary + } + + + @if (alert.StartsAt != DateTimeOffset.MinValue) + { + seit @alert.StartsAt.LocalDateTime.ToString("HH:mm") + } + @alert.Severity + +
+ } +} -
- Konfigurierte Regeln (M4): GpuJunctionTempCritical · GpuFanStalledUnderLoad · - GpuEdgeTempHigh · GpuVramNearFull · NodeExporterDown · NodeExporterAbsent -
+@code +{ + private CancellationTokenSource? _cts; + private string? _error; + private List _alerts = []; + + protected override void OnInitialized() + { + _cts = new CancellationTokenSource(); + _ = RunLoopAsync(_cts.Token); + } + + private async Task RunLoopAsync(CancellationToken ct) + { + try + { + using var timer = new PeriodicTimer(TimeSpan.FromSeconds(5)); + while (!ct.IsCancellationRequested) + { + try + { + _alerts = [.. await Alertmanager.GetAlertsAsync(ct)]; + _error = null; + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + _error = ex.Message; + } + await InvokeAsync(StateHasChanged); + if (!await timer.WaitForNextTickAsync(ct)) + { + break; + } + } + } + catch (OperationCanceledException) + { + } + } + + private static string RowClass(AlertmanagerClient.Alert alert) => alert.Severity switch + { + "critical" => "alert-row sev-critical", + "warning" => "alert-row sev-warning", + _ => "alert-row", + }; + + public void Dispose() => _cts?.Cancel(); +} diff --git a/llm-server/monitoring/src/Ghostnet.Dashboard/Components/Pages/Logs.razor b/llm-server/monitoring/src/Ghostnet.Dashboard/Components/Pages/Logs.razor index c14270d..289860c 100644 --- a/llm-server/monitoring/src/Ghostnet.Dashboard/Components/Pages/Logs.razor +++ b/llm-server/monitoring/src/Ghostnet.Dashboard/Components/Pages/Logs.razor @@ -1,30 +1,177 @@ @page "/logs" @rendermode InteractiveServer +@using Ghostnet.Dashboard.Services +@inject LokiClient Loki +@implements IDisposable Logs - Ghostnet.AI Monitoring

Logs

-

Browser über Loki — Abfrage folgt in M3.5.

Unit / Quelle - + + @foreach (var source in _sources) + { + + }
Suche - +
- - + +
-
-
Ergebnisse
- - Keine Logeinträge geladen (M3.5 verbindet diese Ansicht mit Loki - query_range). - +
+
Ergebnisse (@_entries.Count)
+ @if (_error is not null) + { +
Loki nicht erreichbar: @_error
+ } + else if (_entries.Count == 0) + { + Keine Treffer im Zeitfenster (letzte @WindowMinutes Minuten). + } + else + { +
+ @foreach (var entry in _entries) + { +
+ @entry.Timestamp.LocalDateTime.ToString("HH:mm:ss") + @UnitSummary(entry) + @entry.Line +
+ } +
+ }
+ +@code +{ + private const int WindowMinutes = 15; + private const int MaxEntries = 300; + + private CancellationTokenSource? _cts; + private bool _live = true; + private string? _error; + private string _selectedSource = ""; + private string _search = ""; + private List _sources = []; + private List _entries = []; + + protected override void OnInitialized() + { + _cts = new CancellationTokenSource(); + _ = RunLoopAsync(_cts.Token); + } + + private Task OnSourceChanged(ChangeEventArgs e) + { + _selectedSource = e.Value?.ToString() ?? ""; + return RefreshAsync(_cts?.Token ?? default); + } + + private Task OnSearchChanged(ChangeEventArgs e) + { + _search = e.Value?.ToString() ?? ""; + return RefreshAsync(_cts?.Token ?? default); + } + + private void ToggleLive() + { + _live = !_live; + } + + private async Task RunLoopAsync(CancellationToken ct) + { + // Der erste Lauf laedt Filter und Eintraege; danach fragt der + // Live-Tail alle 3 Sekunden ab. + try + { + await RefreshAsync(ct); + await InvokeAsync(StateHasChanged); + using var timer = new PeriodicTimer(TimeSpan.FromSeconds(3)); + while (!ct.IsCancellationRequested) + { + if (!await timer.WaitForNextTickAsync(ct)) + { + break; + } + if (_live) + { + await RefreshAsync(ct); + await InvokeAsync(StateHasChanged); + } + } + } + catch (OperationCanceledException) + { + } + } + + private bool _sourcesLoaded; + + private async Task RefreshAsync(CancellationToken ct) + { + try + { + if (!_sourcesLoaded) + { + _sources = [.. (await Loki.GetLabelValuesAsync("syslog_identifier", ct)).OrderBy(s => s)]; + _sourcesLoaded = true; + } + + var end = DateTimeOffset.Now; + var start = end.AddMinutes(-WindowMinutes); + var query = BuildQuery(); + _entries = [.. await Loki.QueryRangeAsync(query, start, end, MaxEntries, ct)]; + _error = null; + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + _error = ex.Message; + } + } + + /// + /// LogQL: selector mit Quellen-Filter und optionalem Zeilenfilter. + /// + private string BuildQuery() + { + var selector = string.IsNullOrEmpty(_selectedSource) + ? "{job=~\".+\"}" + : $"{{syslog_identifier=\"{_selectedSource}\"}}"; + return string.IsNullOrWhiteSpace(_search) + ? selector + : $"{selector} |= \"{Escape(_search)}\""; + } + + private static string Escape(string text) => text.Replace("\"", "\\\""); + + private static string UnitSummary(LokiClient.LogEntry entry) + { + var unit = entry.Labels.GetValueOrDefault("unit", ""); + if (!string.IsNullOrEmpty(unit)) + { + return unit; + } + var userUnit = entry.Labels.GetValueOrDefault("user_unit", ""); + if (!string.IsNullOrEmpty(userUnit)) + { + return userUnit; + } + var sid = entry.Labels.GetValueOrDefault("syslog_identifier", ""); + return string.IsNullOrEmpty(sid) + ? entry.Labels.GetValueOrDefault("source", "") + : sid; + } + + public void Dispose() => _cts?.Cancel(); +} diff --git a/llm-server/monitoring/src/Ghostnet.Dashboard/Program.cs b/llm-server/monitoring/src/Ghostnet.Dashboard/Program.cs index ec03f65..2d137ea 100644 --- a/llm-server/monitoring/src/Ghostnet.Dashboard/Program.cs +++ b/llm-server/monitoring/src/Ghostnet.Dashboard/Program.cs @@ -15,6 +15,18 @@ builder.Services.AddHttpClient(client => builder.Configuration["Monitoring:VictoriaMetrics"] ?? "http://127.0.0.1:8428"); client.Timeout = TimeSpan.FromSeconds(5); }); +builder.Services.AddHttpClient(client => +{ + client.BaseAddress = new Uri( + builder.Configuration["Monitoring:Loki"] ?? "http://127.0.0.1:3100"); + client.Timeout = TimeSpan.FromSeconds(5); +}); +builder.Services.AddHttpClient(client => +{ + client.BaseAddress = new Uri( + builder.Configuration["Monitoring:Alertmanager"] ?? "http://127.0.0.1:9093"); + client.Timeout = TimeSpan.FromSeconds(5); +}); var app = builder.Build(); diff --git a/llm-server/monitoring/src/Ghostnet.Dashboard/Services/AlertmanagerClient.cs b/llm-server/monitoring/src/Ghostnet.Dashboard/Services/AlertmanagerClient.cs new file mode 100644 index 0000000..3a7f8e1 --- /dev/null +++ b/llm-server/monitoring/src/Ghostnet.Dashboard/Services/AlertmanagerClient.cs @@ -0,0 +1,57 @@ +using System.Net.Http.Json; +using System.Text.Json; + +namespace Ghostnet.Dashboard.Services; + +/// +/// Alertmanager v2 API client (GNMON-001 M4: active alerts view). +/// +public sealed class AlertmanagerClient +{ + private readonly HttpClient _http; + + public AlertmanagerClient(HttpClient http) => _http = http; + + public sealed record Alert( + string AlertName, + string Severity, + string State, + string Summary, + DateTimeOffset StartsAt); + + public async Task> GetAlertsAsync(CancellationToken ct) + { + using var response = await _http.GetAsync("/api/v2/alerts", ct); + response.EnsureSuccessStatusCode(); + + using var doc = await response.Content.ReadFromJsonAsync(ct); + var alerts = new List(); + if (doc is null || doc.RootElement.ValueKind != JsonValueKind.Array) + { + return alerts; + } + foreach (var item in doc.RootElement.EnumerateArray()) + { + var labels = item.GetProperty("labels"); + var annotations = item.TryGetProperty("annotations", out var ann) + ? ann + : default; + var startsAt = item.TryGetProperty("startsAt", out var starts) + && DateTime.TryParse(starts.GetString(), null, + System.Globalization.DateTimeStyles.RoundtripKind, out var parsed) + ? new DateTimeOffset(parsed, TimeSpan.Zero) + : DateTimeOffset.MinValue; + alerts.Add(new Alert( + labels.TryGetProperty("alertname", out var name) ? name.GetString() ?? "?" : "?", + labels.TryGetProperty("severity", out var severity) ? severity.GetString() ?? "" : "", + item.TryGetProperty("status", out var status) + && status.TryGetProperty("state", out var state) + ? state.GetString() ?? "" : "", + annotations.ValueKind == JsonValueKind.Object + && annotations.TryGetProperty("summary", out var summary) + ? summary.GetString() ?? "" : "", + startsAt)); + } + return alerts.OrderByDescending(a => a.Severity).ThenBy(a => a.AlertName).ToList(); + } +} diff --git a/llm-server/monitoring/src/Ghostnet.Dashboard/Services/LokiClient.cs b/llm-server/monitoring/src/Ghostnet.Dashboard/Services/LokiClient.cs new file mode 100644 index 0000000..5aa7ace --- /dev/null +++ b/llm-server/monitoring/src/Ghostnet.Dashboard/Services/LokiClient.cs @@ -0,0 +1,98 @@ +using System.Globalization; +using System.Net.Http.Json; +using System.Text.Json; + +namespace Ghostnet.Dashboard.Services; + +/// +/// Minimal Loki client (GNMON-001 section 5: query_range browser and +/// label filters). Parse results stay culture-invariant. +/// +public sealed class LokiClient +{ + private readonly HttpClient _http; + + public LokiClient(HttpClient http) => _http = http; + + public sealed record LogEntry( + DateTimeOffset Timestamp, + IReadOnlyDictionary Labels, + string Line); + + /// + /// Runs a LogQL query_range and returns flat entries, newest first. + /// + public async Task> QueryRangeAsync( + string logql, DateTimeOffset start, DateTimeOffset end, int limit, + CancellationToken ct) + { + var url = string.Create(CultureInfo.InvariantCulture, + $"/loki/api/v1/query_range?query={Uri.EscapeDataString(logql)}" + + $"&start={start.ToUnixTimeMilliseconds() * 1_000_000}" + + $"&end={end.ToUnixTimeMilliseconds() * 1_000_000}" + + $"&limit={limit}&direction=backward"); + using var response = await _http.GetAsync(url, ct); + response.EnsureSuccessStatusCode(); + + using var doc = await response.Content.ReadFromJsonAsync(ct); + var entries = new List(); + if (doc is null) + { + return entries; + } + var streams = doc.RootElement + .GetProperty("data") + .GetProperty("result"); + if (streams.ValueKind != JsonValueKind.Array) + { + return entries; + } + foreach (var stream in streams.EnumerateArray()) + { + var labels = new Dictionary(); + foreach (var label in stream.GetProperty("stream").EnumerateObject()) + { + labels[label.Name] = label.Value.GetString() ?? ""; + } + foreach (var value in stream.GetProperty("values").EnumerateArray()) + { + var ns = value[0].GetString(); + var line = value[1].GetString() ?? ""; + if (!long.TryParse(ns, out var nanoseconds)) + { + continue; + } + entries.Add(new LogEntry( + DateTimeOffset.FromUnixTimeMilliseconds(nanoseconds / 1_000_000), + labels, line)); + } + } + return entries.OrderByDescending(e => e.Timestamp).ToList(); + } + + /// + /// Returns the known values of a label (e.g. "unit") for filters. + /// + public async Task> GetLabelValuesAsync( + string label, CancellationToken ct) + { + using var response = await _http.GetAsync( + $"/loki/api/v1/label/{Uri.EscapeDataString(label)}/values", ct); + response.EnsureSuccessStatusCode(); + + using var doc = await response.Content.ReadFromJsonAsync(ct); + var values = new List(); + if (doc is null) + { + return values; + } + foreach (var value in doc.RootElement + .GetProperty("data") + .GetProperty("values") + .EnumerateArray()) + { + values.Add(value.GetString() ?? ""); + } + return values; + } +} diff --git a/llm-server/monitoring/src/Ghostnet.Dashboard/wwwroot/app.css b/llm-server/monitoring/src/Ghostnet.Dashboard/wwwroot/app.css index a753806..5856fb4 100644 --- a/llm-server/monitoring/src/Ghostnet.Dashboard/wwwroot/app.css +++ b/llm-server/monitoring/src/Ghostnet.Dashboard/wwwroot/app.css @@ -552,3 +552,99 @@ button { width: 100%; z-index: 1000; } + +button.btn-active { + border-color: var(--green); + color: var(--green); + box-shadow: 0 0 8px rgba(166, 227, 161, 0.25); +} + +/* ---- log browser ---- */ +.log-panel .log-list { + display: flex; + flex-direction: column; + gap: 0.1rem; + max-height: 70vh; + overflow-y: auto; +} + +.log-row { + display: flex; + gap: 0.7rem; + align-items: baseline; + padding: 0.1rem 0.2rem; +} + +.log-row:nth-child(odd) { + background: rgba(24, 24, 37, 0.5); +} + +.log-time { + color: var(--overlay0); + font-size: 0.78rem; + flex: 0 0 auto; +} + +.log-units { + color: var(--teal); + font-size: 0.78rem; + flex: 0 0 auto; + min-width: 9rem; + max-width: 14rem; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.log-line { + flex: 1 1 auto; + min-width: 0; + color: var(--subtext1); + white-space: pre-wrap; + overflow-wrap: anywhere; +} + +/* ---- alerts view ---- */ +.alert-main { + display: flex; + gap: 0.6rem; + align-items: baseline; + min-width: 0; + overflow: hidden; +} + +.alert-name { + font-weight: 700; + letter-spacing: 0.03em; + white-space: nowrap; +} + +.alert-summary { + color: var(--subtext0); + font-size: 0.82rem; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.alert-meta { + display: flex; + gap: 0.7rem; + align-items: baseline; + flex: 0 0 auto; +} + +.alert-since { + color: var(--overlay0); + font-size: 0.78rem; +} + +.alert-row.sev-critical .badge { + color: var(--red); + border-color: var(--red); +} + +.alert-row.sev-warning .badge { + color: var(--yellow); + border-color: var(--yellow); +}