GNMON-001 M3.5+M3.6: Log-Browser, Alerts-Ansicht, Alloy-Journal-Pipeline repariert

Logs-Seite mit Quelle-Filter, Zeilen-Suche und Live-Tail (3-s-Takt);
Alerts-Seite listet aktive Alerts mit Severity-Farben. Neue Clients:
LokiClient (query_range, Label-Values) und AlertmanagerClient (v2).
Journal-Pipeline: der dokumentierte relabel_rules-Handoff am
loki.source.journal uebertraeg __journal_* in v1.21.3/User-Scope nicht
in stabile Labels (verifiziert); Ersatz format_as_json +
loki.process (stage.json/labels/output) extrahiert unit, user_unit,
syslog_identifier, priority deterministisch. Expliziter Journal-Pfad
notwendig (User-Scope oeffnet sonst nur das User-Journal), als
Inventory-Variable parametrisiert. Fünf Befunde in GNMON-000.

Generated by Mistral Vibe.
Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
2026-11-04 17:47:37 +01:00
parent bad5641b9d
commit 888c1a294c
10 changed files with 577 additions and 44 deletions
@@ -96,6 +96,32 @@ Betreiberentscheidung 04.11.2026: Benachrichtigung per E-Mail.
- Offen bis M5: echter SMTP-Versand gegen den Betreiber-Mailserver
(Operator-Schritt: secrets-Datei + Testmail).
### M3.5 + M3.6: Log-Browser und Alerts-Ansicht (04.11.2026)
- `LokiClient` (query_range + Label-Values) und `AlertmanagerClient`
(v2 API) als weitere typed HTTP-Clients; Logs-Seite mit
Quellen-Filter (`syslog_identifier`), Zeilen-Suche (LogQL `|=`),
Live-Tail-Toggle (3-s-Polling, pausierbar); Alerts-Seite listet
aktive Alerts mit Severity-Farben.
- **Alloy-Journal-Pipeline überarbeitet** (Journald-Ingestion war
nach dem Reboot aktiv, aber ohne stabile Labels):
- Befund 1: In User-Scope öffnet die sd-journal-Auto-Discovery nur
das User-Journal; die Workstation setzt darum einen expliziten
Journal-Pfad (`gnmon_alloy_journal_path`, Inventory pro Host).
- Befund 2: Das in der Dokumentation propagierte
`relabel_rules = loki.relabel.*.rules` am `loki.source.journal`
überträgt die `__journal_*`-Felder in v1.21.3 (User-Scope) nicht
in stabile Labels (systematische Verifikation über statisches
Label und Test-Logger: Labels kamen nie an). Ersatz:
`format_as_json = true` am Source + `loki.process` mit
`stage.json`/`stage.labels`/`stage.output` extrahiert
`unit`, `user_unit`, `syslog_identifier`, `priority` deterministisch
und reduziert die Zeile auf MESSAGE.
- Verifikation: Logger-Probes landen mit `syslog_identifier=ghost`,
`priority=5` in Loki; Kernel-Einträge mit `syslog_identifier=kernel`;
user-unit-Einträge mit `unit=user@1000.service`. Ansible-Rolle
idempotent (`changed=0`).
### M3: Blazor-Dashboard (implementiert 04.11.2026, Workstation)
M3.1–M3.4 in kleinen visuell abnehmbaren Schritten umgesetzt
@@ -8,3 +8,7 @@
# line (the v1.136 LTS line has no bare OSS tags in the Docker registry).
gnmon_victoria_metrics_image_tag: v1.154.1
gnmon_victoria_metrics_image_digest: null
# In user scope the sd-journal auto-discovery only opens the user
# journal; point alloy at the system journal directory explicitly.
gnmon_alloy_journal_path: "/var/log/journal/1a18b069dd7a4191b5f060e72aedae87"
@@ -18,6 +18,12 @@ gnmon_alloy_cache_dir: "{{ gnmon_alloy_user_home }}/.cache/gnmon"
gnmon_alloy_http_listen: "127.0.0.1:12345"
gnmon_alloy_loki_push_url: "http://127.0.0.1:3100/loki/api/v1/push"
# journald source. In user-scope operation the sd-journal auto-discovery
# only opens the user journal; the workstation inventory therefore sets
# an explicit journal path. Empty string = auto-discovery.
gnmon_alloy_journal_path: ""
gnmon_alloy_journal_max_age: "12h"
# journald source (needs systemd-journal group membership).
gnmon_alloy_journal_enabled: true
gnmon_alloy_journal_max_age: "12h"
@@ -8,38 +8,50 @@ loki.write "gnmon" {
}
}
{% if gnmon_alloy_journal_enabled %}
// Relabel journal fields into Loki labels before writing
// (documented pattern: pass the loki.relabel rules export to the
// journal source; internal __journal_* labels are dropped otherwise).
loki.relabel "gnmon_journal" {
// Journal pipeline: the source passes entries as JSON (all journal
// fields); the process stage extracts unit/syslog_identifier/priority
// labels and reduces the line back to MESSAGE. The upstream
// relabel_rules handoff (__journal_* -> stable labels) does not work
// in alloy v1.21.3 in user scope (labels never arrived, found
// 04.11.2026); the JSON pipeline achieves the same deterministically.
loki.process "gnmon_journal" {
forward_to = [loki.write.gnmon.receiver]
rule {
action = "replace"
source_labels = ["__journal_unit"]
target_label = "unit"
stage.json {
expressions = {
unit = "_SYSTEMD_UNIT",
user_unit = "_SYSTEMD_USER_UNIT",
syslog_identifier = "SYSLOG_IDENTIFIER",
priority = "PRIORITY",
message = "MESSAGE",
}
}
rule {
action = "replace"
source_labels = ["__journal_syslog_identifier"]
target_label = "syslog_identifier"
stage.labels {
values = {
unit = "",
user_unit = "",
syslog_identifier = "",
priority = "",
}
}
rule {
action = "replace"
source_labels = ["__journal_priority_keyword"]
target_label = "priority"
stage.output {
source = "message"
}
}
loki.source.journal "gnmon" {
max_age = "{{ gnmon_alloy_journal_max_age }}"
forward_to = [loki.relabel.gnmon_journal.receiver]
relabel_rules = loki.relabel.gnmon_journal.rules
}
max_age = "{{ gnmon_alloy_journal_max_age }}"
format_as_json = true
forward_to = [loki.process.gnmon_journal.receiver]
{% if gnmon_alloy_journal_path | length > 0 %}
// Explicit journal path (see role defaults; auto-discovery in user
// scope only opened the user journal).
path = "{{ gnmon_alloy_journal_path }}"
{% endif %}
}
{% if gnmon_alloy_file_targets | length > 0 %}
local.file_match "gnmon_files" {
path_targets = [
{% for target in gnmon_alloy_file_targets %}
@@ -52,4 +64,3 @@ loki.source.file "gnmon_files" {
targets = local.file_match.gnmon_files.targets
forward_to = [loki.write.gnmon.receiver]
}
{% endif %}
@@ -1,17 +1,93 @@
@page "/alerts"
@rendermode InteractiveServer
@using Ghostnet.Dashboard.Services
@inject AlertmanagerClient Alertmanager
@implements IDisposable
<PageTitle>Alerts - Ghostnet.AI Monitoring</PageTitle>
<h1>Alerts</h1>
<p class="hint">Aktive Alarme aus vmalert / Alertmanager — Anbindung folgt in M3.6.</p>
<div class="alert-row">
<span>Keine aktiven Alerts (Platzhalter)</span>
<span class="badge">--</span>
</div>
@if (_error is not null)
{
<div class="note">Alertmanager nicht erreichbar: @_error</div>
}
else if (_alerts.Count == 0)
{
<div class="alert-row">
<span>Keine aktiven Alerts</span>
<span class="badge">OK</span>
</div>
}
else
{
@foreach (var alert in _alerts)
{
<div class="@RowClass(alert)">
<span class="alert-main">
<span class="alert-name">@alert.AlertName</span>
@if (!string.IsNullOrEmpty(alert.Summary))
{
<span class="alert-summary">&mdash; @alert.Summary</span>
}
</span>
<span class="alert-meta">
@if (alert.StartsAt != DateTimeOffset.MinValue)
{
<span class="alert-since">seit @alert.StartsAt.LocalDateTime.ToString("HH:mm")</span>
}
<span class="badge">@alert.Severity</span>
</span>
</div>
}
}
<div class="note note-accent">
Konfigurierte Regeln (M4): GpuJunctionTempCritical · GpuFanStalledUnderLoad ·
GpuEdgeTempHigh · GpuVramNearFull · NodeExporterDown · NodeExporterAbsent
</div>
@code
{
private CancellationTokenSource? _cts;
private string? _error;
private List<AlertmanagerClient.Alert> _alerts = [];
protected override void OnInitialized()
{
_cts = new CancellationTokenSource();
_ = RunLoopAsync(_cts.Token);
}
private async Task RunLoopAsync(CancellationToken ct)
{
try
{
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(5));
while (!ct.IsCancellationRequested)
{
try
{
_alerts = [.. await Alertmanager.GetAlertsAsync(ct)];
_error = null;
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
{
_error = ex.Message;
}
await InvokeAsync(StateHasChanged);
if (!await timer.WaitForNextTickAsync(ct))
{
break;
}
}
}
catch (OperationCanceledException)
{
}
}
private static string RowClass(AlertmanagerClient.Alert alert) => alert.Severity switch
{
"critical" => "alert-row sev-critical",
"warning" => "alert-row sev-warning",
_ => "alert-row",
};
public void Dispose() => _cts?.Cancel();
}
@@ -1,30 +1,177 @@
@page "/logs"
@rendermode InteractiveServer
@using Ghostnet.Dashboard.Services
@inject LokiClient Loki
@implements IDisposable
<PageTitle>Logs - Ghostnet.AI Monitoring</PageTitle>
<h1>Logs</h1>
<p class="hint">Browser über Loki — Abfrage folgt in M3.5.</p>
<div class="toolbar">
<div class="toolbar-field">
<span class="toolbar-label" id="log-unit-label">Unit / Quelle</span>
<select aria-labelledby="log-unit-label" disabled>
<option>Alle</option>
<select aria-labelledby="log-unit-label" value="@_selectedSource" @onchange="OnSourceChanged">
<option value="">Alle</option>
@foreach (var source in _sources)
{
<option value="@source">@source</option>
}
</select>
</div>
<div class="toolbar-field">
<span class="toolbar-label" id="log-search-label">Suche</span>
<input type="text" aria-labelledby="log-search-label" placeholder="…" disabled />
<input type="text" aria-labelledby="log-search-label" placeholder="&hellip;"
value="@_search" @onchange="OnSearchChanged" />
</div>
<button disabled>Suchen</button>
<button disabled>Live-Tail</button>
<button class="@(_live ? "btn-active" : "")" @onclick="ToggleLive">@(_live ? "II Pause" : "▶ Live-Tail")</button>
<button @onclick="() => _ = RefreshAsync(_cts?.Token ?? default)">↻ Aktualisieren</button>
</div>
<div class="panel">
<div class="panel-header">Ergebnisse</div>
<span class="tile-sub">
Keine Logeinträge geladen (M3.5 verbindet diese Ansicht mit Loki
<code>query_range</code>).
</span>
<div class="panel log-panel">
<div class="panel-header">Ergebnisse (@_entries.Count)</div>
@if (_error is not null)
{
<div class="note">Loki nicht erreichbar: @_error</div>
}
else if (_entries.Count == 0)
{
<span class="tile-sub">Keine Treffer im Zeitfenster (letzte @WindowMinutes Minuten).</span>
}
else
{
<div class="log-list">
@foreach (var entry in _entries)
{
<div class="log-row">
<span class="log-time">@entry.Timestamp.LocalDateTime.ToString("HH:mm:ss")</span>
<span class="log-units">@UnitSummary(entry)</span>
<span class="log-line">@entry.Line</span>
</div>
}
</div>
}
</div>
@code
{
private const int WindowMinutes = 15;
private const int MaxEntries = 300;
private CancellationTokenSource? _cts;
private bool _live = true;
private string? _error;
private string _selectedSource = "";
private string _search = "";
private List<string> _sources = [];
private List<LokiClient.LogEntry> _entries = [];
protected override void OnInitialized()
{
_cts = new CancellationTokenSource();
_ = RunLoopAsync(_cts.Token);
}
private Task OnSourceChanged(ChangeEventArgs e)
{
_selectedSource = e.Value?.ToString() ?? "";
return RefreshAsync(_cts?.Token ?? default);
}
private Task OnSearchChanged(ChangeEventArgs e)
{
_search = e.Value?.ToString() ?? "";
return RefreshAsync(_cts?.Token ?? default);
}
private void ToggleLive()
{
_live = !_live;
}
private async Task RunLoopAsync(CancellationToken ct)
{
// Der erste Lauf laedt Filter und Eintraege; danach fragt der
// Live-Tail alle 3 Sekunden ab.
try
{
await RefreshAsync(ct);
await InvokeAsync(StateHasChanged);
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(3));
while (!ct.IsCancellationRequested)
{
if (!await timer.WaitForNextTickAsync(ct))
{
break;
}
if (_live)
{
await RefreshAsync(ct);
await InvokeAsync(StateHasChanged);
}
}
}
catch (OperationCanceledException)
{
}
}
private bool _sourcesLoaded;
private async Task RefreshAsync(CancellationToken ct)
{
try
{
if (!_sourcesLoaded)
{
_sources = [.. (await Loki.GetLabelValuesAsync("syslog_identifier", ct)).OrderBy(s => s)];
_sourcesLoaded = true;
}
var end = DateTimeOffset.Now;
var start = end.AddMinutes(-WindowMinutes);
var query = BuildQuery();
_entries = [.. await Loki.QueryRangeAsync(query, start, end, MaxEntries, ct)];
_error = null;
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
{
_error = ex.Message;
}
}
/// <summary>
/// LogQL: selector mit Quellen-Filter und optionalem Zeilenfilter.
/// </summary>
private string BuildQuery()
{
var selector = string.IsNullOrEmpty(_selectedSource)
? "{job=~\".+\"}"
: $"{{syslog_identifier=\"{_selectedSource}\"}}";
return string.IsNullOrWhiteSpace(_search)
? selector
: $"{selector} |= \"{Escape(_search)}\"";
}
private static string Escape(string text) => text.Replace("\"", "\\\"");
private static string UnitSummary(LokiClient.LogEntry entry)
{
var unit = entry.Labels.GetValueOrDefault("unit", "");
if (!string.IsNullOrEmpty(unit))
{
return unit;
}
var userUnit = entry.Labels.GetValueOrDefault("user_unit", "");
if (!string.IsNullOrEmpty(userUnit))
{
return userUnit;
}
var sid = entry.Labels.GetValueOrDefault("syslog_identifier", "");
return string.IsNullOrEmpty(sid)
? entry.Labels.GetValueOrDefault("source", "")
: sid;
}
public void Dispose() => _cts?.Cancel();
}
@@ -15,6 +15,18 @@ builder.Services.AddHttpClient<VictoriaMetricsClient>(client =>
builder.Configuration["Monitoring:VictoriaMetrics"] ?? "http://127.0.0.1:8428");
client.Timeout = TimeSpan.FromSeconds(5);
});
builder.Services.AddHttpClient<LokiClient>(client =>
{
client.BaseAddress = new Uri(
builder.Configuration["Monitoring:Loki"] ?? "http://127.0.0.1:3100");
client.Timeout = TimeSpan.FromSeconds(5);
});
builder.Services.AddHttpClient<AlertmanagerClient>(client =>
{
client.BaseAddress = new Uri(
builder.Configuration["Monitoring:Alertmanager"] ?? "http://127.0.0.1:9093");
client.Timeout = TimeSpan.FromSeconds(5);
});
var app = builder.Build();
@@ -0,0 +1,57 @@
using System.Net.Http.Json;
using System.Text.Json;
namespace Ghostnet.Dashboard.Services;
/// <summary>
/// Alertmanager v2 API client (GNMON-001 M4: active alerts view).
/// </summary>
public sealed class AlertmanagerClient
{
private readonly HttpClient _http;
public AlertmanagerClient(HttpClient http) => _http = http;
public sealed record Alert(
string AlertName,
string Severity,
string State,
string Summary,
DateTimeOffset StartsAt);
public async Task<IReadOnlyList<Alert>> GetAlertsAsync(CancellationToken ct)
{
using var response = await _http.GetAsync("/api/v2/alerts", ct);
response.EnsureSuccessStatusCode();
using var doc = await response.Content.ReadFromJsonAsync<JsonDocument>(ct);
var alerts = new List<Alert>();
if (doc is null || doc.RootElement.ValueKind != JsonValueKind.Array)
{
return alerts;
}
foreach (var item in doc.RootElement.EnumerateArray())
{
var labels = item.GetProperty("labels");
var annotations = item.TryGetProperty("annotations", out var ann)
? ann
: default;
var startsAt = item.TryGetProperty("startsAt", out var starts)
&& DateTime.TryParse(starts.GetString(), null,
System.Globalization.DateTimeStyles.RoundtripKind, out var parsed)
? new DateTimeOffset(parsed, TimeSpan.Zero)
: DateTimeOffset.MinValue;
alerts.Add(new Alert(
labels.TryGetProperty("alertname", out var name) ? name.GetString() ?? "?" : "?",
labels.TryGetProperty("severity", out var severity) ? severity.GetString() ?? "" : "",
item.TryGetProperty("status", out var status)
&& status.TryGetProperty("state", out var state)
? state.GetString() ?? "" : "",
annotations.ValueKind == JsonValueKind.Object
&& annotations.TryGetProperty("summary", out var summary)
? summary.GetString() ?? "" : "",
startsAt));
}
return alerts.OrderByDescending(a => a.Severity).ThenBy(a => a.AlertName).ToList();
}
}
@@ -0,0 +1,98 @@
using System.Globalization;
using System.Net.Http.Json;
using System.Text.Json;
namespace Ghostnet.Dashboard.Services;
/// <summary>
/// Minimal Loki client (GNMON-001 section 5: query_range browser and
/// label filters). Parse results stay culture-invariant.
/// </summary>
public sealed class LokiClient
{
private readonly HttpClient _http;
public LokiClient(HttpClient http) => _http = http;
public sealed record LogEntry(
DateTimeOffset Timestamp,
IReadOnlyDictionary<string, string> Labels,
string Line);
/// <summary>
/// Runs a LogQL query_range and returns flat entries, newest first.
/// </summary>
public async Task<IReadOnlyList<LogEntry>> QueryRangeAsync(
string logql, DateTimeOffset start, DateTimeOffset end, int limit,
CancellationToken ct)
{
var url = string.Create(CultureInfo.InvariantCulture,
$"/loki/api/v1/query_range?query={Uri.EscapeDataString(logql)}"
+ $"&start={start.ToUnixTimeMilliseconds() * 1_000_000}"
+ $"&end={end.ToUnixTimeMilliseconds() * 1_000_000}"
+ $"&limit={limit}&direction=backward");
using var response = await _http.GetAsync(url, ct);
response.EnsureSuccessStatusCode();
using var doc = await response.Content.ReadFromJsonAsync<JsonDocument>(ct);
var entries = new List<LogEntry>();
if (doc is null)
{
return entries;
}
var streams = doc.RootElement
.GetProperty("data")
.GetProperty("result");
if (streams.ValueKind != JsonValueKind.Array)
{
return entries;
}
foreach (var stream in streams.EnumerateArray())
{
var labels = new Dictionary<string, string>();
foreach (var label in stream.GetProperty("stream").EnumerateObject())
{
labels[label.Name] = label.Value.GetString() ?? "";
}
foreach (var value in stream.GetProperty("values").EnumerateArray())
{
var ns = value[0].GetString();
var line = value[1].GetString() ?? "";
if (!long.TryParse(ns, out var nanoseconds))
{
continue;
}
entries.Add(new LogEntry(
DateTimeOffset.FromUnixTimeMilliseconds(nanoseconds / 1_000_000),
labels, line));
}
}
return entries.OrderByDescending(e => e.Timestamp).ToList();
}
/// <summary>
/// Returns the known values of a label (e.g. "unit") for filters.
/// </summary>
public async Task<IReadOnlyList<string>> GetLabelValuesAsync(
string label, CancellationToken ct)
{
using var response = await _http.GetAsync(
$"/loki/api/v1/label/{Uri.EscapeDataString(label)}/values", ct);
response.EnsureSuccessStatusCode();
using var doc = await response.Content.ReadFromJsonAsync<JsonDocument>(ct);
var values = new List<string>();
if (doc is null)
{
return values;
}
foreach (var value in doc.RootElement
.GetProperty("data")
.GetProperty("values")
.EnumerateArray())
{
values.Add(value.GetString() ?? "");
}
return values;
}
}
@@ -552,3 +552,99 @@ button {
width: 100%;
z-index: 1000;
}
button.btn-active {
border-color: var(--green);
color: var(--green);
box-shadow: 0 0 8px rgba(166, 227, 161, 0.25);
}
/* ---- log browser ---- */
.log-panel .log-list {
display: flex;
flex-direction: column;
gap: 0.1rem;
max-height: 70vh;
overflow-y: auto;
}
.log-row {
display: flex;
gap: 0.7rem;
align-items: baseline;
padding: 0.1rem 0.2rem;
}
.log-row:nth-child(odd) {
background: rgba(24, 24, 37, 0.5);
}
.log-time {
color: var(--overlay0);
font-size: 0.78rem;
flex: 0 0 auto;
}
.log-units {
color: var(--teal);
font-size: 0.78rem;
flex: 0 0 auto;
min-width: 9rem;
max-width: 14rem;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.log-line {
flex: 1 1 auto;
min-width: 0;
color: var(--subtext1);
white-space: pre-wrap;
overflow-wrap: anywhere;
}
/* ---- alerts view ---- */
.alert-main {
display: flex;
gap: 0.6rem;
align-items: baseline;
min-width: 0;
overflow: hidden;
}
.alert-name {
font-weight: 700;
letter-spacing: 0.03em;
white-space: nowrap;
}
.alert-summary {
color: var(--subtext0);
font-size: 0.82rem;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.alert-meta {
display: flex;
gap: 0.7rem;
align-items: baseline;
flex: 0 0 auto;
}
.alert-since {
color: var(--overlay0);
font-size: 0.78rem;
}
.alert-row.sev-critical .badge {
color: var(--red);
border-color: var(--red);
}
.alert-row.sev-warning .badge {
color: var(--yellow);
border-color: var(--yellow);
}