Fix: User rights and sandboxing problems.

This commit is contained in:
2026-06-12 20:19:09 +02:00
parent 485237440f
commit 415c265db7
13 changed files with 887 additions and 49 deletions
+450
View File
@@ -0,0 +1,450 @@
#!/usr/bin/env bash
set -euo pipefail
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$PROJECT_DIR"
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'
ok() {
echo -e "${GREEN}[OK]${NC} $1"
}
warn() {
echo -e "${YELLOW}[WARN]${NC} $1"
}
fail() {
echo -e "${RED}[FAIL]${NC} $1"
exit 1
}
need_cmd() {
command -v "$1" >/dev/null 2>&1 || fail "Benoetigtes Kommando fehlt: $1"
}
usage() {
cat <<'EOF'
Usage: ./scripts/benchmark-stack.sh [options]
Startet den Stack (optional), fuehrt eine reproduzierbare OpenClaw-Anfrage aus,
validiert den Output und zeigt Metriken (Laufzeit, Token, KV-/VRAM-Metriken).
Optionen:
--model <name> OpenClaw-Modell (default: ollama/qwen-coding-long:latest)
--lines <n> Erwartete Zeilenanzahl LINE_1..LINE_n (default: 120)
--timeout-sec <n> Timeout fuer den Agent-Call in Sekunden (default: 480)
--session-key <key> Session-Key fuer den Lauf (default: auto)
--artifact-dir <path> Zielverzeichnis fuer Ergebnisdateien (default: auto)
--no-start Stack nicht starten, nur vorhandenen Stack nutzen
--no-reload Modell vor Lauf nicht mit 'ollama stop' entladen
-h, --help Hilfe anzeigen
Beispiel:
./scripts/benchmark-stack.sh --model ollama/qwen-coding-long:latest --lines 140
EOF
}
strip_provider_prefix() {
local model="$1"
printf '%s' "${model#ollama/}"
}
wait_for_container_running() {
local name="$1"
local timeout_sec="$2"
local waited=0
while [ "$waited" -lt "$timeout_sec" ]; do
if podman ps --format '{{.Names}}' | grep -qx "$name"; then
return 0
fi
sleep 1
waited=$((waited + 1))
done
return 1
}
wait_for_openclaw_ready() {
local timeout_sec="$1"
local waited=0
while [ "$waited" -lt "$timeout_sec" ]; do
if podman exec openclaw sh -lc 'openclaw models list >/tmp/bench_models 2>&1' >/dev/null 2>&1; then
return 0
fi
sleep 2
waited=$((waited + 2))
done
return 1
}
wait_for_ollama_ready() {
local timeout_sec="$1"
local waited=0
while [ "$waited" -lt "$timeout_sec" ]; do
if podman exec ollama sh -lc 'ollama list >/tmp/bench_ollama_list 2>&1' >/dev/null 2>&1; then
return 0
fi
sleep 2
waited=$((waited + 2))
done
return 1
}
extract_quoted_metric() {
local pattern="$1"
local key="$2"
local line
line="$(rg "$pattern" "$OLLAMA_LOG_FILE" | tail -n 1 || true)"
if [ -z "$line" ]; then
echo "n/a"
return
fi
local value
value="$(printf '%s' "$line" | sed -E "s/.*${key}=\"([^\"]+)\".*/\\1/")"
if [ -z "$value" ] || [ "$value" = "$line" ]; then
echo "n/a"
else
echo "$value"
fi
}
extract_unquoted_metric() {
local pattern="$1"
local sed_expr="$2"
local line
line="$(rg "$pattern" "$OLLAMA_LOG_FILE" | tail -n 1 || true)"
if [ -z "$line" ]; then
echo "n/a"
return
fi
local value
value="$(printf '%s' "$line" | sed -E "$sed_expr")"
if [ -z "$value" ] || [ "$value" = "$line" ]; then
echo "n/a"
else
echo "$value"
fi
}
MODEL="ollama/qwen-coding-long:latest"
EXPECTED_LINES=120
TIMEOUT_SEC=480
SESSION_KEY=""
ARTIFACT_DIR=""
START_STACK=1
FORCE_RELOAD=1
while [ "$#" -gt 0 ]; do
case "$1" in
--model)
MODEL="$2"
shift 2
;;
--lines)
EXPECTED_LINES="$2"
shift 2
;;
--timeout-sec)
TIMEOUT_SEC="$2"
shift 2
;;
--session-key)
SESSION_KEY="$2"
shift 2
;;
--artifact-dir)
ARTIFACT_DIR="$2"
shift 2
;;
--no-start)
START_STACK=0
shift
;;
--no-reload)
FORCE_RELOAD=0
shift
;;
-h|--help)
usage
exit 0
;;
*)
fail "Unbekannte Option: $1"
;;
esac
done
if ! [[ "$EXPECTED_LINES" =~ ^[0-9]+$ ]] || [ "$EXPECTED_LINES" -lt 1 ]; then
fail "--lines muss eine positive Ganzzahl sein"
fi
if ! [[ "$TIMEOUT_SEC" =~ ^[0-9]+$ ]] || [ "$TIMEOUT_SEC" -lt 1 ]; then
fail "--timeout-sec muss eine positive Ganzzahl sein"
fi
need_cmd podman
need_cmd jq
need_cmd rg
need_cmd sed
need_cmd awk
need_cmd timeout
if [ -z "$ARTIFACT_DIR" ]; then
TS="$(date +%Y%m%d-%H%M%S)"
ARTIFACT_DIR="$PROJECT_DIR/storage/workspace/benchmarks/$TS"
fi
mkdir -p "$ARTIFACT_DIR"
JSON_OUT_FILE="$ARTIFACT_DIR/agent-result.json"
ASSISTANT_TEXT_FILE="$ARTIFACT_DIR/assistant-output.txt"
OLLAMA_LOG_FILE="$ARTIFACT_DIR/ollama-since.log"
STATS_BEFORE_FILE="$ARTIFACT_DIR/container-stats-before.txt"
STATS_AFTER_FILE="$ARTIFACT_DIR/container-stats-after.txt"
REQUEST_ERR_FILE="$ARTIFACT_DIR/request-stderr.log"
SUMMARY_FILE="$ARTIFACT_DIR/summary.txt"
if [ -z "$SESSION_KEY" ]; then
SESSION_KEY="agent:main:benchmark:$(date +%s)"
fi
MODEL_OLLAMA="$(strip_provider_prefix "$MODEL")"
echo "=== Ghostnet Benchmark ==="
echo "Projekt: $PROJECT_DIR"
echo "Model: $MODEL"
echo "Session key: $SESSION_KEY"
echo "Erwartete Zeilen: $EXPECTED_LINES"
echo "Timeout: ${TIMEOUT_SEC}s"
echo "Artefakte: $ARTIFACT_DIR"
if [ "$START_STACK" -eq 1 ]; then
echo
echo "[1/6] Starte Stack via ./ghostnet.sh up ..."
./ghostnet.sh up
else
echo
echo "[1/6] Stack-Start uebersprungen (--no-start)"
fi
echo "[2/6] Warte auf Container-Readiness ..."
wait_for_container_running ollama 180 || fail "Container ollama ist nicht running"
wait_for_container_running openclaw 180 || fail "Container openclaw ist nicht running"
wait_for_ollama_ready 120 || fail "Ollama API wurde nicht rechtzeitig bereit"
wait_for_openclaw_ready 180 || fail "OpenClaw wurde nicht rechtzeitig bereit"
ok "Container und APIs sind bereit"
if [ "$FORCE_RELOAD" -eq 1 ]; then
echo "[3/6] Entlade Modell fuer frische KV/VRAM-Load-Metriken ..."
podman exec ollama sh -lc "ollama stop '$MODEL_OLLAMA' >/dev/null 2>&1 || true"
else
echo "[3/6] Modell-Entladung uebersprungen (--no-reload)"
fi
echo "[4/6] Fuehre Benchmark-Anfrage aus ..."
PROMPT="Gib exakt ${EXPECTED_LINES} Zeilen aus. Jede Zeile muss exakt den Text LINE_<n> haben, mit aufsteigender Nummerierung von 1 bis ${EXPECTED_LINES}. Kein weiterer Text."
SINCE_UTC="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
podman stats --no-stream --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}' ollama openclaw > "$STATS_BEFORE_FILE" 2>&1 || true
START_NS="$(date +%s%N)"
REQUEST_RC=0
if ! timeout "${TIMEOUT_SEC}s" podman exec openclaw openclaw agent \
--session-key "$SESSION_KEY" \
--model "$MODEL" \
--message "$PROMPT" \
--json > "$JSON_OUT_FILE" 2> "$REQUEST_ERR_FILE"; then
REQUEST_RC=$?
fi
END_NS="$(date +%s%N)"
WALL_MS=$(( (END_NS - START_NS) / 1000000 ))
podman logs --since "$SINCE_UTC" ollama > "$OLLAMA_LOG_FILE" 2>&1 || true
podman stats --no-stream --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}' ollama openclaw > "$STATS_AFTER_FILE" 2>&1 || true
HAS_VALID_JSON=0
if [ -s "$JSON_OUT_FILE" ] && jq -e . "$JSON_OUT_FILE" >/dev/null 2>&1; then
HAS_VALID_JSON=1
fi
if [ "$HAS_VALID_JSON" -ne 1 ] && [ "$REQUEST_RC" -eq 0 ]; then
REQUEST_RC=65
fi
REQUEST_ERROR_MSG=""
if [ "$REQUEST_RC" -ne 0 ]; then
REQUEST_ERROR_MSG="$(tail -n 20 "$REQUEST_ERR_FILE" 2>/dev/null | tr '\n' ' ' | sed -E 's/[[:space:]]+/ /g' | sed -E 's/^ //; s/ $//')"
if [ -z "$REQUEST_ERROR_MSG" ]; then
if [ "$HAS_VALID_JSON" -ne 1 ]; then
REQUEST_ERROR_MSG="Agent lieferte kein valides JSON"
else
REQUEST_ERROR_MSG="Agent-Lauf fehlgeschlagen ohne stderr-Ausgabe"
fi
fi
warn "Agent-Lauf fehlgeschlagen (rc=$REQUEST_RC): $REQUEST_ERROR_MSG"
fi
echo "[5/6] Validiere Antwortformat ..."
ASSISTANT_TEXT=""
if [ "$HAS_VALID_JSON" -eq 1 ]; then
ASSISTANT_TEXT="$(jq -r '.result.payloads[0].text // ""' "$JSON_OUT_FILE")"
fi
printf '%s' "$ASSISTANT_TEXT" > "$ASSISTANT_TEXT_FILE"
ACTUAL_LINES=()
if [ -n "$ASSISTANT_TEXT" ]; then
mapfile -t ACTUAL_LINES < "$ASSISTANT_TEXT_FILE"
for i in "${!ACTUAL_LINES[@]}"; do
ACTUAL_LINES[$i]="${ACTUAL_LINES[$i]%$'\r'}"
done
fi
VALIDATION_OK=1
VALIDATION_DETAIL="OK"
if [ "$REQUEST_RC" -ne 0 ]; then
VALIDATION_OK=0
VALIDATION_DETAIL="Agent-Lauf fehlgeschlagen (rc=$REQUEST_RC)"
elif [ "${#ACTUAL_LINES[@]}" -ne "$EXPECTED_LINES" ]; then
VALIDATION_OK=0
VALIDATION_DETAIL="Zeilenanzahl abweichend: erwartet=$EXPECTED_LINES, erhalten=${#ACTUAL_LINES[@]}"
else
for ((idx = 1; idx <= EXPECTED_LINES; idx++)); do
expected="LINE_${idx}"
actual="${ACTUAL_LINES[$((idx - 1))]}"
if [ "$actual" != "$expected" ]; then
VALIDATION_OK=0
VALIDATION_DETAIL="Mismatch in Zeile $idx: erwartet='$expected', erhalten='$actual'"
break
fi
done
fi
if [ "$VALIDATION_OK" -eq 1 ]; then
ok "Output validiert: exakt LINE_1..LINE_$EXPECTED_LINES"
else
warn "Output-Validierung fehlgeschlagen: $VALIDATION_DETAIL"
fi
echo "[6/6] Sammle und berechne Metriken ..."
DURATION_MS="n/a"
USAGE_INPUT="n/a"
USAGE_OUTPUT="n/a"
USAGE_TOTAL="n/a"
PROMPT_TOKENS="n/a"
CONTEXT_TOKENS="n/a"
STOP_REASON="n/a"
FINISH_REASON="n/a"
if [ "$HAS_VALID_JSON" -eq 1 ]; then
DURATION_MS="$(jq -r '.result.meta.durationMs // "n/a"' "$JSON_OUT_FILE")"
USAGE_INPUT="$(jq -r '.result.meta.agentMeta.usage.input // "n/a"' "$JSON_OUT_FILE")"
USAGE_OUTPUT="$(jq -r '.result.meta.agentMeta.usage.output // "n/a"' "$JSON_OUT_FILE")"
USAGE_TOTAL="$(jq -r '.result.meta.agentMeta.usage.total // "n/a"' "$JSON_OUT_FILE")"
PROMPT_TOKENS="$(jq -r '.result.meta.agentMeta.promptTokens // "n/a"' "$JSON_OUT_FILE")"
CONTEXT_TOKENS="$(jq -r '.result.meta.agentMeta.contextTokens // "n/a"' "$JSON_OUT_FILE")"
STOP_REASON="$(jq -r '.result.meta.completion.stopReason // "n/a"' "$JSON_OUT_FILE")"
FINISH_REASON="$(jq -r '.result.meta.completion.finishReason // "n/a"' "$JSON_OUT_FILE")"
fi
OUTPUT_TOKENS_PER_SEC="n/a"
if [[ "$DURATION_MS" =~ ^[0-9]+$ ]] && [ "$DURATION_MS" -gt 0 ] && [[ "$USAGE_OUTPUT" =~ ^[0-9]+$ ]]; then
OUTPUT_TOKENS_PER_SEC="$(awk "BEGIN { printf \"%.2f\", ($USAGE_OUTPUT * 1000) / $DURATION_MS }")"
fi
END2END_TOKENS_PER_SEC="n/a"
if [[ "$WALL_MS" =~ ^[0-9]+$ ]] && [ "$WALL_MS" -gt 0 ] && [[ "$USAGE_OUTPUT" =~ ^[0-9]+$ ]]; then
END2END_TOKENS_PER_SEC="$(awk "BEGIN { printf \"%.2f\", ($USAGE_OUTPUT * 1000) / $WALL_MS }")"
fi
GPU_AVAILABLE="$(extract_quoted_metric 'msg="gpu memory"' 'available')"
GPU_FREE="$(extract_quoted_metric 'msg="gpu memory"' 'free')"
GPU_OVERHEAD="$(extract_quoted_metric 'msg="gpu memory"' 'overhead')"
KV_GPU="$(extract_quoted_metric 'msg="kv cache" device=ROCm0' 'size')"
KV_CPU="$(extract_quoted_metric 'msg="kv cache" device=CPU' 'size')"
MODEL_GPU="$(extract_quoted_metric 'msg="model weights" device=ROCm0' 'size')"
MODEL_CPU="$(extract_quoted_metric 'msg="model weights" device=CPU' 'size')"
COMPUTE_GPU="$(extract_quoted_metric 'msg="compute graph" device=ROCm0' 'size')"
COMPUTE_CPU="$(extract_quoted_metric 'msg="compute graph" device=CPU' 'size')"
TOTAL_MEMORY="$(extract_quoted_metric 'msg="total memory"' 'size')"
KV_SIZE_TOKENS="$(extract_unquoted_metric 'KvSize:[0-9]+' 's/.*KvSize:([0-9]+).*/\1/')"
RUNNER_START_SEC="$(extract_unquoted_metric 'llama runner started in [0-9]+(\.[0-9]+)? seconds' 's/.*llama runner started in ([0-9]+(\.[0-9]+)?) seconds.*/\1/')"
OLLAMA_ENV="$(podman exec ollama sh -lc 'echo "KV_CACHE=$OLLAMA_KV_CACHE_TYPE GPU_OVERHEAD=$OLLAMA_GPU_OVERHEAD KEEP_ALIVE=$OLLAMA_KEEP_ALIVE FLASH_ATTN=$OLLAMA_FLASH_ATTENTION MAX_LOADED=$OLLAMA_MAX_LOADED_MODELS NUM_PARALLEL=$OLLAMA_NUM_PARALLEL"' 2>/dev/null || true)"
JOURNAL_HITS="n/a"
if command -v journalctl >/dev/null 2>&1; then
JOURNAL_HITS="$(journalctl -b --since "$SINCE_UTC" --no-pager 2>/dev/null | rg -i 'amdgpu|MESA: error|graphics reset|drm|failed to allocate a buffer|not enough memory for command submission' | wc -l | tr -d ' ' || true)"
[ -n "$JOURNAL_HITS" ] || JOURNAL_HITS="0"
fi
{
echo "=== Ghostnet Benchmark Summary ==="
echo "Model: $MODEL"
echo "Session key: $SESSION_KEY"
echo "Request exit code: $REQUEST_RC"
if [ "$REQUEST_RC" -ne 0 ]; then
echo "Request error: $REQUEST_ERROR_MSG"
fi
echo "Prompt line target: $EXPECTED_LINES"
echo "Validation: $([ "$VALIDATION_OK" -eq 1 ] && echo PASS || echo FAIL)"
echo "Validation detail: $VALIDATION_DETAIL"
echo
echo "Latenz und Token"
echo "- end_to_end_wall_ms: $WALL_MS"
echo "- openclaw_duration_ms: $DURATION_MS"
echo "- usage_input_tokens: $USAGE_INPUT"
echo "- usage_output_tokens: $USAGE_OUTPUT"
echo "- usage_total_tokens: $USAGE_TOTAL"
echo "- prompt_tokens: $PROMPT_TOKENS"
echo "- context_tokens: $CONTEXT_TOKENS"
echo "- stop_reason: $STOP_REASON"
echo "- finish_reason: $FINISH_REASON"
echo "- output_tokens_per_sec_provider: $OUTPUT_TOKENS_PER_SEC"
echo "- output_tokens_per_sec_end_to_end: $END2END_TOKENS_PER_SEC"
echo
echo "Ollama runtime"
echo "- env: $OLLAMA_ENV"
echo "- gpu_available: $GPU_AVAILABLE"
echo "- gpu_free: $GPU_FREE"
echo "- gpu_overhead: $GPU_OVERHEAD"
echo "- kv_cache_gpu: $KV_GPU"
echo "- kv_cache_cpu: $KV_CPU"
echo "- kv_size_tokens: $KV_SIZE_TOKENS"
echo "- model_weights_gpu: $MODEL_GPU"
echo "- model_weights_cpu: $MODEL_CPU"
echo "- compute_graph_gpu: $COMPUTE_GPU"
echo "- compute_graph_cpu: $COMPUTE_CPU"
echo "- total_model_memory: $TOTAL_MEMORY"
echo "- model_load_time_sec: $RUNNER_START_SEC"
echo
echo "Stabilitaet"
echo "- journal_gpu_related_hits_since_start: $JOURNAL_HITS"
echo
echo "Artefakte"
echo "- agent_json: $JSON_OUT_FILE"
echo "- assistant_text: $ASSISTANT_TEXT_FILE"
echo "- ollama_log_since_start: $OLLAMA_LOG_FILE"
echo "- stats_before: $STATS_BEFORE_FILE"
echo "- stats_after: $STATS_AFTER_FILE"
echo "- request_stderr: $REQUEST_ERR_FILE"
} | tee "$SUMMARY_FILE"
echo
echo "Container-Stats vor Lauf:"
cat "$STATS_BEFORE_FILE"
echo
echo "Container-Stats nach Lauf:"
cat "$STATS_AFTER_FILE"
if [ "$REQUEST_RC" -eq 0 ] && [ "$VALIDATION_OK" -eq 1 ]; then
ok "Benchmark erfolgreich abgeschlossen"
exit 0
fi
warn "Benchmark beendet, aber Validierung ist fehlgeschlagen"
exit 2