629 lines
19 KiB
Bash
Executable File
629 lines
19 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$PROJECT_DIR"
|
|
|
|
# Source report generator
|
|
source "$PROJECT_DIR/scripts/lib/report-generator.sh" 2>/dev/null || true
|
|
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m'
|
|
|
|
ok() {
|
|
echo -e "${GREEN}[OK]${NC} $1"
|
|
}
|
|
|
|
warn() {
|
|
echo -e "${YELLOW}[WARN]${NC} $1"
|
|
}
|
|
|
|
fail() {
|
|
echo -e "${RED}[FAIL]${NC} $1"
|
|
exit 1
|
|
}
|
|
|
|
need_cmd() {
|
|
command -v "$1" >/dev/null 2>&1 || fail "Benoetigtes Kommando fehlt: $1"
|
|
}
|
|
|
|
read_env_var() {
|
|
local key="$1"
|
|
local env_file="$PROJECT_DIR/.env"
|
|
local line=""
|
|
|
|
if [ -f "$env_file" ]; then
|
|
line="$(grep -E "^${key}=" "$env_file" | tail -n 1 || true)"
|
|
fi
|
|
|
|
if [ -z "$line" ] && [ -f "$PROJECT_DIR/.env.example" ]; then
|
|
line="$(grep -E "^${key}=" "$PROJECT_DIR/.env.example" | tail -n 1 || true)"
|
|
fi
|
|
|
|
if [ -n "$line" ]; then
|
|
printf '%s' "${line#*=}"
|
|
fi
|
|
}
|
|
|
|
resolve_abs_path() {
|
|
local raw_path="$1"
|
|
if [[ "$raw_path" = /* ]]; then
|
|
printf '%s' "$raw_path"
|
|
return
|
|
fi
|
|
if command -v realpath >/dev/null 2>&1; then
|
|
realpath -m "$PROJECT_DIR/$raw_path"
|
|
return
|
|
fi
|
|
(cd "$PROJECT_DIR" && printf '%s/%s\n' "$PWD" "${raw_path#./}")
|
|
}
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage: ./scripts/benchmark-stack.sh [options]
|
|
|
|
Startet den Stack (optional), fuehrt eine reproduzierbare OpenClaw-Anfrage aus,
|
|
validiert den Output und zeigt Metriken (Laufzeit, Token, KV-/VRAM-Metriken).
|
|
|
|
Optionen:
|
|
--model <name> OpenClaw-Modell (default: ollama/qwen3-coding:latest)
|
|
--lines <n> Erwartete Zeilenanzahl LINE_1..LINE_n (default: 120)
|
|
--timeout-sec <n> Timeout fuer den Agent-Call in Sekunden (default: 600)
|
|
--session-key <key> Session-Key fuer den Lauf (default: auto)
|
|
--runtime-dir <path> Zielverzeichnis fuer temporaere Laufdaten (default: auto)
|
|
--results-dir <path> Zielverzeichnis fuer kuratierte Resultate (default: auto)
|
|
--artifact-dir <path> Legacy-Alias; nutzt denselben Pfad fuer Runtime und Resultate
|
|
--keep-runtime Runtime-Dateien nach dem Export nicht loeschen
|
|
--no-start Stack nicht starten, nur vorhandenen Stack nutzen
|
|
--no-reload Modell vor Lauf nicht mit 'ollama stop' entladen
|
|
-h, --help Hilfe anzeigen
|
|
|
|
Beispiel:
|
|
./scripts/benchmark-stack.sh --model ollama/qwen3-coding:latest --lines 140
|
|
EOF
|
|
}
|
|
|
|
strip_provider_prefix() {
|
|
local model="$1"
|
|
printf '%s' "${model#ollama/}"
|
|
}
|
|
|
|
wait_for_container_running() {
|
|
local name="$1"
|
|
local timeout_sec="$2"
|
|
local waited=0
|
|
while [ "$waited" -lt "$timeout_sec" ]; do
|
|
if podman ps --format '{{.Names}}' | grep -qx "$name"; then
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
waited=$((waited + 1))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
wait_for_openclaw_ready() {
|
|
local timeout_sec="$1"
|
|
local waited=0
|
|
while [ "$waited" -lt "$timeout_sec" ]; do
|
|
if podman exec openclaw sh -lc 'openclaw models list >/tmp/bench_models 2>&1' >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
waited=$((waited + 2))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
wait_for_ollama_ready() {
|
|
local timeout_sec="$1"
|
|
local waited=0
|
|
while [ "$waited" -lt "$timeout_sec" ]; do
|
|
if podman exec ollama sh -lc 'ollama list >/tmp/bench_ollama_list 2>&1' >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
waited=$((waited + 2))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
extract_quoted_metric() {
|
|
local pattern="$1"
|
|
local key="$2"
|
|
local line
|
|
line="$(rg "$pattern" "$OLLAMA_LOG_FILE" | tail -n 1 || true)"
|
|
if [ -z "$line" ]; then
|
|
echo "n/a"
|
|
return
|
|
fi
|
|
|
|
local value
|
|
value="$(printf '%s' "$line" | sed -E "s/.*${key}=\"([^\"]+)\".*/\\1/")"
|
|
if [ -z "$value" ] || [ "$value" = "$line" ]; then
|
|
echo "n/a"
|
|
else
|
|
echo "$value"
|
|
fi
|
|
}
|
|
|
|
extract_unquoted_metric() {
|
|
local pattern="$1"
|
|
local sed_expr="$2"
|
|
local line
|
|
line="$(rg "$pattern" "$OLLAMA_LOG_FILE" | tail -n 1 || true)"
|
|
if [ -z "$line" ]; then
|
|
echo "n/a"
|
|
return
|
|
fi
|
|
|
|
local value
|
|
value="$(printf '%s' "$line" | sed -E "$sed_expr")"
|
|
if [ -z "$value" ] || [ "$value" = "$line" ]; then
|
|
echo "n/a"
|
|
else
|
|
echo "$value"
|
|
fi
|
|
}
|
|
|
|
resolve_workspace_media_path() {
|
|
local media_path="$1"
|
|
local normalized="$media_path"
|
|
|
|
normalized="${normalized#/workspace/}"
|
|
normalized="${normalized#/home/node/.openclaw/workspace/}"
|
|
|
|
if [ -z "$normalized" ] || [ "$normalized" = "$media_path" -a [[ "$media_path" = /* ]] ]; then
|
|
return 1
|
|
fi
|
|
|
|
printf '%s/%s\n' "$HOST_WORKSPACE_ABS" "$normalized"
|
|
}
|
|
|
|
extract_tool_call_text() {
|
|
local raw_text="$1"
|
|
local extracted
|
|
|
|
extracted="$(printf '%s\n' "$raw_text" | sed '/^```json$/d; /^```$/d; /^```$/d' | jq -rs '
|
|
map(select(type == "object" and (.name == "write" or .name == "edit"))) as $calls
|
|
| if ($calls | length) == 0 then ""
|
|
else
|
|
reduce $calls[] as $call ({content: ""};
|
|
if $call.name == "write" then
|
|
.content = ($call.arguments.content // .content)
|
|
elif $call.name == "edit" then
|
|
.content = (
|
|
if (($call.arguments.edits // []) | length) > 0 then
|
|
($call.arguments.edits[-1].newText // .content)
|
|
else
|
|
.content
|
|
end
|
|
)
|
|
else
|
|
.
|
|
end
|
|
)
|
|
| .content
|
|
end
|
|
' 2>/dev/null || true)"
|
|
|
|
if [ -n "$extracted" ]; then
|
|
printf '%s' "$extracted"
|
|
return 0
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
extract_media_payload_text() {
|
|
local media_path=""
|
|
local host_path=""
|
|
|
|
while IFS= read -r media_path; do
|
|
[ -n "$media_path" ] || continue
|
|
host_path="$(resolve_workspace_media_path "$media_path" || true)"
|
|
if [ -n "$host_path" ] && [ -f "$host_path" ]; then
|
|
cat "$host_path"
|
|
return 0
|
|
fi
|
|
done < <(jq -r '.result.payloads[]? | .mediaUrl?, (.mediaUrls[]?)' "$JSON_OUT_FILE" 2>/dev/null | awk 'NF && !seen[$0]++')
|
|
|
|
return 1
|
|
}
|
|
|
|
MODEL="ollama/qwen3-coding:latest"
|
|
EXPECTED_LINES=120
|
|
TIMEOUT_SEC=600
|
|
SESSION_KEY=""
|
|
RUNTIME_DIR=""
|
|
RESULTS_DIR=""
|
|
LEGACY_ARTIFACT_DIR=""
|
|
START_STACK=1
|
|
FORCE_RELOAD=1
|
|
KEEP_RUNTIME=0
|
|
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--model)
|
|
MODEL="$2"
|
|
shift 2
|
|
;;
|
|
--lines)
|
|
EXPECTED_LINES="$2"
|
|
shift 2
|
|
;;
|
|
--timeout-sec)
|
|
TIMEOUT_SEC="$2"
|
|
shift 2
|
|
;;
|
|
--session-key)
|
|
SESSION_KEY="$2"
|
|
shift 2
|
|
;;
|
|
--runtime-dir)
|
|
RUNTIME_DIR="$2"
|
|
shift 2
|
|
;;
|
|
--results-dir)
|
|
RESULTS_DIR="$2"
|
|
shift 2
|
|
;;
|
|
--artifact-dir)
|
|
LEGACY_ARTIFACT_DIR="$2"
|
|
shift 2
|
|
;;
|
|
--keep-runtime)
|
|
KEEP_RUNTIME=1
|
|
shift
|
|
;;
|
|
--no-start)
|
|
START_STACK=0
|
|
shift
|
|
;;
|
|
--no-reload)
|
|
FORCE_RELOAD=0
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
fail "Unbekannte Option: $1"
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if ! [[ "$EXPECTED_LINES" =~ ^[0-9]+$ ]] || [ "$EXPECTED_LINES" -lt 1 ]; then
|
|
fail "--lines muss eine positive Ganzzahl sein"
|
|
fi
|
|
|
|
if ! [[ "$TIMEOUT_SEC" =~ ^[0-9]+$ ]] || [ "$TIMEOUT_SEC" -lt 1 ]; then
|
|
fail "--timeout-sec muss eine positive Ganzzahl sein"
|
|
fi
|
|
|
|
need_cmd podman
|
|
need_cmd jq
|
|
need_cmd rg
|
|
need_cmd sed
|
|
need_cmd awk
|
|
need_cmd timeout
|
|
|
|
RUN_ID="$(date +%Y%m%d-%H%M%S)"
|
|
HOST_WORKSPACE_RAW="$(read_env_var HOST_WORKSPACE_PATH)"
|
|
HOST_WORKSPACE_RAW="${HOST_WORKSPACE_RAW:-./storage/workspace}"
|
|
HOST_WORKSPACE_ABS="$(resolve_abs_path "$HOST_WORKSPACE_RAW")"
|
|
BENCH_OUTPUT_FILENAME="benchmark-output-${RUN_ID}.txt"
|
|
|
|
if [ -n "$LEGACY_ARTIFACT_DIR" ]; then
|
|
RUNTIME_DIR="$LEGACY_ARTIFACT_DIR"
|
|
if [ -z "$RESULTS_DIR" ]; then
|
|
RESULTS_DIR="$LEGACY_ARTIFACT_DIR"
|
|
KEEP_RUNTIME=1
|
|
fi
|
|
fi
|
|
|
|
if [ -z "$RUNTIME_DIR" ]; then
|
|
RUNTIME_DIR="$PROJECT_DIR/.cache/benchmarks/existing-suite/runs/$RUN_ID"
|
|
fi
|
|
|
|
if [ -z "$RESULTS_DIR" ]; then
|
|
RESULTS_DIR="$PROJECT_DIR/docs/benchmarks/existing-suite/$RUN_ID"
|
|
fi
|
|
|
|
mkdir -p "$RUNTIME_DIR" "$RESULTS_DIR"
|
|
|
|
JSON_OUT_FILE="$RUNTIME_DIR/agent-result.json"
|
|
ASSISTANT_TEXT_FILE="$RUNTIME_DIR/assistant-output.txt"
|
|
OLLAMA_LOG_FILE="$RUNTIME_DIR/ollama-since.log"
|
|
STATS_BEFORE_FILE="$RUNTIME_DIR/container-stats-before.txt"
|
|
STATS_AFTER_FILE="$RUNTIME_DIR/container-stats-after.txt"
|
|
REQUEST_ERR_FILE="$RUNTIME_DIR/request-stderr.log"
|
|
SUMMARY_FILE="$RUNTIME_DIR/summary.txt"
|
|
EXPORTED_SUMMARY_FILE="$RESULTS_DIR/summary.txt"
|
|
EXPORTED_ASSISTANT_FILE="$RESULTS_DIR/assistant-output.txt"
|
|
EXPORTED_JSON_FILE="$RESULTS_DIR/agent-result.json"
|
|
EXPORTED_OLLAMA_LOG_FILE="$RESULTS_DIR/ollama-since.log"
|
|
EXPORTED_STATS_BEFORE_FILE="$RESULTS_DIR/container-stats-before.txt"
|
|
EXPORTED_STATS_AFTER_FILE="$RESULTS_DIR/container-stats-after.txt"
|
|
EXPORTED_REQUEST_ERR_FILE="$RESULTS_DIR/request-stderr.log"
|
|
|
|
if [ -z "$SESSION_KEY" ]; then
|
|
SESSION_KEY="agent:main:benchmark:$(date +%s)"
|
|
fi
|
|
|
|
MODEL_OLLAMA="$(strip_provider_prefix "$MODEL")"
|
|
|
|
echo "=== Ghostnet Benchmark ==="
|
|
echo "Projekt: $PROJECT_DIR"
|
|
echo "Model: $MODEL"
|
|
echo "Session key: $SESSION_KEY"
|
|
echo "Erwartete Zeilen: $EXPECTED_LINES"
|
|
echo "Timeout: ${TIMEOUT_SEC}s"
|
|
echo "Runtime-Dateien: $RUNTIME_DIR"
|
|
echo "Kuratierte Resultate: $RESULTS_DIR"
|
|
|
|
if [ "$START_STACK" -eq 1 ]; then
|
|
echo
|
|
echo "[1/6] Starte Stack via ./ghostnet.sh up ..."
|
|
./ghostnet.sh up
|
|
else
|
|
echo
|
|
echo "[1/6] Stack-Start uebersprungen (--no-start)"
|
|
fi
|
|
|
|
echo "[2/6] Warte auf Container-Readiness ..."
|
|
wait_for_container_running ollama 180 || fail "Container ollama ist nicht running"
|
|
wait_for_container_running openclaw 180 || fail "Container openclaw ist nicht running"
|
|
wait_for_ollama_ready 120 || fail "Ollama API wurde nicht rechtzeitig bereit"
|
|
wait_for_openclaw_ready 180 || fail "OpenClaw wurde nicht rechtzeitig bereit"
|
|
ok "Container und APIs sind bereit"
|
|
|
|
if [ "$FORCE_RELOAD" -eq 1 ]; then
|
|
echo "[3/6] Entlade Modell fuer frische KV/VRAM-Load-Metriken ..."
|
|
podman exec ollama sh -lc "ollama stop '$MODEL_OLLAMA' >/dev/null 2>&1 || true"
|
|
else
|
|
echo "[3/6] Modell-Entladung uebersprungen (--no-reload)"
|
|
fi
|
|
|
|
echo "[4/6] Fuehre Benchmark-Anfrage aus ..."
|
|
PROMPT="Gib exakt ${EXPECTED_LINES} Zeilen aus. Jede Zeile muss exakt den Text LINE_<n> haben, mit aufsteigender Nummerierung von 1 bis ${EXPECTED_LINES}. Kein weiterer Text. Nutze keine Tools, keine Dateien, keine Anhaenge und kein JSON. Antworte direkt als reiner Nachrichtentext. Falls du trotzdem Tools benutzt, schreibe exakt diese Zeilen in die Datei ${BENCH_OUTPUT_FILENAME} im Workspace und sonst nirgendwo hin."
|
|
SINCE_UTC="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
|
|
podman stats --no-stream --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}' ollama openclaw > "$STATS_BEFORE_FILE" 2>&1 || true
|
|
|
|
START_NS="$(date +%s%N)"
|
|
REQUEST_RC=0
|
|
if ! timeout "${TIMEOUT_SEC}s" podman exec openclaw openclaw agent \
|
|
--session-key "$SESSION_KEY" \
|
|
--model "$MODEL" \
|
|
--thinking off \
|
|
--message "$PROMPT" \
|
|
--json > "$JSON_OUT_FILE" 2> "$REQUEST_ERR_FILE"; then
|
|
REQUEST_RC=$?
|
|
fi
|
|
END_NS="$(date +%s%N)"
|
|
WALL_MS=$(( (END_NS - START_NS) / 1000000 ))
|
|
|
|
podman logs --since "$SINCE_UTC" ollama > "$OLLAMA_LOG_FILE" 2>&1 || true
|
|
podman stats --no-stream --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}' ollama openclaw > "$STATS_AFTER_FILE" 2>&1 || true
|
|
|
|
HAS_VALID_JSON=0
|
|
if [ -s "$JSON_OUT_FILE" ] && jq -e . "$JSON_OUT_FILE" >/dev/null 2>&1; then
|
|
HAS_VALID_JSON=1
|
|
fi
|
|
if [ "$HAS_VALID_JSON" -ne 1 ] && [ "$REQUEST_RC" -eq 0 ]; then
|
|
REQUEST_RC=65
|
|
fi
|
|
|
|
REQUEST_ERROR_MSG=""
|
|
if [ "$REQUEST_RC" -ne 0 ]; then
|
|
REQUEST_ERROR_MSG="$(tail -n 20 "$REQUEST_ERR_FILE" 2>/dev/null | tr '\n' ' ' | sed -E 's/[[:space:]]+/ /g' | sed -E 's/^ //; s/ $//')"
|
|
if [ -z "$REQUEST_ERROR_MSG" ]; then
|
|
if [ "$HAS_VALID_JSON" -ne 1 ]; then
|
|
REQUEST_ERROR_MSG="Agent lieferte kein valides JSON"
|
|
else
|
|
REQUEST_ERROR_MSG="Agent-Lauf fehlgeschlagen ohne stderr-Ausgabe"
|
|
fi
|
|
fi
|
|
warn "Agent-Lauf fehlgeschlagen (rc=$REQUEST_RC): $REQUEST_ERROR_MSG"
|
|
fi
|
|
|
|
echo "[5/6] Validiere Antwortformat ..."
|
|
ASSISTANT_TEXT=""
|
|
RAW_ASSISTANT_TEXT=""
|
|
if [ "$HAS_VALID_JSON" -eq 1 ]; then
|
|
RAW_ASSISTANT_TEXT="$(jq -r '[.result.payloads[]?.text // empty] | join("\n")' "$JSON_OUT_FILE")"
|
|
ASSISTANT_TEXT="$RAW_ASSISTANT_TEXT"
|
|
if [ -n "$RAW_ASSISTANT_TEXT" ]; then
|
|
TOOL_CALL_TEXT="$(extract_tool_call_text "$RAW_ASSISTANT_TEXT" || true)"
|
|
if [ -n "$TOOL_CALL_TEXT" ]; then
|
|
ASSISTANT_TEXT="$TOOL_CALL_TEXT"
|
|
fi
|
|
fi
|
|
if [ -z "$ASSISTANT_TEXT" ]; then
|
|
ASSISTANT_TEXT="$(extract_media_payload_text || true)"
|
|
fi
|
|
fi
|
|
printf '%s' "$ASSISTANT_TEXT" > "$ASSISTANT_TEXT_FILE"
|
|
|
|
ACTUAL_LINES=()
|
|
if [ -n "$ASSISTANT_TEXT" ]; then
|
|
mapfile -t ACTUAL_LINES < "$ASSISTANT_TEXT_FILE"
|
|
for i in "${!ACTUAL_LINES[@]}"; do
|
|
ACTUAL_LINES[$i]="${ACTUAL_LINES[$i]%$'\r'}"
|
|
done
|
|
fi
|
|
|
|
VALIDATION_OK=1
|
|
VALIDATION_DETAIL="OK"
|
|
|
|
if [ "$REQUEST_RC" -ne 0 ]; then
|
|
VALIDATION_OK=0
|
|
VALIDATION_DETAIL="Agent-Lauf fehlgeschlagen (rc=$REQUEST_RC)"
|
|
elif [ "${#ACTUAL_LINES[@]}" -ne "$EXPECTED_LINES" ]; then
|
|
VALIDATION_OK=0
|
|
VALIDATION_DETAIL="Zeilenanzahl abweichend: erwartet=$EXPECTED_LINES, erhalten=${#ACTUAL_LINES[@]}"
|
|
else
|
|
for ((idx = 1; idx <= EXPECTED_LINES; idx++)); do
|
|
expected="LINE_${idx}"
|
|
actual="${ACTUAL_LINES[$((idx - 1))]}"
|
|
if [ "$actual" != "$expected" ]; then
|
|
VALIDATION_OK=0
|
|
VALIDATION_DETAIL="Mismatch in Zeile $idx: erwartet='$expected', erhalten='$actual'"
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
|
|
if [ "$VALIDATION_OK" -eq 1 ]; then
|
|
ok "Output validiert: exakt LINE_1..LINE_$EXPECTED_LINES"
|
|
else
|
|
warn "Output-Validierung fehlgeschlagen: $VALIDATION_DETAIL"
|
|
fi
|
|
|
|
echo "[6/6] Sammle und berechne Metriken ..."
|
|
DURATION_MS="n/a"
|
|
USAGE_INPUT="n/a"
|
|
USAGE_OUTPUT="n/a"
|
|
USAGE_TOTAL="n/a"
|
|
PROMPT_TOKENS="n/a"
|
|
CONTEXT_TOKENS="n/a"
|
|
STOP_REASON="n/a"
|
|
FINISH_REASON="n/a"
|
|
|
|
if [ "$HAS_VALID_JSON" -eq 1 ]; then
|
|
DURATION_MS="$(jq -r '.result.meta.durationMs // "n/a"' "$JSON_OUT_FILE")"
|
|
USAGE_INPUT="$(jq -r '.result.meta.agentMeta.usage.input // "n/a"' "$JSON_OUT_FILE")"
|
|
USAGE_OUTPUT="$(jq -r '.result.meta.agentMeta.usage.output // "n/a"' "$JSON_OUT_FILE")"
|
|
USAGE_TOTAL="$(jq -r '.result.meta.agentMeta.usage.total // "n/a"' "$JSON_OUT_FILE")"
|
|
PROMPT_TOKENS="$(jq -r '.result.meta.agentMeta.promptTokens // "n/a"' "$JSON_OUT_FILE")"
|
|
CONTEXT_TOKENS="$(jq -r '.result.meta.agentMeta.contextTokens // "n/a"' "$JSON_OUT_FILE")"
|
|
STOP_REASON="$(jq -r '.result.meta.completion.stopReason // "n/a"' "$JSON_OUT_FILE")"
|
|
FINISH_REASON="$(jq -r '.result.meta.completion.finishReason // "n/a"' "$JSON_OUT_FILE")"
|
|
fi
|
|
|
|
OUTPUT_TOKENS_PER_SEC="n/a"
|
|
if [[ "$DURATION_MS" =~ ^[0-9]+$ ]] && [ "$DURATION_MS" -gt 0 ] && [[ "$USAGE_OUTPUT" =~ ^[0-9]+$ ]]; then
|
|
OUTPUT_TOKENS_PER_SEC="$(awk "BEGIN { printf \"%.2f\", ($USAGE_OUTPUT * 1000) / $DURATION_MS }")"
|
|
fi
|
|
|
|
END2END_TOKENS_PER_SEC="n/a"
|
|
if [[ "$WALL_MS" =~ ^[0-9]+$ ]] && [ "$WALL_MS" -gt 0 ] && [[ "$USAGE_OUTPUT" =~ ^[0-9]+$ ]]; then
|
|
END2END_TOKENS_PER_SEC="$(awk "BEGIN { printf \"%.2f\", ($USAGE_OUTPUT * 1000) / $WALL_MS }")"
|
|
fi
|
|
|
|
GPU_AVAILABLE="$(extract_quoted_metric 'msg="gpu memory"' 'available')"
|
|
GPU_FREE="$(extract_quoted_metric 'msg="gpu memory"' 'free')"
|
|
GPU_OVERHEAD="$(extract_quoted_metric 'msg="gpu memory"' 'overhead')"
|
|
KV_GPU="$(extract_quoted_metric 'msg="kv cache" device=ROCm0' 'size')"
|
|
KV_CPU="$(extract_quoted_metric 'msg="kv cache" device=CPU' 'size')"
|
|
MODEL_GPU="$(extract_quoted_metric 'msg="model weights" device=ROCm0' 'size')"
|
|
MODEL_CPU="$(extract_quoted_metric 'msg="model weights" device=CPU' 'size')"
|
|
COMPUTE_GPU="$(extract_quoted_metric 'msg="compute graph" device=ROCm0' 'size')"
|
|
COMPUTE_CPU="$(extract_quoted_metric 'msg="compute graph" device=CPU' 'size')"
|
|
TOTAL_MEMORY="$(extract_quoted_metric 'msg="total memory"' 'size')"
|
|
KV_SIZE_TOKENS="$(extract_unquoted_metric 'KvSize:[0-9]+' 's/.*KvSize:([0-9]+).*/\1/')"
|
|
RUNNER_START_SEC="$(extract_unquoted_metric 'llama runner started in [0-9]+(\.[0-9]+)? seconds' 's/.*llama runner started in ([0-9]+(\.[0-9]+)?) seconds.*/\1/')"
|
|
|
|
OLLAMA_ENV="$(podman exec ollama sh -lc 'echo "KV_CACHE=$OLLAMA_KV_CACHE_TYPE GPU_OVERHEAD=$OLLAMA_GPU_OVERHEAD KEEP_ALIVE=$OLLAMA_KEEP_ALIVE FLASH_ATTN=$OLLAMA_FLASH_ATTENTION MAX_LOADED=$OLLAMA_MAX_LOADED_MODELS NUM_PARALLEL=$OLLAMA_NUM_PARALLEL"' 2>/dev/null || true)"
|
|
|
|
JOURNAL_HITS="n/a"
|
|
if command -v journalctl >/dev/null 2>&1; then
|
|
JOURNAL_HITS="$(journalctl -b --since "$SINCE_UTC" --no-pager 2>/dev/null | rg -i 'amdgpu|MESA: error|graphics reset|drm|failed to allocate a buffer|not enough memory for command submission' | wc -l | tr -d ' ' || true)"
|
|
[ -n "$JOURNAL_HITS" ] || JOURNAL_HITS="0"
|
|
fi
|
|
|
|
{
|
|
echo "=== Ghostnet Benchmark Summary ==="
|
|
echo "Model: $MODEL"
|
|
echo "Session key: $SESSION_KEY"
|
|
echo "Request exit code: $REQUEST_RC"
|
|
if [ "$REQUEST_RC" -ne 0 ]; then
|
|
echo "Request error: $REQUEST_ERROR_MSG"
|
|
fi
|
|
echo "Prompt line target: $EXPECTED_LINES"
|
|
echo "Validation: $([ "$VALIDATION_OK" -eq 1 ] && echo PASS || echo FAIL)"
|
|
echo "Validation detail: $VALIDATION_DETAIL"
|
|
echo
|
|
echo "Latenz und Token"
|
|
echo "- end_to_end_wall_ms: $WALL_MS"
|
|
echo "- openclaw_duration_ms: $DURATION_MS"
|
|
echo "- usage_input_tokens: $USAGE_INPUT"
|
|
echo "- usage_output_tokens: $USAGE_OUTPUT"
|
|
echo "- usage_total_tokens: $USAGE_TOTAL"
|
|
echo "- prompt_tokens: $PROMPT_TOKENS"
|
|
echo "- context_tokens: $CONTEXT_TOKENS"
|
|
echo "- stop_reason: $STOP_REASON"
|
|
echo "- finish_reason: $FINISH_REASON"
|
|
echo "- output_tokens_per_sec_provider: $OUTPUT_TOKENS_PER_SEC"
|
|
echo "- output_tokens_per_sec_end_to_end: $END2END_TOKENS_PER_SEC"
|
|
echo
|
|
echo "Ollama runtime"
|
|
echo "- env: $OLLAMA_ENV"
|
|
echo "- gpu_available: $GPU_AVAILABLE"
|
|
echo "- gpu_free: $GPU_FREE"
|
|
echo "- gpu_overhead: $GPU_OVERHEAD"
|
|
echo "- kv_cache_gpu: $KV_GPU"
|
|
echo "- kv_cache_cpu: $KV_CPU"
|
|
echo "- kv_size_tokens: $KV_SIZE_TOKENS"
|
|
echo "- model_weights_gpu: $MODEL_GPU"
|
|
echo "- model_weights_cpu: $MODEL_CPU"
|
|
echo "- compute_graph_gpu: $COMPUTE_GPU"
|
|
echo "- compute_graph_cpu: $COMPUTE_CPU"
|
|
echo "- total_model_memory: $TOTAL_MEMORY"
|
|
echo "- model_load_time_sec: $RUNNER_START_SEC"
|
|
echo
|
|
echo "Stabilitaet"
|
|
echo "- journal_gpu_related_hits_since_start: $JOURNAL_HITS"
|
|
echo
|
|
echo "Artefakte"
|
|
echo "- agent_json: $EXPORTED_JSON_FILE"
|
|
echo "- assistant_text: $EXPORTED_ASSISTANT_FILE"
|
|
echo "- ollama_log_since_start: $EXPORTED_OLLAMA_LOG_FILE"
|
|
echo "- stats_before: $EXPORTED_STATS_BEFORE_FILE"
|
|
echo "- stats_after: $EXPORTED_STATS_AFTER_FILE"
|
|
echo "- request_stderr: $EXPORTED_REQUEST_ERR_FILE"
|
|
} | tee "$SUMMARY_FILE"
|
|
|
|
echo
|
|
echo "Container-Stats vor Lauf:"
|
|
cat "$STATS_BEFORE_FILE"
|
|
echo
|
|
echo "Container-Stats nach Lauf:"
|
|
cat "$STATS_AFTER_FILE"
|
|
|
|
cp "$SUMMARY_FILE" "$EXPORTED_SUMMARY_FILE"
|
|
cp "$ASSISTANT_TEXT_FILE" "$EXPORTED_ASSISTANT_FILE"
|
|
cp "$JSON_OUT_FILE" "$EXPORTED_JSON_FILE"
|
|
cp "$OLLAMA_LOG_FILE" "$EXPORTED_OLLAMA_LOG_FILE"
|
|
cp "$STATS_BEFORE_FILE" "$EXPORTED_STATS_BEFORE_FILE"
|
|
cp "$STATS_AFTER_FILE" "$EXPORTED_STATS_AFTER_FILE"
|
|
|
|
if [ -s "$REQUEST_ERR_FILE" ]; then
|
|
cp "$REQUEST_ERR_FILE" "$EXPORTED_REQUEST_ERR_FILE"
|
|
fi
|
|
|
|
if [ "$KEEP_RUNTIME" -eq 0 ] && [ "$RUNTIME_DIR" != "$RESULTS_DIR" ]; then
|
|
rm -rf "$RUNTIME_DIR"
|
|
fi
|
|
|
|
# Generate professional reports
|
|
if [ -f "$EXPORTED_SUMMARY_FILE" ]; then
|
|
ok "Generating HTML report..."
|
|
generate_html_report "$MODEL_NAME" "$RESULTS_DIR" "$EXPORTED_SUMMARY_FILE" 2>/dev/null || true
|
|
|
|
ok "Generating Markdown summary..."
|
|
generate_markdown_summary "$MODEL_NAME" "$RESULTS_DIR" "$EXPORTED_SUMMARY_FILE" 2>/dev/null || true
|
|
fi
|
|
|
|
if [ "$REQUEST_RC" -eq 0 ] && [ "$VALIDATION_OK" -eq 1 ]; then
|
|
ok "Benchmark erfolgreich abgeschlossen"
|
|
exit 0
|
|
fi
|
|
|
|
warn "Benchmark beendet, aber Validierung ist fehlgeschlagen"
|
|
exit 2 |