From ce64ef56e402ccdd23fcf7526b286ea160c3f716 Mon Sep 17 00:00:00 2001 From: ghost Date: Wed, 17 Jun 2026 19:14:08 +0200 Subject: [PATCH] Add: CodeNeedle Benchmark. Optimize model handling. --- .env.example | 3 +- AGENT_INSTRUCTIONS.md | 35 +- README.md | 68 +++- compose.amd.yaml | 1 + compose.intel.yaml | 1 + compose.nvidia.yaml | 1 + ollama/modelfiles/qwen3-general.modelfile | 22 ++ openclaw/config/openclaw.json | 2 +- scripts/benchmark-codeneedle-suite.sh | 393 ++++++++++++++++++++++ scripts/benchmark-model-suite.sh | 97 ++++-- scripts/benchmark-stack.sh | 233 +++++++++++-- 11 files changed, 799 insertions(+), 57 deletions(-) create mode 100644 ollama/modelfiles/qwen3-general.modelfile create mode 100755 scripts/benchmark-codeneedle-suite.sh diff --git a/.env.example b/.env.example index fdd85a8..5ad2270 100644 --- a/.env.example +++ b/.env.example @@ -3,7 +3,8 @@ # ========================================== # --- LLM ENGINE & MODEL SELECTION --- -MODEL_LIST=qwen2.5-coder:14b,qwen3:14b,deepseek-coder-v2:16b +MODEL_LIST=gemma4:e4b,qwen2.5-coder:14b,qwen3:14b,deepseek-coder-v2:16b +OPENCLAW_DEFAULT_MODEL=ollama/qwen3-general OLLAMA_KEEP_ALIVE=-1 # KV cache quantization for large contexts (recommended on desktop GPUs): q8_0 or q4_0 # OLLAMA_KV_CACHE_TYPE=q8_0 diff --git a/AGENT_INSTRUCTIONS.md b/AGENT_INSTRUCTIONS.md index 7eaa17f..ff9e83c 100644 --- a/AGENT_INSTRUCTIONS.md +++ b/AGENT_INSTRUCTIONS.md @@ -48,4 +48,37 @@ Wir entwickeln eine lokale, containerisierte AI-Workspace-Infrastruktur namens * 1. **OpenClaw Integration:** Die `openclaw.json` muss so konfiguriert werden, dass der Agent den `ghostnet-router` (SSE-Endpunkt) als primäre Tool-Quelle nutzt. 2. **Modell-Management:** DNS-Problematik in der Ollama-Container-Registry beheben (Registry-Auflösung schlägt aktuell fehl). 3. **Skill Expansion:** Entwicklung weiterer MCP-Bridges (z.B. für lokales Filesystem-Management oder Datenbank-Abfragen), die einfach am Router "angepluggt" werden können. -4. **UX-Automation:** Ein `ghostnet.sh` Wrapper, der die Vendor-Erkennung automatisiert und den Stack konsistent hochfährt. \ No newline at end of file +4. **UX-Automation:** Ein `ghostnet.sh` Wrapper, der die Vendor-Erkennung automatisiert und den Stack konsistent hochfährt. + + +# Stand: 17.06.2026 + +## 📋 Handover & Context Prompt for the Next AI Agent + +**Context:** +Das Repo hat inzwischen mehrere betriebliche Verbesserungen erhalten, die nicht nur die Laufzeit des Stacks, sondern auch Modellprofilierung und Benchmarking betreffen. OpenClaw laeuft weiterhin containerisiert gegen den lokalen Ollama-Stack und den MCP-Router, aber Konfiguration und Benchmark-Ausgaben sind jetzt deutlich sauberer organisiert. + +**Current Architecture & Operational Notes:** +1. **Default-Modellsteuerung:** `agents.defaults.model` in `openclaw/config/openclaw.json` nutzt `${OPENCLAW_DEFAULT_MODEL}`. OpenClaw expandiert `${VAR}` direkt in Config-Strings zur Laufzeit. Compose setzt deshalb einen Fallback, damit fehlende Variablen den Config-Load nicht brechen. +2. **Default-Modellprofil:** Das generelle OpenClaw-Defaultprofil ist jetzt `ollama/qwen3-general`. Das bestehende `qwen3-coding`-Profil bleibt fuer codinglastige Workflows erhalten. +3. **Custom-Modelfiles:** Ollama baut alle Dateien unter `ollama/modelfiles/*.modelfile` automatisch beim Containerstart; der Modellname entspricht jeweils dem Dateinamen ohne Endung. +4. **Benchmark-Artefakte:** Laufzeitdaten liegen unter `.cache/benchmarks/existing-suite/runs/$RUN_ID`, waehrend kuratierte Ergebnisse unter `docs/benchmarks/existing-suite/$RUN_ID` landen. +5. **Benchmark-Reports:** Per-Model werden `report.html` und `RESULTS.md` erzeugt, fuer Suites zusaetzlich `SUITE_COMPARISON.html`. Die Erzeugung laeuft ueber `scripts/lib/report-generator.sh`. + +**Benchmarking State:** +* `scripts/benchmark-stack.sh` und `scripts/benchmark-model-suite.sh` verwenden jetzt 600s Timeout fuer robustere lokale Modelllaeufe. +* `scripts/benchmark-codeneedle-suite.sh` bleibt bei 600s und dient als separater Recall-Benchmark gegen denselben Ollama-Endpunkt. +* Fuer stabile Vergleichslaeufe ist die Ollama-Konfiguration auf `OLLAMA_KEEP_ALIVE=-1`, `OLLAMA_KV_CACHE_TYPE=q8_0`, `OLLAMA_MAX_LOADED_MODELS=1`, `OLLAMA_NUM_PARALLEL=1` und `SHM_SIZE=16gb` ausgerichtet. +* Die Benchmark-Skripte trennen jetzt konsequent zwischen Temp-Artefakten und finaler Ergebnisdoku, damit `docs/` nicht mehr mit Rohlogs zugemuellt wird. + +**Current Repository State:** +* **Model Routing:** OpenClaw-Defaultmodell ist per `.env` und Compose ueberschreibbar, ohne JSON manuell anzupassen. +* **Profiles:** `qwen3-general` ist das allgemeine Defaultprofil fuer Chats; `qwen3-coding`, `qwen25-coding` und `deepseek-coding` bleiben spezialisierte Profile. +* **Benchmark Presentation:** Benchmark-Ergebnisse stehen jetzt als HTML-Dashboard, Markdown-Lab-Notiz und Suite-Vergleich bereit. +* **Operational Validation:** Compose-Render und OpenClaw-Config-Expansion fuer `OPENCLAW_DEFAULT_MODEL` wurden direkt gegen das Upstream-Image verifiziert. + +**Next Tasks / Where to Continue:** +1. **DNS/Ollama Registry:** Die externe Modellauflosung fuer Registry-Pulls im Containernetz weiter haerten. +2. **Additional Profiles:** Weitere nicht-codinglastige oder domänenspezifische Modelfiles auf Basis bestehender Profile ergänzen. +3. **Benchmark Evolution:** Adaptive Timeouts, Warm-up-Runs und historische Vergleichsreports sind sinnvolle naechste Ausbaustufen. +4. **OpenClaw Tooling:** Weitere MCP-Bridges und agentenspezifische Tool-Profile sauber in die bestehende Router-Struktur integrieren. \ No newline at end of file diff --git a/README.md b/README.md index d007e31..b96617e 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Clone the repository and prepare your environment configuration: `cp .env.example .env` Open `.env` and set your `GPU_TYPE` (amd, nvidia, or intel) and your desired models. +Set `OPENCLAW_DEFAULT_MODEL` to the model OpenClaw should use by default. ### 2. Boot the Stack Use the integrated management script, which automatically selects the correct vendor-specific compose file: @@ -45,6 +46,7 @@ Use `ollama list` inside the container or on the host to see the exact model nam Recommended custom profiles for this stack: ```text +qwen3-general -> Qwen3 14B, universal profile for OpenClaw chats qwen3-coding -> Qwen3 14B, balanced default for agentic coding qwen25-coding -> Qwen2.5-Coder 14B, code-first edit and repair profile deepseek-coding -> DeepSeek-Coder-V2 16B, long-context repo analysis profile @@ -78,15 +80,71 @@ If you change the host binding, use `TOOL_ROUTER_BIND` and `TOOL_ROUTER_HOST_POR ## Benchmarking Custom Models -The repository includes a reproducible benchmark script for comparing local models through the OpenClaw agent path. +The repository now ships two complementary benchmark entry points: + +* `./scripts/benchmark-stack.sh` and `./scripts/benchmark-model-suite.sh` exercise the full OpenClaw agent path. +* `./scripts/benchmark-codeneedle-suite.sh` runs the upstream CodeNeedle recall benchmark against the same local Ollama instance. + +The benchmark flow is split intentionally into runtime artifacts and curated results: + +* Runtime artifacts go to `.cache/benchmarks/existing-suite/runs/$RUN_ID/`. +* Final benchmark outputs go to `docs/benchmarks/existing-suite/$RUN_ID/`. + +Generated result sets now include more than raw `summary.txt` files: + +* `report.html` per model for a quick interactive dashboard. +* `RESULTS.md` per model for git-friendly lab notes. +* `SUITE_COMPARISON.html` for side-by-side multi-model comparison. + +These reports are generated automatically at the end of the benchmark scripts through `scripts/lib/report-generator.sh`. + +Current timeout defaults were raised for slower local models and more reliable result capture: + +* `benchmark-stack.sh`: 600s +* `benchmark-model-suite.sh`: 600s +* `benchmark-codeneedle-suite.sh`: 600s + +The current Ollama tuning used for benchmark stability is: + +* `OLLAMA_KEEP_ALIVE=-1` +* `OLLAMA_KV_CACHE_TYPE=q8_0` +* `OLLAMA_MAX_LOADED_MODELS=1` +* `OLLAMA_NUM_PARALLEL=1` +* `SHM_SIZE=16gb` + +OpenClaw path examples: ```bash -./scripts/benchmark-stack.sh --model ollama/qwen3-coding:latest --lines 140 --timeout-sec 300 --no-start -./scripts/benchmark-stack.sh --model ollama/qwen25-coding:latest --lines 140 --timeout-sec 300 --no-start -./scripts/benchmark-stack.sh --model ollama/deepseek-coding:latest --lines 140 --timeout-sec 300 --no-start +./scripts/benchmark-stack.sh --model ollama/qwen3-coding:latest --lines 140 --timeout-sec 600 --no-start +./scripts/benchmark-model-suite.sh --lines 140 --timeout-sec 600 ``` -This benchmark is useful for latency, stability, and tool-path comparison under identical conditions. +CodeNeedle path example: + +```bash +./scripts/benchmark-codeneedle-suite.sh \ + --model qwen3-coding:latest \ + --model qwen25-coding:latest \ + --model deepseek-coding:latest +``` + +`benchmark-codeneedle-suite.sh` clones the upstream CodeNeedle repository into `.cache/benchmarks/codeneedle/upstream`, builds its Dockerfile with the configured container engine, and benchmarks the standard `http_server` and `jquery` corpora against `http://127.0.0.1:11434`. + +Use `--runtime-root`, `--results-root`, or the legacy `--artifact-root` overrides if you want to keep artifacts somewhere else for a run. + +Typical curated benchmark outputs look like this: + +```text +docs/benchmarks/existing-suite/model-suite-TIMESTAMP/ +├── qwen3-coding/ +│ ├── summary.txt +│ ├── assistant-output.txt +│ ├── report.html +│ └── RESULTS.md +├── qwen25-coding/ +│ └── ... +└── SUITE_COMPARISON.html +``` ## Technical Architecture & Key Highlights This setup relies on unique architectural design patterns engineered to overcome container engines boundaries and system strictness: diff --git a/compose.amd.yaml b/compose.amd.yaml index 3432b71..c721232 100644 --- a/compose.amd.yaml +++ b/compose.amd.yaml @@ -64,6 +64,7 @@ services: environment: - OPENCLAW_GATEWAY_MODE=local - OPENCLAW_GATEWAY_PASSWORD=${OPENCLAW_PASSWORD} + - OPENCLAW_DEFAULT_MODEL=${OPENCLAW_DEFAULT_MODEL:-ollama/qwen3-general} - OPENCLAW_CONFIG_PATH=/home/node/.openclaw/openclaw.json - OPENCLAW_STATE_DIR=/home/node/.openclaw - OLLAMA_URL=http://127.0.0.1:11434 diff --git a/compose.intel.yaml b/compose.intel.yaml index 8353a2b..a33b132 100644 --- a/compose.intel.yaml +++ b/compose.intel.yaml @@ -39,6 +39,7 @@ services: environment: - OPENCLAW_GATEWAY_MODE=local - OPENCLAW_GATEWAY_PASSWORD=${OPENCLAW_PASSWORD} + - OPENCLAW_DEFAULT_MODEL=${OPENCLAW_DEFAULT_MODEL:-ollama/qwen3-general} - OPENCLAW_CONFIG_PATH=/home/node/.openclaw/openclaw.json - OPENCLAW_STATE_DIR=/home/node/.openclaw - OLLAMA_URL=http://127.0.0.1:11434 diff --git a/compose.nvidia.yaml b/compose.nvidia.yaml index 1a705d9..1ca054b 100644 --- a/compose.nvidia.yaml +++ b/compose.nvidia.yaml @@ -37,6 +37,7 @@ services: environment: - OPENCLAW_GATEWAY_MODE=local - OPENCLAW_GATEWAY_PASSWORD=${OPENCLAW_PASSWORD} + - OPENCLAW_DEFAULT_MODEL=${OPENCLAW_DEFAULT_MODEL:-ollama/qwen3-general} - OPENCLAW_CONFIG_PATH=/home/node/.openclaw/openclaw.json - OPENCLAW_STATE_DIR=/home/node/.openclaw - OLLAMA_URL=http://127.0.0.1:11434 diff --git a/ollama/modelfiles/qwen3-general.modelfile b/ollama/modelfiles/qwen3-general.modelfile new file mode 100644 index 0000000..2251cb0 --- /dev/null +++ b/ollama/modelfiles/qwen3-general.modelfile @@ -0,0 +1,22 @@ +# Basis-Modell aus der Registry laden +FROM qwen3:14b + +# Allgemeines Profil fuer breit nutzbare OpenClaw-Chats ohne Coding-Fokus +PARAMETER num_ctx 12288 +PARAMETER num_predict 4096 +PARAMETER temperature 0.3 + +# Dem Modell seine primaere Identitaet verpassen +SYSTEM """ +Du bist ein vielseitiger, zuverlaessiger KI-Assistent innerhalb des OpenClaw-Frameworks. +Deine Aufgaben: +- Beantworte allgemeine Fragen klar, praezise und hilfreich. +- Hilf bei Analyse, Recherche, Planung, Zusammenfassungen und alltaeglichen Aufgaben. +- Wenn der Nutzer codingbezogene Hilfe will, unterstuetze dabei sauber und sachlich, aber ohne Coding-Fokus als Grundhaltung. + +Arbeitsregeln: +- Simuliere keine Terminal-Ausgaben, keine "Let me check..."-Monologe und keine erfundenen Befehlslogs. +- Nutze verfuegbare Tools/Befehle tatsaechlich und berichte nur echte Ergebnisse. +- Wenn etwas nicht fertig ist, nenne den konkreten naechsten Schritt statt langer Zwischenkommentare. +- Antworte standardmaessig auf Deutsch, ausser der Nutzer fordert explizit etwas anderes. +""" \ No newline at end of file diff --git a/openclaw/config/openclaw.json b/openclaw/config/openclaw.json index 4f9bf2c..18fb30c 100644 --- a/openclaw/config/openclaw.json +++ b/openclaw/config/openclaw.json @@ -9,7 +9,7 @@ }, "agents": { "defaults": { - "model": "ollama/qwen3-coding", + "model": "${OPENCLAW_DEFAULT_MODEL}", "workspace": "/home/node/.openclaw/workspace", "sandbox": { "mode": "all", diff --git a/scripts/benchmark-codeneedle-suite.sh b/scripts/benchmark-codeneedle-suite.sh new file mode 100755 index 0000000..d916144 --- /dev/null +++ b/scripts/benchmark-codeneedle-suite.sh @@ -0,0 +1,393 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +CODE_NEEDLE_REPO_URL="https://github.com/alexziskind1/codeneedle.git" + +usage() { + cat <<'EOF' +Usage: ./scripts/benchmark-codeneedle-suite.sh [options] + +Vergleicht mehrere lokale Ollama-Modelle mit CodeNeedle gegen ein OpenAI- +kompatibles Endpoint-Schema. Das Skript klont CodeNeedle zur Laufzeit unter +.cache/benchmarks, baut das Upstream-Dockerfile mit Podman oder Docker und +fuehrt die Standard-Corpora http_server und jquery gegen den lokalen Ollama- +Server aus. + +Optionen: + --model Modellname in Ollama; mehrfach nutzbar + (default: qwen3-coding:latest, qwen25-coding:latest, + deepseek-coding:latest) + --corpus CodeNeedle-Corpus; mehrfach nutzbar + (default: http_server, jquery) + --base-url OpenAI-kompatible Basis-URL ohne /v1-Suffix + (default: http://127.0.0.1:11434) + --api-key API-Key fuer den Endpoint (default: not-needed) + --temperature Sampling-Temperatur (default: 0) + --max-tokens Antwortbudget je Query (default: 8000) + --timeout-sec Timeout je Query in Sekunden (default: 600) + --sample-k Anzahl der getesteten Funktionen je Corpus (default: 16) + --sample-seed Seed fuer die Stichprobe (default: 42) + --skip-preflight Ueberspringt CodeNeedle-Preflight-Probe pro Lauf + --runtime-root Zielordner fuer temporaere Laufdaten (default: auto) + --results-root Zielordner fuer kuratierte Resultate (default: auto) + --artifact-root Legacy-Alias; nutzt denselben Pfad fuer Runtime und Resultate + --repo-dir Ablageort fuer den Upstream-Clone (default: auto) + --image-tag Name des lokal gebauten Runner-Images (default: ghostnet-codeneedle:latest) + --container-engine podman oder docker (default: aus .env/.env.example oder auto) + --refresh-upstream Upstream-Clone vor dem Lauf mit git pull aktualisieren + --rebuild-image Image vor dem Lauf neu bauen + --keep-runtime Runtime-Dateien nach dem Export nicht loeschen + -h, --help Hilfe anzeigen +EOF +} + +need_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "ERROR: Missing required command '$1'." >&2 + exit 1 + fi +} + +read_env_var() { + local key="$1" + local file line + + for file in "$PROJECT_DIR/.env" "$PROJECT_DIR/.env.example"; do + if [ ! -f "$file" ]; then + continue + fi + + line="$(grep -E "^${key}=" "$file" | tail -n 1 || true)" + if [ -n "$line" ]; then + printf '%s\n' "${line#*=}" + return 0 + fi + done + + return 1 +} + +slugify() { + printf '%s' "$1" | tr '/:@ ' '----' | tr -cd '[:alnum:]_.-' +} + +MODELS=( + "qwen3-coding:latest" + "qwen25-coding:latest" + "deepseek-coding:latest" +) +CORPORA=( + "http_server" + "jquery" +) +BASE_URL="http://127.0.0.1:11434" +API_KEY="not-needed" +TEMPERATURE="0" +MAX_TOKENS=8000 +TIMEOUT_SEC=600 +SAMPLE_K=16 +SAMPLE_SEED=42 +SKIP_PREFLIGHT=0 +RUNTIME_ROOT="" +RESULTS_ROOT="" +LEGACY_ARTIFACT_ROOT="" +REPO_DIR="" +IMAGE_TAG="ghostnet-codeneedle:latest" +CONTAINER_ENGINE="${CONTAINER_ENGINE:-}" +REFRESH_UPSTREAM=0 +REBUILD_IMAGE=0 +KEEP_RUNTIME=0 + +while [ $# -gt 0 ]; do + case "$1" in + --model) + if [ "${MODELS[0]:-}" = "qwen3-coding:latest" ] && [ "${#MODELS[@]}" -eq 3 ]; then + MODELS=() + fi + MODELS+=("$2") + shift 2 + ;; + --corpus) + if [ "${CORPORA[0]:-}" = "http_server" ] && [ "${#CORPORA[@]}" -eq 2 ]; then + CORPORA=() + fi + CORPORA+=("$2") + shift 2 + ;; + --base-url) + BASE_URL="$2" + shift 2 + ;; + --api-key) + API_KEY="$2" + shift 2 + ;; + --temperature) + TEMPERATURE="$2" + shift 2 + ;; + --max-tokens) + MAX_TOKENS="$2" + shift 2 + ;; + --timeout-sec) + TIMEOUT_SEC="$2" + shift 2 + ;; + --sample-k) + SAMPLE_K="$2" + shift 2 + ;; + --sample-seed) + SAMPLE_SEED="$2" + shift 2 + ;; + --skip-preflight) + SKIP_PREFLIGHT=1 + shift + ;; + --runtime-root) + RUNTIME_ROOT="$2" + shift 2 + ;; + --results-root) + RESULTS_ROOT="$2" + shift 2 + ;; + --artifact-root) + LEGACY_ARTIFACT_ROOT="$2" + shift 2 + ;; + --repo-dir) + REPO_DIR="$2" + shift 2 + ;; + --image-tag) + IMAGE_TAG="$2" + shift 2 + ;; + --container-engine) + CONTAINER_ENGINE="$2" + shift 2 + ;; + --refresh-upstream) + REFRESH_UPSTREAM=1 + shift + ;; + --rebuild-image) + REBUILD_IMAGE=1 + shift + ;; + --keep-runtime) + KEEP_RUNTIME=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "ERROR: Unknown option '$1'." >&2 + usage >&2 + exit 1 + ;; + esac +done + +need_cmd git +need_cmd jq + +if [ -z "$CONTAINER_ENGINE" ]; then + CONTAINER_ENGINE="$(read_env_var CONTAINER_ENGINE || true)" +fi + +if [ -z "$CONTAINER_ENGINE" ]; then + if command -v podman >/dev/null 2>&1; then + CONTAINER_ENGINE="podman" + elif command -v docker >/dev/null 2>&1; then + CONTAINER_ENGINE="docker" + else + echo "ERROR: Neither podman nor docker is available." >&2 + exit 1 + fi +fi + +case "$CONTAINER_ENGINE" in + podman) + ENGINE_CMD=(podman) + MOUNT_SUFFIX=':Z' + ;; + docker) + ENGINE_CMD=(docker) + MOUNT_SUFFIX='' + ;; + *) + echo "ERROR: Invalid container engine '$CONTAINER_ENGINE'. Use 'podman' or 'docker'." >&2 + exit 1 + ;; +esac + +need_cmd "${ENGINE_CMD[0]}" + +RUN_ID="codeneedle-suite-$(date +%Y%m%d-%H%M%S)" + +if [ -n "$LEGACY_ARTIFACT_ROOT" ]; then + RUNTIME_ROOT="$LEGACY_ARTIFACT_ROOT" + if [ -z "$RESULTS_ROOT" ]; then + RESULTS_ROOT="$LEGACY_ARTIFACT_ROOT" + KEEP_RUNTIME=1 + fi +fi + +if [ -z "$RUNTIME_ROOT" ]; then + RUNTIME_ROOT="$PROJECT_DIR/.cache/benchmarks/$RUN_ID" +fi + +if [ -z "$RESULTS_ROOT" ]; then + RESULTS_ROOT="$PROJECT_DIR/docs/benchmarks/codeneedle/$RUN_ID" +fi + +if [ -z "$REPO_DIR" ]; then + REPO_DIR="$PROJECT_DIR/.cache/benchmarks/codeneedle/upstream" +fi + +mkdir -p "$RUNTIME_ROOT" "$RESULTS_ROOT" + +if [ ! -d "$REPO_DIR/.git" ]; then + mkdir -p "$(dirname "$REPO_DIR")" + git clone --depth 1 "$CODE_NEEDLE_REPO_URL" "$REPO_DIR" +elif [ "$REFRESH_UPSTREAM" -eq 1 ]; then + git -C "$REPO_DIR" pull --ff-only +fi + +UPSTREAM_REF="$(git -C "$REPO_DIR" rev-parse HEAD)" + +if [ "$REBUILD_IMAGE" -eq 1 ] || ! "${ENGINE_CMD[@]}" image inspect "$IMAGE_TAG" >/dev/null 2>&1; then + "${ENGINE_CMD[@]}" build -t "$IMAGE_TAG" -f "$REPO_DIR/Dockerfile" "$REPO_DIR" +fi + +SUMMARY_TABLE="$RESULTS_ROOT/summary.tsv" +RUN_METADATA_FILE="$RESULTS_ROOT/run-metadata.txt" +printf 'corpus\tmodel\trun_rc\tpassed\ttotal\terrored\tprimary_matched\tprimary_total\thallucinated\tbonus\n' > "$SUMMARY_TABLE" + +cat > "$RUN_METADATA_FILE" <"$stdout_file" 2>"$stderr_file"; then + run_rc=0 + else + run_rc=$? + fi + + cp "$stdout_file" "$results_dir/stdout.log" + cp "$stderr_file" "$results_dir/stderr.log" + + passed="" + total="" + errored="" + primary_matched="" + primary_total="" + hallucinated="" + bonus="" + + if [ -f "$dump_file" ]; then + cp "$dump_file" "$results_dir/results.json" + passed="$(jq '[.results[] | select(.passed == true and (.error == null or .error == ""))] | length' "$dump_file")" + total="$(jq '.results | length' "$dump_file")" + errored="$(jq '[.results[] | select(.error != null and .error != "")] | length' "$dump_file")" + primary_matched="$(jq '[.results[].primary_matched] | add // 0' "$dump_file")" + primary_total="$(jq '[.results[].primary_total] | add // 0' "$dump_file")" + hallucinated="$(jq '[.results[].hallucinated] | add // 0' "$dump_file")" + bonus="$(jq '[.results[].bonus_matched] | add // 0' "$dump_file")" + fi + + cat > "$results_dir/summary.txt" <> "$SUMMARY_TABLE" + + if [ "$KEEP_RUNTIME" -eq 0 ] && [ "$runtime_dir" != "$results_dir" ]; then + rm -rf "$runtime_dir" + fi + done +done + +echo +echo "CodeNeedle-Vergleich gespeichert unter: $RESULTS_ROOT" +if command -v column >/dev/null 2>&1; then + column -ts $'\t' "$SUMMARY_TABLE" +else + cat "$SUMMARY_TABLE" +fi \ No newline at end of file diff --git a/scripts/benchmark-model-suite.sh b/scripts/benchmark-model-suite.sh index ac415ce..d9311c6 100755 --- a/scripts/benchmark-model-suite.sh +++ b/scripts/benchmark-model-suite.sh @@ -5,6 +5,9 @@ set -euo pipefail PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$PROJECT_DIR" +# Source report generator +source "$PROJECT_DIR/scripts/lib/report-generator.sh" 2>/dev/null || true + usage() { cat <<'EOF' Usage: ./scripts/benchmark-model-suite.sh [options] @@ -14,8 +17,11 @@ und schreibt Artefakte plus eine Vergleichszusammenfassung. Optionen: --lines Erwartete Zeilenanzahl LINE_1..LINE_n (default: 140) - --timeout-sec Timeout je Lauf in Sekunden (default: 300) - --artifact-root

Zielordner fuer alle Benchmark-Artefakte (default: auto) + --timeout-sec Timeout je Lauf in Sekunden (default: 600) + --runtime-root

Zielordner fuer temporaere Laufdaten (default: auto) + --results-root

Zielordner fuer kuratierte Resultate (default: auto) + --artifact-root

Legacy-Alias; nutzt denselben Pfad fuer Runtime und Resultate + --keep-runtime Runtime-Dateien der Einzelruns nicht loeschen --session-prefix Prefix fuer Session-Keys (default: suite:) -h, --help Hilfe anzeigen EOF @@ -46,8 +52,11 @@ extract_metric() { } EXPECTED_LINES=140 -TIMEOUT_SEC=300 -ARTIFACT_ROOT="" +TIMEOUT_SEC=600 +RUNTIME_ROOT="" +RESULTS_ROOT="" +LEGACY_ARTIFACT_ROOT="" +KEEP_RUNTIME=0 SESSION_PREFIX="suite:$(date +%Y%m%d-%H%M%S)" while [ "$#" -gt 0 ]; do @@ -60,10 +69,22 @@ while [ "$#" -gt 0 ]; do TIMEOUT_SEC="$2" shift 2 ;; - --artifact-root) - ARTIFACT_ROOT="$2" + --runtime-root) + RUNTIME_ROOT="$2" shift 2 ;; + --results-root) + RESULTS_ROOT="$2" + shift 2 + ;; + --artifact-root) + LEGACY_ARTIFACT_ROOT="$2" + shift 2 + ;; + --keep-runtime) + KEEP_RUNTIME=1 + shift + ;; --session-prefix) SESSION_PREFIX="$2" shift 2 @@ -80,11 +101,25 @@ while [ "$#" -gt 0 ]; do esac done -if [ -z "$ARTIFACT_ROOT" ]; then - ARTIFACT_ROOT="$PROJECT_DIR/storage/workspace/benchmarks/model-suite-$(date +%Y%m%d-%H%M%S)" +RUN_ID="model-suite-$(date +%Y%m%d-%H%M%S)" + +if [ -n "$LEGACY_ARTIFACT_ROOT" ]; then + RUNTIME_ROOT="$LEGACY_ARTIFACT_ROOT" + if [ -z "$RESULTS_ROOT" ]; then + RESULTS_ROOT="$LEGACY_ARTIFACT_ROOT" + KEEP_RUNTIME=1 + fi fi -mkdir -p "$ARTIFACT_ROOT" +if [ -z "$RUNTIME_ROOT" ]; then + RUNTIME_ROOT="$PROJECT_DIR/.cache/benchmarks/$RUN_ID" +fi + +if [ -z "$RESULTS_ROOT" ]; then + RESULTS_ROOT="$PROJECT_DIR/docs/benchmarks/existing-suite/$RUN_ID" +fi + +mkdir -p "$RUNTIME_ROOT" "$RESULTS_ROOT" MODELS=( "ollama/qwen3-coding:latest" @@ -92,27 +127,36 @@ MODELS=( "ollama/deepseek-coding:latest" ) -SUMMARY_TABLE="$ARTIFACT_ROOT/summary.tsv" +SUMMARY_TABLE="$RESULTS_ROOT/summary.tsv" printf 'model\tbenchmark_rc\tvalidation\trequest_exit\twall_ms\tprovider_tps\tend_to_end_tps\tload_sec\tkv_tokens\n' > "$SUMMARY_TABLE" for model in "${MODELS[@]}"; do slug="$(slugify_model "$model")" - run_dir="$ARTIFACT_ROOT/$slug" + runtime_dir="$RUNTIME_ROOT/$slug" + results_dir="$RESULTS_ROOT/$slug" session_key="$SESSION_PREFIX:$slug" + run_args=( + --model "$model" + --lines "$EXPECTED_LINES" + --timeout-sec "$TIMEOUT_SEC" + --session-key "$session_key" + --runtime-dir "$runtime_dir" + --results-dir "$results_dir" + --no-start + ) + + if [ "$KEEP_RUNTIME" -eq 1 ]; then + run_args+=(--keep-runtime) + fi echo "=== Benchmark fuer $model ===" - benchmark_rc=0 - if ! ./scripts/benchmark-stack.sh \ - --model "$model" \ - --lines "$EXPECTED_LINES" \ - --timeout-sec "$TIMEOUT_SEC" \ - --session-key "$session_key" \ - --artifact-dir "$run_dir" \ - --no-start; then + if ./scripts/benchmark-stack.sh "${run_args[@]}"; then + benchmark_rc=0 + else benchmark_rc=$? fi - summary_file="$run_dir/summary.txt" + summary_file="$results_dir/summary.txt" validation="$(extract_value 'Validation' "$summary_file")" request_exit="$(extract_value 'Request exit code' "$summary_file")" @@ -128,5 +172,14 @@ for model in "${MODELS[@]}"; do done echo -echo "Vergleich gespeichert unter: $ARTIFACT_ROOT" -column -ts $'\t' "$SUMMARY_TABLE" \ No newline at end of file +echo "Runtime-Wurzel: $RUNTIME_ROOT" +echo "Kuratierte Resultate: $RESULTS_ROOT" +column -ts $'\t' "$SUMMARY_TABLE" + +# Generate suite comparison report +if [ -f "$SUMMARY_TABLE" ]; then + echo + echo "📊 Generating suite comparison report..." + generate_suite_comparison_report "$SUMMARY_TABLE" "$RESULTS_ROOT" 2>/dev/null || true + echo "✅ Reports generated in: $RESULTS_ROOT" +fi \ No newline at end of file diff --git a/scripts/benchmark-stack.sh b/scripts/benchmark-stack.sh index a0494db..4e928ff 100755 --- a/scripts/benchmark-stack.sh +++ b/scripts/benchmark-stack.sh @@ -5,6 +5,9 @@ set -euo pipefail PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$PROJECT_DIR" +# Source report generator +source "$PROJECT_DIR/scripts/lib/report-generator.sh" 2>/dev/null || true + RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' @@ -27,6 +30,37 @@ need_cmd() { command -v "$1" >/dev/null 2>&1 || fail "Benoetigtes Kommando fehlt: $1" } +read_env_var() { + local key="$1" + local env_file="$PROJECT_DIR/.env" + local line="" + + if [ -f "$env_file" ]; then + line="$(grep -E "^${key}=" "$env_file" | tail -n 1 || true)" + fi + + if [ -z "$line" ] && [ -f "$PROJECT_DIR/.env.example" ]; then + line="$(grep -E "^${key}=" "$PROJECT_DIR/.env.example" | tail -n 1 || true)" + fi + + if [ -n "$line" ]; then + printf '%s' "${line#*=}" + fi +} + +resolve_abs_path() { + local raw_path="$1" + if [[ "$raw_path" = /* ]]; then + printf '%s' "$raw_path" + return + fi + if command -v realpath >/dev/null 2>&1; then + realpath -m "$PROJECT_DIR/$raw_path" + return + fi + (cd "$PROJECT_DIR" && printf '%s/%s\n' "$PWD" "${raw_path#./}") +} + usage() { cat <<'EOF' Usage: ./scripts/benchmark-stack.sh [options] @@ -37,9 +71,12 @@ validiert den Output und zeigt Metriken (Laufzeit, Token, KV-/VRAM-Metriken). Optionen: --model OpenClaw-Modell (default: ollama/qwen3-coding:latest) --lines Erwartete Zeilenanzahl LINE_1..LINE_n (default: 120) - --timeout-sec Timeout fuer den Agent-Call in Sekunden (default: 480) + --timeout-sec Timeout fuer den Agent-Call in Sekunden (default: 600) --session-key Session-Key fuer den Lauf (default: auto) - --artifact-dir Zielverzeichnis fuer Ergebnisdateien (default: auto) + --runtime-dir Zielverzeichnis fuer temporaere Laufdaten (default: auto) + --results-dir Zielverzeichnis fuer kuratierte Resultate (default: auto) + --artifact-dir Legacy-Alias; nutzt denselben Pfad fuer Runtime und Resultate + --keep-runtime Runtime-Dateien nach dem Export nicht loeschen --no-start Stack nicht starten, nur vorhandenen Stack nutzen --no-reload Modell vor Lauf nicht mit 'ollama stop' entladen -h, --help Hilfe anzeigen @@ -132,13 +169,81 @@ extract_unquoted_metric() { fi } +resolve_workspace_media_path() { + local media_path="$1" + local normalized="$media_path" + + normalized="${normalized#/workspace/}" + normalized="${normalized#/home/node/.openclaw/workspace/}" + + if [ -z "$normalized" ] || [ "$normalized" = "$media_path" -a [[ "$media_path" = /* ]] ]; then + return 1 + fi + + printf '%s/%s\n' "$HOST_WORKSPACE_ABS" "$normalized" +} + +extract_tool_call_text() { + local raw_text="$1" + local extracted + + extracted="$(printf '%s\n' "$raw_text" | sed '/^```json$/d; /^```$/d; /^```$/d' | jq -rs ' + map(select(type == "object" and (.name == "write" or .name == "edit"))) as $calls + | if ($calls | length) == 0 then "" + else + reduce $calls[] as $call ({content: ""}; + if $call.name == "write" then + .content = ($call.arguments.content // .content) + elif $call.name == "edit" then + .content = ( + if (($call.arguments.edits // []) | length) > 0 then + ($call.arguments.edits[-1].newText // .content) + else + .content + end + ) + else + . + end + ) + | .content + end + ' 2>/dev/null || true)" + + if [ -n "$extracted" ]; then + printf '%s' "$extracted" + return 0 + fi + + return 1 +} + +extract_media_payload_text() { + local media_path="" + local host_path="" + + while IFS= read -r media_path; do + [ -n "$media_path" ] || continue + host_path="$(resolve_workspace_media_path "$media_path" || true)" + if [ -n "$host_path" ] && [ -f "$host_path" ]; then + cat "$host_path" + return 0 + fi + done < <(jq -r '.result.payloads[]? | .mediaUrl?, (.mediaUrls[]?)' "$JSON_OUT_FILE" 2>/dev/null | awk 'NF && !seen[$0]++') + + return 1 +} + MODEL="ollama/qwen3-coding:latest" EXPECTED_LINES=120 -TIMEOUT_SEC=480 +TIMEOUT_SEC=600 SESSION_KEY="" -ARTIFACT_DIR="" +RUNTIME_DIR="" +RESULTS_DIR="" +LEGACY_ARTIFACT_DIR="" START_STACK=1 FORCE_RELOAD=1 +KEEP_RUNTIME=0 while [ "$#" -gt 0 ]; do case "$1" in @@ -158,10 +263,22 @@ while [ "$#" -gt 0 ]; do SESSION_KEY="$2" shift 2 ;; - --artifact-dir) - ARTIFACT_DIR="$2" + --runtime-dir) + RUNTIME_DIR="$2" shift 2 ;; + --results-dir) + RESULTS_DIR="$2" + shift 2 + ;; + --artifact-dir) + LEGACY_ARTIFACT_DIR="$2" + shift 2 + ;; + --keep-runtime) + KEEP_RUNTIME=1 + shift + ;; --no-start) START_STACK=0 shift @@ -195,19 +312,44 @@ need_cmd sed need_cmd awk need_cmd timeout -if [ -z "$ARTIFACT_DIR" ]; then - TS="$(date +%Y%m%d-%H%M%S)" - ARTIFACT_DIR="$PROJECT_DIR/storage/workspace/benchmarks/$TS" -fi -mkdir -p "$ARTIFACT_DIR" +RUN_ID="$(date +%Y%m%d-%H%M%S)" +HOST_WORKSPACE_RAW="$(read_env_var HOST_WORKSPACE_PATH)" +HOST_WORKSPACE_RAW="${HOST_WORKSPACE_RAW:-./storage/workspace}" +HOST_WORKSPACE_ABS="$(resolve_abs_path "$HOST_WORKSPACE_RAW")" +BENCH_OUTPUT_FILENAME="benchmark-output-${RUN_ID}.txt" -JSON_OUT_FILE="$ARTIFACT_DIR/agent-result.json" -ASSISTANT_TEXT_FILE="$ARTIFACT_DIR/assistant-output.txt" -OLLAMA_LOG_FILE="$ARTIFACT_DIR/ollama-since.log" -STATS_BEFORE_FILE="$ARTIFACT_DIR/container-stats-before.txt" -STATS_AFTER_FILE="$ARTIFACT_DIR/container-stats-after.txt" -REQUEST_ERR_FILE="$ARTIFACT_DIR/request-stderr.log" -SUMMARY_FILE="$ARTIFACT_DIR/summary.txt" +if [ -n "$LEGACY_ARTIFACT_DIR" ]; then + RUNTIME_DIR="$LEGACY_ARTIFACT_DIR" + if [ -z "$RESULTS_DIR" ]; then + RESULTS_DIR="$LEGACY_ARTIFACT_DIR" + KEEP_RUNTIME=1 + fi +fi + +if [ -z "$RUNTIME_DIR" ]; then + RUNTIME_DIR="$PROJECT_DIR/.cache/benchmarks/existing-suite/runs/$RUN_ID" +fi + +if [ -z "$RESULTS_DIR" ]; then + RESULTS_DIR="$PROJECT_DIR/docs/benchmarks/existing-suite/$RUN_ID" +fi + +mkdir -p "$RUNTIME_DIR" "$RESULTS_DIR" + +JSON_OUT_FILE="$RUNTIME_DIR/agent-result.json" +ASSISTANT_TEXT_FILE="$RUNTIME_DIR/assistant-output.txt" +OLLAMA_LOG_FILE="$RUNTIME_DIR/ollama-since.log" +STATS_BEFORE_FILE="$RUNTIME_DIR/container-stats-before.txt" +STATS_AFTER_FILE="$RUNTIME_DIR/container-stats-after.txt" +REQUEST_ERR_FILE="$RUNTIME_DIR/request-stderr.log" +SUMMARY_FILE="$RUNTIME_DIR/summary.txt" +EXPORTED_SUMMARY_FILE="$RESULTS_DIR/summary.txt" +EXPORTED_ASSISTANT_FILE="$RESULTS_DIR/assistant-output.txt" +EXPORTED_JSON_FILE="$RESULTS_DIR/agent-result.json" +EXPORTED_OLLAMA_LOG_FILE="$RESULTS_DIR/ollama-since.log" +EXPORTED_STATS_BEFORE_FILE="$RESULTS_DIR/container-stats-before.txt" +EXPORTED_STATS_AFTER_FILE="$RESULTS_DIR/container-stats-after.txt" +EXPORTED_REQUEST_ERR_FILE="$RESULTS_DIR/request-stderr.log" if [ -z "$SESSION_KEY" ]; then SESSION_KEY="agent:main:benchmark:$(date +%s)" @@ -221,7 +363,8 @@ echo "Model: $MODEL" echo "Session key: $SESSION_KEY" echo "Erwartete Zeilen: $EXPECTED_LINES" echo "Timeout: ${TIMEOUT_SEC}s" -echo "Artefakte: $ARTIFACT_DIR" +echo "Runtime-Dateien: $RUNTIME_DIR" +echo "Kuratierte Resultate: $RESULTS_DIR" if [ "$START_STACK" -eq 1 ]; then echo @@ -247,7 +390,7 @@ else fi echo "[4/6] Fuehre Benchmark-Anfrage aus ..." -PROMPT="Gib exakt ${EXPECTED_LINES} Zeilen aus. Jede Zeile muss exakt den Text LINE_ haben, mit aufsteigender Nummerierung von 1 bis ${EXPECTED_LINES}. Kein weiterer Text." +PROMPT="Gib exakt ${EXPECTED_LINES} Zeilen aus. Jede Zeile muss exakt den Text LINE_ haben, mit aufsteigender Nummerierung von 1 bis ${EXPECTED_LINES}. Kein weiterer Text. Nutze keine Tools, keine Dateien, keine Anhaenge und kein JSON. Antworte direkt als reiner Nachrichtentext. Falls du trotzdem Tools benutzt, schreibe exakt diese Zeilen in die Datei ${BENCH_OUTPUT_FILENAME} im Workspace und sonst nirgendwo hin." SINCE_UTC="$(date -u +%Y-%m-%dT%H:%M:%SZ)" podman stats --no-stream --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}' ollama openclaw > "$STATS_BEFORE_FILE" 2>&1 || true @@ -257,6 +400,7 @@ REQUEST_RC=0 if ! timeout "${TIMEOUT_SEC}s" podman exec openclaw openclaw agent \ --session-key "$SESSION_KEY" \ --model "$MODEL" \ + --thinking off \ --message "$PROMPT" \ --json > "$JSON_OUT_FILE" 2> "$REQUEST_ERR_FILE"; then REQUEST_RC=$? @@ -290,8 +434,19 @@ fi echo "[5/6] Validiere Antwortformat ..." ASSISTANT_TEXT="" +RAW_ASSISTANT_TEXT="" if [ "$HAS_VALID_JSON" -eq 1 ]; then - ASSISTANT_TEXT="$(jq -r '.result.payloads[0].text // ""' "$JSON_OUT_FILE")" + RAW_ASSISTANT_TEXT="$(jq -r '[.result.payloads[]?.text // empty] | join("\n")' "$JSON_OUT_FILE")" + ASSISTANT_TEXT="$RAW_ASSISTANT_TEXT" + if [ -n "$RAW_ASSISTANT_TEXT" ]; then + TOOL_CALL_TEXT="$(extract_tool_call_text "$RAW_ASSISTANT_TEXT" || true)" + if [ -n "$TOOL_CALL_TEXT" ]; then + ASSISTANT_TEXT="$TOOL_CALL_TEXT" + fi + fi + if [ -z "$ASSISTANT_TEXT" ]; then + ASSISTANT_TEXT="$(extract_media_payload_text || true)" + fi fi printf '%s' "$ASSISTANT_TEXT" > "$ASSISTANT_TEXT_FILE" @@ -426,12 +581,12 @@ fi echo "- journal_gpu_related_hits_since_start: $JOURNAL_HITS" echo echo "Artefakte" - echo "- agent_json: $JSON_OUT_FILE" - echo "- assistant_text: $ASSISTANT_TEXT_FILE" - echo "- ollama_log_since_start: $OLLAMA_LOG_FILE" - echo "- stats_before: $STATS_BEFORE_FILE" - echo "- stats_after: $STATS_AFTER_FILE" - echo "- request_stderr: $REQUEST_ERR_FILE" + echo "- agent_json: $EXPORTED_JSON_FILE" + echo "- assistant_text: $EXPORTED_ASSISTANT_FILE" + echo "- ollama_log_since_start: $EXPORTED_OLLAMA_LOG_FILE" + echo "- stats_before: $EXPORTED_STATS_BEFORE_FILE" + echo "- stats_after: $EXPORTED_STATS_AFTER_FILE" + echo "- request_stderr: $EXPORTED_REQUEST_ERR_FILE" } | tee "$SUMMARY_FILE" echo @@ -441,6 +596,30 @@ echo echo "Container-Stats nach Lauf:" cat "$STATS_AFTER_FILE" +cp "$SUMMARY_FILE" "$EXPORTED_SUMMARY_FILE" +cp "$ASSISTANT_TEXT_FILE" "$EXPORTED_ASSISTANT_FILE" +cp "$JSON_OUT_FILE" "$EXPORTED_JSON_FILE" +cp "$OLLAMA_LOG_FILE" "$EXPORTED_OLLAMA_LOG_FILE" +cp "$STATS_BEFORE_FILE" "$EXPORTED_STATS_BEFORE_FILE" +cp "$STATS_AFTER_FILE" "$EXPORTED_STATS_AFTER_FILE" + +if [ -s "$REQUEST_ERR_FILE" ]; then + cp "$REQUEST_ERR_FILE" "$EXPORTED_REQUEST_ERR_FILE" +fi + +if [ "$KEEP_RUNTIME" -eq 0 ] && [ "$RUNTIME_DIR" != "$RESULTS_DIR" ]; then + rm -rf "$RUNTIME_DIR" +fi + +# Generate professional reports +if [ -f "$EXPORTED_SUMMARY_FILE" ]; then + ok "Generating HTML report..." + generate_html_report "$MODEL_NAME" "$RESULTS_DIR" "$EXPORTED_SUMMARY_FILE" 2>/dev/null || true + + ok "Generating Markdown summary..." + generate_markdown_summary "$MODEL_NAME" "$RESULTS_DIR" "$EXPORTED_SUMMARY_FILE" 2>/dev/null || true +fi + if [ "$REQUEST_RC" -eq 0 ] && [ "$VALIDATION_OK" -eq 1 ]; then ok "Benchmark erfolgreich abgeschlossen" exit 0