Add: CodeNeedle Benchmark. Optimize model handling.

2026-06-17 19:14:08 +02:00
parent 599822036f
commit ce64ef56e4
11 changed files with 799 additions and 57 deletions
@@ -5,6 +5,9 @@ set -euo pipefail
 PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 cd "$PROJECT_DIR"

+# Source report generator
+source "$PROJECT_DIR/scripts/lib/report-generator.sh" 2>/dev/null || true
+
 RED='\033[0;31m'
 GREEN='\033[0;32m'
 YELLOW='\033[1;33m'
@@ -27,6 +30,37 @@ need_cmd() {
  command -v "$1" >/dev/null 2>&1 || fail "Benoetigtes Kommando fehlt: $1"
 }

+read_env_var() {
+  local key="$1"
+  local env_file="$PROJECT_DIR/.env"
+  local line=""
+
+  if [ -f "$env_file" ]; then
+    line="$(grep -E "^${key}=" "$env_file" | tail -n 1 || true)"
+  fi
+
+  if [ -z "$line" ] && [ -f "$PROJECT_DIR/.env.example" ]; then
+    line="$(grep -E "^${key}=" "$PROJECT_DIR/.env.example" | tail -n 1 || true)"
+  fi
+
+  if [ -n "$line" ]; then
+    printf '%s' "${line#*=}"
+  fi
+}
+
+resolve_abs_path() {
+  local raw_path="$1"
+  if [[ "$raw_path" = /* ]]; then
+    printf '%s' "$raw_path"
+    return
+  fi
+  if command -v realpath >/dev/null 2>&1; then
+    realpath -m "$PROJECT_DIR/$raw_path"
+    return
+  fi
+  (cd "$PROJECT_DIR" && printf '%s/%s\n' "$PWD" "${raw_path#./}")
+}
+
 usage() {
  cat <<'EOF'
 Usage: ./scripts/benchmark-stack.sh [options]
@@ -37,9 +71,12 @@ validiert den Output und zeigt Metriken (Laufzeit, Token, KV-/VRAM-Metriken).
 Optionen:
  --model <name>          OpenClaw-Modell (default: ollama/qwen3-coding:latest)
  --lines <n>             Erwartete Zeilenanzahl LINE_1..LINE_n (default: 120)
-  --timeout-sec <n>       Timeout fuer den Agent-Call in Sekunden (default: 480)
+  --timeout-sec <n>       Timeout fuer den Agent-Call in Sekunden (default: 600)
  --session-key <key>     Session-Key fuer den Lauf (default: auto)
-  --artifact-dir <path>   Zielverzeichnis fuer Ergebnisdateien (default: auto)
+  --runtime-dir <path>    Zielverzeichnis fuer temporaere Laufdaten (default: auto)
+  --results-dir <path>    Zielverzeichnis fuer kuratierte Resultate (default: auto)
+  --artifact-dir <path>   Legacy-Alias; nutzt denselben Pfad fuer Runtime und Resultate
+  --keep-runtime          Runtime-Dateien nach dem Export nicht loeschen
  --no-start              Stack nicht starten, nur vorhandenen Stack nutzen
  --no-reload             Modell vor Lauf nicht mit 'ollama stop' entladen
  -h, --help              Hilfe anzeigen
@@ -132,13 +169,81 @@ extract_unquoted_metric() {
  fi
 }

+resolve_workspace_media_path() {
+  local media_path="$1"
+  local normalized="$media_path"
+
+  normalized="${normalized#/workspace/}"
+  normalized="${normalized#/home/node/.openclaw/workspace/}"
+
+  if [ -z "$normalized" ] || [ "$normalized" = "$media_path" -a [[ "$media_path" = /* ]] ]; then
+    return 1
+  fi
+
+  printf '%s/%s\n' "$HOST_WORKSPACE_ABS" "$normalized"
+}
+
+extract_tool_call_text() {
+  local raw_text="$1"
+  local extracted
+
+  extracted="$(printf '%s\n' "$raw_text" | sed '/^```json$/d; /^```$/d; /^```$/d' | jq -rs '
+    map(select(type == "object" and (.name == "write" or .name == "edit"))) as $calls
+    | if ($calls | length) == 0 then ""
+      else
+        reduce $calls[] as $call ({content: ""};
+          if $call.name == "write" then
+            .content = ($call.arguments.content // .content)
+          elif $call.name == "edit" then
+            .content = (
+              if (($call.arguments.edits // []) | length) > 0 then
+                ($call.arguments.edits[-1].newText // .content)
+              else
+                .content
+              end
+            )
+          else
+            .
+          end
+        )
+        | .content
+      end
+  ' 2>/dev/null || true)"
+
+  if [ -n "$extracted" ]; then
+    printf '%s' "$extracted"
+    return 0
+  fi
+
+  return 1
+}
+
+extract_media_payload_text() {
+  local media_path=""
+  local host_path=""
+
+  while IFS= read -r media_path; do
+    [ -n "$media_path" ] || continue
+    host_path="$(resolve_workspace_media_path "$media_path" || true)"
+    if [ -n "$host_path" ] && [ -f "$host_path" ]; then
+      cat "$host_path"
+      return 0
+    fi
+  done < <(jq -r '.result.payloads[]? | .mediaUrl?, (.mediaUrls[]?)' "$JSON_OUT_FILE" 2>/dev/null | awk 'NF && !seen[$0]++')
+
+  return 1
+}
+
 MODEL="ollama/qwen3-coding:latest"
 EXPECTED_LINES=120
-TIMEOUT_SEC=480
+TIMEOUT_SEC=600
 SESSION_KEY=""
-ARTIFACT_DIR=""
+RUNTIME_DIR=""
+RESULTS_DIR=""
+LEGACY_ARTIFACT_DIR=""
 START_STACK=1
 FORCE_RELOAD=1
+KEEP_RUNTIME=0

 while [ "$#" -gt 0 ]; do
  case "$1" in
@@ -158,10 +263,22 @@ while [ "$#" -gt 0 ]; do
      SESSION_KEY="$2"
      shift 2
      ;;
-    --artifact-dir)
-      ARTIFACT_DIR="$2"
+    --runtime-dir)
+      RUNTIME_DIR="$2"
      shift 2
      ;;
+    --results-dir)
+      RESULTS_DIR="$2"
+      shift 2
+      ;;
+    --artifact-dir)
+      LEGACY_ARTIFACT_DIR="$2"
+      shift 2
+      ;;
+    --keep-runtime)
+      KEEP_RUNTIME=1
+      shift
+      ;;
    --no-start)
      START_STACK=0
      shift
@@ -195,19 +312,44 @@ need_cmd sed
 need_cmd awk
 need_cmd timeout

-if [ -z "$ARTIFACT_DIR" ]; then
-  TS="$(date +%Y%m%d-%H%M%S)"
-  ARTIFACT_DIR="$PROJECT_DIR/storage/workspace/benchmarks/$TS"
-fi
-mkdir -p "$ARTIFACT_DIR"
+RUN_ID="$(date +%Y%m%d-%H%M%S)"
+HOST_WORKSPACE_RAW="$(read_env_var HOST_WORKSPACE_PATH)"
+HOST_WORKSPACE_RAW="${HOST_WORKSPACE_RAW:-./storage/workspace}"
+HOST_WORKSPACE_ABS="$(resolve_abs_path "$HOST_WORKSPACE_RAW")"
+BENCH_OUTPUT_FILENAME="benchmark-output-${RUN_ID}.txt"

-JSON_OUT_FILE="$ARTIFACT_DIR/agent-result.json"
-ASSISTANT_TEXT_FILE="$ARTIFACT_DIR/assistant-output.txt"
-OLLAMA_LOG_FILE="$ARTIFACT_DIR/ollama-since.log"
-STATS_BEFORE_FILE="$ARTIFACT_DIR/container-stats-before.txt"
-STATS_AFTER_FILE="$ARTIFACT_DIR/container-stats-after.txt"
-REQUEST_ERR_FILE="$ARTIFACT_DIR/request-stderr.log"
-SUMMARY_FILE="$ARTIFACT_DIR/summary.txt"
+if [ -n "$LEGACY_ARTIFACT_DIR" ]; then
+  RUNTIME_DIR="$LEGACY_ARTIFACT_DIR"
+  if [ -z "$RESULTS_DIR" ]; then
+    RESULTS_DIR="$LEGACY_ARTIFACT_DIR"
+    KEEP_RUNTIME=1
+  fi
+fi
+
+if [ -z "$RUNTIME_DIR" ]; then
+  RUNTIME_DIR="$PROJECT_DIR/.cache/benchmarks/existing-suite/runs/$RUN_ID"
+fi
+
+if [ -z "$RESULTS_DIR" ]; then
+  RESULTS_DIR="$PROJECT_DIR/docs/benchmarks/existing-suite/$RUN_ID"
+fi
+
+mkdir -p "$RUNTIME_DIR" "$RESULTS_DIR"
+
+JSON_OUT_FILE="$RUNTIME_DIR/agent-result.json"
+ASSISTANT_TEXT_FILE="$RUNTIME_DIR/assistant-output.txt"
+OLLAMA_LOG_FILE="$RUNTIME_DIR/ollama-since.log"
+STATS_BEFORE_FILE="$RUNTIME_DIR/container-stats-before.txt"
+STATS_AFTER_FILE="$RUNTIME_DIR/container-stats-after.txt"
+REQUEST_ERR_FILE="$RUNTIME_DIR/request-stderr.log"
+SUMMARY_FILE="$RUNTIME_DIR/summary.txt"
+EXPORTED_SUMMARY_FILE="$RESULTS_DIR/summary.txt"
+EXPORTED_ASSISTANT_FILE="$RESULTS_DIR/assistant-output.txt"
+EXPORTED_JSON_FILE="$RESULTS_DIR/agent-result.json"
+EXPORTED_OLLAMA_LOG_FILE="$RESULTS_DIR/ollama-since.log"
+EXPORTED_STATS_BEFORE_FILE="$RESULTS_DIR/container-stats-before.txt"
+EXPORTED_STATS_AFTER_FILE="$RESULTS_DIR/container-stats-after.txt"
+EXPORTED_REQUEST_ERR_FILE="$RESULTS_DIR/request-stderr.log"

 if [ -z "$SESSION_KEY" ]; then
  SESSION_KEY="agent:main:benchmark:$(date +%s)"
@@ -221,7 +363,8 @@ echo "Model: $MODEL"
 echo "Session key: $SESSION_KEY"
 echo "Erwartete Zeilen: $EXPECTED_LINES"
 echo "Timeout: ${TIMEOUT_SEC}s"
-echo "Artefakte: $ARTIFACT_DIR"
+echo "Runtime-Dateien: $RUNTIME_DIR"
+echo "Kuratierte Resultate: $RESULTS_DIR"

 if [ "$START_STACK" -eq 1 ]; then
  echo
@@ -247,7 +390,7 @@ else
 fi

 echo "[4/6] Fuehre Benchmark-Anfrage aus ..."
-PROMPT="Gib exakt ${EXPECTED_LINES} Zeilen aus. Jede Zeile muss exakt den Text LINE_<n> haben, mit aufsteigender Nummerierung von 1 bis ${EXPECTED_LINES}. Kein weiterer Text."
+PROMPT="Gib exakt ${EXPECTED_LINES} Zeilen aus. Jede Zeile muss exakt den Text LINE_<n> haben, mit aufsteigender Nummerierung von 1 bis ${EXPECTED_LINES}. Kein weiterer Text. Nutze keine Tools, keine Dateien, keine Anhaenge und kein JSON. Antworte direkt als reiner Nachrichtentext. Falls du trotzdem Tools benutzt, schreibe exakt diese Zeilen in die Datei ${BENCH_OUTPUT_FILENAME} im Workspace und sonst nirgendwo hin."
 SINCE_UTC="$(date -u +%Y-%m-%dT%H:%M:%SZ)"

 podman stats --no-stream --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}' ollama openclaw > "$STATS_BEFORE_FILE" 2>&1 || true
@@ -257,6 +400,7 @@ REQUEST_RC=0
 if ! timeout "${TIMEOUT_SEC}s" podman exec openclaw openclaw agent \
  --session-key "$SESSION_KEY" \
  --model "$MODEL" \
+  --thinking off \
  --message "$PROMPT" \
  --json > "$JSON_OUT_FILE" 2> "$REQUEST_ERR_FILE"; then
  REQUEST_RC=$?
@@ -290,8 +434,19 @@ fi

 echo "[5/6] Validiere Antwortformat ..."
 ASSISTANT_TEXT=""
+RAW_ASSISTANT_TEXT=""
 if [ "$HAS_VALID_JSON" -eq 1 ]; then
-  ASSISTANT_TEXT="$(jq -r '.result.payloads[0].text // ""' "$JSON_OUT_FILE")"
+  RAW_ASSISTANT_TEXT="$(jq -r '[.result.payloads[]?.text // empty] | join("\n")' "$JSON_OUT_FILE")"
+  ASSISTANT_TEXT="$RAW_ASSISTANT_TEXT"
+  if [ -n "$RAW_ASSISTANT_TEXT" ]; then
+    TOOL_CALL_TEXT="$(extract_tool_call_text "$RAW_ASSISTANT_TEXT" || true)"
+    if [ -n "$TOOL_CALL_TEXT" ]; then
+      ASSISTANT_TEXT="$TOOL_CALL_TEXT"
+    fi
+  fi
+  if [ -z "$ASSISTANT_TEXT" ]; then
+    ASSISTANT_TEXT="$(extract_media_payload_text || true)"
+  fi
 fi
 printf '%s' "$ASSISTANT_TEXT" > "$ASSISTANT_TEXT_FILE"

@@ -426,12 +581,12 @@ fi
  echo "- journal_gpu_related_hits_since_start: $JOURNAL_HITS"
  echo
  echo "Artefakte"
-  echo "- agent_json: $JSON_OUT_FILE"
-  echo "- assistant_text: $ASSISTANT_TEXT_FILE"
-  echo "- ollama_log_since_start: $OLLAMA_LOG_FILE"
-  echo "- stats_before: $STATS_BEFORE_FILE"
-  echo "- stats_after: $STATS_AFTER_FILE"
-  echo "- request_stderr: $REQUEST_ERR_FILE"
+  echo "- agent_json: $EXPORTED_JSON_FILE"
+  echo "- assistant_text: $EXPORTED_ASSISTANT_FILE"
+  echo "- ollama_log_since_start: $EXPORTED_OLLAMA_LOG_FILE"
+  echo "- stats_before: $EXPORTED_STATS_BEFORE_FILE"
+  echo "- stats_after: $EXPORTED_STATS_AFTER_FILE"
+  echo "- request_stderr: $EXPORTED_REQUEST_ERR_FILE"
 } | tee "$SUMMARY_FILE"

 echo
@@ -441,6 +596,30 @@ echo
 echo "Container-Stats nach Lauf:"
 cat "$STATS_AFTER_FILE"

+cp "$SUMMARY_FILE" "$EXPORTED_SUMMARY_FILE"
+cp "$ASSISTANT_TEXT_FILE" "$EXPORTED_ASSISTANT_FILE"
+cp "$JSON_OUT_FILE" "$EXPORTED_JSON_FILE"
+cp "$OLLAMA_LOG_FILE" "$EXPORTED_OLLAMA_LOG_FILE"
+cp "$STATS_BEFORE_FILE" "$EXPORTED_STATS_BEFORE_FILE"
+cp "$STATS_AFTER_FILE" "$EXPORTED_STATS_AFTER_FILE"
+
+if [ -s "$REQUEST_ERR_FILE" ]; then
+  cp "$REQUEST_ERR_FILE" "$EXPORTED_REQUEST_ERR_FILE"
+fi
+
+if [ "$KEEP_RUNTIME" -eq 0 ] && [ "$RUNTIME_DIR" != "$RESULTS_DIR" ]; then
+  rm -rf "$RUNTIME_DIR"
+fi
+
+# Generate professional reports
+if [ -f "$EXPORTED_SUMMARY_FILE" ]; then
+  ok "Generating HTML report..."
+  generate_html_report "$MODEL_NAME" "$RESULTS_DIR" "$EXPORTED_SUMMARY_FILE" 2>/dev/null || true
+  
+  ok "Generating Markdown summary..."
+  generate_markdown_summary "$MODEL_NAME" "$RESULTS_DIR" "$EXPORTED_SUMMARY_FILE" 2>/dev/null || true
+fi
+
 if [ "$REQUEST_RC" -eq 0 ] && [ "$VALIDATION_OK" -eq 1 ]; then
  ok "Benchmark erfolgreich abgeschlossen"
  exit 0