Add: CodeNeedle Benchmark. Optimize model handling.

scripts/benchmark-model-suite.sh

@@ -5,6 +5,9 @@ set -euo pipefail
 PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 cd "$PROJECT_DIR"
 
+# Source report generator
+source "$PROJECT_DIR/scripts/lib/report-generator.sh" 2>/dev/null || true
+
 usage() {
   cat <<'EOF'
 Usage: ./scripts/benchmark-model-suite.sh [options]
@@ -14,8 +17,11 @@ und schreibt Artefakte plus eine Vergleichszusammenfassung.
 
 Optionen:
   --lines <n>           Erwartete Zeilenanzahl LINE_1..LINE_n (default: 140)
-  --timeout-sec <n>     Timeout je Lauf in Sekunden (default: 300)
-  --artifact-root <p>   Zielordner fuer alle Benchmark-Artefakte (default: auto)
+  --timeout-sec <n>     Timeout je Lauf in Sekunden (default: 600)
+  --runtime-root <p>    Zielordner fuer temporaere Laufdaten (default: auto)
+  --results-root <p>    Zielordner fuer kuratierte Resultate (default: auto)
+  --artifact-root <p>   Legacy-Alias; nutzt denselben Pfad fuer Runtime und Resultate
+  --keep-runtime        Runtime-Dateien der Einzelruns nicht loeschen
   --session-prefix <s>  Prefix fuer Session-Keys (default: suite:<timestamp>)
   -h, --help            Hilfe anzeigen
 EOF
@@ -46,8 +52,11 @@ extract_metric() {
 }
 
 EXPECTED_LINES=140
-TIMEOUT_SEC=300
-ARTIFACT_ROOT=""
+TIMEOUT_SEC=600
+RUNTIME_ROOT=""
+RESULTS_ROOT=""
+LEGACY_ARTIFACT_ROOT=""
+KEEP_RUNTIME=0
 SESSION_PREFIX="suite:$(date +%Y%m%d-%H%M%S)"
 
 while [ "$#" -gt 0 ]; do
@@ -60,10 +69,22 @@ while [ "$#" -gt 0 ]; do
       TIMEOUT_SEC="$2"
       shift 2
       ;;
-    --artifact-root)
-      ARTIFACT_ROOT="$2"
+    --runtime-root)
+      RUNTIME_ROOT="$2"
       shift 2
       ;;
+    --results-root)
+      RESULTS_ROOT="$2"
+      shift 2
+      ;;
+    --artifact-root)
+      LEGACY_ARTIFACT_ROOT="$2"
+      shift 2
+      ;;
+    --keep-runtime)
+      KEEP_RUNTIME=1
+      shift
+      ;;
     --session-prefix)
       SESSION_PREFIX="$2"
       shift 2
@@ -80,11 +101,25 @@ while [ "$#" -gt 0 ]; do
   esac
 done
 
-if [ -z "$ARTIFACT_ROOT" ]; then
-  ARTIFACT_ROOT="$PROJECT_DIR/storage/workspace/benchmarks/model-suite-$(date +%Y%m%d-%H%M%S)"
+RUN_ID="model-suite-$(date +%Y%m%d-%H%M%S)"
+
+if [ -n "$LEGACY_ARTIFACT_ROOT" ]; then
+  RUNTIME_ROOT="$LEGACY_ARTIFACT_ROOT"
+  if [ -z "$RESULTS_ROOT" ]; then
+    RESULTS_ROOT="$LEGACY_ARTIFACT_ROOT"
+    KEEP_RUNTIME=1
+  fi
 fi
 
-mkdir -p "$ARTIFACT_ROOT"
+if [ -z "$RUNTIME_ROOT" ]; then
+  RUNTIME_ROOT="$PROJECT_DIR/.cache/benchmarks/$RUN_ID"
+fi
+
+if [ -z "$RESULTS_ROOT" ]; then
+  RESULTS_ROOT="$PROJECT_DIR/docs/benchmarks/existing-suite/$RUN_ID"
+fi
+
+mkdir -p "$RUNTIME_ROOT" "$RESULTS_ROOT"
 
 MODELS=(
   "ollama/qwen3-coding:latest"
@@ -92,27 +127,36 @@ MODELS=(
   "ollama/deepseek-coding:latest"
 )
 
-SUMMARY_TABLE="$ARTIFACT_ROOT/summary.tsv"
+SUMMARY_TABLE="$RESULTS_ROOT/summary.tsv"
 printf 'model\tbenchmark_rc\tvalidation\trequest_exit\twall_ms\tprovider_tps\tend_to_end_tps\tload_sec\tkv_tokens\n' > "$SUMMARY_TABLE"
 
 for model in "${MODELS[@]}"; do
   slug="$(slugify_model "$model")"
-  run_dir="$ARTIFACT_ROOT/$slug"
+  runtime_dir="$RUNTIME_ROOT/$slug"
+  results_dir="$RESULTS_ROOT/$slug"
   session_key="$SESSION_PREFIX:$slug"
+  run_args=(
+    --model "$model"
+    --lines "$EXPECTED_LINES"
+    --timeout-sec "$TIMEOUT_SEC"
+    --session-key "$session_key"
+    --runtime-dir "$runtime_dir"
+    --results-dir "$results_dir"
+    --no-start
+  )
+
+  if [ "$KEEP_RUNTIME" -eq 1 ]; then
+    run_args+=(--keep-runtime)
+  fi
 
   echo "=== Benchmark fuer $model ==="
-  benchmark_rc=0
-  if ! ./scripts/benchmark-stack.sh \
-    --model "$model" \
-    --lines "$EXPECTED_LINES" \
-    --timeout-sec "$TIMEOUT_SEC" \
-    --session-key "$session_key" \
-    --artifact-dir "$run_dir" \
-    --no-start; then
+  if ./scripts/benchmark-stack.sh "${run_args[@]}"; then
+    benchmark_rc=0
+  else
     benchmark_rc=$?
   fi
 
-  summary_file="$run_dir/summary.txt"
+  summary_file="$results_dir/summary.txt"
 
   validation="$(extract_value 'Validation' "$summary_file")"
   request_exit="$(extract_value 'Request exit code' "$summary_file")"
@@ -128,5 +172,14 @@ for model in "${MODELS[@]}"; do
 done
 
 echo
-echo "Vergleich gespeichert unter: $ARTIFACT_ROOT"
-column -ts $'\t' "$SUMMARY_TABLE"
+echo "Runtime-Wurzel: $RUNTIME_ROOT"
+echo "Kuratierte Resultate: $RESULTS_ROOT"
+column -ts $'\t' "$SUMMARY_TABLE"
+
+# Generate suite comparison report
+if [ -f "$SUMMARY_TABLE" ]; then
+  echo
+  echo "📊 Generating suite comparison report..."
+  generate_suite_comparison_report "$SUMMARY_TABLE" "$RESULTS_ROOT" 2>/dev/null || true
+  echo "✅ Reports generated in: $RESULTS_ROOT"
+fi