ghostnet-openclaw/scripts/benchmark-model-suite.sh

#!/usr/bin/env bash

set -euo pipefail

PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$PROJECT_DIR"

# Source report generator
source "$PROJECT_DIR/scripts/lib/report-generator.sh" 2>/dev/null || true

usage() {
  cat <<'EOF'
Usage: ./scripts/benchmark-model-suite.sh [options]

Fuehrt den bestehenden Stack-Benchmark fuer die drei lokalen Coding-Profile aus
und schreibt Artefakte plus eine Vergleichszusammenfassung.

Optionen:
  --lines <n>           Erwartete Zeilenanzahl LINE_1..LINE_n (default: 140)
  --timeout-sec <n>     Timeout je Lauf in Sekunden (default: 600)
  --runtime-root <p>    Zielordner fuer temporaere Laufdaten (default: auto)
  --results-root <p>    Zielordner fuer kuratierte Resultate (default: auto)
  --artifact-root <p>   Legacy-Alias; nutzt denselben Pfad fuer Runtime und Resultate
  --keep-runtime        Runtime-Dateien der Einzelruns nicht loeschen
  --session-prefix <s>  Prefix fuer Session-Keys (default: suite:<timestamp>)
  -h, --help            Hilfe anzeigen
EOF
}

slugify_model() {
  local model="$1"
  local slug="${model#ollama/}"
  slug="${slug//:/-}"
  slug="${slug//./-}"
  printf '%s' "$slug"
}

extract_value() {
  local label="$1"
  local file="$2"
  local value
  value="$(sed -n "s/^${label}: //p" "$file" | head -n 1)"
  printf '%s' "${value:-n/a}"
}

extract_metric() {
  local label="$1"
  local file="$2"
  local value
  value="$(sed -n "s/^- ${label}: //p" "$file" | head -n 1)"
  printf '%s' "${value:-n/a}"
}

EXPECTED_LINES=140
TIMEOUT_SEC=600
RUNTIME_ROOT=""
RESULTS_ROOT=""
LEGACY_ARTIFACT_ROOT=""
KEEP_RUNTIME=0
SESSION_PREFIX="suite:$(date +%Y%m%d-%H%M%S)"

while [ "$#" -gt 0 ]; do
  case "$1" in
    --lines)
      EXPECTED_LINES="$2"
      shift 2
      ;;
    --timeout-sec)
      TIMEOUT_SEC="$2"
      shift 2
      ;;
    --runtime-root)
      RUNTIME_ROOT="$2"
      shift 2
      ;;
    --results-root)
      RESULTS_ROOT="$2"
      shift 2
      ;;
    --artifact-root)
      LEGACY_ARTIFACT_ROOT="$2"
      shift 2
      ;;
    --keep-runtime)
      KEEP_RUNTIME=1
      shift
      ;;
    --session-prefix)
      SESSION_PREFIX="$2"
      shift 2
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      echo "Unbekannte Option: $1" >&2
      usage >&2
      exit 1
      ;;
  esac
done

RUN_ID="model-suite-$(date +%Y%m%d-%H%M%S)"

if [ -n "$LEGACY_ARTIFACT_ROOT" ]; then
  RUNTIME_ROOT="$LEGACY_ARTIFACT_ROOT"
  if [ -z "$RESULTS_ROOT" ]; then
    RESULTS_ROOT="$LEGACY_ARTIFACT_ROOT"
    KEEP_RUNTIME=1
  fi
fi

if [ -z "$RUNTIME_ROOT" ]; then
  RUNTIME_ROOT="$PROJECT_DIR/.cache/benchmarks/$RUN_ID"
fi

if [ -z "$RESULTS_ROOT" ]; then
  RESULTS_ROOT="$PROJECT_DIR/docs/benchmarks/existing-suite/$RUN_ID"
fi

mkdir -p "$RUNTIME_ROOT" "$RESULTS_ROOT"

MODELS=(
  "ollama/qwen3-coding:latest"
  "ollama/qwen25-coding:latest"
  "ollama/deepseek-coding:latest"
)

SUMMARY_TABLE="$RESULTS_ROOT/summary.tsv"
printf 'model\tbenchmark_rc\tvalidation\trequest_exit\twall_ms\tprovider_tps\tend_to_end_tps\tload_sec\tkv_tokens\n' > "$SUMMARY_TABLE"

for model in "${MODELS[@]}"; do
  slug="$(slugify_model "$model")"
  runtime_dir="$RUNTIME_ROOT/$slug"
  results_dir="$RESULTS_ROOT/$slug"
  session_key="$SESSION_PREFIX:$slug"
  run_args=(
    --model "$model"
    --lines "$EXPECTED_LINES"
    --timeout-sec "$TIMEOUT_SEC"
    --session-key "$session_key"
    --runtime-dir "$runtime_dir"
    --results-dir "$results_dir"
    --no-start
  )

  if [ "$KEEP_RUNTIME" -eq 1 ]; then
    run_args+=(--keep-runtime)
  fi

  echo "=== Benchmark fuer $model ==="
  if ./scripts/benchmark-stack.sh "${run_args[@]}"; then
    benchmark_rc=0
  else
    benchmark_rc=$?
  fi

  summary_file="$results_dir/summary.txt"

  validation="$(extract_value 'Validation' "$summary_file")"
  request_exit="$(extract_value 'Request exit code' "$summary_file")"
  wall_ms="$(extract_metric 'end_to_end_wall_ms' "$summary_file")"
  provider_tps="$(extract_metric 'output_tokens_per_sec_provider' "$summary_file")"
  end_to_end_tps="$(extract_metric 'output_tokens_per_sec_end_to_end' "$summary_file")"
  load_sec="$(extract_metric 'model_load_time_sec' "$summary_file")"
  kv_tokens="$(extract_metric 'kv_size_tokens' "$summary_file")"

  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
    "$model" "$benchmark_rc" "$validation" "$request_exit" "$wall_ms" "$provider_tps" "$end_to_end_tps" "$load_sec" "$kv_tokens" \
    >> "$SUMMARY_TABLE"
done

echo
echo "Runtime-Wurzel: $RUNTIME_ROOT"
echo "Kuratierte Resultate: $RESULTS_ROOT"
column -ts $'\t' "$SUMMARY_TABLE"

# Generate suite comparison report
if [ -f "$SUMMARY_TABLE" ]; then
  echo
  echo "📊 Generating suite comparison report..."
  generate_suite_comparison_report "$SUMMARY_TABLE" "$RESULTS_ROOT" 2>/dev/null || true
  echo "✅ Reports generated in: $RESULTS_ROOT"
fi