Add: Cline integration
This commit is contained in:
Executable
+132
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: ./scripts/benchmark-model-suite.sh [options]
|
||||
|
||||
Fuehrt den bestehenden Stack-Benchmark fuer die drei lokalen Coding-Profile aus
|
||||
und schreibt Artefakte plus eine Vergleichszusammenfassung.
|
||||
|
||||
Optionen:
|
||||
--lines <n> Erwartete Zeilenanzahl LINE_1..LINE_n (default: 140)
|
||||
--timeout-sec <n> Timeout je Lauf in Sekunden (default: 300)
|
||||
--artifact-root <p> Zielordner fuer alle Benchmark-Artefakte (default: auto)
|
||||
--session-prefix <s> Prefix fuer Session-Keys (default: suite:<timestamp>)
|
||||
-h, --help Hilfe anzeigen
|
||||
EOF
|
||||
}
|
||||
|
||||
slugify_model() {
|
||||
local model="$1"
|
||||
local slug="${model#ollama/}"
|
||||
slug="${slug//:/-}"
|
||||
slug="${slug//./-}"
|
||||
printf '%s' "$slug"
|
||||
}
|
||||
|
||||
extract_value() {
|
||||
local label="$1"
|
||||
local file="$2"
|
||||
local value
|
||||
value="$(sed -n "s/^${label}: //p" "$file" | head -n 1)"
|
||||
printf '%s' "${value:-n/a}"
|
||||
}
|
||||
|
||||
extract_metric() {
|
||||
local label="$1"
|
||||
local file="$2"
|
||||
local value
|
||||
value="$(sed -n "s/^- ${label}: //p" "$file" | head -n 1)"
|
||||
printf '%s' "${value:-n/a}"
|
||||
}
|
||||
|
||||
EXPECTED_LINES=140
|
||||
TIMEOUT_SEC=300
|
||||
ARTIFACT_ROOT=""
|
||||
SESSION_PREFIX="suite:$(date +%Y%m%d-%H%M%S)"
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--lines)
|
||||
EXPECTED_LINES="$2"
|
||||
shift 2
|
||||
;;
|
||||
--timeout-sec)
|
||||
TIMEOUT_SEC="$2"
|
||||
shift 2
|
||||
;;
|
||||
--artifact-root)
|
||||
ARTIFACT_ROOT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--session-prefix)
|
||||
SESSION_PREFIX="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unbekannte Option: $1" >&2
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "$ARTIFACT_ROOT" ]; then
|
||||
ARTIFACT_ROOT="$PROJECT_DIR/storage/workspace/benchmarks/model-suite-$(date +%Y%m%d-%H%M%S)"
|
||||
fi
|
||||
|
||||
mkdir -p "$ARTIFACT_ROOT"
|
||||
|
||||
MODELS=(
|
||||
"ollama/qwen3-coding:latest"
|
||||
"ollama/qwen25-coding:latest"
|
||||
"ollama/deepseek-coding:latest"
|
||||
)
|
||||
|
||||
SUMMARY_TABLE="$ARTIFACT_ROOT/summary.tsv"
|
||||
printf 'model\tbenchmark_rc\tvalidation\trequest_exit\twall_ms\tprovider_tps\tend_to_end_tps\tload_sec\tkv_tokens\n' > "$SUMMARY_TABLE"
|
||||
|
||||
for model in "${MODELS[@]}"; do
|
||||
slug="$(slugify_model "$model")"
|
||||
run_dir="$ARTIFACT_ROOT/$slug"
|
||||
session_key="$SESSION_PREFIX:$slug"
|
||||
|
||||
echo "=== Benchmark fuer $model ==="
|
||||
benchmark_rc=0
|
||||
if ! ./scripts/benchmark-stack.sh \
|
||||
--model "$model" \
|
||||
--lines "$EXPECTED_LINES" \
|
||||
--timeout-sec "$TIMEOUT_SEC" \
|
||||
--session-key "$session_key" \
|
||||
--artifact-dir "$run_dir" \
|
||||
--no-start; then
|
||||
benchmark_rc=$?
|
||||
fi
|
||||
|
||||
summary_file="$run_dir/summary.txt"
|
||||
|
||||
validation="$(extract_value 'Validation' "$summary_file")"
|
||||
request_exit="$(extract_value 'Request exit code' "$summary_file")"
|
||||
wall_ms="$(extract_metric 'end_to_end_wall_ms' "$summary_file")"
|
||||
provider_tps="$(extract_metric 'output_tokens_per_sec_provider' "$summary_file")"
|
||||
end_to_end_tps="$(extract_metric 'output_tokens_per_sec_end_to_end' "$summary_file")"
|
||||
load_sec="$(extract_metric 'model_load_time_sec' "$summary_file")"
|
||||
kv_tokens="$(extract_metric 'kv_size_tokens' "$summary_file")"
|
||||
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$model" "$benchmark_rc" "$validation" "$request_exit" "$wall_ms" "$provider_tps" "$end_to_end_tps" "$load_sec" "$kv_tokens" \
|
||||
>> "$SUMMARY_TABLE"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Vergleich gespeichert unter: $ARTIFACT_ROOT"
|
||||
column -ts $'\t' "$SUMMARY_TABLE"
|
||||
@@ -35,7 +35,7 @@ Startet den Stack (optional), fuehrt eine reproduzierbare OpenClaw-Anfrage aus,
|
||||
validiert den Output und zeigt Metriken (Laufzeit, Token, KV-/VRAM-Metriken).
|
||||
|
||||
Optionen:
|
||||
--model <name> OpenClaw-Modell (default: ollama/qwen-coding-long:latest)
|
||||
--model <name> OpenClaw-Modell (default: ollama/qwen3-coding:latest)
|
||||
--lines <n> Erwartete Zeilenanzahl LINE_1..LINE_n (default: 120)
|
||||
--timeout-sec <n> Timeout fuer den Agent-Call in Sekunden (default: 480)
|
||||
--session-key <key> Session-Key fuer den Lauf (default: auto)
|
||||
@@ -45,7 +45,7 @@ Optionen:
|
||||
-h, --help Hilfe anzeigen
|
||||
|
||||
Beispiel:
|
||||
./scripts/benchmark-stack.sh --model ollama/qwen-coding-long:latest --lines 140
|
||||
./scripts/benchmark-stack.sh --model ollama/qwen3-coding:latest --lines 140
|
||||
EOF
|
||||
}
|
||||
|
||||
@@ -132,7 +132,7 @@ extract_unquoted_metric() {
|
||||
fi
|
||||
}
|
||||
|
||||
MODEL="ollama/qwen-coding-long:latest"
|
||||
MODEL="ollama/qwen3-coding:latest"
|
||||
EXPECTED_LINES=120
|
||||
TIMEOUT_SEC=480
|
||||
SESSION_KEY=""
|
||||
|
||||
Reference in New Issue
Block a user