132 lines
3.4 KiB
Bash
Executable File
132 lines
3.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$PROJECT_DIR"
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage: ./scripts/benchmark-model-suite.sh [options]
|
|
|
|
Fuehrt den bestehenden Stack-Benchmark fuer die drei lokalen Coding-Profile aus
|
|
und schreibt Artefakte plus eine Vergleichszusammenfassung.
|
|
|
|
Optionen:
|
|
--lines <n> Erwartete Zeilenanzahl LINE_1..LINE_n (default: 140)
|
|
--timeout-sec <n> Timeout je Lauf in Sekunden (default: 300)
|
|
--artifact-root <p> Zielordner fuer alle Benchmark-Artefakte (default: auto)
|
|
--session-prefix <s> Prefix fuer Session-Keys (default: suite:<timestamp>)
|
|
-h, --help Hilfe anzeigen
|
|
EOF
|
|
}
|
|
|
|
slugify_model() {
|
|
local model="$1"
|
|
local slug="${model#ollama/}"
|
|
slug="${slug//:/-}"
|
|
slug="${slug//./-}"
|
|
printf '%s' "$slug"
|
|
}
|
|
|
|
extract_value() {
|
|
local label="$1"
|
|
local file="$2"
|
|
local value
|
|
value="$(sed -n "s/^${label}: //p" "$file" | head -n 1)"
|
|
printf '%s' "${value:-n/a}"
|
|
}
|
|
|
|
extract_metric() {
|
|
local label="$1"
|
|
local file="$2"
|
|
local value
|
|
value="$(sed -n "s/^- ${label}: //p" "$file" | head -n 1)"
|
|
printf '%s' "${value:-n/a}"
|
|
}
|
|
|
|
EXPECTED_LINES=140
|
|
TIMEOUT_SEC=300
|
|
ARTIFACT_ROOT=""
|
|
SESSION_PREFIX="suite:$(date +%Y%m%d-%H%M%S)"
|
|
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--lines)
|
|
EXPECTED_LINES="$2"
|
|
shift 2
|
|
;;
|
|
--timeout-sec)
|
|
TIMEOUT_SEC="$2"
|
|
shift 2
|
|
;;
|
|
--artifact-root)
|
|
ARTIFACT_ROOT="$2"
|
|
shift 2
|
|
;;
|
|
--session-prefix)
|
|
SESSION_PREFIX="$2"
|
|
shift 2
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "Unbekannte Option: $1" >&2
|
|
usage >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [ -z "$ARTIFACT_ROOT" ]; then
|
|
ARTIFACT_ROOT="$PROJECT_DIR/storage/workspace/benchmarks/model-suite-$(date +%Y%m%d-%H%M%S)"
|
|
fi
|
|
|
|
mkdir -p "$ARTIFACT_ROOT"
|
|
|
|
MODELS=(
|
|
"ollama/qwen3-coding:latest"
|
|
"ollama/qwen25-coding:latest"
|
|
"ollama/deepseek-coding:latest"
|
|
)
|
|
|
|
SUMMARY_TABLE="$ARTIFACT_ROOT/summary.tsv"
|
|
printf 'model\tbenchmark_rc\tvalidation\trequest_exit\twall_ms\tprovider_tps\tend_to_end_tps\tload_sec\tkv_tokens\n' > "$SUMMARY_TABLE"
|
|
|
|
for model in "${MODELS[@]}"; do
|
|
slug="$(slugify_model "$model")"
|
|
run_dir="$ARTIFACT_ROOT/$slug"
|
|
session_key="$SESSION_PREFIX:$slug"
|
|
|
|
echo "=== Benchmark fuer $model ==="
|
|
benchmark_rc=0
|
|
if ! ./scripts/benchmark-stack.sh \
|
|
--model "$model" \
|
|
--lines "$EXPECTED_LINES" \
|
|
--timeout-sec "$TIMEOUT_SEC" \
|
|
--session-key "$session_key" \
|
|
--artifact-dir "$run_dir" \
|
|
--no-start; then
|
|
benchmark_rc=$?
|
|
fi
|
|
|
|
summary_file="$run_dir/summary.txt"
|
|
|
|
validation="$(extract_value 'Validation' "$summary_file")"
|
|
request_exit="$(extract_value 'Request exit code' "$summary_file")"
|
|
wall_ms="$(extract_metric 'end_to_end_wall_ms' "$summary_file")"
|
|
provider_tps="$(extract_metric 'output_tokens_per_sec_provider' "$summary_file")"
|
|
end_to_end_tps="$(extract_metric 'output_tokens_per_sec_end_to_end' "$summary_file")"
|
|
load_sec="$(extract_metric 'model_load_time_sec' "$summary_file")"
|
|
kv_tokens="$(extract_metric 'kv_size_tokens' "$summary_file")"
|
|
|
|
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
|
"$model" "$benchmark_rc" "$validation" "$request_exit" "$wall_ms" "$provider_tps" "$end_to_end_tps" "$load_sec" "$kv_tokens" \
|
|
>> "$SUMMARY_TABLE"
|
|
done
|
|
|
|
echo
|
|
echo "Vergleich gespeichert unter: $ARTIFACT_ROOT"
|
|
column -ts $'\t' "$SUMMARY_TABLE" |