LLM-Server: Effizienz-Auswertung fuer Benchmark-Ergebnisse (GNAI-009 5.2)

eval-efficiency.py/.sh in der benchmark-Rolle: liest die Per-Sample-Logs
von lm-eval, zaehlt die exakt generierten Tokens je Antwort ueber den
/tokenize-Endpunkt der Engine (Qwen-Tokenizer, keine Schaetzung), zieht
den Score und aggregiert Score, Median/Mean/Max-Tokens sowie die
Effizienzkennzahl Score je Median-Token. Smoke-Test auf den AIME-2-Ergebnissen:
Score 1.0, Median 645 generierte Tokens je Aufgabe.

Generated by Mistral Vibe.
Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
2026-11-02 06:44:44 +01:00
parent c6cabd98ef
commit 89e42ecd9b
3 changed files with 155 additions and 0 deletions
@@ -63,6 +63,26 @@
tags:
- bench
- name: Render the efficiency evaluation script
ansible.builtin.template:
src: eval-efficiency.py.j2
dest: "{{ benchmark_base_dir }}/eval-efficiency.py"
owner: "{{ benchmark_user }}"
group: "{{ benchmark_user }}"
mode: "0750"
tags:
- bench
- name: Render the efficiency evaluation wrapper
ansible.builtin.template:
src: eval-efficiency.sh.j2
dest: "{{ benchmark_base_dir }}/eval-efficiency.sh"
owner: "{{ benchmark_user }}"
group: "{{ benchmark_user }}"
mode: "0750"
tags:
- bench
- name: Reload the workload user manager for the bench network
ansible.builtin.command:
argv:
@@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""Efficiency evaluation for lm-eval result directories (GNAI-009 5.2).
Reads the per-sample lm-eval logs (samples_*.jsonl), counts the exact
generated tokens of every response via the engine's /tokenize endpoint
(the Qwen tokenizer, no approximation), extracts the score and aggregates:
score, generated tokens (median/mean/max),
efficiency = score / median generated tokens
Usage: eval-efficiency.py <results-dir> (e.g. /results/<profile>-<ts>)
The results-dir is the lm-eval --output_path directory that contains one
subdirectory per model name with samples_*.jsonl files. The summary is
written to <results-dir>/efficiency-summary.json.
"""
import glob
import json
import os
import statistics
import sys
import urllib.request
ENGINE_URL = os.environ.get("ENGINE_URL", "http://llama-cpp:8080")
TOKENIZE_TIMEOUT = 300
def tokenize(text: str) -> int:
payload = json.dumps({"content": text}).encode()
req = urllib.request.Request(
f"{ENGINE_URL}/tokenize",
data=payload,
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(req, timeout=TOKENIZE_TIMEOUT) as resp:
data = json.load(resp)
return len(data.get("tokens", []))
def response_text(rec: dict) -> str:
resps = rec.get("resps")
if not resps:
return ""
first = resps[0]
if isinstance(first, list):
return first[0] if first else ""
return first
def sample_score(rec: dict):
# exact_match based tasks (aime, math variants)
if isinstance(rec.get("exact_match"), (int, float, bool)):
return float(rec["exact_match"])
metrics = rec.get("metrics")
if isinstance(metrics, dict):
for key in ("prompt_level_strict_acc", "exact_match", "acc"):
if key in metrics and isinstance(metrics[key], (int, float, bool)):
return float(metrics[key])
if isinstance(rec.get("acc"), (int, float, bool)):
return float(rec["acc"])
return None
def main() -> None:
if len(sys.argv) != 2:
print("usage: eval-efficiency.py <results-dir>", file=sys.stderr)
sys.exit(2)
results_dir = sys.argv[1]
summary = {}
for path in sorted(glob.glob(os.path.join(results_dir, "*", "samples_*.jsonl"))):
bench = os.path.basename(path).removeprefix("samples_")
bench = bench.rsplit("_2026", 1)[0]
per_sample = []
with open(path, encoding="utf-8") as fh:
for line in fh:
rec = json.loads(line)
text = response_text(rec)
per_sample.append(
{
"doc_id": rec.get("doc_id"),
"generated_tokens": tokenize(text),
"score": sample_score(rec),
}
)
scores = [s["score"] for s in per_sample if s["score"] is not None]
toks = [s["generated_tokens"] for s in per_sample]
med = statistics.median(toks) if toks else 0
mean = statistics.mean(toks) if toks else 0
score = statistics.mean(scores) if scores else None
summary[bench] = {
"samples": len(per_sample),
"score": score,
"generated_tokens_median": med,
"generated_tokens_mean": round(mean, 1),
"generated_tokens_max": max(toks) if toks else 0,
"efficiency_score_per_median_token": (
round(score / med, 8) if med and score is not None else None
),
"per_sample": per_sample,
}
out_path = os.path.join(results_dir, "efficiency-summary.json")
with open(out_path, "w", encoding="utf-8") as fh:
json.dump(summary, fh, indent=2, ensure_ascii=False)
report = {
k: {kk: vv for kk, vv in v.items() if kk != "per_sample"}
for k, v in summary.items()
}
print(json.dumps(report, indent=2, ensure_ascii=False))
print(f"summary written to {out_path}")
if __name__ == "__main__":
main()
@@ -0,0 +1,22 @@
#!/bin/bash
# Efficiency evaluation wrapper (GNAI-009 5.2).
# Usage: eval-efficiency.sh <results-dir>
# Runs the token-counting evaluation inside the pinned bench container
# against the running engine (tokenizer-only, no generation).
set -euo pipefail
RESULTS=$1
/usr/sbin/runuser --user {{ benchmark_user }} -- \
/usr/bin/env HOME={{ benchmark_user_home }} XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }} \
/usr/bin/systemctl --user start {{ benchmark_bench_network_name }}-network.service
exec /usr/sbin/runuser --user {{ benchmark_user }} -- \
/usr/bin/env HOME={{ benchmark_user_home }} XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }} \
/usr/bin/podman run --rm \
--name gnai-bench-eval \
--network {{ benchmark_engine_network_name }} \
--network {{ benchmark_bench_network_name }} \
-v {{ benchmark_base_dir }}/eval-efficiency.py:/work/eval-efficiency.py:ro \
-v "$RESULTS":/results:Z \
--entrypoint python3 \
{{ benchmark_image_tag }} /work/eval-efficiency.py /results