LLM-Server: Effizienz-Auswertung fuer Benchmark-Ergebnisse (GNAI-009 5.2)
eval-efficiency.py/.sh in der benchmark-Rolle: liest die Per-Sample-Logs von lm-eval, zaehlt die exakt generierten Tokens je Antwort ueber den /tokenize-Endpunkt der Engine (Qwen-Tokenizer, keine Schaetzung), zieht den Score und aggregiert Score, Median/Mean/Max-Tokens sowie die Effizienzkennzahl Score je Median-Token. Smoke-Test auf den AIME-2-Ergebnissen: Score 1.0, Median 645 generierte Tokens je Aufgabe. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
@@ -63,6 +63,26 @@
|
||||
tags:
|
||||
- bench
|
||||
|
||||
- name: Render the efficiency evaluation script
|
||||
ansible.builtin.template:
|
||||
src: eval-efficiency.py.j2
|
||||
dest: "{{ benchmark_base_dir }}/eval-efficiency.py"
|
||||
owner: "{{ benchmark_user }}"
|
||||
group: "{{ benchmark_user }}"
|
||||
mode: "0750"
|
||||
tags:
|
||||
- bench
|
||||
|
||||
- name: Render the efficiency evaluation wrapper
|
||||
ansible.builtin.template:
|
||||
src: eval-efficiency.sh.j2
|
||||
dest: "{{ benchmark_base_dir }}/eval-efficiency.sh"
|
||||
owner: "{{ benchmark_user }}"
|
||||
group: "{{ benchmark_user }}"
|
||||
mode: "0750"
|
||||
tags:
|
||||
- bench
|
||||
|
||||
- name: Reload the workload user manager for the bench network
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Efficiency evaluation for lm-eval result directories (GNAI-009 5.2).
|
||||
|
||||
Reads the per-sample lm-eval logs (samples_*.jsonl), counts the exact
|
||||
generated tokens of every response via the engine's /tokenize endpoint
|
||||
(the Qwen tokenizer, no approximation), extracts the score and aggregates:
|
||||
|
||||
score, generated tokens (median/mean/max),
|
||||
efficiency = score / median generated tokens
|
||||
|
||||
Usage: eval-efficiency.py <results-dir> (e.g. /results/<profile>-<ts>)
|
||||
|
||||
The results-dir is the lm-eval --output_path directory that contains one
|
||||
subdirectory per model name with samples_*.jsonl files. The summary is
|
||||
written to <results-dir>/efficiency-summary.json.
|
||||
"""
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
ENGINE_URL = os.environ.get("ENGINE_URL", "http://llama-cpp:8080")
|
||||
TOKENIZE_TIMEOUT = 300
|
||||
|
||||
|
||||
def tokenize(text: str) -> int:
|
||||
payload = json.dumps({"content": text}).encode()
|
||||
req = urllib.request.Request(
|
||||
f"{ENGINE_URL}/tokenize",
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=TOKENIZE_TIMEOUT) as resp:
|
||||
data = json.load(resp)
|
||||
return len(data.get("tokens", []))
|
||||
|
||||
|
||||
def response_text(rec: dict) -> str:
|
||||
resps = rec.get("resps")
|
||||
if not resps:
|
||||
return ""
|
||||
first = resps[0]
|
||||
if isinstance(first, list):
|
||||
return first[0] if first else ""
|
||||
return first
|
||||
|
||||
|
||||
def sample_score(rec: dict):
|
||||
# exact_match based tasks (aime, math variants)
|
||||
if isinstance(rec.get("exact_match"), (int, float, bool)):
|
||||
return float(rec["exact_match"])
|
||||
metrics = rec.get("metrics")
|
||||
if isinstance(metrics, dict):
|
||||
for key in ("prompt_level_strict_acc", "exact_match", "acc"):
|
||||
if key in metrics and isinstance(metrics[key], (int, float, bool)):
|
||||
return float(metrics[key])
|
||||
if isinstance(rec.get("acc"), (int, float, bool)):
|
||||
return float(rec["acc"])
|
||||
return None
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: eval-efficiency.py <results-dir>", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
results_dir = sys.argv[1]
|
||||
summary = {}
|
||||
for path in sorted(glob.glob(os.path.join(results_dir, "*", "samples_*.jsonl"))):
|
||||
bench = os.path.basename(path).removeprefix("samples_")
|
||||
bench = bench.rsplit("_2026", 1)[0]
|
||||
per_sample = []
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
rec = json.loads(line)
|
||||
text = response_text(rec)
|
||||
per_sample.append(
|
||||
{
|
||||
"doc_id": rec.get("doc_id"),
|
||||
"generated_tokens": tokenize(text),
|
||||
"score": sample_score(rec),
|
||||
}
|
||||
)
|
||||
scores = [s["score"] for s in per_sample if s["score"] is not None]
|
||||
toks = [s["generated_tokens"] for s in per_sample]
|
||||
med = statistics.median(toks) if toks else 0
|
||||
mean = statistics.mean(toks) if toks else 0
|
||||
score = statistics.mean(scores) if scores else None
|
||||
summary[bench] = {
|
||||
"samples": len(per_sample),
|
||||
"score": score,
|
||||
"generated_tokens_median": med,
|
||||
"generated_tokens_mean": round(mean, 1),
|
||||
"generated_tokens_max": max(toks) if toks else 0,
|
||||
"efficiency_score_per_median_token": (
|
||||
round(score / med, 8) if med and score is not None else None
|
||||
),
|
||||
"per_sample": per_sample,
|
||||
}
|
||||
out_path = os.path.join(results_dir, "efficiency-summary.json")
|
||||
with open(out_path, "w", encoding="utf-8") as fh:
|
||||
json.dump(summary, fh, indent=2, ensure_ascii=False)
|
||||
report = {
|
||||
k: {kk: vv for kk, vv in v.items() if kk != "per_sample"}
|
||||
for k, v in summary.items()
|
||||
}
|
||||
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||||
print(f"summary written to {out_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/bin/bash
|
||||
# Efficiency evaluation wrapper (GNAI-009 5.2).
|
||||
# Usage: eval-efficiency.sh <results-dir>
|
||||
# Runs the token-counting evaluation inside the pinned bench container
|
||||
# against the running engine (tokenizer-only, no generation).
|
||||
set -euo pipefail
|
||||
RESULTS=$1
|
||||
|
||||
/usr/sbin/runuser --user {{ benchmark_user }} -- \
|
||||
/usr/bin/env HOME={{ benchmark_user_home }} XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }} \
|
||||
/usr/bin/systemctl --user start {{ benchmark_bench_network_name }}-network.service
|
||||
|
||||
exec /usr/sbin/runuser --user {{ benchmark_user }} -- \
|
||||
/usr/bin/env HOME={{ benchmark_user_home }} XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }} \
|
||||
/usr/bin/podman run --rm \
|
||||
--name gnai-bench-eval \
|
||||
--network {{ benchmark_engine_network_name }} \
|
||||
--network {{ benchmark_bench_network_name }} \
|
||||
-v {{ benchmark_base_dir }}/eval-efficiency.py:/work/eval-efficiency.py:ro \
|
||||
-v "$RESULTS":/results:Z \
|
||||
--entrypoint python3 \
|
||||
{{ benchmark_image_tag }} /work/eval-efficiency.py /results
|
||||
Reference in New Issue
Block a user