LLM-Server: Effizienz-Auswertung fuer Benchmark-Ergebnisse (GNAI-009 5.2)
eval-efficiency.py/.sh in der benchmark-Rolle: liest die Per-Sample-Logs von lm-eval, zaehlt die exakt generierten Tokens je Antwort ueber den /tokenize-Endpunkt der Engine (Qwen-Tokenizer, keine Schaetzung), zieht den Score und aggregiert Score, Median/Mean/Max-Tokens sowie die Effizienzkennzahl Score je Median-Token. Smoke-Test auf den AIME-2-Ergebnissen: Score 1.0, Median 645 generierte Tokens je Aufgabe. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
@@ -63,6 +63,26 @@
|
|||||||
tags:
|
tags:
|
||||||
- bench
|
- bench
|
||||||
|
|
||||||
|
- name: Render the efficiency evaluation script
|
||||||
|
ansible.builtin.template:
|
||||||
|
src: eval-efficiency.py.j2
|
||||||
|
dest: "{{ benchmark_base_dir }}/eval-efficiency.py"
|
||||||
|
owner: "{{ benchmark_user }}"
|
||||||
|
group: "{{ benchmark_user }}"
|
||||||
|
mode: "0750"
|
||||||
|
tags:
|
||||||
|
- bench
|
||||||
|
|
||||||
|
- name: Render the efficiency evaluation wrapper
|
||||||
|
ansible.builtin.template:
|
||||||
|
src: eval-efficiency.sh.j2
|
||||||
|
dest: "{{ benchmark_base_dir }}/eval-efficiency.sh"
|
||||||
|
owner: "{{ benchmark_user }}"
|
||||||
|
group: "{{ benchmark_user }}"
|
||||||
|
mode: "0750"
|
||||||
|
tags:
|
||||||
|
- bench
|
||||||
|
|
||||||
- name: Reload the workload user manager for the bench network
|
- name: Reload the workload user manager for the bench network
|
||||||
ansible.builtin.command:
|
ansible.builtin.command:
|
||||||
argv:
|
argv:
|
||||||
|
|||||||
@@ -0,0 +1,113 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Efficiency evaluation for lm-eval result directories (GNAI-009 5.2).
|
||||||
|
|
||||||
|
Reads the per-sample lm-eval logs (samples_*.jsonl), counts the exact
|
||||||
|
generated tokens of every response via the engine's /tokenize endpoint
|
||||||
|
(the Qwen tokenizer, no approximation), extracts the score and aggregates:
|
||||||
|
|
||||||
|
score, generated tokens (median/mean/max),
|
||||||
|
efficiency = score / median generated tokens
|
||||||
|
|
||||||
|
Usage: eval-efficiency.py <results-dir> (e.g. /results/<profile>-<ts>)
|
||||||
|
|
||||||
|
The results-dir is the lm-eval --output_path directory that contains one
|
||||||
|
subdirectory per model name with samples_*.jsonl files. The summary is
|
||||||
|
written to <results-dir>/efficiency-summary.json.
|
||||||
|
"""
|
||||||
|
import glob
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import statistics
|
||||||
|
import sys
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
ENGINE_URL = os.environ.get("ENGINE_URL", "http://llama-cpp:8080")
|
||||||
|
TOKENIZE_TIMEOUT = 300
|
||||||
|
|
||||||
|
|
||||||
|
def tokenize(text: str) -> int:
|
||||||
|
payload = json.dumps({"content": text}).encode()
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"{ENGINE_URL}/tokenize",
|
||||||
|
data=payload,
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(req, timeout=TOKENIZE_TIMEOUT) as resp:
|
||||||
|
data = json.load(resp)
|
||||||
|
return len(data.get("tokens", []))
|
||||||
|
|
||||||
|
|
||||||
|
def response_text(rec: dict) -> str:
|
||||||
|
resps = rec.get("resps")
|
||||||
|
if not resps:
|
||||||
|
return ""
|
||||||
|
first = resps[0]
|
||||||
|
if isinstance(first, list):
|
||||||
|
return first[0] if first else ""
|
||||||
|
return first
|
||||||
|
|
||||||
|
|
||||||
|
def sample_score(rec: dict):
|
||||||
|
# exact_match based tasks (aime, math variants)
|
||||||
|
if isinstance(rec.get("exact_match"), (int, float, bool)):
|
||||||
|
return float(rec["exact_match"])
|
||||||
|
metrics = rec.get("metrics")
|
||||||
|
if isinstance(metrics, dict):
|
||||||
|
for key in ("prompt_level_strict_acc", "exact_match", "acc"):
|
||||||
|
if key in metrics and isinstance(metrics[key], (int, float, bool)):
|
||||||
|
return float(metrics[key])
|
||||||
|
if isinstance(rec.get("acc"), (int, float, bool)):
|
||||||
|
return float(rec["acc"])
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
if len(sys.argv) != 2:
|
||||||
|
print("usage: eval-efficiency.py <results-dir>", file=sys.stderr)
|
||||||
|
sys.exit(2)
|
||||||
|
results_dir = sys.argv[1]
|
||||||
|
summary = {}
|
||||||
|
for path in sorted(glob.glob(os.path.join(results_dir, "*", "samples_*.jsonl"))):
|
||||||
|
bench = os.path.basename(path).removeprefix("samples_")
|
||||||
|
bench = bench.rsplit("_2026", 1)[0]
|
||||||
|
per_sample = []
|
||||||
|
with open(path, encoding="utf-8") as fh:
|
||||||
|
for line in fh:
|
||||||
|
rec = json.loads(line)
|
||||||
|
text = response_text(rec)
|
||||||
|
per_sample.append(
|
||||||
|
{
|
||||||
|
"doc_id": rec.get("doc_id"),
|
||||||
|
"generated_tokens": tokenize(text),
|
||||||
|
"score": sample_score(rec),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
scores = [s["score"] for s in per_sample if s["score"] is not None]
|
||||||
|
toks = [s["generated_tokens"] for s in per_sample]
|
||||||
|
med = statistics.median(toks) if toks else 0
|
||||||
|
mean = statistics.mean(toks) if toks else 0
|
||||||
|
score = statistics.mean(scores) if scores else None
|
||||||
|
summary[bench] = {
|
||||||
|
"samples": len(per_sample),
|
||||||
|
"score": score,
|
||||||
|
"generated_tokens_median": med,
|
||||||
|
"generated_tokens_mean": round(mean, 1),
|
||||||
|
"generated_tokens_max": max(toks) if toks else 0,
|
||||||
|
"efficiency_score_per_median_token": (
|
||||||
|
round(score / med, 8) if med and score is not None else None
|
||||||
|
),
|
||||||
|
"per_sample": per_sample,
|
||||||
|
}
|
||||||
|
out_path = os.path.join(results_dir, "efficiency-summary.json")
|
||||||
|
with open(out_path, "w", encoding="utf-8") as fh:
|
||||||
|
json.dump(summary, fh, indent=2, ensure_ascii=False)
|
||||||
|
report = {
|
||||||
|
k: {kk: vv for kk, vv in v.items() if kk != "per_sample"}
|
||||||
|
for k, v in summary.items()
|
||||||
|
}
|
||||||
|
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||||||
|
print(f"summary written to {out_path}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Efficiency evaluation wrapper (GNAI-009 5.2).
|
||||||
|
# Usage: eval-efficiency.sh <results-dir>
|
||||||
|
# Runs the token-counting evaluation inside the pinned bench container
|
||||||
|
# against the running engine (tokenizer-only, no generation).
|
||||||
|
set -euo pipefail
|
||||||
|
RESULTS=$1
|
||||||
|
|
||||||
|
/usr/sbin/runuser --user {{ benchmark_user }} -- \
|
||||||
|
/usr/bin/env HOME={{ benchmark_user_home }} XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }} \
|
||||||
|
/usr/bin/systemctl --user start {{ benchmark_bench_network_name }}-network.service
|
||||||
|
|
||||||
|
exec /usr/sbin/runuser --user {{ benchmark_user }} -- \
|
||||||
|
/usr/bin/env HOME={{ benchmark_user_home }} XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }} \
|
||||||
|
/usr/bin/podman run --rm \
|
||||||
|
--name gnai-bench-eval \
|
||||||
|
--network {{ benchmark_engine_network_name }} \
|
||||||
|
--network {{ benchmark_bench_network_name }} \
|
||||||
|
-v {{ benchmark_base_dir }}/eval-efficiency.py:/work/eval-efficiency.py:ro \
|
||||||
|
-v "$RESULTS":/results:Z \
|
||||||
|
--entrypoint python3 \
|
||||||
|
{{ benchmark_image_tag }} /work/eval-efficiency.py /results
|
||||||
Reference in New Issue
Block a user