From 89e42ecd9bcdb2ce636dae2bef2bc56255955b67 Mon Sep 17 00:00:00 2001 From: spektr Date: Mon, 2 Nov 2026 06:44:44 +0100 Subject: [PATCH] LLM-Server: Effizienz-Auswertung fuer Benchmark-Ergebnisse (GNAI-009 5.2) eval-efficiency.py/.sh in der benchmark-Rolle: liest die Per-Sample-Logs von lm-eval, zaehlt die exakt generierten Tokens je Antwort ueber den /tokenize-Endpunkt der Engine (Qwen-Tokenizer, keine Schaetzung), zieht den Score und aggregiert Score, Median/Mean/Max-Tokens sowie die Effizienzkennzahl Score je Median-Token. Smoke-Test auf den AIME-2-Ergebnissen: Score 1.0, Median 645 generierte Tokens je Aufgabe. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../ansible/roles/benchmark/tasks/main.yml | 20 ++++ .../benchmark/templates/eval-efficiency.py.j2 | 113 ++++++++++++++++++ .../benchmark/templates/eval-efficiency.sh.j2 | 22 ++++ 3 files changed, 155 insertions(+) create mode 100644 llm-server/llm/ansible/roles/benchmark/templates/eval-efficiency.py.j2 create mode 100644 llm-server/llm/ansible/roles/benchmark/templates/eval-efficiency.sh.j2 diff --git a/llm-server/llm/ansible/roles/benchmark/tasks/main.yml b/llm-server/llm/ansible/roles/benchmark/tasks/main.yml index 1170cbc..4cb0d38 100644 --- a/llm-server/llm/ansible/roles/benchmark/tasks/main.yml +++ b/llm-server/llm/ansible/roles/benchmark/tasks/main.yml @@ -63,6 +63,26 @@ tags: - bench +- name: Render the efficiency evaluation script + ansible.builtin.template: + src: eval-efficiency.py.j2 + dest: "{{ benchmark_base_dir }}/eval-efficiency.py" + owner: "{{ benchmark_user }}" + group: "{{ benchmark_user }}" + mode: "0750" + tags: + - bench + +- name: Render the efficiency evaluation wrapper + ansible.builtin.template: + src: eval-efficiency.sh.j2 + dest: "{{ benchmark_base_dir }}/eval-efficiency.sh" + owner: "{{ benchmark_user }}" + group: "{{ benchmark_user }}" + mode: "0750" + tags: + - bench + - name: Reload the workload user manager for the bench network ansible.builtin.command: argv: diff --git a/llm-server/llm/ansible/roles/benchmark/templates/eval-efficiency.py.j2 b/llm-server/llm/ansible/roles/benchmark/templates/eval-efficiency.py.j2 new file mode 100644 index 0000000..99308ba --- /dev/null +++ b/llm-server/llm/ansible/roles/benchmark/templates/eval-efficiency.py.j2 @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Efficiency evaluation for lm-eval result directories (GNAI-009 5.2). + +Reads the per-sample lm-eval logs (samples_*.jsonl), counts the exact +generated tokens of every response via the engine's /tokenize endpoint +(the Qwen tokenizer, no approximation), extracts the score and aggregates: + + score, generated tokens (median/mean/max), + efficiency = score / median generated tokens + +Usage: eval-efficiency.py (e.g. /results/-) + +The results-dir is the lm-eval --output_path directory that contains one +subdirectory per model name with samples_*.jsonl files. The summary is +written to /efficiency-summary.json. +""" +import glob +import json +import os +import statistics +import sys +import urllib.request + +ENGINE_URL = os.environ.get("ENGINE_URL", "http://llama-cpp:8080") +TOKENIZE_TIMEOUT = 300 + + +def tokenize(text: str) -> int: + payload = json.dumps({"content": text}).encode() + req = urllib.request.Request( + f"{ENGINE_URL}/tokenize", + data=payload, + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=TOKENIZE_TIMEOUT) as resp: + data = json.load(resp) + return len(data.get("tokens", [])) + + +def response_text(rec: dict) -> str: + resps = rec.get("resps") + if not resps: + return "" + first = resps[0] + if isinstance(first, list): + return first[0] if first else "" + return first + + +def sample_score(rec: dict): + # exact_match based tasks (aime, math variants) + if isinstance(rec.get("exact_match"), (int, float, bool)): + return float(rec["exact_match"]) + metrics = rec.get("metrics") + if isinstance(metrics, dict): + for key in ("prompt_level_strict_acc", "exact_match", "acc"): + if key in metrics and isinstance(metrics[key], (int, float, bool)): + return float(metrics[key]) + if isinstance(rec.get("acc"), (int, float, bool)): + return float(rec["acc"]) + return None + + +def main() -> None: + if len(sys.argv) != 2: + print("usage: eval-efficiency.py ", file=sys.stderr) + sys.exit(2) + results_dir = sys.argv[1] + summary = {} + for path in sorted(glob.glob(os.path.join(results_dir, "*", "samples_*.jsonl"))): + bench = os.path.basename(path).removeprefix("samples_") + bench = bench.rsplit("_2026", 1)[0] + per_sample = [] + with open(path, encoding="utf-8") as fh: + for line in fh: + rec = json.loads(line) + text = response_text(rec) + per_sample.append( + { + "doc_id": rec.get("doc_id"), + "generated_tokens": tokenize(text), + "score": sample_score(rec), + } + ) + scores = [s["score"] for s in per_sample if s["score"] is not None] + toks = [s["generated_tokens"] for s in per_sample] + med = statistics.median(toks) if toks else 0 + mean = statistics.mean(toks) if toks else 0 + score = statistics.mean(scores) if scores else None + summary[bench] = { + "samples": len(per_sample), + "score": score, + "generated_tokens_median": med, + "generated_tokens_mean": round(mean, 1), + "generated_tokens_max": max(toks) if toks else 0, + "efficiency_score_per_median_token": ( + round(score / med, 8) if med and score is not None else None + ), + "per_sample": per_sample, + } + out_path = os.path.join(results_dir, "efficiency-summary.json") + with open(out_path, "w", encoding="utf-8") as fh: + json.dump(summary, fh, indent=2, ensure_ascii=False) + report = { + k: {kk: vv for kk, vv in v.items() if kk != "per_sample"} + for k, v in summary.items() + } + print(json.dumps(report, indent=2, ensure_ascii=False)) + print(f"summary written to {out_path}") + + +if __name__ == "__main__": + main() diff --git a/llm-server/llm/ansible/roles/benchmark/templates/eval-efficiency.sh.j2 b/llm-server/llm/ansible/roles/benchmark/templates/eval-efficiency.sh.j2 new file mode 100644 index 0000000..2c20b43 --- /dev/null +++ b/llm-server/llm/ansible/roles/benchmark/templates/eval-efficiency.sh.j2 @@ -0,0 +1,22 @@ +#!/bin/bash +# Efficiency evaluation wrapper (GNAI-009 5.2). +# Usage: eval-efficiency.sh +# Runs the token-counting evaluation inside the pinned bench container +# against the running engine (tokenizer-only, no generation). +set -euo pipefail +RESULTS=$1 + +/usr/sbin/runuser --user {{ benchmark_user }} -- \ + /usr/bin/env HOME={{ benchmark_user_home }} XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }} \ + /usr/bin/systemctl --user start {{ benchmark_bench_network_name }}-network.service + +exec /usr/sbin/runuser --user {{ benchmark_user }} -- \ + /usr/bin/env HOME={{ benchmark_user_home }} XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }} \ + /usr/bin/podman run --rm \ + --name gnai-bench-eval \ + --network {{ benchmark_engine_network_name }} \ + --network {{ benchmark_bench_network_name }} \ + -v {{ benchmark_base_dir }}/eval-efficiency.py:/work/eval-efficiency.py:ro \ + -v "$RESULTS":/results:Z \ + --entrypoint python3 \ + {{ benchmark_image_tag }} /work/eval-efficiency.py /results