From c6cabd98ef865f6820f2e3dfd2d70cd5a17305d4 Mon Sep 17 00:00:00 2001 From: spektr Date: Mon, 2 Nov 2026 05:58:58 +0100 Subject: [PATCH] LLM-Server: Benchmark-Harness (lm-eval) aufgesetzt und per Smoke-Test verifiziert Neue Rolle benchmark (GNAI-009 5.1/5.2): Egress-Netzwerk-Quadlet gnai-testing-bench (dokumentierte Ausnahme nur fuer Datensatz-Downloads), persistentes results/- und HF-Cache-Verzeichnis, gepinntes Image gnai-bench:lm-eval-0.4.13 (python:3.13-slim per Digest, lm-eval[api]-Pin, bewusst ohne Torch), Runner-Skript mit Network-Unit-Absicherung. Live-Funde aus dem Smoke-Test (2 AIME-Samples gegen Swift 1.5): - lokale Chat-Completions brauchen --apply_chat_template (Server wendet sein --jinja-Template an, kein lokaler Tokenizer noetig) - Request-Timeout 1800 s statt lm-eval-Default 300 s (xhigh-Thinking bei ~18 t/s) - Quadlet-Netzwerke materialisieren erst mit gestarteter Unit; Rolle und Runner starten sie explizit - Ergebnis: exact_match 1.0, Per-Sample-Logs in results/; Tokenzaehlung erfolgt nachtraglich exakt ueber die /tokenize-Endpunkte der Engine Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../testing/group_vars/llm_servers.yml | 6 + llm-server/llm/ansible/playbooks/bench.yml | 5 + .../llm/ansible/roles/benchmark/README.md | 55 +++++++ .../ansible/roles/benchmark/defaults/main.yml | 36 +++++ .../ansible/roles/benchmark/tasks/main.yml | 151 ++++++++++++++++++ .../templates/bench.Containerfile.j2 | 12 ++ .../benchmark/templates/bench.network.j2 | 6 + .../roles/benchmark/templates/run-bench.sh.j2 | 40 +++++ 8 files changed, 311 insertions(+) create mode 100644 llm-server/llm/ansible/playbooks/bench.yml create mode 100644 llm-server/llm/ansible/roles/benchmark/README.md create mode 100644 llm-server/llm/ansible/roles/benchmark/defaults/main.yml create mode 100644 llm-server/llm/ansible/roles/benchmark/tasks/main.yml create mode 100644 llm-server/llm/ansible/roles/benchmark/templates/bench.Containerfile.j2 create mode 100644 llm-server/llm/ansible/roles/benchmark/templates/bench.network.j2 create mode 100644 llm-server/llm/ansible/roles/benchmark/templates/run-bench.sh.j2 diff --git a/llm-server/llm/ansible/inventories/testing/group_vars/llm_servers.yml b/llm-server/llm/ansible/inventories/testing/group_vars/llm_servers.yml index b08a8ac..af9702c 100644 --- a/llm-server/llm/ansible/inventories/testing/group_vars/llm_servers.yml +++ b/llm-server/llm/ansible/inventories/testing/group_vars/llm_servers.yml @@ -13,3 +13,9 @@ inference_engine_network_name: gnai-testing-llama # accepted separately (GNAI-010 section 10.3). 64k bring-ups passed for all # three profiles (10.5, 10.8, 10.9); null runs the 128k target context. inference_engine_ctx_size_override: null + +# Benchmark harness (GNAI-009 5.1); reuses the workload identity. +benchmark_user: gnai-testing +benchmark_uid: 981 +benchmark_user_home: /var/lib/ghostnet-ai/testing +benchmark_engine_network_name: gnai-testing-llama diff --git a/llm-server/llm/ansible/playbooks/bench.yml b/llm-server/llm/ansible/playbooks/bench.yml new file mode 100644 index 0000000..f395823 --- /dev/null +++ b/llm-server/llm/ansible/playbooks/bench.yml @@ -0,0 +1,5 @@ +- name: Prepare the benchmark harness of one environment + hosts: llm_servers + become: true + roles: + - role: benchmark diff --git a/llm-server/llm/ansible/roles/benchmark/README.md b/llm-server/llm/ansible/roles/benchmark/README.md new file mode 100644 index 0000000..c55f73b --- /dev/null +++ b/llm-server/llm/ansible/roles/benchmark/README.md @@ -0,0 +1,55 @@ +# benchmark + +Prepares the lm-evaluation-harness benchmark environment for one KI +environment (GNAI-009 sections 5.1/5.2). Runs in the workload account's +rootless store so the bench container can attach to the internal engine +network. + +## What it creates + +- `gnai-testing-bench.network` quadlet: egress network, deliberately not + `Internal=true`. Its ONLY purpose is benchmark dataset downloads by the + pinned bench container (documented exception, GNAI-009 5.1); the engine + network stays internal. +- `/var/lib/ghostnet-ai/testing/bench/` with `results/` and `hf/` (persistent + HF dataset cache) owned by the workload account. +- `Containerfile` + locally built, pinned image + `gnai-bench:lm-eval-0.4.13` (python:3.13-slim by digest; the build needs + host egress and takes several minutes the first time because of the + lm-eval dependency chain). +- `run-bench.sh` runner. + +## Usage + +```bash +# Prepare / converge the harness: +../../../.venv/bin/ansible-playbook -i inventories/testing playbooks/bench.yml \ + --private-key ~/.ssh/id_ed25519_ghostnet_ai_ansible_deploy + +# Smoke test (2 samples of AIME 2024): +cd /var/lib/ghostnet-ai/testing/bench && sudo ./run-bench.sh smoke-profile 2 aime24 + +# Full run against the currently active engine profile: +cd /var/lib/ghostnet-ai/testing/bench && sudo ./run-bench.sh swift-1.5-qwen3.8-27b-q6k "" aime24 ifeval +``` + +The runner attaches the container to both the internal engine network +(`http://llama-cpp:8080`) and the bench network, writes lm-eval results +with `--log_samples` under `results/-/` and reuses the +persistent HF cache. + +## Token accounting + +llama-server reports `usage.completion_tokens` including reasoning tokens +(no separate `reasoning_tokens` field; verified 02.11.2026). Completion +tokens are therefore the exact generated-token count per sample and form +the basis of the thinking-efficiency metric (score per median generated +tokens, GNAI-009 5.2). + +## Notes + +- The engine profile under test is whatever is currently active; switch + profiles with the `inference_engine` role first (`-e + inference_engine_active_profile=...`). +- The server-side sampling/reasoning configuration comes from the profile + quadlet; the harness deliberately sends no generation overrides. diff --git a/llm-server/llm/ansible/roles/benchmark/defaults/main.yml b/llm-server/llm/ansible/roles/benchmark/defaults/main.yml new file mode 100644 index 0000000..f88ec5f --- /dev/null +++ b/llm-server/llm/ansible/roles/benchmark/defaults/main.yml @@ -0,0 +1,36 @@ +--- +# Benchmark harness for the LLM evaluation (GNAI-009 sections 5.1/5.2). +# Runs the pinned lm-evaluation-harness against the running engine over the +# internal podman network; a dedicated egress network exists ONLY for the +# benchmark dataset downloads (documented exception, GNAI-009 5.1). +# +# The harness reuses the environment identity from the group_vars +# (inference_engine_user/uid/home/network) and runs in the workload +# account's rootless store so it can attach to the engine network. +benchmark_python_image: docker.io/library/python:3.13-slim@sha256:2d44738ebf4ce5f63b9d9ea871e9c7cfdc0426f7b0c97cac915fd88ded1d86bb +benchmark_lm_eval_version: "0.4.13" +benchmark_image_tag: "gnai-bench:lm-eval-0.4.13" +benchmark_image_force_rebuild: false + +benchmark_bench_network_name: gnai-testing-bench +benchmark_quadlet_dir: /etc/containers/systemd/users/981 +benchmark_base_dir: /var/lib/ghostnet-ai/testing/bench +benchmark_results_dir: "{{ benchmark_base_dir }}/results" +benchmark_hf_cache_dir: "{{ benchmark_base_dir }}/hf" + +# Engine endpoint inside the shared podman network. +benchmark_engine_url: http://llama-cpp:8080/v1 + +# lm-eval sample/token accounting: llama-server reports completion_tokens +# including reasoning (no separate reasoning_tokens field, verified +# 02.11.2026). completion_tokens is therefore the exact per-task generated +# token count and the basis of the thinking-efficiency metric +# (GNAI-009 section 5.2). +benchmark_task_args: + model: dummy-model-name + base_url: "{{ benchmark_engine_url }}/chat/completions" + num_concurrent: 1 + max_retries: 2 + tokenized_requests: false + # xhigh thinking on this hardware needs well over the 300 s default. + timeout: 1800 diff --git a/llm-server/llm/ansible/roles/benchmark/tasks/main.yml b/llm-server/llm/ansible/roles/benchmark/tasks/main.yml new file mode 100644 index 0000000..1170cbc --- /dev/null +++ b/llm-server/llm/ansible/roles/benchmark/tasks/main.yml @@ -0,0 +1,151 @@ +--- +# Prepares the benchmark harness (GNAI-009 5.1/5.2): egress network quadlet, +# persistent result/cache directories, pinned lm-eval image and the runner +# script. Benchmark runs themselves are started manually via the script. +- name: Validate benchmark harness declarations + ansible.builtin.assert: + that: + - benchmark_user | length > 0 + - benchmark_uid | int > 0 + - benchmark_user_home is match("^/") + - benchmark_python_image is match("^docker\\.io/library/python(:[A-Za-z0-9._-]+)?@sha256:[0-9a-f]{64}$") + - benchmark_lm_eval_version is match("^[0-9]+\\.[0-9]+\\.[0-9]+$") + - benchmark_bench_network_name | length > 0 + - benchmark_engine_network_name | length > 0 + fail_msg: Benchmark harness declarations are incomplete or ambiguous. + tags: + - bench + +- name: Ensure the bench base and result directories exist + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ benchmark_user }}" + group: "{{ benchmark_user }}" + mode: "0750" + loop: + - "{{ benchmark_base_dir }}" + - "{{ benchmark_results_dir }}" + - "{{ benchmark_hf_cache_dir }}" + tags: + - bench + +- name: Render the benchmark egress network quadlet + # Deliberately NOT Internal=true: this network exists only for the + # benchmark dataset downloads (GNAI-009 5.1). The engine network stays + # internal; the bench container is the only consumer of this network. + ansible.builtin.template: + src: bench.network.j2 + dest: "{{ benchmark_quadlet_dir }}/{{ benchmark_bench_network_name }}.network" + owner: root + group: root + mode: "0644" + tags: + - bench + +- name: Render the benchmark Containerfile + ansible.builtin.template: + src: bench.Containerfile.j2 + dest: "{{ benchmark_base_dir }}/Containerfile" + owner: "{{ benchmark_user }}" + group: "{{ benchmark_user }}" + mode: "0640" + tags: + - bench + +- name: Render the benchmark runner script + ansible.builtin.template: + src: run-bench.sh.j2 + dest: "{{ benchmark_base_dir }}/run-bench.sh" + owner: "{{ benchmark_user }}" + group: "{{ benchmark_user }}" + mode: "0750" + tags: + - bench + +- name: Reload the workload user manager for the bench network + ansible.builtin.command: + argv: + - /usr/sbin/runuser + - --user + - "{{ benchmark_user }}" + - -- + - /usr/bin/env + - "HOME={{ benchmark_user_home }}" + - "XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }}" + - /usr/bin/systemctl + - --user + - daemon-reload + chdir: "{{ benchmark_user_home }}" + changed_when: false + when: not ansible_check_mode + tags: + - bench + +- name: Start the bench network unit + # Quadlet networks only materialize when their generated unit runs; the + # bench container is started with plain `podman run` and has no systemd + # dependency, so the unit must be started explicitly. + ansible.builtin.command: + argv: + - /usr/sbin/runuser + - --user + - "{{ benchmark_user }}" + - -- + - /usr/bin/env + - "HOME={{ benchmark_user_home }}" + - "XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }}" + - /usr/bin/systemctl + - --user + - start + - "{{ benchmark_bench_network_name }}-network.service" + chdir: "{{ benchmark_user_home }}" + changed_when: false + when: not ansible_check_mode + tags: + - bench + +- name: Check whether the pinned bench image is already built + ansible.builtin.command: + argv: + - /usr/sbin/runuser + - --user + - "{{ benchmark_user }}" + - -- + - /usr/bin/env + - "HOME={{ benchmark_user_home }}" + - "XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }}" + - /usr/bin/podman + - image + - exists + - "{{ benchmark_image_tag }}" + chdir: "{{ benchmark_user_home }}" + changed_when: false + failed_when: benchmark_image_present.rc not in [0, 1] + check_mode: false + register: benchmark_image_present + tags: + - bench + +- name: Build the pinned bench image + ansible.builtin.command: + argv: + - /usr/sbin/runuser + - --user + - "{{ benchmark_user }}" + - -- + - /usr/bin/env + - "HOME={{ benchmark_user_home }}" + - "XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }}" + - /usr/bin/podman + - build + - -t + - "{{ benchmark_image_tag }}" + - "{{ benchmark_base_dir }}" + chdir: "{{ benchmark_user_home }}" + when: + - not ansible_check_mode + - benchmark_image_force_rebuild or benchmark_image_present.rc != 0 + register: benchmark_image_build + tags: + - bench diff --git a/llm-server/llm/ansible/roles/benchmark/templates/bench.Containerfile.j2 b/llm-server/llm/ansible/roles/benchmark/templates/bench.Containerfile.j2 new file mode 100644 index 0000000..f623ca8 --- /dev/null +++ b/llm-server/llm/ansible/roles/benchmark/templates/bench.Containerfile.j2 @@ -0,0 +1,12 @@ +# Benchmark image for the KI evaluation harness (GNAI-009 5.1). +# Base image and lm-eval version are pinned in the benchmark role defaults. +FROM docker.io/library/python:3.13-slim@sha256:2d44738ebf4ce5f63b9d9ea871e9c7cfdc0426f7b0c97cac915fd88ded1d86bb + +ENV PIP_NO_CACHE_DIR=1 \ + HF_HOME=/hf \ + PYTHONUNBUFFERED=1 + +RUN pip install --no-cache-dir "lm-eval[api]=={{ benchmark_lm_eval_version }}" + +WORKDIR /work +ENTRYPOINT ["lm_eval"] diff --git a/llm-server/llm/ansible/roles/benchmark/templates/bench.network.j2 b/llm-server/llm/ansible/roles/benchmark/templates/bench.network.j2 new file mode 100644 index 0000000..bc48485 --- /dev/null +++ b/llm-server/llm/ansible/roles/benchmark/templates/bench.network.j2 @@ -0,0 +1,6 @@ +[Network] +# Benchmark egress network (GNAI-009 section 5.1): exists ONLY for benchmark +# dataset downloads by the pinned lm-eval container. Deliberately not +# Internal. The engine network remains internal. +NetworkName={{ benchmark_bench_network_name }} +Driver=bridge diff --git a/llm-server/llm/ansible/roles/benchmark/templates/run-bench.sh.j2 b/llm-server/llm/ansible/roles/benchmark/templates/run-bench.sh.j2 new file mode 100644 index 0000000..215baeb --- /dev/null +++ b/llm-server/llm/ansible/roles/benchmark/templates/run-bench.sh.j2 @@ -0,0 +1,40 @@ +#!/bin/bash +# Benchmark runner for one engine profile (GNAI-009 5.1/5.2). +# Usage: run-bench.sh [ ...] +# profile-label: e.g. swift-1.5-qwen3.8-27b-q6k (used for the result dir) +# limit: max samples per task (e.g. 2 for a smoke test, empty for all) +# +# Runs lm-eval inside the pinned bench image, attached to BOTH the internal +# engine network (to reach http://llama-cpp:8080) and the bench network +# (egress, only for benchmark dataset downloads). Results and the HF +# dataset cache persist under the workload home. +set -euo pipefail +PROFILE=$1; shift +LIMIT=$1; shift +TS=$(date +%Y-%m-%d-%H%M) +OUT=/results/"$PROFILE"-"$TS" + +# Ensure the bench network unit is up (quadlet networks materialize only +# when their unit runs; this container has no systemd dependency). +/usr/sbin/runuser --user {{ benchmark_user }} -- \ + /usr/bin/env HOME={{ benchmark_user_home }} XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }} \ + /usr/bin/systemctl --user start {{ benchmark_bench_network_name }}-network.service + +ARGS=(--model local-chat-completions + --model_args "model={{ benchmark_task_args.model }},base_url={{ benchmark_task_args.base_url }},num_concurrent={{ benchmark_task_args.num_concurrent }},max_retries={{ benchmark_task_args.max_retries }},tokenized_requests={{ benchmark_task_args.tokenized_requests }},timeout={{ benchmark_task_args.timeout }}" + --apply_chat_template + --tasks "$*" + --log_samples + --output_path "$OUT") +if [ -n "$LIMIT" ]; then + ARGS+=(--limit "$LIMIT") +fi +exec /usr/sbin/runuser --user {{ benchmark_user }} -- \ + /usr/bin/env HOME={{ benchmark_user_home }} XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }} \ + /usr/bin/podman run --rm \ + --name gnai-bench-run \ + --network {{ benchmark_engine_network_name }} \ + --network {{ benchmark_bench_network_name }} \ + -v {{ benchmark_results_dir }}:/results:Z \ + -v {{ benchmark_hf_cache_dir }}:/hf:Z \ + {{ benchmark_image_tag }} "${ARGS[@]}"