LLM-Server: BFCL-Harness aufgesetzt (GNAI-009 5.1, GPQA-Ersatz)

Separates gepinntes Image gnai-bench:bfcl-2026.3.23 (python:3.10-slim per
Digest, bfcl-eval 2026.3.23 - Python 3.10 wegen BFCL-Zielumgebung), Runner
run-bfcl.sh mit Remote-Modus gegen die OpenAI-kompatible Engine-API
(REMOTE_OPENAI_BASE_URL=http://llama-cpp:8080/v1). Kategorien: simple,
multiple, irrelevance (Irrelevance-Detection: Modell soll bei unpassenden
Anfragen NICHT rufen). Smoke und Serienlauf bewusst NACH Abschluss der
laufenden IFEval/MATH-Serie - Engine-Kontakt waehrend eines Benchmark-Laufs
wuerde die Messung kontaminieren.

Generated by Mistral Vibe.
Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
2026-11-03 06:17:05 +01:00
parent f1dcc8f6a9
commit 0c6e302e7b
4 changed files with 125 additions and 0 deletions
@@ -38,3 +38,14 @@ benchmark_task_args:
max_retries: 1
tokenized_requests: false
timeout: 2700
# BFCL (Berkeley Function Calling Leaderboard, GNAI-009 5.1 replacement for
# GPQA, operator decision 03.11.2026). Separate image because bfcl-eval
# targets Python 3.10 while the lm-eval image is 3.13-based.
benchmark_bfcl_python_image: docker.io/library/python:3.10-slim@sha256:6ff506466f8b1e981719b468cc0023b8ef000b983cdec0eafe119d01f379e31d
benchmark_bfcl_version: "2026.3.23"
benchmark_bfcl_image_tag: "gnai-bench:bfcl-2026.3.23"
benchmark_bfcl_force_rebuild: false
# Core function-calling categories; irrelevance checks that the model does
# NOT call tools on unsuitable requests (GNAI-009 5.1).
benchmark_bfcl_categories: simple,multiple,irrelevance
@@ -147,6 +147,74 @@
tags:
- bench
- name: Render the BFCL Containerfile
ansible.builtin.template:
src: bfcl.Containerfile.j2
dest: "{{ benchmark_base_dir }}/bfcl.Containerfile"
owner: "{{ benchmark_user }}"
group: "{{ benchmark_user }}"
mode: "0640"
tags:
- bench
- name: Render the BFCL runner script
ansible.builtin.template:
src: run-bfcl.sh.j2
dest: "{{ benchmark_base_dir }}/run-bfcl.sh"
owner: "{{ benchmark_user }}"
group: "{{ benchmark_user }}"
mode: "0750"
tags:
- bench
- name: Check whether the pinned BFCL image is already built
ansible.builtin.command:
argv:
- /usr/sbin/runuser
- --user
- "{{ benchmark_user }}"
- --
- /usr/bin/env
- "HOME={{ benchmark_user_home }}"
- "XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }}"
- /usr/bin/podman
- image
- exists
- "{{ benchmark_bfcl_image_tag }}"
chdir: "{{ benchmark_user_home }}"
changed_when: false
failed_when: benchmark_bfcl_image_present.rc not in [0, 1]
check_mode: false
register: benchmark_bfcl_image_present
tags:
- bench
- name: Build the pinned BFCL image
# Built directly on the bench dir with the bfcl Containerfile; needs host
# egress for the pip install.
ansible.builtin.command:
argv:
- /usr/sbin/runuser
- --user
- "{{ benchmark_user }}"
- --
- /usr/bin/env
- "HOME={{ benchmark_user_home }}"
- "XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }}"
- /usr/bin/podman
- build
- -f
- "{{ benchmark_base_dir }}/bfcl.Containerfile"
- -t
- "{{ benchmark_bfcl_image_tag }}"
- "{{ benchmark_base_dir }}"
chdir: "{{ benchmark_user_home }}"
when:
- not ansible_check_mode
- benchmark_bfcl_force_rebuild or benchmark_bfcl_image_present.rc != 0
tags:
- bench
- name: Build the pinned bench image
ansible.builtin.command:
argv:
@@ -0,0 +1,13 @@
# BFCL benchmark image (GNAI-009 5.1, replaces GPQA, operator decision
# 03.11.2026). bfcl-eval targets Python 3.10; the lm-eval image stays on 3.13.
FROM docker.io/library/python:3.10-slim@sha256:6ff506466f8b1e981719b468cc0023b8ef000b983cdec0eafe119d01f379e31d
ENV PIP_NO_CACHE_DIR=1 \
PYTHONUNBUFFERED=1 \
REMOTE_OPENAI_BASE_URL=http://llama-cpp:8080/v1 \
REMOTE_OPENAI_API_KEY=no-key-set
RUN pip install --no-cache-dir bfcl-eval=={{ benchmark_bfcl_version }}
WORKDIR /work
ENTRYPOINT ["bfcl"]
@@ -0,0 +1,33 @@
#!/bin/bash
# BFCL runner (GNAI-009 5.1). Usage: run-bfcl.sh <profile-label> <model-handler> [<categories>]
# profile-label: result dir label, e.g. swift-1.5-qwen3.8-27b-q6k
# model-handler: BFCL model handler name (an OpenAI-API-style handler;
# the engine ignores the model name in requests)
# categories: comma-separated BFCL categories (default: {{ benchmark_bfcl_categories }})
#
# Runs against the running engine via REMOTE_OPENAI_BASE_URL (OpenAI-
# compatible chat completions with native tool calls).
set -euo pipefail
PROFILE=$1; shift
MODEL=$1; shift
CATEGORIES=${1:-{{ benchmark_bfcl_categories }}}
TS=$(date +%Y-%m-%d-%H%M)
OUT=/results/bfcl-"$PROFILE"-"$TS"
/usr/sbin/runuser --user {{ benchmark_user }} -- \
/usr/bin/env HOME={{ benchmark_user_home }} XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }} \
/usr/bin/systemctl --user start {{ benchmark_bench_network_name }}-network.service
exec /usr/sbin/runuser --user {{ benchmark_user }} -- \
/usr/bin/env HOME={{ benchmark_user_home }} XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }} \
/usr/bin/podman run --rm \
--name gnai-bfcl-run \
--network {{ benchmark_engine_network_name }} \
--network {{ benchmark_bench_network_name }} \
-v {{ benchmark_results_dir }}:/results:Z \
{{ benchmark_bfcl_image_tag }} \
evaluate \
--model "$MODEL" \
--test-category "$CATEGORIES" \
--include-input-token \
--result-path "$OUT"