diff --git a/llm-server/llm/ansible/roles/benchmark/defaults/main.yml b/llm-server/llm/ansible/roles/benchmark/defaults/main.yml index d912d57..dd31ec8 100644 --- a/llm-server/llm/ansible/roles/benchmark/defaults/main.yml +++ b/llm-server/llm/ansible/roles/benchmark/defaults/main.yml @@ -38,3 +38,14 @@ benchmark_task_args: max_retries: 1 tokenized_requests: false timeout: 2700 + +# BFCL (Berkeley Function Calling Leaderboard, GNAI-009 5.1 replacement for +# GPQA, operator decision 03.11.2026). Separate image because bfcl-eval +# targets Python 3.10 while the lm-eval image is 3.13-based. +benchmark_bfcl_python_image: docker.io/library/python:3.10-slim@sha256:6ff506466f8b1e981719b468cc0023b8ef000b983cdec0eafe119d01f379e31d +benchmark_bfcl_version: "2026.3.23" +benchmark_bfcl_image_tag: "gnai-bench:bfcl-2026.3.23" +benchmark_bfcl_force_rebuild: false +# Core function-calling categories; irrelevance checks that the model does +# NOT call tools on unsuitable requests (GNAI-009 5.1). +benchmark_bfcl_categories: simple,multiple,irrelevance diff --git a/llm-server/llm/ansible/roles/benchmark/tasks/main.yml b/llm-server/llm/ansible/roles/benchmark/tasks/main.yml index 4cb0d38..d7c931a 100644 --- a/llm-server/llm/ansible/roles/benchmark/tasks/main.yml +++ b/llm-server/llm/ansible/roles/benchmark/tasks/main.yml @@ -147,6 +147,74 @@ tags: - bench +- name: Render the BFCL Containerfile + ansible.builtin.template: + src: bfcl.Containerfile.j2 + dest: "{{ benchmark_base_dir }}/bfcl.Containerfile" + owner: "{{ benchmark_user }}" + group: "{{ benchmark_user }}" + mode: "0640" + tags: + - bench + +- name: Render the BFCL runner script + ansible.builtin.template: + src: run-bfcl.sh.j2 + dest: "{{ benchmark_base_dir }}/run-bfcl.sh" + owner: "{{ benchmark_user }}" + group: "{{ benchmark_user }}" + mode: "0750" + tags: + - bench + +- name: Check whether the pinned BFCL image is already built + ansible.builtin.command: + argv: + - /usr/sbin/runuser + - --user + - "{{ benchmark_user }}" + - -- + - /usr/bin/env + - "HOME={{ benchmark_user_home }}" + - "XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }}" + - /usr/bin/podman + - image + - exists + - "{{ benchmark_bfcl_image_tag }}" + chdir: "{{ benchmark_user_home }}" + changed_when: false + failed_when: benchmark_bfcl_image_present.rc not in [0, 1] + check_mode: false + register: benchmark_bfcl_image_present + tags: + - bench + +- name: Build the pinned BFCL image + # Built directly on the bench dir with the bfcl Containerfile; needs host + # egress for the pip install. + ansible.builtin.command: + argv: + - /usr/sbin/runuser + - --user + - "{{ benchmark_user }}" + - -- + - /usr/bin/env + - "HOME={{ benchmark_user_home }}" + - "XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }}" + - /usr/bin/podman + - build + - -f + - "{{ benchmark_base_dir }}/bfcl.Containerfile" + - -t + - "{{ benchmark_bfcl_image_tag }}" + - "{{ benchmark_base_dir }}" + chdir: "{{ benchmark_user_home }}" + when: + - not ansible_check_mode + - benchmark_bfcl_force_rebuild or benchmark_bfcl_image_present.rc != 0 + tags: + - bench + - name: Build the pinned bench image ansible.builtin.command: argv: diff --git a/llm-server/llm/ansible/roles/benchmark/templates/bfcl.Containerfile.j2 b/llm-server/llm/ansible/roles/benchmark/templates/bfcl.Containerfile.j2 new file mode 100644 index 0000000..91c6e18 --- /dev/null +++ b/llm-server/llm/ansible/roles/benchmark/templates/bfcl.Containerfile.j2 @@ -0,0 +1,13 @@ +# BFCL benchmark image (GNAI-009 5.1, replaces GPQA, operator decision +# 03.11.2026). bfcl-eval targets Python 3.10; the lm-eval image stays on 3.13. +FROM docker.io/library/python:3.10-slim@sha256:6ff506466f8b1e981719b468cc0023b8ef000b983cdec0eafe119d01f379e31d + +ENV PIP_NO_CACHE_DIR=1 \ + PYTHONUNBUFFERED=1 \ + REMOTE_OPENAI_BASE_URL=http://llama-cpp:8080/v1 \ + REMOTE_OPENAI_API_KEY=no-key-set + +RUN pip install --no-cache-dir bfcl-eval=={{ benchmark_bfcl_version }} + +WORKDIR /work +ENTRYPOINT ["bfcl"] diff --git a/llm-server/llm/ansible/roles/benchmark/templates/run-bfcl.sh.j2 b/llm-server/llm/ansible/roles/benchmark/templates/run-bfcl.sh.j2 new file mode 100644 index 0000000..ccfe45c --- /dev/null +++ b/llm-server/llm/ansible/roles/benchmark/templates/run-bfcl.sh.j2 @@ -0,0 +1,33 @@ +#!/bin/bash +# BFCL runner (GNAI-009 5.1). Usage: run-bfcl.sh [] +# profile-label: result dir label, e.g. swift-1.5-qwen3.8-27b-q6k +# model-handler: BFCL model handler name (an OpenAI-API-style handler; +# the engine ignores the model name in requests) +# categories: comma-separated BFCL categories (default: {{ benchmark_bfcl_categories }}) +# +# Runs against the running engine via REMOTE_OPENAI_BASE_URL (OpenAI- +# compatible chat completions with native tool calls). +set -euo pipefail +PROFILE=$1; shift +MODEL=$1; shift +CATEGORIES=${1:-{{ benchmark_bfcl_categories }}} +TS=$(date +%Y-%m-%d-%H%M) +OUT=/results/bfcl-"$PROFILE"-"$TS" + +/usr/sbin/runuser --user {{ benchmark_user }} -- \ + /usr/bin/env HOME={{ benchmark_user_home }} XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }} \ + /usr/bin/systemctl --user start {{ benchmark_bench_network_name }}-network.service + +exec /usr/sbin/runuser --user {{ benchmark_user }} -- \ + /usr/bin/env HOME={{ benchmark_user_home }} XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }} \ + /usr/bin/podman run --rm \ + --name gnai-bfcl-run \ + --network {{ benchmark_engine_network_name }} \ + --network {{ benchmark_bench_network_name }} \ + -v {{ benchmark_results_dir }}:/results:Z \ + {{ benchmark_bfcl_image_tag }} \ + evaluate \ + --model "$MODEL" \ + --test-category "$CATEGORIES" \ + --include-input-token \ + --result-path "$OUT"