diff --git a/llm-server/llm/ansible/roles/benchmark/defaults/main.yml b/llm-server/llm/ansible/roles/benchmark/defaults/main.yml index dd31ec8..d2e2355 100644 --- a/llm-server/llm/ansible/roles/benchmark/defaults/main.yml +++ b/llm-server/llm/ansible/roles/benchmark/defaults/main.yml @@ -46,6 +46,7 @@ benchmark_bfcl_python_image: docker.io/library/python:3.10-slim@sha256:6ff506466 benchmark_bfcl_version: "2026.3.23" benchmark_bfcl_image_tag: "gnai-bench:bfcl-2026.3.23" benchmark_bfcl_force_rebuild: false -# Core function-calling categories; irrelevance checks that the model does -# NOT call tools on unsuitable requests (GNAI-009 5.1). -benchmark_bfcl_categories: simple,multiple,irrelevance +# Core function-calling categories (BFCL 2026.3.23 naming: language-specific +# simple categories); irrelevance checks that the model does NOT call tools on +# unsuitable requests (GNAI-009 5.1). +benchmark_bfcl_categories: simple_python,multiple,irrelevance diff --git a/llm-server/llm/ansible/roles/benchmark/templates/bfcl.Containerfile.j2 b/llm-server/llm/ansible/roles/benchmark/templates/bfcl.Containerfile.j2 index 91c6e18..16de7a8 100644 --- a/llm-server/llm/ansible/roles/benchmark/templates/bfcl.Containerfile.j2 +++ b/llm-server/llm/ansible/roles/benchmark/templates/bfcl.Containerfile.j2 @@ -5,9 +5,12 @@ FROM docker.io/library/python:3.10-slim@sha256:6ff506466f8b1e981719b468cc0023b8e ENV PIP_NO_CACHE_DIR=1 \ PYTHONUNBUFFERED=1 \ REMOTE_OPENAI_BASE_URL=http://llama-cpp:8080/v1 \ - REMOTE_OPENAI_API_KEY=no-key-set + REMOTE_OPENAI_API_KEY=no-key-set \ + OPENAI_API_KEY=no-key-set \ + OPENAI_BASE_URL=http://llama-cpp:8080/v1 -RUN pip install --no-cache-dir bfcl-eval=={{ benchmark_bfcl_version }} +RUN pip install --no-cache-dir bfcl-eval=={{ benchmark_bfcl_version }} \ + && pip install --no-cache-dir soundfile WORKDIR /work ENTRYPOINT ["bfcl"] diff --git a/llm-server/llm/ansible/roles/benchmark/templates/run-bfcl.sh.j2 b/llm-server/llm/ansible/roles/benchmark/templates/run-bfcl.sh.j2 index ccfe45c..e422f54 100644 --- a/llm-server/llm/ansible/roles/benchmark/templates/run-bfcl.sh.j2 +++ b/llm-server/llm/ansible/roles/benchmark/templates/run-bfcl.sh.j2 @@ -20,14 +20,15 @@ OUT=/results/bfcl-"$PROFILE"-"$TS" exec /usr/sbin/runuser --user {{ benchmark_user }} -- \ /usr/bin/env HOME={{ benchmark_user_home }} XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }} \ - /usr/bin/podman run --rm \ + /usr/bin/podman run --rm --replace \ --name gnai-bfcl-run \ --network {{ benchmark_engine_network_name }} \ --network {{ benchmark_bench_network_name }} \ -v {{ benchmark_results_dir }}:/results:Z \ - {{ benchmark_bfcl_image_tag }} \ - evaluate \ - --model "$MODEL" \ - --test-category "$CATEGORIES" \ - --include-input-token \ - --result-path "$OUT" + -e BFCL_PROFILE="$PROFILE" \ + -e BFCL_MODEL="$MODEL" \ + -e BFCL_CATEGORIES="$CATEGORIES" \ + --entrypoint /bin/bash \ + {{ benchmark_bfcl_image_tag }} -c '\ + bfcl generate --model "$BFCL_MODEL" --test-category "$BFCL_CATEGORIES" --num-threads 1 --result-dir /results/bfcl-"$BFCL_PROFILE" && \ + bfcl evaluate --model "$BFCL_MODEL" --test-category "$BFCL_CATEGORIES" --result-dir /results/bfcl-"$BFCL_PROFILE" --score-dir /results/bfcl-"$BFCL_PROFILE"-scores'