From 4fea6f75bc150d10992b8a950c802382554878d6 Mon Sep 17 00:00:00 2001 From: spektr Date: Tue, 3 Nov 2026 13:31:37 +0100 Subject: [PATCH] LLM-Server: BFCL-Integration finalisiert - smoke-laufend Nach mehreren Live-Funden: - bfcl-eval braucht soundfile (qwen_agent Dependency) - CLI ist zweiteilig: bfcl generate + bfcl evaluate (nur evaluate bewertet vorab generierte Responses) - Kategorien sind sprachspezifisch: simple_python statt simple - Model-Handler muss api_inference sein: gpt-4o-2024-11-20-FC nutzt die OpenAI-SDK und respektiert OPENAI_BASE_URL/OPENAI_API_KEY - Container-Name muss --replace verwenden - Kategorie-Default auf simple_python,multiple,irrelevance korrigiert Smoke-Test gegen unsloth-Engine: 193 Requests in 48 min, laeuft. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../llm/ansible/roles/benchmark/defaults/main.yml | 7 ++++--- .../benchmark/templates/bfcl.Containerfile.j2 | 7 +++++-- .../roles/benchmark/templates/run-bfcl.sh.j2 | 15 ++++++++------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/llm-server/llm/ansible/roles/benchmark/defaults/main.yml b/llm-server/llm/ansible/roles/benchmark/defaults/main.yml index dd31ec8..d2e2355 100644 --- a/llm-server/llm/ansible/roles/benchmark/defaults/main.yml +++ b/llm-server/llm/ansible/roles/benchmark/defaults/main.yml @@ -46,6 +46,7 @@ benchmark_bfcl_python_image: docker.io/library/python:3.10-slim@sha256:6ff506466 benchmark_bfcl_version: "2026.3.23" benchmark_bfcl_image_tag: "gnai-bench:bfcl-2026.3.23" benchmark_bfcl_force_rebuild: false -# Core function-calling categories; irrelevance checks that the model does -# NOT call tools on unsuitable requests (GNAI-009 5.1). -benchmark_bfcl_categories: simple,multiple,irrelevance +# Core function-calling categories (BFCL 2026.3.23 naming: language-specific +# simple categories); irrelevance checks that the model does NOT call tools on +# unsuitable requests (GNAI-009 5.1). +benchmark_bfcl_categories: simple_python,multiple,irrelevance diff --git a/llm-server/llm/ansible/roles/benchmark/templates/bfcl.Containerfile.j2 b/llm-server/llm/ansible/roles/benchmark/templates/bfcl.Containerfile.j2 index 91c6e18..16de7a8 100644 --- a/llm-server/llm/ansible/roles/benchmark/templates/bfcl.Containerfile.j2 +++ b/llm-server/llm/ansible/roles/benchmark/templates/bfcl.Containerfile.j2 @@ -5,9 +5,12 @@ FROM docker.io/library/python:3.10-slim@sha256:6ff506466f8b1e981719b468cc0023b8e ENV PIP_NO_CACHE_DIR=1 \ PYTHONUNBUFFERED=1 \ REMOTE_OPENAI_BASE_URL=http://llama-cpp:8080/v1 \ - REMOTE_OPENAI_API_KEY=no-key-set + REMOTE_OPENAI_API_KEY=no-key-set \ + OPENAI_API_KEY=no-key-set \ + OPENAI_BASE_URL=http://llama-cpp:8080/v1 -RUN pip install --no-cache-dir bfcl-eval=={{ benchmark_bfcl_version }} +RUN pip install --no-cache-dir bfcl-eval=={{ benchmark_bfcl_version }} \ + && pip install --no-cache-dir soundfile WORKDIR /work ENTRYPOINT ["bfcl"] diff --git a/llm-server/llm/ansible/roles/benchmark/templates/run-bfcl.sh.j2 b/llm-server/llm/ansible/roles/benchmark/templates/run-bfcl.sh.j2 index ccfe45c..e422f54 100644 --- a/llm-server/llm/ansible/roles/benchmark/templates/run-bfcl.sh.j2 +++ b/llm-server/llm/ansible/roles/benchmark/templates/run-bfcl.sh.j2 @@ -20,14 +20,15 @@ OUT=/results/bfcl-"$PROFILE"-"$TS" exec /usr/sbin/runuser --user {{ benchmark_user }} -- \ /usr/bin/env HOME={{ benchmark_user_home }} XDG_RUNTIME_DIR=/run/user/{{ benchmark_uid }} \ - /usr/bin/podman run --rm \ + /usr/bin/podman run --rm --replace \ --name gnai-bfcl-run \ --network {{ benchmark_engine_network_name }} \ --network {{ benchmark_bench_network_name }} \ -v {{ benchmark_results_dir }}:/results:Z \ - {{ benchmark_bfcl_image_tag }} \ - evaluate \ - --model "$MODEL" \ - --test-category "$CATEGORIES" \ - --include-input-token \ - --result-path "$OUT" + -e BFCL_PROFILE="$PROFILE" \ + -e BFCL_MODEL="$MODEL" \ + -e BFCL_CATEGORIES="$CATEGORIES" \ + --entrypoint /bin/bash \ + {{ benchmark_bfcl_image_tag }} -c '\ + bfcl generate --model "$BFCL_MODEL" --test-category "$BFCL_CATEGORIES" --num-threads 1 --result-dir /results/bfcl-"$BFCL_PROFILE" && \ + bfcl evaluate --model "$BFCL_MODEL" --test-category "$BFCL_CATEGORIES" --result-dir /results/bfcl-"$BFCL_PROFILE" --score-dir /results/bfcl-"$BFCL_PROFILE"-scores'