Files
infrastructure/llm-server/llm/ansible/roles/inference_engine/defaults/main.yml
T
spektr bc37972a09 GNAI-009 v0.5: Swift 1.5 als dritter Kandidat + Benchmark-Design; Rollen erweitert
Betreiberbeschluss 01.11.2026: Swift 1.5 Q6_K aufgenommen (GNAI-009 3.4,
Artefakt gepinnt: Revision 14bfe4b, SHA-256 cd2eb65 aus HF-Blob-Metadaten,
22,2 GiB). Testmatrix um S-Reihe und AB3/AB4 erweitert (AB3 isoliert die
Thinking-Optimierung Swift 1.0 vs 1.5, AB4 ist der Praxisvergleich).
Benchmark-Design ergaenzt: lm-evaluation-harness gegen die interne Engine-API
(AIME, GPQA Diamond, MATH-500, IFEval) plus Kernkennzahl Score je
Median-Reasoning-Token aus der getrennten reasoning_content.

model_store: drittes Artefakt; inference_engine: Profil swift15
(Unit gnai-testing-llama-swift15, Alias swift-1.5-qwen3.8-27b-q6k).

Generated by Mistral Vibe.
Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
2026-11-01 17:46:47 +01:00

129 lines
4.7 KiB
YAML

---
# Inference engine deployment (GNAI-008 section 7.4, GNAI-010 sections 3, 5,
# 6 and 9). The first engine profile is llama.cpp; Strata follows as an
# Experimental candidate and is intentionally not modelled here yet.
#
# Container image: immutable digest, never a moving tag (GNAI-010 section 3.1).
inference_engine_image: ghcr.io/ggml-org/llama.cpp:server-rocm
inference_engine_image_ref: ghcr.io/ggml-org/llama.cpp@sha256:b429b8be10d3d0353a09c98f4ab840fe516f779858a28f9fd4e5bdf10bc63cb4
inference_engine_image_build: "b11058"
inference_engine_image_revision: f072b103714dfa1eee531f80b24512faf38e3dd2
# Environment identity (overridden per inventory, GNAI-008 section 7.4).
inference_engine_user: gnai-testing
inference_engine_uid: 981
inference_engine_user_home: /var/lib/ghostnet-ai/testing
inference_engine_unit_prefix: gnai-testing-llama
inference_engine_network_name: gnai-testing-llama
inference_engine_network_alias: llama-cpp
# Paths
inference_engine_quadlet_user_dir: /etc/containers/systemd/users
inference_engine_generator_path: /usr/lib/systemd/system-generators/podman-system-generator
inference_engine_container_port: 8080
# Bring-up context override: when set, it replaces the profile ctx_size for
# every profile (GNAI-008 section 7.5: 64k bring-up before the 128k target).
# Set to null once the 128k acceptance (GNAI-010 section 10.3) has passed.
inference_engine_ctx_size_override: null
# Shared serving parameters (GNAI-010 sections 6.1-6.5, 8).
inference_engine_common:
n_gpu_layers: all
split_mode: layer
tensor_split: "1,1"
flash_attn: "on"
cache_type_k: q8_0
cache_type_v: q8_0
reasoning: "on"
reasoning_budget: -1
# Exactly one profile is active at a time; the other quadlet stays rendered
# but stopped (GNAI-010 section 5.3). Unit names follow GNAI-010 section 5.1.
inference_engine_active_profile: swift-qwen3.8-27b-q6k
inference_engine_profiles:
swift-qwen3.8-27b-q6k:
unit_suffix: swift
description: Swift efficiency finetune of Qwen3.8-27B (Q6_K)
model_directory: swift-qwen3.8-27b
model_file: Swift-Qwen3.8-27B-Q6_K.gguf
alias: swift-qwen3.8-27b-q6k
ctx_size: 131072
n_predict: 32768
parallel: 1
reasoning_effort: xhigh
temp: 1.0
top_p: 0.95
top_k: 20
min_p: 0.0
presence_penalty: 0.0
repeat_penalty: 1.0
mtp:
enabled: false
spec_draft_n_max: 2
unsloth-qwen3.8-27b-ud-q6k:
unit_suffix: unsloth
description: Unsloth UD quant of Qwen3.8-27B (Q6_K)
model_directory: unsloth-qwen3.8-27b
model_file: Qwen3.8-27B-UD-Q6_K.gguf
alias: unsloth-qwen3.8-27b-ud-q6k
ctx_size: 131072
n_predict: 32768
parallel: 1
# xhigh for the controlled A/B comparison; switch to medium for the
# practice comparison (GNAI-010 section 6.1).
reasoning_effort: xhigh
temp: 1.0
top_p: 0.95
top_k: 20
min_p: 0.0
presence_penalty: 0.0
repeat_penalty: 1.0
mtp:
enabled: false
spec_draft_n_max: 2
swift-1.5-qwen3.8-27b-q6k:
unit_suffix: swift15
description: Swift 1.5 reasoning-efficient finetune of Qwen3.8-27B (Q6_K)
model_directory: swift-1.5-qwen3.8-27b
model_file: Swift-1.5-Qwen3.8-27B-Q6_K.gguf
alias: swift-1.5-qwen3.8-27b-q6k
ctx_size: 131072
n_predict: 32768
parallel: 1
reasoning_effort: xhigh
temp: 1.0
top_p: 0.95
top_k: 20
min_p: 0.0
presence_penalty: 0.0
repeat_penalty: 1.0
mtp:
enabled: false
spec_draft_n_max: 2
# Derived paths (do not override).
inference_engine_quadlet_dir: "{{ inference_engine_quadlet_user_dir }}/{{ inference_engine_uid }}"
# Health-check commands per available tool (GNAI-010 section 8). curl -f
# fails on HTTP error statuses, so rc == 0 is a valid health signal; the
# bash fallback prints the status line, the wget variant fails on errors.
inference_engine_curl_health_cmd: >-
curl -fsS -o /dev/null http://127.0.0.1:{{ inference_engine_container_port }}/health
inference_engine_curl_models_cmd: >-
curl -fsS http://127.0.0.1:{{ inference_engine_container_port }}/v1/models
inference_engine_wget_health_cmd: >-
wget -qO- http://127.0.0.1:{{ inference_engine_container_port }}/health
inference_engine_wget_models_cmd: >-
wget -qO- http://127.0.0.1:{{ inference_engine_container_port }}/v1/models
# bash /dev/tcp fallback for health checks when the image contains neither
# curl nor wget. printf receives the backslash escapes as literal text.
inference_engine_bash_health_script: >-
exec 3<>/dev/tcp/127.0.0.1:{{ inference_engine_container_port }};
printf 'GET /health HTTP/1.0\r\n\r\n' >&3; cat <&3
inference_engine_bash_models_script: >-
exec 3<>/dev/tcp/127.0.0.1:{{ inference_engine_container_port }};
printf 'GET /v1/models HTTP/1.0\r\n\r\n' >&3; cat <&3