Betreiberbeschluss 01.11.2026: Swift 1.5 Q6_K aufgenommen (GNAI-009 3.4, Artefakt gepinnt: Revision 14bfe4b, SHA-256 cd2eb65 aus HF-Blob-Metadaten, 22,2 GiB). Testmatrix um S-Reihe und AB3/AB4 erweitert (AB3 isoliert die Thinking-Optimierung Swift 1.0 vs 1.5, AB4 ist der Praxisvergleich). Benchmark-Design ergaenzt: lm-evaluation-harness gegen die interne Engine-API (AIME, GPQA Diamond, MATH-500, IFEval) plus Kernkennzahl Score je Median-Reasoning-Token aus der getrennten reasoning_content. model_store: drittes Artefakt; inference_engine: Profil swift15 (Unit gnai-testing-llama-swift15, Alias swift-1.5-qwen3.8-27b-q6k). Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
129 lines
4.7 KiB
YAML
129 lines
4.7 KiB
YAML
---
|
|
# Inference engine deployment (GNAI-008 section 7.4, GNAI-010 sections 3, 5,
|
|
# 6 and 9). The first engine profile is llama.cpp; Strata follows as an
|
|
# Experimental candidate and is intentionally not modelled here yet.
|
|
#
|
|
# Container image: immutable digest, never a moving tag (GNAI-010 section 3.1).
|
|
inference_engine_image: ghcr.io/ggml-org/llama.cpp:server-rocm
|
|
inference_engine_image_ref: ghcr.io/ggml-org/llama.cpp@sha256:b429b8be10d3d0353a09c98f4ab840fe516f779858a28f9fd4e5bdf10bc63cb4
|
|
inference_engine_image_build: "b11058"
|
|
inference_engine_image_revision: f072b103714dfa1eee531f80b24512faf38e3dd2
|
|
|
|
# Environment identity (overridden per inventory, GNAI-008 section 7.4).
|
|
inference_engine_user: gnai-testing
|
|
inference_engine_uid: 981
|
|
inference_engine_user_home: /var/lib/ghostnet-ai/testing
|
|
inference_engine_unit_prefix: gnai-testing-llama
|
|
inference_engine_network_name: gnai-testing-llama
|
|
inference_engine_network_alias: llama-cpp
|
|
|
|
# Paths
|
|
inference_engine_quadlet_user_dir: /etc/containers/systemd/users
|
|
inference_engine_generator_path: /usr/lib/systemd/system-generators/podman-system-generator
|
|
inference_engine_container_port: 8080
|
|
|
|
# Bring-up context override: when set, it replaces the profile ctx_size for
|
|
# every profile (GNAI-008 section 7.5: 64k bring-up before the 128k target).
|
|
# Set to null once the 128k acceptance (GNAI-010 section 10.3) has passed.
|
|
inference_engine_ctx_size_override: null
|
|
|
|
# Shared serving parameters (GNAI-010 sections 6.1-6.5, 8).
|
|
inference_engine_common:
|
|
n_gpu_layers: all
|
|
split_mode: layer
|
|
tensor_split: "1,1"
|
|
flash_attn: "on"
|
|
cache_type_k: q8_0
|
|
cache_type_v: q8_0
|
|
reasoning: "on"
|
|
reasoning_budget: -1
|
|
|
|
# Exactly one profile is active at a time; the other quadlet stays rendered
|
|
# but stopped (GNAI-010 section 5.3). Unit names follow GNAI-010 section 5.1.
|
|
inference_engine_active_profile: swift-qwen3.8-27b-q6k
|
|
|
|
inference_engine_profiles:
|
|
swift-qwen3.8-27b-q6k:
|
|
unit_suffix: swift
|
|
description: Swift efficiency finetune of Qwen3.8-27B (Q6_K)
|
|
model_directory: swift-qwen3.8-27b
|
|
model_file: Swift-Qwen3.8-27B-Q6_K.gguf
|
|
alias: swift-qwen3.8-27b-q6k
|
|
ctx_size: 131072
|
|
n_predict: 32768
|
|
parallel: 1
|
|
reasoning_effort: xhigh
|
|
temp: 1.0
|
|
top_p: 0.95
|
|
top_k: 20
|
|
min_p: 0.0
|
|
presence_penalty: 0.0
|
|
repeat_penalty: 1.0
|
|
mtp:
|
|
enabled: false
|
|
spec_draft_n_max: 2
|
|
unsloth-qwen3.8-27b-ud-q6k:
|
|
unit_suffix: unsloth
|
|
description: Unsloth UD quant of Qwen3.8-27B (Q6_K)
|
|
model_directory: unsloth-qwen3.8-27b
|
|
model_file: Qwen3.8-27B-UD-Q6_K.gguf
|
|
alias: unsloth-qwen3.8-27b-ud-q6k
|
|
ctx_size: 131072
|
|
n_predict: 32768
|
|
parallel: 1
|
|
# xhigh for the controlled A/B comparison; switch to medium for the
|
|
# practice comparison (GNAI-010 section 6.1).
|
|
reasoning_effort: xhigh
|
|
temp: 1.0
|
|
top_p: 0.95
|
|
top_k: 20
|
|
min_p: 0.0
|
|
presence_penalty: 0.0
|
|
repeat_penalty: 1.0
|
|
mtp:
|
|
enabled: false
|
|
spec_draft_n_max: 2
|
|
swift-1.5-qwen3.8-27b-q6k:
|
|
unit_suffix: swift15
|
|
description: Swift 1.5 reasoning-efficient finetune of Qwen3.8-27B (Q6_K)
|
|
model_directory: swift-1.5-qwen3.8-27b
|
|
model_file: Swift-1.5-Qwen3.8-27B-Q6_K.gguf
|
|
alias: swift-1.5-qwen3.8-27b-q6k
|
|
ctx_size: 131072
|
|
n_predict: 32768
|
|
parallel: 1
|
|
reasoning_effort: xhigh
|
|
temp: 1.0
|
|
top_p: 0.95
|
|
top_k: 20
|
|
min_p: 0.0
|
|
presence_penalty: 0.0
|
|
repeat_penalty: 1.0
|
|
mtp:
|
|
enabled: false
|
|
spec_draft_n_max: 2
|
|
|
|
# Derived paths (do not override).
|
|
inference_engine_quadlet_dir: "{{ inference_engine_quadlet_user_dir }}/{{ inference_engine_uid }}"
|
|
|
|
# Health-check commands per available tool (GNAI-010 section 8). curl -f
|
|
# fails on HTTP error statuses, so rc == 0 is a valid health signal; the
|
|
# bash fallback prints the status line, the wget variant fails on errors.
|
|
inference_engine_curl_health_cmd: >-
|
|
curl -fsS -o /dev/null http://127.0.0.1:{{ inference_engine_container_port }}/health
|
|
inference_engine_curl_models_cmd: >-
|
|
curl -fsS http://127.0.0.1:{{ inference_engine_container_port }}/v1/models
|
|
inference_engine_wget_health_cmd: >-
|
|
wget -qO- http://127.0.0.1:{{ inference_engine_container_port }}/health
|
|
inference_engine_wget_models_cmd: >-
|
|
wget -qO- http://127.0.0.1:{{ inference_engine_container_port }}/v1/models
|
|
|
|
# bash /dev/tcp fallback for health checks when the image contains neither
|
|
# curl nor wget. printf receives the backslash escapes as literal text.
|
|
inference_engine_bash_health_script: >-
|
|
exec 3<>/dev/tcp/127.0.0.1:{{ inference_engine_container_port }};
|
|
printf 'GET /health HTTP/1.0\r\n\r\n' >&3; cat <&3
|
|
inference_engine_bash_models_script: >-
|
|
exec 3<>/dev/tcp/127.0.0.1:{{ inference_engine_container_port }};
|
|
printf 'GET /v1/models HTTP/1.0\r\n\r\n' >&3; cat <&3
|