diff --git a/docs/platform/llm-server/ai/GNAI-010-TECH_Llama-Serving.md b/docs/platform/llm-server/ai/GNAI-010-TECH_Llama-Serving.md index c993c58..1cb93b1 100644 --- a/docs/platform/llm-server/ai/GNAI-010-TECH_Llama-Serving.md +++ b/docs/platform/llm-server/ai/GNAI-010-TECH_Llama-Serving.md @@ -22,6 +22,7 @@ | 2026-09-21 | 0.2 | spektr / Codex | Unsloth UD-Q6_K als zweites installiertes Modellprofil und kontrollierten Profilwechsel für A/B-Tests ergänzt | | 2026-09-21 | 0.1 | spektr / Codex | erstes technisches Profil für Swift Q6_K, 128k, Q8-KV, rootless Quadlet und gestufte MTP-Abnahme erstellt | | 2026-10-31 | 0.5 | spektr / Mistral Vibe | Swift-Repository-Revision aktualisiert: `eb0e3a7…` wurde vom Herausgeber aus dem Repository entfernt (resolve liefert 404); Repin auf den aktuellen `main`-Commit `98a6100…`. Die Q6_K-Datei ist laut Herausgeber-SHA256SUMS unverändert (`7f4de8…`), der Vision-Projektor ebenfalls (`daa111…`); beide Prüfsummen bleiben unverändert und werden beim Download verifiziert. | +| 2026-10-31 | 0.6 | spektr / Mistral Vibe | Implementierungsbefund vom ersten Live-Start: Das gepinnte AMD-Image definiert `ENTRYPOINT /app/llama-server`; `Exec=` im Quadlet trägt deshalb nur die Argumente ohne Binärnamen. Das Image enthält weder `curl` noch `wget` – die Health-Prüfung darf nicht von einem im Image vorhandenen HTTP-Werkzeug ausgehen (Abschnitt 8 bestätigt). | --- diff --git a/llm-server/llm/ansible/roles/inference_engine/defaults/main.yml b/llm-server/llm/ansible/roles/inference_engine/defaults/main.yml index 9fffd9e..23d7c53 100644 --- a/llm-server/llm/ansible/roles/inference_engine/defaults/main.yml +++ b/llm-server/llm/ansible/roles/inference_engine/defaults/main.yml @@ -86,3 +86,24 @@ inference_engine_profiles: # Derived paths (do not override). inference_engine_quadlet_dir: "{{ inference_engine_quadlet_user_dir }}/{{ inference_engine_uid }}" + +# Health-check commands per available tool (GNAI-010 section 8). curl -f +# fails on HTTP error statuses, so rc == 0 is a valid health signal; the +# bash fallback prints the status line, the wget variant fails on errors. +inference_engine_curl_health_cmd: >- + curl -fsS -o /dev/null http://127.0.0.1:{{ inference_engine_container_port }}/health +inference_engine_curl_models_cmd: >- + curl -fsS http://127.0.0.1:{{ inference_engine_container_port }}/v1/models +inference_engine_wget_health_cmd: >- + wget -qO- http://127.0.0.1:{{ inference_engine_container_port }}/health +inference_engine_wget_models_cmd: >- + wget -qO- http://127.0.0.1:{{ inference_engine_container_port }}/v1/models + +# bash /dev/tcp fallback for health checks when the image contains neither +# curl nor wget. printf receives the backslash escapes as literal text. +inference_engine_bash_health_script: >- + exec 3<>/dev/tcp/127.0.0.1:{{ inference_engine_container_port }}; + printf 'GET /health HTTP/1.0\r\n\r\n' >&3; cat <&3 +inference_engine_bash_models_script: >- + exec 3<>/dev/tcp/127.0.0.1:{{ inference_engine_container_port }}; + printf 'GET /v1/models HTTP/1.0\r\n\r\n' >&3; cat <&3 diff --git a/llm-server/llm/ansible/roles/inference_engine/tasks/main.yml b/llm-server/llm/ansible/roles/inference_engine/tasks/main.yml index 165cb00..edef3bf 100644 --- a/llm-server/llm/ansible/roles/inference_engine/tasks/main.yml +++ b/llm-server/llm/ansible/roles/inference_engine/tasks/main.yml @@ -234,6 +234,36 @@ tags: - service +- name: Apply rendered quadlet changes before touching units + # The daemon-reload handler must run before unit states are read or units + # are started; otherwise systemd does not know the freshly rendered units + # and `is-active` exits with rc 4 (unknown unit). + ansible.builtin.meta: flush_handlers + tags: + - service + +- name: Ensure the workload user manager knows the rendered units + # Deterministic reload: handler notifications only fire when a quadlet + # changed in the same run, but a previous run may have rendered units + # without reloading (failed run) — so reload unconditionally. + ansible.builtin.command: + argv: + - /usr/sbin/runuser + - --user + - "{{ inference_engine_user }}" + - -- + - /usr/bin/env + - "HOME={{ inference_engine_user_home }}" + - "XDG_RUNTIME_DIR=/run/user/{{ inference_engine_uid }}" + - /usr/bin/systemctl + - --user + - daemon-reload + chdir: "{{ inference_engine_user_home }}" + changed_when: false + when: not ansible_check_mode + tags: + - service + - name: Read the current state of every engine unit ansible.builtin.command: argv: @@ -250,7 +280,7 @@ - "{{ inference_engine_unit_names[item.key] }}" chdir: "{{ inference_engine_user_home }}" changed_when: false - failed_when: inference_engine_unit_states.rc not in [0, 3] + failed_when: inference_engine_unit_states.rc not in [0, 3, 4] check_mode: false loop: "{{ inference_engine_profiles | dict2items }}" loop_control: @@ -277,7 +307,7 @@ chdir: "{{ inference_engine_user_home }}" when: - not ansible_check_mode - - item.stdout | trim != 'active' + - item.stdout | trim == 'active' - item.item.key != inference_engine_active_profile loop: "{{ inference_engine_unit_states.results }}" loop_control: @@ -324,7 +354,7 @@ - "{{ inference_engine_unit_names[inference_engine_active_profile] }}" - /bin/sh - -c - - "command -v curl || command -v wget || true" + - "command -v curl || command -v wget || command -v bash || true" chdir: "{{ inference_engine_user_home }}" changed_when: false check_mode: false @@ -333,29 +363,40 @@ tags: - service +- name: Record the HTTP client for health checks + ansible.builtin.set_fact: + inference_engine_http_tool: "{{ inference_engine_http_client.stdout | split('/') | last | trim }}" + when: not ansible_check_mode + tags: + - service + - name: Require an HTTP client in the pinned engine image ansible.builtin.assert: that: - - inference_engine_http_client.stdout | trim in ['curl', 'wget'] + - inference_engine_http_tool in ['curl', 'wget', 'bash'] fail_msg: >- - The pinned engine image contains neither curl nor wget; health checks - cannot run (GNAI-010 section 8 requires checking the image tool first). + The pinned engine image contains none of curl, wget or bash; health + checks cannot run (GNAI-010 section 8 requires checking the image + tool first; confirmed for this image 31.10.2026). when: not ansible_check_mode tags: - service - name: Wait for the engine health endpoint ansible.builtin.command: - argv: "{{ [ '/usr/sbin/runuser', '--user', inference_engine_user, '--', - '/usr/bin/env', 'HOME=' ~ inference_engine_user_home, - 'XDG_RUNTIME_DIR=/run/user/' ~ inference_engine_uid, - '/usr/bin/podman', 'exec', - inference_engine_unit_names[inference_engine_active_profile], - '/bin/sh', '-c' ] + - (['curl', '-fsS', health_url] if inference_engine_http_client.stdout | trim == 'curl' - else ['wget', '-qO-', health_url]) }}" + argv: >- + {{ ['/usr/sbin/runuser', '--user', inference_engine_user, '--', + '/usr/bin/env', 'HOME=' ~ inference_engine_user_home, + 'XDG_RUNTIME_DIR=/run/user/' ~ inference_engine_uid, + '/usr/bin/podman', 'exec', + inference_engine_unit_names[inference_engine_active_profile], + '/bin/sh', '-c', health_cmd] }} + chdir: "{{ inference_engine_user_home }}" vars: - health_url: "http://127.0.0.1:{{ inference_engine_container_port }}/health" + health_cmd: >- + {{ inference_engine_curl_health_cmd if inference_engine_http_tool == 'curl' + else inference_engine_wget_health_cmd if inference_engine_http_tool == 'wget' + else inference_engine_bash_health_script }} changed_when: false check_mode: false register: inference_engine_health @@ -368,16 +409,19 @@ - name: Verify the served model alias ansible.builtin.command: - argv: "{{ [ '/usr/sbin/runuser', '--user', inference_engine_user, '--', - '/usr/bin/env', 'HOME=' ~ inference_engine_user_home, - 'XDG_RUNTIME_DIR=/run/user/' ~ inference_engine_uid, - '/usr/bin/podman', 'exec', - inference_engine_unit_names[inference_engine_active_profile], - '/bin/sh', '-c' ] + - (['curl', '-fsS', models_url] if inference_engine_http_client.stdout | trim == 'curl' - else ['wget', '-qO-', models_url]) }}" + argv: >- + {{ ['/usr/sbin/runuser', '--user', inference_engine_user, '--', + '/usr/bin/env', 'HOME=' ~ inference_engine_user_home, + 'XDG_RUNTIME_DIR=/run/user/' ~ inference_engine_uid, + '/usr/bin/podman', 'exec', + inference_engine_unit_names[inference_engine_active_profile], + '/bin/sh', '-c', models_cmd] }} + chdir: "{{ inference_engine_user_home }}" vars: - models_url: "http://127.0.0.1:{{ inference_engine_container_port }}/v1/models" + models_cmd: >- + {{ inference_engine_curl_models_cmd if inference_engine_http_tool == 'curl' + else inference_engine_wget_models_cmd if inference_engine_http_tool == 'wget' + else inference_engine_bash_models_script }} changed_when: false check_mode: false register: inference_engine_models_endpoint diff --git a/llm-server/llm/ansible/roles/inference_engine/templates/llama-server.container.j2 b/llm-server/llm/ansible/roles/inference_engine/templates/llama-server.container.j2 index 38cf4e2..0ad981b 100644 --- a/llm-server/llm/ansible/roles/inference_engine/templates/llama-server.container.j2 +++ b/llm-server/llm/ansible/roles/inference_engine/templates/llama-server.container.j2 @@ -1,5 +1,7 @@ [Unit] Description=LLM inference engine ({{ entry.profile_key }}) +Wants={{ inference_engine_network_name }}-network.service +After={{ inference_engine_network_name }}-network.service {% for other in entry.other_units %} Conflicts={{ other }} {% endfor %} @@ -19,7 +21,10 @@ GroupAdd=keep-groups Volume={{ model_store_root }}/{{ entry.profile.model_directory }}:/models:ro Network={{ inference_engine_network_name }} NetworkAlias={{ inference_engine_network_alias }} -Exec=llama-server --host 0.0.0.0 --port {{ inference_engine_container_port }} --model /models/{{ entry.profile.model_file }} --alias {{ entry.profile.alias }} --parallel {{ entry.profile.parallel }} --ctx-size {{ entry.ctx_size }} --n-predict {{ entry.profile.n_predict }} --n-gpu-layers {{ inference_engine_common.n_gpu_layers }} --split-mode {{ inference_engine_common.split_mode }} --tensor-split {{ inference_engine_common.tensor_split }} --flash-attn {{ inference_engine_common.flash_attn }} --cache-type-k {{ inference_engine_common.cache_type_k }} --cache-type-v {{ inference_engine_common.cache_type_v }} --jinja --reasoning {{ inference_engine_common.reasoning }} --reasoning-effort {{ entry.profile.reasoning_effort }} --reasoning-budget {{ inference_engine_common.reasoning_budget }} --temp {{ entry.profile.temp }} --top-p {{ entry.profile.top_p }} --top-k {{ entry.profile.top_k }} --min-p {{ entry.profile.min_p }} --presence-penalty {{ entry.profile.presence_penalty }} --repeat-penalty {{ entry.profile.repeat_penalty }}{% if entry.profile.mtp.enabled %} --spec-type draft-mtp --spec-draft-n-max {{ entry.profile.mtp.spec_draft_n_max }}{% endif %} --no-webui --metrics +# The pinned AMD image defines ENTRYPOINT /app/llama-server; Exec= therefore +# carries only the arguments (observed 31.10.2026: a leading "llama-server" +# is passed as an invalid positional argument and aborts within a second). +Exec=--host 0.0.0.0 --port {{ inference_engine_container_port }} --model /models/{{ entry.profile.model_file }} --alias {{ entry.profile.alias }} --parallel {{ entry.profile.parallel }} --ctx-size {{ entry.ctx_size }} --n-predict {{ entry.profile.n_predict }} --n-gpu-layers {{ inference_engine_common.n_gpu_layers }} --split-mode {{ inference_engine_common.split_mode }} --tensor-split {{ inference_engine_common.tensor_split }} --flash-attn {{ inference_engine_common.flash_attn }} --cache-type-k {{ inference_engine_common.cache_type_k }} --cache-type-v {{ inference_engine_common.cache_type_v }} --jinja --reasoning {{ inference_engine_common.reasoning }} --reasoning-effort {{ entry.profile.reasoning_effort }} --reasoning-budget {{ inference_engine_common.reasoning_budget }} --temp {{ entry.profile.temp }} --top-p {{ entry.profile.top_p }} --top-k {{ entry.profile.top_k }} --min-p {{ entry.profile.min_p }} --presence-penalty {{ entry.profile.presence_penalty }} --repeat-penalty {{ entry.profile.repeat_penalty }}{% if entry.profile.mtp.enabled %} --spec-type draft-mtp --spec-draft-n-max {{ entry.profile.mtp.spec_draft_n_max }}{% endif %} --no-webui --metrics [Service] Restart=on-failure