diff --git a/llm-server/llm/ansible/README.md b/llm-server/llm/ansible/README.md new file mode 100644 index 0000000..b49908b --- /dev/null +++ b/llm-server/llm/ansible/README.md @@ -0,0 +1,54 @@ +# LLM serving stack – Ansible control project + +Executes the LLM serving stack for exactly one environment on +`ghostnet-ki` (GNAI-008 section 7.4). Host baseline, Podman runtime, +workload accounts and AMD GPU access are managed by +[`../../host/ansible`](../../host/ansible/README.md); this project never +changes host state. + +## Layout + +- `inventories//` – one inventory per KI environment + (currently `testing`); deployment uses the `ansible-deploy` account with + `become: true`. +- `playbooks/ai.yml` – applies `model_store` then `inference_engine`. +- `roles/model_store/` – shared, pinned model weights under + `/var/lib/ghostnet-ai/shared/models` (GNAI-010 sections 3 and 4). +- `roles/inference_engine/` – rootless Quadlets, network, image provisioning + and the exclusive profile switch for llama.cpp (GNAI-010 sections 5 and 6). + +## Control node setup + +Use the repository-wide control venv built from +[`../../../shared/ansible/requirements-control.txt`](../../../shared/ansible/requirements-control.txt) +(`ansible-core==2.21.4`), for example: + +```bash +python3 -m venv .venv +.venv/bin/pip install -r shared/ansible/requirements-control.txt +cd llm-server/llm/ansible +../../../.venv/bin/ansible-playbook -i inventories/testing playbooks/ai.yml --syntax-check +``` + +## Usage (Testing) + +```bash +# Preview only – no changes, no downloads, no service start: +../../../.venv/bin/ansible-playbook -i inventories/testing playbooks/ai.yml --check --diff + +# Provision model weights only (tag models): +../../../.venv/bin/ansible-playbook -i inventories/testing playbooks/ai.yml --tags models + +# Render quadlets and start/verify the selected profile (tag service): +../../../.venv/bin/ansible-playbook -i inventories/testing playbooks/ai.yml --tags service +``` + +`inference_engine_active_profile` selects which rendered quadlet is started; +all other profiles stay rendered but stopped. Check mode downloads no model +and starts no service. A second identical run must report `changed=0` +(GNAI-010 section 10.4). + +The full technical acceptance (64k/128k start, VRAM distribution, prefill +ladder, restart behaviour) is an operator-run protocol documented in +[GNAI-010](../../../docs/platform/llm-server/ai/GNAI-010-TECH_Llama-Serving.md); +this role automates deployment and basic health only. diff --git a/llm-server/llm/ansible/ansible.cfg b/llm-server/llm/ansible/ansible.cfg new file mode 100644 index 0000000..b13fcf0 --- /dev/null +++ b/llm-server/llm/ansible/ansible.cfg @@ -0,0 +1,12 @@ +[defaults] +home = .ansible +host_key_checking = True +interpreter_python = auto_silent +local_tmp = .ansible/tmp +retry_files_enabled = False +roles_path = roles:../../../shared/ansible/roles +timeout = 15 + +[ssh_connection] +pipelining = True +ssh_args = -o ControlMaster=auto -o ControlPersist=60s diff --git a/llm-server/llm/ansible/inventories/testing/group_vars/llm_servers.yml b/llm-server/llm/ansible/inventories/testing/group_vars/llm_servers.yml new file mode 100644 index 0000000..dd5641c --- /dev/null +++ b/llm-server/llm/ansible/inventories/testing/group_vars/llm_servers.yml @@ -0,0 +1,14 @@ +--- +# Environment-specific values for the Testing stack (GNAI-008 section 7.4, +# GNAI-010 section 5). The workload account, quadlet directory, unit prefix +# and network exist once per environment; Production and Experimental use the +# same role logic with their own values. +inference_engine_user: gnai-testing +inference_engine_uid: 981 +inference_engine_user_home: /var/lib/ghostnet-ai/testing +inference_engine_unit_prefix: gnai-testing-llama +inference_engine_network_name: gnai-testing-llama + +# GNAI-008 section 7.5: bring-up starts at 64k; the 128k target profile is +# accepted separately (GNAI-010 section 10.3). Set to null after acceptance. +inference_engine_ctx_size_override: 65536 diff --git a/llm-server/llm/ansible/inventories/testing/hosts.yml b/llm-server/llm/ansible/inventories/testing/hosts.yml new file mode 100644 index 0000000..8c61b48 --- /dev/null +++ b/llm-server/llm/ansible/inventories/testing/hosts.yml @@ -0,0 +1,7 @@ +all: + children: + llm_servers: + hosts: + ghostnet-ki: + ansible_host: 10.0.0.90 + ansible_user: ansible-deploy diff --git a/llm-server/llm/ansible/playbooks/ai.yml b/llm-server/llm/ansible/playbooks/ai.yml new file mode 100644 index 0000000..79983d8 --- /dev/null +++ b/llm-server/llm/ansible/playbooks/ai.yml @@ -0,0 +1,16 @@ +--- +# Deploys the LLM serving stack for exactly one explicitly selected +# environment (GNAI-008 section 7.4). Host baseline, Podman runtime, +# workload accounts and GPU access are managed by the host project +# (llm-server/host/ansible); this playbook never changes host state. +# +# Usage (Testing): +# ansible-playbook -i inventories/testing playbooks/ai.yml --check --diff +# ansible-playbook -i inventories/testing playbooks/ai.yml --tags models +# ansible-playbook -i inventories/testing playbooks/ai.yml --tags service +- name: Bring the LLM serving stack of one environment into its desired state + hosts: llm_servers + become: true + roles: + - role: model_store + - role: inference_engine diff --git a/llm-server/llm/ansible/roles/inference_engine/README.md b/llm-server/llm/ansible/roles/inference_engine/README.md new file mode 100644 index 0000000..430267c --- /dev/null +++ b/llm-server/llm/ansible/roles/inference_engine/README.md @@ -0,0 +1,54 @@ +# inference_engine + +Deploys the inference engine for one KI environment as rootless Podman +Quadlets (GNAI-010 sections 5 and 6; first profile: llama.cpp). Runs after +`model_store` in the same play and reuses `model_store_root`. + +## Variables + +- `inference_engine_image_ref` – immutable image digest (`ghcr.io/...@sha256:...`). +- `inference_engine_user` / `inference_engine_uid` / `inference_engine_user_home` + – workload account created by the host `workload_accounts` role. +- `inference_engine_unit_prefix` – unit and container prefix of the + environment (Testing: `gnai-testing-llama`). +- `inference_engine_network_name` / `inference_engine_network_alias` – + internal-only Quadlet network (`Internal=true`), alias `llama-cpp` for the + later API policy layer. +- `inference_engine_profiles` – map of model profiles; each needs `unit_suffix`, + `model_directory`, `model_file`, `alias`, `ctx_size`, `n_predict`, + `reasoning_effort`, sampling values and an `mtp` block. +- `inference_engine_active_profile` – the single profile to start. +- `inference_engine_ctx_size_override` – when not `null`, replaces every + profile's `ctx_size` (Testing bring-up: 65536; set to `null` after the 128k + acceptance, GNAI-008 section 7.5). + +## Behaviour + +- Validates declarations, requires `/dev/kfd` plus exactly two render nodes, + and requires every profile's model artifact (run `--tags models` first). +- Pulls the pinned image by digest into the environment's rootless store + only when missing. +- Renders `.network` and one `.container` per profile into + `/etc/containers/systemd/users//`; units are never enabled for + `default.target`. +- Units mutually exclude each other via `Conflicts=`/`Before=` + (GNAI-010 section 5.3). +- Exercises the Quadlet generator with `--dryrun` before any reload; quadlet + changes notify a `daemon-reload` of the workload user manager. Services + are never restarted automatically. +- Switch/start (only with the `service` tag, never in check mode): stop any + other active engine unit, start the selected unit, wait for `/health`, and + verify the served alias via `/v1/models`. The HTTP check uses whichever of + `curl`/`wget` exists in the pinned image (GNAI-010 section 8). + +## Tags + +- `service` – everything in this role. Model provisioning belongs to the + `models` tag of the `model_store` role. + +## Notes + +- There is no `PublishPort=`; the engine is reachable only inside the + environment network. +- The role does not start the engine in check mode and never downloads + models. diff --git a/llm-server/llm/ansible/roles/inference_engine/defaults/main.yml b/llm-server/llm/ansible/roles/inference_engine/defaults/main.yml new file mode 100644 index 0000000..9fffd9e --- /dev/null +++ b/llm-server/llm/ansible/roles/inference_engine/defaults/main.yml @@ -0,0 +1,88 @@ +--- +# Inference engine deployment (GNAI-008 section 7.4, GNAI-010 sections 3, 5, +# 6 and 9). The first engine profile is llama.cpp; Strata follows as an +# Experimental candidate and is intentionally not modelled here yet. +# +# Container image: immutable digest, never a moving tag (GNAI-010 section 3.1). +inference_engine_image: ghcr.io/ggml-org/llama.cpp:server-rocm +inference_engine_image_ref: ghcr.io/ggml-org/llama.cpp@sha256:b429b8be10d3d0353a09c98f4ab840fe516f779858a28f9fd4e5bdf10bc63cb4 +inference_engine_image_build: "b11058" +inference_engine_image_revision: f072b103714dfa1eee531f80b24512faf38e3dd2 + +# Environment identity (overridden per inventory, GNAI-008 section 7.4). +inference_engine_user: gnai-testing +inference_engine_uid: 981 +inference_engine_user_home: /var/lib/ghostnet-ai/testing +inference_engine_unit_prefix: gnai-testing-llama +inference_engine_network_name: gnai-testing-llama +inference_engine_network_alias: llama-cpp + +# Paths +inference_engine_quadlet_user_dir: /etc/containers/systemd/users +inference_engine_generator_path: /usr/lib/systemd/system-generators/podman-system-generator +inference_engine_container_port: 8080 + +# Bring-up context override: when set, it replaces the profile ctx_size for +# every profile (GNAI-008 section 7.5: 64k bring-up before the 128k target). +# Set to null once the 128k acceptance (GNAI-010 section 10.3) has passed. +inference_engine_ctx_size_override: null + +# Shared serving parameters (GNAI-010 sections 6.1-6.5, 8). +inference_engine_common: + n_gpu_layers: all + split_mode: layer + tensor_split: "1,1" + flash_attn: "on" + cache_type_k: q8_0 + cache_type_v: q8_0 + reasoning: "on" + reasoning_budget: -1 + +# Exactly one profile is active at a time; the other quadlet stays rendered +# but stopped (GNAI-010 section 5.3). Unit names follow GNAI-010 section 5.1. +inference_engine_active_profile: swift-qwen3.8-27b-q6k + +inference_engine_profiles: + swift-qwen3.8-27b-q6k: + unit_suffix: swift + description: Swift efficiency finetune of Qwen3.8-27B (Q6_K) + model_directory: swift-qwen3.8-27b + model_file: Swift-Qwen3.8-27B-Q6_K.gguf + alias: swift-qwen3.8-27b-q6k + ctx_size: 131072 + n_predict: 32768 + parallel: 1 + reasoning_effort: xhigh + temp: 1.0 + top_p: 0.95 + top_k: 20 + min_p: 0.0 + presence_penalty: 0.0 + repeat_penalty: 1.0 + mtp: + enabled: false + spec_draft_n_max: 2 + unsloth-qwen3.8-27b-ud-q6k: + unit_suffix: unsloth + description: Unsloth UD quant of Qwen3.8-27B (Q6_K) + model_directory: unsloth-qwen3.8-27b + model_file: Qwen3.8-27B-UD-Q6_K.gguf + alias: unsloth-qwen3.8-27b-ud-q6k + ctx_size: 131072 + n_predict: 32768 + parallel: 1 + # xhigh for the controlled A/B comparison; switch to medium for the + # practice comparison (GNAI-010 section 6.1). + reasoning_effort: xhigh + temp: 1.0 + top_p: 0.95 + top_k: 20 + min_p: 0.0 + presence_penalty: 0.0 + repeat_penalty: 1.0 + mtp: + enabled: false + spec_draft_n_max: 2 + +# Derived paths (do not override). +inference_engine_quadlet_dir: "{{ inference_engine_quadlet_user_dir }}/{{ inference_engine_uid }}" diff --git a/llm-server/llm/ansible/roles/inference_engine/handlers/main.yml b/llm-server/llm/ansible/roles/inference_engine/handlers/main.yml new file mode 100644 index 0000000..0201ef8 --- /dev/null +++ b/llm-server/llm/ansible/roles/inference_engine/handlers/main.yml @@ -0,0 +1,20 @@ +--- +# Reloads the persistent user manager of the workload account after quadlet +# changes (GNAI-010 section 9). Handlers never run in check mode. Services +# are intentionally not restarted automatically: switching profiles is an +# explicit sequential operation (GNAI-010 section 5.3). +- name: Reload the workload user manager + ansible.builtin.command: + argv: + - /usr/sbin/runuser + - --user + - "{{ inference_engine_user }}" + - -- + - /usr/bin/env + - "HOME={{ inference_engine_user_home }}" + - "XDG_RUNTIME_DIR=/run/user/{{ inference_engine_uid }}" + - /usr/bin/systemctl + - --user + - daemon-reload + chdir: "{{ inference_engine_user_home }}" + changed_when: false diff --git a/llm-server/llm/ansible/roles/inference_engine/tasks/main.yml b/llm-server/llm/ansible/roles/inference_engine/tasks/main.yml new file mode 100644 index 0000000..43c8532 --- /dev/null +++ b/llm-server/llm/ansible/roles/inference_engine/tasks/main.yml @@ -0,0 +1,388 @@ +--- +# Deploys the inference engine (llama.cpp profile) for one environment +# (GNAI-010 sections 5, 6 and 9). Rendering and provisioning run under the +# `service` tag, model artifacts under the `models` tag of the model_store +# role. The selected unit is only started by the explicit switch/start tasks +# below and is never enabled for default.target (GNAI-010 section 5.3). +- name: Validate inference engine declarations + ansible.builtin.assert: + that: + - inference_engine_user | length > 0 + - inference_engine_uid | int > 0 + - inference_engine_user_home is match("^/") + - inference_engine_unit_prefix | length > 0 + - inference_engine_network_name | length > 0 + - inference_engine_image_ref is match("^ghcr\\.io/.+@sha256:[0-9a-f]{64}$") + - inference_engine_image_revision is match("^[0-9a-f]{40}$") + - inference_engine_profiles | length >= 2 + - inference_engine_active_profile in inference_engine_profiles + - model_store_root is match("^/") + fail_msg: Inference engine declarations are incomplete or ambiguous. + tags: + - service + +- name: Derive the systemd unit name of every profile + ansible.builtin.set_fact: + inference_engine_unit_names: >- + {{ inference_engine_unit_names | default({}) + | combine({ item.key: + inference_engine_unit_prefix ~ '-' ~ item.value.unit_suffix }) }} + loop: "{{ inference_engine_profiles | dict2items }}" + loop_control: + label: "{{ item.key }}" + tags: + - service + +- name: Require the rootless runtime utilities + ansible.builtin.stat: + path: "{{ item }}" + loop: + - /usr/sbin/runuser + - /usr/bin/podman + - /usr/bin/systemctl + - "{{ inference_engine_generator_path }}" + register: inference_engine_tools + tags: + - service + +- name: Verify the rootless runtime utilities are executable + ansible.builtin.assert: + that: + - item.stat.exists + - item.stat.executable + loop: "{{ inference_engine_tools.results }}" + fail_msg: >- + {{ item.item }} is missing or not executable; run the host baseline + roles first (llm-server/host/ansible). + tags: + - service + +- name: Require the KFD compute device + ansible.builtin.stat: + path: /dev/kfd + register: inference_engine_kfd + tags: + - service + +- name: Discover the AMD render nodes + ansible.builtin.find: + paths: /dev/dri + patterns: renderD* + register: inference_engine_render_node_find + tags: + - service + +- name: Record the render nodes + ansible.builtin.set_fact: + inference_engine_render_nodes: >- + {{ inference_engine_render_node_find.files + | map(attribute='path') | sort }} + tags: + - service + +- name: Verify KFD and exactly two render nodes + ansible.builtin.assert: + that: + - inference_engine_kfd.stat.exists + - inference_engine_render_nodes | length == 2 + fail_msg: >- + Expected /dev/kfd and exactly two render nodes (2x RX 6900 XT + baseline); found {{ inference_engine_render_nodes | length }} + render node(s): {{ inference_engine_render_nodes }}. + tags: + - service + +- name: Require every profile model artifact to be present + ansible.builtin.stat: + path: "{{ model_store_root }}/{{ item.value.model_directory }}/{{ item.value.model_file }}" + loop: "{{ inference_engine_profiles | dict2items }}" + loop_control: + label: "{{ item.key }}" + register: inference_engine_models + tags: + - service + +- name: Verify every profile model artifact is present + ansible.builtin.assert: + that: + - item.stat.exists + loop: "{{ inference_engine_models.results }}" + fail_msg: >- + Model artifact for {{ item.item.key }} is missing under + {{ model_store_root }}; run the model_store role (--tags models) first. + tags: + - service + +- name: Check whether the pinned engine image is in the workload store + ansible.builtin.command: + argv: + - /usr/sbin/runuser + - --user + - "{{ inference_engine_user }}" + - -- + - /usr/bin/env + - "HOME={{ inference_engine_user_home }}" + - "XDG_RUNTIME_DIR=/run/user/{{ inference_engine_uid }}" + - /usr/bin/podman + - image + - exists + - "{{ inference_engine_image_ref }}" + chdir: "{{ inference_engine_user_home }}" + changed_when: false + failed_when: inference_engine_image_present.rc not in [0, 1] + check_mode: false + register: inference_engine_image_present + tags: + - service + +- name: Pull the pinned engine image into the workload store + ansible.builtin.command: + argv: + - /usr/sbin/runuser + - --user + - "{{ inference_engine_user }}" + - -- + - /usr/bin/env + - "HOME={{ inference_engine_user_home }}" + - "XDG_RUNTIME_DIR=/run/user/{{ inference_engine_uid }}" + - /usr/bin/podman + - pull + - "{{ inference_engine_image_ref }}" + chdir: "{{ inference_engine_user_home }}" + when: + - not ansible_check_mode + - inference_engine_image_present.rc != 0 + tags: + - service + +- name: Ensure the environment quadlet directory exists + ansible.builtin.file: + path: "{{ inference_engine_quadlet_dir }}" + state: directory + owner: root + group: root + mode: "0755" + tags: + - service + +- name: Render the environment Podman network quadlet + ansible.builtin.template: + src: llama-cpp.network.j2 + dest: "{{ inference_engine_quadlet_dir }}/{{ inference_engine_network_name }}.network" + owner: root + group: root + mode: "0644" + notify: Reload the workload user manager + tags: + - service + +- name: Render one engine quadlet per model profile + ansible.builtin.template: + src: llama-server.container.j2 + dest: "{{ inference_engine_quadlet_dir }}/{{ inference_engine_unit_names[quadlet_item.key] }}.container" + owner: root + group: root + mode: "0644" + vars: + entry: + profile_key: "{{ quadlet_item.key }}" + profile: "{{ quadlet_item.value }}" + unit_name: "{{ inference_engine_unit_names[quadlet_item.key] }}" + container_name: "{{ inference_engine_unit_names[quadlet_item.key] }}" + ctx_size: >- + {{ (inference_engine_ctx_size_override | int) + if inference_engine_ctx_size_override is not none + else quadlet_item.value.ctx_size }} + other_units: >- + {{ inference_engine_unit_names.values() + | difference([inference_engine_unit_names[quadlet_item.key]]) }} + loop: "{{ inference_engine_profiles | dict2items }}" + loop_control: + loop_var: quadlet_item + label: "{{ quadlet_item.key }}" + notify: Reload the workload user manager + tags: + - service + +- name: Exercise the Quadlet generator without starting anything + ansible.builtin.command: + argv: + - /usr/sbin/runuser + - --user + - "{{ inference_engine_user }}" + - -- + - /usr/bin/env + - "HOME={{ inference_engine_user_home }}" + - "XDG_RUNTIME_DIR=/run/user/{{ inference_engine_uid }}" + - "QUADLET_UNIT_DIRS={{ inference_engine_quadlet_dir }}" + - "{{ inference_engine_generator_path }}" + - --dryrun + chdir: "{{ inference_engine_user_home }}" + changed_when: false + when: not ansible_check_mode + tags: + - service + +- name: Read the current state of every engine unit + ansible.builtin.command: + argv: + - /usr/sbin/runuser + - --user + - "{{ inference_engine_user }}" + - -- + - /usr/bin/env + - "HOME={{ inference_engine_user_home }}" + - "XDG_RUNTIME_DIR=/run/user/{{ inference_engine_uid }}" + - /usr/bin/systemctl + - --user + - is-active + - "{{ inference_engine_unit_names[item.key] }}" + chdir: "{{ inference_engine_user_home }}" + changed_when: false + failed_when: inference_engine_unit_states.rc not in [0, 3] + check_mode: false + loop: "{{ inference_engine_profiles | dict2items }}" + loop_control: + label: "{{ item.key }}" + register: inference_engine_unit_states + when: not ansible_check_mode + tags: + - service + +- name: Stop any other active engine unit before the switch + ansible.builtin.command: + argv: + - /usr/sbin/runuser + - --user + - "{{ inference_engine_user }}" + - -- + - /usr/bin/env + - "HOME={{ inference_engine_user_home }}" + - "XDG_RUNTIME_DIR=/run/user/{{ inference_engine_uid }}" + - /usr/bin/systemctl + - --user + - stop + - "{{ inference_engine_unit_names[item.item.key] }}" + chdir: "{{ inference_engine_user_home }}" + when: + - not ansible_check_mode + - item.stdout | trim != 'active' + - item.item.key != inference_engine_active_profile + loop: "{{ inference_engine_unit_states.results }}" + loop_control: + label: "{{ item.item.key }}" + tags: + - service + +- name: Start the selected engine unit + ansible.builtin.command: + argv: + - /usr/sbin/runuser + - --user + - "{{ inference_engine_user }}" + - -- + - /usr/bin/env + - "HOME={{ inference_engine_user_home }}" + - "XDG_RUNTIME_DIR=/run/user/{{ inference_engine_uid }}" + - /usr/bin/systemctl + - --user + - start + - "{{ inference_engine_unit_names[inference_engine_active_profile] }}" + chdir: "{{ inference_engine_user_home }}" + when: + - not ansible_check_mode + - >- + (inference_engine_unit_states.results + | selectattr('item.key', 'equalto', inference_engine_active_profile) + | map(attribute='stdout') | join('') | trim) != 'active' + tags: + - service + +- name: Detect an HTTP client inside the engine container + ansible.builtin.command: + argv: + - /usr/sbin/runuser + - --user + - "{{ inference_engine_user }}" + - -- + - /usr/bin/env + - "HOME={{ inference_engine_user_home }}" + - "XDG_RUNTIME_DIR=/run/user/{{ inference_engine_uid }}" + - /usr/bin/podman + - exec + - "{{ inference_engine_unit_names[inference_engine_active_profile] }}" + - /bin/sh + - -c + - "command -v curl || command -v wget || true" + chdir: "{{ inference_engine_user_home }}" + changed_when: false + check_mode: false + register: inference_engine_http_client + when: not ansible_check_mode + tags: + - service + +- name: Require an HTTP client in the pinned engine image + ansible.builtin.assert: + that: + - inference_engine_http_client.stdout | trim in ['curl', 'wget'] + fail_msg: >- + The pinned engine image contains neither curl nor wget; health checks + cannot run (GNAI-010 section 8 requires checking the image tool first). + when: not ansible_check_mode + tags: + - service + +- name: Wait for the engine health endpoint + ansible.builtin.command: + argv: "{{ [ '/usr/sbin/runuser', '--user', inference_engine_user, '--', + '/usr/bin/env', 'HOME=' ~ inference_engine_user_home, + 'XDG_RUNTIME_DIR=/run/user/' ~ inference_engine_uid, + '/usr/bin/podman', 'exec', + inference_engine_unit_names[inference_engine_active_profile], + '/bin/sh', '-c' ] + + (['curl', '-fsS', health_url] if inference_engine_http_client.stdout | trim == 'curl' + else ['wget', '-qO-', health_url]) }}" + vars: + health_url: "http://127.0.0.1:{{ inference_engine_container_port }}/health" + changed_when: false + check_mode: false + register: inference_engine_health + until: inference_engine_health.rc == 0 + retries: 60 + delay: 5 + when: not ansible_check_mode + tags: + - service + +- name: Verify the served model alias + ansible.builtin.command: + argv: "{{ [ '/usr/sbin/runuser', '--user', inference_engine_user, '--', + '/usr/bin/env', 'HOME=' ~ inference_engine_user_home, + 'XDG_RUNTIME_DIR=/run/user/' ~ inference_engine_uid, + '/usr/bin/podman', 'exec', + inference_engine_unit_names[inference_engine_active_profile], + '/bin/sh', '-c' ] + + (['curl', '-fsS', models_url] if inference_engine_http_client.stdout | trim == 'curl' + else ['wget', '-qO-', models_url]) }}" + vars: + models_url: "http://127.0.0.1:{{ inference_engine_container_port }}/v1/models" + changed_when: false + check_mode: false + register: inference_engine_models_endpoint + when: not ansible_check_mode + tags: + - service + +- name: Verify the active profile alias is served + ansible.builtin.assert: + that: + - >- + inference_engine_profiles[inference_engine_active_profile].alias + in inference_engine_models_endpoint.stdout + fail_msg: >- + The /v1/models response does not contain the expected alias + {{ inference_engine_profiles[inference_engine_active_profile].alias }}. + when: not ansible_check_mode + tags: + - service diff --git a/llm-server/llm/ansible/roles/inference_engine/templates/llama-cpp.network.j2 b/llm-server/llm/ansible/roles/inference_engine/templates/llama-cpp.network.j2 new file mode 100644 index 0000000..0860320 --- /dev/null +++ b/llm-server/llm/ansible/roles/inference_engine/templates/llama-cpp.network.j2 @@ -0,0 +1,4 @@ +[Network] +NetworkName={{ inference_engine_network_name }} +Driver=bridge +Internal=true diff --git a/llm-server/llm/ansible/roles/inference_engine/templates/llama-server.container.j2 b/llm-server/llm/ansible/roles/inference_engine/templates/llama-server.container.j2 new file mode 100644 index 0000000..38cf4e2 --- /dev/null +++ b/llm-server/llm/ansible/roles/inference_engine/templates/llama-server.container.j2 @@ -0,0 +1,27 @@ +[Unit] +Description=LLM inference engine ({{ entry.profile_key }}) +{% for other in entry.other_units %} +Conflicts={{ other }} +{% endfor %} +{% if entry.other_units | length == 1 %} +Before={{ entry.other_units | first }} +{% endif %} + +[Container] +Image={{ inference_engine_image_ref }} +Pull=never +ContainerName={{ entry.container_name }} +AddDevice=/dev/kfd +{% for device in inference_engine_render_nodes %} +AddDevice={{ device }} +{% endfor %} +GroupAdd=keep-groups +Volume={{ model_store_root }}/{{ entry.profile.model_directory }}:/models:ro +Network={{ inference_engine_network_name }} +NetworkAlias={{ inference_engine_network_alias }} +Exec=llama-server --host 0.0.0.0 --port {{ inference_engine_container_port }} --model /models/{{ entry.profile.model_file }} --alias {{ entry.profile.alias }} --parallel {{ entry.profile.parallel }} --ctx-size {{ entry.ctx_size }} --n-predict {{ entry.profile.n_predict }} --n-gpu-layers {{ inference_engine_common.n_gpu_layers }} --split-mode {{ inference_engine_common.split_mode }} --tensor-split {{ inference_engine_common.tensor_split }} --flash-attn {{ inference_engine_common.flash_attn }} --cache-type-k {{ inference_engine_common.cache_type_k }} --cache-type-v {{ inference_engine_common.cache_type_v }} --jinja --reasoning {{ inference_engine_common.reasoning }} --reasoning-effort {{ entry.profile.reasoning_effort }} --reasoning-budget {{ inference_engine_common.reasoning_budget }} --temp {{ entry.profile.temp }} --top-p {{ entry.profile.top_p }} --top-k {{ entry.profile.top_k }} --min-p {{ entry.profile.min_p }} --presence-penalty {{ entry.profile.presence_penalty }} --repeat-penalty {{ entry.profile.repeat_penalty }}{% if entry.profile.mtp.enabled %} --spec-type draft-mtp --spec-draft-n-max {{ entry.profile.mtp.spec_draft_n_max }}{% endif %} --no-webui --metrics + +[Service] +Restart=on-failure +RestartSec=10 +TimeoutStartSec=900 diff --git a/llm-server/llm/ansible/roles/model_store/README.md b/llm-server/llm/ansible/roles/model_store/README.md new file mode 100644 index 0000000..196c141 --- /dev/null +++ b/llm-server/llm/ansible/roles/model_store/README.md @@ -0,0 +1,35 @@ +# model_store + +Provides the shared, pinned model store under +`/var/lib/ghostnet-ai/shared/models` (GNAI-010 sections 3 and 4). Weights are +large, immutable, non-secret artifacts; every KI environment mounts its +profile directory read-only. + +## Variables + +- `model_store_root` – root of the shared store (default + `/var/lib/ghostnet-ai/shared/models`). +- `model_store_artifacts` – list of pinned artifacts with `profile`, + `directory`, `file`, `sha256`, `url`, `repository`, `revision`, `origin` + and `license`. +- `model_store_include_vision` – additionally provisions the Swift vision + projector when `true` (default `false`; vision stage only, GNAI-010 + section 3.4). +- `model_store_required_free_mib` – free-space reserve checked before any + pending download (default 71680). + +## Behaviour + +- Manifest is validated (unique profiles, 64-hex checksums, HTTPS URLs). +- Directories are `root:root 0755`, artifact files `root:root 0644`; a + `SOURCE.yml` provenance record is rendered per directory. +- Downloads go to a hidden `.part` file with `get_url` checksum verification + and are moved into place only after the checksum passed (atomic). +- An existing file with a mismatching checksum fails the run; it is never + overwritten or used. +- Check mode downloads nothing; already-present files are still verified. + +## Tags + +`models` – everything in this role. The `service` tag of `inference_engine` +also re-verifies present artifacts. diff --git a/llm-server/llm/ansible/roles/model_store/defaults/main.yml b/llm-server/llm/ansible/roles/model_store/defaults/main.yml new file mode 100644 index 0000000..c24b563 --- /dev/null +++ b/llm-server/llm/ansible/roles/model_store/defaults/main.yml @@ -0,0 +1,51 @@ +--- +# Shared, pinned model store (GNAI-010 sections 3 and 4). Weights are large, +# immutable, non-secret artifacts; every AI environment mounts them read-only. +model_store_root: /var/lib/ghostnet-ai/shared/models + +# Minimum free space on the target filesystem before downloading missing +# artifacts (GNAI-010 section 4: ~45 GiB final stock, the largest single +# temporary download and an operating buffer). +model_store_required_free_mib: 71680 + +# The vision projector is intentionally not part of the first text/tool +# bring-up (GNAI-010 section 3.4). Enable only for the dedicated vision stage. +model_store_include_vision: false + +model_store_artifacts: + - profile: swift-qwen3.8-27b-q6k + directory: swift-qwen3.8-27b + file: Swift-Qwen3.8-27B-Q6_K.gguf + sha256: 7f4de8abd5446c08b0f975a1b38e43d02639f4ae9ecdd2ddfd5c5b0f612bda59 + url: https://huggingface.co/ukisai/Swift-Qwen3.8-27B-GGUF/resolve/eb0e3a7dc70643c2ab920f5133750d02c20849ff/Swift-Qwen3.8-27B-Q6_K.gguf + repository: ukisai/Swift-Qwen3.8-27B-GGUF + revision: eb0e3a7dc70643c2ab920f5133750d02c20849ff + origin: Swift efficiency finetune of Qwen3.8-27B (UkisAI), Q6_K + license: >- + Qwen base model Apache-2.0; Swift changes under Swift Open License 1.0 + (free personal, research, education, evaluation and commercial use up to + the revenue limit defined in the model card). + - profile: unsloth-qwen3.8-27b-ud-q6k + directory: unsloth-qwen3.8-27b + file: Qwen3.8-27B-UD-Q6_K.gguf + sha256: c9c206812fbe4ac7b76a729e25928b63f2ae89d37f69da7a71c20aec763cd436 + url: https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/4ca720788d1e01f1bff70c033e0d0028fd02e502/Qwen3.8-27B-UD-Q6_K.gguf + repository: unsloth/Qwen3.8-27B-GGUF + revision: 4ca720788d1e01f1bff70c033e0d0028fd02e502 + origin: Unsloth dynamic quant (UD) of Qwen3.8-27B, Q6_K + license: Apache-2.0 + +model_store_vision_artifact: + profile: swift-qwen3.8-27b-vision-mmproj + directory: swift-qwen3.8-27b + file: mmproj-Swift-Qwen3.8-27B-F16.gguf + sha256: daa1116c9422fa390cc8688495da0e91781f92841dfc3b31a378ff252571745a + url: https://huggingface.co/ukisai/Swift-Qwen3.8-27B-GGUF/resolve/eb0e3a7dc70643c2ab920f5133750d02c20849ff/mmproj-Swift-Qwen3.8-27B-F16.gguf + repository: ukisai/Swift-Qwen3.8-27B-GGUF + revision: eb0e3a7dc70643c2ab920f5133750d02c20849ff + origin: Swift vision projector, F16 (vision stage only) + license: >- + Qwen base model Apache-2.0; Swift changes under Swift Open License 1.0. + +model_store_directory_mode: "0755" +model_store_file_mode: "0644" diff --git a/llm-server/llm/ansible/roles/model_store/tasks/main.yml b/llm-server/llm/ansible/roles/model_store/tasks/main.yml new file mode 100644 index 0000000..f906682 --- /dev/null +++ b/llm-server/llm/ansible/roles/model_store/tasks/main.yml @@ -0,0 +1,176 @@ +--- +# Builds the shared, pinned model store (GNAI-010 sections 3 and 4). +# Downloads are atomic and checksum-verified; an existing file with a +# mismatching checksum aborts the run instead of being overwritten. +- name: Assemble the effective artifact manifest + ansible.builtin.set_fact: + model_store_all_artifacts: >- + {{ model_store_artifacts + + (model_store_vision_artifact | ternary( + [model_store_vision_artifact], [])) + if model_store_include_vision + else model_store_artifacts }} + tags: + - models + - service + +- name: Validate the artifact manifest + ansible.builtin.assert: + that: + - model_store_root is match("^/") + - model_store_all_artifacts | length > 0 + - model_store_all_artifacts | map(attribute='profile') | list | unique | length + == model_store_all_artifacts | length + - model_store_all_artifacts | map(attribute='directory') | list | length > 0 + - model_store_all_artifacts | map(attribute='file') | list | length > 0 + - model_store_all_artifacts | map(attribute='sha256') | select('match', '^[0-9a-f]{64}$') | list | length + == model_store_all_artifacts | length + - model_store_all_artifacts | map(attribute='url') | select('match', '^https://') | list | length + == model_store_all_artifacts | length + fail_msg: The model store manifest is incomplete or ambiguous. + tags: + - models + - service + +- name: Require the checksum utility + ansible.builtin.stat: + path: /usr/bin/sha256sum + register: model_store_sha256sum_tool + tags: + - models + - service + +- name: Verify the checksum utility is executable + ansible.builtin.assert: + that: + - model_store_sha256sum_tool.stat.exists + - model_store_sha256sum_tool.stat.executable + fail_msg: /usr/bin/sha256sum is required to verify model artifacts. + tags: + - models + - service + +- name: Create the model profile directories + ansible.builtin.file: + path: "{{ model_store_root }}/{{ item }}" + state: directory + owner: root + group: root + mode: "{{ model_store_directory_mode }}" + loop: "{{ model_store_all_artifacts | map(attribute='directory') | unique }}" + tags: + - models + - service + +- name: Check which model artifacts are already present + ansible.builtin.stat: + path: "{{ model_store_root }}/{{ item.directory }}/{{ item.file }}" + get_checksum: false + loop: "{{ model_store_all_artifacts }}" + loop_control: + label: "{{ item.profile }}" + register: model_store_present + tags: + - models + - service + +- name: Require sufficient free space before downloading missing artifacts + ansible.builtin.command: + argv: + - /bin/df + - -m + - --output=avail + - "{{ model_store_root }}" + changed_when: false + check_mode: false + register: model_store_free_space + when: >- + model_store_present.results + | selectattr('stat.exists', 'equalto', false) + | list | length > 0 + tags: + - models + +- name: Verify the free-space reserve for pending downloads + ansible.builtin.assert: + that: + - model_store_free_space.stdout_lines | last | int >= model_store_required_free_mib | int + fail_msg: >- + Less than {{ model_store_required_free_mib }} MiB free on + {{ model_store_root }}; refusing to download model artifacts + (GNAI-010 section 4). + when: model_store_free_space is not skipped + tags: + - models + +- name: Verify the checksum of already present artifacts + ansible.builtin.command: + argv: + - /usr/bin/sha256sum + - "{{ model_store_root }}/{{ item.item.directory }}/{{ item.item.file }}" + changed_when: false + check_mode: false + loop: "{{ model_store_present.results | selectattr('stat.exists', 'equalto', true) | list }}" + loop_control: + label: "{{ item.item.profile }}" + register: model_store_existing_checksums + failed_when: model_store_existing_checksums.stdout.split()[0] != item.item.sha256 + tags: + - models + - service + +- name: Download missing model artifacts atomically + ansible.builtin.get_url: + url: "{{ item.item.url }}" + dest: "{{ model_store_root }}/{{ item.item.directory }}/.{{ item.item.file }}.part" + checksum: "sha256:{{ item.item.sha256 }}" + mode: "0644" + owner: root + group: root + timeout: 120 + loop: "{{ model_store_present.results | selectattr('stat.exists', 'equalto', false) | list }}" + loop_control: + label: "{{ item.item.profile }}" + when: not ansible_check_mode + register: model_store_downloads + tags: + - models + +- name: Move verified downloads to their final model path + ansible.builtin.command: + argv: + - /bin/mv + - "{{ model_store_root }}/{{ item.item.directory }}/.{{ item.item.file }}.part" + - "{{ model_store_root }}/{{ item.item.directory }}/{{ item.item.file }}" + loop: "{{ model_store_present.results | selectattr('stat.exists', 'equalto', false) | list }}" + loop_control: + label: "{{ item.item.profile }}" + when: not ansible_check_mode + tags: + - models + +- name: Enforce model artifact ownership and permissions + ansible.builtin.file: + path: "{{ model_store_root }}/{{ item.directory }}/{{ item.file }}" + state: file + owner: root + group: root + mode: "{{ model_store_file_mode }}" + loop: "{{ model_store_all_artifacts }}" + loop_control: + label: "{{ item.profile }}" + tags: + - models + - service + +- name: Write the per-directory origin manifest SOURCE.yml + ansible.builtin.template: + src: source.yml.j2 + dest: "{{ model_store_root }}/{{ item }}/SOURCE.yml" + owner: root + group: root + mode: "{{ model_store_file_mode }}" + loop: "{{ model_store_all_artifacts | map(attribute='directory') | unique }}" + tags: + - models + - service diff --git a/llm-server/llm/ansible/roles/model_store/templates/source.yml.j2 b/llm-server/llm/ansible/roles/model_store/templates/source.yml.j2 new file mode 100644 index 0000000..7084c2c --- /dev/null +++ b/llm-server/llm/ansible/roles/model_store/templates/source.yml.j2 @@ -0,0 +1,15 @@ +# Provenance record for the artifacts of this model directory. +# Rendered by the model_store role; do not edit on the host. +{% set entries = model_store_all_artifacts | selectattr('directory', 'equalto', item) | list %} +{% for entry in entries %} +{{ entry.profile }}: + file: {{ entry.file }} + sha256: {{ entry.sha256 }} + repository: {{ entry.repository }} + revision: {{ entry.revision }} + url: {{ entry.url }} + origin: >- + {{ entry.origin }} + license: >- + {{ entry.license | replace("\n", " ") }} +{% endfor %}